├── s3st.gif ├── .eslintrc.js ├── scripts ├── build.sh └── publish-release.js ├── src ├── index.js ├── transformers │ └── decompress.js └── cli.js ├── LICENSE ├── .gitignore ├── package.json ├── .circleci └── config.yml └── README.md /s3st.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lmammino/s3st/HEAD/s3st.gif -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | 'env': { 3 | 'browser': true, 4 | 'commonjs': true, 5 | 'es6': true 6 | }, 7 | 'extends': 'standard', 8 | 'globals': { 9 | 'Atomics': 'readonly', 10 | 'SharedArrayBuffer': 'readonly' 11 | }, 12 | 'parserOptions': { 13 | 'ecmaVersion': 2018 14 | }, 15 | 'rules': { 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | VERSION=$1 4 | V=$(echo $VERSION | tr . _) 5 | NODE_VERSION=12.0.0 6 | 7 | for PLATFORM in "macos-x64" "linux-x64" "alpine-x64" "windows-x86" 8 | do 9 | SUFFIX=$(echo $PLATFORM | cut -d'-' -f1) 10 | if [ "$SUFFIX" == "windows" ] 11 | then 12 | SUFFIX="${SUFFIX}.exe" 13 | fi 14 | DEST="build/s3st-${V}-${SUFFIX}" 15 | node_modules/.bin/nexe . -t ${PLATFORM}-${NODE_VERSION} -o ${DEST} --temp .nexe 16 | gzip ${DEST} 17 | done 18 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const S3ListBucketStream = require('s3-list-bucket-stream') 4 | const S3ObjectContentStream = require('s3-object-content-stream') 5 | const pumpify = require('pumpify') 6 | 7 | const createS3stStream = (s3, bucketName, prefix = '', transformer = undefined) => { 8 | // create the instance for the list bucket stream 9 | const listBucketStream = new S3ListBucketStream( 10 | s3, 11 | bucketName, 12 | prefix 13 | ) 14 | 15 | // create the instance for the object content stream 16 | const objectContentStream = new S3ObjectContentStream( 17 | s3, 18 | bucketName, 19 | transformer 20 | ) 21 | 22 | return pumpify(listBucketStream, objectContentStream) 23 | } 24 | 25 | module.exports = createS3stStream 26 | -------------------------------------------------------------------------------- /src/transformers/decompress.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { extname } = require('path') 4 | const { createGunzip, createDeflate, createBrotliDecompress } = require('zlib') 5 | const { PassThrough } = require('readable-stream') 6 | 7 | const decompress = (key) => { 8 | const extension = extname(key.toString()) 9 | if (['.gz', '.gzip'].includes(extension)) { 10 | return createGunzip() // if the file is gzip return a transform stream to decompress it 11 | } 12 | 13 | if (['.zz', '.deflate'].includes(extension)) { 14 | return createDeflate() // if the file is deflate return a transform stream to decompress it 15 | } 16 | 17 | if (createBrotliDecompress && ['.br', '.brotli'].includes(extension)) { 18 | return createBrotliDecompress() // if the file is brotli return a transform stream to decompress it 19 | } 20 | 21 | // otherwise returns a passthrough stream (do not modify the content) 22 | return new PassThrough() 23 | } 24 | 25 | module.exports = decompress 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Luciano Mammino 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # TypeScript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # next.js build output 61 | .next 62 | 63 | build/ 64 | .nexe/ 65 | -------------------------------------------------------------------------------- /src/cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | 'use strict' 4 | 5 | const AWS = require('aws-sdk') 6 | const program = require('commander') 7 | const { pipeline } = require('readable-stream') 8 | const createS3stStream = require('./') 9 | const decompress = require('./transformers/decompress') 10 | const pkg = require('../package.json') 11 | 12 | program 13 | .version(pkg.version, '-v, --version') 14 | .option('-D, --do-not-decompress', 'Do not try to decompress files automatically (gzip, deflate, brotli)') 15 | .usage('[options] [prefix]') 16 | .parse(process.argv) 17 | 18 | const [bucket, prefix] = program.args 19 | 20 | if (!bucket) { 21 | console.error('Error: expected "bucket" argument. Run with --help for more details on usage') 22 | process.exit(1) 23 | } 24 | 25 | const transform = program.doNotDecompress ? undefined : decompress 26 | 27 | const s3 = new AWS.S3() 28 | const stream = createS3stStream(s3, bucket, prefix || '', transform) 29 | 30 | const handleErr = (err) => { 31 | console.error('Error:', err.message) 32 | process.exit(1) 33 | } 34 | 35 | stream.on('error', handleErr) 36 | 37 | pipeline( 38 | stream, 39 | process.stdout, 40 | (err) => { 41 | if (err) { 42 | handleErr(err) 43 | } 44 | } 45 | ) 46 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "s3st", 3 | "version": "1.1.0", 4 | "description": "A command line utility that allows you to stream data from multiple S3 objects directly into your terminal", 5 | "main": "src/index.js", 6 | "bin": { 7 | "s3st": "src/cli.js" 8 | }, 9 | "files": [ 10 | "src/" 11 | ], 12 | "engines": { 13 | "node": ">=10" 14 | }, 15 | "engineStrict": true, 16 | "scripts": { 17 | "test:lint": "eslint src", 18 | "build": "scripts/build.sh $npm_package_version", 19 | "test": "npm run test:lint", 20 | "release:tag": "git tag $npm_package_version && git push --tags", 21 | "package:publish": "scripts/publish-release.js $GITHUB_TOKEN lmammino/s3st $npm_package_version" 22 | }, 23 | "repository": { 24 | "type": "git", 25 | "url": "git+https://github.com/lmammino/s3st.git" 26 | }, 27 | "keywords": [ 28 | "Stream", 29 | "Streams", 30 | "AWS", 31 | "S3", 32 | "Bucket", 33 | "List bucket", 34 | "Object" 35 | ], 36 | "author": "Luciano Mammino", 37 | "license": "MIT", 38 | "bugs": { 39 | "url": "https://github.com/lmammino/s3st/issues" 40 | }, 41 | "homepage": "https://github.com/lmammino/s3st#readme", 42 | "dependencies": { 43 | "aws-sdk": "^2.694.0", 44 | "commander": "^5.1.0", 45 | "pumpify": "^2.0.1", 46 | "readable-stream": "^3.6.0", 47 | "s3-list-bucket-stream": "^1.0.0", 48 | "s3-object-content-stream": "^1.0.1" 49 | }, 50 | "devDependencies": { 51 | "acorn": "^7.3.1", 52 | "eslint": "^7.2.0", 53 | "eslint-config-standard": "^14.1.1", 54 | "eslint-plugin-import": "^2.21.2", 55 | "eslint-plugin-node": "^11.1.0", 56 | "eslint-plugin-promise": "^4.2.1", 57 | "eslint-plugin-standard": "^4.0.1", 58 | "nexe": "^3.3.2", 59 | "request": "^2.88.2", 60 | "request-promise": "^4.2.5", 61 | "request-promise-core": "^1.1.3" 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /scripts/publish-release.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const fs = require('fs') 4 | const path = require('path') 5 | const request = require('request-promise') 6 | 7 | const USER_AGENT = 'publish-release-script-1' 8 | const GH_API_ENDPOINT = 'https://api.github.com' 9 | const GH_UPLOAD_URL = id => `https://uploads.github.com/repos/lmammino/s3st/releases/${id}/assets` 10 | 11 | const token = process.argv[2] 12 | const repo = process.argv[3] 13 | const release = process.argv[4] 14 | const buildFolder = process.argv[5] || './build' 15 | 16 | console.log(`Creating release "${release}" from "${buildFolder}"`) 17 | 18 | const uploadFile = releaseId => file => new Promise((resolve, reject) => { 19 | console.log('attaching', file.path) 20 | return fs.readFile(file.path, (error, buffer) => { 21 | if (error) { 22 | return reject(error) 23 | } 24 | 25 | return resolve(request.post({ 26 | body: buffer, 27 | url: `${GH_UPLOAD_URL(releaseId)}?name=${file.name}`, 28 | headers: { 29 | 'User-Agent': USER_AGENT, 30 | 'Content-Type': 'application/zip', 31 | Authorization: `token ${token}` 32 | } 33 | })) 34 | }) 35 | }) 36 | 37 | let releaseId 38 | 39 | request({ 40 | uri: `${GH_API_ENDPOINT}/repos/${repo}/releases`, 41 | method: 'POST', 42 | body: JSON.stringify({ 43 | tag_name: release, 44 | name: `v${release}` 45 | }), 46 | headers: { 47 | 'User-Agent': USER_AGENT, 48 | 'Content-Type': 'application/json', 49 | Authorization: `token ${token}` 50 | } 51 | }) 52 | .then((response) => { 53 | const data = JSON.parse(response) 54 | releaseId = data.id 55 | console.log(`Created release: ${data.url}`) 56 | 57 | return new Promise((resolve, reject) => { 58 | fs.readdir(buildFolder, (err, items) => { 59 | if (err) { 60 | return reject(err) 61 | } 62 | 63 | return resolve(items.map(item => ({ 64 | name: item, 65 | path: path.resolve(path.join(buildFolder, item)) 66 | }))) 67 | }) 68 | }) 69 | }) 70 | .then(files => Promise.all(files.map(uploadFile(releaseId)))) 71 | .then(() => { 72 | console.log('✅ Done') 73 | }) 74 | .catch((err) => { 75 | console.error('Failed to create release', err) 76 | process.exit(1) 77 | }) 78 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build_node11: 4 | docker: 5 | - image: circleci/node:11 6 | 7 | working_directory: ~/node11 8 | 9 | steps: 10 | - checkout 11 | 12 | - restore_cache: 13 | keys: 14 | - v1-dependencies-v11-{{ checksum "package.json" }} 15 | - v1-dependencies-v11- 16 | - v1-dependencies- 17 | 18 | - run: 19 | name: install dependencies 20 | command: npm install 21 | 22 | - save_cache: 23 | paths: 24 | - node_modules 25 | key: v1-dependencies-v11-{{ checksum "package.json" }} 26 | 27 | - run: 28 | name: run tests 29 | command: npm test 30 | 31 | - persist_to_workspace: 32 | root: "." 33 | paths: 34 | - "." 35 | 36 | 37 | deploy: 38 | docker: 39 | - image: circleci/node:11 40 | 41 | steps: 42 | - attach_workspace: 43 | at: ~/node11 44 | 45 | - run: 46 | name: configure NPM registry 47 | command: echo "Running in master. Attempting release" 48 | 49 | - run: 50 | name: configure NPM registry 51 | command: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc 52 | 53 | - run: 54 | name: configure GitHub SSH 55 | command: mkdir -p ~/.ssh && ssh-keyscan -H github.com > ~/.ssh/known_hosts 56 | 57 | - run: 58 | working_directory: ~/node11 59 | name: Setup Git 60 | command: git config user.email $GIT_EMAIL && git config user.name $GIT_NAME 61 | 62 | - run: 63 | working_directory: ~/node11 64 | name: build 65 | command: | 66 | npm run build; 67 | 68 | - run: 69 | working_directory: ~/node11 70 | name: publish 71 | command: | 72 | npm run release:tag && npm run package:publish && npm publish; 73 | 74 | - persist_to_workspace: 75 | root: "." 76 | paths: 77 | - "." 78 | 79 | workflows: 80 | version: 2 81 | build_test_deploy: 82 | jobs: 83 | - build_node11 84 | - deploy: 85 | requires: 86 | - build_node11 87 | filters: 88 | branches: 89 | only: master 90 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # s3st 2 | 3 | A command line utility that allows you to stream data from multiple S3 objects 4 | directly into your terminal. 5 | 6 | [![npm version](https://badge.fury.io/js/s3st.svg)](https://badge.fury.io/js/s3st) 7 | [![CircleCI](https://circleci.com/gh/lmammino/s3st.svg?style=shield)](https://circleci.com/gh/lmammino/s3st) 8 | [![JavaScript Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://standardjs.com) 9 | 10 | ## Demo! 11 | 12 | [![Demo image terminal](s3st.gif)](https://asciinema.org/a/dWJtrXA0HRqDJxndId9Xauz0e) 13 | 14 | [See the FULL demo on asciinema](https://asciinema.org/a/dWJtrXA0HRqDJxndId9Xauz0e) 15 | 16 | ## Rationale 17 | 18 | This utility is particularly useful when you are storing data in S3 and you want 19 | to easily process the content of your S3 objects from your command line, 20 | for instance if you are storing your CloudTrail logs in an S3 buckets and you 21 | want to grep over them you can do something like this: 22 | 23 | ```bash 24 | s3st mybucket AWSLogs/123456789/CloudTrail/eu-west-1/2019/01/17/ | jq . | grep "lambda" 25 | ``` 26 | 27 | By default the command line will be able to decompress most compressed files in 28 | realtime (gzip, brotli and deflate). 29 | 30 | 31 | ## Install 32 | 33 | There are several ways to install `s3st`: 34 | 35 | ### Install global with NPM 36 | 37 | (Requires Node v10+): 38 | 39 | ```bash 40 | npm i -g s3st 41 | ``` 42 | 43 | ### Precompiled binaries 44 | 45 | Alternatively you can download one of the pre-compiled binaries for linux, 46 | windows, mac or alpine from the [Releases page](https://github.com/lmammino/s3st/releases). 47 | 48 | These binaries do not require you to have Node installed. 49 | 50 | ### With [npx](https://www.npmjs.com/package/npx) (use without install) 51 | 52 | ```bash 53 | npx s3st some-s3-bucket 54 | ``` 55 | 56 | 57 | ## Usage 58 | 59 | ```bash 60 | Usage: s3st [options] [prefix] 61 | 62 | Options: 63 | -v, --version output the version number 64 | -D, --do-not-decompress Do not try to decompress files automatically (gzip, deflate, brotli) 65 | -h, --help output usage information 66 | ``` 67 | 68 | `bucket` represents the name of the bucket to iterate over 69 | `prefix` is an optional argument that you can pass to select a subset of object 70 | that match the given prefix. 71 | 72 | 73 | ## Automatic Decompression 74 | 75 | The command will automatically try to decompress compressed files based on their 76 | extension, as per the following mapping: 77 | 78 | - `.gz` or `.gzip`: decompress using gzip 79 | - `.zz` or `.deflate`: decompress using deflate 80 | - `.br` or `.brotli`: decompress using brotli (available only if using Node v11.7+) 81 | 82 | If you want to disable this option you can specify the flag `--do-not-decompress` 83 | 84 | 85 | ## AWS Authentication 86 | 87 | The tool will assume you have the proper environment variables or configuration 88 | files properly set as per the [AWS CLI documentation](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) 89 | in order to authenticate requests to AWS. 90 | 91 | 92 | ## Programmatic usage 93 | 94 | This package can also be used programmatically as per the following example: 95 | 96 | ```javascript 97 | 'use strict' 98 | 99 | const createS3stStream = require('s3st') 100 | const AWS = require('aws-sdk') 101 | 102 | // creates an s3 client using the AWS SDK 103 | const s3 = new AWS.S3() 104 | 105 | const stream = createS3stStream(s3, 'mybucket', 'some-prefix') 106 | 107 | stream.pipe(process.stdout) // attach the stream to standard output 108 | ``` 109 | 110 | `createS3stStream` exposes accepts the following arguments: 111 | 112 | - `s3`: an s3 client instance from the AWS SDK or a compatible implementation 113 | - `bucketName`: the name of the bucket 114 | - `prefix` (optional): an object prefix to filter objects in the bucket 115 | - `transform` (optional): a function that allows you to transform the content of 116 | objects as they get streamed (useful for instance for decompression or decryption). 117 | 118 | ### Transform function 119 | 120 | If you want to provide a custom transform function, it should respect the following 121 | signature. 122 | 123 | #### Arguments 124 | - `key` (string): the name of the current object (object key) 125 | 126 | #### Return value 127 | - a `Transform` stream that manipulates the object 128 | 129 | If you want to use the default decompression implementation available by the 130 | default in the command line client, you can import that from [`s3st/src/transformers/decompress`](/src/transformers/decompress.js). 131 | 132 | 133 | ## Data Transfer costs 134 | 135 | If you are using this tool to stream large amount of data be aware that this might have an impact on your [data transfer costs](https://blog.cloudability.com/aws-data-transfer-costs/). In such cases, using an alternative approach like [S3 Select](https://docs.aws.amazon.com/AmazonS3/latest/dev/selecting-content-from-objects.html), could be a way to save on cost. 136 | 137 | Make sure you are aware of alternatives and that you make careful costs considerations before running any heavy workload in the cloud. 138 | 139 | 140 | ## Contributing 141 | 142 | Everyone is very welcome to contribute to this project. You can contribute just by submitting bugs or 143 | suggesting improvements by [opening an issue on GitHub](https://github.com/lmammino/s3st/issues). 144 | 145 | You can also submit PRs as long as you adhere with the code standards and write tests for the proposed changes. 146 | 147 | ## License 148 | 149 | Licensed under [MIT License](LICENSE). © Luciano Mammino. 150 | --------------------------------------------------------------------------------