├── .prettierignore ├── .gitignore ├── test-image.jpg ├── .env.example ├── .prettierrc ├── docker-compose.yml ├── test-upload.js ├── README.md ├── aws.js ├── .eslintrc.js ├── package.json └── article.md /.prettierignore: -------------------------------------------------------------------------------- 1 | package.json 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | .localstack 3 | -------------------------------------------------------------------------------- /test-image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/good-idea/localstack-demo/HEAD/test-image.jpg -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | AWS_ACCESS_KEY_ID='abc' 2 | AWS_SECRET_KEY='123' 3 | AWS_BUCKET_NAME='demo-bucket' -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "useTabs": true, 3 | "semi": false, 4 | "singleQuote": true, 5 | "trailingComma": "all", 6 | "printWidth": 130, 7 | "arrowParens": "always" 8 | } 9 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.2' 2 | services: 3 | localstack: 4 | image: localstack/localstack:latest 5 | container_name: localstack_demo 6 | ports: 7 | - '4563-4599:4563-4599' 8 | - '8055:8080' 9 | environment: 10 | - SERVICES=s3 11 | - DEBUG=1 12 | - DATA_DIR=/tmp/localstack/data 13 | volumes: 14 | - './.localstack:/tmp/localstack' 15 | -------------------------------------------------------------------------------- /test-upload.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | const path = require('path') 3 | const uploadFile = require('./aws') 4 | 5 | const testUpload = () => { 6 | const filePath = path.resolve(__dirname, 'test-image.jpg') 7 | const fileStream = fs.createReadStream(filePath) 8 | const now = new Date() 9 | const fileName = `test-image-${now.toISOString()}.jpg` 10 | uploadFile(fileStream, fileName).then((response) => { 11 | console.log(":)") 12 | console.log(response) 13 | }).catch((err) => { 14 | console.log(":|") 15 | console.log(err) 16 | }) 17 | } 18 | 19 | testUpload() -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | A basic demo setting up Localstack's S3. 2 | 3 | Read the tutorial [here](https://dev.to/goodidea/how-to-fake-aws-locally-with-localstack-27me). 4 | 5 | Or, spin things up here: 6 | 7 | 1. Install [Docker](https://docs.docker.com/install/) if you haven't already. 8 | 2. Install the [AWS CLI](https://aws.amazon.com/cli/). Even though we aren't going to be working with "real" AWS, we'll use this to talk to our local docker containers. 9 | - Run `aws configure` to set up some credentials. You can enter dummy credentials here if you'd like. 10 | 3. Copy the contents of `.env.example` into a new `.env` file. 11 | 4. Initialize Localstack: `npm run localstack:init`. 12 | - This will create a new container, then stream the logs as it is setting up. It will start with `Waiting for all LocalStack services to be ready`. After a few moments, you'll see a final `Ready`. When you do, press Ctrl+C to exit the logs. 13 | 6. Configure the bucket: run `npm run localstack:config` 14 | - *Note: If you used a different BUCKET_NAME in your `.env` file, make sure to change the instances of `demo_bucket` in the `localstack:config` script in package.json to match.*. 15 | 16 | Upload the test file: 17 | 18 | `npm run test-upload` 19 | -------------------------------------------------------------------------------- /aws.js: -------------------------------------------------------------------------------- 1 | const AWS = require('aws-sdk') 2 | require('dotenv').config() 3 | 4 | const credentials = { 5 | accessKeyId: process.env.AWS_ACCESS_KEY_ID, 6 | secretAccessKey: process.env.AWS_SECRET_KEY, 7 | } 8 | 9 | const useLocal = process.env.NODE_ENV !== 'production' 10 | 11 | const bucketName = process.env.AWS_BUCKET_NAME 12 | 13 | const s3client = new AWS.S3({ 14 | credentials, 15 | /** 16 | * When working locally, we'll use the Localstack endpoints. This is the one for S3. 17 | * A full list of endpoints for each service can be found in the Localstack docs. 18 | */ 19 | endpoint: useLocal ? 'http://localhost:4572' : undefined, 20 | }) 21 | 22 | 23 | const uploadFile = async (data, name) => 24 | new Promise((resolve) => { 25 | s3client.upload( 26 | { 27 | Bucket: bucketName, 28 | /* 29 | include the bucket name here. For some reason Localstack needs it. 30 | see: https://github.com/localstack/localstack/issues/1180 31 | */ 32 | Key: `${bucketName}/${name}`, 33 | Body: data, 34 | }, 35 | (err, response) => { 36 | if (err) throw err 37 | resolve(response) 38 | }, 39 | ) 40 | }) 41 | 42 | module.exports = uploadFile -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | 3 | module.exports = { 4 | parser: 'babel-eslint', 5 | env: { 6 | browser: true, 7 | node: true, 8 | }, 9 | extends: ['airbnb', 'prettier', 'plugin:flowtype/recommended'], 10 | rules: { 11 | 'no-underscore-dangle': 0, 12 | 'no-nested-ternary': 0, 13 | 'jsx-a11y/anchor-is-valid': 0, 14 | 'import/no-cycle': 0, 15 | 'react/jsx-indent': [2, 'tab'], 16 | 'react/jsx-indent-props': [2, 'tab'], 17 | 'react/jsx-filename-extension': [ 18 | 1, 19 | { 20 | extensions: ['.js', '.jsx'], 21 | }, 22 | ], 23 | 'import/prefer-default-export': 0, 24 | 'import/no-extraneous-dependencies': [ 25 | 'error', 26 | { 27 | devDependencies: true, 28 | }, 29 | ], 30 | }, 31 | plugins: ['react', 'jsx-a11y', 'import', 'flowtype'], 32 | settings: { 33 | 'import/resolver': { 34 | alias: [ 35 | ['GraphQL', path.resolve(__dirname, 'src', 'graphql')], 36 | ['Models', path.resolve(__dirname, 'src', 'models')], 37 | ['Utils', path.resolve(__dirname, 'src', 'utils')], 38 | ['Database', path.resolve(__dirname, 'src', 'database')], 39 | ['Config', path.resolve(__dirname, 'src', 'config')], 40 | ['Errors', path.resolve(__dirname, 'src', 'errorTypes')], 41 | ['Types', path.resolve(__dirname, 'src', 'types')], 42 | ['Services', path.resolve(__dirname, 'src', 'types')], 43 | ['Shared', path.resolve(__dirname, '..', 'shared')], 44 | ], 45 | }, 46 | }, 47 | } 48 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "localstack-demo", 3 | "version": "1.0.0", 4 | "description": "Demo for setting up Localstack S3 in Node", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "localstack:init": "npm run localstack:up && docker logs localstack_demo -f", 9 | "localstack:up": "docker-compose up -d", 10 | "localstack:config": "aws --endpoint-url=http://localhost:4572 s3 mb s3://demo-bucket && aws --endpoint-url=http://localhost:4572 s3api put-bucket-acl --bucket demo-bucket --acl public-read", 11 | "test-upload": "node test-upload.js" 12 | }, 13 | "engines": { 14 | "node": ">=8.12.0" 15 | }, 16 | "repository": { 17 | "type": "git", 18 | "url": "git+https://github.com/good-idea/localstack-demo.git" 19 | }, 20 | "keywords": [ 21 | "node", 22 | "aws", 23 | "localstack", 24 | "docker" 25 | ], 26 | "author": "Joseph Thomas | Good Idea Studio", 27 | "license": "MIT", 28 | "bugs": { 29 | "url": "https://github.com/good-idea/localstack-demo/issues" 30 | }, 31 | "homepage": "https://github.com/good-idea/localstack-demo#readme", 32 | "dependencies": { 33 | "aws-sdk": "^2.418.0", 34 | "dotenv": "^6.2.0" 35 | }, 36 | "devDependencies": { 37 | "eslint": "^5.13.0", 38 | "eslint-config-airbnb": "^17.1.0", 39 | "eslint-config-prettier": "^4.0.0", 40 | "eslint-import-resolver-webpack": "^0.11.0", 41 | "eslint-plugin-import": "^2.16.0", 42 | "eslint-plugin-jest": "^22.2.2", 43 | "eslint-plugin-jsx-a11y": "^6.2.1", 44 | "eslint-plugin-react": "^7.12.4" 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /article.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How to fake AWS locally with LocalStack 3 | published: true 4 | description: A brief tutorial on setting up LocalStack + Node to simulate Amazon S3 locally 5 | tags: node, tutorial, aws, docker 6 | --- 7 | 8 | If you're anything like me, you prefer to avoid logging into the AWS console as much as possible. Did you set up your IAM root user with 2FA and correctly configure the CORS and ACL settings on your S3 bucket? 9 | 10 | # 🤷‍♂️ nah. 11 | 12 | I also prefer to keep my local development environment as close as possible to how it's going to work in production. Additionally, I'm always looking for new ways to fill up my small hard drive. I can't think of a better away to achieve all of the above than putting a bunch of S3 servers inside my computer. 13 | 14 | This tutorial will cover setting up [Localstack](https://github.com/localstack/localstack) within a node app. Localstack allows you to emulate a number of AWS services on your computer, but we're just going to use S3 in this example. Also, Localstack isn't specific to Node - so even if you aren't working in Node, a good portion of this tutorial will still be relevant. This also covers a little bit about Docker - if you don't really know what you're doing with Docker or how it works, don't worry. Neither do I. 15 | 16 | You can see the [demo repo](https://github.com/good-idea/localstack-demo) for the finished code. 17 | 18 | A few benefits of this approach are: 19 | 20 | - You can work offline 21 | - You don't need a shared 'dev' bucket that everyone on your team uses 22 | - You can easily wipe & replace your local buckets 23 | - You don't need to worry about paying for AWS usage 24 | - You don't need to log into AWS 😛 25 | 26 | ## Initial Setup 27 | 28 | First, we'll need to install a few things. 29 | 30 | 1. Install [Docker](https://docs.docker.com/install/) if you haven't already. 31 | 2. Install the [AWS CLI](https://aws.amazon.com/cli/). Even though we aren't going to be working with "real" AWS, we'll use this to talk to our local docker containers. 32 | 3. Make a few files. Create a new directory for your project, and within it: `touch index.js docker-compose.yml .env && mkdir .localstack` 33 | 4. Add an image to your project directory and rename it to `test-upload.jpg` 34 | 5. `npm init` to set up a package.json, then `npm install aws-sdk dotenv` 35 | 36 | ## Docker 37 | 38 | (disclaimer: I'm not a docker expert. If anyone has any suggestions on how to improve or better explain any of this, please let me know in the comments!) 39 | 40 | 41 | ### Docker Config 42 | 43 | You can run Localstack directly from the command line, but I like using Docker because it makes me feel smart. It's also nice because you don't need to worry about installing Localstack on your system. I prefer to use docker-compose to set this up. Here's the config: 44 | 45 | `docker-compose.yml` 46 | 47 | ``` 48 | version: '3.2' 49 | services: 50 | localstack: 51 | image: localstack/localstack:latest 52 | container_name: localstack_demo 53 | ports: 54 | - '4563-4584:4563-4584' 55 | - '8055:8080' 56 | environment: 57 | - SERVICES=s3 58 | - DEBUG=1 59 | - DATA_DIR=/tmp/localstack/data 60 | volumes: 61 | - './.localstack:/tmp/localstack' 62 | - '/var/run/docker.sock:/var/run/docker.sock' 63 | ``` 64 | 65 | Breaking some of these lines down: 66 | 67 | #### `image: localstack/localstack:latest` 68 | 69 | Use the latest [Localstack image from Dockerhub](https://hub.docker.com/r/localstack/localstack/) 70 | 71 | 72 | #### `container_name: localstack_demo`: 73 | 74 | This gives our container a specific name that we can refer to later in the CLI. 75 | 76 | 77 | #### `ports: '4563-4584:4563-4584'` and `'8055:8080'`: 78 | 79 | When your docker container starts, it will open up a few ports. The number on the **left** binds the port on your `localhost` to the port within the container, which is the number on the **right**. In most cases, these two numbers can be the same, i.e. `8080:8080`. I often have some other things running on `localhost:8080`, so here, I've changed the default to `8055:8080`. This means that when I connect to `http://localhost:8050` within my app, it's going to talk to port `8080` on the container. 80 | 81 | The line `'4563-4584:4563-4584'` does the same thing, but binds a whole range of ports. These particular port numbers are what Localstack uses as endpoints for the various APIs. We'll see more about this in a little bit. 82 | 83 | #### `environment` 84 | 85 | These are environment variables that are supplied to the container. Localstack will use these to set some things up internally: 86 | 87 | - `SERVICES=s3`: You can define a list of AWS services to emulate. In our case, we're just using S3, but you can include additional APIs, i.e. `SERVICES=s3,lambda`. There's more on this in the Localstack docs. 88 | - `DEBUG=1`: 🧻 Show me all of the logs! 89 | - `DATA_DIR=/tmp/localstack/data`: This is the directory where Localstack will save its data *internally*. More in this next: 90 | 91 | 92 | #### `volumes` 93 | 94 | `'./.localstack:/tmp/localstack'` 95 | 96 | Remember when set up the `DATA_DIR` to be `/tmp/localstack/data` about 2 seconds ago? Just like the `localhost:container` syntax we used on the ports, this allows your containers to access a portion of your hard drive. Your computer's directory on the left, the container's on the right. 97 | 98 | Here, we're telling the container to use our `.localstack` directory for its `/tmp/localstack`. It's like a symlink, or a magical portal, or something. 99 | 100 | In our case, this makes sure that any data created by the container will still be present once the container restarts. Note that `/tmp` is cleared frequently and isn't a good place to store. If you want to put it in a more secure place 101 | - `'/var/run/docker.sock:/var/run/docker.sock'` 102 | 103 | (edit! I accidentally published an unfinished draft. This post isn't complete. Check back in a little bit!) 104 | 105 | ### Starting our Container 106 | 107 | Now that we have our `docker-compose.yml` in good shape, we can spin up the container: `docker-compose up -d`. 108 | 109 | To make sure it's working, we can visit http://localhost:8055 to see Localstack's web UI. Right now it will look pretty empty: 110 | 111 | ![Empty Localstack UI](https://thepracticaldev.s3.amazonaws.com/i/0ym8w6yneym9nh98xo0e.png) 112 | 113 | Similarly, our S3 endpoint http://localhost:4572 will show some basic AWS info: 114 | 115 | ![Empty S3 bucket](https://thepracticaldev.s3.amazonaws.com/i/mmqvbgjrcgs2bhqguxi4.png) 116 | 117 | 118 | (If you don't see something similar to these, check the logs for your docker containers) 119 | 120 | 121 | ## Working with Localstack 122 | 123 | AWS is now inside our computer. You might already be feeling a little bit like you are [the richest person in the world](https://www.businessinsider.com/amazon-ceo-jeff-bezos-richest-person-net-worth-billions-2018-12). (If not, don't worry, just keep reading 😛) 124 | 125 | Before we start uploading files, we need to create and configure a bucket. We'll do this using the AWS CLI that we installed earlier, using the `--endpoint-url` flag to talk to Localstack instead. 126 | 127 | 1. Create a bucket: `aws --endpoint-url=http://localhost:4572 s3 mb s3://demo-bucket` 128 | 2. Attach an [ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html) to the bucket so it is readable: `aws --endpoint-url=http://localhost:4572 s3api put-bucket-acl --bucket demo-bucket --acl public-read` 129 | 130 | Now, when we visit the web UI, we will see our bucket: 131 | 132 | ![Localstack UI with S3 Bucket](https://thepracticaldev.s3.amazonaws.com/i/7lrpsp1n71xhead0xll1.png) 133 | 134 | If you used `volumes` in your docker settings, let's pause for a moment to look at what's going on in `./.localstack/data`. 135 | 136 | ![Localstack S3 JSON](https://thepracticaldev.s3.amazonaws.com/i/12dwa5s12m1of06w4qcu.png) 137 | 138 | Here, we can see that Localstack is recording all API calls in this JSON file. When the container restarts, it will re-apply these calls - this is how we are able to keep our data between restarts. Once we start uploading, we won't see new files appear in this directory. Instead, our uploads will be recorded in this file *as raw data*. (You could include this file in your repo if you wanted to share the state of the container with others - but depending on how much you upload, it's going to become a pretty big file) 139 | 140 | If you want to be able to "restore" your bucket later, you can make a backup of this file. When you're ready to restore, just remove the updated `s3_api_calls.json` file, replace it with your backup, and restart your container. 141 | 142 | ### Uploading from our app 143 | 144 | There are a lot of S3 uploading tutorials out there, so this section won't be as in-depth. We'll just make a simple `upload` function and try uploading an image a few times. 145 | 146 | Copy these contents into their files: 147 | 148 | **.env**, our environment variables 149 | 150 | ``` 151 | AWS_ACCESS_KEY_ID='123' 152 | AWS_SECRET_KEY='xyz' 153 | AWS_BUCKET_NAME='demo-bucket' 154 | ``` 155 | 156 | *Note: it doesn't matter what your AWS key & secret are, as long as they aren't empty.* 157 | 158 | **aws.js**, the module for our upload function 159 | 160 | ```js 161 | const AWS = require('aws-sdk') 162 | require('dotenv').config() 163 | 164 | const credentials = { 165 | accessKeyId: process.env.AWS_ACCESS_KEY_ID, 166 | secretAccessKey: process.env.AWS_SECRET_KEY, 167 | } 168 | 169 | const useLocal = process.env.NODE_ENV !== 'production' 170 | 171 | const bucketName = process.env.AWS_BUCKET_NAME 172 | 173 | const s3client = new AWS.S3({ 174 | credentials, 175 | /** 176 | * When working locally, we'll use the Localstack endpoints. This is the one for S3. 177 | * A full list of endpoints for each service can be found in the Localstack docs. 178 | */ 179 | endpoint: useLocal ? 'http://localhost:4572' : undefined, 180 | }) 181 | 182 | 183 | const uploadFile = async (data, name) => 184 | new Promise((resolve) => { 185 | s3client.upload( 186 | { 187 | Bucket: bucketName, 188 | /* 189 | include the bucket name here. For some reason Localstack needs it. 190 | see: https://github.com/localstack/localstack/issues/1180 191 | */ 192 | Key: `${bucketName}/${name}`, 193 | Body: data, 194 | }, 195 | (err, response) => { 196 | if (err) throw err 197 | resolve(response) 198 | }, 199 | ) 200 | }) 201 | 202 | module.exports = uploadFile 203 | ``` 204 | 205 | **test-upload.js**, which implements the upload function 206 | 207 | ```js 208 | const fs = require('fs') 209 | const path = require('path') 210 | const uploadFile = require('./aws') 211 | 212 | const testUpload = () => { 213 | const filePath = path.resolve(__dirname, 'test-image.jpg') 214 | const fileStream = fs.createReadStream(filePath) 215 | const now = new Date() 216 | const fileName = `test-image-${now.toISOString()}.jpg` 217 | uploadFile(fileStream, fileName).then((response) => { 218 | console.log(":)") 219 | console.log(response) 220 | }).catch((err) => { 221 | console.log(":|") 222 | console.log(err) 223 | }) 224 | } 225 | 226 | testUpload() 227 | ``` 228 | 229 | the `testUpload()` function gets the file contents, gives it a unique name based on the current time, and uploads it. Let's give it a shot: 230 | 231 | `node test-upload.js` 232 | 233 | ![testing the upload](https://thepracticaldev.s3.amazonaws.com/i/yx94y5fj0j8rq4y67foy.png) 234 | 235 | Copy the URL in the `Location` property of the response and paste it into your browser. The browser will immediately download the image. If you want to see it in your browser, you can use something like JS Bin: 236 | 237 | ![I love my dog](https://thepracticaldev.s3.amazonaws.com/i/v1vnvbzo7z0wlabm3o43.png) 238 | 239 | Then, if you look at `.localstack/data/s3_api_calls.json` again, you'll see it filled up with the binary data of the image: 240 | 241 | ![Image binary data](https://thepracticaldev.s3.amazonaws.com/i/qcydypsebmmcb8r855jg.png) 242 | 243 | 244 | **Finally**, let's restart the container to make sure our uploads still work. To do this, run `docker restart localstack_demo`. After it has restarted, run `docker logs -f localstack_demo`. This will show you the logs of the container (the `-f` flag will "follow" them). 245 | 246 | After it initializes Localstack, it will re-apply the API calls found in `s3_api_calls.json`: 247 | 248 | ![Localstack Logs](https://thepracticaldev.s3.amazonaws.com/i/ydjjg14zztr5vvqv9q6s.png) 249 | 250 | When you reload your browser, you should see the image appear just as before. 251 | 252 | 🎉 That's it! Thanks for sticking around. This is my first tutorial and I'd love to know what you think. If you have any questions or suggestions, let me know in the comments! --------------------------------------------------------------------------------