├── .gitignore
├── README.md
├── app
├── README.md
├── app.py
├── components
│ ├── app.js
│ ├── brush_size_slider.js
│ ├── canvas.js
│ ├── display_size_nag_bar.js
│ ├── download_button.js
│ ├── fancy_button.js
│ ├── output_picture.js
│ ├── purecanvas.js
│ ├── speech_bubble.js
│ ├── toolbox.js
│ ├── toolbox_label_button.js
│ ├── toolbox_tool_button.js
│ └── tweet_button.js
├── index.html
├── models
│ ├── 102.yaml
│ ├── 62.yaml
│ └── 79.yaml
├── package-lock.json
├── package.json
├── requirements.txt
├── robots.txt
├── scripts
│ ├── upload_to_gh.sh
│ └── upload_to_s3.sh
└── static
│ ├── fonts.css
│ ├── img
│ ├── default.png
│ ├── example.png
│ ├── favicon.png
│ ├── favicon.svg
│ └── social.jpg
│ ├── index.css
│ └── index.js
├── imgs
├── model-lineage.png
├── model-lineage.svg
└── screenshot.png
├── lib
├── .gitignore
├── README.md
├── SPADE-master
│ ├── .gitignore
│ ├── LICENSE.md
│ ├── README.md
│ ├── data
│ │ ├── __init__.py
│ │ ├── ade20k_dataset.py
│ │ ├── base_dataset.py
│ │ ├── cityscapes_dataset.py
│ │ ├── coco_dataset.py
│ │ ├── custom_dataset.py
│ │ ├── facades_dataset.py
│ │ ├── image_folder.py
│ │ └── pix2pix_dataset.py
│ ├── datasets
│ │ └── coco_generate_instance_map.py
│ ├── models
│ │ ├── __init__.py
│ │ ├── networks
│ │ │ ├── __init__.py
│ │ │ ├── architecture.py
│ │ │ ├── base_network.py
│ │ │ ├── discriminator.py
│ │ │ ├── encoder.py
│ │ │ ├── generator.py
│ │ │ ├── loss.py
│ │ │ ├── normalization.py
│ │ │ └── sync_batchnorm
│ │ │ │ ├── __init__.py
│ │ │ │ ├── batchnorm.py
│ │ │ │ ├── batchnorm_reimpl.py
│ │ │ │ ├── comm.py
│ │ │ │ ├── replicate.py
│ │ │ │ └── unittest.py
│ │ └── pix2pix_model.py
│ ├── options
│ │ ├── __init__.py
│ │ ├── base_options.py
│ │ ├── test_options.py
│ │ └── train_options.py
│ ├── requirements.txt
│ ├── test.py
│ ├── train.py
│ ├── trainers
│ │ ├── __init__.py
│ │ └── pix2pix_trainer.py
│ └── util
│ │ ├── __init__.py
│ │ ├── coco.py
│ │ ├── html.py
│ │ ├── iter_counter.py
│ │ ├── util.py
│ │ └── visualizer.py
└── model_server
│ ├── latest_net_G.yaml
│ ├── requirements.txt
│ ├── scripts
│ └── load_test.py
│ ├── server.py
│ └── test_payload.json
├── models
├── README.md
├── model_0.py
├── model_1.py
├── model_10.py
├── model_11.py
├── model_12.py
├── model_13.py
├── model_14.py
├── model_2.py
├── model_3.py
├── model_4.py
├── model_5.py
├── model_6.py
├── model_7.py
├── model_8.py
└── model_9.py
└── requirements.txt
/.gitignore:
--------------------------------------------------------------------------------
1 | lib/SPADE-master/datasets/coco_stuff/
2 | lib/SPADE-master/docs/
3 | .ipynb_checkpoints
4 | .DS_Store
5 | .vscode
6 | *.pyc
7 | __pycache__/
8 | node_modules/
9 | dist/
10 | checkpoints/
11 | temp
12 | *.pt
13 | *.egg-info/
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # paint-with-ml
2 |
3 | 
4 |
5 | This app uses a version of the GauGAN deep learning model to transform an input segmentation masks into a landscape painting.
6 |
7 | GauGAN, released in 2019, is the most powerful image-to-image translation algorithm currently known. This version of the model has been trained on the popular ADE20K dataset, then fine-tuned on a dataset of 250 paintings from Bob Ross's beloved PBS series, "The Joy of Painting".
8 |
9 | Choose from nine different semantic brushes to craft your painting. Then click on the "Run" button to generate a result! Here are some example outputs:
10 |
11 | 
12 |
13 | Try it out yourself by visiting [paintwith.spell.ml](http://paintwith.spell.ml/). To learn more about how this model was built and served, check out [the accompanying article](https://spell.ml/blog/paint-with-machine-learning-X10i3BAAACQAUI_o) on the Spell blog.
14 |
15 | ## Project hierarchy
16 |
17 | ```
18 | ├── LICENSE
19 | ├── README.md <- You're reading it!
20 | ├── lib/ <- Vendored model code (Python).
21 | │ ├── SPADE-master/ <- Vendored copy of NVlabs/SPADE.* **
22 | │ └── model_server/ <- Model server code (uses Spell model serving).
23 | ├── models <- Model assets
24 | │ ├── model_0.py <- The build script for the first model trained.
25 | │ ├── ...
26 | │ ├── model_N.py <- The build script for the last model trained.
27 | │ └── README.md <- Reference to the models builds.
28 | ├── notebooks <- Jupyter notebooks discussing the model build process.
29 | ├── requirements.txt <- Project environment requirements, installable with pip.
30 | ├── app/ <- User-facing demo React web app.
31 | │ ├── README.md <- Reference on how to build and serve the web app.
32 | │ ├── index.html
33 | │ ├── components/ <- React components.
34 | │ ├── models/ <- Model configs (see model_server/).
35 | │ ├── static/
36 | │ ├── requirements.txt <- Web app Python requirements, installable with pip.
37 | | └── package.json <- Web app JS requirements, installable with npm.
38 | ├── Dockerfile <- Dockerfile bundling the web application.
39 | └── .gitignore
40 |
41 | * Also contains a copy of vacancy/Synchronized-BatchNorm-PyTorch, a NVlabs/SPADE requirement
42 | ** Code has minor modifications made for compatibility with the Jupyter environment
43 | ```
44 |
--------------------------------------------------------------------------------
/app/README.md:
--------------------------------------------------------------------------------
1 | # app
2 |
3 | This folder defines our consumer-facing web application.
4 |
5 | The application frontend is in React. The core of the application is a drawable HTML Canvas with click-based interaction logic. The application backend is a [model server](https://spell.ml/docs/model_servers) running on Spell.
6 |
7 | ## Deployment
8 |
9 | ### Local
10 |
11 | 1. Run `npm install` to install the JS packages.
12 | 2. Run `npm run-script build` to build the app JS assets.
13 | 3. Run `pip install -r requirements.txt` (preferably in a `virtualenv` or `conda` environment) to install the Python packages.
14 | 4. Export the Flask environment variables (you may set `FLASK_ENV=production` instead, if you are so inclined):
15 |
16 | ```bash
17 | export FLASK_APP=app.py
18 | export FLASK_ENV=development
19 | ```
20 | 5. Start the web service using `flask run --no-reload`.
21 |
22 | ### Remote
23 |
24 | We're hosting this site statically on AWS S3. See [Enabling website hosting](https://docs.aws.amazon.com/AmazonS3/latest/dev/EnableWebsiteHosting.html) in the AWS documentation for details, and `app/scripts/upload_to_s3.sh` for the deploy script.
25 |
--------------------------------------------------------------------------------
/app/app.py:
--------------------------------------------------------------------------------
1 | import os
2 | import requests
3 |
4 | from flask import Flask, send_file
5 | from flask import request
6 | app = Flask('neural-painter', static_folder='static')
7 |
8 | @app.route('/', methods=['GET'])
9 | def index():
10 | return send_file('index.html')
11 |
12 | print('App ready!')
13 |
--------------------------------------------------------------------------------
/app/components/brush_size_slider.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import { Component } from 'react';
3 |
4 |
5 | class BrushSizeSlider extends Component {
6 | render() {
7 | return
8 | this.props.onChange(e.target.value)}
12 | />
13 |
14 | }
15 | }
16 |
17 | export default BrushSizeSlider;
--------------------------------------------------------------------------------
/app/components/display_size_nag_bar.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import { Component } from 'react';
3 |
4 |
5 | class DisplaySizeNagBar extends Component {
6 | render() {
7 | return
8 |
9 | This app is optimized for large screens and may not work in your current browser!
10 |
11 |
46 |
47 | {this.getButtonImage()}
48 |
49 |
50 | {this.props.buttonFunction}
51 |
52 |
12 | }
13 | }
14 |
15 | export default OutputPicture;
--------------------------------------------------------------------------------
/app/components/purecanvas.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import { Component } from 'react';
3 |
4 |
5 | class PureCanvas extends Component {
6 | shouldComponentUpdate() {
7 | return false;
8 | }
9 |
10 | render() {
11 | return (
12 |