├── .dvc ├── .gitignore ├── config └── plots │ ├── confusion.json │ ├── default.json │ ├── scatter.json │ └── smooth.json ├── .github └── workflows │ └── cml.yaml ├── README.md ├── dvc.lock ├── dvc.yaml ├── final_owl.png ├── imgs ├── georgia_nature_forms.jpg ├── georgia_okeefe.jpg └── owl_photo.jpg ├── metrics.txt ├── neural_style_transfer.py ├── params.yaml ├── requirements.txt └── results └── owl_at_iteration_0.png /.dvc/.gitignore: -------------------------------------------------------------------------------- 1 | /config.local 2 | /tmp 3 | /cache 4 | -------------------------------------------------------------------------------- /.dvc/config: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iterative/cml_cloud_case/bd8923be49b223ce29c6e05649b5979d0618996e/.dvc/config -------------------------------------------------------------------------------- /.dvc/plots/confusion.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://vega.github.io/schema/vega-lite/v4.json", 3 | "data": { 4 | "values": "" 5 | }, 6 | "title": "", 7 | "mark": "rect", 8 | "encoding": { 9 | "x": { 10 | "field": "", 11 | "type": "nominal", 12 | "sort": "ascending", 13 | "title": "" 14 | }, 15 | "y": { 16 | "field": "", 17 | "type": "nominal", 18 | "sort": "ascending", 19 | "title": "" 20 | }, 21 | "color": { 22 | "aggregate": "count", 23 | "type": "quantitative" 24 | }, 25 | "facet": { 26 | "field": "rev", 27 | "type": "nominal" 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /.dvc/plots/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://vega.github.io/schema/vega-lite/v4.json", 3 | "data": { 4 | "values": "" 5 | }, 6 | "title": "", 7 | "mark": { 8 | "type": "line" 9 | }, 10 | "encoding": { 11 | "x": { 12 | "field": "", 13 | "type": "quantitative", 14 | "title": "" 15 | }, 16 | "y": { 17 | "field": "", 18 | "type": "quantitative", 19 | "title": "", 20 | "scale": { 21 | "zero": false 22 | } 23 | }, 24 | "color": { 25 | "field": "rev", 26 | "type": "nominal" 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /.dvc/plots/scatter.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://vega.github.io/schema/vega-lite/v4.json", 3 | "data": { 4 | "values": "" 5 | }, 6 | "title": "", 7 | "mark": "point", 8 | "encoding": { 9 | "x": { 10 | "field": "", 11 | "type": "quantitative", 12 | "title": "" 13 | }, 14 | "y": { 15 | "field": "", 16 | "type": "quantitative", 17 | "title": "", 18 | "scale": { 19 | "zero": false 20 | } 21 | }, 22 | "color": { 23 | "field": "rev", 24 | "type": "nominal" 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /.dvc/plots/smooth.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://vega.github.io/schema/vega-lite/v4.json", 3 | "data": { 4 | "values": "" 5 | }, 6 | "title": "", 7 | "mark": { 8 | "type": "line" 9 | }, 10 | "encoding": { 11 | "x": { 12 | "field": "", 13 | "type": "quantitative", 14 | "title": "" 15 | }, 16 | "y": { 17 | "field": "", 18 | "type": "quantitative", 19 | "title": "", 20 | "scale": { 21 | "zero": false 22 | } 23 | }, 24 | "color": { 25 | "field": "rev", 26 | "type": "nominal" 27 | } 28 | }, 29 | "transform": [ 30 | { 31 | "loess": "", 32 | "on": "", 33 | "groupby": [ 34 | "rev" 35 | ], 36 | "bandwidth": 0.3 37 | } 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /.github/workflows/cml.yaml: -------------------------------------------------------------------------------- 1 | name: train-my-model 2 | 3 | on: [push] 4 | 5 | jobs: 6 | deploy-runner: 7 | runs-on: [ubuntu-latest] 8 | steps: 9 | - uses: actions/checkout@v2 10 | 11 | - uses: iterative/setup-cml@v1 12 | 13 | - name: deploy 14 | shell: bash 15 | env: 16 | repo_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} 17 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 18 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 19 | run: | 20 | cml-runner \ 21 | --cloud aws \ 22 | --cloud-region us-west \ 23 | --cloud-type=g3.4xlarge \ 24 | --cloud-hdd-size 64 \ 25 | --labels=cml-runner 26 | 27 | run: 28 | needs: deploy-runner 29 | runs-on: [self-hosted,cml-runner] 30 | container: 31 | image: docker://iterativeai/cml 32 | options: --gpus all 33 | 34 | steps: 35 | - uses: actions/checkout@v2 36 | 37 | - uses: actions/setup-python@v2 38 | with: 39 | python-version: '3.6' 40 | 41 | - name: cml 42 | env: 43 | repo_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} 44 | run: | 45 | apt-get update -y 46 | apt install imagemagick -y 47 | pip install -r requirements.txt 48 | 49 | # DVC stuff 50 | git fetch --prune 51 | dvc repro 52 | 53 | echo "# Style transfer" >> report.md 54 | git show origin/master:final_owl.png > master_owl.png 55 | convert +append final_owl.png master_owl.png out.png 56 | convert out.png -resize 75% out_shrink.png 57 | echo "### Workspace vs. Main" >> report.md 58 | cml-publish out_shrink.png --md --title 'compare' >> report.md 59 | 60 | echo "## Training metrics" >> report.md 61 | dvc params diff master --show-md >> report.md 62 | 63 | echo >> report.md 64 | echo "## GPU info" >> report.md 65 | cat gpu_info.txt >> report.md 66 | 67 | cml-send-comment report.md 68 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CML with cloud compute 2 | 3 | 4 | This repository contains a sample project using [CML](https://github.com/iterative/cml) with Terraform (via the `cml-runner` function) to launch an AWS EC2 instance and then run a neural style transfer on that instance. On a pull request, the following actions will occur: 5 | - GitHub will deploy a runner with a custom CML Docker image 6 | - `cml-runner` will provision an EC2 instance and pass the neural style transfer workflow to it. DVC is used to version the workflow and dependencies. 7 | - Neural style transfer will be executed on the EC2 instance 8 | - CML will report results of the style transfer as a comment in the pull request. 9 | 10 | The key file enabling these actions is `.github/workflows/cml.yaml`. 11 | 12 | ## Variables 13 | You must create a [personal access token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) with repository and workflow privileges to supply as a secret, in addition to your AWS credentials (`AWS_ACCESS_KEY_ID`,`AWS_SECRET_ACCESS_KEY`, and optionally `AWS_SESSION_TOKEN`). 14 | -------------------------------------------------------------------------------- /dvc.lock: -------------------------------------------------------------------------------- 1 | transfer: 2 | cmd: python neural_style_transfer.py imgs/owl_photo.jpg imgs/georgia_okeefe.jpg 3 | results/owl 4 | deps: 5 | - path: imgs 6 | md5: ae79bcc8a88e962400402d20d4976e89.dir 7 | params: 8 | params.yaml: 9 | content_weight: 0.025 10 | iter: 1 11 | style_weight: 10.0 12 | tv_weight: 1.0 13 | outs: 14 | - path: final_owl.png 15 | md5: 9997e951f45a7459e825216fc17c38ce 16 | -------------------------------------------------------------------------------- /dvc.yaml: -------------------------------------------------------------------------------- 1 | stages: 2 | transfer: 3 | cmd: python neural_style_transfer.py imgs/owl_photo.jpg imgs/georgia_okeefe.jpg 4 | results/owl 5 | deps: 6 | - imgs 7 | params: 8 | - content_weight 9 | - iter 10 | - style_weight 11 | - tv_weight 12 | outs: 13 | - final_owl.png: 14 | cache: false 15 | - gpu_info.txt: 16 | cache: false 17 | 18 | -------------------------------------------------------------------------------- /final_owl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iterative/cml_cloud_case/bd8923be49b223ce29c6e05649b5979d0618996e/final_owl.png -------------------------------------------------------------------------------- /imgs/georgia_nature_forms.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iterative/cml_cloud_case/bd8923be49b223ce29c6e05649b5979d0618996e/imgs/georgia_nature_forms.jpg -------------------------------------------------------------------------------- /imgs/georgia_okeefe.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iterative/cml_cloud_case/bd8923be49b223ce29c6e05649b5979d0618996e/imgs/georgia_okeefe.jpg -------------------------------------------------------------------------------- /imgs/owl_photo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iterative/cml_cloud_case/bd8923be49b223ce29c6e05649b5979d0618996e/imgs/owl_photo.jpg -------------------------------------------------------------------------------- /metrics.txt: -------------------------------------------------------------------------------- 1 | | Metric | Value | 2 | |:-------------|-------------:| 3 | | Iterations | 0 | 4 | | Run time (s) | 49.8537 | 5 | | Final loss | 2.62031e+10 | -------------------------------------------------------------------------------- /neural_style_transfer.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from keras.preprocessing.image import load_img, save_img, img_to_array 3 | import numpy as np 4 | from scipy.optimize import fmin_l_bfgs_b 5 | import time 6 | import argparse 7 | import nvgpu 8 | from keras.applications import vgg19 9 | from keras import backend as K 10 | import pandas as pd 11 | import yaml 12 | import tensorflow as tf 13 | 14 | tf.compat.v1.disable_eager_execution() 15 | 16 | 17 | parser = argparse.ArgumentParser(description='Neural style transfer with Keras.') 18 | parser.add_argument('base_image_path', metavar='base', type=str, 19 | help='Path to the image to transform.') 20 | parser.add_argument('style_reference_image_path', metavar='ref', type=str, 21 | help='Path to the style reference image.') 22 | parser.add_argument('result_prefix', metavar='res_prefix', type=str, 23 | help='Prefix for the saved results.') 24 | 25 | args = parser.parse_args() 26 | base_image_path = args.base_image_path 27 | style_reference_image_path = args.style_reference_image_path 28 | result_prefix = args.result_prefix 29 | 30 | # Model fitting params 31 | with open('params.yaml') as file: 32 | model_args = yaml.load(file, Loader=yaml.FullLoader) 33 | 34 | iterations = model_args['iter'] 35 | # these are the weights of the different loss components 36 | total_variation_weight = model_args['tv_weight'] 37 | style_weight = model_args['style_weight'] 38 | content_weight = model_args['tv_weight'] 39 | 40 | # dimensions of the generated picture. 41 | width, height = load_img(base_image_path).size 42 | img_nrows = 300 43 | img_ncols = int(width * img_nrows / height) 44 | 45 | # util function to open, resize and format pictures into appropriate tensors 46 | 47 | 48 | def preprocess_image(image_path): 49 | img = load_img(image_path, target_size=(img_nrows, img_ncols)) 50 | img = img_to_array(img) 51 | img = np.expand_dims(img, axis=0) 52 | img = vgg19.preprocess_input(img) 53 | return img 54 | 55 | # util function to convert a tensor into a valid image 56 | 57 | 58 | def deprocess_image(x): 59 | if K.image_data_format() == 'channels_first': 60 | x = x.reshape((3, img_nrows, img_ncols)) 61 | x = x.transpose((1, 2, 0)) 62 | else: 63 | x = x.reshape((img_nrows, img_ncols, 3)) 64 | # Remove zero-center by mean pixel 65 | x[:, :, 0] += 103.939 66 | x[:, :, 1] += 116.779 67 | x[:, :, 2] += 123.68 68 | # 'BGR'->'RGB' 69 | x = x[:, :, ::-1] 70 | x = np.clip(x, 0, 255).astype('uint8') 71 | return x 72 | 73 | # get tensor representations of our images 74 | base_image = K.variable(preprocess_image(base_image_path)) 75 | style_reference_image = K.variable(preprocess_image(style_reference_image_path)) 76 | 77 | # this will contain our generated image 78 | if K.image_data_format() == 'channels_first': 79 | combination_image = K.placeholder((1, 3, img_nrows, img_ncols)) 80 | else: 81 | combination_image = K.placeholder((1, img_nrows, img_ncols, 3)) 82 | 83 | # combine the 3 images into a single Keras tensor 84 | input_tensor = K.concatenate([base_image, 85 | style_reference_image, 86 | combination_image], axis=0) 87 | 88 | # build the VGG19 network with our 3 images as input 89 | # the model will be loaded with pre-trained ImageNet weights 90 | model = vgg19.VGG19(input_tensor=input_tensor, 91 | weights='imagenet', include_top=False) 92 | print('Model loaded.') 93 | 94 | # get the symbolic outputs of each "key" layer (we gave them unique names). 95 | outputs_dict = dict([(layer.name, layer.output) for layer in model.layers]) 96 | 97 | # compute the neural style loss 98 | # first we need to define 4 util functions 99 | 100 | # the gram matrix of an image tensor (feature-wise outer product) 101 | 102 | 103 | def gram_matrix(x): 104 | assert K.ndim(x) == 3 105 | if K.image_data_format() == 'channels_first': 106 | features = K.batch_flatten(x) 107 | else: 108 | features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1))) 109 | gram = K.dot(features, K.transpose(features)) 110 | return gram 111 | 112 | # the "style loss" is designed to maintain 113 | # the style of the reference image in the generated image. 114 | # It is based on the gram matrices (which capture style) of 115 | # feature maps from the style reference image 116 | # and from the generated image 117 | 118 | 119 | def style_loss(style, combination): 120 | assert K.ndim(style) == 3 121 | assert K.ndim(combination) == 3 122 | S = gram_matrix(style) 123 | C = gram_matrix(combination) 124 | channels = 3 125 | size = img_nrows * img_ncols 126 | return K.sum(K.square(S - C)) / (4.0 * (channels ** 2) * (size ** 2)) 127 | 128 | # an auxiliary loss function 129 | # designed to maintain the "content" of the 130 | # base image in the generated image 131 | 132 | 133 | def content_loss(base, combination): 134 | return K.sum(K.square(combination - base)) 135 | 136 | # the 3rd loss function, total variation loss, 137 | # designed to keep the generated image locally coherent 138 | 139 | 140 | def total_variation_loss(x): 141 | assert K.ndim(x) == 4 142 | if K.image_data_format() == 'channels_first': 143 | a = K.square( 144 | x[:, :, :img_nrows - 1, :img_ncols - 1] - x[:, :, 1:, :img_ncols - 1]) 145 | b = K.square( 146 | x[:, :, :img_nrows - 1, :img_ncols - 1] - x[:, :, :img_nrows - 1, 1:]) 147 | else: 148 | a = K.square( 149 | x[:, :img_nrows - 1, :img_ncols - 1, :] - x[:, 1:, :img_ncols - 1, :]) 150 | b = K.square( 151 | x[:, :img_nrows - 1, :img_ncols - 1, :] - x[:, :img_nrows - 1, 1:, :]) 152 | return K.sum(K.pow(a + b, 1.25)) 153 | 154 | 155 | # combine these loss functions into a single scalar 156 | loss = K.variable(0.0) 157 | layer_features = outputs_dict['block5_conv2'] 158 | base_image_features = layer_features[0, :, :, :] 159 | combination_features = layer_features[2, :, :, :] 160 | loss = loss + content_weight * content_loss(base_image_features, 161 | combination_features) 162 | 163 | feature_layers = ['block1_conv1', 'block2_conv1', 164 | 'block3_conv1', 'block4_conv1', 165 | 'block5_conv1'] 166 | for layer_name in feature_layers: 167 | layer_features = outputs_dict[layer_name] 168 | style_reference_features = layer_features[1, :, :, :] 169 | combination_features = layer_features[2, :, :, :] 170 | sl = style_loss(style_reference_features, combination_features) 171 | loss = loss + (style_weight / len(feature_layers)) * sl 172 | loss = loss + total_variation_weight * total_variation_loss(combination_image) 173 | 174 | # get the gradients of the generated image wrt the loss 175 | grads = K.gradients(loss, combination_image) 176 | 177 | outputs = [loss] 178 | if isinstance(grads, (list, tuple)): 179 | outputs += grads 180 | else: 181 | outputs.append(grads) 182 | 183 | f_outputs = K.function([combination_image], outputs) 184 | 185 | 186 | def eval_loss_and_grads(x): 187 | if K.image_data_format() == 'channels_first': 188 | x = x.reshape((1, 3, img_nrows, img_ncols)) 189 | else: 190 | x = x.reshape((1, img_nrows, img_ncols, 3)) 191 | outs = f_outputs([x]) 192 | loss_value = outs[0] 193 | if len(outs[1:]) == 1: 194 | grad_values = outs[1].flatten().astype('float64') 195 | else: 196 | grad_values = np.array(outs[1:]).flatten().astype('float64') 197 | return loss_value, grad_values 198 | 199 | # this Evaluator class makes it possible 200 | # to compute loss and gradients in one pass 201 | # while retrieving them via two separate functions, 202 | # "loss" and "grads". This is done because scipy.optimize 203 | # requires separate functions for loss and gradients, 204 | # but computing them separately would be inefficient. 205 | 206 | 207 | class Evaluator(object): 208 | 209 | def __init__(self): 210 | self.loss_value = None 211 | self.grads_values = None 212 | 213 | def loss(self, x): 214 | assert self.loss_value is None 215 | loss_value, grad_values = eval_loss_and_grads(x) 216 | self.loss_value = loss_value 217 | self.grad_values = grad_values 218 | return self.loss_value 219 | 220 | def grads(self, x): 221 | assert self.loss_value is not None 222 | grad_values = np.copy(self.grad_values) 223 | self.loss_value = None 224 | self.grad_values = None 225 | return grad_values 226 | 227 | 228 | evaluator = Evaluator() 229 | 230 | # run scipy-based optimization (L-BFGS) over the pixels of the generated image 231 | # so as to minimize the neural style loss 232 | x = preprocess_image(base_image_path) 233 | 234 | train_start = time.time() 235 | for i in range(iterations): 236 | print('Start of iteration', i) 237 | start_time = time.time() 238 | x, min_val, info = fmin_l_bfgs_b(evaluator.loss, x.flatten(), 239 | fprime=evaluator.grads, maxfun=20) 240 | print('Current loss value:', min_val) 241 | # save current generated image 242 | if i % 10 == 0: 243 | img = deprocess_image(x.copy()) 244 | fname = result_prefix + '_at_iteration_%d.png' % i 245 | save_img(fname, img) 246 | print('Image saved as', fname) 247 | end_time = time.time() 248 | print('Iteration %d completed in %ds' % (i, end_time - start_time)) 249 | 250 | train_duration = end_time - train_start 251 | # Save a final image 252 | save_img("final_owl.png", img) 253 | 254 | 255 | df = pd.DataFrame(data = {"Value":[i,train_duration,min_val]}, index=["Iterations","Run time (s)","Final loss"]) 256 | df.index.names= ['Metric'] 257 | with open("metrics.txt", "w") as outfile: 258 | outfile.write(df.to_markdown()) 259 | 260 | # Make a table of GPU info 261 | g = nvgpu.gpu_info() 262 | df = pd.DataFrame.from_dict(g[0], orient="index", columns=["Value"]) 263 | with open("gpu_info.txt", "w") as outfile: 264 | outfile.write(df.to_markdown()) 265 | -------------------------------------------------------------------------------- /params.yaml: -------------------------------------------------------------------------------- 1 | iter: 1 2 | content_weight: 0.025 3 | style_weight: 100.0 4 | tv_weight: 10.0 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow-gpu==2.2.2 2 | keras 3 | pillow 4 | pandas 5 | tabulate 6 | nvgpu 7 | dvc 8 | -------------------------------------------------------------------------------- /results/owl_at_iteration_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iterative/cml_cloud_case/bd8923be49b223ce29c6e05649b5979d0618996e/results/owl_at_iteration_0.png --------------------------------------------------------------------------------