└── dashboard-api ├── LICENSE ├── README.md ├── app.py ├── dashboard-api-dev-1572477459.zip ├── dashboard-api-dev-template-1571105490.json ├── deploy.py ├── dynamodb.py ├── output.txt ├── package-lock.json ├── package.json ├── postsetup.js ├── requirements.txt ├── serverless.yml ├── test.py └── zappa_settings.json /dashboard-api/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Alex DeBrie 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /dashboard-api/README.md: -------------------------------------------------------------------------------- 1 | # Dashboard API 2 | 3 | The Dashboard API is a RESTful and serverless service used to create and manage DataAgora Repos. 4 | 5 | ## Set up 6 | 7 | Before anything you need to install the [serverless](https://github.com/serverless/serverless) package by running: 8 | 9 | ``` 10 | npm install -g serverless 11 | ``` 12 | 13 | ## Running locally 14 | 15 | To run a server locally, you first need to install your Python dependencies: 16 | 17 | ``` 18 | pip install -r requirements.txt 19 | ``` 20 | 21 | Then, you can just run the following command to get the server running: 22 | 23 | ``` 24 | sls wsgi serve 25 | ``` 26 | 27 | ## Deploying 28 | 29 | To deploy, just run the following command: 30 | 31 | ``` 32 | sls deploy 33 | ``` 34 | 35 | Note that you need AWS properly configured in your machine for this to work. 36 | 37 | 38 | ## Tests 39 | 40 | There aren't any tests available for this repo yet. Please help us write them! 41 | -------------------------------------------------------------------------------- /dashboard-api/app.py: -------------------------------------------------------------------------------- 1 | import re 2 | import time 3 | import secrets 4 | import decimal 5 | import hashlib 6 | 7 | import jwt 8 | import boto3 9 | import requests 10 | from flask_cors import CORS 11 | from boto3.dynamodb.conditions import Key, Attr 12 | from flask import Flask, request, jsonify 13 | 14 | from deploy import run_deploy_routine, run_delete_routine 15 | from dynamodb import DB 16 | 17 | 18 | JWT_SECRET = "datajbsnmd5h84rbewvzx6*cax^jgmqw@m3$ds_%z-4*qy0n44fjr5shark" 19 | JWT_ALGO = "HS256" 20 | 21 | APPLICATION_NAME = "cloud-node" 22 | app = Flask(__name__) 23 | db = DB() 24 | CORS(app) 25 | 26 | @app.route("/") 27 | def home(): 28 | return "This is the dashboard api homepage!" 29 | 30 | @app.route('/reset_state/', methods=['POST', 'GET']) 31 | def reset_state(repo_id): 32 | """ 33 | Authorize request, then reset state for cloud node corresponding to given repo_id 34 | """ 35 | claims = authorize_user(request, use_header=False) 36 | if claims is None: return jsonify(make_unauthorized_error()), 400 37 | try: 38 | response = requests.get(CLOUD_BASE_ENDPOINT.format(repo_id)) 39 | except Exception as e: 40 | print("Error: " + str(e)) 41 | return jsonify(make_error(str(e))) 42 | 43 | return jsonify(make_success(response.text)) 44 | 45 | @app.route('/get_username/', methods=['POST', 'GET']) 46 | def get_username(repo_id): 47 | """ 48 | Authorize request, then retrieve username for given repo_id 49 | """ 50 | claims = authorize_user(request, use_header=False) 51 | if claims is None: return jsonify(make_unauthorized_error()), 400 52 | user_id = claims["pk"] 53 | try: 54 | username = db.get_username(user_id, repo_id) 55 | except Exception as e: 56 | print("Error: " + str(e)) 57 | return jsonify(make_error(str(e))) 58 | 59 | return jsonify(make_success(username)) 60 | 61 | @app.route("/userdata", methods=["GET"]) 62 | def get_user_data(): 63 | """ 64 | Returns the authenticated user's data. 65 | """ 66 | # Check authorization 67 | claims = authorize_user(request) 68 | if claims is None: return jsonify(make_unauthorized_error()), 400 69 | 70 | # Get data 71 | user_id = claims["pk"] 72 | try: 73 | user_data = _get_user_data(user_id) 74 | repos_remaining = user_data['ReposRemaining'] 75 | except: 76 | return jsonify(make_error("Error while getting user's permissions.")), 400 77 | return jsonify({"ReposRemaining": True if repos_remaining > 0 else False}) 78 | 79 | @app.route("/repo/", methods=["GET"]) 80 | def get_repo(repo_id): 81 | """ 82 | Returns a Repo's information (if the user has access to it). 83 | """ 84 | # Check authorization 85 | claims = authorize_user(request) 86 | if claims is None: return jsonify(make_unauthorized_error()), 400 87 | 88 | # Get data 89 | user_id = claims["pk"] 90 | try: 91 | repo_details = _get_repo_details(user_id, repo_id) 92 | except: 93 | return jsonify(make_error("Error while getting the details for this repo.")), 400 94 | return jsonify(repo_details) 95 | 96 | @app.route("/repo", methods=["POST"]) 97 | def create_new_repo(): 98 | """ 99 | Creates a new repo under the authenticated user. 100 | 101 | Example HTTP POST Request body (JSON format): 102 | { 103 | "RepoName": "repo_name", 104 | "RepoDescription": "Some description here." 105 | } 106 | """ 107 | # Check authorization 108 | claims = authorize_user(request) 109 | if claims is None: return jsonify(make_unauthorized_error()), 400 110 | 111 | # Get parameters 112 | # TODO: Sanitize inputs. 113 | params = request.get_json() 114 | if "RepoName" not in params: 115 | return jsonify(make_error("Missing repo name from request.")), 400 116 | if "RepoDescription" not in params: 117 | return jsonify(make_error("Missing repo description from request.")), 400 118 | repo_name = params["RepoName"][:20] 119 | repo_description = params["RepoDescription"][:80] 120 | 121 | # TODO: Check repo doesn't already exist. 122 | 123 | user_id = claims["pk"] 124 | repo_name = re.sub('[^a-zA-Z0-9-]', '-', repo_name) 125 | try: 126 | _assert_user_has_repos_left(user_id) 127 | repo_id = _create_new_repo_document(user_id, repo_name, repo_description) 128 | api_key, true_api_key = _create_new_api_key(user_id, repo_id) 129 | _update_user_data_with_new_repo(user_id, repo_id, api_key) 130 | _create_new_cloud_node(repo_id, api_key) 131 | except Exception as e: 132 | # TODO: Revert things. 133 | return jsonify(make_error(str(e))), 200 134 | 135 | return jsonify({ 136 | "Error": False, 137 | "Results": { 138 | "RepoId": repo_id, 139 | "TrueApiKey": true_api_key 140 | } 141 | }) 142 | 143 | @app.route("/delete/", methods=["POST"]) 144 | def delete_repo(repo_id): 145 | """ 146 | Deletes a repo under the authenticated user. 147 | """ 148 | # Check authorization 149 | claims = authorize_user(request, use_header=False) 150 | if not claims: return jsonify(make_unauthorized_error()), 200 151 | 152 | user_id = claims["pk"] 153 | try: 154 | run_delete_routine(repo_id) 155 | api_key = _remove_api_key(user_id, repo_id) 156 | _remove_logs(repo_id) 157 | _remove_repo_from_user_details(user_id, repo_id, api_key) 158 | _remove_creds_from_repo(user_id, repo_id) 159 | _remove_repo_details(user_id, repo_id) 160 | except Exception as e: 161 | # TODO: Revert things. 162 | return jsonify(make_error(str(e))), 200 163 | 164 | return jsonify({ 165 | "Error": False, 166 | }) 167 | 168 | @app.route("/repos", methods=["GET"]) 169 | def get_all_repos(): 170 | """ 171 | Returns all repos the user has access to. 172 | """ 173 | # Check authorization 174 | claims = authorize_user(request) 175 | if claims is None: return jsonify(make_unauthorized_error()), 200 176 | 177 | # Get data 178 | user_id = claims["pk"] 179 | try: 180 | repo_list = _get_all_repos(user_id) 181 | except: 182 | return jsonify(make_error("Error while getting list of repos.")), 200 183 | return jsonify(repo_list) 184 | 185 | @app.route("/logs/", methods=["GET"]) 186 | def get_logs(repo_id): 187 | """ 188 | Returns all logs for a Repo the user has access to. 189 | """ 190 | # Check authorization 191 | claims = authorize_user(request) 192 | if claims is None: return jsonify(make_unauthorized_error()), 400 193 | 194 | # Get data 195 | user_id = claims["pk"] 196 | try: 197 | _assert_user_can_read_repo(user_id, repo_id) 198 | logs = _get_logs(repo_id) 199 | except Exception as e: 200 | return jsonify(make_error(str(e))), 400 201 | return jsonify(logs) 202 | 203 | @app.route("/coordinator/status/", methods=["GET"]) 204 | def get_coordinator_status(repo_id): 205 | """ 206 | Returns the status of the Cloud Node associated to a Repo the user has 207 | access to. 208 | """ 209 | # Check authorization 210 | claims = authorize_user(request) 211 | if claims is None: return jsonify(make_unauthorized_error()), 400 212 | 213 | # Get data 214 | user_id = claims["pk"] 215 | try: 216 | repo_details = _get_repo_details(user_id, repo_id) 217 | cloud_node_url = repo_details['CoordinatorAddress'] 218 | # TODO: Remove the 'http' here and add 'https' to the URL constructor. 219 | r = requests.get("http://" + cloud_node_url + "/status") 220 | status_data = r.json() 221 | assert "Busy" in status_data 222 | except Exception as e: 223 | return jsonify(make_error("Error while checking coordinator's status.")), 400 224 | return jsonify(status_data) 225 | 226 | @app.route("/model", methods=["POST"]) 227 | def download_model(): 228 | """ 229 | Returns the download url of a model. 230 | 231 | Example HTTP POST Request body (JSON format): 232 | { 233 | "RepoId": "test", 234 | "SessionId": "c257efa4-791d-4130-bdc6-b0d3cee5fa25", 235 | "Round": 5 236 | } 237 | """ 238 | # Check authorization 239 | claims = authorize_user(request) 240 | if claims is None: return jsonify(make_unauthorized_error()), 400 241 | 242 | # Get parameters 243 | params = request.get_json() 244 | if "RepoId" not in params: 245 | return jsonify(make_error("Missing repo id from request.")), 400 246 | if "SessionId" not in params: 247 | return jsonify(make_error("Missing session id from request.")), 400 248 | if "Round" not in params: 249 | return jsonify(make_error("Missing round from request.")), 400 250 | repo_id = params.get('RepoId', None) 251 | session_id = params.get('SessionId', None) 252 | round = params.get('Round', None) 253 | bucket_name = "updatestore" 254 | object_name = "{0}/{1}/{2}/model.h5".format(repo_id, session_id, round) 255 | 256 | # Get presigned URL 257 | user_id = claims["pk"] 258 | try: 259 | _assert_user_can_read_repo(user_id, repo_id) 260 | url = _create_presigned_url(bucket_name, object_name) 261 | except Exception as e: 262 | return jsonify(make_error(str(e))), 400 263 | 264 | # Return url 265 | return jsonify({'DownloadUrl': url}) 266 | 267 | 268 | # NOTE: This function was added in the auth-enterprise app instead. 269 | # Feel free to remove if you'd like. 270 | # @app.route("/userdata", methods=["POST"]) 271 | # def create_user_data(): 272 | # # Check authorization 273 | # claims = authorize_user(request) 274 | # if claims is None: return jsonify(make_unauthorized_error()), 400 275 | # 276 | # # Create document 277 | # user_id = claims["pk"] 278 | # try: 279 | # _create_user_data(user_id) 280 | # except Exception as e: 281 | # return jsonify(make_error(str(e))), 400 282 | # return jsonify({}) 283 | # 284 | # def _create_user_data(user_id): 285 | # """Only creates it if doesn't exist already.""" 286 | # table = _get_dynamodb_table("UsersDashboardData") 287 | # try: 288 | # item = { 289 | # 'UserId': user_id, 290 | # 'ReposManaged': set(["null"]), 291 | # 'ApiKeys': set(["null"]), 292 | # 'ReposRemaining': 5, 293 | # } 294 | # table.put_item( 295 | # Item=item, 296 | # ConditionExpression="attribute_not_exists(UserId)" 297 | # ) 298 | # except: 299 | # raise Exception("Error while creating the user data.") 300 | 301 | def _assert_user_has_repos_left(user_id): 302 | """ 303 | Asserts that the user has any repos available to create. 304 | """ 305 | user_data = _get_user_data(user_id) 306 | assert int(user_data['ReposRemaining']) > 0, "You don't have any repos left." 307 | 308 | def _assert_user_can_read_repo(user_id, repo_id): 309 | """ 310 | Asserts the user can read a particular repo. 311 | """ 312 | try: 313 | user_data = _get_user_data(user_id) 314 | repos_managed = user_data['ReposManaged'] 315 | except: 316 | raise Exception("Error while getting user's permissions.") 317 | assert repo_id in repos_managed, "You don't have permissions for this repo." 318 | 319 | def _get_logs(repo_id): 320 | """ 321 | Returns the logs for a repo. 322 | """ 323 | logs_table = _get_dynamodb_table("UpdateStore") 324 | try: 325 | response = logs_table.query( 326 | KeyConditionExpression=Key('RepoId').eq(repo_id) 327 | ) 328 | logs = response["Items"] 329 | except Exception as e: 330 | raise Exception("Error while getting logs for repo. " + str(e)) 331 | return logs 332 | 333 | def _remove_logs(repo_id): 334 | """ 335 | Removes the logs for a repo. 336 | """ 337 | logs_table = _get_dynamodb_table("UpdateStore") 338 | try: 339 | response = logs_table.query( 340 | KeyConditionExpression=Key('RepoId').eq(repo_id) 341 | ) 342 | with logs_table.batch_writer() as batch: 343 | for item in response["Items"]: 344 | batch.delete_item( 345 | Key={ 346 | 'RepoId': item['RepoId'], 347 | 'Timestamp': item['Timestamp'] 348 | } 349 | ) 350 | print("Successfully deleted logs!") 351 | return True 352 | except Exception as e: 353 | raise Exception("Error while getting logs for repo. " + str(e)) 354 | 355 | def _get_user_data(user_id): 356 | """ 357 | Returns the user's data. 358 | """ 359 | table = _get_dynamodb_table("UsersDashboardData") 360 | try: 361 | response = table.get_item( 362 | Key={ 363 | "UserId": user_id, 364 | } 365 | ) 366 | data = response["Item"] 367 | except: 368 | raise Exception("Error while getting user dashboard data.") 369 | return data 370 | 371 | def _remove_repo_from_user_details(user_id, repo_id, api_key): 372 | """ 373 | Upon removing a repo, remove the repo from the user's details. 374 | """ 375 | table = _get_dynamodb_table("UsersDashboardData") 376 | try: 377 | response = table.get_item( 378 | Key={ 379 | "UserId": user_id, 380 | } 381 | ) 382 | data = response["Item"] 383 | repos_managed = data['ReposManaged'] 384 | api_keys = data['ApiKeys'] 385 | if repo_id in repos_managed and api_key in api_keys: 386 | repos_managed.remove(repo_id) 387 | api_keys.remove(api_key) 388 | if len(repos_managed) == 0: 389 | data.pop('ReposManaged') 390 | data.pop('ApiKeys') 391 | data['ReposRemaining'] += 1 392 | table.put_item( 393 | Item=data 394 | ) 395 | print("Successfully removed repo_id from user details!") 396 | return True 397 | else: 398 | raise Exception("Could not find corresponding API key or repo id!") 399 | except Exception as e: 400 | raise Exception("Error while removing user dashboard data: " + str(e)) 401 | 402 | 403 | def _get_repo_details(user_id, repo_id): 404 | """ 405 | Returns a repo's details. 406 | """ 407 | repos_table = _get_dynamodb_table("Repos") 408 | try: 409 | response = repos_table.get_item( 410 | Key={ 411 | "Id": repo_id, 412 | "OwnerId": user_id, 413 | } 414 | ) 415 | repo_details = response["Item"] 416 | except: 417 | raise Exception("Error while getting repo details.") 418 | return repo_details 419 | 420 | def _remove_repo_details(user_id, repo_id): 421 | """ 422 | Removes a repo's details. 423 | """ 424 | repos_table = _get_dynamodb_table("Repos") 425 | try: 426 | response = repos_table.delete_item( 427 | Key={ 428 | "Id": repo_id, 429 | "OwnerId": user_id, 430 | } 431 | ) 432 | print("Removed repo details!") 433 | return True 434 | except: 435 | raise Exception("Error while getting repo details.") 436 | 437 | def _remove_creds_from_repo(user_id, repo_id): 438 | repos_table = _get_dynamodb_table("Repos") 439 | users_table = _get_dynamodb_table('jupyterhub-users') 440 | try: 441 | response = repos_table.get_item( 442 | Key={ 443 | "Id": repo_id, 444 | "OwnerId": user_id, 445 | } 446 | ) 447 | repo_details = response["Item"] 448 | username = repo_details['creds'] 449 | repos_table.update_item( 450 | Key={ 451 | "Id": repo_id, 452 | "OwnerId": user_id, 453 | }, 454 | UpdateExpression='SET creds = :val1', 455 | ExpressionAttributeValues={ 456 | ':val1': 'N/A', 457 | } 458 | ) 459 | users_table.update_item( 460 | Key = { 461 | 'username': username 462 | }, 463 | UpdateExpression='SET in_use = :val1', 464 | ExpressionAttributeValues={ 465 | ':val1': False 466 | } 467 | ) 468 | except: 469 | raise Exception("Error while removing repo creds.") 470 | 471 | 472 | def _update_user_data_with_new_repo(user_id, repo_id, api_key): 473 | """ 474 | Updates a user with a new repo and its metadata. 475 | """ 476 | table = _get_dynamodb_table("UsersDashboardData") 477 | try: 478 | response = table.get_item( 479 | Key={ 480 | "UserId": user_id, 481 | } 482 | ) 483 | data = response["Item"] 484 | 485 | repos_managed = data.get('ReposManaged', set([])) 486 | api_keys = data.get('ApiKeys', set([])) 487 | 488 | repos_managed.add(repo_id) 489 | api_keys.add(api_key) 490 | 491 | response = table.update_item( 492 | Key={ 493 | 'UserId': user_id, 494 | }, 495 | UpdateExpression="SET ReposRemaining = ReposRemaining - :val, ReposManaged = :val2, ApiKeys = :val3", 496 | ExpressionAttributeValues={ 497 | ':val': decimal.Decimal(1), 498 | ':val2': repos_managed, 499 | ':val3': api_keys, 500 | } 501 | ) 502 | except Exception as e: 503 | raise Exception("Error while updating user data with new repo data: " + str(e)) 504 | 505 | def _create_new_repo_document(user_id, repo_name, repo_description): 506 | """ 507 | Creates a new repo document in the DB. 508 | """ 509 | table = _get_dynamodb_table("Repos") 510 | repo_id = secrets.token_hex(16) 511 | try: 512 | item = { 513 | 'Id': repo_id, 514 | 'Name': repo_name, 515 | 'Description': repo_description, 516 | 'OwnerId': user_id, 517 | 'ContributorsId': [], 518 | 'CoordinatorAddress': _construct_cloud_node_url(repo_id), 519 | 'CreatedAt': int(time.time()), 520 | 'creds': 'N/A' 521 | # 'ExploratoryData': None, 522 | } 523 | table.put_item(Item=item) 524 | except: 525 | raise Exception("Error while creating the new repo document.") 526 | return repo_id 527 | 528 | def _create_new_api_key(user_id, repo_id): 529 | """ 530 | Creates a new API Key for an user and repo. 531 | """ 532 | table = _get_dynamodb_table("ApiKeys") 533 | true_api_key = secrets.token_urlsafe(32) 534 | h = hashlib.sha256() 535 | h.update(true_api_key.encode('utf-8')) 536 | api_key = h.hexdigest() 537 | try: 538 | item = { 539 | 'Key': api_key, 540 | 'OwnerId': user_id, 541 | 'RepoId': repo_id, 542 | 'CreatedAt': int(time.time()), 543 | } 544 | table.put_item(Item=item) 545 | except: 546 | raise Exception("Error while creating a new API key.") 547 | return api_key, true_api_key 548 | 549 | def _remove_api_key(user_id, repo_id): 550 | table = _get_dynamodb_table("ApiKeys") 551 | try: 552 | response = table.scan( 553 | FilterExpression=Attr('RepoId').eq(repo_id) 554 | ) 555 | items = response['Items'] 556 | assert len(items) > 0, "Should have found repo in API keys table!" 557 | item = items[0] 558 | api_key = item['Key'] 559 | table.delete_item( 560 | Key={ 561 | 'Key': item['Key'], 562 | 'OwnerId': item['OwnerId'] 563 | } 564 | ) 565 | return api_key 566 | except: 567 | raise Exception("Error while deleting API key.") 568 | 569 | 570 | def _get_all_repos(user_id): 571 | """ 572 | Returns all repos for a user. 573 | """ 574 | try: 575 | user_data = _get_user_data(user_id) 576 | repos_managed = user_data.get('ReposManaged', 'None') 577 | 578 | repos_table = _get_dynamodb_table("Repos") 579 | all_repos = [] 580 | for repo_id in repos_managed: 581 | if repo_id == "null": continue 582 | response = repos_table.get_item( 583 | Key={ 584 | "Id": repo_id, 585 | "OwnerId": user_id, 586 | } 587 | ) 588 | 589 | if 'Item' in response: 590 | all_repos.append(response['Item']) 591 | except: 592 | raise Exception("Error while getting all repos.") 593 | return all_repos 594 | 595 | def _create_new_cloud_node(repo_id, api_key): 596 | """ 597 | Creates a new cloud node. 598 | """ 599 | try: 600 | run_deploy_routine(repo_id) 601 | except Exception as e: 602 | raise Exception("Error while creating new cloud node: " + str(e)) 603 | 604 | def _create_presigned_url(bucket_name, object_name, expiration=3600): 605 | """Generate a presigned URL to share an S3 object 606 | 607 | :param bucket_name: string 608 | :param object_name: string 609 | :param expiration: Time in seconds for the presigned URL to remain valid 610 | :return: Presigned URL as string. If error, returns None. 611 | """ 612 | 613 | # Generate a presigned URL for the S3 object 614 | s3_client = boto3.client('s3') 615 | try: 616 | response = s3_client.generate_presigned_url( 617 | 'get_object', 618 | Params={ 619 | 'Bucket': bucket_name, 620 | 'Key': object_name 621 | }, 622 | ExpiresIn=expiration 623 | ) 624 | except ClientError as e: 625 | raise Exception("Error while creating S3 presigned url.") 626 | 627 | # The response contains the presigned URL 628 | return response 629 | 630 | def _get_dynamodb_table(table_name): 631 | """ 632 | Helper function that returns an AWS DynamoDB table object. 633 | """ 634 | dynamodb = boto3.resource('dynamodb', region_name='us-west-1') 635 | table = dynamodb.Table(table_name) 636 | return table 637 | 638 | def _construct_cloud_node_url(repo_id): 639 | """ 640 | Helper function that constructs a Cloud Node url given a Repo ID. 641 | """ 642 | CLOUD_NODE_ADDRESS_TEMPLATE = "{0}.au4c4pd2ch.us-west-1.elasticbeanstalk.com" 643 | return CLOUD_NODE_ADDRESS_TEMPLATE.format(repo_id) 644 | 645 | def authorize_user(request, use_header=True): 646 | """ 647 | Helper function that authorizes a request/user based on the JWT Token 648 | provided. Return the claims if successful, `None` otherwise. 649 | """ 650 | try: 651 | if use_header: 652 | jwt_string = request.headers.get("Authorization").split('Bearer ')[1] 653 | else: 654 | jwt_string = request.get_json().get("token") 655 | claims = jwt.decode(jwt_string, JWT_SECRET, algorithms=[JWT_ALGO]) 656 | except Exception as e: 657 | return None 658 | 659 | return claims 660 | 661 | def make_unauthorized_error(): 662 | """ 663 | Helper function that returns an unauthorization error. 664 | """ 665 | return make_error('Authorization failed.') 666 | 667 | def make_error(msg): 668 | """ 669 | Helper function to create an error message to return on failed requests. 670 | """ 671 | return {'Error': True, 'Message': msg} 672 | 673 | 674 | if __name__ == "__main__": 675 | app.run(port=5001) 676 | -------------------------------------------------------------------------------- /dashboard-api/dashboard-api-dev-1572477459.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DiscreetAI/dashboard-api/918e7e00689852190d4795ef00ed47a91cb377cc/dashboard-api/dashboard-api-dev-1572477459.zip -------------------------------------------------------------------------------- /dashboard-api/dashboard-api-dev-template-1571105490.json: -------------------------------------------------------------------------------- 1 | {"Description":"Automatically generated with Zappa","Resources":{"ANY0":{"Properties":{"ApiKeyRequired":false,"AuthorizationType":"NONE","HttpMethod":"ANY","Integration":{"CacheKeyParameters":[],"CacheNamespace":"none","Credentials":"arn:aws:iam::880058582700:role/dashboard-api-dev-ZappaLambdaExecutionRole","IntegrationHttpMethod":"POST","IntegrationResponses":[],"PassthroughBehavior":"NEVER","Type":"AWS_PROXY","Uri":"arn:aws:apigateway:us-west-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-1:880058582700:function:dashboard-api-dev/invocations"},"MethodResponses":[],"ResourceId":{"Fn::GetAtt":["Api","RootResourceId"]},"RestApiId":{"Ref":"Api"}},"Type":"AWS::ApiGateway::Method"},"ANY1":{"Properties":{"ApiKeyRequired":false,"AuthorizationType":"NONE","HttpMethod":"ANY","Integration":{"CacheKeyParameters":[],"CacheNamespace":"none","Credentials":"arn:aws:iam::880058582700:role/dashboard-api-dev-ZappaLambdaExecutionRole","IntegrationHttpMethod":"POST","IntegrationResponses":[],"PassthroughBehavior":"NEVER","Type":"AWS_PROXY","Uri":"arn:aws:apigateway:us-west-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-1:880058582700:function:dashboard-api-dev/invocations"},"MethodResponses":[],"ResourceId":{"Ref":"ResourceAnyPathSlashed"},"RestApiId":{"Ref":"Api"}},"Type":"AWS::ApiGateway::Method"},"Api":{"Properties":{"Description":"Created automatically by Zappa.","Name":"dashboard-api-dev"},"Type":"AWS::ApiGateway::RestApi"},"ResourceAnyPathSlashed":{"Properties":{"ParentId":{"Fn::GetAtt":["Api","RootResourceId"]},"PathPart":"{proxy+}","RestApiId":{"Ref":"Api"}},"Type":"AWS::ApiGateway::Resource"}}} -------------------------------------------------------------------------------- /dashboard-api/deploy.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | import sys 4 | # import git 5 | # import shutil 6 | from time import strftime, sleep 7 | import boto3 8 | from botocore.exceptions import ClientError 9 | 10 | # REPO_URL = "https://github.com/georgymh/decentralized-ml-js" 11 | # REPO_PATH = "/tmp/decentralized-ml-js" 12 | # ZIP_PATH = '/tmp/cloud-node.zip' 13 | 14 | APPLICATION_NAME = "cloud-node" 15 | S3_BUCKET = "cloud-node-deployment" 16 | AWS_REGION = 'us-west-1' 17 | 18 | BUCKET_KEY = APPLICATION_NAME + '/' + 'cloudnode_build.zip' 19 | DEPLOYMENT_NAME = 'deploy-elb-{}' 20 | PIPELINE_NAME = 'cloud-node-deploy' 21 | 22 | 23 | # def upload_to_s3(artifact): 24 | # """ 25 | # Uploads an artifact to Amazon S3 26 | # """ 27 | # try: 28 | # client = boto3.client('s3') 29 | # except ClientError as err: 30 | # print("Failed to create boto3 client.\n" + str(err)) 31 | # return False 32 | # 33 | # try: 34 | # client.put_object( 35 | # Body=open(artifact, 'rb'), 36 | # Bucket=S3_BUCKET, 37 | # Key=BUCKET_KEY 38 | # ) 39 | # except ClientError as err: 40 | # print("Failed to upload artifact to S3.\n" + str(err)) 41 | # return False 42 | # except IOError as err: 43 | # print("Failed to access artifact.zip in this directory.\n" + str(err)) 44 | # return False 45 | # 46 | # return True 47 | 48 | 49 | # def pre_steps(): 50 | # """ 51 | # Removes temporary folders. 52 | # """ 53 | # try: 54 | # shutil.rmtree(REPO_PATH) 55 | # shutil.rmtree(ZIP_PATH) 56 | # except: 57 | # pass 58 | 59 | # def clone_repo(): 60 | # """ 61 | # Clones the repo locally. 62 | # """ 63 | # git.exec_command('clone', REPO_URL) 64 | # return REPO_PATH 65 | 66 | # def zip_server_directory(): 67 | # """ 68 | # Zips the cloned repo so it can be uploaded to S3. 69 | # """ 70 | # shutil.make_archive(ZIP_PATH.split('.zip')[0], 'zip', REPO_PATH + "/server") 71 | # return ZIP_PATH 72 | 73 | 74 | def create_new_version(version_label): 75 | """ 76 | Helper function that creates a new application version of an environment in 77 | AWS Elastic Beanstalk. 78 | """ 79 | try: 80 | client = boto3.client('elasticbeanstalk') 81 | except ClientError as err: 82 | raise Exception("Failed to create boto3 client.\n" + str(err)) 83 | 84 | try: 85 | response = client.create_application_version( 86 | ApplicationName=APPLICATION_NAME, 87 | VersionLabel=version_label, 88 | Description='New build', 89 | SourceBundle={ 90 | 'S3Bucket': S3_BUCKET, 91 | 'S3Key': BUCKET_KEY 92 | }, 93 | Process=True 94 | ) 95 | except ClientError as err: 96 | raise Exception("Failed to create application version.\n" + str(err)) 97 | 98 | try: 99 | if response['ResponseMetadata']['HTTPStatusCode'] is 200: 100 | return True, "success" 101 | else: 102 | raise Exception("Response did not return 200. Response: \n" + str(response)) 103 | except (KeyError, TypeError) as err: 104 | raise Exception(str(err)) 105 | 106 | def deploy_new_version(env_name, version_label): 107 | """ 108 | Helper function to deploy a created version of the environment to AWS 109 | Elastic Beanstalk. 110 | 111 | This needs to run after `create_new_version()`. 112 | """ 113 | try: 114 | client = boto3.client('elasticbeanstalk') 115 | except ClientError as err: 116 | raise Exception("Failed to create boto3 client.\n" + str(err)) 117 | 118 | try: 119 | response = client.create_environment( 120 | ApplicationName=APPLICATION_NAME, 121 | EnvironmentName=env_name, 122 | VersionLabel=version_label, 123 | SolutionStackName="64bit Amazon Linux 2018.03 v2.12.10 running Docker 18.06.1-ce", 124 | OptionSettings=[ 125 | { 126 | 'ResourceName':'AWSEBLoadBalancer', 127 | 'Namespace':'aws:elb:listener:80', 128 | 'OptionName':'InstanceProtocol', 129 | 'Value':'TCP' 130 | }, 131 | { 132 | 'ResourceName':'AWSEBAutoScalingLaunchConfiguration', 133 | 'Namespace':'aws:autoscaling:launchconfiguration', 134 | 'OptionName':'SecurityGroups', 135 | 'Value':'ebs-websocket' 136 | }, 137 | { 138 | 'ResourceName': 'AWSEBLoadBalancer', 139 | 'Namespace': 'aws:elb:listener:80', 140 | 'OptionName': 'ListenerProtocol' , 141 | 'Value': 'TCP' 142 | }, 143 | ], 144 | ) 145 | except ClientError as err: 146 | raise Exception("Failed to update environment.\n" + str(err)) 147 | 148 | return response 149 | 150 | def deploy_cloud_node(env_name): 151 | """ 152 | Creates and then deploys a new version of the Cloud Node to AWS Elastic 153 | Beanstalk. 154 | """ 155 | # if not upload_to_s3(ZIP_PATH): 156 | # sys.exit(1) 157 | version_label = strftime("%Y%m%d%H%M%S") 158 | _ = create_new_version(version_label) 159 | # Wait for the new version to be consistent before deploying 160 | sleep(5) 161 | _ = deploy_new_version(env_name, version_label) 162 | return True 163 | 164 | 165 | def make_codepipeline_elb_deploy_action(env_name): 166 | """ 167 | Crafts a Deployment Action for AWS CodePipeline in JSON format. 168 | """ 169 | return { 170 | 'name':'deploy-elb-' + env_name, 171 | 'actionTypeId':{ 172 | 'category':'Deploy', 173 | 'owner':'AWS', 174 | 'provider':'ElasticBeanstalk', 175 | 'version':'1' 176 | }, 177 | 'runOrder':1, 178 | 'configuration':{ 179 | 'ApplicationName': APPLICATION_NAME, 180 | 'EnvironmentName': env_name 181 | }, 182 | 'outputArtifacts':[ 183 | 184 | ], 185 | 'inputArtifacts':[ 186 | { 187 | 'name':'SourceArtifact' 188 | } 189 | ], 190 | 'region': AWS_REGION 191 | } 192 | 193 | def add_codepipeline_deploy_step(env_name): 194 | """ 195 | Makes a Cloud Node self-updateable by updating the AWS CodePipeline 196 | pipeline. 197 | """ 198 | try: 199 | client = boto3.client('codepipeline') 200 | pipeline_response = client.get_pipeline( 201 | name=PIPELINE_NAME, 202 | ) 203 | 204 | pipeline_data = pipeline_response['pipeline'] 205 | for stage in pipeline_data['stages']: 206 | if stage['name'] == 'Deploy': 207 | new_action = make_codepipeline_elb_deploy_action(env_name) 208 | stage['actions'].append(new_action) 209 | _ = client.update_pipeline( 210 | pipeline=pipeline_data 211 | ) 212 | print("Updated CodeDeploy pipeline.") 213 | except Exception as e: 214 | raise Exception("Error adding deploy step: " + str(e)) 215 | 216 | return True 217 | 218 | 219 | def run_deploy_routine(repo_id): 220 | """ 221 | Runs the Deploy routine 222 | """ 223 | # pre_steps() 224 | # _ = clone_repo() 225 | # _ = zip_server_directory() 226 | _ = deploy_cloud_node(repo_id) 227 | _ = add_codepipeline_deploy_step(repo_id) 228 | 229 | def _delete_cloud_node(repo_id): 230 | """ 231 | Deletes cloud node 232 | """ 233 | try: 234 | client = boto3.client('elasticbeanstalk') 235 | except ClientError as err: 236 | raise Exception("Failed to create boto3 client.\n" + str(err)) 237 | 238 | try: 239 | response = client.terminate_environment( 240 | EnvironmentName=repo_id, 241 | TerminateResources=True, 242 | ) 243 | 244 | return response 245 | except ClientError as err: 246 | raise Exception("Failed to delete environment.\n" + str(err)) 247 | 248 | def _remove_codepipeline_deploy_step(env_name): 249 | """ 250 | Removes Cloud Node from the AWS CodePipeline pipeline. 251 | """ 252 | try: 253 | target_name = DEPLOYMENT_NAME.format(env_name) 254 | client = boto3.client('codepipeline') 255 | pipeline_response = client.get_pipeline( 256 | name=PIPELINE_NAME, 257 | ) 258 | 259 | pipeline_data = pipeline_response['pipeline'] 260 | for stage in pipeline_data['stages']: 261 | if stage['name'] == 'Deploy': 262 | stage['actions'] = [action for action in stage['actions'] \ 263 | if action['name'] != target_name] 264 | _ = client.update_pipeline( 265 | pipeline=pipeline_data 266 | ) 267 | print("Updated CodeDeploy pipeline.") 268 | 269 | return True 270 | except Exception as e: 271 | raise Exception("Error removing deploy step: " + str(e)) 272 | 273 | def run_delete_routine(repo_id): 274 | """ 275 | Runs delete cloud node routine 276 | """ 277 | _ = _delete_cloud_node(repo_id) 278 | _ = _remove_codepipeline_deploy_step(repo_id) 279 | -------------------------------------------------------------------------------- /dashboard-api/dynamodb.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import os 3 | from boto3.dynamodb.conditions import Key, Attr 4 | import logging 5 | 6 | class DB(object): 7 | """ 8 | DynamoDB class used to manage two tables. 9 | 10 | 'jupyterhub-users' (users_table) maps usernames to whether they are in use or not 11 | 'Repos' (repos_table) maps repo_ids to usernames (N/A if no username mapping exists yet) 12 | """ 13 | def __init__(self): 14 | """ 15 | Set up DB and tables. 16 | """ 17 | access_key = os.environ["AWS_ACCESS_KEY_ID"] 18 | secret_key = os.environ["AWS_SECRET_ACCESS_KEY"] 19 | self.dynamodb = boto3.resource('dynamodb', aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name='us-west-1') 20 | self.users_table = self.dynamodb.Table('jupyterhub-users') 21 | self.repos_table = self.dynamodb.Table('Repos') 22 | 23 | def create_table(self): 24 | """ 25 | Recreate users table (ONLY TO BE USED AFTER DELETING) 26 | """ 27 | self.users_table = self.dynamodb.create_table( 28 | TableName='jupyterhub-users', 29 | KeySchema=[ 30 | { 31 | 'AttributeName': 'username', 32 | 'KeyType': 'HASH' 33 | }, 34 | { 35 | 'AttributeName': 'repo_id', 36 | 'KeyType': 'SORT' 37 | }, 38 | ], 39 | AttributeDefinitions=[ 40 | { 41 | 'AttributeName': 'username', 42 | 'AttributeType': 'S' 43 | }, 44 | { 45 | 'AttributeName': 'repo_id', 46 | 'AttributeType': 'S' 47 | }, 48 | ], 49 | ProvisionedThroughput={ 50 | 'ReadCapacityUnits': 5, 51 | 'WriteCapacityUnits': 5 52 | } 53 | ) 54 | 55 | table.meta.client.get_waiter('table_exists').wait(TableName='jupyterhub-users') 56 | 57 | def delete_table(self): 58 | """ 59 | Delete the users table 60 | """ 61 | self.users_table.delete() 62 | self.users_table = None 63 | 64 | def add_user(self, username, batch=None): 65 | """ 66 | Add a single user to the users table 67 | """ 68 | if not batch: 69 | batch = self.users_table 70 | assert batch != None, "Table not initialized!" 71 | batch.put_item( 72 | Item={ 73 | 'username': username, 74 | 'in_use': False 75 | } 76 | ) 77 | 78 | def add_multiple_users(self, usernames): 79 | """ 80 | Add multiple users in a single batch update 81 | """ 82 | with self.users_table.batch_writer() as batch: 83 | for username in usernames: 84 | self.add_user(username, batch=batch) 85 | 86 | 87 | def get_username(self, user_id, repo_id): 88 | """ 89 | Get username for given repo_id. If a username doesn't exist for this repo, retrieve an 90 | available username and make it unavailable for other repos. 91 | """ 92 | creds = self._get_creds(user_id, repo_id) 93 | if creds != 'N/A': 94 | return creds 95 | return self._get_next_available_username(user_id, repo_id) 96 | 97 | def _get_next_available_username(self, user_id, repo_id): 98 | """ 99 | Retrieve next available username and make it unavailable for other repos. 100 | """ 101 | response = self.users_table.scan( 102 | FilterExpression=Attr('in_use').eq(False) 103 | ) 104 | items = response['Items'] 105 | assert items, "No more usernames available!" 106 | self._set_user_creds(user_id, repo_id, items[0]['username']) 107 | self._set_creds_in_use(items[0]['username'], True) 108 | return items[0]['username'] 109 | 110 | def _set_user_creds(self, user_id, repo_id, username): 111 | """ 112 | Set username for given repo_id 113 | """ 114 | self.repos_table.update_item( 115 | Key={ 116 | 'Id': repo_id, 117 | 'OwnerId': user_id 118 | }, 119 | UpdateExpression='SET creds = :val1', 120 | ExpressionAttributeValues={ 121 | ':val1': username 122 | } 123 | ) 124 | 125 | def _set_creds_in_use(self, username, in_use): 126 | """ 127 | Set the 'in_use' boolean for this username 128 | """ 129 | self.users_table.update_item( 130 | Key={ 131 | 'username': username, 132 | }, 133 | UpdateExpression='SET in_use = :val1', 134 | ExpressionAttributeValues={ 135 | ':val1': in_use 136 | } 137 | ) 138 | 139 | def _get_creds(self, user_id, repo_id): 140 | """ 141 | Get username for given repo_id (N/A if it doesn't exist) 142 | """ 143 | response = self.repos_table.get_item( 144 | Key={ 145 | 'Id': repo_id, 146 | 'OwnerId': user_id 147 | } 148 | ) 149 | item = response['Item'] 150 | return item['creds'] -------------------------------------------------------------------------------- /dashboard-api/output.txt: -------------------------------------------------------------------------------- 1 | a5fac81a-61e1-4b19-89fd-072f070c85f8 2 | 8f8f9434-c01b-4b5d-8c43-4c07bce57e1a 3 | 345733c0-2fe7-40d0-bcfb-e910c88b0f9f 4 | 182178da-debc-4275-92c1-9803255e75be 5 | 7e0b4a21-035f-4e1d-af33-c1f9ea6e666a 6 | 04c287f6-67ec-4f39-b5aa-febcac7ba5d7 7 | b7147034-792b-44b2-a5d8-81cbe6d1db26 8 | 7c324e17-a690-4ac2-9ee6-e728d4b82a54 9 | b0e0697b-35bd-461e-b244-d1c107f4433b 10 | 10465a91-ddd4-41d9-80a5-4b5b1d8d552b 11 | 773ccaa3-62a0-4eea-8e65-4091eace3ada 12 | 0d9c3866-e22d-44e9-8c8f-1530f9401f5c 13 | 73800267-b6c8-462b-957b-3eb030d4d8d3 14 | 20159828-0ad1-4d52-a6df-22df97e2fbd9 15 | 009ed80b-e3e8-4c81-9144-a5245b8506c6 16 | f80c838f-d395-4689-bedc-20c5b4761e3c 17 | e997f12e-eef1-4b8b-989b-58f597490f18 18 | 5e0b59a2-27e8-495c-b1b2-0c954795c5f1 19 | d9f0d5bb-5fbf-46c9-b6ce-050b7d3ddd1c 20 | 1191d8a3-156e-41af-86d3-449a1e8308d5 21 | 9cb5c69c-f656-472d-b773-b949424ab54b 22 | d2518942-d545-4fe8-ac6a-fdc5a4b61d13 23 | 878a33ea-b899-4041-8754-f479199b8154 24 | c278bcaf-79e6-4f94-86c3-70aece54cdd3 25 | 869ca015-0bbd-4644-8736-7de5b67e5f23 26 | 760b3ac4-fad8-45e1-8074-498b6074b1e7 27 | b6fc8b32-eebe-419b-a8d7-a069cd758e2b 28 | 52e6e458-d53c-4402-bed1-5eb4ec686fcc 29 | fda50c87-e564-458c-a6ab-441fb1656961 30 | 2be3aa7c-e087-4d08-87ae-2bed9f9d0b7b 31 | 7acf310e-eb53-4d5c-bc4c-76b77f41ffde 32 | a366a0b6-bf81-4d86-9d24-a91e341b792c 33 | 264da496-5c89-4242-b425-8e676e2aa1bf 34 | d08d0f89-26eb-428d-85f2-f480a29e7ced 35 | 147d2160-ddfc-4422-ab90-e0485fdd7853 36 | e8a44679-e961-483e-a690-2d6590e7b986 37 | e61d5417-1473-4c29-a092-f936772b402d 38 | 66270e14-9cb6-4356-8749-67b476de30b8 39 | 1ec7533a-d391-4de0-8e93-c4bee4980e18 40 | 6e85f121-acd7-4c37-a95f-9629506fe03b 41 | eec5f920-0a4e-4a8f-99b4-008494ca6f73 42 | d5ed3863-9e2e-432d-8195-20f8a1c3386c 43 | accf68bb-eb06-4a50-8465-33262826685f 44 | 8648ee5c-97c4-4816-bde5-02a58ce0a855 45 | 7a717475-5838-45de-b662-8c98e00fe680 46 | 035b84f9-ce99-4b0a-9ac0-f026e284c77e 47 | 1f1f6bc2-0303-4cd2-99c2-9017f7ae694c 48 | 09451235-ffd6-4e88-8ea8-247666ee26ff 49 | 9c9b007c-e9c2-4196-bd9b-f1d85cb0f20e 50 | d5fde7b7-6638-44d6-afec-68603a2a9297 51 | a2512f4f-73b8-4dad-84ff-a47c3f980897 52 | a3c4f5b0-2402-42f8-8bc2-9c0a45fc6a0c 53 | 50318ba8-ff9b-4565-a111-2d6cccbe5c70 54 | d3903ba6-2fa2-4189-9c94-c0cb674df772 55 | 171639a0-5f48-4ee6-ab1e-def4a866e0d6 56 | c7aa25bc-f2db-4bf3-98ef-b8cda93281e1 57 | 04cac1e4-a64c-4792-99d2-268ab85b2bd0 58 | ae28ebb9-879b-4997-8761-b5eee5e65b27 59 | 3cf6292e-19b1-4646-8aad-980a58af53e9 60 | a1a6aa23-5901-4844-8ae0-7d091d631924 61 | 53fbed50-a1a2-4df9-9e1e-cfa135939549 62 | 505a06f5-2776-48d4-8f42-7d6b3e9e1dd0 63 | 6eb38caa-bd58-4274-97e8-ce85db5869a9 64 | e42fae8f-9045-4ecc-abb5-342a78f10684 65 | 6a3d24ca-3d9e-4462-a698-874335c73a3f 66 | c35c35cb-ead0-48d6-8644-0e13b53afff8 67 | f144815e-890c-410f-ba9d-045766f13000 68 | 2f26077f-49c9-43a2-84ef-ca87d251ba0b 69 | cef6296e-2eed-4222-8e68-c1eb1fe63af2 70 | d434bbfe-5c4f-4840-9796-7637123d4344 71 | 51ebc589-622a-4cb3-960b-4f4b689652e4 72 | fc7c6a67-d916-4aaf-9646-2a2300827687 73 | 157b727e-2876-4107-a8c5-e9d5579a7b8c 74 | e0c80679-9669-43bc-98dd-78e57d9bab38 75 | 837af662-cf41-491e-836a-403d7d4e5d96 76 | 954317ab-efd2-4bad-b84d-76a86d03f5e5 77 | ad09d0b2-896b-4801-9941-ee61d6d90163 78 | f5ecdb7f-cb09-461c-baf5-ba482c4e1f62 79 | 690533dd-d38e-4872-b392-340c19d28c6f 80 | 8917aaf7-bade-488c-9b73-899dfc22dc78 81 | a82595d2-9a88-4622-8d12-88232c4e9cfa 82 | e5675a53-6b40-4aef-93df-35d34cce24d6 83 | a7c2813b-8bc6-4ba1-b534-c6c808f83dff 84 | b84c04fc-ee41-4b16-a166-f744ce6b0bdf 85 | a2e29621-5de9-43ff-a819-5a308b372c66 86 | 3c750766-2877-470e-bcf0-85baf22f20e8 87 | 63afc185-ab3d-49d7-846b-eda111dac2a4 88 | 34719f8a-8b8b-4e32-a159-f5f28958f040 89 | f400b6ef-5eb8-4267-b0e2-a51d3386efe1 90 | 8138df70-d0de-4451-b005-8b16fec7283c 91 | 30bef364-d96e-4c3d-a288-6a98469ac732 92 | 1bcca343-eef0-4b0e-bd8e-68006a4fa558 93 | 4302be28-17e5-4447-9251-83bc71da3631 94 | 6aac9e3c-950c-4c60-b862-94a88995e5a8 95 | f3a8dec5-9e24-4eb7-aa98-b89d29928cda 96 | dc150531-0483-43f5-a89c-2ccd3e3b520a 97 | 77bb61f6-624b-464a-92ad-fc6c19532ece 98 | 3e6872c2-1293-45b0-a98b-ec3cea511b7c 99 | da1751ec-9240-4852-bfc4-f629abe759ec 100 | 1fd0f02d-0a7d-4d97-915e-b38513d4440a 101 | 4ef3c172-7891-4d9f-8fbd-d2cdd3a8423c 102 | 6490c9ce-3503-4de8-86ef-a3785bbb79db 103 | 1c330918-e4ba-4139-a667-9ad885a30c5b 104 | 4aefe3af-d450-46dc-a62c-e609375be6f4 105 | b4b3612e-a9a9-44b4-a612-b3e443695e12 106 | 67dd1f89-9c82-408a-b9f4-1ecf51640618 107 | 5fef37db-9122-43b6-b487-305d63056c0e 108 | bed7801f-e012-4353-a1b3-7abf2be7d7bc 109 | 5d140853-f08e-4257-8693-e1a353ebbf25 110 | aa9313cb-d05a-40d2-b161-34f3f40c36b7 111 | 08c122b0-bb22-4536-995d-32137a57ed23 112 | f6b69ca7-bfc2-4c96-805e-5dfe639e5890 113 | 37c0094f-4b2b-4200-80d2-492955f93b6b 114 | b6cccaeb-b633-4006-916a-a4f4bf25a001 115 | 53f77dc1-c465-46e5-945a-577480de1a33 116 | 3d06416c-81a6-49c3-bcef-32a46846823c 117 | 0f0ea682-293d-470b-9981-6f06f017bec5 118 | d6fb3fc4-9f8b-478a-9236-21c475b5e3a9 119 | 0f116c54-9890-4eb0-b769-4f8a93332f29 120 | 7d5b0ad1-3a14-40b0-88ff-1a26e4c39f41 121 | 344cf1be-2571-435d-a22b-cab762d87f85 122 | cc2bddff-a15a-4bb0-bf8f-5849a9e61a26 123 | 9a8904e7-4cc5-40b4-b48e-533025b8e8e6 124 | d4312079-e172-43c8-b0ff-552a53611e5b 125 | 301ceba6-b63d-4bde-bb71-d7593a612f34 126 | 569f27e5-299c-47f6-85f7-349ec1d7c0c2 127 | 6550ee2b-5295-4bd1-914f-f2cd75607d76 128 | e3a26bb3-1627-463b-8d5b-5ebc3967342c 129 | 4b7114c6-9343-4391-bd55-6fe8c04759ae 130 | 59eb69b5-1a0f-4fef-acff-006ce6e453ca 131 | aafc3601-33db-4098-92b1-7b79f9a5f799 132 | c416a74e-a650-4297-b21c-9b77a48b894c 133 | f6755bef-54a7-4872-a024-85499bde0fef 134 | 586be86d-7396-469e-b6e6-152904ef4512 135 | 9f578fb5-9a3f-443f-b6db-88fde339153d 136 | faf95961-6730-40d6-86f6-a1d5343f8a04 137 | 7d56f4c2-2ca5-4734-8517-bc5146010fae 138 | 95600b76-3529-47bb-a8a0-d2a250c09d4e 139 | 4b809fb4-30c9-49b4-a36e-d57d81846237 140 | 053ba75b-481e-46a9-9493-1de34f8eeeac 141 | 23d3c308-f4a2-40bf-9ad7-a1b5c65cea76 142 | 62f741b6-529c-4ff5-98ba-f605d65a7579 143 | df8c8c95-efe7-459e-ab48-d60359335b08 144 | 41a51254-06d2-4243-94d6-addfada1b22e 145 | 5fc820e1-a7fd-445e-9559-afa01c682a58 146 | e9c87768-fdab-41bb-8a08-caf684ac45aa 147 | 5c5a210f-6083-4bae-b9bf-c33b4bbcdb99 148 | e5239179-1028-4a13-beb0-2d757731f2b1 149 | e5c59d7f-4d88-4747-be95-3e75d05f1367 150 | aed1c5b4-62e4-4a93-9e57-1d269af442a7 151 | b008ca2f-0eb7-465c-8251-35418dec9381 152 | 5b9533cb-ea53-41a5-a760-50296bf98813 153 | 17fd34dd-35ea-4d3c-b166-1ff03a91f7e1 154 | f4a40437-4cf5-4b63-9307-a0299885ac56 155 | a68f3f21-a1bd-4989-b155-9ed19fff8891 156 | f5f0fd68-a232-4e14-a7dc-0e71b6e08eff 157 | 01f2fdf9-cf7a-4726-ae47-8748360375ea 158 | 4a6520c4-004b-4b9d-90ab-368e2dd36037 159 | 00043620-dce5-4c33-a289-d9e774166bc8 160 | 5fdbfe35-7186-4be4-933b-48391e0078c6 161 | bce7480e-4711-4299-a9a4-51f3bfa214f7 162 | ceebd752-4ba9-452a-8f4e-f40184925938 163 | 47ec0080-3775-4de4-913d-a55da0b6ad38 164 | c3f34e05-63d2-4685-81d5-9e860474cd77 165 | eac77a67-32c0-465f-814f-9267e8e72703 166 | a8acf582-ab0b-4601-986e-5861c628c2a0 167 | f368de80-a29f-4a4c-b1fb-e08cb1bbf467 168 | d1bd244f-479f-4753-9438-aa6b8e38324f 169 | e3ea4c17-d7ec-4ded-8ddc-a4261e35cb85 170 | 81021987-5ac6-417b-bfb3-98a9f1daab51 171 | f90bb7df-58bf-407b-9902-e47d2858e42c 172 | 02a3ca03-11d5-46ad-b1e8-47d9c35ecbbb 173 | c1594ea0-e020-4c65-9873-19cd568191db 174 | 9ed99460-f7a6-4ec7-83c7-3e16b390b4bc 175 | 4e56ee72-186a-4a12-ab71-c06e816368f5 176 | 0f5a09a3-96d4-440d-9cb0-931465f5bf86 177 | 547a0924-9830-4c8f-adc0-cc45f8d03c0c 178 | 4ec7d57a-f7de-41fa-bff1-e12f0c531e90 179 | 38bee614-80af-4f1b-843b-1b59ddd231ef 180 | f30d2f3c-78a5-48cf-8590-5db05015254b 181 | 6c379723-9ee0-41be-86a2-5ac8140a41c5 182 | 86744c70-036e-4c29-9dc7-8b37db64cbeb 183 | c1193948-ad5a-45f8-a878-370028ab97c7 184 | feabcc1a-f226-4cbd-aa13-87006a89be9b 185 | 28380e0e-f283-4724-a05a-03b5913a5087 186 | 6ee79694-a07b-4304-9d3b-29477a26c35a 187 | 1c4bb845-74cc-4363-b606-36d4d4925d87 188 | e4039b3c-1bd8-4b6f-aa07-7cac47d2302a 189 | 17a056c2-b68a-49bf-9c64-06be14b305b4 190 | f9f2d446-246e-498a-93c0-8790ba447ad8 191 | e8144042-db25-4427-a552-d4a5324c1c97 192 | d08f29ff-0b48-466c-8350-0036fe95b185 193 | 9832c5eb-f08e-43db-86c6-135986a023e2 194 | eef22509-fac2-4b05-af23-6c74e8a79abf 195 | 294fdb56-7bb4-43c9-930b-1662b808514d 196 | 818ba4d2-24ce-4df8-a832-2a70c4f99f53 197 | e5effc59-9cd6-4629-b7b6-cb35dc56fafc 198 | 3a64147a-a15a-4aa7-9ac9-1202204f6615 199 | e0eb5545-7672-47df-8d8e-ae214181a4d2 200 | a9cdc4ff-0db7-45a2-9eaf-7fa45b795374 201 | -------------------------------------------------------------------------------- /dashboard-api/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dashboard-api", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "setup": "npm install", 8 | "postsetup": "node postsetup.js", 9 | "test": "echo \"Error: no test specified\" && exit 1" 10 | }, 11 | "author": "", 12 | "license": "ISC", 13 | "devDependencies": { 14 | "chalk": "^2.1.0", 15 | "inquirer": "^3.2.3", 16 | "js-yaml": "^3.9.1", 17 | "serverless-python-requirements": "^2.5.0", 18 | "serverless-wsgi": "^1.3.0" 19 | }, 20 | "dependencies": { 21 | "serverless": "^1.54.0" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /dashboard-api/postsetup.js: -------------------------------------------------------------------------------- 1 | // Check to see if the user has Docker installed and which version of Python they prefer. 2 | 3 | 'use strict'; 4 | var inquirer = require('inquirer'); 5 | var chalk = require('chalk'); 6 | var YAML = require('js-yaml'); 7 | var fs = require('fs'); 8 | 9 | console.log(chalk.yellow('Hi, a few quick questions before we start:')); 10 | 11 | var questions = [ 12 | { 13 | type: 'list', 14 | name: 'python', 15 | message: 'What Python version?', 16 | choices: ['python2.7', 'python3.6'], 17 | filter: function (val) { 18 | return val.toLowerCase(); 19 | } 20 | }, 21 | { 22 | type: 'confirm', 23 | name: 'docker', 24 | message: 'Do you have Docker installed? Recommended, but not required.', 25 | default: true 26 | }, 27 | { 28 | type: 'confirm', 29 | name: 'wantsDomain', 30 | message: 'Do you want to set up a custom domain? e.g. api.mycompany.com? Requires a domain in Route53.', 31 | default: false 32 | }, 33 | { 34 | type: 'input', 35 | name: 'domainName', 36 | message: 'What is your domain name? e.g. api.mycompany.com', 37 | when: function(answers) { 38 | return answers.wantsDomain 39 | } 40 | }, 41 | ] 42 | 43 | inquirer.prompt(questions).then(function (answers) { 44 | 45 | var doc = YAML.safeLoad(fs.readFileSync('serverless.yml', 'utf8')); 46 | doc.custom.pythonRequirements.dockerizePip = answers.docker; 47 | doc.provider.runtime = answers.python; 48 | if (answers.wantsDomain) { 49 | doc.plugins.push('serverless-domain-manager'); 50 | doc.custom.customDomain = createCustomDomain(answers.domainName); 51 | } 52 | saveServerlessYml(doc); 53 | console.log(chalk.yellow("All set! Run `sls deploy` to send your code to the cloud")); 54 | if (answers.wantsDomain) { 55 | console.log(chalk.yellow("Run `sls create_domain` to set up your custom domain.")) 56 | } 57 | }); 58 | 59 | const createCustomDomain = (domainName) => { 60 | return { 61 | domainName, 62 | basePath: '', 63 | stage: '${self:provider.stage}', 64 | createRoute53Record: true 65 | } 66 | } 67 | 68 | const saveServerlessYml = (config) => { 69 | /* 70 | When dumping to yml, it puts quotation marks around Serverless variables. This 71 | breaks variable resolution, so I clean it out after dumping to yml but before 72 | writing to the config file. 73 | */ 74 | const dumped = YAML.dump(config); 75 | const cleaned = dumped.replace("\'${self:provider.stage}\'", "${self:provider.stage}") 76 | fs.writeFileSync('serverless.yml', cleaned); 77 | } 78 | -------------------------------------------------------------------------------- /dashboard-api/requirements.txt: -------------------------------------------------------------------------------- 1 | click==6.7 2 | Flask==0.12.2 3 | itsdangerous==0.24 4 | Jinja2==2.9.6 5 | MarkupSafe==1.0 6 | Werkzeug==0.12.2 7 | PyJWT==1.7.1 8 | boto3==1.9.145 9 | simplejson==3.16.0 10 | requests==2.21.0 11 | Flask-Cors==3.0.7 12 | lambda-git==0.1.1 13 | -------------------------------------------------------------------------------- /dashboard-api/serverless.yml: -------------------------------------------------------------------------------- 1 | org: ndodda 2 | app: dashboard-api-app 3 | service: dashboard-api 4 | plugins: 5 | - serverless-python-requirements 6 | - serverless-wsgi 7 | custom: 8 | wsgi: 9 | app: app.app 10 | packRequirements: false 11 | pythonRequirements: 12 | dockerizePip: true 13 | package: 14 | exclude: 15 | - node_modules/** 16 | - venv/** 17 | - __pycache__/** 18 | provider: 19 | name: aws 20 | runtime: python3.6 21 | stage: dev 22 | region: us-west-1 23 | functions: 24 | app: 25 | handler: wsgi.handler 26 | events: 27 | - http: ANY / 28 | - http: 'ANY {proxy+}' 29 | -------------------------------------------------------------------------------- /dashboard-api/test.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | from boto3.dynamodb.conditions import Key, Attr 3 | 4 | APPLICATION_NAME = "cloud-node" 5 | PIPELINE_NAME = 'cloud-node-deploy' 6 | DEPLOYMENT_NAME = 'deploy-elb-{}' 7 | AWS_REGION = 'us-west-1' 8 | 9 | def _get_dynamodb_table(table_name): 10 | """ 11 | Helper function that returns an AWS DynamoDB table object. 12 | """ 13 | dynamodb = boto3.resource('dynamodb', region_name='us-west-1') 14 | table = dynamodb.Table(table_name) 15 | return table 16 | 17 | table_name = "CandidateAuth" 18 | table = _get_dynamodb_table(table_name) 19 | 20 | response = table.get_item( 21 | Key={ 22 | 'uuid': '0f0ea682-293d-470b-9981-6f06f017bec5', 23 | } 24 | ) 25 | 26 | # items = table.scan() 27 | # print(items) 28 | # print(response) 29 | print('Item' in response) 30 | # print(response['Item']) 31 | 32 | # item = response['Item'] 33 | # print(item) 34 | # with table.batch_writer() as batch: 35 | # with open('output.txt', 'r') as f: 36 | # for uuid in f.readlines(): 37 | # print(uuid.split("\n")[0]) 38 | # batch.put_item( 39 | # Item={ 40 | # 'uuid': uuid.split("\n")[0], 41 | # 'status': False 42 | # } 43 | # ) -------------------------------------------------------------------------------- /dashboard-api/zappa_settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "dev": { 3 | "app_function": "app.app", 4 | "aws_region": "us-west-1", 5 | "profile_name": "default", 6 | "project_name": "dashboard-api", 7 | "runtime": "python3.6", 8 | "s3_bucket": "zappa-rjnc07ko2" 9 | } 10 | } --------------------------------------------------------------------------------