├── .babelrc ├── .gitignore ├── .travis.yml ├── .vscode ├── launch.json └── settings.json ├── Dockerfile ├── LICENSE ├── README.md ├── config └── services.json ├── docker-compose.yaml ├── jsconfig.json ├── killrvideo.env ├── lib └── killrvideo-service-protos │ ├── LICENSE │ ├── README.md │ └── src │ ├── comments │ ├── comments_events.proto │ └── comments_service.proto │ ├── common │ └── common_types.proto │ ├── ratings │ ├── ratings_events.proto │ └── ratings_service.proto │ ├── search │ └── search_service.proto │ ├── statistics │ └── statistics_service.proto │ ├── suggested-videos │ └── suggested_videos_service.proto │ ├── uploads │ ├── uploads_events.proto │ └── uploads_service.proto │ ├── user-management │ ├── user_management_events.proto │ └── user_management_service.proto │ └── video-catalog │ ├── video_catalog_events.proto │ └── video_catalog_service.proto ├── npm-shrinkwrap.json ├── package.json ├── scripts ├── copy-google-protos.js ├── git-subtree-pull.ps1 └── git-subtree-pull.sh └── src ├── common ├── cassandra.js ├── config.js ├── extendable-error.js ├── logging.js ├── message-bus.js ├── service-discovery.js └── with-retries.js ├── data └── README.md ├── grpc └── server.js ├── index.js ├── protos └── README.md ├── server-listeners ├── index.js └── log-services.js └── services ├── comments ├── comment-on-video.js ├── events.js ├── get-user-comments.js ├── get-video-comments.js ├── index.js └── protos.js ├── common ├── grpc-errors.js ├── load.js └── protobuf-conversions.js ├── index.js ├── ratings ├── events.js ├── get-rating.js ├── get-user-rating.js ├── index.js ├── protos.js └── rate-video.js ├── search ├── get-query-suggestions.js ├── handlers.js ├── index.js ├── protos.js └── search-videos.js ├── statistics ├── get-number-of-plays.js ├── index.js ├── protos.js └── record-playback-started.js ├── suggested-videos ├── get-related-videos.js ├── get-suggested-for-user.js ├── index.js └── protos.js ├── uploads ├── events.js ├── index.js └── protos.js ├── user-management ├── create-user.js ├── events.js ├── get-user-profile.js ├── index.js ├── password-hashing.js ├── protos.js └── verify-credentials.js └── video-catalog ├── constants.js ├── events.js ├── get-latest-video-previews.js ├── get-user-video-previews.js ├── get-video-previews.js ├── get-video.js ├── index.js ├── protos.js ├── submit-uploaded-video.js └── submit-youtube-video.js /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | "env" 4 | ], 5 | "plugins": [ 6 | "transform-async-to-generator" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log.* 5 | 6 | # Dependency directory 7 | # https://docs.npmjs.com/misc/faq#should-i-check-my-node-modules-folder-into-git 8 | node_modules 9 | 10 | # VS Code stuff 11 | /typings 12 | 13 | # Build and package output 14 | /dist 15 | 16 | # Docker .env settings 17 | .env 18 | 19 | # Webstorm/IDEA 20 | .idea 21 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "8" 4 | 5 | # Sudo required for doing docker build 6 | sudo: required 7 | services: 8 | - docker 9 | 10 | # Build the app and a docker image 11 | script: 12 | - docker build -t ${TRAVIS_COMMIT} . 13 | 14 | # If successful, see if we need to publish also 15 | after_success: 16 | - test -z $TRAVIS_TAG && travis_terminate 0 17 | - docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-nodejs:${TRAVIS_TAG} 18 | - echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin 19 | - docker push killrvideo/killrvideo-nodejs:${TRAVIS_TAG} 20 | - "[ \"$(git tag --sort=-v:refname | grep -P \"^\\d+.\\d+.\\d+$\" | head -n1)\" == \"$TRAVIS_TAG\" ] && { docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-nodejs:latest; docker push killrvideo/killrvideo-nodejs:latest; }" 21 | 22 | env: 23 | global: 24 | # DOCKER_USER 25 | - secure: hL9GzKnAuHP130bLzB1nK9eF6bfPD4yFdQ1IdRRp7QDZ+AJ2MUw483w5Uacw6/VNk+KlItO9ySxkAI8gaBcxjxcp+TmFkmD0o4rRQixCoZiqlNeTCMmkx5J6KffNALiFBFPjOvVNXLEYh6lbyIsPJR0/eHGlhCbfpx9Ok9PxzVV4AtNNXVcqCl4hZFWPU8OX7nL0pyQ86MD1WiC/1nfTaE/9zaZ8M/qJhv558KmsSXnFN1eGrwVJLt8XPbZ5aKEvJq3cngwVJ3/hmMQnA6ScgANZFbFrDZo9gDmQLDAbGzQ4mg1wpmvdCGB9HEo14o+ZK3utmnURZDGLHcbgKitIp7fd+FyIktxF3h+hYm7yMV/P67ixLXWxay9F0XXkKF1CPhRU+uEijcWQY/txUItvfHJtDGQbHhChUPmzv2eTQD5QoevFByZ6c3ekQew6hgCPaOsV2FspQQ+XVtdjiiOzmWzeFux2Obc46K85BSyeEp9SjuTJ0772dFBZjWn9UP+M5B8THs4BNPhkSVhceHQBvv65H8DZYEazue0aJYwJhMd1qx5tq/3dXItyDVqxCj0J0LFI4TsFbSf4R+snYOxjc2dkpO1l9aZyLNBTbRn//MTki8o/tdIjAzNYKKWHHBWCYN7OU4Ej4t7XI/SKs3Wf131ebqLlTYU0ppJD7KQUvCU= 26 | # DOCKER_PASS 27 | - secure: hxakhOfACkkXcGc5T9cT0+a+ICw6X2ybsOXzJsgdju4QbdP0nD2iBSZCypDzfdzlF/y83yJGcSznegMega37ABomPd8BF04IJTY1ZSvfj8LiUiLLfo/YSsz68r/X1G/AxVmS/7x9x5xaCPMnCHaH2jK5Qiogawzlqgqw/h3EAQlnJyKdjwwmFpATUAuRJidQ/95pn4KYvERg3eZ7/DzqbAO4IRjSuxHRNqDsHXDR4xlBxXAwqt0SE9qfr8rIbuA264vwStltPi4RDfTWWBooxFTGWHpu7rS/N1xEg7+48Il0zckVqmbwMJeAfl8abXhqK5RRrpv77icHeqxi5bev/SL28PnNOhj5gXn6a4/1/JPuhzmKp1omNtlJsjjZhs7YidSOVQ2nEJjDoeQ9/d+Z+g6DosnkDbiS7+ijvIPD9SlRbyUlf7Z3wD8Hf91/fUgkaUoIVkBUFvW8i0fArfcWSIQErvixIGhj41yNOwIAheUFM4Y39NsuilJZZmqZ2Zh0/9AHUiY4fA7C6CajFC06CpjUCpDhb3m0uy1vJcUiG7Y3J6iTrRzaW8eqPnGBQcVTS7/dXPj3WtEP/xOF93sYeZMKE7hk489WAUJNLV9kmasSaVTzbu80kfM5Ml9m675dN3ztUAhaAPT0IqHnycngMBKe2Y7+EDalsDBsffOpO0k= 28 | 29 | notifications: 30 | slack: 31 | rooms: 32 | secure: Eh/HI7+7KAUlzFpl9ViMUpExt4YXXWa5rEcwW3h/zIAo/UVghstuoCnk/CWbY/4SyrBeFO7iGcAc396GrBHbkJA25Iz+z3M7VYI/v22di1P8DeWguLNxbe1/wwCAbneYlxLbpPmkOKuFVho5NWDF1QgGszWONRM6DyiFiPlMF84nUcZEHP9exRELGqY64Cg3gfaKMXBn+QSZ5dzNH5cgVrWsGmPSeVBSTWZnZUYHWFKTbgnkY0bs1aHQefRAvl2CCZY/eFPJtcM9oIyIOiuX0NVz7h971tFJdJ9zpTesmaSVOjhCT00mUEJnPPI+zFJxLyIiXHjwROZXzCZL3J21DuWAb3+3ndVe6f47ao/4VNR/Q1TNJGIw5ewkA7RHHW9fegF27OzF+ddfd88ExFvBh1PvJXlAO3c050/2dIqDw7ovhIZWHFchUjSaoV6o5mkgv8/rEu76o3cpdAa78kgZ2tqzDQSquIY55eyHY1vDRusLYii5QZkXzblN9Hm/VvWuM//X99XtoFfnloeLcARWu6ze0fnsBeexK6iQMuUqseLZTjF3tnFqVe2cArA7Swwkh77fRWT3jm35FX1SHaT/kecsMiPY/E7sBfFGHjHVqSfEwMYXhJ6O3aur0UssB+UpuzWu+yBCkf7TienwzimXMrvOGnE++dSOxx6mXRaJBhI= 33 | 34 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Launch", 6 | "type": "node", 7 | "request": "launch", 8 | "program": "${workspaceRoot}/src/index.js", 9 | "stopOnEntry": false, 10 | "args": [], 11 | "cwd": "${workspaceRoot}", 12 | "preLaunchTask": null, 13 | "runtimeExecutable": null, 14 | "runtimeArgs": [ 15 | "--nolazy" 16 | ], 17 | "env": { 18 | "NODE_ENV": "development" 19 | }, 20 | "externalConsole": true, 21 | "sourceMaps": true, 22 | "outDir": "${workspaceRoot}/dist" 23 | } 24 | ] 25 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | // Place your settings in this file to overwrite default and user settings. 2 | { 3 | "editor.tabSize": 2, 4 | "search.exclude": { 5 | "**/node_modules": true, 6 | "**/bower_components": true, 7 | "dist": true, 8 | "lib": true 9 | } 10 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:8 2 | WORKDIR /usr/src/app 3 | 4 | COPY . . 5 | 6 | RUN npm install --unsafe-perm 7 | RUN npm run build 8 | 9 | CMD node dist/index.js 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2016 Luke Tillman 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # KillrVideo Node.js 2 | 3 | A reference application for Node.js developers looking to learn more about using 4 | [Apache Cassandra][cassandra] and [DataStax Enterprise][dse] in their applications and 5 | services. Learn more at [killrvideo.github.io][killrvideo]. 6 | 7 | ## Running Locally using Docker 8 | 9 | Running should be pretty straight-forward if you are familiar with docker and docker-compose 10 | 11 | ``` 12 | > docker-compose pull 13 | > docker-compose build 14 | > docker-compose up -d 15 | ``` 16 | 17 | ## Contributing, Requests for More Examples 18 | 19 | This project will continue to evolve along with Cassandra and you can expect that as 20 | Cassandra, DSE, and the drivers add new features, this application will try and provide 21 | examples of those. We gladly accept any pull requests for bug fixes, new features, etc. and 22 | if you have a request for an example that you don't see in the code currently, feel free to 23 | open an issue here on GitHub or send a message to [@LukeTillman][twitter] on Twitter. 24 | 25 | [cassandra]: http://cassandra.apache.org/ 26 | [dse]: http://www.datastax.com/products/datastax-enterprise 27 | [killrvideo]: https://killrvideo.github.io/ 28 | [getting-started]: https://killrvideo.github.io/getting-started/ 29 | [getting-started-node]: https://killrvideo.github.io/docs/languages/nodejs/ 30 | [twitter]: https://twitter.com/LukeTillman 31 | -------------------------------------------------------------------------------- /config/services.json: -------------------------------------------------------------------------------- 1 | { 2 | "services": { 3 | "web": [ 4 | "web:3000" 5 | ], 6 | "cassandra": [ 7 | "dse" 8 | ], 9 | "dse-search": [ 10 | "dse:8983" 11 | ], 12 | "UploadsService": [ 13 | "backend:50101" 14 | ], 15 | "RatingsService": [ 16 | "backend:50101" 17 | ], 18 | "CommentsService": [ 19 | "backend:50101" 20 | ], 21 | "SearchService": [ 22 | "backend:50101" 23 | ], 24 | "StatisticsService": [ 25 | "backend:50101" 26 | ], 27 | "VideoCatalogService": [ 28 | "backend:50101" 29 | ], 30 | "UserManagementService": [ 31 | "backend:50101" 32 | ], 33 | "SuggestedVideoService": [ 34 | "backend:50101" 35 | ] 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | # NodeJS Backend 4 | backend: 5 | build: . 6 | volumes: 7 | - .:/usr/src/app 8 | depends_on: 9 | - dse 10 | env_file: ./killrvideo.env 11 | environment: 12 | NODE_ENV: development 13 | 14 | # Datastax Enterprise 15 | dse: 16 | image: datastax/dse-server:6.7.0 17 | command: [ -s -g ] 18 | ports: 19 | - "9042:9042" 20 | - "8983:8983" 21 | - "8182:8182" 22 | env_file: ./killrvideo.env 23 | environment: 24 | DS_LICENSE: accept 25 | # Allow DSE to lock memory with mlock 26 | cap_add: 27 | - IPC_LOCK 28 | ulimits: 29 | memlock: -1 30 | 31 | # Frontend 32 | web: 33 | image: killrvideo/killrvideo-web:3.0.1 34 | env_file: ./killrvideo.env 35 | ports: 36 | - "3000:3000" 37 | depends_on: 38 | - dse 39 | 40 | # DSE Configurator to setup DSE 41 | dse-config: 42 | image: killrvideo/killrvideo-dse-config:3.0.0 43 | env_file: ./killrvideo.env 44 | depends_on: 45 | - dse 46 | 47 | # Sample Data Generator 48 | generator: 49 | image: killrvideo/killrvideo-generator:3.0.2 50 | env_file: ./killrvideo.env 51 | depends_on: 52 | - dse 53 | - backend 54 | 55 | # DataStax Studio 56 | studio: 57 | image: datastax/dse-studio:6.7.0 58 | ports: 59 | - "9091:9091" 60 | depends_on: 61 | - dse 62 | environment: 63 | DS_LICENSE: accept 64 | -------------------------------------------------------------------------------- /jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=759670 3 | // for the documentation about the jsconfig.json format 4 | "compilerOptions": { 5 | "target": "es6", 6 | "module": "commonjs", 7 | "allowSyntheticDefaultImports": true 8 | }, 9 | "exclude": [ 10 | "node_modules", 11 | "bower_components", 12 | "jspm_packages", 13 | "tmp", 14 | "temp" 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /killrvideo.env: -------------------------------------------------------------------------------- 1 | KILLRVIDEO_DSE_USERNAME=cassandra 2 | KILLRVIDEO_DSE_PASSWORD=cassandra 3 | KILLRVIDEO_LOGGING_LEVEL=debug 4 | DS_LICENSE: accept 5 | -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2016 Luke Tillman 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/README.md: -------------------------------------------------------------------------------- 1 | # KillrVideo Service Definitions 2 | 3 | The definitions for the services used in KillrVideo, defined using 4 | [Google's Protocol Buffers](https://developers.google.com/protocol-buffers/). This project can be used 5 | along with [grpc](https://github.com/grpc/grpc) to generate client and server code in multiple 6 | programming languages, allowing the KillrVideo backend services to be implemented in those languages. -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/src/comments/comments_events.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package killrvideo.comments.events; 4 | option csharp_namespace = "KillrVideo.Comments.Events"; 5 | 6 | import "google/protobuf/timestamp.proto"; 7 | import "common/common_types.proto"; 8 | 9 | // Message published when a user commented on a video 10 | message UserCommentedOnVideo { 11 | killrvideo.common.Uuid user_id = 1; 12 | killrvideo.common.Uuid video_id = 2; 13 | killrvideo.common.TimeUuid comment_id = 3; 14 | google.protobuf.Timestamp comment_timestamp = 4; 15 | } -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/src/comments/comments_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package killrvideo.comments; 4 | option csharp_namespace = "KillrVideo.Comments"; 5 | 6 | import "google/protobuf/timestamp.proto"; 7 | import "common/common_types.proto"; 8 | 9 | // Manages comments 10 | service CommentsService { 11 | // Add a new comment to a video 12 | rpc CommentOnVideo(CommentOnVideoRequest) returns (CommentOnVideoResponse); 13 | 14 | // Get comments made by a user 15 | rpc GetUserComments(GetUserCommentsRequest) returns (GetUserCommentsResponse); 16 | 17 | // Get comments made on a video 18 | rpc GetVideoComments(GetVideoCommentsRequest) returns (GetVideoCommentsResponse); 19 | } 20 | 21 | // Add a comment to a video 22 | message CommentOnVideoRequest { 23 | killrvideo.common.Uuid video_id = 1; 24 | killrvideo.common.Uuid user_id = 2; 25 | killrvideo.common.TimeUuid comment_id = 3; 26 | string comment = 4; 27 | } 28 | 29 | // Response to adding a comment to a video 30 | message CommentOnVideoResponse { 31 | } 32 | 33 | // Get a page of comments made by a specific user 34 | message GetUserCommentsRequest { 35 | killrvideo.common.Uuid user_id = 1; 36 | int32 page_size = 2; 37 | killrvideo.common.TimeUuid starting_comment_id = 3; 38 | string paging_state = 16; 39 | } 40 | 41 | // Response when getting a page of comments made by a user 42 | message GetUserCommentsResponse { 43 | killrvideo.common.Uuid user_id = 1; 44 | repeated UserComment comments = 2; 45 | string paging_state = 3; 46 | } 47 | 48 | // A comment made by a user 49 | message UserComment { 50 | killrvideo.common.TimeUuid comment_id = 1; 51 | killrvideo.common.Uuid video_id = 2; 52 | string comment = 3; 53 | google.protobuf.Timestamp comment_timestamp = 4; 54 | } 55 | 56 | // Request for getting a page of comments on a video 57 | message GetVideoCommentsRequest { 58 | killrvideo.common.Uuid video_id = 1; 59 | int32 page_size = 2; 60 | killrvideo.common.TimeUuid starting_comment_id = 3; 61 | string paging_state = 16; 62 | } 63 | 64 | // Response when getting a page of comments for a video 65 | message GetVideoCommentsResponse { 66 | killrvideo.common.Uuid video_id = 1; 67 | repeated VideoComment comments = 2; 68 | string paging_state = 3; 69 | } 70 | 71 | // A comment on a video 72 | message VideoComment { 73 | killrvideo.common.TimeUuid comment_id = 1; 74 | killrvideo.common.Uuid user_id = 2; 75 | string comment = 3; 76 | google.protobuf.Timestamp comment_timestamp = 4; 77 | } 78 | -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/src/common/common_types.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package killrvideo.common; 4 | option csharp_namespace = "KillrVideo.Protobuf"; 5 | 6 | // Represents a v4 UUID/GUID 7 | message Uuid { 8 | // Use string for simplicity sake since most programming languages provide a way 9 | // to parse to/from a UUID string (e.g. 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX') 10 | string value = 1; 11 | } 12 | 13 | // Represents a v1 UUID/GUID (i.e. time-based UUID) 14 | message TimeUuid { 15 | // Just like Uuid, use string to represent TimeUuids (see Uuid comment) 16 | string value = 1; 17 | } -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/src/ratings/ratings_events.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package killrvideo.ratings.events; 4 | option csharp_namespace = "KillrVideo.Ratings.Events"; 5 | 6 | import "common/common_types.proto"; 7 | import "google/protobuf/timestamp.proto"; 8 | 9 | // Event published when a user rates a video 10 | message UserRatedVideo { 11 | killrvideo.common.Uuid video_id = 1; 12 | killrvideo.common.Uuid user_id = 2; 13 | int32 rating = 3; 14 | google.protobuf.Timestamp rating_timestamp = 4; 15 | } -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/src/ratings/ratings_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package killrvideo.ratings; 4 | option csharp_namespace = "KillrVideo.Ratings"; 5 | 6 | import "common/common_types.proto"; 7 | 8 | // Service that manages user's ratings of videos 9 | service RatingsService { 10 | // Rate a video 11 | rpc RateVideo(RateVideoRequest) returns (RateVideoResponse); 12 | 13 | // Gets the current rating stats for a video 14 | rpc GetRating(GetRatingRequest) returns (GetRatingResponse); 15 | 16 | // Gets a user's rating of a specific video and returns 0 if the user hasn't rated the video 17 | rpc GetUserRating(GetUserRatingRequest) returns (GetUserRatingResponse); 18 | } 19 | 20 | // Request for a user rating a video 21 | message RateVideoRequest { 22 | killrvideo.common.Uuid video_id = 1; 23 | killrvideo.common.Uuid user_id = 2; 24 | int32 rating = 3; 25 | } 26 | 27 | // Response when a user rates a video 28 | message RateVideoResponse { 29 | } 30 | 31 | // Request to get the ratings stats for a video 32 | message GetRatingRequest { 33 | killrvideo.common.Uuid video_id = 1; 34 | } 35 | 36 | // Response when getting the ratings stats for a video 37 | message GetRatingResponse { 38 | killrvideo.common.Uuid video_id = 1; 39 | int64 ratings_count = 2; 40 | int64 ratings_total = 3; 41 | } 42 | 43 | // Request to get a specific user's rating of a video 44 | message GetUserRatingRequest { 45 | killrvideo.common.Uuid video_id = 1; 46 | killrvideo.common.Uuid user_id = 2; 47 | } 48 | 49 | // Response when getting a specific user's rating of a video 50 | message GetUserRatingResponse { 51 | killrvideo.common.Uuid video_id = 1; 52 | killrvideo.common.Uuid user_id = 2; 53 | int32 rating = 3; 54 | } -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/src/search/search_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package killrvideo.search; 4 | option csharp_namespace = "KillrVideo.Search"; 5 | 6 | import "google/protobuf/timestamp.proto"; 7 | import "common/common_types.proto"; 8 | 9 | // Searches for videos 10 | service SearchService { 11 | // Searches for videos by a given query term 12 | rpc SearchVideos(SearchVideosRequest) returns (SearchVideosResponse); 13 | 14 | // Gets search query suggestions (could be used for typeahead support) 15 | rpc GetQuerySuggestions(GetQuerySuggestionsRequest) returns (GetQuerySuggestionsResponse); 16 | } 17 | 18 | // Request when searching for videos by a query term 19 | message SearchVideosRequest { 20 | string query = 1; 21 | int32 page_size = 2; 22 | string paging_state = 16; 23 | } 24 | 25 | // Response when searching for videos 26 | message SearchVideosResponse { 27 | string query = 1; 28 | repeated SearchResultsVideoPreview videos = 2; 29 | string paging_state = 3; 30 | } 31 | 32 | // A video preview returned in search results 33 | message SearchResultsVideoPreview { 34 | killrvideo.common.Uuid video_id = 1; 35 | google.protobuf.Timestamp added_date = 2; 36 | string name = 3; 37 | string preview_image_location = 4; 38 | killrvideo.common.Uuid user_id = 5; 39 | } 40 | 41 | // Request for getting query suggestions based on some user input 42 | message GetQuerySuggestionsRequest { 43 | string query = 1; 44 | int32 page_size = 2; 45 | } 46 | 47 | // Response with top query suggestions for the user input 48 | message GetQuerySuggestionsResponse { 49 | string query = 1; 50 | repeated string suggestions = 2; 51 | } -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/src/statistics/statistics_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package killrvideo.statistics; 4 | option csharp_namespace = "KillrVideo.Statistics"; 5 | 6 | import "common/common_types.proto"; 7 | 8 | // Service that tracks playback statistics for videos 9 | service StatisticsService { 10 | // Record that playback started for a given video 11 | rpc RecordPlaybackStarted(RecordPlaybackStartedRequest) returns (RecordPlaybackStartedResponse); 12 | 13 | // Get the number of plays for a given video or set of videos 14 | rpc GetNumberOfPlays(GetNumberOfPlaysRequest) returns (GetNumberOfPlaysResponse); 15 | } 16 | 17 | // Request for recording that a user started playing back a video 18 | message RecordPlaybackStartedRequest { 19 | killrvideo.common.Uuid video_id = 1; 20 | } 21 | 22 | // Response when recording that a user started playing back a video 23 | message RecordPlaybackStartedResponse { 24 | } 25 | 26 | // Request for getting the number of times a video or set of videos has been played back 27 | message GetNumberOfPlaysRequest { 28 | repeated killrvideo.common.Uuid video_ids = 1; 29 | } 30 | 31 | // Response when getting playback stats for a video or set of videos 32 | message GetNumberOfPlaysResponse { 33 | repeated PlayStats stats = 1; 34 | } 35 | 36 | // The playback stats for a given video id 37 | message PlayStats { 38 | killrvideo.common.Uuid video_id = 1; 39 | int64 views = 2; 40 | } -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/src/suggested-videos/suggested_videos_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package killrvideo.suggested_videos; 4 | option csharp_namespace = "KillrVideo.SuggestedVideos"; 5 | 6 | import "google/protobuf/timestamp.proto"; 7 | import "common/common_types.proto"; 8 | 9 | // Service responsible for generating video suggestions 10 | service SuggestedVideoService { 11 | // Gets videos related to another video 12 | rpc GetRelatedVideos(GetRelatedVideosRequest) returns (GetRelatedVideosResponse); 13 | 14 | // Gets personalized video suggestions for a user 15 | rpc GetSuggestedForUser(GetSuggestedForUserRequest) returns (GetSuggestedForUserResponse); 16 | } 17 | 18 | // Request to get videos related to another video 19 | message GetRelatedVideosRequest { 20 | killrvideo.common.Uuid video_id = 1; 21 | int32 page_size = 2; 22 | string paging_state = 16; 23 | } 24 | 25 | // Response when getting messages related to another video 26 | message GetRelatedVideosResponse { 27 | killrvideo.common.Uuid video_id = 1; 28 | repeated SuggestedVideoPreview videos = 2; 29 | string paging_state = 3; 30 | } 31 | 32 | // Request to get personalized video suggestions for a user 33 | message GetSuggestedForUserRequest { 34 | killrvideo.common.Uuid user_id = 1; 35 | int32 page_size = 2; 36 | string paging_state = 16; 37 | } 38 | 39 | // Response when getting personalized suggestions for a user 40 | message GetSuggestedForUserResponse { 41 | killrvideo.common.Uuid user_id = 1; 42 | repeated SuggestedVideoPreview videos = 2; 43 | string paging_state = 3; 44 | } 45 | 46 | // Video preview data for a video returned as a video suggestion 47 | message SuggestedVideoPreview { 48 | killrvideo.common.Uuid video_id = 1; 49 | google.protobuf.Timestamp added_date = 2; 50 | string name = 3; 51 | string preview_image_location = 4; 52 | killrvideo.common.Uuid user_id = 5; 53 | } -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/src/uploads/uploads_events.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package killrvideo.uploads.events; 4 | option csharp_namespace = "KillrVideo.Uploads.Events"; 5 | 6 | import "google/protobuf/timestamp.proto"; 7 | import "common/common_types.proto"; 8 | 9 | // Event that's published when there's a problem processing an uploaded video 10 | message UploadedVideoProcessingFailed { 11 | killrvideo.common.Uuid video_id = 1; 12 | google.protobuf.Timestamp timestamp = 2; 13 | } 14 | 15 | // Event that's published when an uploaded video has started being processed 16 | message UploadedVideoProcessingStarted { 17 | killrvideo.common.Uuid video_id = 1; 18 | google.protobuf.Timestamp timestamp = 2; 19 | } 20 | 21 | // Event that's published when an uploaded video has been successfully processed 22 | message UploadedVideoProcessingSucceeded { 23 | killrvideo.common.Uuid video_id = 1; 24 | google.protobuf.Timestamp timestamp = 2; 25 | } 26 | 27 | // Event that's published when an uploaded video is available and ready for playback 28 | message UploadedVideoPublished { 29 | killrvideo.common.Uuid video_id = 1; 30 | google.protobuf.Timestamp timestamp = 2; 31 | string video_url = 3; 32 | string thumbnail_url = 4; 33 | } -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/src/uploads/uploads_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package killrvideo.uploads; 4 | option csharp_namespace = "KillrVideo.Uploads"; 5 | 6 | import "google/protobuf/timestamp.proto"; 7 | import "common/common_types.proto"; 8 | 9 | // Service that handles processing/re-encoding of uploaded videos 10 | service UploadsService { 11 | // Gets an upload destination for a user to upload a video 12 | rpc GetUploadDestination(GetUploadDestinationRequest) returns (GetUploadDestinationResponse); 13 | 14 | // Marks an upload as complete 15 | rpc MarkUploadComplete(MarkUploadCompleteRequest) returns (MarkUploadCompleteResponse); 16 | 17 | // Gets the status of an uploaded video 18 | rpc GetStatusOfVideo(GetStatusOfVideoRequest) returns (GetStatusOfVideoResponse); 19 | } 20 | 21 | // Request to get/generate a location where a video can be uploaded 22 | message GetUploadDestinationRequest { 23 | string file_name = 1; 24 | } 25 | 26 | // Response that has the location where a video can be uploaded 27 | message GetUploadDestinationResponse { 28 | string upload_url = 1; 29 | } 30 | 31 | // Request to tell the upload service that a video is finished uploading 32 | message MarkUploadCompleteRequest { 33 | string upload_url = 1; 34 | } 35 | 36 | // Response when marking an upload complete 37 | message MarkUploadCompleteResponse { 38 | } 39 | 40 | // Get the current status of an uploaded video 41 | message GetStatusOfVideoRequest { 42 | killrvideo.common.Uuid video_id = 1; 43 | } 44 | 45 | // Response with the current status of an uploaded video 46 | message GetStatusOfVideoResponse { 47 | google.protobuf.Timestamp status_date = 1; 48 | string current_state = 2; 49 | } -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/src/user-management/user_management_events.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package killrvideo.user_management.events; 4 | option csharp_namespace = "KillrVideo.UserManagement.Events"; 5 | 6 | import "google/protobuf/timestamp.proto"; 7 | import "common/common_types.proto"; 8 | 9 | // Event published when a new user is created 10 | message UserCreated { 11 | killrvideo.common.Uuid user_id = 1; 12 | string first_name = 2; 13 | string last_name = 3; 14 | string email = 4; 15 | google.protobuf.Timestamp timestamp = 5; 16 | } -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/src/user-management/user_management_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package killrvideo.user_management; 4 | option csharp_namespace = "KillrVideo.UserManagement"; 5 | 6 | import "common/common_types.proto"; 7 | 8 | // The service responsible for managing user information 9 | service UserManagementService { 10 | // Creates a new user 11 | rpc CreateUser(CreateUserRequest) returns (CreateUserResponse); 12 | 13 | // Verify a user's username and password 14 | rpc VerifyCredentials(VerifyCredentialsRequest) returns (VerifyCredentialsResponse); 15 | 16 | // Gets a user or group of user's profiles 17 | rpc GetUserProfile(GetUserProfileRequest) returns (GetUserProfileResponse); 18 | } 19 | 20 | // Request to create a new user 21 | message CreateUserRequest { 22 | killrvideo.common.Uuid user_id = 1; 23 | string first_name = 2; 24 | string last_name = 3; 25 | string email = 4; 26 | string password = 5; 27 | } 28 | 29 | // Response when creating a new user 30 | message CreateUserResponse { 31 | } 32 | 33 | // Request to verify a user's credentials (i.e. for logging them in) 34 | message VerifyCredentialsRequest { 35 | string email = 1; 36 | string password = 2; 37 | } 38 | 39 | // Response that indicates the user's id if the credentials were correct 40 | message VerifyCredentialsResponse { 41 | killrvideo.common.Uuid user_id = 1; 42 | } 43 | 44 | // Request to get a user or multiple users profiles 45 | message GetUserProfileRequest { 46 | repeated killrvideo.common.Uuid user_ids = 1; 47 | } 48 | 49 | // Response with user profiles 50 | message GetUserProfileResponse { 51 | repeated UserProfile profiles = 1; 52 | } 53 | 54 | // A user's profile information 55 | message UserProfile { 56 | killrvideo.common.Uuid user_id = 1; 57 | string first_name = 2; 58 | string last_name = 3; 59 | string email = 4; 60 | } -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/src/video-catalog/video_catalog_events.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package killrvideo.video_catalog.events; 4 | option csharp_namespace = "KillrVideo.VideoCatalog.Events"; 5 | 6 | import "google/protobuf/timestamp.proto"; 7 | import "common/common_types.proto"; 8 | 9 | // Event published when uploaded video has been accepted by the catalog, but isn't ready for playback yet 10 | message UploadedVideoAccepted { 11 | killrvideo.common.Uuid video_id = 1; 12 | string upload_url = 2; 13 | google.protobuf.Timestamp timestamp = 3; 14 | } 15 | 16 | // Event published when an uploaded video has been added to the catalog and is ready for playback 17 | message UploadedVideoAdded { 18 | killrvideo.common.Uuid video_id = 1; 19 | killrvideo.common.Uuid user_id = 2; 20 | string name = 3; 21 | string description = 4; 22 | string location = 5; 23 | string preview_image_location = 6; 24 | repeated string tags = 7; 25 | google.protobuf.Timestamp added_date = 8; 26 | google.protobuf.Timestamp timestamp = 9; 27 | } 28 | 29 | // Event published when a YouTube video has been added to the catalog and is ready for playback 30 | message YouTubeVideoAdded { 31 | killrvideo.common.Uuid video_id = 1; 32 | killrvideo.common.Uuid user_id = 2; 33 | string name = 3; 34 | string description = 4; 35 | string location = 5; 36 | string preview_image_location = 6; 37 | repeated string tags = 7; 38 | google.protobuf.Timestamp added_date = 8; 39 | google.protobuf.Timestamp timestamp = 9; 40 | } -------------------------------------------------------------------------------- /lib/killrvideo-service-protos/src/video-catalog/video_catalog_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package killrvideo.video_catalog; 4 | option csharp_namespace = "KillrVideo.VideoCatalog"; 5 | 6 | import "google/protobuf/timestamp.proto"; 7 | import "common/common_types.proto"; 8 | 9 | // Service responsible for tracking the catalog of available videos for playback 10 | service VideoCatalogService { 11 | // Submit an uploaded video to the catalog 12 | rpc SubmitUploadedVideo(SubmitUploadedVideoRequest) returns (SubmitUploadedVideoResponse); 13 | 14 | // Submit a YouTube video to the catalog 15 | rpc SubmitYouTubeVideo(SubmitYouTubeVideoRequest) returns (SubmitYouTubeVideoResponse); 16 | 17 | // Gets a video from the catalog 18 | rpc GetVideo(GetVideoRequest) returns (GetVideoResponse); 19 | 20 | // Gets video previews for a limited number of videos from the catalog 21 | rpc GetVideoPreviews(GetVideoPreviewsRequest) returns (GetVideoPreviewsResponse); 22 | 23 | // Gets video previews for the latest (i.e. newest) videos from the catalog 24 | rpc GetLatestVideoPreviews(GetLatestVideoPreviewsRequest) returns (GetLatestVideoPreviewsResponse); 25 | 26 | // Gets video previews for videos added to the site by a particular user 27 | rpc GetUserVideoPreviews(GetUserVideoPreviewsRequest) returns (GetUserVideoPreviewsResponse); 28 | } 29 | 30 | // Request to submit a new uploaded video to the catalog 31 | message SubmitUploadedVideoRequest { 32 | killrvideo.common.Uuid video_id = 1; 33 | killrvideo.common.Uuid user_id = 2; 34 | string name = 3; 35 | string description = 4; 36 | repeated string tags = 5; 37 | string upload_url = 6; 38 | } 39 | 40 | // Response when submitting a new uploaded video to the catalog 41 | message SubmitUploadedVideoResponse { 42 | } 43 | 44 | // Request to submit a new YouTube video to the catalog 45 | message SubmitYouTubeVideoRequest { 46 | killrvideo.common.Uuid video_id = 1; 47 | killrvideo.common.Uuid user_id = 2; 48 | string name = 3; 49 | string description = 4; 50 | repeated string tags = 5; 51 | string you_tube_video_id = 6; 52 | } 53 | 54 | // Response when submitting a new YouTube video to the catalog 55 | message SubmitYouTubeVideoResponse { 56 | } 57 | 58 | // Request to get a video and all its details from the catalog 59 | message GetVideoRequest { 60 | killrvideo.common.Uuid video_id = 1; 61 | } 62 | 63 | // Response when getting a video and all its details from the catalog 64 | message GetVideoResponse { 65 | killrvideo.common.Uuid video_id = 1; 66 | killrvideo.common.Uuid user_id = 2; 67 | string name = 3; 68 | string description = 4; 69 | string location = 5; 70 | VideoLocationType location_type = 6; 71 | repeated string tags = 7; 72 | google.protobuf.Timestamp added_date = 8; 73 | } 74 | 75 | // Enum representing what kind of video location is present for a video 76 | enum VideoLocationType { 77 | YOUTUBE = 0; 78 | UPLOAD = 1; 79 | } 80 | 81 | // Request for getting some video previews by the video ids 82 | message GetVideoPreviewsRequest { 83 | repeated killrvideo.common.Uuid video_ids = 1; 84 | } 85 | 86 | // Response when getting some video previews by their ids 87 | message GetVideoPreviewsResponse { 88 | repeated VideoPreview video_previews = 1; 89 | } 90 | 91 | // A video preview (i.e. limited details about a video) 92 | message VideoPreview { 93 | killrvideo.common.Uuid video_id = 1; 94 | google.protobuf.Timestamp added_date = 2; 95 | string name = 3; 96 | string preview_image_location = 4; 97 | killrvideo.common.Uuid user_id = 5; 98 | } 99 | 100 | // Request for getting a page of the latest (i.e. newest) videos in the catalog 101 | message GetLatestVideoPreviewsRequest { 102 | int32 page_size = 1; 103 | google.protobuf.Timestamp starting_added_date = 2; 104 | killrvideo.common.Uuid starting_video_id = 3; 105 | string paging_state = 16; 106 | } 107 | 108 | // Response when getting a page of the latest videos in the catalog 109 | message GetLatestVideoPreviewsResponse { 110 | repeated VideoPreview video_previews = 1; 111 | string paging_state = 2; 112 | } 113 | 114 | // Request for getting videos a particular user has added to the catalog 115 | message GetUserVideoPreviewsRequest { 116 | killrvideo.common.Uuid user_id = 1; 117 | int32 page_size = 2; 118 | google.protobuf.Timestamp starting_added_date = 3; 119 | killrvideo.common.Uuid starting_video_id = 4; 120 | string paging_state = 16; 121 | } 122 | 123 | // Response when getting videos for a particular user 124 | message GetUserVideoPreviewsResponse { 125 | killrvideo.common.Uuid user_id = 1; 126 | repeated VideoPreview video_previews = 2; 127 | string paging_state = 3; 128 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "killrvideo-nodejs", 3 | "version": "0.0.0", 4 | "description": "Reference application for using Cassandra, DataStax Enterprise, and NodeJS", 5 | "main": "dist/index.js", 6 | "scripts": { 7 | "start": "node dist/index.js", 8 | "clean": "rimraf dist", 9 | "build": "npm-run-all --parallel \"build:*\"", 10 | "build:js": "babel src --out-dir dist --source-maps", 11 | "build:protos": "cpx \"lib/killrvideo-service-protos/src/**/*\" dist/protos --verbose", 12 | "build:googleprotos": "node scripts/copy-google-protos.js dist/protos", 13 | "watch": "npm-run-all clean --parallel \"build:* -- --watch\"" 14 | }, 15 | "repository": { 16 | "type": "git", 17 | "url": "git+ssh://git@github.com/KillrVideo/killrvideo-nodejs.git" 18 | }, 19 | "author": "Luke Tillman (http://www.luketillman.com)", 20 | "license": "Apache-2.0", 21 | "bugs": { 22 | "url": "https://github.com/KillrVideo/killrvideo-nodejs/issues" 23 | }, 24 | "homepage": "https://killrvideo.github.io", 25 | "private": true, 26 | "dependencies": { 27 | "async": "^2.5.0", 28 | "bluebird": "^3.5.1", 29 | "cassandra-driver": "^3.3.0", 30 | "convict": "^4.0.1", 31 | "dotenv": "^4.0.0", 32 | "grpc": "^1.7.1", 33 | "moment": "^2.19.1", 34 | "regenerator-runtime": "^0.11.0", 35 | "request": "^2.83.0", 36 | "request-promise": "^4.2.2", 37 | "winston": "^2.4.0" 38 | }, 39 | "devDependencies": { 40 | "babel-cli": "^6.26.0", 41 | "babel-plugin-transform-async-to-generator": "^6.24.1", 42 | "babel-preset-env": "^1.6.1", 43 | "cpx": "^1.5.0", 44 | "grpc-tools": "^1.6.6", 45 | "npm-run-all": "^4.1.1", 46 | "rimraf": "^2.6.2" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /scripts/copy-google-protos.js: -------------------------------------------------------------------------------- 1 | var path = require('path'); 2 | var cpx = require('cpx'); 3 | 4 | // Resolve location of grpc-tools package, then add location of google .proto files 5 | var grpcToolsPath = path.dirname(require.resolve('grpc-tools')); 6 | var googlePath = path.resolve(grpcToolsPath, 'bin'); 7 | 8 | var srcGlob = googlePath + path.sep + '**' + path.sep + '*.proto'; 9 | 10 | // Copy all protocol buffers files to destination specified 11 | console.log('Copying "' + srcGlob + '" to "' + process.argv[2] + '"'); 12 | cpx.copySync(srcGlob, process.argv[2]); 13 | -------------------------------------------------------------------------------- /scripts/git-subtree-pull.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | .DESCRIPTION 3 | Pull the latest from the projects that are git subtrees. 4 | #> 5 | 6 | git subtree pull --prefix lib/killrvideo-docker-common git@github.com:KillrVideo/killrvideo-docker-common.git master --squash 7 | git subtree pull --prefix lib/killrvideo-service-protos git@github.com:KillrVideo/killrvideo-service-protos.git master --squash -------------------------------------------------------------------------------- /scripts/git-subtree-pull.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Pull the latest from projects that are git subtrees 4 | git subtree pull --prefix lib/killrvideo-docker-common git@github.com:KillrVideo/killrvideo-docker-common.git master --squash 5 | git subtree pull --prefix lib/killrvideo-service-protos git@github.com:KillrVideo/killrvideo-service-protos.git master --squash -------------------------------------------------------------------------------- /src/common/cassandra.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import { Client, types as CassandraTypes } from 'cassandra-driver'; 3 | import { lookupServiceAsync } from './service-discovery'; 4 | import { withRetries } from './with-retries'; 5 | import { logger } from './logging'; 6 | 7 | /** 8 | * Looks up the location of the cassandra service and creates a client with the options/keyspace 9 | * specified. Returns a Promise of the created client. 10 | */ 11 | function createClientAsync(keyspace, queryOptions) { 12 | return lookupServiceAsync('cassandra') 13 | .then(contactPoints => { 14 | let client = new Client({ contactPoints, keyspace, queryOptions }); 15 | Promise.promisifyAll(client); // This creates "Async" versions of methods that return promises 16 | return client; 17 | }); 18 | } 19 | 20 | // A singleton client instance to be reused throughout the application 21 | let clientInstance = null; 22 | 23 | /** 24 | * Gets a Cassandra client instance. 25 | */ 26 | export function getCassandraClient() { 27 | if (clientInstance === null) { 28 | throw new Error('No client instance found. Did you forget to call initCassandraAsync?'); 29 | } 30 | 31 | return clientInstance; 32 | }; 33 | 34 | /** 35 | * Initializes Cassandra by creating any keyspace and tables necessary, and making sure the 36 | * client can connect to the cluster. Should be called before using the getCassandraClient 37 | * method in this module. Returns a Promise. 38 | */ 39 | export function initCassandraAsync() { 40 | logger.log('info', 'Initializing cassandra...'); 41 | 42 | // Create the client with some default query options 43 | return createClientAsync('killrvideo', { 44 | prepare: true, 45 | consistency: CassandraTypes.consistencies.localQuorum 46 | }) 47 | .tap(client => { 48 | // Wait until Cassandra is ready and we can connect (could be delayed if starting up for 1st time) 49 | return withRetries(() => client.connectAsync(), 10, 10, 'Connecting to cassandra...', false, true); 50 | }) 51 | .tap(client => { 52 | // Wait until Cassandra is bootstrapped and we can use it (dse-config needs time to initialise it) 53 | return withRetries(() => 54 | new Promise ( 55 | function(resolve, reject){ 56 | client.execute( 57 | 'SELECT keyspace_name FROM system_schema.keyspaces WHERE keyspace_name=\'kv_init_done\';', [], [], 58 | function(err, result) { 59 | if (err || result.rowLength != 1) { 60 | reject(new Error('DB is not initialised')); 61 | } 62 | resolve(); 63 | } 64 | ) 65 | } 66 | ), 10, 10, 'Waiting for dse-config to bootstrap cassandra...', false, false 67 | ); 68 | }) 69 | .tap(client => { 70 | // Save client instance for reuse everywhere and log 71 | clientInstance = client; 72 | logger.log('info', 'Cassandra initialized') 73 | }); 74 | }; 75 | -------------------------------------------------------------------------------- /src/common/config.js: -------------------------------------------------------------------------------- 1 | import convict from 'convict'; 2 | 3 | // Create application config 4 | const conf = convict({ 5 | loggingLevel: { 6 | doc: 'The default logger output level', 7 | format: String, 8 | default: 'verbose', 9 | env: 'KILLRVIDEO_LOGGING_LEVEL' 10 | }, 11 | 12 | dseEnabled: { 13 | doc: 'Whether or not to use DataStax Enterprise service implementations of services', 14 | format: Boolean, 15 | default: true, 16 | env: 'KILLRVIDEO_DSE_ENABLED' 17 | }, 18 | 19 | appName: { 20 | doc: 'The name of this application', 21 | format: String, 22 | default: 'killrvideo-nodejs', 23 | env: 'NODE_APP' 24 | }, 25 | 26 | appInstance: { 27 | doc: 'A unique instance number for this application', 28 | format: 'nat', // Natural number (positive integer) 29 | default: 1, 30 | env: 'NODE_APP_INSTANCE' 31 | }, 32 | 33 | services: { 34 | doc: 'Service definitions', 35 | format: 'Object', 36 | default: {} 37 | }, 38 | 39 | listen: { 40 | ip: { 41 | doc: 'The IP address for Grpc services to listen on', 42 | format: 'ipaddress', 43 | default: '0.0.0.0', 44 | env: 'KILLRVIDEO_LISTEN_IP' 45 | }, 46 | 47 | port: { 48 | doc: 'The Port for Grpc services to listen on', 49 | format: 'port', 50 | default: 50101, 51 | env: 'KILLRVIDEO_LISTEN_PORT' 52 | } 53 | } 54 | }); 55 | 56 | conf.loadFile('./config/services.json'); 57 | 58 | // Validate the config and export it 59 | conf.validate(); 60 | 61 | export default conf; 62 | -------------------------------------------------------------------------------- /src/common/extendable-error.js: -------------------------------------------------------------------------------- 1 | /** 2 | * An ES6 class that can be used to create custom error classes 3 | */ 4 | export class ExtendableError extends Error { 5 | constructor(message) { 6 | super(); 7 | this.message = message; 8 | this.stack = (new Error()).stack; 9 | this.name = this.constructor.name; 10 | } 11 | }; 12 | 13 | export default ExtendableError; -------------------------------------------------------------------------------- /src/common/logging.js: -------------------------------------------------------------------------------- 1 | import { Logger, transports as CoreTransports } from 'winston'; 2 | 3 | /** 4 | * The default winston logger instance. 5 | */ 6 | export const logger = new Logger({ 7 | transports: [ new CoreTransports.Console({ level: 'verbose', timestamp: true, colorize: true, stderrLevels: [ 'error' ] }) ] 8 | }); 9 | 10 | /** 11 | * Adjust the logging level of the default logger instance. 12 | */ 13 | export function setLoggingLevel(level) { 14 | logger.transports.console.level = level; 15 | }; 16 | 17 | export default logger; -------------------------------------------------------------------------------- /src/common/message-bus.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import { logger } from './logging'; 3 | import util from 'util'; 4 | 5 | /** 6 | * Recursively gets the fully qualified name of a probuf.js reflection value 7 | */ 8 | function getFullyQualifiedName(value) { 9 | if (value === null || value === undefined) { 10 | return ''; 11 | } 12 | 13 | let name = value.name; 14 | const parentName = getFullyQualifiedName(value.parent); 15 | if (parentName !== '') { 16 | name = parentName + '.' + name; 17 | } 18 | return name; 19 | } 20 | 21 | /** 22 | * Map of subscribers by event fully-qualified name. 23 | */ 24 | const handlersByEventName = {}; 25 | 26 | /** 27 | * Publishes an event to the message bus and returns a Promise that is resolved 28 | * when the publish is complete. The event object should be a Protobuf message. 29 | */ 30 | export function publishAsync(event) { 31 | return Promise.try(() => { 32 | let eventName = getFullyQualifiedName(event.$type); 33 | 34 | // Get handlers (or empty array if none) 35 | let handlers = handlersByEventName[eventName] || []; 36 | 37 | logger.log('debug', `Publish ${eventName} to ${handlers.length} handlers`); 38 | logger.log('debug', util.inspect(event)); 39 | 40 | // Invoke each handler with event 41 | return handlers.map(h => h(event)); 42 | }) 43 | .all() // Publish is complete when all handlers are done 44 | .return(); // Just return undefined value 45 | }; 46 | 47 | /** 48 | * Publishes an event to the message bus and invokes the specified callback when 49 | * complete. The event object should be a Protobuf message. 50 | */ 51 | export function publish(event, cb) { 52 | publishAsync(event).asCallback(cb); 53 | }; 54 | 55 | /** 56 | * Subscribes the specified handler to the specified event type on the message bus. 57 | * Returns a Promise that is resolved when subscribe is complete. The eventType 58 | * should be a Protobuf message definition. 59 | */ 60 | export function subscribeAsync(eventType, handler) { 61 | return Promise.try(() => { 62 | let eventName = getFullyQualifiedName(eventType.$type); 63 | 64 | // Get handlers (or empty array if none) and add handler, wrapping with Promise.method since it could be sync or async 65 | let handlers = handlersByEventName[eventName] || []; 66 | handlers.push(Promise.method(handler)); 67 | 68 | logger.log('debug', `Subscribe ${eventName} handler ${handlers.length}`); 69 | 70 | // Make sure index is updated in case we created a new array 71 | handlersByEventName[eventName] = handlers; 72 | }); 73 | }; 74 | 75 | /** 76 | * Subscribes the specified handler to the specified event type on the message bus and 77 | * invokes the callback when complete. The eventType should be a Profobuf message 78 | * definition. 79 | */ 80 | export function subscribe(eventType, handler, cb) { 81 | subscribeAsync(eventType, handler).asCallback(cb); 82 | }; -------------------------------------------------------------------------------- /src/common/service-discovery.js: -------------------------------------------------------------------------------- 1 | import { logger } from './logging'; 2 | import Promise from 'bluebird'; 3 | import { ExtendableError } from './extendable-error'; 4 | import config from './config'; 5 | 6 | let registry = config.get('services'); 7 | 8 | /** 9 | * Error thrown when a service can't be found 10 | */ 11 | export class ServiceNotFoundError extends ExtendableError { 12 | constructor(serviceName) { 13 | super(`Could not find service ${serviceName}`); 14 | } 15 | }; 16 | 17 | /** 18 | * Looks up a service with a given name. Returns a Promise with an array of strings in the format of 'ip:port' or throws ServiceNotFoundError. 19 | */ 20 | export function lookupServiceAsync(serviceName) { 21 | logger.log('verbose', `Looking up service ${serviceName}`); 22 | 23 | if (!(serviceName in registry)) { 24 | logger.log('error', `Found no service ${serviceName}`); 25 | throw new ServiceNotFoundError(serviceName); 26 | } 27 | 28 | logger.log('verbose', `Found service ${serviceName} at ${registry[serviceName]}`); 29 | 30 | return new Promise (function(resolve, reject){resolve(registry[serviceName])}); 31 | }; 32 | -------------------------------------------------------------------------------- /src/common/with-retries.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import { logger } from './logging'; 3 | 4 | /** 5 | * Do a promise returning function with retries. 6 | */ 7 | export function withRetries(promiseFn, maxRetries, delaySeconds, errMsg, expBackoff, suppressError = false) { 8 | let retryCount = 0; 9 | 10 | function doIt() { 11 | return promiseFn().catch(err => { 12 | // If we've hit the max, just propagate the error 13 | if (retryCount >= maxRetries) { 14 | throw err; 15 | } 16 | 17 | // Calculate delay time in MS 18 | let delayMs = expBackoff === true 19 | ? Math.pow(delaySeconds, retryCount) * 1000 20 | : delaySeconds * 1000; 21 | 22 | // Log, delay, and try again 23 | retryCount++; 24 | 25 | logger.log('verbose', `${errMsg}. Retry ${retryCount} in ${delayMs}ms.`); 26 | if (! suppressError) { 27 | logger.log('debug', '', err); 28 | } 29 | 30 | return Promise.delay(delayMs).then(doIt); 31 | }); 32 | } 33 | 34 | return doIt(); 35 | }; 36 | -------------------------------------------------------------------------------- /src/data/README.md: -------------------------------------------------------------------------------- 1 | This folder is just a placeholder. The actual data files (like the `schema.cql`) are copied 2 | as part of the build scripts. See the `package.json` file under the `scripts` property for 3 | those commands. -------------------------------------------------------------------------------- /src/grpc/server.js: -------------------------------------------------------------------------------- 1 | import EventEmitter from 'events'; 2 | import Promise from 'bluebird'; 3 | import { Server, ServerCredentials } from 'grpc'; 4 | import { logger } from '../common/logging'; 5 | import config from '../common/config'; 6 | 7 | /** 8 | * Class that wraps an underlying Grpc server object and allows it to be started/stopped. Emits 9 | * 'start' and 'stop' events for when this happens. 10 | */ 11 | export class GrpcServer extends EventEmitter { 12 | constructor(services) { 13 | super(); 14 | this._services = services; 15 | this._stopAsync = null; 16 | } 17 | 18 | /** 19 | * Start the server and all Grpc services. 20 | */ 21 | start() { 22 | // Already started? 23 | if (this._stopAsync !== null) return; 24 | 25 | let server = new Server(); 26 | 27 | // Add all available services to the Grpc Server 28 | this._services.forEach(service => { 29 | logger.log('debug', `Adding Grpc service for ${service.service.name}`); 30 | server.addService(service.service, service.implementation); 31 | }); 32 | 33 | // Figure out where to listen 34 | let listen = config.get('listen'); 35 | let ipAndPort = `${listen.ip}:${listen.port}`; 36 | 37 | // Bind the server and start all services 38 | server.bind(ipAndPort, ServerCredentials.createInsecure()); 39 | 40 | // Save stop method 41 | this._stopAsync = Promise.promisify(server.tryShutdown, { context: server }); 42 | 43 | logger.log('debug', `Starting Grpc server on ${ipAndPort}`); 44 | server.start(); 45 | this._emitEvent('start'); 46 | logger.log('debug', 'Started Grpc server'); 47 | } 48 | 49 | /** 50 | * Stop the server, trying to allow all outstanding requests to complete. Returns a Promise 51 | * that resolves once stopped. 52 | */ 53 | stopAsync() { 54 | // Not started? 55 | if (this._stopAsync === null) return; 56 | 57 | logger.log('debug', 'Stopping Grpc server'); 58 | 59 | // Stop, then let everyone know we're stopped 60 | return this._stopAsync() 61 | .then(() => { 62 | this._emitEvent('stop'); 63 | return null; // In case any listeners are creating promises, return null to prevent bluebird warning 64 | }) 65 | .finally(() => logger.log('debug', 'Stopped Grpc server')); 66 | } 67 | 68 | _emitEvent(eventName) { 69 | this.emit(eventName, this._services.map(s => s.service)); 70 | } 71 | }; 72 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | // Import the regenerator runtime so async generators work in the transpiled code 2 | import 'regenerator-runtime/runtime'; 3 | 4 | // Regular imports 5 | import { createInterface } from 'readline'; 6 | import Promise from 'bluebird'; 7 | import config from './common/config'; 8 | import { initCassandraAsync } from './common/cassandra'; 9 | import { logger, setLoggingLevel } from './common/logging'; 10 | import { lookupServiceAsync } from './common/service-discovery'; 11 | import { subscribeAsync } from './common/message-bus'; 12 | import { GrpcServer } from './grpc/server'; 13 | import { services } from './services'; 14 | import { listeners } from './server-listeners'; 15 | 16 | // Allow bluebird promise cancellation 17 | Promise.config({ cancellation: true }); 18 | 19 | /** 20 | * Start the application and all Grpc services. 21 | */ 22 | function startAsync() { 23 | // Initialize logging 24 | let loggingLevel = config.get('loggingLevel'); 25 | setLoggingLevel(loggingLevel); 26 | logger.log(loggingLevel, `Logging initialized at ${loggingLevel}`); 27 | 28 | // Start by initializing cassandra 29 | return initCassandraAsync() 30 | // Register all service event handlers 31 | .then(() => { 32 | return services.reduce((allSubs, serviceDef) => { 33 | // If service has no event handlers, just continue 34 | if (serviceDef.hasOwnProperty('handlers') === false) { 35 | return allSubs; 36 | } 37 | 38 | // Subscribe and add subscription promise to allSubs 39 | serviceDef.handlers.forEach(handlerDef => { 40 | allSubs.push(subscribeAsync(handlerDef.eventType, handlerDef.handler)); 41 | }); 42 | return allSubs; 43 | }, []); 44 | }) 45 | // Wait for all subscriptions to finish 46 | .all() 47 | // Find the web UI's host and port 48 | .then(() => lookupServiceAsync('web')) 49 | // Start the Grpc server to process requests 50 | .then(webIpAndPorts => { 51 | // Create a Grpc server and register all listeners 52 | logger.log('info', 'Starting all Grpc services'); 53 | 54 | let server = new GrpcServer(services); 55 | listeners.forEach(listener => listener(server)); 56 | 57 | // Start the server and return it 58 | server.start(); 59 | logger.log('info', `Open http://${webIpAndPorts[0]} in a web browser to see the UI`); 60 | logger.log('info', 'KillrVideo has started. Press Ctrl+C to exit.'); 61 | return server; 62 | }) 63 | .catch(err => { 64 | // Use console to log error since logger might write asynchronously 65 | console.error(err); 66 | process.exit(1); 67 | }); 68 | } 69 | 70 | let startPromise = startAsync(); 71 | 72 | /** 73 | * Handle stopping everything. 74 | */ 75 | function stop() { 76 | logger.log('info', 'Attempting to shutdown'); 77 | if (startPromise.isFulfilled()) { 78 | let server = startPromise.value(); 79 | server.stopAsync().then(() => process.exit(0)); 80 | } else { 81 | startPromise.cancel(); 82 | process.exit(0); 83 | } 84 | } 85 | 86 | // Try to gracefully shutdown on SIGTERM and SIGINT 87 | process.on('SIGTERM', stop); 88 | process.on('SIGINT', stop); 89 | 90 | // Graceful shutdown attempt in Windows 91 | if (process.platform === 'win32') { 92 | // Simulate SIGINT on Windows (see http://stackoverflow.com/questions/10021373/what-is-the-windows-equivalent-of-process-onsigint-in-node-js) 93 | createInterface({ 94 | input: process.stdin, 95 | output: process.stdout 96 | }) 97 | .on('SIGINT', () => process.emit('SIGINT')); 98 | } 99 | -------------------------------------------------------------------------------- /src/protos/README.md: -------------------------------------------------------------------------------- 1 | This folder is just a placeholder. The actual Protocol Buffers files are copied from their 2 | respective `/lib` folders as part of the build scripts. See the `package.json` file under 3 | the `scripts` property for those commands. -------------------------------------------------------------------------------- /src/server-listeners/index.js: -------------------------------------------------------------------------------- 1 | import { logServices } from './log-services'; 2 | 3 | /** 4 | * An array of all listener registration functions that want to subscribe to Grpc server events. 5 | */ 6 | export const listeners = [ 7 | logServices, 8 | ]; 9 | 10 | export default listeners; -------------------------------------------------------------------------------- /src/server-listeners/log-services.js: -------------------------------------------------------------------------------- 1 | import { logger } from '../common/logging'; 2 | import config from '../common/config'; 3 | import {serviceNameFromDefinition} from '../services'; 4 | 5 | /** 6 | * Listener that will log all services and the host/port they're running on. 7 | */ 8 | export function logServices(grpcServer) { 9 | let listen = config.get('listen'); 10 | let ipAndPort = `${listen.ip}:${listen.port}`; 11 | 12 | grpcServer.on('start', function logServicesOnStart(services) { 13 | services.forEach(s => { 14 | logger.log('verbose', `Service ${serviceNameFromDefinition(s)} is listening on ${ipAndPort}`); 15 | }); 16 | }); 17 | }; 18 | -------------------------------------------------------------------------------- /src/services/comments/comment-on-video.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import { types as CassandraTypes } from 'cassandra-driver'; 3 | import { toCassandraUuid, toCassandraTimeUuid, toProtobufTimestamp } from '../common/protobuf-conversions'; 4 | import { getCassandraClient } from '../../common/cassandra'; 5 | import { publishAsync } from '../../common/message-bus'; 6 | import { CommentOnVideoResponse } from './protos'; 7 | import { UserCommentedOnVideo } from './events'; 8 | 9 | /** 10 | * Records a user comment on a video. 11 | */ 12 | export function commentOnVideo(call, cb) { 13 | // Invoke async function and convert to bluebird promise, then invoke callback appropriately when complete 14 | return Promise.resolve(commentOnVideoImpl(call)).asCallback(cb); 15 | }; 16 | 17 | // The actual method implementation using async/await 18 | async function commentOnVideoImpl(call) { 19 | let { request } = call; 20 | 21 | // UTC timestamp of the comment 22 | let timestamp = new Date(Date.now()); 23 | 24 | // Convert some request values to Cassandra values 25 | let videoId = toCassandraUuid(request.videoId); 26 | let userId = toCassandraUuid(request.userId); 27 | let commentId = toCassandraTimeUuid(request.commentId); 28 | 29 | // Create an array of statement to execute as a batch 30 | let queries = [ 31 | { 32 | query: 'INSERT INTO comments_by_video (videoid, commentid, userid, comment) VALUES (?, ?, ?, ?)', 33 | params: [ videoId, commentId, userId, request.comment ] 34 | }, 35 | { 36 | query: 'INSERT INTO comments_by_user (userid, commentid, videoid, comment) VALUES (?, ?, ?, ?)', 37 | params: [ userId, commentId, videoId, request.comment ] 38 | } 39 | ]; 40 | 41 | // Use a client-side timestamp for the batch 42 | let queryOpts = { timestamp: CassandraTypes.generateTimestamp(timestamp) }; 43 | 44 | // Execute the batch using our Promise-returning async method 45 | let client = getCassandraClient(); 46 | await client.batchAsync(queries, queryOpts); 47 | 48 | // Tell the world about the new comment 49 | let event = new UserCommentedOnVideo({ 50 | userId: request.userId, 51 | videoId: request.videoId, 52 | commentId: request.commentId, 53 | commentTimestamp: toProtobufTimestamp(timestamp) 54 | }); 55 | await publishAsync(event); 56 | 57 | // Return the response 58 | return new CommentOnVideoResponse(); 59 | } 60 | 61 | -------------------------------------------------------------------------------- /src/services/comments/events.js: -------------------------------------------------------------------------------- 1 | import { load } from '../common/load'; 2 | 3 | // Load events published by this service 4 | const eventsFile = 'comments/comments_events.proto'; 5 | const { 6 | UserCommentedOnVideo 7 | } = load(eventsFile).killrvideo.comments.events; 8 | 9 | export { 10 | UserCommentedOnVideo 11 | }; -------------------------------------------------------------------------------- /src/services/comments/get-user-comments.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import { toCassandraUuid, toCassandraTimeUuid, toProtobufTimestamp, toProtobufUuid } from '../common/protobuf-conversions'; 3 | import { getCassandraClient } from '../../common/cassandra'; 4 | import { GetUserCommentsResponse, UserComment } from './protos'; 5 | 6 | /** 7 | * Gets a page of the latest comments for a user. 8 | */ 9 | export function getUserComments(call, cb) { 10 | // Use async function, convert to bluebird promise, and invoke callback when finished 11 | return Promise.resolve(getUserCommentsImpl(call)).asCallback(cb); 12 | }; 13 | 14 | // Actual implementation using async/await 15 | async function getUserCommentsImpl(call) { 16 | let { request } = call; 17 | 18 | let startingCommentId = toCassandraTimeUuid(request.startingCommentId); 19 | let userId = toCassandraUuid(request.userId); 20 | 21 | // Choose which CQL to execute based on whether we have a starting point for the list of comments 22 | let query, queryParams; 23 | if (startingCommentId === null) { 24 | // No starting point so just get the latest comments overall 25 | query = 'SELECT commentid, videoid, comment, dateOf(commentid) AS comment_timestamp FROM comments_by_user WHERE userid = ?'; 26 | queryParams = [ userId ]; 27 | } else { 28 | // Since commentId is a TimeUuid, we can get that comment and older comments given the starting commentId by using <= 29 | query = 'SELECT commentid, videoid, comment, dateOf(commentid) AS comment_timestamp FROM comments_by_user WHERE userid = ? AND commentid <= ?'; 30 | queryParams = [ userId, startingCommentId ]; 31 | } 32 | 33 | // Set the options for the query, including paging state from previous page if provided in the request 34 | let queryOpts = { autoPage: false, fetchSize: request.pageSize }; 35 | if (request.pagingState !== null && request.pagingState !== '') { 36 | queryOpts.pageState = request.pagingState; 37 | } 38 | 39 | // Execute the query using our Promise returning async method 40 | let client = getCassandraClient(); 41 | let resultSet = await client.executeAsync(query, queryParams, queryOpts); 42 | 43 | // Build the response object from the rows returned 44 | return new GetUserCommentsResponse({ 45 | userId: request.userId, 46 | comments: resultSet.rows.map(row => new UserComment({ 47 | commentId: toProtobufUuid(row.commentid), 48 | videoId: toProtobufUuid(row.videoid), 49 | comment: row.comment, 50 | commentTimestamp: toProtobufTimestamp(row.comment_timestamp) 51 | })), 52 | pagingState: resultSet.pageState !== null ? resultSet.pageState : '' 53 | }); 54 | } -------------------------------------------------------------------------------- /src/services/comments/get-video-comments.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import { toCassandraUuid, toCassandraTimeUuid, toProtobufTimestamp, toProtobufUuid } from '../common/protobuf-conversions'; 3 | import { getCassandraClient } from '../../common/cassandra'; 4 | import { GetVideoCommentsResponse, VideoComment } from './protos'; 5 | 6 | /** 7 | * Gets a page of the latest comments for a video. 8 | */ 9 | export function getVideoComments(call, cb) { 10 | // Use async function, convert to bluebird promise, and invoke callback when finished 11 | return Promise.resolve(getVideoCommentsImpl(call)).asCallback(cb); 12 | }; 13 | 14 | // Implementation using async/await 15 | async function getVideoCommentsImpl(call) { 16 | let { request } = call; 17 | 18 | let startingCommentId = toCassandraTimeUuid(request.startingCommentId); 19 | let videoId = toCassandraUuid(request.videoId); 20 | 21 | // Choose which CQL to execute based on whether we have a starting point for the list of comments 22 | let query, queryParams; 23 | if (startingCommentId === null) { 24 | // No starting point so just get the latest comments overall 25 | query = 'SELECT commentid, userid, comment, dateOf(commentid) AS comment_timestamp FROM comments_by_video WHERE videoid = ?'; 26 | queryParams = [ videoId ]; 27 | } else { 28 | // Since commentId is a TimeUuid, we can get that comment and older comments given the starting commentId by using <= 29 | query = 'SELECT commentid, userid, comment, dateOf(commentid) AS comment_timestamp FROM comments_by_video WHERE videoid = ? AND commentid <= ?'; 30 | queryParams = [ videoId, startingCommentId ]; 31 | } 32 | 33 | // Set the options for the query, including paging state from previous page if provided in the request 34 | let queryOpts = { autoPage: false, fetchSize: request.pageSize }; 35 | if (request.pagingState !== null && request.pagingState !== '') { 36 | queryOpts.pageState = request.pagingState; 37 | } 38 | 39 | // Execute the query using our Promise returning async method 40 | let client = getCassandraClient(); 41 | let resultSet = await client.executeAsync(query, queryParams, queryOpts); 42 | 43 | // Build the response object from the rows returned 44 | return new GetVideoCommentsResponse({ 45 | videoId: request.videoId, 46 | comments: resultSet.rows.map(row => new VideoComment({ 47 | commentId: toProtobufUuid(row.commentid), 48 | userId: toProtobufUuid(row.userid), 49 | comment: row.comment, 50 | commentTimestamp: toProtobufTimestamp(row.comment_timestamp) 51 | })), 52 | pagingState: resultSet.pageState !== null ? resultSet.pageState : '' 53 | }); 54 | } 55 | 56 | -------------------------------------------------------------------------------- /src/services/comments/index.js: -------------------------------------------------------------------------------- 1 | import { CommentsService } from './protos'; 2 | import { commentOnVideo } from './comment-on-video'; 3 | import { getVideoComments } from './get-video-comments'; 4 | import { getUserComments } from './get-user-comments'; 5 | 6 | /** 7 | * The comments service implementation. 8 | */ 9 | const implementation = { 10 | commentOnVideo, 11 | getUserComments, 12 | getVideoComments 13 | }; 14 | 15 | /** 16 | * Comments service, responsible for managing comments on videos. 17 | */ 18 | 19 | export default { 20 | name: 'CommentsService', 21 | service: CommentsService.service, 22 | implementation 23 | }; 24 | -------------------------------------------------------------------------------- /src/services/comments/protos.js: -------------------------------------------------------------------------------- 1 | import { load } from '../common/load'; 2 | 3 | // Load the protobuf definition to get the service and response objects 4 | const serviceFile = 'comments/comments_service.proto'; 5 | const { 6 | CommentsService, 7 | CommentOnVideoResponse, 8 | GetUserCommentsResponse, 9 | UserComment, 10 | GetVideoCommentsResponse, 11 | VideoComment 12 | } = load(serviceFile).killrvideo.comments; 13 | 14 | export { 15 | CommentsService, 16 | CommentOnVideoResponse, 17 | GetUserCommentsResponse, 18 | UserComment, 19 | GetVideoCommentsResponse, 20 | VideoComment 21 | }; -------------------------------------------------------------------------------- /src/services/common/grpc-errors.js: -------------------------------------------------------------------------------- 1 | import { status } from 'grpc'; 2 | 3 | /** 4 | * An extendable error class for errors that want to provide a Grpc status code. 5 | */ 6 | class GrpcError extends Error { 7 | constructor(message, statusCode) { 8 | super(message); 9 | this.stack = (new Error()).stack; 10 | this.name = this.constructor.name; 11 | this.message = message; 12 | this.code = statusCode; 13 | } 14 | } 15 | 16 | /** 17 | * An argument provided in a request was invalid. 18 | */ 19 | export class InvalidArgumentError extends GrpcError { 20 | constructor(message) { 21 | super(message, status.INVALID_ARGUMENT); 22 | } 23 | }; 24 | 25 | /** 26 | * Something we're trying to create already exists. 27 | */ 28 | export class AlreadyExistsError extends GrpcError { 29 | constructor(message) { 30 | super(message, status.ALREADY_EXISTS); 31 | } 32 | }; 33 | 34 | /** 35 | * Something requested was not found. 36 | */ 37 | export class NotFoundError extends GrpcError { 38 | constructor(message) { 39 | super(message, status.NOT_FOUND); 40 | } 41 | }; 42 | 43 | /** 44 | * The authentication provided is not valid for something you're trying to do. 45 | */ 46 | export class UnauthenticatedError extends GrpcError { 47 | constructor(message) { 48 | super(message, status.UNAUTHENTICATED); 49 | } 50 | }; 51 | 52 | /** 53 | * The service doesn't support some operation. 54 | */ 55 | export class NotImplementedError extends GrpcError { 56 | constructor(message) { 57 | super(message, status.UNIMPLEMENTED); 58 | } 59 | }; -------------------------------------------------------------------------------- /src/services/common/load.js: -------------------------------------------------------------------------------- 1 | import { resolve } from 'path'; 2 | import { load as grpcLoad } from 'grpc'; 3 | 4 | // The proto files are copied as part of the build script (see package.json) to a '/protos' 5 | // folder under the root of the project 6 | const root = resolve(__dirname, '../../protos'); 7 | 8 | /** 9 | * Common function for loading a protobuf file. Returns the proto object. 10 | */ 11 | export function load(file) { 12 | return grpcLoad({ root, file }, 'proto', { convertFieldsToCamelCase: true }); 13 | }; 14 | 15 | export default load; -------------------------------------------------------------------------------- /src/services/common/protobuf-conversions.js: -------------------------------------------------------------------------------- 1 | import { types as CassandraTypes } from 'cassandra-driver'; 2 | 3 | /** 4 | * Converts an object representing a google.protobuf.Timestamp to a JavaScript Date. If the 5 | * timestamp supplied is null, returns null. 6 | */ 7 | export function toJavaScriptDate(timestamp) { 8 | if (timestamp === null) return null; 9 | if (!timestamp.seconds || !timestamp.nanos) { 10 | throw new Error('Object is not a google.protobuf.Timestamp'); 11 | } 12 | let millis = (timestamp.seconds * 1000) + Math.trunc(timestamp.nanos / 1000000); 13 | return new Date(millis); 14 | }; 15 | 16 | /** 17 | * Converts a JavaScript Date object to a google.protobuf.Timestamp. If the date object supplied 18 | * is null, returns null. 19 | */ 20 | export function toProtobufTimestamp(date) { 21 | if (date === null) return null; 22 | let millis = date.valueOf(); 23 | let seconds = Math.trunc(millis / 1000); 24 | let nanos = (millis % 1000) * 1000000; 25 | return { seconds, nanos }; 26 | }; 27 | 28 | /** 29 | * Converts a killrvideo.common.Uuid to a Cassandra driver Uuid. If the protobuf Uuid is null, 30 | * returns null. 31 | */ 32 | export function toCassandraUuid(protobufUuid) { 33 | if (protobufUuid === null) return null; 34 | return CassandraTypes.Uuid.fromString(protobufUuid.value); 35 | }; 36 | 37 | /** 38 | * Converts a killrvideo.common.TimeUuid to a Cassandra driver TimeUuid. If the protobuf TimeUuid 39 | * is null, returns null. 40 | */ 41 | export function toCassandraTimeUuid(protobufTimeUuid) { 42 | if (protobufTimeUuid === null) return null; 43 | return CassandraTypes.TimeUuid.fromString(protobufTimeUuid.value); 44 | }; 45 | 46 | /** 47 | * Converts a Cassandra driver Uuid or TimeUuid to an object compatible with killrvideo.common.Uuid 48 | * and killrvideo.common.TimeUuid. If the Cassandra Uuid is null, returns null. 49 | */ 50 | export function toProtobufUuid(cassandraUuid) { 51 | if (cassandraUuid === null) return null; 52 | return { value: cassandraUuid.toString() }; 53 | }; -------------------------------------------------------------------------------- /src/services/index.js: -------------------------------------------------------------------------------- 1 | import { logger } from '../common/logging'; 2 | 3 | import comments from './comments'; 4 | import ratings from './ratings'; 5 | import search from './search'; 6 | import stats from './statistics'; 7 | import suggestedVideos from './suggested-videos'; 8 | import uploads from './uploads'; 9 | import userManagement from './user-management'; 10 | import videoCatalog from './video-catalog'; 11 | 12 | /** 13 | * A helper function for logging errors in Grpc service calls. 14 | */ 15 | function executeAndLogErrors(fn, call, cb) { 16 | // Call the original function with the call argument, but replace the callback 17 | fn(call, function logErrorsCallback(...args) { 18 | // Log any errors, then execute the original callback 19 | if (args[0]) { 20 | logger.log('error', '', args[0]); 21 | } 22 | cb(...args); 23 | }); 24 | } 25 | 26 | /** 27 | * An array of all available service objects. 28 | */ 29 | export const services = [ 30 | comments, 31 | ratings, 32 | search, 33 | stats, 34 | suggestedVideos, 35 | uploads, 36 | userManagement, 37 | videoCatalog 38 | ].map(serviceDef => { 39 | // Wrap each service call with error logging 40 | let impl = serviceDef.implementation; 41 | Object.keys(impl).forEach(fnName => { 42 | // Get the original function 43 | let fn = impl[fnName]; 44 | 45 | // Wrap by partially applying the original function to our helper above 46 | let value = executeAndLogErrors.bind(undefined, fn); 47 | Object.defineProperty(impl, fnName, { value }); 48 | }); 49 | 50 | return serviceDef; 51 | }); 52 | 53 | function matchServiceDef(a, b) { 54 | const aKeys = Object.keys(a); 55 | const bKeys = Object.keys(b); 56 | 57 | if (aKeys.length === bKeys.length) { 58 | let index = bKeys.length; 59 | while (index--) { 60 | let bKey = bKeys[index]; 61 | if (!a.hasOwnProperty(bKey)) { 62 | return false; 63 | } 64 | } 65 | return true; 66 | } 67 | return false; 68 | }; 69 | 70 | export function serviceNameFromDefinition(serviceDef) { 71 | let index = services.length 72 | while (index--) { 73 | let service = services[index] 74 | if (matchServiceDef(serviceDef, service.service)) { 75 | return service.name; 76 | } 77 | } 78 | logger.log('error',`failed to find name for serviceDef with keys ${JSON.stringify(Object.keys(serviceDef))}`) 79 | return undefined; 80 | } 81 | -------------------------------------------------------------------------------- /src/services/ratings/events.js: -------------------------------------------------------------------------------- 1 | import { load } from '../common/load'; 2 | 3 | // Load events published by this service 4 | const eventsFile = 'ratings/ratings_events.proto'; 5 | const { 6 | UserRatedVideo 7 | } = load(eventsFile).killrvideo.ratings.events; 8 | 9 | export { 10 | UserRatedVideo 11 | }; -------------------------------------------------------------------------------- /src/services/ratings/get-rating.js: -------------------------------------------------------------------------------- 1 | import async from 'async'; 2 | import { GetRatingResponse } from './protos'; 3 | import { toCassandraUuid } from '../common/protobuf-conversions'; 4 | import { getCassandraClient } from '../../common/cassandra'; 5 | 6 | /** 7 | * Gets the current ratings stats for the specified video. 8 | */ 9 | export function getRating(call, cb) { 10 | let { request } = call; 11 | async.waterfall([ 12 | async.asyncify(getCassandraClient), 13 | (client, next) => { 14 | let queryParams = [ toCassandraUuid(request.videoId) ]; 15 | client.execute('SELECT * FROM video_ratings WHERE videoid = ?', queryParams, next); 16 | }, 17 | (resultSet, next) => { 18 | // Init to 0 in case we don't have any stats yet for the requested video 19 | let ratingsCount = 0; 20 | let ratingsTotal = 0; 21 | 22 | let row = resultSet.first(); 23 | if (row !== null) { 24 | ratingsCount = row.rating_counter; 25 | ratingsTotal = row.rating_total; 26 | } 27 | 28 | next(null, new GetRatingResponse({ 29 | videoId: request.videoId, 30 | ratingsCount, 31 | ratingsTotal 32 | })); 33 | } 34 | ], cb); 35 | }; -------------------------------------------------------------------------------- /src/services/ratings/get-user-rating.js: -------------------------------------------------------------------------------- 1 | import async from 'async'; 2 | import { GetUserRatingResponse } from './protos'; 3 | import { toCassandraUuid } from '../common/protobuf-conversions'; 4 | import { getCassandraClient } from '../../common/cassandra'; 5 | 6 | /** 7 | * Gets the rating given by a user for a specific video. Returns 0 if the user hasn't rated 8 | * the video yet. 9 | */ 10 | export function getUserRating(call, cb) { 11 | let { request } = call; 12 | async.waterfall([ 13 | async.asyncify(getCassandraClient), 14 | (client, next) => { 15 | let queryParams = [ 16 | toCassandraUuid(request.videoId), 17 | toCassandraUuid(request.userId) 18 | ]; 19 | client.execute('SELECT rating FROM video_ratings_by_user WHERE videoid = ? AND userid = ?', queryParams, next); 20 | }, 21 | (resultSet, next) => { 22 | // Init to 0 in case user hasn't rated video yet 23 | let rating = 0; 24 | 25 | let row = resultSet.first(); 26 | if (row !== null) { 27 | rating = row.rating; 28 | } 29 | 30 | next(null, new GetUserRatingResponse({ 31 | videoId: request.videoId, 32 | userId: request.userId, 33 | rating 34 | })); 35 | } 36 | ], cb); 37 | }; -------------------------------------------------------------------------------- /src/services/ratings/index.js: -------------------------------------------------------------------------------- 1 | import { RatingsService } from './protos'; 2 | import { getRating } from './get-rating'; 3 | import { getUserRating } from './get-user-rating'; 4 | import { rateVideo } from './rate-video'; 5 | 6 | /** 7 | * The ratings service implementation. 8 | */ 9 | const implementation = { 10 | rateVideo, 11 | getRating, 12 | getUserRating 13 | }; 14 | 15 | /** 16 | * Ratings service, responsible for tracking user's rating of videos. 17 | */ 18 | export default { 19 | name: 'RatingsService', 20 | service: RatingsService.service, 21 | implementation 22 | }; 23 | -------------------------------------------------------------------------------- /src/services/ratings/protos.js: -------------------------------------------------------------------------------- 1 | import { load } from '../common/load'; 2 | 3 | // Load the protobuf definition to get the service and response objects 4 | const serviceFile = 'ratings/ratings_service.proto'; 5 | const { 6 | RatingsService, 7 | RateVideoResponse, 8 | GetRatingResponse, 9 | GetUserRatingResponse 10 | } = load(serviceFile).killrvideo.ratings; 11 | 12 | export { 13 | RatingsService, 14 | RateVideoResponse, 15 | GetRatingResponse, 16 | GetUserRatingResponse 17 | }; -------------------------------------------------------------------------------- /src/services/ratings/rate-video.js: -------------------------------------------------------------------------------- 1 | import async from 'async'; 2 | import { RateVideoResponse } from './protos'; 3 | import { UserRatedVideo } from './events'; 4 | import { toCassandraUuid, toProtobufTimestamp } from '../common/protobuf-conversions'; 5 | import { getCassandraClient } from '../../common/cassandra'; 6 | import { publish } from '../../common/message-bus'; 7 | 8 | // Update the video_ratings counter table 9 | const updateRatingsCql = ` 10 | UPDATE video_ratings 11 | SET rating_counter = rating_counter + 1, rating_total = rating_total + ? 12 | WHERE videoid = ?`; 13 | 14 | // Insert rating for a user and specific video 15 | const insertUserRatingCql = ` 16 | INSERT INTO video_ratings_by_user ( 17 | videoid, userid, rating) 18 | VALUES (?, ?, ?)`; 19 | 20 | /** 21 | * Adds a user's rating of a video. 22 | */ 23 | export function rateVideo(call, cb) { 24 | let { request } = call; 25 | async.waterfall([ 26 | // Get client 27 | async.asyncify(getCassandraClient), 28 | 29 | // Execute CQL 30 | (client, next) => { 31 | // Get some bind variable values for the CQL we're going to run 32 | let videoId = toCassandraUuid(request.videoId); 33 | let userId = toCassandraUuid(request.userId); 34 | let { rating } = request; 35 | 36 | // We can't use a batch to do inserts to multiple tables here because one the video_ratings table 37 | // is a counter table (and Cassandra doesn't let us mix counter DML with regular DML in a batch), 38 | // but we can execute the inserts in parallel 39 | async.parallel([ 40 | execCb => client.execute(updateRatingsCql, [ rating, videoId ], execCb), 41 | execCb => client.execute(insertUserRatingCql, [ videoId, userId, rating ], execCb) 42 | ], next); 43 | }, 44 | 45 | // If successful with inserts, publish an event 46 | (resultSets, next) => { 47 | // Tell the world about the user rating the video 48 | let event = new UserRatedVideo({ 49 | videoId: request.videoId, 50 | userId: request.userId, 51 | rating: request.rating, 52 | ratingTimestamp: toProtobufTimestamp(new Date(Date.now())) 53 | }); 54 | 55 | publish(event, next); 56 | }, 57 | 58 | // Finally, return a response object 59 | next => { 60 | next(null, new RateVideoResponse()); 61 | } 62 | ], cb); 63 | }; -------------------------------------------------------------------------------- /src/services/search/get-query-suggestions.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import rp from 'request-promise'; 3 | import config from '../../common/config'; 4 | import { lookupServiceAsync } from '../../common/service-discovery'; 5 | import { getCassandraClient } from '../../common/cassandra'; 6 | import { NotImplementedError } from '../common/grpc-errors'; 7 | import { GetQuerySuggestionsResponse } from './protos'; 8 | 9 | /** 10 | * Gets a list of query suggestions for providing typeahead support. 11 | */ 12 | export function getQuerySuggestions(call, cb) { 13 | // Pick appropriate implementation 14 | let fn = config.get('dseEnabled') === true 15 | ? getQuerySuggestionsWithDseSearch 16 | : getTagSuggestions; 17 | 18 | // Invoke async function, wrap with bluebird Promise, and invoke callback when finished 19 | return Promise.resolve(fn(call)).asCallback(cb); 20 | }; 21 | 22 | /** 23 | * Gets suggested tags based on current input using just Cassandra. 24 | */ 25 | async function getTagSuggestions(call) { 26 | let { request } = call; 27 | 28 | let firstLetter = request.query.substr(0, 1); 29 | 30 | // Do query that gets tags starting with the first letter of the query 31 | let client = getCassandraClient(); 32 | let queryParams = [ firstLetter, request.query, request.pageSize ]; 33 | let resultSet = await client.executeAsync('SELECT tag FROM tags_by_letter WHERE first_letter = ? AND tag >= ? LIMIT ?', queryParams); 34 | 35 | // Convert results to response 36 | return new GetQuerySuggestionsResponse({ 37 | query: request.query, 38 | suggestions: resultSet.rows.map(row => row.tag) 39 | }); 40 | } 41 | 42 | // Cache promise 43 | let getSearchClientPromise = null; 44 | 45 | function getSearchClientAsync() { 46 | if (getSearchClientPromise === null) { 47 | getSearchClientPromise = lookupServiceAsync('dse-search') 48 | .then(hostAndPorts => { 49 | // Just use the first host:port returned 50 | return rp.defaults({ 51 | baseUrl: `http://${hostAndPorts[0]}/solr` 52 | }); 53 | }) 54 | .catch(err => { 55 | // Remove cached promise and rethrow 56 | getSearchClientPromise = null; 57 | throw err; 58 | }); 59 | } 60 | return getSearchClientPromise; 61 | } 62 | 63 | /** 64 | * Uses the Suggester module in DSE Search to get typeahead suggestions. 65 | */ 66 | async function getQuerySuggestionsWithDseSearch(call) { 67 | let { request } = call; 68 | 69 | // Get HTTP client for DSE search Solr API 70 | let doRequest = await getSearchClientAsync(); 71 | 72 | // Make request 73 | let requestOpts = { 74 | url: '/killrvideo.videos/suggest', 75 | method: 'POST', 76 | form: { 77 | 'wt': 'json', 78 | // We'll build on every request but in a real app, we'd probably take advantage of Solr config 79 | // buildOnCommit or buildOnOptimize settings 80 | 'suggest.build': 'true', 81 | 'suggest.q': request.query 82 | }, 83 | json: true 84 | }; 85 | let searchResponse = await doRequest(requestOpts); 86 | 87 | // Convert results to response 88 | return new GetQuerySuggestionsResponse({ 89 | query: request.query, 90 | suggestions: searchResponse.suggest.searchSuggester[request.query].suggestions.map(s => s.term) 91 | }); 92 | } 93 | -------------------------------------------------------------------------------- /src/services/search/handlers.js: -------------------------------------------------------------------------------- 1 | import { types as CassandraTypes } from 'cassandra-driver'; 2 | import { getCassandraClient } from '../../common/cassandra'; 3 | import { load } from '../common/load'; 4 | import { toCassandraUuid, toJavaScriptDate } from '../common/protobuf-conversions'; 5 | 6 | // Load protobuf definitions we want to handle 7 | const { 8 | UploadedVideoAdded, 9 | YouTubeVideoAdded 10 | } = load('video-catalog/video_catalog_events.proto').killrvideo.video_catalog.events; 11 | 12 | /** 13 | * Updates the search by tags data when new videos are added to the video catalog. 14 | */ 15 | function updateSearchOnVideoAdded(event) { 16 | // Nothing to do if no tags on the video 17 | if (event.tags.length === 0) { 18 | return; 19 | } 20 | 21 | // Convert some values 22 | let videoId = toCassandraUuid(event.videoId); 23 | let userId = toCassandraUuid(event.userId); 24 | let addedDate = toJavaScriptDate(event.addedDate); 25 | let timestamp = toJavaScriptDate(event.timestamp); 26 | 27 | // Create a batch for updating tag data tables 28 | let queries = []; 29 | event.tags.forEach(tag => { 30 | // Videos by tag table 31 | queries.push({ 32 | query: 'INSERT INTO videos_by_tag (tag, videoid, added_date, userid, name, preview_image_location, tagged_date) VALUES (?, ?, ?, ?, ?, ?, ?)', 33 | params: [ tag, videoId, addedDate, userId, event.name, event.previewImageLocation, timestamp ] 34 | }); 35 | 36 | // Tags by letter table 37 | let firstLetter = tag.substr(0, 1); 38 | queries.push({ 39 | query: 'INSERT INTO tags_by_letter (first_letter, tag) VALUES (?, ?)', 40 | params: [ firstLetter, tag ] 41 | }); 42 | }); 43 | 44 | // Use event's timestamp as the write time in Cassandra 45 | let queryOpts = {}; 46 | 47 | let client = getCassandraClient(); 48 | return client.batchAsync(queries, queryOpts); 49 | } 50 | 51 | /** 52 | * An array of message bus handler definitions (i.e. the event type and handler function). 53 | */ 54 | export default [ 55 | { eventType: UploadedVideoAdded, handler: updateSearchOnVideoAdded }, 56 | { eventType: YouTubeVideoAdded, handler: updateSearchOnVideoAdded } 57 | ]; -------------------------------------------------------------------------------- /src/services/search/index.js: -------------------------------------------------------------------------------- 1 | import { SearchService } from './protos'; 2 | import { searchVideos } from './search-videos'; 3 | import { getQuerySuggestions } from './get-query-suggestions'; 4 | import handlers from './handlers'; 5 | 6 | /** 7 | * The search service implementation. 8 | */ 9 | const implementation = { 10 | searchVideos, 11 | getQuerySuggestions 12 | }; 13 | 14 | /** 15 | * Search service which allows searching for videos. 16 | */ 17 | export default { 18 | name: 'SearchService', 19 | service: SearchService.service, 20 | implementation, 21 | handlers 22 | }; 23 | -------------------------------------------------------------------------------- /src/services/search/protos.js: -------------------------------------------------------------------------------- 1 | import { load } from '../common/load'; 2 | 3 | // Load the protobuf definition to get the service and response objects 4 | const serviceFile = 'search/search_service.proto'; 5 | const { 6 | SearchService, 7 | SearchVideosResponse, 8 | SearchResultsVideoPreview, 9 | GetQuerySuggestionsResponse 10 | } = load(serviceFile).killrvideo.search; 11 | 12 | export { 13 | SearchService, 14 | SearchVideosResponse, 15 | SearchResultsVideoPreview, 16 | GetQuerySuggestionsResponse 17 | }; -------------------------------------------------------------------------------- /src/services/search/search-videos.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import { types as CassandraTypes } from 'cassandra-driver'; 3 | import config from '../../common/config'; 4 | import { getCassandraClient } from '../../common/cassandra'; 5 | import { NotImplementedError } from '../common/grpc-errors'; 6 | import { toProtobufTimestamp, toProtobufUuid } from '../common/protobuf-conversions'; 7 | import { SearchVideosResponse, SearchResultsVideoPreview } from './protos'; 8 | 9 | /** 10 | * Gets a page of video preview search results for a query. 11 | */ 12 | export function searchVideos(call, cb) { 13 | // Pick appropriate implementation 14 | let fn = config.get('dseEnabled') === true 15 | ? searchVideosWithDseSearch 16 | : searchVideosByTag; 17 | 18 | // Invoke async function, wrap with bluebird Promise, and invoke callback when finished 19 | return Promise.resolve(fn(call)).asCallback(cb); 20 | }; 21 | 22 | /** 23 | * Helper function to map a row from C* to a SearchResultsVideoPreview response object. 24 | */ 25 | function toSearchResultsVideoPreview(row) { 26 | return new SearchResultsVideoPreview({ 27 | videoId: toProtobufUuid(row.videoid), 28 | addedDate: toProtobufTimestamp(row.added_date), 29 | name: row.name, 30 | previewImageLocation: row.preview_image_location, 31 | userId: toProtobufUuid(row.userid) 32 | }); 33 | } 34 | 35 | /** 36 | * Searches videos by tag using just Cassandra. 37 | */ 38 | async function searchVideosByTag(call) { 39 | let { request } = call; 40 | 41 | // Options for the query 42 | let queryOpts = { autoPage: false, fetchSize: request.pageSize }; 43 | if (request.pagingState !== null && request.pagingState !== '') { 44 | queryOpts.pageState = request.pagingState; 45 | } 46 | 47 | // Do the query 48 | let client = getCassandraClient(); 49 | let resultSet = await client.executeAsync('SELECT * FROM videos_by_tag WHERE tag = ?', [ request.query ], queryOpts); 50 | 51 | // Convert the rows in the ResultSet to a response 52 | return new SearchVideosResponse({ 53 | query: request.query, 54 | videos: resultSet.rows.map(toSearchResultsVideoPreview), 55 | pagingState: resultSet.pageState !== null ? resultSet.pageState : '' 56 | }); 57 | } 58 | 59 | /** 60 | * Searches videos using Solr indexes in DSE Search. 61 | */ 62 | async function searchVideosWithDseSearch(call) { 63 | let { request } = call; 64 | 65 | // Do a Solr query against DSE search to find videos using Solr's ExtendedDisMax query parser. Query the 66 | // name, tags, and description fields in the videos table giving a boost to matches in the name and tags 67 | // fields as opposed to the description field 68 | // More info on ExtendedDisMax: http://wiki.apache.org/solr/ExtendedDisMax 69 | let solrQuery = `{ "q": "{!edismax qf=\\"name^2 tags^1 description\\"}${request.query}" }`; 70 | 71 | // Options for the query 72 | let queryOpts = { 73 | autoPage: false, 74 | fetchSize: request.pageSize, 75 | consistency: CassandraTypes.consistencies.localOne // Search only supports one/localOne 76 | }; 77 | if (request.pagingState !== null && request.pagingState !== '') { 78 | queryOpts.pageState = request.pagingState; 79 | } 80 | 81 | // Do the query 82 | let client = getCassandraClient(); 83 | let resultSet = await client.executeAsync( 84 | 'SELECT videoid, userid, name, preview_image_location, added_date FROM videos WHERE solr_query=?', 85 | [ solrQuery ], queryOpts); 86 | 87 | // Convert the rows in the resultset to a response 88 | return new SearchVideosResponse({ 89 | query: request.query, 90 | videos: resultSet.rows.map(toSearchResultsVideoPreview), 91 | pagingState: resultSet.pageState !== null ? resultSet.pageState : '' 92 | }); 93 | } -------------------------------------------------------------------------------- /src/services/statistics/get-number-of-plays.js: -------------------------------------------------------------------------------- 1 | import async from 'async'; 2 | import { GetNumberOfPlaysResponse, PlayStats } from './protos'; 3 | import { InvalidArgumentError } from '../common/grpc-errors'; 4 | import { toCassandraUuid } from '../common/protobuf-conversions'; 5 | import { getCassandraClient } from '../../common/cassandra'; 6 | 7 | /** 8 | * Get the number of times the requested videos have been played. 9 | */ 10 | export function getNumberOfPlays(call, cb) { 11 | let { request } = call; 12 | 13 | // Enforce some sanity on this method to avoid doing a big multi-get 14 | if (request.videoIds.length > 20) { 15 | cb(new InvalidArgumentError('Cannot do a get on more than 20 videos at once')); 16 | return; 17 | } 18 | 19 | // Get the cassandra-driver client and map protobuf uuids to cassandra uuids 20 | let client, videoIds; 21 | try { 22 | client = getCassandraClient(); 23 | videoIds = request.videoIds.map(id => toCassandraUuid(id)); 24 | } catch (err) { 25 | cb(err); 26 | return; 27 | } 28 | 29 | // Run parallel queries on each video Id using the async library's map function 30 | async.map(videoIds, (videoId, done) => { 31 | client.execute('SELECT views FROM video_playback_stats WHERE videoid = ?', [ videoId ], done); 32 | }, 33 | (err, resultSets) => { 34 | if (err) { 35 | cb(err); 36 | return; 37 | } 38 | 39 | // For each ResultSet, get the row returned and add to response 40 | let stats = []; 41 | for (let i = 0; i < resultSets.length; i++) { 42 | let row = resultSets[i].first(); 43 | stats.push(new PlayStats({ 44 | videoId: request.videoIds[i], 45 | 46 | // Return 0 if there was no row returned from cassandra, otherwise the views 47 | views: row === null ? 0 : row.views 48 | })); 49 | } 50 | 51 | cb(null, new GetNumberOfPlaysResponse({ stats })); 52 | }); 53 | }; -------------------------------------------------------------------------------- /src/services/statistics/index.js: -------------------------------------------------------------------------------- 1 | import { StatisticsService } from './protos'; 2 | import { getNumberOfPlays } from './get-number-of-plays'; 3 | import { recordPlaybackStarted } from './record-playback-started'; 4 | 5 | /** 6 | * The stats service implementation. 7 | */ 8 | const implementation = { 9 | recordPlaybackStarted, 10 | getNumberOfPlays 11 | }; 12 | 13 | /** 14 | * Statistics service that tracks stats for videos. 15 | */ 16 | export default { 17 | name: 'StatisticsService', 18 | service: StatisticsService.service, 19 | implementation 20 | }; 21 | -------------------------------------------------------------------------------- /src/services/statistics/protos.js: -------------------------------------------------------------------------------- 1 | import { load } from '../common/load'; 2 | 3 | // Load the protobuf definition to get the service object 4 | const file = 'statistics/statistics_service.proto'; 5 | const { 6 | StatisticsService, 7 | RecordPlaybackStartedResponse, 8 | GetNumberOfPlaysResponse, 9 | PlayStats 10 | } = load(file).killrvideo.statistics; 11 | 12 | export { 13 | StatisticsService, 14 | RecordPlaybackStartedResponse, 15 | GetNumberOfPlaysResponse, 16 | PlayStats 17 | }; -------------------------------------------------------------------------------- /src/services/statistics/record-playback-started.js: -------------------------------------------------------------------------------- 1 | import async from 'async'; 2 | import { RecordPlaybackStartedResponse } from './protos'; 3 | import { toCassandraUuid } from '../common/protobuf-conversions'; 4 | import { getCassandraClient } from '../../common/cassandra'; 5 | 6 | /** 7 | * Records that playback was started for a given video. 8 | */ 9 | export function recordPlaybackStarted(call, cb) { 10 | let { request } = call; 11 | async.waterfall([ 12 | // Get the cassandra client 13 | async.asyncify(getCassandraClient), 14 | 15 | // Update the playback stats 16 | (client, next) => { 17 | let videoId = toCassandraUuid(request.videoId); 18 | client.execute('UPDATE video_playback_stats SET views = views + 1 WHERE videoid = ?', [ videoId ], next); 19 | }, 20 | 21 | // Return the response 22 | (resultSet, next) => { 23 | next(null, new RecordPlaybackStartedResponse()); 24 | } 25 | ], cb); 26 | }; -------------------------------------------------------------------------------- /src/services/suggested-videos/get-related-videos.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import rp from 'request-promise'; 3 | import { lookupServiceAsync } from '../../common/service-discovery'; 4 | import { getCassandraClient } from '../../common/cassandra'; 5 | import config from '../../common/config'; 6 | import { toCassandraUuid, toProtobufUuid, toProtobufTimestamp } from '../common/protobuf-conversions'; 7 | import { NotImplementedError } from '../common/grpc-errors'; 8 | import { GetRelatedVideosResponse, SuggestedVideoPreview } from './protos'; 9 | 10 | /** 11 | * Gets a list of videos related to another video. 12 | */ 13 | export function getRelatedVideos(call, cb) { 14 | // Pick appropriate implementation 15 | let fn = config.get('dseEnabled') === true 16 | ? getRelatedVideosWithDseSearch 17 | : getRelatedVideosByTag; 18 | 19 | // Invoke async function, wrap with bluebird Promise, and invoke callback when finished 20 | return Promise.resolve(fn(call)).asCallback(cb); 21 | }; 22 | 23 | /** 24 | * Helper function that returns a new empty response. 25 | */ 26 | function emptyResponse(videoId) { 27 | return new GetRelatedVideosResponse({ 28 | videoId, 29 | videos: [], 30 | pagingState: '' 31 | }); 32 | } 33 | 34 | /** 35 | * Helper function to convert a Cassandra row to a SuggestedVideoPreview object. 36 | */ 37 | function toSuggestedVideoPreview(row) { 38 | return new SuggestedVideoPreview({ 39 | videoId: toProtobufUuid(row.videoid), 40 | addedDate: toProtobufTimestamp(row.added_date), 41 | name: row.name, 42 | previewImageLocation: row.preview_image_location, 43 | userId: toProtobufUuid(row.userid) 44 | }); 45 | } 46 | 47 | /** 48 | * Number of videos to return when doing related videos by tag. 49 | */ 50 | const RELATED_BY_TAG_RETURN_COUNT = 4; 51 | 52 | /** 53 | * Gets related videos based on the tags in the specified video. Does not support paging. 54 | */ 55 | async function getRelatedVideosByTag(call) { 56 | // TODO: Stop "cheating" and using data directly from other services 57 | 58 | let { request } = call; 59 | let videoId = toCassandraUuid(request.videoId); 60 | 61 | // Get tags for the given video 62 | let client = getCassandraClient(); 63 | let tagsResultSet = await client.executeAsync('SELECT tags FROM videos WHERE videoid = ?', [ videoId ]); 64 | 65 | // Make sure we have tags 66 | let tagRow = tagsResultSet.first(); 67 | if (tagRow === null) { 68 | return emptyResponse(); 69 | } 70 | 71 | let { tags } = tagRow; 72 | if (tags.length === 0) { 73 | return emptyResponse(); 74 | } 75 | 76 | // Use the number of results we want to return * 2 when querying so that we can account for potentially having 77 | // to filter out the video Id we're looking up, as well as duplicates 78 | const pageSize = RELATED_BY_TAG_RETURN_COUNT * 2; 79 | 80 | // Kick off queries in parallel for the tags 81 | let inFlight = []; 82 | let videos = {}; 83 | let videoCount = 0; 84 | for (let i = 0; i < tags.length; i++) { 85 | let tag = tags[i]; 86 | let promise = client.executeAsync('SELECT * FROM videos_by_tag WHERE tag = ? LIMIT ?', [ tag, pageSize ]); 87 | inFlight.push(promise); 88 | 89 | // If we don't have at least three in-flight queries and this isn't the last tag, keep kicking off queries 90 | if (inFlight.length < 3 && i !== tags.length - 1) { 91 | continue; 92 | } 93 | 94 | // Otherwise, wait for all in-flight queries to complete 95 | let resultSets = await Promise.all(inFlight); 96 | 97 | // Process the results 98 | for (let resultSet of resultSets) { 99 | for (let row of resultSet.rows) { 100 | let video = toSuggestedVideoPreview(row); 101 | 102 | // Skip self 103 | if (video.videoId.value === request.videoId.value) { 104 | continue; 105 | } 106 | 107 | // Skip it if we already have it in the results 108 | if (videos.hasOwnProperty(video.videoId.value)) { 109 | continue; 110 | } 111 | 112 | // Add to results 113 | videos[video.videoId.value] = video; 114 | videoCount++; 115 | 116 | // Do we have enough results? 117 | if (videoCount >= RELATED_BY_TAG_RETURN_COUNT) 118 | break; 119 | } 120 | 121 | // Do we have enough results? 122 | if (videoCount >= RELATED_BY_TAG_RETURN_COUNT) 123 | break; 124 | } 125 | 126 | // Do we have enough results? 127 | if (videoCount >= RELATED_BY_TAG_RETURN_COUNT) 128 | break; 129 | 130 | // Not enough yet so clear the in-flight query list and start again 131 | inFlight = []; 132 | } 133 | 134 | // Return the response 135 | return new GetRelatedVideosResponse({ 136 | videoId: request.videoId, 137 | videos: Object.keys(videos).map(id => videos[id]), 138 | pagingState: '' // Does not support paging 139 | }); 140 | } 141 | 142 | // Cache promise 143 | let getSearchClientPromise = null; 144 | 145 | function getSearchClientAsync() { 146 | if (getSearchClientPromise === null) { 147 | getSearchClientPromise = lookupServiceAsync('dse-search') 148 | .then(hostAndPorts => { 149 | // Just use the first host:port returned 150 | return rp.defaults({ 151 | baseUrl: `http://${hostAndPorts[0]}/solr` 152 | }); 153 | }) 154 | .catch(err => { 155 | // Remove cached promise and rethrow 156 | getSearchClientPromise = null; 157 | throw err; 158 | }); 159 | } 160 | return getSearchClientPromise; 161 | } 162 | 163 | /** 164 | * Gets related videos using DSE Search "More Like This" functionality. 165 | */ 166 | async function getRelatedVideosWithDseSearch(call) { 167 | let { request } = call; 168 | 169 | // Parse paging state if present 170 | let start = 0; 171 | if (request.pagingState !== null && request.pagingState !== '') { 172 | start = parseInt(request.pagingState); 173 | } 174 | 175 | // Get HTTP client for DSE search Solr API 176 | let doRequest = await getSearchClientAsync(); 177 | 178 | // Make request 179 | let requestOpts = { 180 | url: '/killrvideo.videos/mlt', 181 | method: 'POST', 182 | form: { 183 | 'wt': 'json', 184 | 'q': `videoid:"${request.videoId.value}"`, // Just use string representation of UUID in query 185 | 'start': start, 186 | 'rows': request.pageSize, 187 | // More like this fields to consider 188 | 'mlt.fl': 'name,description,tags', 189 | // MLT Minimum Document Frequency - the frequency at which words will be ignored which do not occur in at least this many docs 190 | 'mlt.mindf': 2, 191 | // MLT Minimum Term Frequency - the frequency below which terms will be ignored in the source doc 192 | 'mlt.mintf': 2 193 | }, 194 | json: true 195 | }; 196 | let searchResponse = await doRequest(requestOpts); 197 | 198 | // Get the starting index for the next page, then compare against total results available to determine paging state 199 | let nextPageStartIdx = searchResponse.response 200 | ? searchResponse.response.start + searchResponse.response.docs.length 201 | : 0; 202 | let pagingState = nextPageStartIdx === searchResponse.numFound 203 | ? '' 204 | : nextPageStartIdx.toString(); 205 | 206 | // Convert the search response to gRPC response object 207 | return new GetRelatedVideosResponse({ 208 | videoId: request.videoId, 209 | videos: searchResponse.response 210 | ? searchResponse.response.docs.map(doc => new SuggestedVideoPreview({ 211 | videoId: { value: doc.videoid }, 212 | addedDate: toProtobufTimestamp(new Date(doc.added_date)), 213 | name: doc.name, 214 | previewImageLocation: doc.preview_image_location, 215 | userId: { value: doc.userid } 216 | })) 217 | : [], 218 | pagingState 219 | }); 220 | } 221 | -------------------------------------------------------------------------------- /src/services/suggested-videos/get-suggested-for-user.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import { getCassandraClient } from '../../common/cassandra'; 3 | import config from '../../common/config'; 4 | import { NotImplementedError } from '../common/grpc-errors'; 5 | import { GetSuggestedForUserResponse, SuggestedVideoPreview } from './protos'; 6 | 7 | /** 8 | * Gets personalized video suggestions for a specific user. 9 | */ 10 | export function getSuggestedForUser(call, cb) { 11 | // Pick appropriate implementation 12 | let fn = config.get('dseEnabled') === true 13 | ? getSuggestedWithDseAnalytics 14 | : emptySuggestions; 15 | 16 | // Invoke async function, wrap with bluebird Promise, and invoke callback when finished 17 | return Promise.resolve(fn(call)).asCallback(cb); 18 | }; 19 | 20 | /** 21 | * When not using DSE, we don't have a way to do personalized suggestions so just return an 22 | * empty response. 23 | */ 24 | function emptySuggestions(call) { 25 | let { request } = call; 26 | return new GetSuggestedForUserResponse({ 27 | userId: request.userId, 28 | videos: [], 29 | pagingState: '' 30 | }); 31 | } 32 | 33 | /** 34 | * Use DSE analytics to get personalized suggestions for a user. 35 | */ 36 | async function getSuggestedWithDseAnalytics(call) { 37 | // TODO: Since the Spark job that populates this data is not currently running, 38 | // just return an empty response 39 | return emptySuggestions(call); 40 | } -------------------------------------------------------------------------------- /src/services/suggested-videos/index.js: -------------------------------------------------------------------------------- 1 | import { SuggestedVideoService } from './protos'; 2 | import { getRelatedVideos } from './get-related-videos'; 3 | import { getSuggestedForUser } from './get-suggested-for-user'; 4 | 5 | /** 6 | * The suggested video service implementation. 7 | */ 8 | const implementation = { 9 | getRelatedVideos, 10 | getSuggestedForUser 11 | }; 12 | 13 | /** 14 | * Suggested video service that's responsible for generating video suggestions for users and videos. 15 | */ 16 | export default { 17 | name: 'SuggestedVideoService', 18 | service: SuggestedVideoService.service, 19 | implementation 20 | }; 21 | -------------------------------------------------------------------------------- /src/services/suggested-videos/protos.js: -------------------------------------------------------------------------------- 1 | import { load } from '../common/load'; 2 | 3 | // Load the protobuf definition to get the service and response objects 4 | const serviceFile = 'suggested-videos/suggested_videos_service.proto'; 5 | const { 6 | SuggestedVideoService, 7 | GetRelatedVideosResponse, 8 | GetSuggestedForUserResponse, 9 | SuggestedVideoPreview 10 | } = load(serviceFile).killrvideo.suggested_videos; 11 | 12 | export { 13 | SuggestedVideoService, 14 | GetRelatedVideosResponse, 15 | GetSuggestedForUserResponse, 16 | SuggestedVideoPreview 17 | }; -------------------------------------------------------------------------------- /src/services/uploads/events.js: -------------------------------------------------------------------------------- 1 | import { load } from '../common/load'; 2 | 3 | // Load events published by this service 4 | const eventsFile = 'uploads/uploads_events.proto'; 5 | const { 6 | UploadedVideoProcessingFailed, 7 | UploadedVideoProcessingStarted, 8 | UploadedVideoProcessingSucceeded, 9 | UploadedVideoPublished 10 | } = load(eventsFile).killrvideo.uploads.events; 11 | 12 | export { 13 | UploadedVideoProcessingFailed, 14 | UploadedVideoProcessingStarted, 15 | UploadedVideoProcessingSucceeded, 16 | UploadedVideoPublished 17 | }; -------------------------------------------------------------------------------- /src/services/uploads/index.js: -------------------------------------------------------------------------------- 1 | import { UploadsService } from './protos'; 2 | import { NotImplementedError } from '../common/grpc-errors'; 3 | 4 | /** 5 | * The uploads service implementation. 6 | */ 7 | const implementation = { 8 | getUploadDestination(call, cb) { 9 | cb(new NotImplementedError('Not implemented')); 10 | }, 11 | 12 | markUploadComplete(call, cb) { 13 | cb(new NotImplementedError('Not implemented')); 14 | }, 15 | 16 | getStatusOfVideo(call, cb) { 17 | cb(new NotImplementedError('Not implemented')); 18 | } 19 | }; 20 | 21 | /** 22 | * Uploads service, that handles processing/re-encoding of user uploaded videos. 23 | */ 24 | export default { 25 | name: 'UploadsService', 26 | service: UploadsService.service, 27 | implementation 28 | }; 29 | -------------------------------------------------------------------------------- /src/services/uploads/protos.js: -------------------------------------------------------------------------------- 1 | import { load } from '../common/load'; 2 | 3 | // Load the protobuf definition to get the service and response objects 4 | const serviceFile = 'uploads/uploads_service.proto'; 5 | const { 6 | UploadsService, 7 | GetUploadDestinationResponse, 8 | MarkUploadCompleteResponse, 9 | GetStatusOfVideoResponse 10 | } = load(serviceFile).killrvideo.uploads; 11 | 12 | export { 13 | UploadsService, 14 | GetUploadDestinationResponse, 15 | MarkUploadCompleteResponse, 16 | GetStatusOfVideoResponse 17 | }; -------------------------------------------------------------------------------- /src/services/user-management/create-user.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import { types as CassandraTypes } from 'cassandra-driver'; 3 | import { CreateUserResponse } from './protos'; 4 | import { UserCreated } from './events'; 5 | import { createHashAsync } from './password-hashing'; 6 | import { toCassandraUuid, toProtobufTimestamp } from '../common/protobuf-conversions'; 7 | import { AlreadyExistsError } from '../common/grpc-errors'; 8 | import { getCassandraClient } from '../../common/cassandra'; 9 | import { publishAsync } from '../../common/message-bus'; 10 | 11 | /** 12 | * Creates a new user account. 13 | */ 14 | export function createUser(call, cb) { 15 | let { request } = call; 16 | 17 | // Hash the password provided 18 | return createHashAsync(request.password) 19 | .then(passwordHash => { 20 | // Insert the user credentials using a Lighweight Transactions (will fail if there is already a user 21 | // with the requested email address) 22 | let client = getCassandraClient(); 23 | let insertParams = [ 24 | request.email, 25 | passwordHash, 26 | toCassandraUuid(request.userId) 27 | ]; 28 | return client.executeAsync( 29 | 'INSERT INTO user_credentials (email, password, userid) VALUES (?, ?, ?) IF NOT EXISTS', insertParams); 30 | }) 31 | .then(resultSet => { 32 | let row = resultSet.first(); 33 | // Sanity check 34 | if (row === null) { 35 | throw new Error('No row returned when creating user'); 36 | } 37 | 38 | // See if the insert succeeded by checking the [applied] column in the first row returned 39 | if (row['[applied]'] === false) { 40 | throw new AlreadyExistsError('A user with that email address already exists'); 41 | } 42 | 43 | // UTC creation date of the user and use that as client-side generated timestamp for the write in C* 44 | let createdDate = new Date(Date.now()); 45 | let insertOpts = { timestamp: CassandraTypes.generateTimestamp(createdDate) }; 46 | 47 | // Go ahead and insert the user profile record and return the createdDate when successful 48 | let client = getCassandraClient(); 49 | let insertParams = [ 50 | toCassandraUuid(request.userId), 51 | request.firstName, 52 | request.lastName, 53 | request.email, 54 | createdDate 55 | ]; 56 | return client.executeAsync( 57 | 'INSERT INTO users (userid, firstname, lastname, email, created_date) VALUES (?, ?, ?, ?, ?)', 58 | insertParams, insertOpts).return(createdDate); 59 | }) 60 | .then(createdDate => { 61 | // Publish an event to tell the world about the new user 62 | let event = new UserCreated({ 63 | userId: request.userId, 64 | firstName: request.firstName, 65 | lastName: request.lastName, 66 | email: request.email, 67 | timestamp: toProtobufTimestamp(createdDate) 68 | }); 69 | return publishAsync(event); 70 | }) 71 | .asCallback(cb); 72 | }; -------------------------------------------------------------------------------- /src/services/user-management/events.js: -------------------------------------------------------------------------------- 1 | import { load } from '../common/load'; 2 | 3 | // Load events published by this service 4 | const eventsFile = 'user-management/user_management_events.proto'; 5 | const { 6 | UserCreated 7 | } = load(eventsFile).killrvideo.user_management.events; 8 | 9 | export { 10 | UserCreated 11 | }; -------------------------------------------------------------------------------- /src/services/user-management/get-user-profile.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import { InvalidArgumentError } from '../common/grpc-errors'; 3 | import { GetUserProfileResponse, UserProfile } from './protos'; 4 | import { toCassandraUuid } from '../common/protobuf-conversions'; 5 | import { getCassandraClient } from '../../common/cassandra'; 6 | 7 | /** 8 | * Gets profiles for the user(s) specified in the request. 9 | */ 10 | export function getUserProfile(call, cb) { 11 | let { request } = call; 12 | 13 | return Promise.try(() => { 14 | if (request.userIds.length === 0) { 15 | throw new InvalidArgumentError('Must specify at least one user id to get profiles for'); 16 | } 17 | 18 | // Try to enforce some sanity on the number we can get for any given request 19 | if (request.userIds.length > 20) { 20 | throw new InvalidArgumentError('Cannot get more than 20 user profiles at once'); 21 | } 22 | 23 | // Do multiple queries in parallel 24 | let client = getCassandraClient(); 25 | return Promise.map(request.userIds, userId => { 26 | let uid = toCassandraUuid(userId); 27 | return client.executeAsync('SELECT userid, firstname, lastname, email FROM users WHERE userid = ?', [ uid ]); 28 | }); 29 | }) 30 | .then(resultSets => { 31 | // Map each ResultSet and its single row to a profile to build the response 32 | let profiles = resultSets.map((resultSet, idx) => { 33 | let row = resultSet.first(); 34 | if (row === null) return null; 35 | 36 | return new UserProfile({ 37 | userId: request.userIds[idx], 38 | firstName: row.firstname, 39 | lastName: row.lastname, 40 | email: row.email 41 | }); 42 | }); 43 | return new GetUserProfileResponse({ profiles }); 44 | }) 45 | .asCallback(cb); 46 | }; -------------------------------------------------------------------------------- /src/services/user-management/index.js: -------------------------------------------------------------------------------- 1 | import { UserManagementService } from './protos'; 2 | import { getUserProfile } from './get-user-profile'; 3 | import { createUser } from './create-user'; 4 | import { verifyCredentials } from './verify-credentials'; 5 | 6 | /** 7 | * The user management service implementation. 8 | */ 9 | const implementation = { 10 | createUser, 11 | verifyCredentials, 12 | getUserProfile 13 | }; 14 | 15 | /** 16 | * User Management Service, responsible for managing/authenticating users. 17 | */ 18 | export default { 19 | name: 'UserManagementService', 20 | service: UserManagementService.service, 21 | implementation 22 | }; 23 | -------------------------------------------------------------------------------- /src/services/user-management/password-hashing.js: -------------------------------------------------------------------------------- 1 | import crypto from 'crypto'; 2 | import Promise from 'bluebird'; 3 | 4 | // Add Promise returning funtions to crypto lib 5 | Promise.promisifyAll(crypto); 6 | 7 | // These constants can change without breaking existing hashes that are stored somewhere 8 | const SALT_BYTES = 24; 9 | const HASH_BYTES = 24; 10 | const PBKDF2_ITERATIONS = 1000; 11 | const DIGEST = 'sha1'; 12 | 13 | // These constants can NOT be changed 14 | const DIGEST_IDX = 0; 15 | const ITERATION_IDX = 1; 16 | const SALT_IDX = 2; 17 | const HASH_IDX = 3; 18 | 19 | /** 20 | * Do a length-constant time comparison between two node Buffers. 21 | */ 22 | function slowEquals(a, b) { 23 | let diff = a.length ^ b.length; 24 | for (let i = 0; i < a.length && i < b.length; i++) { 25 | diff |= a[i] ^ b[i]; 26 | } 27 | return diff === 0; 28 | } 29 | 30 | /** 31 | * Creates a salted PBKDF2 password hash for the specified password. Returns a 32 | * Promise that resolves to a string with the hashed password. 33 | */ 34 | export function createHashAsync(password) { 35 | // Generate a random salt 36 | let saltPromise = crypto.randomBytesAsync(SALT_BYTES); 37 | 38 | // Hash the password 39 | let hashPromise = saltPromise.then(saltBuffer => { 40 | return crypto.pbkdf2Async(password, saltBuffer, PBKDF2_ITERATIONS, HASH_BYTES, DIGEST); 41 | }); 42 | 43 | return Promise.join(saltPromise, hashPromise, (saltBuffer, hashBuffer) => { 44 | // Concat salt and hash as base64 encoded strings into final result 45 | return `${DIGEST}:${PBKDF2_ITERATIONS}:${saltBuffer.toString('base64')}:${hashBuffer.toString('base64')}`; 46 | }); 47 | }; 48 | 49 | /** 50 | * Validates the given password against a hash that is known to be good. Returns 51 | * a Promise that resolves to true/false indicating whether the password is valid. 52 | */ 53 | export function validatePasswordAsync(password, goodHash) { 54 | // Take the good hash and split into its parts 55 | let hashPartsPromise = Promise.try(() => { 56 | let hashParts = goodHash.split(':'); 57 | if (hashParts.length !== 4) 58 | throw new Error('Invalid good hash'); 59 | return hashParts; 60 | }); 61 | 62 | // Hash the password provided 63 | let hashPromise = hashPartsPromise.then(hashParts => { 64 | let digest = hashParts[DIGEST_IDX]; 65 | let iterations = parseInt(hashParts[ITERATION_IDX]); 66 | let salt = new Buffer(hashParts[SALT_IDX], 'base64'); 67 | let hash = new Buffer(hashParts[HASH_IDX], 'base64'); 68 | return crypto.pbkdf2Async(password, salt, iterations, hash.length, digest); 69 | }); 70 | 71 | // Compare the password hash to the known good hash 72 | return Promise.join(hashPartsPromise, hashPromise, (hashParts, hashBuffer) => { 73 | let goodHashBuffer = new Buffer(hashParts[HASH_IDX], 'base64'); 74 | return slowEquals(goodHashBuffer, hashBuffer); 75 | }); 76 | }; -------------------------------------------------------------------------------- /src/services/user-management/protos.js: -------------------------------------------------------------------------------- 1 | import { load } from '../common/load'; 2 | 3 | // Load the protobuf definition to get the service and response objects 4 | const file = 'user-management/user_management_service.proto'; 5 | const { 6 | UserManagementService, 7 | CreateUserResponse, 8 | VerifyCredentialsResponse, 9 | GetUserProfileResponse, 10 | UserProfile 11 | } = load(file).killrvideo.user_management; 12 | 13 | export { 14 | UserManagementService, 15 | CreateUserResponse, 16 | VerifyCredentialsResponse, 17 | GetUserProfileResponse, 18 | UserProfile 19 | }; -------------------------------------------------------------------------------- /src/services/user-management/verify-credentials.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import { getCassandraClient } from '../../common/cassandra'; 3 | import { toProtobufUuid } from '../common/protobuf-conversions'; 4 | import { UnauthenticatedError } from '../common/grpc-errors'; 5 | import { validatePasswordAsync } from './password-hashing'; 6 | import { VerifyCredentialsResponse } from './protos'; 7 | 8 | /** 9 | * Verifies a user's credentials and returns the user's Id if successful or throws a Grpc 10 | * Unauthenticated error if credentials are incorrect. 11 | */ 12 | export function verifyCredentials(call, cb) { 13 | let { request } = call; 14 | 15 | // Find the user in Cassandra 16 | let getUser = Promise.try(() => { 17 | let client = getCassandraClient(); 18 | return client.executeAsync('SELECT email, password, userid FROM user_credentials WHERE email = ?', [ request.email ]); 19 | }) 20 | .then(resultSet => resultSet.first()); 21 | 22 | // Validate the password provided against the hash stored in Cassandra 23 | let validatePassword = getUser.then(row => { 24 | // If no row, then we definitely don't have a valid email/pass 25 | if (row === null) { 26 | return false; 27 | } 28 | 29 | return validatePasswordAsync(request.password, row.password); 30 | }); 31 | 32 | // Once we have user info and have validated the password... 33 | return Promise.join(getUser, validatePassword, (row, isValid) => { 34 | if (isValid !== true) { 35 | throw new UnauthenticatedError('Email address or password are not correct'); 36 | } 37 | 38 | // Return the user's Id from Cassandra 39 | return new VerifyCredentialsResponse({ 40 | userId: toProtobufUuid(row.userid) 41 | }); 42 | }) 43 | .asCallback(cb); 44 | }; -------------------------------------------------------------------------------- /src/services/video-catalog/constants.js: -------------------------------------------------------------------------------- 1 | /** 2 | * The length of time a video will stay in the latest videos table 3 | */ 4 | export const LATEST_VIDEOS_MAX_DAYS = 7; -------------------------------------------------------------------------------- /src/services/video-catalog/events.js: -------------------------------------------------------------------------------- 1 | import { load } from '../common/load'; 2 | 3 | // Load events published by this service 4 | const eventsFile = 'video-catalog/video_catalog_events.proto'; 5 | const { 6 | UploadedVideoAccepted, 7 | UploadedVideoAdded, 8 | YouTubeVideoAdded 9 | } = load(eventsFile).killrvideo.video_catalog.events; 10 | 11 | export { 12 | UploadedVideoAccepted, 13 | UploadedVideoAdded, 14 | YouTubeVideoAdded 15 | }; -------------------------------------------------------------------------------- /src/services/video-catalog/get-latest-video-previews.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import moment from 'moment'; 3 | import { GetLatestVideoPreviewsResponse, VideoPreview } from './protos'; 4 | import { LATEST_VIDEOS_MAX_DAYS } from './constants'; 5 | import { toCassandraUuid, toJavaScriptDate, toProtobufTimestamp, toProtobufUuid } from '../common/protobuf-conversions'; 6 | import { getCassandraClient } from '../../common/cassandra'; 7 | 8 | /** 9 | * Helper function for creating latest videos paging state tokens. 10 | */ 11 | function createPagingState(buckets, bucketIndex, rowsPagingState) { 12 | // Are there more rows in the current bucket? 13 | if (rowsPagingState !== null) { 14 | return `${buckets.join('')}${bucketIndex}${rowsPagingState}`; 15 | } 16 | 17 | // Do we still have buckets left? 18 | if (bucketIndex !== buckets.length - 1) { 19 | return `${buckets.join('')}${bucketIndex + 1}`; 20 | } 21 | 22 | // No more data available 23 | return ''; 24 | } 25 | 26 | /** 27 | * The pattern for parsing paging state which is: 28 | * - yyyyMMdd bucket integers repeated LATEST_VIDEOS_MAX_DAYS + 1 times 29 | * - a current bucket index integer 30 | * - an optional rowset paging state string for the position in the current bucket 31 | */ 32 | const PAGING_STATE_REGEX_PATTERN = `((?:[0-9]{8}){${LATEST_VIDEOS_MAX_DAYS + 1}})([0-9]{1})(.*)`; 33 | 34 | /** 35 | * Helper function for parsing latest videos paging state tokens. 36 | */ 37 | function parsePagingState(pagingState) { 38 | let buckets, bucketIndex, rowsPagingState; 39 | 40 | // If not empty, parse the parts 41 | if (pagingState !== '') { 42 | // The paging state will be yyyyMMdd buckets 8 times, followed by 1 bucket index int, 43 | // followed by the row paging state string 44 | let match = new RegExp(PAGING_STATE_REGEX_PATTERN).exec(pagingState); 45 | if (match === null) 46 | throw new Error('Bad paging state'); 47 | 48 | // Split the bucket string matched into the array of buckets 49 | buckets = match[1].match(/[0-9]{8}/g); 50 | 51 | // Index should just be an integer value 52 | bucketIndex = parseInt(match[2]); 53 | 54 | // We may or may not have row paging state 55 | rowsPagingState = match.length === 4 ? match[3] : null; 56 | 57 | return { buckets, bucketIndex, rowsPagingState }; 58 | } 59 | 60 | // Generate initial values since no paging state was provided 61 | buckets = []; 62 | bucketIndex = 0; 63 | rowsPagingState = null; 64 | 65 | // For each day in the past that we want to search, generate a YYYYMMDD string 66 | let now = moment.utc(); 67 | for (let i = 0; i <= LATEST_VIDEOS_MAX_DAYS; i++) { 68 | let b = now.clone().subtract(i, 'days').format('YYYYMMDD'); 69 | buckets.push(b); 70 | } 71 | 72 | return { buckets, bucketIndex, rowsPagingState }; 73 | } 74 | 75 | /** 76 | * Helper function to map a Cassandra row to a protobuf VideoPreview. 77 | */ 78 | function mapRowToVideoPreview(row) { 79 | return new VideoPreview({ 80 | videoId: toProtobufUuid(row.videoid), 81 | addedDate: toProtobufTimestamp(row.added_date), 82 | name: row.name, 83 | previewImageLocation: row.preview_image_location, 84 | userId: toProtobufUuid(row.userid) 85 | }); 86 | } 87 | 88 | /** 89 | * Gets the latest video previews. 90 | */ 91 | export function getLatestVideoPreviews(call, cb) { 92 | return Promise.try(() => { 93 | let { request } = call; 94 | let results = []; 95 | 96 | // Get or initialize the parts of our paging state 97 | let { buckets, bucketIndex, rowsPagingState } = parsePagingState(request.pagingState); 98 | let startingAddedDate = toJavaScriptDate(request.startingAddedDate); 99 | let startingVideoId = toCassandraUuid(request.startingVideoId); 100 | 101 | // Sanity check (if we're on last bucket and have no paging state for that bucket, nothing to do) 102 | if (bucketIndex === buckets.length - 1 && rowsPagingState === null) { 103 | return [ results, '' ]; // Empty results, empty paging state 104 | } 105 | 106 | // Run different select statements depending on whether we have a starting point in the list 107 | let hasStartingPoint = startingAddedDate !== null && startingVideoId !== null; 108 | let cql = hasStartingPoint 109 | ? 'SELECT * FROM latest_videos WHERE yyyymmdd = ? AND (added_date, videoid) <= (?, ?)' 110 | : 'SELECT * FROM latest_videos WHERE yyyymmdd = ?'; 111 | 112 | // Define a function for running a query 113 | function runQuery() { 114 | let recordsStillNeeded = request.pageSize - results.length; 115 | 116 | // Get parameter values for the query 117 | let bucket = buckets[bucketIndex]; 118 | let cqlParams = hasStartingPoint 119 | ? [ bucket, startingAddedDate, startingVideoId ] 120 | : [ bucket ]; 121 | 122 | // Set options for the query 123 | let queryOpts = { autoPage: false, fetchSize: recordsStillNeeded }; 124 | if (rowsPagingState !== null) 125 | queryOpts.pageState = rowsPagingState; 126 | 127 | // Run the query using the promisified client method 128 | let client = getCassandraClient(); 129 | return client.executeAsync(cql, cqlParams, queryOpts); 130 | }; 131 | 132 | // Define a function for collecting the results from the query 133 | function collectResults(resultSet) { 134 | // Add all rows to results 135 | resultSet.rows.forEach(row => results.push(row)); 136 | 137 | // Do we have enough rows? 138 | if (results.length === request.pageSize) { 139 | return [ results, createPagingState(buckets, bucketIndex, resultSet.pageState) ]; 140 | } 141 | 142 | // Are we out of buckets? 143 | if (bucketIndex === buckets.length - 1) { 144 | return [ results, '' ]; 145 | } 146 | 147 | // Keep querying next bucket 148 | bucketIndex++; 149 | rowsPagingState = null; 150 | return runQuery().then(collectResults); 151 | } 152 | 153 | // Kick off querying 154 | return runQuery().then(collectResults); 155 | }) 156 | .then(([ results, pagingState ]) => { 157 | // Create the Grpc response from the rows and paging state 158 | return new GetLatestVideoPreviewsResponse({ 159 | videoPreviews: results.map(mapRowToVideoPreview), 160 | pagingState 161 | }); 162 | }) 163 | .asCallback(cb); 164 | }; -------------------------------------------------------------------------------- /src/services/video-catalog/get-user-video-previews.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import { toJavaScriptDate, toCassandraUuid, toProtobufUuid, toProtobufTimestamp } from '../common/protobuf-conversions'; 3 | import { getCassandraClient } from '../../common/cassandra'; 4 | import { GetUserVideoPreviewsResponse, VideoPreview } from './protos'; 5 | 6 | /** 7 | * Gets a page of video previews that were authored by a particular user. 8 | */ 9 | export function getUserVideoPreviews(call, cb) { 10 | let { request } = call; 11 | return Promise.try(() => { 12 | let startingAddedDate = toJavaScriptDate(request.startingAddedDate); 13 | let startingVideoId = toCassandraUuid(request.startingVideoId); 14 | let userId = toCassandraUuid(request.userId); 15 | 16 | // Figure out what query to run based on whether we have starting info in the request 17 | let query, queryParams; 18 | if (startingVideoId === null) { 19 | // Just get the latest overall for the given user id 20 | query = 'SELECT * FROM user_videos WHERE userid = ?'; 21 | queryParams = [ userId ]; 22 | } else { 23 | // Get the latest starting from the point specified for the given user and going back in time 24 | query = 'SELECT * FROM user_videos WHERE userid = ? AND (added_date, videoid) <= (?, ?)'; 25 | queryParams = [ userId, startingAddedDate, startingVideoId ]; 26 | } 27 | 28 | // Get the number of records requested and include paging state if requested 29 | let queryOpts = { autoPage: false, fetchSize: request.pageSize }; 30 | if (request.pagingState !== null && request.pagingState !== '') { 31 | queryOpts.pageState = request.pagingState; 32 | } 33 | 34 | // Do the query 35 | let client = getCassandraClient(); 36 | return client.executeAsync(query, queryParams, queryOpts); 37 | }) 38 | .then(resultSet => { 39 | // Convert ResultSet rows to VideoPreviews and return the response 40 | return new GetUserVideoPreviewsResponse({ 41 | userId: request.userId, 42 | videoPreviews: resultSet.rows.map(row => new VideoPreview({ 43 | videoId: toProtobufUuid(row.videoid), 44 | addedDate: toProtobufTimestamp(row.added_date), 45 | name: row.name, 46 | previewImageLocation: row.preview_image_location, 47 | userId: toProtobufUuid(row.userid), 48 | })), 49 | pagingState: resultSet.pageState !== null ? resultSet.pageState : '' 50 | }) 51 | }) 52 | .asCallback(cb); 53 | }; -------------------------------------------------------------------------------- /src/services/video-catalog/get-video-previews.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import { InvalidArgumentError } from '../common/grpc-errors'; 3 | import { toCassandraUuid, toProtobufUuid, toProtobufTimestamp } from '../common/protobuf-conversions'; 4 | import { getCassandraClient } from '../../common/cassandra'; 5 | import { GetVideoPreviewsResponse, VideoPreview } from './protos'; 6 | 7 | /** 8 | * Get a limited number of video previews by id. 9 | */ 10 | export function getVideoPreviews(call, cb) { 11 | let { request } = call; 12 | return Promise.try(() => { 13 | // We're doing a multi-get here so try and enforce some sanity on the number we can get in 14 | // a single request 15 | if (request.videoIds.length > 20) { 16 | throw new InvalidArgumentError('Cannot fetch more than 20 videos at once'); 17 | } 18 | 19 | // Execute multiple selects in parallel for each requested video 20 | let client = getCassandraClient(); 21 | return request.videoIds.map(id => { 22 | let videoId = toCassandraUuid(id); 23 | return client.executeAsync( 24 | 'SELECT videoid, userid, added_date, name, preview_image_location FROM videos WHERE videoid = ?', [ videoId ]); 25 | }); 26 | }) 27 | .all() 28 | .then(resultSets => { 29 | // Each ResultSet should be a single row with a video's data 30 | let videoPreviews = resultSets.map(resultSet => { 31 | let row = resultSet.first(); 32 | return new VideoPreview({ 33 | videoId: toProtobufUuid(row.videoid), 34 | addedDate: toProtobufTimestamp(row.added_date), 35 | name: row.name, 36 | previewImageLocation: row.preview_image_location, 37 | userId: toProtobufUuid(row.userid) 38 | }); 39 | }); 40 | 41 | return new GetVideoPreviewsResponse({ 42 | videoPreviews 43 | }); 44 | }) 45 | .asCallback(cb); 46 | }; -------------------------------------------------------------------------------- /src/services/video-catalog/get-video.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import { GetVideoResponse, VideoLocationType } from './protos'; 3 | import { toCassandraUuid, toProtobufTimestamp, toProtobufUuid } from '../common/protobuf-conversions'; 4 | import { NotFoundError } from '../common/grpc-errors'; 5 | import { getCassandraClient } from '../../common/cassandra'; 6 | 7 | /** 8 | * Gets the details of a specific video from the catalog. 9 | */ 10 | export function getVideo(call, cb) { 11 | return Promise.try(() => { 12 | let { request } = call; 13 | 14 | let client = getCassandraClient(); 15 | let requestParams = [ 16 | toCassandraUuid(request.videoId) 17 | ]; 18 | return client.executeAsync('SELECT * FROM videos WHERE videoid = ?', requestParams); 19 | }) 20 | .then(resultSet => { 21 | let row = resultSet.first(); 22 | if (row === null) { 23 | throw new NotFoundError(`A video with id ${call.request.videoId.value} was not found`); 24 | } 25 | 26 | return new GetVideoResponse({ 27 | videoId: toProtobufUuid(row.videoid), 28 | userId: toProtobufUuid(row.userid), 29 | name: row.name, 30 | description: row.description, 31 | location: row.location, 32 | locationType: row.location_type, 33 | tags: row.tags === null ? [] : row.tags, 34 | addedDate: toProtobufTimestamp(row.added_date) 35 | }); 36 | }) 37 | .asCallback(cb); 38 | }; 39 | -------------------------------------------------------------------------------- /src/services/video-catalog/index.js: -------------------------------------------------------------------------------- 1 | import { VideoCatalogService } from './protos'; 2 | import { getLatestVideoPreviews } from './get-latest-video-previews'; 3 | import { getVideo } from './get-video'; 4 | import { submitYouTubeVideo } from './submit-youtube-video'; 5 | import { getVideoPreviews } from './get-video-previews'; 6 | import { getUserVideoPreviews } from './get-user-video-previews'; 7 | import { submitUploadedVideo } from './submit-uploaded-video'; 8 | 9 | /** 10 | * The video catalog service implementation. 11 | */ 12 | const implementation = { 13 | submitUploadedVideo, 14 | submitYouTubeVideo, 15 | getVideo, 16 | getVideoPreviews, 17 | getLatestVideoPreviews, 18 | getUserVideoPreviews 19 | }; 20 | 21 | /** 22 | * Video Catalog Service, responsible for tracking the catalog of videos available for playback. 23 | */ 24 | export default { 25 | name: 'VideoCatalogService', 26 | service: VideoCatalogService.service, 27 | implementation 28 | }; 29 | -------------------------------------------------------------------------------- /src/services/video-catalog/protos.js: -------------------------------------------------------------------------------- 1 | import { load } from '../common/load'; 2 | 3 | // Load the protobuf definition to get the service and response objects 4 | const serviceFile = 'video-catalog/video_catalog_service.proto'; 5 | const { 6 | VideoCatalogService, 7 | SubmitUploadedVideoResponse, 8 | SubmitYouTubeVideoResponse, 9 | GetVideoResponse, 10 | VideoLocationType, 11 | GetVideoPreviewsResponse, 12 | VideoPreview, 13 | GetLatestVideoPreviewsResponse, 14 | GetUserVideoPreviewsResponse 15 | } = load(serviceFile).killrvideo.video_catalog; 16 | 17 | export { 18 | VideoCatalogService, 19 | SubmitUploadedVideoResponse, 20 | SubmitYouTubeVideoResponse, 21 | GetVideoResponse, 22 | VideoLocationType, 23 | GetVideoPreviewsResponse, 24 | VideoPreview, 25 | GetLatestVideoPreviewsResponse, 26 | GetUserVideoPreviewsResponse 27 | }; -------------------------------------------------------------------------------- /src/services/video-catalog/submit-uploaded-video.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import { types as CassandraTypes } from 'cassandra-driver'; 3 | import { getCassandraClient } from '../../common/cassandra'; 4 | import { publishAsync } from '../../common/message-bus'; 5 | import { toCassandraUuid, toProtobufTimestamp } from '../common/protobuf-conversions'; 6 | import { SubmitUploadedVideoResponse, VideoLocationType } from './protos'; 7 | import { UploadedVideoAccepted } from './events'; 8 | 9 | /** 10 | * Submits an uploaded video to the catalog. 11 | */ 12 | export function submitUploadedVideo(call, cb) { 13 | let { request } = call; 14 | return Promise.try(() => { 15 | // Convert some request params to Cassandra values 16 | let videoId = toCassandraUuid(request.videoId); 17 | let userId = toCassandraUuid(request.userId); 18 | 19 | // Added date is current UTC date and time 20 | let addedDate = new Date(Date.now()); 21 | 22 | // Create the event we'll publish if our write is successful 23 | let event = new UploadedVideoAccepted({ 24 | videoId: request.videoId, 25 | uploadUrl: request.uploadUrl, 26 | timestamp: toProtobufTimestamp(addedDate) 27 | }); 28 | 29 | // Create the queries we're going to run in a batch (here, we're storing only the data we know now at submit time) 30 | let queries = [ 31 | { 32 | query: 'INSERT INTO videos (videoid, userid, name, description, tags, location_type, added_date) VALUES (?, ?, ?, ?, ?, ?, ?)', 33 | params: [ videoId, userId, request.name, request.description, request.tags, VideoLocationType.UPLOAD, addedDate ] 34 | }, 35 | { 36 | query: 'INSERT INTO user_videos (userid, added_date, videoid, name) VALUES (?, ?, ?, ?)', 37 | params: [ userId, addedDate, videoId, request.name ] 38 | } 39 | ]; 40 | 41 | // Use addedDate as a client-provided timestamp for the write in Cassandra 42 | let queryOpts = { timestamp: CassandraTypes.generateTimestamp(addedDate) }; 43 | 44 | // Execute the batch and if successful, return the event so it can be published 45 | let client = getCassandraClient(); 46 | return client.batchAsync(queries, queryOpts).return(event); 47 | }) 48 | .then(event => { 49 | // Tell the world about the uploaded video that was accepted 50 | return publishAsync(event); 51 | }) 52 | .then(() => { 53 | // Return the response 54 | return new SubmitUploadedVideoResponse(); 55 | }) 56 | .asCallback(cb); 57 | }; -------------------------------------------------------------------------------- /src/services/video-catalog/submit-youtube-video.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird'; 2 | import moment from 'moment'; 3 | import { types as CassandraTypes } from 'cassandra-driver'; 4 | import { getCassandraClient } from '../../common/cassandra'; 5 | import { publishAsync } from '../../common/message-bus'; 6 | import { toCassandraUuid, toProtobufTimestamp } from '../common/protobuf-conversions'; 7 | import { SubmitYouTubeVideoResponse, VideoLocationType } from './protos'; 8 | import { YouTubeVideoAdded } from './events'; 9 | import { LATEST_VIDEOS_MAX_DAYS } from './constants'; 10 | 11 | const videosCql = ` 12 | INSERT INTO videos ( 13 | videoid, userid, name, description, location, preview_image_location, tags, added_date, location_type) 14 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`; 15 | 16 | const userVideosCql = ` 17 | INSERT INTO user_videos ( 18 | userid, added_date, videoid, name, preview_image_location) 19 | VALUES (?, ?, ?, ?, ?)`; 20 | 21 | const latestVideosCql = ` 22 | INSERT INTO latest_videos ( 23 | yyyymmdd, added_date, videoid, userid, name, preview_image_location) 24 | VALUES (?, ?, ?, ?, ?, ?) 25 | USING TTL ?`; 26 | 27 | // Number of seconds a record should stay in the latest videos table 28 | const LATEST_VIDEOS_TTL_SECONDS = LATEST_VIDEOS_MAX_DAYS * 24 * 60 * 60; 29 | 30 | /** 31 | * Submits a YouTube video to the catalog. 32 | */ 33 | export function submitYouTubeVideo(call, cb) { 34 | return Promise.try(() => { 35 | let { request } = call; 36 | 37 | // Convert user and video Ids to Cassandra Uuids 38 | let videoId = toCassandraUuid(request.videoId); 39 | let userId = toCassandraUuid(request.userId); 40 | 41 | // Added date is current UTC date and time 42 | let addedDate = new Date(Date.now()); 43 | let yyyymmdd = moment(addedDate).format('YYYYMMDD'); 44 | 45 | // Use YouTubeVideoId from request along with well-known location of thumbnails as the preview image 46 | let previewImageLocation = `//img.youtube.com/vi/${request.youTubeVideoId}/hqdefault.jpg`; 47 | 48 | // Create the event to publish if we successfully write the data 49 | let event = new YouTubeVideoAdded({ 50 | videoId: request.videoId, 51 | userId: request.userId, 52 | name: request.name, 53 | description: request.description, 54 | location: request.youTubeVideoId, 55 | previewImageLocation, 56 | tags: request.tags, 57 | addedDate: toProtobufTimestamp(addedDate), 58 | timestamp: toProtobufTimestamp(addedDate) 59 | }); 60 | 61 | // Create the array for the batch of queries and their bind values 62 | let queries = [ 63 | { 64 | query: videosCql, 65 | params: [ 66 | videoId, userId, request.name, request.description, request.youTubeVideoId, previewImageLocation, 67 | request.tags, addedDate, VideoLocationType.YOUTUBE 68 | ] 69 | }, 70 | { query: userVideosCql, params: [ userId, addedDate, videoId, request.name, previewImageLocation ] }, 71 | { 72 | query: latestVideosCql, 73 | params: [ yyyymmdd, addedDate, videoId, userId, request.name, previewImageLocation, LATEST_VIDEOS_TTL_SECONDS ] 74 | } 75 | ]; 76 | 77 | // Generate a timestamp for the queries in the batch from the added date 78 | let queryOpts = { timestamp: CassandraTypes.generateTimestamp(addedDate) }; 79 | 80 | // Send the batch and return the event to publish if successful 81 | let client = getCassandraClient(); 82 | return client.batchAsync(queries, queryOpts).return(event); 83 | }) 84 | .then(event => { 85 | // Tell the world about the new YouTube video 86 | return publishAsync(event); 87 | }) 88 | .then(() => { 89 | // Return the response 90 | return new SubmitYouTubeVideoResponse(); 91 | }) 92 | .asCallback(cb); 93 | }; --------------------------------------------------------------------------------