├── .travis.yml ├── Makefile ├── README.md ├── basic-microservice ├── .dockerignore ├── .editorconfig ├── .eslintignore ├── .eslintrc ├── .gitignore ├── .npmrc ├── .nvmrc ├── Dockerfile ├── README.md ├── components │ ├── app │ │ └── index.js │ ├── config │ │ ├── confabulous.js │ │ └── index.js │ ├── express │ │ └── index.js │ ├── logging │ │ ├── bunyan.js │ │ ├── console.js │ │ ├── index.js │ │ ├── prepper-middleware.js │ │ └── prepper.js │ └── routes │ │ ├── admin-routes.js │ │ └── index.js ├── config │ ├── build.js │ ├── default.js │ ├── live.js │ ├── local.js │ └── test.js ├── docker-compose.yml ├── index.js ├── package.json ├── system.js ├── test │ ├── .eslintrc │ ├── env.js │ ├── mocha.opts │ ├── service.tests.js │ └── test-system.js └── yarn.lock ├── desktop-setup.md ├── followme.md ├── recipes-api ├── .dockerignore ├── .editorconfig ├── .eslintignore ├── .eslintrc ├── .gitignore ├── .npmrc ├── .nvmrc ├── Dockerfile ├── Makefile ├── README.md ├── components │ ├── config │ │ ├── confabulous.js │ │ └── index.js │ ├── express │ │ └── index.js │ ├── logging │ │ ├── bunyan.js │ │ ├── console.js │ │ ├── index.js │ │ ├── prepper-middleware.js │ │ ├── prepper.js │ │ └── sumo.js │ ├── main │ │ └── index.js │ ├── mongo │ │ ├── index.js │ │ └── initCollections.js │ ├── rabbitmq │ │ ├── index.js │ │ ├── initBroker.js │ │ └── initSubscriptions.js │ ├── routes │ │ ├── admin-routes.js │ │ ├── api-routes.js │ │ └── index.js │ └── store │ │ ├── index.js │ │ └── types │ │ ├── in-memory.js │ │ ├── index.js │ │ └── mongo.js ├── config │ ├── build.js │ ├── default.js │ ├── live.js │ ├── local.js │ └── test.js ├── docker-compose.yml ├── docker │ └── supervisor.conf ├── fixtures │ └── recipe_sample.json ├── index.js ├── infra │ └── deploy.sh ├── package.json ├── system.js ├── test │ ├── .eslintrc │ ├── e2e │ │ └── api.tests.js │ ├── env.js │ ├── mocha.opts │ ├── test-system.js │ └── unit │ │ └── store │ │ └── all.tests.js └── yarn.lock ├── recipes-crawler ├── .dockerignore ├── .editorconfig ├── .eslintignore ├── .eslintrc ├── .gitignore ├── .npmrc ├── .nvmrc ├── Dockerfile ├── Makefile ├── README.md ├── components │ ├── config │ │ ├── confabulous.js │ │ └── index.js │ ├── express │ │ └── index.js │ ├── lib │ │ ├── crawler.js │ │ └── index.js │ ├── logging │ │ ├── bunyan.js │ │ ├── console.js │ │ ├── index.js │ │ ├── prepper-middleware.js │ │ ├── prepper.js │ │ └── sumo.js │ ├── rabbitmq │ │ ├── index.js │ │ └── initBroker.js │ └── routes │ │ ├── admin-routes.js │ │ └── index.js ├── config │ ├── build.js │ ├── default.js │ ├── live.js │ ├── local.js │ └── test.js ├── docker-compose.yml ├── docker │ └── supervisor.conf ├── fixtures │ ├── recipe.json │ └── recipesSearch.json ├── index.js ├── infra │ └── deploy.sh ├── package.json ├── system.js ├── test │ ├── .eslintrc │ ├── e2e │ │ └── crawler.tests.js │ ├── env.js │ ├── mocha.opts │ ├── service.tests.js │ └── test-system.js └── yarn.lock ├── recipes-id-generator ├── .dockerignore ├── .editorconfig ├── .eslintignore ├── .eslintrc ├── .gitignore ├── .npmrc ├── .nvmrc ├── Dockerfile ├── Makefile ├── README.md ├── components │ ├── app │ │ └── index.js │ ├── config │ │ ├── confabulous.js │ │ └── index.js │ ├── express │ │ └── index.js │ ├── generators │ │ ├── index.js │ │ └── strategies │ │ │ ├── block.js │ │ │ ├── index.js │ │ │ └── memory.js │ ├── logging │ │ ├── bunyan.js │ │ ├── console.js │ │ ├── index.js │ │ ├── prepper-middleware.js │ │ ├── prepper.js │ │ └── sumo.js │ └── routes │ │ ├── admin-routes.js │ │ ├── api-routes.js │ │ └── index.js ├── config │ ├── build.js │ ├── default.js │ ├── live.js │ ├── local.js │ └── test.js ├── docker-compose.yml ├── docker │ └── supervisor.conf ├── index.js ├── infra │ └── deploy.sh ├── package.json ├── system.js ├── test │ ├── .eslintrc │ ├── env.js │ ├── mocha.opts │ ├── service.tests.js │ └── unit │ │ └── generators.tests.js └── yarn.lock ├── recipes-infra ├── .gitignore ├── .nvmrc ├── Makefile ├── config │ └── ec2.json ├── deploy.js ├── lib │ ├── aws.js │ ├── ec2.js │ ├── s3.js │ ├── ssh.js │ └── utils.js ├── package.json ├── prepare-ec2-instance.js └── yarn.lock └── ubuntu-setup.sh /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: true 2 | language: node_js 3 | env: 4 | global: 5 | # DOCKER_USERNAME 6 | - secure: hnjhn1EoTzu/B72kMakHlzQIrAERbV6IcDtDJWTnGLxXeFwK/Dtyurqrvuc/dTzWff37Gss+VtMEk1i+jTtDdTYpYUpAuiBNY9bTjGuRTS3oyehrbOrmh7iQDZXf+HB8/Xf6r1WQ+j/Be8sdat6aFgmacIjHvS10ABPFWm77YQuOvpC66LU8Vt5PTmW4k/BWtWWcZSTQPLLd7kVkSEih+b3nhgW+JMiwwsbcTni6y3DTcW37NVqkaz0blZgMQTgK1TLK3DvZ+7MS3rvKaDysiV++kOkoLLiYMDs5aKQWPEmxmdXT8Ry4fpRGZ37Cv0zDlrP8EO54Cc14cDHEJenwGqjxorCrdWYpzQvzg5V1zUnSZRHpO+VPFHidPijvu/TieL1McsBJlOFh0ciI7nCGE5rg7f5F2rFCOUtnmbAmZdm7Nb/g1mTW22YGvMLq33Lv341luwyMY/cmsQoR4ewBwW/nRmp4c4NpteciO3Vk/JwcqrK2WGMF4rJAYATS1xi0oz5D0Swhn1luwYFsjbnpT0H2w0PFXgXSzWMyQg949EAmUPsCB9LQC5Vn11mFnceVRCa1QiI0q42yg0og3k0ZxqPtnB8JVuwxUX3Gog89s4DVc6WshyQiEWgclZbQxbt78PFCDSXnVpUiY7fl0+GDhjk6MLlQnYhRFSHkJJZ0/1Y= 7 | # DOCKER_PASSWORD 8 | - secure: dY8jGLJayMC05Nw8RgMbxKf0FCCd1T/qh48sUxvT1Uwi1IfraYQP9FQgXIWAUGrU+n1I7K7TaUWgXikaQpKEebQLgtSSkdIchWp6V191cbEOfPIbzSJ6lDcPgHgWRk/LNJ29MoaqSqshnLPxfhE5vw9Jf9gs+hukUKYogUczEG6N+M7uaydDd9m76pgm7SailmJyqthIkxiJtzDz+E0BLMXbfnrjfnNRedYE5If2ZhCWEUulhAi3pMdA4NxW0uul4drOqZuZlfBqW5jQzfQKao3dj9oHFzKalgAukUIoSd1TkQMcOqvEPVhx13/EU8gkB0H4TDyAlUEUkLs2BewBEDpikf6posJkxA8lUbM4L1nIh19vFfLw75ZrfOEUvDwfKJGbkziBVQnBnlgCNAUxVwPPDOLoXPa5ZOHsMQcmaNoJ6leglg/i91OksTcD8tQFpNN0lFPd2phP8KVNbHAp8Yd4zqHANZPN+7lXYJeFmSrZJ41eh+SMIN+QQwTLMzZy+056ROp8bqBizpykv0+WPkhmzXy2ExAOMvb7puQ0cZur+uJ7niggc0h8rOdt9CjH8BOzG4tNcHgs1t0ziotYDGFHVAd6E6+3+UfWyCkVovPb1C+5Gmp1/OrYUmFTsY+YZ/RJIvdkwe37qDr3Df1RJPEFqsTvkakif9JzRyc4v14= 9 | # F2F_KEY (https://food2fork.com/about/api) 10 | - secure: NWOOfmeaia2X8NPRxA/QM3WowYVhsnjltqYpcRPrHqPiyRP1GSqezm38G0Hs6wMwYgCZ/SOmT/0iv/1ILrQTgt8rWg930NanJ9Bki1Gh66onmHUg3OvvFCOSlBmreYih3zJUa7+QwZyUDVsxjaJ54frINgILM7wx4fFPyIoYgS4panT2wLe1XLnQvQGlZS/0UfcGD90s2H0BI5JFFXoyIBRTFMrlqtD2lSqmcthg3zsvTTSyW3Y2ny4RAGdR0O1rD8w8qfrMBY5byfSyQN63a/jrUKAwvu5xASN8ROS4pnYL4SSzx8fX2uHzIWGerUQ7IkMv+95hm8ZD8I0odYEZ678uAdL/QXwOuc1ZnzWOwrXaDzFt9M//r4zrpkXKx1SArvCViSq+XSN6674JKRmLZAUdUWgmHbowZqC/MIay2NbDBXrHnCDAN1mdhkyjAARPRYZ8grBQBVdvitrq8zOYDO77jRpUQT4er4GkxW5d2TkfGteahMS7ex5ctl+Jek0tdkS0Ze6ahr4LpNvJE28bNSBi2rEFn/osNbHvkCLK/fJt6Q1bnUCax8BQsqBtuTMvod+LkSjMomeP2vuFpgy83ZVhz0rM6MIJbpypiifZ1HLwn6aTME7xp6TdkQwE4EnUHGGMTB65BMm4XOY/r58gufoPgrDpB2bgDWhYqR/cSbk= 11 | # AWS_ACCESS_KEY_ID 12 | - secure: fmRxv71O35DPMnDSLY8tRMCe/JuXwlIW8LubegAH9cWdNYsojD/dNJAWuKivQGGPQYBxy/e0i7WVigRWx6JRz+fnc/FmBWxydVpZc/wbAWeotWwYske0EGTyv2BQEVzJPCGnMEBWBNVrmkErITozyVbroLXGwVQJ7XMCoQ6lboz30ICUvFWP8We590HvkpIR9oEqUB9BPZBqA+Z/2YnNRzGRppfISFNoOQ0fR3XTDYeEK/uHN8xLX8zzcIhjZMDA/AJ2IE94fTo5ju6AIoSom00Y+vUFHg7mSiomJVMP6qtngUJ2ajVYoE14ZcQpQNpYu61s6dxCsUGwJy0Jm8w9n+AsQjL2KoftIjypLznSs2zRG5t1Dmtiv/UbLrR4AgzavhVKHQ1UcRHQkrtaJCtTI2rRka25Vwk2cRSQNsu2+FHlMB3g5sCTkkf7qjA3PPjkAhguvmfOUPCir6H3xOUeCkebELZNKzhpMeRBtPXf+huaVI8JYujQP0iq9JMDQKOoKaKnRSwTa6OgYr2r4NIGdBHRJjry7bheb2CQtGxEr40TE79DvOvz9qm9Uvie3ylYVfzD0atpzeU3Ayw8+hdp9UukJ8uszZlJ2MKF/ZvM8VahUh2Fh9G3IXq0YCEN8s7CZz+L3U5eZCRqUWaQb0fThB0JGXG9gbzTxBqeTVZM0Xg= 13 | # AWS_SECRET_ACCESS_KEY 14 | - secure: SLbpzJ/NiNYxH+GfB1bp08scCR3f5xRxEcTkd6trEQertrSgbPD8KXVbWy5eJfA6j8VBnnSiOGdQiSRL9G+MUCcjlLJEsNM6qJcP0cxCjgyhsYiA1qp1sE16a5/c8C2AptWfHz7yKuoPl5Bgul30X0ALKGVWPDJw4UgYKHO/93KCcuUcwl5mIN1XOl7lgPozFCIVEUHEVVReoMW6QwbZiyo/awYGuFzpNVMqJSrQ8Ex1zv77U8aqYQPHYyHh9xapjb7uqydY58xaEn/VPYUbI/25/6orbj2MHDj9Z9Xy+H9q79LWjyZ/tZso83+MF4ddJoxcNO36txnY0IbIUEuUzCYx7Ca6+GD4dNlMIlePfY5snnM00bfQXeLUJv0e8keewKNqeXHaWJUNLis3XQAp25kVc2Pg4jT72SJJOMyzxANonzClGWiuJaLaKKiLs38JvKNMJJFJzc5ZsoVsc+uyNoeWf/lqbUZQ7HBH+4mr3IoNyjsd0DmG52lF43mbe81DOgx/kOvR88QZcdVP3zdZNh59OALSQwjAJjaHPo9fsG8J6KEAxBPtv4EvLpAdaXqaKFWQmqRHFCfQJUrsjGDXlFZXnBAp3h/XxjEZFDUetuHRC9h8aNxVTjYAQB1xT1j0ErSJp/H4tHSqj/r3sK1SH9HVEEsVkDNHxgcZ65Wucxs= 15 | # MONGO_URL 16 | - secure: Ua1xsuk9g+DfUl/O5w3ZsE5yRrdC+vPreBkc28of73Udw5cuW5MAqzcxUfdKlzKEWYMaEYQ9GbIbpIj66zuLhugmuf40AXWLIqJmkgW0QLXoBfGJOWTkac30BwkF1Sv1AEypqX3xBPnmDqnmntlmwZYGKnUYOk84Z5LTUT4Dtx+c3+mztqTJhwqJG/KP6qMUR19H7PvGekZoXyzDvBOL5bpu+eSu8/U5U+GXIbWY0XK6efIN5KmLPN2Of0axyeGuTJHnNp6yfEugtM08NYWli9SRWfg5uelemYj0Vn/DNxPsOukdWmrQPjOwJNaRPLKSpNeMJ54HicP+DasrqjDmq1o5B6lOm0/193ypRIWiuMr6JKIf9Oyt8ZJB3ruZ19VmRfpVdS/bjGS46LQMHTZUCyhxGxn0j7pg8/8nKhzs/wqkFTmeAv6ZAbtF9kJ+VOIQd+zQOH6mxSt3U7c3Ceo5acfY4bIKBAplWk+JdWLjlrfqhPCv5OiRkSXAETv1/f45cOvLZWQ9FGNM3/k8Fw3fmu1rprwiZx5DmXTHhiImlZ/zW6EGN7tERFCpe7Y6pe4KN/ia8a39ORlJRzpnbErD2peky5u0p6ivRKRbKsnaehnaacMp0KoXUW6dC4ZlEGoP+Y4VhcKUS/2Xv4U35oAaEYXuv5xilg2LjEGwgV56kEE= 17 | # RABBIT_PWD 18 | - secure: VPNGA7NhIKxDETXZDjZDgiuICv+9HfyThYlrh2idQnwDsjstYDWiPEwxuGR8xHLmQy0VRy+67XjfVIYf/bWutazkmDGxKnELS669Ik7DrMb+dc+Hw7RHyXX0+WiwHCNXDDYodd5JsypvBWpWFB+r45VDQaAovO8gaj9PjdcSvpVugYW1Ni9vpVhYYBRLlfO3hj+FvLuYzIRLbJbAYPhaDGCh3HX23WjxA2/NgN68xOQHQ5nV1E6F8DqX9qXi0GSDfvVxWrlJ3NyyfB+jPpaSCUysrQx1/ec/SZ9DDd+oQbhUcX5xp91ZbkTBiLlSJacVZaYqbJwBbl6GQc72kOZyZcByk2Cja1w5B2as46slhq6imkpnnIZhhvsZo/75weaesiBgtIdaA2Pcug4LH9NItfUiJosBdSN2qJoQuA7lkH3vgOxkUuxxFlUCa6ZS+h3tIBOxxonO1K1dk/gw8Y9JbxdaUSM8KDAyI5LUGjehGCs2QccA6xtRQFgzyS0gblw9X2aGywG0VGnF3Wupnu4il9RtqHpHz0pO9gcALJzJjJqs4FzBmn61Qoo+gxAcoHtm1xr7aHE43H1pVzBvFvA/slVE9iuYFrinsYY793Wp2+sX16XfD0vplf5pC2JNe2OPn/STSWQGClxtQkN/fGVHgTB4Fix+Yc5d0YIP5RalPTk= 19 | # SUMO_URL 20 | - secure: K2q7SIL2/LRGq/ilksAjtwjnj1pnTPqpoT1zMpwywTcO6SkiOaVi+lPfxaR1izXUjt56ZOnNsERoOTZ2/nSHTDTsZvhTI29Kf+uord5vDWG2aXwLIHRtZnXK5MaDndy27W5esDgNMH4B6MUx9XZqRfkIfx+nsH7j/H3XA4vxwgn4hjJuZh2xrGW+PfnJ8WTdLqlCFvVidoDzvUIuysfi1NcVC8kGejNavtF8BAYWi439xTCaZo0SKlG30oZAi5Fu4CGHMwm+JdX4ccD7QqkUjSma5YyTsHVmKFncri3AegLzcduXRqQ0YuchJ7oUwdjag/ejiff4qbj0I2gp+yuUWOFLkloMH2IggTyTjqXTQyBL8Gwh9sK5tc/HZQhW/tGu5mPaR5aU3Sd1IoCY5NmBI31CFQmfqjX5zBQUOTOLlGJlFyVN0Bjn3/nwvhIZsUd50r1qz1J3k06Bh+0eqaa6Ceh2ki2CsVa2dO5wqF9cFDppjpFdG32HVRxWbvS2gunN4Cx35Cs8ZEc2l07A4mZDnBlWgjmX/Z3y5QNNCqmmwzz2/yiFBaizbFEcoUHd82O2vqAY63BeLXMxIsRx+P3AAZdSPc4G/4XDM9Hrs5viirV3ZkwhzgeawGXeeh37UWRwiCyoo3Vh0sgbHiIlRKjoVlgv3i2M7PCalGgz3ZoZebk= 21 | branches: 22 | only: 23 | - master 24 | 25 | install: # needed to run deployment scripts 26 | - cd ./$SERVICE 27 | - nvm install 28 | - nvm use 29 | - node --version 30 | - npm install --only=production 31 | before_script: 32 | - make ensure-dependencies 33 | script: 34 | - make brand 35 | - make package 36 | - make qa 37 | - make archive 38 | - make check 39 | after_script: 40 | - make deploy 41 | matrix: 42 | include: 43 | - env: SERVICE=recipes-infra 44 | - env: SERVICE=recipes-api 45 | services: 46 | - docker 47 | - env: SERVICE=recipes-crawler 48 | services: 49 | - docker 50 | - env: SERVICE=recipes-id-generator 51 | services: 52 | - docker 53 | notifications: 54 | slack: 55 | secure: eJtVJ7Kj2e3vjJazLAayQ7in1zgY/7/+yWd0lPckymkEQFbbgpN8AGWCNePG+9AWMjzb7icQ7tHGi8ZBxCEd6cJp6bTBgBMRZjBagMzfMwmej193Sg0GGvg2DVKjTzDY3x0btU+h0K23raAFc6UIpYqm84CiD7RhGh5GRDbXSpynhD9GWuGkZWj3eezr/ZIzfRAeV63y/2mE8e0Sx29rS367ktW/0mN65J1AlcDtsMiwlHVNX9CQh6FQBOK4o3BbmTvjpz8Gz5mpBKdq5JBhoSAI+/QRK9+Vy3oA/cpdJoMT96nxuvJIVqNOw24JxrBK+mp8V6xvk31oOOsJeQCqysiRTNImujCJ26xgsty/HDeLIA2AyCNqEqkSZhbYHpBXwjenbrperRl3twiAGpnbYBrE4lBHi1dGNYtde4c5q2fac/t9Q3fsNGua92ryRvlJhs+jU7Cvgebg3x2FjJ0fzmLOHkXDdhMETPYFVBspMhBw7gSN3CQhO10gGEoh7CqwZuL7xTLsnj9745bfbISHdiTph4fo442XG9Vs6eUJ9fp/HZ/J9jU2OB0LOmx1Ei1yCBhTR8agDAqQGXwCnTxXxhBJo0lF4t5dOV3eIvEL2pwnNbw+vCYWaV6FZrNEcIG6/2FnTTnmwvTuDmbf3njTi72Omaikn9upptciOPKEDyg= 56 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | DOCKER_HOST=quay.io 2 | DOCKER_ACCOUNT=feliun 3 | 4 | # CONTINUOS INTEGRATION 5 | 6 | package: 7 | @docker build --tag $(SERVICE):$(TRAVIS_BUILD_NUMBER) . 8 | @docker images 9 | 10 | brand: 11 | @node_modules/make-manifest/bin/make-manifest --extra "build.url: https://travis-ci.org/feliun/microservices-school/builds/"$(TRAVIS_BUILD_ID) --extra "build.number: "$(TRAVIS_BUILD_NUMBER) 12 | @cat ./manifest.json 13 | 14 | qa: 15 | @docker run --name $(SERVICE) --env SERVICE_ENV=build --rm --network=local --entrypoint npm $(SERVICE):$(TRAVIS_BUILD_NUMBER) run qa -- 16 | 17 | archive: start 18 | @docker login -u=$(DOCKER_USERNAME) -p=$(DOCKER_PASSWORD) $(DOCKER_HOST) 19 | docker ps 20 | @CONTAINER_ID=`docker ps | grep $(SERVICE) | awk '{print $$1}'` && \ 21 | docker commit $$CONTAINER_ID $(DOCKER_HOST)/$(DOCKER_ACCOUNT)/$(SERVICE) 22 | docker push $(DOCKER_HOST)/$(DOCKER_ACCOUNT)/$(SERVICE) 23 | 24 | check: 25 | @echo "Checking our $(SERVICE) container is up and running..." 26 | @curl http://localhost:$(SERVICE_PORT)/__/manifest 27 | 28 | ensure-dependencies: 29 | @npm run docker 30 | 31 | # CONTINUOUS DEPLOYMENT 32 | 33 | copy-infra: 34 | @echo "Copying infra tools inside service $(SERVICE)" 35 | @cp -r ../recipes-infra/lib ./infra 36 | @cp ../recipes-infra/deploy.js ./infra/ 37 | 38 | deploy: copy-infra 39 | @SERVICE_PORT=$(SERVICE_PORT) node ./infra/deploy.js 40 | @echo "Cleaning up..." 41 | @rm ./infra/deploy.js 42 | @rm -rf ./infra/lib -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # microservices-school 2 | Some material for my microservices workshop 3 | 4 | -------------------------------------------------------------------------------- /basic-microservice/.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | -------------------------------------------------------------------------------- /basic-microservice/.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | 15 | [Makefile] 16 | indent_style = tab -------------------------------------------------------------------------------- /basic-microservice/.eslintignore: -------------------------------------------------------------------------------- 1 | client/* 2 | -------------------------------------------------------------------------------- /basic-microservice/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "imperative-es6" 3 | } -------------------------------------------------------------------------------- /basic-microservice/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | manifest.json 3 | *.log 4 | -------------------------------------------------------------------------------- /basic-microservice/.npmrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feliun/microservices-school/0f4b67d3f64d53ec3e9957695803e9856db28498/basic-microservice/.npmrc -------------------------------------------------------------------------------- /basic-microservice/.nvmrc: -------------------------------------------------------------------------------- 1 | 7 2 | -------------------------------------------------------------------------------- /basic-microservice/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/feliun/docker-nvm-yarn 2 | COPY manifest.json /root/app/manifest.json 3 | -------------------------------------------------------------------------------- /basic-microservice/README.md: -------------------------------------------------------------------------------- 1 | # basic-microservice 2 | An example service using 3 | 4 | * [systemic](https://github.com/guidesmiths/systemic) 5 | * [confabulous](https://github.com/guidesmiths/confabulous) 6 | * [prepper](https://github.com/guidesmiths/prepper) 7 | * [systemic-express](https://github.com/guidesmiths/systemic-express) 8 | 9 | ## Features 10 | * Environmental configuration 11 | * Secrets obtained from a runtime location 12 | * Automatically applies schema changes on startup 13 | * Orderly startup / shutdown (e.g. establishes database connections before setting up http listeners and vice versa) 14 | * Graceful shutdown on errors, unhandled rejections, unhandled exceptions, SIGINT and SIGTERM 15 | * Useful log decorators, including request scoped logging 16 | * JSON logging to stdout in "proper" environments, human friendly logging locally 17 | * The Dockerfile uses settings from .npmrc and .nvmrc 18 | * The docker build cache busts using package.json and npm-shrinkwrap.json so npm install only runs when necessary 19 | * Deployed artifact (a docker image) is traceable back to SCM commit via manifest.json, exposed via /__/manifest endpoint 20 | 21 | ## Running locally 22 | ``` 23 | npm run docker 24 | npm start 25 | ``` 26 | 27 | ## Running tests 28 | ``` 29 | npm test 30 | ``` 31 | 32 | -------------------------------------------------------------------------------- /basic-microservice/components/app/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const optional = require('optional'); 3 | const { join } = require('path'); 4 | const manifest = optional(join(process.cwd(), 'manifest.json')) || {}; 5 | const pkg = require(join(process.cwd(), 'package.json')); 6 | 7 | module.exports = new System({ name: 'app' }) 8 | .add('manifest', manifest) 9 | .add('pkg', pkg); 10 | -------------------------------------------------------------------------------- /basic-microservice/components/config/confabulous.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = ({ confabulous } = {}) => { 4 | 5 | const Confabulous = confabulous || require('confabulous'); 6 | const loaders = Confabulous.loaders; 7 | let config; 8 | 9 | const start = (cb) => { 10 | if (config) return cb(null, config); 11 | 12 | new Confabulous() 13 | .add(config => loaders.require({ path: path.join(process.cwd(), 'config', 'default.js'), watch: true })) 14 | .add(config => loaders.require({ path: path.join(process.cwd(), 'config', `${process.env.SERVICE_ENV}.js`), mandatory: false })) 15 | .add(config => loaders.require({ path: path.join(process.cwd(), 'secrets', 'secrets.json'), watch: true, mandatory: false })) 16 | .add(config => loaders.args()) 17 | .on('loaded', cb) 18 | .on('error', cb) 19 | .end(cb); 20 | }; 21 | 22 | return { start }; 23 | }; 24 | -------------------------------------------------------------------------------- /basic-microservice/components/config/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const confabulous = require('./confabulous'); 3 | 4 | module.exports = new System({ name: 'config' }).add('config', confabulous(), { scoped: true }); 5 | -------------------------------------------------------------------------------- /basic-microservice/components/express/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const defaultMiddleware = require('systemic-express').defaultMiddleware; 3 | const app = require('systemic-express').app; 4 | const server = require('systemic-express').server; 5 | 6 | module.exports = new System({ name: 'express' }) 7 | .add('app', app()).dependsOn('config', 'logger') 8 | .add('middleware.default', defaultMiddleware()).dependsOn('logger', 'app', 'routes') 9 | .add('server', server()).dependsOn('config', 'app', 'middleware.default'); 10 | -------------------------------------------------------------------------------- /basic-microservice/components/logging/bunyan.js: -------------------------------------------------------------------------------- 1 | const bunyan = require('bunyan'); 2 | const R = require('ramda'); 3 | 4 | module.exports = () => { 5 | 6 | let log; 7 | 8 | const onMessage = (event) => { 9 | log[event.level](R.omit(['level', 'message'], event), event.message); 10 | }; 11 | 12 | const start = ({ pkg }, cb) => { 13 | log = bunyan.createLogger({ name: pkg.name }); 14 | return cb(null, onMessage); 15 | }; 16 | 17 | return { start }; 18 | }; 19 | -------------------------------------------------------------------------------- /basic-microservice/components/logging/console.js: -------------------------------------------------------------------------------- 1 | const chalk = require('chalk'); 2 | const hogan = require('hogan.js'); 3 | const R = require('ramda'); 4 | 5 | const response = hogan.compile('{{{displayTracer}}} {{{displayLevel}}} {{package.name}} {{{request.method}}} {{{response.statusCode}}} {{{request.url}}}'); 6 | const error = hogan.compile('{{{displayTracer}}} {{{displayLevel}}} {{package.name}} {{{message}}} {{{code}}}\n{{{error.stack}}} {{{details}}}'); 7 | const info = hogan.compile('{{{displayTracer}}} {{{displayLevel}}} {{package.name}} {{{message}}} {{{details}}}'); 8 | 9 | const colours = { 10 | debug: chalk.gray, 11 | info: chalk.white, 12 | warn: chalk.yellow, 13 | error: chalk.red, 14 | default: chalk.white 15 | }; 16 | 17 | module.exports = () => { 18 | 19 | const onMessage = (event) => { 20 | const details = R.pluck(event, []); 21 | const data = R.merge(event, { 22 | displayTracer: R.has('tracer', event) ? event.tracer.substr(0, 6) : '------', 23 | displayLevel: event.level.toUpperCase(), 24 | details: Object.keys(details).length ? `\n ${JSON.stringify(details, null, 2)}` : '' 25 | }); 26 | const colour = colours[event.level] || colours.default; 27 | const log = console[event.level] || console.info; // eslint-disable-line no-console 28 | if (R.has('response.statusCode', event)) log(colour(response.render(data))); 29 | else if (R.has('error.message', event)) log(colour(error.render(data))); 30 | else log(colour(info.render(data))); 31 | }; 32 | 33 | const start = (cb) => cb(null, onMessage); 34 | 35 | return { start }; 36 | }; 37 | -------------------------------------------------------------------------------- /basic-microservice/components/logging/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const prepper = require('./prepper'); 3 | const console = require('./console'); 4 | const bunyan = require('./bunyan'); 5 | const prepperMiddleware = require('./prepper-middleware'); 6 | 7 | module.exports = new System({ name: 'logging' }) 8 | .add('transports.console', console()) 9 | .add('transports.bunyan', bunyan()).dependsOn('pkg') 10 | .add('transports').dependsOn( 11 | { component: 'transports.console', destination: 'console' }, 12 | { component: 'transports.bunyan', destination: 'bunyan' }) 13 | .add('logger', prepper()).dependsOn('config', 'pkg', 'transports') 14 | .add('middleware.prepper', prepperMiddleware()).dependsOn('app') 15 | -------------------------------------------------------------------------------- /basic-microservice/components/logging/prepper-middleware.js: -------------------------------------------------------------------------------- 1 | const onHeaders = require('on-headers'); 2 | const R = require('ramda'); 3 | 4 | module.exports = ({ prepper } = {}) => { 5 | const handlers = (prepper || require('prepper')).handlers; 6 | 7 | const start = ({ app }, cb) => { 8 | app.use((req, res, next) => { 9 | const logger = req.app.locals.logger.child({ handlers: [ 10 | new handlers.Tracer(), 11 | new handlers.Merge(R.pick(['url', 'method', 'headers', 'params'], req), { key: 'request' }) 12 | ]}); 13 | 14 | onHeaders(res, () => { 15 | const response = { response: { statusCode: res.statusCode, headers: res.headers } }; 16 | if (res.statusCode === 400) logger.error(req.url, response); 17 | if (res.statusCode < 500) logger.info(req.url, response); 18 | else logger.error(req.url, response); 19 | }); 20 | 21 | res.locals.logger = logger; 22 | 23 | next(); 24 | }); 25 | 26 | cb(); 27 | }; 28 | 29 | return { start }; 30 | }; 31 | -------------------------------------------------------------------------------- /basic-microservice/components/logging/prepper.js: -------------------------------------------------------------------------------- 1 | const merge = require('lodash.merge'); 2 | const R = require('ramda'); 3 | 4 | module.exports = ({ prepper, transport } = {}) => { 5 | 6 | const prepperFn = prepper || require('prepper'); 7 | const handlers = prepperFn.handlers; 8 | 9 | const start = ({ config, transports, pkg = { name: 'unknown' } }, cb) => { 10 | const transportFn = transport || R.path([config.transport], transports); 11 | config = merge({ include: [], exclude: [] }, config); 12 | 13 | const logger = new prepperFn.Logger({ handlers: [ 14 | new handlers.Merge({ package: pkg }), 15 | new handlers.Merge({ service: { env: process.env.SERVICE_ENV } }), 16 | new handlers.Process(), 17 | new handlers.System(), 18 | new handlers.Timestamp(), 19 | new handlers.Flatten(), 20 | new handlers.KeyFilter({ include: config.include, exclude: config.exclude }), 21 | new handlers.Unflatten() 22 | ]}).on('message', event => { 23 | if (transportFn) transportFn(event); 24 | }); 25 | 26 | cb(null, logger); 27 | }; 28 | 29 | return { start }; 30 | }; 31 | -------------------------------------------------------------------------------- /basic-microservice/components/routes/admin-routes.js: -------------------------------------------------------------------------------- 1 | module.exports = () => { 2 | 3 | const start = ({ manifest = {}, app }, cb) => { 4 | app.get('/__/manifest', (req, res) => res.json(manifest)); 5 | app.post('/__/error', (req, res) => { 6 | setTimeout(() => process.emit('error', new Error('On Noes'))); 7 | }); 8 | app.post('/__/crash', (req, res) => { 9 | setTimeout(() => undefined.meh); 10 | }); 11 | app.post('/__/reject', (req, res) => { 12 | setTimeout(() => Promise.reject(new Error('Oh Noes'))); 13 | }); 14 | cb(); 15 | }; 16 | 17 | return { start }; 18 | }; 19 | -------------------------------------------------------------------------------- /basic-microservice/components/routes/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const adminRoutes = require('./admin-routes'); 3 | 4 | module.exports = new System({ name: 'routes' }) 5 | .add('routes.admin', adminRoutes()).dependsOn('config', 'logger', 'app', 'middleware.prepper', 'manifest') 6 | .add('routes').dependsOn('routes.admin'); 7 | -------------------------------------------------------------------------------- /basic-microservice/config/build.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logger: { 3 | transport: null 4 | } 5 | }; 6 | -------------------------------------------------------------------------------- /basic-microservice/config/default.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | server: { 3 | host: '0.0.0.0', 4 | port: 3000 5 | }, 6 | routes: { 7 | proxy: { 8 | routes: { 9 | '/api/1.0/other': 'http://other.example.com' 10 | } 11 | } 12 | }, 13 | logger: { 14 | transport: 'bunyan', 15 | include: [ 16 | 'tracer', 17 | 'timestamp', 18 | 'level', 19 | 'message', 20 | 'error.message', 21 | 'error.code', 22 | 'error.stack', 23 | 'request.url', 24 | 'request.headers', 25 | 'request.params', 26 | 'request.method', 27 | 'response.statusCode', 28 | 'response.headers', 29 | 'response.time', 30 | 'process', 31 | 'system', 32 | 'package.name', 33 | 'service' 34 | ], 35 | exclude: [ 36 | 'password', 37 | 'secret', 38 | 'token', 39 | 'request.headers.cookie', 40 | 'dependencies', 41 | 'devDependencies' 42 | ] 43 | } 44 | }; 45 | -------------------------------------------------------------------------------- /basic-microservice/config/live.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | } 3 | -------------------------------------------------------------------------------- /basic-microservice/config/local.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logger: { 3 | transport: 'console' 4 | } 5 | }; 6 | -------------------------------------------------------------------------------- /basic-microservice/config/test.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logger: { 3 | transport: null 4 | } 5 | }; 6 | -------------------------------------------------------------------------------- /basic-microservice/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | 5 | networks: 6 | default: 7 | external: 8 | name: local 9 | 10 | -------------------------------------------------------------------------------- /basic-microservice/index.js: -------------------------------------------------------------------------------- 1 | process.env.SERVICE_ENV = process.env.SERVICE_ENV || 'local'; 2 | 3 | const system = require('./system'); 4 | const runner = require('systemic-domain-runner'); 5 | const bunyan = require('bunyan'); 6 | const name = require('./package.json').name; 7 | const emergencyLogger = process.env.SERVICE_ENV === 'local' ? console : bunyan.createLogger({ name: name }); 8 | 9 | const die = (message, err) => { 10 | emergencyLogger.error(err, message); 11 | process.exit(1); 12 | }; 13 | 14 | runner(system(), { logger: emergencyLogger }).start((err, dependencies) => { 15 | if (err) die('Error starting system', err); 16 | dependencies.logger.info(`${dependencies.pkg.name} has started`); 17 | }); 18 | 19 | -------------------------------------------------------------------------------- /basic-microservice/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "basic-microservice", 3 | "version": "1.0.3", 4 | "description": "An example microservice using systemic, confabulous and prepper", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "node index.js", 8 | "server": "supervisor .", 9 | "test": "mocha test", 10 | "lint": "eslint .", 11 | "qa": "npm run lint && npm run test", 12 | "docker": "bash -c '(docker network inspect local 2>&1 > /dev/null || docker network create local) && docker-compose --file docker-compose.yml pull && docker-compose --file docker-compose.yml up -d --force-recreate'", 13 | "precommit": "npm run lint", 14 | "prepush": "npm run test" 15 | }, 16 | "keywords": [ 17 | "microservice", 18 | "systemic", 19 | "confabulous", 20 | "prepper", 21 | "example" 22 | ], 23 | "author": "GuideSmiths Ltd", 24 | "license": "ISC", 25 | "devDependencies": { 26 | "eslint": "^3.0.1", 27 | "eslint-config-imperative-es6": "^1.0.0", 28 | "eslint-plugin-mocha": "^4.0.0", 29 | "expect.js": "^0.3.1", 30 | "husky": "^0.11.4", 31 | "mocha": "^2.5.3", 32 | "supertest": "^3.0.0", 33 | "supertest-as-promised": "^4.0.2" 34 | }, 35 | "dependencies": { 36 | "R": "0.0.1", 37 | "aws-sdk": "^2.100.0", 38 | "body-parser": "^1.15.2", 39 | "boom": "^3.2.2", 40 | "bunyan": "^1.8.5", 41 | "chalk": "^1.1.3", 42 | "confabulous": "^1.1.0", 43 | "debug": "^2.2.0", 44 | "hogan.js": "^3.0.2", 45 | "http-proxy-middleware": "^0.17.4", 46 | "make-manifest": "^1.0.1", 47 | "node-ssh": "^4.2.3", 48 | "on-headers": "^1.0.1", 49 | "optimist": "^0.6.1", 50 | "optional": "^0.1.3", 51 | "pify": "^3.0.0", 52 | "prepper": "^1.1.0", 53 | "ramda": "^0.21.0", 54 | "systemic": "^1.3.0", 55 | "systemic-domain-runner": "^1.1.0", 56 | "systemic-express": "^1.0.1", 57 | "systemic-pg": "^1.0.6" 58 | }, 59 | "repository": { 60 | "type": "git", 61 | "url": "git+https://github.com/guidesmiths/svc-example.git" 62 | }, 63 | "bugs": { 64 | "url": "https://github.com/guidesmiths/svc-example/issues" 65 | }, 66 | "homepage": "https://github.com/guidesmiths/svc-example#readme" 67 | } 68 | -------------------------------------------------------------------------------- /basic-microservice/system.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const { join } = require('path'); 3 | 4 | module.exports = () => new System({ name: 'svc-example' }).bootstrap(join(__dirname, 'components')); 5 | -------------------------------------------------------------------------------- /basic-microservice/test/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "mocha": true 4 | }, 5 | "plugins": [ 6 | "mocha" 7 | ], 8 | "rules": { 9 | "mocha/no-exclusive-tests": 2 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /basic-microservice/test/env.js: -------------------------------------------------------------------------------- 1 | process.env.SERVICE_ENV = process.env.SERVICE_ENV || 'test' 2 | process.env.NODE_ENV = 'test' 3 | -------------------------------------------------------------------------------- /basic-microservice/test/mocha.opts: -------------------------------------------------------------------------------- 1 | --recursive 2 | --require ./test/env.js 3 | --timeout 5000 4 | -------------------------------------------------------------------------------- /basic-microservice/test/service.tests.js: -------------------------------------------------------------------------------- 1 | const expect = require('expect.js'); 2 | const system = require('../system'); 3 | const supertest = require('supertest-as-promised'); 4 | 5 | describe('Service Tests', () => { 6 | let request; 7 | let sys; 8 | 9 | before(done => { 10 | sys = system().start((err, { app }) => { 11 | if (err) return done(err); 12 | request = supertest(Promise)(app); 13 | done(); 14 | }); 15 | }); 16 | 17 | after(done => sys.stop(done)); 18 | 19 | it('should return manifest', () => 20 | request 21 | .get('/__/manifest') 22 | .expect(200) 23 | .then((response) => { 24 | expect(response.headers['content-type']).to.equal('application/json; charset=utf-8'); 25 | }) 26 | ); 27 | }); 28 | -------------------------------------------------------------------------------- /basic-microservice/test/test-system.js: -------------------------------------------------------------------------------- 1 | const system = require('../system'); 2 | 3 | module.exports = () => { 4 | return system(); 5 | }; 6 | 7 | -------------------------------------------------------------------------------- /desktop-setup.md: -------------------------------------------------------------------------------- 1 | # UBUNTU DESKTOP SETUP 2 | 3 | 1. Connect ssh to ec2 instance. 4 | 5 | 2. Become the super user after executing the command `sudo -s` 6 | 7 | 3. Type the following commands to install vncserver: 8 | 9 | ``` 10 | sudo apt-get update 11 | sudo apt-get -y install ubuntu-desktop 12 | sudo apt install -y xfce4 xfce4-goodies 13 | sudo apt install -y tightvncserver 14 | sudo apt-get install gnome-panel gnome-settings-daemon metacity nautilus gnome-terminal 15 | ``` 16 | 17 | 4. Type the command vncserver once. 18 | 19 | 5. Remember the password you use for accessing the vncserver. Kill vncserver by typing the command `vncserver -kill :1` 20 | 21 | 6. Backup the vnc config files: 22 | 23 | ``` 24 | mv ~/.vnc/xstartup ~/.vnc/xstartup.bak 25 | nano ~/.vnc/xstartup 26 | ``` 27 | 28 | Commands in this file are executed automatically whenever you start or restart the VNC server. We need VNC to start our desktop environment if it's not already started. Add these commands to the file: 29 | 30 | ``` 31 | #!/bin/sh 32 | 33 | export XKL_XMODMAP_DISABLE=1 34 | unset SESSION_MANAGER 35 | unset DBUS_SESSION_BUS_ADDRESS 36 | 37 | [ -x /etc/vnc/xstartup ] && exec /etc/vnc/xstartup 38 | [ -r $HOME/.Xresources ] && xrdb $HOME/.Xresources 39 | xsetroot -solid grey 40 | 41 | vncconfig -iconic & 42 | gnome-panel & 43 | gnome-settings-daemon & 44 | metacity & 45 | nautilus & 46 | gnome-terminal & 47 | ``` 48 | 49 | To ensure that the VNC server will be able to use this new startup file properly, we'll need to make it executable. 50 | 51 | `sudo chmod +x ~/.vnc/xstartup` 52 | 53 | 7. Now, restart the VNC server. 54 | 55 | `vncserver` 56 | 57 | You'll see output similar to this: 58 | 59 | ``` 60 | Output 61 | New 'X' desktop is your_hostname:1 62 | 63 | Starting applications specified in /home/sammy/.vnc/xstartup 64 | Log file is /home/ubuntu/.vnc/your_hostname:1.log 65 | ``` 66 | 67 | 8. Download and install tightvnc to connect remote desktop from [the following link](http://www.tightvnc.com/download.php) 68 | 69 | 9. Now run tightvnc viewer 70 | 71 | 10. Add the port no 5901 in your ec2 security group 72 | 73 | 11. Write your public ip in remote host text box and port no. `publicIp::port` 74 | 75 | 12. Your desktop in ec2 instance is ready and execute the command vncserver after every restart. 76 | 77 | 13. Install Firefox: 78 | 79 | ``` 80 | wget http://ftp.mozilla.org/pub/firefox/releases/50.0/linux-$(uname -m)/en-US/firefox-50.0.tar.bz2 81 | tar -xjf firefox-50.0.tar.bz2 82 | sudo mv firefox /opt/ 83 | sudo mv /usr/bin/firefox /usr/bin/firefox_old 84 | sudo ln -s /opt/firefox/firefox /usr/bin/firefox 85 | ``` 86 | -------------------------------------------------------------------------------- /followme.md: -------------------------------------------------------------------------------- 1 | # Preflight 2 | 3 | 1. Install nvm: https://github.com/creationix/nvm#install-script 4 | 2. Install node/npm: https://nodejs.org/en/ 5 | 3. Install docker: https://docs.docker.com/engine/installation/ 6 | 4. Create a slack account 7 | 5. Setup your github account 8 | 6. Fork https://github.com/feliun/microservices-school 9 | 10 | # Introduction 11 | 12 | ## Node JS Introduction 13 | 14 | ### A systemic-based service 15 | 16 | ``` 17 | cd basic-microservice 18 | npm i 19 | npm run start 20 | curl http://localhost:3000/__/manifest 21 | ``` 22 | 23 | ### in-memory RESTful API for recipes 24 | 25 | ``` 26 | cd recipes-api 27 | npm i 28 | npm t 29 | ``` 30 | - [Basic setup](https://github.com/feliun/microservices-school/commit/7f8ea46f69017f2b3748313fdabbe98d1d91b792) 31 | - [Choosing store: strategy pattern](https://github.com/feliun/microservices-school/commit/910cb1283606d6d95ba71dd822501001d8942a71) 32 | - [In-memory store](https://github.com/feliun/microservices-school/commit/8b07859deb97d5d98a3ae230b6af74923772e50f) 33 | - [Recipes API + tests](https://github.com/feliun/microservices-school/commit/12ecc27016a67a05e51a43c6853786857a04d0db) 34 | - [Optimistic control based on versions](https://github.com/feliun/microservices-school/commit/8949ea7119156e4eb6e279aa75f770545daa144f) 35 | 36 | # Interaction & comunication 37 | 38 | ## Mongo Introduction 39 | 40 | ### Mongo RESTful API for recipes 41 | - [Docker config](https://github.com/feliun/microservices-school/commit/829bbc3ced32e701136f94f55c9f0344abfcd377) 42 | - [Adding mongo systemic component](https://github.com/feliun/microservices-school/commit/2deb5b311ee785781d20c098f55a52d32ec5e5a4) 43 | - [Mongo store + tests](https://github.com/feliun/microservices-school/commit/5423f7f0f9c0acd358e8181bcc988e5434e26a1d) 44 | - [Refactor: proxy store](https://github.com/feliun/microservices-school/commit/77b26f343442a9c45f1c029a462db54a776bfc15) 45 | - [Refactor: multiple automatic tests](https://github.com/feliun/microservices-school/commit/ef2eb0d9e000dc63fbab2dce4c761ef089c4d28f) 46 | 47 | ## RabbitMQ Introduction 48 | 49 | ### Publishing conclusions 50 | - [Docker config](https://github.com/feliun/microservices-school/commit/efbec01dde74d9ae07a190c801166367660d9da1) 51 | - [Wiring up rabbitmq](https://github.com/feliun/microservices-school/commit/de850c4a9e45aef527e3b0fdb5a7c0d726a9f250) 52 | - [Publishing conclusions on every store action](https://github.com/feliun/microservices-school/commit/a466f9d5d08f510a18919ae3bd94f0965ffe1c59) 53 | - [Subscribing to conclusions to test published messages](https://github.com/feliun/microservices-school/commit/9035623f0742660f56430bfa5437a74e5cc61599) 54 | 55 | ## Building a recipes crawler 56 | 57 | ### Recipe crawler 58 | ``` 59 | mkdir recipes-crawler 60 | cp -r basic-microservice/* recipes-crawler/ && cp -r basic-microservice/.* recipes-crawler/ 61 | rm -rf node_modules/ 62 | nvm use && npm i 63 | npm t 64 | ``` 65 | - [Initial commit](https://github.com/feliun/microservices-school/commit/4aa6fb767a751480eeccb667d2bccd73f4e70228) 66 | - [Docker config](https://github.com/feliun/microservices-school/commit/2851c9323f9fd4d794f37091735777c1d4dfca1b) 67 | - [Wiring up rabbitmq](https://github.com/feliun/microservices-school/commit/cf5b166f2f69c20bfa60bb4f30d4bdb0bc68f326) 68 | - [Basic crawler set up](https://github.com/feliun/microservices-school/commit/072825d0bee2e3e46a21963d109a3bcd49b65130) 69 | - [Using config](https://github.com/feliun/microservices-school/commit/cd5b8f8342c24e6adf85572415511cb5fb377dff) 70 | - [Crawling recipes from the source](https://github.com/feliun/microservices-school/commit/1e2005bc386435a3ced034b59d3572278c9b01a3) 71 | - [Preparation for tests](https://github.com/feliun/microservices-school/commit/d2cde89e0405713e18f2a77e50603ac8083e4347) 72 | - [Testing crawling](https://github.com/feliun/microservices-school/commit/52a55c751d130242c2db977a5f60eefe93a33705) 73 | 74 | ### Wiring up both services 75 | - [Subsystem to initialise subscriptions](https://github.com/feliun/microservices-school/commit/1e87161d2d073e9cd42505a59bb8618a1f72c261) 76 | - [Rabbit config to subscribe to crawler](https://github.com/feliun/microservices-school/commit/3711a1d05f96a31f8b373e4165be976b66fa6746) 77 | - [Subscribing to crawled recipes](https://github.com/feliun/microservices-school/commit/a5468ce5750a8c6b9c1351211467f12ff2c4d787) 78 | 79 | ### Testing locally 80 | - Create your own spy queue and check mongo content 81 | - [First architecture problem: we need our own ids](https://github.com/feliun/microservices-school/commit/3ec8c312f7468689b537ef4d77aae214979a9773) 82 | - [Errors handling - recoverable, irrecoverable, absolved](https://github.com/feliun/microservices-school/commit/caaf2b38121591366fbe799a9c40eb358705883e) 83 | 84 | # DevOps 85 | 86 | ## Continuos integration (commit, build, test, brand, package, archive) 87 | - Setting up travis for our project 88 | - [Basic CI pipeline using Makefile: COMMIT, BUILD, TEST](https://github.com/feliun/microservices-school/commit/09ee8ba01300d70ef557694aa3d432c7a81708a6), outcome could be check [here](https://travis-ci.org/feliun/microservices-school/builds/252189365) - you might need to add `sleep 5` after `ensure-dependencies` if Travis is flaky 89 | - [Adding linting to qa process](https://github.com/feliun/microservices-school/commit/590035a42f7eab7ebca6dff67ac61e8d815da4b6) and [a small fix](https://github.com/feliun/microservices-school/commit/5427a38c41e6740cad25709d22331600ff91f864) 90 | - [Brand step](https://github.com/feliun/microservices-school/commit/3d2e01d008d1e831220e756d441f721bb8ea7bf4) 91 | - [Add docker login](https://github.com/feliun/microservices-school/commit/62197436c1bb18e9828d53c0f609647ecd3a20a0) 92 | - [Package step using Dockerfiles](https://github.com/feliun/microservices-school/commit/db5d8b1bef578817c6002f93afc07255e72f5968). Build output could be seen [here](https://travis-ci.org/feliun/microservices-school/builds/252595747) 93 | - Saving building time. Building a basic image from a [different repository](https://github.com/feliun/docker-nvm-yarn/commit/1dccb1a679d9a3aa71efe30cde3e24f1a6fcbb8e). Some instructions [here](https://github.com/feliun/docker-nvm-yarn#docker-nvm-yarn). The build output could be seen [here](https://quay.io/repository/feliun/docker-nvm-yarn/build/a5c5ecdd-fea8-436e-9898-dfb2ac60eeba). The image could be retrieved by doing `docker pull quay.io/feliun/docker-nvm-yarn`. 94 | - [Simpler build as docker is in charge of installation, nvm management etc](https://github.com/feliun/microservices-school/commit/b7d8440b5525037b39a651aaf74714a5a04bc3e9) 95 | - [Still not ready for previous step. We need to replace tests as well first](https://github.com/feliun/microservices-school/commit/4eed25035e34c4589ac7f52dfcac64bb0f0734a9) 96 | - Making it faster: running tests inside container [here](https://github.com/feliun/microservices-school/commit/29e0ee35fea4dd458d3a94d8a6748495685fcd7a) and [here](https://github.com/feliun/microservices-school/commit/e122515d7321c3f50f3851673d29db2e106a48a8). This could be seen [in this build](https://travis-ci.org/feliun/microservices-school/builds/253291881). 97 | - Fix for the brand step [here](https://github.com/feliun/microservices-school/commit/26f112be3b55e0c446dfc9c49b563a05be5f28f5) and [here](https://github.com/feliun/microservices-school/commit/444e35a168c819f5a6c447c676429ab9f38d4607). 98 | - Last step: [archiving the artefact](https://github.com/feliun/microservices-school/commit/6faae4e62912794ebd4a8d28059b6c869cb5efb6) and [enabling it in the build](https://github.com/feliun/microservices-school/commit/1f7ffb977a8d6b000b330aad67dc3d7c4039b1c6) and a [fix](https://github.com/feliun/microservices-school/commit/fadb27a833274ea5a4149d611ef64578c4b51503).Build could be checked [here](https://travis-ci.org/feliun/microservices-school/builds/253555540). 99 | - Checking containers [here](https://github.com/feliun/microservices-school/commit/dd1a2f57d9871aa21413c00dfcc56e06b6b2b65d) and [missing command here](https://github.com/feliun/microservices-school/commit/31c0cc8793a5b3063656beb8a07d9b5775cf4bcc). 100 | - [Fixing port allocation](https://github.com/feliun/microservices-school/commit/2cc5a893414a2061e5c92665ea67b93c80ef599c) 101 | - [Using build.json to test in CI, live.json to start the container](https://github.com/feliun/microservices-school/commit/bb9cf881cd9b8333303de115626773d28a79766f) and another [fix](https://github.com/feliun/microservices-school/commit/0c8cf8027bae8b1bfb54536153af40f0e569dd70). 102 | - [Adding slack integration for notifications](https://github.com/feliun/microservices-school/commit/f94577d67c1d8706be2207ab84c98887d41ff150) 103 | 104 | ## Continuos Deployment 105 | - Preparing live instance for mongo [here](https://github.com/feliun/microservices-school/commit/9a715f43cf8bfc534078b03b2ee13c07e5f8bc3f). Create your account here: https://mlab.com 106 | - Preparing live instance for rabbitmq [here](https://github.com/feliun/microservices-school/commit/e7fe1a99303dae575cf4601269f80a08b7582439). Create your account here: https://www.cloudamqp.com/ 107 | - [Preparing EC2 instance](https://github.com/feliun/microservices-school/commit/f456141230c4c50a16ff60f8059cbef2e5dac59f) 108 | - Deployment starts [here](https://github.com/feliun/microservices-school/blob/master/.travis.yml#L38). This uses [this main script](https://github.com/feliun/microservices-school/blob/master/Makefile#L38-L42) and [a file like this on each project(https://github.com/feliun/microservices-school/blob/master/recipes-api/infra/deploy.sh). 109 | 110 | ## Check-ins after deployment 111 | 1. Create a spy queue 112 | 2. Enter docker container for recipes-crawler `docker exec -it recipes-crawler bash` 113 | 3. Change config so the service starts immediately 114 | 4. Check the spy queue 115 | 5. Check the mongo collection 116 | 6. Enter docker container for recipes-crawler and check logs in /var/log/supervisor 117 | 118 | ## Logging 119 | - Create a free account in https://www.sumologic.com 120 | - Add sumo to your service [like this](https://github.com/feliun/microservices-school/commit/c0fe69cca7838e57c9decc6e77cc170f97ec8b30) 121 | - Few logging strategies implemented [here](https://github.com/feliun/microservices-school/tree/master/recipes-id-generator/components/generators/strategies). 122 | 123 | # Fixing an architectural issue 124 | 125 | ## Creating a infrastructure microservice: ID generator 126 | - Check out its code [here](https://github.com/feliun/microservices-school/tree/master/recipes-id-generator) 127 | - recipes-crawler to use the id generator if recipe is repeated: https://github.com/feliun/microservices-school/commit/0f210204a6142b08b633a4d5c80e068ef7d2e401 128 | - recipes-api to support getting recipes by source id: https://github.com/feliun/microservices-school/commit/21a0215a22b66c0846aa8d66559fa3705f220b28 129 | 130 | # By Branches 131 | 132 | ## all-services-developed 133 | 134 | ## devops1 135 | ### [CI: basic pipeline - commit, build, test](https://github.com/feliun/microservices-school/commit/83d3d449aae450891e9e9d580d5a7c8aecd4504d) 136 | ### [CI: running lint as part of the qa process](https://github.com/feliun/microservices-school/commit/70c6a79f8b7d61fefc5b12aef39fadd2ca7b9f25) 137 | ### [Installing prod and dev dependencies](https://github.com/feliun/microservices-school/commit/6bc9f259a0271a87ce2f61b26a7dc0eea87fe79d) 138 | ### [Giving some time to docker containers](https://github.com/feliun/microservices-school/commit/9ca77b28fb2db94fb5f5bab1dff1c55b2e7c3c00) 139 | ### [Brand step](https://github.com/feliun/microservices-school/commit/2f322b5c7a4edbb781f853adbb34431d5c0b9545) 140 | 141 | ## devops-2-docker 142 | ### [CI: logging into quay.io to be able to build](https://github.com/feliun/microservices-school/commit/dd11b848e1d0c9a07a58c6e698cabe8aa7f2ce33) 143 | ### [Package step using Dockerfiles](https://github.com/feliun/microservices-school/commit/903ebff460a9cccb362582db214690cea6f5a30e) 144 | ### [Simpler and faster build reusing the base image](https://github.com/feliun/microservices-school/commit/a63bfc3f57a950e5cc93b18b32c2bf88ba907466) 145 | ### [Refining package step](https://github.com/feliun/microservices-school/commit/317b5dbb9e3edfa442ad725360a1fc68f3276303) 146 | ### [Running tests inside container](https://github.com/feliun/microservices-school/commit/bc337034ed33053c17c20f8694c52c98ade19283) 147 | ### [Archiving new version artefact](https://github.com/feliun/microservices-school/commit/2ee6de0a8175fda83f1cded00a26e019e5b79945) 148 | ### [Defining start for each service](https://github.com/feliun/microservices-school/commit/442794c318e3f71b104fa78569eedf8ca003f147) 149 | ### [Checking services are up and running](https://github.com/feliun/microservices-school/commit/df13574494d04ecf3a804a0147b486237e045cdc) 150 | ### [Adding slack notifications](https://github.com/feliun/microservices-school/commit/8af9c5ed2eeef4cf52e2c861ef1bb75e566d24fd) 151 | ### [Cleaner Makefile using export variables](https://github.com/feliun/microservices-school/commit/865ccf40f6bcb6f72b9b0a8ccb857ab08339920b) 152 | -------------------------------------------------------------------------------- /recipes-api/.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | -------------------------------------------------------------------------------- /recipes-api/.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | 15 | [Makefile] 16 | indent_style = tab -------------------------------------------------------------------------------- /recipes-api/.eslintignore: -------------------------------------------------------------------------------- 1 | client/* 2 | -------------------------------------------------------------------------------- /recipes-api/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "imperative-es6" 3 | } -------------------------------------------------------------------------------- /recipes-api/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | manifest.json 3 | *.log 4 | -------------------------------------------------------------------------------- /recipes-api/.npmrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feliun/microservices-school/0f4b67d3f64d53ec3e9957695803e9856db28498/recipes-api/.npmrc -------------------------------------------------------------------------------- /recipes-api/.nvmrc: -------------------------------------------------------------------------------- 1 | 7 2 | -------------------------------------------------------------------------------- /recipes-api/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/feliun/docker-nvm-yarn 2 | COPY manifest.json /root/app/manifest.json 3 | -------------------------------------------------------------------------------- /recipes-api/Makefile: -------------------------------------------------------------------------------- 1 | SERVICE_PORT=3000 2 | 3 | start: 4 | @docker run -d -p $(SERVICE_PORT):$(SERVICE_PORT) -e SERVICE_ENV=live -e MONGO_URL=$(MONGO_URL) -e RABBIT_PWD=$(RABBIT_PWD) -e SUMO_URL=$(SUMO_URL) --name $(SERVICE) --network=local $(SERVICE):$(TRAVIS_BUILD_NUMBER) 5 | 6 | include ../Makefile 7 | -------------------------------------------------------------------------------- /recipes-api/README.md: -------------------------------------------------------------------------------- 1 | # svc-example 2 | An example service using 3 | 4 | * [systemic](https://github.com/guidesmiths/systemic) 5 | * [confabulous](https://github.com/guidesmiths/confabulous) 6 | * [prepper](https://github.com/guidesmiths/prepper) 7 | * [systemic-express](https://github.com/guidesmiths/systemic-express) 8 | 9 | ## Features 10 | * Environmental configuration 11 | * Secrets obtained from a runtime location 12 | * Automatically applies schema changes on startup 13 | * Orderly startup / shutdown (e.g. establishes database connections before setting up http listeners and vice versa) 14 | * Graceful shutdown on errors, unhandled rejections, unhandled exceptions, SIGINT and SIGTERM 15 | * Useful log decorators, including request scoped logging 16 | * JSON logging to stdout in "proper" environments, human friendly logging locally 17 | * The Dockerfile uses settings from .npmrc and .nvmrc 18 | * The docker build cache busts using package.json and npm-shrinkwrap.json so npm install only runs when necessary 19 | * Deployed artifact (a docker image) is traceable back to SCM commit via manifest.json, exposed via /__/manifest endpoint 20 | 21 | ## Running locally 22 | ``` 23 | npm run docker 24 | npm start 25 | ``` 26 | 27 | ## Running tests 28 | ``` 29 | npm test 30 | ``` 31 | 32 | -------------------------------------------------------------------------------- /recipes-api/components/config/confabulous.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = ({ confabulous } = {}) => { 4 | 5 | const Confabulous = confabulous || require('confabulous'); 6 | const loaders = Confabulous.loaders; 7 | let config; 8 | 9 | const start = (cb) => { 10 | if (config) return cb(null, config); 11 | 12 | new Confabulous() 13 | .add(config => loaders.require({ path: path.join(process.cwd(), 'config', 'default.js'), watch: true })) 14 | .add(config => loaders.require({ path: path.join(process.cwd(), 'config', `${process.env.SERVICE_ENV}.js`), mandatory: false })) 15 | .add(config => loaders.require({ path: path.join(process.cwd(), 'secrets', 'secrets.json'), watch: true, mandatory: false })) 16 | .add(config => loaders.args()) 17 | .on('loaded', cb) 18 | .on('error', cb) 19 | .end(cb); 20 | }; 21 | 22 | return { start }; 23 | }; 24 | -------------------------------------------------------------------------------- /recipes-api/components/config/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const confabulous = require('./confabulous'); 3 | 4 | module.exports = new System({ name: 'config' }).add('config', confabulous(), { scoped: true }); 5 | -------------------------------------------------------------------------------- /recipes-api/components/express/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const defaultMiddleware = require('systemic-express').defaultMiddleware; 3 | const app = require('systemic-express').app; 4 | const server = require('systemic-express').server; 5 | 6 | module.exports = new System({ name: 'express' }) 7 | .add('app', app()).dependsOn('config', 'logger') 8 | .add('middleware.default', defaultMiddleware()).dependsOn('logger', 'app', 'routes') 9 | .add('server', server()).dependsOn('config', 'app', 'middleware.default'); 10 | -------------------------------------------------------------------------------- /recipes-api/components/logging/bunyan.js: -------------------------------------------------------------------------------- 1 | const bunyan = require('bunyan'); 2 | const R = require('ramda'); 3 | 4 | module.exports = () => { 5 | 6 | let log; 7 | 8 | const onMessage = (event) => { 9 | log[event.level](R.omit(['level', 'message'], event), event.message); 10 | }; 11 | 12 | const start = ({ pkg }, cb) => { 13 | log = bunyan.createLogger({ name: pkg.name }); 14 | return cb(null, onMessage); 15 | }; 16 | 17 | return { start }; 18 | }; 19 | -------------------------------------------------------------------------------- /recipes-api/components/logging/console.js: -------------------------------------------------------------------------------- 1 | const chalk = require('chalk'); 2 | const hogan = require('hogan.js'); 3 | const R = require('ramda'); 4 | 5 | const response = hogan.compile('{{{displayTracer}}} {{{displayLevel}}} {{package.name}} {{{request.method}}} {{{response.statusCode}}} {{{request.url}}}'); 6 | const error = hogan.compile('{{{displayTracer}}} {{{displayLevel}}} {{package.name}} {{{message}}} {{{code}}}\n{{{error.stack}}} {{{details}}}'); 7 | const info = hogan.compile('{{{displayTracer}}} {{{displayLevel}}} {{package.name}} {{{message}}} {{{details}}}'); 8 | 9 | const colours = { 10 | debug: chalk.gray, 11 | info: chalk.white, 12 | warn: chalk.yellow, 13 | error: chalk.red, 14 | default: chalk.white 15 | }; 16 | 17 | module.exports = () => { 18 | 19 | const onMessage = (event) => { 20 | const details = R.pluck(event, []); 21 | const data = R.merge(event, { 22 | displayTracer: R.has('tracer', event) ? event.tracer.substr(0, 6) : '------', 23 | displayLevel: event.level.toUpperCase(), 24 | details: Object.keys(details).length ? `\n ${JSON.stringify(details, null, 2)}` : '' 25 | }); 26 | const colour = colours[event.level] || colours.default; 27 | const log = console[event.level] || console.info; // eslint-disable-line no-console 28 | if (R.has('response.statusCode', event)) log(colour(response.render(data))); 29 | else if (R.has('error.message', event)) log(colour(error.render(data))); 30 | else log(colour(info.render(data))); 31 | }; 32 | 33 | const start = (cb) => cb(null, onMessage); 34 | 35 | return { start }; 36 | }; 37 | -------------------------------------------------------------------------------- /recipes-api/components/logging/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const prepper = require('./prepper'); 3 | const console = require('./console'); 4 | const bunyan = require('./bunyan'); 5 | const sumo = require('./sumo'); 6 | const prepperMiddleware = require('./prepper-middleware'); 7 | 8 | module.exports = new System({ name: 'logging' }) 9 | .add('transports.console', console()) 10 | .add('transports.bunyan', bunyan()).dependsOn('pkg') 11 | .add('transports.sumo', sumo()) 12 | .add('transports').dependsOn( 13 | { component: 'transports.console', destination: 'console' }, 14 | { component: 'transports.sumo', destination: 'sumo' }, 15 | { component: 'transports.bunyan', destination: 'bunyan' }) 16 | .add('logger', prepper()).dependsOn('config', 'pkg', 'transports') 17 | .add('middleware.prepper', prepperMiddleware()).dependsOn('app') 18 | -------------------------------------------------------------------------------- /recipes-api/components/logging/prepper-middleware.js: -------------------------------------------------------------------------------- 1 | const onHeaders = require('on-headers'); 2 | const R = require('ramda'); 3 | 4 | module.exports = ({ prepper } = {}) => { 5 | const handlers = (prepper || require('prepper')).handlers; 6 | 7 | const start = ({ app }, cb) => { 8 | app.use((req, res, next) => { 9 | const logger = req.app.locals.logger.child({ handlers: [ 10 | new handlers.Tracer(), 11 | new handlers.Merge(R.pick(['url', 'method', 'headers', 'params'], req), { key: 'request' }) 12 | ]}); 13 | 14 | onHeaders(res, () => { 15 | const response = { response: { statusCode: res.statusCode, headers: res.headers } }; 16 | if (res.statusCode === 400) logger.error(req.url, response); 17 | if (res.statusCode < 500) logger.info(req.url, response); 18 | else logger.error(req.url, response); 19 | }); 20 | 21 | res.locals.logger = logger; 22 | 23 | next(); 24 | }); 25 | 26 | cb(); 27 | }; 28 | 29 | return { start }; 30 | }; 31 | -------------------------------------------------------------------------------- /recipes-api/components/logging/prepper.js: -------------------------------------------------------------------------------- 1 | const merge = require('lodash.merge'); 2 | const R = require('ramda'); 3 | 4 | module.exports = ({ prepper, transport } = {}) => { 5 | 6 | const prepperFn = prepper || require('prepper'); 7 | const handlers = prepperFn.handlers; 8 | 9 | const start = ({ config, transports, pkg = { name: 'unknown' } }, cb) => { 10 | const transportFn = transport || R.path([config.transport], transports); 11 | config = merge({ include: [], exclude: [] }, config); 12 | 13 | const logger = new prepperFn.Logger({ handlers: [ 14 | new handlers.Merge({ app: pkg.name }), 15 | new handlers.Merge({ env: process.env.SERVICE_ENV }), 16 | new handlers.Process(), 17 | new handlers.System(), 18 | new handlers.Timestamp(), 19 | new handlers.Flatten(), 20 | new handlers.KeyFilter({ include: config.include, exclude: config.exclude }), 21 | new handlers.Unflatten() 22 | ]}).on('message', event => { 23 | if (transportFn) transportFn(event); 24 | }); 25 | 26 | cb(null, logger); 27 | }; 28 | 29 | return { start }; 30 | }; 31 | -------------------------------------------------------------------------------- /recipes-api/components/logging/sumo.js: -------------------------------------------------------------------------------- 1 | const R = require('ramda'); 2 | const Sumologic = require('logs-to-sumologic'); 3 | 4 | let logger; 5 | 6 | module.exports = () => { 7 | 8 | const onMessage = (event) => { 9 | logger.log(event, (err) => { 10 | if (err) console.log(`Error posting to sumo! ${err.message}`); 11 | }); 12 | }; 13 | 14 | const start = (cb) => { 15 | const url = process.env.SUMO_URL; 16 | if (!url) return cb(null, R.identity); 17 | logger = Sumologic.createClient({ url, name: "RecipesCollector" }); 18 | return cb(null, onMessage); 19 | }; 20 | 21 | return { start }; 22 | }; 23 | -------------------------------------------------------------------------------- /recipes-api/components/main/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const optional = require('optional'); 3 | const { join } = require('path'); 4 | const manifest = optional(join(process.cwd(), 'manifest.json')) || {}; 5 | const pkg = require(join(process.cwd(), 'package.json')); 6 | 7 | module.exports = new System({ name: 'main' }) 8 | .add('manifest', manifest) 9 | .add('pkg', pkg); 10 | -------------------------------------------------------------------------------- /recipes-api/components/mongo/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const mongodb = require('systemic-mongodb'); 3 | const initCollections = require('./initCollections'); 4 | 5 | module.exports = new System({ name: 'mongodb' }) 6 | .add('mongo', mongodb()).dependsOn('config', 'logger') 7 | .add('collections', initCollections()).dependsOn('mongo'); 8 | -------------------------------------------------------------------------------- /recipes-api/components/mongo/initCollections.js: -------------------------------------------------------------------------------- 1 | module.exports = () => { 2 | 3 | const collections = { 4 | recipes: null 5 | }; 6 | 7 | const start = ({ mongo }, cb) => { 8 | collections.recipes = mongo.collection('recipes'); 9 | collections.recipes.createIndex({ id: 1 }, { unique: true, sparse: true }); 10 | return cb(null, collections); 11 | }; 12 | 13 | return { start }; 14 | }; 15 | -------------------------------------------------------------------------------- /recipes-api/components/rabbitmq/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const rabbitmq = require('systemic-rabbitmq'); 3 | const initBroker = require('./initBroker'); 4 | const initSubscriptions = require('./initSubscriptions'); 5 | 6 | module.exports = new System({ name: 'rabbit' }) 7 | .add('rabbitmq', rabbitmq()).dependsOn('config', 'logger') 8 | .add('broker', initBroker()).dependsOn('rabbitmq') 9 | .add('subscriptions', initSubscriptions()).dependsOn('broker', 'store', 'logger'); 10 | -------------------------------------------------------------------------------- /recipes-api/components/rabbitmq/initBroker.js: -------------------------------------------------------------------------------- 1 | const pify = require('pify'); 2 | 3 | module.exports = () => { 4 | 5 | const start = ({ rabbitmq }, cb) => { 6 | 7 | const publish = (...args) => new Promise((resolve, reject) => { 8 | rabbitmq.broker.publish(...args, (err, publication) => { 9 | if (err) return reject(err); 10 | publication 11 | .on('success', () => resolve()) 12 | .on('error', reject); 13 | }); 14 | }); 15 | 16 | const broker = { 17 | config: rabbitmq.broker.config, 18 | publish, 19 | subscribe: rabbitmq.broker.subscribe, 20 | nuke: pify(rabbitmq.broker.nuke), 21 | purge: pify(rabbitmq.broker.purge), 22 | }; 23 | return cb(null, broker); 24 | }; 25 | 26 | return { start }; 27 | }; 28 | -------------------------------------------------------------------------------- /recipes-api/components/rabbitmq/initSubscriptions.js: -------------------------------------------------------------------------------- 1 | module.exports = () => { 2 | const start = ({ broker, logger, store }, cb) => { 3 | logger.info('Initialising subscriptions'); 4 | 5 | const recipeHandler = (message, content, ackOrNack) => 6 | store.saveRecipe(content) 7 | .then(() => { 8 | ackOrNack(); 9 | logger.info(`Just saved correctly recipe with id ${content.id}`); 10 | }) 11 | .catch((error) => { 12 | logger.error(`There was an error digesting a recipe: ${error.message}, ${error.stack}`); 13 | const recoveryStrategy = error.recoverable ? broker.config.recovery.deferred_retry : broker.config.recovery.dead_letter; 14 | ackOrNack(error, recoveryStrategy); 15 | }); 16 | 17 | broker.subscribe('recipes_api', (err, subscription) => { 18 | if (err) return cb(err); 19 | subscription 20 | .on('error', (err) => logger.error(`Error receiving message: ${err.message}`)) 21 | .on('message', recipeHandler); 22 | cb(); 23 | }); 24 | }; 25 | 26 | return { start }; 27 | }; 28 | -------------------------------------------------------------------------------- /recipes-api/components/routes/admin-routes.js: -------------------------------------------------------------------------------- 1 | module.exports = () => { 2 | 3 | const start = ({ manifest = {}, app }, cb) => { 4 | app.get('/__/manifest', (req, res) => res.json(manifest)); 5 | app.post('/__/error', (req, res) => { 6 | setTimeout(() => process.emit('error', new Error('On Noes'))); 7 | }); 8 | app.post('/__/crash', (req, res) => { 9 | setTimeout(() => undefined.meh); 10 | }); 11 | app.post('/__/reject', (req, res) => { 12 | setTimeout(() => Promise.reject(new Error('Oh Noes'))); 13 | }); 14 | cb(); 15 | }; 16 | 17 | return { start }; 18 | }; 19 | -------------------------------------------------------------------------------- /recipes-api/components/routes/api-routes.js: -------------------------------------------------------------------------------- 1 | const Boom = require('boom'); 2 | const bodyParser = require('body-parser'); 3 | 4 | module.exports = () => { 5 | 6 | const start = ({ app, store }, cb) => { 7 | 8 | app.use(bodyParser.json()); 9 | 10 | app.get('/api/v1/recipes/:id', (req, res, next) => { 11 | const method = req.query.key && req.query.key === 'source_id' ? 'getRecipeBySourceId' : 'getRecipe'; 12 | store[method](req.params.id) 13 | .then((recipe) => recipe ? res.json(recipe) : next()) 14 | .catch(next); 15 | }); 16 | 17 | app.post('/api/v1/recipes', (req, res, next) => { 18 | // TODO apply Joi validation 19 | if(!req.body) return next(Boom.badRequest('Invalid or missing body')); 20 | store.saveRecipe(req.body) 21 | .then(() => res.status(204).send()) 22 | .catch(next); 23 | }); 24 | 25 | app.delete('/api/v1/recipes/:id', (req, res, next) => { 26 | store.deleteRecipe(req.params.id) 27 | .then(() => res.status(204).send()) 28 | .catch(next); 29 | }); 30 | 31 | cb(); 32 | }; 33 | 34 | return { start }; 35 | }; 36 | -------------------------------------------------------------------------------- /recipes-api/components/routes/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const adminRoutes = require('./admin-routes'); 3 | const apiRoutes = require('./api-routes'); 4 | 5 | module.exports = new System({ name: 'routes' }) 6 | .add('routes.admin', adminRoutes()).dependsOn('config', 'logger', 'app', 'middleware.prepper', 'manifest') 7 | .add('routes.api', apiRoutes()).dependsOn('config', 'logger', 'app', 'middleware.prepper', 'store') 8 | .add('routes').dependsOn('routes.admin', 'routes.api'); 9 | -------------------------------------------------------------------------------- /recipes-api/components/store/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const store = require('./types'); 3 | 4 | module.exports = new System({ name: 'store' }) 5 | .add('store', store()) 6 | .dependsOn('config', 'collections', 'logger', 'broker'); 7 | -------------------------------------------------------------------------------- /recipes-api/components/store/types/in-memory.js: -------------------------------------------------------------------------------- 1 | const R = require('ramda'); 2 | 3 | module.exports = () => { 4 | 5 | let recipes = {}; 6 | 7 | const saveRecipe = (recipe) => new Promise((resolve, reject) => { 8 | recipes[recipe.id] = recipe; 9 | return resolve(recipe); 10 | }); 11 | 12 | const updateRecipe = (update) => new Promise((resolve, reject) => { 13 | recipes[update.id] = update; 14 | return resolve(update); 15 | }); 16 | 17 | const deleteRecipe = (id) => new Promise((resolve, reject) => { 18 | delete recipes[id]; 19 | return resolve(id); 20 | }); 21 | 22 | const getRecipe = (id) => Promise.resolve(recipes[id]); 23 | 24 | const getRecipeBySourceId = (sourceId) => { 25 | const found = R.find(({ source_id }) => source_id === sourceId, R.values(recipes)) 26 | return Promise.resolve(found); 27 | }; 28 | 29 | const flush = () => new Promise((resolve) => { 30 | recipes = {}; 31 | return resolve(); 32 | }); 33 | 34 | return { 35 | saveRecipe, 36 | deleteRecipe, 37 | updateRecipe, 38 | getRecipe, 39 | getRecipeBySourceId, 40 | flush 41 | }; 42 | 43 | }; 44 | -------------------------------------------------------------------------------- /recipes-api/components/store/types/index.js: -------------------------------------------------------------------------------- 1 | const R = require('ramda'); 2 | const request = require('superagent'); 3 | const stores = require('require-all')({ 4 | dirname: __dirname, 5 | filter: (fileName) => fileName === 'index.js' ? undefined : fileName.replace('.js', '') 6 | }); 7 | 8 | const TIMEOUT = 3000; 9 | 10 | module.exports = () => { 11 | 12 | const start = ({ config, collections, logger, broker }, cb) => { 13 | const { strategy, idGenerator } = config; 14 | const pickedStore = stores[strategy] && stores[strategy](collections); 15 | if (!pickedStore) return cb(new Error(`No available store with name ${strategy}`)); 16 | 17 | const publish = (obj, action) => broker.publish('conclusions', obj, `recipes_api.v1.notifications.recipe.${action}`); 18 | 19 | const makeItRecoverable = (err) => { 20 | logger.error(`There was an error on the recipes api store: ${err.message}`); 21 | const recoverableErr = R.merge(err, { recoverable: true }); 22 | throw recoverableErr; 23 | }; 24 | 25 | const insert = (recipe) => { 26 | logger.info(`Inserting new recipe with id ${recipe.id}`); 27 | return pickedStore.saveRecipe(recipe) 28 | .then(() => publish(recipe, 'saved')); 29 | }; 30 | 31 | const getId = () => { 32 | const { host, path } = idGenerator; 33 | const idUrl = `${host}${path}`; 34 | return request 35 | .get(idUrl) 36 | .timeout(TIMEOUT) 37 | .catch((error) => { 38 | logger.error(`Error trying to reach ${idUrl}: ${error.message}`); 39 | throw error; 40 | }) 41 | .then(({ body }) => ({ id: body.id })); 42 | }; 43 | 44 | const saveRecipe = (recipe) => { 45 | if (recipe.id) { 46 | return pickedStore.getRecipe(recipe.id) 47 | .then((existing) => (existing ? updateRecipe(existing, recipe) : insert(recipe))) 48 | .catch(makeItRecoverable); 49 | } 50 | return getId() 51 | .then(({ id }) => { 52 | const newRecipe = R.merge(recipe, { id }); 53 | return insert(newRecipe); 54 | }) 55 | .catch(makeItRecoverable); 56 | }; 57 | 58 | const updateRecipe = (current, update) => { 59 | if (!update.id) return Promise.reject(new Error('Could not update recipe with no id')); 60 | if (!update.version) return Promise.reject(new Error('Could not update recipe with no version')); 61 | if (update.version < current.version) return discard(current); 62 | logger.info(`Updating recipe with id ${update.id} and version ${update.version}`); 63 | return pickedStore.updateRecipe(update) 64 | .then(() => publish(update, 'updated')); 65 | }; 66 | 67 | const deleteRecipe = (id) => { 68 | if (!id) return Promise.reject(new Error('Could not delete recipe with no id')); 69 | logger.info(`Deleting recipe with id ${id}`); 70 | const recipeId = parseInt(id, 10); 71 | return pickedStore.deleteRecipe(recipeId) 72 | .then(() => publish({ id: recipeId }, 'deleted')) 73 | .catch(makeItRecoverable); 74 | }; 75 | 76 | const getRecipe = (id) => { 77 | if (!id) return Promise.reject(new Error('Could not get recipe with no id')); 78 | const recipeId = parseInt(id, 10); 79 | return pickedStore.getRecipe(recipeId) 80 | .catch(makeItRecoverable); 81 | }; 82 | 83 | const getRecipeBySourceId = (sourceId) => { 84 | if (!sourceId) return Promise.reject(new Error('Could not get recipe with no source id')); 85 | return pickedStore.getRecipeBySourceId(sourceId) 86 | .catch(makeItRecoverable); 87 | }; 88 | 89 | const flush = () => { 90 | if (process.env.NODE_ENV !== 'test') throw new Error('Flushing the store is not allowed!'); 91 | return pickedStore.flush(); 92 | }; 93 | 94 | const discard = (recipe) => { 95 | logger.warn(`Discarding recipe with id ${recipe.id} since its version is outdated`); 96 | return Promise.resolve(recipe); 97 | }; 98 | 99 | const api = { 100 | saveRecipe, 101 | deleteRecipe, 102 | getRecipe, 103 | getRecipeBySourceId, 104 | flush 105 | }; 106 | 107 | cb(null, api); 108 | }; 109 | 110 | return { start }; 111 | 112 | }; 113 | -------------------------------------------------------------------------------- /recipes-api/components/store/types/mongo.js: -------------------------------------------------------------------------------- 1 | module.exports = ({ recipes }) => { 2 | 3 | const saveRecipe = (recipe) => recipes.insert(recipe); 4 | 5 | const updateRecipe = (update) => recipes.findOneAndUpdate({ id: update.id, version: { $lte: update.version } }, update, {}); 6 | 7 | const deleteRecipe = (id) => recipes.remove({ id }); 8 | 9 | const getRecipe = (id) => recipes.findOne({ id }); 10 | 11 | const getRecipeBySourceId = (sourceId) => recipes.findOne({ source_id: sourceId }); 12 | 13 | const flush = (query = {}) => recipes.remove(query); 14 | 15 | return { 16 | saveRecipe, 17 | deleteRecipe, 18 | updateRecipe, 19 | getRecipe, 20 | getRecipeBySourceId, 21 | flush 22 | }; 23 | 24 | }; 25 | -------------------------------------------------------------------------------- /recipes-api/config/build.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logger: { 3 | transport: null 4 | }, 5 | mongo: { 6 | url: 'mongodb://mongo:27017/ysojkvfe' 7 | }, 8 | rabbitmq: { 9 | defaults: { 10 | vhosts: { 11 | namespace: true, 12 | queues: { 13 | options: { 14 | durable: false 15 | } 16 | }, 17 | exchanges: { 18 | options: { 19 | durable: false 20 | } 21 | } 22 | } 23 | }, 24 | vhosts: { 25 | ysojkvfe: { 26 | connection: { 27 | hostname: 'rabbitmq' 28 | }, 29 | queues: { 30 | 'dead_letters:snoop': {}, 31 | 'retry:snoop': {}, 32 | 'delay:1ms': { 33 | options: { 34 | arguments: { 35 | 'x-message-ttl': 1, 36 | 'x-dead-letter-exchange': 'retry' 37 | } 38 | } 39 | }, 40 | 'recipes_api:snoop': {} 41 | }, 42 | bindings: { 43 | 'delay[delay.#] -> delay:1ms': {}, 44 | 'retry -> retry:snoop': {}, 45 | 'dead_letters -> dead_letters:snoop': {}, 46 | 'internal[recipes_api.v1.notifications.#.#] -> recipes_api:snoop': {} 47 | }, 48 | subscriptions: { 49 | dead_letters: { 50 | queue: 'dead_letters:snoop' 51 | }, 52 | retries: { 53 | queue: 'retry:snoop' 54 | }, 55 | recipes_snoop: { 56 | queue: 'recipes_api:snoop', 57 | contentType: 'application/json' 58 | } 59 | } 60 | } 61 | } 62 | } 63 | }; 64 | -------------------------------------------------------------------------------- /recipes-api/config/default.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | server: { 3 | host: '0.0.0.0', 4 | port: 3000 5 | }, 6 | mongo: { 7 | url: 'mongodb://127.0.0.1/recipes' 8 | }, 9 | store: { 10 | strategy: 'mongo', 11 | idGenerator: { 12 | host: 'http://127.0.0.1:3002', 13 | path: '/api/v1/id' 14 | } 15 | }, 16 | logger: { 17 | transport: 'bunyan', 18 | include: [ 19 | 'tracer', 20 | 'timestamp', 21 | 'level', 22 | 'message', 23 | 'error.message', 24 | 'error.code', 25 | 'error.stack', 26 | 'request.url', 27 | 'request.headers', 28 | 'request.params', 29 | 'request.method', 30 | 'response.statusCode', 31 | 'response.headers', 32 | 'response.time', 33 | 'process', 34 | 'system', 35 | 'env', 36 | 'app' 37 | ], 38 | exclude: [ 39 | 'password', 40 | 'secret', 41 | 'token', 42 | 'request.headers.cookie', 43 | 'dependencies', 44 | 'devDependencies' 45 | ] 46 | }, 47 | rabbitmq: { 48 | defaults: {}, 49 | vhosts: { 50 | ysojkvfe: { 51 | connection: { 52 | hostname: '127.0.0.1', 53 | user: 'rabbitmq', 54 | password: 'rabbitmq' 55 | }, 56 | exchanges: [ 57 | 'internal', 58 | 'delay', 59 | 'retry', 60 | 'dead_letters' 61 | ], 62 | queues: { 63 | 'recipes_api:recipe:received': { 64 | 'options': { 65 | 'arguments': { 66 | 'x-dead-letter-exchange': 'dead_letters', 67 | 'x-dead-letter-routing-key': 'recipes_api.dead_letter' 68 | } 69 | } 70 | }, 71 | 'dead_letter:recipes_api': {} 72 | }, 73 | bindings: { 74 | 'internal[recipes_crawler.v1.notifications.recipe.crawled] -> recipes_api:recipe:received': {}, 75 | 'retry[recipes_api:recipe:received.#] -> recipes_api:recipe:received': {}, 76 | 'dead_letters[recipes_api.dead_letter] -> dead_letter:recipes_api': {}, 77 | }, 78 | subscriptions: { 79 | recipes_api: { 80 | queue: 'recipes_api:recipe:received', 81 | prefetch: 5, 82 | contentType: 'application/json' 83 | } 84 | }, 85 | publications: { 86 | conclusions: { 87 | exchange: 'internal' 88 | }, 89 | 'retry_in_5m': { 90 | exchange: 'delay', 91 | options: { 92 | CC: [ 93 | 'delay.5m' 94 | ] 95 | } 96 | } 97 | } 98 | } 99 | }, 100 | recovery: { 101 | 'deferred_retry': [ 102 | { 103 | strategy: 'forward', 104 | attempts: 10, 105 | publication: 'retry_in_5m', 106 | xDeathFix: true 107 | }, 108 | { 109 | strategy: 'nack' 110 | } 111 | ], 112 | 'dead_letter': [ 113 | { 114 | strategy: 'republish', 115 | immediateNack: true 116 | } 117 | ] 118 | } 119 | } 120 | }; 121 | -------------------------------------------------------------------------------- /recipes-api/config/live.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logger: { 3 | transport: 'sumo' 4 | }, 5 | mongo: { 6 | url: process.env.MONGO_URL || 'mongodb://mongo:27017/ysojkvfe' 7 | }, 8 | store: { 9 | idGenerator: { 10 | host: 'http://recipes-id-generator:3002', 11 | } 12 | }, 13 | rabbitmq: { 14 | defaults: {}, 15 | vhosts: { 16 | ysojkvfe: { 17 | connection: { 18 | hostname: 'swan.rmq.cloudamqp.com', 19 | user: 'ysojkvfe', 20 | password: process.env.RABBIT_PWD || 'N/A' 21 | } 22 | } 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /recipes-api/config/local.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logger: { 3 | transport: 'console' 4 | } 5 | }; 6 | -------------------------------------------------------------------------------- /recipes-api/config/test.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logger: { 3 | transport: null 4 | }, 5 | rabbitmq: { 6 | defaults: { 7 | vhosts: { 8 | namespace: true, 9 | queues: { 10 | options: { 11 | durable: false 12 | } 13 | }, 14 | exchanges: { 15 | options: { 16 | durable: false 17 | } 18 | } 19 | } 20 | }, 21 | vhosts: { 22 | ysojkvfe: { 23 | queues: { 24 | 'dead_letters:snoop': {}, 25 | 'retry:snoop': {}, 26 | 'delay:1ms': { 27 | options: { 28 | arguments: { 29 | 'x-message-ttl': 1, 30 | 'x-dead-letter-exchange': 'retry' 31 | } 32 | } 33 | }, 34 | 'recipes_api:snoop': {} 35 | }, 36 | bindings: { 37 | 'delay[delay.#] -> delay:1ms': {}, 38 | 'retry -> retry:snoop': {}, 39 | 'dead_letters -> dead_letters:snoop': {}, 40 | 'internal[recipes_api.v1.notifications.#.#] -> recipes_api:snoop': {} 41 | }, 42 | subscriptions: { 43 | dead_letters: { 44 | queue: 'dead_letters:snoop' 45 | }, 46 | retries: { 47 | queue: 'retry:snoop' 48 | }, 49 | recipes_snoop: { 50 | queue: 'recipes_api:snoop', 51 | contentType: 'application/json' 52 | } 53 | } 54 | } 55 | } 56 | } 57 | }; 58 | -------------------------------------------------------------------------------- /recipes-api/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | 5 | mongo: 6 | image: mongo 7 | container_name: mongo 8 | ports: 9 | - '27017:27017' 10 | command: "--logpath=/dev/null" 11 | 12 | rabbitmq: 13 | image: "rabbitmq:3-management" 14 | container_name: rabbitmq 15 | hostname: "rabbitmq" 16 | environment: 17 | RABBITMQ_ERLANG_COOKIE: "SWQOKODSQALRPCLNMEQG" 18 | RABBITMQ_DEFAULT_USER: "rabbitmq" 19 | RABBITMQ_DEFAULT_PASS: "rabbitmq" 20 | RABBITMQ_DEFAULT_VHOST: "ysojkvfe" 21 | ports: 22 | - "15672:15672" 23 | - "5672:5672" 24 | 25 | networks: 26 | default: 27 | external: 28 | name: local 29 | 30 | -------------------------------------------------------------------------------- /recipes-api/docker/supervisor.conf: -------------------------------------------------------------------------------- 1 | [program:recipes-api] 2 | directory=/root/app 3 | command=/usr/bin/npm run start 4 | autorestart=true 5 | stdout_logfile=/var/log/supervisor/recipes-api.out.log 6 | stderr_logfile=/var/log/supervisor/recipes-api.err.log 7 | stdout_logfile_backups=5 8 | stderr_logfile_backups=5 9 | -------------------------------------------------------------------------------- /recipes-api/fixtures/recipe_sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "publisher": "Elana's Pantry", 3 | "ingredients": [ 4 | "1 cup blanched almond flour", 5 | " teaspoon celtic sea salt", 6 | " teaspoon baking soda", 7 | "1 tablespoon cinnamon", 8 | "1 teaspoon nutmeg", 9 | " teaspoon cloves", 10 | " cup roasted pumpkin", 11 | "2 tablespoons honey", 12 | " teaspoon stevia", 13 | "3 eggs" 14 | ], 15 | "recipeUrl": "http://www.elanaspantry.com/paleo-pumpkin-bread/", 16 | "imageUrl": "http://static.food2fork.com/paleopumpkinbreadglutenfreegrainfreerecipe413x575b26e.jpg", 17 | "socialRank": 100, 18 | "source_id": "abcd", 19 | "publisherUrl": "http://www.elanaspantry.com", 20 | "title": "Paleo Pumpkin Bread", 21 | "version": 1497113136363 22 | } 23 | -------------------------------------------------------------------------------- /recipes-api/index.js: -------------------------------------------------------------------------------- 1 | process.env.SERVICE_ENV = process.env.SERVICE_ENV || 'local'; 2 | 3 | const system = require('./system'); 4 | const runner = require('systemic-domain-runner'); 5 | const bunyan = require('bunyan'); 6 | const name = require('./package.json').name; 7 | const emergencyLogger = process.env.SERVICE_ENV === 'local' ? console : bunyan.createLogger({ name: name }); 8 | 9 | const die = (message, err) => { 10 | emergencyLogger.error(err, message); 11 | process.exit(1); 12 | }; 13 | 14 | runner(system(), { logger: emergencyLogger }).start((err, dependencies) => { 15 | if (err) die('Error starting system', err); 16 | dependencies.logger.info(`${dependencies.pkg.name} has started`); 17 | }); 18 | 19 | -------------------------------------------------------------------------------- /recipes-api/infra/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker run -d -p 3000:3000 -e SERVICE_ENV=live -e MONGO_URL=$MONGO_URL -e RABBIT_PWD=$RABBIT_PWD -e SUMO_URL=$SUMO_URL --name recipes-api --network=local quay.io/feliun/recipes-api:latest 4 | -------------------------------------------------------------------------------- /recipes-api/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "recipes-api", 3 | "version": "1.0.3", 4 | "description": "A RESTful API for recipes", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "node index.js", 8 | "server": "supervisor .", 9 | "test": "mocha test", 10 | "lint": "eslint .", 11 | "qa": "npm run lint && npm run test", 12 | "docker": "bash -c '(docker network inspect local 2>&1 > /dev/null || docker network create local) && docker-compose --file docker-compose.yml pull && docker-compose --file docker-compose.yml up -d --force-recreate'", 13 | "precommit": "npm run lint", 14 | "prepush": "npm run test" 15 | }, 16 | "keywords": [ 17 | "microservice", 18 | "systemic", 19 | "confabulous", 20 | "prepper", 21 | "example" 22 | ], 23 | "author": "GuideSmiths Ltd", 24 | "license": "ISC", 25 | "devDependencies": { 26 | "eslint": "^3.0.1", 27 | "eslint-config-imperative-es6": "^1.0.0", 28 | "eslint-plugin-mocha": "^4.0.0", 29 | "expect.js": "^0.3.1", 30 | "husky": "^0.11.4", 31 | "mocha": "^2.5.3", 32 | "nock": "^9.0.14", 33 | "supertest": "^3.0.0", 34 | "supertest-as-promised": "^4.0.2" 35 | }, 36 | "dependencies": { 37 | "R": "0.0.1", 38 | "aws-sdk": "^2.94.0", 39 | "body-parser": "^1.17.2", 40 | "boom": "^5.2.0", 41 | "bunyan": "^1.8.5", 42 | "chalk": "^1.1.3", 43 | "confabulous": "^1.1.0", 44 | "debug": "^2.6.8", 45 | "hogan.js": "^3.0.2", 46 | "http-proxy-middleware": "^0.17.4", 47 | "http-status-codes": "^1.1.6", 48 | "logs-to-sumologic": "^1.1.0", 49 | "make-manifest": "^1.0.1", 50 | "node-ssh": "^4.2.3", 51 | "on-headers": "^1.0.1", 52 | "optimist": "^0.6.1", 53 | "optional": "^0.1.3", 54 | "pify": "^3.0.0", 55 | "prepper": "^1.1.0", 56 | "ramda": "^0.21.0", 57 | "superagent": "^3.6.0", 58 | "systemic": "^1.3.0", 59 | "systemic-domain-runner": "^1.1.0", 60 | "systemic-express": "^1.0.1", 61 | "systemic-mongodb": "^1.0.4", 62 | "systemic-pg": "^1.0.6", 63 | "systemic-rabbitmq": "^1.0.1" 64 | }, 65 | "repository": { 66 | "type": "git", 67 | "url": "git+https://github.com/guidesmiths/svc-example.git" 68 | }, 69 | "bugs": { 70 | "url": "https://github.com/guidesmiths/svc-example/issues" 71 | }, 72 | "homepage": "https://github.com/guidesmiths/svc-example#readme" 73 | } 74 | -------------------------------------------------------------------------------- /recipes-api/system.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const { join } = require('path'); 3 | 4 | module.exports = () => new System({ name: 'recipes-api' }).bootstrap(join(__dirname, 'components')); 5 | -------------------------------------------------------------------------------- /recipes-api/test/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "mocha": true 4 | }, 5 | "plugins": [ 6 | "mocha" 7 | ], 8 | "rules": { 9 | "mocha/no-exclusive-tests": 2 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /recipes-api/test/e2e/api.tests.js: -------------------------------------------------------------------------------- 1 | const R = require('ramda'); 2 | const { join } = require('path'); 3 | const expect = require('expect.js'); 4 | const nock = require('nock'); 5 | const statusCodes = require('http-status-codes'); 6 | const supertest = require('supertest-as-promised'); 7 | const configSystem = require('../../components/config'); 8 | const system = require('../../system'); 9 | const recipe = require('../../fixtures/recipe_sample.json'); 10 | const stores = require('require-all')({ 11 | dirname: join(__dirname, '..', '..', 'components', 'store', 'types'), 12 | filter: (fileName) => fileName === 'index.js' ? undefined : fileName.replace('.js', '') 13 | }); 14 | 15 | const test = (strategy) => { 16 | 17 | describe(`Recipes API based on ${strategy} store`, () => { 18 | let request; 19 | let sys; 20 | let myStore; 21 | let myConfig; 22 | 23 | const normalise = R.omit(['_id', 'id']); 24 | 25 | const mockFn = (system) => 26 | system() 27 | .set('config', { start: (cb) => cb(null, myConfig) }); 28 | 29 | before(done => { 30 | configSystem.start((err, { config }) => { 31 | if (err) return done(err); 32 | myConfig = R.merge(config, { store: { strategy, idGenerator: config.store.idGenerator } }); 33 | sys = system(mockFn).start((err, { app, store }) => { 34 | if (err) return done(err); 35 | request = supertest(Promise)(app); 36 | myStore = store; 37 | done(); 38 | }); 39 | }); 40 | }); 41 | 42 | beforeEach(() => myStore.flush()); 43 | after(done => sys.stop(done)); 44 | 45 | const nockIdGenerator = (id, expectedStatusCode = statusCodes.OK) => { 46 | const { host, path } = myConfig.store.idGenerator; 47 | nock(host) 48 | .get(path) 49 | .reply(expectedStatusCode, { id }); 50 | }; 51 | 52 | const post = (recipe, id = 1) => { 53 | nockIdGenerator(id); 54 | return request 55 | .post('/api/v1/recipes') 56 | .send(recipe) 57 | .expect(statusCodes.NO_CONTENT); 58 | }; 59 | 60 | const get = (id, expectation, query = {}) => { 61 | return request 62 | .get(`/api/v1/recipes/${id}`) 63 | .query(query) 64 | .expect(expectation || statusCodes.OK) 65 | } 66 | 67 | const erase = (id) => 68 | request 69 | .delete(`/api/v1/recipes/${id}`) 70 | .expect(statusCodes.NO_CONTENT) 71 | 72 | it('should POST a recipe', () => post(recipe)); 73 | 74 | it('should GET a recipe', () => { 75 | const EXPECTED_ID = 2; 76 | return post(recipe, EXPECTED_ID) 77 | .then(() => 78 | get(EXPECTED_ID) 79 | .then((response) => { 80 | expect(response.headers['content-type']).to.equal('application/json; charset=utf-8'); 81 | expect(normalise(response.body)).to.eql(normalise(recipe)); 82 | }) 83 | ) 84 | }); 85 | 86 | it('should GET a recipe by source id', () => { 87 | const EXPECTED_ID = 2; 88 | return post(recipe, EXPECTED_ID) 89 | .then(() => 90 | get(recipe.source_id, statusCodes.OK, { key: 'source_id' }) 91 | .then((response) => { 92 | expect(response.headers['content-type']).to.equal('application/json; charset=utf-8'); 93 | expect(normalise(response.body)).to.eql(normalise(recipe)); 94 | }) 95 | ) 96 | }); 97 | 98 | it('should DELETE a recipe', () => { 99 | const EXPECTED_ID = 2; 100 | return post(recipe, EXPECTED_ID) 101 | .then(() => 102 | erase(EXPECTED_ID) 103 | .then(() => get(EXPECTED_ID, statusCodes.NOT_FOUND)) 104 | ); 105 | }); 106 | }); 107 | }; 108 | 109 | const runAll = R.pipe( 110 | R.keys, 111 | R.map(test) 112 | ); 113 | 114 | runAll(stores); 115 | -------------------------------------------------------------------------------- /recipes-api/test/env.js: -------------------------------------------------------------------------------- 1 | process.env.SERVICE_ENV = process.env.SERVICE_ENV || 'test' 2 | process.env.NODE_ENV = 'test' 3 | -------------------------------------------------------------------------------- /recipes-api/test/mocha.opts: -------------------------------------------------------------------------------- 1 | --recursive 2 | --require ./test/env.js 3 | --timeout 5000 4 | -------------------------------------------------------------------------------- /recipes-api/test/test-system.js: -------------------------------------------------------------------------------- 1 | const system = require('../system'); 2 | 3 | module.exports = (mockFn) => mockFn(system()); 4 | 5 | -------------------------------------------------------------------------------- /recipes-api/test/unit/store/all.tests.js: -------------------------------------------------------------------------------- 1 | const R = require('ramda'); 2 | const { join } = require('path'); 3 | const expect = require('expect.js'); 4 | const nock = require('nock'); 5 | const statusCodes = require('http-status-codes'); 6 | const system = require('../../../system'); 7 | const configSystem = require('../../../components/config'); 8 | const recipe = require('../../../fixtures/recipe_sample.json'); 9 | const stores = require('require-all')({ 10 | dirname: join(__dirname, '..', '..', '..', 'components', 'store', 'types'), 11 | filter: (fileName) => fileName === 'index.js' ? undefined : fileName.replace('.js', '') 12 | }); 13 | 14 | const test = (strategy) => { 15 | describe(`Testing store ${strategy}`, () => { 16 | let myStore; 17 | let sys; 18 | let myConfig; 19 | let myBroker; 20 | 21 | const mockFn = (system) => 22 | system() 23 | .set('config', { start: (cb) => cb(null, myConfig) }); 24 | 25 | before(done => { 26 | configSystem.start((err, { config }) => { 27 | if (err) return done(err); 28 | myConfig = R.merge(config, { store: { strategy, idGenerator: config.store.idGenerator } }); 29 | sys = system(mockFn).start((err, { store, broker }) => { 30 | if (err) return done(err); 31 | myBroker = broker; 32 | myStore = store; 33 | done(); 34 | }); 35 | }); 36 | }); 37 | 38 | beforeEach(() => 39 | myStore.flush() 40 | .then(() => myBroker.purge())); 41 | 42 | afterEach(() => nock.cleanAll()); 43 | 44 | after(done => 45 | myBroker.nuke() 46 | .then(() => sys.stop(done))); 47 | 48 | const nockIdGenerator = (id, expectedStatusCode = statusCodes.OK) => { 49 | const { host, path } = myConfig.store.idGenerator; 50 | nock(host) 51 | .get(path) 52 | .reply(expectedStatusCode, { id }); 53 | }; 54 | 55 | const shouldReceive = (expectedRK) => new Promise((resolve, reject) => { 56 | myBroker.subscribe('recipes_snoop', (err, subscription) => { 57 | if (err) return reject(err); 58 | subscription 59 | .on('message', (message, content, ackOrNack) => { 60 | ackOrNack(); 61 | if (message.fields.routingKey !== expectedRK) return shouldReceive(expectedRK); 62 | return subscription.cancel(() => resolve({ message, content })); 63 | }) 64 | .on('error', reject); 65 | }); 66 | }); 67 | 68 | const shouldNotReceive = (expectedRK) => new Promise((resolve, reject) => { 69 | myBroker.subscribe('recipes_snoop', (err, subscription) => { 70 | if (err) return reject(err); 71 | subscription 72 | .on('message', (message, content, ackOrNack) => { 73 | ackOrNack(); 74 | if (message.fields.routingKey !== expectedRK) return setTimeout(() => subscription.cancel(resolve), 500); 75 | return shouldNotReceive(expectedRK); 76 | }) 77 | .on('error', (err) => { throw err }); 78 | }); 79 | }); 80 | 81 | const normalise = R.omit(['_id', 'id']); 82 | 83 | it('should get a recipe by id', () => { 84 | const expectedId = 1; 85 | nockIdGenerator(expectedId); 86 | return myStore.saveRecipe(recipe) 87 | .then(() => myStore.getRecipe(expectedId)) 88 | .then((saved) => expect(normalise(saved)).to.eql(normalise(recipe))) 89 | }); 90 | 91 | it('should get a recipe by source id', () => { 92 | const expectedId = 1; 93 | nockIdGenerator(expectedId); 94 | return myStore.saveRecipe(recipe) 95 | .then(() => myStore.getRecipeBySourceId(recipe.source_id)) 96 | .then((saved) => expect(normalise(saved)).to.eql(normalise(recipe))) 97 | }); 98 | 99 | it('should save a recipe with no id, requesting one for it', () => { 100 | const expectedId = 1; 101 | nockIdGenerator(expectedId); 102 | return myStore.saveRecipe(recipe) 103 | .then(() => myStore.getRecipe(expectedId)) 104 | .then((saved) => expect(normalise(saved)).to.eql(normalise(recipe))) 105 | .then(() => shouldReceive('recipes_api.v1.notifications.recipe.saved')) 106 | .then(({ message, content }) => expect(normalise(content)).to.eql(normalise(recipe))) 107 | }); 108 | 109 | it('should update a recipe when the recipe exists and the new version is greater than the saved one', () => { 110 | const myRecipe = R.merge(recipe, { id: 1 }); 111 | const greaterVersion = new Date().getTime(); 112 | const update = R.merge(myRecipe, { version: greaterVersion }); 113 | return myStore.saveRecipe(myRecipe) 114 | .then(() => shouldReceive('recipes_api.v1.notifications.recipe.saved')) 115 | .then(() => myStore.saveRecipe(update)) 116 | .then(() => shouldReceive('recipes_api.v1.notifications.recipe.updated')) 117 | .then(({ message, content }) => expect(normalise(content)).to.eql(normalise(update))) 118 | .then(() => myStore.getRecipe(myRecipe.id)) 119 | .then((saved) => expect(saved.version).to.eql(greaterVersion)) 120 | }); 121 | 122 | it('should not update a recipe when the recipe exists and the new version is lower than the saved one', () => { 123 | const lowerVersion = 1; 124 | const myRecipe = R.merge(recipe, { id: 1 }); 125 | const update = R.merge(myRecipe, { version: lowerVersion }); 126 | return myStore.saveRecipe(myRecipe) 127 | .then(() => myStore.saveRecipe(update)) 128 | .then(() => myStore.getRecipe(myRecipe.id)) 129 | .then((saved) => expect(saved.version).to.eql(myRecipe.version)) 130 | .then(() => shouldNotReceive('recipes_api.v1.notifications.recipe.updated')); 131 | }); 132 | 133 | it('should throw an error when deleting a recipe with no id', () => 134 | myStore.deleteRecipe(null) 135 | .catch((err) => expect(err.message).to.equal('Could not delete recipe with no id')) 136 | ); 137 | 138 | it('should delete a recipe', () => { 139 | const expectedId = 1; 140 | nockIdGenerator(expectedId); 141 | return myStore.saveRecipe(recipe) 142 | .then(() => myStore.deleteRecipe(expectedId)) 143 | .then(() => myStore.getRecipe(expectedId)) 144 | .then((saved) => expect(saved).to.eql(null)) 145 | .then(() => shouldReceive('recipes_api.v1.notifications.recipe.deleted')) 146 | .then(({ message, content }) => expect(content.id).to.eql(expectedId)) 147 | }); 148 | }); 149 | }; 150 | 151 | const runAll = R.pipe( 152 | R.keys, 153 | R.map(test) 154 | ); 155 | 156 | runAll(stores); 157 | -------------------------------------------------------------------------------- /recipes-crawler/.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | -------------------------------------------------------------------------------- /recipes-crawler/.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | 15 | [Makefile] 16 | indent_style = tab -------------------------------------------------------------------------------- /recipes-crawler/.eslintignore: -------------------------------------------------------------------------------- 1 | client/* 2 | -------------------------------------------------------------------------------- /recipes-crawler/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "imperative-es6" 3 | } -------------------------------------------------------------------------------- /recipes-crawler/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | manifest.json 3 | *.log 4 | -------------------------------------------------------------------------------- /recipes-crawler/.npmrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feliun/microservices-school/0f4b67d3f64d53ec3e9957695803e9856db28498/recipes-crawler/.npmrc -------------------------------------------------------------------------------- /recipes-crawler/.nvmrc: -------------------------------------------------------------------------------- 1 | 7 2 | -------------------------------------------------------------------------------- /recipes-crawler/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/feliun/docker-nvm-yarn 2 | COPY manifest.json /root/app/manifest.json 3 | -------------------------------------------------------------------------------- /recipes-crawler/Makefile: -------------------------------------------------------------------------------- 1 | SERVICE_PORT=3001 2 | 3 | start: 4 | @docker run -d -p $(SERVICE_PORT):$(SERVICE_PORT) -e SERVICE_ENV=live -e RABBIT_PWD=$(RABBIT_PWD) -e F2F_KEY=$(F2F_KEY) -e SUMO_URL=$(SUMO_URL) --name $(SERVICE) --network=local $(SERVICE):$(TRAVIS_BUILD_NUMBER) 5 | 6 | include ../Makefile 7 | -------------------------------------------------------------------------------- /recipes-crawler/README.md: -------------------------------------------------------------------------------- 1 | # svc-example 2 | An example service using 3 | 4 | * [systemic](https://github.com/guidesmiths/systemic) 5 | * [confabulous](https://github.com/guidesmiths/confabulous) 6 | * [prepper](https://github.com/guidesmiths/prepper) 7 | * [systemic-express](https://github.com/guidesmiths/systemic-express) 8 | 9 | ## Features 10 | * Environmental configuration 11 | * Secrets obtained from a runtime location 12 | * Automatically applies schema changes on startup 13 | * Orderly startup / shutdown (e.g. establishes database connections before setting up http listeners and vice versa) 14 | * Graceful shutdown on errors, unhandled rejections, unhandled exceptions, SIGINT and SIGTERM 15 | * Useful log decorators, including request scoped logging 16 | * JSON logging to stdout in "proper" environments, human friendly logging locally 17 | * The Dockerfile uses settings from .npmrc and .nvmrc 18 | * The docker build cache busts using package.json and npm-shrinkwrap.json so npm install only runs when necessary 19 | * Deployed artifact (a docker image) is traceable back to SCM commit via manifest.json, exposed via /__/manifest endpoint 20 | 21 | ## Running locally 22 | ``` 23 | npm run docker 24 | npm start 25 | ``` 26 | 27 | ## Running tests 28 | ``` 29 | npm test 30 | ``` 31 | 32 | -------------------------------------------------------------------------------- /recipes-crawler/components/config/confabulous.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = ({ confabulous } = {}) => { 4 | 5 | const Confabulous = confabulous || require('confabulous'); 6 | const loaders = Confabulous.loaders; 7 | let config; 8 | 9 | const start = (cb) => { 10 | if (config) return cb(null, config); 11 | 12 | new Confabulous() 13 | .add(config => loaders.require({ path: path.join(process.cwd(), 'config', 'default.js'), watch: true })) 14 | .add(config => loaders.require({ path: path.join(process.cwd(), 'config', `${process.env.SERVICE_ENV}.js`), mandatory: false })) 15 | .add(config => loaders.require({ path: path.join(process.cwd(), 'secrets', 'secrets.json'), watch: true, mandatory: false })) 16 | .add(config => loaders.args()) 17 | .on('loaded', cb) 18 | .on('error', cb) 19 | .end(cb); 20 | }; 21 | 22 | return { start }; 23 | }; 24 | -------------------------------------------------------------------------------- /recipes-crawler/components/config/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const confabulous = require('./confabulous'); 3 | 4 | module.exports = new System({ name: 'config' }).add('config', confabulous(), { scoped: true }); 5 | -------------------------------------------------------------------------------- /recipes-crawler/components/express/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const defaultMiddleware = require('systemic-express').defaultMiddleware; 3 | const app = require('systemic-express').app; 4 | const server = require('systemic-express').server; 5 | 6 | module.exports = new System({ name: 'express' }) 7 | .add('app', app()).dependsOn('config', 'logger') 8 | .add('middleware.default', defaultMiddleware()).dependsOn('logger', 'app', 'routes') 9 | .add('server', server()).dependsOn('config', 'app', 'middleware.default'); 10 | -------------------------------------------------------------------------------- /recipes-crawler/components/lib/crawler.js: -------------------------------------------------------------------------------- 1 | const R = require('ramda'); 2 | const debug = require('debug')('recipes-crawler'); 3 | const request = require('superagent'); 4 | 5 | module.exports = () => { 6 | 7 | const start = ({ config, logger, broker }, cb) => { 8 | 9 | const API_KEY = config.key || process.env.F2F_KEY; 10 | 11 | // TODO extract these functions to a controller to encapsulate APIs accesses 12 | 13 | const getPage = (baseUrl, page) => { 14 | const url = `${baseUrl}?key=${API_KEY}&page=${page}`; 15 | logger.info(`Getting recipes batch from url ${url}`); 16 | return request.get(url); 17 | }; 18 | 19 | const getRecipe = R.curry((baseUrl, recipeId) => { 20 | const url = `${baseUrl}?key=${API_KEY}&rId=${recipeId}`; 21 | debug(`Rquesting recipe with url: ${url}`); 22 | return request.get(url); 23 | }); 24 | 25 | const findByRecipeId = (recipeId) => { 26 | const { host, path, query } = config.recipesApi; 27 | const url = `${host}${path}?${query.key}=${query.value}`.replace(':id', recipeId); 28 | debug(`Trying to find recipe with url: ${url}`); 29 | return request.get(url) 30 | .then(({ body }) => body) 31 | .catch((err) => { 32 | if (err.status !== 404) logger.error(`Error when finding recipe on url ${url}: ${err.message}`); 33 | throw err; 34 | }); 35 | }; 36 | 37 | const generateId = () => { 38 | const { host, path } = config.idGenerator; 39 | const url = `${host}${path}`; 40 | debug(`Generating a new id from url: ${url}`); 41 | return request.get(url) 42 | .then(({ body }) => body) 43 | .catch((err) => { 44 | logger.error(`Error when generating a new id from url ${url}: ${err.message}`); 45 | throw err; 46 | }); 47 | }; 48 | 49 | const extractIds = R.pipe( 50 | R.prop('recipes'), 51 | R.pluck('recipe_id') 52 | ); 53 | 54 | const extractRecipes = R.pipe( 55 | R.pluck('text'), 56 | R.map(JSON.parse), 57 | R.pluck('recipe') 58 | ); 59 | 60 | const publish = (recipe) => broker.publish('conclusions', recipe, 'recipes_crawler.v1.notifications.recipe.crawled'); 61 | 62 | const translate = ({ publisher, ingredients, source_url, recipe_id, image_url, social_rank, title, id }) => ({ 63 | publisher, 64 | ingredients, 65 | source_url, 66 | image_url, 67 | social_rank, 68 | title, 69 | id, 70 | version: new Date().getTime(), 71 | source_id: recipe_id, 72 | source: 'F2F' 73 | }); 74 | 75 | const adapt = (recipe) => 76 | findByRecipeId(recipe.recipe_id) 77 | .then(translate) 78 | .catch((err) => { 79 | if (err.status !== 404) throw err; 80 | // non found - we generate a new id 81 | return generateId() 82 | .then(({ id }) => translate(R.merge(recipe, { id }))); 83 | }); 84 | 85 | const crawl = () => { 86 | logger.info('Crawling in search of new recipes...'); 87 | const random = Math.floor(Math.random() * 100); 88 | const url = `${config.baseUrl}${config.searchSuffix}`; 89 | return getPage(url, config.page || random) 90 | .catch((err) => { 91 | logger.error(`Error accessing url ${url}: ${err.message} ${err.stack}`); 92 | throw err; 93 | }) 94 | .then(({ text }) => { 95 | const ids = extractIds(JSON.parse(text)); 96 | logger.info(`Getting details for ${ids.length} recipes`); 97 | const recipeRequests = R.map(getRecipe(`${config.baseUrl}${config.recipeSuffix}`), ids); 98 | return Promise.all(recipeRequests); 99 | }) 100 | .then((recipeResponse) => extractRecipes(recipeResponse)) 101 | .then((recipes) => Promise.all(R.map(adapt, recipes))) 102 | .then((recipeList) => Promise.all(R.map(publish, recipeList)) 103 | .then(() => logger.info('New recipes ingested correctly')) 104 | .catch((err) => logger.error(`Error when pulling new recipes: ${err.message} ${err.stack}`))) 105 | }; 106 | setInterval(crawl, config.frequency); 107 | if (config.autostart) crawl(); 108 | cb(); 109 | }; 110 | 111 | return { start }; 112 | 113 | }; 114 | -------------------------------------------------------------------------------- /recipes-crawler/components/lib/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const optional = require('optional'); 3 | const { join } = require('path'); 4 | const manifest = optional(join(process.cwd(), 'manifest.json')) || {}; 5 | const pkg = require(join(process.cwd(), 'package.json')); 6 | const crawler = require('./crawler'); 7 | 8 | module.exports = new System({ name: 'app' }) 9 | .add('manifest', manifest) 10 | .add('pkg', pkg) 11 | .add('crawler', crawler()) 12 | .dependsOn('config', 'logger', 'broker'); 13 | -------------------------------------------------------------------------------- /recipes-crawler/components/logging/bunyan.js: -------------------------------------------------------------------------------- 1 | const bunyan = require('bunyan'); 2 | const R = require('ramda'); 3 | 4 | module.exports = () => { 5 | 6 | let log; 7 | 8 | const onMessage = (event) => { 9 | log[event.level](R.omit(['level', 'message'], event), event.message); 10 | }; 11 | 12 | const start = ({ pkg }, cb) => { 13 | log = bunyan.createLogger({ name: pkg.name }); 14 | return cb(null, onMessage); 15 | }; 16 | 17 | return { start }; 18 | }; 19 | -------------------------------------------------------------------------------- /recipes-crawler/components/logging/console.js: -------------------------------------------------------------------------------- 1 | const chalk = require('chalk'); 2 | const hogan = require('hogan.js'); 3 | const R = require('ramda'); 4 | 5 | const response = hogan.compile('{{{displayTracer}}} {{{displayLevel}}} {{package.name}} {{{request.method}}} {{{response.statusCode}}} {{{request.url}}}'); 6 | const error = hogan.compile('{{{displayTracer}}} {{{displayLevel}}} {{package.name}} {{{message}}} {{{code}}}\n{{{error.stack}}} {{{details}}}'); 7 | const info = hogan.compile('{{{displayTracer}}} {{{displayLevel}}} {{package.name}} {{{message}}} {{{details}}}'); 8 | 9 | const colours = { 10 | debug: chalk.gray, 11 | info: chalk.white, 12 | warn: chalk.yellow, 13 | error: chalk.red, 14 | default: chalk.white 15 | }; 16 | 17 | module.exports = () => { 18 | 19 | const onMessage = (event) => { 20 | const details = R.pluck(event, []); 21 | const data = R.merge(event, { 22 | displayTracer: R.has('tracer', event) ? event.tracer.substr(0, 6) : '------', 23 | displayLevel: event.level.toUpperCase(), 24 | details: Object.keys(details).length ? `\n ${JSON.stringify(details, null, 2)}` : '' 25 | }); 26 | const colour = colours[event.level] || colours.default; 27 | const log = console[event.level] || console.info; // eslint-disable-line no-console 28 | if (R.has('response.statusCode', event)) log(colour(response.render(data))); 29 | else if (R.has('error.message', event)) log(colour(error.render(data))); 30 | else log(colour(info.render(data))); 31 | }; 32 | 33 | const start = (cb) => cb(null, onMessage); 34 | 35 | return { start }; 36 | }; 37 | -------------------------------------------------------------------------------- /recipes-crawler/components/logging/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const prepper = require('./prepper'); 3 | const console = require('./console'); 4 | const bunyan = require('./bunyan'); 5 | const sumo = require('./sumo'); 6 | const prepperMiddleware = require('./prepper-middleware'); 7 | 8 | module.exports = new System({ name: 'logging' }) 9 | .add('transports.console', console()) 10 | .add('transports.bunyan', bunyan()).dependsOn('pkg') 11 | .add('transports.sumo', sumo()) 12 | .add('transports').dependsOn( 13 | { component: 'transports.console', destination: 'console' }, 14 | { component: 'transports.sumo', destination: 'sumo' }, 15 | { component: 'transports.bunyan', destination: 'bunyan' }) 16 | .add('logger', prepper()).dependsOn('config', 'pkg', 'transports') 17 | .add('middleware.prepper', prepperMiddleware()).dependsOn('app') 18 | -------------------------------------------------------------------------------- /recipes-crawler/components/logging/prepper-middleware.js: -------------------------------------------------------------------------------- 1 | const onHeaders = require('on-headers'); 2 | const R = require('ramda'); 3 | 4 | module.exports = ({ prepper } = {}) => { 5 | const handlers = (prepper || require('prepper')).handlers; 6 | 7 | const start = ({ app }, cb) => { 8 | app.use((req, res, next) => { 9 | const logger = req.app.locals.logger.child({ handlers: [ 10 | new handlers.Tracer(), 11 | new handlers.Merge(R.pick(['url', 'method', 'headers', 'params'], req), { key: 'request' }) 12 | ]}); 13 | 14 | onHeaders(res, () => { 15 | const response = { response: { statusCode: res.statusCode, headers: res.headers } }; 16 | if (res.statusCode === 400) logger.error(req.url, response); 17 | if (res.statusCode < 500) logger.info(req.url, response); 18 | else logger.error(req.url, response); 19 | }); 20 | 21 | res.locals.logger = logger; 22 | 23 | next(); 24 | }); 25 | 26 | cb(); 27 | }; 28 | 29 | return { start }; 30 | }; 31 | -------------------------------------------------------------------------------- /recipes-crawler/components/logging/prepper.js: -------------------------------------------------------------------------------- 1 | const merge = require('lodash.merge'); 2 | const R = require('ramda'); 3 | 4 | module.exports = ({ prepper, transport } = {}) => { 5 | 6 | const prepperFn = prepper || require('prepper'); 7 | const handlers = prepperFn.handlers; 8 | 9 | const start = ({ config, transports, pkg = { name: 'unknown' } }, cb) => { 10 | const transportFn = transport || R.path([config.transport], transports); 11 | config = merge({ include: [], exclude: [] }, config); 12 | 13 | const logger = new prepperFn.Logger({ handlers: [ 14 | new handlers.Merge({ app: pkg.name }), 15 | new handlers.Merge({ env: process.env.SERVICE_ENV }), 16 | new handlers.Process(), 17 | new handlers.System(), 18 | new handlers.Timestamp(), 19 | new handlers.Flatten(), 20 | new handlers.KeyFilter({ include: config.include, exclude: config.exclude }), 21 | new handlers.Unflatten() 22 | ]}).on('message', event => { 23 | if (transportFn) transportFn(event); 24 | }); 25 | 26 | cb(null, logger); 27 | }; 28 | 29 | return { start }; 30 | }; 31 | -------------------------------------------------------------------------------- /recipes-crawler/components/logging/sumo.js: -------------------------------------------------------------------------------- 1 | const R = require('ramda'); 2 | const Sumologic = require('logs-to-sumologic'); 3 | 4 | let logger; 5 | 6 | module.exports = () => { 7 | 8 | const onMessage = (event) => { 9 | logger.log(event, (err) => { 10 | if (err) console.log(`Error posting to sumo! ${err.message}`); 11 | }); 12 | }; 13 | 14 | const start = (cb) => { 15 | const url = process.env.SUMO_URL; 16 | if (!url) return cb(null, R.identity); 17 | logger = Sumologic.createClient({ url, name: "RecipesCollector" }); 18 | return cb(null, onMessage); 19 | }; 20 | 21 | return { start }; 22 | }; 23 | -------------------------------------------------------------------------------- /recipes-crawler/components/rabbitmq/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const rabbitmq = require('systemic-rabbitmq'); 3 | const initBroker = require('./initBroker'); 4 | 5 | module.exports = new System({ name: 'rabbit' }) 6 | .add('rabbitmq', rabbitmq()).dependsOn('config', 'logger') 7 | .add('broker', initBroker()).dependsOn('rabbitmq'); 8 | -------------------------------------------------------------------------------- /recipes-crawler/components/rabbitmq/initBroker.js: -------------------------------------------------------------------------------- 1 | const pify = require('pify'); 2 | 3 | module.exports = () => { 4 | 5 | const start = ({ rabbitmq }, cb) => { 6 | 7 | const publish = (...args) => new Promise((resolve, reject) => { 8 | rabbitmq.broker.publish(...args, (err, publication) => { 9 | if (err) return reject(err); 10 | publication 11 | .on('success', () => resolve()) 12 | .on('error', reject); 13 | }); 14 | }); 15 | 16 | const subscribe = (...args) => new Promise((resolve, reject) => { 17 | rabbitmq.broker.subscribe(...args, (err, subscription) => { 18 | if (err) return reject(err); 19 | const cancel = new Promise(subscription.cancel); 20 | subscription 21 | .on('message', (message, content, ackOrNack) => resolve({ message, content, ackOrNack, cancel })) 22 | .on('error', reject); 23 | }); 24 | }); 25 | 26 | const broker = { 27 | publish, 28 | subscribe, 29 | nuke: pify(rabbitmq.broker.nuke), 30 | purge: pify(rabbitmq.broker.purge), 31 | }; 32 | return cb(null, broker); 33 | }; 34 | 35 | return { start }; 36 | }; 37 | -------------------------------------------------------------------------------- /recipes-crawler/components/routes/admin-routes.js: -------------------------------------------------------------------------------- 1 | module.exports = () => { 2 | 3 | const start = ({ manifest = {}, app }, cb) => { 4 | app.get('/__/manifest', (req, res) => res.json(manifest)); 5 | app.post('/__/error', (req, res) => { 6 | setTimeout(() => process.emit('error', new Error('On Noes'))); 7 | }); 8 | app.post('/__/crash', (req, res) => { 9 | setTimeout(() => undefined.meh); 10 | }); 11 | app.post('/__/reject', (req, res) => { 12 | setTimeout(() => Promise.reject(new Error('Oh Noes'))); 13 | }); 14 | cb(); 15 | }; 16 | 17 | return { start }; 18 | }; 19 | -------------------------------------------------------------------------------- /recipes-crawler/components/routes/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const adminRoutes = require('./admin-routes'); 3 | 4 | module.exports = new System({ name: 'routes' }) 5 | .add('routes.admin', adminRoutes()).dependsOn('config', 'logger', 'app', 'middleware.prepper', 'manifest') 6 | .add('routes').dependsOn('routes.admin'); 7 | -------------------------------------------------------------------------------- /recipes-crawler/config/build.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logger: { 3 | transport: null 4 | }, 5 | crawler: { 6 | baseUrl: 'http://localhost:6000', 7 | searchSuffix: '/api/search', 8 | recipeSuffix: '/api/get', 9 | autostart: true, 10 | key: 'some_key', 11 | page: 10 12 | }, 13 | rabbitmq: { 14 | defaults: { 15 | vhosts: { 16 | namespace: true, 17 | queues: { 18 | options: { 19 | durable: false 20 | } 21 | }, 22 | exchanges: { 23 | options: { 24 | durable: false 25 | } 26 | } 27 | } 28 | }, 29 | vhosts: { 30 | ysojkvfe: { 31 | connection: { 32 | hostname: 'rabbitmq' 33 | }, 34 | queues: { 35 | 'dead_letters:snoop': {}, 36 | 'retry:snoop': {}, 37 | 'delay:1ms': { 38 | options: { 39 | arguments: { 40 | 'x-message-ttl': 1, 41 | 'x-dead-letter-exchange': 'retry' 42 | } 43 | } 44 | }, 45 | 'recipes_crawler:snoop': {} 46 | }, 47 | bindings: { 48 | 'delay[delay.#] -> delay:1ms': {}, 49 | 'retry -> retry:snoop': {}, 50 | 'dead_letters -> dead_letters:snoop': {}, 51 | 'internal[recipes_crawler.v1.notifications.#.#] -> recipes_crawler:snoop': {} 52 | }, 53 | subscriptions: { 54 | dead_letters: { 55 | queue: 'dead_letters:snoop' 56 | }, 57 | retries: { 58 | queue: 'retry:snoop' 59 | }, 60 | recipes_crawler: { 61 | queue: 'recipes_crawler:snoop', 62 | contentType: 'application/json' 63 | } 64 | }, 65 | } 66 | } 67 | } 68 | }; 69 | -------------------------------------------------------------------------------- /recipes-crawler/config/default.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | server: { 3 | host: '0.0.0.0', 4 | port: 3001 5 | }, 6 | logger: { 7 | transport: 'bunyan', 8 | include: [ 9 | 'tracer', 10 | 'timestamp', 11 | 'level', 12 | 'message', 13 | 'error.message', 14 | 'error.code', 15 | 'error.stack', 16 | 'request.url', 17 | 'request.headers', 18 | 'request.params', 19 | 'request.method', 20 | 'response.statusCode', 21 | 'response.headers', 22 | 'response.time', 23 | 'process', 24 | 'system', 25 | 'env', 26 | 'app' 27 | ], 28 | exclude: [ 29 | 'password', 30 | 'secret', 31 | 'token', 32 | 'request.headers.cookie', 33 | 'dependencies', 34 | 'devDependencies' 35 | ] 36 | }, 37 | crawler: { // once every 2 hours 38 | frequency: 7200000, 39 | baseUrl: 'http://food2fork.com/api', 40 | searchSuffix: '/search', 41 | recipeSuffix: '/get', 42 | recipesApi: { 43 | host: 'http://127.0.0.1:3000', 44 | path: '/api/v1/recipes/:id', 45 | query: { 46 | key: 'key', 47 | value: 'source_id' 48 | } 49 | }, 50 | idGenerator: { 51 | host: 'http://127.0.0.1:3002', 52 | path: '/api/v1/id' 53 | } 54 | }, 55 | rabbitmq: { 56 | defaults: {}, 57 | vhosts: { 58 | ysojkvfe: { 59 | connection: { 60 | hostname: '127.0.0.1', 61 | user: 'rabbitmq', 62 | password: 'rabbitmq' 63 | }, 64 | exchanges: [ 65 | 'internal', 66 | 'delay', 67 | 'retry', 68 | 'dead_letters' 69 | ], 70 | queues: {}, 71 | bindings: {}, 72 | subscriptions: {}, 73 | publications: { 74 | conclusions: { 75 | exchange: 'internal' 76 | } 77 | } 78 | } 79 | } 80 | } 81 | }; 82 | -------------------------------------------------------------------------------- /recipes-crawler/config/live.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logger: { 3 | transport: 'sumo' 4 | }, 5 | crawler: { 6 | recipesApi: { 7 | host: 'http://recipes-api:3000' 8 | }, 9 | idGenerator: { 10 | host: 'http://recipes-id-generator:3002' 11 | } 12 | }, 13 | rabbitmq: { 14 | defaults: {}, 15 | vhosts: { 16 | ysojkvfe: { 17 | connection: { 18 | hostname: 'swan.rmq.cloudamqp.com', 19 | user: 'ysojkvfe', 20 | password: process.env.RABBIT_PWD || 'N/A' 21 | } 22 | } 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /recipes-crawler/config/local.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logger: { 3 | transport: 'console' 4 | }, 5 | crawler: { 6 | frequency: 10000, 7 | autostart: true 8 | } 9 | }; 10 | -------------------------------------------------------------------------------- /recipes-crawler/config/test.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logger: { 3 | transport: null 4 | }, 5 | crawler: { 6 | baseUrl: 'http://localhost:6000', 7 | searchSuffix: '/api/search', 8 | recipeSuffix: '/api/get', 9 | autostart: true, 10 | key: 'some_key', 11 | page: 10 12 | }, 13 | rabbitmq: { 14 | defaults: { 15 | vhosts: { 16 | namespace: true, 17 | queues: { 18 | options: { 19 | durable: false 20 | } 21 | }, 22 | exchanges: { 23 | options: { 24 | durable: false 25 | } 26 | } 27 | } 28 | }, 29 | vhosts: { 30 | ysojkvfe: { 31 | queues: { 32 | 'dead_letters:snoop': {}, 33 | 'retry:snoop': {}, 34 | 'delay:1ms': { 35 | options: { 36 | arguments: { 37 | 'x-message-ttl': 1, 38 | 'x-dead-letter-exchange': 'retry' 39 | } 40 | } 41 | }, 42 | 'recipes_crawler:snoop': {} 43 | }, 44 | bindings: { 45 | 'delay[delay.#] -> delay:1ms': {}, 46 | 'retry -> retry:snoop': {}, 47 | 'dead_letters -> dead_letters:snoop': {}, 48 | 'internal[recipes_crawler.v1.notifications.#.#] -> recipes_crawler:snoop': {} 49 | }, 50 | subscriptions: { 51 | dead_letters: { 52 | queue: 'dead_letters:snoop' 53 | }, 54 | retries: { 55 | queue: 'retry:snoop' 56 | }, 57 | recipes_crawler: { 58 | queue: 'recipes_crawler:snoop', 59 | contentType: 'application/json' 60 | } 61 | }, 62 | } 63 | } 64 | } 65 | }; 66 | -------------------------------------------------------------------------------- /recipes-crawler/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | 5 | mongo: 6 | image: mongo 7 | container_name: mongo 8 | ports: 9 | - '27017:27017' 10 | command: "--logpath=/dev/null" 11 | 12 | rabbitmq: 13 | image: "rabbitmq:3-management" 14 | container_name: rabbitmq 15 | hostname: "rabbitmq" 16 | environment: 17 | RABBITMQ_ERLANG_COOKIE: "SWQOKODSQALRPCLNMEQG" 18 | RABBITMQ_DEFAULT_USER: "rabbitmq" 19 | RABBITMQ_DEFAULT_PASS: "rabbitmq" 20 | RABBITMQ_DEFAULT_VHOST: "ysojkvfe" 21 | ports: 22 | - "15672:15672" 23 | - "5672:5672" 24 | 25 | networks: 26 | default: 27 | external: 28 | name: local 29 | 30 | -------------------------------------------------------------------------------- /recipes-crawler/docker/supervisor.conf: -------------------------------------------------------------------------------- 1 | [program:recipes-crawler] 2 | directory=/root/app 3 | command=/usr/bin/npm run start 4 | autorestart=true 5 | stdout_logfile=/var/log/supervisor/recipes-crawler.out.log 6 | stderr_logfile=/var/log/supervisor/recipes-crawler.err.log 7 | stdout_logfile_backups=5 8 | stderr_logfile_backups=5 9 | -------------------------------------------------------------------------------- /recipes-crawler/fixtures/recipe.json: -------------------------------------------------------------------------------- 1 | { 2 | "recipe": { 3 | "publisher": "Closet Cooking", 4 | "f2f_url": "http://food2fork.com/view/35075", 5 | "ingredients": [ 6 | "3 tablespoons butter", 7 | "3 pounds onions, sliced", 8 | "1/2 cup water", 9 | "1 apple, peeled and julienned", 10 | "1 teaspoon thyme, chopped", 11 | "3 tablespoons flour", 12 | "1 cup white wine", 13 | "2 cups beef or vegetable broth", 14 | "1 cup apple cider", 15 | "2 bay leaves", 16 | "1 splash brandy (optional)", 17 | "salt & pepper to taste", 18 | "4 1/2 inch thick slices of day old bread, toasted", 19 | "1 1/2 cup gruyere or fontina, grated\\n" 20 | ], 21 | "source_url": "http://www.closetcooking.com/2011/12/apple-french-onion-soup.html", 22 | "recipe_id": "35075", 23 | "image_url": "http://static.food2fork.com/Apple2BFrench2BOnion2BSoup2B5002B13714856193e.jpg", 24 | "social_rank": 99.99973904876121, 25 | "publisher_url": "http://closetcooking.com", 26 | "title": "Apple French Onion Soup" 27 | } 28 | } -------------------------------------------------------------------------------- /recipes-crawler/fixtures/recipesSearch.json: -------------------------------------------------------------------------------- 1 | { 2 | "count": 30, 3 | "recipes": [ 4 | { 5 | "publisher": "All Recipes", 6 | "f2f_url": "http://food2fork.com/view/16373", 7 | "title": "Herb-Rubbed Sirloin Tip Roast", 8 | "source_url": "http://allrecipes.com/Recipe/Herb-Rubbed-Sirloin-Tip-Roast/Detail.aspx", 9 | "recipe_id": "16373", 10 | "image_url": "http://static.food2fork.com/10114e104.jpg", 11 | "social_rank": 99.99992779285354, 12 | "publisher_url": "http://allrecipes.com" 13 | }, 14 | { 15 | "publisher": "BBC Good Food", 16 | "f2f_url": "http://food2fork.com/view/b07a47", 17 | "title": "Chicken tikka masala", 18 | "source_url": "http://www.bbcgoodfood.com/recipes/12798/chicken-tikka-masala", 19 | "recipe_id": "b07a47", 20 | "image_url": "http://static.food2fork.com/12798_MEDIUM5c64.jpg", 21 | "social_rank": 99.9999267759075, 22 | "publisher_url": "http://www.bbcgoodfood.com" 23 | }, 24 | { 25 | "publisher": "All Recipes", 26 | "f2f_url": "http://food2fork.com/view/5123", 27 | "title": "Burrito Pie", 28 | "source_url": "http://allrecipes.com/Recipe/Burrito-Pie/Detail.aspx", 29 | "recipe_id": "5123", 30 | "image_url": "http://static.food2fork.com/76304025.jpg", 31 | "social_rank": 99.99992668658831, 32 | "publisher_url": "http://allrecipes.com" 33 | }, 34 | { 35 | "publisher": "101 Cookbooks", 36 | "f2f_url": "http://food2fork.com/view/47936", 37 | "title": "Cashew Curry", 38 | "source_url": "http://www.101cookbooks.com/archives/cashew-curry-recipe.html", 39 | "recipe_id": "47936", 40 | "image_url": "http://static.food2fork.com/cashew_curry_recipeeb34.jpg", 41 | "social_rank": 99.9999262111281, 42 | "publisher_url": "http://www.101cookbooks.com" 43 | }, 44 | { 45 | "publisher": "The Pioneer Woman", 46 | "f2f_url": "http://food2fork.com/view/e01b04", 47 | "title": "Olive Cheese Bread", 48 | "source_url": "http://thepioneerwoman.com/cooking/2007/06/olive_cheese_br/", 49 | "recipe_id": "e01b04", 50 | "image_url": "http://static.food2fork.com/549578972_8f563494d551ac.jpg", 51 | "social_rank": 99.99992609954744, 52 | "publisher_url": "http://thepioneerwoman.com" 53 | }, 54 | { 55 | "publisher": "Simply Recipes", 56 | "f2f_url": "http://food2fork.com/view/37102", 57 | "title": "Turkey Soup with Lemon and Barley", 58 | "source_url": "http://www.simplyrecipes.com/recipes/turkey_soup_with_lemon_and_barley/", 59 | "recipe_id": "37102", 60 | "image_url": "http://static.food2fork.com/turkeysouplemonbarley520a300x20003493bfb.jpg", 61 | "social_rank": 99.99992608375084, 62 | "publisher_url": "http://simplyrecipes.com" 63 | }, 64 | { 65 | "publisher": "101 Cookbooks", 66 | "f2f_url": "http://food2fork.com/view/47666", 67 | "title": "Strawberry Rhubarb Crumble", 68 | "source_url": "http://www.101cookbooks.com/archives/strawberry-rhubarb-crumble-recipe.html", 69 | "recipe_id": "47666", 70 | "image_url": "http://static.food2fork.com/strawberry_rhubarb_crumble_recipefd88.jpg", 71 | "social_rank": 99.99992603237148, 72 | "publisher_url": "http://www.101cookbooks.com" 73 | }, 74 | { 75 | "publisher": "Two Peas and Their Pod", 76 | "f2f_url": "http://food2fork.com/view/5d3368", 77 | "title": "Brie, Fig, and Apple Grilled Cheese", 78 | "source_url": "http://www.twopeasandtheirpod.com/brie-fig-and-apple-grilled-cheese/", 79 | "recipe_id": "5d3368", 80 | "image_url": "http://static.food2fork.com/BrieFigandAppleGrilledCheese5b79c.jpg", 81 | "social_rank": 99.99992574848615, 82 | "publisher_url": "http://www.twopeasandtheirpod.com" 83 | }, 84 | { 85 | "publisher": "Bon Appetit", 86 | "f2f_url": "http://food2fork.com/view/48976", 87 | "title": "Weeknight Chorizo and Clam Paella", 88 | "source_url": "http://www.bonappetit.com/recipes/quick-recipes/2012/10/weeknight-chorizo-and-clam-paella", 89 | "recipe_id": "48976", 90 | "image_url": "http://static.food2fork.com/weeknightchorizoandclampaella646b191.jpg", 91 | "social_rank": 99.99992571471638, 92 | "publisher_url": "http://www.bonappetit.com" 93 | }, 94 | { 95 | "publisher": "All Recipes", 96 | "f2f_url": "http://food2fork.com/view/2425", 97 | "title": "Baked Chicken Nuggets", 98 | "source_url": "http://allrecipes.com/Recipe/Baked-Chicken-Nuggets/Detail.aspx", 99 | "recipe_id": "2425", 100 | "image_url": "http://static.food2fork.com/9880830335.jpg", 101 | "social_rank": 99.99992455567956, 102 | "publisher_url": "http://allrecipes.com" 103 | }, 104 | { 105 | "publisher": "BBC Good Food", 106 | "f2f_url": "http://food2fork.com/view/876d19", 107 | "title": "Coffee & walnut cake", 108 | "source_url": "http://www.bbcgoodfood.com/recipes/4738/coffee-and-walnut-cake", 109 | "recipe_id": "876d19", 110 | "image_url": "http://static.food2fork.com/4738_MEDIUMb674.jpg", 111 | "social_rank": 99.99992446206328, 112 | "publisher_url": "http://www.bbcgoodfood.com" 113 | }, 114 | { 115 | "publisher": "Cookie and Kate", 116 | "f2f_url": "http://food2fork.com/view/2cef01", 117 | "title": "All Recipes", 118 | "source_url": "http://cookieandkate.com/2014/healthy-maple-pumpkin-muffins/", 119 | "recipe_id": "2cef01", 120 | "image_url": "http://static.food2fork.com/healthymaplepumpkinmuffinsbdc6.jpg", 121 | "social_rank": 99.9999244356148, 122 | "publisher_url": "http://cookieandkate.com" 123 | }, 124 | { 125 | "publisher": "Two Peas and Their Pod", 126 | "f2f_url": "http://food2fork.com/view/2c9347", 127 | "title": "Green Chile Enchilada Quinoa Bake", 128 | "source_url": "http://www.twopeasandtheirpod.com/green-chile-enchilada-quinoa-bake/", 129 | "recipe_id": "2c9347", 130 | "image_url": "http://static.food2fork.com/GreenChileEnchiladaQuinoaBake2424e.jpg", 131 | "social_rank": 99.99992397392855, 132 | "publisher_url": "http://www.twopeasandtheirpod.com" 133 | }, 134 | { 135 | "publisher": "101 Cookbooks", 136 | "f2f_url": "http://food2fork.com/view/42118d", 137 | "title": "Momo Dumplings", 138 | "source_url": "http://www.101cookbooks.com/archives/momo-dumplings-recipe.html", 139 | "recipe_id": "42118d", 140 | "image_url": "http://static.food2fork.com/momo_dumplings519b.jpg", 141 | "social_rank": 99.99992385174366, 142 | "publisher_url": "http://www.101cookbooks.com" 143 | }, 144 | { 145 | "publisher": "Two Peas and Their Pod", 146 | "f2f_url": "http://food2fork.com/view/bcf043", 147 | "title": "Brown Butter Pasta with Sweet Potatoes and Brussels Sprouts", 148 | "source_url": "http://www.twopeasandtheirpod.com/brown-butter-pasta-with-sweet-potatoes-and-brussels-sprouts/", 149 | "recipe_id": "bcf043", 150 | "image_url": "http://static.food2fork.com/BrownButterPastawithSweetPotatoesandBrusselsSprouts157c4.jpg", 151 | "social_rank": 99.99992355693259, 152 | "publisher_url": "http://www.twopeasandtheirpod.com" 153 | }, 154 | { 155 | "publisher": "Tasty Kitchen", 156 | "f2f_url": "http://food2fork.com/view/0b005f", 157 | "title": "Slutty Brownies (So Delicious and Completely Inappropriate!)", 158 | "source_url": "http://tastykitchen.com/recipes/desserts/slutty-brownies-so-delicious-and-completely-inappropriate/", 159 | "recipe_id": "0b005f", 160 | "image_url": "http://static.food2fork.com/SluttyBrownies1410x2733734.jpg", 161 | "social_rank": 99.99992235267638, 162 | "publisher_url": "http://tastykitchen.com" 163 | }, 164 | { 165 | "publisher": "All Recipes", 166 | "f2f_url": "http://food2fork.com/view/23163", 167 | "title": "Molasses Cookies", 168 | "source_url": "http://allrecipes.com/Recipe/Molasses-Cookies/Detail.aspx", 169 | "recipe_id": "23163", 170 | "image_url": "http://static.food2fork.com/18538d21e.jpg", 171 | "social_rank": 99.99992221551013, 172 | "publisher_url": "http://allrecipes.com" 173 | }, 174 | { 175 | "publisher": "Chow", 176 | "f2f_url": "http://food2fork.com/view/811e02", 177 | "title": "Fettuccine with Pesto, Asparagus, and Artichoke Recipe", 178 | "source_url": "http://www.chow.com/recipes/29517-fettuccine-with-pesto-asparagus-and-artichoke", 179 | "recipe_id": "811e02", 180 | "image_url": "http://static.food2fork.com/29517_fettuccine_pesto_asparagus_2_6202352.jpg", 181 | "social_rank": 99.99992092310303, 182 | "publisher_url": "http://www.chow.com" 183 | }, 184 | { 185 | "publisher": "101 Cookbooks", 186 | "f2f_url": "http://food2fork.com/view/47681", 187 | "title": "Carrot Oatmeal Cookies", 188 | "source_url": "http://www.101cookbooks.com/archives/carrot-oatmeal-cookies-recipe.html", 189 | "recipe_id": "47681", 190 | "image_url": "http://static.food2fork.com/carrot_oatmeal_cookie_recipec076.jpg", 191 | "social_rank": 99.99992062288737, 192 | "publisher_url": "http://www.101cookbooks.com" 193 | }, 194 | { 195 | "publisher": "101 Cookbooks", 196 | "f2f_url": "http://food2fork.com/view/47862", 197 | "title": "A Simple Tomato Soup", 198 | "source_url": "http://www.101cookbooks.com/archives/a-simple-tomato-soup-recipe.html", 199 | "recipe_id": "47862", 200 | "image_url": "http://static.food2fork.com/brothy_tomato_soup_reciped6e2.jpg", 201 | "social_rank": 99.99992058071909, 202 | "publisher_url": "http://www.101cookbooks.com" 203 | }, 204 | { 205 | "publisher": "All Recipes", 206 | "f2f_url": "http://food2fork.com/view/20872", 207 | "title": "Pesto Sauce", 208 | "source_url": "http://allrecipes.com/Recipe/Pesto-Sauce/Detail.aspx", 209 | "recipe_id": "20872", 210 | "image_url": "http://static.food2fork.com/409147069d.jpg", 211 | "social_rank": 99.99991997699081, 212 | "publisher_url": "http://allrecipes.com" 213 | }, 214 | { 215 | "publisher": "Two Peas and Their Pod", 216 | "f2f_url": "http://food2fork.com/view/0e4e29", 217 | "title": "Summer Zucchini Noodle Salad", 218 | "source_url": "http://www.twopeasandtheirpod.com/summer-zucchini-noodle-salad/", 219 | "recipe_id": "0e4e29", 220 | "image_url": "http://static.food2fork.com/SummerZucchiniNoodleSalad1e9b9.jpg", 221 | "social_rank": 99.99991978463643, 222 | "publisher_url": "http://www.twopeasandtheirpod.com" 223 | }, 224 | { 225 | "publisher": "Bon Appetit", 226 | "f2f_url": "http://food2fork.com/view/50480", 227 | "title": "Pork Meatball Banh Mi", 228 | "source_url": "http://www.bonappetit.com/recipes/2010/01/pork_meatball_banh_mi", 229 | "recipe_id": "50480", 230 | "image_url": "http://static.food2fork.com/mare_pork_meatball_banh_mi_vb507.jpg", 231 | "social_rank": 99.99991887683383, 232 | "publisher_url": "http://www.bonappetit.com" 233 | }, 234 | { 235 | "publisher": "Smitten Kitchen", 236 | "f2f_url": "http://food2fork.com/view/e6717a", 237 | "title": "mushroom bourguignon", 238 | "source_url": "http://smittenkitchen.com/blog/2009/01/mushroom-bourguignon/", 239 | "recipe_id": "e6717a", 240 | "image_url": "http://static.food2fork.com/3197332671_43b2c5e2ff_m774b.jpg", 241 | "social_rank": 99.9999186263604, 242 | "publisher_url": "http://www.smittenkitchen.com" 243 | }, 244 | { 245 | "publisher": "BBC Good Food", 246 | "f2f_url": "http://food2fork.com/view/46ff71", 247 | "title": "Chocolate marquise", 248 | "source_url": "http://www.bbcgoodfood.com/recipes/4806/chocolate-marquise", 249 | "recipe_id": "46ff71", 250 | "image_url": "http://static.food2fork.com/4806_MEDIUMbac8.jpg", 251 | "social_rank": 99.99991846593565, 252 | "publisher_url": "http://www.bbcgoodfood.com" 253 | }, 254 | { 255 | "publisher": "Simply Recipes", 256 | "f2f_url": "http://food2fork.com/view/36996", 257 | "title": "Spinach Frittata", 258 | "source_url": "http://www.simplyrecipes.com/recipes/spinach_frittata/", 259 | "recipe_id": "36996", 260 | "image_url": "http://static.food2fork.com/spinachfrittatagoatcheese300x2000783992b.jpg", 261 | "social_rank": 99.9999175823382, 262 | "publisher_url": "http://simplyrecipes.com" 263 | }, 264 | { 265 | "publisher": "Two Peas and Their Pod", 266 | "f2f_url": "http://food2fork.com/view/6bbae5", 267 | "title": "Parmesan Crusted Scalloped Potatoes", 268 | "source_url": "http://www.twopeasandtheirpod.com/parmesan-crusted-scalloped-potatoes/", 269 | "recipe_id": "6bbae5", 270 | "image_url": "http://static.food2fork.com/ScallopedPotatoes52356.jpg", 271 | "social_rank": 99.99991731279583, 272 | "publisher_url": "http://www.twopeasandtheirpod.com" 273 | }, 274 | { 275 | "publisher": "Chow", 276 | "f2f_url": "http://food2fork.com/view/7cfa2d", 277 | "title": "Tres Leches de Ron con Chocolate (Chocolate Rum Tres Leches Cake) Recipe", 278 | "source_url": "http://www.chow.com/recipes/29006-tres-leches-de-ron-con-chocolate-chocolate-rum-tres-leches-cake", 279 | "recipe_id": "7cfa2d", 280 | "image_url": "http://static.food2fork.com/29006_tres_leches_cake_62021a1.jpg", 281 | "social_rank": 99.99991627768803, 282 | "publisher_url": "http://www.chow.com" 283 | }, 284 | { 285 | "publisher": "All Recipes", 286 | "f2f_url": "http://food2fork.com/view/19886", 287 | "title": "Marie's Lentil Soup", 288 | "source_url": "http://allrecipes.com/Recipe/Lentil-Soup/Detail.aspx", 289 | "recipe_id": "19886", 290 | "image_url": "http://static.food2fork.com/1997443c50.jpg", 291 | "social_rank": 99.99991610289698, 292 | "publisher_url": "http://allrecipes.com" 293 | }, 294 | { 295 | "publisher": "Closet Cooking", 296 | "f2f_url": "http://food2fork.com/view/35267", 297 | "title": "Crispy Beer Battered Fish Sandwich", 298 | "source_url": "http://www.closetcooking.com/2011/03/crispy-beer-battered-fish-sandwich.html", 299 | "recipe_id": "35267", 300 | "image_url": "http://static.food2fork.com/Crispy2BBeer2BBattered2BFish2BSandwich2Bwith2BColeslaw2Band2BTartar2BSauce2Band2Ba2BSide2Bof2BFries2B2B12B500a7dee810.jpg", 301 | "social_rank": 99.99991582841521, 302 | "publisher_url": "http://closetcooking.com" 303 | } 304 | ] 305 | } -------------------------------------------------------------------------------- /recipes-crawler/index.js: -------------------------------------------------------------------------------- 1 | process.env.SERVICE_ENV = process.env.SERVICE_ENV || 'local'; 2 | 3 | const system = require('./system'); 4 | const runner = require('systemic-domain-runner'); 5 | const bunyan = require('bunyan'); 6 | const name = require('./package.json').name; 7 | const emergencyLogger = process.env.SERVICE_ENV === 'local' ? console : bunyan.createLogger({ name: name }); 8 | 9 | const die = (message, err) => { 10 | emergencyLogger.error(err, message); 11 | process.exit(1); 12 | }; 13 | 14 | runner(system(), { logger: emergencyLogger }).start((err, dependencies) => { 15 | if (err) die('Error starting system', err); 16 | dependencies.logger.info(`${dependencies.pkg.name} has started`); 17 | }); 18 | 19 | -------------------------------------------------------------------------------- /recipes-crawler/infra/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker run -d -p 3001:3001 -e SERVICE_ENV=live -e F2F_KEY=$F2F_KEY -e RABBIT_PWD=$RABBIT_PWD -e SUMO_URL=$SUMO_URL --name recipes-crawler --network=local quay.io/feliun/recipes-crawler:latest 4 | -------------------------------------------------------------------------------- /recipes-crawler/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "recipes-crawler", 3 | "version": "1.0.3", 4 | "description": "A crawler to pull recipes into our system", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "node index.js", 8 | "server": "supervisor .", 9 | "test": "mocha test", 10 | "lint": "eslint .", 11 | "qa": "npm run lint && npm run test", 12 | "docker": "bash -c '(docker network inspect local 2>&1 > /dev/null || docker network create local) && docker-compose --file docker-compose.yml pull && docker-compose --file docker-compose.yml up -d --force-recreate'", 13 | "precommit": "npm run lint", 14 | "prepush": "npm run test" 15 | }, 16 | "keywords": [ 17 | "microservice", 18 | "systemic", 19 | "confabulous", 20 | "prepper", 21 | "example" 22 | ], 23 | "author": "GuideSmiths Ltd", 24 | "license": "ISC", 25 | "devDependencies": { 26 | "eslint": "^3.0.1", 27 | "eslint-config-imperative-es6": "^1.0.0", 28 | "eslint-plugin-mocha": "^4.0.0", 29 | "expect.js": "^0.3.1", 30 | "husky": "^0.11.4", 31 | "mocha": "^2.5.3", 32 | "supertest": "^3.0.0", 33 | "supertest-as-promised": "^4.0.2" 34 | }, 35 | "dependencies": { 36 | "R": "0.0.1", 37 | "aws-sdk": "^2.94.0", 38 | "body-parser": "^1.15.2", 39 | "boom": "^3.2.2", 40 | "bunyan": "^1.8.5", 41 | "chalk": "^1.1.3", 42 | "confabulous": "^1.1.0", 43 | "debug": "^2.6.8", 44 | "hogan.js": "^3.0.2", 45 | "http-proxy-middleware": "^0.17.4", 46 | "logs-to-sumologic": "^1.1.0", 47 | "make-manifest": "^1.0.1", 48 | "nock": "^9.0.13", 49 | "node-ssh": "^4.2.3", 50 | "on-headers": "^1.0.1", 51 | "optimist": "^0.6.1", 52 | "optional": "^0.1.3", 53 | "pify": "^3.0.0", 54 | "prepper": "^1.1.0", 55 | "ramda": "^0.21.0", 56 | "superagent": "^3.5.2", 57 | "systemic": "^1.3.0", 58 | "systemic-domain-runner": "^1.1.0", 59 | "systemic-express": "^1.0.1", 60 | "systemic-pg": "^1.0.6", 61 | "systemic-rabbitmq": "^1.0.1" 62 | }, 63 | "repository": { 64 | "type": "git", 65 | "url": "git+https://github.com/guidesmiths/svc-example.git" 66 | }, 67 | "bugs": { 68 | "url": "https://github.com/guidesmiths/svc-example/issues" 69 | }, 70 | "homepage": "https://github.com/guidesmiths/svc-example#readme" 71 | } 72 | -------------------------------------------------------------------------------- /recipes-crawler/system.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const { join } = require('path'); 3 | 4 | module.exports = () => new System({ name: 'svc-example' }).bootstrap(join(__dirname, 'components')); 5 | -------------------------------------------------------------------------------- /recipes-crawler/test/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "mocha": true 4 | }, 5 | "plugins": [ 6 | "mocha" 7 | ], 8 | "rules": { 9 | "mocha/no-exclusive-tests": 2 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /recipes-crawler/test/e2e/crawler.tests.js: -------------------------------------------------------------------------------- 1 | const nock = require('nock'); 2 | const R = require('ramda'); 3 | const expect = require('expect.js'); 4 | const configSystem = require('../../components/config'); 5 | const system = require('../../system'); 6 | const recipesSearch = require('../../fixtures/recipesSearch.json'); 7 | 8 | describe('Crawls a source url to get recipes and publishes them', () => { 9 | let sys; 10 | let myBroker; 11 | let myConfig; 12 | 13 | before((done) => { 14 | nock.cleanAll(); 15 | configSystem.start((err, { config }) => { 16 | if (err) return done(err); 17 | myConfig = config.crawler; 18 | done(); 19 | }); 20 | }); 21 | 22 | afterEach(() => { 23 | nock.cleanAll(); 24 | if (sys) return stopSystem(); 25 | }); 26 | 27 | const startSystem = () => new Promise((resolve, reject) => { 28 | sys = system().start((err, { broker }) => { 29 | if (err) return reject(err); 30 | myBroker = broker; 31 | resolve(); 32 | }); 33 | }); 34 | 35 | const stopSystem = () => new Promise((resolve, reject) => { 36 | sys.stop((err) => { 37 | if (err) return reject(err); 38 | resolve(); 39 | }); 40 | }); 41 | 42 | const random = () => Math.floor(Math.random() * 100000); 43 | 44 | const createSearchResponse = (recipes) => JSON.stringify(recipes || recipesSearch); 45 | 46 | const nockExternalApiSearch = (statusCode, searchResponse) => 47 | nock(myConfig.baseUrl) 48 | .get(`${myConfig.searchSuffix}`) 49 | .query({ 50 | key: `${myConfig.key}`, 51 | page: `${myConfig.page}` 52 | }) 53 | .reply(statusCode, searchResponse); 54 | 55 | const nockRecipesApi = R.curry((statusCode, findResponse) => { 56 | const { host, path, query } = myConfig.recipesApi; 57 | nock(host) 58 | .get(path.replace(':id', findResponse.recipe_id)) 59 | .query({ 60 | key: query.value 61 | }) 62 | .reply(statusCode, findResponse); 63 | }); 64 | 65 | const nockIdGenerator = (statusCode, expectedId) => { 66 | const { host, path } = myConfig.idGenerator; 67 | nock(host) 68 | .get(path) 69 | .reply(statusCode, { id: expectedId }); 70 | }; 71 | 72 | const nockExternalApiGet = R.curry((statusCode, recipe_id) => { 73 | const getRecipe = R.pipe( 74 | R.find(R.propEq('recipe_id', recipe_id)), 75 | R.merge({ ingredients: [] }) 76 | ); 77 | const recipe = getRecipe(recipesSearch.recipes); 78 | return nock(myConfig.baseUrl) 79 | .get(`${myConfig.recipeSuffix}`) 80 | .query({ 81 | key: `${myConfig.key}`, 82 | rId: recipe_id 83 | }) 84 | .reply(statusCode, JSON.stringify({ recipe })) 85 | }); 86 | 87 | const shouldReceive = (expectedRK, times, received = []) => { 88 | return myBroker.subscribe('recipes_crawler') 89 | .then(({ message, content, ackOrNack, cancel }) => { 90 | ackOrNack(); 91 | if (message.fields.routingKey === expectedRK) received.push(content); 92 | if (times === received.length) return cancel.then(() => Promise.resolve(received)); 93 | return shouldReceive(expectedRK, times, received); 94 | }); 95 | }; 96 | 97 | const shouldNotReceive = () => new Promise((resolve, reject) => { 98 | setTimeout(() => resolve(), 1000); 99 | return myBroker.subscribe('recipes_crawler') 100 | .then(({ message, ackOrNack, cancel }) => { 101 | ackOrNack(); 102 | return reject('A message has been received and this is wrong!'); 103 | }); 104 | }); 105 | 106 | const wait = (ms) => new Promise((resolve, reject) => setTimeout(resolve, ms)); 107 | 108 | const PAUSE = 500; 109 | 110 | const checkFields = (recipe) => { 111 | ['publisher','ingredients','source_url','image_url', 112 | 'social_rank','title','source_id','source'].forEach((field) => expect(recipe[field]).to.be.ok()); 113 | expect(recipe.source).to.equal('F2F'); 114 | expect(recipe.id).to.be.a('number'); 115 | expect(recipe.version).to.be.greaterThan(0); 116 | }; 117 | 118 | it('fails when search endpoint is not available', () => startSystem().then(() => shouldNotReceive())); 119 | 120 | it('fails when recipes endpoint is not available', () => { 121 | nockExternalApiSearch(200, createSearchResponse()); 122 | return startSystem().then(() => shouldNotReceive()); 123 | }); 124 | 125 | it('publishes a recipe with the recorded id whenever the recipes API returns that it already exists', () => { 126 | const myRecipe = R.head(recipesSearch.recipes); 127 | const sampleMsg = { 128 | count: 1, 129 | recipes: [ myRecipe ] 130 | }; 131 | const existentId = myRecipe.recipe_id; 132 | nockExternalApiSearch(200, createSearchResponse(sampleMsg)); 133 | nockExternalApiGet(200, existentId); 134 | nockRecipesApi(200, R.merge(myRecipe, { id: existentId })); 135 | return startSystem() 136 | .then(() => wait(PAUSE)) 137 | .then(() => shouldReceive('recipes_crawler.v1.notifications.recipe.crawled', sampleMsg.recipes.length)) 138 | .then(([ received ]) => { 139 | expect(myRecipe.id).to.equal(undefined); 140 | expect(received.id).to.equal(myRecipe.recipe_id); 141 | }); 142 | }); 143 | 144 | it('publishes a recipe with a new generated id whenever the API cannot find it', () => { 145 | const myRecipe = R.head(recipesSearch.recipes); 146 | const sampleMsg = { 147 | count: 1, 148 | recipes: [ myRecipe ] 149 | }; 150 | const existentId = myRecipe.recipe_id; 151 | nockExternalApiSearch(200, createSearchResponse(sampleMsg)); 152 | nockExternalApiGet(200, existentId); 153 | nockRecipesApi(404, R.merge(myRecipe, { id: existentId })); 154 | const newId = 1234567890; 155 | nockIdGenerator(200, newId); 156 | return startSystem() 157 | .then(() => wait(PAUSE)) 158 | .then(() => shouldReceive('recipes_crawler.v1.notifications.recipe.crawled', sampleMsg.recipes.length)) 159 | .then(([ received ]) => { 160 | expect(received.id).to.equal(newId); 161 | }); 162 | }); 163 | 164 | it('publishes as many recipes as the search page returned when everything goes OK', () => { 165 | nockExternalApiSearch(200, createSearchResponse()); 166 | R.map(nockExternalApiGet(200), R.pluck('recipe_id', recipesSearch.recipes)); 167 | R.map(nockRecipesApi(200), recipesSearch.recipes); 168 | return startSystem() 169 | .then(() => wait(PAUSE)) 170 | .then(() => shouldReceive('recipes_crawler.v1.notifications.recipe.crawled', recipesSearch.recipes.length)) 171 | .then((received) => expect(received.length).to.equal(recipesSearch.recipes.length)); 172 | }); 173 | 174 | it('publishes recipes applying a transformation on all of them', () => { 175 | nockExternalApiSearch(200, createSearchResponse()); 176 | R.map(nockExternalApiGet(200), R.pluck('recipe_id', recipesSearch.recipes)); 177 | const recipesWithIds = R.map(R.merge({ id: random(), ingredients: [] }), recipesSearch.recipes); 178 | R.map(nockRecipesApi(200), recipesWithIds); 179 | return startSystem() 180 | .then(() => wait(PAUSE)) 181 | .then(() => shouldReceive('recipes_crawler.v1.notifications.recipe.crawled', recipesSearch.recipes.length)) 182 | .then((received) => { 183 | expect(received.length).to.equal(recipesSearch.recipes.length); 184 | R.map(checkFields, received); 185 | }); 186 | }); 187 | }); 188 | -------------------------------------------------------------------------------- /recipes-crawler/test/env.js: -------------------------------------------------------------------------------- 1 | process.env.SERVICE_ENV = process.env.SERVICE_ENV || 'test' 2 | process.env.NODE_ENV = 'test' 3 | -------------------------------------------------------------------------------- /recipes-crawler/test/mocha.opts: -------------------------------------------------------------------------------- 1 | --recursive 2 | --require ./test/env.js 3 | --timeout 5000 4 | -------------------------------------------------------------------------------- /recipes-crawler/test/service.tests.js: -------------------------------------------------------------------------------- 1 | const expect = require('expect.js'); 2 | const system = require('../system'); 3 | const supertest = require('supertest-as-promised'); 4 | 5 | describe('Service Tests', () => { 6 | let request; 7 | let sys; 8 | 9 | before(done => { 10 | sys = system().start((err, { app }) => { 11 | if (err) return done(err); 12 | request = supertest(Promise)(app); 13 | done(); 14 | }); 15 | }); 16 | 17 | after(done => sys.stop(done)); 18 | 19 | it('should return manifest', () => 20 | request 21 | .get('/__/manifest') 22 | .expect(200) 23 | .then((response) => { 24 | expect(response.headers['content-type']).to.equal('application/json; charset=utf-8'); 25 | }) 26 | ); 27 | }); 28 | -------------------------------------------------------------------------------- /recipes-crawler/test/test-system.js: -------------------------------------------------------------------------------- 1 | const R = require('ramda'); 2 | const system = require('../system'); 3 | 4 | module.exports = (mockFn = R.identity) => mockFn(system()); 5 | 6 | -------------------------------------------------------------------------------- /recipes-id-generator/.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | -------------------------------------------------------------------------------- /recipes-id-generator/.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | 15 | [Makefile] 16 | indent_style = tab -------------------------------------------------------------------------------- /recipes-id-generator/.eslintignore: -------------------------------------------------------------------------------- 1 | client/* 2 | -------------------------------------------------------------------------------- /recipes-id-generator/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "imperative-es6" 3 | } -------------------------------------------------------------------------------- /recipes-id-generator/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | manifest.json 3 | *.log 4 | -------------------------------------------------------------------------------- /recipes-id-generator/.npmrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feliun/microservices-school/0f4b67d3f64d53ec3e9957695803e9856db28498/recipes-id-generator/.npmrc -------------------------------------------------------------------------------- /recipes-id-generator/.nvmrc: -------------------------------------------------------------------------------- 1 | 7 2 | -------------------------------------------------------------------------------- /recipes-id-generator/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/feliun/docker-nvm-yarn 2 | COPY manifest.json /root/app/manifest.json 3 | -------------------------------------------------------------------------------- /recipes-id-generator/Makefile: -------------------------------------------------------------------------------- 1 | SERVICE_PORT=3002 2 | 3 | start: 4 | @docker run -d -p $(SERVICE_PORT):$(SERVICE_PORT) -e SERVICE_ENV=live -e MONGO_URL=$(MONGO_URL) -e SUMO_URL=$(SUMO_URL) --name $(SERVICE) --network=local $(SERVICE):$(TRAVIS_BUILD_NUMBER) 5 | 6 | include ../Makefile 7 | -------------------------------------------------------------------------------- /recipes-id-generator/README.md: -------------------------------------------------------------------------------- 1 | # recipes-id-generator 2 | 3 | A service to generate ids sequentially in a microservice ecosystem 4 | -------------------------------------------------------------------------------- /recipes-id-generator/components/app/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const optional = require('optional'); 3 | const { join } = require('path'); 4 | const manifest = optional(join(process.cwd(), 'manifest.json')) || {}; 5 | const pkg = require(join(process.cwd(), 'package.json')); 6 | 7 | module.exports = new System({ name: 'app' }) 8 | .add('manifest', manifest) 9 | .add('pkg', pkg); 10 | -------------------------------------------------------------------------------- /recipes-id-generator/components/config/confabulous.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = ({ confabulous } = {}) => { 4 | 5 | const Confabulous = confabulous || require('confabulous'); 6 | const loaders = Confabulous.loaders; 7 | let config; 8 | 9 | const start = (cb) => { 10 | if (config) return cb(null, config); 11 | 12 | new Confabulous() 13 | .add(config => loaders.require({ path: path.join(process.cwd(), 'config', 'default.js'), watch: true })) 14 | .add(config => loaders.require({ path: path.join(process.cwd(), 'config', `${process.env.SERVICE_ENV}.js`), mandatory: false })) 15 | .add(config => loaders.require({ path: path.join(process.cwd(), 'secrets', 'secrets.json'), watch: true, mandatory: false })) 16 | .add(config => loaders.args()) 17 | .on('loaded', cb) 18 | .on('error', cb) 19 | .end(cb); 20 | }; 21 | 22 | return { start }; 23 | }; 24 | -------------------------------------------------------------------------------- /recipes-id-generator/components/config/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const confabulous = require('./confabulous'); 3 | 4 | module.exports = new System({ name: 'config' }).add('config', confabulous(), { scoped: true }); 5 | -------------------------------------------------------------------------------- /recipes-id-generator/components/express/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const defaultMiddleware = require('systemic-express').defaultMiddleware; 3 | const app = require('systemic-express').app; 4 | const server = require('systemic-express').server; 5 | 6 | module.exports = new System({ name: 'express' }) 7 | .add('app', app()).dependsOn('config', 'logger') 8 | .add('middleware.default', defaultMiddleware()).dependsOn('logger', 'app', 'routes') 9 | .add('server', server()).dependsOn('config', 'app', 'middleware.default'); 10 | -------------------------------------------------------------------------------- /recipes-id-generator/components/generators/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const generator = require('./strategies'); 3 | 4 | module.exports = new System({ name: 'generator' }) 5 | .add('generator', generator()).dependsOn('config', 'logger'); 6 | -------------------------------------------------------------------------------- /recipes-id-generator/components/generators/strategies/block.js: -------------------------------------------------------------------------------- 1 | const R = require('ramda'); 2 | const { BlockArray } = require('block-sequence'); 3 | const init = require('block-sequence-mongo'); 4 | 5 | module.exports = (options) => new Promise((resolve, reject) => { 6 | 7 | let idGenerator; 8 | 9 | const name = options.block.sequence.name; 10 | 11 | const generate = () => new Promise((resolve, reject) => { 12 | idGenerator.next((err, id) => { 13 | if (err) return reject(err); 14 | resolve(id); 15 | }); 16 | }); 17 | 18 | const mongoOptions = {}; 19 | 20 | init({ url: options.url, options: mongoOptions }, (err, driver) => { 21 | if (err) return reject(err); 22 | driver.ensure({ name }, (err, sequence) => { 23 | if (err) return reject(err); 24 | const driverOpts = { block: { sequence, driver, size: options.size }}; 25 | const blockConfig = R.merge(options, driverOpts); 26 | idGenerator = new BlockArray(blockConfig); 27 | resolve(generate); 28 | }); 29 | }); 30 | 31 | }); 32 | -------------------------------------------------------------------------------- /recipes-id-generator/components/generators/strategies/index.js: -------------------------------------------------------------------------------- 1 | const strategies = require('require-all')({ 2 | dirname: __dirname, 3 | filter: (fileName) => fileName === 'index.js' ? undefined : fileName.replace('.js', '') 4 | }); 5 | 6 | module.exports = () => { 7 | const start = ({ config, logger }, cb) => { 8 | const { strategy, options } = config; 9 | const init = strategies[strategy]; 10 | if (!init) return cb(`No generator strategy for ${strategy}`); 11 | init(options, logger) 12 | .then((generate) => cb(null, { generate })) 13 | .catch(cb); 14 | }; 15 | 16 | return { start }; 17 | 18 | }; 19 | -------------------------------------------------------------------------------- /recipes-id-generator/components/generators/strategies/memory.js: -------------------------------------------------------------------------------- 1 | let currentId = 1; 2 | 3 | const generate = () => 4 | Promise.resolve(currentId) 5 | .then(() => currentId++); 6 | 7 | module.exports = () => Promise.resolve(generate); 8 | -------------------------------------------------------------------------------- /recipes-id-generator/components/logging/bunyan.js: -------------------------------------------------------------------------------- 1 | const bunyan = require('bunyan'); 2 | const R = require('ramda'); 3 | 4 | module.exports = () => { 5 | 6 | let log; 7 | 8 | const onMessage = (event) => { 9 | log[event.level](R.omit(['level', 'message'], event), event.message); 10 | }; 11 | 12 | const start = ({ pkg }, cb) => { 13 | log = bunyan.createLogger({ name: pkg.name }); 14 | return cb(null, onMessage); 15 | }; 16 | 17 | return { start }; 18 | }; 19 | -------------------------------------------------------------------------------- /recipes-id-generator/components/logging/console.js: -------------------------------------------------------------------------------- 1 | const chalk = require('chalk'); 2 | const hogan = require('hogan.js'); 3 | const R = require('ramda'); 4 | 5 | const response = hogan.compile('{{{displayTracer}}} {{{displayLevel}}} {{package.name}} {{{request.method}}} {{{response.statusCode}}} {{{request.url}}}'); 6 | const error = hogan.compile('{{{displayTracer}}} {{{displayLevel}}} {{package.name}} {{{message}}} {{{code}}}\n{{{error.stack}}} {{{details}}}'); 7 | const info = hogan.compile('{{{displayTracer}}} {{{displayLevel}}} {{package.name}} {{{message}}} {{{details}}}'); 8 | 9 | const colours = { 10 | debug: chalk.gray, 11 | info: chalk.white, 12 | warn: chalk.yellow, 13 | error: chalk.red, 14 | default: chalk.white 15 | }; 16 | 17 | module.exports = () => { 18 | 19 | const onMessage = (event) => { 20 | const details = R.pluck(event, []); 21 | const data = R.merge(event, { 22 | displayTracer: R.has('tracer', event) ? event.tracer.substr(0, 6) : '------', 23 | displayLevel: event.level.toUpperCase(), 24 | details: Object.keys(details).length ? `\n ${JSON.stringify(details, null, 2)}` : '' 25 | }); 26 | const colour = colours[event.level] || colours.default; 27 | const log = console[event.level] || console.info; // eslint-disable-line no-console 28 | if (R.has('response.statusCode', event)) log(colour(response.render(data))); 29 | else if (R.has('error.message', event)) log(colour(error.render(data))); 30 | else log(colour(info.render(data))); 31 | }; 32 | 33 | const start = (cb) => cb(null, onMessage); 34 | 35 | return { start }; 36 | }; 37 | -------------------------------------------------------------------------------- /recipes-id-generator/components/logging/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const prepper = require('./prepper'); 3 | const console = require('./console'); 4 | const bunyan = require('./bunyan'); 5 | const sumo = require('./sumo'); 6 | const prepperMiddleware = require('./prepper-middleware'); 7 | 8 | module.exports = new System({ name: 'logging' }) 9 | .add('transports.console', console()) 10 | .add('transports.sumo', sumo()) 11 | .add('transports.bunyan', bunyan()).dependsOn('pkg') 12 | .add('transports').dependsOn( 13 | { component: 'transports.console', destination: 'console' }, 14 | { component: 'transports.sumo', destination: 'sumo' }, 15 | { component: 'transports.bunyan', destination: 'bunyan' }) 16 | .add('logger', prepper()).dependsOn('config', 'pkg', 'transports') 17 | .add('middleware.prepper', prepperMiddleware()).dependsOn('app') 18 | -------------------------------------------------------------------------------- /recipes-id-generator/components/logging/prepper-middleware.js: -------------------------------------------------------------------------------- 1 | const onHeaders = require('on-headers'); 2 | const R = require('ramda'); 3 | 4 | module.exports = ({ prepper } = {}) => { 5 | const handlers = (prepper || require('prepper')).handlers; 6 | 7 | const start = ({ app }, cb) => { 8 | app.use((req, res, next) => { 9 | const logger = req.app.locals.logger.child({ handlers: [ 10 | new handlers.Tracer(), 11 | new handlers.Merge(R.pick(['url', 'method', 'headers', 'params'], req), { key: 'request' }) 12 | ]}); 13 | 14 | onHeaders(res, () => { 15 | const response = { response: { statusCode: res.statusCode, headers: res.headers } }; 16 | if (res.statusCode === 400) logger.error(req.url, response); 17 | if (res.statusCode < 500) logger.info(req.url, response); 18 | else logger.error(req.url, response); 19 | }); 20 | 21 | res.locals.logger = logger; 22 | 23 | next(); 24 | }); 25 | 26 | cb(); 27 | }; 28 | 29 | return { start }; 30 | }; 31 | -------------------------------------------------------------------------------- /recipes-id-generator/components/logging/prepper.js: -------------------------------------------------------------------------------- 1 | const merge = require('lodash.merge'); 2 | const R = require('ramda'); 3 | 4 | module.exports = ({ prepper, transport } = {}) => { 5 | 6 | const prepperFn = prepper || require('prepper'); 7 | const handlers = prepperFn.handlers; 8 | 9 | const start = ({ config, transports, pkg = { name: 'unknown' } }, cb) => { 10 | const transportFn = transport || R.path([config.transport], transports); 11 | config = merge({ include: [], exclude: [] }, config); 12 | 13 | const logger = new prepperFn.Logger({ handlers: [ 14 | new handlers.Merge({ app: pkg.name }), 15 | new handlers.Merge({ env: process.env.SERVICE_ENV }), 16 | new handlers.Process(), 17 | new handlers.System(), 18 | new handlers.Timestamp(), 19 | new handlers.Flatten(), 20 | new handlers.KeyFilter({ include: config.include, exclude: config.exclude }), 21 | new handlers.Unflatten() 22 | ]}).on('message', event => { 23 | if (transportFn) transportFn(event); 24 | }); 25 | 26 | cb(null, logger); 27 | }; 28 | 29 | return { start }; 30 | }; 31 | -------------------------------------------------------------------------------- /recipes-id-generator/components/logging/sumo.js: -------------------------------------------------------------------------------- 1 | const R = require('ramda'); 2 | const Sumologic = require('logs-to-sumologic'); 3 | 4 | let logger; 5 | 6 | module.exports = () => { 7 | 8 | const onMessage = (event) => { 9 | logger.log(event, (err) => { 10 | if (err) console.log(`Error posting to sumo! ${err.message}`); 11 | }); 12 | }; 13 | 14 | const start = (cb) => { 15 | const url = process.env.SUMO_URL; 16 | if (!url) return cb(null, R.identity); 17 | logger = Sumologic.createClient({ url, name: "RecipesCollector" }); 18 | return cb(null, onMessage); 19 | }; 20 | 21 | return { start }; 22 | }; 23 | -------------------------------------------------------------------------------- /recipes-id-generator/components/routes/admin-routes.js: -------------------------------------------------------------------------------- 1 | module.exports = () => { 2 | 3 | const start = ({ manifest = {}, app }, cb) => { 4 | app.get('/__/manifest', (req, res) => res.json(manifest)); 5 | app.post('/__/error', (req, res) => { 6 | setTimeout(() => process.emit('error', new Error('On Noes'))); 7 | }); 8 | app.post('/__/crash', (req, res) => { 9 | setTimeout(() => undefined.meh); 10 | }); 11 | app.post('/__/reject', (req, res) => { 12 | setTimeout(() => Promise.reject(new Error('Oh Noes'))); 13 | }); 14 | cb(); 15 | }; 16 | 17 | return { start }; 18 | }; 19 | -------------------------------------------------------------------------------- /recipes-id-generator/components/routes/api-routes.js: -------------------------------------------------------------------------------- 1 | const bodyParser = require('body-parser'); 2 | 3 | module.exports = () => { 4 | 5 | const start = ({ app, generator }, cb) => { 6 | 7 | const { generate } = generator; 8 | 9 | app.use(bodyParser.json()); 10 | 11 | app.get('/api/v1/id', (req, res, next) => { 12 | generate() 13 | .then((id) => res.json({ id })) 14 | .catch(next); 15 | }); 16 | 17 | cb(); 18 | }; 19 | 20 | return { start }; 21 | }; 22 | -------------------------------------------------------------------------------- /recipes-id-generator/components/routes/index.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const adminRoutes = require('./admin-routes'); 3 | const apiRoutes = require('./api-routes'); 4 | 5 | module.exports = new System({ name: 'routes' }) 6 | .add('routes.api', apiRoutes()).dependsOn('config', 'logger', 'app', 'middleware.prepper', 'manifest', 'generator') 7 | .add('routes.admin', adminRoutes()).dependsOn('config', 'logger', 'app', 'middleware.prepper', 'manifest') 8 | .add('routes').dependsOn('routes.admin', 'routes.api'); 9 | -------------------------------------------------------------------------------- /recipes-id-generator/config/build.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | generator: { 3 | options: { 4 | url: 'mongodb://mongo:27017/ysojkvfe' 5 | } 6 | }, 7 | logger: { 8 | transport: null 9 | } 10 | }; 11 | -------------------------------------------------------------------------------- /recipes-id-generator/config/default.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | server: { 3 | host: '0.0.0.0', 4 | port: 3002 5 | }, 6 | generator: { 7 | strategy: 'block', 8 | options: { 9 | size: 1, 10 | url: 'mongodb://127.0.0.1/recipes', 11 | block: { 12 | prime: false, 13 | size: 1, 14 | retry: { 15 | limit: 100, 16 | interval: 100 17 | }, 18 | sequence: { 19 | name: "recipes", 20 | value: 0, 21 | metadata: { 22 | environment: process.env.SERVICE_ENV 23 | } 24 | }, 25 | template: "{{=sequence.name}}-{{=id}}-{{=sequence.metadata.environment}}" 26 | } 27 | } 28 | }, 29 | logger: { 30 | transport: 'bunyan', 31 | include: [ 32 | 'tracer', 33 | 'timestamp', 34 | 'level', 35 | 'message', 36 | 'error.message', 37 | 'error.code', 38 | 'error.stack', 39 | 'request.url', 40 | 'request.headers', 41 | 'request.params', 42 | 'request.method', 43 | 'response.statusCode', 44 | 'response.headers', 45 | 'response.time', 46 | 'process', 47 | 'app', 48 | 'env' 49 | ], 50 | exclude: [ 51 | 'password', 52 | 'secret', 53 | 'token', 54 | 'request.headers.cookie', 55 | 'dependencies', 56 | 'devDependencies' 57 | ] 58 | } 59 | }; 60 | -------------------------------------------------------------------------------- /recipes-id-generator/config/live.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logger: { 3 | transport: 'sumo' 4 | }, 5 | generator: { 6 | options: { 7 | url: process.env.MONGO_URL || 'mongodb://mongo:27017/ysojkvfe' 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /recipes-id-generator/config/local.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logger: { 3 | transport: 'console' 4 | } 5 | }; 6 | -------------------------------------------------------------------------------- /recipes-id-generator/config/test.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logger: { 3 | transport: null 4 | } 5 | }; 6 | -------------------------------------------------------------------------------- /recipes-id-generator/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | 5 | mongo: 6 | image: mongo 7 | container_name: mongo 8 | ports: 9 | - '27017:27017' 10 | command: "--logpath=/dev/null" 11 | 12 | networks: 13 | default: 14 | external: 15 | name: local 16 | -------------------------------------------------------------------------------- /recipes-id-generator/docker/supervisor.conf: -------------------------------------------------------------------------------- 1 | [program:recipes-id-generator] 2 | directory=/root/app 3 | command=/usr/bin/npm run start 4 | autorestart=true 5 | stdout_logfile=/var/log/supervisor/recipes-id-generator.out.log 6 | stderr_logfile=/var/log/supervisor/recipes-id-generator.err.log 7 | stdout_logfile_backups=5 8 | stderr_logfile_backups=5 9 | -------------------------------------------------------------------------------- /recipes-id-generator/index.js: -------------------------------------------------------------------------------- 1 | process.env.SERVICE_ENV = process.env.SERVICE_ENV || 'local'; 2 | 3 | const system = require('./system'); 4 | const runner = require('systemic-domain-runner'); 5 | const bunyan = require('bunyan'); 6 | const name = require('./package.json').name; 7 | const emergencyLogger = process.env.SERVICE_ENV === 'local' ? console : bunyan.createLogger({ name: name }); 8 | 9 | const die = (message, err) => { 10 | emergencyLogger.error(err, message); 11 | process.exit(1); 12 | }; 13 | 14 | runner(system(), { logger: emergencyLogger }).start((err, dependencies) => { 15 | if (err) die('Error starting system', err); 16 | dependencies.logger.info(`${dependencies.pkg.name} has started`); 17 | }); 18 | 19 | -------------------------------------------------------------------------------- /recipes-id-generator/infra/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker run -d -p 3002:3002 -e SERVICE_ENV=live -e MONGO_URL=$MONGO_URL -e SUMO_URL=$SUMO_URL --name recipes-id-generator --network=local quay.io/feliun/recipes-id-generator:latest 4 | -------------------------------------------------------------------------------- /recipes-id-generator/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "recipes-id-generator", 3 | "version": "0.0.1", 4 | "description": "An ID generator", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "node index.js", 8 | "server": "supervisor .", 9 | "test": "mocha test", 10 | "lint": "eslint .", 11 | "qa": "npm run lint && npm run test", 12 | "docker": "bash -c '(docker network inspect local 2>&1 > /dev/null || docker network create local) && docker-compose --file docker-compose.yml pull && docker-compose --file docker-compose.yml up -d --force-recreate'", 13 | "precommit": "npm run lint", 14 | "prepush": "npm run test" 15 | }, 16 | "author": "GuideSmiths Ltd", 17 | "license": "ISC", 18 | "devDependencies": { 19 | "eslint": "^3.0.1", 20 | "eslint-config-imperative-es6": "^1.0.0", 21 | "eslint-plugin-mocha": "^4.0.0", 22 | "expect.js": "^0.3.1", 23 | "husky": "^0.11.4", 24 | "mocha": "^2.5.3", 25 | "supertest": "^3.0.0", 26 | "supertest-as-promised": "^4.0.2", 27 | "systemic-mongodb": "^1.0.4" 28 | }, 29 | "dependencies": { 30 | "R": "0.0.1", 31 | "aws-sdk": "^2.100.0", 32 | "block-sequence": "^1.0.0", 33 | "block-sequence-mongo": "^1.1.0", 34 | "body-parser": "^1.15.2", 35 | "boom": "^3.2.2", 36 | "bunyan": "^1.8.5", 37 | "chalk": "^1.1.3", 38 | "confabulous": "^1.1.0", 39 | "debug": "^2.2.0", 40 | "hogan.js": "^3.0.2", 41 | "http-proxy-middleware": "^0.17.4", 42 | "http-status-codes": "^1.2.0", 43 | "logs-to-sumologic": "^1.1.0", 44 | "make-manifest": "^1.0.1", 45 | "node-ssh": "^4.2.3", 46 | "on-headers": "^1.0.1", 47 | "optimist": "^0.6.1", 48 | "optional": "^0.1.3", 49 | "pify": "^3.0.0", 50 | "prepper": "^1.1.0", 51 | "ramda": "^0.21.0", 52 | "require-all": "^2.2.0", 53 | "systemic": "^1.3.0", 54 | "systemic-domain-runner": "^1.1.0", 55 | "systemic-express": "^1.0.1", 56 | "systemic-pg": "^1.0.6" 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /recipes-id-generator/system.js: -------------------------------------------------------------------------------- 1 | const System = require('systemic'); 2 | const { join } = require('path'); 3 | 4 | module.exports = () => new System({ name: 'recipes-id-generator' }).bootstrap(join(__dirname, 'components')); 5 | -------------------------------------------------------------------------------- /recipes-id-generator/test/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "mocha": true 4 | }, 5 | "plugins": [ 6 | "mocha" 7 | ], 8 | "rules": { 9 | "mocha/no-exclusive-tests": 2 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /recipes-id-generator/test/env.js: -------------------------------------------------------------------------------- 1 | process.env.SERVICE_ENV = process.env.SERVICE_ENV || 'test' 2 | process.env.NODE_ENV = 'test' 3 | -------------------------------------------------------------------------------- /recipes-id-generator/test/mocha.opts: -------------------------------------------------------------------------------- 1 | --recursive 2 | --require ./test/env.js 3 | --timeout 5000 4 | -------------------------------------------------------------------------------- /recipes-id-generator/test/service.tests.js: -------------------------------------------------------------------------------- 1 | const expect = require('expect.js'); 2 | const system = require('../system'); 3 | const supertest = require('supertest-as-promised'); 4 | 5 | describe('Service Tests', () => { 6 | let request; 7 | let sys; 8 | 9 | before(done => { 10 | sys = system().start((err, { app }) => { 11 | if (err) return done(err); 12 | request = supertest(Promise)(app); 13 | done(); 14 | }); 15 | }); 16 | 17 | after(done => sys.stop(done)); 18 | 19 | it('should return manifest', () => 20 | request 21 | .get('/__/manifest') 22 | .expect(200) 23 | .then((response) => { 24 | expect(response.headers['content-type']).to.equal('application/json; charset=utf-8'); 25 | }) 26 | ); 27 | }); 28 | -------------------------------------------------------------------------------- /recipes-id-generator/test/unit/generators.tests.js: -------------------------------------------------------------------------------- 1 | const R = require('ramda'); 2 | const { join } = require('path'); 3 | const expect = require('expect.js'); 4 | const configSystem = require('../../components/config'); 5 | const idGenerators = require('require-all')({ 6 | dirname: join(__dirname, '..', '..', 'components', 'generators', 'strategies'), 7 | filter: (fileName) => fileName === 'index.js' ? undefined : fileName.replace('.js', '') 8 | }); 9 | 10 | const test = (generator, id) => { 11 | 12 | const NUM_IDS = 100; 13 | 14 | describe(`Recipes ID generator based on ${id} strategy`, () => { 15 | let initialConfig; 16 | 17 | before(done => { 18 | configSystem.start((err, { config }) => { 19 | if (err) return done(err); 20 | initialConfig = config; 21 | done(); 22 | }); 23 | }); 24 | 25 | it(`should generate an id greater than 0 with strategy ${id}`, () => 26 | generator(initialConfig.generator.options) 27 | .then((generateFn) => generateFn()) 28 | .then((id) => expect(id).to.be.greaterThan(0)) 29 | ); 30 | 31 | const checkMultipleGeneration = (currentId, generateFn, times) => { 32 | if (times === 0) return Promise.resolve(); 33 | return generateFn() 34 | .then((id) => { 35 | expect(id).to.be.equal(currentId + 1); 36 | return checkMultipleGeneration(currentId + 1, generateFn, times - 1); 37 | }); 38 | }; 39 | 40 | it(`should generate multiple consecutive ids with strategy ${id}`, () => { 41 | let generateFn; 42 | return generator(initialConfig.generator.options) 43 | .then((_generateFn) => { 44 | generateFn = _generateFn; 45 | return generateFn(); 46 | }) 47 | .then((id) => { 48 | expect(id).to.be.greaterThan(0); 49 | return checkMultipleGeneration(id, generateFn, NUM_IDS); 50 | }); 51 | }); 52 | }); 53 | }; 54 | 55 | const runAll = R.mapObjIndexed(test); 56 | runAll(idGenerators); 57 | -------------------------------------------------------------------------------- /recipes-infra/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | manifest.json 3 | *.log 4 | -------------------------------------------------------------------------------- /recipes-infra/.nvmrc: -------------------------------------------------------------------------------- 1 | 7 2 | -------------------------------------------------------------------------------- /recipes-infra/Makefile: -------------------------------------------------------------------------------- 1 | ensure-dependencies: 2 | @npm install 3 | @node prepare-ec2-instance 4 | 5 | brand: 6 | @echo "Nothing to brand here..." 7 | 8 | package: 9 | @echo "Nothing to package here..." 10 | 11 | qa: 12 | @echo "Nothing to QA here..." 13 | 14 | archive: 15 | @echo "Nothing to archive here..." 16 | 17 | check: 18 | @echo "Nothing to check here..." -------------------------------------------------------------------------------- /recipes-infra/config/ec2.json: -------------------------------------------------------------------------------- 1 | { 2 | "ImageId": "ami-d7b9a2b1", 3 | "MaxCount": 1, 4 | "MinCount": 1, 5 | "DryRun": false, 6 | "InstanceType": "t2.micro", 7 | "KeyName": "micro-school-ec2", 8 | "Placement": { "AvailabilityZone": "eu-west-1a"}, 9 | "Monitoring": { 10 | "Enabled": true 11 | }, 12 | "IamInstanceProfile": { 13 | "Name": "deployment" 14 | }, 15 | "SecurityGroupIds": [ 16 | "ssh-and-http-from-anywhere" 17 | ] 18 | } -------------------------------------------------------------------------------- /recipes-infra/deploy.js: -------------------------------------------------------------------------------- 1 | const { join } = require('path'); 2 | const { downloadPemFile, downloadInstanceRegistry } = require('./lib/s3'); 3 | const { replaceServiceContainer } = require('./lib/ssh'); 4 | const { removeFile } = require('./lib/utils'); 5 | 6 | if (!process.env.SERVICE) throw new Error("No SERVICE environment variable has been specified"); 7 | 8 | const PEM_KEY_PATH = join(__dirname, 'micro-school-ec2.pem'); 9 | const INSTANCE_REGISTRY_PATH = join(__dirname, 'instances-registry.json'); 10 | 11 | const cleanUp = () => { 12 | console.log('Cleaning up...'); 13 | return Promise.all([ removeFile(PEM_KEY_PATH), removeFile(INSTANCE_REGISTRY_PATH) ]); 14 | }; 15 | 16 | Promise.all([ 17 | downloadPemFile(PEM_KEY_PATH), 18 | downloadInstanceRegistry(INSTANCE_REGISTRY_PATH) 19 | ]) 20 | .then((result) => { 21 | const { publicDns } = require(INSTANCE_REGISTRY_PATH); 22 | console.log(`All needed files dowloaded. Replacing service docker container in ${publicDns}...`); 23 | return replaceServiceContainer(publicDns, PEM_KEY_PATH, process.env) 24 | .then(() => cleanUp()) 25 | .then(() => { 26 | console.log(`The service ${process.env.SERVICE} has been deployed`); 27 | process.exit(0) 28 | }) 29 | }) 30 | .catch((err) => { 31 | console.error(err); 32 | process.exit(1); 33 | }); 34 | -------------------------------------------------------------------------------- /recipes-infra/lib/aws.js: -------------------------------------------------------------------------------- 1 | const R = require('ramda'); 2 | const AWS = require('aws-sdk'); 3 | const pify = require('pify'); 4 | 5 | const region = 'eu-west-1'; 6 | 7 | AWS.config.update({ 8 | accessKeyId: process.env.AWS_ACCESS_KEY_ID, 9 | secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, 10 | region 11 | }); 12 | 13 | const pifyProtoFns = (obj, ...fns) => R.mergeAll(fns.map(fn => { 14 | const result = {}; 15 | result[fn] = pify(obj[fn].bind(obj)); 16 | return result 17 | })); 18 | 19 | const ec2 = pifyProtoFns(new AWS.EC2(), 'describeInstances', 'runInstances'); 20 | const s3 = new AWS.S3(); 21 | 22 | module.exports = { ec2, s3 }; -------------------------------------------------------------------------------- /recipes-infra/lib/ec2.js: -------------------------------------------------------------------------------- 1 | // your role should contain 2 | // - AmazonEC2FullAccess 3 | // - AmazonS3FullAccess 4 | 5 | const R = require('ramda'); 6 | const { ec2 } = require('../lib/aws'); 7 | const { retry } = require('../lib/utils'); 8 | const ec2Config = require('../config/ec2.json'); 9 | 10 | const INSTANCE_NAME = 'recipes-ec2-instance'; 11 | 12 | const DELAY = 5000; 13 | 14 | const checkInstances = () => { 15 | console.log('Checking running instances...'); 16 | const params = { Filters: [ { Name: 'instance-state-name', Values: [ 'running'] } ] }; 17 | return ec2.describeInstances(params); 18 | }; 19 | 20 | const createInstance = () => { 21 | console.log('Creating new EC2 instance...'); 22 | const tags = { 23 | TagSpecifications: [{ 24 | ResourceType: 'instance', 25 | Tags: [{ 26 | Key: 'Name', 27 | Value: INSTANCE_NAME, 28 | }] 29 | }] 30 | }; 31 | const instanceConfig = R.merge(ec2Config, tags); 32 | return ec2.runInstances(instanceConfig); 33 | }; 34 | 35 | const extractPublicDns = (Reservations) => Reservations 36 | && Reservations[0] 37 | && Reservations[0].Instances 38 | && Reservations[0].Instances[0] 39 | && Reservations[0].Instances[0].PublicDnsName; 40 | 41 | const findPublicDns = () => 42 | checkInstances() 43 | .then(({ Reservations }) => { 44 | const publicDns = extractPublicDns(Reservations); 45 | return publicDns ? publicDns : retry(DELAY, findPublicDns); 46 | }); 47 | 48 | module.exports = { 49 | checkInstances, 50 | createInstance, 51 | extractPublicDns, 52 | findPublicDns 53 | }; -------------------------------------------------------------------------------- /recipes-infra/lib/s3.js: -------------------------------------------------------------------------------- 1 | const R = require('ramda'); 2 | const fs = require('fs'); 3 | const { s3 } = require('./aws'); 4 | 5 | const S3_BUCKET = 'microservices-school-recipes'; 6 | 7 | const registerInstance = (publicDns) => { 8 | const registry = { 9 | creationDate: new Date(), 10 | publicDns 11 | }; 12 | const params = { 13 | Body: JSON.stringify(registry), 14 | Bucket: S3_BUCKET, 15 | Key: 'instances-registry' 16 | }; 17 | return new Promise((resolve, reject) => { 18 | s3.putObject(params, (err) => (err ? reject(err) : resolve())); 19 | }); 20 | }; 21 | 22 | const download = R.curry((s3Key, outputFile) => new Promise((resolve, reject) => { 23 | const s3Stream = s3.getObject({ Bucket: S3_BUCKET, Key: s3Key }).createReadStream(); 24 | const fileStream = fs.createWriteStream(outputFile); 25 | s3Stream.on('error', reject); 26 | fileStream.on('error', reject); 27 | fileStream.on('close', () => resolve(outputFile)); 28 | s3Stream.pipe(fileStream); 29 | })); 30 | 31 | const downloadPemFile = download('keys/micro-school-ec2.pem'); 32 | const downloadInstanceRegistry = download('instances-registry'); 33 | 34 | module.exports = { 35 | downloadPemFile, 36 | downloadInstanceRegistry, 37 | registerInstance 38 | }; -------------------------------------------------------------------------------- /recipes-infra/lib/ssh.js: -------------------------------------------------------------------------------- 1 | const R = require('ramda'); 2 | const { join } = require('path'); 3 | const nodeSSH = require('node-ssh'); 4 | const { wait } = require('./utils'); 5 | 6 | const EC2_USER = 'ec2-user'; 7 | const MAX_ATTEMPTS = 3; 8 | const MIN_STARTUP_TIME = 5000; 9 | const ENV_VARS = ['MONGO_URL', 'RABBIT_PWD', 'SERVICE', 'F2F_KEY', 'SERVICE_PORT', 'SUMO_URL']; 10 | 11 | const runInstallation = (publicDns, pemKeyPath) => { 12 | const ssh = new nodeSSH(); 13 | return ssh 14 | .connect({ 15 | host: publicDns, 16 | username: EC2_USER, 17 | privateKey: pemKeyPath, 18 | }) 19 | .then(() => 20 | ssh 21 | .execCommand('sudo yum update -y', { cwd: '.' }) 22 | .then(() => ssh.execCommand('sudo yum install -y docker', { cwd: '.' })) 23 | .then(() => ssh.execCommand('sudo service docker start', { cwd: '.' })) 24 | .then(() => ssh.execCommand('sudo usermod -a -G docker ec2-user', { cwd: '.' })) 25 | .then(() => ssh.execCommand('sudo docker network create local', { cwd: '.' })) 26 | .then(({ stdout }) => console.log(stdout)) 27 | .then(() => ssh.execCommand('sudo docker network inspect local', { cwd: '.' })) 28 | .then(({ stdout }) => console.log(stdout)) 29 | .then(() => ssh.execCommand('sudo docker ps', { cwd: '.' })) 30 | .then(({ stdout }) => console.log(stdout)) 31 | ) 32 | .catch(e => { 33 | if (e.code !== 'ECONNREFUSED') throw e; 34 | console.log('Instance not ready yet, retrying...'); 35 | return wait(MIN_STARTUP_TIME).then(() => runInstallation(publicDns, pemKeyPath)); 36 | }); 37 | }; 38 | 39 | const replaceServiceContainer = (publicDns, pemKeyPath, environment) => { 40 | const { SERVICE } = environment; 41 | 42 | const applyEnv = () => { 43 | const usefulVars = R.intersection(R.keys(environment), ENV_VARS); 44 | return R.reduce((acc, key) => `${acc} ${key}=${environment[key]}`, '', usefulVars); 45 | }; 46 | 47 | const copyRunScripts = () => 48 | ssh 49 | .putFiles([{ local: join(__dirname, '..', 'deploy.sh'), remote: './deploy.sh' }]) 50 | .then(() => ssh.execCommand('chmod +x deploy.sh', { cwd: '.' })); 51 | 52 | const checkStability = (attempts = 0) => { 53 | return wait(MIN_STARTUP_TIME) 54 | .then(() => { 55 | const checkUrl = `curl http://localhost:${environment.SERVICE_PORT}/__/manifest`; 56 | console.log(`Checking url: ${checkUrl}...`); 57 | return ssh.execCommand(checkUrl, { cwd: '.' }); 58 | }) 59 | .then(res => { 60 | if (!res.stdout) throw new Error(`Service ${SERVICE} not available yet`); 61 | const response = JSON.parse(res.stdout); 62 | if (response.name !== SERVICE) throw new Error(`Expected ${SERVICE} but got ${response.name}`); 63 | }) 64 | .catch(err => { 65 | if (attempts >= MAX_ATTEMPTS) throw new Error(`Something went wrong deploying service ${SERVICE}`); 66 | console.log(`Error on attempt ${attempts}: ${err.message}`); 67 | return checkStability(attempts++); 68 | }); 69 | }; 70 | 71 | const ssh = new nodeSSH(); 72 | console.log(`Connecting to ${publicDns} with ${EC2_USER} and pem file in ${pemKeyPath}...`); 73 | return ssh 74 | .connect({ 75 | host: publicDns, 76 | username: EC2_USER, 77 | privateKey: pemKeyPath, 78 | readyTimeout: 99999, 79 | }) 80 | .then(() => 81 | ssh 82 | .execCommand(`docker stop ${SERVICE} && docker rm ${SERVICE}`, { cwd: '.' }) 83 | .then(() => { 84 | console.log('Coyping run scripts...'); 85 | return copyRunScripts(); 86 | }) 87 | .then(() => { 88 | console.log('Deploying service...'); 89 | return ssh.execCommand(`${applyEnv()} ./deploy.sh`, { cwd: '.' }); 90 | }) 91 | .then(({ stdout }) => console.log(stdout)) 92 | .then(() => checkStability()) 93 | ); 94 | }; 95 | 96 | module.exports = { 97 | runInstallation, 98 | replaceServiceContainer, 99 | }; 100 | -------------------------------------------------------------------------------- /recipes-infra/lib/utils.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | 3 | const removeFile = (filePath) => new Promise((resolve, reject) => { 4 | fs.unlink(filePath, (err) => (err ? reject(err) : resolve())); 5 | }); 6 | 7 | const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); 8 | 9 | const retry = (delay, fn, ...args) => new Promise((resolve) => { 10 | setTimeout(() => fn(args).then(resolve), delay); 11 | }); 12 | 13 | module.exports = { 14 | removeFile, 15 | wait, 16 | retry 17 | }; -------------------------------------------------------------------------------- /recipes-infra/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "recipes-infra", 3 | "version": "0.0.1", 4 | "description": "Scripts to create infrastructure as code", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "node index.js" 8 | }, 9 | "keywords": [ 10 | "microservice", 11 | "infra" 12 | ], 13 | "author": "GuideSmiths Ltd", 14 | "license": "ISC", 15 | "dependencies": { 16 | "aws-sdk": "^2.94.0", 17 | "node-ssh": "^4.2.3", 18 | "pify": "^3.0.0", 19 | "ramda": "^0.24.1" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /recipes-infra/prepare-ec2-instance.js: -------------------------------------------------------------------------------- 1 | const { join } = require('path'); 2 | 3 | const { downloadPemFile, registerInstance } = require('./lib/s3'); 4 | const { runInstallation } = require('./lib/ssh'); 5 | const { checkInstances, createInstance, extractPublicDns, findPublicDns } = require('./lib/ec2'); 6 | const { removeFile, wait } = require('./lib/utils'); 7 | 8 | const PEM_KEY_PATH = join(__dirname, 'micro-school-ec2.pem'); 9 | 10 | const DELAY = 5000; 11 | 12 | const setup = (publicDnsName) => 13 | downloadPemFile(PEM_KEY_PATH) 14 | .then(() => 15 | wait(DELAY) 16 | .then(() => { 17 | console.log('About to run setup commands via ssh...'); 18 | return runInstallation(publicDnsName, PEM_KEY_PATH) 19 | .then(() => removeFile(PEM_KEY_PATH)) 20 | .then(() => registerInstance(publicDnsName)); 21 | })); 22 | 23 | checkInstances() 24 | .then(({ Reservations }) => { 25 | if (Reservations.length > 0) return extractPublicDns(Reservations); 26 | return createInstance() 27 | .then(() => 28 | findPublicDns() 29 | .then((publicDns) => 30 | setup(publicDns) 31 | .then(() => publicDns) 32 | ) 33 | ) 34 | }) 35 | .then((publicDns) => { 36 | console.log(`The EC2 ${publicDns} instance has been configured and it is running!`); 37 | process.exit(0); 38 | }) 39 | .catch((err) => { 40 | console.error(err); 41 | process.exit(1); 42 | }); 43 | -------------------------------------------------------------------------------- /recipes-infra/yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | asn1@~0.2.0: 6 | version "0.2.3" 7 | resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.3.tgz#dac8787713c9966849fc8180777ebe9c1ddf3b86" 8 | 9 | aws-sdk@^2.94.0: 10 | version "2.94.0" 11 | resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.94.0.tgz#7043de3ef8c24cb6ab4bf235f08d87d84173e174" 12 | dependencies: 13 | buffer "4.9.1" 14 | crypto-browserify "1.0.9" 15 | events "^1.1.1" 16 | jmespath "0.15.0" 17 | querystring "0.2.0" 18 | sax "1.2.1" 19 | url "0.10.3" 20 | uuid "3.0.1" 21 | xml2js "0.4.17" 22 | xmlbuilder "4.2.1" 23 | 24 | base64-js@^1.0.2: 25 | version "1.2.1" 26 | resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.1.tgz#a91947da1f4a516ea38e5b4ec0ec3773675e0886" 27 | 28 | buffer@4.9.1: 29 | version "4.9.1" 30 | resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" 31 | dependencies: 32 | base64-js "^1.0.2" 33 | ieee754 "^1.1.4" 34 | isarray "^1.0.0" 35 | 36 | crypto-browserify@1.0.9: 37 | version "1.0.9" 38 | resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-1.0.9.tgz#cc5449685dfb85eb11c9828acc7cb87ab5bbfcc0" 39 | 40 | events@^1.1.1: 41 | version "1.1.1" 42 | resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" 43 | 44 | ieee754@^1.1.4: 45 | version "1.1.8" 46 | resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.8.tgz#be33d40ac10ef1926701f6f08a2d86fbfd1ad3e4" 47 | 48 | isarray@^1.0.0: 49 | version "1.0.0" 50 | resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" 51 | 52 | jmespath@0.15.0: 53 | version "0.15.0" 54 | resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" 55 | 56 | lodash@^4.0.0: 57 | version "4.17.4" 58 | resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" 59 | 60 | node-ssh@^4.2.3: 61 | version "4.2.3" 62 | resolved "https://registry.yarnpkg.com/node-ssh/-/node-ssh-4.2.3.tgz#93f1bb1a6721015d8a2d32980eb2e85c1614825b" 63 | dependencies: 64 | sb-promisify "^2.0.1" 65 | sb-scandir "^1.0.0" 66 | shell-escape "^0.2.0" 67 | ssh2 "^0.5.0" 68 | 69 | pify@^3.0.0: 70 | version "3.0.0" 71 | resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176" 72 | 73 | punycode@1.3.2: 74 | version "1.3.2" 75 | resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" 76 | 77 | querystring@0.2.0: 78 | version "0.2.0" 79 | resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" 80 | 81 | ramda@^0.24.1: 82 | version "0.24.1" 83 | resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.24.1.tgz#c3b7755197f35b8dc3502228262c4c91ddb6b857" 84 | 85 | sax@1.2.1, sax@>=0.6.0: 86 | version "1.2.1" 87 | resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" 88 | 89 | sb-promisify@^1.3.0: 90 | version "1.3.0" 91 | resolved "https://registry.yarnpkg.com/sb-promisify/-/sb-promisify-1.3.0.tgz#3af6f1fa9ffc833f14de86916eefc1f559b1b051" 92 | 93 | sb-promisify@^2.0.1: 94 | version "2.0.2" 95 | resolved "https://registry.yarnpkg.com/sb-promisify/-/sb-promisify-2.0.2.tgz#4277a54754488aa9675d886e354db894c9bdc981" 96 | 97 | sb-scandir@^1.0.0: 98 | version "1.0.0" 99 | resolved "https://registry.yarnpkg.com/sb-scandir/-/sb-scandir-1.0.0.tgz#102b4666c7b644f4287f0d33ceb4374fac1bfa15" 100 | dependencies: 101 | sb-promisify "^1.3.0" 102 | 103 | semver@^5.1.0: 104 | version "5.4.1" 105 | resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" 106 | 107 | shell-escape@^0.2.0: 108 | version "0.2.0" 109 | resolved "https://registry.yarnpkg.com/shell-escape/-/shell-escape-0.2.0.tgz#68fd025eb0490b4f567a027f0bf22480b5f84133" 110 | 111 | ssh2-streams@~0.1.18: 112 | version "0.1.19" 113 | resolved "https://registry.yarnpkg.com/ssh2-streams/-/ssh2-streams-0.1.19.tgz#f80ececc2de1a39e1aa64469851ec32bc96b83f9" 114 | dependencies: 115 | asn1 "~0.2.0" 116 | semver "^5.1.0" 117 | streamsearch "~0.1.2" 118 | 119 | ssh2@^0.5.0: 120 | version "0.5.5" 121 | resolved "https://registry.yarnpkg.com/ssh2/-/ssh2-0.5.5.tgz#c7781ecd2ece7304a253cf620fab5a5c22bb2235" 122 | dependencies: 123 | ssh2-streams "~0.1.18" 124 | 125 | streamsearch@~0.1.2: 126 | version "0.1.2" 127 | resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a" 128 | 129 | url@0.10.3: 130 | version "0.10.3" 131 | resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" 132 | dependencies: 133 | punycode "1.3.2" 134 | querystring "0.2.0" 135 | 136 | uuid@3.0.1: 137 | version "3.0.1" 138 | resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1" 139 | 140 | xml2js@0.4.17: 141 | version "0.4.17" 142 | resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.17.tgz#17be93eaae3f3b779359c795b419705a8817e868" 143 | dependencies: 144 | sax ">=0.6.0" 145 | xmlbuilder "^4.1.0" 146 | 147 | xmlbuilder@4.2.1, xmlbuilder@^4.1.0: 148 | version "4.2.1" 149 | resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-4.2.1.tgz#aa58a3041a066f90eaa16c2f5389ff19f3f461a5" 150 | dependencies: 151 | lodash "^4.0.0" 152 | -------------------------------------------------------------------------------- /ubuntu-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sudo apt update 3 | 4 | #installing sublime 5 | wget -qO - https://download.sublimetext.com/sublimehq-pub.gpg | sudo apt-key add - 6 | sudo apt-add-repository "deb https://download.sublimetext.com/ apt/stable/" 7 | sudo apt install sublime-text 8 | 9 | #installing nvm 10 | wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash 11 | export NVM_DIR="$HOME/.nvm" 12 | source ~/.profile 13 | node -v 14 | 15 | #installing docker & docker-compose 16 | sudo apt install -y docker.io 17 | sudo curl -o /usr/local/bin/docker-compose -L "https://github.com/docker/compose/releases/download/1.15.0/docker-compose-$(uname -s)-$(uname -m)" 18 | sudo chmod +x /usr/local/bin/docker-compose 19 | docker-compose -v 20 | 21 | #installing nodejs environment 22 | nvm install 7 23 | nvm use 7 24 | 25 | #building project 26 | npm install -g yarn 27 | pushd recipes-api 28 | yarn install 29 | popd 30 | pushd recipes-crawler 31 | yarn install 32 | popd 33 | pushd recipes-id-generator 34 | yarn install 35 | popd 36 | pushd recipes-infra 37 | yarn install 38 | popd 39 | 40 | #downloading all branches 41 | git branch -r | grep -v '\->' | while read remote; do git branch --track "${remote#origin/}" "$remote"; done 42 | git fetch --all 43 | git pull --all --------------------------------------------------------------------------------