├── .gitignore ├── .project ├── Amazon_ECS_Java_Starter_Kit.drawio ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── amazon-ecs-java-starter-kit-cdk ├── .gitignore ├── cdk.json ├── delete_ddb_items.sh ├── pom.xml ├── scratchpad.sh ├── src │ └── main │ │ └── java │ │ └── software │ │ └── aws │ │ └── ecs │ │ └── java │ │ └── starterkit │ │ └── cdk │ │ ├── CdkApp.java │ │ ├── ECSTaskSubmissionFromLambdaPattern.java │ │ └── ECSTaskSubmissionFromStepFunctionsPattern.java ├── workflow_specs_pattern_1.json └── workflow_specs_pattern_2.json ├── amazon-ecs-java-starter-kit-task ├── .classpath ├── .gitignore ├── .project ├── Dockerfile ├── LICENSE ├── pom.xml ├── runner.sh └── src │ ├── main │ ├── java │ │ └── software │ │ │ └── aws │ │ │ └── ecs │ │ │ └── java │ │ │ └── starterkit │ │ │ ├── task │ │ │ └── ECSTask.java │ │ │ └── util │ │ │ └── DDBUtil.java │ └── resources │ │ ├── docker_scripts │ │ └── build-docker.sh │ │ ├── ecs_task_metadata_response_sample.json │ │ ├── iam_policy_dynamodb.json │ │ └── iam_policy_s3.json │ └── test │ └── java │ └── software │ └── aws │ └── ecs │ └── java │ └── starterkit │ └── task │ ├── ECSTaskMetadataParserTest.java │ └── RandomTest.java ├── amazon-ecs-java-starter-kit-tasklauncher ├── .classpath ├── .gitignore ├── .project ├── LICENSE ├── dependency-reduced-pom.xml ├── pom.xml └── src │ └── main │ ├── java │ └── software │ │ └── aws │ │ └── ecs │ │ └── java │ │ └── starterkit │ │ ├── launcher │ │ └── ECSTaskLauncher.java │ │ └── util │ │ ├── DDBUtil.java │ │ ├── TaskConfig.java │ │ └── WorkflowSpecs.java │ └── resources │ ├── amazon_cloudwatch_logs_policy.json │ ├── amazon_dynamodb_policy.json │ ├── amazon_ecs_policy.json │ ├── aws_lambda_assume_role_policy_doc.json │ └── iam_pass_role_policy.json ├── amazon-ecs-java-starter-kit-taskmonitor ├── .classpath ├── .gitignore ├── .project ├── dependency-reduced-pom.xml ├── pom.xml └── src │ └── main │ └── java │ └── software │ └── aws │ └── ecs │ └── java │ └── starterkit │ ├── monitor │ ├── ECSTaskMonitor.java │ └── model │ │ ├── Input.java │ │ ├── Iterator.java │ │ └── WorkflowStatus.java │ └── util │ └── DDBUtil.java ├── pom.xml └── resources ├── Amazon_ECS_Java_Starter_Kit-Architecture_Pattern_1.png ├── Amazon_ECS_Java_Starter_Kit-Architecture_Pattern_2.png ├── Pattern_1_execution_outcome.png ├── Pattern_2_execution_outcome.png ├── output_of_bootstrap.png └── pattern_1_test_output.png /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | .settings/ 3 | build/ 4 | target/classes/ -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | amazon-ecs-and-aws-step-functions-design-patterns-starter-kit 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.m2e.core.maven2Builder 10 | 11 | 12 | 13 | 14 | 15 | org.eclipse.m2e.core.maven2Nature 16 | 17 | 18 | -------------------------------------------------------------------------------- /Amazon_ECS_Java_Starter_Kit.drawio: -------------------------------------------------------------------------------- 1 | 7V1bc6O4Ev41qTr7EBd3zGPiJHOylcxJbWY2M+clJYNsmADyAJ7E++tXEgKDJPAN2yTj2VrHFkJI6q8v6m6JM30UvX1KwMy/Rx4MzzTFezvTr840TVN0C/8hJYu8xNKGecE0Cby8SF0WPAb/QFaosNJ54MG0VjFDKMyCWb3QRXEM3axWBpIEvdarTVBYf+oMTKFQ8OiCUCx9CrzMz0uHprIs/y8Mpn7xZFVhVyJQVGYFqQ889Fop0q/P9FGCUJZ/i95GMCSTV8xLft9Nw9WyYwmMs3VuUN7u5tFkAi8+P47Te/vbz6f007nFqJFmi2LE0MMTwH6iJPPRFMUgvF6WXiZoHnuQNKvgX8s6dwjNcKGKC3/ALFswaoJ5hnCRn0Uhu4p7nCy+sfvpj+/kx8AyzKLg6q16+WpR/fUAkyCCGUzKQu+CkBr/dEOQpoGbF94EYfHECYqzGxAFIWnoIgL/oBiP+joak0boVdZZVWO/RyhECZ0RXaH/cLkHUp+OW2U/HkCGuxHTEk0hpSJVGKFSNE9c2EYKncEbJFPI7g2+3k8v4d/B159aYjrOn7O77O9zO69H6FR5AiP6J4jw1CQLXCGBIciCX3UgA8YP07LeEjL4C0PNJghi3f4Fwjl71GMGZ7jkZh67WYDi9Iwg0wrxmC7HCf42Jd/SDGSQ8onrBzH5liXBdIrpwQNygslYUCNGMaQ3J+gFriTRL5hkAeblizCYEiKNUZahqCA4e4LaQvAKZkMwhuEDSgMyJII0TGAKnuIhd1yFjDDDJWCPLqs3AoS0A99aKcqunlsGkymLQhwxTLwuRZRayB2/Ip70glq7wKANlDUUcATehaw8JfPJrZJRaSFjhb81ZaU06IJCulOnUEmxCoXKOlUKWYX+6JxCpkCh44r+ga0YNfG/jejHUiTJROFPiyviv6IhGNSW6mEHQHQngtmtDyjAjyhhpDt2DUW2adSbyJUKu6uq+1c0NDQ4lOVKR2iIAq0cz/ZKwtQk4Ms1QjoDcQ2F1s85MYkoTc5TiqMLolzV2dvyYqFF7tAUfwURkQbxOJ3ll8K8yk3edl5TADtm5ayOzQTix4FxKU5mZDroBJmXZ+aVTJrzcikKPI8yClUXl8B9mVKWqUilCf3HWR3GWjaKTLaJ4Gzlfl6ElZYzG3fNOJWKNmWg6ZpVg9M5a2pbuBdV0GSSwl0BKB2+JeDvf5lPbA3l4ukRfz7C5Ffg4oUGj5L1TQhRnbUTtEcGhcErK1VQVqbMnFBVo5nGOymr4WpldWD7Ya/2Am/PSawFOQG0fVkLjkCActyPukALNM9CbOONykV422xOE+AFsHbNUi503a5cuwoS6DLcx8TM4DgM36PZtqpaMqqXErZcDCgtbFVytBwjAr9VFIZs3VjFE0hn+XRMgjfSj8vUBzNyMXqbEnfJALymxgCrHarGb13SH6KF8m/1Wql+ECAOOVFgiUg0TBGIRVnnOCz8QIe1UmtmZSlcljZlJ16HRnJVXQXySWFid6WroPC+9cRX0Ctn0ypXkztPfpWehKNTctgrQhZcKbHnyUytZc9bMnteq1jveUt9tt43QUQbQ3RilZuqqnJW+bDvZnkB/wqOrkfEHv8C0hf85w7MY9eX+AR3MjgK87xuT1wp5ki1BXuCVe6dKUHhDZPrXzBHudpkXoQgGnvgecK8sQdxf2mmWYOiXTgAqgatxJ4192XOlmOQ2LOib3oUorn3BDLX79zSvdENY6htZulejlTd/H0sXZfM/iud/YMsvXpm8RZL3ZorHRNIMJ3CMJilkDNc2bRjOzT3vwtgKsEpAVNluSu7Y2OPRQm4lZp3uR7fr2BS9bojVFVNRyC3Louc7E0yiavpnixwOvaGrLSFtXVt4cJqOLwx3NrveuwryTbjWDr7J56VGRNG3ZiwhsdmWTGUxuxVVaD5OvYC/u+GPP43N1Shmz5nZB4PACrOzTW0JUpfZp9qg6G9L1SJMYqVMTI3pz1ZTyfT8X90k5qz+PmKYZdfLe0PcaHNAKutGy07Lbh6iWPV4oBsOQNFFI97xLLUr6HJ4jf7xrJ+wvK7xrKmO33EsnEMuWycsPyusayr/cSyGNbdP5bNE5bfNZZ5D1lPsKw3x6D2h+XPJyy/bywP1SNjWR6nF7HM8tKEVPoPFI/IQdeYYbctGlUZGnfJvME0KMNnmPLNPrV+5JUbQxG8RVn3yBUDa/VMgFw+tgTbrkd/rRv03xns9vCaJp1vAPYGqftRg2/QPQiQsX084Ly6x4676eJGpgKXwRKRecFcal3wGS6aLMOlZIDtm7iDUxh7FaYZ843jsnmHD+xi2ORac+dkgyAiYudZ3qENdUAwL09HURIIPLKx7QklL5MQvZLHYC4kRYDQRpnADFelv/H/YZCSjBo0KZskFg+5miH8MSYbpeAbdOcZ9Lrou9bW93Q+joIsrXeF3PDFD0hpkI8Cf1w83OJPLKzCLjqlk2c8gAQ3R3cJ5wMOqGzhZ4b2H5AEDPp7OT20g7Tz43mKlUFKvoZoGrhNG0F26bIhzOM9ioMMkSazBLg5CX1CPhdFsxCy4ZDdjfO0Pq4vjOIBqXC1iEGEri7p+EiSVwedNfnO1udj2fP5zAP5TPoUuRGIFxVIspFAb8A3MfKh+3I7oSMZFbU4rb5t7yPwUrKL66PAhSUvzRLkQvqg1yDzyxl/LXmvFCIVSc3ZDixhcI0swXmGUqZtK4o5hJOsUXvjpaAbxNMvNPp+bjTo7M2D6RUjoAulW/dJOY5E48r2JKrqviKpuinQ6bB7EMsk4G52IG6fKn6QbOMiHXxlhoXBbKG+pBtrKxYZ26YbV12THyvduJXdds83VgaqXvek9GoPoBRFzjFkzdFZuUip7XivMpZnYFGpwOAv0rXwXPBb3GyjSuOV9S1H4TCR96BThKirnBnbyhnjw8qZNk7rZluDY9bNlj7JmVbWbNjUwC9SliuKUwjhPe9xGBpiHOGgexx08QyW5cAJALfAV2Nq4pG9qe8kdEAcQseMF2hctEsTzwmShbr400G6i9l+JBOsWCRVTTC5WdGrxZQkXf0z6qchwrPpzoZJ4eLvYP2jKE7dLun/Aki1jsF+23pbjs+2Zr/4VoxPfZccWPMxGbeAbiecqzr144uc3Ri3MAiHdXHg1BvY33rDEIGxaSIUuYOkPpGdLeSvrv6Rr03GJanP+VtMs7hn+SW/qykYt/YaWhofaIg9NKyr18vSyq3Q4mBX7Uy6M6uw6EiowfVBkg086AZpvs549YMMPs4AFTmvuN5h1h4K58q3xBObVFsXLbvhvvJADHFPVJ7BdIPFLD0IssvV7SmXY53Vx4RN/RFy94+ezSHJdL6NU0h2ZRYBZTEVCQMIJd6RjSSVptadjsVs1rROXdNufSom184eD8Vs9cxvordFtdiSWbfMMThAzjK+ZlzZ2o2+mUjG/4xL57cRyR4liTc+hEy2eyaSTdFOrR60IksNJY5ffOWvedixASF3jzckNr979zgkNZ7zzzFI6c37N1ItReFyPIf28GAQlC8jjxID7jq3o83JeIQjAbeLB+s6jw2DI+8+AryCBHoCgXhQB89E41kUD1hhlbArhcAa7pI15cSsYuGxc0QqRt+lVBiywjN6uEjsgYQqr0U0RmQAGb75QJsWuE3qjl0mfx8gViYHgn1kK//QrtBWD+dKoUFq9cgVqorOhiskecXEkmBqg6tm9zDRJodPWXW/4/JlQRU2GMp27+yLDSRnjvw1j8slMm+P0Rlpcuj0mH8+9nJY51Kltn9LhF7H57oL4k11P9/h4jl7Vf2amB1xA/OFRxCRl4AdF8+6cgyvT8cHV+8b6RqXCrY10vmGDv0+lLZUnduLewGLOzvLTcM2nM08MyNb1dWb38YzE3i4A0G2eMaW8jNwXZimzxGIsWSIKMw7Oa3JPJ5LRn4eiJgL0k3Cq94YmOOQvVW4uvMdOsZKmdfEbB2gwuLd0JpkdSR7+0cXL/+Qv65JclryPPMJg7jU9GveaFfGX/Ma+efqGzBJfZQQahS250kb910bqzZ/yDf3csF1tTHf0LAIJh7q7WRirDDfIXl2ihW+98WRWai4XU1GvqFDm4yWKJMTulSvbIMWUco2Rp9A2muQGrbKgdTcDqR8Q0NDPSxINdFwoFlAo7vbrRY1bfsPTuHm1a9SQFFE1jNknp9psHBC3Z97iu+p5fvojrWgsZrS0O7oDqGtMHjaY3W0PVaWonMQc4qs+wNEji4mi9n3228/vLfs2/9/OuD15fz53NhCm9YCwDur1jXeQr+mupRqt9VZ9b56d/H1UzS+c/98jJORNTKf1w4/5+9uOHi02ShefF5oWNvmkLCby1kKFdkbt7twrqjvZjdxA8M3Je3XodfMfi3J/abNUbo8PKIX+3LaeKealbA8VeoxP1WKo2xNjUiVWIXoAkm3fylIg3ON0xevcIwXHigd/EiJSmjAgCBmmvWAMawT1XGGEh9ZR1sZ8c8EEYZb0piM6h55kNT4Fw==7V1bc6M4Fv41qdp5iAshMPgxceLtVKV7s5Ou7emnlAyyTQeQF+Rc5tePJO4S+IKNcdJOV9tGCCF0vnN0bhIXcBy8/TtCy8VX4mL/Qtfctwt4c6HrugaH7IuXvCclQ91OCuaR5yZFoCh49P7GaaGWlq48F8eVipQQn3rLaqFDwhA7tFKGooi8VqvNiF+96xLNsVLw6CBfLf3huXSRlNqmVpR/wd58kd0ZaOmZAGWV04J4gVzyWiqCtxdwHBFCk1/B2xj7fPCycfliPs+N2Rxo6PKvSTj676+H/325TBqb7HJJ/ggRDulhmzaSpl+Qv0rH65EiisXjOwsvxOmj0/dsPGee74+JTyJ2GBJWAV7HNCLPOCu80KEm/tiZFxxRj9HiyvfmITtHyZKVzkhIH9MWtfS47mpRLwGUnh1PUOD5HIpXAfqbhKxvt8EUsyuv0ydhd8RvEuk3jBvIicm4AJMA0+idXZe2cglHKQAyDjDS49cCT3mdRQlLpp0WohTD87zxgk7sR0qqHchmKmRT6IRdxgbpIYnogsxJiPzbovQ6IqvQxW5KhKLOPeFUugGs8Bem9D0lAVpRwooWNPDTs2xUo/e/+PUDSzOygp9pg+Lg5q1y9F4+esCRx4aDE08UxhRF9IqzPCtwfBTHnpMVTzy/uKubVUrxx0rS89uihA/OQTASYR9R76XaWB2500sfiMduk2MLjqwKtCzTqDYRk1Xk4PSqMnNvaMg2JOixQZxjqjQk0Jc/T3tADmsAOfQpJ98ShRVkDv+/4hJT0OkyFti6YhUAWL4VJ9mvOf++J3P2EwVcbITTeJmc8pMqk6TtpKbCAEwM0CpeI8xuh6a53Fny0RDjY15fmDesBKVyymFgEHCRBVjgua5gHh9NsX+NnOe5YKOS+JqJv6r4AsZGYDYJwUaxlk6Y6fMU09RGKJvrxZ020KE+rKApm4Haoj2rQmazGHeCP0vB33/ogo2rrl39eGSfjzh68Rymh8gokQk8JZSSQJqkOHhq5r31BBUIeSCxRz1Si6l7qUIyOSoQ7GpeM+RZDSizmqnXzGoAGM2E32tWszfPakfWPo6vbUhUsWt0jXqq6F3pGiOFKvlgPEKFQGRFfaY2jnNtft0QzyPkerhybqhdQWiVzt14EWso4ZCQKykSL7JrdMsCYFgHhVwWuyhe5KpOEwPmvF8PHIUzS1NLWdbrNSBD8TIZjpn3xvtxzYyJJT8ZvM253TVAr7ExYBOUmO/vHN4fPl8lv6q1YtgfOm1JaAxVeBqmis6s7ODgzKzM4yq+FU01F0OFmqpV1dRCly00VbAlDVtqqmxEBJLW1UuZN9EM11SE++q++1EY9EHhwrSpmDUDc4Nh46yiF3HfD0Rfu1fy6o2WAx++rSyHYZ3loJfshKSlU7YTWojy1vo/2ODvYCgHGdflBoB96hZABvcSkG7HXPX/juJn9nWPVqEjTIJDaiyZJVBVSG40cwwsRSFJK5+cLiLwjaPbF5zAHDTpJz4Kpi56mrGRTLrZl0tON80KPi1TV9XkOo9cV0oyUB2p+WhkgmgaZTJo7JOV+wNRZ3Fw/XkCDcPWd9Ofr8cAmr+P/uzw0X8Vo9+flXdqerTqUX7kVFNUL9/3ljGW1OGUFky7TaIHCsJyxNYgrGRu112xsxslR+HGSbrwB/QgwgCsemwBMEcKBmCNDIOdybDhqdpS+7hoOlSwrS0V7EQ16U3DVn2jj3x8d+NtQZIzd2+toBhVBWVo983cqnM1VYxV+3obHYT9m/Db/+YaMXbiJ8rHsS+kSQ4526pRJOoUYX1gW11BTfUYbwwGOgkguDkfzaf/gqbQm9n9NcPKfw71P1Q7P0Wxvm1Y8GzufRxwg6GE7uFooKmC9NgAzzwtRwU4PAP88wFch6OTBDjoAeDGGeCfD+AQnCbAm6MO3QHcPAP88wFc9uSdCsDVaEj3AP92BvgnBLgNThLgNQGXJNOP4iX7mqQxKjXj7yNHWBIkNuYstoUoqIPoPhlKjAZ5lJBRvtnNd8Ip/YatIjorOzyc6zL6y2kQiSRdE1O8Hf+5bcbD3hxg2bci338HDmiQz581xoid/tDNdO6B5H3uO7yoN68P8AqYJgWrWuVEzvnR63J+cq5o38Q9nuPQLXHSVG6cla0OeMNDPDY/19y5uofgcmPvUd6jDTDgjFCfn6NFGLls3tZ+kOh55pNXfhvGmrwIcdpoM0xZVXHM/vtezFOMyCxvkitM/Cwl7GPKV7PhN+ysKHYP0Xd9bd9XXOUo94PX/r7weKmXPAL7uHq4Y59MfPmH6BHk93hAEWtOrN5MntYT0kYeFtF5xDNPxHExNqKDlH9OVzGbHmL+0ydzz2lagbNPlw1lEL+S0KOEN0kj5CT0W3DaOSRY+jh9nJgiuoqrz/U9JbfHK9y8hyggN9fi+Xhu3AE6a8qdrY5H0fPV0kXJSC4EbAMUvpfwmD4JdgdyE+MFdp7vZuJJxlktaZ5v2/sAPee84iyI5+CckZYRcbC40atHF/mIv+aMJ2sZa0qygm+E4qpEKkmfGvW8glZHcIdgWRFxFn2bRWzG17Xx/V3eccE/TV1htyzNKpLyk6Z7bpHjuaIkTtWFkmbh4xltVD+Y1et44fy7yH24NBqUjt1TGUpaTGdaQ9VRNxrVqAx1i1yBvNLwcEqDpRDvuIta82Tvwyxpbb9Q4ISyymG2hn/jqgGrHm7HSXrR69aSHSKtvOzEPaeVq+zaKF+0AYBVT9LpryrVR31IoA/C4IZWT+/dyMlEH3ovVUiZQqV25rqRF0haRpnyG+sPR5qElKQHB8UNbE4Q2E/6GGfps5ZV1y5qGZlVFefkxQ9Uw/DNRkjZgjqHWz7dChfbUGMuR13hAtWQeTEaHJUtQNeYRNqzk/mDhFm4S+zkYiu6FC7MrLgNscLOrEio7lHw+XW4bVcmGL2uTIBqDPcbOU1VRmbpY6o2sIFIuWGlaaOqZnP6lhU0++DKtr6dj8LNDTs7HYmb1Rjfz5oNl87svH7/LcbOYFTdfmu0HzdnKqVdlRGjagMd8rq6AG3XXDR+Bc8+42kS/BuCPxKTZ5qT+lK+xDSza4ofyVVNAc2tbfPaMEtDCKfBXt8uUS7RY7N9S/WL2qV5mU7IIzbOAkV04GLHixPz5XXhUfy4RELCvLJ6PZo0mhRiGKo7jgELqrqh3VXWDVS9xEkS2YSJWrEN6iEt6XPmzDZGzSwd+lNZktF77kzWnxJI78IY8wW8WQRfDYUyVJHI7VnHAiLl8bwp7I4T9ag6UbfeE1Zq59hbwhrNTvDGaV+dVddkPBaZHkfIOmfnjBtLn8DdhDf7M65Hv43wdgVJ3Glv0ts6MeFt6H3I4BPKFzCOtstgu3AihJKfvRpP7CY+mA1KSTT+QJ66I4fMidNlEA7SwjK9N0qNLcziLQXLsjQ/pxuGlKbs61oZmxZeiF1EQhdFQsq8B1PCH4Cyi/tcCiCtMR9ZefZ0X6EWw+hZcftAzrFMcm/ehKdX55ihru+4ITVv1yioCBqM9x2DDF3sXDWsuqeKV6aUGMauWz3TGcOorsc/V2FuH8mKoximJhP/hDntN7SFoJS+0/4FGbAK2m2toV0VCrnD2X261SdUD+sEi90lNS/g70fqF+RQ68MP0MOu2weHvy6lLLWGv9zQ0Z0BNZ7WbOjvrr4qAN3b0WoaljHazVYfWwCCyW9jq3su64BH35+YSv6EHAfH8VOAQiYuAgH17nZ1Mk/NSG/esGy/dE3YabrmwdenGBulYxMHdgWVoezC1Gtss7rXoXT2NhRT9WpereiCs5Ij1MnmpXJ56C+pkXxuvoDReUEiTqJMnz1P5h9yMgeWvGk5bDeZyw3ZWYjoSJO5qeYlJwsfL84RqU9phZmZR2pfNVRu6NhqaPaagBJyI+EoKC15VqGbLoI+I/fjIdewgIRcsx1y5YZsAxwXub0sRSsAWAHfYGhtehFS6+Wwe+FnC4exuW/Kc1MgSdpSCGjQPi5C1KBSkswk1s23sK/Xrc44x8I3v2aEBAE3rfk4P4nlRzPhyO/OMT/UNGnjH6ABs8ZiOqp1bbZJ9a5EsPcWYzmssoMHRBk9QlHCd8Dfem7sXDod7Z0S7cLlhi3ZMJYlAaQD97bZvHvUfv4Z8GGW0zbKjPZvgd6Uhm5aQ1ihdf6uvtNdVmKqgZDSJlKPySZSEm0rc07tvFgiu0LU9i8wafDQSRPOK54y84PEg18xV7M7mjlMrcrWo5FdN20cc91e5hQ+ssL75tFSZJYd/SydKXRdflAblu1JqzW2nDaMPWeNBpKyw4hw0VowM0fvV+JiXuMf7V1bc6M4Fv41rt15iAsQGPyYOPFOqrp7u9pd1dtPKRlkmwkgD8hxMr9+JXGXwBcwxkmcrmobIQTofOd+JA/AxH/9TwjXq6/YQd5AU5zXAbgfaJqmgBH9YC1vcctIs+KGZeg6cZOaN8zcf1DSqCStG9dBUakjwdgj7rrcaOMgQDYptcEwxNtytwX2ynddwyWSGmY29OTWX65DVnGrZSh5+5/IXa7SO6tKcsaHaeekIVpBB28LTeBhACYhxiT+5r9OkMcmL50X82/sb35PoxvyQt/2EeLfP/68iQebHnNJ9gohCshph9bioV+gt0nma0YgQfz17ZUboOTVyVs6nwvX8ybYwyE9DDDtAO4iEuJnlDYONKDwP3rmBYXEpbS49dxlQM8RvKatCxyQWTKikhxXXc37xYDS0uMp9F2PQfHWh//ggD7bgz9H9Mq75E3oHdGrQPo986ZmxKRcgLCPSPhGr0tGuQHjBAApB+jJ8TbHU9ZnVcCSYSWNMMHwMhs8pxP9kpDqCLIBiWwSnZBD2SA5xCFZ4SUOoPeQt96FeBM4yEmIkPf5ghmV7lXa+Bci5C0hAdwQTJtWxPeSs3RWw7f/seuHpqKnDb+TAfnB/Wvp6K149B2FLp0ORjzeGBEYklvG8rTB9mAUuXbaPHW9/K5O2inBH21Jzh+KEjY5J8FIiDxI3JfyYFXkTi79jl16mwxbYGyWoGUaenmICG9CGyVXFZl7z0CWLkCPTuISEWkgjr7sfZoDUq8A5MgjjHxrGJSQOfp7wyQmp9NNxLF1Szuo6vo1P0m/LdnnF7ykX6HPxEYwj9bxKS/uMo3HjntKDEDFACnjNUT0dnCeyZ01mw0+P8bdwLinLTCRUzYFA4eLKMB813E483hwjrw7aD8vORsVxNeC/5XFl6rvBWadEKwVa4nCTN4nV1N7oQx2iztlqAFtVEJTqoGaoj3tgheLCHWCP0PC33/Jis6rptz+mtH/Zyh8cW1qh4goEQk8x4RgX1BSDDwVem83QTlCvuPIJS6uxNQXoUOsHCUIdqXXdFGrqZJWM7QKraaqej3hW2m10X6tdmbr4/zWhkAVq8LWqKaK1pWtYUpUySZjBiQC4Q3xqNk4yaz5XVO8DKHjotK5kXILgFk4d++GdKCYQwJmpAi8SK/RTFNVR1VQyGSxA6NVZurUMWDG+9XAkTizoFqKsl6rABmM1vF0LNxX9hx31JlYs5P+65L5XUO4jfQhVVBc3z/a7HmYvoq/lXtFoD90WoLQGMnw1A0ZnWnbycGpyg5MNhmpoTAPUxth4uGN8wsSe3Vy3E6Brlvacbi9m6jA+Dy4tdnsb/ns9yddLw2/sic3Y1STvDnPoxehMvFSWlD/J/baJYRliK1AWEHNVV1xtPmSoXCvEZ3r4R68eRWUPSVVNcYSBkCFhgVdKdjUnjqv817ytjNTKne125pGDV1sOg1clOzqZ1RTuKUvfpNROHPGtfIYHfvQquzEzBhBjhMGnIZXcXCoOMhEe6oRrL6lgewF/YTRM4uQNDJa6L8pu71ktKRObBki94oxUU0JIknnizNHeAgHhQ8vKI7kFDmhZHwgO3oibB77QppgOVtmheVRFUPWhpbZFdRk125v1M6OAcEiduFy/m9gcEOb3l/RzezrSPtDDuUlKNYOjd+1ssiv4D4ruNWRgO7ReKjIgvTsALd6ADi4AvzjAVwD44sE+LgHgOtXgH88gAP1IgGeRrXPCnDjCvCPB3Ax9HcpAFd7APi3K8A/IMAt9SIBXpGhiVPyBK3px3QT8DSJnJp/zymZGIm1xQVNIapWQbRNKpHS4GmRUWBHmO+Ca+90S0Z02nZ6OFeV3sXClE1RKkl3JCEfJj8K4rd40ek5wLQeeGHeERxQI58/alIS2f2hm9rcQyH63Hc+Uqsv5HNzmMYNm0rjRCzr06rK+jKuaD7EF7REgVPgpLk4OG3bnPCGp3htdq7+4apegsmN1rPcYgx1yFU1JCgpCP8XVdTKdxhCz+MrBdJ68RBBh536hcPnhYe37JaUTaNKGagN80H4s8VjRJu57xJ2ycOEmQjMnopY158rl7VuIlbBpwRxmo9CmcoOystc4mjKHJEtQkGlaaFAhpV42Opy0vzhALvjA2RFIvwJ6Ad6RfaG8JvHzzffRHQuIvbVw0vX3jVq06nXh9lEJHb7Vxy4BLMhSQjtZ3Z3smITYWN/7aFkHthsbtg5vCgNwJ+edbh/C6CP7+/4+7Hy0RM8rCE+bHk+8iffrB0Yz+SKg8SHwVtK6vxNkDMUh5iskP38uOBvMkl7Cehq+vQ+fOaPBNkTrLBrowwy6xDbiN9o65JVNuPbDOYifOpbKH9LQlRqyCTAN0xQWWAUhEMVxGOQxhiwYcwRMRzCeKYWIVXImjL58pi93e33x+Oebs9LCQZMUoN9QOH1huAoUfkF68BDC1JrQlDP1XaD5U9e8HCj1xgOx9cvFCyRzjR/Odg2Hstq36jKOqtiWf/pFL8hEe+8K0iMwUnXj9RUtBQWj+QrTPL1I+yZ7E34wt9J3YuUQceVLloCk7jwZEdH0LYkph12qgq3iz7RAWJ5VKlUDnWULmKtRwMHovEaDq2G4MnwylAF5WjQ5S/h0Mw+JNB7YXCrJYPzS6nog2+FDglTyNROwy/iagRTL1J+b//RWBGQEj/BaXFTn+RvJ330q/TZyaq7lpAZY6Ns4ly++JFT6fXuTNEXu6ZMmqZMPOjPHZjFn5sDuXURqFEOw1m6nDcxqvImna3vltPe+WwwVDYAXW0haM+B4neSKmGhrIvLj2hCyi+1ofbk+zrzIlOj8zPZcMA40IbTlWpansdJA3Ie9hu+TFNGZOlzmjZgN8MpQ0UZly2by/esgLxM94JjO++Fm7VeuVnO0/2u2N3gys67N7ug7KyOy3tdjNtxc2pSWmUZMS4P0CGvy4vIjq0nY1ewCjIWY2KfQP0jdnnmGalvxEsMI70m/xJfVZeUPNg3r0zY1CSDavz1w4rdYjs23SRMG1Qur0ttQpb7sVcwJEMH2W4Uuy/blUvQbA25hNnSfj26NIqQYhjJ23uoJpBtQ6uryhkgR4njQrApFbU8/3tKT/pa/XKIU7NIpv5SllX0Xv8C5CVxj0GE2CLctBZAToVSVOHQ6dnGUnnZ4nUHtiMV9bisqBtvwCaMc+7910CDlW6yVt1RtZjXjJyhcpye0+9NbQqOE970T78bfxrh7XCSOPPepLd5acJ73IcMvqB6gVTo7HVeYy/n7OlEAIQ4ezmf2E1+UJdj6b+gK++qIXLifO0Hw6SxSO+9UuMAt/hAwbIu6Odk04+Cyr6rlLFJ44DvBBI4MORS5s2fY/YChF7cZzm/sE58bGYV0H2lWvReQtbvMzimawfKF7Um7HKe4Fj6mAWOv8cVW1nnVFRrnPcjkwxdbFc1Koen8v3JCwxjVa2A6Yxh5BUwPzZB5h+JhiOfpjoX/4I57RP6QkAo32m+GzUog/ZQb+hYg0J84PQ+3doTcuh9iviWkorrsx8j6BfkQOkjDlBXTPue4K8JJUuN4S8OdPbN2OUcQDb1j7dfJYC2DrQauqmPj/PVJ6YK1Omn8dVdhz6AS96eqEn+BG0bRdGTDwMqLnwO9e52ZjIuzEnXuyoWB52Wa558fYq+Vzoe/YsALaEyEkOYWoVvVrX3eGdbj+sVe49vyIqxks3NyX1L+bIe8f/7L6B0XuGQkSi1Z6/K/F0qc9Ush5pMAzRT5uJAVhplPJcylyP78RLKwTUj9SG9MCN1MNqaoeJAZzdD5Yr6kAcKCounZegmy6mvyH1/yNVNVUCu0Qy54kCWrp4VuWmwuS8AlsA3HJlGawDWLIdthZ8DAsZGN0CTtgVSFWCdFyHytm1xMRNfXN/Av961OuOaC9//2yLY95lrzeb5iS8/WvBAfneB+ZGiCJv3qIpqVHhMZ/WujV4W0dZJrn1y6xARVUqvdy6vEotpb4LLGJ1CsB2/HtcSvJrEOek04G00qC06KGKjHlpM1HsZe60UaVyYbuxeQku1PvP9i7ROa8gvd52JIWdGCrtBzeLdoATSlpRQpaIsUF2iafNfJakJ2QkaaIvm1B/B0fCviNndHakSQylz9XhsVemRcy7kM/rZDubVJYVULT36XTiTKxF2cIwO6VptaAdqDa2l0mhH0l6q4y6BPMboUPq0LYtr9/On2tV2OweV25pu7Yhcv3/sbmNqkQSBbuOoWZrQoG4eU4xpYAjxwJB4ml3C/z/ILNM+sVk2qhHRhZ1NTCE4daNe/AJgQ865yjt4CsQ9qjiuM9NIk9I4smE0Po1hRA9DzBghn3lmA37FDmI9/g8=7Vtbd+I2EP41nNM+kOMLNuQxhiS9ZNttaZvuo7CFUSJbriQC9NdXsiVfsNdAgI3pYXP2xJoZy9LMfDOjS3r2OFo/UpAsPpEA4p5lBOuePelZlmmOXPFLUjaKMrRGGSWkKFC0gjBF/0JFNBR1iQLIKoKcEMxRUiX6JI6hzys0QClZVcXmBFe/moAQ1ghTH+A69RkFfJFRR45R0H+AKFzoL5uG4kRACysCW4CArEok+75njykhPHuK1mOIpfa0Xn5aoF+eXoL7356Z8fuvrjn6K3nqZ509HPJKPgUKY37aru2s6zeAl0pf003sq/nyjVYiJcs4gLIjo2d7qwXicJoAX3JXwm8EbcEjLFqmeJwjjMcEEyraMYmFkLfnBNRE3yDlcF0yn5rQIyQR5HQjRBTX0WZU3tl3VXtVmNocKtqiZGbbUH4NlHuFed+FCsWD0uIBGrVqGhXqdLH4rscSEFf06v6zlM7jzUnM+3MQISzmcdeT3uWCKEkVZ9sDqV6I3yBHPqhxUulx0zuAIoC/Ls9AzPoMUjSXpJSRjoOlEJajMJJ1zkpt3meZ0SUzJqnl8zmIpzD77WyHE0eoT1JTqOQtrU4nVaigTOSzHJ0jNegIK+ySNXNZ7Vvv6sYqusnMlnMKRma0nKEBIQnyUxVQSGKmHEcBQ1LMtFmAQ9JSeEhyaQSZ8+8zx5yVT69AiCMxUki6hT6MTUHvl+kZZop3hiWewk7OtM0SU2AoZ4SlEWyrP23mNigTq56h5GoulKFIoDIDkva3K7iu4LqC60zgupBCwHbfWwjoOvWYQsB6eZmuX7+4U+8O/Tx8Mf98flzoOZSUBwNRmaomoXxBQhIDfF9Qvap6C5knQhKl1BfI+UaV2WDJSVXlQq1087d8/8ZydPuL6i9tTNaV1ka1DjMMI0vqwxa5gar0AQ1ha3+qDJOqabUzhRhw9FYt6ptslr56RynYlAQSgmLOSj1/loTCfayBU3Ef29iqpbfkB2arvHjIRlB4Tz6V91eWg1pleTHwND8Uno1zcHZrczde/SV9S9VtngK8TMCF38k1ryD4GDCGfE1+QLjAeKCFlNkERfHPAOXhnlAenRrJR5nX3W9hK/yXV41AoajRwCwVkNpUoUNIO17PmcjSD6MwlhYS+oUCPJ7Egage8Z1iRCgIUvfAYAaxB/zXMPUNDTZRLc7Tf/sbS3vrNuLyfRM14srWRBMSjRtzqDo7LKQWMVCLkPmcCY/YttQJIt2wZrz78bQl2JndCnajarAz7YZgZzUEO/dcsW5UB8NyFiEu3fdStdo3P1qrt9cMsl8GMfW+785qcNipHKI97P+SRLTDHp1E+iKLWNUKWO8kdzepaC+80Kxidy6rmNd4176o3RnvrG6Fu0G37KlMVTamUTVmYfHCnua3sWfjlo9xans27z64t9VQYOtYrLvIBqreOkMkdbrlKJcHfPssjnLortdg24/M9l0v13ba5M+z62XWV/KfCeP9hBIfMkYuZwVjj7bU53x4Bq8vtHcepzWcVdEoPQrbOqsqLTEPPVvomt26tvI06wv6a0RulrvdtxQ7ee4+zsL1zYVLWqw4nVusWPW1/DXU7bTbx4c6jbCuRLbOLEqsfY9YB50KbB3IVKUz8oOOyC/cD7q12WDZVz/4GD84+drzOD9outVwzcs7lo79/LLEt0jMjXtd74njl7/IaNv22328dXtq6L1r22e4dXlJ+8hXt32GRpv80ds+bTo9JCz4GRZlQKDh7DvRg7qMW3r6vh4kJpsYRGQi7+H2xf9nQl/nWPiOvtsbz1h2xbf1+vGMnnJM55rrJ8hBADgQ7D/kYR/bNzCSJccohuP8j3QkjOSN5tJxpPh5kBb2QgoCBCu8wWQomCXeBFHRESJxilEqY1gloMob1bY98G5TeFPyCpsOPgPAFnk80SeoT/LA9DNhSHU/I5yTqFc/YuUy6tRPYsshXsxQBSPT0m2lFflJwJJMHXO0luPw2AIkkhmtQ3nB9gas2OCGwiyE/OjL8cjD4eypKhWkfhjMzphGagcHRj2LOA1JxDk8ifTUveNSUChuHNv3/wE= -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this 4 | software and associated documentation files (the "Software"), to deal in the Software 5 | without restriction, including without limitation the rights to use, copy, modify, 6 | merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 10 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 11 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 12 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 13 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 14 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Amazon ECS and AWS Step Functions Design Patterns Starter kit 2 | 3 | This starter kit demonstrates how to run [Amazon Elastic Container Service](https://aws.amazon.com/ecs/) (ECS) tasks using [AWS Step Functions](https://aws.amazon.com/step-functions/). We will implement the following design patterns: 4 | 5 | 1. Running ECS tasks using AWS Lambda 6 | 1. Running ECS tasks using Step Functions native integration 7 | 8 | We use [AWS Cloud Development Kit](https://aws.amazon.com/cdk/) (CDK) to deploy application resources. 9 | 10 | --- 11 | 12 | ## Contents 13 | 14 | * [Prerequisites](#prerequisites) 15 | * [Overview](#Overview) 16 | * [ECS Task Business Logic](#ecs-task-business-logic) 17 | * [Amazon DynamoDB Tables](#Amazon-dynamoDB-tables) 18 | * [Workflow Specification](#Workflow-Specification) 19 | * [AWS CDK Stacks](#aws-cdk-stacks) 20 | * [Patterns](#Patterns) 21 | * [Pattern 1: Running ECS tasks using AWS Lambda](#Running-ECS-tasks-using-aws-lambda) 22 | * [Pattern 2: Running ECS tasks using Step Functions native integration](#Running-ECS-tasks-using-Step-Functions-native-integration) 23 | * [Build](#build) 24 | * [Deploy](#deploy) 25 | * [Test](#Test) 26 | * [Pattern 1: Testing ECS tasks using AWS Lambda](#Testing-ECS-tasks-using-aws-lambda) 27 | * [Pattern 2: Testing ECS tasks using Step Functions native integration](#Testing-ECS-tasks-using-Step-Functions-native-integration) 28 | * [Cleanup](#cleanup) 29 | * [Contributors](#contributors) 30 | 31 | --- 32 | 33 | ## Prerequisites 34 | 35 | 1. Docker software is installed on your MacBook / Laptop 36 | 1. Docker daemon is running 37 | 1. You have AWS account credentials 38 | 39 | --- 40 | 41 | ## Overview 42 | 43 | ### ECS Task Business Logic 44 | 45 | We run a simple business logic within an ECS task. It creates a copy of the input file in S3 bucket. We will run multiple instances of the task simultaneously. 46 | 47 | --- 48 | 49 | ### Amazon DynamoDB Tables 50 | 51 | Each pattern requires 2 DynamoDB tables. They are workflow_summary and workflow_details. workflow_summary is used to audit the status of overall workflow execution status. workflow_details is used to audit the status of individual ECS tasks. The schema of the DynamoDB tables is described in the below table. 52 | 53 | | Table | Schema | Capacity | 54 | |----------| ------ | ----------- | 55 | | workflow_summary_pattern_x | Partition key = workflow_name (String), Sort key = workflow_run_id (Number) | Provisioned read capacity units = 5, Provisioned write capacity units = 5 | 56 | | workflow_details_pattern_y | Partition key = workflow_run_id (Number), Sort key = ecs_task_id (String) | Provisioned read capacity units = 5, Provisioned write capacity units = 5 | 57 | 58 | **Note:** here, x and y represent either 1 or 2. 59 | 60 | --- 61 | 62 | ### Workflow Specification 63 | 64 | We create 2 Step Functions State machines to demonstrate the design patterns. State machine is executed with a JSON specs as an input. The specs have two parts - 1) values for ECS cluster, DynamoDB tables, subnets, security groups, S3 bucket etc. 2) list of ECS tasks to run. Table below describes the specs. 65 | 66 | | JSON Attribute | Description | 67 | |------------------- | ------------- | 68 | | region | AWS region used | 69 | | s3BucketName | Amazon S3 bucket used demonstrate ECS Task Business Logic | 70 | | subnetIdLiteral | List of Subnet Ids separated by a separator | 71 | | separator | The separator used in subnetIdLiteral | 72 | | workflowName | Name of the workflow name for e.g. ```amazon_ecs_starter_kit-pattern-1``` | 73 | | securityGroupId | The security group id used to run ECS tasks | 74 | | ddbTableNameWFSummary | Name of the DynamoDB table for workflow summary | 75 | | hashKeyWFSummary | The hash key of workflow summary table | 76 | | rangeKeyWFSummary | The sort key of workflow summary table | 77 | | ddbTableNameWFDetails | Name of the DynamoDB table for workflow details | 78 | | hashKeyWFDetails | The hash key of workflow details table | 79 | | rangeKeyWFDetails | The sort key of workflow details table | 80 | | clusterName | Name of the ECS cluster | 81 | | containerName | Name of the container | 82 | | taskDefinition | Name of the ECS task definition name | 83 | | taskList | It has specs for one more ECS tasks. These specs drive the business logic of a task. Each task has three attributes - 1) taskName (Name of the ECS task) 2) s3BucketName (S3 bucket name) 3) objectKey (Object key) | 84 | 85 | --- 86 | 87 | ### AWS CDK Stacks 88 | 89 | [CdkApp](./amazon-ecs-java-starter-kit-cdk/src/main/java/software/aws/ecs/java/starterkit/cdk/CdkApp.java) runs the following stacks 90 | 91 | | Stack Name | Purpose | 92 | |---------------| --------- | 93 | | [ECSTaskSubmissionFromLambdaPattern](./amazon-ecs-java-starter-kit-cdk/src/main/java/software/aws/ecs/java/starterkit/cdk/ECSTaskSubmissionFromLambdaPattern.java) | This stack provisions resources needed to demonstrate Pattern 1 | 94 | | [ECSTaskSubmissionFromStepFunctionsPattern](./amazon-ecs-java-starter-kit-cdk/src/main/java/software/aws/ecs/java/starterkit/cdk/ECSTaskSubmissionFromStepFunctionsPattern.java) | This stack provisions resources needed to demonstrate Pattern 2 | 95 | 96 | --- 97 | 98 | ## Patterns 99 | 100 | ### Running ECS tasks using AWS Lambda 101 | 102 | As show in the below figure, this pattern (Pattern 1) uses AWS Lambda function to run ECS tasks. We call the Lambda function as **ECS Task Launcher**. It parses workflow specs, submits ECS tasks to ECS Cluster and invokes second AWS Lambda function called **ECS Task Monitor**. 103 | 104 | ECS Task Monitor tracks the completion status of running ECS tasks. Each time it runs, it checks the number of completed tasks versus the total number of tasks submitted and updates the DynamoDB table **workflow_summary**. 105 | 106 | The task executed on ECS cluster is called **ECS Task**. It takes the following actions - 1) reads input parameters 2) inserts a record in DynamoDB table for auditing 3) copies the input file to a target folder 4) marks the status of its job to Complete in the the DynamoDB table **workflow_detail**. 107 | 108 | ![Alt](./resources/Amazon_ECS_Java_Starter_Kit-Architecture_Pattern_1.png) 109 | 110 | --- 111 | 112 | ### Running ECS tasks using Step Functions native integration 113 | 114 | As shown in the below figure, this pattern (Pattern 2) uses AWS Step Functions' native integration with Amazon ECS. Unlike the usage of a Lambda function in Pattern 1, we use [Parallel state](https://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-parallel-state.html) to run ECS tasks. The number of tasks run depends on the size of ```"taskList":[]``` in [workflow_specs_pattern_2.json](./amazon-ecs-java-starter-kit-cdk/workflow_specs_pattern_2.json). The role of ECS Task Monitor and the way ECS Task executes are similar to Pattern 1. 115 | 116 | ![Alt](./resources/Amazon_ECS_Java_Starter_Kit-Architecture_Pattern_2.png) 117 | 118 | --- 119 | 120 | ## Build 121 | 122 | 1. Clone this repository to your Mac/Laptop 123 | 124 | 1. Open your IDE for e.g. [Eclipse](https://www.eclipse.org/) or [Spring Tools](https://spring.io/tools) or [Intellij IDEA](https://www.jetbrains.com/idea/) 125 | 126 | 1. Import the project as a Maven project by pointing to ```/Amazon-ecs-java-starter-kit/pom.xml``` | This imports 4 module projects. 127 | 128 | 1. Select parent project **Amazon-ecs-java-starter-kit** and build it using the below instructions 129 | 130 | 1. Using standalone Maven, go to project home directory and run command ```mvn -X clean install``` 131 | 1. From Eclipse or STS, run command ```-X clean install```. Navigation: Project right click --> Run As --> Maven Build (Option 4) 132 | 133 | 1. Expected output 1: In your IDE, you will see the following output 134 | 135 | ```bash 136 | [INFO] Reactor Summary for amazon-ecs-java-starter-kit 1.0: 137 | [INFO] 138 | [INFO] amazon-ecs-java-starter-kit ........................ [SUCCESS [ 0.717 s] 139 | [INFO] amazon-ecs-java-starter-kit-cdk .................... [SUCCESS [ 14.230 s] 140 | [INFO] amazon-ecs-java-starter-kit-tasklauncher ........... [SUCCESS [ 8.418 s] 141 | [INFO] amazon-ecs-java-starter-kit-task ................... [SUCCESS [ 21.857 s] 142 | [INFO] amazon-ecs-java-starter-kit-taskmonitor ............ [SUCCESS [ 4.587 s] 143 | [INFO] ------------------------------------------------------------------------ 144 | [INFO] BUILD SUCCESS 145 | [INFO] ------------------------------------------------------------------------ 146 | [INFO] Total time: 49.979 s 147 | [INFO] Finished at: 2020-12-21T13:03:30-06:00 148 | ``` 149 | 150 | 1. Expected output 2: Build process generates the following jar file in their respective directories 151 | 152 | | Module artifact name | Approximate Size | 153 | |--------------------------------------------------------|-------| 154 | | ```amazon-ecs-java-starter-kit-cdk-1.0.jar``` | 32 KB | 155 | | ```amazon-ecs-java-starter-kit-tasklauncher-1.0.jar``` | 21 MB | 156 | | ```amazon-ecs-java-starter-kit-task-1.0.jar``` | 19 MB | 157 | | ```amazon-ecs-java-starter-kit-taskmonitor-1.0.jar``` | 21 MB | 158 | 159 | --- 160 | 161 | ## Deploy 162 | 163 | 1. In the terminal, go to path ```//amazon-ecs-and-aws-step-functions-design-patterns-starter-kit/amazon-ecs-java-starter-kit-cdk```. Now, you are in the CDK module of this project. 164 | 165 | 1. Replace **1234567890** with your AWS Account Id wherever applicable in the following steps. 166 | 167 | 1. Set these to your account and region 168 | 169 | ```bash 170 | export AWS_ACCOUNT_ID=1234567890 171 | export AWS_REGION=us-east-2 172 | ``` 173 | 174 | 1. Bootstrap CDK 175 | 176 | ```bash 177 | cdk bootstrap aws://${AWS_ACCOUNT_ID}/$AWS_REGION 178 | ``` 179 | 180 | 1. Output 1: In the command line, you will get the following output 181 | 182 | ```bash 183 | (node:63268) ExperimentalWarning: The fs.promises API is experimental 184 | ⏳ Bootstrapping environment aws://AWS_ACCOUNT_ID/us-west-2... 185 | ✅ Environment aws://AWS_ACCOUNT_ID/us-west-2 bootstrapped (no changes). 186 | ``` 187 | 188 | 1. Output 2: In the AWS console under CloudFormation, you will see a Stack created as follows 189 | 190 | ![Alt](./resources/output_of_bootstrap.png) 191 | 192 | 1. Output 3: In the AWS console under S3, you will see a bucket created with name ```cdktoolkit-stagingbucket-*``` 193 | 194 | 1. Deploy both stacks 195 | 196 | ```bash 197 | cdk deploy --require-approval never --all --outputs-file outputs.json 198 | ``` 199 | 200 | 1. Expected output 1: Stack for **amazon-ecs-java-starter-pattern-1** created with the following resources: 201 | 202 | | Resource Type | Resource Details | 203 | |---------------|-------------------| 204 | | VPC | 1 VPC to launch resources needed by the starter kit | 205 | | Subnet | 2 public subnets and 2 private subnets | 206 | | Route Table | 1 route table per public, and private subnet | 207 | | Security Group | 1 security group per ECR, ECS, and ECS Agent endpoints | 208 | | Security Group | 1 security group per ECS Task Launcher and ECS Task Monitor | 209 | | VPC Endpoint | 1 VPC endpoint per Amazon DynamoDB, Amazon S3, Amazon ECS, Amazon ECS Agent, and Amazon ECR API. | 210 | | ECS Cluster | 1 ECS cluster to run ECS tasks | 211 | | ECR Repository | 1 ECR repository to store Docker image for ECS Task binary | 212 | | ECS Task Definition | 1 ECS task definition for ECS Task | 213 | | Amazon DynamoDB | 2 tables. Refer to [Amazon DynamoDB Tables](#Amazon-dynamoDB-tables) for more details | 214 | | Step Functions state machine | 1 State machine for orchestration | 215 | | AWS Lambda | Lambda Function to submit ECS tasks | 216 | | AWS Lambda | Lambda Function to monitor the progress of ECS tasks | 217 | | Amazon IAM Role | 1 IAM role per Step Functions State machine, ECS Task Launcher, ECS Task Monitor. 2 IAM roles for ECS Task Definition - 1) ECS Task Role 2) ECS Task Execution Role | 218 | 219 | 1. Expected output 2: Stack for **amazon-ecs-java-starter-pattern-2** created with the following resources: 220 | 221 | | Resource Type | Resource Details | 222 | |---------------|-------------------| 223 | | VPC | 1 VPC to launch resources needed by the starter kit | 224 | | Subnet | 2 public subnets and 2 private subnets | 225 | | Route Table | 1 route table per public, and private subnet | 226 | | Security Group | 1 security group per ECR, ECS, and ECS Agent endpoints | 227 | | Security Group | 1 security group per ECS Task Launcher and ECS Task Monitor | 228 | | VPC Endpoint | 1 VPC endpoint per Amazon DynamoDB, Amazon S3, Amazon ECS, Amazon ECS Agent, and Amazon ECR API. | 229 | | ECS Cluster | 1 ECS cluster to run ECS tasks | 230 | | ECR Repository | 1 ECR repository to store Docker image for ECS Task binary | 231 | | ECS Task Definition | 1 ECS task definition for ECS Task | 232 | | Amazon DynamoDB | 2 tables. Refer to [Amazon DynamoDB Tables](#Amazon-dynamoDB-tables) for more details | 233 | | Step Functions state machine | 1 State machine for orchestration | 234 | | AWS Lambda | Lambda Function to monitor the progress of ECS tasks | 235 | | Amazon IAM Role | 1 IAM role per Step Functions State machine, and ECS Task Launcher. 2 IAM roles for ECS Task Definition - 1) ECS Task Role 2) ECS Task Execution Role | 236 | 237 | 1. Expected output 3: A file [outputs.json](./amazon-ecs-java-starter-kit-cdk/outputs.json) is created with a list of AWS resource values provisioned by the CDK. 238 | 239 | --- 240 | 241 | ## Test 242 | 243 | ### Testing ECS tasks using AWS Lambda 244 | 245 | 1. Open file [workflow_specs_pattern_1.json](./amazon-ecs-java-starter-kit-cdk/workflow_specs_pattern_1.json) in your IDE and update JSON attributes based on the values for ```amazon-ecs-java-starter-pattern-1``` of ```outputs.json```. 246 | 247 | 1. Open your command prompt / Mac terminal 248 | 249 | 1. Go to path ```//amazon-ecs-and-aws-step-functions-design-patterns-starter-kit/amazon-ecs-java-starter-kit-tasklauncher/``` 250 | 251 | 1. Copy jar file to S3 bucket. Use the following command 252 | 253 | ```bash 254 | aws s3 cp ../amazon-ecs-java-starter-kit-tasklauncher/target/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar s3://${AWS_ACCOUNT_ID}-amazon-ecs-java-starter-kit-pattern-1-bucket/amazon_ecs_java_starter_kit_jar/ 255 | ``` 256 | 257 | 1. Go to path ```//amazon-ecs-and-aws-step-functions-design-patterns-starter-kit/amazon-ecs-java-starter-kit-cdk``` 258 | 259 | 1. Start Step Functions execution. Use the following command 260 | 261 | ```bash 262 | aws stepfunctions start-execution --state-machine-arn "arn:aws:states:${AWS_REGION}:${AWS_ACCOUNT_ID}:stateMachine:amazon-ecs-java-starter-kit-pattern-1" --input "$(cat workflow_specs_pattern_1.json)" 263 | ``` 264 | 265 | 1. Expected output 1: You will get a response as follows 266 | 267 | ```bash 268 | { 269 | "executionArn": "arn:aws:states:us-east-2:1234567890:execution:amazon-ecs-java-starter-kit-pattern-1:4ea1f256-a0bb-4692-b63a-6b80edc02cb7", 270 | "startDate": "2020-12-21T09:54:29.385000-06:00" 271 | } 272 | ``` 273 | 274 | 1. Expected output 2: In Step Functions console, state machine ```amazon-ecs-java-starter-kit-pattern-1``` changes to Running state 275 | 276 | 1. Expected output 3: After a minute or two, your State machine execution status will be successful as follows. 277 | 278 | ![Alt](./resources/Pattern_1_execution_outcome.png) 279 | 280 | 1. Expected output 4: In S3 bucket, you will see 10 extra jar files which are copied by ECS task instances. 281 | 282 | ![Alt](./resources/pattern_1_test_output.png) 283 | 284 | 1. Expected output 5: In DynamoDB tables, you will find new items as follows: 285 | 286 | | Table | Number of items | Sample column values | 287 | |----------| ----------------| ------------- | 288 | | workflow_summary_pattern_1 | 1 | number_of_tasks = 10, completed_tasks = 10 | 289 | | workflow_details_pattern_1 | 10 | status = Completed | 290 | 291 | ### Testing ECS tasks using Step Functions native integration 292 | 293 | 1. Open file [workflow_specs_pattern_2.json](./amazon-ecs-java-starter-kit-cdk/workflow_specs_pattern_2.json) in your IDE and update JSON attributes based on the values for ```amazon-ecs-java-starter-pattern-2``` of ```outputs.json```. 294 | 295 | 1. Open your command prompt or Mac terminal 296 | 297 | 1. Go to path ```//amazon-ecs-and-aws-step-functions-design-patterns-starter-kit/amazon-ecs-java-starter-kit-tasklauncher/``` 298 | 299 | 1. Copy jar file to S3 bucket. Use the following command 300 | 301 | ```bash 302 | aws s3 cp ../amazon-ecs-java-starter-kit-tasklauncher/target/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar s3://${AWS_ACCOUNT_ID}-amazon-ecs-java-starter-kit-pattern-2-bucket/amazon_ecs_java_starter_kit_jar/ 303 | ``` 304 | 305 | 1. Go to path ```//amazon-ecs-and-aws-step-functions-design-patterns-starter-kit/amazon-ecs-java-starter-kit-cdk``` 306 | 307 | 1. Start Step Functions execution. Use the following command 308 | 309 | ```bash 310 | aws stepfunctions start-execution --state-machine-arn "arn:aws:states:${AWS_REGION}:${AWS_ACCOUNT_ID}:stateMachine:amazon-ecs-java-starter-kit-pattern-1" --input "$(cat workflow_specs_pattern_2.json)" 311 | 312 | 1. Expected outputs are similar to Pattern 1 313 | 314 | --- 315 | 316 | ## Cleanup 317 | 318 | 1. Go to ```//Amazon-ecs-java-starter-kit/amazon-ecs-java-starter-kit-cdk``` 319 | 320 | 1. Delete DynamoDB tables 321 | 322 | ```bash 323 | ./delete_ddb_items.sh workflow_details_pattern_1 workflow_summary_pattern_1 324 | ``` 325 | 326 | ```bash 327 | ./delete_ddb_items.sh workflow_details_pattern_2 workflow_summary_pattern_2 328 | ``` 329 | 330 | 1. Empty S3 buckets 331 | 332 | ```bash 333 | aws s3 ls s3://${AWS_ACCOUNT_ID}-amazon-ecs-java-starter-kit-pattern-1-bucket/amazon_ecs_java_starter_kit_jar/ | grep _ | awk '{print $NF}' | while read OBJ; do aws s3 rm s3://${AWS_ACCOUNT_ID-}amazon-ecs-java-starter-kit-pattern-2-bucket/amazon_ecs_java_starter_kit_jar/$OBJ;done 334 | ``` 335 | 336 | ```bash 337 | aws s3 ls s3://${AWS_ACCOUNT_ID-}-amazon-ecs-java-starter-kit-pattern-2-bucket/amazon_ecs_java_starter_kit_jar/ | grep _ | awk '{print $NF}' | while read OBJ; do aws s3 rm s3://${AWS_ACCOUNT_ID-}amazon-ecs-java-starter-kit-pattern-2-bucket/amazon_ecs_java_starter_kit_jar/$OBJ;done 338 | ``` 339 | 340 | 1. Delete S3 buckets 341 | 342 | ```bash 343 | aws s3 rm s3://${AWS_ACCOUNT_ID-}amazon-ecs-java-starter-kit-pattern-1-bucket/amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar 344 | ``` 345 | 346 | ```bash 347 | aws s3 rm s3://${AWS_ACCOUNT_ID-}amazon-ecs-java-starter-kit-pattern-2-bucket/amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar 348 | ``` 349 | 350 | 1. Delete ECR Repositories 351 | 352 | ```bash 353 | aws ecr delete-repository --force --repository-name amazon-ecs-java-starter-kit-pattern-1 354 | ``` 355 | 356 | ```bash 357 | aws ecr delete-repository --force --repository-name amazon-ecs-java-starter-kit-pattern-2 358 | ``` 359 | 360 | 1. Cleanup stacks 361 | 362 | ```bash 363 | cdk destroy --force --all 364 | ``` 365 | 366 | --- 367 | 368 | ## Contributors 369 | 370 | 1. **Sarma Palli**, Senior DevOps Cloud Architect, Amazon Web Services 371 | 1. **Ravi Itha**, Senior Big Data Consultant, Amazon Web Services 372 | 373 | --- 374 | 375 | ## License Summary 376 | 377 | This sample code is made available under the MIT-0 license. See the LICENSE file. 378 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-cdk/.gitignore: -------------------------------------------------------------------------------- 1 | .classpath.txt 2 | target 3 | .classpath 4 | .project 5 | .idea 6 | .settings 7 | .vscode 8 | *.iml 9 | 10 | # CDK asset staging directory 11 | .cdk.staging 12 | cdk.out 13 | package.json 14 | package-lock.json 15 | outputs.json 16 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-cdk/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "mvn -q -e compile exec:java", 3 | "context": { 4 | "@aws-cdk/core:enableStackNameDuplicates": "true", 5 | "aws-cdk:enableDiffNoFail": "true", 6 | "@aws-cdk/core:stackRelativeExports": "true", 7 | "@aws-cdk/aws-ecr-assets:dockerIgnoreSupport": true 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-cdk/delete_ddb_items.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | set -x 4 | 5 | export AWS_PAGER="" 6 | 7 | for TABLE_NAME in $@ 8 | do 9 | HASH_KEY=$(aws dynamodb describe-table \ 10 | --table-name $TABLE_NAME \ 11 | --output text \ 12 | --query 'Table.KeySchema[?KeyType==`HASH`].AttributeName') 13 | RANGE_KEY=$(aws dynamodb describe-table \ 14 | --table-name $TABLE_NAME \ 15 | --output text \ 16 | --query 'Table.KeySchema[?KeyType==`RANGE`].AttributeName') 17 | 18 | aws dynamodb scan \ 19 | --attributes-to-get $HASH_KEY $RANGE_KEY \ 20 | --table-name $TABLE_NAME --query "Items[*]" \ 21 | | jq --compact-output '.[]' \ 22 | | tr '\n' '\0' \ 23 | | xargs -0 -t -P 5 -I keyItem \ 24 | aws dynamodb delete-item --table-name $TABLE_NAME --key=keyItem 25 | done 26 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-cdk/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | software.aws.ecs.samples 7 | amazon-ecs-and-aws-step-functions-design-patterns-starter-kit 8 | 1.0 9 | 10 | amazon-ecs-java-starter-kit-cdk 11 | amazon-ecs-java-starter-kit-cdk 12 | 13 | 14 | UTF-8 15 | 1.75.0 16 | 17 | 18 | 19 | 20 | 21 | org.apache.maven.plugins 22 | maven-compiler-plugin 23 | 3.8.1 24 | 25 | 1.8 26 | 1.8 27 | 28 | 29 | 30 | 31 | org.codehaus.mojo 32 | exec-maven-plugin 33 | 3.0.0 34 | 35 | software.aws.ecs.java.starterkit.cdk.CdkApp 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | software.amazon.awscdk 45 | core 46 | ${cdk.version} 47 | 48 | 49 | software.amazon.awscdk 50 | ec2 51 | ${cdk.version} 52 | 53 | 54 | software.amazon.awscdk 55 | s3 56 | ${cdk.version} 57 | 58 | 59 | software.amazon.awscdk 60 | dynamodb 61 | ${cdk.version} 62 | 63 | 64 | software.amazon.awscdk 65 | lambda 66 | ${cdk.version} 67 | 68 | 69 | software.amazon.awscdk 70 | ecs 71 | ${cdk.version} 72 | 73 | 74 | software.amazon.awscdk 75 | stepfunctions 76 | ${cdk.version} 77 | 78 | 79 | software.amazon.awscdk 80 | stepfunctions-tasks 81 | ${cdk.version} 82 | 83 | 84 | 102 | 103 | 104 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-cdk/scratchpad.sh: -------------------------------------------------------------------------------- 1 | # CLI commands 2 | 3 | # Set these to your account and region 4 | export AAWS_ACCOUNT_ID=1234567890 5 | export AWS_REGION=us-east-2 6 | 7 | # BootStrap CDK 8 | cdk bootstrap aws://${AAWS_ACCOUNT_ID}/$AWS_REGION 9 | 10 | # Deploy both stacks 11 | cdk deploy --require-approval never --all --outputs-file outputs.json 12 | 13 | # Edit workflow_specs_pattern_1.json and workflow_specs_pattern_2.json based on outputs.json 14 | 15 | # Copy jar files to S3 Buckets 16 | aws s3 cp ../amazon-ecs-java-starter-kit-tasklauncher/target/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar s3://${AWS_ACCOUNT_ID}-amazon-ecs-java-starter-kit-pattern-1-bucket/amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar 17 | aws s3 cp ../amazon-ecs-java-starter-kit-tasklauncher/target/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar s3://${AWS_ACCOUNT_ID}-amazon-ecs-java-starter-kit-pattern-2-bucket/amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar 18 | 19 | # Launch stepfunctions 20 | aws stepfunctions start-execution --state-machine-arn "arn:aws:states:${AWS_REGION}:${AWS_ACCOUNT_ID}:stateMachine:amazon-ecs-java-starter-kit-pattern-1" --input "$(cat workflow_specs_pattern_1.json )" 21 | aws stepfunctions start-execution --state-machine-arn "arn:aws:states:${AWS_REGION}:${AWS_ACCOUNT_ID}:stateMachine:amazon-ecs-java-starter-kit-pattern-2" --input "$(cat workflow_specs_pattern_2.json )" 22 | 23 | # Sample output 24 | 25 | REGION}:${AWS_ACCOUNT_ID}:stateMachine:amazon-ecs-java-starter-kit-pattern-1" --input "$(cat workflow_specs_pattern_1.json )" 26 | { 27 | "executionArn": "arn:aws:states:us-east-2:1234567890:execution:amazon-ecs-java-starter-kit-pattern-1:4ea1f256-a0bb-4692-b63a-6b80edc02cb7", 28 | "startDate": "2020-12-21T09:54:29.385000-06:00" 29 | } 30 | 31 | { 32 | "executionArn": "arn:aws:states:us-east-2:1234567890:execution:amazon-ecs-java-starter-kit-pattern-2:17e02b1f-636b-4ffc-b061-96c9ab8e27db", 33 | "startDate": "2020-12-21T10:16:30.717000-06:00" 34 | } 35 | 36 | 37 | 38 | # ============= CLEANUP ============= 39 | 40 | # Cleanup DDB Tables (if needed) 41 | ./delete_ddb_items.sh workflow_details_pattern_1 workflow_summary_pattern_1 42 | ./delete_ddb_items.sh workflow_details_pattern_2 workflow_summary_pattern_2 43 | 44 | # Cleanup S3 Buckets of copied objects 45 | aws s3 ls s3://${AWS_ACCOUNT_ID}-amazon-ecs-java-starter-kit-pattern-1-bucket/amazon_ecs_java_starter_kit_jar/ | grep _ | awk '{print $NF}' | while read OBJ; do aws s3 rm s3://${AWS_ACCOUNT_ID-}amazon-ecs-java-starter-kit-pattern-2-bucket/amazon_ecs_java_starter_kit_jar/$OBJ;done 46 | aws s3 ls s3://${AWS_ACCOUNT_ID-}-amazon-ecs-java-starter-kit-pattern-2-bucket/amazon_ecs_java_starter_kit_jar/ | grep _ | awk '{print $NF}' | while read OBJ; do aws s3 rm s3://${AWS_ACCOUNT_ID-}amazon-ecs-java-starter-kit-pattern-2-bucket/amazon_ecs_java_starter_kit_jar/$OBJ;done 47 | 48 | # Cleanup S3 Buckets 49 | aws s3 rm s3://${AWS_ACCOUNT_ID-}amazon-ecs-java-starter-kit-pattern-1-bucket/amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar 50 | aws s3 rm s3://${AWS_ACCOUNT_ID-}amazon-ecs-java-starter-kit-pattern-2-bucket/amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar 51 | 52 | # Cleanup stacks 53 | cdk destroy --force --all 54 | 55 | # Delete ECR Repositories 56 | aws ecr delete-repository --force --repository-name amazon-ecs-java-starter-kit-pattern-1 57 | aws ecr delete-repository --force --repository-name amazon-ecs-java-starter-kit-pattern-2 58 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-cdk/src/main/java/software/aws/ecs/java/starterkit/cdk/CdkApp.java: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | package software.aws.ecs.java.starterkit.cdk; 5 | 6 | import software.amazon.awscdk.core.App; 7 | 8 | /** 9 | * 10 | * This CDK Application runs CloudFormation stacks. This is the entry point of 11 | * the application. 12 | * 13 | * @author Sarma Palli, Senior DevOps Cloud Architect 14 | * 15 | */ 16 | public class CdkApp { 17 | public static void main(final String[] args) { 18 | App app = new App(); 19 | new ECSTaskSubmissionFromLambdaPattern(app, "amazon-ecs-java-starter-pattern-1"); 20 | new ECSTaskSubmissionFromStepFunctionsPattern(app, "amazon-ecs-java-starter-pattern-2"); 21 | app.synth(); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-cdk/src/main/java/software/aws/ecs/java/starterkit/cdk/ECSTaskSubmissionFromLambdaPattern.java: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | package software.aws.ecs.java.starterkit.cdk; 5 | 6 | import java.util.ArrayList; 7 | import java.util.Collections; 8 | import java.util.HashMap; 9 | 10 | import software.amazon.awscdk.core.CfnOutput; 11 | import software.amazon.awscdk.core.Construct; 12 | import software.amazon.awscdk.core.Duration; 13 | import software.amazon.awscdk.core.Fn; 14 | import software.amazon.awscdk.core.RemovalPolicy; 15 | import software.amazon.awscdk.core.Stack; 16 | import software.amazon.awscdk.core.StackProps; 17 | import software.amazon.awscdk.services.dynamodb.Attribute; 18 | import software.amazon.awscdk.services.dynamodb.AttributeType; 19 | import software.amazon.awscdk.services.dynamodb.Table; 20 | import software.amazon.awscdk.services.ec2.GatewayVpcEndpointAwsService; 21 | import software.amazon.awscdk.services.ec2.GatewayVpcEndpointOptions; 22 | import software.amazon.awscdk.services.ec2.InterfaceVpcEndpointAwsService; 23 | import software.amazon.awscdk.services.ec2.InterfaceVpcEndpointOptions; 24 | import software.amazon.awscdk.services.ec2.SecurityGroup; 25 | import software.amazon.awscdk.services.ec2.SubnetSelection; 26 | import software.amazon.awscdk.services.ec2.SubnetType; 27 | import software.amazon.awscdk.services.ec2.Vpc; 28 | import software.amazon.awscdk.services.ecr.assets.DockerImageAsset; 29 | import software.amazon.awscdk.services.ecs.AwsLogDriver; 30 | import software.amazon.awscdk.services.ecs.AwsLogDriverProps; 31 | import software.amazon.awscdk.services.ecs.Cluster; 32 | import software.amazon.awscdk.services.ecs.ContainerDefinition; 33 | import software.amazon.awscdk.services.ecs.ContainerImage; 34 | import software.amazon.awscdk.services.ecs.FargateTaskDefinition; 35 | import software.amazon.awscdk.services.iam.PolicyStatement; 36 | import software.amazon.awscdk.services.lambda.Code; 37 | import software.amazon.awscdk.services.lambda.Function; 38 | import software.amazon.awscdk.services.lambda.Runtime; 39 | import software.amazon.awscdk.services.logs.LogGroup; 40 | import software.amazon.awscdk.services.logs.RetentionDays; 41 | import software.amazon.awscdk.services.s3.BlockPublicAccess; 42 | import software.amazon.awscdk.services.s3.Bucket; 43 | import software.amazon.awscdk.services.stepfunctions.Chain; 44 | import software.amazon.awscdk.services.stepfunctions.Choice; 45 | import software.amazon.awscdk.services.stepfunctions.Condition; 46 | import software.amazon.awscdk.services.stepfunctions.StateMachine; 47 | import software.amazon.awscdk.services.stepfunctions.StateMachineType; 48 | import software.amazon.awscdk.services.stepfunctions.Succeed; 49 | import software.amazon.awscdk.services.stepfunctions.TaskInput; 50 | import software.amazon.awscdk.services.stepfunctions.Wait; 51 | import software.amazon.awscdk.services.stepfunctions.WaitTime; 52 | import software.amazon.awscdk.services.stepfunctions.tasks.LambdaInvoke; 53 | 54 | /** 55 | * 56 | * This CDK Application runs a CloudFormation stack to demonstrate pattern 57 | * ECS Task submission from Lambda 58 | * 59 | * @author Sarma Palli, Senior DevOps Cloud Architect 60 | * 61 | */ 62 | public class ECSTaskSubmissionFromLambdaPattern extends Stack { 63 | 64 | public ECSTaskSubmissionFromLambdaPattern(final Construct scope, final String id) { 65 | this(scope, id, null); 66 | } 67 | 68 | public ECSTaskSubmissionFromLambdaPattern(final Construct scope, final String id, final StackProps props) { 69 | super(scope, id, props); 70 | 71 | SubnetSelection privateSubnets = SubnetSelection.builder().subnetType(SubnetType.PRIVATE).build(); 72 | // VPC 73 | Vpc vpc = Vpc.Builder.create(this, "StarterKitVPC").cidr("10.110.0.0/16").maxAzs(2) 74 | .gatewayEndpoints(new HashMap() { 75 | private static final long serialVersionUID = -2650622903964203923L; 76 | { 77 | put("S3EndPoint", GatewayVpcEndpointOptions.builder().service(GatewayVpcEndpointAwsService.S3) 78 | .subnets(new ArrayList() { 79 | private static final long serialVersionUID = 1687752884565877349L; 80 | { 81 | add(privateSubnets); 82 | } 83 | }).build()); 84 | put("DDBEndPoint", 85 | GatewayVpcEndpointOptions.builder().service(GatewayVpcEndpointAwsService.DYNAMODB) 86 | .subnets(new ArrayList() { 87 | private static final long serialVersionUID = -746781466183802042L; 88 | { 89 | add(privateSubnets); 90 | } 91 | }).build()); 92 | } 93 | }).build(); 94 | vpc.addInterfaceEndpoint("ECSEndPoint", InterfaceVpcEndpointOptions.builder() 95 | .service(InterfaceVpcEndpointAwsService.ECS).subnets(privateSubnets).build()); 96 | vpc.addInterfaceEndpoint("ECSAgentEndPoint", InterfaceVpcEndpointOptions.builder() 97 | .service(InterfaceVpcEndpointAwsService.ECS_AGENT).subnets(privateSubnets).build()); 98 | vpc.addInterfaceEndpoint("ECREndPoint", InterfaceVpcEndpointOptions.builder() 99 | .service(InterfaceVpcEndpointAwsService.ECR).subnets(privateSubnets).build()); 100 | 101 | // S3 Bucket 102 | Bucket s3Bucket = Bucket.Builder.create(this, "S3Bucket") 103 | .bucketName(this.getAccount() + "-amazon-ecs-java-starter-kit-pattern-1-bucket") 104 | .blockPublicAccess(BlockPublicAccess.Builder.create() 105 | .blockPublicAcls(true) 106 | .blockPublicPolicy(true) 107 | .ignorePublicAcls(true) 108 | .restrictPublicBuckets(true) 109 | .build()) 110 | .removalPolicy(RemovalPolicy.DESTROY) 111 | .build(); 112 | 113 | // DynamoDB Tables 114 | String workflowSummaryPartitionKeyName = "workflow_name"; 115 | String workflowSummarySortKeyName = "workflow_run_id"; 116 | Table workflow_summary = Table.Builder.create(this, "DDBWorkFlowSummary").tableName("workflow_summary_pattern_1") 117 | .removalPolicy(RemovalPolicy.DESTROY) 118 | .partitionKey( 119 | Attribute.builder().name(workflowSummaryPartitionKeyName).type(AttributeType.STRING).build()) 120 | .sortKey(Attribute.builder().name(workflowSummarySortKeyName).type(AttributeType.NUMBER).build()) 121 | .build(); 122 | 123 | String workflowDetailsPartitionKeyName = "workflow_run_id"; 124 | String workflowDetailsSortKeyName = "ecs_task_id"; 125 | Table workflow_details = Table.Builder.create(this, "DDBWorkFlowDetails").tableName("workflow_details_pattern_1") 126 | .removalPolicy(RemovalPolicy.DESTROY) 127 | .partitionKey( 128 | Attribute.builder().name(workflowDetailsPartitionKeyName).type(AttributeType.NUMBER).build()) 129 | .sortKey(Attribute.builder().name(workflowDetailsSortKeyName).type(AttributeType.STRING).build()) 130 | .build(); 131 | 132 | // ECS Cluster 133 | Cluster cluster = Cluster.Builder.create(this, "StarterKitCluster").clusterName("amazon-ecs-java-starter-kit-pattern-1") 134 | .vpc(vpc).build(); 135 | 136 | // ECR Image 137 | String ecrRepoName = "amazon-ecs-java-starter-kit-pattern-1"; 138 | @SuppressWarnings("deprecation") 139 | DockerImageAsset dockerImageAsset = DockerImageAsset.Builder.create(this, "StarterKitECRImage") 140 | .directory("../amazon-ecs-java-starter-kit-task").repositoryName(ecrRepoName).build(); 141 | 142 | // Fargate Task Definition 143 | FargateTaskDefinition fargateTaskDefinition = FargateTaskDefinition.Builder 144 | .create(this, "StarterKitFargateTaskDefinition").family("amazon-ecs-java-starter-kit-pattern-1").cpu(1024) 145 | .memoryLimitMiB(2048).build(); 146 | 147 | // Container Definition 148 | ContainerDefinition containerDefinition = ContainerDefinition.Builder 149 | .create(this, "amazon-ecs-java-starter-kit").taskDefinition(fargateTaskDefinition).essential(true) 150 | .image(ContainerImage.fromDockerImageAsset(dockerImageAsset)) 151 | .logging(AwsLogDriver.awsLogs(AwsLogDriverProps.builder() 152 | .logGroup(LogGroup.Builder.create(this, "ECSLogGroup") 153 | .logGroupName("/ecs/amazon-ecs-java-starter-kit-pattern-1").removalPolicy(RemovalPolicy.DESTROY) 154 | .retention(RetentionDays.ONE_DAY).build()) 155 | .streamPrefix("amazon-ecs-java-starter-kit").build())) 156 | .build(); 157 | 158 | // Container IAM permissions 159 | workflow_details.grantReadWriteData(fargateTaskDefinition.getTaskRole()); 160 | s3Bucket.grantReadWrite(fargateTaskDefinition.getTaskRole()); 161 | 162 | // TaskLauncher Lambda 163 | Function taskLauncher = Function.Builder.create(this, "TaskLauncherLambda") 164 | .functionName("amazon-ecs-java-starter-kit-pattern-1-ecs-task-launcher") 165 | .code(Code.fromAsset( 166 | "../amazon-ecs-java-starter-kit-tasklauncher/target/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar")) 167 | .handler("software.aws.ecs.java.starterkit.launcher.ECSTaskLauncher").runtime(Runtime.JAVA_8_CORRETTO) 168 | .timeout(Duration.minutes(5)).memorySize(256).logRetention(RetentionDays.ONE_DAY).vpc(vpc) 169 | .vpcSubnets(privateSubnets) 170 | .securityGroups(Collections.singletonList(SecurityGroup.Builder.create(this, "TaskLauncherSG").vpc(vpc) 171 | .securityGroupName("amazon-ecs-java-starter-kit-pattern-1-ecs-task-launcher").allowAllOutbound(true) 172 | .build())) 173 | .build(); 174 | 175 | // Permissions to run ECS Task 176 | taskLauncher.getRole().addToPrincipalPolicy(PolicyStatement.Builder.create() 177 | .actions(Collections.singletonList("ecs:RunTask")).resources(new ArrayList() { 178 | private static final long serialVersionUID = 1133379137898541366L; 179 | 180 | { 181 | add(fargateTaskDefinition.getTaskDefinitionArn()); 182 | add(cluster.getClusterArn()); 183 | } 184 | }).build()); 185 | taskLauncher.getRole().addToPrincipalPolicy(PolicyStatement.Builder.create() 186 | .actions(Collections.singletonList("iam:PassRole")).resources(new ArrayList() { 187 | private static final long serialVersionUID = 306435034815975097L; 188 | 189 | { 190 | add(fargateTaskDefinition.getTaskRole().getRoleArn()); 191 | add(fargateTaskDefinition.getExecutionRole().getRoleArn()); 192 | } 193 | }).build()); 194 | 195 | // TaskMonitor Lambda 196 | Function taskMonitor = Function.Builder.create(this, "TaskMonitorLambda") 197 | .functionName("amazon-ecs-java-starter-kit-pattern-1-ecs-task-monitor") 198 | .code(Code.fromAsset( 199 | "../amazon-ecs-java-starter-kit-taskmonitor/target/amazon-ecs-java-starter-kit-taskmonitor-1.0.jar")) 200 | .handler("software.aws.ecs.java.starterkit.monitor.ECSTaskMonitor").runtime(Runtime.JAVA_8_CORRETTO) 201 | .timeout(Duration.minutes(5)).memorySize(256).logRetention(RetentionDays.ONE_DAY).vpc(vpc) 202 | .vpcSubnets(privateSubnets) 203 | .securityGroups(Collections.singletonList(SecurityGroup.Builder.create(this, "TaskMonitorSG").vpc(vpc) 204 | .securityGroupName("amazon-ecs-java-starter-kit-pattern-1-ecs-task-Monitor").allowAllOutbound(true) 205 | .build())) 206 | .environment(new HashMap() { 207 | private static final long serialVersionUID = -8778366953471384771L; 208 | { 209 | put("region", getRegion()); 210 | put("workflow_summary_ddb_table_name", workflow_summary.getTableName()); 211 | put("workflow_summary_hash_key", workflowSummaryPartitionKeyName); 212 | put("workflow_summary_range_key", workflowSummarySortKeyName); 213 | put("workflow_details_ddb_table_name", workflow_details.getTableName()); 214 | put("workflow_details_hash_key", workflowDetailsPartitionKeyName); 215 | put("workflow_details_range_key", workflowDetailsSortKeyName); 216 | } 217 | }).build(); 218 | 219 | // IAM permissions for Lambdas 220 | workflow_details.grantReadWriteData(taskLauncher.getRole()); 221 | workflow_details.grantReadWriteData(taskMonitor.getRole()); 222 | workflow_summary.grantReadWriteData(taskLauncher.getRole()); 223 | workflow_summary.grantReadWriteData(taskMonitor.getRole()); 224 | 225 | // Monitor State in StateMachine 226 | LambdaInvoke invokeMonitorState = LambdaInvoke.Builder.create(this, "InvokeTaskMonitor") 227 | .payloadResponseOnly(true).lambdaFunction(taskMonitor) 228 | .payload(TaskInput.fromObject(new HashMap() { 229 | private static final long serialVersionUID = -4995384961752093935L; 230 | { 231 | put("iterator.$", "$.iterator"); 232 | } 233 | })).resultPath("$.iterator").payloadResponseOnly(true).build(); 234 | 235 | // StateMachine 236 | StateMachine.Builder.create(this, "amazon-ecs-java-starter-kit-state-machine") 237 | .stateMachineName("amazon-ecs-java-starter-kit-pattern-1").stateMachineType(StateMachineType.STANDARD) 238 | .definition(Chain 239 | .start(LambdaInvoke.Builder.create(this, "InvokeTaskLauncher").lambdaFunction(taskLauncher) 240 | .resultPath("$.iterator").payloadResponseOnly(true).build()) 241 | .next(invokeMonitorState) 242 | .next(Choice.Builder.create(this, "CheckIfTasksCompleted").build().when( 243 | Condition.booleanEquals("$.iterator.continue", true), 244 | Wait.Builder.create(this, "WaitForECS").time(WaitTime.duration(Duration.seconds(120))) 245 | .build().next(invokeMonitorState)) 246 | .otherwise(Succeed.Builder.create(this, "Done").build()))) 247 | .build(); 248 | 249 | // Outputs 250 | CfnOutput.Builder.create(this, "region").value(this.getRegion()).build(); 251 | CfnOutput.Builder.create(this, "clusterName").value(cluster.getClusterName()).build(); 252 | CfnOutput.Builder.create(this, "containerName").value(containerDefinition.getContainerName()).build(); 253 | CfnOutput.Builder.create(this, "taskDefinition") 254 | .value(Fn.select(1, Fn.split("/", fargateTaskDefinition.getTaskDefinitionArn()))).build(); 255 | CfnOutput.Builder.create(this, "securityGroupId").value(vpc.getVpcDefaultSecurityGroup()).build(); 256 | CfnOutput.Builder.create(this, "subnetIdLiteral").value(vpc.getPrivateSubnets().get(0).getSubnetId()).build(); 257 | CfnOutput.Builder.create(this, "ddbTableNameWFSummary").value(workflow_summary.getTableName()).build(); 258 | CfnOutput.Builder.create(this, "hashKeyWFSummary").value(workflowSummaryPartitionKeyName).build(); 259 | CfnOutput.Builder.create(this, "rangeKeyWFSummary").value(workflowSummarySortKeyName).build(); 260 | CfnOutput.Builder.create(this, "ddbTableNameWFDetails").value(workflow_details.getTableName()).build(); 261 | CfnOutput.Builder.create(this, "hashKeyWFDetails").value(workflowDetailsPartitionKeyName).build(); 262 | CfnOutput.Builder.create(this, "rangeKeyWFDetails").value(workflowDetailsSortKeyName).build(); 263 | CfnOutput.Builder.create(this, "s3BucketName").value(s3Bucket.getBucketName()).build(); 264 | CfnOutput.Builder.create(this, "workflowName").value("amazon_ecs_starter_kit-pattern-1").build(); 265 | CfnOutput.Builder.create(this, "separator").value("$").build(); 266 | } 267 | } 268 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-cdk/src/main/java/software/aws/ecs/java/starterkit/cdk/ECSTaskSubmissionFromStepFunctionsPattern.java: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | package software.aws.ecs.java.starterkit.cdk; 5 | 6 | import java.util.ArrayList; 7 | import java.util.Collections; 8 | import java.util.HashMap; 9 | 10 | import software.amazon.awscdk.core.Aws; 11 | import software.amazon.awscdk.core.CfnOutput; 12 | import software.amazon.awscdk.core.Construct; 13 | import software.amazon.awscdk.core.Duration; 14 | import software.amazon.awscdk.core.Fn; 15 | import software.amazon.awscdk.core.RemovalPolicy; 16 | import software.amazon.awscdk.core.Stack; 17 | import software.amazon.awscdk.core.StackProps; 18 | import software.amazon.awscdk.services.dynamodb.Attribute; 19 | import software.amazon.awscdk.services.dynamodb.AttributeType; 20 | import software.amazon.awscdk.services.dynamodb.Table; 21 | import software.amazon.awscdk.services.ec2.GatewayVpcEndpointAwsService; 22 | import software.amazon.awscdk.services.ec2.GatewayVpcEndpointOptions; 23 | import software.amazon.awscdk.services.ec2.InterfaceVpcEndpointAwsService; 24 | import software.amazon.awscdk.services.ec2.InterfaceVpcEndpointOptions; 25 | import software.amazon.awscdk.services.ec2.SecurityGroup; 26 | import software.amazon.awscdk.services.ec2.SubnetSelection; 27 | import software.amazon.awscdk.services.ec2.SubnetType; 28 | import software.amazon.awscdk.services.ec2.Vpc; 29 | import software.amazon.awscdk.services.ecr.assets.DockerImageAsset; 30 | import software.amazon.awscdk.services.ecs.AwsLogDriver; 31 | import software.amazon.awscdk.services.ecs.AwsLogDriverProps; 32 | import software.amazon.awscdk.services.ecs.Cluster; 33 | import software.amazon.awscdk.services.ecs.ContainerDefinition; 34 | import software.amazon.awscdk.services.ecs.ContainerDefinitionOptions; 35 | import software.amazon.awscdk.services.ecs.ContainerImage; 36 | import software.amazon.awscdk.services.ecs.FargatePlatformVersion; 37 | import software.amazon.awscdk.services.ecs.FargateTaskDefinition; 38 | import software.amazon.awscdk.services.lambda.Code; 39 | import software.amazon.awscdk.services.lambda.Function; 40 | import software.amazon.awscdk.services.lambda.Runtime; 41 | import software.amazon.awscdk.services.logs.LogGroup; 42 | import software.amazon.awscdk.services.logs.RetentionDays; 43 | import software.amazon.awscdk.services.s3.BlockPublicAccess; 44 | import software.amazon.awscdk.services.s3.Bucket; 45 | import software.amazon.awscdk.services.stepfunctions.Chain; 46 | import software.amazon.awscdk.services.stepfunctions.Choice; 47 | import software.amazon.awscdk.services.stepfunctions.Condition; 48 | import software.amazon.awscdk.services.stepfunctions.JsonPath; 49 | import software.amazon.awscdk.services.stepfunctions.Map; 50 | import software.amazon.awscdk.services.stepfunctions.Parallel; 51 | import software.amazon.awscdk.services.stepfunctions.Pass; 52 | import software.amazon.awscdk.services.stepfunctions.StateMachine; 53 | import software.amazon.awscdk.services.stepfunctions.StateMachineType; 54 | import software.amazon.awscdk.services.stepfunctions.Succeed; 55 | import software.amazon.awscdk.services.stepfunctions.Wait; 56 | import software.amazon.awscdk.services.stepfunctions.WaitTime; 57 | import software.amazon.awscdk.services.stepfunctions.tasks.ContainerOverride; 58 | import software.amazon.awscdk.services.stepfunctions.tasks.EcsFargateLaunchTarget; 59 | import software.amazon.awscdk.services.stepfunctions.tasks.EcsRunTask; 60 | import software.amazon.awscdk.services.stepfunctions.tasks.LambdaInvoke; 61 | import software.amazon.awscdk.services.stepfunctions.tasks.TaskEnvironmentVariable; 62 | 63 | /** 64 | * 65 | * This CDK Application runs a CloudFormation stack to demonstrate pattern 66 | * ECS Task submission from StepFunctions 67 | * 68 | * @author Sarma Palli, Senior DevOps Cloud Architect 69 | * 70 | */ 71 | public class ECSTaskSubmissionFromStepFunctionsPattern extends Stack { 72 | 73 | public ECSTaskSubmissionFromStepFunctionsPattern(final Construct scope, final String id) { 74 | this(scope, id, null); 75 | } 76 | 77 | private TaskEnvironmentVariable EnvVarBuilder(String envKey, String envValue) { 78 | return TaskEnvironmentVariable.builder().name(envKey).value(envValue).build(); 79 | } 80 | 81 | public ECSTaskSubmissionFromStepFunctionsPattern(final Construct scope, final String id, final StackProps props) { 82 | super(scope, id, props); 83 | 84 | SubnetSelection privateSubnets = SubnetSelection.builder().subnetType(SubnetType.PRIVATE).build(); 85 | // VPC 86 | Vpc vpc = Vpc.Builder.create(this, "StarterKitVPC").cidr("10.120.0.0/16").maxAzs(2) 87 | .gatewayEndpoints(new HashMap() { 88 | private static final long serialVersionUID = 2479535941382804947L; 89 | { 90 | put("S3EndPoint", GatewayVpcEndpointOptions.builder().service(GatewayVpcEndpointAwsService.S3) 91 | .subnets(new ArrayList() { 92 | private static final long serialVersionUID = 1454955270027154519L; 93 | { 94 | add(privateSubnets); 95 | } 96 | }).build()); 97 | put("DDBEndPoint", 98 | GatewayVpcEndpointOptions.builder().service(GatewayVpcEndpointAwsService.DYNAMODB) 99 | .subnets(new ArrayList() { 100 | private static final long serialVersionUID = -4389876763264722986L; 101 | { 102 | add(privateSubnets); 103 | } 104 | }).build()); 105 | } 106 | }).build(); 107 | vpc.addInterfaceEndpoint("ECSEndPoint", InterfaceVpcEndpointOptions.builder() 108 | .service(InterfaceVpcEndpointAwsService.ECS).subnets(privateSubnets).build()); 109 | vpc.addInterfaceEndpoint("ECSAgentEndPoint", InterfaceVpcEndpointOptions.builder() 110 | .service(InterfaceVpcEndpointAwsService.ECS_AGENT).subnets(privateSubnets).build()); 111 | vpc.addInterfaceEndpoint("ECREndPoint", InterfaceVpcEndpointOptions.builder() 112 | .service(InterfaceVpcEndpointAwsService.ECR).subnets(privateSubnets).build()); 113 | 114 | // ECS Tasks SecurityGroup 115 | SecurityGroup ecsSecurityGroup = SecurityGroup.Builder.create(this, "FargateSecurityGroup").vpc(vpc) 116 | .securityGroupName("amazon-ecs-java-starter-kit-pattern-2").allowAllOutbound(true).build(); 117 | 118 | // S3 Bucket 119 | Bucket s3Bucket = Bucket.Builder.create(this, "S3Bucket") 120 | .bucketName(this.getAccount() + "-amazon-ecs-java-starter-kit-pattern-2-bucket") 121 | .blockPublicAccess(BlockPublicAccess.Builder.create() 122 | .blockPublicAcls(true) 123 | .blockPublicPolicy(true) 124 | .ignorePublicAcls(true) 125 | .restrictPublicBuckets(true) 126 | .build()) 127 | .removalPolicy(RemovalPolicy.DESTROY) 128 | .build(); 129 | 130 | // DynamoDB Tables 131 | String workflowSummaryPartitionKeyName = "workflow_name"; 132 | String workflowSummarySortKeyName = "workflow_run_id"; 133 | Table workflow_summary = Table.Builder.create(this, "DDBWorkFlowSummary").tableName("workflow_summary_pattern_2") 134 | .removalPolicy(RemovalPolicy.DESTROY) 135 | .partitionKey( 136 | Attribute.builder().name(workflowSummaryPartitionKeyName).type(AttributeType.STRING).build()) 137 | .sortKey(Attribute.builder().name(workflowSummarySortKeyName).type(AttributeType.NUMBER).build()) 138 | .build(); 139 | 140 | String workflowDetailsPartitionKeyName = "workflow_run_id"; 141 | String workflowDetailsSortKeyName = "ecs_task_id"; 142 | Table workflow_details = Table.Builder.create(this, "DDBWorkFlowDetails").tableName("workflow_details_pattern_2") 143 | .removalPolicy(RemovalPolicy.DESTROY) 144 | .partitionKey( 145 | Attribute.builder().name(workflowDetailsPartitionKeyName).type(AttributeType.NUMBER).build()) 146 | .sortKey(Attribute.builder().name(workflowDetailsSortKeyName).type(AttributeType.STRING).build()) 147 | .build(); 148 | 149 | // ECS Cluster 150 | Cluster cluster = Cluster.Builder.create(this, "StarterKitCluster").clusterName("amazon-ecs-java-starter-kit-pattern-2") 151 | .vpc(vpc).build(); 152 | 153 | // ECR Image 154 | String ecrRepoName = "amazon-ecs-java-starter-kit-pattern-2"; 155 | @SuppressWarnings("deprecation") 156 | DockerImageAsset dockerImageAsset = DockerImageAsset.Builder.create(this, "StarterKitECRImage") 157 | .directory("../amazon-ecs-java-starter-kit-task").repositoryName(ecrRepoName).build(); 158 | 159 | // Fargate Task Definition 160 | FargateTaskDefinition fargateTaskDefinition = FargateTaskDefinition.Builder 161 | .create(this, "StarterKitFargateTaskDefinition").family("amazon-ecs-java-starter-kit-pattern-2").cpu(1024) 162 | .memoryLimitMiB(2048).build(); 163 | 164 | // Container Definition 165 | ContainerDefinition containerDefinition = fargateTaskDefinition.addContainer("amazon-ecs-java-starter-kit", 166 | ContainerDefinitionOptions.builder().essential(true) 167 | .image(ContainerImage.fromDockerImageAsset(dockerImageAsset)) 168 | .logging(AwsLogDriver.awsLogs(AwsLogDriverProps.builder() 169 | .logGroup(LogGroup.Builder.create(this, "ECSLogGroup") 170 | .logGroupName("/ecs/amazon-ecs-java-starter-kit-pattern-2") 171 | .removalPolicy(RemovalPolicy.DESTROY).retention(RetentionDays.ONE_DAY).build()) 172 | .streamPrefix("amazon-ecs-java-starter-kit").build())) 173 | .build()); 174 | 175 | // Container IAM permissions 176 | workflow_details.grantReadWriteData(fargateTaskDefinition.getTaskRole()); 177 | s3Bucket.grantReadWrite(fargateTaskDefinition.getTaskRole()); 178 | 179 | // TaskMonitor Lambda 180 | Function taskMonitor = Function.Builder.create(this, "TaskMonitorLambda") 181 | .functionName("amazon-ecs-java-starter-kit-pattern-2-ecs-task-monitor") 182 | .code(Code.fromAsset( 183 | "../amazon-ecs-java-starter-kit-taskmonitor/target/amazon-ecs-java-starter-kit-taskmonitor-1.0.jar")) 184 | .handler("software.aws.ecs.java.starterkit.monitor.ECSTaskMonitor").runtime(Runtime.JAVA_8_CORRETTO) 185 | .timeout(Duration.minutes(5)).memorySize(256).logRetention(RetentionDays.ONE_DAY).vpc(vpc) 186 | .vpcSubnets(privateSubnets) 187 | .securityGroups(Collections.singletonList(SecurityGroup.Builder.create(this, "TaskMonitorSG").vpc(vpc) 188 | .securityGroupName("amazon-ecs-java-starter-kit-pattern-2-ecs-task-Monitor").allowAllOutbound(true) 189 | .build())) 190 | .environment(new HashMap() { 191 | private static final long serialVersionUID = -4232375236129537678L; 192 | { 193 | put("region", getRegion()); 194 | put("workflow_summary_ddb_table_name", workflow_summary.getTableName()); 195 | put("workflow_summary_hash_key", workflowSummaryPartitionKeyName); 196 | put("workflow_summary_range_key", workflowSummarySortKeyName); 197 | put("workflow_details_ddb_table_name", workflow_details.getTableName()); 198 | put("workflow_details_hash_key", workflowDetailsPartitionKeyName); 199 | put("workflow_details_range_key", workflowDetailsSortKeyName); 200 | } 201 | }).build(); 202 | 203 | // IAM permissions for Lambdas 204 | workflow_details.grantReadWriteData(taskMonitor.getRole()); 205 | workflow_summary.grantReadWriteData(taskMonitor.getRole()); 206 | 207 | // Environment variables being passed to ECS Tasks 208 | ArrayList containerEnvVars = new ArrayList() { 209 | private static final long serialVersionUID = -7266629031441090205L; 210 | { 211 | add(EnvVarBuilder("region", Aws.REGION)); 212 | add(EnvVarBuilder("workflow_details_ddb_table_name", workflow_details.getTableName())); 213 | add(EnvVarBuilder("workflow_details_hash_key", workflowDetailsPartitionKeyName)); 214 | add(EnvVarBuilder("workflow_details_range_key", workflowDetailsSortKeyName)); 215 | add(EnvVarBuilder("workflow_name", JsonPath.stringAt("$.workflowName"))); 216 | add(EnvVarBuilder("workflow_run_id", JsonPath.stringAt("$.workflowRunId"))); 217 | add(EnvVarBuilder("task_name", JsonPath.stringAt("$.taskName"))); 218 | add(EnvVarBuilder("s3_bucket_name", JsonPath.stringAt("$.s3BucketName"))); 219 | add(EnvVarBuilder("object_key", JsonPath.stringAt("$.objectKey"))); 220 | } 221 | }; 222 | 223 | // ECS Run Task State 224 | EcsRunTask ecsRunTask = EcsRunTask.Builder.create(this, "SubmitECSTasks").assignPublicIp(false).cluster(cluster) 225 | .taskDefinition(fargateTaskDefinition).subnets(privateSubnets) 226 | .securityGroups(Collections.singletonList(ecsSecurityGroup)) 227 | .launchTarget(EcsFargateLaunchTarget.Builder.create().platformVersion(FargatePlatformVersion.VERSION1_4) 228 | .build()) 229 | .containerOverrides(Collections.singletonList(ContainerOverride.builder() 230 | .containerDefinition(containerDefinition).environment(containerEnvVars).build())) 231 | .build(); 232 | 233 | // Submit ECS tasks simultaneously using Map state 234 | Map ecsTasksSubmitter = Map.Builder.create(this, "S3CopyTaskRunner").parameters(new HashMap() { 235 | private static final long serialVersionUID = -2748074145208985587L; 236 | { 237 | put("workflowRunId.$", "$.workflowRunId"); 238 | put("workflowName.$", "$.workflowName"); 239 | put("s3BucketName.$", "$.s3BucketName"); 240 | put("taskName.$", "$$.Map.Item.Value.taskName"); 241 | put("objectKey.$", "$$.Map.Item.Value.objectKey"); 242 | } 243 | }).itemsPath("$.taskList").outputPath("$.[*].Tasks.[*].TaskArn") 244 | /** 245 | * TODO: This line is not supported yet. See https://github.com/aws/aws-cdk/issues/9904 246 | * .resultSelection("$.[*].Tasks.[*].TaskArn") 247 | */ 248 | .maxConcurrency(5).build().iterator(ecsRunTask); 249 | /** 250 | * TODO: Parallel Wrapper to work-around this issue with CDK 251 | * See https://github.com/aws/aws-cdk/issues/9904, So that we can pass through the input variables 252 | */ 253 | Parallel parallelWrapper = Parallel.Builder.create(this, "KickOffInParallel").resultPath("$.paralleloutput") 254 | .build().branch(ecsTasksSubmitter); 255 | 256 | // Pass-Through State for passing through the ECS Task Arns 257 | Pass passThroughECSTasksArns = Pass.Builder.create(this, "PassThroughECSArns") 258 | .parameters(new HashMap() { 259 | private static final long serialVersionUID = 4528431724317351553L; 260 | { 261 | put("iterator", new HashMap() { 262 | private static final long serialVersionUID = 5348720279215832325L; 263 | { 264 | put("continue", false); 265 | put("workflowRunId.$", "$.workflowRunId"); 266 | put("workflowName.$", "$.workflowName"); 267 | put("ecsTaskArns.$", "$.paralleloutput.[0]"); 268 | } 269 | }); 270 | } 271 | }).build(); 272 | 273 | // Monitor State Lambda invocation 274 | LambdaInvoke invokeMonitorState = LambdaInvoke.Builder.create(this, "InvokeTaskMonitor") 275 | .lambdaFunction(taskMonitor).resultPath("$.iterator").payloadResponseOnly(true).build(); 276 | 277 | // State for sleeping for sometime 278 | Chain waitState = Wait.Builder.create(this, "WaitForECS").time(WaitTime.duration(Duration.seconds(120))).build() 279 | .next(invokeMonitorState); 280 | 281 | // Success State 282 | Succeed doneState = Succeed.Builder.create(this, "Done").build(); 283 | 284 | // State for checking if tasks completed 285 | Choice checkTasksCompleted = Choice.Builder.create(this, "CheckIfTasksCompleted").build() 286 | .when(Condition.booleanEquals("$.iterator.continue", true), waitState).otherwise(doneState); 287 | 288 | // StateMachine 289 | StateMachine.Builder.create(this, "amazon-ecs-java-starter-kit") 290 | .stateMachineName("amazon-ecs-java-starter-kit-pattern-2").stateMachineType(StateMachineType.STANDARD) 291 | .definition(Chain.start(parallelWrapper).next(passThroughECSTasksArns).next(invokeMonitorState) 292 | .next(checkTasksCompleted)) 293 | .build(); 294 | 295 | // Outputs 296 | CfnOutput.Builder.create(this, "region").value(this.getRegion()).build(); 297 | CfnOutput.Builder.create(this, "clusterName").value(cluster.getClusterName()).build(); 298 | CfnOutput.Builder.create(this, "containerName").value(containerDefinition.getContainerName()).build(); 299 | CfnOutput.Builder.create(this, "taskDefinition") 300 | .value(Fn.select(1, Fn.split("/", fargateTaskDefinition.getTaskDefinitionArn()))).build(); 301 | CfnOutput.Builder.create(this, "securityGroupId").value(vpc.getVpcDefaultSecurityGroup()).build(); 302 | CfnOutput.Builder.create(this, "subnetIdLiteral").value(vpc.getPrivateSubnets().get(0).getSubnetId()).build(); 303 | CfnOutput.Builder.create(this, "ddbTableNameWFSummary").value(workflow_summary.getTableName()).build(); 304 | CfnOutput.Builder.create(this, "hashKeyWFSummary").value(workflowSummaryPartitionKeyName).build(); 305 | CfnOutput.Builder.create(this, "rangeKeyWFSummary").value(workflowSummarySortKeyName).build(); 306 | CfnOutput.Builder.create(this, "ddbTableNameWFDetails").value(workflow_details.getTableName()).build(); 307 | CfnOutput.Builder.create(this, "hashKeyWFDetails").value(workflowDetailsPartitionKeyName).build(); 308 | CfnOutput.Builder.create(this, "rangeKeyWFDetails").value(workflowDetailsSortKeyName).build(); 309 | CfnOutput.Builder.create(this, "s3BucketName").value(s3Bucket.getBucketName()).build(); 310 | CfnOutput.Builder.create(this, "separator").value("$").build(); 311 | CfnOutput.Builder.create(this, "workflowName").value("amazon_ecs_starter_kit_pattern_2").build(); 312 | CfnOutput.Builder.create(this, "workflowRunId").value("100001").build(); 313 | } 314 | } 315 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-cdk/workflow_specs_pattern_1.json: -------------------------------------------------------------------------------- 1 | { 2 | "hashKeyWFDetails": "workflow_run_id", 3 | "subnetIdLiteral": "subnet-id-literal", 4 | "workflowName": "amazon_ecs_starter_kit-pattern-1", 5 | "separator": "$", 6 | "securityGroupId": "sg-id", 7 | "rangeKeyWFDetails": "ecs_task_id", 8 | "ddbTableNameWFDetails": "workflow_details_pattern_1", 9 | "containerName": "amazon-ecs-java-starter-kit", 10 | "clusterName": "amazon-ecs-java-starter-kit-pattern-1", 11 | "hashKeyWFSummary": "workflow_name", 12 | "s3BucketName": "1234567890-amazon-ecs-java-starter-kit-pattern-1-bucket", 13 | "taskDefinition": "amazon-ecs-java-starter-kit-pattern-1:2", 14 | "rangeKeyWFSummary": "workflow_run_id", 15 | "region": "us-east-2", 16 | "ddbTableNameWFSummary": "workflow_summary_pattern_1", 17 | "taskList": [ 18 | { 19 | "taskName": "ECS_task_01", 20 | "s3BucketName": "1234567890-amazon-ecs-java-starter-kit-pattern-1-bucket", 21 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 22 | }, 23 | { 24 | "taskName": "ECS_task_02", 25 | "s3BucketName": "1234567890-amazon-ecs-java-starter-kit-pattern-1-bucket", 26 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 27 | }, 28 | { 29 | "taskName": "ECS_task_03", 30 | "s3BucketName": "1234567890-amazon-ecs-java-starter-kit-pattern-1-bucket", 31 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 32 | }, 33 | { 34 | "taskName": "ECS_task_04", 35 | "s3BucketName": "1234567890-amazon-ecs-java-starter-kit-pattern-1-bucket", 36 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 37 | }, 38 | { 39 | "taskName": "ECS_task_05", 40 | "s3BucketName": "1234567890-amazon-ecs-java-starter-kit-pattern-1-bucket", 41 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 42 | }, 43 | { 44 | "taskName": "ECS_task_06", 45 | "s3BucketName": "1234567890-amazon-ecs-java-starter-kit-pattern-1-bucket", 46 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 47 | }, 48 | { 49 | "taskName": "ECS_task_07", 50 | "s3BucketName": "1234567890-amazon-ecs-java-starter-kit-pattern-1-bucket", 51 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 52 | }, 53 | { 54 | "taskName": "ECS_task_08", 55 | "s3BucketName": "1234567890-amazon-ecs-java-starter-kit-pattern-1-bucket", 56 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 57 | }, 58 | { 59 | "taskName": "ECS_task_09", 60 | "s3BucketName": "1234567890-amazon-ecs-java-starter-kit-pattern-1-bucket", 61 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 62 | }, 63 | { 64 | "taskName": "ECS_task_10", 65 | "s3BucketName": "1234567890-amazon-ecs-java-starter-kit-pattern-1-bucket", 66 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 67 | } 68 | ] 69 | } -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-cdk/workflow_specs_pattern_2.json: -------------------------------------------------------------------------------- 1 | { 2 | "hashKeyWFDetails": "workflow_run_id", 3 | "subnetIdLiteral": "subnet-id-literal", 4 | "workflowName": "amazon_ecs_starter_kit_pattern_2", 5 | "separator": "$", 6 | "securityGroupId": "sg-id", 7 | "rangeKeyWFDetails": "ecs_task_id", 8 | "ddbTableNameWFDetails": "workflow_details_pattern_2", 9 | "containerName": "amazon-ecs-java-starter-kit", 10 | "clusterName": "amazon-ecs-java-starter-kit-pattern-2", 11 | "hashKeyWFSummary": "workflow_name", 12 | "s3BucketName": "1234567890-amazon-ecs-java-starter-kit-pattern-2-bucket", 13 | "taskDefinition": "amazon-ecs-java-starter-kit-pattern-2:2", 14 | "rangeKeyWFSummary": "workflow_run_id", 15 | "region": "us-east-2", 16 | "ddbTableNameWFSummary": "workflow_summary_pattern_2", 17 | "workflowRunId": "100001", 18 | "taskList": [ 19 | { 20 | "taskName": "ECS_task_01", 21 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 22 | }, 23 | { 24 | "taskName": "ECS_task_02", 25 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 26 | }, 27 | { 28 | "taskName": "ECS_task_03", 29 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 30 | }, 31 | { 32 | "taskName": "ECS_task_04", 33 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 34 | }, 35 | { 36 | "taskName": "ECS_task_05", 37 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 38 | }, 39 | { 40 | "taskName": "ECS_task_06", 41 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 42 | }, 43 | { 44 | "taskName": "ECS_task_07", 45 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 46 | }, 47 | { 48 | "taskName": "ECS_task_08", 49 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 50 | }, 51 | { 52 | "taskName": "ECS_task_09", 53 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 54 | }, 55 | { 56 | "taskName": "ECS_task_10", 57 | "objectKey": "amazon_ecs_java_starter_kit_jar/amazon-ecs-java-starter-kit-tasklauncher-1.0.jar" 58 | } 59 | ] 60 | } -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-task/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-task/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | .settings/ 3 | build/ 4 | target/classes/ -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-task/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | amazon-ecs-java-starter-kit-task 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-task/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazonlinux:latest 2 | 3 | RUN yum -y install perl which unzip aws-cli java-1.8.0-openjdk.x86_64 4 | 5 | # Add Java Jar 6 | ADD target/amazon-ecs-java-starter-kit-task-1.0.jar /java-app/ 7 | RUN chmod 744 /java-app/* 8 | 9 | # Docker execution entry point 10 | ADD runner.sh /java-app/ 11 | RUN chmod 755 /java-app/runner.sh 12 | 13 | WORKDIR /java-app 14 | USER root 15 | ENV program_executable /java-app/amazon-ecs-java-starter-kit-task-1.0.jar 16 | ENTRYPOINT ["/java-app/runner.sh"] 17 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-task/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this 4 | software and associated documentation files (the "Software"), to deal in the Software 5 | without restriction, including without limitation the rights to use, copy, modify, 6 | merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 10 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 11 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 12 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 13 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 14 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-task/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 4.0.0 7 | 8 | software.aws.ecs.samples 9 | amazon-ecs-and-aws-step-functions-design-patterns-starter-kit 10 | 1.0 11 | 12 | amazon-ecs-java-starter-kit-task 13 | amazon-ecs-java-starter-kit-task 14 | 15 | 16 | 17 | 18 | software.amazon.awssdk 19 | s3 20 | 2.15.19 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | true 29 | org.apache.maven.plugins 30 | maven-compiler-plugin 31 | 3.5.1 32 | 33 | 1.8 34 | 1.8 35 | 36 | 37 | 38 | 40 | 41 | org.apache.maven.plugins 42 | maven-assembly-plugin 43 | 3.1.0 44 | 45 | 46 | 47 | software.aws.ecs.java.starterkit.task.ECSTask 48 | 49 | 50 | 51 | jar-with-dependencies 52 | 53 | amazon-ecs-java-starter-kit-task-1.0 54 | false 55 | 56 | 57 | 58 | make-assembly 59 | package 60 | 61 | single 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-task/runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | 4 | PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin" 5 | BASENAME="${0##*/}" 6 | 7 | echo "Program Executable Jar: '$program_executable'" 8 | 9 | java -jar $program_executable -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-task/src/main/java/software/aws/ecs/java/starterkit/task/ECSTask.java: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | package software.aws.ecs.java.starterkit.task; 5 | 6 | import java.io.BufferedReader; 7 | import java.io.IOException; 8 | import java.io.InputStreamReader; 9 | import java.io.Reader; 10 | import java.io.UnsupportedEncodingException; 11 | import java.net.HttpURLConnection; 12 | import java.net.MalformedURLException; 13 | import java.net.ProtocolException; 14 | import java.net.URL; 15 | import java.net.URLEncoder; 16 | import java.nio.charset.StandardCharsets; 17 | import java.util.Date; 18 | import java.util.Iterator; 19 | import java.util.List; 20 | import java.util.Random; 21 | import java.util.UUID; 22 | 23 | import com.google.gson.JsonElement; 24 | import com.google.gson.JsonParser; 25 | 26 | import software.amazon.awssdk.regions.Region; 27 | import software.amazon.awssdk.services.dynamodb.DynamoDbClient; 28 | import software.amazon.awssdk.services.s3.S3Client; 29 | import software.amazon.awssdk.services.s3.model.CopyObjectRequest; 30 | import software.amazon.awssdk.services.s3.model.CopyObjectResponse; 31 | import software.amazon.awssdk.services.s3.model.S3Exception; 32 | import software.aws.ecs.java.starterkit.util.DDBUtil; 33 | 34 | public class ECSTask { 35 | 36 | public static void main(String[] args) { 37 | 38 | String regionPassed = System.getenv("region"); 39 | String tableName = System.getenv("workflow_details_ddb_table_name"); 40 | String hashKey = System.getenv("workflow_details_hash_key"); 41 | String rangeKey = System.getenv("workflow_details_range_key"); 42 | String workflowName = System.getenv("workflow_name"); 43 | long workflowRunId = Long.parseLong(System.getenv("workflow_run_id")); 44 | String taskName = System.getenv("task_name"); 45 | String bucketName = System.getenv("s3_bucket_name"); 46 | String objectKey = System.getenv("object_key"); 47 | String taskMetadataEndpoint = System.getenv("ECS_CONTAINER_METADATA_URI"); 48 | 49 | long startTime = System.currentTimeMillis(); 50 | 51 | // print runtime properties of the task 52 | printInputParameters(regionPassed, tableName, hashKey, rangeKey, workflowName, workflowRunId, taskName, 53 | bucketName, objectKey, taskMetadataEndpoint); 54 | String destinationKey = objectKey.concat("_").concat(UUID.randomUUID().toString()); 55 | 56 | // Create objects 57 | Region region = Region.regions().stream().filter(r -> r.toString().equalsIgnoreCase(regionPassed)).findFirst() 58 | .orElse(Region.US_EAST_1); 59 | S3Client s3 = S3Client.builder().region(region).build(); 60 | DynamoDbClient dynamoDB = DynamoDbClient.builder().region(region).build(); 61 | DDBUtil ddbUtil = new DDBUtil(); 62 | 63 | // get Task ARN 64 | String response = ""; 65 | HttpURLConnection con = getHTTPConnectionForTaskMetadataEndpoint(taskMetadataEndpoint.concat("/task")); 66 | System.out.println("HTTP Connection: " + con.getURL().toString()); 67 | try { 68 | response = getFullResponse(con); 69 | System.out.println("Response from HTTP Connection: " + response); 70 | } catch (IOException e) { 71 | e.printStackTrace(); 72 | } 73 | String taskARN = getTaskARN(response); 74 | System.out.println("Task ARN: " + taskARN); 75 | 76 | // insert job running status in DynamoDB table 77 | String status = "Running"; 78 | String insertTime = new Date().toString(); 79 | ddbUtil.insertTaskStatus(dynamoDB, tableName, hashKey, rangeKey, workflowRunId, taskARN, taskName, status, insertTime); 80 | 81 | // perform the task - actual business logic 82 | boolean objectCopied = copyFile(s3, bucketName, objectKey, destinationKey); 83 | 84 | // a random sleep interval from 1 to 3 minutes 85 | int waitTime = (1 + new Random().nextInt(3)) * 60000; 86 | System.out.printf("Task sleeping for %s seconds \n", waitTime); 87 | try { 88 | Thread.sleep(waitTime); 89 | } catch (NumberFormatException e) { 90 | e.printStackTrace(); 91 | } catch (InterruptedException e) { 92 | e.printStackTrace(); 93 | } 94 | 95 | // update job completion status in DynamoDB table 96 | String updateTime = new Date().toString(); 97 | if (objectCopied) 98 | status = "Completed"; 99 | else 100 | status = "Failed"; 101 | 102 | long endTime = System.currentTimeMillis(); 103 | long execTimeinSeconds = (endTime - startTime)/1000; 104 | ddbUtil.updateTaskStatus(dynamoDB, tableName, hashKey, rangeKey, workflowRunId, taskARN, status, updateTime, execTimeinSeconds); 105 | } 106 | 107 | /** 108 | * This method retrieves TaskARN from TaskMetadataEndpoint's response 109 | * @param response 110 | * @return 111 | */ 112 | public static String getTaskARN(String response) { 113 | String taskARN = null; 114 | try { 115 | JsonParser parser = new JsonParser(); 116 | JsonElement rootNode = parser.parse(response); 117 | if (rootNode.isJsonObject()) { 118 | JsonElement jsonElement = rootNode.getAsJsonObject().get("TaskARN"); 119 | taskARN = jsonElement.getAsString(); 120 | } 121 | } catch (Exception e) { 122 | e.printStackTrace(); 123 | } 124 | return taskARN; 125 | } 126 | 127 | /** 128 | * This method prints runtime properties sent to the ECS Task 129 | * @param regionPassed 130 | * @param tableName 131 | * @param hashKey 132 | * @param rangeKey 133 | * @param workflowName 134 | * @param workflowRunId 135 | * @param taskName 136 | * @param bucketName 137 | * @param sourceKey 138 | * @param taskMetadataEndpoint 139 | */ 140 | public static void printInputParameters(String regionPassed, String tableName, String hashKey, 141 | String rangeKey, String workflowName, long workflowRunId, String taskName, String bucketName, 142 | String sourceKey, String taskMetadataEndpoint) { 143 | System.out.println("regionPassed: " + regionPassed); 144 | System.out.println("tableName: " + tableName); 145 | System.out.println("hashKey: " + hashKey); 146 | System.out.println("rangeKey: " + rangeKey); 147 | System.out.println("workflowName: " + workflowName); 148 | System.out.println("workflowRunId: " + workflowRunId); 149 | System.out.println("taskName: " + taskName); 150 | System.out.println("bucketName: " + bucketName); 151 | System.out.println("sourceKey: " + sourceKey); 152 | System.out.println("taskMetadataEndpoint: " + taskMetadataEndpoint); 153 | } 154 | 155 | /** 156 | * This is a representation business logic of the ECS Task. This method creates 157 | * a copy of the input object 158 | * 159 | * @param s3 160 | * @param bucketName 161 | * @param objectKey 162 | * @param destinationKey 163 | */ 164 | public static boolean copyFile(S3Client s3, String bucketName, String objectKey, String destinationKey) { 165 | // Build S3 object key 166 | String encodedUrl = null; 167 | boolean objectCopied = false; 168 | try { 169 | encodedUrl = URLEncoder.encode(bucketName + "/" + objectKey, StandardCharsets.UTF_8.toString()); 170 | } catch (UnsupportedEncodingException e) { 171 | System.out.println("URL could not be encoded: " + e.getMessage()); 172 | } 173 | // Copy object request 174 | CopyObjectRequest copyReq = CopyObjectRequest.builder().copySource(encodedUrl).destinationBucket(bucketName) 175 | .destinationKey(destinationKey).build(); 176 | try { 177 | CopyObjectResponse copyRes = s3.copyObject(copyReq); 178 | if (copyRes.sdkHttpResponse().isSuccessful()) { 179 | System.out.println("Copy operation successful"); 180 | objectCopied = true; 181 | } 182 | else 183 | System.out.println("Copy operation not successful"); 184 | } catch (S3Exception e) { 185 | System.out.println("Exception thrown while copying S3 object"); 186 | System.err.println(e.awsErrorDetails().errorMessage()); 187 | } 188 | return objectCopied; 189 | } 190 | 191 | /** 192 | * This method gets HTTP Connection from ECS TaskMetadataEndpoint 193 | * 194 | * @param taskMetadataEndpoint 195 | * @return 196 | */ 197 | public static HttpURLConnection getHTTPConnectionForTaskMetadataEndpoint(String taskMetadataEndpoint) { 198 | URL url = null; 199 | HttpURLConnection con = null; 200 | try { 201 | url = new URL(taskMetadataEndpoint); 202 | con = (HttpURLConnection) url.openConnection(); 203 | con.setRequestMethod("GET"); 204 | } catch (MalformedURLException e) { 205 | e.printStackTrace(); 206 | } catch (ProtocolException e) { 207 | e.printStackTrace(); 208 | } catch (IOException e) { 209 | e.printStackTrace(); 210 | } 211 | return con; 212 | } 213 | 214 | /** 215 | * This methods gets a response from ECS Task Metadata endpoint 216 | * 217 | * @param con 218 | * @return 219 | * @throws IOException 220 | */ 221 | public static String getFullResponse(HttpURLConnection con) throws IOException { 222 | StringBuilder fullResponseBuilder = new StringBuilder(); 223 | fullResponseBuilder.append(con.getResponseCode()).append(" ").append(con.getResponseMessage()).append("\n"); 224 | con.getHeaderFields().entrySet().stream().filter(entry -> entry.getKey() != null).forEach(entry -> { 225 | fullResponseBuilder.append(entry.getKey()).append(": "); 226 | List headerValues = entry.getValue(); 227 | Iterator it = headerValues.iterator(); 228 | if (it.hasNext()) { 229 | fullResponseBuilder.append(it.next()); 230 | while (it.hasNext()) { 231 | fullResponseBuilder.append(", ").append(it.next()); 232 | } 233 | } 234 | fullResponseBuilder.append("\n"); 235 | }); 236 | Reader streamReader = null; 237 | if (con.getResponseCode() > 299) { 238 | streamReader = new InputStreamReader(con.getErrorStream()); 239 | } else { 240 | streamReader = new InputStreamReader(con.getInputStream()); 241 | } 242 | BufferedReader in = new BufferedReader(streamReader); 243 | String inputLine; 244 | StringBuilder content = new StringBuilder(); 245 | while ((inputLine = in.readLine()) != null) { 246 | content.append(inputLine); 247 | } 248 | in.close(); 249 | return content.toString(); 250 | } 251 | } -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-task/src/main/java/software/aws/ecs/java/starterkit/util/DDBUtil.java: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | package software.aws.ecs.java.starterkit.util; 5 | 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | import software.amazon.awssdk.services.dynamodb.DynamoDbClient; 10 | import software.amazon.awssdk.services.dynamodb.model.AttributeAction; 11 | import software.amazon.awssdk.services.dynamodb.model.AttributeValue; 12 | import software.amazon.awssdk.services.dynamodb.model.AttributeValueUpdate; 13 | import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; 14 | import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; 15 | import software.amazon.awssdk.services.dynamodb.model.ResourceNotFoundException; 16 | import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest; 17 | import software.amazon.awssdk.services.dynamodb.model.UpdateItemResponse; 18 | 19 | public class DDBUtil { 20 | 21 | /** 22 | * This method inserts a record to workflow_details table 23 | * 24 | * @param ddbClient 25 | * @param tableName 26 | * @param hashKey 27 | * @param rangeKey 28 | * @param workflowId 29 | * @param ecsTaskId 30 | * @param status 31 | * @param time 32 | * @return 33 | */ 34 | public boolean insertTaskStatus(DynamoDbClient ddbClient, String tableName, String hashKey, String rangeKey, 35 | long workflowId, String ecsTaskId, String taskName, String status, String time) { 36 | 37 | boolean itemInserted = false; 38 | // Populate item 39 | HashMap itemValues = new HashMap(); 40 | itemValues.put(hashKey, AttributeValue.builder().n(Long.toString(workflowId)).build()); 41 | itemValues.put(rangeKey, AttributeValue.builder().s(ecsTaskId).build()); 42 | itemValues.put("task_name", AttributeValue.builder().s(taskName).build()); 43 | itemValues.put("start_time", AttributeValue.builder().s(time).build()); 44 | itemValues.put("status", AttributeValue.builder().s(status).build()); 45 | // Create a PutItemRequest object 46 | PutItemRequest request = PutItemRequest.builder().tableName(tableName).item(itemValues).build(); 47 | try { 48 | ddbClient.putItem(request); 49 | itemInserted = true; 50 | System.out.printf("An item added to %s successfully. \n", tableName); 51 | 52 | } catch (DynamoDbException e) { 53 | System.err.println(e.getMessage()); 54 | System.exit(1); 55 | } 56 | return itemInserted; 57 | } 58 | 59 | /** 60 | * This method updates a record in workflow_details table 61 | * 62 | * @param ddbClient 63 | * @param tableName 64 | * @param hashKey 65 | * @param rangeKey 66 | * @param workflowId 67 | * @param ecsTaskId 68 | * @param status 69 | * @param time 70 | * @return 71 | */ 72 | public boolean updateTaskStatus(DynamoDbClient ddbClient, String tableName, String hashKey, String rangeKey, 73 | long workflowId, String ecsTaskId, String status, String time, long execTimeinSeconds) { 74 | boolean operationSuccess = false; 75 | // populate Hash Key and Range Key 76 | Map key = new HashMap(); 77 | key.put(hashKey, AttributeValue.builder().n(Long.toString(workflowId)).build()); 78 | key.put(rangeKey, AttributeValue.builder().s(ecsTaskId).build()); 79 | 80 | AttributeAction action = AttributeAction.PUT; 81 | Map attributeUpdates = new HashMap(); 82 | attributeUpdates.put("status", AttributeValueUpdate.builder().action(action) 83 | .value(AttributeValue.builder().s(status).build()).build()); 84 | attributeUpdates.put("update_time", 85 | AttributeValueUpdate.builder().action(action).value(AttributeValue.builder().s(time).build()).build()); 86 | attributeUpdates.put("exec_time_in_seconds", AttributeValueUpdate.builder().action(action) 87 | .value(AttributeValue.builder().n(Long.toString(execTimeinSeconds)).build()).build()); 88 | 89 | UpdateItemRequest updateItemRequest = UpdateItemRequest.builder().tableName(tableName).key(key) 90 | .attributeUpdates(attributeUpdates).build(); 91 | try { 92 | UpdateItemResponse updateItemResponse = ddbClient.updateItem(updateItemRequest); 93 | if (updateItemResponse.sdkHttpResponse().isSuccessful()) { 94 | operationSuccess = true; 95 | System.out.printf("Update item operation with hash_key: %d and range_key: %s was successful. \n", 96 | workflowId, ecsTaskId); 97 | } else 98 | System.out.printf("Update item operation with hash_key: %d and range_key: %s was not successful. \n", 99 | workflowId, ecsTaskId); 100 | } catch (ResourceNotFoundException e) { 101 | e.printStackTrace(); 102 | System.out.println("Table not found"); 103 | } 104 | return operationSuccess; 105 | } 106 | 107 | } 108 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-task/src/main/resources/docker_scripts/build-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | set -e 4 | export AWS_REGION=$1 5 | export AWS_ACCOUNT_ID=$2 6 | export ECR_REPO_NAME=$3 7 | export ECS_TASK_BINARY=$4 8 | aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com 9 | echo "Copying ECS Task binary from ->" $ECS_TASK_BINARY 10 | cp $ECS_TASK_BINARY . 11 | docker build -t $ECR_REPO_NAME . 12 | docker tag amazon-ecs-java-starter-kit-ecr:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPO_NAME:version1.8 13 | docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPO_NAME:version1.8 14 | rm -rfr *.jar -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-task/src/main/resources/ecs_task_metadata_response_sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "Cluster": "arn:aws:ecs:us-east-1:1234567890:cluster/amazon-ecs-java-starter-kit", 3 | "TaskARN": "arn:aws:ecs:us-east-1:1234567890:task/amazon-ecs-java-starter-kit/54ffee5791c74a918b349002f133d6b3", 4 | "Family": "amazon-ecs-java-starter-kit-task-def", 5 | "Revision": "5", 6 | "DesiredStatus": "RUNNING", 7 | "KnownStatus": "RUNNING", 8 | "Containers": [ 9 | { 10 | "DockerId": "7ad0cd7ea75761ce7023eb159cb3d0a3c3008017e04d9d056c8d906141b34497", 11 | "Name": "~internal~ecs~pause", 12 | "DockerName": "ecs-amazon-ecs-java-starter-kit-task-def-5-internalecspause-96d7e1e9b98f9bdf9401", 13 | "Image": "fg-proxy:tinyproxy", 14 | "ImageID": "", 15 | "Labels": { 16 | "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-east-1:1234567890:cluster/amazon-ecs-java-starter-kit", 17 | "com.amazonaws.ecs.container-name": "~internal~ecs~pause", 18 | "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-1:1234567890:task/amazon-ecs-java-starter-kit/54ffee5791c74a918b349002f133d6b3", 19 | "com.amazonaws.ecs.task-definition-family": "amazon-ecs-java-starter-kit-task-def", 20 | "com.amazonaws.ecs.task-definition-version": "5" 21 | }, 22 | "DesiredStatus": "RESOURCES_PROVISIONED", 23 | "KnownStatus": "RESOURCES_PROVISIONED", 24 | "Limits": { 25 | "CPU": 0, 26 | "Memory": 0 27 | }, 28 | "CreatedAt": "2020-11-10T15:57:23.111119079Z", 29 | "StartedAt": "2020-11-10T15:57:23.677759612Z", 30 | "Type": "CNI_PAUSE", 31 | "Networks": [ 32 | { 33 | "NetworkMode": "awsvpc", 34 | "IPv4Addresses": [ 35 | "10.0.1.215" 36 | ] 37 | } 38 | ] 39 | }, 40 | { 41 | "DockerId": "54543ff716f5caa68cc944568a562479cfe3cc25f072ba7d4839f76f073245b7", 42 | "Name": "amazon-ecs-java-starter-kit-ecr", 43 | "DockerName": "ecs-amazon-ecs-java-starter-kit-task-def-5-amazon-ecs-java-starter-kit-ecr-c0eeabc1fbf7b5bf1f00", 44 | "Image": "1234567890.dkr.ecr.us-east-1.amazonaws.com/amazon-ecs-java-starter-kit-ecr:version1.2", 45 | "ImageID": "sha256:f0cfce3b647a530031166a527ebf2582fab91615ccd79c2713dd2ade09834a21", 46 | "Labels": { 47 | "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-east-1:1234567890:cluster/amazon-ecs-java-starter-kit", 48 | "com.amazonaws.ecs.container-name": "amazon-ecs-java-starter-kit-ecr", 49 | "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-east-1:1234567890:task/amazon-ecs-java-starter-kit/54ffee5791c74a918b349002f133d6b3", 50 | "com.amazonaws.ecs.task-definition-family": "amazon-ecs-java-starter-kit-task-def", 51 | "com.amazonaws.ecs.task-definition-version": "5" 52 | }, 53 | "DesiredStatus": "RUNNING", 54 | "KnownStatus": "RUNNING", 55 | "Limits": { 56 | "CPU": 0, 57 | "Memory": 0 58 | }, 59 | "CreatedAt": "2020-11-10T15:59:42.532801596Z", 60 | "StartedAt": "2020-11-10T15:59:53.897514965Z", 61 | "Type": "NORMAL", 62 | "Networks": [ 63 | { 64 | "NetworkMode": "awsvpc", 65 | "IPv4Addresses": [ 66 | "10.0.1.215" 67 | ] 68 | } 69 | ] 70 | } 71 | ], 72 | "Limits": { 73 | "CPU": 1, 74 | "Memory": 2048 75 | }, 76 | "PullStartedAt": "2020-11-10T15:57:23.849651626Z", 77 | "PullStoppedAt": "2020-11-10T15:59:42.517600903Z" 78 | } 79 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-task/src/main/resources/iam_policy_dynamodb.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "VisualEditor0", 6 | "Effect": "Allow", 7 | "Action": [ 8 | "dynamodb:BatchWriteItem", 9 | "dynamodb:PutItem", 10 | "dynamodb:DeleteItem", 11 | "dynamodb:UpdateItem" 12 | ], 13 | "Resource": "*" 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-task/src/main/resources/iam_policy_s3.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "VisualEditor0", 6 | "Effect": "Allow", 7 | "Action": [ 8 | "s3:PutObject", 9 | "s3:GetObject" 10 | ], 11 | "Resource": "*" 12 | } 13 | ] 14 | } -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-task/src/test/java/software/aws/ecs/java/starterkit/task/ECSTaskMetadataParserTest.java: -------------------------------------------------------------------------------- 1 | package software.aws.ecs.java.starterkit.task; 2 | 3 | import java.io.IOException; 4 | import java.nio.charset.StandardCharsets; 5 | import java.nio.file.Files; 6 | import java.nio.file.Paths; 7 | 8 | import org.junit.jupiter.api.Test; 9 | 10 | import com.google.gson.JsonElement; 11 | import com.google.gson.JsonObject; 12 | import com.google.gson.JsonParser; 13 | 14 | class ECSTaskMetadataParserTest { 15 | 16 | @Test 17 | void test() { 18 | String filePath = "./src/main/resources/ecs_task_metadata_response_sample.json"; 19 | String jsonString = ""; 20 | try { 21 | jsonString = new String(Files.readAllBytes(Paths.get(filePath)), StandardCharsets.UTF_8); 22 | System.out.println(jsonString); 23 | } catch (IOException e) { 24 | e.printStackTrace(); 25 | } 26 | 27 | JsonParser parser = new JsonParser(); 28 | JsonElement rootNode = parser.parse(jsonString); 29 | if (rootNode.isJsonObject()) { 30 | JsonObject details = rootNode.getAsJsonObject(); 31 | JsonElement taskARN = details.get("TaskARN"); 32 | System.out.println("TaskARN: " +taskARN.getAsString()); 33 | String taskARNString = taskARN.getAsString(); 34 | System.out.println("String: " + taskARNString); 35 | } 36 | } 37 | 38 | 39 | 40 | 41 | } 42 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-task/src/test/java/software/aws/ecs/java/starterkit/task/RandomTest.java: -------------------------------------------------------------------------------- 1 | package software.aws.ecs.java.starterkit.task; 2 | 3 | import java.util.Random; 4 | 5 | import org.junit.jupiter.api.Test; 6 | 7 | public class RandomTest { 8 | 9 | @Test 10 | void test() { 11 | for (int i = 0; i < 10; i++) { 12 | int waitTime = (1 + new Random().nextInt(3)) * 60000; 13 | System.out.println("Wait_time - " + i + ": " + waitTime); 14 | } 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-tasklauncher/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-tasklauncher/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | .settings/ 3 | build/ 4 | target/classes/ -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-tasklauncher/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | amazon-ecs-java-starter-kit-tasklauncher 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-tasklauncher/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this 4 | software and associated documentation files (the "Software"), to deal in the Software 5 | without restriction, including without limitation the rights to use, copy, modify, 6 | merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 10 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 11 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 12 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 13 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 14 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-tasklauncher/dependency-reduced-pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | amazon-ecs-and-aws-step-functions-design-patterns-starter-kit 5 | software.aws.ecs.samples 6 | 1.0 7 | 8 | 4.0.0 9 | amazon-ecs-java-starter-kit-tasklauncher 10 | amazon-ecs-java-starter-kit-tasklauncher 11 | 12 | 13 | 14 | maven-compiler-plugin 15 | 3.7.0 16 | 17 | 1.8 18 | 1.8 19 | 20 | 21 | 22 | maven-shade-plugin 23 | 3.1.0 24 | 25 | 26 | package 27 | 28 | shade 29 | 30 | 31 | 32 | 33 | software.aws.ecs.java.starterkit.launcher.ECSTaskLauncher 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | org.junit.jupiter 45 | junit-jupiter-engine 46 | 5.6.2 47 | test 48 | 49 | 50 | apiguardian-api 51 | org.apiguardian 52 | 53 | 54 | junit-platform-engine 55 | org.junit.platform 56 | 57 | 58 | junit-jupiter-api 59 | org.junit.jupiter 60 | 61 | 62 | 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-tasklauncher/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 4.0.0 7 | 8 | software.aws.ecs.samples 9 | amazon-ecs-and-aws-step-functions-design-patterns-starter-kit 10 | 1.0 11 | 12 | amazon-ecs-java-starter-kit-tasklauncher 13 | amazon-ecs-java-starter-kit-tasklauncher 14 | 15 | 16 | 17 | 18 | com.amazonaws 19 | aws-lambda-java-events 20 | 1.3.0 21 | 22 | 23 | 24 | com.amazonaws 25 | aws-lambda-java-core 26 | 1.1.0 27 | 28 | 29 | 30 | 31 | 32 | 33 | org.apache.maven.plugins 34 | maven-compiler-plugin 35 | 3.7.0 36 | 37 | 1.8 38 | 1.8 39 | 40 | 41 | 42 | org.apache.maven.plugins 43 | maven-shade-plugin 44 | 3.1.0 45 | 46 | 47 | package 48 | 49 | shade 50 | 51 | 52 | 53 | 55 | software.aws.ecs.java.starterkit.launcher.ECSTaskLauncher 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-tasklauncher/src/main/java/software/aws/ecs/java/starterkit/launcher/ECSTaskLauncher.java: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | package software.aws.ecs.java.starterkit.launcher; 5 | 6 | import com.amazonaws.services.lambda.runtime.Context; 7 | import com.amazonaws.services.lambda.runtime.RequestHandler; 8 | import com.google.gson.Gson; 9 | import software.amazon.awssdk.regions.Region; 10 | import software.amazon.awssdk.services.dynamodb.DynamoDbClient; 11 | import software.amazon.awssdk.services.ecs.EcsClient; 12 | import software.amazon.awssdk.services.ecs.model.*; 13 | import software.aws.ecs.java.starterkit.util.DDBUtil; 14 | import software.aws.ecs.java.starterkit.util.TaskConfig; 15 | import software.aws.ecs.java.starterkit.util.WorkflowSpecs; 16 | 17 | import java.util.*; 18 | import java.util.stream.Collectors; 19 | 20 | /** 21 | * ECSTaskLauncher implemented as an AWS Lambda function. It launches ECS tasks 22 | * based on a Workflow specifications provided as a JSON input. 23 | * 24 | * @author Ravi Itha, Sr. Big Data Consultant 25 | * 26 | */ 27 | public class ECSTaskLauncher implements RequestHandler> { 28 | 29 | @Override 30 | public Map handleRequest(WorkflowSpecs workflowSpecs, Context context) { 31 | 32 | context.getLogger().log("Input event: " + new Gson().toJson(workflowSpecs)); 33 | String regionString = workflowSpecs.getRegion(); 34 | String clusterName = workflowSpecs.getClusterName(); 35 | String containerName = workflowSpecs.getContainerName(); 36 | String taskDefinition = workflowSpecs.getTaskDefinition(); 37 | String securityGroupId = workflowSpecs.getSecurityGroupId(); 38 | String subnetIdLiteral = workflowSpecs.getSubnetIdLiteral(); 39 | String separator = workflowSpecs.getSeparator(); 40 | String ddbTableNameWFSummary = workflowSpecs.getDdbTableNameWFSummary(); 41 | String hashKeyWFSummary = workflowSpecs.getHashKeyWFSummary(); 42 | String rangeKeyWFSummary = workflowSpecs.getRangeKeyWFSummary(); 43 | String ddbTableNameWFDetails = workflowSpecs.getDdbTableNameWFDetails(); 44 | String hashKeyWFDetails = workflowSpecs.getHashKeyWFDetails(); 45 | String rangeKeyWFDetails = workflowSpecs.getRangeKeyWFDetails(); 46 | 47 | printEnvVariables(regionString, clusterName, containerName, taskDefinition, securityGroupId, subnetIdLiteral, 48 | separator, ddbTableNameWFSummary, hashKeyWFSummary, rangeKeyWFSummary, ddbTableNameWFDetails, 49 | hashKeyWFDetails, rangeKeyWFDetails); 50 | 51 | Collection subnetIds = tokenizeStrings(subnetIdLiteral, separator); 52 | Collection securityGroupIds = tokenizeStrings(securityGroupId, separator); 53 | 54 | Region region = Region.regions().stream().filter(r -> r.toString().equalsIgnoreCase(regionString)).findFirst() 55 | .orElse(Region.US_EAST_1); 56 | 57 | List tasks = new ArrayList(); 58 | List ecsTaskArns = new ArrayList(); 59 | DDBUtil ddbUtil = new DDBUtil(); 60 | EcsClient ecs = EcsClient.builder().region(region).build(); 61 | DynamoDbClient dynamoDB = DynamoDbClient.builder().region(region).build(); 62 | 63 | long workflowRunId = System.currentTimeMillis(); 64 | 65 | // TODO: validate the parsing 66 | List taskList = workflowSpecs.getTaskList(); 67 | for (TaskConfig taskConfig : taskList) { 68 | // Prepare Container Overrides -for each ECS task 69 | Collection environment = Arrays.asList( 70 | KeyValuePair.builder().name("region").value(regionString).build(), 71 | KeyValuePair.builder().name("workflow_details_ddb_table_name").value(ddbTableNameWFDetails).build(), 72 | KeyValuePair.builder().name("workflow_details_hash_key").value(hashKeyWFDetails).build(), 73 | KeyValuePair.builder().name("workflow_details_range_key").value(rangeKeyWFDetails).build(), 74 | KeyValuePair.builder().name("workflow_name").value(workflowSpecs.getWorkflowName()).build(), 75 | KeyValuePair.builder().name("workflow_run_id").value(Long.toString(workflowRunId)).build(), 76 | KeyValuePair.builder().name("task_name").value(taskConfig.getTaskName()).build(), 77 | KeyValuePair.builder().name("s3_bucket_name").value(taskConfig.getS3BucketName()).build(), 78 | KeyValuePair.builder().name("object_key").value(taskConfig.getObjectKey()).build()); 79 | 80 | ContainerOverride co = ContainerOverride.builder().environment(environment).name(containerName).build(); 81 | Collection containerOverrides = Arrays.asList(co); 82 | TaskOverride overrides = TaskOverride.builder().containerOverrides(containerOverrides).build(); 83 | 84 | // Submit ECS Task 85 | Task task = submitECSTask(ecs, subnetIds, securityGroupIds, overrides, clusterName, taskDefinition); 86 | tasks.add(task); 87 | ecsTaskArns.add(task.taskArn()); 88 | } 89 | // Insert status to DynamoDB Table 90 | String startTime = new Date().toString(); 91 | ddbUtil.insertWorkflowSummary(dynamoDB, ddbTableNameWFSummary, hashKeyWFSummary, rangeKeyWFSummary, 92 | workflowSpecs.getWorkflowName(), new Gson().toJson(workflowSpecs), workflowRunId, tasks.size(), 93 | "Running", startTime); 94 | 95 | /** 96 | * Prepare response to AWS Step Functions State Machine. This will model 97 | * Iterator design pattern. 98 | * 99 | */ 100 | Map map = new HashMap(); 101 | map.put("workflowName", workflowSpecs.getWorkflowName()); 102 | map.put("workflowRunId", workflowRunId); 103 | map.put("ecsTaskArns", ecsTaskArns); 104 | return map; 105 | } 106 | 107 | /** 108 | * This method runs an ECS Task 109 | * 110 | * @param ecs 111 | * @param subnetIds 112 | * @param securityGroupIds 113 | * @param taskOverrides 114 | * @param clusterName 115 | * @param taskDefinition 116 | * @return 117 | */ 118 | public Task submitECSTask(EcsClient ecs, Collection subnetIds, Collection securityGroupIds, 119 | TaskOverride taskOverrides, String clusterName, String taskDefinition) { 120 | 121 | System.out.println("Submitting ECS Tasks"); 122 | List tasks = null; 123 | AwsVpcConfiguration awsvpcConfiguration = AwsVpcConfiguration.builder().subnets(subnetIds) 124 | .securityGroups(securityGroupIds).build(); 125 | NetworkConfiguration networkConfiguration = NetworkConfiguration.builder() 126 | .awsvpcConfiguration(awsvpcConfiguration).build(); 127 | RunTaskRequest runTaskRequest = RunTaskRequest.builder().cluster(clusterName).taskDefinition(taskDefinition) 128 | .launchType(LaunchType.FARGATE).networkConfiguration(networkConfiguration).overrides(taskOverrides) 129 | .build(); 130 | try { 131 | RunTaskResponse response = ecs.runTask(runTaskRequest); 132 | // Process the response 133 | tasks = response.tasks(); 134 | for (Task task : tasks) { 135 | System.out.println("Task ARN: " + task.taskArn()); 136 | System.out.println("Task Def ARN: " + task.taskDefinitionArn()); 137 | System.out.println("Cluster ARN: " + task.clusterArn()); 138 | System.out.println("Task CPU: " + task.cpu()); 139 | System.out.println("Task Memory: " + task.memory()); 140 | System.out.println("Task Last Status: " + task.lastStatus()); 141 | System.out.println("Task Start Time: " + task.startedAt()); 142 | } 143 | } catch (Exception e) { 144 | e.printStackTrace(); 145 | System.out.println("Cannot run ECS Task."); 146 | } 147 | return tasks.get(0); 148 | } 149 | 150 | /** 151 | * This method tokenizes strings using a provided separator 152 | * 153 | * @param str 154 | * @param separator 155 | * @return 156 | */ 157 | public static List tokenizeStrings(String str, String separator) { 158 | List tokenList = Collections.list(new StringTokenizer(str, separator)).stream() 159 | .map(token -> (String) token).collect(Collectors.toList()); 160 | return tokenList; 161 | } 162 | 163 | /** 164 | * This method prints all input parameters 165 | * 166 | * @param regionString 167 | * @param clusterName 168 | * @param containerName 169 | * @param taskDefinition 170 | * @param securityGroupId 171 | * @param subnetIdLiteral 172 | * @param separator 173 | * @param ddbTableNameWFSummary 174 | * @param hashKeyWFSummary 175 | * @param rangeKeyWFSummary 176 | * @param ddbTableNameWFDetails 177 | * @param hashKeyWFDetails 178 | * @param rangeKeyWFDetails 179 | */ 180 | public static void printEnvVariables(String regionString, String clusterName, String containerName, 181 | String taskDefinition, String securityGroupId, String subnetIdLiteral, String separator, 182 | String ddbTableNameWFSummary, String hashKeyWFSummary, String rangeKeyWFSummary, 183 | String ddbTableNameWFDetails, String hashKeyWFDetails, String rangeKeyWFDetails) { 184 | System.out.println("regionString: " + regionString); 185 | System.out.println("clusterName: " + clusterName); 186 | System.out.println("containerName: " + containerName); 187 | System.out.println("taskDefinition: " + taskDefinition); 188 | System.out.println("securityGroupId: " + securityGroupId); 189 | System.out.println("subnetIdLiteral: " + subnetIdLiteral); 190 | System.out.println("separator: " + separator); 191 | System.out.println("ddbTableNameWFSummary: " + ddbTableNameWFSummary); 192 | System.out.println("hashKeyWFSummary: " + hashKeyWFSummary); 193 | System.out.println("rangeKeyWFSummary: " + rangeKeyWFSummary); 194 | System.out.println("ddbTableNameWFDetails: " + ddbTableNameWFDetails); 195 | System.out.println("hashKeyWFDetails: " + hashKeyWFDetails); 196 | System.out.println("rangeKeyWFDetails: " + rangeKeyWFDetails); 197 | } 198 | 199 | public static WorkflowSpecs parseWorkflowSpecs(String jsonString) { 200 | WorkflowSpecs workflowSpecs = null; 201 | try { 202 | workflowSpecs = new Gson().fromJson(jsonString, WorkflowSpecs.class); 203 | } catch (Exception e) { 204 | e.printStackTrace(); 205 | } 206 | return workflowSpecs; 207 | } 208 | 209 | } 210 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-tasklauncher/src/main/java/software/aws/ecs/java/starterkit/util/DDBUtil.java: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | package software.aws.ecs.java.starterkit.util; 5 | 6 | import java.util.HashMap; 7 | 8 | import software.amazon.awssdk.services.dynamodb.DynamoDbClient; 9 | import software.amazon.awssdk.services.dynamodb.model.AttributeValue; 10 | import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; 11 | import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; 12 | 13 | public class DDBUtil { 14 | 15 | /** 16 | * This method inserts an item to DynamoDB Table 17 | * @param tableName 18 | * @param workflowId 19 | * @param status 20 | * @param numberOfTasks 21 | * @return 22 | */ 23 | public boolean insertWorkflowSummary(DynamoDbClient dynamoDB, String tableName, String hashKey, String rangeKey, 24 | String workflowName, String workflowSpecs, long workflowRunId, int numberOfTasks, String status, 25 | String time) { 26 | boolean itemInserted = false; 27 | HashMap itemValues = new HashMap(); 28 | itemValues.put(hashKey, AttributeValue.builder().s(workflowName).build()); 29 | itemValues.put(rangeKey, AttributeValue.builder().n(Long.toString(workflowRunId)).build()); 30 | itemValues.put("workflow_specs", AttributeValue.builder().s(workflowSpecs).build()); 31 | itemValues.put("number_of_tasks", AttributeValue.builder().n(Integer.toString(numberOfTasks)).build()); 32 | itemValues.put("status", AttributeValue.builder().s(status).build()); 33 | itemValues.put("start_time", AttributeValue.builder().s(time).build()); 34 | 35 | // Create a PutItemRequest object 36 | PutItemRequest request = PutItemRequest.builder().tableName(tableName).item(itemValues).build(); 37 | try { 38 | dynamoDB.putItem(request); 39 | itemInserted = true; 40 | System.out.printf("An item added to %s successfully. \n", tableName); 41 | 42 | } catch (DynamoDbException e) { 43 | System.err.println(e.getMessage()); 44 | } 45 | return itemInserted; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-tasklauncher/src/main/java/software/aws/ecs/java/starterkit/util/TaskConfig.java: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | package software.aws.ecs.java.starterkit.util; 5 | 6 | public class TaskConfig { 7 | 8 | private String taskName; 9 | private String s3BucketName; 10 | private String objectKey; 11 | 12 | public String getTaskName() { 13 | return taskName; 14 | } 15 | public void setTaskName(String taskName) { 16 | this.taskName = taskName; 17 | } 18 | public String getS3BucketName() { 19 | return s3BucketName; 20 | } 21 | public void setS3BucketName(String s3BucketName) { 22 | this.s3BucketName = s3BucketName; 23 | } 24 | public String getObjectKey() { 25 | return objectKey; 26 | } 27 | public void setObjectKey(String objectKey) { 28 | this.objectKey = objectKey; 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-tasklauncher/src/main/java/software/aws/ecs/java/starterkit/util/WorkflowSpecs.java: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | package software.aws.ecs.java.starterkit.util; 5 | 6 | import java.util.List; 7 | 8 | public class WorkflowSpecs { 9 | 10 | private String workflowName; 11 | private String region; 12 | private String clusterName; 13 | private String containerName; 14 | private String taskDefinition; 15 | private String securityGroupId; 16 | private String subnetIdLiteral; 17 | private String separator; 18 | private String ddbTableNameWFSummary; 19 | private String hashKeyWFSummary; 20 | private String rangeKeyWFSummary; 21 | private String ddbTableNameWFDetails; 22 | private String hashKeyWFDetails; 23 | private String rangeKeyWFDetails; 24 | private List taskList; 25 | 26 | public String getWorkflowName() { 27 | return workflowName; 28 | } 29 | 30 | public void setWorkflowName(String workflowName) { 31 | this.workflowName = workflowName; 32 | } 33 | 34 | public List getTaskList() { 35 | return taskList; 36 | } 37 | 38 | public void setTaskList(List taskList) { 39 | this.taskList = taskList; 40 | } 41 | 42 | public String getRegion() { 43 | return region; 44 | } 45 | 46 | public void setRegion(String region) { 47 | this.region = region; 48 | } 49 | 50 | public String getClusterName() { 51 | return clusterName; 52 | } 53 | 54 | public void setClusterName(String clusterName) { 55 | this.clusterName = clusterName; 56 | } 57 | 58 | public String getContainerName() { 59 | return containerName; 60 | } 61 | 62 | public void setContainerName(String containerName) { 63 | this.containerName = containerName; 64 | } 65 | 66 | public String getTaskDefinition() { 67 | return taskDefinition; 68 | } 69 | 70 | public void setTaskDefinition(String taskDefinition) { 71 | this.taskDefinition = taskDefinition; 72 | } 73 | 74 | 75 | 76 | public String getSecurityGroupId() { 77 | return securityGroupId; 78 | } 79 | 80 | public void setSecurityGroupId(String securityGroupId) { 81 | this.securityGroupId = securityGroupId; 82 | } 83 | 84 | public String getSubnetIdLiteral() { 85 | return subnetIdLiteral; 86 | } 87 | 88 | public void setSubnetIdLiteral(String subnetIdLiteral) { 89 | this.subnetIdLiteral = subnetIdLiteral; 90 | } 91 | 92 | public String getSeparator() { 93 | return separator; 94 | } 95 | 96 | public void setSeparator(String separator) { 97 | this.separator = separator; 98 | } 99 | 100 | public String getDdbTableNameWFSummary() { 101 | return ddbTableNameWFSummary; 102 | } 103 | 104 | public void setDdbTableNameWFSummary(String ddbTableNameWFSummary) { 105 | this.ddbTableNameWFSummary = ddbTableNameWFSummary; 106 | } 107 | 108 | public String getHashKeyWFSummary() { 109 | return hashKeyWFSummary; 110 | } 111 | 112 | public void setHashKeyWFSummary(String hashKeyWFSummary) { 113 | this.hashKeyWFSummary = hashKeyWFSummary; 114 | } 115 | 116 | public String getRangeKeyWFSummary() { 117 | return rangeKeyWFSummary; 118 | } 119 | 120 | public void setRangeKeyWFSummary(String rangeKeyWFSummary) { 121 | this.rangeKeyWFSummary = rangeKeyWFSummary; 122 | } 123 | 124 | public String getDdbTableNameWFDetails() { 125 | return ddbTableNameWFDetails; 126 | } 127 | 128 | public void setDdbTableNameWFDetails(String ddbTableNameWFDetails) { 129 | this.ddbTableNameWFDetails = ddbTableNameWFDetails; 130 | } 131 | 132 | public String getHashKeyWFDetails() { 133 | return hashKeyWFDetails; 134 | } 135 | 136 | public void setHashKeyWFDetails(String hashKeyWFDetails) { 137 | this.hashKeyWFDetails = hashKeyWFDetails; 138 | } 139 | 140 | public String getRangeKeyWFDetails() { 141 | return rangeKeyWFDetails; 142 | } 143 | 144 | public void setRangeKeyWFDetails(String rangeKeyWFDetails) { 145 | this.rangeKeyWFDetails = rangeKeyWFDetails; 146 | } 147 | 148 | } 149 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-tasklauncher/src/main/resources/amazon_cloudwatch_logs_policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "VisualEditor0", 6 | "Effect": "Allow", 7 | "Action": [ 8 | "logs:CreateLogStream", 9 | "logs:CreateLogGroup", 10 | "logs:PutLogEvents" 11 | ], 12 | "Resource": "*" 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-tasklauncher/src/main/resources/amazon_dynamodb_policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "VisualEditor0", 6 | "Effect": "Allow", 7 | "Action": [ 8 | "dynamodb:BatchGetItem", 9 | "dynamodb:BatchWriteItem", 10 | "dynamodb:PutItem", 11 | "dynamodb:GetItem", 12 | "dynamodb:Query", 13 | "dynamodb:UpdateItem" 14 | ], 15 | "Resource": "*" 16 | } 17 | ] 18 | } -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-tasklauncher/src/main/resources/amazon_ecs_policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "VisualEditor0", 6 | "Effect": "Allow", 7 | "Action": "ecs:RunTask", 8 | "Resource": "*" 9 | } 10 | ] 11 | } -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-tasklauncher/src/main/resources/aws_lambda_assume_role_policy_doc.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Principal": { 7 | "Service": "lambda.amazonaws.com" 8 | }, 9 | "Action": "sts:AssumeRole" 10 | } 11 | ] 12 | } -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-tasklauncher/src/main/resources/iam_pass_role_policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "VisualEditor0", 6 | "Effect": "Allow", 7 | "Action": "iam:PassRole", 8 | "Resource": "*" 9 | } 10 | ] 11 | } -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-taskmonitor/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-taskmonitor/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | .settings/ 3 | build/ 4 | target/classes/ -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-taskmonitor/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | amazon-ecs-java-starter-kit-taskmonitor 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-taskmonitor/dependency-reduced-pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | amazon-ecs-and-aws-step-functions-design-patterns-starter-kit 5 | software.aws.ecs.samples 6 | 1.0 7 | 8 | 4.0.0 9 | amazon-ecs-java-starter-kit-taskmonitor 10 | amazon-ecs-java-starter-kit-taskmonitor 11 | 12 | 13 | 14 | maven-compiler-plugin 15 | 3.7.0 16 | 17 | 1.8 18 | 1.8 19 | 20 | 21 | 22 | maven-shade-plugin 23 | 3.1.0 24 | 25 | 26 | package 27 | 28 | shade 29 | 30 | 31 | 32 | 33 | software.aws.ecs.java.starterkit.monitor.ECSTaskMonitor 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | org.junit.jupiter 45 | junit-jupiter-engine 46 | 5.6.2 47 | test 48 | 49 | 50 | apiguardian-api 51 | org.apiguardian 52 | 53 | 54 | junit-platform-engine 55 | org.junit.platform 56 | 57 | 58 | junit-jupiter-api 59 | org.junit.jupiter 60 | 61 | 62 | 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-taskmonitor/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 4.0.0 7 | 8 | software.aws.ecs.samples 9 | amazon-ecs-and-aws-step-functions-design-patterns-starter-kit 10 | 1.0 11 | 12 | amazon-ecs-java-starter-kit-taskmonitor 13 | amazon-ecs-java-starter-kit-taskmonitor 14 | 15 | 16 | 17 | com.amazonaws 18 | aws-lambda-java-events 19 | 1.3.0 20 | 21 | 22 | 23 | 24 | com.amazonaws 25 | aws-lambda-java-core 26 | 1.1.0 27 | 28 | 29 | 30 | 31 | 32 | org.apache.maven.plugins 33 | maven-compiler-plugin 34 | 3.7.0 35 | 36 | 1.8 37 | 1.8 38 | 39 | 40 | 41 | org.apache.maven.plugins 42 | maven-shade-plugin 43 | 3.1.0 44 | 45 | 46 | package 47 | 48 | shade 49 | 50 | 51 | 52 | 54 | software.aws.ecs.java.starterkit.monitor.ECSTaskMonitor 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-taskmonitor/src/main/java/software/aws/ecs/java/starterkit/monitor/ECSTaskMonitor.java: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | package software.aws.ecs.java.starterkit.monitor; 5 | 6 | import com.amazonaws.regions.Regions; 7 | import com.amazonaws.services.lambda.runtime.Context; 8 | import com.amazonaws.services.lambda.runtime.LambdaLogger; 9 | import com.amazonaws.services.lambda.runtime.RequestHandler; 10 | import com.google.gson.Gson; 11 | import com.google.gson.GsonBuilder; 12 | import software.amazon.awssdk.regions.Region; 13 | import software.amazon.awssdk.services.dynamodb.DynamoDbClient; 14 | import software.amazon.awssdk.services.dynamodb.model.AttributeValue; 15 | import software.aws.ecs.java.starterkit.monitor.model.Input; 16 | import software.aws.ecs.java.starterkit.monitor.model.WorkflowStatus; 17 | import software.aws.ecs.java.starterkit.util.DDBUtil; 18 | 19 | import java.util.*; 20 | 21 | public class ECSTaskMonitor implements RequestHandler> { 22 | @Override 23 | public Map handleRequest(Input input, Context context) { 24 | Gson gson = new GsonBuilder().setPrettyPrinting().create(); 25 | LambdaLogger logger = context.getLogger(); 26 | logger.log("Input event type: " + input.getClass().toString()); 27 | logger.log("Input event: " + gson.toJson(input)); 28 | logger.log("Workflow Run Id: " + input.getIterator().getWorkflowRunId()); 29 | logger.log("Workflow Name: " + input.getIterator().getWorkflowName()); 30 | logger.log("Numm of Task ARNs: " + input.getIterator().getEcsTaskArns().size()); 31 | for (String taskARN : input.getIterator().getEcsTaskArns()) { 32 | logger.log("Task ARN: " + taskARN); 33 | } 34 | 35 | String regionString = Optional.ofNullable(System.getenv("region")).orElse(Regions.US_EAST_1.getName()); 36 | String ddbTableNameWFSummary = Optional.ofNullable(System.getenv("workflow_summary_ddb_table_name")) 37 | .orElse("workflow_summary"); 38 | String hashKeyWFSummary = Optional.ofNullable(System.getenv("workflow_summary_hash_key")) 39 | .orElse("workflow_name"); 40 | String rangeKeyWFSummary = Optional.ofNullable(System.getenv("workflow_summary_range_key")) 41 | .orElse("workflow_run_id"); 42 | String ddbTableNameWFDetails = Optional.ofNullable(System.getenv("workflow_details_ddb_table_name")) 43 | .orElse("workflow_details"); 44 | String hashKeyWFDetails = Optional.ofNullable(System.getenv("workflow_details_hash_key")) 45 | .orElse("workflow_run_id"); 46 | String rangeKeyWFDetails = Optional.ofNullable(System.getenv("workflow_details_range_key")) 47 | .orElse("ecs_task_id"); 48 | 49 | printEnvVariables(logger, regionString, ddbTableNameWFSummary, hashKeyWFSummary, rangeKeyWFSummary, 50 | ddbTableNameWFDetails, hashKeyWFDetails, rangeKeyWFDetails); 51 | 52 | List completedTasks = new ArrayList(); 53 | List failedTasks = new ArrayList(); 54 | List runningTasks = new ArrayList(); 55 | WorkflowStatus workflowStatus = new WorkflowStatus(); 56 | 57 | Region region = Region.regions().stream().filter(r -> r.toString().equalsIgnoreCase(regionString)).findFirst() 58 | .orElse(Region.US_EAST_1); 59 | DDBUtil ddbUtil = new DDBUtil(); 60 | DynamoDbClient dynamoDB = DynamoDbClient.builder().region(region).build(); 61 | 62 | // Populate the iterator object 63 | Map map = new HashMap(); 64 | map.put("workflowName", input.getIterator().getWorkflowName()); 65 | map.put("workflowRunId", input.getIterator().getWorkflowRunId()); 66 | map.put("ecsTaskArns", input.getIterator().getEcsTaskArns()); 67 | 68 | // get all completed tasks from Workflow Details table 69 | List> tasks = ddbUtil.getWorkflowDetails(dynamoDB, ddbTableNameWFDetails, 70 | hashKeyWFDetails, input.getIterator().getWorkflowRunId()); 71 | System.out.printf("Number of Tasks retrieved from DDB: %d\n", tasks.size()); 72 | 73 | // iterate the items and derive statistics 74 | for (Map item : tasks) { 75 | String status = ""; 76 | String ecsTaskId = ""; 77 | for (Map.Entry entry : item.entrySet()) { 78 | logger.log(entry.getKey() + ":" + entry.getValue()); 79 | if (entry.getKey().equalsIgnoreCase("status")) { 80 | status = entry.getValue().s(); 81 | } else { 82 | ecsTaskId = entry.getValue().s(); 83 | } 84 | } 85 | if (status.equalsIgnoreCase("Completed")) 86 | completedTasks.add(ecsTaskId); 87 | else if (status.equalsIgnoreCase("Failed")) 88 | failedTasks.add(ecsTaskId); 89 | else 90 | runningTasks.add(ecsTaskId); 91 | } 92 | workflowStatus.setCompletedTasks(completedTasks); 93 | workflowStatus.setFailedTasks(failedTasks); 94 | workflowStatus.setRunningTasks(runningTasks); 95 | 96 | if (input.getIterator().getEcsTaskArns().size() == completedTasks.size() + failedTasks.size()) { 97 | System.out.printf("ECS Workflow Status: Completed tasks = %d, Failed tasks = %d, Running tasks = %d \n", 98 | completedTasks.size(), failedTasks.size(), runningTasks.size()); 99 | workflowStatus.setStatus("Completed"); 100 | map.put("continue", false); 101 | } else { 102 | System.out.printf("ECS Workflow Status: Completed tasks = %d, Failed tasks = %d, Running tasks = %d \n", 103 | completedTasks.size(), failedTasks.size(), runningTasks.size()); 104 | workflowStatus.setStatus("Running"); 105 | map.put("continue", true); 106 | } 107 | // updated workflow summary in DynamoDB 108 | ddbUtil.updateWorkflowSummary(dynamoDB, ddbTableNameWFSummary, hashKeyWFSummary, rangeKeyWFSummary, 109 | input.getIterator().getWorkflowName(), input.getIterator().getWorkflowRunId(), workflowStatus.getStatus(), new Date().toString(), 110 | completedTasks.size(), failedTasks.size(), runningTasks.size()); 111 | 112 | return map; 113 | } 114 | 115 | /** 116 | * This method prints environment variables 117 | * 118 | * @param regionString 119 | * @param ddbTableNameWFSummary 120 | * @param hashKeyWFSummary 121 | * @param rangeKeyWFSummary 122 | * @param ddbTableNameWFDetails 123 | * @param hashKeyWFDetails 124 | * @param rangeKeyWFDetails 125 | */ 126 | public static void printEnvVariables(LambdaLogger logger, String regionString, String ddbTableNameWFSummary, String hashKeyWFSummary, 127 | String rangeKeyWFSummary, String ddbTableNameWFDetails, String hashKeyWFDetails, String rangeKeyWFDetails) { 128 | logger.log("regionString: " + regionString); 129 | logger.log("ddbTableNameWFSummary: " + ddbTableNameWFSummary); 130 | logger.log("hashKeyWFSummary: " + hashKeyWFSummary); 131 | logger.log("rangeKeyWFSummary: " + rangeKeyWFSummary); 132 | logger.log("ddbTableNameWFDetails: " + ddbTableNameWFDetails); 133 | logger.log("hashKeyWFDetails: " + hashKeyWFDetails); 134 | logger.log("rangeKeyWFDetails: " + rangeKeyWFDetails); 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-taskmonitor/src/main/java/software/aws/ecs/java/starterkit/monitor/model/Input.java: -------------------------------------------------------------------------------- 1 | package software.aws.ecs.java.starterkit.monitor.model; 2 | 3 | public class Input { 4 | 5 | private Iterator iterator; 6 | 7 | public Iterator getIterator() { 8 | return iterator; 9 | } 10 | 11 | public void setIterator(Iterator iterator) { 12 | this.iterator = iterator; 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-taskmonitor/src/main/java/software/aws/ecs/java/starterkit/monitor/model/Iterator.java: -------------------------------------------------------------------------------- 1 | package software.aws.ecs.java.starterkit.monitor.model; 2 | 3 | import java.util.List; 4 | 5 | public class Iterator { 6 | 7 | private String workflowName; 8 | private long workflowRunId; 9 | private List ecsTaskArns; 10 | public String getWorkflowName() { 11 | return workflowName; 12 | } 13 | public void setWorkflowName(String workflowName) { 14 | this.workflowName = workflowName; 15 | } 16 | public long getWorkflowRunId() { 17 | return workflowRunId; 18 | } 19 | public void setWorkflowRunId(long workflowRunId) { 20 | this.workflowRunId = workflowRunId; 21 | } 22 | public List getEcsTaskArns() { 23 | return ecsTaskArns; 24 | } 25 | public void setEcsTaskArns(List ecsTaskArns) { 26 | this.ecsTaskArns = ecsTaskArns; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-taskmonitor/src/main/java/software/aws/ecs/java/starterkit/monitor/model/WorkflowStatus.java: -------------------------------------------------------------------------------- 1 | package software.aws.ecs.java.starterkit.monitor.model; 2 | 3 | import java.util.List; 4 | 5 | public class WorkflowStatus { 6 | 7 | private String status; 8 | private List completedTasks; 9 | private List failedTasks; 10 | private List runningTasks; 11 | 12 | public String getStatus() { 13 | return status; 14 | } 15 | public void setStatus(String status) { 16 | this.status = status; 17 | } 18 | public List getCompletedTasks() { 19 | return completedTasks; 20 | } 21 | public void setCompletedTasks(List completedTasks) { 22 | this.completedTasks = completedTasks; 23 | } 24 | public List getFailedTasks() { 25 | return failedTasks; 26 | } 27 | public void setFailedTasks(List failedTasks) { 28 | this.failedTasks = failedTasks; 29 | } 30 | public List getRunningTasks() { 31 | return runningTasks; 32 | } 33 | public void setRunningTasks(List runningTasks) { 34 | this.runningTasks = runningTasks; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /amazon-ecs-java-starter-kit-taskmonitor/src/main/java/software/aws/ecs/java/starterkit/util/DDBUtil.java: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | package software.aws.ecs.java.starterkit.util; 5 | 6 | import java.util.Arrays; 7 | import java.util.Collection; 8 | import java.util.HashMap; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | import software.amazon.awssdk.services.dynamodb.DynamoDbClient; 13 | import software.amazon.awssdk.services.dynamodb.model.AttributeAction; 14 | import software.amazon.awssdk.services.dynamodb.model.AttributeValue; 15 | import software.amazon.awssdk.services.dynamodb.model.AttributeValueUpdate; 16 | import software.amazon.awssdk.services.dynamodb.model.Condition; 17 | import software.amazon.awssdk.services.dynamodb.model.QueryRequest; 18 | import software.amazon.awssdk.services.dynamodb.model.QueryResponse; 19 | import software.amazon.awssdk.services.dynamodb.model.ResourceNotFoundException; 20 | import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest; 21 | import software.amazon.awssdk.services.dynamodb.model.UpdateItemResponse; 22 | 23 | public class DDBUtil { 24 | 25 | /** 26 | * This method gets the status of ECS tasks for a given workflow_run_id 27 | * @param dynamoDB 28 | * @param tableName 29 | * @param hashKey 30 | * @param hashKeyValue 31 | * @param statusKey 32 | * @param statusKeyValue 33 | * @return 34 | */ 35 | public List> getWorkflowDetails(DynamoDbClient dynamoDB, String tableName, 36 | String hashKey, long hashKeyValue, String statusKey, String statusKeyValue) { 37 | 38 | String keyConditionExpression = "#part_key = :workflowRunId"; 39 | String filterExpression = "#status = :required_status"; 40 | 41 | Map expressionAttributeNames = new HashMap(); 42 | expressionAttributeNames.put("#part_key", hashKey); 43 | expressionAttributeNames.put("#status", statusKey); 44 | 45 | Map expressionAttributeValues = new HashMap(); 46 | expressionAttributeValues.put(":workflowRunId", AttributeValue.builder().n(Long.toString(hashKeyValue)).build()); 47 | expressionAttributeValues.put(":required_status", AttributeValue.builder().s(statusKeyValue).build()); 48 | 49 | QueryRequest queryRequest = QueryRequest.builder().tableName(tableName) 50 | .keyConditionExpression(keyConditionExpression).filterExpression(filterExpression) 51 | .expressionAttributeNames(expressionAttributeNames).expressionAttributeValues(expressionAttributeValues) 52 | .build(); 53 | QueryResponse response = dynamoDB.query(queryRequest); 54 | List> items = response.items(); 55 | return items; 56 | } 57 | 58 | /** 59 | * This method gets the status of ECS tasks for a given workflow_run_id. It 60 | * retrieves data for all attributes. 61 | * @param dynamoDB 62 | * @param tableName 63 | * @param hashKey 64 | * @param hashKeyValue 65 | * @return 66 | */ 67 | public List> getWorkflowDetails(DynamoDbClient dynamoDB, 68 | String tableName, String hashKey, long hashKeyValue) { 69 | 70 | String keyConditionExpression = "#part_key = :workflowRunId"; 71 | Map expressionAttributeNames = new HashMap(); 72 | expressionAttributeNames.put("#part_key", hashKey); 73 | Map expressionAttributeValues = new HashMap(); 74 | expressionAttributeValues.put(":workflowRunId", 75 | AttributeValue.builder().n(Long.toString(hashKeyValue)).build()); 76 | 77 | Collection attributesToGet = Arrays.asList("status"); 78 | 79 | QueryRequest queryRequest = QueryRequest.builder().tableName(tableName) 80 | .keyConditionExpression(keyConditionExpression).expressionAttributeNames(expressionAttributeNames) 81 | .expressionAttributeValues(expressionAttributeValues).build(); 82 | 83 | Map keyConditions = new HashMap(); 84 | Collection attributeValueList = Arrays.asList(AttributeValue.builder().n(Long.toString(hashKeyValue)).build()); 85 | keyConditions.put(hashKey, Condition.builder().attributeValueList(attributeValueList).comparisonOperator("EQ").build()); 86 | 87 | QueryRequest queryRequest2 = QueryRequest.builder().tableName(tableName).keyConditions(keyConditions).attributesToGet(attributesToGet).build(); 88 | dynamoDB.query(queryRequest2); 89 | 90 | QueryResponse response = dynamoDB.query(queryRequest); 91 | List> items = response.items(); 92 | return items; 93 | } 94 | 95 | /** 96 | * This method gets the status of ECS tasks for a given workflow_run_id. It 97 | * retrieves data for only few attributes. 98 | * 99 | * @param dynamoDB 100 | * @param tableName 101 | * @param hashKey 102 | * @param hashKeyValue 103 | * @param rangeKey 104 | * @return 105 | */ 106 | public List> getWorkflowDetails(DynamoDbClient dynamoDB, 107 | String tableName, String hashKey, long hashKeyValue, String rangeKey) { 108 | 109 | Collection attributesToGet = Arrays.asList("status", rangeKey); 110 | Map keyConditions = new HashMap(); 111 | Collection attributeValueList = Arrays 112 | .asList(AttributeValue.builder().n(Long.toString(hashKeyValue)).build()); 113 | keyConditions.put(hashKey, 114 | Condition.builder().attributeValueList(attributeValueList).comparisonOperator("EQ").build()); 115 | 116 | QueryRequest queryRequest = QueryRequest.builder().tableName(tableName).keyConditions(keyConditions) 117 | .attributesToGet(attributesToGet).build(); 118 | 119 | QueryResponse response = dynamoDB.query(queryRequest); 120 | List> items = response.items(); 121 | return items; 122 | } 123 | 124 | /** 125 | * This method updates the status of Workflow Summary 126 | * @param dynamoDB 127 | * @param tableName 128 | * @param hashKey 129 | * @param rangeKey 130 | * @param workflowName 131 | * @param workflowRunId 132 | * @param status 133 | * @param time 134 | * @return 135 | */ 136 | public boolean updateWorkflowSummary(DynamoDbClient dynamoDB, String tableName, String hashKey, String rangeKey, 137 | String workflowName, long workflowRunId, String status, String time, int completedTasks, int failedTasks, 138 | int runningTasks) { 139 | boolean operationSuccess = false; 140 | 141 | // populate Hash Key and Range Key 142 | Map key = new HashMap(); 143 | key.put(hashKey, AttributeValue.builder().s(workflowName).build()); 144 | key.put(rangeKey, AttributeValue.builder().n(Long.toString(workflowRunId)).build()); 145 | 146 | AttributeAction action = AttributeAction.PUT; 147 | Map attributeUpdates = new HashMap(); 148 | attributeUpdates.put("status", AttributeValueUpdate.builder().action(action) 149 | .value(AttributeValue.builder().s(status).build()).build()); 150 | attributeUpdates.put("update_time", 151 | AttributeValueUpdate.builder().action(action).value(AttributeValue.builder().s(time).build()).build()); 152 | attributeUpdates.put("completed_tasks", 153 | AttributeValueUpdate.builder().action(action).value(AttributeValue.builder().n(Integer.toString(completedTasks)).build()).build()); 154 | attributeUpdates.put("failed_tasks", 155 | AttributeValueUpdate.builder().action(action).value(AttributeValue.builder().n(Integer.toString(failedTasks)).build()).build()); 156 | attributeUpdates.put("running_tasks", 157 | AttributeValueUpdate.builder().action(action).value(AttributeValue.builder().n(Integer.toString(runningTasks)).build()).build()); 158 | 159 | UpdateItemRequest updateItemRequest = UpdateItemRequest.builder().tableName(tableName).key(key) 160 | .attributeUpdates(attributeUpdates).build(); 161 | try { 162 | UpdateItemResponse updateItemResponse = dynamoDB.updateItem(updateItemRequest); 163 | if (updateItemResponse.sdkHttpResponse().isSuccessful()) { 164 | operationSuccess = true; 165 | System.out.printf("Update item operation with hash_key: %s and range_key: %d was successful. \n", 166 | workflowName, workflowRunId); 167 | } else 168 | System.out.printf("Update item operation with hash_key: %s and range_key: %d was not successful. \n", 169 | workflowName, workflowRunId); 170 | } catch (ResourceNotFoundException e) { 171 | e.printStackTrace(); 172 | System.out.println("Table not found"); 173 | } 174 | return operationSuccess; 175 | } 176 | 177 | } 178 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | software.aws.ecs.samples 7 | amazon-ecs-and-aws-step-functions-design-patterns-starter-kit 8 | pom 9 | 1.0 10 | amazon-ecs-java-starter-kit 11 | 12 | 13 | 14 | 15 | software.amazon.awssdk 16 | ecs 17 | 2.15.19 18 | 19 | 20 | 21 | software.amazon.awssdk 22 | dynamodb 23 | 2.15.19 24 | 25 | 26 | 27 | com.google.code.gson 28 | gson 29 | 2.8.9 30 | 31 | 32 | 33 | com.google.guava 34 | guava 35 | 30.0-jre 36 | 37 | 38 | 39 | org.junit.jupiter 40 | junit-jupiter-engine 41 | 5.6.2 42 | test 43 | 44 | 45 | 46 | 47 | amazon-ecs-java-starter-kit-cdk 48 | amazon-ecs-java-starter-kit-tasklauncher 49 | amazon-ecs-java-starter-kit-task 50 | amazon-ecs-java-starter-kit-taskmonitor 51 | 52 | 53 | -------------------------------------------------------------------------------- /resources/Amazon_ECS_Java_Starter_Kit-Architecture_Pattern_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-ecs-and-aws-step-functions-design-patterns-starter-kit/b951bec451f2b4780758da7e0e8b188fe7780858/resources/Amazon_ECS_Java_Starter_Kit-Architecture_Pattern_1.png -------------------------------------------------------------------------------- /resources/Amazon_ECS_Java_Starter_Kit-Architecture_Pattern_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-ecs-and-aws-step-functions-design-patterns-starter-kit/b951bec451f2b4780758da7e0e8b188fe7780858/resources/Amazon_ECS_Java_Starter_Kit-Architecture_Pattern_2.png -------------------------------------------------------------------------------- /resources/Pattern_1_execution_outcome.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-ecs-and-aws-step-functions-design-patterns-starter-kit/b951bec451f2b4780758da7e0e8b188fe7780858/resources/Pattern_1_execution_outcome.png -------------------------------------------------------------------------------- /resources/Pattern_2_execution_outcome.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-ecs-and-aws-step-functions-design-patterns-starter-kit/b951bec451f2b4780758da7e0e8b188fe7780858/resources/Pattern_2_execution_outcome.png -------------------------------------------------------------------------------- /resources/output_of_bootstrap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-ecs-and-aws-step-functions-design-patterns-starter-kit/b951bec451f2b4780758da7e0e8b188fe7780858/resources/output_of_bootstrap.png -------------------------------------------------------------------------------- /resources/pattern_1_test_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-ecs-and-aws-step-functions-design-patterns-starter-kit/b951bec451f2b4780758da7e0e8b188fe7780858/resources/pattern_1_test_output.png --------------------------------------------------------------------------------