├── .gitattributes ├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── Pipfile ├── Pipfile.lock ├── README.md ├── awspack ├── build.sh ├── compile.sh ├── deploy.sh └── entrypoint.sh ├── build.sh ├── deploy.sh ├── docker_build.sh ├── example ├── .gitignore ├── matrix.R └── script.R ├── integration_test.sh ├── r ├── build.sh └── compile.sh ├── recommended ├── build.sh └── deploy.sh ├── remote_compile_and_deploy.sh ├── runtime ├── build.sh ├── deploy.sh └── src │ ├── bootstrap │ ├── bootstrap.R │ └── runtime.R ├── template.yaml ├── test-template.yaml └── tests ├── R ├── api.R ├── aws.R ├── lowercase.r ├── matrix.R └── script.R ├── __init__.py ├── sam.py ├── test_api.py ├── test_aws.py ├── test_matrix.py └── test_runtime.py /.gitattributes: -------------------------------------------------------------------------------- 1 | *.sh text eol=lf 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.zip 2 | *.log 3 | __pycache__/ 4 | .idea/ 5 | *.iml 6 | build/ 7 | venv/ 8 | packaged.yaml 9 | .Rhistory 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.7" 4 | dist: xenial 5 | services: 6 | - docker 7 | cache: 8 | directories: 9 | - $HOME/.cache/pip 10 | matrix: 11 | include: 12 | - name: R 3.5.1 13 | env: R_VERSION=3.5.1 14 | - name: R 3.5.3 15 | env: R_VERSION=3.5.3 16 | - name: R 3.6.0 17 | env: R_VERSION=3.6.0 18 | install: 19 | - pipenv install --dev 20 | script: 21 | - ./build.sh ${R_VERSION} && LOGLEVEL=INFO VERSION=${R_VERSION//\./_} pipenv run python -m unittest && if [ "$TRAVIS_BRANCH" = "master" ]; then ./integration_test.sh ${R_VERSION} ; fi 22 | deploy: 23 | - provider: script 24 | script: ./deploy.sh $R_VERSION 25 | skip_cleanup: true 26 | on: 27 | branch: master 28 | env: 29 | global: 30 | - AWS_DEFAULT_REGION=eu-central-1 31 | - PIPENV_VERBOSITY=-1 32 | - secure: lcvzavlau1syMM1uMZVSIHMUu8nGA9zN3KLqfJ4OiVVNEYxK0wrRtJKJyLCZM+SfEGoUkreLRLjEcQePDAV5+0A1xPEc+hnCWbDjAsl5Cj2t3ei76dKM5xwEAspMGaw+D1kwgWrreJXYzDvVDbgNc74XH49TpGMyKc4XGxFpvoixY5CjsYgigrvNn9OGfNjZkku99TRdxjBE14cmxtAwS3tMM5PL45Of4ZyrJ4Jl1Jeo5qncihNfHWDvSySb0MhdSBpYvXLPLffUS8tTurvwkRWEwcdPmjok04g+gKElvyEwUx1YmRvALBW7YdJ8Z58OaYIDhi7MGL6AM5fguPy+ldHy97zzVzhExIo5gdSM+IHVtjzPwTeICIT/Jh1OjbjN2wcnbV197LBdIHqSaaLibTuxlFwig6BFPq2hmWYu3ChtmheyVza8pZ9XTuutoWhO92uTvClJzI3cuD+pAFRKslnhIgClP5ifd1JA9HnzB/29bSGVwG5SD7dP11eTxfhXX44xfDkRPzs+GOTZH7Tc63AjuSayR7RM2IE//036LoRNRDZC4NCNrjKyBW2tROtR+BW06cY077kEmt2I4jEyHXb74ivsmD2JLSjCX5kf3oBI71zk/xWkM0NHV4ASVODzyyUShaYar+q2K/j4JqJQnATAMrwpK/yJopHMIa4Jx9A= 33 | - secure: GIVFD4H5E1Azi+41/lZqg3DX1dYu0oMaIOkr9GOY1UrzAmCgXRJ9A3P3b6UxGTltANy26jVROrDbeBhKrys5xUyPaUJTWV2oJEn8aMXbIQU9hQlxgRwJRLxBA1BkD6G59N52hlvV1K6p+jsrklHpB6NOvHrzH8Dsf17A9vtXb3FhPrDM+3aZIWNhib69w56kGqF9scJcBEAnjK9GAaz80jzwS620h3jn4gtft0gWKm80DB9wytBC/MuckfrMp9XkRKuRGFeFcgJaXRJzfd1Od4UnJCmbWdRB8bhHZUhW/0BtbgS+kjHm+zDAqqInYyx9Fv1dH1AeanLGxu3CsJJe5obNjeMtXKBKzTWf9n8IbLCi3jg050CVZ7zh2MP/gPZVdVvkmPAi1ZZDMwZhrAaLauZN9giFnOssE1lVkzZGmXgg24QWUFMh6x/ZKPj+krlb1V0aqoeTRwLHSupgrAj4q7qmkRFmxnb1RBeVrYvq8FnmV98dp8PsCNYvtaeqMK6A04WRA8hQwo+9nk/bTOpC9gzEGDZuMicpT6M0NIRsE0QCoYAreaWYz+ahBKBSprjwpQc8y+fBWGlWKT9Xs5eJvyrUVF5/2AJKYX/gUhfIMYrVKNZeub9iVksKl5xDbo/IS78M3r2/qLP+9SQQxvArsZ7ru1c52x+wY3sXMb6XQLM= 34 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM lambci/lambda:build-provided 2 | 3 | RUN yum install -q -y wget \ 4 | readline-devel \ 5 | xorg-x11-server-devel libX11-devel libXt-devel \ 6 | curl-devel \ 7 | gcc-c++ gcc-gfortran \ 8 | zlib-devel bzip2 bzip2-libs \ 9 | java-1.8.0-openjdk-devel 10 | 11 | ARG VERSION=3.6.0 12 | ARG R_DIR=/opt/R/ 13 | 14 | RUN wget -q https://cran.r-project.org/src/base/R-3/R-${VERSION}.tar.gz && \ 15 | mkdir ${R_DIR} && \ 16 | tar -xf R-${VERSION}.tar.gz && \ 17 | mv R-${VERSION}/* ${R_DIR} && \ 18 | rm R-${VERSION}.tar.gz 19 | 20 | WORKDIR ${R_DIR} 21 | RUN ./configure --prefix=${R_DIR} --exec-prefix=${R_DIR} --with-libpth-prefix=/opt/ --enable-R-shlib && \ 22 | make && \ 23 | cp /usr/lib64/libgfortran.so.3 lib/ && \ 24 | cp /usr/lib64/libgomp.so.1 lib/ && \ 25 | cp /usr/lib64/libquadmath.so.0 lib/ && \ 26 | cp /usr/lib64/libstdc++.so.6 lib/ 27 | RUN yum install -q -y openssl-devel libxml2-devel && \ 28 | ./bin/Rscript -e 'install.packages(c("httr", "aws.s3", "logging"), repos="http://cran.r-project.org")' 29 | CMD mkdir -p /var/r/ && \ 30 | cp -r bin/ lib/ etc/ library/ doc/ modules/ share/ /var/r/ 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 bakdata 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | awscli = ">=1.16.164" 8 | 9 | [dev-packages] 10 | aws-sam-cli = ">=0.17.0" 11 | 12 | [requires] 13 | python_version = "3.7" 14 | -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "2da1df2a420a4f031f89d715d28b66d6c30745131cedde229dee073e37885a7b" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.7" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "awscli": { 20 | "hashes": [ 21 | "sha256:09fc351d4695facef972156645b9dbc560946c73c1e097d80aa83b9a01ac0002", 22 | "sha256:39b89d6f9c165d25d0c9406ee33162ba2077aeadfb641b72a995f9973debfd61" 23 | ], 24 | "index": "pypi", 25 | "version": "==1.16.175" 26 | }, 27 | "botocore": { 28 | "hashes": [ 29 | "sha256:d60fde28b199e588fa2e8ccae899b59b271f66c479af2b3a5951df3ea26cb73e", 30 | "sha256:f9556deba345a8cb5fa040342101dff2bbf95e53424c5e7002560c8fb29dac2d" 31 | ], 32 | "version": "==1.12.165" 33 | }, 34 | "colorama": { 35 | "hashes": [ 36 | "sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda", 37 | "sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1" 38 | ], 39 | "version": "==0.3.9" 40 | }, 41 | "docutils": { 42 | "hashes": [ 43 | "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", 44 | "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", 45 | "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" 46 | ], 47 | "version": "==0.14" 48 | }, 49 | "jmespath": { 50 | "hashes": [ 51 | "sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6", 52 | "sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c" 53 | ], 54 | "version": "==0.9.4" 55 | }, 56 | "pyasn1": { 57 | "hashes": [ 58 | "sha256:da2420fe13a9452d8ae97a0e478adde1dee153b11ba832a95b223a2ba01c10f7", 59 | "sha256:da6b43a8c9ae93bc80e2739efb38cc776ba74a886e3e9318d65fe81a8b8a2c6e" 60 | ], 61 | "version": "==0.4.5" 62 | }, 63 | "python-dateutil": { 64 | "hashes": [ 65 | "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", 66 | "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" 67 | ], 68 | "markers": "python_version >= '2.7'", 69 | "version": "==2.8.0" 70 | }, 71 | "pyyaml": { 72 | "hashes": [ 73 | "sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b", 74 | "sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf", 75 | "sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a", 76 | "sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3", 77 | "sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1", 78 | "sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1", 79 | "sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613", 80 | "sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04", 81 | "sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f", 82 | "sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537", 83 | "sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531" 84 | ], 85 | "version": "==3.13" 86 | }, 87 | "rsa": { 88 | "hashes": [ 89 | "sha256:25df4e10c263fb88b5ace923dd84bf9aa7f5019687b5e55382ffcdb8bede9db5", 90 | "sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd" 91 | ], 92 | "version": "==3.4.2" 93 | }, 94 | "s3transfer": { 95 | "hashes": [ 96 | "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d", 97 | "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba" 98 | ], 99 | "version": "==0.2.1" 100 | }, 101 | "six": { 102 | "hashes": [ 103 | "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", 104 | "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" 105 | ], 106 | "version": "==1.12.0" 107 | }, 108 | "urllib3": { 109 | "hashes": [ 110 | "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", 111 | "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" 112 | ], 113 | "markers": "python_version >= '3.4'", 114 | "version": "==1.25.3" 115 | } 116 | }, 117 | "develop": { 118 | "arrow": { 119 | "hashes": [ 120 | "sha256:03404b624e89ac5e4fc19c52045fa0f3203419fd4dd64f6e8958c522580a574a", 121 | "sha256:41be7ea4c53c2cf57bf30f2d614f60c411160133f7a0a8c49111c30fb7e725b5" 122 | ], 123 | "version": "==0.14.2" 124 | }, 125 | "aws-lambda-builders": { 126 | "hashes": [ 127 | "sha256:427724b039409a05a706a3f8125cb88c3901527b1192f4da2588714103f68b08", 128 | "sha256:9222fc6d6ac481bf5fa1849e2ae2b353eb38d310eea76606824c971b39482705", 129 | "sha256:e7c06c79a9f031a461b00a7241396115cf3e5bed0db0d6c9763cfb2cde7779dd" 130 | ], 131 | "version": "==0.3.0" 132 | }, 133 | "aws-sam-cli": { 134 | "hashes": [ 135 | "sha256:1da003711aab7fa668f2fbf82d078e407b64533a8d356d4452d2e38bdab3a55e", 136 | "sha256:95be2293373f323975198f7695eb712d687920d1e376918b54df24d9c4ef1580", 137 | "sha256:b487e4e3c6947f4d7a1694a18ff35a080ad540caf118cbdfca86657729e3a9ea" 138 | ], 139 | "index": "pypi", 140 | "version": "==0.17.0" 141 | }, 142 | "aws-sam-translator": { 143 | "hashes": [ 144 | "sha256:0e1fa094c6791b233f5e73f2f0803ec6e0622f2320ec5a969f0986855221b92b" 145 | ], 146 | "version": "==1.10.0" 147 | }, 148 | "binaryornot": { 149 | "hashes": [ 150 | "sha256:359501dfc9d40632edc9fac890e19542db1a287bbcfa58175b66658392018061", 151 | "sha256:b8b71173c917bddcd2c16070412e369c3ed7f0528926f70cac18a6c97fd563e4" 152 | ], 153 | "version": "==0.4.4" 154 | }, 155 | "boto3": { 156 | "hashes": [ 157 | "sha256:a9a4ea9f81e6f1cd2e3b6c472dab8d1dff96aeb5a1f7c30c2570a1c1482f7084", 158 | "sha256:e3247ef6ca868f1fb6759e26e992caf84758660ec9be293b42e3dd91133a2662" 159 | ], 160 | "version": "==1.9.165" 161 | }, 162 | "botocore": { 163 | "hashes": [ 164 | "sha256:d60fde28b199e588fa2e8ccae899b59b271f66c479af2b3a5951df3ea26cb73e", 165 | "sha256:f9556deba345a8cb5fa040342101dff2bbf95e53424c5e7002560c8fb29dac2d" 166 | ], 167 | "version": "==1.12.165" 168 | }, 169 | "certifi": { 170 | "hashes": [ 171 | "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", 172 | "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" 173 | ], 174 | "version": "==2019.3.9" 175 | }, 176 | "chardet": { 177 | "hashes": [ 178 | "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", 179 | "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" 180 | ], 181 | "version": "==3.0.4" 182 | }, 183 | "chevron": { 184 | "hashes": [ 185 | "sha256:95b0a055ef0ada5eb061d60be64a7f70670b53372ccd221d1b88adf1c41a9094", 186 | "sha256:f95054a8b303268ebf3efd6bdfc8c1b428d3fc92327913b4e236d062ec61c989" 187 | ], 188 | "version": "==0.13.1" 189 | }, 190 | "click": { 191 | "hashes": [ 192 | "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d", 193 | "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b" 194 | ], 195 | "version": "==6.7" 196 | }, 197 | "cookiecutter": { 198 | "hashes": [ 199 | "sha256:1316a52e1c1f08db0c9efbf7d876dbc01463a74b155a0d83e722be88beda9a3e", 200 | "sha256:ed8f54a8fc79b6864020d773ce11539b5f08e4617f353de1f22d23226f6a0d36" 201 | ], 202 | "version": "==1.6.0" 203 | }, 204 | "dateparser": { 205 | "hashes": [ 206 | "sha256:42d51be54e74a8e80a4d76d1fa6e4edd997098fce24ad2d94a2eab5ef247193e", 207 | "sha256:78124c458c461ea7198faa3c038f6381f37588b84bb42740e91a4cbd260b1d09" 208 | ], 209 | "version": "==0.7.1" 210 | }, 211 | "docker": { 212 | "hashes": [ 213 | "sha256:3db499d4d25847fed86acf8e100c989f7bc0f75a6fff6c52855726ada1d124f6", 214 | "sha256:f61c37d721b489b7d55ef631b241be2d6a5884c3ffe63dc8f7dd9a3c3cd60489" 215 | ], 216 | "version": "==4.0.1" 217 | }, 218 | "docutils": { 219 | "hashes": [ 220 | "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", 221 | "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", 222 | "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" 223 | ], 224 | "version": "==0.14" 225 | }, 226 | "flask": { 227 | "hashes": [ 228 | "sha256:ad7c6d841e64296b962296c2c2dabc6543752985727af86a975072dea984b6f3", 229 | "sha256:e7d32475d1de5facaa55e3958bc4ec66d3762076b074296aa50ef8fdc5b9df61" 230 | ], 231 | "version": "==1.0.3" 232 | }, 233 | "future": { 234 | "hashes": [ 235 | "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8" 236 | ], 237 | "version": "==0.17.1" 238 | }, 239 | "idna": { 240 | "hashes": [ 241 | "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", 242 | "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" 243 | ], 244 | "version": "==2.8" 245 | }, 246 | "itsdangerous": { 247 | "hashes": [ 248 | "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", 249 | "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" 250 | ], 251 | "version": "==1.1.0" 252 | }, 253 | "jinja2": { 254 | "hashes": [ 255 | "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", 256 | "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" 257 | ], 258 | "version": "==2.10.1" 259 | }, 260 | "jinja2-time": { 261 | "hashes": [ 262 | "sha256:d14eaa4d315e7688daa4969f616f226614350c48730bfa1692d2caebd8c90d40", 263 | "sha256:d3eab6605e3ec8b7a0863df09cc1d23714908fa61aa6986a845c20ba488b4efa" 264 | ], 265 | "version": "==0.2.0" 266 | }, 267 | "jmespath": { 268 | "hashes": [ 269 | "sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6", 270 | "sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c" 271 | ], 272 | "version": "==0.9.4" 273 | }, 274 | "jsonschema": { 275 | "hashes": [ 276 | "sha256:000e68abd33c972a5248544925a0cae7d1125f9bf6c58280d37546b946769a08", 277 | "sha256:6ff5f3180870836cae40f06fa10419f557208175f13ad7bc26caa77beb1f6e02" 278 | ], 279 | "version": "==2.6.0" 280 | }, 281 | "markupsafe": { 282 | "hashes": [ 283 | "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", 284 | "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", 285 | "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", 286 | "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", 287 | "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", 288 | "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", 289 | "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", 290 | "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", 291 | "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", 292 | "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", 293 | "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", 294 | "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", 295 | "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", 296 | "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", 297 | "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", 298 | "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", 299 | "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", 300 | "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", 301 | "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", 302 | "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", 303 | "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", 304 | "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", 305 | "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", 306 | "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", 307 | "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", 308 | "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", 309 | "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", 310 | "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" 311 | ], 312 | "version": "==1.1.1" 313 | }, 314 | "poyo": { 315 | "hashes": [ 316 | "sha256:c34a5413191210ed564640510e9c4a4ba3b698746d6b454d46eb5bfb30edcd1d", 317 | "sha256:d1c317054145a6b1ca0608b5e676b943ddc3bfd671f886a2fe09288b98221edb" 318 | ], 319 | "version": "==0.4.2" 320 | }, 321 | "python-dateutil": { 322 | "hashes": [ 323 | "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", 324 | "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" 325 | ], 326 | "markers": "python_version >= '2.7'", 327 | "version": "==2.8.0" 328 | }, 329 | "pytz": { 330 | "hashes": [ 331 | "sha256:303879e36b721603cc54604edcac9d20401bdbe31e1e4fdee5b9f98d5d31dfda", 332 | "sha256:d747dd3d23d77ef44c6a3526e274af6efeb0a6f1afd5a69ba4d5be4098c8e141" 333 | ], 334 | "version": "==2019.1" 335 | }, 336 | "pyyaml": { 337 | "hashes": [ 338 | "sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b", 339 | "sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf", 340 | "sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a", 341 | "sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3", 342 | "sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1", 343 | "sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1", 344 | "sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613", 345 | "sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04", 346 | "sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f", 347 | "sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537", 348 | "sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531" 349 | ], 350 | "version": "==3.13" 351 | }, 352 | "regex": { 353 | "hashes": [ 354 | "sha256:1c70ccb8bf4ded0cbe53092e9f56dcc9d6b0efcf6e80b6ef9b0ece8a557d6635", 355 | "sha256:2948310c01535ccb29bb600dd033b07b91f36e471953889b7f3a1e66b39d0c19", 356 | "sha256:2ab13db0411cb308aa590d33c909ea4efeced40188d8a4a7d3d5970657fe73bc", 357 | "sha256:38e6486c7e14683cd1b17a4218760f0ea4c015633cf1b06f7c190fb882a51ba7", 358 | "sha256:80dde4ff10b73b823da451687363cac93dd3549e059d2dc19b72a02d048ba5aa", 359 | "sha256:84daedefaa56320765e9c4d43912226d324ef3cc929f4d75fa95f8c579a08211", 360 | "sha256:b98e5876ca1e63b41c4aa38d7d5cc04a736415d4e240e9ae7ebc4f780083c7d5", 361 | "sha256:ca4f47131af28ef168ff7c80d4b4cad019cb4cabb5fa26143f43aa3dbd60389c", 362 | "sha256:cf7838110d3052d359da527372666429b9485ab739286aa1a11ed482f037a88c", 363 | "sha256:dd4e8924915fa748e128864352875d3d0be5f4597ab1b1d475988b8e3da10dd7", 364 | "sha256:f2c65530255e4010a5029eb11138f5ecd5aa70363f57a3444d83b3253b0891be" 365 | ], 366 | "version": "==2019.6.8" 367 | }, 368 | "requests": { 369 | "hashes": [ 370 | "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", 371 | "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" 372 | ], 373 | "version": "==2.22.0" 374 | }, 375 | "s3transfer": { 376 | "hashes": [ 377 | "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d", 378 | "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba" 379 | ], 380 | "version": "==0.2.1" 381 | }, 382 | "serverlessrepo": { 383 | "hashes": [ 384 | "sha256:533389d41a51450e50cc01405ab766550170149c08e1c85b3a1559b0fab4cb25", 385 | "sha256:d40e83d29175ba3eddaa02f1dac57332d1dec495e012cd2e80366be9ad7c94a5" 386 | ], 387 | "version": "==0.1.8" 388 | }, 389 | "six": { 390 | "hashes": [ 391 | "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", 392 | "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" 393 | ], 394 | "version": "==1.12.0" 395 | }, 396 | "tzlocal": { 397 | "hashes": [ 398 | "sha256:4ebeb848845ac898da6519b9b31879cf13b6626f7184c496037b818e238f2c4e" 399 | ], 400 | "version": "==1.5.1" 401 | }, 402 | "urllib3": { 403 | "hashes": [ 404 | "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", 405 | "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" 406 | ], 407 | "markers": "python_version >= '3.4'", 408 | "version": "==1.25.3" 409 | }, 410 | "websocket-client": { 411 | "hashes": [ 412 | "sha256:1151d5fb3a62dc129164292e1227655e4bbc5dd5340a5165dfae61128ec50aa9", 413 | "sha256:1fd5520878b68b84b5748bb30e592b10d0a91529d5383f74f4964e72b297fd3a" 414 | ], 415 | "version": "==0.56.0" 416 | }, 417 | "werkzeug": { 418 | "hashes": [ 419 | "sha256:865856ebb55c4dcd0630cdd8f3331a1847a819dda7e8c750d3db6f2aa6c0209c", 420 | "sha256:a0b915f0815982fb2a09161cb8f31708052d0951c3ba433ccc5e1aa276507ca6" 421 | ], 422 | "version": "==0.15.4" 423 | }, 424 | "wheel": { 425 | "hashes": [ 426 | "sha256:5e79117472686ac0c4aef5bad5172ea73a1c2d1646b808c35926bd26bdfb0c08", 427 | "sha256:62fcfa03d45b5b722539ccbc07b190e4bfff4bb9e3a4d470dd9f6a0981002565" 428 | ], 429 | "version": "==0.33.4" 430 | }, 431 | "whichcraft": { 432 | "hashes": [ 433 | "sha256:7533870f751901a0ce43c93cc9850186e9eba7fe58c924dfb435968ba9c9fa4e", 434 | "sha256:fecddd531f237ffc5db8b215409afb18fa30300699064cca4817521b4fc81815" 435 | ], 436 | "version": "==0.5.2" 437 | } 438 | } 439 | } 440 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # aws-lambda-r-runtime 2 | 3 | [![Build Status](https://travis-ci.com/bakdata/aws-lambda-r-runtime.svg?branch=master)](https://travis-ci.com/bakdata/aws-lambda-r-runtime) 4 | 5 | This project makes it easy to run AWS Lambda Functions written in R. 6 | 7 | ## Example 8 | To run the example, we need to create a IAM role executing our lambda. 9 | This role should have the following properties: 10 | - Trusted entity – Lambda. 11 | - Permissions – AWSLambdaBasicExecutionRole. 12 | 13 | Furthermore you need a current version of the AWS CLI. 14 | 15 | Then create a lambda function which uses the R runtime layer: 16 | ```bash 17 | cd example/ 18 | chmod 755 script.R 19 | zip function.zip script.R 20 | # current region 21 | region=$(aws configure get region) 22 | # latest runtime layer ARN for R 3.6.0 in most regions 23 | # for an accurate list, please have a look at the deploy section of the travis ci build log 24 | # https://travis-ci.com/bakdata/aws-lambda-r-runtime 25 | runtime_layer=arn:aws:lambda:$region:131329294410:layer:r-runtime-3_6_0:13 26 | aws lambda create-function --function-name r-example \ 27 | --zip-file fileb://function.zip --handler script.handler \ 28 | --runtime provided --timeout 60 \ 29 | --layers ${runtime_layer} \ 30 | --role 31 | ``` 32 | 33 | The function simply increments 'x' by 1. 34 | Invoke the function: 35 | ```bash 36 | aws lambda invoke --function-name r-example \ 37 | --payload '{"x":1}' response.txt 38 | cat response.txt 39 | ``` 40 | 41 | The expected result should look similar to this: 42 | ```json 43 | 2 44 | ``` 45 | 46 | ### Using packages 47 | 48 | We also provide a layer which ships with some recommended R packages, such as `Matrix`. 49 | This example lambda shows how to use them: 50 | ```bash 51 | cd example/ 52 | chmod 755 matrix.R 53 | zip function.zip matrix.R 54 | # current region 55 | region=$(aws configure get region) 56 | # latest runtime layer ARN for R 3.6.0 in most regions 57 | # for an accurate list, please have a look at the deploy section of the travis ci build log 58 | # https://travis-ci.com/bakdata/aws-lambda-r-runtime 59 | runtime_layer=arn:aws:lambda:$region:131329294410:layer:r-runtime-3_6_0:13 60 | # latest recommended layer ARN for R 3.6.0 in most regions 61 | # for an accurate list, please have a look at the deploy section of the travis ci build log 62 | # https://travis-ci.com/bakdata/aws-lambda-r-runtime 63 | recommended_layer=arn:aws:lambda:$region:131329294410:layer:r-recommended-3_6_0:13 64 | aws lambda create-function --function-name r-matrix-example \ 65 | --zip-file fileb://function.zip --handler matrix.handler \ 66 | --runtime provided --timeout 60 --memory-size 3008 \ 67 | --layers ${runtime_layer} ${recommended_layer} \ 68 | --role 69 | ``` 70 | 71 | The function returns the second column of some static matrix. 72 | Invoke the function: 73 | ```bash 74 | aws lambda invoke --function-name r-matrix-example response.txt 75 | cat response.txt 76 | ``` 77 | 78 | The expected result should look similar to this: 79 | ```json 80 | [4,5,6] 81 | ``` 82 | 83 | ## Provided layers 84 | 85 | Layers are only accessible in the AWS region they were published. 86 | We provide the following layers: 87 | 88 | ### r-runtime 89 | 90 | R, 91 | [httr](https://cran.r-project.org/package=httr), 92 | [jsonlite](https://cran.r-project.org/package=jsonlite), 93 | [aws.s3](https://cran.r-project.org/package=aws.s3), 94 | [logging](https://cran.r-project.org/package=logging) 95 | 96 | Available AWS regions: 97 | - ap-northeast-1 98 | - ap-northeast-2 99 | - ap-south-1 100 | - ap-southeast-1 101 | - ap-southeast-2 102 | - ca-central-1 103 | - eu-central-1 104 | - eu-north-1 105 | - eu-west-1 106 | - eu-west-2 107 | - eu-west-3 108 | - sa-east-1 109 | - us-east-1 110 | - us-east-2 111 | - us-west-1 112 | - us-west-2 113 | 114 | Available R versions: 115 | - 3_5_1 116 | - 3_5_3 117 | - 3_6_0 118 | 119 | Latest ARN can be retrieved from the [Travis CI build log](https://travis-ci.com/bakdata/aws-lambda-r-runtime). In general, it looks this: 120 | 121 | `arn:aws:lambda:$region:131329294410:layer:r-runtime-$r_version:$layer_version` 122 | 123 | Automated command for retrieving the ARN does not work currently: 124 | ```bash 125 | aws lambda list-layer-versions --max-items 1 --no-paginate \ 126 | --layer-name arn:aws:lambda:${region}:131329294410:layer:r-runtime-${r_version} \ 127 | --query 'LayerVersions[0].LayerVersionArn' --output text 128 | ``` 129 | 130 | ### r-recommended 131 | 132 | The recommended packages that ship with R: 133 | boot, 134 | class, 135 | cluster, 136 | codetools, 137 | foreign, 138 | KernSmooth, 139 | lattice, 140 | MASS, 141 | Matrix, 142 | mgcv, 143 | nlme, 144 | nnet, 145 | rpart, 146 | spatial, 147 | survival 148 | 149 | Available AWS regions: 150 | - ap-northeast-1 151 | - ap-northeast-2 152 | - ap-south-1 153 | - ap-southeast-1 154 | - ap-southeast-2 155 | - ca-central-1 156 | - eu-central-1 157 | - eu-north-1 158 | - eu-west-1 159 | - eu-west-2 160 | - eu-west-3 161 | - sa-east-1 162 | - us-east-1 163 | - us-east-2 164 | - us-west-1 165 | - us-west-2 166 | 167 | Available R versions: 168 | - 3_5_1 169 | - 3_5_3 170 | - 3_6_0 171 | 172 | Latest ARN can be retrieved from the [Travis CI build log](https://travis-ci.com/bakdata/aws-lambda-r-runtime). In general, it looks this: 173 | 174 | `arn:aws:lambda:$region:131329294410:layer:r-recommended-$r_version:$layer_version` 175 | 176 | Automated command for retrieving the ARN does not work currently: 177 | ```bash 178 | aws lambda list-layer-versions --max-items 1 --no-paginate \ 179 | --layer-name arn:aws:lambda:${region}:131329294410:layer:r-recommended-${r_version} \ 180 | --query 'LayerVersions[0].LayerVersionArn' --output text 181 | ``` 182 | 183 | ### r-awspack 184 | 185 | The [aws.s3](https://cran.r-project.org/package=aws.s3) package. 186 | It used to contain the [awspack](https://cran.r-project.org/package=awspack) package but unfortunately this package has been retired. 187 | You can still find it in old versions of the layer that have been published before 2020. 188 | 189 | Available AWS regions: 190 | - ap-northeast-1 191 | - ap-northeast-2 192 | - ap-south-1 193 | - ap-southeast-1 194 | - ap-southeast-2 195 | - ca-central-1 196 | - eu-central-1 197 | - eu-north-1 198 | - eu-west-1 199 | - eu-west-2 200 | - eu-west-3 201 | - sa-east-1 202 | - us-east-1 203 | - us-east-2 204 | - us-west-1 205 | - us-west-2 206 | 207 | Available R versions: 208 | - 3_5_1 209 | - 3_5_3 210 | - 3_6_0 211 | 212 | Latest ARN can be retrieved from the [Travis CI build log](https://travis-ci.com/bakdata/aws-lambda-r-runtime). In general, it looks this: 213 | 214 | `arn:aws:lambda:$region:131329294410:layer:r-awspack-$r_version:$layer_version` 215 | 216 | Automated command for retrieving the ARN does not work currently: 217 | ```bash 218 | aws lambda list-layer-versions --max-items 1 --no-paginate \ 219 | --layer-name arn:aws:lambda:${region}:131329294410:layer:r-awspack-${r_version} \ 220 | --query 'LayerVersions[0].LayerVersionArn' --output text 221 | ``` 222 | 223 | ## Documentation 224 | 225 | The lambda handler is used to determine both the file name of the R script and the function to call. 226 | The handler must be separated by `.`, e.g., `script.handler`. 227 | 228 | The lambda payload is unwrapped as named arguments to the R function to call, e.g., `{"x":1}` is unwrapped to `handler(x=1)`. 229 | 230 | The lambda function returns whatever is returned by the R function as a JSON object. 231 | 232 | ### Building custom layers 233 | 234 | In order to install additional R packages, you can create a lambda layer containing the libraries, just as in the second example. 235 | You must use the the compiled package files. 236 | The easiest way is to install the package with `install.packages()` and copy the resulting folder in `$R_LIBS`. 237 | Using only the package sources does not suffice. 238 | The file structure must be `R/library/`. 239 | If your package requires system libraries, place them in `R/lib/`. 240 | 241 | You can use Docker for building your layer. 242 | You need to run `./docker_build.sh` first. 243 | Then you can install your packages inside the container and copy the files to your machine. 244 | See `awspack/` for an example. 245 | The `build.sh` script is used to run the docker container and copy sources to your machine. 246 | The `entrypoint.sh` script is used for installing packages inside the container. 247 | 248 | ### Debugging 249 | 250 | In order to make the runtime log debugging messages, you can set the environment variable `LOGLEVEL` to `DEBUG`. 251 | 252 | ## Limitations 253 | 254 | AWS Lambda is limited to running with 3GB RAM and must finish within 15 minutes. 255 | It is therefore not feasible to execute long running R scripts with this runtime. 256 | Furthermore, only the `/tmp/` directory is writeable on AWS Lambda. 257 | This must be considered when writing to the local disk. 258 | 259 | 260 | ## Building 261 | 262 | To build the layer yourself, you need to first build R from source. 263 | We provide a Docker image which uses the great [docker-lambda](https://github.com/lambci/docker-lambda) project. 264 | Just run `./build.sh ` and everything should be build properly. 265 | 266 | If you plan to publish the runtime, you need to have a recent version of aws cli (>=1.16). 267 | Now run the `/deploy.sh` script. 268 | This creates a lambda layer named `r--` in your AWS account. 269 | You can use it as shown in the example. 270 | 271 | ### Compiling on EC2 272 | 273 | In case the Docker image does not properly represent the lambda environment, 274 | we also provide a script which launches an EC2 instance, compiles R, and uploads the zipped distribution to S3. 275 | You need to specify the R version, e.g., `3.6.0`, as well as the S3 bucket to upload the distribution to. 276 | Finally, you need to create an EC2 instance profile which is capable of uploading to the S3 bucket. 277 | See the [AWS documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#create-iam-role) for details. 278 | With everything prepared, you can run the script: 279 | ```bash 280 | ./remote_compile_and_deploy.sh 281 | ``` 282 | The script will also take care of terminating the launched EC2 instance. 283 | 284 | To manually build R from source, follow these steps: 285 | 286 | Start an EC2 instance which uses the [Lambda AMI](https://console.aws.amazon.com/ec2/v2/home#Images:visibility=public-images;search=amzn-ami-hvm-2017.03.1.20170812-x86_64-gp2): 287 | ```bash 288 | aws ec2 run-instances --image-id ami-657bd20a --count 1 --instance-type t2.medium --key-name 289 | ``` 290 | Now run the `compile.sh` script in `r/`. 291 | You must pass the R version as a parameter to the script, e.g., `3.6.0`. 292 | The script produces a zip containing a functional R installation in `/opt/R/`. 293 | The relevant files can be found in `r/build/bin/`. 294 | Use this R distribution for building the layers. 295 | 296 | ### Testing 297 | 298 | After building all layers, you can test it locally with SAM CLI and Docker. 299 | Install it via `pipenv install --dev`. 300 | Then run `python3 -m unittest`. 301 | This will spawn a local lambda server via Docker and invokes the lambdas defined in `template.yaml`. 302 | -------------------------------------------------------------------------------- /awspack/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | if [[ -z ${1+x} ]]; 6 | then 7 | echo 'version number required' 8 | exit 1 9 | else 10 | VERSION=$1 11 | fi 12 | 13 | BASE_DIR=$(pwd) 14 | BUILD_DIR=${BASE_DIR}/build/ 15 | 16 | rm -rf ${BUILD_DIR} 17 | 18 | mkdir -p ${BUILD_DIR}/layer/ 19 | docker run -v ${BUILD_DIR}/layer/:/var/awspack -v ${BASE_DIR}/entrypoint.sh:/entrypoint.sh \ 20 | lambda-r:build-${VERSION} /entrypoint.sh 21 | sudo chown -R $(whoami):$(whoami) ${BUILD_DIR}/layer/ 22 | chmod -R 755 ${BUILD_DIR}/layer/ 23 | -------------------------------------------------------------------------------- /awspack/compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | if [[ -z ${1+x} ]]; 6 | then 7 | echo 'version number required' 8 | exit 1 9 | else 10 | VERSION=$1 11 | fi 12 | 13 | BASE_DIR=$(pwd) 14 | BUILD_DIR=${BASE_DIR}/build/ 15 | R_DIR=/opt/R/ 16 | 17 | rm -rf ${BUILD_DIR} 18 | 19 | export R_LIBS=${BUILD_DIR}/layer/R/library 20 | mkdir -p ${R_LIBS} 21 | ${R_DIR}/bin/Rscript -e 'install.packages("aws.s3", repos="http://cran.r-project.org")' 22 | chmod -R 755 ${BUILD_DIR}/layer/ 23 | -------------------------------------------------------------------------------- /awspack/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | if [[ -z ${1+x} ]]; 6 | then 7 | echo 'version number required' 8 | exit 1 9 | else 10 | VERSION=$1 11 | fi 12 | 13 | BASE_DIR=$(pwd) 14 | BUILD_DIR=${BASE_DIR}/build/ 15 | 16 | cd ${BUILD_DIR}/layer/ 17 | zip -r -q awspack-${VERSION}.zip . 18 | mkdir -p ${BUILD_DIR}/dist/ 19 | mv awspack-${VERSION}.zip ${BUILD_DIR}/dist/ 20 | version_="${VERSION//\./_}" 21 | aws lambda publish-layer-version \ 22 | --layer-name r-awspack-${version_} \ 23 | --zip-file fileb://${BUILD_DIR}/dist/awspack-${VERSION}.zip 24 | -------------------------------------------------------------------------------- /awspack/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | BUILD_DIR=/var/awspack/ 6 | R_DIR=/opt/R/ 7 | 8 | export R_LIBS=${BUILD_DIR}/R/library 9 | mkdir -p ${R_LIBS} 10 | ${R_DIR}/bin/Rscript -e 'install.packages("aws.s3", repos="http://cran.r-project.org")' 11 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | if [[ -z ${1+x} ]]; 6 | then 7 | echo 'version number required' 8 | exit 1 9 | else 10 | VERSION=$1 11 | fi 12 | 13 | BASE_DIR=$(pwd) 14 | ./docker_build.sh ${VERSION} 15 | cd ${BASE_DIR}/r 16 | ./build.sh ${VERSION} 17 | cd ${BASE_DIR}/runtime 18 | ./build.sh 19 | cd ${BASE_DIR}/recommended 20 | ./build.sh 21 | cd ${BASE_DIR}/awspack 22 | ./build.sh ${VERSION} 23 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | if [[ -z ${1+x} ]]; 6 | then 7 | echo 'version number required' 8 | exit 1 9 | else 10 | VERSION=$1 11 | fi 12 | 13 | function releaseToRegion { 14 | version=$1 15 | region=$2 16 | bucket="aws-lambda-r-runtime.$region" 17 | echo "publishing layers to region $region" 18 | sam package \ 19 | --output-template-file packaged.yaml \ 20 | --s3-bucket ${bucket} \ 21 | --s3-prefix R-${version} \ 22 | --region ${region} 23 | version_="${version//\./_}" 24 | stack_name=r-${version//\./-} 25 | sam deploy \ 26 | --template-file packaged.yaml \ 27 | --stack-name ${stack_name} \ 28 | --parameter-overrides Version=${version_} \ 29 | --no-fail-on-empty-changeset \ 30 | --region ${region} \ 31 | --capabilities CAPABILITY_IAM 32 | layers=(runtime recommended awspack) 33 | echo "Published layers:" 34 | aws cloudformation describe-stack-resources \ 35 | --stack-name ${stack_name} \ 36 | --query "StackResources[?ResourceType=='AWS::Lambda::LayerVersion'].PhysicalResourceId" \ 37 | --region ${region} 38 | } 39 | 40 | regions=( 41 | us-east-1 us-east-2 42 | us-west-1 us-west-2 43 | ap-south-1 44 | ap-northeast-1 ap-northeast-2 45 | ap-southeast-1 ap-southeast-2 46 | ca-central-1 47 | eu-central-1 48 | eu-north-1 49 | eu-west-1 eu-west-2 eu-west-3 50 | sa-east-1 51 | ) 52 | 53 | for region in "${regions[@]}" 54 | do 55 | releaseToRegion ${VERSION} ${region} 56 | done 57 | -------------------------------------------------------------------------------- /docker_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | if [[ -z ${1+x} ]]; 6 | then 7 | echo 'version number required' 8 | exit 1 9 | else 10 | VERSION=$1 11 | fi 12 | 13 | docker build -t lambda-r:build-${VERSION} --build-arg VERSION=${VERSION} . 14 | -------------------------------------------------------------------------------- /example/.gitignore: -------------------------------------------------------------------------------- 1 | response.txt 2 | -------------------------------------------------------------------------------- /example/matrix.R: -------------------------------------------------------------------------------- 1 | library(Matrix) 2 | 3 | handler <- function() { 4 | return(Matrix(1:6, 3, 2)[, 2]) 5 | } 6 | -------------------------------------------------------------------------------- /example/script.R: -------------------------------------------------------------------------------- 1 | handler <- function(x) { 2 | return(x + 1) 3 | } 4 | -------------------------------------------------------------------------------- /integration_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | if [[ -z ${1+x} ]]; 6 | then 7 | echo 'version number required' 8 | exit 1 9 | else 10 | R_VERSION=$1 11 | fi 12 | 13 | function integrationTest { 14 | version=$1 15 | region=$2 16 | bucket="aws-lambda-r-runtime.$region" 17 | echo "Integration testing in region $region" 18 | sam package \ 19 | --output-template-file packaged.yaml \ 20 | --s3-bucket ${bucket} \ 21 | --s3-prefix R-${version} \ 22 | --template-file test-template.yaml \ 23 | --region ${region} 24 | version_="${version//\./_}" 25 | stack_name=r-${version//\./-}-test 26 | sam deploy \ 27 | --template-file packaged.yaml \ 28 | --stack-name ${stack_name} \ 29 | --capabilities CAPABILITY_IAM \ 30 | --parameter-overrides Version=${version_} \ 31 | --no-fail-on-empty-changeset \ 32 | --region ${region} 33 | VERSION=${version_} INTEGRATION_TEST=True AWS_DEFAULT_REGION=${region} pipenv run python -m unittest 34 | } 35 | 36 | regions=( 37 | us-east-1 38 | ) 39 | 40 | for region in "${regions[@]}" 41 | do 42 | integrationTest ${R_VERSION} ${region} 43 | done 44 | -------------------------------------------------------------------------------- /r/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | if [[ -z ${1+x} ]]; 6 | then 7 | echo 'version number required' 8 | exit 1 9 | else 10 | VERSION=$1 11 | fi 12 | 13 | BASE_DIR=$(pwd) 14 | BUILD_DIR=${BASE_DIR}/build/ 15 | R_DIR=/opt/R/ 16 | 17 | rm -rf ${BUILD_DIR} 18 | 19 | mkdir -p ${BUILD_DIR}/bin/ 20 | docker run -v ${BUILD_DIR}/bin/:/var/r lambda-r:build-${VERSION} 21 | sudo chown -R $(whoami):$(whoami) ${BUILD_DIR}/bin/ 22 | -------------------------------------------------------------------------------- /r/compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | if [[ -z ${1+x} ]]; 6 | then 7 | echo 'version number required' 8 | exit 1 9 | else 10 | VERSION=$1 11 | fi 12 | 13 | BASE_DIR=$(pwd) 14 | BUILD_DIR=${BASE_DIR}/build/ 15 | R_DIR=/opt/R/ 16 | 17 | mkdir -p ${BUILD_DIR} 18 | cd ${BUILD_DIR} 19 | wget https://cran.r-project.org/src/base/R-3/R-${VERSION}.tar.gz 20 | sudo mkdir ${R_DIR} 21 | sudo chown $(whoami) ${R_DIR} 22 | tar -xf R-${VERSION}.tar.gz 23 | mv R-${VERSION}/* ${R_DIR} 24 | rm R-${VERSION}.tar.gz 25 | sudo yum install -y readline-devel \ 26 | xorg-x11-server-devel libX11-devel libXt-devel \ 27 | curl-devel \ 28 | gcc-c++ gcc-gfortran \ 29 | zlib-devel bzip2 bzip2-libs 30 | # workaround for making R build work 31 | # issue seems similar to https://stackoverflow.com/questions/40639138/configure-error-installing-r-3-3-2-on-ubuntu-checking-whether-bzip2-support-suf 32 | sudo yum install -y R 33 | 34 | cd ${R_DIR} 35 | ./configure --prefix=${R_DIR} --exec-prefix=${R_DIR} --with-libpth-prefix=/opt/ --enable-R-shlib 36 | make 37 | cp /usr/lib64/libgfortran.so.3 lib/ 38 | cp /usr/lib64/libgomp.so.1 lib/ 39 | cp /usr/lib64/libquadmath.so.0 lib/ 40 | cp /usr/lib64/libstdc++.so.6 lib/ 41 | sudo yum install -y openssl-devel libxml2-devel 42 | ./bin/Rscript -e 'install.packages(c("httr", "aws.s3", "logging"), repos="http://cran.r-project.org")' 43 | 44 | mkdir -p ${BUILD_DIR}/bin/ 45 | cp -r bin/ lib/ etc/ library/ doc/ modules/ share/ ${BUILD_DIR}/bin/ 46 | -------------------------------------------------------------------------------- /recommended/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | BASE_DIR=$(pwd) 6 | BUILD_DIR=${BASE_DIR}/build/ 7 | 8 | rm -rf ${BUILD_DIR} 9 | mkdir -p ${BUILD_DIR}/layer/R.orig/ 10 | cd ${BUILD_DIR}/layer/ 11 | cp -r ${BASE_DIR}/../r/build/bin/* R.orig/ 12 | mkdir -p R/library 13 | 14 | recommended=(boot class cluster codetools foreign KernSmooth lattice MASS Matrix mgcv nlme nnet rpart spatial survival) 15 | for package in "${recommended[@]}" 16 | do 17 | mv R.orig/library/${package}/ R/library/${package}/ 18 | done 19 | rm -rf R.orig/ 20 | chmod -R 755 . 21 | -------------------------------------------------------------------------------- /recommended/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | if [[ -z ${1+x} ]]; 6 | then 7 | echo 'version number required' 8 | exit 1 9 | else 10 | VERSION=$1 11 | fi 12 | 13 | BASE_DIR=$(pwd) 14 | BUILD_DIR=${BASE_DIR}/build/ 15 | 16 | cd ${BUILD_DIR}/layer/ 17 | zip -r -q recommended-${VERSION}.zip . 18 | mkdir -p ${BUILD_DIR}/dist/ 19 | mv recommended-${VERSION}.zip ${BUILD_DIR}/dist/ 20 | version_="${VERSION//\./_}" 21 | aws lambda publish-layer-version \ 22 | --layer-name r-recommended-${version_} \ 23 | --zip-file fileb://${BUILD_DIR}/dist/recommended-${VERSION}.zip 24 | -------------------------------------------------------------------------------- /remote_compile_and_deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | if [[ -z ${1+x} ]]; 6 | then 7 | echo 'version number required' 8 | exit 1 9 | else 10 | VERSION=$1 11 | fi 12 | 13 | if [[ -z ${2+x} ]]; 14 | then 15 | echo 'bucket name required' 16 | exit 1 17 | else 18 | BUCKET=$2 19 | fi 20 | 21 | if [[ -z ${3+x} ]]; 22 | then 23 | echo 'instance profile required' 24 | exit 1 25 | else 26 | PROFILE=$3 27 | fi 28 | 29 | instance_id=$(aws ec2 run-instances --image-id ami-657bd20a --count 1 --instance-type t2.medium \ 30 | --instance-initiated-shutdown-behavior terminate --iam-instance-profile Name='"'${PROFILE}'"' \ 31 | --user-data '#!/bin/bash 32 | yum install -y git 33 | git clone https://github.com/bakdata/aws-lambda-r-runtime.git 34 | cd aws-lambda-r-runtime/r/ 35 | ./compile.sh '"$VERSION"' 36 | cd build/bin/ 37 | zip -r R-'"$VERSION"'.zip . 38 | aws s3 cp R-'"$VERSION"'.zip s3://'"$BUCKET"'/R-'"$VERSION"'/ 39 | cd ../../../awspack/ 40 | ./compile_and_deploy.sh '"$VERSION"' 41 | cd build/bin/ 42 | zip -r awspack-'"$VERSION"'.zip . 43 | aws s3 cp awspack-'"$VERSION"'.zip s3://'"$BUCKET"'/R-'"$VERSION"'/ 44 | shutdown -h now' \ 45 | --query 'Instances[0].InstanceId' --output text) 46 | 47 | until aws ec2 wait instance-terminated --instance-ids ${instance_id} 2>/dev/null 48 | do 49 | echo "Still waiting for $instance_id to terminate" 50 | sleep 10 51 | done 52 | 53 | -------------------------------------------------------------------------------- /runtime/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | BASE_DIR=$(pwd) 6 | BUILD_DIR=${BASE_DIR}/build/ 7 | 8 | rm -rf ${BUILD_DIR} 9 | mkdir -p ${BUILD_DIR}/layer/R/ 10 | cp ${BASE_DIR}/src/* ${BUILD_DIR}/layer/ 11 | cd ${BUILD_DIR}/layer/ 12 | cp -r ${BASE_DIR}/../r/build/bin/* R/ 13 | rm -r R/doc/manual/ 14 | #remove some libraries to save space 15 | recommended=(boot class cluster codetools foreign KernSmooth lattice MASS Matrix mgcv nlme nnet rpart spatial survival) 16 | for package in "${recommended[@]}" 17 | do 18 | rm -r R/library/${package}/ 19 | done 20 | chmod -R 755 . 21 | -------------------------------------------------------------------------------- /runtime/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | if [[ -z ${1+x} ]]; 6 | then 7 | echo 'version number required' 8 | exit 1 9 | else 10 | VERSION=$1 11 | fi 12 | 13 | BASE_DIR=$(pwd) 14 | BUILD_DIR=${BASE_DIR}/build/ 15 | 16 | cd ${BUILD_DIR}/layer/ 17 | zip -r -q runtime-${VERSION}.zip . 18 | mkdir -p ${BUILD_DIR}/dist/ 19 | mv runtime-${VERSION}.zip ${BUILD_DIR}/dist/ 20 | version_="${VERSION//\./_}" 21 | aws lambda publish-layer-version \ 22 | --layer-name r-runtime-${version_} \ 23 | --zip-file fileb://${BUILD_DIR}/dist/runtime-${VERSION}.zip 24 | -------------------------------------------------------------------------------- /runtime/src/bootstrap: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -euo pipefail 4 | 5 | /opt/R/bin/Rscript /opt/bootstrap.R 6 | -------------------------------------------------------------------------------- /runtime/src/bootstrap.R: -------------------------------------------------------------------------------- 1 | source('/opt/runtime.R') 2 | tryCatch({ 3 | function_name <- initializeRuntime() 4 | while (TRUE) { 5 | handle_request(function_name) 6 | logReset() 7 | rm(list=ls()) 8 | source('/opt/runtime.R') 9 | function_name <- initializeRuntime() 10 | } 11 | }, error = throwInitError) 12 | -------------------------------------------------------------------------------- /runtime/src/runtime.R: -------------------------------------------------------------------------------- 1 | to_str <- function(x) { 2 | return(paste(capture.output(print(x)), collapse = "\n")) 3 | } 4 | 5 | error_to_payload <- function(error) { 6 | return(list(errorMessage = toString(error), errorType = class(error)[1])) 7 | } 8 | 9 | post_error <- function(error, url) { 10 | logerror(error, logger = 'runtime') 11 | res <- POST(url, 12 | add_headers("Lambda-Runtime-Function-Error-Type" = "Unhandled"), 13 | body = error_to_payload(error), 14 | encode = "json") 15 | logdebug("Posted result:\n%s", to_str(res), logger = 'runtime') 16 | } 17 | 18 | get_source_file_name <- function(file_base_name) { 19 | file_name <- paste0(file_base_name, ".R") 20 | if (! file.exists(file_name)) { 21 | file_name <- paste0(file_base_name, ".r") 22 | } 23 | if (! file.exists(file_name)) { 24 | stop(paste0('Source file does not exist: ', file_base_name, '.[R|r]')) 25 | } 26 | return(file_name) 27 | } 28 | 29 | invoke_lambda <- function(EVENT_DATA, function_name) { 30 | params <- fromJSON(EVENT_DATA) 31 | logdebug("Invoking function '%s' with parameters:\n%s", function_name, to_str(params), logger = 'runtime') 32 | result <- do.call(function_name, params) 33 | logdebug("Function returned:\n%s", to_str(result), logger = 'runtime') 34 | return(result) 35 | } 36 | 37 | initializeLogging <- function() { 38 | library(logging) 39 | 40 | basicConfig() 41 | addHandler(writeToConsole, logger='runtime') 42 | log_level <- Sys.getenv('LOGLEVEL', unset = NA) 43 | if (!is.na(log_level)) { 44 | setLevel(log_level, 'runtime') 45 | } 46 | } 47 | 48 | initializeRuntime <- function() { 49 | library(httr) 50 | library(jsonlite) 51 | 52 | initializeLogging() 53 | HANDLER <- Sys.getenv("_HANDLER") 54 | HANDLER_split <- strsplit(HANDLER, ".", fixed = TRUE)[[1]] 55 | file_base_name <- HANDLER_split[1] 56 | file_name <- get_source_file_name(file_base_name) 57 | logdebug("Sourcing '%s'", file_name, logger = 'runtime') 58 | source(file_name) 59 | function_name <- HANDLER_split[2] 60 | if (!exists(function_name, mode = "function")) { 61 | stop(paste0("Function \"", function_name, "\" does not exist")) 62 | } 63 | return(function_name) 64 | } 65 | 66 | AWS_LAMBDA_RUNTIME_API <- Sys.getenv("AWS_LAMBDA_RUNTIME_API") 67 | API_ENDPOINT <- paste0("http://", AWS_LAMBDA_RUNTIME_API, "/2018-06-01/runtime/") 68 | 69 | throwInitError <- function(error) { 70 | url <- paste0(API_ENDPOINT, "init/error") 71 | post_error(error, url) 72 | stop() 73 | } 74 | 75 | throwRuntimeError <- function(error, REQUEST_ID) { 76 | url <- paste0(API_ENDPOINT, "invocation/", REQUEST_ID, "/error") 77 | post_error(error, url) 78 | } 79 | 80 | postResult <- function(result, REQUEST_ID) { 81 | url <- paste0(API_ENDPOINT, "invocation/", REQUEST_ID, "/response") 82 | res <- POST(url, body = toJSON(result, auto_unbox = TRUE), encode = "raw", content_type_json()) 83 | logdebug("Posted result:\n%s", to_str(res), logger = 'runtime') 84 | } 85 | 86 | handle_request <- function(function_name) { 87 | event_url <- paste0(API_ENDPOINT, "invocation/next") 88 | event_response <- GET(event_url) 89 | REQUEST_ID <- event_response$headers$`Lambda-Runtime-Aws-Request-Id` 90 | tryCatch({ 91 | EVENT_DATA <- rawToChar(event_response$content) 92 | result <- invoke_lambda(EVENT_DATA, function_name) 93 | postResult(result, REQUEST_ID) 94 | }, 95 | error = function(error) { 96 | throwRuntimeError(error, REQUEST_ID) 97 | }) 98 | } 99 | -------------------------------------------------------------------------------- /template.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Transform: 'AWS::Serverless-2016-10-31' 3 | Parameters: 4 | Version: 5 | Type: String 6 | Globals: 7 | Function: 8 | Runtime: provided 9 | Timeout: 300 10 | MemorySize: 3008 11 | Layers: 12 | - !Ref RuntimeLayer 13 | Resources: 14 | RuntimeLayer: 15 | Type: AWS::Serverless::LayerVersion 16 | Properties: 17 | LayerName: !Sub r-runtime-${Version} 18 | ContentUri: runtime/build/layer/ 19 | LicenseInfo: MIT 20 | RuntimeLayerPermission: 21 | Type: AWS::Lambda::LayerVersionPermission 22 | Properties: 23 | Action: lambda:GetLayerVersion 24 | LayerVersionArn: !Ref RuntimeLayer 25 | Principal: "*" 26 | RecommendedLayer: 27 | Type: AWS::Serverless::LayerVersion 28 | Properties: 29 | LayerName: !Sub r-recommended-${Version} 30 | ContentUri: recommended/build/layer/ 31 | LicenseInfo: MIT 32 | RecommendedLayerPermission: 33 | Type: AWS::Lambda::LayerVersionPermission 34 | Properties: 35 | Action: lambda:GetLayerVersion 36 | LayerVersionArn: !Ref RecommendedLayer 37 | Principal: "*" 38 | AWSLayer: 39 | Type: AWS::Serverless::LayerVersion 40 | Properties: 41 | LayerName: !Sub r-awspack-${Version} 42 | ContentUri: awspack/build/layer/ 43 | LicenseInfo: MIT 44 | AWSLayerPermission: 45 | Type: AWS::Lambda::LayerVersionPermission 46 | Properties: 47 | Action: lambda:GetLayerVersion 48 | LayerVersionArn: !Ref AWSLayer 49 | Principal: "*" 50 | -------------------------------------------------------------------------------- /test-template.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Transform: 'AWS::Serverless-2016-10-31' 3 | Parameters: 4 | Version: 5 | Type: String 6 | Globals: 7 | Function: 8 | Runtime: provided 9 | Timeout: 900 10 | MemorySize: 3008 11 | Layers: 12 | - !Ref RuntimeLayer 13 | Resources: 14 | ExampleFunction: 15 | Type: 'AWS::Serverless::Function' 16 | Properties: 17 | Handler: script.handler 18 | CodeUri: tests/R/ 19 | FunctionName: !Sub ExampleFunction-${Version} 20 | LowerCaseExtensionFunction: 21 | Type: 'AWS::Serverless::Function' 22 | Properties: 23 | Handler: lowercase.handler 24 | CodeUri: tests/R/ 25 | FunctionName: !Sub LowerCaseExtensionFunction-${Version} 26 | MissingFunctionFunction: 27 | Type: 'AWS::Serverless::Function' 28 | Properties: 29 | Handler: script.handler_missing 30 | CodeUri: tests/R/ 31 | FunctionName: !Sub MissingFunctionFunction-${Version} 32 | HandlerAsVariableFunction: 33 | Type: 'AWS::Serverless::Function' 34 | Properties: 35 | Handler: script.handler_as_variable 36 | CodeUri: tests/R/ 37 | FunctionName: !Sub HandlerAsVariableFunction-${Version} 38 | MissingSourceFileFunction: 39 | Type: 'AWS::Serverless::Function' 40 | Properties: 41 | Handler: missing.handler 42 | CodeUri: tests/R/ 43 | FunctionName: !Sub MissingSourceFileFunction-${Version} 44 | MultipleArgumentsFunction: 45 | Type: 'AWS::Serverless::Function' 46 | Properties: 47 | Handler: script.handler_with_multiple_arguments 48 | CodeUri: tests/R/ 49 | FunctionName: !Sub MultipleArgumentsFunction-${Version} 50 | VariableArgumentsFunction: 51 | Type: 'AWS::Serverless::Function' 52 | Properties: 53 | Handler: script.handler_with_variable_arguments 54 | CodeUri: tests/R/ 55 | FunctionName: !Sub VariableArgumentsFunction-${Version} 56 | LoggingFunction: 57 | Type: 'AWS::Serverless::Function' 58 | Properties: 59 | Handler: script.handler_with_debug_logging 60 | CodeUri: tests/R/ 61 | Environment: 62 | Variables: 63 | LOGLEVEL: DEBUG 64 | FunctionName: !Sub LoggingFunction-${Version} 65 | MatrixFunction: 66 | Type: 'AWS::Serverless::Function' 67 | Properties: 68 | Handler: matrix.handler 69 | CodeUri: tests/R/ 70 | Layers: 71 | - !Ref RecommendedLayer 72 | FunctionName: !Sub MatrixFunction-${Version} 73 | MissingLibraryFunction: 74 | Type: 'AWS::Serverless::Function' 75 | Properties: 76 | Handler: matrix.handler 77 | CodeUri: tests/R/ 78 | FunctionName: !Sub MissingLibraryFunction-${Version} 79 | AWSFunction: 80 | Type: 'AWS::Serverless::Function' 81 | Properties: 82 | Handler: aws.handler 83 | CodeUri: tests/R/ 84 | Layers: 85 | - !Ref AWSLayer 86 | FunctionName: !Sub AWSFunction-${Version} 87 | ApiFunction: 88 | Type: 'AWS::Serverless::Function' 89 | Properties: 90 | Handler: api.handler 91 | CodeUri: tests/R/ 92 | FunctionName: !Sub ApiFunction-${Version} 93 | Events: 94 | Api: 95 | Type: Api 96 | Properties: 97 | Path: '/hello' 98 | Method: GET 99 | RuntimeLayer: 100 | Type: AWS::Serverless::LayerVersion 101 | Properties: 102 | LayerName: !Sub r-runtime-${Version}-test 103 | ContentUri: runtime/build/layer/ 104 | LicenseInfo: MIT 105 | RecommendedLayer: 106 | Type: AWS::Serverless::LayerVersion 107 | Properties: 108 | LayerName: !Sub r-recommended-${Version}-test 109 | ContentUri: recommended/build/layer/ 110 | LicenseInfo: MIT 111 | AWSLayer: 112 | Type: AWS::Serverless::LayerVersion 113 | Properties: 114 | LayerName: !Sub r-awspack-${Version}-test 115 | ContentUri: awspack/build/layer/ 116 | LicenseInfo: MIT 117 | -------------------------------------------------------------------------------- /tests/R/api.R: -------------------------------------------------------------------------------- 1 | library(jsonlite) 2 | 3 | handler <- function(headers, multiValueHeaders, queryStringParameters, multiValueQueryStringParameters, pathParameters, body, ...) { 4 | return( 5 | list( 6 | statusCode = 200, 7 | headers = list("Content-Type" = "application/json"), 8 | body = toJSON(list(hello = queryStringParameters$who), auto_unbox = TRUE) 9 | ) 10 | ) 11 | } 12 | -------------------------------------------------------------------------------- /tests/R/aws.R: -------------------------------------------------------------------------------- 1 | library(aws.s3) 2 | 3 | handler <- function() { 4 | usercsvobj <- get_object(object = 'examples/medicare/Medicare_Hospital_Provider.csv', bucket = 'awsglue-datasets', check_region = FALSE, region = 'us-east-1') 5 | csvcharobj <- rawToChar(usercsvobj) 6 | con <- textConnection(csvcharobj) 7 | data <- read.csv(con) 8 | close(con) 9 | return(data[1,]) 10 | } 11 | -------------------------------------------------------------------------------- /tests/R/lowercase.r: -------------------------------------------------------------------------------- 1 | handler <- function(x) { 2 | return(x + 1) 3 | } 4 | -------------------------------------------------------------------------------- /tests/R/matrix.R: -------------------------------------------------------------------------------- 1 | library(Matrix) 2 | 3 | handler <- function() { 4 | return(Matrix(1:6, 3, 2)[, 2]) 5 | } 6 | -------------------------------------------------------------------------------- /tests/R/script.R: -------------------------------------------------------------------------------- 1 | handler <- function(x) { 2 | return(x + 1) 3 | } 4 | 5 | handler_with_multiple_arguments <- function(x, y) { 6 | return(list(x = x, y = y)) 7 | } 8 | 9 | handler_with_variable_arguments <- function(...) { 10 | return(1) 11 | } 12 | 13 | handler_as_variable <- "foo" 14 | 15 | handler_with_debug_logging <- function(x) { 16 | return(1) 17 | } -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import socket 4 | import time 5 | 6 | LOGLEVEL = os.environ.get('LOGLEVEL', 'WARNING').upper() 7 | logging.basicConfig(level=LOGLEVEL) 8 | 9 | 10 | def wait_for_port(port: int, host: str = 'localhost', interval: int = 10, retries: int = 6) -> bool: 11 | for i in range(1, retries + 1): 12 | try: 13 | logging.info("Try %s/%s: Connecting to %s:%s", i, retries, host, port) 14 | s = socket.create_connection((host, port)) 15 | s.close() 16 | logging.info("Try %s/%s: Connection succeeded", i, retries) 17 | return True 18 | except ConnectionRefusedError as e: 19 | logging.info("Try %s/%s: Connection to %s:%s not possible: %s. Waiting %ss", 20 | i, retries, host, port, e, interval) 21 | time.sleep(interval) 22 | return False 23 | 24 | 25 | def get_function_name(name: str) -> str: 26 | return name if is_local() else '{0}-{1}'.format(name, get_version()) 27 | 28 | 29 | def get_version() -> str: 30 | return os.getenv('VERSION', '3_6_0') 31 | 32 | 33 | def is_local() -> bool: 34 | return os.getenv('INTEGRATION_TEST') != 'True' 35 | -------------------------------------------------------------------------------- /tests/sam.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from subprocess import Popen 3 | 4 | import boto3 5 | import botocore 6 | 7 | from tests import wait_for_port 8 | 9 | 10 | class LocalApi: 11 | 12 | def __init__(self, 13 | host: str = '127.0.0.1', 14 | port: int = 3000, 15 | template_path: str = None, 16 | parameter_overrides: dict = None, 17 | ): 18 | self.host = host 19 | self.port = port 20 | command = ['sam', 'local', 'start-api', '--host', self.host, '--port', str(self.port)] 21 | if template_path: 22 | command += ['--template', template_path] 23 | if parameter_overrides: 24 | command += ['--parameter-overrides', create_parameter_overrides(parameter_overrides)] 25 | self.process = Popen(command) 26 | 27 | def kill(self): 28 | self.process.kill() 29 | return_code = self.process.wait() 30 | logging.info('Killed server with code %s', return_code) 31 | 32 | def wait(self, interval: int = 10, retries: int = 6): 33 | wait_for_port(self.port, self.host, interval=interval, retries=retries) 34 | 35 | def get_uri(self) -> str: 36 | return 'http://{}:{}'.format(self.host, self.port) 37 | 38 | 39 | def start_local_api(host: str = '127.0.0.1', 40 | port: int = 3000, 41 | template_path: str = None, 42 | parameter_overrides: dict = None) -> LocalApi: 43 | server = LocalApi(host=host, 44 | port=port, 45 | template_path=template_path, 46 | parameter_overrides=parameter_overrides) 47 | server.wait() 48 | return server 49 | 50 | 51 | def create_parameter_overrides(parameter_overrides): 52 | return "'" + ' '.join(['ParameterKey={},ParameterValue={}'.format(key, value) for key, value in 53 | parameter_overrides.items()]) + "'" 54 | 55 | 56 | class LocalLambdaServer: 57 | 58 | def __init__(self, 59 | host: str = '127.0.0.1', 60 | port: int = 3001, 61 | template_path: str = None, 62 | parameter_overrides: dict = None, 63 | ): 64 | self.host = host 65 | self.port = port 66 | command = ['sam', 'local', 'start-lambda', '--host', self.host, '--port', str(self.port)] 67 | if template_path: 68 | command += ['--template', template_path] 69 | if parameter_overrides: 70 | command += ['--parameter-overrides', create_parameter_overrides(parameter_overrides)] 71 | self.process = Popen(command) 72 | 73 | def get_client(self): 74 | config = botocore.client.Config(signature_version=botocore.UNSIGNED, 75 | read_timeout=900, 76 | retries={'max_attempts': 0}, 77 | ) 78 | return boto3.client('lambda', 79 | endpoint_url="http://{}:{}".format(self.host, self.port), 80 | use_ssl=False, 81 | verify=False, 82 | config=config, 83 | ) 84 | 85 | def kill(self): 86 | self.process.kill() 87 | return_code = self.process.wait() 88 | logging.info('Killed server with code %s', return_code) 89 | 90 | def wait(self, interval: int = 10, retries: int = 6): 91 | wait_for_port(self.port, self.host, interval=interval, retries=retries) 92 | 93 | 94 | def start_local_lambda(host: str = '127.0.0.1', 95 | port: int = 3001, 96 | template_path: str = None, 97 | parameter_overrides: dict = None) -> LocalLambdaServer: 98 | server = LocalLambdaServer(host=host, 99 | port=port, 100 | template_path=template_path, 101 | parameter_overrides=parameter_overrides) 102 | server.wait() 103 | return server 104 | -------------------------------------------------------------------------------- /tests/test_api.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import requests 4 | 5 | from tests import get_version, is_local 6 | from tests.sam import LocalApi, start_local_api 7 | 8 | 9 | class TestApi(unittest.TestCase): 10 | api: LocalApi = None 11 | 12 | @classmethod 13 | def setUpClass(cls): 14 | if is_local(): 15 | cls.api = start_local_api(template_path="test-template.yaml", 16 | parameter_overrides={'Version': get_version()}, 17 | ) 18 | 19 | @unittest.skipUnless(is_local(), 'Only works locally') 20 | def test_api(self): 21 | response = requests.get('%s/hello' % self.api.get_uri(), params={'who': 'World'}) 22 | result = response.json() 23 | self.assertDictEqual({'hello': 'World'}, result) 24 | 25 | @classmethod 26 | def tearDownClass(cls): 27 | if is_local(): 28 | cls.api.kill() 29 | -------------------------------------------------------------------------------- /tests/test_aws.py: -------------------------------------------------------------------------------- 1 | import json 2 | import unittest 3 | 4 | import boto3 5 | 6 | from tests import get_function_name, get_version, is_local 7 | from tests.sam import LocalLambdaServer, start_local_lambda 8 | 9 | 10 | class TestAWSLayer(unittest.TestCase): 11 | lambda_server: LocalLambdaServer = None 12 | 13 | @classmethod 14 | def setUpClass(cls): 15 | if is_local(): 16 | cls.lambda_server = start_local_lambda(template_path="test-template.yaml", 17 | parameter_overrides={'Version': get_version()}, 18 | ) 19 | 20 | def get_client(self): 21 | return self.lambda_server.get_client() if is_local() else boto3.client('lambda') 22 | 23 | @unittest.skipUnless(is_local(), "Credentials missing for remote Lambda") 24 | def test_s3_get_object(self): 25 | lambda_client = self.get_client() 26 | response = lambda_client.invoke(FunctionName=get_function_name("AWSFunction")) 27 | raw_payload = response['Payload'].read().decode('utf-8') 28 | result = json.loads(raw_payload) 29 | self.assertEqual(1, len(result)) 30 | self.assertDictEqual({ 31 | "DRG.Definition": "039 - EXTRACRANIAL PROCEDURES W/O CC/MCC", 32 | "Provider.Id": "10001", 33 | "Provider.Name": "SOUTHEAST ALABAMA MEDICAL CENTER", 34 | "Provider.Street.Address": "1108 ROSS CLARK CIRCLE", 35 | "Provider.City": "DOTHAN", 36 | "Provider.State": "AL", 37 | "Provider.Zip.Code": 36301, 38 | "Hospital.Referral.Region.Description": "AL - Dothan", 39 | "Total.Discharges": 91, 40 | "Average.Covered.Charges": "$32963.07", 41 | "Average.Total.Payments": "$5777.24", 42 | "Average.Medicare.Payments": "$4763.73" 43 | }, result[0]) 44 | 45 | @classmethod 46 | def tearDownClass(cls): 47 | if is_local(): 48 | cls.lambda_server.kill() 49 | -------------------------------------------------------------------------------- /tests/test_matrix.py: -------------------------------------------------------------------------------- 1 | import json 2 | import unittest 3 | 4 | import boto3 5 | 6 | from tests import get_version, get_function_name, is_local 7 | from tests.sam import LocalLambdaServer, start_local_lambda 8 | 9 | 10 | class TestRecommendedLayer(unittest.TestCase): 11 | lambda_server: LocalLambdaServer = None 12 | 13 | @classmethod 14 | def setUpClass(cls): 15 | if is_local(): 16 | cls.lambda_server = start_local_lambda(template_path="test-template.yaml", 17 | parameter_overrides={'Version': get_version()}, 18 | ) 19 | 20 | def get_client(self): 21 | return self.lambda_server.get_client() if is_local() else boto3.client('lambda') 22 | 23 | def test_matrix(self): 24 | lambda_client = self.get_client() 25 | response = lambda_client.invoke(FunctionName=get_function_name("MatrixFunction")) 26 | raw_payload = response['Payload'].read().decode('utf-8') 27 | result = json.loads(raw_payload) 28 | self.assertEqual(3, len(result)) 29 | self.assertIn(4, result) 30 | self.assertIn(5, result) 31 | self.assertIn(6, result) 32 | 33 | @classmethod 34 | def tearDownClass(cls): 35 | if is_local(): 36 | cls.lambda_server.kill() 37 | -------------------------------------------------------------------------------- /tests/test_runtime.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import json 3 | import re 4 | import unittest 5 | 6 | import boto3 7 | 8 | from tests import get_version, get_function_name, is_local 9 | from tests.sam import LocalLambdaServer, start_local_lambda 10 | 11 | 12 | class TestRuntimeLayer(unittest.TestCase): 13 | lambda_server: LocalLambdaServer = None 14 | 15 | @classmethod 16 | def setUpClass(cls): 17 | if is_local(): 18 | cls.lambda_server = start_local_lambda(template_path="test-template.yaml", 19 | parameter_overrides={'Version': get_version()}, 20 | ) 21 | 22 | def get_client(self): 23 | return self.lambda_server.get_client() if is_local() else boto3.client('lambda') 24 | 25 | def test_script(self): 26 | lambda_client = self.get_client() 27 | response = lambda_client.invoke(FunctionName=get_function_name("ExampleFunction"), 28 | Payload=json.dumps({'x': 1}), 29 | ) 30 | raw_payload = response['Payload'].read().decode('utf-8') 31 | result = json.loads(raw_payload) 32 | self.assertEqual(2, result) 33 | 34 | def test_lowercase_extension(self): 35 | lambda_client = self.get_client() 36 | response = lambda_client.invoke(FunctionName=get_function_name("LowerCaseExtensionFunction"), 37 | Payload=json.dumps({'x': 1}), 38 | ) 39 | raw_payload = response['Payload'].read().decode('utf-8') 40 | result = json.loads(raw_payload) 41 | self.assertEqual(2, result) 42 | 43 | def test_multiple_arguments(self): 44 | lambda_client = self.get_client() 45 | payload = {'x': 'bar', 'y': 1} 46 | response = lambda_client.invoke(FunctionName=get_function_name("MultipleArgumentsFunction"), 47 | Payload=json.dumps(payload), 48 | ) 49 | raw_payload = response['Payload'].read().decode('utf-8') 50 | result = json.loads(raw_payload) 51 | self.assertDictEqual(payload, result) 52 | 53 | @unittest.skipIf(is_local(), 'Lambda local does not support log retrieval') 54 | def test_debug_logging(self): 55 | lambda_client = self.get_client() 56 | response = lambda_client.invoke(FunctionName=get_function_name("LoggingFunction"), 57 | LogType='Tail', 58 | Payload=json.dumps({'x': 1}), 59 | ) 60 | raw_payload = response['Payload'].read().decode('utf-8') 61 | result = json.loads(raw_payload) 62 | self.assertEqual(1, result) 63 | log = base64.b64decode(response['LogResult']).decode('utf-8') 64 | self.assertIn("runtime:Sourcing 'script.R'", log) 65 | self.assertIn("runtime:Invoking function 'handler_with_debug_logging' with parameters:\n$x\n[1] 1", log) 66 | self.assertIn("runtime:Function returned:\n[1] 1", log) 67 | self.assertIn("runtime:Posted result:\n", log) 68 | 69 | @unittest.skipIf(is_local(), 'Lambda local does not support log retrieval') 70 | def test_no_debug_logging(self): 71 | lambda_client = self.get_client() 72 | response = lambda_client.invoke(FunctionName=get_function_name("ExampleFunction"), 73 | LogType='Tail', 74 | Payload=json.dumps({'x': 1}), 75 | ) 76 | raw_payload = response['Payload'].read().decode('utf-8') 77 | result = json.loads(raw_payload) 78 | self.assertEqual(2, result) 79 | log = base64.b64decode(response['LogResult']).decode('utf-8') 80 | self.assertNotIn("Sourcing ", log) 81 | self.assertNotIn("Invoking function ", log) 82 | self.assertNotIn("Function returned:", log) 83 | self.assertNotIn("Posted result:", log) 84 | 85 | @unittest.skipIf(is_local(), 'Lambda local does not pass errors properly') 86 | def test_missing_source_file(self): 87 | lambda_client = self.get_client() 88 | response = lambda_client.invoke(FunctionName=get_function_name("MissingSourceFileFunction"), 89 | Payload=json.dumps({'y': 1}), 90 | ) 91 | raw_payload = response['Payload'].read().decode('utf-8') 92 | json_payload = json.loads(raw_payload) 93 | self.assertEqual('Unhandled', response['FunctionError']) 94 | self.assertIn('Source file does not exist: missing.[R|r]', json_payload['errorMessage']) 95 | self.assertEqual('simpleError', json_payload['errorType']) 96 | 97 | @unittest.skipIf(is_local(), 'Lambda local does not pass errors properly') 98 | def test_missing_function(self): 99 | lambda_client = self.get_client() 100 | response = lambda_client.invoke(FunctionName=get_function_name("MissingFunctionFunction"), 101 | Payload=json.dumps({'y': 1}), 102 | ) 103 | raw_payload = response['Payload'].read().decode('utf-8') 104 | json_payload = json.loads(raw_payload) 105 | self.assertEqual('Unhandled', response['FunctionError']) 106 | self.assertIn('Function "handler_missing" does not exist', json_payload['errorMessage']) 107 | self.assertEqual('simpleError', json_payload['errorType']) 108 | 109 | @unittest.skipIf(is_local(), 'Lambda local does not pass errors properly') 110 | def test_function_as_variable(self): 111 | lambda_client = self.get_client() 112 | response = lambda_client.invoke(FunctionName=get_function_name("HandlerAsVariableFunction"), 113 | Payload=json.dumps({'y': 1}), 114 | ) 115 | raw_payload = response['Payload'].read().decode('utf-8') 116 | json_payload = json.loads(raw_payload) 117 | self.assertEqual('Unhandled', response['FunctionError']) 118 | self.assertIn('Function "handler_as_variable" does not exist', json_payload['errorMessage']) 119 | self.assertEqual('simpleError', json_payload['errorType']) 120 | 121 | @unittest.skipIf(is_local(), 'Lambda local does not pass errors properly') 122 | def test_missing_argument(self): 123 | lambda_client = self.get_client() 124 | response = lambda_client.invoke(FunctionName=get_function_name("ExampleFunction")) 125 | raw_payload = response['Payload'].read().decode('utf-8') 126 | json_payload = json.loads(raw_payload) 127 | self.assertEqual('Unhandled', response['FunctionError']) 128 | self.assertIn('argument "x" is missing, with no default', json_payload['errorMessage']) 129 | self.assertEqual('simpleError', json_payload['errorType']) 130 | 131 | @unittest.skipIf(is_local(), 'Lambda local does not pass errors properly') 132 | def test_unused_argument(self): 133 | lambda_client = self.get_client() 134 | response = lambda_client.invoke(FunctionName=get_function_name("ExampleFunction"), 135 | Payload=json.dumps({'x': 1, 'y': 1}), 136 | ) 137 | raw_payload = response['Payload'].read().decode('utf-8') 138 | json_payload = json.loads(raw_payload) 139 | self.assertEqual('Unhandled', response['FunctionError']) 140 | self.assertIn('unused argument (y = 1)', json_payload['errorMessage']) 141 | self.assertEqual('simpleError', json_payload['errorType']) 142 | 143 | # @unittest.skipIf(is_local(), 'Fails locally with "argument list too long"') 144 | @unittest.skip('Fails with timeout') 145 | def test_long_argument(self): 146 | lambda_client = self.get_client() 147 | payload = {x: x for x in range(0, 100000)} 148 | response = lambda_client.invoke(FunctionName=get_function_name("VariableArgumentsFunction"), 149 | Payload=json.dumps(payload), 150 | ) 151 | raw_payload = response['Payload'].read().decode('utf-8') 152 | result = json.loads(raw_payload) 153 | self.assertEqual(1, result) 154 | 155 | @unittest.skipIf(is_local(), 'Lambda local does not pass errors properly') 156 | def test_missing_library(self): 157 | lambda_client = self.get_client() 158 | response = lambda_client.invoke(FunctionName=get_function_name("MissingLibraryFunction"), 159 | Payload=json.dumps({'y': 1}), 160 | ) 161 | raw_payload = response['Payload'].read().decode('utf-8') 162 | json_payload = json.loads(raw_payload) 163 | self.assertEqual('Unhandled', response['FunctionError']) 164 | self.assertIn('there is no package called ‘Matrix’', json_payload['errorMessage']) 165 | error_type = 'packageNotFoundError' if get_version() == '3_6_0' else 'simpleError' 166 | self.assertEqual(error_type, json_payload['errorType']) 167 | 168 | @classmethod 169 | def tearDownClass(cls): 170 | if is_local(): 171 | cls.lambda_server.kill() 172 | --------------------------------------------------------------------------------