├── .gitignore ├── LICENCE.md ├── README.md ├── deps.edn ├── example ├── bb.edn ├── build.clj ├── deps.edn ├── src │ └── demo │ │ ├── app.clj │ │ └── handler.clj └── terraform │ ├── .terraform.lock.hcl │ ├── main.tf │ ├── outputs.tf │ ├── setup.tf │ └── variables.tf └── src └── clj_lambda_reload └── core.clj /.gitignore: -------------------------------------------------------------------------------- 1 | .cpcache 2 | .nrepl-port 3 | .lsp/.cache 4 | .clj-kondo/.cache 5 | target 6 | .terraform 7 | terraform.tfstate 8 | terraform.tfstate.backup 9 | example/out 10 | .rebel_readline_history 11 | -------------------------------------------------------------------------------- /LICENCE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Kimmo Koskinen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JVM/Clojure AWS Lambda Sideloader 2 | 3 | A library to sideload code into a live JVM/Clojure AWS Lambda. 4 | 5 | The idea is to first build a JVM/Clojure AWS Lambda in the usual way, by AOT compiling Clojure code into Java bytecode, but in the handler, call a sideloader, that puts new Clojure code into the Classpath, by downloading a zip from S3. 6 | 7 | If there is new Clojure code available, the sideloader calls `(require ' :reload-all)` to reload application namespaces. The idea is to exclude Clojure source code from the AOT bundle, so when reloading with sideloaded Clojure code on the classpath, we let the Clojure compiler [pick up the new source code](https://github.com/clojure/clojure/blob/master/src/jvm/clojure/lang/RT.java#L440-L460), if it is newer than the compiled class files 8 | 9 | There is an example in the [example](example/) folder, that uses [pod-babashka-fswatcher](https://github.com/babashka/pod-babashka-fswatcher) to package application Clojure code into a zip file and upload to S3, when there are changes in the source folder. 10 | 11 | This way, you can write new code, and have it loaded just before handling an event in the Lambda. 12 | 13 | https://user-images.githubusercontent.com/57011/226206498-606be685-0760-4186-a551-4e7802ffa1f1.mov 14 | 15 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src"]} 2 | -------------------------------------------------------------------------------- /example/bb.edn: -------------------------------------------------------------------------------- 1 | {:pods {org.babashka/fswatcher {:version "0.0.3"}} 2 | :tasks 3 | {:init (def src "src.zip") 4 | :requires ([babashka.process :as p]) 5 | build {:doc "Build lambda" 6 | :task (clojure "-T:build uber")} 7 | deploy {:doc "Deploy lambda" 8 | :task (shell {:dir "terraform"} "terraform" "apply" (str "-var=src_file=" src) "-auto-approve")} 9 | invoke {:doc "Invoke lambda" 10 | :task (let [lambda-name (:out (shell {:out :string :dir "terraform"} "terraform" "output" "-raw" "lambda_name"))] 11 | (shell "aws" "lambda" "invoke" "--function-name" lambda-name "out") 12 | (println (slurp "out")))} 13 | tail-logs {:doc "Tail lambda logs" 14 | :task (let [lambda-name (:out (shell {:out :string :dir "terraform"} "terraform" "output" "-raw" "lambda_name"))] 15 | (shell "aws" "logs" "tail" "--follow" (str "/aws/lambda/" lambda-name)))} 16 | watch {:doc "Watch changes and package source code to S3 for sideloading" 17 | :requires ([babashka.fs :as fs] 18 | [pod.babashka.fswatcher :as fw]) 19 | :task (let [sideload-bucket (:out (shell {:out :string :dir "terraform"} "terraform" "output" "-raw" "sideload_bucket")) 20 | watcher (fw/watch "src" (fn [{:keys [path] :as event}] 21 | (when (and (not (.contains path ".#")) 22 | (.contains (name (:type event)) "write")) 23 | (println "Making sideload zip") 24 | (fs/zip (str "target/" src) "src" {:root "src"}) 25 | (shell "aws" "s3" "cp" (str "target/" src) (str "s3://" sideload-bucket "/" src)) 26 | (println "Upload done"))) 27 | {:recursive true})] 28 | (.addShutdownHook (Runtime/getRuntime) (Thread. (fn [] 29 | (println "Stopping watcher") 30 | (fw/unwatch watcher)))) 31 | (deref (promise)))}}} 32 | -------------------------------------------------------------------------------- /example/build.clj: -------------------------------------------------------------------------------- 1 | (ns build 2 | (:require [clojure.tools.build.api :as b])) 3 | 4 | (def basis (b/create-basis {:project "deps.edn"})) 5 | 6 | (defn uber [_] 7 | (b/delete {:path "target"}) 8 | (b/compile-clj {:basis basis 9 | :src-dirs ["src"] 10 | :class-dir "target/classes"}) 11 | (b/uber {:uber-file "target/demo.jar" 12 | :class-dir "target/classes" 13 | :basis basis})) 14 | -------------------------------------------------------------------------------- /example/deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["../src" ;; For clj-lambda-reload 2 | "src"] 3 | :deps {com.amazonaws/aws-lambda-java-core {:mvn/version "1.2.1"} 4 | io.github.crac/org-crac {:mvn/version "0.1.3"}} 5 | :aliases 6 | {:build {:extra-deps {io.github.clojure/tools.build {:git/tag "v0.9.3" 7 | :git/sha "e537cd1"} 8 | babashka/fs {:mvn/version "0.3.17"}} 9 | :ns-default build}}} 10 | -------------------------------------------------------------------------------- /example/src/demo/app.clj: -------------------------------------------------------------------------------- 1 | (ns demo.app) 2 | 3 | (defn main [] 4 | (let [message "Hello from application entrypoint! 18"] 5 | (println message) 6 | message)) 7 | 8 | (defn warmup [] 9 | (let [message "Warming up"] 10 | (println message) 11 | message)) 12 | -------------------------------------------------------------------------------- /example/src/demo/handler.clj: -------------------------------------------------------------------------------- 1 | (ns demo.handler 2 | (:require [clj-lambda-reload.core :as core] 3 | [demo.app :as app])) 4 | 5 | (gen-class 6 | :name "demo.handler" 7 | :implements [com.amazonaws.services.lambda.runtime.RequestStreamHandler 8 | org.crac.Resource] 9 | :post-init register-crac) 10 | 11 | (defn -handleRequest [this in out ctx] 12 | (core/sideload 'demo.app) 13 | (let [result (app/main)] 14 | (spit out result))) 15 | 16 | ;; crac stuff 17 | 18 | (defn -register-crac [this] 19 | (.register (org.crac.Core/getGlobalContext) this)) 20 | 21 | (defn -beforeCheckpoint [this context] 22 | (println "Before checkpoint") 23 | #_(core/sideload) 24 | (app/warmup) 25 | (println "Before checkpoint done")) 26 | 27 | (defn -afterRestore [this context] 28 | (println "After restore") 29 | (println "After restore done")) 30 | -------------------------------------------------------------------------------- /example/terraform/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/aws" { 5 | version = "4.59.0" 6 | constraints = "4.59.0" 7 | hashes = [ 8 | "h1:uBpb5w197ACmg0JGuouIR8Dzbtg7V9OxcsabBZ8JxgQ=", 9 | "zh:0341a460210463a0bebd5c12ce13dc49bd8cae2399b215418c5efa607fed84e4", 10 | "zh:0544e9bbdd31d3551e7273bed7326d26a28653fd9c26b5cd06ac8ed76f188798", 11 | "zh:3d13acd0363f0a48d2725cae9d224481df38dddb90ef4a66eb82303f0aa45a99", 12 | "zh:416f5b92d41dce1d7ee1a1acb06ba8b0f10679eecee2fcc134853adbb09d9757", 13 | "zh:80c9c3b901151cd697caa58bfa196816d4622e4ce11aa789e36efc460695313b", 14 | "zh:8fc3659ebdae1ac9de899f57e5a3a50274a2e96c46aa2cf74be51ffdac56300a", 15 | "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", 16 | "zh:a235b44ad074446a6138b3fb454dd0d234aacf7a1efea89d1eafac7284689d19", 17 | "zh:a36a7f1cd7f9f6c45127d916a65b5441cc0430393535a5a3de4b646405c50c41", 18 | "zh:c161c38727902271efa19020b95b69ebe0282989d575f31dff603a1d551bafd2", 19 | "zh:d1562223347c49cbe3ff6e7295e25816a35dfef862d28cd8a7870e7be6ec8093", 20 | "zh:e7a1d08bfe91d3789755ee587fc816907c3bea203342c717144c7459111ce20c", 21 | "zh:e89d5a668c391669ed323d493c5ea131fe8833d562a6fe31f525bdcbe959056e", 22 | "zh:f268ccd3e1a32ba7fd59bbf0c8d85611201c0c87462a2a5cddd02babde7b5fe8", 23 | "zh:fe8c2eae8c367d2cb7cade250a8d5f6c411ac4a8214c46df0a1fd90d9eaf7152", 24 | ] 25 | } 26 | 27 | provider "registry.terraform.io/hashicorp/random" { 28 | version = "3.4.3" 29 | hashes = [ 30 | "h1:saZR+mhthL0OZl4SyHXZraxyaBNVMxiZzks78nWcZ2o=", 31 | "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752", 32 | "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b", 33 | "zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53", 34 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 35 | "zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3", 36 | "zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5", 37 | "zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda", 38 | "zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6", 39 | "zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1", 40 | "zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d", 41 | "zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8", 42 | "zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93", 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /example/terraform/main.tf: -------------------------------------------------------------------------------- 1 | resource "random_pet" "demo" { 2 | } 3 | 4 | resource "aws_lambda_function" "demo" { 5 | function_name = "${random_pet.demo.id}-demo" 6 | role = aws_iam_role.demo.arn 7 | 8 | handler = "demo.handler" 9 | filename = "../target/demo.jar" 10 | source_code_hash = filebase64sha256("../target/demo.jar") 11 | 12 | runtime = "java11" 13 | 14 | memory_size = 3008 15 | 16 | environment { 17 | variables = { 18 | SIDELOAD_BUCKET = aws_s3_bucket.sideload.id 19 | SIDELOAD_SRC = var.src_file 20 | SIDELOAD_ENABLED = "true" 21 | } 22 | } 23 | } 24 | 25 | resource "aws_iam_role" "demo" { 26 | name = "${random_pet.demo.id}-demo" 27 | assume_role_policy = jsonencode({ 28 | Version = "2012-10-17" 29 | Statement = [ 30 | { 31 | Effect = "Allow" 32 | Principal = { 33 | Service = "lambda.amazonaws.com" 34 | } 35 | Action = "sts:AssumeRole" 36 | } 37 | ] 38 | }) 39 | 40 | inline_policy { 41 | name = "sideload" 42 | 43 | policy = jsonencode({ 44 | Version = "2012-10-17" 45 | Statement = [ 46 | { 47 | Action = [ 48 | "s3:GetObject*" 49 | ] 50 | Effect = "Allow" 51 | Resource = [ 52 | "${aws_s3_bucket.sideload.arn}/*" 53 | ] 54 | }, 55 | ] 56 | }) 57 | } 58 | } 59 | 60 | resource "aws_iam_role_policy_attachment" "demo" { 61 | role = aws_iam_role.demo.name 62 | policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" 63 | } 64 | 65 | resource "aws_s3_bucket" "sideload" { 66 | bucket = "${random_pet.demo.id}-demo" 67 | } 68 | 69 | resource "aws_s3_bucket_public_access_block" "sideload" { 70 | bucket = aws_s3_bucket.sideload.id 71 | 72 | block_public_acls = true 73 | block_public_policy = true 74 | ignore_public_acls = true 75 | restrict_public_buckets = true 76 | } 77 | -------------------------------------------------------------------------------- /example/terraform/outputs.tf: -------------------------------------------------------------------------------- 1 | output "lambda_name" { 2 | value = aws_lambda_function.demo.function_name 3 | } 4 | 5 | output "sideload_bucket" { 6 | value = aws_s3_bucket.sideload.id 7 | } 8 | -------------------------------------------------------------------------------- /example/terraform/setup.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | aws = { 4 | source = "hashicorp/aws" 5 | version = "4.59.0" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /example/terraform/variables.tf: -------------------------------------------------------------------------------- 1 | variable "src_file" { 2 | description = "Sideload source archive" 3 | } 4 | -------------------------------------------------------------------------------- /src/clj_lambda_reload/core.clj: -------------------------------------------------------------------------------- 1 | (ns clj-lambda-reload.core 2 | (:require [clojure.string :as str] 3 | [clojure.java.io :as io]) 4 | (:import (java.net URL) 5 | (java.time ZonedDateTime ZoneOffset) 6 | (java.time.format DateTimeFormatter) 7 | (java.security MessageDigest) 8 | (javax.crypto Mac) 9 | (javax.crypto.spec SecretKeySpec))) 10 | 11 | (def src-last-modified (atom nil)) 12 | 13 | (def date-time-format (DateTimeFormatter/ofPattern "yyyyMMdd'T'HHmmss'Z'")) 14 | (def date-format (DateTimeFormatter/ofPattern "yyyyMMdd")) 15 | 16 | (defn sha-256 [s] 17 | (let [md (MessageDigest/getInstance "SHA-256")] 18 | (.update md (.getBytes s)) 19 | (.digest md))) 20 | 21 | (defn- hmac-sha256 [key-bytes data] 22 | (let [mac (Mac/getInstance "HmacSHA256")] 23 | (doto mac 24 | (.init (SecretKeySpec. key-bytes "HmacSHA256")) 25 | (.update (.getBytes data))) 26 | (.doFinal mac))) 27 | 28 | (defn bytes->hex [bs] 29 | (str/join (map #(format "%02x" %) bs))) 30 | 31 | ;; Sign HTTP request as per https://docs.aws.amazon.com/general/latest/gr/create-signed-request.html 32 | (defn open-signed-connection [url] 33 | (let [access-key-id (System/getenv "AWS_ACCESS_KEY_ID") 34 | secret-access-key (System/getenv "AWS_SECRET_ACCESS_KEY") 35 | session-token (System/getenv "AWS_SESSION_TOKEN") 36 | region (System/getenv "AWS_REGION") 37 | 38 | now (ZonedDateTime/now ZoneOffset/UTC) 39 | x-amz-date (.format date-time-format now) 40 | date (.format date-format now) 41 | service "s3" 42 | algorithm "AWS4-HMAC-SHA256" 43 | hashed-payload (bytes->hex (sha-256 "")) 44 | host (.getHost url) 45 | headers-to-sign (sort-by first (remove nil? 46 | [["host" host] 47 | ["x-amz-content-sha256" hashed-payload] 48 | ["x-amz-date" x-amz-date] 49 | (when session-token 50 | ["x-amz-security-token" session-token])])) 51 | signed-headers (str/join ";" (map first headers-to-sign)) 52 | credential-scope (str/join "/" [date 53 | region 54 | service 55 | "aws4_request"]) 56 | canonical-request (str/join "\n" ["GET" ;; HTTPMethod 57 | (.getPath url) ;; CanonicalUri 58 | "" ;; CanonicalQueryString 59 | (str/join (map (fn [[header value]] 60 | (str header ":" value "\n")) 61 | headers-to-sign)) ;; CanonicalHeaders 62 | signed-headers ;; SignedHeaders 63 | hashed-payload ;; HashedPayload 64 | ]) 65 | canonical-request-hash (bytes->hex (sha-256 canonical-request)) 66 | string-to-sign (str/join "\n" [algorithm ;; Algorithm 67 | x-amz-date ;; RequestDateTime 68 | credential-scope ;; CredentialScope 69 | canonical-request-hash]) 70 | signature (-> (.getBytes (str "AWS4" secret-access-key)) 71 | (hmac-sha256 date) 72 | (hmac-sha256 region) 73 | (hmac-sha256 service) 74 | (hmac-sha256 "aws4_request") 75 | (hmac-sha256 string-to-sign) 76 | bytes->hex) 77 | authorization-header (str algorithm " " 78 | "Credential=" access-key-id "/" credential-scope ", " 79 | "SignedHeaders=" signed-headers ", " 80 | "Signature=" signature) 81 | url-connection (.openConnection url)] 82 | ;; Prevent caching 83 | (.setUseCaches url-connection false) 84 | (.setDefaultUseCaches url-connection false) 85 | (.addRequestProperty url-connection "authorization" authorization-header) 86 | (.addRequestProperty url-connection "x-amz-date" x-amz-date) 87 | (.addRequestProperty url-connection "x-amz-content-sha256" hashed-payload) 88 | (when session-token 89 | (.addRequestProperty url-connection "x-amz-security-token" session-token)) 90 | url-connection)) 91 | 92 | (defn do-sideload [bucket src ns-symbol] 93 | (println (format "Sideloader active for s3://%s/%s" bucket src)) 94 | (let [t (Thread/currentThread)] 95 | (when-not (instance? clojure.lang.DynamicClassLoader (.getContextClassLoader t)) 96 | (println "Installing DynamicClassLoader") 97 | (.setContextClassLoader t (clojure.lang.DynamicClassLoader. (.getContextClassLoader t))))) 98 | (let [url-spec (format "https://%s.s3.%s.amazonaws.com/%s" 99 | bucket 100 | (System/getenv "AWS_REGION") 101 | src) 102 | timestamp (swap! src-last-modified 103 | (fn [v] 104 | (if (not v) 105 | (do 106 | (println "First load") 107 | (let [url (URL. url-spec)] 108 | (with-open [in (.getInputStream (open-signed-connection url))] 109 | (io/copy in (io/file "/tmp/src.zip"))) 110 | (.addURL (.getContextClassLoader (Thread/currentThread)) (URL. "file:/tmp/src.zip")) 111 | (let [url-connection (open-signed-connection url) 112 | status (.getResponseCode url-connection) 113 | last-modified (.getLastModified url-connection)] 114 | (when (not= 200 status) 115 | (println "Sideloader failed to new code load: " status)) 116 | (.disconnect url-connection) 117 | last-modified))) 118 | (do 119 | (println "Successive load") 120 | (let [url (URL. url-spec) 121 | url-connection (open-signed-connection url) 122 | status (.getResponseCode url-connection) 123 | last-modified (.getLastModified url-connection)] 124 | (.disconnect url-connection) 125 | (when (not= 200 status) 126 | (println "Sideloader failed to load new code load: " status)) 127 | (when (not= last-modified v) 128 | (println "New source available, reloading") 129 | (with-open [in (.getInputStream (open-signed-connection url))] 130 | (io/copy in (io/file "/tmp/src.zip"))) 131 | (.setDefaultUseCaches (.openConnection (URL. "file:/tmp/src.zip")) false) 132 | (require ns-symbol :reload-all) 133 | (println "reload done")) 134 | last-modified)))))] 135 | (println (str "Sideload src timestamp: " timestamp)))) 136 | 137 | (defn sideload [ns-symbol] 138 | (let [sideload-bucket (System/getenv "SIDELOAD_BUCKET") 139 | sideload-src (System/getenv "SIDELOAD_SRC") 140 | sideload-enabled (System/getenv "SIDELOAD_ENABLED")] 141 | (when (and sideload-bucket sideload-src sideload-enabled) 142 | (do-sideload sideload-bucket sideload-src ns-symbol)))) 143 | --------------------------------------------------------------------------------