├── version-prefix.txt ├── test ├── resources │ ├── .gitignore │ ├── lib2 │ │ ├── lib2 │ │ │ └── resource2.json │ │ ├── data_readers.clj │ │ └── resource1.edn │ ├── my-app │ │ ├── config │ │ │ └── config.edn │ │ ├── resources │ │ │ └── resource1.edn │ │ ├── src │ │ │ └── my_app │ │ │ │ ├── compilation_error.clj │ │ │ │ └── server.clj │ │ └── deps.edn │ ├── greeting │ │ └── greeting.txt │ ├── badlib │ │ └── bad-input.edn │ ├── lib1 │ │ ├── data_readers.clj │ │ └── resource1.edn │ ├── lib4 │ │ └── data_readers.cljc │ ├── lib5 │ │ └── with-reader-macros.edn │ ├── lib6 │ │ └── with-reader-macros.edn │ ├── fake-app.tar │ └── lib3 │ │ └── lib3.jar ├── unit │ └── vessel │ │ ├── jib │ │ ├── helpers_test.clj │ │ ├── containerizer_test.clj │ │ └── pusher_test.clj │ │ ├── test_helpers.clj │ │ ├── resource_merge_test.clj │ │ ├── api_test.clj │ │ ├── misc_test.clj │ │ ├── cli_test.clj │ │ ├── builder_test.clj │ │ └── image_test.clj ├── vessel │ └── jib │ │ └── credentials_test.clj └── integration │ └── vessel │ └── program_integration_test.clj ├── .dir-locals.el ├── .gitignore ├── .github ├── issue-template.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ └── test.yaml ├── Makefile ├── deps.edn ├── src └── vessel │ ├── jib │ ├── helpers.clj │ ├── credentials.clj │ ├── containerizer.clj │ └── pusher.clj │ ├── api.clj │ ├── sh.clj │ ├── resource_merge.clj │ ├── image.clj │ ├── cli.clj │ ├── misc.clj │ ├── program.clj │ └── builder.clj ├── README.md ├── CHANGELOG.md └── LICENSE /version-prefix.txt: -------------------------------------------------------------------------------- 1 | 0.2 2 | -------------------------------------------------------------------------------- /test/resources/.gitignore: -------------------------------------------------------------------------------- 1 | !fake-app.tar 2 | -------------------------------------------------------------------------------- /test/resources/lib2/lib2/resource2.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /test/resources/my-app/config/config.edn: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /test/resources/greeting/greeting.txt: -------------------------------------------------------------------------------- 1 | Hello world! 2 | -------------------------------------------------------------------------------- /test/resources/badlib/bad-input.edn: -------------------------------------------------------------------------------- 1 | {:foo :bar 2 | :baz #_ :biff} 3 | -------------------------------------------------------------------------------- /test/resources/lib1/data_readers.clj: -------------------------------------------------------------------------------- 1 | {lib1/url lib1.url/string->url} 2 | -------------------------------------------------------------------------------- /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ((nil . ((cider-clojure-cli-global-options . "-Adev")))) 2 | -------------------------------------------------------------------------------- /test/resources/lib4/data_readers.cljc: -------------------------------------------------------------------------------- 1 | {lib4/usd lib4.money/bigdec->money} 2 | -------------------------------------------------------------------------------- /test/resources/lib2/data_readers.clj: -------------------------------------------------------------------------------- 1 | {lib2/time lib2.time/string->local-date-time} 2 | -------------------------------------------------------------------------------- /test/resources/lib5/with-reader-macros.edn: -------------------------------------------------------------------------------- 1 | {:k1 :v2 2 | :k2 #unknown/macro :v2} 3 | -------------------------------------------------------------------------------- /test/resources/my-app/resources/resource1.edn: -------------------------------------------------------------------------------- 1 | {:app-name "my-app", :version "1.0.0"} 2 | -------------------------------------------------------------------------------- /test/resources/lib6/with-reader-macros.edn: -------------------------------------------------------------------------------- 1 | {:k1 :override-k1 2 | :k3 #weird/macro :v3} 3 | -------------------------------------------------------------------------------- /test/resources/fake-app.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nubank/vessel/HEAD/test/resources/fake-app.tar -------------------------------------------------------------------------------- /test/resources/lib3/lib3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nubank/vessel/HEAD/test/resources/lib3/lib3.jar -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.tar 2 | .vessel 3 | .cpcache 4 | .nrepl-port 5 | classes 6 | pom.xml 7 | scratch.clj 8 | target 9 | -------------------------------------------------------------------------------- /test/resources/lib1/resource1.edn: -------------------------------------------------------------------------------- 1 | {:list [1 2 3 4] 2 | :map {:a 1 3 | :c 4} 4 | :set #{1 2} 5 | :old-key 4 6 | :same-key :old-value} 7 | -------------------------------------------------------------------------------- /test/resources/lib2/resource1.edn: -------------------------------------------------------------------------------- 1 | {:list [4 5 6] 2 | :map {:a 2 3 | :b 3} 4 | :set #{1 3} 5 | :new-key 4 6 | :same-key :new-value} 7 | -------------------------------------------------------------------------------- /test/resources/my-app/src/my_app/compilation_error.clj: -------------------------------------------------------------------------------- 1 | (ns my-app.compilation-error 2 | (:gen-class)) 3 | 4 | ;; Intentionally has a compilation error 5 | (boom!) 6 | -------------------------------------------------------------------------------- /test/resources/my-app/deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src" "resources"] 2 | :deps 3 | {org.clojure/clojure #:mvn{:version "1.10.1"} 4 | org.clojure/data.json #:mvn{:version "0.2.6"} 5 | org.eclipse.jetty/jetty-server #:mvn{:version "9.4.25.v20191220"}} 6 | :aliases {:dev {}}} 7 | -------------------------------------------------------------------------------- /.github/issue-template.md: -------------------------------------------------------------------------------- 1 | ## Expected Behavior 2 | 3 | ## Actual Behavior 4 | 5 | ## Steps to Reproduce the Problem (if it's a bug) 6 | 7 | 1. 8 | 2. 9 | 3. 10 | 11 | ## Additional Info 12 | 13 | - Vessel version: 14 | - Clojure version: 15 | - Java version: 16 | 17 | 18 | -------------------------------------------------------------------------------- /test/resources/my-app/src/my_app/server.clj: -------------------------------------------------------------------------------- 1 | (ns my-app.server 2 | (:gen-class) 3 | (:require [clojure.data.json :as json] 4 | [clojure.edn :as edn] 5 | [clojure.java.io :as io])) 6 | 7 | (defn -main [& args] 8 | (-> (io/resource "resource1.edn") 9 | slurp 10 | edn/read-string 11 | json/write-str 12 | println)) 13 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | * **Please check if the PR fulfills these requirements** 4 | - [ ] There is an open issue describing the problem that this pr intents to solve. 5 | - [ ] You have a descriptive commit message with a short title (first line). 6 | - [ ] Tests for the changes have been added (for bug fixes / features). 7 | - [ ] Docs have been added / updated (for bug fixes / features). 8 | 9 | * **What kind of change does this PR introduce?** (Bug fix, feature, docs update, ...) 10 | 11 | 12 | * **What is the current behavior?** 13 | 14 | 15 | 16 | * **What is the new behavior (if this is a feature change)?** 17 | 18 | 19 | 20 | * **Does this PR introduce a breaking change?** (What changes might users need to make in their application due to this PR?) 21 | 22 | 23 | 24 | * **Other information**: 25 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Nubank 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | VERSION = $(shell cat version-prefix.txt).$(shell git rev-list --count master) 16 | 17 | .PHONY: build 18 | 19 | build: 20 | @./build/uberjar.sh $(VERSION) 21 | 22 | clean: 23 | @rm -rf target 24 | 25 | unit-test: 26 | @./build/test.sh -d test/unit 27 | 28 | integration-test: 29 | @./build/test.sh -d test/integration 30 | 31 | release: clean 32 | @./build/release.sh $(VERSION) 33 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src" "resources"] 2 | :deps 3 | {org.clojure/data.json #:mvn{:version "0.2.6"} 4 | org.clojure/clojure #:mvn{:version "1.10.0"} 5 | com.cognitect.aws/api #:mvn{:version "0.8.456"} 6 | com.cognitect.aws/endpoints #:mvn{:version "1.1.11.789"} 7 | org.clojure/tools.cli #:mvn{:version "0.4.2"} 8 | org.clojure/tools.namespace #:mvn{:version "0.3.1"} 9 | com.cognitect.aws/ecr #:mvn{:version "798.2.678.0"} 10 | progrock/progrock #:mvn{:version "0.1.2"} 11 | com.google.cloud.tools/jib-core #:mvn{:version "0.13.0"} 12 | org.clojure/core.async #:mvn{:version "1.0.567"}} 13 | :aliases 14 | {:dev 15 | {:extra-paths ["dev/resources" "test/unit" "test/integration"] 16 | :extra-deps 17 | {cognitect/test-runner 18 | {:git/url "https://github.com/cognitect-labs/test-runner.git" 19 | :sha "cb96e80f6f3d3b307c59cbeb49bb0dcb3a2a780b"} 20 | org.clojure/test.check #:mvn{:version "0.10.0-RC1"} 21 | nubank/mockfn #:mvn{:version "0.6.1"} 22 | org.apache.commons/commons-vfs2 #:mvn{:version "2.4.1"} 23 | babashka/fs {:mvn/version "0.5.22"} 24 | io.github.tonsky/clj-reload {:mvn/version "0.7.1"} 25 | nubank/matcher-combinators #:mvn{:version "3.9.1"}}}}} 26 | -------------------------------------------------------------------------------- /test/unit/vessel/jib/helpers_test.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.jib.helpers-test 2 | (:require [clojure.java.io :as io] 3 | [clojure.test :refer :all] 4 | [matcher-combinators.matchers :as m] 5 | [matcher-combinators.test :refer [match?]] 6 | [vessel.jib.helpers :as jib.helpers] 7 | [vessel.misc :as misc] 8 | [vessel.test-helpers :refer [ensure-clean-test-dir]]) 9 | (:import com.google.cloud.tools.jib.api.AbsoluteUnixPath)) 10 | 11 | (use-fixtures :once (ensure-clean-test-dir)) 12 | 13 | (deftest string->absolute-unix-path-test 14 | (is (instance? AbsoluteUnixPath (jib.helpers/string->absolute-unix-path "/")))) 15 | 16 | (deftest extract-tarball-test 17 | (let [tarball (io/file "test/resources/fake-app.tar") 18 | destination (io/file "target/tests/helpers-test")] 19 | (jib.helpers/extract-tarball tarball destination) 20 | (is (match? (m/in-any-order ["030a57e84b5be8d31b3c061ff7d7653836673f50475be0a507188ced9d0763d1.tar.gz" 21 | "051334be9afdd6a54c28ef9f063d2cddf7dbf79fcc9b1b0965cb1f69403db6b5.tar.gz" 22 | "config.json" 23 | "manifest.json"]) 24 | (map #(.getName %) (misc/filter-files (file-seq destination))))))) 25 | -------------------------------------------------------------------------------- /test/unit/vessel/test_helpers.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.test-helpers 2 | (:require [clojure.java.io :as io] 3 | [clojure.java.shell :as shell] 4 | [clojure.string :as string] 5 | [vessel.misc :as misc]) 6 | (:import java.io.File)) 7 | 8 | (defmacro ensure-clean-test-dir 9 | "Expands to a fixture function that creates a clean directory under target/tests/. 10 | 11 | Therefore, by caling (use-fixture :each (ensure-clean-dir)) in a 12 | namespace named my-feature-test, will create a fresh directory named 13 | target/tests/my-feature-test before each test in this namespace." 14 | [] 15 | `(fn [test#] 16 | (misc/make-empty-dir "target" "tests" ~(last (string/split (name (ns-name *ns*)) #"\."))) 17 | (test#))) 18 | 19 | (defn ^String classpath 20 | "Runs the clojure -Spath command to determine the classpath of the 21 | project at working-dir." 22 | [^File working-dir] 23 | (let [{:keys [exit err out]} 24 | (shell/sh "clojure" "-Spath" 25 | :dir working-dir 26 | :env (into {} (System/getenv)))] 27 | (if (zero? exit) 28 | (->> (string/split (string/trim out) #":") 29 | (map (fn [path] 30 | (if (.isAbsolute (io/file path)) 31 | path 32 | (str (io/file (.getAbsoluteFile working-dir) path))))) 33 | (string/join ":")) 34 | (throw (ex-info "clojure -Spath failed" 35 | {:process-output err}))))) 36 | -------------------------------------------------------------------------------- /src/vessel/jib/helpers.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.jib.helpers 2 | "Helper functions for dealing with orthogonal features of Google Jib." 3 | (:require [vessel.misc :as misc]) 4 | (:import com.google.cloud.tools.jib.api.AbsoluteUnixPath 5 | com.google.cloud.tools.jib.event.events.ProgressEvent 6 | com.google.cloud.tools.jib.tar.TarExtractor 7 | java.io.File)) 8 | 9 | (defn ^AbsoluteUnixPath string->absolute-unix-path 10 | [^String path] 11 | (AbsoluteUnixPath/get path)) 12 | 13 | (defn log-event-handler 14 | "Returns a consumer that handles log events triggered by Jib." 15 | [^String handler-name] 16 | (misc/java-consumer 17 | #(misc/log* (.getLevel %) handler-name (.getMessage %)))) 18 | 19 | (defn progress-event-handler 20 | "Returns a consumer that handles progress events triggered by Jib." 21 | [^String handler-name] 22 | (misc/java-consumer (fn [^ProgressEvent progress-event] 23 | (misc/log* :progress handler-name "%s (%.2f%%)" 24 | (.. progress-event getAllocation getDescription) 25 | (* (.. progress-event getAllocation getFractionOfRoot) 26 | (.getUnits progress-event) 27 | 100))))) 28 | 29 | (defn extract-tarball 30 | "Extracts the tarball into the specified destination." 31 | [^File tarball ^File destination] 32 | (TarExtractor/extract (misc/string->java-path (str tarball)) 33 | (misc/string->java-path (str destination)))) 34 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: Vessel Tests 2 | 3 | on: 4 | pull_request: 5 | types: [opened, reopened, synchronize] 6 | 7 | jobs: 8 | unit-test: 9 | runs-on: ubuntu-latest 10 | container: 11 | image: cimg/clojure:1.11.1 12 | options: -u 1001:115 13 | 14 | steps: 15 | - name: Checkout repository 16 | uses: actions/checkout@v4 17 | 18 | - name: Cache Maven deps 19 | uses: actions/cache@v4 20 | env: 21 | cache-name: maven-deps 22 | with: 23 | path: /__w/vessel/vessel/?/.m2 24 | key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('deps.edn') }} 25 | restore-keys: | 26 | ${{ runner.os }}-build-${{ env.cache-name }}- 27 | ${{ runner.os }}-build- 28 | ${{ runner.os }}- 29 | 30 | - name: Run unit tests 31 | run: 'make unit-test' 32 | 33 | integration-test: 34 | runs-on: ubuntu-latest 35 | container: 36 | image: cimg/clojure:1.11.1 37 | env: 38 | VESSEL_TEST_REGISTRY: registry 39 | options: -u 1001:115 40 | 41 | services: 42 | registry: 43 | image: registry:2 44 | ports: 45 | - 5000:5000 46 | 47 | steps: 48 | - name: Checkout repository 49 | uses: actions/checkout@v4 50 | 51 | - name: Cache Maven deps 52 | uses: actions/cache@v4 53 | env: 54 | cache-name: maven-deps 55 | with: 56 | path: /__w/vessel/vessel/?/.m2 57 | key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('deps.edn') }} 58 | restore-keys: | 59 | ${{ runner.os }}-build-${{ env.cache-name }}- 60 | ${{ runner.os }}-build- 61 | ${{ runner.os }}- 62 | 63 | - name: Run integration tests 64 | run: 'make integration-test' 65 | -------------------------------------------------------------------------------- /test/unit/vessel/resource_merge_test.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.resource-merge-test 2 | (:require [clojure.java.io :as io] 3 | [clojure.test :refer [deftest is testing]] 4 | [vessel.resource-merge :as merge])) 5 | 6 | (deftest find-matching-rule-test 7 | (let [data-readers-file (io/file "test/resources/lib1/data_readers.clj") 8 | edn-file (io/file "test/resources/lib1/resource1.edn")] 9 | (testing "if rules are empty, it returns nil" 10 | (is (nil? (merge/find-matching-rule [] edn-file)))) 11 | 12 | (testing "if no rules match, it returns nil" 13 | (let [rule {:match-fn (constantly false) 14 | :read-fn (constantly "rule") 15 | :merge-fn (constantly "rule") 16 | :write-fn (constantly "rule")}] 17 | (is (nil? (merge/find-matching-rule [rule] edn-file))))) 18 | 19 | (testing "if multiple rules match, it returns the first one" 20 | (let [first-rule {:match-fn (constantly true) 21 | :read-fn (constantly "first-rule") 22 | :merge-fn (constantly "first-rule") 23 | :write-fn (constantly "first-rule")} 24 | second-rule {:match-fn (constantly true) 25 | :read-fn (constantly "second-rule") 26 | :merge-fn (constantly "second-rule") 27 | :write-fn (constantly "second-rule")}] 28 | (is (= first-rule 29 | (merge/find-matching-rule [first-rule second-rule] edn-file))))) 30 | 31 | (testing "with base-rules" 32 | (testing "if file matches data_readers pattern, it returns the data-readers-base-rule" 33 | (is (= merge/data-readers-base-rule 34 | (merge/find-matching-rule merge/base-rules data-readers-file)))) 35 | 36 | (testing "if file ends with .edn, it returns the edn-base-rule" 37 | (is (= merge/edn-base-rule 38 | (merge/find-matching-rule merge/base-rules edn-file))))))) 39 | -------------------------------------------------------------------------------- /src/vessel/api.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.api 2 | (:require [clojure.data.json :as json] 3 | [clojure.java.io :as io] 4 | [vessel.builder :as builder] 5 | [vessel.image :as image] 6 | [vessel.jib.containerizer :as jib.containerizer] 7 | [vessel.jib.pusher :as jib.pusher] 8 | [vessel.misc :as misc]) 9 | (:import java.io.Writer)) 10 | 11 | (def vessel-dir (io/file ".vessel")) 12 | 13 | (defmacro ^:private with-elapsed-time 14 | "Evaluates body and shows a log message by displaying the elapsed time in the process. 15 | 16 | Returns whatever body yelds." 17 | [^String message & body] 18 | `(let [start# (misc/now) 19 | result# (do ~@body)] 20 | (misc/log :info "%s %s" 21 | ~message 22 | (misc/duration->string (misc/duration-between start# (misc/now)))) 23 | result#)) 24 | 25 | (defn containerize 26 | "Containerizes a Clojure application." 27 | [{:keys [verbose?] :as options}] 28 | (binding [misc/*verbose-logs* verbose?] 29 | (with-elapsed-time "Successfully containerized in" 30 | (let [opts (assoc options :target-dir (misc/make-empty-dir vessel-dir))] 31 | (-> (builder/build-app opts) 32 | (image/render-image-spec opts) 33 | jib.containerizer/containerize))))) 34 | 35 | (defn build 36 | "Builds a Clojure application." 37 | [{:keys [verbose?] :as options}] 38 | (binding [misc/*verbose-logs* verbose?] 39 | (with-elapsed-time "Successfully built in" 40 | (builder/build-app options)))) 41 | 42 | (defn- write-manifest 43 | "Writes the manifest to the output as a JSON object." 44 | [^Writer output manifest] 45 | (binding [*out* output] 46 | (println (json/write-str manifest)))) 47 | 48 | (defn manifest 49 | [{:keys [attributes object output]}] 50 | {:pre [attributes object output]} 51 | (->> (into {} attributes) 52 | (assoc {} object) 53 | (write-manifest output))) 54 | 55 | (defn image 56 | [{:keys [attributes base-image manifests output registry repository tag]}] 57 | {:pre [output registry repository]} 58 | (let [merge-all (partial apply merge) 59 | image-manifest (-> {:image (into {:repository repository :registry registry} attributes)} 60 | (misc/assoc-some :base-image base-image) 61 | (merge-all manifests)) 62 | image-tag (or tag (misc/sha-256 image-manifest))] 63 | (write-manifest output (assoc-in image-manifest [:image :tag] image-tag)))) 64 | 65 | (defn push 66 | "Pushes a tarball to a registry." 67 | [{:keys [verbose-logs?] :as options}] 68 | (binding [misc/*verbose-logs* verbose-logs?] 69 | (with-elapsed-time "Successfully pushed in" 70 | (jib.pusher/push 71 | (assoc options :temp-dir (misc/make-empty-dir vessel-dir)))))) 72 | -------------------------------------------------------------------------------- /test/unit/vessel/jib/containerizer_test.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.jib.containerizer-test 2 | (:require [clojure.java.io :as io] 3 | [clojure.test :refer :all] 4 | [matcher-combinators.test :refer [match?]] 5 | [vessel.jib.containerizer :as jib.containerizer] 6 | [vessel.misc :as misc] 7 | [vessel.test-helpers :refer [ensure-clean-test-dir]]) 8 | (:import org.apache.commons.vfs2.VFS)) 9 | 10 | (def tar-path "target/tests/containerizer-test/my-app.tar") 11 | 12 | (defn read-from-tarball 13 | [^String file-name] 14 | (let [cwd (str (.getCanonicalFile (io/file "."))) 15 | tar-file (format "tar:%s/%s!/%s" cwd tar-path file-name)] 16 | (.. VFS getManager 17 | (resolveFile tar-file) 18 | getContent 19 | getInputStream))) 20 | 21 | (use-fixtures :once (ensure-clean-test-dir)) 22 | 23 | (deftest containerize-test 24 | (testing "calls Google Jib and containerize the files in question" 25 | (let [greeting-file (io/file "test/resources/greeting/greeting.txt") 26 | layer-re #"^[0-9a-f]{64}\.tar.gz$"] 27 | (binding [misc/*verbose-logs* true] 28 | (jib.containerizer/containerize #:image{:from 29 | #:image {:repository "openjdk" :tag "sha256:1fd5a77d82536c88486e526da26ae79b6cd8a14006eb3da3a25eb8d2d682ccd6"} 30 | :user "jetty" 31 | :name 32 | #:image {:repository "nubank/my-app" :tag "v1"} 33 | :layers 34 | [#:image.layer{:name "resources" 35 | :entries [#:layer.entry{:source (.getPath greeting-file) 36 | :target "/opt/app/WEB-INF/classes/greeting.txt" 37 | :file-permissions #{"OTHERS_READ" 38 | "OWNER_WRITE" 39 | "OWNER_READ" 40 | "GROUP_READ"} 41 | :modification-time (misc/last-modified-time greeting-file)}]}] 42 | :tar-path tar-path})) 43 | 44 | (is (true? (misc/file-exists? (io/file tar-path)))) 45 | 46 | (is (match? [{:config "config.json" 47 | :repoTags ["nubank/my-app:v1"] 48 | :layers 49 | [layer-re 50 | layer-re 51 | layer-re 52 | layer-re]}] 53 | (misc/read-json (read-from-tarball "manifest.json"))))))) 54 | -------------------------------------------------------------------------------- /test/unit/vessel/api_test.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.api-test 2 | (:require [clojure.java.io :as io] 3 | [clojure.test :refer :all] 4 | [matcher-combinators.test :refer [match?]] 5 | [vessel.api :as api] 6 | [vessel.misc :as misc])) 7 | 8 | (def my-app-manifest (io/file "target/my-app.json")) 9 | 10 | (use-fixtures :each 11 | (fn [f] 12 | (io/make-parents my-app-manifest) 13 | (f) 14 | (.delete my-app-manifest))) 15 | 16 | (deftest manifest-test 17 | (testing "generates a manifest by writing it to the stdout or the specified 18 | file" 19 | (let [expected-manifest "{\"service\":{\"name\":\"my-app\",\"version\":\"v1\"}}\n"] 20 | (is (= expected-manifest 21 | (with-out-str 22 | (api/manifest {:attributes [[:name "my-app"] 23 | [:version "v1"]] 24 | :object :service 25 | :output *out*})))) 26 | 27 | (is (= expected-manifest 28 | (do (api/manifest {:attributes [[:name "my-app"] 29 | [:version "v1"]] 30 | :object :service 31 | :output (io/writer (io/file "target/my-app.json"))}) 32 | (slurp my-app-manifest))))))) 33 | 34 | (defn- gen-image-manifest 35 | [options] 36 | (api/image (assoc options :output (io/writer my-app-manifest))) 37 | (misc/read-json my-app-manifest)) 38 | 39 | (deftest image-test 40 | (let [base-image {:image 41 | {:registry "docker.io" 42 | :repository "openjdk" 43 | :tag "alpine"}} 44 | options {:registry "my-registry.com" 45 | :repository "my-app"}] 46 | (testing "generates an image manifest according to the provided options" 47 | (is (= {:image {:registry "my-registry.com" 48 | :repository "my-app" 49 | :tag "9965bb9aad0efdaf499a35368f338ea053689e8d44cadb748991a84fd1eb355d"}} 50 | (gen-image-manifest options))) 51 | 52 | (is (match? {:base-image base-image 53 | :image {:registry "my-registry.com" 54 | :repository "my-app" 55 | :tag string?}} 56 | (gen-image-manifest (assoc options :base-image base-image))) 57 | "assoc's the base image's manifest") 58 | 59 | (is (match? {:image {:registry "my-registry.com" 60 | :repository "my-app" 61 | :tag string? 62 | :git-commit "4c52b901c6"}} 63 | (gen-image-manifest (assoc options :attributes #{[:git-commit "4c52b901c6"]}))) 64 | "assoc's arbitrary attributes into the resulting manifest") 65 | 66 | (is (match? {:image {:registry "my-registry.com" 67 | :repository "my-app" 68 | :tag string?} 69 | :service {:name "my-service" 70 | :type "clojure"}} 71 | (gen-image-manifest (assoc options :manifests #{{:service {:name "my-service" 72 | :type "clojure"}}}))) 73 | "merges arbitrary manifests into the generated one") 74 | 75 | (is (match? {:image {:tag "v1"}} 76 | (gen-image-manifest (assoc options :tag "v1"))) 77 | "overrides the auto-generated tag when one is provided")))) 78 | -------------------------------------------------------------------------------- /src/vessel/sh.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.sh 2 | "Shell with support for streaming stdout and stderr." 3 | (:require [clojure.java.io :as io] 4 | [clojure.string :as string]) 5 | (:import java.io.File)) 6 | 7 | (def ^:const encoding 8 | "Shell encoding." 9 | "UTF-8") 10 | 11 | (def ^:const working-dir 12 | "Directory where the shell will be started at." 13 | ".") 14 | 15 | (defn- read-from-stream 16 | "Reads lines from the stream and sent them to the supplied output function as 17 | they become available." 18 | [process-stream output-fn] 19 | (with-open [stream (io/reader process-stream)] 20 | (loop [] 21 | (when-let [line (.readLine stream)] 22 | (output-fn line) 23 | (recur))))) 24 | 25 | (defn- env-vars-map->string-array 26 | "Takes a map of environment variables and turns it into an array of strings in 27 | the form var-name=var-value." 28 | [env-vars] 29 | (->> env-vars 30 | (reduce-kv #(conj %1 (str %2 "=" %3)) []) 31 | (into-array String))) 32 | 33 | (defn- env-vars 34 | "Returns a map of environment variables." 35 | [] 36 | (into {} (System/getenv))) 37 | 38 | (defn exec 39 | [cmd & args] 40 | (let [args (into-array String (cons cmd args)) 41 | process (.. Runtime getRuntime 42 | (exec args (env-vars-map->string-array (env-vars)) (io/file working-dir))) 43 | stdout (future (read-from-stream (.getInputStream process) println)) 44 | stderr (future (read-from-stream (.getErrorStream process) #(binding [*err* *out*] 45 | (println %)))) 46 | exit-code (.waitFor process)] 47 | @stdout 48 | @stderr 49 | {:args args 50 | :exit-code exit-code})) 51 | 52 | (defn- ^String classpath 53 | "Takes a seq of java.io.File objects representing a classpath and 54 | returns a string suited to be passed to java and javac commands." 55 | [classpath-files] 56 | (string/join ":" (map str classpath-files))) 57 | 58 | (defn- java-cmd 59 | "Returns a vector containing the arguments to spawn a Java sub-process." 60 | [classpath-files] 61 | [(or (System/getProperty "vessel.sh.java.cmd") "java") 62 | "-classpath" (classpath classpath-files)]) 63 | 64 | (defn clojure 65 | "Calls clojure.main with the supplied classpath and arguments. Throws an 66 | exception if the sub-process exits with an error." 67 | [classpath-files & args] 68 | (let [{:keys [exit-code] :as result} 69 | (apply exec (concat (java-cmd classpath-files) 70 | ["clojure.main"] 71 | args))] 72 | (when-not (zero? exit-code) 73 | (throw (ex-info (str "Sub-process exited with code " exit-code) 74 | result))))) 75 | 76 | (defn- javac-cmd 77 | "Returns a vector containing the arguments to spawn a Javac sub-process." 78 | [classpath-files ^File target-dir sources] 79 | (into 80 | [(or (System/getProperty "vessel.sh.javac.cmd") "javac") 81 | "-classpath" (classpath classpath-files) 82 | "-d" (str target-dir)] 83 | (map str sources))) 84 | 85 | (defn javac 86 | "Calls javac command with the supplied classpath, target directory and 87 | source files. Throws an exception if the sub-process exits with an 88 | error." 89 | [classpath-files ^File target-dir sources] 90 | (let [{:keys [exit-code] :as result} 91 | (apply exec (javac-cmd classpath-files target-dir sources))] 92 | (when-not (zero? exit-code) 93 | (throw (ex-info (str "Sub-process exited with code " exit-code) 94 | result))))) 95 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Vessel 2 | 3 | ![Workflow](https://github.com/nubank/vessel/workflows/Vessel%20Tests/badge.svg?branch=master) 4 | 5 | A containerization tool for Clojure applications. It uses [Google Jib](https://github.com/GoogleContainerTools/jib) to perform this task. 6 | 7 | ## Usage 8 | 9 | > `build`: 10 | 11 | Build a Clojure application 12 | 13 | | Option | Default | Description | 14 | | -- | -- | -- | 15 | | `-c` `--classpath PATHS` | `--` | Directories and zip/jar files on the classpath in the same format expected by the java command | 16 | | `-s` `--source-path PATH` | `--` | Directories containing source files. This option can be repeated many times | 17 | | `-m` `--main-class NAMESPACE` | `--` | Namespace that contains the application's entrypoint, with a :gen-class directive and a -main function | 18 | | `-r` `--resource-path PATH` | `--` | Directories containing resource files. This option can be repeated many times | 19 | | `-o` `--output PATH` | `--` | Directory where the application's files will be written to | 20 | | `-C` `--compiler-options OPTIONS` | No | Options provided to the Clojure compiler, see `clojure.core/*compiler-options*` | 21 | 22 | > `containerize`: 23 | 24 | Containerize a Clojure application 25 | 26 | | Option | Default | Description | 27 | | -- | -- | -- | 28 | | `-a` `--app-root PATH` | `"/app"` | app root of the container image. Classes and resource files will be copied to relative paths to the app root | 29 | | `-c` `--classpath PATHS` | `--` | Directories and zip/jar files on the classpath in the same format expected by the java command | 30 | | `-e` `--extra-path PATH` | `--` | extra files to be copied to the container image. The value must be passed in the form source:target or source:target@churn and this option can be repeated many times | 31 | | `-i` `--internal-deps REGEX` | `--` | java regex to determine internal dependencies. Can be repeated many times for a logical or effect | 32 | | `-m` `--main-class NAMESPACE` | `--` | Namespace that contains the application's entrypoint, with a :gen-class directive and a -main function | 33 | | `-M` `--manifest PATH` | `--` | manifest file describing the image to be built | 34 | | `-o` `--output PATH` | `"image.tar"` | path to save the tarball containing the built image | 35 | | `-p` `--project-root PATH` | `--` | root dir of the Clojure project to be built | 36 | | `-P` `--preserve-file-permissions` | `--` | Preserve original file permissions when copying files to the container. If not enabled, the default permissions for files are 644 | 37 | | `-s` `--source-path PATH` | `--` | Directories containing source files. This option can be repeated many times | 38 | | `-r` `--resource-path PATH` | `--` | Directories containing resource files. This option can be repeated many times | 39 | | `-u` `--user USER` | `root` | Define the default user for the image | 40 | | `-C` `--compiler-options OPTIONS` | `nil` | Options provided to the Clojure compiler, see `clojure.core/*compiler-options*` | 41 | 42 | > `image`: 43 | 44 | Generate an image manifest, optionally by extending a base image and/or merging other manifests 45 | 46 | | Option | Default | Description | 47 | | -- | -- | -- | 48 | | `-a` `--attribute KEY-VALUE` | `--` | Add the attribute in the form key:value to the manifest. This option can be repeated multiple times | 49 | | `-b` `--base-image PATH` | `--` | Manifest file describing the base image | 50 | | `-m` `--merge-into PATH` | -- | Manifest file to be merged into the manifest being created. This option can be repeated multiple times | 51 | | `-o` `--output PATH` | `stdout` | Write the manifest to path instead of stdout | 52 | | `-r` `--registry REGISTRY` | `"docker.io"` | Image registry | 53 | | `-R` `--repository REPOSITORY` | -- | Image repository | 54 | | `-t` `--tag TAG` | -- | Image tag. When omitted uses a SHA-256 digest of the resulting manifest | 55 | 56 | > `manifest`: 57 | 58 | Generate arbitrary manifests 59 | 60 | | Option | Default | Description | 61 | | -- | -- | -- | 62 | | `-a` `--attribute KEY-VALUE` | `[]` | Add the attribute in the form key:value to the manifest. This option can be repeated multiple times | 63 | | `-o` `--output PATH` | `stdout` | "Write the manifest to path instead of stdout" | 64 | | `-O` `--object OBJECT` | `--` | Object under which attributes will be added | 65 | 66 | > `push`: 67 | 68 | Push a tarball to a registry 69 | 70 | | Option | Default | Description | 71 | | -- | -- | -- | 72 | | `-t` `--tarball PATH` | `--` | Tar archive containing image layers and metadata files | 73 | | `-a` `--allow-insecure-registries` | `--` | Allow pushing images to insecure registries | 74 | | `-A` `--anonymous` | `--` | Do not authenticate on the registry; push anonymously | 75 | -------------------------------------------------------------------------------- /test/vessel/jib/credentials_test.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.jib.credentials-test 2 | (:require [clojure.test :refer :all] 3 | [cognitect.aws.client.api :as aws] 4 | [matcher-combinators.standalone :as standalone] 5 | [mockfn.macros :refer [providing]] 6 | [vessel.jib.credentials :as credentials]) 7 | (:import [com.google.cloud.tools.jib.api Credential CredentialRetriever ImageReference] 8 | com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException 9 | java.util.Optional)) 10 | 11 | (deftest get-ecr-credential-test 12 | (providing [(aws/client (standalone/match? {:api :ecr 13 | :region "us-east-1"})) 'client] 14 | 15 | (testing "returns username and password to access the ECR registry that the 16 | image in question is associated to" 17 | (providing [(aws/invoke 'client {:op :GetAuthorizationToken 18 | :request {:registryIds ["591385309914"]}}) 19 | {:authorizationData [{:authorizationToken "QVdTOnBhc3N3b3Jk"}]}] 20 | (is (= {:username "AWS" 21 | :password "password"} 22 | (credentials/get-ecr-credential 23 | (ImageReference/parse "591385309914.dkr.ecr.us-east-1.amazonaws.com/application:v1.0.1")))))) 24 | 25 | (testing "throws a CredentialRetrievalException when the 26 | authentication on ECR fails" 27 | (providing [(aws/invoke 'client {:op :GetAuthorizationToken 28 | :request {:registryIds ["591385309914"]}}) 29 | {:cognitect.anomalies/category :cognitect.anomalies/incorrect}] 30 | (is (thrown? CredentialRetrievalException 31 | (credentials/get-ecr-credential 32 | (ImageReference/parse "591385309914.dkr.ecr.us-east-1.amazonaws.com/application:v1.0.1")))))))) 33 | 34 | (deftest ecr-credential-retriever-test 35 | (testing "returns the credential to access Amazon ECR" 36 | (let [image (ImageReference/parse "591385309914.dkr.ecr.us-east-1.amazonaws.com/application:v1.0.1")] 37 | (providing [(credentials/get-ecr-credential image) {:username "AWS" :password "password"}] 38 | (let [retriever (credentials/ecr-credential-retriever image) 39 | credential (.. retriever retrieve get)] 40 | (is (= "AWS" 41 | (.getUsername credential))) 42 | (is (= "password" 43 | (.getPassword credential))))))) 44 | 45 | (testing "returns Optional/empty when the image isn't associated to an ECR registry" 46 | (let [image (ImageReference/parse "docker.io/repo/application:v1.0.1")] 47 | (let [retriever (credentials/ecr-credential-retriever image)] 48 | (is (false? (.. retriever retrieve isPresent))))))) 49 | 50 | (defn make-retriever 51 | ([] 52 | (fn [_] 53 | (reify CredentialRetriever 54 | (retrieve [this] 55 | (Optional/empty))))) 56 | ([username password] 57 | (fn [_] 58 | (reify CredentialRetriever 59 | (retrieve [this] 60 | (Optional/of (Credential/from username password))))))) 61 | 62 | (deftest retriever-chain-test 63 | (testing "returns the first non-empty credential retrieved by the supplied 64 | retrievers" 65 | (let [get-credentials (fn [credentials] 66 | [(.getUsername credentials) (.getPassword credentials)])] 67 | (are [retrievers username password] (= [username password] 68 | (get-credentials (.. (credentials/retriever-chain (ImageReference/parse "repo/application:v1.0.1") retrievers) retrieve get))) 69 | [(make-retriever "john.doe" "abc123")] "john.doe" "abc123" 70 | [(make-retriever "john.doe" "abc123") (make-retriever "jd" "def456")] "john.doe" "abc123" 71 | [(make-retriever "jd" "def456") (make-retriever "john.doe" "abc123")] "jd" "def456" 72 | [(make-retriever) (make-retriever "john.doe" "abc123")] "john.doe" "abc123" 73 | [(make-retriever "john.doe" "abc123") (make-retriever)] "john.doe" "abc123"))) 74 | 75 | (testing "returns Optional/empty when all retrievers return empty credentials" 76 | (is (= (Optional/empty) 77 | (.. (credentials/retriever-chain (ImageReference/parse "repo/application:v1.0.1") [(make-retriever) (make-retriever)]) 78 | retrieve))))) 79 | -------------------------------------------------------------------------------- /src/vessel/jib/credentials.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.jib.credentials 2 | (:require [clojure.string :as string] 3 | [cognitect.aws.client.api :as aws] 4 | [vessel.jib.helpers :as jib.helpers]) 5 | (:import [com.google.cloud.tools.jib.api Credential CredentialRetriever ImageReference] 6 | com.google.cloud.tools.jib.frontend.CredentialRetrieverFactory 7 | com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException 8 | [java.util Base64 Base64$Decoder Optional])) 9 | 10 | (def ^:private ecr-url #"^(\d+)\.dkr.ecr\.([^\.]+)\..*") 11 | 12 | (defn- account-id-and-region 13 | "Given an ImageReference object, returns the Amazon account id and the 14 | region where the registry lives." 15 | [^ImageReference image-reference] 16 | (let [registry (.getRegistry image-reference)] 17 | (some->> registry 18 | (re-matches ecr-url) 19 | rest 20 | (zipmap [:account-id :region])))) 21 | 22 | (defn- check-authentication-error 23 | "Throws a CredentialRetrievalException when the response returned by 24 | aws-api contains Cognitect anomalies." 25 | [response] 26 | (if-not (:cognitect.anomalies/category response) 27 | response 28 | (throw (CredentialRetrievalException. 29 | (ex-info "Unable to authenticate on Amazon ECR" response))))) 30 | 31 | (defn- get-ecr-authorization-token 32 | "Given an ImageReference object, calls the ECR API to obtain an 33 | authorization token that, presumably, grants access to pull and/or 34 | push images from/to the registry in question." 35 | [^ImageReference image-reference] 36 | (let [{:keys [account-id region]} (account-id-and-region image-reference) 37 | client (aws/client {:api :ecr 38 | :region region})] 39 | (-> (aws/invoke client {:op :GetAuthorizationToken 40 | :request {:registryIds [account-id]}}) 41 | check-authentication-error 42 | :authorizationData 43 | first 44 | :authorizationToken))) 45 | 46 | (defn get-ecr-credential 47 | "Given an ImageReference object, calls the ECR API and returns a map 48 | containing the keys :username and :password representing a 49 | credential to access the ECR registry in question." 50 | [^ImageReference image-reference] 51 | (let [^Base64$Decoder decoder (Base64/getDecoder) 52 | ^String username-and-password (->> (get-ecr-authorization-token image-reference) 53 | (.decode decoder) 54 | (String.))] 55 | (zipmap [:username :password] (string/split username-and-password #":")))) 56 | 57 | (defn- ecr-registry? 58 | "True if the ImageReference is associated to an ECR registry." 59 | [^ImageReference image-reference] 60 | (re-find ecr-url (.getRegistry image-reference))) 61 | 62 | (defn ^CredentialRetriever ecr-credential-retriever 63 | "Returns an instance of Credentialretriever interface that attempts to 64 | retrieve credential from Amazon Elastic Container Registry. 65 | 66 | If the supplied ImageReference isn't related to ECR, returns 67 | Optional/empty. Otherwise, attempts to retrieve the credential by 68 | calling the ECR API. Credentials to interact with Amazon API are 69 | obtained through the same chain supported by awscli or AWS SDK." 70 | [^ImageReference image-reference] 71 | (reify CredentialRetriever 72 | (retrieve [this] 73 | (if-not (ecr-registry? image-reference) 74 | (Optional/empty) 75 | (let [{:keys [username password]} (get-ecr-credential image-reference)] 76 | (Optional/of (Credential/from username password))))))) 77 | 78 | (defn ^CredentialRetriever docker-config-retriever 79 | "Returns an instance of the CredentialRetriever interface that 80 | retrieves credentials from the Docker config." 81 | [^ImageReference image-reference] 82 | (.. CredentialRetrieverFactory 83 | (forImage image-reference (jib.helpers/log-event-handler "vessel.jib.helpers")) 84 | dockerConfig)) 85 | 86 | (def ^:private default-retrievers 87 | [ecr-credential-retriever 88 | docker-config-retriever]) 89 | 90 | (defn ^CredentialRetriever retriever-chain 91 | "Returns an instance of Credentialretriever interface that attempts to 92 | retrieve credentials to deal with the supplied image by calling a 93 | chain of credential retrievers. 94 | 95 | The argument retrievers is a sequence of 1-arity functions that take 96 | an ImageReference and returns a CredentialRetriever. If omitted, the 97 | default chain (ECR and Docker config) will be assumed" 98 | ([^ImageReference image-reference] 99 | (retriever-chain image-reference default-retrievers)) 100 | ([^ImageReference image-reference retrievers] 101 | (reify CredentialRetriever 102 | (retrieve [this] 103 | (or (some (fn [retriever] 104 | (let [^Credential credential (.. (retriever image-reference) retrieve)] 105 | (when (.isPresent credential) 106 | credential))) 107 | retrievers) 108 | (Optional/empty)))))) 109 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | ## [Unreleased] 6 | 7 | ## [0.2.161] - 2024-11-04 8 | 9 | ### Changed 10 | 11 | - Replace hub with gh in the release script: [#42](https://github.com/nubank/vessel/pull/42). 12 | 13 | ### Fixed 14 | 15 | - Detect and report errors reading mergeable sources: [#41](https://github.com/nubank/vessel/pull/41). 16 | 17 | ## [0.2.157] - 2024-10-16 18 | 19 | ### Fixed 20 | 21 | - Fix GitHub Actions Workflows: [#40](https://github.com/nubank/vessel/pull/40). 22 | - Handle reader macros when merging EDN files [#39](https://github.com/nubank/vessel/pull/39). 23 | 24 | ## [0.2.154] - 2024-02-15 25 | 26 | - Fix the logic in the function that checks if a file ends with `.edn`: [#36](https://github.com/nubank/vessel/pull/36). 27 | 28 | ## [0.2.152] - 2024-02-01 29 | 30 | ## [0.2.151] - 2024-02-01 31 | 32 | ### Added 33 | 34 | - Add option to define options for the Clojure compiler when building and containerizing an application: [#32](https://github.com/nubank/vessel/pull/32). 35 | See the [Clojure documentation](https://clojure.org/reference/compilation#_compiler_options) for more details. 36 | 37 | ## [0.2.148] - 2023-09-04 38 | 39 | ### Changed 40 | 41 | - Use the newly created function misc/read-data to read/merge data_readers instead of the misc/read-edn function. The former preserves reader conditionals [#31](https://github.com/nubank/vessel/pull/31). 42 | 43 | ## [0.2.146] - 2023-08-10 44 | 45 | - Resources with the extension `.edn` are deep merged together: [#29](https://github.com/nubank/vessel/pull/29). 46 | 47 | ## [0.2.144] - 2022-11-18 48 | 49 | ### Added 50 | - Exposes command for building Clojure application `build`: [#26](https://github.com/nubank/vessel/pull/26). 51 | 52 | ## [0.2.142] - 2021-04-01 53 | 54 | ### Added 55 | - Add support for compiling Java source files present on the classpath: [#25](https://github.com/nubank/vessel/pull/25). 56 | 57 | ### Changed 58 | - Along with the changes made in [#25](https://github.com/nubank/vessel/pull/25), a new namespace named `sh` was introduced. As of this pull request, Vessel spawns a subshell to run Java in order to call `clojure.core/compile` rather than using a separate classloader. 59 | 60 | ## [0.2.140] - 2020-12-19 61 | 62 | ### Added 63 | - Accept tarballs as base images: [#23](https://github.com/nubank/vessel/pull/23). This feature will be less obscure and easier to be used in the version 1 of Vessel. 64 | 65 | ### Fixed 66 | - Preserve the order of classpath entries to avoid introducing conflicts: [#24](https://github.com/nubank/vessel/pull/24). 67 | 68 | ## [0.2.137] - 2020-12-07 69 | 70 | ### Fixed 71 | - Avoid blowing up the build process when a non-mapped class file is found [#22](https://github.com/nubank/vessel/pull/22). Now, the non-mapped file is assigned to the first known source directory in order to force the file to be copied to the corresponding image layer. 72 | 73 | ## [0.2.135] - 2020-08-10 74 | 75 | ### Fixed 76 | - Preserve timestamps when copying files: 77 | [#21](https://github.com/nubank/vessel/pull/21). The corresponding issue was 78 | slowing down the startup of containerized applications and causing runtime 79 | problems due to conflicts between AOT and JIT compiled Clojure namespaces. 80 | 81 | ## [0.2.134] - 2020-07-03 82 | 83 | ### Added 84 | - [#15](https://github.com/nubank/vessel/pull/15): added the user option 85 | to set the default user image 86 | 87 | ## [0.2.128] - 2020-05-28 88 | 89 | ### Added 90 | - [#10](https://github.com/nubank/vessel/pull/10): added out of the box 91 | integration with Amazon Elastic Container Registry. Vessel looks up 92 | credentials to access AWS API the same way the `awscli` or `Java SDK` 93 | do. Thus, Vessel is capable of obtaining credentials to access `ECR` 94 | repositories through instance profiles without additional configurations, what 95 | might be useful to eliminate extra steps on CI pipelines. 96 | 97 | ## [0.2.126] - 2020-05-21 98 | 99 | ### Added 100 | - Integration tests. 101 | - Workflow file to run tests automatically on Github Actions. 102 | - Normalize keys of the manifest.json file in order to avoid incongruities. Thus, Vessel can be employed as a generic alternative to push tarballs to remote registries. 103 | 104 | ## [0.2.107] - 2020-03-09 105 | 106 | ### Added 107 | - Added the `--preserve-file-permissions` flag to the `containerize` command. By 108 | default, files are copied to the container with the permissions 644. When this 109 | flag is enabled, Vessel copies files with their original permissions. This 110 | feature is useful, for instance, to keep executable scripts (e.g. wrappers for 111 | calling the java command) with their original permissions within the 112 | container. 113 | 114 | ## [0.2.99] - 2020-03-02 115 | 116 | ### Added 117 | * Push command to upload a tarball to a registry. 118 | 119 | ### Changed 120 | * Internal: rename vessel.jib namespace to vessel.jib.containerizer and move 121 | functions that deal with common aspects of Jib API to the new 122 | vessel.jib.helpers namespace. 123 | * Upgrade Jib to version 0.13.0. 124 | 125 | ## [0.1.90] - 2020-02-20 126 | 127 | ### Added 128 | * Set the directory for caching base image and application layers to 129 | ~/.vessel-cache. Eventually, Vessel can take this directory as a parameter to 130 | allow a more fine-grained customization. 131 | 132 | ## [0.1.84] - 2020-02-18 133 | 134 | ### Added 135 | * Containerize, image and manifest commands (Vessel is in an alpha stage; the 136 | API is subject to changes). 137 | -------------------------------------------------------------------------------- /src/vessel/resource_merge.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.resource-merge 2 | "Support for merging duplicated resources on the classpath." 3 | (:require [clojure.edn :as edn] 4 | [clojure.java.io :as io] 5 | [vessel.misc :as misc]) 6 | (:import (java.io File InputStream PushbackReader))) 7 | 8 | 9 | (defn- write-edn 10 | [path value] 11 | (spit path (pr-str value))) 12 | 13 | (declare ^:private deep-merge) 14 | 15 | (defn- deep-merge 16 | [left right] 17 | (cond 18 | (map? left) 19 | (merge-with deep-merge left right) 20 | 21 | (sequential? left) 22 | (into left right) 23 | 24 | (set? left) 25 | (into left right) 26 | 27 | :else 28 | right)) 29 | 30 | (defn read-edn 31 | "Reads an InputStream as EDN but uses a tagged literal for any reader macros found in the stream." 32 | [input-stream] 33 | (->> input-stream 34 | io/reader 35 | PushbackReader. 36 | (edn/read {:default tagged-literal}))) 37 | 38 | ;; A rule is a map: 39 | ;; :match-fn (fn [File]) -> boolean (File is the output file) 40 | ;; :read-fn (fn [InputStream]) -> value 41 | ;; :merge-fn (fn [old-value, new-value]) -> merged-value 42 | ;; :write-fn (fn [File, value]) -> nil (but writes the merged value) 43 | 44 | (def data-readers-base-rule 45 | "data_raders.clj/cljc - merged together" 46 | {:match-fn #(re-find #"/data_readers.cljc?$" (.getPath ^File %)) 47 | :read-fn misc/read-data 48 | :merge-fn merge 49 | :write-fn write-edn}) 50 | 51 | (def edn-base-rule 52 | "*.edn - deep merged together" 53 | {:match-fn #(.endsWith (.getPath ^File %) ".edn") 54 | :read-fn read-edn 55 | :merge-fn deep-merge 56 | :write-fn write-edn}) 57 | 58 | (def base-rules 59 | [data-readers-base-rule 60 | edn-base-rule]) 61 | 62 | (defn new-merge-set 63 | "Creates a new merge set, with the provided rules (or [[base-rules]] as a default). 64 | Merge sets are impure: they contain a mutable atom to track files that may be merged 65 | if multiple copies are found." 66 | ([] 67 | (new-merge-set base-rules)) 68 | ([rules] 69 | {::rules rules 70 | ::*merged-paths (atom {})})) 71 | 72 | (defn- apply-rule 73 | [classpath-root input-source ^InputStream input-stream target-file last-modified rule merge-set] 74 | (let [{::keys [*merged-paths]} merge-set 75 | {:keys [read-fn merge-fn]} rule 76 | new-value (try 77 | (read-fn input-stream) 78 | (catch Throwable t 79 | (throw (ex-info (str "Unable to read " input-source ": " 80 | (or (ex-message t) 81 | (-> t class .getName))) 82 | {:classpath-root classpath-root 83 | :input-source input-source 84 | :target-file target-file}))) 85 | (finally 86 | (.close input-stream)))] 87 | (swap! *merged-paths 88 | (fn [merged-paths] 89 | (if (contains? merged-paths target-file) 90 | (update merged-paths target-file 91 | #(-> % 92 | (update :value merge-fn new-value) 93 | (assoc :classpath-root classpath-root 94 | :last-modified last-modified))) 95 | (assoc merged-paths target-file {:rule rule 96 | :classpath-root classpath-root 97 | :last-modified last-modified 98 | :value new-value})))))) 99 | 100 | (defn find-matching-rule 101 | [rules file] 102 | (some (fn [rule] 103 | (when ((:match-fn rule) file) 104 | rule)) 105 | rules)) 106 | 107 | (defn execute-merge-rules 108 | "Evaluates the inputs against the rules in the provided merge set. Returns true 109 | if the file is subject to a merge rule (in which case, it should not be simply copied). 110 | 111 | classpath-root - origin of the input, a File; either a directory or a JAR file 112 | input-source - string describing where the input-stream comes from 113 | input-stream - InputStream containing content of the file 114 | target-file - File to write from the input stream (or merged) 115 | last-modified - timestamp of last modified time to be applied to the final output 116 | merge-set - mutable object containing details about merging" 117 | [classpath-root input-source input-stream target-file last-modified merge-set] 118 | (let [{::keys [rules]} merge-set 119 | matched-rule (find-matching-rule rules target-file)] 120 | (if matched-rule 121 | (do 122 | (apply-rule classpath-root input-source input-stream target-file last-modified matched-rule merge-set) 123 | true) 124 | false))) 125 | 126 | (defn write-merged-paths 127 | "After all other file reading and copying has completed, this function writes the merged files whose 128 | data has accumulated in the merged paths. 129 | 130 | Returns a map of output files to classpath root (the last classpath 131 | root identified for any file with the same target path)." 132 | [merge-set] 133 | (reduce-kv 134 | (fn [result target-file merged-path] 135 | (let [{:keys [value last-modified rule classpath-root]} merged-path 136 | {:keys [write-fn]} rule] 137 | (io/make-parents target-file) 138 | (write-fn target-file value) 139 | (misc/set-timestamp target-file last-modified) 140 | (assoc result target-file classpath-root))) 141 | {} 142 | (-> merge-set ::*merged-paths deref))) 143 | -------------------------------------------------------------------------------- /src/vessel/jib/containerizer.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.jib.containerizer 2 | "Containerization API built on top of Google Jib." 3 | (:require [vessel.jib.credentials :as credentials] 4 | [vessel.jib.helpers :as jib.helpers] 5 | [vessel.misc :as misc]) 6 | (:import [com.google.cloud.tools.jib.api Containerizer FilePermissions ImageFormat ImageReference Jib JibContainerBuilder LayerConfiguration LayerConfiguration$Builder LayerEntry LogEvent RegistryImage TarImage] 7 | com.google.cloud.tools.jib.event.events.ProgressEvent 8 | java.nio.file.attribute.PosixFilePermission)) 9 | 10 | (defn ^ImageReference make-image-reference 11 | "Returns a new image reference from the provided values." 12 | [{:image/keys [^String registry ^String repository ^String tag]}] 13 | (ImageReference/of registry repository tag)) 14 | 15 | (defn ^TarImage make-tar-image 16 | "Returns a new tar image from the provided tarball." 17 | [^String tar-path] 18 | (TarImage/at (misc/string->java-path tar-path))) 19 | 20 | (defn- ^Containerizer make-containerizer 21 | "Makes a new Jib containerizer object to containerize the application 22 | to a given tarball." 23 | [{:image/keys [name tar-path]}] 24 | {:pre [name tar-path]} 25 | (let [cache-dir (-> (misc/home-dir) 26 | (misc/make-dir ".vessel-cache") 27 | str 28 | misc/string->java-path) 29 | handler-name "vessel.jib.containerizer"] 30 | (.. Containerizer 31 | (to (.. (make-tar-image tar-path) 32 | (named (make-image-reference name)))) 33 | (setBaseImageLayersCache cache-dir) 34 | (setApplicationLayersCache cache-dir) 35 | (setToolName "vessel") 36 | (addEventHandler LogEvent (jib.helpers/log-event-handler handler-name)) 37 | (addEventHandler ProgressEvent (jib.helpers/progress-event-handler handler-name))))) 38 | 39 | (defn- containerize* 40 | [^JibContainerBuilder container-builder image-spec] 41 | (.containerize container-builder (make-containerizer image-spec))) 42 | 43 | (defn- ^LayerEntry make-layer-entry 44 | "Creates a new LayerEntry object from the supplied values." 45 | [{:layer.entry/keys [source target file-permissions modification-time]}] 46 | (let [permissions (some->> file-permissions 47 | (map #(PosixFilePermission/valueOf %)) 48 | set 49 | (FilePermissions/fromPosixFilePermissions))] 50 | (LayerEntry. (misc/string->java-path source) 51 | (jib.helpers/string->absolute-unix-path target) 52 | (or permissions FilePermissions/DEFAULT_FILE_PERMISSIONS) 53 | modification-time))) 54 | 55 | (defn- ^LayerConfiguration make-layer-configuration 56 | "Makes a LayerConfiguration object from the supplied data structure." 57 | [{:image.layer/keys [name entries]}] 58 | (loop [^LayerConfiguration$Builder layer (.. LayerConfiguration builder (setName name)) 59 | entries entries] 60 | (if-not (seq entries) 61 | (.build layer) 62 | (let [layer-entry (first entries)] 63 | (.addEntry layer (make-layer-entry layer-entry)) 64 | (recur layer (rest entries)))))) 65 | 66 | (defn- ^JibContainerBuilder add-layers 67 | "Adds the supplied layers to the JibContainerBuilder object as a set 68 | of LayerConfiguration instances." 69 | [^JibContainerBuilder container-builder layers] 70 | (reduce (fn [builder layer] 71 | (.addLayer builder (make-layer-configuration layer))) 72 | container-builder layers)) 73 | 74 | (defn- ^JibContainerBuilder set-user 75 | "Set the user for the image" 76 | [^JibContainerBuilder container-builder user] 77 | (.setUser container-builder user)) 78 | 79 | (defn- ^RegistryImage make-registry-image 80 | "Given an ImageReference instance, returns a new registry image 81 | object." 82 | [^ImageReference image-reference] 83 | (let [^CredentialRetriever retriever (credentials/retriever-chain image-reference)] 84 | (.. RegistryImage (named image-reference) 85 | (addCredentialRetriever retriever)))) 86 | 87 | (defn- ^Boolean is-in-docker-hub? 88 | "Is the image in question stored in the official Docker hub?" 89 | [^ImageReference image-reference] 90 | (= "registry-1.docker.io" 91 | (.getRegistry image-reference))) 92 | 93 | (defn- ^JibContainerBuilder make-container-builder 94 | "Returns a new container builder to start building the image. 95 | 96 | from is a map representing the base image descriptor. The following keys are 97 | meaningful: :image/registry, :image/repository, :image/tag and 98 | :image/tar-path. The :image/registry and :image/tag are optional for registry 99 | images. The key :image/tar-path indicates that the base image should be loaded 100 | from a tarball." 101 | [from] 102 | (let [base-image (if (:image/tar-path from) 103 | (make-tar-image (:image/tar-path from)) 104 | (let [^ImageReference reference (make-image-reference from)] 105 | (if (is-in-docker-hub? reference) 106 | (str reference) 107 | (make-registry-image reference))))] 108 | (.. Jib (from base-image) 109 | (setCreationTime (misc/now)) 110 | (setFormat ImageFormat/Docker)))) 111 | 112 | (defn containerize 113 | "Given an image spec, containerize the application in question by 114 | producing a tarball containing image layers and metadata files." 115 | [{:image/keys [from user layers] :as image-spec}] 116 | {:pre [from layers]} 117 | (-> (make-container-builder from) 118 | (add-layers layers) 119 | (set-user user) 120 | (containerize* image-spec))) 121 | -------------------------------------------------------------------------------- /test/unit/vessel/jib/pusher_test.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.jib.pusher-test 2 | (:require [clojure.core.async :as async] 3 | [clojure.java.io :as io] 4 | [clojure.test :refer :all] 5 | [mockfn.macros :refer [calling providing verifying]] 6 | [mockfn.matchers :refer [a any exactly pred]] 7 | [vessel.jib.credentials :as credentials] 8 | [vessel.jib.pusher :as pusher] 9 | [vessel.test-helpers :refer [ensure-clean-test-dir]]) 10 | (:import [com.google.cloud.tools.jib.api Credential CredentialRetriever DescriptorDigest ImageReference] 11 | com.google.cloud.tools.jib.blob.BlobDescriptor 12 | com.google.cloud.tools.jib.image.json.BuildableManifestTemplate 13 | com.google.cloud.tools.jib.registry.RegistryClient 14 | java.util.Optional)) 15 | 16 | (use-fixtures :once (ensure-clean-test-dir)) 17 | 18 | (def credential-retriever 19 | (reify CredentialRetriever 20 | (retrieve [this] 21 | (Optional/of (Credential/from "user" "password"))))) 22 | 23 | (deftest make-registry-client-test 24 | (testing "by default, attempts to retrieve credentials and to authenticate on 25 | the registry" 26 | (verifying [(credentials/retriever-chain (a ImageReference)) credential-retriever (exactly 1) 27 | (pusher/authenticate (a RegistryClient)) any (exactly 1)] 28 | (is (instance? RegistryClient 29 | (pusher/make-registry-client (ImageReference/parse "library/my-app:v1") {}))))) 30 | 31 | (testing "when :anonymous? is set to true, neither attempts to retrieve 32 | credentials nor to authenticate on the registry" 33 | (verifying [(credentials/retriever-chain (a ImageReference)) any (exactly 0) 34 | (pusher/authenticate (a RegistryClient)) any (exactly 0)] 35 | (is (instance? RegistryClient 36 | (pusher/make-registry-client (ImageReference/parse "library/my-app:v1") {:anonymous? true})))))) 37 | 38 | (defn closed-channel [] 39 | (let [channel (async/chan)] 40 | (async/close! channel) 41 | channel)) 42 | 43 | (deftest push-layer-test 44 | (let [^DescriptorDigest digest (DescriptorDigest/fromDigest "sha256:8e3ba11ec2a2b39ab372c60c16b421536e50e5ce64a0bc81765c2e38381bcff6") 45 | ^BlobDescriptor descriptor (BlobDescriptor. digest) 46 | blob-data #:blob {:descriptor descriptor :reader (constantly nil)} 47 | channel (closed-channel)] 48 | (testing "when the layer already exists on the registry, skips the push" 49 | (providing [(#'pusher/check-blob 'client digest) true] 50 | (verifying [(#'pusher/push-blob 'client channel blob-data) any (exactly 0)] 51 | (is (any? 52 | (pusher/push-layer 'client channel blob-data)))))) 53 | 54 | (testing "when the layer doesn't exist on the registry, pushes it" 55 | (providing [(#'pusher/check-blob 'client digest) false] 56 | (verifying [(#'pusher/push-blob 'client channel blob-data) any (exactly 1)] 57 | (is (any? 58 | (pusher/push-layer 'client channel blob-data)))))) 59 | 60 | (testing "returns a Throwable object when the push throws an exception" 61 | (providing [(#'pusher/check-blob 'client digest) (calling (fn [_ _] 62 | (throw (Exception. "Boom!"))))] 63 | (is (instance? Throwable 64 | (pusher/push-layer 'client channel blob-data))))))) 65 | 66 | (defn manifest-of-digest 67 | [^DescriptorDigest digest] 68 | (pred (fn [^BuildableManifestTemplate manifest] 69 | (= digest 70 | (.. manifest getContainerConfiguration getDigest))))) 71 | 72 | (deftest push-test 73 | (let [^DescriptorDigest layer1-digest (DescriptorDigest/fromDigest "sha256:030a57e84b5be8d31b3c061ff7d7653836673f50475be0a507188ced9d0763d1") 74 | ^DescriptorDigest layer2-digest (DescriptorDigest/fromDigest "sha256:051334be9afdd6a54c28ef9f063d2cddf7dbf79fcc9b1b0965cb1f69403db6b5") 75 | tarball (io/file "test/resources/fake-app.tar") 76 | temp-dir (io/file "target/tests/pusher-test")] 77 | 78 | (testing "ensures that all steps needed to push an image are being performed 79 | accordingly" 80 | (let [^DescriptorDigest image-digest (DescriptorDigest/fromDigest "sha256:ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356")] 81 | (providing [(credentials/retriever-chain (a ImageReference)) credential-retriever 82 | (pusher/authenticate (a RegistryClient)) any 83 | (#'pusher/check-blob (a RegistryClient) layer1-digest) true 84 | (#'pusher/check-blob (a RegistryClient) layer2-digest) true 85 | ;; Called from push-container-config 86 | (#'pusher/push-blob (a RegistryClient) (any) (pred map?)) any 87 | (pusher/push-manifest (a RegistryClient) (any) (manifest-of-digest image-digest) "v1") image-digest] 88 | (is (any? 89 | (pusher/push {:tarball tarball 90 | :temp-dir temp-dir})))))) 91 | 92 | (testing "throws an exception when one of the layers can't be pushed" 93 | (providing [(credentials/retriever-chain (a ImageReference)) credential-retriever 94 | (pusher/authenticate (a RegistryClient)) any 95 | (#'pusher/check-blob (a RegistryClient) layer1-digest) true 96 | (#'pusher/check-blob (a RegistryClient) layer2-digest) (calling (fn [_ _] 97 | (throw (Exception. "Unknown error"))))] 98 | (is (thrown-with-msg? clojure.lang.ExceptionInfo 99 | #"One or more layers could not be pushed into remote registry" 100 | (pusher/push {:tarball tarball 101 | :temp-dir temp-dir}))))))) 102 | -------------------------------------------------------------------------------- /src/vessel/image.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.image 2 | (:require [clojure.java.io :as io] 3 | [clojure.string :as string] 4 | [vessel.misc :as misc]) 5 | (:import java.io.File)) 6 | 7 | (defn- image-layers-for-extra-paths 8 | "Given a set of extra-paths (maps containing the keys :source, :target 9 | and :churn), returns a sequence of layers by grouping files 10 | according to their churn." 11 | [{:keys [extra-paths preserve-file-permissions?]}] 12 | (letfn [(layer-entry [{:keys [source target]}] 13 | (misc/assoc-some #:layer.entry{:source (.getPath source) 14 | :target (.getPath target) 15 | :modification-time (misc/last-modified-time source)} 16 | :layer.entry/file-permissions (when preserve-file-permissions? 17 | (misc/posix-file-permissions source))))] 18 | (->> extra-paths 19 | (group-by :churn) 20 | (map (fn [[churn files]] 21 | #:image.layer{:entries (map layer-entry files) 22 | :churn churn})) 23 | (sort-by :image.layer/churn) 24 | (map-indexed (fn [index layer] 25 | (assoc layer :image.layer/name (str "extra-files-" (inc index)))))))) 26 | 27 | (defn- concat-with-extra-paths 28 | [{:keys [extra-paths] :as options} layers] 29 | (if (seq extra-paths) 30 | (into (image-layers-for-extra-paths options) layers) 31 | layers)) 32 | 33 | (defn internal-dep? 34 | "Is the file in question an internal dependency?" 35 | [^File file {:keys [internal-deps-re]}] 36 | (boolean 37 | (when internal-deps-re 38 | (some #(re-find % (.getPath file)) 39 | internal-deps-re)))) 40 | 41 | (defn- subpath? 42 | "Returns true if this is a subpath of that." 43 | [^File this ^File that] 44 | (string/includes? (.getPath that) (.getPath this))) 45 | 46 | (defn resource? 47 | "Is the file in question a resource file?" 48 | [^File file {:keys [resource-paths]}] 49 | (boolean 50 | (some #(subpath? % file) resource-paths))) 51 | 52 | (defn source-file? 53 | "Is the file in question a source file?" 54 | [^File file {:keys [source-paths]}] 55 | (boolean 56 | (some #(subpath? % file) source-paths))) 57 | 58 | (def ^:private classifiers 59 | "Map of classifiers for files that will be part of each image layer. 60 | 61 | Keys are layer names and values are a tuple of [predicate 62 | churn]. The churn determines whether the layer will be topmost or 63 | undermost positioned at the resulting image." 64 | {"external-deps" [(constantly false) 1] ;; default layer 65 | "internal-deps" [internal-dep? 3] 66 | "resources" [resource? 5] 67 | "sources" [source-file? 7]}) 68 | 69 | (defn- ^Boolean apply-classifier-predicate 70 | "Applies the predicate on the file or map entry (whose value is a 71 | java.io.File object to be classified)." 72 | [pred file-or-map-entry options] 73 | (cond 74 | (instance? File file-or-map-entry) (pred file-or-map-entry options) 75 | (map-entry? file-or-map-entry) (pred (val file-or-map-entry) options))) 76 | 77 | (defn- classify 78 | "Given a java.io.File or a map entry whose value is a java.io.File, 79 | classifies it according to heuristics extracted from the supplied 80 | options." 81 | [file-or-map-entry options] 82 | (or (some (fn [[layer-name [pred]]] 83 | (when (apply-classifier-predicate pred file-or-map-entry options) 84 | layer-name)) 85 | classifiers) 86 | "external-deps")) 87 | 88 | (defn- image-layer 89 | "Creates an image layer map from the supplied arguments." 90 | [[layer-name files] {:keys [app-root preserve-file-permissions? target-dir]}] 91 | #:image.layer{:name layer-name 92 | :entries (map (fn [file-or-map-entry] 93 | (let [^File file (if (map-entry? file-or-map-entry) 94 | (key file-or-map-entry) 95 | file-or-map-entry)] 96 | (misc/assoc-some #:layer.entry{:source (.getPath file) 97 | :target (.getPath (io/file app-root (misc/relativize file target-dir))) 98 | :modification-time (misc/last-modified-time file)} 99 | :layer.entry/file-permissions (when preserve-file-permissions? 100 | (misc/posix-file-permissions file))))) 101 | files) 102 | :churn (second (get classifiers layer-name))}) 103 | 104 | (defn- layer-comparator 105 | "Compares two image layers." 106 | [this that] 107 | (compare (get this :image.layer/churn) 108 | (get that :image.layer/churn))) 109 | 110 | (defn- organize-image-layers 111 | "Takes a sequence of java.io.File objects or map entries (whose values 112 | are java.io.File objects) and organize them into image layers 113 | according to known heuristics." 114 | [files options] 115 | (->> files 116 | (group-by #(classify % options)) 117 | (map #(image-layer % options)) 118 | (concat-with-extra-paths options) 119 | (sort layer-comparator))) 120 | 121 | (defn- image-reference 122 | "Turns image information read from a manifest into an image reference 123 | map. " 124 | [{:keys [registry repository tag tar-path]}] 125 | (misc/assoc-some #:image{:registry registry 126 | :repository repository 127 | :tag tag} 128 | :image/tar-path tar-path)) 129 | 130 | (defn render-image-spec 131 | [{:app/keys [classes lib]} {:keys [manifest user ^File tarball] :as options}] 132 | (let [files (into (vec classes) lib)] 133 | #:image{:from (image-reference (get-in manifest [:base-image :image])) 134 | :name (image-reference (get manifest :image)) 135 | :user user 136 | :layers (organize-image-layers files options) 137 | :tar-path (.getPath tarball)})) 138 | -------------------------------------------------------------------------------- /test/unit/vessel/misc_test.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.misc-test 2 | (:require [clojure.java.io :as io] 3 | [clojure.test :refer :all] 4 | [clojure.test.check.clojure-test :refer [defspec]] 5 | [clojure.test.check.generators :as gen] 6 | [clojure.test.check.properties :as prop] 7 | [vessel.misc :as misc] 8 | [vessel.test-helpers :refer [ensure-clean-test-dir]]) 9 | (:import [java.io StringReader StringWriter] 10 | java.nio.file.Path 11 | [java.time Duration Instant] 12 | java.util.function.Consumer)) 13 | 14 | (use-fixtures :once (ensure-clean-test-dir)) 15 | 16 | (def kv-gen (gen/tuple gen/keyword (gen/such-that (complement nil?) gen/any))) 17 | 18 | (defspec assoc-some-spec-test 19 | {:num-tests 25} 20 | (prop/for-all [kvs (gen/fmap (partial mapcat identity) 21 | (gen/not-empty (gen/vector kv-gen)))] 22 | (testing "assoc-some behaves like assoc for non-nil values" 23 | (is (= (apply assoc {} kvs) 24 | (apply misc/assoc-some {} kvs)))))) 25 | 26 | (deftest assoc-some-test 27 | (is (= {:a 1} 28 | (misc/assoc-some {} 29 | :a 1 :b nil)))) 30 | 31 | (deftest kebab-case-test 32 | (are [value result] (= result (misc/kebab-case value)) 33 | "repo-tags" "repo-tags" 34 | "repo_tags" "repo-tags" 35 | "repoTags" "repo-tags" 36 | "RepoTags" "repo-tags" 37 | "Layers" "layers")) 38 | 39 | (deftest string->java-path-test 40 | (is (instance? Path (misc/string->java-path "/")))) 41 | 42 | (deftest java-consumer-test 43 | (is (instance? Consumer 44 | (misc/java-consumer identity))) 45 | 46 | (is (= "Hello world!" 47 | (with-out-str 48 | (.. (misc/java-consumer #(printf %)) 49 | (accept "Hello world!")))))) 50 | 51 | (deftest now-test 52 | (is (instance? Instant (misc/now)))) 53 | 54 | (deftest duration-between-test 55 | (is (instance? Duration (misc/duration-between (misc/now) 56 | (misc/now))))) 57 | 58 | (deftest duration->string-test 59 | (are [duration result] (= result (misc/duration->string duration)) 60 | (Duration/ZERO) "0 milliseconds" 61 | (Duration/ofMillis 1) "1 millisecond" 62 | (Duration/ofMillis 256) "256 milliseconds" 63 | (Duration/ofMillis 1000) "1 second" 64 | (Duration/ofMillis 6537) "6.54 seconds" 65 | (Duration/ofMinutes 1) "1 minute" 66 | (Duration/ofMillis 63885) "1.06 minutes" 67 | (Duration/ofMinutes 4) "4 minutes")) 68 | 69 | (deftest with-stderr-test 70 | (let [writer (StringWriter.)] 71 | (binding [*err* writer] 72 | (misc/with-stderr 73 | (print "Error!")) 74 | (is (= "Error!" 75 | (str writer)))))) 76 | 77 | (deftest log*-test 78 | (testing "prints or omits the message depending on the value bound to 79 | *verbose-logs* and the supplied log level" 80 | (are [verbose? stream level result] 81 | (= result 82 | (let [writer (java.io.StringWriter.)] 83 | (binding [misc/*verbose-logs* verbose? 84 | stream writer] 85 | (misc/log* level "me" "Hello!") 86 | (str writer)))) 87 | true *out* :info "INFO [me] Hello!\n" 88 | true *out* "info" "INFO [me] Hello!\n" 89 | true *out* :debug "DEBUG [me] Hello!\n" 90 | true *err* :error "ERROR [me] Hello!\n" 91 | true *err* :fatal "FATAL [me] Hello!\n" 92 | false *out* :info "Hello!\n" 93 | false *err* :error "Hello!\n" 94 | false *err* :fatal "Hello!\n" 95 | false *out* :debug "")) 96 | 97 | (testing "supports formatting through clojure.core/format" 98 | (is (= "INFO [me] Hello John Doe!\n" 99 | (with-out-str 100 | (binding [misc/*verbose-logs* true] 101 | (misc/log* :info "me" "Hello %s!" "John Doe"))))))) 102 | 103 | (deftest log-test 104 | (testing "shows the log message displaying the current namespace as the 105 | emitter" 106 | (is (= "INFO [vessel.misc-test] Hello John Doe!\n" 107 | (with-out-str 108 | (binding [misc/*verbose-logs* true] 109 | (misc/log :info "Hello %s!" "John Doe"))))))) 110 | 111 | (def cwd (io/file (.getCanonicalPath (io/file ".")))) 112 | 113 | (deftest filter-files-test 114 | (is (every? #(.isFile %) 115 | (misc/filter-files (file-seq cwd))))) 116 | 117 | (deftest home-dir-test 118 | (is (true? (misc/file-exists? (misc/home-dir))))) 119 | 120 | (deftest last-modified-time-test 121 | (is (instance? Instant 122 | (misc/last-modified-time (io/file "deps.edn"))))) 123 | 124 | (deftest make-dir-test 125 | (let [dir (misc/make-dir (io/file "target") "tests" "misc-test" "dir1" "dir2")] 126 | (is (true? (misc/file-exists? dir))))) 127 | 128 | (deftest make-empty-dir-test 129 | (testing "creates a new directory" 130 | (let [dir (misc/make-empty-dir (io/file "target") "tests" "misc-test" "dir3" "dir4")] 131 | (is (true? (misc/file-exists? dir))))) 132 | 133 | (testing "when the directory in question already exists, guarantees that the 134 | returned directory is empty" 135 | (let [old-dir (misc/make-empty-dir (io/file "target") "tests" "misc-test" "dir5") 136 | _ (spit (io/file old-dir "file.txt") "Lorem Ipsum") 137 | new-dir (misc/make-empty-dir old-dir)] 138 | (is (true? (misc/file-exists? new-dir))) 139 | (is (empty? (.listFiles new-dir)))))) 140 | 141 | (deftest posix-file-permissions-test 142 | (is (= #{"OTHERS_READ" 143 | "OWNER_WRITE" 144 | "OWNER_READ" 145 | "GROUP_READ"} 146 | (misc/posix-file-permissions (io/file "deps.edn"))))) 147 | 148 | (deftest relativize-test 149 | (is (= (io/file "deps.edn") 150 | (misc/relativize (io/file cwd "deps.edn") cwd)))) 151 | 152 | (deftest read-edn-test 153 | (is (= {:greeting "Hello!"} 154 | (misc/read-edn (StringReader. "{:greeting \"Hello!\"}"))))) 155 | 156 | (deftest read-data-test 157 | (is (= {'x (reader-conditional '(:clj foo.bar/x-clj :cljs foo.bar/x-cljs) false)} 158 | (misc/read-data 159 | (StringReader. "{x #?(:clj foo.bar/x-clj :cljs foo.bar/x-cljs)}"))))) 160 | 161 | (deftest read-json-test 162 | (is (= {:greeting "Hello!"} 163 | (misc/read-json (StringReader. "{\"greeting\" : \"Hello!\"}"))))) 164 | 165 | (deftest sha-256-test 166 | (is (= "d2cf1a50c1a07db39d8397d4815da14aa7c7230775bb3c94ea62c9855cf9488d" 167 | (misc/sha-256 {:image 168 | {:name "my-app" 169 | :registry "docker.io" 170 | :version "v1"}})))) 171 | -------------------------------------------------------------------------------- /src/vessel/cli.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.cli 2 | "Functions for dealing with command line input and output." 3 | (:require [clojure.java.io :as io] 4 | [clojure.string :as string] 5 | [clojure.tools.cli :as tools.cli] 6 | [vessel.misc :as misc :refer [with-stderr]])) 7 | 8 | (def ^:private help 9 | "Help option." 10 | ["-?" "--help" 11 | :id :help 12 | :desc "Show this help message and exit"]) 13 | 14 | (defn- show-program-help? 15 | "Shall Vessel show the help message?" 16 | [{:keys [arguments options]}] 17 | (or (and (empty? options) 18 | (empty? arguments)) 19 | (options :help))) 20 | 21 | (defn- show-help 22 | "Prints the help message. 23 | 24 | Return 0 indicating success." 25 | [{:keys [commands desc summary usage]}] 26 | (printf "Usage: %s%n%n" usage) 27 | (println desc) 28 | (println) 29 | (println "Options:") 30 | (println summary) 31 | (when commands 32 | (println) 33 | (println "Commands:") 34 | (run! println commands) 35 | (println) 36 | (println "See \"vessel COMMAND --help\" for more information on a command")) 37 | 0) 38 | 39 | (defn- show-errors 40 | "Prints the error messages in the stderr. 41 | 42 | Returns 1 indicating an error in the execution." 43 | [{:keys [errors tip]}] 44 | (with-stderr 45 | (print "Vessel: ") 46 | (run! println errors) 47 | (when tip 48 | (printf "See \"%s\"%n" tip)) 49 | 1)) 50 | 51 | (defn- command-not-found 52 | "Shows an error message saying that the command in question could not be found. 53 | 54 | Returns 127 (the Linux error code for non-existing commands)." 55 | [{:keys [cmd] :as result}] 56 | (show-errors (assoc result :errors [(format "\"%s\" isn't a Vessel command" cmd)])) 57 | 127) 58 | 59 | (defn- run-command* 60 | "Calls the function assigned to the command in question. 61 | 62 | If the options map contains the `:help` key, shows the command's 63 | help message instead. 64 | 65 | Returns 0 indicating success." 66 | [{:keys [fn options] :as spec}] 67 | (if-not (options :help) 68 | (do (fn options) 69 | 0) 70 | (show-help spec))) 71 | 72 | (defn- parse-args 73 | [{:keys [cmd desc fn opts]} args] 74 | (-> (tools.cli/parse-opts args (conj opts help)) 75 | (assoc :desc desc 76 | :fn fn 77 | :usage (format "vessel %s [OPTIONS]" cmd) 78 | :tip (format "vessel %s --help" cmd)))) 79 | 80 | (defn- run-command 81 | [command args] 82 | (let [{:keys [errors] :as result} (parse-args command args)] 83 | (if-not errors 84 | (run-command* result) 85 | (show-errors result)))) 86 | 87 | (defn- formatted-commands 88 | "Given a program spec, returns a sequence of formatted commands to be 89 | shown in the help message." 90 | [program] 91 | (let [lines (->> program 92 | :commands 93 | (map #(vector (first %) (:desc (second %)))) 94 | (sort-by first)) 95 | lengths (map count (apply map (partial max-key count) lines))] 96 | (tools.cli/format-lines lengths lines))) 97 | 98 | (defn- parse-input 99 | [{:keys [desc] :as program} args] 100 | (let [{:keys [arguments] :as result} 101 | (tools.cli/parse-opts args 102 | [help] 103 | :in-order true) 104 | [cmd & args] arguments] 105 | (assoc result 106 | :args args 107 | :cmd cmd 108 | :commands (formatted-commands program) 109 | :desc desc 110 | :tip "vessel --help" 111 | :usage (format "vessel [OPTIONS] COMMAND")))) 112 | 113 | (defn- run* 114 | [program input] 115 | (let [{:keys [errors cmd args] :as result} (parse-input program input) 116 | spec (get-in program [:commands cmd])] 117 | (cond 118 | errors (show-errors result) 119 | (show-program-help? result) (show-help result) 120 | (nil? spec) (command-not-found result) 121 | :else (run-command (assoc spec :cmd cmd) args)))) 122 | 123 | (defn run 124 | [program args] 125 | (try 126 | (run* program args) 127 | (catch Throwable t 128 | (show-errors {:errors [(.getMessage t)]}) 129 | (when-let [cause (:vessel.error/throwable (ex-data t))] 130 | (.printStackTrace cause)) 131 | 1))) 132 | 133 | (defn- split-at-colon 134 | "Splits the input into two parts divided by the first colon. 135 | 136 | Throws an IllegalArgumentException with the supplied message if the 137 | input is malformed (i.e. can't be split as explained above)." 138 | [^String input ^String message] 139 | (let [parts (vec (rest (re-matches #"([^:]+):(.*)" input)))] 140 | (if (= 2 (count parts)) 141 | parts 142 | (throw (IllegalArgumentException. message))))) 143 | 144 | (defn parse-attribute 145 | "Takes an attribute specification in the form key:value and returns a 146 | tuple where the first element is the key (as a keyword) and the 147 | second one is the value." 148 | [^String input] 149 | (let [key+value (split-at-colon input "Invalid attribute format. Please, 150 | specify attributes in the form key:value")] 151 | (update key+value 0 keyword))) 152 | 153 | (defn- parse-churn 154 | [^String input] 155 | (if-not input 156 | 0 157 | (try 158 | (Integer/parseInt input) 159 | (catch NumberFormatException _ 160 | (throw (IllegalArgumentException. (format "Expected an integer but got '%s' in the churn field of the extra-path specification." input))))))) 161 | 162 | (defn parse-extra-path 163 | "Takes an extra path specification in the form `source:target` or 164 | `source:target@churn` and returns a map containing the following 165 | keys: 166 | 167 | :source java.io.File 168 | 169 | The file to be copied to the resulting image. 170 | 171 | :target java.io.File 172 | 173 | The absolute path to which the file in question must be copied. 174 | 175 | :churn Integer 176 | 177 | An integer indicating how often this file changes. Defaults to 0." 178 | [^String input] 179 | (let [[source rest] (split-at-colon input 180 | "Invalid extra-path format. 181 | Please, specify extra paths in the form source:target or source:target@churn.") 182 | [target churn] (string/split rest #"@")] 183 | {:source (io/file source) 184 | :target (io/file target) 185 | :churn (parse-churn churn)})) 186 | 187 | (def file-or-dir-must-exist 188 | [misc/file-exists? "no such file or directory"]) 189 | 190 | (def source-must-exist 191 | [#(misc/file-exists? (:source %)) "no such file or directory"]) 192 | 193 | (defn repeat-option 194 | [m k v] 195 | (update-in m [k] (fnil conj #{}) v)) 196 | 197 | (def compiler-options-must-be-nil-or-map 198 | [#(or (nil? %) (map? %)) "Compiler options must be a valid Clojure map"]) 199 | -------------------------------------------------------------------------------- /test/unit/vessel/cli_test.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.cli-test 2 | (:require [clojure.java.io :as io] 3 | [clojure.string :as string] 4 | [clojure.test :refer :all] 5 | [vessel.cli :as cli])) 6 | 7 | (defmacro with-err-str 8 | [& body] 9 | `(let [writer# (java.io.StringWriter.)] 10 | (binding [*err* writer#] 11 | ~@body 12 | (str writer#)))) 13 | 14 | (defn greet 15 | [{:keys [names]}] 16 | (println "Hello" 17 | (string/join ", " names))) 18 | 19 | (defn goodbye 20 | [{:keys [names]}] 21 | (println "Goodbye" 22 | (string/join ", " names))) 23 | 24 | (def vessel 25 | {:desc "FIXME" 26 | :commands 27 | {"greet" 28 | {:desc "Say a hello message for someone" 29 | :fn greet 30 | :opts [["-n" "--name NAME" 31 | :id :names 32 | :desc "Name of the person to greet" 33 | :assoc-fn cli/repeat-option]]} 34 | "goodbye" 35 | {:desc "Print a goodbye message" 36 | :fn greet 37 | :opts [["-n" "--name NAME" 38 | :id :names 39 | :desc "Name of the person to say goodbye to" 40 | :assoc-fn cli/repeat-option]]} 41 | "boom" 42 | {:desc "Simply blows up" 43 | :fn (fn [_] (throw (Exception. "Boom!")))}}}) 44 | 45 | (deftest run-test 46 | (testing "calls the function assigned to the command in question" 47 | (is (= "Hello John Doe\n" 48 | (with-out-str (cli/run vessel ["greet" 49 | "-n" "John Doe"]))))) 50 | 51 | (testing "the function `repeat-option`, when assigned to the `:assoc-fn` 52 | option, allows the flag to be repeated multiple times" 53 | (is (= "Hello John Doe, Jane Doe\n" 54 | (with-out-str (cli/run vessel ["greet" 55 | "-n" "John Doe" 56 | "-n" "Jane Doe"]))))) 57 | 58 | (testing "returns 0 indicating success" 59 | (is (= 0 60 | (cli/run vessel ["greet" 61 | "-n" "John Doe"])))) 62 | 63 | (testing "shows the help message when one calls Vessel with no arguments or with 64 | the help flag" 65 | (are [args] (= "Usage: vessel [OPTIONS] COMMAND\n\nFIXME\n\nOptions:\n -?, --help Show this help message and exit\n\nCommands:\n boom Simply blows up\n goodbye Print a goodbye message\n greet Say a hello message for someone\n\nSee \"vessel COMMAND --help\" for more information on a command\n" 66 | (with-out-str 67 | (cli/run vessel args))) 68 | [] 69 | ["-?"] 70 | ["--help"])) 71 | 72 | (testing "shows the help message for the command in question" 73 | (is (= "Usage: vessel greet [OPTIONS] 74 | 75 | Say a hello message for someone 76 | 77 | Options: 78 | -n, --name NAME Name of the person to greet 79 | -?, --help Show this help message and exit\n" 80 | (with-out-str 81 | (cli/run vessel ["greet" "--help"]))))) 82 | 83 | (testing "returns 0 after showing the help message" 84 | (is (= 0 85 | (cli/run vessel ["--help"]))) 86 | (is (= 0 87 | (cli/run vessel ["goodbye" "--help"])))) 88 | 89 | (testing "shows a meaningful message when Vessel is called with wrong options" 90 | (is (= "Vessel: Unknown option: \"--foo\"\nSee \"vessel --help\"\n" 91 | (with-err-str 92 | (cli/run vessel ["--foo"]))))) 93 | 94 | (testing "returns 1 indicating the error" 95 | (is (= 1 96 | (cli/run vessel ["--foo"])))) 97 | 98 | (testing "shows a meaningful message when the command in question doesn't 99 | exist" 100 | (is (= "Vessel: \"build\" isn't a Vessel command\nSee \"vessel --help\"\n" 101 | (with-err-str 102 | (cli/run vessel ["build" "--help"]))))) 103 | 104 | (testing "returns 127 indicating that the command could not be found" 105 | (is (= 127 106 | (cli/run vessel ["build" "--help"])))) 107 | 108 | (testing "shows a meaningful message when a command is mistakenly called" 109 | (is (= "Vessel: Missing required argument for \"-n NAME\"\nSee \"vessel greet --help\"\n" 110 | (with-err-str 111 | (cli/run vessel ["greet" "-n"]))))) 112 | 113 | (testing "returns 1 indicating an error" 114 | (is (= 1 115 | (cli/run vessel ["greet" "-n"])))) 116 | 117 | (testing "shows an error message when the command throws an exception" 118 | (is (= "Vessel: Boom!\n" 119 | (with-err-str 120 | (cli/run vessel ["boom"]))))) 121 | 122 | (testing "returns 1 indicating an error" 123 | (is (= 1 124 | (cli/run vessel ["boom"]))))) 125 | 126 | (deftest parse-attribute-test 127 | (testing "parses the input in the form `key:value`" 128 | (is (= [:name "my-app"] 129 | (cli/parse-attribute "name:my-app")))) 130 | 131 | (testing "the value can contain colons" 132 | (is (= [:build-date "Fri Jan 31 12:04:26"] 133 | (cli/parse-attribute "build-date:Fri Jan 31 12:04:26")))) 134 | 135 | (testing "throws an exception when the input is malformed" 136 | (is (thrown-with-msg? IllegalArgumentException #"^Invalid attribute format.*" 137 | (cli/parse-attribute "name"))))) 138 | 139 | (deftest parse-extra-path-test 140 | (testing "parses the input in the form `source:target`" 141 | (is (= {:source (io/file "web.xml") 142 | :target (io/file "/app/web.xml") 143 | :churn 0} 144 | (cli/parse-extra-path "web.xml:/app/web.xml")))) 145 | 146 | (testing "parses the input in the form `source:target@churn`" 147 | (is (= {:source (io/file "web.xml") 148 | :target (io/file "/app/web.xml") 149 | :churn 2} 150 | (cli/parse-extra-path "web.xml:/app/web.xml@2")))) 151 | 152 | (testing "throws an exception when the input is malformed" 153 | (is (thrown-with-msg? IllegalArgumentException #"^Invalid extra-path format.*" 154 | (cli/parse-extra-path "web.xml")))) 155 | 156 | (testing "throws an exception when the churn isn't an integer" 157 | (is (thrown-with-msg? IllegalArgumentException #"Expected an integer but got 'foo' in the churn field of the extra-path specification\." 158 | (cli/parse-extra-path "web.xml:/app/web.xml@foo"))))) 159 | 160 | (defn validate 161 | [[validate-fn message] input] 162 | (when-not (validate-fn input) 163 | message)) 164 | 165 | (deftest file-or-dir-must-exist-test 166 | (is (= "no such file or directory" 167 | (validate cli/file-or-dir-must-exist (io/file "foo.txt")))) 168 | 169 | (is (nil? 170 | (validate cli/file-or-dir-must-exist (io/file "deps.edn"))))) 171 | 172 | (deftest source-must-exist-test 173 | (is (= "no such file or directory" 174 | (validate cli/source-must-exist {:source (io/file "foo.txt")}))) 175 | 176 | (is (nil? 177 | (validate cli/source-must-exist {:source (io/file "deps.edn")})))) 178 | -------------------------------------------------------------------------------- /test/integration/vessel/program_integration_test.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.program-integration-test 2 | (:require [clojure.java.io :as io] 3 | [clojure.test :refer :all] 4 | [matcher-combinators.test :refer [match?]] 5 | [mockfn.macros :refer [calling providing]] 6 | [vessel.misc :as misc] 7 | [vessel.program :as vessel] 8 | [vessel.test-helpers :refer [classpath ensure-clean-test-dir]])) 9 | 10 | (def project-dir (io/file "test/resources/my-app")) 11 | (def target-dir (io/file "target/tests/program-integration-test")) 12 | 13 | (def registry 14 | "Name of the Docker registry to be used in the test suite. 15 | 16 | Defaults to localhost, but it may be overwritten through the 17 | environment variable VESSEL_TEST_REGISTRY." 18 | (or (System/getenv "VESSEL_TEST_REGISTRY") 19 | "localhost")) 20 | 21 | (use-fixtures :once (ensure-clean-test-dir)) 22 | 23 | (deftest vessel-test 24 | (providing [(#'vessel/exit int?) (calling identity)] 25 | 26 | (testing "generates a manifest file containing metadata about the 27 | application in question" 28 | (is (zero? (vessel/-main "manifest" 29 | "--attribute" "name:my-app" 30 | "--attribute" "git-commit:07dccc801700cbe28e4a428e455b4627e0ab4ba9" 31 | "--object" "service" 32 | "--output" (str (io/file target-dir "my-app.json"))))) 33 | 34 | (is (= {:service {:name "my-app" 35 | :git-commit "07dccc801700cbe28e4a428e455b4627e0ab4ba9"}} 36 | (misc/read-json (io/file target-dir "my-app.json"))))) 37 | 38 | (testing "generates a manifest file describing the base image" 39 | (is (zero? (vessel/-main "image" 40 | "--repository" "openjdk" 41 | "--tag" "alpine" 42 | "--attribute" "comment:OpenJDK Alpine image" 43 | "--output" (str (io/file target-dir "openjdk-alpine.json"))))) 44 | 45 | (is (= {:image 46 | {:repository "openjdk" 47 | :registry "docker.io" 48 | :tag "alpine" 49 | :comment "OpenJDK Alpine image"}} 50 | (misc/read-json (io/file target-dir "openjdk-alpine.json"))))) 51 | 52 | (testing "generates a manifest file describing the application's 53 | image to be built; merges the manifests created previously" 54 | (is (zero? (vessel/-main "image" 55 | "--repository" "nubank/my-app" 56 | "--registry" (str registry ":5000") 57 | "--attribute" "comment:My Clojure application" 58 | "--base-image" (str (io/file target-dir "openjdk-alpine.json")) 59 | "--merge-into" (str (io/file target-dir "my-app.json")) 60 | "--output" (str (io/file target-dir "image.json"))))) 61 | 62 | (is (match? {:image 63 | {:repository "nubank/my-app" 64 | :registry (str registry ":5000") 65 | :comment "My Clojure application" 66 | :tag #"^[0-9a-f]{64}$"} 67 | :base-image 68 | {:image 69 | {:repository "openjdk" 70 | :registry "docker.io" 71 | :comment "OpenJDK Alpine image" 72 | :tag "alpine"}} 73 | :service 74 | {:name "my-app" 75 | :git-commit "07dccc801700cbe28e4a428e455b4627e0ab4ba9"}} 76 | (misc/read-json (io/file target-dir "image.json"))))) 77 | 78 | (testing "containerizes the application" 79 | (is (zero? 80 | (vessel/-main "containerize" 81 | "--app-root" "/opt/my-app" 82 | "--classpath" (classpath project-dir) 83 | "--extra-path" (str (io/file project-dir "config/config.edn") ":/etc/my-app/config.edn") 84 | "--main-class" "my-app.server" 85 | "--manifest" (str (io/file target-dir "image.json")) 86 | "--output" (str (io/file target-dir "my-app.tar")) 87 | "--preserve-file-permissions" 88 | "--source-path" (str (io/file project-dir "src")) 89 | "--resource-path" (str (io/file project-dir "resources")) 90 | "--verbose"))) 91 | 92 | (is (true? (misc/file-exists? (io/file target-dir "my-app.tar"))))) 93 | 94 | (testing "pushes the built image to the registry" 95 | (is (zero? 96 | (vessel/-main "push" 97 | "--tarball" (str (io/file target-dir "my-app.tar")) 98 | "--allow-insecure-registries" 99 | "--anonymous")))) 100 | 101 | (testing "containerizes a new image, now using a tarball as the base image" 102 | (is (zero? (vessel/-main "manifest" 103 | "--attribute" (str "tar-path:" (io/file target-dir "my-app.tar")) 104 | "--object" "image" 105 | "--output" (str (io/file target-dir "tarball.json"))))) 106 | 107 | (is (zero? (vessel/-main "image" 108 | "--repository" "nubank/my-app2" 109 | "--registry" (str registry ":5000") 110 | "--base-image" (str (io/file target-dir "tarball.json")) 111 | "--output" (str (io/file target-dir "image2.json"))))) 112 | 113 | (is (zero? 114 | (vessel/-main "containerize" 115 | "--app-root" "/opt/my-app2" 116 | "--classpath" (classpath project-dir) 117 | "--main-class" "my-app.server" 118 | "--manifest" (str (io/file target-dir "image2.json")) 119 | "--output" (str (io/file target-dir "my-app2.tar")) 120 | "--source-path" (str (io/file project-dir "src")) 121 | "--resource-path" (str (io/file project-dir "resources")) 122 | "--verbose")))))) 123 | -------------------------------------------------------------------------------- /src/vessel/misc.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.misc 2 | (:require [clojure.data.json :as json] 3 | [clojure.edn :as edn] 4 | [clojure.java.io :as io] 5 | [clojure.string :as string]) 6 | (:import java.io.File 7 | [java.nio.file Files LinkOption Path Paths] 8 | java.security.MessageDigest 9 | java.text.DecimalFormat 10 | (java.nio.file.attribute FileTime) 11 | [java.time Duration Instant] 12 | java.util.function.Consumer 13 | java.util.Locale)) 14 | 15 | (defn assoc-some 16 | "Assoc's key and value into the associative data structure only when 17 | the value isn't nil." 18 | [m & kvs] 19 | {:pre [(even? (count kvs))]} 20 | (reduce (fn [result [key val]] 21 | (if-not (nil? val) 22 | (assoc result key val) 23 | result)) 24 | m (partition 2 kvs))) 25 | 26 | (def kebab-case 27 | "Converts a string in camel or snake case to kebap case." 28 | (comp string/lower-case #(string/replace % #"([a-z])([A-Z])|_+" "$1-$2"))) 29 | 30 | ;; Java interop functions 31 | 32 | (defn ^Path string->java-path 33 | [^String path] 34 | (Paths/get path (into-array String []))) 35 | 36 | (defn ^Consumer java-consumer 37 | "Returns a java.util.function.Consumer instance that calls the function f. 38 | 39 | f is a 1-arity function that returns nothing." 40 | [f] 41 | (reify Consumer 42 | (accept [_ arg] 43 | (f arg)))) 44 | 45 | (defn now 46 | "Return a java.time.Instant object representing the current instant." 47 | [] 48 | (Instant/now)) 49 | 50 | (defn duration-between 51 | "Return a java.time.Duration object representing the duration between two temporal objects." 52 | [start end] 53 | (Duration/between start end)) 54 | 55 | (def ^:private formatter 56 | "Instance of java.text.Decimalformat used internally to format decimal 57 | values." 58 | (let [decimal-format (DecimalFormat/getInstance (Locale/ENGLISH))] 59 | (.applyPattern decimal-format "#.##") 60 | decimal-format)) 61 | 62 | (defn- format-duration 63 | [value time-unit] 64 | (str (.format formatter value) " " 65 | (if (= (float value) 1.0) 66 | (name time-unit) 67 | (str (name time-unit) "s")))) 68 | 69 | (defn ^String duration->string 70 | "Returns a friendly representation of the duration object in question." 71 | [^Duration duration] 72 | (let [millis (.toMillis duration)] 73 | (cond 74 | (<= millis 999) (format-duration millis :millisecond) 75 | (<= millis 59999) (format-duration (float (/ millis 1000)) :second) 76 | :else (format-duration (float (/ millis 60000)) :minute)))) 77 | 78 | ;; I/O functions. 79 | 80 | (defmacro with-stderr 81 | "Binds *out* to *err* and evaluates body." 82 | [& body] 83 | `(binding [*out* *err*] 84 | ~@body)) 85 | 86 | (def ^:dynamic *verbose-logs* false) 87 | 88 | (defn- emit-log 89 | [level emitter message] 90 | (cond 91 | *verbose-logs* (println (format "%s [%s] %s" level emitter message)) 92 | (#{"ERROR" "FATAL" "INFO"} level) (println message))) 93 | 94 | (defn log* 95 | [level emitter message & args] 96 | (let [the-level (string/upper-case (if (keyword? level) 97 | (name level) (str level))) 98 | the-message (apply format message args)] 99 | (if (#{"ERROR" "FATAL"} the-level) 100 | (with-stderr (emit-log the-level emitter the-message)) 101 | (emit-log the-level emitter the-message)))) 102 | 103 | (defmacro log 104 | [level message & args] 105 | `(apply log* 106 | ~level ~(str (ns-name *ns*)) ~message 107 | [~@args])) 108 | 109 | (defn file-exists? 110 | "Returns true if the file exists or false otherwise." 111 | [^File file] 112 | (.exists file)) 113 | 114 | (defn filter-files 115 | "Given a sequence of java.io.File objects (either files or 116 | directories), returns just the files." 117 | [fs] 118 | (filter #(.isFile %) fs)) 119 | 120 | (defn ^File home-dir 121 | "Returns a file object representing the home directory of the current 122 | user." 123 | [] 124 | (io/file (System/getProperty "user.home"))) 125 | 126 | (defn ^Instant last-modified-time 127 | "Returns the file's last modified time as a java.time.Instant." 128 | [^File file] 129 | (.toInstant (Files/getLastModifiedTime (.toPath file) (make-array LinkOption 0)))) 130 | 131 | (defn ^File make-dir 132 | "Creates the directory in question and all of its parents. 133 | 134 | The arguments are the same taken by clojure.java.io/file. Returns 135 | the created directory." 136 | [f & others] 137 | {:pos [(.isDirectory %)]} 138 | (let [dir (apply io/file f others)] 139 | (.mkdirs dir) 140 | dir)) 141 | 142 | (defn make-empty-dir 143 | "Creates the directory in question and all of its parents. 144 | 145 | If the directory already exists, delete all existing files and 146 | sub-directories. 147 | 148 | The arguments are the same taken by clojure.java.io/file. Returns 149 | the created directory." 150 | [f & others] 151 | (let [dir (apply io/file f others)] 152 | (when (file-exists? dir) 153 | (run! #(io/delete-file %) (reverse (file-seq dir)))) 154 | (make-dir dir))) 155 | 156 | (defn posix-file-permissions 157 | "Returns a set containing posix file permissions for the supplied 158 | file." 159 | [^File file] 160 | (->> (Files/getPosixFilePermissions (.toPath file) (make-array LinkOption 0)) 161 | (map str) 162 | set)) 163 | 164 | (defn ^File relativize 165 | "Given a file and a base directory, returns a new file representing 166 | the relative path of the provided file in relation to the base 167 | directory." 168 | [^File file ^File base] 169 | (.. base toPath (relativize (.toPath file)) toFile)) 170 | 171 | (defn read-edn 172 | "Reads an EDN object and parses it as Clojure data. 173 | 174 | input can be any object supported by clojure.core/slurp." 175 | [input] 176 | (edn/read-string (slurp input))) 177 | 178 | (defn read-data 179 | "Reads one object from the provided input. 180 | 181 | input can be any object supported by clojure.core/slurp. 182 | 183 | Note: this function is meant to be used to parse objects from 184 | data_readers files." 185 | [input] 186 | (binding [*read-eval* false] 187 | (read-string {:read-cond :preserve :features #{:clj}} 188 | (slurp input)))) 189 | 190 | (defn read-json 191 | "Reads a JSON object and parses it as Clojure data. 192 | 193 | input can be any object supported by clojure.core/slurp." 194 | [input] 195 | (json/read-str (slurp input) :key-fn keyword)) 196 | 197 | (defn- hex 198 | "Returns the hexadecimal representation of the provided array of 199 | bytes." 200 | ^String 201 | [bytes] 202 | (let [builder (StringBuilder.)] 203 | (run! #(.append builder (format "%02x" %)) bytes) 204 | (str builder))) 205 | 206 | (defn sha-256 207 | "Returns the SHA-256 digest for the Clojure object in question." 208 | ^String 209 | [data] 210 | (let [message-digest (MessageDigest/getInstance "SHA-256") 211 | input (.getBytes (pr-str data) "UTF-8")] 212 | (.update message-digest input) 213 | (hex (.digest message-digest)))) 214 | 215 | (defn set-timestamp 216 | [^File file ^Long last-modified-time] 217 | (let [^FileTime file-time (FileTime/fromMillis last-modified-time)] 218 | (Files/setAttribute (.toPath file) "lastModifiedTime" file-time (make-array LinkOption 0)))) 219 | -------------------------------------------------------------------------------- /src/vessel/program.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.program 2 | (:gen-class) 3 | (:require [clojure.edn :as edn] 4 | [clojure.java.io :as io] 5 | [clojure.string :as string] 6 | [vessel.api :as api] 7 | [vessel.cli :as cli] 8 | [vessel.misc :as misc])) 9 | 10 | (def ^:private cwd 11 | "Current working directory." 12 | (.getCanonicalFile (io/file "."))) 13 | 14 | (def verbose ["-v" "--verbose" 15 | :id :verbose? 16 | :desc "Show verbose messages"]) 17 | 18 | (defn- exit 19 | "Terminates the JVM with the status code." 20 | [status] 21 | (System/exit status)) 22 | 23 | (def vessel 24 | {:desc "A containerization tool for Clojure applications" 25 | :commands 26 | {"build" 27 | {:desc "Build a Clojure application" 28 | :fn api/build 29 | :opts [["-c" "--classpath PATHS" 30 | :id :classpath-files 31 | :desc "Directories and zip/jar files on the classpath in the same format expected by the java command" 32 | :parse-fn (comp (partial map io/file) #(string/split % #":"))] 33 | ["-s" "--source-path PATH" 34 | :id :source-paths 35 | :desc "Directories containing source files. This option can be repeated many times" 36 | :parse-fn io/file 37 | :validate cli/file-or-dir-must-exist 38 | :assoc-fn cli/repeat-option] 39 | ["-m" "--main-class NAMESPACE" 40 | :id :main-class 41 | :desc "Namespace that contains the application's entrypoint, with a :gen-class directive and a -main function" 42 | :parse-fn symbol] 43 | ["-r" "--resource-path PATH" 44 | :id :resource-paths 45 | :desc "Directories containing resource files. This option can be repeated many times" 46 | :parse-fn io/file 47 | :validate cli/file-or-dir-must-exist 48 | :assoc-fn cli/repeat-option] 49 | ["-o" "--output PATH" 50 | :id :target-dir 51 | :desc "Directory where the application's files will be written to" 52 | :parse-fn io/file 53 | :validate cli/file-or-dir-must-exist] 54 | ["-C" "--compiler-options OPTIONS" 55 | :id :compiler-options 56 | :desc "Options provided to the Clojure compiler, see clojure.core/*compiler-options*" 57 | :default nil 58 | :parse-fn edn/read-string 59 | :validate cli/compiler-options-must-be-nil-or-map] 60 | verbose]} 61 | 62 | "containerize" 63 | {:desc "Containerize a Clojure application" 64 | :fn api/containerize 65 | :opts [["-a" "--app-root PATH" 66 | :id :app-root 67 | :desc "app root of the container image. Classes and resource files will be copied to relative paths to the app root" 68 | :default (io/file "/app") 69 | :parse-fn io/file] 70 | ["-c" "--classpath PATHS" 71 | :id :classpath-files 72 | :desc "Directories and zip/jar files on the classpath in the same format expected by the java command" 73 | :parse-fn (comp (partial map io/file) #(string/split % #":"))] 74 | ["-e" "--extra-path PATH" 75 | :id :extra-paths 76 | :desc "extra files to be copied to the container image. The value must be passed in the form source:target or source:target@churn and this option can be repeated many times" 77 | :parse-fn cli/parse-extra-path 78 | :validate cli/source-must-exist 79 | :assoc-fn cli/repeat-option] 80 | ["-i" "--internal-deps REGEX" 81 | :id :internal-deps-re 82 | :desc "java regex to determine internal dependencies. Can be repeated many times for a logical or effect" 83 | :parse-fn re-pattern 84 | :assoc-fn cli/repeat-option] 85 | ["-m" "--main-class NAMESPACE" 86 | :id :main-class 87 | :desc "Namespace that contains the application's entrypoint, with a :gen-class directive and a -main function" 88 | :parse-fn symbol] 89 | ["-M" "--manifest PATH" 90 | :id :manifest 91 | :desc "manifest file describing the image to be built" 92 | :parse-fn io/file 93 | :validate cli/file-or-dir-must-exist 94 | :assoc-fn #(assoc %1 %2 (misc/read-json %3))] 95 | ["-o" "--output PATH" 96 | :id :tarball 97 | :desc "path to save the tarball containing the built image" 98 | :default (io/file "image.tar") 99 | :parse-fn io/file] 100 | ["-p" "--project-root PATH" 101 | :id :project-root 102 | :desc "root dir of the Clojure project to be built" 103 | :default cwd 104 | :parse-fn io/file 105 | :validate cli/file-or-dir-must-exist] 106 | ["-P" "--preserve-file-permissions" 107 | :id :preserve-file-permissions? 108 | :desc "Preserve original file permissions when copying files to the container. If not enabled, the default permissions for files are 644"] 109 | ["-s" "--source-path PATH" 110 | :id :source-paths 111 | :desc "Directories containing source files. This option can be repeated many times" 112 | :parse-fn io/file 113 | :validate cli/file-or-dir-must-exist 114 | :assoc-fn cli/repeat-option] 115 | ["-r" "--resource-path PATH" 116 | :id :resource-paths 117 | :desc "Directories containing resource files. This option can be repeated many times" 118 | :parse-fn io/file 119 | :validate cli/file-or-dir-must-exist 120 | :assoc-fn cli/repeat-option] 121 | ["-u" "--user USER" 122 | :id :user 123 | :desc "Define the default user for the image" 124 | :default "root"] 125 | ["-C" "--compiler-options OPTIONS" 126 | :id :compiler-options 127 | :desc "Options provided to the Clojure compiler, see clojure.core/*compiler-options*" 128 | :default nil 129 | :parse-fn edn/read-string 130 | :validate cli/compiler-options-must-be-nil-or-map] 131 | verbose]} 132 | 133 | "image" 134 | {:desc "Generate an image manifest, optionally by extending a base image and/or merging other manifests" 135 | :fn api/image 136 | :opts 137 | [["-a" "--attribute KEY-VALUE" 138 | :id :attributes 139 | :desc "Add the attribute in the form key:value to the manifest. This option can be repeated multiple times" 140 | :parse-fn cli/parse-attribute 141 | :assoc-fn cli/repeat-option] 142 | ["-b" "--base-image PATH" 143 | :id :base-image 144 | :desc "Manifest file describing the base image" 145 | :parse-fn io/file 146 | :validate cli/file-or-dir-must-exist 147 | :assoc-fn #(assoc %1 %2 (misc/read-json %3))] 148 | ["-m" "--merge-into PATH" 149 | :id :manifests 150 | :desc "Manifest file to be merged into the manifest being created. This option can be repeated multiple times" 151 | :parse-fn io/file 152 | :validate cli/file-or-dir-must-exist 153 | :assoc-fn #(cli/repeat-option %1 %2 (misc/read-json %3))] 154 | ["-o" "--output PATH" 155 | :id :output 156 | :desc "Write the manifest to path instead of stdout" 157 | :default *out* 158 | :default-desc "stdout" 159 | :parse-fn (comp io/writer io/file)] 160 | ["-r" "--registry REGISTRY" 161 | :id :registry 162 | :desc "Image registry" 163 | :default "docker.io"] 164 | ["-R" "--repository REPOSITORY" 165 | :id :repository 166 | :desc "Image repository"] 167 | ["-t" "--tag TAG" 168 | :id :tag 169 | :desc "Image tag. When omitted uses a SHA-256 digest of the resulting manifest"]]} 170 | 171 | "manifest" 172 | {:desc "Generate arbitrary manifests" 173 | :fn api/manifest 174 | :opts 175 | [["-a" "--attribute KEY-VALUE" 176 | :id :attributes 177 | :desc "Add the attribute in the form key:value to the manifest. This option can be repeated multiple times" 178 | :default [] 179 | :parse-fn cli/parse-attribute 180 | :assoc-fn cli/repeat-option] 181 | ["-o" "--output PATH" 182 | :id :output 183 | :desc "Write the manifest to path instead of stdout" 184 | :default *out* 185 | :default-desc "stdout" 186 | :parse-fn (comp io/writer io/file)] 187 | ["-O" "--object OBJECT" 188 | :id :object 189 | :desc "Object under which attributes will be added" 190 | :parse-fn keyword]]} 191 | 192 | "push" 193 | {:desc "Push a tarball to a registry" 194 | :fn api/push 195 | :opts [["-t" "--tarball PATH" 196 | :id :tarball 197 | :desc "Tar archive containing image layers and metadata files" 198 | :parse-fn io/file 199 | :validate cli/file-or-dir-must-exist] 200 | ["-a" "--allow-insecure-registries" 201 | :id :allow-insecure-registries? 202 | :desc "Allow pushing images to insecure registries"] 203 | ["-A" "--anonymous" 204 | :id :anonymous? 205 | :desc "Do not authenticate on the registry; push anonymously"] 206 | verbose]}}}) 207 | 208 | (defn -main 209 | [& args] 210 | (exit (cli/run vessel args))) 211 | -------------------------------------------------------------------------------- /test/unit/vessel/builder_test.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.builder-test 2 | (:require [clojure.java.io :as io] 3 | [clojure.string :as string] 4 | [clojure.test :refer :all] 5 | [matcher-combinators.matchers :as m] 6 | [matcher-combinators.test :refer [match?]] 7 | [vessel.builder :as builder] 8 | [vessel.misc :as misc] 9 | [vessel.sh :as sh] 10 | [babashka.fs :as fs] 11 | [vessel.test-helpers :refer [classpath ensure-clean-test-dir]]) 12 | (:import java.io.File 13 | (clojure.lang ExceptionInfo))) 14 | 15 | (use-fixtures :once (ensure-clean-test-dir)) 16 | 17 | (deftest get-class-file-source-test 18 | (let [namespaces {'clj-jwt.base64 (io/file "clj-jwt.jar") 19 | 'clj-tuple (io/file "clj-tuple.jar") 20 | 'zookeeper (io/file "zookeeper-clj.jar")}] 21 | 22 | (testing "given a class file, returns its source" 23 | (are [file-path src] (= (io/file src) 24 | (builder/get-class-file-source namespaces (io/file file-path))) 25 | "clj_jwt/base64$decode.class" "clj-jwt.jar" 26 | "clj_jwt/base64/ByteArrayInput.class" "clj-jwt.jar" 27 | "clj_tuple$fn__18034.class" "clj-tuple.jar" 28 | "zookeeper$host_acl.class" "zookeeper-clj.jar" 29 | "zookeeper$set_data$fn__54225.class" "zookeeper-clj.jar" 30 | "zookeeper__init.class" "zookeeper-clj.jar")))) 31 | 32 | (deftest copy-files-test 33 | (let [src (io/file "test/resources") 34 | target (io/file "target/tests/builder-test/copy-files-test") 35 | output (builder/copy-files #{(io/file src "lib1") 36 | (io/file src "lib2") 37 | (io/file src "lib3/lib3.jar") 38 | (io/file src "lib4")} 39 | target)] 40 | 41 | (testing "copies all src files (typically resources) to the target directory 42 | under the `classes` folder" 43 | (is (match? (m/in-any-order ["classes/META-INF/MANIFEST.MF" 44 | "classes/data_readers.clj" 45 | "classes/data_readers.cljc" 46 | "classes/lib2/resource2.json" 47 | "classes/lib3/resource3.edn" 48 | "classes/resource1.edn"]) 49 | (->> target 50 | file-seq 51 | misc/filter-files 52 | (map (comp #(.getPath %) #(misc/relativize % target))))))) 53 | 54 | (testing "after copying files, returns a map from target to src files" 55 | (is (= {(io/file target "classes/lib2/resource2.json") (io/file src "lib2") 56 | (io/file target "classes/lib3/resource3.edn") (io/file src "lib3/lib3.jar") 57 | (io/file target "classes/META-INF/MANIFEST.MF") (io/file src "lib3/lib3.jar") 58 | (io/file target "classes/data_readers.clj") (io/file src "lib2") 59 | (io/file target "classes/data_readers.cljc") (io/file src "lib4") 60 | ;; resource1 is merged, last read file is in lib2 61 | (io/file target "classes/resource1.edn") (io/file src "lib2")} 62 | output))) 63 | 64 | (testing "edn files are merged" 65 | ;; This is the deep merge of the two copies of resource1: 66 | (is (= {:list [1 2 3 4 4 5 6] 67 | :map {:a 2 68 | :b 3 69 | :c 4} 70 | :set #{1 2 3} 71 | :old-key 4 72 | :new-key 4 73 | :same-key :new-value} 74 | (-> (io/file target "classes/resource1.edn") 75 | misc/read-edn)))) 76 | 77 | (testing "multiple data-readers (either data_readers.clj or 78 | data_readers.cljc files) found at the root of the classpath are merged into 79 | their respective files" 80 | (is (= {'lib1/url 'lib1.url/string->url 81 | 'lib2/time 'lib2.time/string->local-date-time} 82 | (misc/read-edn (io/file target "classes/data_readers.clj")))) 83 | (is (= {'lib3/date 'lib3.date/string->local-date 84 | 'lib4/usd 'lib4.money/bigdec->money} 85 | (misc/read-edn (io/file target "classes/data_readers.cljc"))))) 86 | 87 | (testing "preserve timestamps when copying files" 88 | ;; This file is simply copied: 89 | (is (= (.lastModified (io/file src "lib2/libs2/resource2.edn")) 90 | (.lastModified (io/file target "classes/lib2/resource2.edn")))) 91 | ;; resource1 is merged, last version in lib2 92 | (is (= (.lastModified (io/file src "lib2/resource1.edn")) 93 | (.lastModified (io/file target "classes/resource1.edn"))))))) 94 | 95 | (deftest merge-with-reader-macros 96 | (let [src (io/file "test/resources") 97 | target (io/file "target/tests/builder-test/reader-macros") 98 | _ (builder/copy-files #{(io/file src "lib5") 99 | (io/file src "lib6")} 100 | target)] 101 | 102 | (testing "edn files with reader macros are merged" 103 | (is (= (slurp (io/file target "classes" "with-reader-macros.edn")) 104 | "{:k1 :override-k1, :k2 #unknown/macro :v2, :k3 #weird/macro :v3}"))))) 105 | 106 | (defn get-file-names 107 | [^File dir] 108 | (map #(.getName %) (.listFiles dir))) 109 | 110 | (deftest build-app-test 111 | (let [project-dir (io/file "test/resources/my-app") 112 | target (io/file "target/tests/builder-test/build-app-test") 113 | classpath-files (map io/file (string/split (classpath project-dir) #":")) 114 | options {:classpath-files classpath-files 115 | :source-paths #{(io/file project-dir "src")} 116 | :resource-paths #{(io/file project-dir "resources")} 117 | :target-dir target} 118 | output (builder/build-app (assoc options :main-class 'my-app.server))] 119 | (testing "the classes directory has the expected files and directories" 120 | (is (match? (m/in-any-order ["clojure" 121 | "META-INF" 122 | "my_app" 123 | "resource1.edn"]) 124 | (get-file-names (io/file target "WEB-INF/classes"))))) 125 | 126 | (testing "the lib directory has the expected files" 127 | (is (match? (m/in-any-order ["core.specs.alpha-0.2.44.jar" 128 | "javax.servlet-api-3.1.0.jar" 129 | "jetty-http-9.4.25.v20191220.jar" 130 | "jetty-io-9.4.25.v20191220.jar" 131 | "jetty-server-9.4.25.v20191220.jar" 132 | "jetty-util-9.4.25.v20191220.jar" 133 | "spec.alpha-0.2.176.jar"]) 134 | (get-file-names (io/file target "WEB-INF/lib"))))) 135 | 136 | (testing "the output data contains a map in the :app/classes key whose keys 137 | match the existing files at the classes directory" 138 | (is (= (set (keys (:app/classes output))) 139 | (set (misc/filter-files (file-seq (io/file target "WEB-INF/classes"))))))) 140 | 141 | (testing "the output data contains a :app/lib key whose values match the 142 | existing files at the lib directory" 143 | (is (= (set (:app/lib output)) 144 | (set (misc/filter-files (file-seq (io/file target "WEB-INF/lib"))))))) 145 | 146 | ;; TODO: uncomment after fixing https://github.com/nubank/vessel/issues/7. 147 | #_(testing "throws an exception describing the underwing compilation error" 148 | (is (thrown-match? clojure.lang.ExceptionInfo 149 | #:vessel.error{:category :vessel/compilation-error} 150 | (builder/build-app (assoc options :main-class 'my-app.compilation-error))))))) 151 | 152 | (deftest use-provided-compiler-options-when-building-app-test 153 | (let [project-dir (io/file "test/resources/my-app") 154 | target (io/file "target/tests/builder-test/build-app-test") 155 | classpath-files (map io/file (string/split (classpath project-dir) #":")) 156 | options {:classpath-files classpath-files 157 | :resource-paths #{(io/file project-dir "src")} 158 | :source-paths #{(io/file project-dir "resources")} 159 | :target-dir target 160 | :main-class 'my-app.server 161 | :compiler-options {:direct-linking true 162 | :testing? true}} 163 | sh-args (atom [])] 164 | (with-redefs [sh/javac (constantly nil) 165 | sh/clojure #(reset! sh-args [%1 %2 %3])] 166 | (builder/build-app options)) 167 | 168 | (is (re-matches #".*clojure\.core/\*compiler-options\* \(clojure\.core/merge clojure\.core/\*compiler-options\* \{:direct-linking true, :testing\? true\}\).*" (last @sh-args))))) 169 | 170 | (deftest reports-errors-when-merging-from-dir 171 | (let [src (io/file "test/resources") 172 | badlib (io/file src "badlib") 173 | target (io/file "target/tests/build-test/merge-errors") 174 | e (is (thrown? ExceptionInfo 175 | (builder/copy-files #{badlib} target)))] 176 | (when e 177 | (is (= "Unable to read test/resources/badlib/bad-input.edn: Map literal must contain an even number of forms" (ex-message e))) 178 | (is (match? {:classpath-root badlib 179 | :input-source "test/resources/badlib/bad-input.edn" 180 | :target-file (m/via str (str target "/classes/bad-input.edn"))} 181 | (ex-data e)))))) 182 | 183 | (deftest reports-errors-when-merging-from-jar 184 | (let [src (io/file "test/resources") 185 | badlib (io/file src "badlib") 186 | badlib-jar (io/file "target/badlib.jar") 187 | _ (fs/zip badlib-jar badlib {:root "test/resources/badlib"}) 188 | target (io/file "target/tests/build-test/merge-errors") 189 | e (is (thrown? ExceptionInfo 190 | (builder/copy-files #{badlib-jar} target)))] 191 | (when e 192 | (is (= "Unable to read target/badlib.jar#bad-input.edn: Map literal must contain an even number of forms" (ex-message e))) 193 | (is (match? {:classpath-root (m/via str "target/badlib.jar") 194 | :input-source "target/badlib.jar#bad-input.edn" 195 | :target-file (m/via str (str target "/classes/bad-input.edn"))} 196 | (ex-data e)))))) 197 | -------------------------------------------------------------------------------- /src/vessel/jib/pusher.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.jib.pusher 2 | "Push API built on top of Google Jib." 3 | (:require [clojure.core.async :as async :refer [!!]] 4 | [clojure.data.json :as json] 5 | [clojure.java.io :as io] 6 | [progrock.core :as progrock] 7 | [vessel.jib.credentials :as credentials] 8 | [vessel.jib.helpers :as jib.helpers] 9 | [vessel.misc :as misc]) 10 | (:import [com.google.cloud.tools.jib.api DescriptorDigest ImageReference LogEvent] 11 | [com.google.cloud.tools.jib.blob Blob BlobDescriptor Blobs] 12 | com.google.cloud.tools.jib.event.EventHandlers 13 | com.google.cloud.tools.jib.event.events.ProgressEvent 14 | com.google.cloud.tools.jib.hash.Digests 15 | com.google.cloud.tools.jib.http.FailoverHttpClient 16 | [com.google.cloud.tools.jib.image.json BuildableManifestTemplate V22ManifestTemplate] 17 | com.google.cloud.tools.jib.registry.RegistryClient 18 | java.io.File)) 19 | 20 | (defn- show-progress 21 | [progress-bar] 22 | (progrock/print progress-bar) 23 | (println) 24 | progress-bar) 25 | 26 | (defn progress-monitor 27 | [^File temp-dir] 28 | (let [channel (async/chan) 29 | total-size (apply + (map #(.length %) (.listFiles temp-dir)))] 30 | (async/go-loop [progress-bar (show-progress (progrock/progress-bar total-size))] 31 | (let [{:progress/keys [status value]} ( (RegistryClient/factory event-handlers (.getRegistry image-reference) (.getRepository image-reference) http-client) 67 | (cond-> (not anonymous?) 68 | (.setCredential credential)) 69 | (.setUserAgentSuffix "vessel") 70 | (.newRegistryClient))] 71 | (when-not anonymous? 72 | (authenticate client)) 73 | (misc/log :info "The push refers to repository [%s]" image-reference) 74 | client)) 75 | 76 | (defn- push-blob 77 | "Pushes the blob into the target registry." 78 | [^RegistryClient client progress-channel {:blob/keys [descriptor reader]}] 79 | (let [^Blob blob (reader) 80 | byte-listener (misc/java-consumer (fn [^Long bytes] 81 | >!! progress-channel #:progress {:value bytes 82 | :status :progress}))] 83 | (.pushBlob client (.getDigest descriptor) 84 | blob 85 | nil 86 | byte-listener))) 87 | 88 | (defn- ^Boolean check-blob 89 | "Checks whether the supplied digest exists on the target registry." 90 | [^RegistryClient client ^DescriptorDigest digest] 91 | (.. client (checkBlob digest) isPresent)) 92 | 93 | (defn- ^BlobDescriptor push-layer* 94 | [^RegistryClient client progress-channel {:blob/keys [descriptor] :as blob-data}] 95 | (let [^DescriptorDigest digest (.getDigest descriptor)] 96 | (if (check-blob client digest) 97 | (do (>!! progress-channel #:progress{:value (.getSize descriptor) 98 | :status :progress}) 99 | (misc/log :info "%s: layer already exists on target repository" digest)) 100 | (do (push-blob client progress-channel blob-data) 101 | (misc/log :info "%s: layer pushed" digest))) 102 | descriptor)) 103 | 104 | (defn ^BlobDescriptor push-layer 105 | "Pushes a layer into the target registry. 106 | 107 | First, checks whether the layer in question already exists in the 108 | repository. If so, skips the push. Otherwise, reads the layer from 109 | disk and transfers it to the registry. 110 | 111 | Returns the blob descriptor that represents the layer if the push 112 | succeeds or a Throwable object if the same fails." 113 | [^RegistryClient client progress-channel {:blob/keys [descriptor] :as blob-data}] 114 | (try 115 | (push-layer* client progress-channel blob-data) 116 | (catch Throwable error 117 | (misc/log :error "%s: push failed" (.getDigest descriptor)) 118 | error))) 119 | 120 | (defn ^BlobDescriptor push-container-config 121 | "Pushes the container configuration into the registry. 122 | 123 | Returns the BlobDescriptor object that represents the container 124 | configuration." 125 | [^RegistryClient client progress-channel {:blob/keys [descriptor] :as blob-data}] 126 | (push-blob client progress-channel blob-data) 127 | (misc/log :info "%s: container configuration pushed" (.getDigest descriptor)) 128 | descriptor) 129 | 130 | (defn ^DescriptorDigest push-manifest 131 | "Pushes the image manifest for a specific tag into the target repository. 132 | 133 | Returns the digest of the pushed image." 134 | [^RegistryClient client progress-channel ^BuildableManifestTemplate manifest ^String tag] 135 | (let [^DescriptorDigest digest (.pushManifest client manifest tag)] 136 | (>!! progress-channel #:progress{:status :done}) 137 | digest)) 138 | 139 | (defn- make-blob-data 140 | "Given a temporary directory and a file name contained therein, 141 | returns a map with the following namespaced keys: 142 | 143 | :blob/descriptor BlobDescriptor 144 | 145 | Object that contains properties describing a blob. 146 | 147 | :blob/read Function 148 | 149 | Function of no args that reads a blob from disk when invoked." 150 | [^File temp-dir ^String file-name] 151 | (let [make-input-stream #(io/input-stream (io/file temp-dir file-name)) 152 | ^BlobDescriptor blob-descriptor (Digests/computeDigest (make-input-stream))] 153 | #:blob{:descriptor blob-descriptor 154 | :reader #(Blobs/from (make-input-stream))})) 155 | 156 | (defn ^BuildableManifestTemplate make-buildable-manifest-template 157 | "Returns an instance of BuildableManifestTemplate containing the size 158 | and digest of the container configuration and the layers that 159 | compose the image to be pushed to the remote repository." 160 | [^BlobDescriptor container-config-descriptor layer-descriptors] 161 | (let [^BuildableManifestTemplate manifest (V22ManifestTemplate.)] 162 | (.setContainerConfiguration manifest (.getSize container-config-descriptor) (.getDigest container-config-descriptor)) 163 | (run! (fn [^BlobDescriptor layer-descriptor] 164 | (.addLayer manifest (.getSize layer-descriptor) (.getDigest layer-descriptor))) 165 | layer-descriptors) 166 | manifest)) 167 | 168 | (defn- read-image-manifest 169 | "Reads the manifest.json extracted from the built tarball. 170 | 171 | This is the manifest.json generated by Jib. It has nothing to do 172 | with Vessel manifests." 173 | [^File source-dir] 174 | (first 175 | (json/read-str (slurp (io/file source-dir "manifest.json")) 176 | :key-fn (comp keyword misc/kebab-case)))) 177 | 178 | (defn- throw-some 179 | "Throws the first Throwable found in coll as an ExceptionInfo or 180 | simply returns coll unchanged if no Throwable has been found." 181 | [coll] 182 | (some (fn [candidate] 183 | (when (instance? Throwable candidate) 184 | (throw (ex-info "One or more layers could not be pushed into remote registry" 185 | #:vessel.error{:category :vessel/push-error 186 | :throwable candidate})))) 187 | coll) 188 | coll) 189 | 190 | (defn- push-layers 191 | "Pushes all layers into target registry in parallel. 192 | 193 | Returns a sequence of BlobDescriptor objects representing the pushed 194 | layers." 195 | [^RegistryClient client progress-channel ^File temp-dir layers] 196 | (let [input-channel (async/to-chan layers) 197 | output-channel (async/chan) 198 | parallelism (count layers) 199 | transducer (comp (map (partial make-blob-data temp-dir)) 200 | (map (partial push-layer client progress-channel)))] 201 | (async/pipeline-blocking parallelism 202 | output-channel 203 | transducer 204 | input-channel) 205 | (->> output-channel 206 | (async/into []) 207 | 'clj-jwt.base64" 22 | [^Symbol ns] 23 | (->> (string/split (str ns) #"\.") 24 | drop-last 25 | (string/join ".") 26 | symbol)) 27 | 28 | (defn- ^Symbol class-file->ns 29 | "Turns a .class file into the corresponding namespace symbol." 30 | [^File class-file] 31 | (let [ns (first (string/split (.getPath class-file) #"\$|__init|\.class"))] 32 | (-> ns 33 | (string/replace "/" ".") 34 | (string/replace "_" "-") 35 | symbol))) 36 | 37 | (defn ^File get-class-file-source 38 | "Given a map from ns symbols to their sources (either directories or 39 | jar files on the classpath) and a file representing a compiled 40 | class, returns the source where the class in question comes from." 41 | [namespaces ^File class-file] 42 | (let [ns (class-file->ns class-file)] 43 | (get namespaces ns 44 | (get namespaces (parent-ns ns))))) 45 | 46 | (defn compile-java-sources 47 | "Compiles Java sources present in the source paths and writes 48 | resulting .class files to the supplied target directory." 49 | [classpath source-paths ^File target-dir] 50 | (let [java-sources (->> source-paths 51 | (map file-seq) 52 | flatten 53 | (filter #(string/ends-with? (.getName %) ".java")))] 54 | (when (seq java-sources) 55 | (sh/javac classpath target-dir java-sources)))) 56 | 57 | (defn- do-compile 58 | "Compiles the main-ns by writing compiled .class files to target-dir." 59 | [^Symbol main-ns ^Sequential classpath source-paths ^File target-dir compiler-options] 60 | (let [forms `(try 61 | (binding [*compile-path* ~(str target-dir) 62 | *compiler-options* (merge *compiler-options* ~compiler-options)] 63 | (clojure.core/compile (symbol ~(name main-ns)))) 64 | (catch Throwable err# 65 | (println) 66 | (.printStackTrace err#) 67 | (System/exit 1))) 68 | _ (misc/log :info "Compiling %s..." main-ns)] 69 | (try 70 | (compile-java-sources classpath source-paths target-dir) 71 | (sh/clojure classpath 72 | "--eval" 73 | (pr-str forms))))) 74 | 75 | (defn- find-namespaces 76 | "Returns all namespaces declared within the file (either a directory 77 | or a jar file)." 78 | [^File file] 79 | (cond 80 | (.isDirectory file) (namespace.find/find-namespaces-in-dir file) 81 | (java.classpath/jar-file? file) (namespace.find/find-namespaces-in-jarfile (JarFile. file)))) 82 | 83 | (defn- find-namespaces-on-classpath 84 | "Given a sequence of files on the classpath, returns a map from 85 | namespace symbols to their sources (either directories or jar 86 | files)." 87 | [classpath-files] 88 | (->> classpath-files 89 | (mapcat #(map vector (find-namespaces %) (repeat %))) 90 | (remove (comp nil? first)) 91 | (into {}))) 92 | 93 | (defn compile 94 | "Compiles the ns symbol (that must have a gen-class directive) into a 95 | set of .class files. 96 | 97 | Returns a map of compiled class files (as instances of 98 | java.io.File) to their sources (instances of java.io.File as well 99 | representing directories or jar files on the classpath)." 100 | [classpath-files ^Symbol main-class source-paths ^File target-dir compiler-options] 101 | (let [namespaces (find-namespaces-on-classpath classpath-files) 102 | classes-dir (misc/make-dir target-dir "classes") 103 | classpath (cons classes-dir classpath-files) 104 | _ (do-compile main-class classpath source-paths classes-dir compiler-options)] 105 | (reduce (fn [result ^File class-file] 106 | (let [source-file (or (get-class-file-source namespaces (misc/relativize class-file classes-dir)) 107 | ;; Defaults to the first element of source-paths if the class file doesn't match any known source. 108 | (first source-paths))] 109 | (assoc result class-file source-file))) 110 | {} 111 | (misc/filter-files (file-seq classes-dir))))) 112 | 113 | (defn copy-libs 114 | "Copies third-party libraries to the lib directory under target-dir. 115 | 116 | Returns a sequence of the copied libraries as java.io.File objects." 117 | [libs ^File target-dir] 118 | (let [lib-dir (misc/make-dir target-dir "lib")] 119 | (mapv (fn [^File lib] 120 | (let [^File dest-file (io/file lib-dir (.getName lib))] 121 | (io/copy lib dest-file) 122 | (misc/set-timestamp dest-file (.lastModified dest-file)) 123 | dest-file)) 124 | libs))) 125 | 126 | (defn- copy-or-merge-files 127 | "Copies or merges the provided files, using the merge set. Returns a seq of the files 128 | that are copied." 129 | [classpath-root merge-set files] 130 | (let [f (fn [result file] 131 | (let [{:keys [target-file input-stream last-modified input-source]} file 132 | input-stream' ^InputStream (input-stream) 133 | merged? (merge/execute-merge-rules classpath-root 134 | input-source 135 | input-stream' 136 | target-file 137 | last-modified 138 | merge-set)] 139 | (if merged? 140 | result 141 | (do 142 | (io/make-parents target-file) 143 | (io/copy input-stream' target-file) 144 | (.close input-stream') 145 | (misc/set-timestamp target-file last-modified) 146 | (conj result target-file)))))] 147 | (reduce f [] files))) 148 | 149 | (defn- copy-files-from-jar 150 | "Copies Jar file resources to the target directory." 151 | [^File jar-root target-dir merge-set] 152 | (with-open [jar-file (JarFile. jar-root)] 153 | (->> jar-file 154 | .entries 155 | enumeration-seq 156 | (remove #(.isDirectory ^JarEntry %)) 157 | (map (fn [^JarEntry entry] 158 | {:target-file (io/file target-dir (.getName entry)) 159 | :input-source (str jar-root "#" entry) 160 | :input-stream #(.getInputStream jar-file entry) 161 | :last-modified (.getTime entry)})) 162 | (copy-or-merge-files jar-root merge-set)))) 163 | 164 | (defn- copy-files-from-dir 165 | "Copies file system files (typically resources) from dir-root to target-dir." 166 | [file-system-root target-dir merge-set] 167 | (->> file-system-root 168 | file-seq 169 | misc/filter-files 170 | (map (fn [^File file] 171 | {:target-file (io/file target-dir (misc/relativize file file-system-root)) 172 | :input-source (str file) 173 | :input-stream #(io/input-stream file) 174 | :last-modified (.lastModified file)})) 175 | (copy-or-merge-files file-system-root merge-set))) 176 | 177 | (defn- copy-files* 178 | "Copies files from a classpath root to the target directory. 179 | 180 | Returns a tuple of [copied-files merged-paths]; copied files are files in the target 181 | directory, merged-paths is a map of data for any files that are require merge logic." 182 | [classpath-root target-dir merge-set] 183 | (let [f (if (.isDirectory classpath-root) 184 | copy-files-from-dir 185 | copy-files-from-jar)] 186 | (f classpath-root target-dir merge-set))) 187 | 188 | (defn copy-files 189 | "Iterates over the classpath roots (source directories or library jars) and copies 190 | all files within to the target directory (on the file system). 191 | 192 | Certain kinds of files (such as `data_readers.clj`) may occur multiple times, and 193 | must be merged. 194 | 195 | Output files have the same last modified time as source files. 196 | 197 | Returns a map from target files (a File) to their source classpath root." 198 | [classpath-roots target-dir] 199 | (let [merge-set (merge/new-merge-set) 200 | classes-dir (misc/make-dir target-dir "classes") 201 | f (fn [result classpath-root] 202 | (->> (copy-files* classpath-root classes-dir merge-set) 203 | (reduce #(assoc %1 %2 classpath-root) result))) 204 | copy-mappings (reduce f {} classpath-roots) 205 | merge-mappings (merge/write-merged-paths merge-set)] 206 | (merge copy-mappings merge-mappings))) 207 | 208 | (defn build-app 209 | "Builds the Clojure application. 210 | 211 | Performs an ahead-of-time (AOT) compilation of the ns 212 | symbol (:main-class) into a set of class files. The compiled classes 213 | will be written at :target-dir/WEB-INF/classes along with any 214 | resource files (including .clj or .cljc ones) found in the 215 | respective jars declared on the classpath (:classpath-files). Other 216 | libraries present on the classpath will by copied to 217 | :target-dir/WEB-INF/lib. 218 | 219 | The options map expects the following keys: 220 | 221 | * :classpath-files (seq of java.io.File objects) representing the 222 | classpath of the Clojure application; 223 | * :main-class (symbol) application entrypoint containing 224 | a :gen-class directive and a -main function; 225 | * :resource-paths (set of java.io.File objects) a set of paths 226 | containing resource files of the Clojure application; 227 | * :target-dir (java.io.File) an existing directory where the 228 | application's files will be written to. 229 | 230 | And it accepts the following optional keys: 231 | 232 | * :compiler-options (map) options for the Clojure compiler, refer 233 | to `clojure.core/*compiler-options*` for the supported keys. 234 | 235 | Returns a map containing the following namespaced keys: 236 | * :app/classes - a map from target files (compiled class files and 237 | resources) to their source classpath root (either directories or jar files present 238 | on the classpath); 239 | * :app/lib - a sequence of java.io.File objects containing libraries 240 | that the application depends on." 241 | [{:keys [classpath-files ^Symbol main-class resource-paths source-paths ^File target-dir compiler-options]}] 242 | {:pre [classpath-files main-class source-paths target-dir]} 243 | (let [web-inf (misc/make-dir target-dir "WEB-INF") 244 | classes (compile classpath-files main-class source-paths web-inf compiler-options) 245 | dirs+jar-files (set/union resource-paths (set (vals classes))) 246 | libs (misc/filter-files (set/difference (set classpath-files) dirs+jar-files)) 247 | resource-files (copy-files dirs+jar-files web-inf)] 248 | #:app{:classes (merge classes resource-files) 249 | :lib (copy-libs libs web-inf)})) 250 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /test/unit/vessel/image_test.clj: -------------------------------------------------------------------------------- 1 | (ns vessel.image-test 2 | (:require [clojure.java.io :as io] 3 | [clojure.test :refer :all] 4 | [matcher-combinators.test :refer [match?]] 5 | [mockfn.macros :refer [providing]] 6 | [vessel.image :as image] 7 | [vessel.misc :as misc]) 8 | (:import java.io.File)) 9 | 10 | (deftest internal-dep?-test 11 | (let [m2 (io/file "/home/builder/.m2/repository") 12 | common-core (io/file m2 "common-core/common-core/1.0.0/common-core-1.0.0.jar") 13 | k8s-clj (io/file m2 "nubank/kubernetes-clj/5.0.0/kubernetes-clj-5.0.0.jar")] 14 | (are [predicate file regexes] (predicate (image/internal-dep? file {:internal-deps-re regexes})) 15 | false? common-core nil 16 | false? common-core #{} 17 | true? common-core #{#"common-.*\.jar$"} 18 | true? common-core #{#"common-.*\.jar$" #"nubank"} 19 | true? k8s-clj #{#"common-.*\.jar$" #"nubank"}))) 20 | 21 | (deftest resource?-test 22 | (let [resources (io/file "/home/builder/my-app/resources") 23 | more-resources (io/file "/home/builder/my-app/src/clj/resources")] 24 | (are [predicate file resource-paths] (predicate (image/resource? file {:resource-paths resource-paths})) 25 | false? (io/file resources "resource1.edn") nil 26 | false? (io/file resources "resource1.edn") #{} 27 | true? (io/file resources "resource1.edn") #{resources} 28 | false? (io/file more-resources "resource2.edn") #{resources} 29 | true? (io/file more-resources "resource2.edn") #{resources more-resources}))) 30 | 31 | (deftest source-file?-test 32 | (let [sources (io/file "/home/builder/my-app/src") 33 | more-sources (io/file "/home/builder/my-app/clj/src")] 34 | (are [predicate file source-paths] (predicate (image/source-file? file {:source-paths source-paths})) 35 | false? (io/file sources "main_app/server.clj") nil 36 | false? (io/file sources "my_app/server.clj") #{} 37 | true? (io/file sources "my_app/server.clj") #{sources} 38 | false? (io/file more-sources "my_app/misc.clj") #{sources} 39 | true? (io/file more-sources "my_app/misc.clj") #{sources more-sources}))) 40 | 41 | (deftest render-image-spec-test 42 | (let [target-dir (io/file "/home/builder/projects/my-app/.vessel") 43 | web-inf-dir (io/file target-dir "WEB-INF") 44 | project-dir (io/file "/home/builder/projects/my-app") 45 | m2-dir (io/file "/home/builder/.m2/repository") 46 | app #:app {:classes 47 | {(io/file web-inf-dir "classes/my_app/server.class") (io/file project-dir "src") 48 | (io/file web-inf-dir "classes/config.edn") (io/file project-dir "resources") 49 | (io/file web-inf-dir "classes/my_company/core__init.class") (io/file m2-dir "my-company/my-company/1.0.0/my-company-1.0.0.jar")} 50 | :lib [(io/file web-inf-dir "lib/aws-java-sdk-1.11.602.jar")]} 51 | options {:app-root (io/file "/opt/app") 52 | :internal-deps-re #{#"my-company.*\.jar$"} 53 | :resource-paths #{(io/file project-dir "resources")} 54 | :source-paths [(io/file project-dir "src")] 55 | :manifest 56 | {:base-image 57 | {:image {:repository "jetty" :registry "my-company.com" :tag "v1"}} 58 | :image {:repository "my-app" :registry "my-company.com" :tag "v2"}} 59 | :tarball (io/file "my-app.tar") 60 | :user "jetty" 61 | :target-dir target-dir} 62 | modification-time (misc/now)] 63 | (providing [(misc/last-modified-time (partial instance? File)) modification-time] 64 | 65 | (testing "takes an options map and returns a map representing an image spec 66 | for the files in question" 67 | (is (= #:image{:from 68 | #:image {:registry "my-company.com" :repository "jetty" :tag "v1"} 69 | :user "jetty" 70 | :name 71 | #:image {:registry "my-company.com" :repository "my-app" :tag "v2"} 72 | :layers 73 | [#:image.layer{:name "external-deps" 74 | :entries 75 | [#:layer.entry{:source "/home/builder/projects/my-app/.vessel/WEB-INF/lib/aws-java-sdk-1.11.602.jar" 76 | :target "/opt/app/WEB-INF/lib/aws-java-sdk-1.11.602.jar" 77 | :modification-time modification-time}] 78 | :churn 1} 79 | #:image.layer{:name "internal-deps" 80 | :entries 81 | [#:layer.entry{:source "/home/builder/projects/my-app/.vessel/WEB-INF/classes/my_company/core__init.class" 82 | :target "/opt/app/WEB-INF/classes/my_company/core__init.class" 83 | :modification-time modification-time}] 84 | :churn 3} 85 | #:image.layer{:name "resources" 86 | :entries 87 | [#:layer.entry{:source "/home/builder/projects/my-app/.vessel/WEB-INF/classes/config.edn" 88 | :target "/opt/app/WEB-INF/classes/config.edn" 89 | :modification-time modification-time}] 90 | :churn 5} 91 | #:image.layer{:name "sources" 92 | :entries 93 | [#:layer.entry{:source "/home/builder/projects/my-app/.vessel/WEB-INF/classes/my_app/server.class" 94 | :target "/opt/app/WEB-INF/classes/my_app/server.class" 95 | :modification-time modification-time}] 96 | :churn 7}] 97 | :tar-path "my-app.tar"} 98 | (image/render-image-spec app options)))) 99 | 100 | (testing "accepts a set of extra-paths that will beget extra layers" 101 | (is (= [#:image.layer{:name "extra-files-1" 102 | :entries [#:layer.entry{:source "/home/builder/logback.xml" 103 | :target "/opt/app/WEB-INF/classes/logback.xml" 104 | :modification-time modification-time} 105 | #:layer.entry{:source "/home/builder/web.xml" 106 | :target "/opt/app/web.xml" 107 | :modification-time modification-time}] 108 | :churn 0} 109 | #:image.layer{:name "external-deps" 110 | :entries 111 | [#:layer.entry{:source "/home/builder/projects/my-app/.vessel/WEB-INF/lib/aws-java-sdk-1.11.602.jar" 112 | :target "/opt/app/WEB-INF/lib/aws-java-sdk-1.11.602.jar" 113 | :modification-time modification-time}] 114 | :churn 1} 115 | #:image.layer{:name "internal-deps" 116 | :entries 117 | [#:layer.entry{:source "/home/builder/projects/my-app/.vessel/WEB-INF/classes/my_company/core__init.class" 118 | :target "/opt/app/WEB-INF/classes/my_company/core__init.class" 119 | :modification-time modification-time}] 120 | :churn 3} 121 | #:image.layer{:name "resources" 122 | :entries 123 | [#:layer.entry{:source "/home/builder/projects/my-app/.vessel/WEB-INF/classes/config.edn" 124 | :target "/opt/app/WEB-INF/classes/config.edn" 125 | :modification-time modification-time}] 126 | :churn 5} 127 | #:image.layer{:name "sources" 128 | :entries 129 | [#:layer.entry{:source "/home/builder/projects/my-app/.vessel/WEB-INF/classes/my_app/server.class" 130 | :target "/opt/app/WEB-INF/classes/my_app/server.class" 131 | :modification-time modification-time}] 132 | :churn 7} 133 | #:image.layer{:name "extra-files-2" 134 | :entries 135 | [#:layer.entry{:source "/home/builder/context.xml" 136 | :target "/opt/app/context.xml" 137 | :modification-time modification-time}] 138 | :churn 8}] 139 | (:image/layers (image/render-image-spec app (assoc options :extra-paths 140 | #{{:source (io/file "/home/builder/logback.xml") 141 | :target (io/file "/opt/app/WEB-INF/classes/logback.xml") 142 | :churn 0} 143 | {:source (io/file "/home/builder/web.xml") 144 | :target (io/file "/opt/app/web.xml") 145 | :churn 0} 146 | {:source (io/file "/home/builder/context.xml") 147 | :target (io/file "/opt/app/context.xml") 148 | :churn 8}})))))) 149 | 150 | (testing "when the key :preserve-file-permissions? is set to true, assoc's 151 | the original posix file permissions of the files that will be copied to the 152 | container" 153 | (let [file-permissions #{"OTHERS_READ" 154 | "OWNER_WRITE" 155 | "OWNER_READ" 156 | "GROUP_READ"}] 157 | (providing [(misc/posix-file-permissions #(instance? File %)) file-permissions] 158 | (is (match? #:image{:layers 159 | [#:image.layer{:name "extra-files-1" 160 | :entries [#:layer.entry{:source "/home/builder/run-app.sh" 161 | :target "/usr/local/bin/run-app" 162 | :file-permissions file-permissions}] 163 | :churn 0} 164 | #:image.layer{:name "external-deps" 165 | :entries 166 | [#:layer.entry{:source "/home/builder/projects/my-app/.vessel/WEB-INF/lib/aws-java-sdk-1.11.602.jar" 167 | :target "/opt/app/WEB-INF/lib/aws-java-sdk-1.11.602.jar" 168 | :file-permissions file-permissions}] 169 | :churn 1} 170 | #:image.layer{:name "internal-deps" 171 | :entries 172 | [#:layer.entry{:source "/home/builder/projects/my-app/.vessel/WEB-INF/classes/my_company/core__init.class" 173 | :target "/opt/app/WEB-INF/classes/my_company/core__init.class" 174 | :file-permissions file-permissions}] 175 | :churn 3} 176 | #:image.layer{:name "resources" 177 | :entries 178 | [#:layer.entry{:source "/home/builder/projects/my-app/.vessel/WEB-INF/classes/config.edn" 179 | :target "/opt/app/WEB-INF/classes/config.edn" 180 | :file-permissions file-permissions}] 181 | :churn 5} 182 | #:image.layer{:name "sources" 183 | :entries 184 | [#:layer.entry{:source "/home/builder/projects/my-app/.vessel/WEB-INF/classes/my_app/server.class" 185 | :target "/opt/app/WEB-INF/classes/my_app/server.class" 186 | :file-permissions file-permissions}] 187 | :churn 7}]} 188 | (image/render-image-spec app (assoc options :extra-paths 189 | #{{:source (io/file "/home/builder/run-app.sh") 190 | :target (io/file "/usr/local/bin/run-app") 191 | :churn 0}} 192 | :preserve-file-permissions? true)))))))))) 193 | --------------------------------------------------------------------------------