Combine tools.deps and Leiningen images but still provide them under separate tags

Author: alexander-yakushev

src/docker_clojure/config.clj

@@ -22,11 +22,15 @@
 (s/def ::distro qualified-keyword?)
 (s/def ::distros (s/coll-of ::distro :distinct true :into #{}))
 
-(s/def ::build-tool (s/or ::specific-tool ::non-blank-string
-                          ::all-tools #(= ::core/all %)))
+(s/def ::build-tool keyword?)
 (s/def ::build-tool-version
   (s/nilable (s/and ::non-blank-string #(re-matches #"[\d\.]+" %))))
-(s/def ::build-tools (s/map-of ::build-tool ::build-tool-version))
+(s/def ::installer-hash ::non-blank-string)
+(s/def ::version ::build-tool-version)
+(s/def ::build-tools (s/map-of ::build-tool
+                               (s/keys :req-un [::version ::installer-hash])))
+
+
 
 (s/def ::maintainers
   (s/coll-of ::non-blank-string :distinct true :into #{}))
@@ -71,17 +75,13 @@
    17       :ubuntu/jammy
    :default :debian/bookworm})
 
-(def build-tools
-  {"lein"       "2.11.2"
-   "tools-deps" "1.11.1.1435"})
-
-(def default-build-tool "tools-deps")
+(def ^:dynamic *build-tools*
+  {:lein       {:version "2.11.2"
+                :installer-hash "28a1a62668c5f427b413a8677e376affaa995f023b1fcd06e2d4c98ac1df5f3e"}
+   :tools-deps {:version "1.11.1.1435"
+                :installer-hash "7edee5b12197a2dbe6338e672b109b18164cde84bea1f049ceceed41fc4dd10a"}})
 
-(def installer-hashes
-  {"lein"       {"2.11.1" "03b3fbf7e6fac262f88f843a87b712a2b37f39cffc4f4f384436a30d8b01d6e4"
-                 "2.11.2" "28a1a62668c5f427b413a8677e376affaa995f023b1fcd06e2d4c98ac1df5f3e"}
-   "tools-deps" {"1.11.1.1429" "bf08cfeb007118b7277aa7423734f5d507604b868f7fc44c0f9929ca9cd94ed4"
-                 "1.11.1.1435" "7edee5b12197a2dbe6338e672b109b18164cde84bea1f049ceceed41fc4dd10a"}})
+(def default-build-tool :tools-deps)
 
 (def exclusions ; don't build these for whatever reason(s)
   #{; no more focal builds for JDK 20+

src/docker_clojure/core.clj

@@ -56,7 +56,7 @@
     m))
 
 (defn variant-map [[base-image jdk-version distro
-                    [build-tool build-tool-version]]]
+                    [build-tool build-tool-info]]]
   (let [variant-arch (get cfg/distro-architectures
                           (-> distro namespace keyword))
         base         {:jdk-version        jdk-version
@@ -65,34 +65,31 @@
                                                           jdk-version distro)
                       :distro             distro
                       :build-tool         build-tool
-                      :build-tool-version build-tool-version
+                      :build-tool-version (:version build-tool-info)
                       :maintainer         (str/join " & " cfg/maintainers)}]
-    (-> base
-        (assoc :docker-tag (default-docker-tag base))
-        (assoc-if #(nil? (:build-tool-version base)) :build-tool-versions
-                  cfg/build-tools)
-        (assoc-if #(seq variant-arch) :architectures variant-arch))))
+    (cond-> base
+      true         (assoc :docker-tag (default-docker-tag base))
+      variant-arch (assoc :architectures variant-arch))))
 
 (defn pull-image [image]
   (sh "docker" "pull" image))
 
-(defn generate-dockerfile! [installer-hashes variant]
+(defn generate-dockerfile! [variant]
   (let [build-dir (df/build-dir variant)
         filename  "Dockerfile"]
     (log "Generating" (str build-dir "/" filename))
-    (df/write-file build-dir filename installer-hashes variant)
+    (df/write-file build-dir filename variant)
     (assoc variant
-      :build-dir build-dir
-      :dockerfile filename)))
+           :build-dir build-dir
+           :dockerfile filename)))
 
-(defn build-image
-  [installer-hashes {:keys [docker-tag base-image architectures] :as variant}]
+(defn build-image [{:keys [docker-tag base-image architectures] :as variant}]
   (let [image-tag     (str "clojure:" docker-tag)
         _             (log "Pulling base image" base-image)
         _             (pull-image base-image)
 
         {:keys [dockerfile build-dir]}
-        (generate-dockerfile! installer-hashes variant)
+        (generate-dockerfile! variant)
 
         host-arch     (let [jvm-arch (System/getProperty "os.arch")]
                         (if (= "aarch64" jvm-arch)
@@ -126,33 +123,24 @@
 
 (defn image-variant-combinations
   [base-images jdk-versions distros build-tools]
-  (reduce
-   (fn [variants jdk-version]
-     (concat
-      variants
-      (let [jdk-base-images (get-or-default base-images jdk-version)]
-        (loop [[bi & r] jdk-base-images
-               acc #{}]
-          (let [vs   (combo/cartesian-product #{bi}
-                                              #{jdk-version}
-                                              (get-or-default distros bi)
-                                              build-tools)
-                acc' (concat acc vs)]
-            (if (seq r)
-              (recur r acc')
-              acc'))))))
-   #{} jdk-versions))
+  (mapcat
+   (fn [jdk-version]
+     (let [jdk-base-images (get-or-default base-images jdk-version)]
+       (mapcat #(combo/cartesian-product #{%}
+                                         #{jdk-version}
+                                         (get-or-default distros %)
+                                         build-tools)
+               jdk-base-images)))
+   jdk-versions))
 
 (defn image-variants
   [base-images jdk-versions distros build-tools]
-  (into #{}
+  (into []
         (comp
          (map variant-map)
          (remove #(= ::s/invalid (s/conform ::variant %))))
-        (conj
-         (image-variant-combinations base-images jdk-versions distros
-                                     build-tools)
-         latest-variant)))
+        (image-variant-combinations base-images jdk-versions distros
+                                    build-tools)))
 
 (defn rand-delay
   "Runs argument f w/ any supplied args after a random delay of 100-1000 ms"
@@ -162,27 +150,26 @@
     (apply f args)))
 
 (defn build-images
-  [parallelization installer-hashes variants]
+  [parallelization variants]
   (log "Building images" parallelization "at a time")
   (let [variants-ch   (to-chan! variants)
         builds-ch     (chan parallelization)]
     ;; Kick off builds with a random delay so we don't have Docker race
     ;; conditions (e.g. build container name collisions)
     (async/thread (pipeline-blocking parallelization builds-ch
-                                     (map (partial rand-delay build-image
-                                                   installer-hashes))
+                                     (map (partial rand-delay build-image))
                                      variants-ch))
     (while (<!! builds-ch))))
 
-(defn generate-dockerfiles! [installer-hashes variants]
+(defn generate-dockerfiles! [variants]
   (log "Generated" (count variants) "variants")
   (doseq [variant variants]
-    (generate-dockerfile! installer-hashes variant)))
+    (generate-dockerfile! variant)))
 
 (defn valid-variants []
   (remove (partial exclude? cfg/exclusions)
           (image-variants cfg/base-images cfg/jdk-versions cfg/distros
-                          cfg/build-tools)))
+                          cfg/*build-tools*)))
 
 (defn generate-manifest! [variants args]
   (let [git-head    (->> ["git" "rev-parse" "HEAD"] (apply sh) :out)
@@ -238,9 +225,9 @@
   (let [variants (generate-variants args)]
     (case cmd
       :clean (df/clean-all)
-      :dockerfiles (generate-dockerfiles! cfg/installer-hashes variants)
+      :dockerfiles (generate-dockerfiles! variants)
       :manifest (-> variants sort-variants (generate-manifest! args))
-      :build-images (build-images parallelization cfg/installer-hashes variants)))
+      :build-images (build-images parallelization variants)))
   (logger/stop))
 
 (defn -main

src/docker_clojure/dockerfile.clj

@@ -1,38 +1,17 @@
 (ns docker-clojure.dockerfile
   (:require
+   [clojure.java.io :as io]
    [clojure.java.shell :refer [sh]]
    [clojure.string :as str]
-   [docker-clojure.dockerfile.lein :as lein]
-   [docker-clojure.dockerfile.tools-deps :as tools-deps]
+   [docker-clojure.dockerfile.combined :as combined]
    [docker-clojure.dockerfile.shared :refer [copy-resource-file! entrypoint]]))
 
 (defn build-dir [{:keys [base-image-tag jdk-version build-tool]}]
-  (str/join "/" ["target"
-                 (str (str/replace base-image-tag ":" "-")
-                      (when-not (str/includes? base-image-tag (str jdk-version))
-                        (str "-" jdk-version)))
-                 (if (= :docker-clojure.core/all build-tool)
-                   "latest"
-                   build-tool)]))
-
-(defn all-prereqs [dir variant]
-  (tools-deps/prereqs dir variant))
-
-(defn all-contents [installer-hashes variant]
-  (concat
-    ["" "### INSTALL LEIN ###"]
-    (lein/install
-      installer-hashes
-      (assoc variant :build-tool-version
-             (get-in variant [:build-tool-versions "lein"])))
-    ["" "### INSTALL TOOLS-DEPS ###"]
-    (tools-deps/install
-      installer-hashes
-     (assoc variant :build-tool-version
-            (get-in variant [:build-tool-versions "tools-deps"])))
-    [""]
-    (entrypoint variant)
-    ["" "CMD [\"-M\", \"--repl\"]"]))
+  (io/file "target"
+           (str (str/replace base-image-tag ":" "-")
+                (when-not (str/includes? base-image-tag (str jdk-version))
+                  (str "-" jdk-version)))
+           (name build-tool)))
 
 (defn copy-java-from-temurin-contents
   [{:keys [jdk-version] :as _variant}]
@@ -41,43 +20,30 @@
    "ENV PATH=\"${JAVA_HOME}/bin:${PATH}\""
    ""])
 
-(defn contents [installer-hashes {:keys [build-tool distro] :as variant}]
-  (str/join "\n"
-            (concat [(format "FROM %s" (:base-image-tag variant))
-                     ""]
-                    (case (-> distro namespace keyword)
-                      (:debian :debian-slim) (copy-java-from-temurin-contents variant)
-                      [])
-                    (case build-tool
-                      :docker-clojure.core/all (all-contents installer-hashes variant)
-                      "lein" (lein/contents installer-hashes variant)
-                      "tools-deps" (tools-deps/contents installer-hashes variant)))))
+(defn contents [{:keys [distro base-image-tag] :as variant}]
+  (->> (concat [(format "FROM %s" base-image-tag)
+                ""]
+               (case (-> distro namespace keyword)
+                 (:debian :debian-slim) (copy-java-from-temurin-contents variant)
+                 [])
+               (combined/contents variant))
+       (str/join "\n")))
 
-(defn shared-prereqs [dir {:keys [build-tool]}]
+(defn do-prereqs [dir {:keys [build-tool] :as variant}]
   (let [entrypoint (case build-tool
-                     "tools-deps"             "clj"
-                     :docker-clojure.core/all "clj"
-                     build-tool)]
+                     :tools-deps "clj"
+                     :lein       "lein")]
     (copy-resource-file! dir "entrypoint"
                          #(str/replace % "@@entrypoint@@" entrypoint)
-                         #(.setExecutable % true false))))
-
-(defn do-prereqs [dir {:keys [build-tool] :as variant}]
-  (shared-prereqs dir variant)
-  (case build-tool
-    :docker-clojure.core/all (all-prereqs dir variant)
-    "lein" (lein/prereqs dir variant)
-    "tools-deps" (tools-deps/prereqs dir variant)))
-
-(defn write-file [dir file installer-hashes variant]
-  (let [{:keys [exit err]} (sh "mkdir" "-p" dir)]
-    (if (zero? exit)
-      (do
-        (do-prereqs dir variant)
-        (spit (str/join "/" [dir file])
-              (str (contents installer-hashes variant) "\n")))
-      (throw (ex-info (str "Error creating directory " dir)
-                      {:error err})))))
+                         #(.setExecutable % true false)))
+  (copy-resource-file! dir "rlwrap.retry" identity
+                       #(.setExecutable % true false)))
+
+(defn write-file [^java.io.File dir file variant]
+  (.mkdirs dir)
+  (do-prereqs dir variant)
+  (spit (io/file dir file)
+        (str (contents variant) "\n")))
 
 (defn clean-all []
   (sh "sh" "-c" "rm -rf target/*"))

src/docker_clojure/dockerfile/combined.clj

@@ -0,0 +1,43 @@
+(ns docker-clojure.dockerfile.combined
+  (:require
+   [docker-clojure.dockerfile.lein :as lein]
+   [docker-clojure.dockerfile.shared :as shared]
+   [docker-clojure.dockerfile.tools-deps :as tools-deps]))
+
+(def distro-deps
+  {:debian-slim {:build   #{"curl" "gnupg"}
+                 :runtime #{"rlwrap" "make" "git"}}
+   :debian      {:build   #{"curl" "gnupg"}
+                 :runtime #{"rlwrap" "make" "git"}}
+   :ubuntu      {:build   #{"gnupg"}
+                 ;; install curl as a runtime dep b/c we need it at build time
+                 ;; but upstream includes it so we don't want to uninstall it
+                 :runtime #{"rlwrap" "make" "git" "curl"}}
+   :alpine      {:build   #{"curl" "tar" "gnupg" "openssl" "ca-certificates"}
+                 :runtime #{"bash" "make" "git"}}})
+
+(defn install [variant]
+  (concat
+   (lein/env-preamble)
+   (tools-deps/env-preamble)
+   [""
+    "WORKDIR /tmp"
+    ""]
+   (shared/multiline-RUN
+    (concat (shared/install-distro-deps distro-deps variant)
+            (lein/installation-commands)
+            (tools-deps/installation-commands)
+            (shared/uninstall-distro-build-deps distro-deps variant)))
+   [""]
+   (lein/installation-postamble)
+   (tools-deps/installation-postamble)))
+
+(defn contents [{:keys [build-tool] :as variant}]
+  (concat (install variant)
+          [""]
+          (shared/entrypoint variant)
+          (case build-tool
+            :lein (lein/command variant)
+            :tools-deps (tools-deps/command variant))))
+
+#_(contents {:build-tool :lein, :build-tool-version "2.11.2", :jdk-version 17})