Merge branch 'docker-enhancements' into 'master'

fix: Avoid unnecessary re-work in docker build See merge request mythic-insight/legendary-doc-site!5
2021-06-10 21:01:10 +00:00 · 2021-06-10 21:01:10 +00:00 · 110c38c939
commit 110c38c939
parent b1038b1de9 a208fd7ebe
12 changed files with 122 additions and 55 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -15,9 +15,8 @@ variables:

 # Test stage. Runs various tests and speculatively builds docker image in
 #   parallel, in case the build passes.
-test:
+.test_template: &test_template
  stage: test
-  image: "elixir:1.10.4-alpine"
  cache:
    paths:
      - _build/
@ -26,6 +25,18 @@ test:
  - name: postgres:12
  script: script/cibuild

+test:
+  <<: *test_template
+  image: "elixir:1.10.4-alpine"
+
+test_1.11.4:
+  <<: *test_template
+  image: "elixir:1.11.4-alpine"
+
+test_1.12.1:
+  <<: *test_template
+  image: "elixir:1.12.1-alpine"
+
 build_image_for_commit:
  stage: test
  image: "docker:20.10"
@ -37,20 +48,11 @@ build_image_for_commit:
      - deps/
  services:
    - name: docker:20.10-dind
+  before_script:
+    - apk add git
+    - script/ci-docker-prebuild
  script:
-    - docker login "https://${CI_REGISTRY}" -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD
-    - docker pull $CI_REGISTRY_IMAGE:latest || true
-    # This enables fast parallel builds
-    - export DOCKER_BUILDKIT=1
-    - mkdir -p _build/prod
-    - docker build --cache-from $CI_REGISTRY_IMAGE:latest -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
-    # Push the commit SHA tagged version to registry. We will later choose to tag that as stable
-    #   if everything passes.
-    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
-    # Pull out the built _build/prod directory so we can cache it!
-    - CONTAINER_HANDLE=`docker create $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA`
-    - docker cp $CONTAINER_HANDLE:/root/app/_build/prod _build
-    - docker cp $CONTAINER_HANDLE:/root/app/deps deps
+    - script/ci-docker-build

 # If tests pass, tag the commit and update package versions
 deploy_to_tags:
@ -92,13 +94,16 @@ deploy_commit_image_to_tag:
    - name: docker:19.03.12-dind
  variables:
    GIT_STRATEGY: none # this job does not need the project files, only docker
+  parallel:
+    matrix:
+      - IMAGE_TYPE: ["", "-elixir-builder", "-asset-builder"]
  script:
    - echo "BUILD_VERSION is ${BUILD_VERSION}"
    - docker login "https://${CI_REGISTRY}" -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD
    # Run the docker pull, but retry if it doesn't work at first because layers may be
    # still be loading to storage
-    - for i in 1 2 3 4 5; do docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA && break || sleep 15; done
-    - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$BUILD_VERSION
-    - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest
-    - docker push $CI_REGISTRY_IMAGE:$BUILD_VERSION
-    - docker push $CI_REGISTRY_IMAGE:latest
+    - for i in 1 2 3 4 5; do docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA$IMAGE_TYPE && break || sleep 15; done
+    - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA$IMAGE_TYPE $CI_REGISTRY_IMAGE:$BUILD_VERSION$IMAGE_TYPE
+    - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA$IMAGE_TYPE $CI_REGISTRY_IMAGE:latest$IMAGE_TYPE
+    - docker push $CI_REGISTRY_IMAGE:$BUILD_VERSION$IMAGE_TYPE
+    - docker push $CI_REGISTRY_IMAGE:latest$IMAGE_TYPE
--- a/38
+++ b/38
@ -1,9 +1,4 @@
-FROM elixir:1.10.4-alpine AS elixir1
-
-RUN apk add make gcc libc-dev
-
-ENV CC=gcc
-ENV MIX_HOME=/opt/mix
+FROM elixir:1.10.4-alpine AS elixir-builder

 RUN mix local.hex --force \
  && mix local.rebar --force
@ -21,23 +16,18 @@ WORKDIR /root/app

 # We load these things one by one so that we can load the deps first and
 #   cache those layers, before we do the app build itself
-ADD ./config /root/app/config
-ADD ./mix.exs /root/app/
-ADD ./mix.lock /root/app/
-ADD ./apps/admin/mix.exs /root/app/apps/admin/
-ADD ./apps/app/mix.exs /root/app/apps/app/
-ADD ./apps/content/mix.exs /root/app/apps/content/
-ADD ./apps/core/mix.exs /root/app/apps/core/
-ADD ./_build/${MIX_ENV}/ /root/app/_build/${MIX_ENV}/
-ADD ./deps/ /root/app/deps/
-ADD ./script/ /root/app/script/
-RUN script/restore-timestamps
+ADD ./mix.exs ./mix.lock ./
+ADD ./config ./config
+ADD ./apps/admin/mix.exs ./apps/admin/
+ADD ./apps/app/mix.exs ./apps/app/
+ADD ./apps/content/mix.exs ./apps/content/
+ADD ./apps/core/mix.exs ./apps/core/
+
 RUN mix deps.get
-RUN mix deps.compile

 # Leave off here so that we can built assets and compile the elixir app in parallel

-FROM node:15.0
+FROM node:15.0 AS asset-builder

 # Build assets in a node container
 ADD ./apps/app/assets/ /root/app/apps/app/assets/
@ -47,7 +37,9 @@ COPY --from=0 /root/app/ /root/app/
 RUN npm install
 RUN npm run deploy

-FROM elixir1
+FROM elixir-builder
+
+RUN mix deps.compile

 ADD ./apps /root/app/apps

@ -55,11 +47,7 @@ ADD ./apps /root/app/apps
 RUN MAKE=cmake mix compile

 # Copy in the built assets & fingerprint them
-COPY --from=1 /root/app/apps/app/priv/static/ /root/app/apps/app/priv/static
+COPY --from=asset-builder /root/app/apps/app/priv/static/ /root/app/apps/app/priv/static
 RUN mix phx.digest

-RUN script/restore-timestamps
-
-RUN mkdir -p /root/app/priv/
-
 CMD elixir --name ${NAME:=legendary}@$(hostname -f) -S mix phx.server
--- a/README.md
+++ b/README.md
@ -55,3 +55,12 @@ The CI script will automatically tag successful builds. To do this, you will
 need to configure a [CI variable](-/settings/ci_cd) named `GITLAB_TOKEN`. This
 token should be a [personal access token](/-/profile/personal_access_tokens) with
 `read_repository, write_repository` permissions.
+
+
+# Support
+
+Want to support development? Chip in on buymeacoffee:
+
+<a href="https://www.buymeacoffee.com/prehnra" target="_blank">
+  <img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" style="height: 60px !important;width: 217px !important;" >
+</a>
--- a/apps/core/lib/i18n.ex
+++ b/apps/core/lib/i18n.ex
@ -12,6 +12,7 @@ defmodule Legendary.I18n do
      path
      |> Path.basename()
      |> Path.rootname()
+      |> String.to_atom()

    locale locale, path
  end)
--- a/infrastructure_templates/kube.yaml.dot
+++ b/infrastructure_templates/kube.yaml.dot
@ -78,6 +78,21 @@ spec:
                secretKeyRef:
                  name: legendary-doc-site
                  key: smtp-password
+            - name: SMTP_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: legendary
+                  key: smtp-host
+            - name: SMTP_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: legendary
+                  key: smtp-username
+            - name: SMTP_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: legendary
+                  key: smtp-password
 ---
 apiVersion: v1
 kind: Service
--- a/mix.lock
+++ b/mix.lock
@ -68,7 +68,7 @@
  "plug_cowboy": {:hex, :plug_cowboy, "2.5.0", "51c998f788c4e68fc9f947a5eba8c215fbb1d63a520f7604134cab0270ea6513", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5b2c8925a5e2587446f33810a58c01e66b3c345652eeec809b76ba007acde71a"},
  "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"},
  "postgrex": {:hex, :postgrex, "0.15.5", "aec40306a622d459b01bff890fa42f1430dac61593b122754144ad9033a2152f", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "ed90c81e1525f65a2ba2279dbcebf030d6d13328daa2f8088b9661eb9143af7f"},
-  "pow": {:hex, :pow, "1.0.20", "b99993811af5233681bfc521e81ca706d25a56f2be54bad6424db327ce840ab9", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.3.0 and < 1.6.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, ">= 2.0.0 and <= 3.0.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:plug, ">= 1.5.0 and < 2.0.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "4b6bd271399ccb353abbdbdc316199fe7fd7ae36bbf47059d53e366831c34fc8"},
+  "pow": {:hex, :pow, "1.0.24", "d834f5ce40c9b15a810a942de6492314e0defa157b6503e63775014b165a9c10", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.3.0 and < 1.6.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, ">= 2.0.0 and <= 3.0.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:plug, ">= 1.5.0 and < 2.0.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "70fe212f2b9a362a73e317cb33f506dbebd5d3bbfd3f64883898e77cc29ea9e5"},
  "quantum": {:hex, :quantum, "2.4.0", "f2ad4b20988f848455d35ed0e884ba0c7629a27ee86cbec6a6e0fc214b6e69cf", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12 or ~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}, {:tzdata, "~> 1.0", [hex: :tzdata, repo: "hexpm", optional: true]}], "hexpm", "a125a9e65a5af740a1198f3b05c1a736fce3942f5e0dc2901e0f9be5745bea99"},
  "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
  "sitemap": {:hex, :sitemap, "1.1.0", "23a019cccef7c17090d0b493354ee47a94549db64fd1cf39bda7eb41c567729c", [:mix], [{:xml_builder, ">= 0.0.0", [hex: :xml_builder, repo: "hexpm", optional: false]}], "hexpm", "d21f2c3ac65567fbdbe231f9faaf802a48405aa487d24052964d3d818a3d8c22"},
--- a/script/ci-docker-build
+++ b/script/ci-docker-build
@ -0,0 +1,36 @@
+#!/bin/sh
+
+export DOCKER_BUILDKIT=1
+
+docker build \
+  --target elixir-builder \
+  --cache-from $CI_REGISTRY_IMAGE:$PARENT_COMMIT-elixir-builder \
+  --cache-from $CI_REGISTRY_IMAGE:latest-elixir-builder \
+  --build-arg BUILDKIT_INLINE_CACHE=1 \
+  -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-elixir-builder .
+
+docker build \
+  --target asset-builder \
+  --cache-from $CI_REGISTRY_IMAGE:$PARENT_COMMIT-elixir-builder \
+  --cache-from $CI_REGISTRY_IMAGE:latest-elixir-builder \
+  --cache-from $CI_REGISTRY_IMAGE:$PARENT_COMMIT-asset-builder \
+  --cache-from $CI_REGISTRY_IMAGE:latest-asset-builder \
+  --build-arg BUILDKIT_INLINE_CACHE=1 \
+  -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-asset-builder .
+
+docker build \
+  --target asset-builder \
+  --cache-from $CI_REGISTRY_IMAGE:$PARENT_COMMIT-elixir-builder \
+  --cache-from $CI_REGISTRY_IMAGE:latest-elixir-builder \
+  --cache-from $CI_REGISTRY_IMAGE:$PARENT_COMMIT-asset-builder \
+  --cache-from $CI_REGISTRY_IMAGE:latest-asset-builder \
+  --cache-from $CI_REGISTRY_IMAGE:$PARENT_COMMIT \
+  --cache-from $CI_REGISTRY_IMAGE:latest \
+  --build-arg BUILDKIT_INLINE_CACHE=1 \
+  -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
+
+# Push the commit SHA tagged version to registry. We will later choose to tag that as stable
+#   if everything passes.
+docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-elixir-builder
+docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-asset-builder
+docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
--- a/script/ci-docker-prebuild
+++ b/script/ci-docker-prebuild
@ -0,0 +1,17 @@
+#!/bin/sh
+
+PARENT_COMMIT=`git rev-list --parents -n 1 HEAD  | cut -d' ' -f2`
+
+docker login "https://${CI_REGISTRY}" -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD
+
+docker pull $CI_REGISTRY_IMAGE:$PARENT_COMMIT-elixir-builder || \
+  docker pull $CI_REGISTRY_IMAGE:latest-elixir-builder || \
+  true
+
+docker pull $CI_REGISTRY_IMAGE:$PARENT_COMMIT-asset-builder || \
+  docker pull $CI_REGISTRY_IMAGE:latest-asset-builder || \
+  true
+
+docker pull $CI_REGISTRY_IMAGE:$PARENT_COMMIT || \
+  docker pull $CI_REGISTRY_IMAGE:latest || \
+  true
--- a/script/console
+++ b/script/console
@ -3,9 +3,8 @@
 set -x
 set -e

-FULL_PATH=$(realpath $0)
-DIR_PATH=$(dirname $FULL_PATH)
+DIR_PATH=$(dirname $0)

 $DIR_PATH/update

-iex -S mix
+iex -S mix
--- a/script/server
+++ b/script/server
@ -3,8 +3,7 @@
 set -x
 set -e

-FULL_PATH=$(realpath $0)
-DIR_PATH=$(dirname $FULL_PATH)
+DIR_PATH=$(dirname $0)

 $DIR_PATH/update

--- a/script/test
+++ b/script/test
@ -3,8 +3,7 @@
 set -x
 set -e

-FULL_PATH=$(realpath $0)
-DIR_PATH=$(dirname $FULL_PATH)
+DIR_PATH=$(dirname $0)

 $DIR_PATH/bootstrap

--- a/script/update
+++ b/script/update
@ -3,8 +3,7 @@
 set -x
 set -e

-FULL_PATH=$(realpath $0)
-DIR_PATH=$(dirname $FULL_PATH)
+DIR_PATH=$(dirname $0)

 $DIR_PATH/bootstrap