Merge branch 'ci-speedup' into 'master'

Ci speedup

See merge request mythic-insight/legendary!49

.gitlab-ci.yml

@@ -1,15 +1,10 @@
 stages:
+  - application_dependencies
+  - asset_dependencies
   - test
   - deploy_tags
   - deploy
 
-cache:
-  paths:
-    - _build/
-    - deps/
-    - node_modules/
-    - apps/app/assets/node_modules/
-
 variables:
   POSTGRES_PASSWORD: "postgres"
   POSTGRES_USER: "postgres"
@@ -17,35 +12,117 @@ variables:
   MIX_ENV: "test"
   DOCKER_TLS_CERTDIR: "/certs"
   DOCKER_HOST: tcp://docker:2376
+  # fetch & clean the repo rather than completely cloning (faster)
+  GIT_STRATEGY: fetch
 
+# Dependency stages-- fetch these first so that we can cache them and reuse them
+#   across jobs and pipelines. Note that elixir deps need to go first, because
+#   we need the phoenix and phoenix_html hex packages to install their JS.
+fetch_application_dependencies:
+  stage: application_dependencies
+  image: "elixir:1.10"
+  cache:
+    key:
+      files:
+        - mix.lock
+    paths:
+      - _build/
+      - deps/
+  script:
+    - mix local.hex --force
+    - mix local.rebar --force
+    - mix deps.get
+    - mix deps.compile
+  # Make results available to other jobs
+  artifacts:
+    paths:
+      - deps/phoenix
+      - deps/phoenix_html
+    exclude:
+      - deps/
+
+fetch_asset_dependencies:
+  stage: asset_dependencies
+  image: "node:15.0"
+  needs:
+    - fetch_application_dependencies
+  only:
+    - master
+  cache:
+    key:
+      files:
+        - apps/app/assets/package-lock.json
+    paths:
+      - apps/app/assets/node_modules/
+  script:
+    - cd apps/app/assets/ && npm install
+  # Make results available to other jobs
+  artifacts:
+    paths:
+      - apps/app/assets/node_modules
+    exclude:
+      - apps/app/assets/node_modules
+
+# Test stage. Runs various tests and speculatively builds docker image in
+#   parallel, in case the build passes.
 test:
   stage: test
-  needs: []
+  needs:
+    - fetch_application_dependencies
   image: "elixir:1.10"
   services:
   - name: postgres:12
+  cache:
+    key:
+      files:
+        - mix.lock
+    paths:
+      - _build/
+      - deps/
   script: script/cibuild
 
 build_image_for_commit:
   stage: test
-  needs: []
-  image: "docker:19.03.12"
+  needs:
+    - fetch_asset_dependencies
+    - fetch_application_dependencies
+  image: "docker:20.10"
   only:
     - master
   services:
-    - name: docker:19.03.12-dind
+    - name: docker:20.10-dind
+  cache:
+    key:
+      files:
+        - mix.lock
+    paths:
+      - _build/
+      - deps/
   script:
     - docker login "https://${CI_REGISTRY}" -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD
     - docker pull $CI_REGISTRY_IMAGE:latest || true
-    - docker build --cache-from $CI_REGISTRY_IMAGE:latest -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
+    # This enables fast parallel builds
+    - export DOCKER_BUILDKIT=1
+    - docker build --cache-from $CI_REGISTRY_IMAGE:latest -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA --build-arg BUILDKIT_INLINE_CACHE=1 .
+    # Push the commit SHA tagged version to registry. We will later choose to tag that as stable
+    #   if everything passes.
     - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+    # This copies the elixir build artifacts for deps and app so that we can cache them
+    - docker cp `docker create $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA`:/root/app/_build/prod _build
 
+# If tests pass, tag the commit and update package versions
 deploy_to_tags:
   stage: deploy_tags
   needs: ['test']
   image: "node:15.0"
   only:
     - master
+  cache:
+    key:
+      files:
+        - package-lock.json
+    paths:
+      - node_modules/
   script:
     - npm install
     - npx semantic-release --repository-url=$CI_REPOSITORY_URL
@@ -54,6 +131,8 @@ deploy_to_tags:
     reports:
       dotenv: build.env
 
+# If the tests passed, we take the image for this SHA and tag it with the version
+#   and latest, so we can signal that it is ready for prod
 deploy_commit_image_to_tag:
   stage: deploy
   needs:
@@ -65,6 +144,8 @@ deploy_commit_image_to_tag:
   image: "docker:19.03.12"
   services:
     - name: docker:19.03.12-dind
+  variables:
+    GIT_STRATEGY: none # this job does not need the project files, only docker
   script:
     - echo "BUILD_VERSION is ${BUILD_VERSION}"
     - docker login "https://${CI_REGISTRY}" -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD

Dockerfile

@@ -17,6 +17,8 @@ ENV PORT=4000
 
 WORKDIR /root/app
 
+# We load these things one by one so that we can load the deps first and
+#   cache those layers, before we do the app build itself
 ADD ./config /root/app/config
 ADD ./mix.exs /root/app/
 ADD ./mix.lock /root/app/
@@ -24,25 +26,32 @@ ADD ./apps/admin/mix.exs /root/app/apps/admin/
 ADD ./apps/app/mix.exs /root/app/apps/app/
 ADD ./apps/content/mix.exs /root/app/apps/content/
 ADD ./apps/core/mix.exs /root/app/apps/core/
+ADD ./deps /root/app/deps
+ADD ./_build /root/app/_build
 RUN mix deps.get
 RUN mix deps.compile
 
-ADD ./script /root/app/script
 ADD ./apps /root/app/apps
 
-RUN MAKE=cmake mix compile
+# Leave off here so that we can built assets and compile the elixir app in parallel
 
 FROM node:15.0
 
+# Build assets in a node container
 WORKDIR /root/app/apps/app/assets/
+ADD ./apps/app/assets/node_modules /root/app/apps/app/assets/node_modules
 COPY --from=0 /root/app/ /root/app/
 RUN npm install
 RUN npm run deploy
 
 FROM elixir1
 
-COPY --from=1 /root/app/apps/app/priv/static/ /root/app/apps/app/priv/static
+# Resume compilation of the elixir app
+ADD ./script /root/app/script
+RUN MAKE=cmake mix compile
 
+# Copy in the built assets & fingerprint them
+COPY --from=1 /root/app/apps/app/priv/static/ /root/app/apps/app/priv/static
 RUN mix phx.digest
 
 CMD ["mix", "phx.server"]

mix.lock

@@ -62,7 +62,7 @@
   "php_serializer": {:hex, :php_serializer, "0.9.2", "59c5fd6bd3096671fd89358fb8229341ac7423b50ad8d45a15213b02ea2edab2", [:mix], [], "hexpm", "34eb835a460944f7fc216773b363c02e7dcf8ac0390c9e9ccdbd92b31a7ca59a"},
   "plug": {:hex, :plug, "1.11.1", "f2992bac66fdae679453c9e86134a4201f6f43a687d8ff1cd1b2862d53c80259", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "23524e4fefbb587c11f0833b3910bfb414bf2e2534d61928e920f54e3a1b881f"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.4.1", "779ba386c0915027f22e14a48919a9545714f849505fa15af2631a0d298abf0f", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d72113b6dff7b37a7d9b2a5b68892808e3a9a752f2bf7e503240945385b70507"},
-  "plug_crypto": {:hex, :plug_crypto, "1.2.1", "5c854427528bf61d159855cedddffc0625e2228b5f30eff76d5a4de42d896ef4", [:mix], [], "hexpm", "6961c0e17febd9d0bfa89632d391d2545d2e0eb73768f5f50305a23961d8782c"},
+  "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"},
   "postgrex": {:hex, :postgrex, "0.15.5", "aec40306a622d459b01bff890fa42f1430dac61593b122754144ad9033a2152f", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "ed90c81e1525f65a2ba2279dbcebf030d6d13328daa2f8088b9661eb9143af7f"},
   "pow": {:hex, :pow, "1.0.20", "b99993811af5233681bfc521e81ca706d25a56f2be54bad6424db327ce840ab9", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.3.0 and < 1.6.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, ">= 2.0.0 and <= 3.0.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:plug, ">= 1.5.0 and < 2.0.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "4b6bd271399ccb353abbdbdc316199fe7fd7ae36bbf47059d53e366831c34fc8"},
   "quantum": {:hex, :quantum, "2.4.0", "f2ad4b20988f848455d35ed0e884ba0c7629a27ee86cbec6a6e0fc214b6e69cf", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12 or ~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}, {:tzdata, "~> 1.0", [hex: :tzdata, repo: "hexpm", optional: true]}], "hexpm", "a125a9e65a5af740a1198f3b05c1a736fce3942f5e0dc2901e0f9be5745bea99"},