Merge branch 'master' of gitlab.com:mythic-insight/legendary

.gitlab-ci.yml

@@ -1,15 +1,10 @@
 stages:
+  - application_dependencies
+  - asset_dependencies
   - test
   - deploy_tags
   - deploy
 
-cache:
-  paths:
-    - _build/
-    - deps/
-    - node_modules/
-    - apps/app/assets/node_modules/
-
 variables:
   POSTGRES_PASSWORD: "postgres"
   POSTGRES_USER: "postgres"
@@ -17,35 +12,117 @@ variables:
   MIX_ENV: "test"
   DOCKER_TLS_CERTDIR: "/certs"
   DOCKER_HOST: tcp://docker:2376
+  # fetch & clean the repo rather than completely cloning (faster)
+  GIT_STRATEGY: fetch
 
+# Dependency stages-- fetch these first so that we can cache them and reuse them
+#   across jobs and pipelines. Note that elixir deps need to go first, because
+#   we need the phoenix and phoenix_html hex packages to install their JS.
+fetch_application_dependencies:
+  stage: application_dependencies
+  image: "elixir:1.10"
+  cache:
+    key:
+      files:
+        - mix.lock
+    paths:
+      - _build/
+      - deps/
+  script:
+    - mix local.hex --force
+    - mix local.rebar --force
+    - mix deps.get
+    - mix deps.compile
+  # Make results available to other jobs
+  artifacts:
+    paths:
+      - deps/phoenix
+      - deps/phoenix_html
+    exclude:
+      - deps/
+
+fetch_asset_dependencies:
+  stage: asset_dependencies
+  image: "node:15.0"
+  needs:
+    - fetch_application_dependencies
+  only:
+    - master
+  cache:
+    key:
+      files:
+        - apps/app/assets/package-lock.json
+    paths:
+      - apps/app/assets/node_modules/
+  script:
+    - cd apps/app/assets/ && npm install
+  # Make results available to other jobs
+  artifacts:
+    paths:
+      - apps/app/assets/node_modules
+    exclude:
+      - apps/app/assets/node_modules
+
+# Test stage. Runs various tests and speculatively builds docker image in
+#   parallel, in case the build passes.
 test:
   stage: test
-  needs: []
+  needs:
+    - fetch_application_dependencies
   image: "elixir:1.10"
   services:
   - name: postgres:12
+  cache:
+    key:
+      files:
+        - mix.lock
+    paths:
+      - _build/
+      - deps/
   script: script/cibuild
 
 build_image_for_commit:
   stage: test
-  needs: []
+  needs:
-  image: "docker:19.03.12"
+    - fetch_asset_dependencies
+    - fetch_application_dependencies
+  image: "docker:20.10"
   only:
     - master
   services:
-    - name: docker:19.03.12-dind
+    - name: docker:20.10-dind
+  cache:
+    key:
+      files:
+        - mix.lock
+    paths:
+      - _build/
+      - deps/
   script:
     - docker login "https://${CI_REGISTRY}" -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD
     - docker pull $CI_REGISTRY_IMAGE:latest || true
-    - docker build --cache-from $CI_REGISTRY_IMAGE:latest -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
+    # This enables fast parallel builds
+    - export DOCKER_BUILDKIT=1
+    - docker build --cache-from $CI_REGISTRY_IMAGE:latest -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA --build-arg BUILDKIT_INLINE_CACHE=1 .
+    # Push the commit SHA tagged version to registry. We will later choose to tag that as stable
+    #   if everything passes.
     - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+    # This copies the elixir build artifacts for deps and app so that we can cache them
+    - docker cp `docker create $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA`:/root/app/_build/prod _build
 
+# If tests pass, tag the commit and update package versions
 deploy_to_tags:
   stage: deploy_tags
   needs: ['test']
   image: "node:15.0"
   only:
     - master
+  cache:
+    key:
+      files:
+        - package-lock.json
+    paths:
+      - node_modules/
   script:
     - npm install
     - npx semantic-release --repository-url=$CI_REPOSITORY_URL
@@ -54,6 +131,8 @@ deploy_to_tags:
     reports:
       dotenv: build.env
 
+# If the tests passed, we take the image for this SHA and tag it with the version
+#   and latest, so we can signal that it is ready for prod
 deploy_commit_image_to_tag:
   stage: deploy
   needs:
@@ -65,6 +144,8 @@ deploy_commit_image_to_tag:
   image: "docker:19.03.12"
   services:
     - name: docker:19.03.12-dind
+  variables:
+    GIT_STRATEGY: none # this job does not need the project files, only docker
   script:
     - echo "BUILD_VERSION is ${BUILD_VERSION}"
     - docker login "https://${CI_REGISTRY}" -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD

Dockerfile

@@ -17,6 +17,8 @@ ENV PORT=4000
 
 WORKDIR /root/app
 
+# We load these things one by one so that we can load the deps first and
+#   cache those layers, before we do the app build itself
 ADD ./config /root/app/config
 ADD ./mix.exs /root/app/
 ADD ./mix.lock /root/app/
@@ -24,25 +26,32 @@ ADD ./apps/admin/mix.exs /root/app/apps/admin/
 ADD ./apps/app/mix.exs /root/app/apps/app/
 ADD ./apps/content/mix.exs /root/app/apps/content/
 ADD ./apps/core/mix.exs /root/app/apps/core/
+ADD ./deps /root/app/deps
+ADD ./_build /root/app/_build
 RUN mix deps.get
 RUN mix deps.compile
 
-ADD ./script /root/app/script
 ADD ./apps /root/app/apps
 
-RUN MAKE=cmake mix compile
+# Leave off here so that we can built assets and compile the elixir app in parallel
 
 FROM node:15.0
 
+# Build assets in a node container
 WORKDIR /root/app/apps/app/assets/
+ADD ./apps/app/assets/node_modules /root/app/apps/app/assets/node_modules
 COPY --from=0 /root/app/ /root/app/
 RUN npm install
 RUN npm run deploy
 
 FROM elixir1
 
-COPY --from=1 /root/app/apps/app/priv/static/ /root/app/apps/app/priv/static
+# Resume compilation of the elixir app
+ADD ./script /root/app/script
+RUN MAKE=cmake mix compile
 
+# Copy in the built assets & fingerprint them
+COPY --from=1 /root/app/apps/app/priv/static/ /root/app/apps/app/priv/static
 RUN mix phx.digest
 
 CMD ["mix", "phx.server"]

apps/admin/mix.exs

@@ -1,7 +1,7 @@
 defmodule Legendary.Admin.MixProject do
   use Mix.Project
 
-  @version "2.1.3"
+  @version "2.3.0"
 
   def project do
     [

apps/content/lib/content/sitemap.ex

@@ -36,7 +36,7 @@ defmodule Legendary.Content.Sitemaps do
         if page_count > 1 do
           (2..page_count)
           |> Enum.each(fn page ->
-            add Helpers.paged_post_path(Endpoint, :show, post, page), priority: 0.5, changefreq: "hourly", expires: nil
+            add Helpers.posts_path(Endpoint, :show, post, page: page), priority: 0.5, changefreq: "hourly", expires: nil
           end)
         end
       end

apps/content/lib/content_web/controllers/posts_controller.ex

@@ -58,7 +58,7 @@ defmodule Legendary.Content.PostsController do
     conn |> index_posts(%{"id" => "blog", "page" => page_string})
   end
 
-  def show(conn, %{"id" => id, "page" => page_string}) do
+  def show(conn, %{"id" => id, "page" => page_string}) when is_binary(id) or is_integer(id) do
     post = Posts.get_post(id)
 
     if is_nil(post) do
@@ -67,6 +67,9 @@ defmodule Legendary.Content.PostsController do
       conn |> show_one(post, page_string)
     end
   end
+  def show(conn, %{"id" => id, "page" => page_string}) when is_list(id) do
+    show(conn, %{"id" => Enum.join(id, "/"), "page" => page_string})
+  end
   def show(conn, %{"id" => id}), do: show(conn, %{"id" => id, "page" => "1"})
 
   defp try_static_post(conn, id) do

apps/content/lib/content_web/routes.ex

@@ -33,7 +33,7 @@ defmodule Legendary.Content.Routes do
         get "/", PostsController, :index
         resources "/sitemap", SitemapController, only: [:index]
         get "/:id", PostsController, :show
-        get "/:id/:page", PostsController, :show, as: :paged_post
+        get "/*id", PostsController, :show, as: :nested_posts
       end
     end
   end

apps/content/lib/content_web/templates/posts/pagination.html.eex

@@ -5,7 +5,7 @@
         1..Legendary.Content.Post.content_page_count(@post),
         fn page ->
           if assigns[:current_page] == nil || assigns[:current_page] != page do
-            link page, to: Routes.paged_post_path(@conn, :show, @post, page)
+            link page, to: Routes.posts_path(@conn, :show, @post, page: page)
           else
             content_tag :span, page, class: "paginator-page"
           end

apps/content/mix.exs

@@ -1,7 +1,7 @@
 defmodule Legendary.Content.MixProject do
   use Mix.Project
 
-  @version "2.1.3"
+  @version "2.3.0"
 
   def project do
     [

apps/content/test/content_web/controllers/posts_controller_test.exs

@@ -154,7 +154,7 @@ defmodule Legendary.Content.PostsControllerTest do
     end
 
     test "shows the post with pagination", %{conn: conn, posts: posts} do
-      conn = get conn, Routes.paged_post_path(conn, :show, posts, "2")
+      conn = get conn, Routes.posts_path(conn, :show, posts, page: "2")
 
       assert html_response(conn, 200) =~ posts.title
     end

apps/core/mix.exs

@@ -1,7 +1,7 @@
 defmodule Legendary.Core.MixProject do
   use Mix.Project
 
-  @version "2.1.3"
+  @version "2.3.0"
 
   def project do
     [

infrastructure/kube.yaml

@@ -20,7 +20,7 @@ spec:
     spec:
       containers:
         - name: app
-          image: registry.gitlab.com/mythic-insight/legendary:2.1.3
+          image: registry.gitlab.com/mythic-insight/legendary:2.3.0
           ports:
             - containerPort: 4000
           env:

mix.exs

@@ -1,7 +1,7 @@
 defmodule Legendary.Mixfile do
   use Mix.Project
 
-  @version "2.1.3"
+  @version "2.3.0"
 
   def project do
     [

mix.lock

@@ -62,7 +62,7 @@
   "php_serializer": {:hex, :php_serializer, "0.9.2", "59c5fd6bd3096671fd89358fb8229341ac7423b50ad8d45a15213b02ea2edab2", [:mix], [], "hexpm", "34eb835a460944f7fc216773b363c02e7dcf8ac0390c9e9ccdbd92b31a7ca59a"},
   "plug": {:hex, :plug, "1.11.1", "f2992bac66fdae679453c9e86134a4201f6f43a687d8ff1cd1b2862d53c80259", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "23524e4fefbb587c11f0833b3910bfb414bf2e2534d61928e920f54e3a1b881f"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.4.1", "779ba386c0915027f22e14a48919a9545714f849505fa15af2631a0d298abf0f", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d72113b6dff7b37a7d9b2a5b68892808e3a9a752f2bf7e503240945385b70507"},
-  "plug_crypto": {:hex, :plug_crypto, "1.2.1", "5c854427528bf61d159855cedddffc0625e2228b5f30eff76d5a4de42d896ef4", [:mix], [], "hexpm", "6961c0e17febd9d0bfa89632d391d2545d2e0eb73768f5f50305a23961d8782c"},
+  "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"},
   "postgrex": {:hex, :postgrex, "0.15.5", "aec40306a622d459b01bff890fa42f1430dac61593b122754144ad9033a2152f", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "ed90c81e1525f65a2ba2279dbcebf030d6d13328daa2f8088b9661eb9143af7f"},
   "pow": {:hex, :pow, "1.0.20", "b99993811af5233681bfc521e81ca706d25a56f2be54bad6424db327ce840ab9", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.3.0 and < 1.6.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, ">= 2.0.0 and <= 3.0.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:plug, ">= 1.5.0 and < 2.0.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "4b6bd271399ccb353abbdbdc316199fe7fd7ae36bbf47059d53e366831c34fc8"},
   "quantum": {:hex, :quantum, "2.4.0", "f2ad4b20988f848455d35ed0e884ba0c7629a27ee86cbec6a6e0fc214b6e69cf", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12 or ~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}, {:tzdata, "~> 1.0", [hex: :tzdata, repo: "hexpm", optional: true]}], "hexpm", "a125a9e65a5af740a1198f3b05c1a736fce3942f5e0dc2901e0f9be5745bea99"},

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mythic-insight/legendary",
-  "version": "2.1.3",
+  "version": "2.3.0",
   "private": true,
   "description": "The Legendary Phoenix Boilerplate.",
   "main": "index.js",