fix: Use fewer db connections

README.md

@@ -1,5 +1,23 @@
 # Legendary
 
+# Up and Running
+
+In order to start the server, run `script/server`. Any dependencies required
+will be installed automatically using [brew](https://brew.sh/),
+[asdf](https://asdf-vm.com/#/), and [hex](https://hex.pm/).
+
+Now you can visit [`localhost:4000`](http://localhost:4000) from your browser.
+
+# CI Configuration
+
+Legendary comes with gitlab CI settings which should work for you with minimal
+setup.
+
+The CI script will automatically tag successful builds. To do this, you will
+need to configure a [CI variable](-/settings/ci_cd) named `GITLAB_TOKEN`. This
+token should be a [personal access token](/-/profile/personal_access_tokens) with
+`read_repository, write_repository` permissions.
+
 To start your Phoenix server:
 
   * Install dependencies with `mix deps.get`
@@ -7,11 +25,7 @@ To start your Phoenix server:
   * Install Node.js dependencies with `npm install` inside the `assets` directory
   * Start Phoenix endpoint with `mix phx.server`
 
-Now you can visit [`localhost:4000`](http://localhost:4000) from your browser.
-
-Ready to run in production? Please [check our deployment guides](https://hexdocs.pm/phoenix/deployment.html).
-
-## Learn more
+# Learn more
 
   * Official website: https://www.phoenixframework.org/
   * Guides: https://hexdocs.pm/phoenix/overview.html

apps/core/lib/core/shared_db_connection_pool.ex

@@ -0,0 +1,46 @@
+defmodule Core.SharedDBConnectionPool do
+  @moduledoc """
+  A shareable connection pool. We use this so that all the apps connecting to
+  one database can use on connection pool, even if they have different repos.
+
+  This allows a reasonable number of connections to be available per application
+  without requiring a huge number of connections to the database.
+  """
+  alias DBConnection.ConnectionPool
+
+  def start_link({mod, opts}) do
+    case GenServer.start_link(ConnectionPool, {mod, opts}, Keyword.take(opts, [:name, :spawn_opt])) do
+      {:ok, pid} -> {:ok, pid}
+      {:error, {:already_started, pid}} -> {:ok, pid}
+      error -> error
+    end
+  end
+
+  def child_spec({mod, opts}) do
+    opts = Keyword.put_new(opts, :name, key(opts))
+    Supervisor.Spec.worker(Core.SharedDBConnectionPool, [{mod, opts}])
+  end
+
+  defp key(opts) do
+    key_hash =
+      case opts do
+        %{url: url} ->
+          Ecto.Repo.Supervisor.parse_url(url)
+        _ ->
+          opts
+      end
+      |> hash_opts()
+
+    String.to_atom("Core.SharedDBConnectionPool.#{key_hash}")
+  end
+
+  defp hash_opts(opts) do
+    unhashed_key =
+      opts
+      |> Keyword.take([:hostname, :username, :password, :database])
+      |> Keyword.values()
+      |> Enum.join("/")
+
+    :crypto.hash(:sha3_256, unhashed_key) |> Base.encode16()
+  end
+end

config/config.exs

@@ -32,10 +32,8 @@ end)
       ecto_repos: [repo],
       generators: [context_app: otp_app]
 
-  {otp_app, repo, context_app} ->
-    config otp_app,
-      ecto_repos: [repo],
-      generators: [context_app: context_app]
+    config otp_app, repo,
+      pool: Core.SharedDBConnectionPool
 end)
 
 config :core, :pow,

mix.lock

@@ -45,7 +45,7 @@
   "neotomex": {:hex, :neotomex, "0.1.7", "64f76513653aa87ea7abdde0fd600e56955d838020a13d88f2bf334c88ac3e7a", [:mix], [], "hexpm", "4b87b8f614d1cd89dc8ba80ba0e559bedb3ebf6f6d74cd774fcfdd215e861445"},
   "oban": {:hex, :oban, "2.1.0", "034144686f7e76a102b5d67731f098d98a9e4a52b07c25ad580a01f83a7f1cf5", [:mix], [{:ecto_sql, ">= 3.4.3", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c6f067fa3b308ed9e0e6beb2b34277c9c4e48bf95338edabd8f4a757a26e04c2"},
   "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"},
-  "phoenix": {:hex, :phoenix, "1.5.7", "2923bb3af924f184459fe4fa4b100bd25fa6468e69b2803dfae82698269aa5e0", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.13", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.1.2 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "774cd64417c5a3788414fdbb2be2eb9bcd0c048d9e6ad11a0c1fd67b7c0d0978"},
+  "phoenix": {:hex, :phoenix, "1.5.8", "71cfa7a9bb9a37af4df98939790642f210e35f696b935ca6d9d9c55a884621a4", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.13", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.1.2 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "35ded0a32f4836168c7ab6c33b88822eccd201bcd9492125a9bea4c54332d955"},
   "phoenix_ecto": {:hex, :phoenix_ecto, "4.1.0", "a044d0756d0464c5a541b4a0bf4bcaf89bffcaf92468862408290682c73ae50d", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "c5e666a341ff104d0399d8f0e4ff094559b2fde13a5985d4cb5023b2c2ac558b"},
   "phoenix_html": {:hex, :phoenix_html, "2.14.3", "51f720d0d543e4e157ff06b65de38e13303d5778a7919bcc696599e5934271b8", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "efd697a7fff35a13eeeb6b43db884705cba353a1a41d127d118fda5f90c8e80f"},
   "phoenix_html_sanitizer": {:hex, :phoenix_html_sanitizer, "1.0.2", "e2c8cfbc83660e362753de127cc957bec3442a8aecdf271fb65a684a906fccf5", [:mix], [{:html_sanitize_ex, "~> 1.0.0", [hex: :html_sanitize_ex, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}], "hexpm", "47aebb08fa954b7ad95f295fb701df9800ee3a489212119c9c6074a65e1e5a10"},
@@ -56,7 +56,7 @@
   "php_serializer": {:hex, :php_serializer, "0.9.2", "59c5fd6bd3096671fd89358fb8229341ac7423b50ad8d45a15213b02ea2edab2", [:mix], [], "hexpm", "34eb835a460944f7fc216773b363c02e7dcf8ac0390c9e9ccdbd92b31a7ca59a"},
   "plug": {:hex, :plug, "1.11.0", "f17217525597628298998bc3baed9f8ea1fa3f1160aa9871aee6df47a6e4d38e", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2d9c633f0499f9dc5c2fd069161af4e2e7756890b81adcbb2ceaa074e8308876"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.4.1", "779ba386c0915027f22e14a48919a9545714f849505fa15af2631a0d298abf0f", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d72113b6dff7b37a7d9b2a5b68892808e3a9a752f2bf7e503240945385b70507"},
-  "plug_crypto": {:hex, :plug_crypto, "1.2.0", "1cb20793aa63a6c619dd18bb33d7a3aa94818e5fd39ad357051a67f26dfa2df6", [:mix], [], "hexpm", "a48b538ae8bf381ffac344520755f3007cc10bd8e90b240af98ea29b69683fc2"},
+  "plug_crypto": {:hex, :plug_crypto, "1.2.1", "5c854427528bf61d159855cedddffc0625e2228b5f30eff76d5a4de42d896ef4", [:mix], [], "hexpm", "6961c0e17febd9d0bfa89632d391d2545d2e0eb73768f5f50305a23961d8782c"},
   "postgrex": {:hex, :postgrex, "0.15.5", "aec40306a622d459b01bff890fa42f1430dac61593b122754144ad9033a2152f", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "ed90c81e1525f65a2ba2279dbcebf030d6d13328daa2f8088b9661eb9143af7f"},
   "pow": {:hex, :pow, "1.0.20", "b99993811af5233681bfc521e81ca706d25a56f2be54bad6424db327ce840ab9", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.3.0 and < 1.6.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, ">= 2.0.0 and <= 3.0.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:plug, ">= 1.5.0 and < 2.0.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "4b6bd271399ccb353abbdbdc316199fe7fd7ae36bbf47059d53e366831c34fc8"},
   "quantum": {:hex, :quantum, "2.4.0", "f2ad4b20988f848455d35ed0e884ba0c7629a27ee86cbec6a6e0fc214b6e69cf", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12 or ~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}, {:tzdata, "~> 1.0", [hex: :tzdata, repo: "hexpm", optional: true]}], "hexpm", "a125a9e65a5af740a1198f3b05c1a736fce3942f5e0dc2901e0f9be5745bea99"},

script/update

@@ -12,9 +12,13 @@ ORIGIN=`git remote get-url origin || echo ''`
 FRAMEWORK=`git remote get-url framework || echo ''`
 
 if [ "$ORIGIN" == 'git@gitlab.com:mythic-insight/legendary.git' ] && [ "$FRAMEWORK" != 'git@gitlab.com:mythic-insight/legendary.git' ]; then
+  echo "Removing Legendary repo as 'origin' remote"
+  git remote remove origin
+fi;
+
+if [ "$FRAMEWORK" != 'git@gitlab.com:mythic-insight/legendary.git' ]; then
   echo "Setting Legendary repo as 'framework' remote"
   git remote add framework git@gitlab.com:mythic-insight/legendary.git
-  git remote remove origin
 fi;
 
 mix deps.get