feat: Commit initially

.formatter.exs

@@ -0,0 +1,4 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
+]

.github/workflows/main.yml

@@ -0,0 +1,53 @@
+name: CI
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+    env:
+      MIX_ENV: test
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - pair:
+              elixir: '1.11'
+              otp: 22
+          - pair:
+              elixir: '1.14'
+              otp: 25
+            lint: lint
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: erlef/setup-beam@v1
+        with:
+          otp-version: ${{matrix.pair.otp}}
+          elixir-version: ${{matrix.pair.elixir}}
+
+      - uses: actions/cache@v2
+        with:
+          path: deps
+          key: mix-deps-${{ hashFiles('**/mix.lock') }}
+
+      - run: mix deps.get
+
+      - run: mix format --check-formatted
+        if: ${{ matrix.lint }}
+
+      - run: mix deps.unlock --check-unused
+        if: ${{ matrix.lint }}
+
+      - run: mix deps.compile
+
+      - run: mix compile --warnings-as-errors
+        if: ${{ matrix.lint }}
+
+      - run: mix test
+        if: ${{ ! matrix.lint }}
+
+      - run: mix test --warnings-as-errors
+        if: ${{ matrix.lint }}

.gitignore

@@ -0,0 +1,26 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+tree_sitter-*.tar
+
+# Temporary files, for example, from tests.
+/tmp/

LICENSE.md

@@ -0,0 +1,22 @@
+# MIT License
+
+Copyright (c) 2023 Robert Prehn.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -0,0 +1,39 @@
+# TreeSitter
+
+Mix tasks for installing and invoking [tree-sitter](https://tree-sitter.github.io/tree-sitter/).
+
+## Installation
+
+If you are going to build assets in production, then you add
+`tree_sitter` as dependency on all environments but only start it
+in dev:
+
+```elixir
+def deps do
+  [
+    {:tree_sitter, "~> 0.1.1"}
+  ]
+end
+```
+
+Once installed, change your `config/config.exs` and [pick a version
+for the tree_sitter CLI](https://github.com/tree-sitter/tree-sitter/releases) of your choice:
+
+```elixir
+config :tree_sitter, version: "0.20.8"
+```
+
+Now you can install tree_sitter by running:
+
+```bash
+$ mix tree_sitter.install
+```
+
+The executable is kept at `_build/tree_sitter-TARGET`.
+Where `TARGET` is your system target architecture.
+
+# License
+
+Copyright (c) 2023 Robert Prehn.
+
+tree_sitter source code is licensed under the [MIT License](LICENSE.md).

config/config.exs

@@ -0,0 +1,3 @@
+import Config
+
+config :tree_sitter, version: "0.20.8"

lib/mix/tasks/tree_sitter.ex

@@ -0,0 +1,60 @@
+defmodule Mix.Tasks.TreeSitter do
+  @moduledoc """
+  Invokes tree_sitter with the given args.
+
+  Usage:
+
+      $ mix tree_sitter TASK_OPTIONS PROFILE TREE_SITTER_ARGS
+
+  Example:
+
+      $ mix tree_sitter default assets/js/app.js --bundle --minify --target=es2016 --outdir=priv/static/assets
+
+  If tree_sitter is not installed, it is automatically downloaded.
+  Note the arguments given to this task will be appended
+  to any configured arguments.
+
+  ## Options
+
+    * `--runtime-config` - load the runtime configuration
+      before executing command
+
+  Note flags to control this Mix task must be given before the
+  profile:
+
+      $ mix tree_sitter --runtime-config default assets/js/app.js
+
+  """
+
+  @shortdoc "Invokes tree_sitter with the profile and args"
+  @compile {:no_warn_undefined, Mix}
+
+  use Mix.Task
+
+  @impl true
+  def run(args) do
+    switches = [runtime_config: :boolean]
+    {opts, remaining_args} = OptionParser.parse_head!(args, switches: switches)
+
+    if function_exported?(Mix, :ensure_application!, 1) do
+      Mix.ensure_application!(:inets)
+      Mix.ensure_application!(:ssl)
+    end
+
+    if opts[:runtime_config] do
+      Mix.Task.run("app.config")
+    else
+      Application.ensure_all_started(:tree_sitter)
+    end
+
+    Mix.Task.reenable("tree_sitter")
+    install_and_run(remaining_args)
+  end
+
+  defp install_and_run(args) do
+    case TreeSitter.install_and_run(args) do
+      0 -> :ok
+      status -> Mix.raise("`mix tree_sitter #{Enum.join(args, " ")}` exited with #{status}")
+    end
+  end
+end

lib/mix/tasks/tree_sitter.install.ex

@@ -0,0 +1,63 @@
+defmodule Mix.Tasks.TreeSitter.Install do
+  @moduledoc """
+  Installs tree_sitter under `_build`.
+
+  ```bash
+  $ mix tree_sitter.install
+  $ mix tree_sitter.install --if-missing
+  ```
+
+  By default, it installs #{TreeSitter.latest_version()} but you
+  can configure it in your config files, such as:
+
+      config :tree_sitter, :version, "#{TreeSitter.latest_version()}"
+
+  ## Options
+
+      * `--runtime-config` - load the runtime configuration
+        before executing command
+
+      * `--if-missing` - install only if the given version
+        does not exist
+  """
+
+  @shortdoc "Installs tree_sitter under _build"
+  @compile {:no_warn_undefined, Mix}
+
+  use Mix.Task
+
+  @impl true
+  def run(args) do
+    valid_options = [runtime_config: :boolean, if_missing: :boolean]
+
+    case OptionParser.parse_head!(args, strict: valid_options) do
+      {opts, []} ->
+        if opts[:runtime_config], do: Mix.Task.run("app.config")
+
+        if opts[:if_missing] && latest_version?() do
+          :ok
+        else
+          if function_exported?(Mix, :ensure_application!, 1) do
+            Mix.ensure_application!(:inets)
+            Mix.ensure_application!(:ssl)
+          end
+
+          TreeSitter.install()
+        end
+
+      {_, _} ->
+        Mix.raise("""
+        Invalid arguments to tree_sitter.install, expected one of:
+
+            mix tree_sitter.install
+            mix tree_sitter.install --runtime-config
+            mix tree_sitter.install --if-missing
+        """)
+    end
+  end
+
+  defp latest_version?() do
+    version = TreeSitter.configured_version()
+    match?({:ok, ^version}, TreeSitter.bin_version())
+  end
+end

lib/tree_sitter.ex

@@ -0,0 +1,336 @@
+defmodule TreeSitter do
+  # https://registry.npmjs.org/tree-sitter/latest
+  @latest_version "0.20.8"
+
+  @moduledoc """
+  TreeSitter is an installer and runner for [tree_sitter](https://tree-sitter.github.io/tree-sitter/).
+
+  ## TreeSitter configuration
+
+  There are two global configurations for the tree_sitter application:
+
+    * `:version` - the expected tree_sitter version
+
+    * `:cacerts_path` - the directory to find certificates for
+      https connections
+
+    * `:path` - the path to find the tree_sitter executable at. By
+      default, it is automatically downloaded and placed inside
+      the `_build` directory of your current app
+
+  Overriding the `:path` is not recommended, as we will automatically
+  download and manage `tree-sitter` for you. But in case you can't download
+  it (for example, the npm registry is behind a proxy), you may want to
+  set the `:path` to a configurable system location.
+
+  For instance, you can install `tree-sitter` globally with `npm`:
+
+      $ npm install -g tree-sitter
+
+  On Unix, the executable will be at:
+
+      NPM_ROOT/tree-sitter/node_modules/@tree-sitter/TARGET/bin/tree_sitter
+
+  On Windows, it will be at:
+
+      NPM_ROOT/tree-sitter/node_modules/@tree-sitter/win32-x(32|64)/tree_sitter.exe
+
+  Where `NPM_ROOT` is the result of `npm root -g` and `TARGET` is your system
+  target architecture.
+
+  Once you find the location of the executable, you can store it in a
+  `MIX_TREE_SITTER_PATH` environment variable, which you can then read in
+  your configuration file:
+
+      config :tree_sitter, path: System.get_env("MIX_TREE_SITTER_PATH")
+
+  """
+
+  use Application
+  require Logger
+
+  @doc false
+  def start(_, _) do
+    unless Application.get_env(:tree_sitter, :version) do
+      Logger.warning("""
+      tree_sitter version is not configured. Please set it in your config files:
+
+          config :tree_sitter, :version, "#{latest_version()}"
+      """)
+    end
+
+    configured_version = configured_version()
+
+    case bin_version() do
+      {:ok, version} ->
+        if version =~ configured_version do
+          :ok
+        else
+          Logger.warning("""
+          Outdated tree_sitter version. Expected #{configured_version}, got #{version}. \
+          Please run `mix tree_sitter.install` or update the version in your config files.\
+          """)
+        end
+
+      :error ->
+        :ok
+    end
+
+    Supervisor.start_link([], strategy: :one_for_one, name: __MODULE__.Supervisor)
+  end
+
+  @doc false
+  # Latest known version at the time of publishing.
+  def latest_version, do: @latest_version
+
+  @doc """
+  Returns the configured tree_sitter version.
+  """
+  def configured_version do
+    Application.get_env(:tree_sitter, :version, latest_version())
+  end
+
+  @doc """
+  Returns the path to the executable.
+
+  The executable may not be available if it was not yet installed.
+  """
+  def bin_path do
+    name = "tree_sitter-#{target()}"
+
+    Application.get_env(:tree_sitter, :path) ||
+      if Code.ensure_loaded?(Mix.Project) do
+        Path.join(Path.dirname(Mix.Project.build_path()), name)
+      else
+        Path.expand("_build/#{name}")
+      end
+  end
+
+  @doc """
+  Returns the version of the tree_sitter executable.
+
+  Returns `{:ok, version_string}` on success or `:error` when the executable
+  is not available.
+  """
+  def bin_version do
+    path = bin_path()
+
+    with true <- File.exists?(path),
+         {result, 0} <- System.cmd(path, ["--version"]) do
+      {:ok, String.trim(result)}
+    else
+      _ -> :error
+    end
+  end
+
+  @doc """
+  Runs the given command with `args`.
+
+  The given args will be appended to the configured args.
+  The task output will be streamed directly to stdio. It
+  returns the status of the underlying call.
+  """
+  def run(extra_args) when is_list(extra_args) do
+    opts = [
+      into: IO.stream(:stdio, :line),
+      stderr_to_stdout: true
+    ]
+
+    bin_path()
+    |> System.cmd(extra_args, opts)
+    |> elem(1)
+  end
+
+  defp start_unique_install_worker() do
+    ref =
+      __MODULE__.Supervisor
+      |> Supervisor.start_child(
+        Supervisor.child_spec({Task, &install/0}, restart: :transient, id: __MODULE__.Installer)
+      )
+      |> case do
+        {:ok, pid} -> pid
+        {:error, {:already_started, pid}} -> pid
+      end
+      |> Process.monitor()
+
+    receive do
+      {:DOWN, ^ref, _, _, _} -> :ok
+    end
+  end
+
+  @doc """
+  Installs, if not available, and then runs `tree_sitter`.
+
+  Returns the same as `run/1`.
+  """
+  def install_and_run(args) do
+    File.exists?(bin_path()) || start_unique_install_worker()
+
+    run(args)
+  end
+
+  @doc """
+  Installs tree_sitter with `configured_version/0`.
+  """
+  def install do
+    version = configured_version()
+    tmp_opts = if System.get_env("MIX_XDG"), do: %{os: :linux}, else: %{}
+
+    tmp_dir =
+      freshdir_p(:filename.basedir(:user_cache, "tree_sitter", tmp_opts)) ||
+        freshdir_p(Path.join(System.tmp_dir!(), "tree_sitter")) ||
+        raise "could not install tree-sitter. Set MIX_XGD=1 and then set XDG_CACHE_HOME to the path you want to use as cache"
+
+    target = target()
+
+    url =
+      "https://github.com/tree-sitter/tree-sitter/releases/download/v#{version}/tree-sitter-#{target}.gz"
+
+    gz = fetch_body!(url)
+
+    result_path =
+      case :os.type() do
+        {:win32, _} ->
+          Path.join([tmp_dir, "tree_sitter.exe"])
+
+        _ ->
+          Path.join([tmp_dir, "tree_sitter"])
+      end
+
+    gz
+    |> :zlib.gunzip()
+    |> then(fn data ->
+      File.write!(result_path, data)
+    end)
+
+    File.chmod!(result_path, 0o700)
+
+    bin_path = bin_path()
+    File.mkdir_p!(Path.dirname(bin_path))
+
+    File.cp!(result_path, bin_path)
+  end
+
+  defp freshdir_p(path) do
+    with {:ok, _} <- File.rm_rf(path),
+         :ok <- File.mkdir_p(path) do
+      path
+    else
+      _ -> nil
+    end
+  end
+
+  # Available targets: https://github.com/evanw/tree_sitter/tree/main/npm/@tree_sitter
+  defp target do
+    case :os.type() do
+      # Assuming it's an x86 CPU
+      {:win32, _} ->
+        wordsize = :erlang.system_info(:wordsize)
+
+        if wordsize == 8 do
+          "windows-x64"
+        else
+          "windows-x86"
+        end
+
+      {:unix, osname} ->
+        arch_str = :erlang.system_info(:system_architecture)
+        [arch | _] = arch_str |> List.to_string() |> String.split("-")
+
+        osname =
+          if osname == :darwin do
+            "macos"
+          else
+            "linux"
+          end
+
+        case arch do
+          "amd64" -> "#{osname}-arm64"
+          "x86_64" -> "#{osname}-x64"
+          "i686" -> "#{osname}-x86"
+          "i386" -> "#{osname}-x86"
+          "aarch64" -> "#{osname}-arm64"
+          "arm" when osname == "macos" -> "darwin-arm64"
+          "arm" -> "#{osname}-arm"
+          "armv7" <> _ -> "#{osname}-arm"
+          _ -> raise "tree_sitter is not available for architecture: #{arch_str}"
+        end
+    end
+  end
+
+  defp fetch_body!(url) do
+    scheme = URI.parse(url).scheme
+    url = String.to_charlist(url)
+    Logger.debug("Downloading tree_sitter from #{url}")
+
+    {:ok, _} = Application.ensure_all_started(:inets)
+    {:ok, _} = Application.ensure_all_started(:ssl)
+
+    if proxy = proxy_for_scheme(scheme) do
+      %{host: host, port: port} = URI.parse(proxy)
+      Logger.debug("Using #{String.upcase(scheme)}_PROXY: #{proxy}")
+      set_option = if "https" == scheme, do: :https_proxy, else: :proxy
+      :httpc.set_options([{set_option, {{String.to_charlist(host), port}, []}}])
+    end
+
+    # https://erlef.github.io/security-wg/secure_coding_and_deployment_hardening/inets
+    cacertfile = cacertfile() |> String.to_charlist()
+
+    http_options =
+      [
+        ssl: [
+          verify: :verify_peer,
+          cacertfile: cacertfile,
+          depth: 2,
+          customize_hostname_check: [
+            match_fun: :public_key.pkix_verify_hostname_match_fun(:https)
+          ]
+        ]
+      ]
+      |> maybe_add_proxy_auth(scheme)
+
+    options = [body_format: :binary]
+
+    case :httpc.request(:get, {url, []}, http_options, options) do
+      {:ok, {{_, 200, _}, _headers, body}} ->
+        body
+
+      other ->
+        raise """
+        couldn't fetch #{url}: #{inspect(other)}
+
+        You may also install the "tree_sitter" executable manually, \
+        see the docs: https://hexdocs.pm/tree_sitter
+        """
+    end
+  end
+
+  defp proxy_for_scheme("http") do
+    System.get_env("HTTP_PROXY") || System.get_env("http_proxy")
+  end
+
+  defp proxy_for_scheme("https") do
+    System.get_env("HTTPS_PROXY") || System.get_env("https_proxy")
+  end
+
+  defp maybe_add_proxy_auth(http_options, scheme) do
+    case proxy_auth(scheme) do
+      nil -> http_options
+      auth -> [{:proxy_auth, auth} | http_options]
+    end
+  end
+
+  defp proxy_auth(scheme) do
+    with proxy when is_binary(proxy) <- proxy_for_scheme(scheme),
+         %{userinfo: userinfo} when is_binary(userinfo) <- URI.parse(proxy),
+         [username, password] <- String.split(userinfo, ":") do
+      {String.to_charlist(username), String.to_charlist(password)}
+    else
+      _ -> nil
+    end
+  end
+
+  defp cacertfile() do
+    Application.get_env(:tree_sitter, :cacerts_path) || CAStore.file_path()
+  end
+end

mix.exs

@@ -0,0 +1,44 @@
+defmodule TreeSitter.MixProject do
+  use Mix.Project
+
+  @version "0.0.1"
+  @source_url "https://gitlab.com/mythic-insight/tree_sitter"
+
+  def project do
+    [
+      app: :tree_sitter,
+      version: @version,
+      elixir: "~> 1.11",
+      deps: deps(),
+      description: "Mix tasks for installing and invoking tree_sitter",
+      package: [
+        links: %{
+          "GitHub" => @source_url,
+          "tree_sitter" => "https://tree-sitter.github.io/tree-sitter/"
+        },
+        licenses: ["MIT"]
+      ],
+      docs: [
+        main: "TreeSitter",
+        source_url: @source_url,
+        source_ref: "v#{@version}"
+      ],
+      aliases: [test: ["tree_sitter.install --if-missing", "test"]]
+    ]
+  end
+
+  def application do
+    [
+      extra_applications: [:logger, inets: :optional, ssl: :optional],
+      mod: {TreeSitter, []},
+      env: [default: []]
+    ]
+  end
+
+  defp deps do
+    [
+      {:castore, ">= 0.0.0"},
+      {:ex_doc, ">= 0.0.0", only: :docs}
+    ]
+  end
+end

mix.lock

@@ -0,0 +1,9 @@
+%{
+  "castore": {:hex, :castore, "0.1.11", "c0665858e0e1c3e8c27178e73dffea699a5b28eb72239a3b2642d208e8594914", [:mix], [], "hexpm", "91b009ba61973b532b84f7c09ce441cba7aa15cb8b006cf06c6f4bba18220081"},
+  "earmark_parser": {:hex, :earmark_parser, "1.4.32", "fa739a0ecfa34493de19426681b23f6814573faee95dfd4b4aafe15a7b5b32c6", [:mix], [], "hexpm", "b8b0dd77d60373e77a3d7e8afa598f325e49e8663a51bcc2b88ef41838cca755"},
+  "ex_doc": {:hex, :ex_doc, "0.29.4", "6257ecbb20c7396b1fe5accd55b7b0d23f44b6aa18017b415cb4c2b91d997729", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "2c6699a737ae46cb61e4ed012af931b57b699643b24dabe2400a8168414bc4f5"},
+  "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"},
+  "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"},
+  "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"},
+  "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"},
+}

test/test_helper.exs

@@ -0,0 +1 @@
+ExUnit.start()

test/tree_sitter_test.exs

@@ -0,0 +1,31 @@
+defmodule TreeSitterTest do
+  use ExUnit.Case, async: true
+
+  @version TreeSitter.latest_version()
+
+  test "runs" do
+    assert ExUnit.CaptureIO.capture_io(fn ->
+             assert TreeSitter.run(["--version"]) == 0
+           end) =~ @version
+  end
+
+  test "updates on install" do
+    Application.put_env(:tree_sitter, :version, "0.20.6")
+
+    Mix.Task.rerun("tree_sitter.install", ["--if-missing"])
+
+    assert ExUnit.CaptureIO.capture_io(fn ->
+             assert TreeSitter.run(["--version"]) == 0
+           end) =~ "0.20.6"
+
+    Application.delete_env(:tree_sitter, :version)
+
+    Mix.Task.rerun("tree_sitter.install", ["--if-missing"])
+
+    assert ExUnit.CaptureIO.capture_io(fn ->
+             assert TreeSitter.run(["--version"]) == 0
+           end) =~ @version
+  after
+    Application.delete_env(:tree_sitter, :version)
+  end
+end