Support loading extensions (closes #136) (#160)

* Implements SQLite extensions loading
  As initial selection for SQLITE extensions we take https://github.com/nalgeon/sqlean with some additions in this fork (https://github.com/mindreframer/sqlean) + packaged as Hex package here: https://hex.pm/packages/ex_sqlean. 
  To keep the usage for Exqlite somewhat simpler and consistent, a new module is introduced: BasicAPI. It allows simple operations without the confusion between Exqlite.Sqlite3 and Exqlite.Connection modules to execute queries and dealing with results.
* Bump the oldest supported elixir version to 1.9 (from 1.8)
* Rename module: BasicAPI -> Basic
* Bump lowest supported OTP to `21` + Elixir `1.8`

Author: mindreframer

.github/workflows/ci.yml

@@ -5,6 +5,7 @@ on:
   push:
     branches:
       - main
+      - "*"
 
 jobs:
   lint:
@@ -42,7 +43,7 @@ jobs:
           - elixir: "1.10"
             otp: "23"
           - elixir: "1.8"
-            otp: "20"
+            otp: "21"
     steps:
       - uses: actions/checkout@v2
       - uses: erlef/setup-elixir@v1

README.md

@@ -85,6 +85,30 @@ The `Exqlite.Sqlite3` module usage is fairly straight forward.
 :ok = Exqlite.Sqlite3.release(conn, statement)
 ```
 
+### Using SQLite3 native extensions
+
+Exqlite supports loading [run-time loadable SQLite3 extensions](https://www.sqlite.org/loadext.html).
+A selection of precompiled extensions for popular CPU types / architectures is available by installing the [ExSqlean](https://github.com/mindreframer/ex_sqlean) package. This package wraps [SQLean: all the missing SQLite functions](https://github.com/nalgeon/sqlean).
+
+```elixir
+alias Exqlite.Basic
+{:ok, conn} = Basic.open("db.sqlite3")
+:ok = Basic.enable_load_extension(conn)
+
+# load the regexp extension - https://github.com/nalgeon/sqlean/blob/main/docs/re.md
+Basic.load_extension(conn, ExSqlean.path_for("re"))
+
+# run some queries to test the new `regexp_like` function
+{:ok, [[1]], ["value"]} = Basic.exec(conn, "select regexp_like('the year is 2021', ?) as value", ["2021"]) |> Basic.rows()
+{:ok, [[0]], ["value"]} = Basic.exec(conn, "select regexp_like('the year is 2021', ?) as value", ["2020"]) |> Basic.rows()
+
+# prevent loading further extensions
+:ok = Basic.disable_load_extension(conn)
+{:error, %Exqlite.Error{message: "not authorized"}, _} = Basic.load_extension(conn, ExSqlean.path_for("re"))
+
+# close connection
+Basic.close(conn)
+```
 
 ## Why SQLite3
 

c_src/sqlite3_nif.c

@@ -798,6 +798,38 @@ on_load(ErlNifEnv* env, void** priv, ERL_NIF_TERM info)
     return 0;
 }
 
+
+//
+// Enable extension loading
+//
+
+static ERL_NIF_TERM
+exqlite_enable_load_extension(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    assert(env);
+    connection_t* conn = NULL;
+    int rc             = SQLITE_OK;
+    int enable_load_extension_value;
+
+    if (argc != 2) {
+        return enif_make_badarg(env);
+    }
+
+    if (!enif_get_resource(env, argv[0], connection_type, (void**)&conn)) {
+        return make_error_tuple(env, "invalid_connection");
+    }
+
+    if (!enif_get_int(env, argv[1], &enable_load_extension_value)) {
+        return make_error_tuple(env, "invalid_enable_load_extension_value");
+    }
+
+    rc = sqlite3_enable_load_extension(conn->db, enable_load_extension_value);
+    if (rc != SQLITE_OK) {
+        return make_sqlite3_error_tuple(env, rc, conn->db);
+    }
+    return make_atom(env, "ok");
+}
+
 //
 // Most of our nif functions are going to be IO bounded
 //
@@ -817,6 +849,7 @@ static ErlNifFunc nif_funcs[] = {
   {"serialize", 2, exqlite_serialize, ERL_NIF_DIRTY_JOB_IO_BOUND},
   {"deserialize", 3, exqlite_deserialize, ERL_NIF_DIRTY_JOB_IO_BOUND},
   {"release", 2, exqlite_release, ERL_NIF_DIRTY_JOB_IO_BOUND},
+  {"enable_load_extension", 2, exqlite_enable_load_extension, ERL_NIF_DIRTY_JOB_IO_BOUND},
 };
 
 ERL_NIF_INIT(Elixir.Exqlite.Sqlite3NIF, nif_funcs, on_load, NULL, NULL, NULL)

lib/exqlite/basic.ex

@@ -0,0 +1,49 @@
+defmodule Exqlite.Basic do
+  @moduledoc """
+  A very basis API without lots of options to allow simpler usage for basic needs.
+  """
+
+  alias Exqlite.Connection
+  alias Exqlite.Query
+  alias Exqlite.Sqlite3
+  alias Exqlite.Error
+  alias Exqlite.Result
+
+  def open(path) do
+    Connection.connect(database: path)
+  end
+
+  def close(conn = %Connection{}) do
+    with :ok <- Sqlite3.close(conn.db) do
+      :ok
+    else
+      {:error, reason} -> {:error, %Error{message: reason}}
+    end
+  end
+
+  def exec(conn = %Connection{}, stmt, args \\ []) do
+    %Query{statement: stmt} |> Connection.handle_execute(args, [], conn)
+  end
+
+  def rows(exec_result) do
+    case exec_result do
+      {:ok, %Query{}, %Result{rows: rows, columns: columns}, %Connection{}} ->
+        {:ok, rows, columns}
+
+      {:error, %Error{message: message}, %Connection{}} ->
+        {:error, message}
+    end
+  end
+
+  def load_extension(conn, path) do
+    exec(conn, "select load_extension(?)", [path])
+  end
+
+  def enable_load_extension(conn) do
+    Sqlite3.enable_load_extension(conn.db, true)
+  end
+
+  def disable_load_extension(conn) do
+    Sqlite3.enable_load_extension(conn.db, false)
+  end
+end

lib/exqlite/sqlite3.ex

@@ -184,6 +184,18 @@ defmodule Exqlite.Sqlite3 do
     Sqlite3NIF.release(conn, statement)
   end
 
+  @doc """
+  Allow loading native extensions.
+  """
+  @spec enable_load_extension(db(), boolean) :: :ok | {:error, any}
+  def enable_load_extension(conn, flag) do
+    if flag do
+      Sqlite3NIF.enable_load_extension(conn, 1)
+    else
+      Sqlite3NIF.enable_load_extension(conn, 0)
+    end
+  end
+
   defp convert(%Date{} = val), do: Date.to_iso8601(val)
   defp convert(%Time{} = val), do: Time.to_iso8601(val)
   defp convert(%NaiveDateTime{} = val), do: NaiveDateTime.to_iso8601(val)

lib/exqlite/sqlite3_nif.ex

@@ -59,5 +59,8 @@ defmodule Exqlite.Sqlite3NIF do
   @spec release(db(), statement()) :: :ok | {:error, reason()}
   def release(_conn, _statement), do: :erlang.nif_error(:not_loaded)
 
+  @spec enable_load_extension(db(), boolean()) :: :ok | {:error, reason()}
+  def enable_load_extension(_conn, _flag), do: :erlang.nif_error(:not_loaded)
+
   # TODO: add statement inspection tooling https://sqlite.org/c3ref/expanded_sql.html
 end

mix.exs

@@ -37,9 +37,10 @@ defmodule Exqlite.MixProject do
   defp deps do
     [
       {:db_connection, "~> 2.1"},
+      {:ex_sqlean, "~> 0.8.4", only: [:dev, :test]},
       {:elixir_make, "~> 0.6", runtime: false},
       {:ex_doc, "~> 0.24", only: :dev, runtime: false},
-      {:temp, "~> 0.4", only: [:test]}
+      {:temp, "~> 0.4", only: [:dev, :test]}
     ]
   end
 

mix.lock

@@ -4,6 +4,7 @@
   "earmark_parser": {:hex, :earmark_parser, "1.4.12", "b245e875ec0a311a342320da0551da407d9d2b65d98f7a9597ae078615af3449", [:mix], [], "hexpm", "711e2cc4d64abb7d566d43f54b78f7dc129308a63bc103fbd88550d2174b3160"},
   "elixir_make": {:hex, :elixir_make, "0.6.2", "7dffacd77dec4c37b39af867cedaabb0b59f6a871f89722c25b28fcd4bd70530", [:mix], [], "hexpm", "03e49eadda22526a7e5279d53321d1cced6552f344ba4e03e619063de75348d9"},
   "ex_doc": {:hex, :ex_doc, "0.24.2", "e4c26603830c1a2286dae45f4412a4d1980e1e89dc779fcd0181ed1d5a05c8d9", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "e134e1d9e821b8d9e4244687fb2ace58d479b67b282de5158333b0d57c6fb7da"},
+  "ex_sqlean": {:hex, :ex_sqlean, "0.8.4", "eaef82ac52e8525e597cf258471fe7bdb11cbed52b9bfba4ff9492bff80e4bab", [:mix], [], "hexpm", "7225ff5461d6d5ea8cf664278ccaa432cba233216d6805cd76a34ff2c533656d"},
   "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.15.1", "b5888c880d17d1cc3e598f05cdb5b5a91b7b17ac4eaf5f297cb697663a1094dd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "db68c173234b07ab2a07f645a5acdc117b9f99d69ebf521821d89690ae6c6ec8"},
   "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"},

test/exqlite/extensions_test.exs

@@ -0,0 +1,55 @@
+defmodule Exqlite.ExtensionsTest do
+  use ExUnit.Case
+  alias Exqlite.Basic
+
+  describe "enable_load_extension" do
+    test "loading can be enabled / disabled" do
+      {:ok, path} = Temp.path()
+      {:ok, conn} = Basic.open(path)
+      :ok = Basic.enable_load_extension(conn)
+
+      {:ok, [[nil]], _} =
+        Basic.load_extension(conn, ExSqlean.path_for("re")) |> Basic.rows()
+
+      {:ok, [[1]], _} =
+        Basic.exec(conn, "select regexp_like('the year is 2021', '2021')")
+        |> Basic.rows()
+
+      :ok = Basic.disable_load_extension(conn)
+
+      {:error, "not authorized"} =
+        Basic.load_extension(conn, ExSqlean.path_for("re")) |> Basic.rows()
+    end
+
+    test "works for 're' (regex)" do
+      {:ok, path} = Temp.path()
+      {:ok, conn} = Basic.open(path)
+
+      :ok = Basic.enable_load_extension(conn)
+
+      {:ok, [[nil]], _} =
+        Basic.load_extension(conn, ExSqlean.path_for("re")) |> Basic.rows()
+
+      {:ok, [[0]], _} =
+        Basic.exec(conn, "select regexp_like('the year is 2021', '2k21')")
+        |> Basic.rows()
+
+      {:ok, [[1]], _} =
+        Basic.exec(conn, "select regexp_like('the year is 2021', '2021')")
+        |> Basic.rows()
+    end
+
+    test "stats extension" do
+      {:ok, path} = Temp.path()
+      {:ok, conn} = Basic.open(path)
+
+      :ok = Basic.enable_load_extension(conn)
+      Basic.load_extension(conn, ExSqlean.path_for("stats"))
+      Basic.load_extension(conn, ExSqlean.path_for("series"))
+
+      {:ok, [[50.5]], ["median(value)"]} =
+        Basic.exec(conn, "select median(value) from generate_series(1, 100)")
+        |> Basic.rows()
+    end
+  end
+end