move to language binder

DEVELOPMENT.md

@@ -1,7 +1,7 @@
 # Development
 
-`dotenvx` is a mixed Ruby/Rust package. Magnus exposes the published
-`dotenvx-primitives` crate to Ruby, and `rb-sys` compiles the extension into
+`dotenvx` is a mixed Ruby/Rust package. Magnus exposes the private
+`dotenvx-ll` shared loader to Ruby, and `rb-sys` compiles the extension into
 `lib/dotenvx`.
 
 ## Setup
@@ -44,9 +44,8 @@ cargo check
 bundle install
 ```
 
-The `dotenvx-primitives` dependency is versioned independently. Update it in
-`ext/dotenvx/Cargo.toml` only when the gem should embed a newer primitives
-release.
+The shared loader owns the published `dotenvx-primitives` dependency. Update
+it in Forge when the consumers should embed a newer primitives release.
 
 ## Publishing
 
@@ -73,3 +72,15 @@ git push origin v4.0.0
 ```
 
 The tag must match `Dotenvx::VERSION`. CI verifies this before packaging.
+
+## Shared loader checkout
+
+The native extension currently depends on private `dotenvx-ll` through a local
+Cargo path. Keep this checkout beside `forge/`, which contains `dotenvx-ll/`.
+It embeds published primitives 4.x through that loader, instead of depending on
+primitives directly. Rebuild the extension after changing the loader.
+
+These are local development paths, not a finished release-distribution setup.
+Standalone CI/source-package builds need private access to the loader source
+before publishing. Do not publish `dotenvx-ll` publicly; compiled platform gems
+and wheels can embed it without distributing its Rust source.

README.md

@@ -2,9 +2,8 @@
 
 *a secure dotenv–from the creator of [`dotenv`](https://github.com/motdotla/dotenv).*
 
-`dotenvx` loads plaintext and encrypted dotenv files through the shared
-[`dotenvx-primitives`](https://crates.io/crates/dotenvx-primitives) Rust
-implementation. The native extension is bundled in the gem; users do not need
+`dotenvx` loads plaintext and encrypted dotenv files through the shared `dotenvx-ll` Rust loader, backed by published
+[`dotenvx-primitives`](https://crates.io/crates/dotenvx-primitives). The native extension is bundled in the gem; users do not need
 Node.js, a dotenvx CLI, or a Rust toolchain.
 
 ## Install
@@ -61,12 +60,6 @@ Raise when a file is missing:
 Dotenvx.load!(".env")
 ```
 
-Require configuration keys:
-
-```ruby
-Dotenvx.require_keys("DATABASE_URL", "SECRET_KEY")
-```
-
 Encrypted values are decrypted automatically when the matching private key is
 available through the process environment, `<filename>.keys`, or `.env.keys`.
 
@@ -81,3 +74,25 @@ Tagged releases publish variants of the same `dotenvx` gem for:
 - Windows x64
 
 RubyGems selects the correct variant during `gem install dotenvx`.
+
+## Loading errors and policy
+
+Loading errors are reported and loading continues by default. `strict: true`,
+`load!`, or strictness in the current directory's Dotenvxfile stops loading and
+raises before updating `ENV`. Dotenvxfile required/type/file rules are shared
+with the CLI. Selected error codes can be ignored with `ignore: ["CODE"]`.
+
+The shared loader returns redaction metadata without changing real application
+values. Application-output redaction hooks are not installed by this adapter.
+
+## Shared file selection
+
+File discovery, directory expansion, conventions, and key-based filename
+inference are resolved by `dotenvx-ll`. The CLI's file-selection and error-ignore
+environment variables apply here too. Rails supplies its root and environment;
+the shared loader chooses the Rails files and their order.
+
+```ruby
+Dotenvx.load(convention: "nextjs")
+Dotenvx.load("app", key_files: "shared/.env.keys")
+```

ext/dotenvx/Cargo.toml

@@ -10,8 +10,7 @@ name = "dotenvx_native"
 crate-type = ["cdylib"]
 
 [dependencies]
-dotenvx-primitives = "=3.0.0"
+dotenvx-ll = { path = "../../../forge/dotenvx-ll" }
 magnus = "0.8"
 rb-sys = { version = "0.9.128", default-features = false, features = ["stable-api-compiled-force"] }
-serde = { version = "1", features = ["derive"] }
 serde_json = "1"

ext/dotenvx/src/lib.rs

@@ -1,57 +1,21 @@
-use dotenvx_primitives::{parse, ParseOptions, ParseResult};
 use magnus::{function, prelude::*, Error, Ruby};
-use serde::Deserialize;
-use std::collections::HashMap;
-use std::path::PathBuf;
 
-type StringPairs = Vec<(String, String)>;
-type ErrorPairs = Vec<(String, String)>;
-
-fn runtime_error(message: impl Into<String>) -> Error {
-    let ruby = Ruby::get().expect("Ruby VM is not available");
-    Error::new(ruby.exception_runtime_error(), message.into())
-}
-
-fn parse_result(result: ParseResult) -> (StringPairs, StringPairs, ErrorPairs) {
-    let errors = result
-        .errors
-        .iter()
-        .map(|error| (error.code().to_owned(), error.to_string()))
-        .collect();
-    (
-        result.parsed.into_iter().collect(),
-        result.injected.into_iter().collect(),
-        errors,
-    )
-}
-
-#[derive(Deserialize)]
-struct ParseInput {
-    source: String,
-    process_env: HashMap<String, String>,
-    overwrite: bool,
-    key_files: Vec<String>,
+fn load_env(input_json: String) -> Result<String, Error> {
+    let options = serde_json::from_str::<dotenvx_ll::LoadOptions>(&input_json)
+        .map_err(|error| runtime_error(error.to_string()))?;
+    serde_json::to_string(&dotenvx_ll::load(&options))
+        .map_err(|error| runtime_error(error.to_string()))
 }
 
-fn parse_dotenv(input_json: String) -> Result<(StringPairs, StringPairs, ErrorPairs), Error> {
-    let input = serde_json::from_str::<ParseInput>(&input_json)
-        .map_err(|error| runtime_error(error.to_string()))?;
-    let process_env = input.process_env;
-    Ok(parse_result(parse(
-        &input.source,
-        &ParseOptions {
-            process_env,
-            overload: input.overwrite,
-            key_files: input.key_files.into_iter().map(PathBuf::from).collect(),
-            ..Default::default()
-        },
-    )))
+fn runtime_error(message: String) -> Error {
+    let ruby = Ruby::get().expect("Ruby VM is not available");
+    Error::new(ruby.exception_runtime_error(), message)
 }
 
 #[magnus::init]
 fn init(ruby: &Ruby) -> Result<(), Error> {
     let dotenvx = ruby.define_module("Dotenvx")?;
     let native = dotenvx.define_module("Native")?;
-    native.define_singleton_method("parse_dotenv", function!(parse_dotenv, 1))?;
+    native.define_singleton_method("load_env", function!(load_env, 1))?;
     Ok(())
 }

lib/dotenvx.rb

@@ -4,15 +4,6 @@
 require "pathname"
 
 module Dotenvx
-  class MissingKeys < RuntimeError
-    attr_reader :keys
-
-    def initialize(keys)
-      @keys = keys
-      super("Missing required configuration keys: #{keys.join(", ")}")
-    end
-  end
-
   class ParseError < RuntimeError
     attr_reader :code
 
@@ -25,12 +16,15 @@ def initialize(code, message)
   class << self
     attr_accessor :instrumenter
 
-    def load(*filenames, overwrite: false, ignore: true, strict: false)
+    def load(*filenames, overwrite: false, ignore: [], strict: false, convention: nil, environment: nil, key_files: nil)
       env, injected_keys, loaded_paths = parse_files(
         *filenames,
         overwrite: overwrite,
         ignore: ignore,
-        strict: strict
+        strict: strict,
+        convention: convention,
+        environment: environment,
+        key_files: key_files
       )
       changed = update(env, overwrite: overwrite)
       log_injected(injected_keys, loaded_paths)
@@ -51,12 +45,15 @@ def overwrite!(*filenames)
     end
     alias overload! overwrite!
 
-    def parse(*filenames, overwrite: false, ignore: true, strict: false)
+    def parse(*filenames, overwrite: false, ignore: [], strict: false, convention: nil, environment: nil, key_files: nil)
       parse_files(
         *filenames,
         overwrite: overwrite,
         ignore: ignore,
-        strict: strict
+        strict: strict,
+        convention: convention,
+        environment: environment,
+        key_files: key_files
       ).first
     end
 
@@ -80,60 +77,35 @@ def update(env = {}, overwrite: false)
       changed
     end
 
-    def require_keys(*keys)
-      missing = keys.flatten.map(&:to_s) - ENV.keys
-      raise MissingKeys, missing unless missing.empty?
-    end
-
     private
 
-    def parse_files(*filenames, overwrite: false, ignore: true, strict: false)
-      filenames = [".env"] if filenames.empty?
-      filenames = filenames.flatten.reverse if overwrite
-
-      process_env = ENV.to_h
-      injected_keys = {}
-      loaded_paths = []
-      values = filenames.reduce({}) do |accumulator, filename|
-        path = File.expand_path(filename)
-        begin
-          source = File.binread(path).sub(/\A\xEF\xBB\xBF/, "").force_encoding(Encoding::UTF_8)
-        rescue Errno::ENOENT, Errno::EISDIR
-          raise unless ignore_missing?(ignore)
-          next accumulator
-        end
-
-        parsed, injected, errors = Native.parse_dotenv(JSON.generate(
-          source: source,
-          process_env: process_env,
-          overwrite: overwrite == true,
-          key_files: key_files(path)
-        ))
-        handle_errors(errors, ignore: ignore, strict: strict)
-        parsed = parsed.to_h
-        injected.each { |key, _value| injected_keys[key] = true }
-        loaded_paths << path
-        process_env.merge!(parsed)
-        accumulator.merge!(parsed)
-        yield Environment.new(path, parsed) if block_given?
-        accumulator
-      end
-      [values, injected_keys.keys, loaded_paths]
-    end
-
-    def handle_errors(errors, ignore:, strict:)
+    def parse_files(*filenames, overwrite: false, ignore: [], strict: false, convention: nil, environment: nil, key_files: nil)
+      filenames = filenames.flatten
       ignored_codes = Array(ignore).grep(String)
-      errors.each do |code, message|
-        next if ignored_codes.include?(code)
-
-        raise ParseError.new(code, message) if strict
-
-        warn "☠ #{message}"
+      ignored_codes << "MISSING_ENV_FILE" if ignore == true
+      result = JSON.parse(Native.load_env(JSON.generate(
+        sources: filenames.map { |filename| { type: "file", value: filename.to_s } },
+        cwd: Dir.pwd,
+        convention: convention,
+        environment_name: environment,
+        key_files: key_files.nil? ? nil : Array(key_files).map(&:to_s),
+        process_env: ENV.to_h,
+        overload: overwrite == true,
+        strict: strict,
+        ignore: ignored_codes,
+        adjacent_keys: true
+      )))
+      result.fetch("diagnostics").each do |error|
+        if error.fetch("strict")
+          if error["code"] == "MISSING_ENV_FILE" && ignore == false
+            raise Errno::ENOENT, error["path"]
+          end
+          raise ParseError.new(error.fetch("code"), error.fetch("message"))
+        end
+        warn "☠ #{error.fetch("message")}"
       end
-    end
-
-    def ignore_missing?(ignore)
-      ignore == true || Array(ignore).map(&:to_s).include?("MISSING_ENV_FILE")
+      [result.fetch("values"), result.fetch("injected").keys,
+        result.fetch("files").map { |file| file.fetch("filepath") }]
     end
 
     def log_injected(injected_keys, loaded_paths)
@@ -152,12 +124,6 @@ def readable_path(path)
     rescue ArgumentError
       pathname.to_s
     end
-
-    def key_files(path)
-      candidates = ["#{path}.keys", File.join(File.dirname(path), ".env.keys")]
-      candidates.select { |candidate| File.file?(candidate) }
-        .uniq
-    end
   end
 
   class Environment < Hash

lib/dotenvx/rails.rb

@@ -39,25 +39,12 @@ class Railtie < Rails::Railtie
     def load
       return if @loaded
 
-      Dotenvx.load(*dotenvx_files)
+      Dotenvx.load(root.to_s, convention: "rails", environment: Rails.env.to_s)
       @loaded = true
     end
 
     private
 
-    def dotenvx_files
-      environment = Rails.env
-      files = [
-        root.join(".env.#{environment}.local")
-      ]
-      files << root.join(".env.local") unless environment == "test"
-      files.concat([
-        root.join(".env.#{environment}"),
-        root.join(".env")
-      ])
-      files
-    end
-
     def root
       Pathname.new(Rails.root || ENV["RAILS_ROOT"] || Dir.pwd)
     end

spec/dotenvx/rails_spec.rb

@@ -1,4 +1,5 @@
 require "spec_helper"
+require "tmpdir"
 require "rails"
 require "dotenvx/rails"
 
@@ -136,4 +137,23 @@ def add(*items)
   #     end
   #   end
   # end
+
+  it "loads Rails test files through the shared selector" do
+    Dir.mktmpdir do |directory|
+      File.write(File.join(directory, ".env.test"), "LL_RAILS=shared\n")
+      File.write(File.join(directory, ".env.local"), "LL_RAILS_LOCAL=excluded\n")
+      allow(Rails).to receive(:root).and_return(Pathname.new(directory))
+      railtie = Dotenvx::Railtie.instance
+      previously_loaded = railtie.instance_variable_get(:@loaded)
+      begin
+        railtie.instance_variable_set(:@loaded, false)
+        expect { railtie.load }.to output(/injected env/).to_stderr
+        expect(ENV["LL_RAILS"]).to eq("shared")
+        expect(ENV).not_to have_key("LL_RAILS_LOCAL")
+      ensure
+        railtie.instance_variable_set(:@loaded, previously_loaded)
+      end
+    end
+  end
+
 end