feat: Add the override store, overlay, and data system wiring

lib/ldclient-rb/config.rb

@@ -736,14 +736,18 @@ class DataSystemConfig
     # @param data_store [LaunchDarkly::Interfaces::FeatureStore, nil] The (optional) data store
     # @param fdv1_fallback_synchronizer [#build(String, Config), nil]
     #   The (optional) builder for FDv1-compatible fallback synchronizer
+    # @param overrides [#build(String, Config), nil] The (optional) builder for an override source.
+    #   Flag overrides are currently experimental and subject to change.
     #
     def initialize(initializers: nil, synchronizers: nil,
-                   data_store_mode: LaunchDarkly::Interfaces::DataSystem::DataStoreMode::READ_ONLY, data_store: nil, fdv1_fallback_synchronizer: nil)
+                   data_store_mode: LaunchDarkly::Interfaces::DataSystem::DataStoreMode::READ_ONLY, data_store: nil, fdv1_fallback_synchronizer: nil,
+                   overrides: nil)
       @initializers = initializers
       @synchronizers = synchronizers
       @data_store_mode = data_store_mode
       @data_store = data_store
       @fdv1_fallback_synchronizer = fdv1_fallback_synchronizer
+      @overrides = overrides
     end
 
     # The initializer builders for the data system. Each builder responds to build(sdk_key, config) and returns an Initializer.
@@ -765,5 +769,11 @@ def initialize(initializers: nil, synchronizers: nil,
     # The FDv1-compatible fallback synchronizer builder. Responds to build(sdk_key, config) and returns a Synchronizer.
     # @return [#build(String, Config), nil]
     attr_reader :fdv1_fallback_synchronizer
+
+    # The override source builder. Responds to build(sdk_key, config) and returns an
+    # {LaunchDarkly::Interfaces::Overrides::OverrideSource}. Flag overrides are currently
+    # experimental and subject to change.
+    # @return [#build(String, Config), nil]
+    attr_reader :overrides
   end
 end

lib/ldclient-rb/data_system/config_builder.rb

@@ -23,6 +23,7 @@ def initialize
         @fdv1_fallback_synchronizer = nil
         @data_store_mode = LaunchDarkly::Interfaces::DataSystem::DataStoreMode::READ_ONLY
         @data_store = nil
+        @overrides = nil
       end
 
       #
@@ -85,6 +86,35 @@ def data_store(data_store, store_mode)
         self
       end
 
+      #
+      # Configures the SDK with an override source. Flag overrides are currently experimental and
+      # subject to change.
+      #
+      # Overrides are flag and segment definitions that take precedence over data received from
+      # LaunchDarkly at evaluation time, on a per-key basis. They exist for resilience during an
+      # incident. Overrides let an operator force one or more flags to a known state on a running
+      # client, whether or not the client can reach LaunchDarkly. Flags not present in the override
+      # data are completely unaffected.
+      #
+      # The override source is not a data source. It has no effect on the client's initialization
+      # status, data availability, or data source status. Without a configured source, or while
+      # the source supplies no overrides, the SDK behaves as it would without this feature.
+      #
+      # At most one override source can be configured. A later call replaces an earlier one.
+      #
+      # @example
+      #   overrides = LaunchDarkly::Integrations::FileData.override_source(paths: ["/etc/launchdarkly/overrides.json"])
+      #   config = LaunchDarkly::Config.new(data_system: LaunchDarkly::DataSystem.default.overrides(overrides))
+      #
+      # @param source [#build(String, Config), nil] a builder that responds to build(sdk_key, config)
+      #   and returns a {LaunchDarkly::Interfaces::Overrides::OverrideSource}, or nil to remove one
+      # @return [ConfigBuilder] self for chaining
+      #
+      def overrides(source)
+        @overrides = source
+        self
+      end
+
       #
       # Builds the data system configuration.
       #
@@ -96,7 +126,8 @@ def build
           synchronizers: @synchronizers,
           data_store_mode: @data_store_mode,
           data_store: @data_store,
-          fdv1_fallback_synchronizer: @fdv1_fallback_synchronizer
+          fdv1_fallback_synchronizer: @fdv1_fallback_synchronizer,
+          overrides: @overrides
         )
       end
     end

lib/ldclient-rb/impl/data_store/store.rb

@@ -38,9 +38,14 @@ def initialize(flag_change_broadcaster, change_set_broadcaster, logger)
           # Source of truth for flag evaluations once initialized
           @memory_store = InMemoryFeatureStoreV2.new(logger)
 
-          # Used to track dependencies between items in the store
+          # Used to track dependencies between items in the store. Items are keyed by string,
+          # the form in which the data model names prerequisites and segments.
           @dependency_tracker = LaunchDarkly::Impl::DependencyTracker.new(logger)
 
+          # A second dependency graph, such as the override layer's, whose items may depend on
+          # items in this store. Set once when the data system is built, before any data.
+          @external_dependencies = nil
+
           # Broadcasters for events
           @flag_change_broadcaster = flag_change_broadcaster
           @change_set_broadcaster = change_set_broadcaster
@@ -179,6 +184,41 @@ def initialized?
           get_active_store.initialized?
         end
 
+        #
+        # Retrieves an item from the active store.
+        #
+        # @param kind [LaunchDarkly::Impl::DataStore::DataKind]
+        # @param key [String, Symbol]
+        # @return [Object, nil] the item, or nil if it is unknown or deleted
+        #
+        def get(kind, key)
+          get_active_store.get(kind, key)
+        end
+
+        #
+        # Retrieves all items of a kind from the active store.
+        #
+        # @param kind [LaunchDarkly::Impl::DataStore::DataKind]
+        # @return [Hash{Symbol => Object}]
+        #
+        def all(kind)
+          get_active_store.all(kind)
+        end
+
+        #
+        # Registers a second dependency graph whose items may depend on items in this store. A
+        # change to an item in this store then also notifies the external items that depend on
+        # it, directly or through any chain of dependencies in either graph.
+        #
+        # @param dependencies [#add_affected_items, nil] a graph answering `add_affected_items(items_out, { kind:, key: })`
+        # @return [void]
+        #
+        def set_external_dependencies(dependencies)
+          @lock.synchronize do
+            @external_dependencies = dependencies
+          end
+        end
+
         #
         # Get the data store status provider for the persistent store, if configured.
         #
@@ -254,9 +294,9 @@ def get_data_store_status_provider
 
           collections.each do |kind, collection|
             collection.each do |key, item|
-              @dependency_tracker.update_dependencies_from(kind, key, item)
+              @dependency_tracker.update_dependencies_from(kind, key.to_s, item)
               if has_listeners
-                @dependency_tracker.add_affected_items(affected_items, { kind: kind, key: key })
+                add_affected_items(affected_items, { kind: kind, key: key })
               end
             end
           end
@@ -338,7 +378,38 @@ def get_data_store_status_provider
           @dependency_tracker.reset
           all_data.each do |kind, items|
             items.each do |key, item|
-              @dependency_tracker.update_dependencies_from(kind, key, item)
+              @dependency_tracker.update_dependencies_from(kind, key.to_s, item)
+            end
+          end
+        end
+
+        #
+        # Adds the item and everything that depends on it to the set, following this store's own
+        # dependency graph and the external one. Every item reached through either graph is
+        # expanded through both, so a dependency chain may cross between them any number of
+        # times. Keys are compared as strings, so a change keyed by symbol reaches the same
+        # dependents as one keyed by string.
+        #
+        # @param items_out [Set<Hash>] Set of {kind:, key:} hashes
+        # @param initial_modified_item [Hash] {kind:, key:} hash
+        # @return [void]
+        #
+        private def add_affected_items(items_out, initial_modified_item)
+          pending = [{ kind: initial_modified_item[:kind], key: initial_modified_item[:key].to_s }]
+          until pending.empty?
+            current = pending.pop
+            # Each graph gets its own working set: a tracker does not expand an item that is
+            # already present, so a shared set would stop the second graph at the start.
+            via_store = Set.new
+            @dependency_tracker.add_affected_items(via_store, current)
+            via_external = Set.new
+            @external_dependencies&.add_affected_items(via_external, current)
+            (via_store | via_external).each do |reached|
+              item = { kind: reached[:kind], key: reached[:key].to_s }
+              next if items_out.include?(item)
+
+              items_out.add(item)
+              pending << item
             end
           end
         end
@@ -380,9 +451,9 @@ def get_data_store_status_provider
 
               # If either is missing or versions differ, it's a change
               if old_item.nil? || new_item.nil?
-                @dependency_tracker.add_affected_items(affected_items, { kind: kind, key: key })
+                add_affected_items(affected_items, { kind: kind, key: key })
               elsif old_item[:version] != new_item[:version]
-                @dependency_tracker.add_affected_items(affected_items, { kind: kind, key: key })
+                add_affected_items(affected_items, { kind: kind, key: key })
               end
             end
           end

lib/ldclient-rb/impl/data_system.rb

@@ -108,6 +108,16 @@ def store
         raise NotImplementedError, "#{self.class} must implement #store"
       end
 
+      #
+      # Returns whether the data system was built with an override source. The value is fixed at
+      # construction. Flag overrides are currently experimental and subject to change.
+      #
+      # @return [Boolean]
+      #
+      def override_source_configured?
+        raise NotImplementedError, "#{self.class} must implement #override_source_configured?"
+      end
+
       #
       # Sets the diagnostic accumulator for streaming initialization metrics.
       # This should be called before start() to ensure metrics are collected.

lib/ldclient-rb/impl/data_system/fdv1.rb

@@ -101,6 +101,11 @@ def store
           @store_wrapper
         end
 
+        # (see DataSystem#override_source_configured?)
+        def override_source_configured?
+          false
+        end
+
         # (see DataSystem#set_diagnostic_accumulator)
         def set_diagnostic_accumulator(diagnostic_accumulator)
           @diagnostic_accumulator = diagnostic_accumulator

lib/ldclient-rb/impl/data_system/fdv2.rb

@@ -8,6 +8,7 @@
 require "ldclient-rb/impl/data_source/status_provider"
 require "ldclient-rb/impl/data_store/status_provider"
 require "ldclient-rb/impl/broadcaster"
+require "ldclient-rb/impl/overrides"
 require "ldclient-rb/impl/repeating_task"
 require "ldclient-rb/interfaces/data_system"
 
@@ -118,6 +119,25 @@ def initialize(sdk_key, config, data_system_config)
           # Track configuration
           @configured_with_data_sources = (@data_system_config.initializers && !@data_system_config.initializers.empty?) ||
             !@synchronizer_builders.empty?
+
+          # The following are nil unless an override source is configured. The layer holds override
+          # entries. The overlay serves them in preference to the store's data at the store read
+          # boundary. The source populates the layer at runtime. The override system is separate
+          # from the initializer and synchronizer pipeline and never affects data availability.
+          @override_layer = nil
+          @overlay = nil
+          @override_source = nil
+          if @data_system_config.overrides && !@disabled
+            # An invalid override source configuration raises here, so LDClient.new reports it
+            # the same way as other invalid component configuration.
+            @override_source = @data_system_config.overrides.build(@sdk_key, @config)
+            @override_layer = LaunchDarkly::Impl::Overrides::Layer.new
+            @overlay = LaunchDarkly::Impl::Overrides::Overlay.new(@store, @override_layer)
+            # A LaunchDarkly change to a flag or segment that an override entry depends on must
+            # notify that entry's flag as well. The store reaches those entries through the
+            # layer's dependency graph.
+            @store.set_external_dependencies(@override_layer)
+          end
         end
 
         # (see DataSystem#start)
@@ -131,6 +151,13 @@ def start
           @stop_event.reset
           @ready_event.reset
 
+          # The override source starts before the run loop, so a source that loads synchronously has
+          # its overrides in place before the client begins evaluating.
+          if @override_source
+            sink = LaunchDarkly::Impl::Overrides::Sink.new(@override_layer, @store, @flag_change_broadcaster, @logger)
+            @override_source.start(sink)
+          end
+
           # Start the main coordination thread
           main_thread = Thread.new { run_main_loop }
           main_thread.name = "FDv2-main"
@@ -141,6 +168,14 @@ def start
 
         # (see DataSystem#stop)
         def stop
+          if @override_source
+            begin
+              @override_source.stop
+            rescue => e
+              @logger.error { "[LDClient] Error stopping override source: #{e.message}" }
+            end
+          end
+
           @stop_event.set
 
           @lock.synchronize do
@@ -175,9 +210,16 @@ def set_diagnostic_accumulator(diagnostic_accumulator)
 
         # (see DataSystem#store)
         def store
+          return @overlay if @overlay
+
           @store.get_active_store
         end
 
+        # (see DataSystem#override_source_configured?)
+        def override_source_configured?
+          !@override_source.nil?
+        end
+
         # (see DataSystem#data_source_status_provider)
         def data_source_status_provider
           @data_source_status_provider

lib/ldclient-rb/impl/overrides.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require "ldclient-rb/impl/overrides/layer"
+require "ldclient-rb/impl/overrides/overlay"
+require "ldclient-rb/impl/overrides/sink"