From 377783125d3d9cdb419b05fbb9cdbad7e2be9f0d Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 26 Aug 2026 13:12:07 -0400 Subject: [PATCH 1/5] Fall back to the backup coarse universe file for fundamental properties in live trading The universe subscription already falls back to the backup coarse universe file when the expected one is not available, but the per symbol fundamental properties read through the CoarseFundamentalDataProvider did not, returning defaults. --- .../CoarseFundamentalDataProvider.cs | 13 ++ .../CoarseFundamentalDataProviderTests.cs | 135 ++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 Tests/Common/Data/UniverseSelection/CoarseFundamentalDataProviderTests.cs diff --git a/Common/Data/UniverseSelection/CoarseFundamentalDataProvider.cs b/Common/Data/UniverseSelection/CoarseFundamentalDataProvider.cs index 15ee5258dd74..98df69021318 100644 --- a/Common/Data/UniverseSelection/CoarseFundamentalDataProvider.cs +++ b/Common/Data/UniverseSelection/CoarseFundamentalDataProvider.cs @@ -17,6 +17,7 @@ using System.IO; using System.Collections.Generic; using QuantConnect.Data.Fundamental; +using QuantConnect.Logging; namespace QuantConnect.Data.UniverseSelection { @@ -49,6 +50,18 @@ public override T Get(DateTime time, SecurityIdentifier securityIdentifier, F var path = Path.Combine(Globals.DataFolder, "equity", "usa", "fundamental", "coarse", $"{time:yyyyMMdd}.csv"); var fileStream = DataProvider.Fetch(path); + if (fileStream == null && LiveMode) + { + // in live trading, fall back to the backup universe file, if any, as a last resort, consistent with the + // universe selection data itself, see LiveCustomDataSubscriptionEnumeratorFactory.BackupUniverseFileDataProvider + var backupPath = path + ".backup"; + fileStream = DataProvider.Fetch(backupPath); + if (fileStream != null) + { + Log.Trace($"CoarseFundamentalDataProvider.Get(): coarse fundamental file '{path}' is not available, " + + $"falling back to backup file '{backupPath}'"); + } + } if (fileStream == null) { return GetDefault(); diff --git a/Tests/Common/Data/UniverseSelection/CoarseFundamentalDataProviderTests.cs b/Tests/Common/Data/UniverseSelection/CoarseFundamentalDataProviderTests.cs new file mode 100644 index 000000000000..239129cd6e6b --- /dev/null +++ b/Tests/Common/Data/UniverseSelection/CoarseFundamentalDataProviderTests.cs @@ -0,0 +1,135 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.IO; +using System.Text; +using NUnit.Framework; +using QuantConnect.Data.Fundamental; +using QuantConnect.Data.UniverseSelection; +using QuantConnect.Interfaces; + +namespace QuantConnect.Tests.Common.Data.UniverseSelection +{ + [TestFixture] + public class CoarseFundamentalDataProviderTests + { + private static readonly DateTime Date = new DateTime(2014, 03, 26); + private const string CoarseLine = "SPY R735QTJ8XC9X,SPY,537.46,5483955,3490219402,True,0.5,0.25"; + + [Test] + public void FallsBackToBackupUniverseFileInLiveModeWhenExpectedFileIsNotAvailable() + { + var dataProvider = new BackupCoarseFileDataProvider(coarseFileAvailable: false, backupCoarseFileAvailable: true); + var provider = CreateProvider(dataProvider, liveMode: true); + + var price = provider.Get(Date, Symbols.SPY.ID, FundamentalProperty.Value); + + Assert.AreEqual(537.46m, price); + Assert.AreEqual(1, dataProvider.CoarseFileRequests); + Assert.AreEqual(1, dataProvider.BackupCoarseFileRequests); + + // the file contents are cached for the date, no further fetches + var priceFactor = provider.Get(Date, Symbols.SPY.ID, FundamentalProperty.PriceFactor); + Assert.AreEqual(0.5m, priceFactor); + Assert.AreEqual(1, dataProvider.CoarseFileRequests); + Assert.AreEqual(1, dataProvider.BackupCoarseFileRequests); + } + + [Test] + public void DoesNotFallBackToBackupUniverseFileWhenExpectedFileIsAvailable() + { + var dataProvider = new BackupCoarseFileDataProvider(coarseFileAvailable: true, backupCoarseFileAvailable: true); + var provider = CreateProvider(dataProvider, liveMode: true); + + var price = provider.Get(Date, Symbols.SPY.ID, FundamentalProperty.Value); + + Assert.AreEqual(537.46m, price); + Assert.AreEqual(1, dataProvider.CoarseFileRequests); + Assert.AreEqual(0, dataProvider.BackupCoarseFileRequests); + } + + [Test] + public void DoesNotFallBackToBackupUniverseFileWhenNotInLiveMode() + { + var dataProvider = new BackupCoarseFileDataProvider(coarseFileAvailable: false, backupCoarseFileAvailable: true); + var provider = CreateProvider(dataProvider, liveMode: false); + + var price = provider.Get(Date, Symbols.SPY.ID, FundamentalProperty.Value); + + Assert.AreEqual(decimal.Zero, price); + Assert.AreEqual(1, dataProvider.CoarseFileRequests); + Assert.AreEqual(0, dataProvider.BackupCoarseFileRequests); + } + + [Test] + public void ReturnsDefaultsInLiveModeWhenNeitherTheExpectedNorTheBackupUniverseFilesAreAvailable() + { + var dataProvider = new BackupCoarseFileDataProvider(coarseFileAvailable: false, backupCoarseFileAvailable: false); + var provider = CreateProvider(dataProvider, liveMode: true); + + var price = provider.Get(Date, Symbols.SPY.ID, FundamentalProperty.Value); + + Assert.AreEqual(decimal.Zero, price); + Assert.AreEqual(1, dataProvider.CoarseFileRequests); + Assert.AreEqual(1, dataProvider.BackupCoarseFileRequests); + } + + private static CoarseFundamentalDataProvider CreateProvider(IDataProvider dataProvider, bool liveMode) + { + var provider = new CoarseFundamentalDataProvider(); + provider.Initialize(dataProvider, liveMode); + return provider; + } + + private class BackupCoarseFileDataProvider : IDataProvider + { + private readonly string _coarseFilePath = Path.Combine(Globals.DataFolder, "equity", "usa", "fundamental", "coarse", $"{Date:yyyyMMdd}.csv"); + private readonly bool _coarseFileAvailable; + private readonly bool _backupCoarseFileAvailable; + + public int CoarseFileRequests { get; private set; } + + public int BackupCoarseFileRequests { get; private set; } + +#pragma warning disable 0067 // the event is never used + public event EventHandler NewDataRequest; +#pragma warning restore 0067 + + public BackupCoarseFileDataProvider(bool coarseFileAvailable, bool backupCoarseFileAvailable) + { + _coarseFileAvailable = coarseFileAvailable; + _backupCoarseFileAvailable = backupCoarseFileAvailable; + } + + public Stream Fetch(string key) + { + if (key == _coarseFilePath) + { + CoarseFileRequests++; + return _coarseFileAvailable ? new MemoryStream(Encoding.UTF8.GetBytes(CoarseLine)) : null; + } + + if (key == _coarseFilePath + ".backup") + { + BackupCoarseFileRequests++; + return _backupCoarseFileAvailable ? new MemoryStream(Encoding.UTF8.GetBytes(CoarseLine)) : null; + } + + return null; + } + } + } +} From 28a3091660398f665dc6f5125184a081201f6e0c Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 26 Aug 2026 13:12:08 -0400 Subject: [PATCH 2/5] Cover future chain universes in the backup universe file fallback tests --- Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs b/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs index 4c4cc80e799a..1fb116700e6a 100644 --- a/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs +++ b/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs @@ -287,6 +287,8 @@ public void LiveChainSelection(SecurityType securityType, Resolution resolution, [TestCase("OptionChain", true)] [TestCase("IndexOptionChain", false)] [TestCase("IndexOptionChain", true)] + [TestCase("FutureChain", false)] + [TestCase("FutureChain", true)] [TestCase("CoarseFundamental", false)] [TestCase("CoarseFundamental", true)] [TestCase("EtfConstituents", false)] @@ -298,6 +300,7 @@ public void UniverseSelectionFallsBackToBackupUniverseFileCloseToMarketOpen(stri { "OptionChain" => new DateTime(2014, 6, 9, 13, 15, 0), "IndexOptionChain" => new DateTime(2021, 1, 4, 14, 15, 0), + "FutureChain" => new DateTime(2014, 6, 9, 13, 15, 0), "CoarseFundamental" => new DateTime(2014, 3, 26, 13, 15, 0), "EtfConstituents" => new DateTime(2020, 12, 1, 14, 15, 0), _ => throw new ArgumentException($"Unexpected universe kind: {universeKind}") @@ -336,6 +339,16 @@ IEnumerable CoarseFilter(IEnumerable coarse) }); break; + case "FutureChain": + var future = _algorithm.AddFuture("ES"); + future.SetFilter(universe => + { + selectionHappened++; + selectedCount = universe.Count(); + return universe; + }); + break; + case "CoarseFundamental": _algorithm.UniverseSettings.Resolution = Resolution.Daily; _algorithm.AddUniverse(CoarseFilter); From 07febb35f15131c3bbce90c3c9ce5e1b25474de8 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 26 Aug 2026 13:23:25 -0400 Subject: [PATCH 3/5] Centralize the backup universe file fallback in a shared data provider wrapper Moves the backup universe file data provider out of the live custom data subscription enumerator factory so the coarse fundamental data provider can reuse it instead of duplicating the fallback logic. --- .../BackupUniverseFileDataProvider.cs | 83 ++++++++++++++ .../CoarseFundamentalDataProvider.cs | 30 ++--- ...CustomDataSubscriptionEnumeratorFactory.cs | 47 -------- .../BackupUniverseFileDataProviderTests.cs | 103 ++++++++++++++++++ 4 files changed, 203 insertions(+), 60 deletions(-) create mode 100644 Common/Data/UniverseSelection/BackupUniverseFileDataProvider.cs create mode 100644 Tests/Common/Data/UniverseSelection/BackupUniverseFileDataProviderTests.cs diff --git a/Common/Data/UniverseSelection/BackupUniverseFileDataProvider.cs b/Common/Data/UniverseSelection/BackupUniverseFileDataProvider.cs new file mode 100644 index 000000000000..a85427d58a95 --- /dev/null +++ b/Common/Data/UniverseSelection/BackupUniverseFileDataProvider.cs @@ -0,0 +1,83 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.IO; +using QuantConnect.Interfaces; +using QuantConnect.Logging; + +namespace QuantConnect.Data.UniverseSelection +{ + /// + /// Data provider wrapper that falls back to the backup universe file ("*.backup"), if any, + /// when the expected universe file can't be fetched, as a last resort + /// + public class BackupUniverseFileDataProvider : IDataProvider + { + private IDataProvider _dataProvider; + + /// + /// Event raised each time data fetch is finished (successfully or not) + /// + public event EventHandler NewDataRequest + { + add => _dataProvider?.NewDataRequest += value; + remove => _dataProvider?.NewDataRequest -= value; + } + + /// + /// Creates a new instance + /// + /// The data provider to wrap, can be set later with + public BackupUniverseFileDataProvider(IDataProvider dataProvider = null) + { + _dataProvider = dataProvider; + } + + /// + /// Sets the data provider to wrap, forwarding its events + /// + /// The data provider to wrap + public void SetDataProvider(IDataProvider dataProvider) + { + _dataProvider = dataProvider; + } + + /// + /// Retrieves data to be used in an algorithm, falling back to the backup universe file, if any, + /// when the requested file is not available + /// + /// A string representing where the data is stored + /// A of the data requested, or null if none is available + public Stream Fetch(string key) + { + var stream = _dataProvider.Fetch(key); + if (stream != null) + { + return stream; + } + + var backupKey = key + ".backup"; + stream = _dataProvider.Fetch(backupKey); + if (stream != null) + { + Log.Trace($"BackupUniverseFileDataProvider.Fetch(): universe file '{key}' is not available, " + + $"falling back to backup universe file '{backupKey}'"); + } + + return stream; + } + } +} diff --git a/Common/Data/UniverseSelection/CoarseFundamentalDataProvider.cs b/Common/Data/UniverseSelection/CoarseFundamentalDataProvider.cs index 98df69021318..3aeac762dd11 100644 --- a/Common/Data/UniverseSelection/CoarseFundamentalDataProvider.cs +++ b/Common/Data/UniverseSelection/CoarseFundamentalDataProvider.cs @@ -17,7 +17,7 @@ using System.IO; using System.Collections.Generic; using QuantConnect.Data.Fundamental; -using QuantConnect.Logging; +using QuantConnect.Interfaces; namespace QuantConnect.Data.UniverseSelection { @@ -29,6 +29,22 @@ public class CoarseFundamentalDataProvider : BaseFundamentalDataProvider private DateTime _date; private readonly Dictionary _coarseFundamental = new(); + /// + /// Initializes the service + /// + /// The data provider instance to use + /// True if running in live mode + public override void Initialize(IDataProvider dataProvider, bool liveMode) + { + base.Initialize(dataProvider, liveMode); + if (liveMode) + { + // in live trading, fall back to the backup coarse universe file, if any, as a last resort, + // consistent with the universe selection data itself + DataProvider = new BackupUniverseFileDataProvider(dataProvider); + } + } + /// /// Will fetch the requested fundamental information for the requested time and symbol /// @@ -50,18 +66,6 @@ public override T Get(DateTime time, SecurityIdentifier securityIdentifier, F var path = Path.Combine(Globals.DataFolder, "equity", "usa", "fundamental", "coarse", $"{time:yyyyMMdd}.csv"); var fileStream = DataProvider.Fetch(path); - if (fileStream == null && LiveMode) - { - // in live trading, fall back to the backup universe file, if any, as a last resort, consistent with the - // universe selection data itself, see LiveCustomDataSubscriptionEnumeratorFactory.BackupUniverseFileDataProvider - var backupPath = path + ".backup"; - fileStream = DataProvider.Fetch(backupPath); - if (fileStream != null) - { - Log.Trace($"CoarseFundamentalDataProvider.Get(): coarse fundamental file '{path}' is not available, " + - $"falling back to backup file '{backupPath}'"); - } - } if (fileStream == null) { return GetDefault(); diff --git a/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactory.cs b/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactory.cs index 7b7f837db017..b20bfa03342d 100644 --- a/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactory.cs +++ b/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactory.cs @@ -16,14 +16,12 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using Python.Runtime; using QuantConnect.Configuration; using QuantConnect.Data; using QuantConnect.Data.UniverseSelection; using QuantConnect.Interfaces; -using QuantConnect.Logging; using QuantConnect.Securities; using QuantConnect.Util; @@ -261,50 +259,5 @@ private static TimeSpan GetMaximumDataAge(TimeSpan increment) { return TimeSpan.FromTicks(Math.Max(increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); } - - /// - /// Data provider wrapper that falls back to the backup universe file ("*.backup"), if any, - /// when the expected universe file can't be fetched, as a last resort - /// - private sealed class BackupUniverseFileDataProvider : IDataProvider - { - private IDataProvider _dataProvider; - - /// - /// Event raised each time data fetch is finished (successfully or not) - /// - public event EventHandler NewDataRequest - { - add => _dataProvider?.NewDataRequest += value; - remove => _dataProvider?.NewDataRequest -= value; - } - - /// - /// Sets the data provider to wrap, forwarding its events - /// - public void SetDataProvider(IDataProvider dataProvider) - { - _dataProvider = dataProvider; - } - - public Stream Fetch(string key) - { - var stream = _dataProvider.Fetch(key); - if (stream != null) - { - return stream; - } - - var backupKey = key + ".backup"; - stream = _dataProvider.Fetch(backupKey); - if (stream != null) - { - Log.Trace($"LiveCustomDataSubscriptionEnumeratorFactory.BackupUniverseFileDataProvider.Fetch(): universe file '{key}' is not available, " + - $"falling back to backup universe file '{backupKey}'"); - } - - return stream; - } - } } } diff --git a/Tests/Common/Data/UniverseSelection/BackupUniverseFileDataProviderTests.cs b/Tests/Common/Data/UniverseSelection/BackupUniverseFileDataProviderTests.cs new file mode 100644 index 000000000000..22c91b092d2a --- /dev/null +++ b/Tests/Common/Data/UniverseSelection/BackupUniverseFileDataProviderTests.cs @@ -0,0 +1,103 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.IO; +using Moq; +using NUnit.Framework; +using QuantConnect.Data.UniverseSelection; +using QuantConnect.Interfaces; + +namespace QuantConnect.Tests.Common.Data.UniverseSelection +{ + [TestFixture] + public class BackupUniverseFileDataProviderTests + { + private const string Key = "universes/20250815.csv"; + private const string BackupKey = "universes/20250815.csv.backup"; + + [Test] + public void ReturnsTheExpectedFileWithoutTouchingTheBackupFile() + { + var dataProvider = new Mock(); + dataProvider.Setup(dp => dp.Fetch(Key)).Returns(() => new MemoryStream()); + var backupDataProvider = new BackupUniverseFileDataProvider(dataProvider.Object); + + using var stream = backupDataProvider.Fetch(Key); + + Assert.IsNotNull(stream); + dataProvider.Verify(dp => dp.Fetch(Key), Times.Once); + dataProvider.Verify(dp => dp.Fetch(BackupKey), Times.Never); + } + + [Test] + public void FallsBackToTheBackupFileWhenTheExpectedFileIsNotAvailable() + { + var dataProvider = new Mock(); + dataProvider.Setup(dp => dp.Fetch(BackupKey)).Returns(() => new MemoryStream()); + var backupDataProvider = new BackupUniverseFileDataProvider(dataProvider.Object); + + using var stream = backupDataProvider.Fetch(Key); + + Assert.IsNotNull(stream); + dataProvider.Verify(dp => dp.Fetch(Key), Times.Once); + dataProvider.Verify(dp => dp.Fetch(BackupKey), Times.Once); + } + + [Test] + public void ReturnsNullWhenNeitherTheExpectedNorTheBackupFilesAreAvailable() + { + var dataProvider = new Mock(); + var backupDataProvider = new BackupUniverseFileDataProvider(dataProvider.Object); + + using var stream = backupDataProvider.Fetch(Key); + + Assert.IsNull(stream); + dataProvider.Verify(dp => dp.Fetch(Key), Times.Once); + dataProvider.Verify(dp => dp.Fetch(BackupKey), Times.Once); + } + + [Test] + public void UsesTheDataProviderSetAfterConstruction() + { + var dataProvider = new Mock(); + dataProvider.Setup(dp => dp.Fetch(BackupKey)).Returns(() => new MemoryStream()); + var backupDataProvider = new BackupUniverseFileDataProvider(); + backupDataProvider.SetDataProvider(dataProvider.Object); + + using var stream = backupDataProvider.Fetch(Key); + + Assert.IsNotNull(stream); + dataProvider.Verify(dp => dp.Fetch(BackupKey), Times.Once); + } + + [Test] + public void ForwardsNewDataRequestEventsToTheWrappedDataProvider() + { + var dataProvider = new Mock(); + var backupDataProvider = new BackupUniverseFileDataProvider(dataProvider.Object); + + var raised = 0; + EventHandler handler = (_, _) => raised++; + backupDataProvider.NewDataRequest += handler; + dataProvider.Raise(dp => dp.NewDataRequest += null, new DataProviderNewDataRequestEventArgs(Key, true, "")); + Assert.AreEqual(1, raised); + + backupDataProvider.NewDataRequest -= handler; + dataProvider.Raise(dp => dp.NewDataRequest += null, new DataProviderNewDataRequestEventArgs(Key, true, "")); + Assert.AreEqual(1, raised); + } + } +} From ed6c76b303993785915ad4938792d9faaf01314f Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 26 Aug 2026 14:31:22 -0400 Subject: [PATCH 4/5] Pace the backup universe file fallback logging --- .../BackupUniverseFileDataProvider.cs | 28 ++++++++++++++++- .../BackupUniverseFileDataProviderTests.cs | 31 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/Common/Data/UniverseSelection/BackupUniverseFileDataProvider.cs b/Common/Data/UniverseSelection/BackupUniverseFileDataProvider.cs index a85427d58a95..52eb54de0253 100644 --- a/Common/Data/UniverseSelection/BackupUniverseFileDataProvider.cs +++ b/Common/Data/UniverseSelection/BackupUniverseFileDataProvider.cs @@ -26,6 +26,14 @@ namespace QuantConnect.Data.UniverseSelection /// public class BackupUniverseFileDataProvider : IDataProvider { + // the fallback is retried on every universe refresh for as long as the expected file is missing, + // so the fallback trace is paced to avoid flooding the logs + private const int MaximumLogsPerWindow = 30; + private static readonly TimeSpan LogWindow = TimeSpan.FromMinutes(5); + private static readonly object LogLock = new(); + private DateTime _logWindowStartUtc; + private int _logCount; + private IDataProvider _dataProvider; /// @@ -71,7 +79,7 @@ public Stream Fetch(string key) var backupKey = key + ".backup"; stream = _dataProvider.Fetch(backupKey); - if (stream != null) + if (stream != null && ShouldLog()) { Log.Trace($"BackupUniverseFileDataProvider.Fetch(): universe file '{key}' is not available, " + $"falling back to backup universe file '{backupKey}'"); @@ -79,5 +87,23 @@ public Stream Fetch(string key) return stream; } + + /// + /// Determines whether the fallback should be logged, allowing up to logs per + /// + private bool ShouldLog() + { + lock (LogLock) + { + var utcNow = DateTime.UtcNow; + if (utcNow - _logWindowStartUtc >= LogWindow) + { + _logWindowStartUtc = utcNow; + _logCount = 0; + } + + return _logCount++ < MaximumLogsPerWindow; + } + } } } diff --git a/Tests/Common/Data/UniverseSelection/BackupUniverseFileDataProviderTests.cs b/Tests/Common/Data/UniverseSelection/BackupUniverseFileDataProviderTests.cs index 22c91b092d2a..d19ac6feee7f 100644 --- a/Tests/Common/Data/UniverseSelection/BackupUniverseFileDataProviderTests.cs +++ b/Tests/Common/Data/UniverseSelection/BackupUniverseFileDataProviderTests.cs @@ -15,10 +15,12 @@ using System; using System.IO; +using System.Linq; using Moq; using NUnit.Framework; using QuantConnect.Data.UniverseSelection; using QuantConnect.Interfaces; +using QuantConnect.Logging; namespace QuantConnect.Tests.Common.Data.UniverseSelection { @@ -83,6 +85,35 @@ public void UsesTheDataProviderSetAfterConstruction() dataProvider.Verify(dp => dp.Fetch(BackupKey), Times.Once); } + [Test] + public void PacesTheFallbackLogging() + { + var dataProvider = new Mock(); + dataProvider.Setup(dp => dp.Fetch(BackupKey)).Returns(() => new MemoryStream()); + var backupDataProvider = new BackupUniverseFileDataProvider(dataProvider.Object); + + var previousLogHandler = Log.LogHandler; + var logHandler = new QueueLogHandler(); + Log.LogHandler = logHandler; + try + { + for (var i = 0; i < 100; i++) + { + using var stream = backupDataProvider.Fetch(Key); + Assert.IsNotNull(stream); + } + } + finally + { + Log.LogHandler = previousLogHandler; + } + + // every fetch fell back to the backup file, but only the first few of them were logged + dataProvider.Verify(dp => dp.Fetch(BackupKey), Times.Exactly(100)); + var fallbackLogs = logHandler.Logs.Count(log => log.Message.Contains("falling back to backup universe file", StringComparison.InvariantCulture)); + Assert.AreEqual(30, fallbackLogs); + } + [Test] public void ForwardsNewDataRequestEventsToTheWrappedDataProvider() { From 1a3a518e5f5c7f6f50b252e62d551937ffc8fed1 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 26 Aug 2026 14:34:00 -0400 Subject: [PATCH 5/5] Use distinct universe file keys in the fallback log pacing test --- .../BackupUniverseFileDataProviderTests.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Tests/Common/Data/UniverseSelection/BackupUniverseFileDataProviderTests.cs b/Tests/Common/Data/UniverseSelection/BackupUniverseFileDataProviderTests.cs index d19ac6feee7f..86f751f8a934 100644 --- a/Tests/Common/Data/UniverseSelection/BackupUniverseFileDataProviderTests.cs +++ b/Tests/Common/Data/UniverseSelection/BackupUniverseFileDataProviderTests.cs @@ -89,7 +89,7 @@ public void UsesTheDataProviderSetAfterConstruction() public void PacesTheFallbackLogging() { var dataProvider = new Mock(); - dataProvider.Setup(dp => dp.Fetch(BackupKey)).Returns(() => new MemoryStream()); + dataProvider.Setup(dp => dp.Fetch(It.Is(key => key.EndsWith(".backup")))).Returns(() => new MemoryStream()); var backupDataProvider = new BackupUniverseFileDataProvider(dataProvider.Object); var previousLogHandler = Log.LogHandler; @@ -97,9 +97,10 @@ public void PacesTheFallbackLogging() Log.LogHandler = logHandler; try { + // distinct keys so that the Log's own identical consecutive message protection does not kick in for (var i = 0; i < 100; i++) { - using var stream = backupDataProvider.Fetch(Key); + using var stream = backupDataProvider.Fetch($"universes/{i:D8}.csv"); Assert.IsNotNull(stream); } } @@ -109,7 +110,7 @@ public void PacesTheFallbackLogging() } // every fetch fell back to the backup file, but only the first few of them were logged - dataProvider.Verify(dp => dp.Fetch(BackupKey), Times.Exactly(100)); + dataProvider.Verify(dp => dp.Fetch(It.Is(key => key.EndsWith(".backup"))), Times.Exactly(100)); var fallbackLogs = logHandler.Logs.Count(log => log.Message.Contains("falling back to backup universe file", StringComparison.InvariantCulture)); Assert.AreEqual(30, fallbackLogs); }