diff --git a/Common/Data/UniverseSelection/BackupUniverseFileDataProvider.cs b/Common/Data/UniverseSelection/BackupUniverseFileDataProvider.cs
new file mode 100644
index 000000000000..52eb54de0253
--- /dev/null
+++ b/Common/Data/UniverseSelection/BackupUniverseFileDataProvider.cs
@@ -0,0 +1,109 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.IO;
+using QuantConnect.Interfaces;
+using QuantConnect.Logging;
+
+namespace QuantConnect.Data.UniverseSelection
+{
+ ///
+ /// Data provider wrapper that falls back to the backup universe file ("*.backup"), if any,
+ /// when the expected universe file can't be fetched, as a last resort
+ ///
+ public class BackupUniverseFileDataProvider : IDataProvider
+ {
+ // the fallback is retried on every universe refresh for as long as the expected file is missing,
+ // so the fallback trace is paced to avoid flooding the logs
+ private const int MaximumLogsPerWindow = 30;
+ private static readonly TimeSpan LogWindow = TimeSpan.FromMinutes(5);
+ private static readonly object LogLock = new();
+ private DateTime _logWindowStartUtc;
+ private int _logCount;
+
+ private IDataProvider _dataProvider;
+
+ ///
+ /// Event raised each time data fetch is finished (successfully or not)
+ ///
+ public event EventHandler NewDataRequest
+ {
+ add => _dataProvider?.NewDataRequest += value;
+ remove => _dataProvider?.NewDataRequest -= value;
+ }
+
+ ///
+ /// Creates a new instance
+ ///
+ /// The data provider to wrap, can be set later with
+ public BackupUniverseFileDataProvider(IDataProvider dataProvider = null)
+ {
+ _dataProvider = dataProvider;
+ }
+
+ ///
+ /// Sets the data provider to wrap, forwarding its events
+ ///
+ /// The data provider to wrap
+ public void SetDataProvider(IDataProvider dataProvider)
+ {
+ _dataProvider = dataProvider;
+ }
+
+ ///
+ /// Retrieves data to be used in an algorithm, falling back to the backup universe file, if any,
+ /// when the requested file is not available
+ ///
+ /// A string representing where the data is stored
+ /// A of the data requested, or null if none is available
+ public Stream Fetch(string key)
+ {
+ var stream = _dataProvider.Fetch(key);
+ if (stream != null)
+ {
+ return stream;
+ }
+
+ var backupKey = key + ".backup";
+ stream = _dataProvider.Fetch(backupKey);
+ if (stream != null && ShouldLog())
+ {
+ Log.Trace($"BackupUniverseFileDataProvider.Fetch(): universe file '{key}' is not available, " +
+ $"falling back to backup universe file '{backupKey}'");
+ }
+
+ return stream;
+ }
+
+ ///
+ /// Determines whether the fallback should be logged, allowing up to logs per
+ ///
+ private bool ShouldLog()
+ {
+ lock (LogLock)
+ {
+ var utcNow = DateTime.UtcNow;
+ if (utcNow - _logWindowStartUtc >= LogWindow)
+ {
+ _logWindowStartUtc = utcNow;
+ _logCount = 0;
+ }
+
+ return _logCount++ < MaximumLogsPerWindow;
+ }
+ }
+ }
+}
diff --git a/Common/Data/UniverseSelection/CoarseFundamentalDataProvider.cs b/Common/Data/UniverseSelection/CoarseFundamentalDataProvider.cs
index 15ee5258dd74..3aeac762dd11 100644
--- a/Common/Data/UniverseSelection/CoarseFundamentalDataProvider.cs
+++ b/Common/Data/UniverseSelection/CoarseFundamentalDataProvider.cs
@@ -17,6 +17,7 @@
using System.IO;
using System.Collections.Generic;
using QuantConnect.Data.Fundamental;
+using QuantConnect.Interfaces;
namespace QuantConnect.Data.UniverseSelection
{
@@ -28,6 +29,22 @@ public class CoarseFundamentalDataProvider : BaseFundamentalDataProvider
private DateTime _date;
private readonly Dictionary _coarseFundamental = new();
+ ///
+ /// Initializes the service
+ ///
+ /// The data provider instance to use
+ /// True if running in live mode
+ public override void Initialize(IDataProvider dataProvider, bool liveMode)
+ {
+ base.Initialize(dataProvider, liveMode);
+ if (liveMode)
+ {
+ // in live trading, fall back to the backup coarse universe file, if any, as a last resort,
+ // consistent with the universe selection data itself
+ DataProvider = new BackupUniverseFileDataProvider(dataProvider);
+ }
+ }
+
///
/// Will fetch the requested fundamental information for the requested time and symbol
///
diff --git a/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactory.cs b/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactory.cs
index 7b7f837db017..b20bfa03342d 100644
--- a/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactory.cs
+++ b/Engine/DataFeeds/Enumerators/Factories/LiveCustomDataSubscriptionEnumeratorFactory.cs
@@ -16,14 +16,12 @@
using System;
using System.Collections.Generic;
-using System.IO;
using System.Linq;
using Python.Runtime;
using QuantConnect.Configuration;
using QuantConnect.Data;
using QuantConnect.Data.UniverseSelection;
using QuantConnect.Interfaces;
-using QuantConnect.Logging;
using QuantConnect.Securities;
using QuantConnect.Util;
@@ -261,50 +259,5 @@ private static TimeSpan GetMaximumDataAge(TimeSpan increment)
{
return TimeSpan.FromTicks(Math.Max(increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
}
-
- ///
- /// Data provider wrapper that falls back to the backup universe file ("*.backup"), if any,
- /// when the expected universe file can't be fetched, as a last resort
- ///
- private sealed class BackupUniverseFileDataProvider : IDataProvider
- {
- private IDataProvider _dataProvider;
-
- ///
- /// Event raised each time data fetch is finished (successfully or not)
- ///
- public event EventHandler NewDataRequest
- {
- add => _dataProvider?.NewDataRequest += value;
- remove => _dataProvider?.NewDataRequest -= value;
- }
-
- ///
- /// Sets the data provider to wrap, forwarding its events
- ///
- public void SetDataProvider(IDataProvider dataProvider)
- {
- _dataProvider = dataProvider;
- }
-
- public Stream Fetch(string key)
- {
- var stream = _dataProvider.Fetch(key);
- if (stream != null)
- {
- return stream;
- }
-
- var backupKey = key + ".backup";
- stream = _dataProvider.Fetch(backupKey);
- if (stream != null)
- {
- Log.Trace($"LiveCustomDataSubscriptionEnumeratorFactory.BackupUniverseFileDataProvider.Fetch(): universe file '{key}' is not available, " +
- $"falling back to backup universe file '{backupKey}'");
- }
-
- return stream;
- }
- }
}
}
diff --git a/Tests/Common/Data/UniverseSelection/BackupUniverseFileDataProviderTests.cs b/Tests/Common/Data/UniverseSelection/BackupUniverseFileDataProviderTests.cs
new file mode 100644
index 000000000000..86f751f8a934
--- /dev/null
+++ b/Tests/Common/Data/UniverseSelection/BackupUniverseFileDataProviderTests.cs
@@ -0,0 +1,135 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.IO;
+using System.Linq;
+using Moq;
+using NUnit.Framework;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+using QuantConnect.Logging;
+
+namespace QuantConnect.Tests.Common.Data.UniverseSelection
+{
+ [TestFixture]
+ public class BackupUniverseFileDataProviderTests
+ {
+ private const string Key = "universes/20250815.csv";
+ private const string BackupKey = "universes/20250815.csv.backup";
+
+ [Test]
+ public void ReturnsTheExpectedFileWithoutTouchingTheBackupFile()
+ {
+ var dataProvider = new Mock();
+ dataProvider.Setup(dp => dp.Fetch(Key)).Returns(() => new MemoryStream());
+ var backupDataProvider = new BackupUniverseFileDataProvider(dataProvider.Object);
+
+ using var stream = backupDataProvider.Fetch(Key);
+
+ Assert.IsNotNull(stream);
+ dataProvider.Verify(dp => dp.Fetch(Key), Times.Once);
+ dataProvider.Verify(dp => dp.Fetch(BackupKey), Times.Never);
+ }
+
+ [Test]
+ public void FallsBackToTheBackupFileWhenTheExpectedFileIsNotAvailable()
+ {
+ var dataProvider = new Mock();
+ dataProvider.Setup(dp => dp.Fetch(BackupKey)).Returns(() => new MemoryStream());
+ var backupDataProvider = new BackupUniverseFileDataProvider(dataProvider.Object);
+
+ using var stream = backupDataProvider.Fetch(Key);
+
+ Assert.IsNotNull(stream);
+ dataProvider.Verify(dp => dp.Fetch(Key), Times.Once);
+ dataProvider.Verify(dp => dp.Fetch(BackupKey), Times.Once);
+ }
+
+ [Test]
+ public void ReturnsNullWhenNeitherTheExpectedNorTheBackupFilesAreAvailable()
+ {
+ var dataProvider = new Mock();
+ var backupDataProvider = new BackupUniverseFileDataProvider(dataProvider.Object);
+
+ using var stream = backupDataProvider.Fetch(Key);
+
+ Assert.IsNull(stream);
+ dataProvider.Verify(dp => dp.Fetch(Key), Times.Once);
+ dataProvider.Verify(dp => dp.Fetch(BackupKey), Times.Once);
+ }
+
+ [Test]
+ public void UsesTheDataProviderSetAfterConstruction()
+ {
+ var dataProvider = new Mock();
+ dataProvider.Setup(dp => dp.Fetch(BackupKey)).Returns(() => new MemoryStream());
+ var backupDataProvider = new BackupUniverseFileDataProvider();
+ backupDataProvider.SetDataProvider(dataProvider.Object);
+
+ using var stream = backupDataProvider.Fetch(Key);
+
+ Assert.IsNotNull(stream);
+ dataProvider.Verify(dp => dp.Fetch(BackupKey), Times.Once);
+ }
+
+ [Test]
+ public void PacesTheFallbackLogging()
+ {
+ var dataProvider = new Mock();
+ dataProvider.Setup(dp => dp.Fetch(It.Is(key => key.EndsWith(".backup")))).Returns(() => new MemoryStream());
+ var backupDataProvider = new BackupUniverseFileDataProvider(dataProvider.Object);
+
+ var previousLogHandler = Log.LogHandler;
+ var logHandler = new QueueLogHandler();
+ Log.LogHandler = logHandler;
+ try
+ {
+ // distinct keys so that the Log's own identical consecutive message protection does not kick in
+ for (var i = 0; i < 100; i++)
+ {
+ using var stream = backupDataProvider.Fetch($"universes/{i:D8}.csv");
+ Assert.IsNotNull(stream);
+ }
+ }
+ finally
+ {
+ Log.LogHandler = previousLogHandler;
+ }
+
+ // every fetch fell back to the backup file, but only the first few of them were logged
+ dataProvider.Verify(dp => dp.Fetch(It.Is(key => key.EndsWith(".backup"))), Times.Exactly(100));
+ var fallbackLogs = logHandler.Logs.Count(log => log.Message.Contains("falling back to backup universe file", StringComparison.InvariantCulture));
+ Assert.AreEqual(30, fallbackLogs);
+ }
+
+ [Test]
+ public void ForwardsNewDataRequestEventsToTheWrappedDataProvider()
+ {
+ var dataProvider = new Mock();
+ var backupDataProvider = new BackupUniverseFileDataProvider(dataProvider.Object);
+
+ var raised = 0;
+ EventHandler handler = (_, _) => raised++;
+ backupDataProvider.NewDataRequest += handler;
+ dataProvider.Raise(dp => dp.NewDataRequest += null, new DataProviderNewDataRequestEventArgs(Key, true, ""));
+ Assert.AreEqual(1, raised);
+
+ backupDataProvider.NewDataRequest -= handler;
+ dataProvider.Raise(dp => dp.NewDataRequest += null, new DataProviderNewDataRequestEventArgs(Key, true, ""));
+ Assert.AreEqual(1, raised);
+ }
+ }
+}
diff --git a/Tests/Common/Data/UniverseSelection/CoarseFundamentalDataProviderTests.cs b/Tests/Common/Data/UniverseSelection/CoarseFundamentalDataProviderTests.cs
new file mode 100644
index 000000000000..239129cd6e6b
--- /dev/null
+++ b/Tests/Common/Data/UniverseSelection/CoarseFundamentalDataProviderTests.cs
@@ -0,0 +1,135 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.IO;
+using System.Text;
+using NUnit.Framework;
+using QuantConnect.Data.Fundamental;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Tests.Common.Data.UniverseSelection
+{
+ [TestFixture]
+ public class CoarseFundamentalDataProviderTests
+ {
+ private static readonly DateTime Date = new DateTime(2014, 03, 26);
+ private const string CoarseLine = "SPY R735QTJ8XC9X,SPY,537.46,5483955,3490219402,True,0.5,0.25";
+
+ [Test]
+ public void FallsBackToBackupUniverseFileInLiveModeWhenExpectedFileIsNotAvailable()
+ {
+ var dataProvider = new BackupCoarseFileDataProvider(coarseFileAvailable: false, backupCoarseFileAvailable: true);
+ var provider = CreateProvider(dataProvider, liveMode: true);
+
+ var price = provider.Get(Date, Symbols.SPY.ID, FundamentalProperty.Value);
+
+ Assert.AreEqual(537.46m, price);
+ Assert.AreEqual(1, dataProvider.CoarseFileRequests);
+ Assert.AreEqual(1, dataProvider.BackupCoarseFileRequests);
+
+ // the file contents are cached for the date, no further fetches
+ var priceFactor = provider.Get(Date, Symbols.SPY.ID, FundamentalProperty.PriceFactor);
+ Assert.AreEqual(0.5m, priceFactor);
+ Assert.AreEqual(1, dataProvider.CoarseFileRequests);
+ Assert.AreEqual(1, dataProvider.BackupCoarseFileRequests);
+ }
+
+ [Test]
+ public void DoesNotFallBackToBackupUniverseFileWhenExpectedFileIsAvailable()
+ {
+ var dataProvider = new BackupCoarseFileDataProvider(coarseFileAvailable: true, backupCoarseFileAvailable: true);
+ var provider = CreateProvider(dataProvider, liveMode: true);
+
+ var price = provider.Get(Date, Symbols.SPY.ID, FundamentalProperty.Value);
+
+ Assert.AreEqual(537.46m, price);
+ Assert.AreEqual(1, dataProvider.CoarseFileRequests);
+ Assert.AreEqual(0, dataProvider.BackupCoarseFileRequests);
+ }
+
+ [Test]
+ public void DoesNotFallBackToBackupUniverseFileWhenNotInLiveMode()
+ {
+ var dataProvider = new BackupCoarseFileDataProvider(coarseFileAvailable: false, backupCoarseFileAvailable: true);
+ var provider = CreateProvider(dataProvider, liveMode: false);
+
+ var price = provider.Get(Date, Symbols.SPY.ID, FundamentalProperty.Value);
+
+ Assert.AreEqual(decimal.Zero, price);
+ Assert.AreEqual(1, dataProvider.CoarseFileRequests);
+ Assert.AreEqual(0, dataProvider.BackupCoarseFileRequests);
+ }
+
+ [Test]
+ public void ReturnsDefaultsInLiveModeWhenNeitherTheExpectedNorTheBackupUniverseFilesAreAvailable()
+ {
+ var dataProvider = new BackupCoarseFileDataProvider(coarseFileAvailable: false, backupCoarseFileAvailable: false);
+ var provider = CreateProvider(dataProvider, liveMode: true);
+
+ var price = provider.Get(Date, Symbols.SPY.ID, FundamentalProperty.Value);
+
+ Assert.AreEqual(decimal.Zero, price);
+ Assert.AreEqual(1, dataProvider.CoarseFileRequests);
+ Assert.AreEqual(1, dataProvider.BackupCoarseFileRequests);
+ }
+
+ private static CoarseFundamentalDataProvider CreateProvider(IDataProvider dataProvider, bool liveMode)
+ {
+ var provider = new CoarseFundamentalDataProvider();
+ provider.Initialize(dataProvider, liveMode);
+ return provider;
+ }
+
+ private class BackupCoarseFileDataProvider : IDataProvider
+ {
+ private readonly string _coarseFilePath = Path.Combine(Globals.DataFolder, "equity", "usa", "fundamental", "coarse", $"{Date:yyyyMMdd}.csv");
+ private readonly bool _coarseFileAvailable;
+ private readonly bool _backupCoarseFileAvailable;
+
+ public int CoarseFileRequests { get; private set; }
+
+ public int BackupCoarseFileRequests { get; private set; }
+
+#pragma warning disable 0067 // the event is never used
+ public event EventHandler NewDataRequest;
+#pragma warning restore 0067
+
+ public BackupCoarseFileDataProvider(bool coarseFileAvailable, bool backupCoarseFileAvailable)
+ {
+ _coarseFileAvailable = coarseFileAvailable;
+ _backupCoarseFileAvailable = backupCoarseFileAvailable;
+ }
+
+ public Stream Fetch(string key)
+ {
+ if (key == _coarseFilePath)
+ {
+ CoarseFileRequests++;
+ return _coarseFileAvailable ? new MemoryStream(Encoding.UTF8.GetBytes(CoarseLine)) : null;
+ }
+
+ if (key == _coarseFilePath + ".backup")
+ {
+ BackupCoarseFileRequests++;
+ return _backupCoarseFileAvailable ? new MemoryStream(Encoding.UTF8.GetBytes(CoarseLine)) : null;
+ }
+
+ return null;
+ }
+ }
+ }
+}
diff --git a/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs b/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs
index 4c4cc80e799a..1fb116700e6a 100644
--- a/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs
+++ b/Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs
@@ -287,6 +287,8 @@ public void LiveChainSelection(SecurityType securityType, Resolution resolution,
[TestCase("OptionChain", true)]
[TestCase("IndexOptionChain", false)]
[TestCase("IndexOptionChain", true)]
+ [TestCase("FutureChain", false)]
+ [TestCase("FutureChain", true)]
[TestCase("CoarseFundamental", false)]
[TestCase("CoarseFundamental", true)]
[TestCase("EtfConstituents", false)]
@@ -298,6 +300,7 @@ public void UniverseSelectionFallsBackToBackupUniverseFileCloseToMarketOpen(stri
{
"OptionChain" => new DateTime(2014, 6, 9, 13, 15, 0),
"IndexOptionChain" => new DateTime(2021, 1, 4, 14, 15, 0),
+ "FutureChain" => new DateTime(2014, 6, 9, 13, 15, 0),
"CoarseFundamental" => new DateTime(2014, 3, 26, 13, 15, 0),
"EtfConstituents" => new DateTime(2020, 12, 1, 14, 15, 0),
_ => throw new ArgumentException($"Unexpected universe kind: {universeKind}")
@@ -336,6 +339,16 @@ IEnumerable CoarseFilter(IEnumerable coarse)
});
break;
+ case "FutureChain":
+ var future = _algorithm.AddFuture("ES");
+ future.SetFilter(universe =>
+ {
+ selectionHappened++;
+ selectedCount = universe.Count();
+ return universe;
+ });
+ break;
+
case "CoarseFundamental":
_algorithm.UniverseSettings.Resolution = Resolution.Daily;
_algorithm.AddUniverse(CoarseFilter);