diff --git a/Algorithm/QCAlgorithm.Python.cs b/Algorithm/QCAlgorithm.Python.cs index 9859cbc4094b..64861e343676 100644 --- a/Algorithm/QCAlgorithm.Python.cs +++ b/Algorithm/QCAlgorithm.Python.cs @@ -637,7 +637,9 @@ public void RegisterIndicator(Symbol symbol, PyObject indicator, PyObject pyObje try { - dataConsolidator = new DataConsolidatorPythonWrapper(pyObject); + // reuse the wrapper of an earlier registration, a new one would never be disposed on removal + dataConsolidator = SubscriptionManager.FindPythonConsolidator(symbol, pyObject) + ?? new DataConsolidatorPythonWrapper(pyObject); } catch { diff --git a/Common/Data/SubscriptionManager.cs b/Common/Data/SubscriptionManager.cs index 30770d779bc5..8feb72811f78 100644 --- a/Common/Data/SubscriptionManager.cs +++ b/Common/Data/SubscriptionManager.cs @@ -233,7 +233,9 @@ public void AddConsolidator(Symbol symbol, PyObject pyConsolidator) { var consolidator = PythonUtil.CreateInstanceOrWrapper( pyConsolidator, - py => new DataConsolidatorPythonWrapper(py) + // reuse the wrapper of an earlier registration: a new one would subscribe to the python event, be + // discarded by the consolidator collections as a duplicate, and so never be disposed on removal + py => FindPythonConsolidator(symbol, py) ?? new DataConsolidatorPythonWrapper(py) ); AddConsolidator(symbol, consolidator); } @@ -313,10 +315,10 @@ public void RemoveConsolidator(Symbol symbol, PyObject pyConsolidator) /// /// Finds the previously created for the given python - /// consolidator so it can be removed and disposed, rather than a throwaway wrapper that would churn - /// the live python object's event subscription and leak the original one. + /// consolidator so it can be reused or removed and disposed, rather than a throwaway wrapper that would + /// subscribe to the live python object's event and leak that subscription or the original one. /// - private IDataConsolidator FindPythonConsolidator(Symbol symbol, PyObject pyConsolidator) + internal IDataConsolidator FindPythonConsolidator(Symbol symbol, PyObject pyConsolidator) { var configs = symbol != null ? _subscriptionManager.GetSubscriptionDataConfigs(symbol) diff --git a/Common/Python/DataConsolidatorPythonWrapper.cs b/Common/Python/DataConsolidatorPythonWrapper.cs index 199b22e50292..7eeff6e3d575 100644 --- a/Common/Python/DataConsolidatorPythonWrapper.cs +++ b/Common/Python/DataConsolidatorPythonWrapper.cs @@ -34,7 +34,9 @@ public class DataConsolidatorPythonWrapper : ConsolidatorBase /// public override IBaseData WorkingData { - get { return _pythonWrapper.GetProperty(nameof(WorkingData)); } + // a disposed wrapper has released its python object, e.g. a consolidator shared by indicators that are + // deregistered one after the other is removed again after its first removal disposed it + get { return _disposed ? null : _pythonWrapper.GetProperty(nameof(WorkingData)); } } /// diff --git a/Tests/Common/Data/SubscriptionManagerTests.cs b/Tests/Common/Data/SubscriptionManagerTests.cs index ad5ca35d5157..0ce73697999e 100644 --- a/Tests/Common/Data/SubscriptionManagerTests.cs +++ b/Tests/Common/Data/SubscriptionManagerTests.cs @@ -28,6 +28,7 @@ using QuantConnect.Data.UniverseSelection; using QuantConnect.Indicators; using QuantConnect.Logging; +using QuantConnect.Python; using QuantConnect.Statistics; using QuantConnect.Tests.Engine.DataFeeds; using QuantConnect.Util; @@ -710,6 +711,120 @@ def get_consolidator(): Assert.AreEqual(0, algorithm.SubscriptionManager.Subscriptions.Sum(x => x.Consolidators.Count)); } + [Test] + public void RemovingAPythonConsolidatorAddedTwiceUnsubscribesFromItsEvent() + { + using var _ = Py.GIL(); + var algorithm = new AlgorithmStub(); + var symbol = algorithm.AddEquity("SPY").Symbol; + using var pyConsolidator = CreateCustomTradeBarPythonConsolidator(); + var source = pyConsolidator.As(); + + algorithm.SubscriptionManager.AddConsolidator(symbol, pyConsolidator); + algorithm.SubscriptionManager.AddConsolidator(symbol, pyConsolidator); + Assert.AreEqual(1, GetDataConsolidatedHandlerCount(source)); + + algorithm.SubscriptionManager.RemoveConsolidator(symbol, pyConsolidator); + Assert.IsEmpty(GetConsolidators(algorithm, symbol)); + Assert.AreEqual(0, GetDataConsolidatedHandlerCount(source)); + } + + [Test] + public void RemovingAPythonConsolidatorSharedByIndicatorsUnsubscribesFromItsEvent() + { + using var _ = Py.GIL(); + var algorithm = new AlgorithmStub(); + var symbol = algorithm.AddEquity("SPY").Symbol; + using var pyConsolidator = CreateCustomTradeBarPythonConsolidator(); + var source = pyConsolidator.As(); + + var sma = new SimpleMovingAverage(2); + var std = new StandardDeviation(2); + using var pySma = sma.ToPython(); + using var pyStd = std.ToPython(); + algorithm.RegisterIndicator(symbol, pySma, pyConsolidator); + algorithm.RegisterIndicator(symbol, pyStd, pyConsolidator); + Assert.AreEqual(1, GetDataConsolidatedHandlerCount(source)); + + // both indicators receive the bars the python consolidator emits + source.OnDataConsolidated(pyConsolidator, new TradeBar(algorithm.Time, symbol, 1, 1, 1, 1, 1, TimeSpan.FromMinutes(1))); + Assert.AreEqual(1, sma.Samples); + Assert.AreEqual(1, std.Samples); + + algorithm.SubscriptionManager.RemoveConsolidator(symbol, pyConsolidator); + Assert.IsEmpty(GetConsolidators(algorithm, symbol)); + Assert.AreEqual(0, GetDataConsolidatedHandlerCount(source)); + + source.OnDataConsolidated(pyConsolidator, new TradeBar(algorithm.Time, symbol, 1, 1, 1, 1, 1, TimeSpan.FromMinutes(1))); + Assert.AreEqual(1, sma.Samples); + Assert.AreEqual(1, std.Samples); + } + + [Test] + public void DeregisteringIndicatorsSharingAPythonConsolidatorUnsubscribesFromItsEvent() + { + using var _ = Py.GIL(); + using var module = PyModule.FromString(Guid.NewGuid().ToString(), @" +from AlgorithmImports import * + +def register(algorithm, symbol, consolidator, sma, std): + algorithm.register_indicator(symbol, sma, consolidator) + algorithm.register_indicator(symbol, std, consolidator) + +def deregister(algorithm, sma, std): + algorithm.deregister_indicator(sma) + algorithm.deregister_indicator(std) +"); + var algorithm = new AlgorithmStub(); + var symbol = algorithm.AddEquity("SPY").Symbol; + using var otherConsolidator = new TradeBarConsolidator(TimeSpan.FromMinutes(1)); + algorithm.SubscriptionManager.AddConsolidator(symbol, otherConsolidator); + using var pyConsolidator = CreateCustomTradeBarPythonConsolidator(); + var source = pyConsolidator.As(); + using var pyAlgorithm = algorithm.ToPython(); + using var pySymbol = symbol.ToPython(); + using var pySma = new SimpleMovingAverage(2).ToPython(); + using var pyStd = new StandardDeviation(2).ToPython(); + + module.GetAttr("register").Invoke(pyAlgorithm, pySymbol, pyConsolidator, pySma, pyStd).Dispose(); + Assert.AreEqual(1, GetDataConsolidatedHandlerCount(source)); + + // the shared consolidator has not emitted a bar yet, both deregistrations remove it + module.GetAttr("deregister").Invoke(pyAlgorithm, pySma, pyStd).Dispose(); + + CollectionAssert.AreEqual(new[] { otherConsolidator }, GetConsolidators(algorithm, symbol)); + Assert.AreEqual(0, GetDataConsolidatedHandlerCount(source)); + } + + private static PyObject CreateCustomTradeBarPythonConsolidator() + { + using var module = PyModule.FromString(Guid.NewGuid().ToString(), @" +from AlgorithmImports import * + +class CustomTradeBarConsolidator(PythonConsolidator): + def __init__(self): + self.consolidated = None + self.working_data = None + self.input_type = TradeBar + self.output_type = TradeBar + + def update(self, data): + pass + + def scan(self, time): + pass +"); + return module.GetAttr("CustomTradeBarConsolidator").Invoke(); + } + + private static int GetDataConsolidatedHandlerCount(PythonConsolidator consolidator) + { + var handlers = (Delegate)typeof(PythonConsolidator) + .GetField(nameof(PythonConsolidator.DataConsolidated), System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance) + .GetValue(consolidator); + return handlers?.GetInvocationList().Length ?? 0; + } + [Test] public void RemoveConsolidatorDisposesScanWrapperEvenWithoutSubscriptions() {