Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion Algorithm/QCAlgorithm.Python.cs
Original file line number Diff line number Diff line change
Expand Up @@ -637,7 +637,9 @@ public void RegisterIndicator(Symbol symbol, PyObject indicator, PyObject pyObje

try
{
dataConsolidator = new DataConsolidatorPythonWrapper(pyObject);
// reuse the wrapper of an earlier registration, a new one would never be disposed on removal
dataConsolidator = SubscriptionManager.FindPythonConsolidator(symbol, pyObject)
Comment thread
quantconnect-kronos-reviewer-2[bot] marked this conversation as resolved.
Resolved
?? new DataConsolidatorPythonWrapper(pyObject);
}
catch
{
Expand Down
10 changes: 6 additions & 4 deletions Common/Data/SubscriptionManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,9 @@ public void AddConsolidator(Symbol symbol, PyObject pyConsolidator)
{
var consolidator = PythonUtil.CreateInstanceOrWrapper<IDataConsolidator>(
pyConsolidator,
py => new DataConsolidatorPythonWrapper(py)
// reuse the wrapper of an earlier registration: a new one would subscribe to the python event, be
// discarded by the consolidator collections as a duplicate, and so never be disposed on removal
py => FindPythonConsolidator(symbol, py) ?? new DataConsolidatorPythonWrapper(py)
);
AddConsolidator(symbol, consolidator);
}
Expand Down Expand Up @@ -313,10 +315,10 @@ public void RemoveConsolidator(Symbol symbol, PyObject pyConsolidator)

/// <summary>
/// Finds the <see cref="DataConsolidatorPythonWrapper"/> previously created for the given python
/// consolidator so it can be removed and disposed, rather than a throwaway wrapper that would churn
/// the live python object's event subscription and leak the original one.
/// consolidator so it can be reused or removed and disposed, rather than a throwaway wrapper that would
/// subscribe to the live python object's event and leak that subscription or the original one.
/// </summary>
private IDataConsolidator FindPythonConsolidator(Symbol symbol, PyObject pyConsolidator)
internal IDataConsolidator FindPythonConsolidator(Symbol symbol, PyObject pyConsolidator)
{
var configs = symbol != null
? _subscriptionManager.GetSubscriptionDataConfigs(symbol)
Expand Down
4 changes: 3 additions & 1 deletion Common/Python/DataConsolidatorPythonWrapper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,9 @@ public class DataConsolidatorPythonWrapper : ConsolidatorBase
/// </summary>
public override IBaseData WorkingData
{
get { return _pythonWrapper.GetProperty<IBaseData>(nameof(WorkingData)); }
// a disposed wrapper has released its python object, e.g. a consolidator shared by indicators that are
// deregistered one after the other is removed again after its first removal disposed it
get { return _disposed ? null : _pythonWrapper.GetProperty<IBaseData>(nameof(WorkingData)); }
}

/// <summary>
Expand Down
115 changes: 115 additions & 0 deletions Tests/Common/Data/SubscriptionManagerTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
using QuantConnect.Data.UniverseSelection;
using QuantConnect.Indicators;
using QuantConnect.Logging;
using QuantConnect.Python;
using QuantConnect.Statistics;
using QuantConnect.Tests.Engine.DataFeeds;
using QuantConnect.Util;
Expand Down Expand Up @@ -710,6 +711,120 @@ def get_consolidator():
Assert.AreEqual(0, algorithm.SubscriptionManager.Subscriptions.Sum(x => x.Consolidators.Count));
}

[Test]
public void RemovingAPythonConsolidatorAddedTwiceUnsubscribesFromItsEvent()
{
using var _ = Py.GIL();
var algorithm = new AlgorithmStub();
var symbol = algorithm.AddEquity("SPY").Symbol;
using var pyConsolidator = CreateCustomTradeBarPythonConsolidator();
var source = pyConsolidator.As<PythonConsolidator>();

algorithm.SubscriptionManager.AddConsolidator(symbol, pyConsolidator);
algorithm.SubscriptionManager.AddConsolidator(symbol, pyConsolidator);
Assert.AreEqual(1, GetDataConsolidatedHandlerCount(source));

algorithm.SubscriptionManager.RemoveConsolidator(symbol, pyConsolidator);
Assert.IsEmpty(GetConsolidators(algorithm, symbol));
Assert.AreEqual(0, GetDataConsolidatedHandlerCount(source));
}

[Test]
public void RemovingAPythonConsolidatorSharedByIndicatorsUnsubscribesFromItsEvent()
{
using var _ = Py.GIL();
var algorithm = new AlgorithmStub();
var symbol = algorithm.AddEquity("SPY").Symbol;
using var pyConsolidator = CreateCustomTradeBarPythonConsolidator();
var source = pyConsolidator.As<PythonConsolidator>();

var sma = new SimpleMovingAverage(2);
var std = new StandardDeviation(2);
using var pySma = sma.ToPython();
using var pyStd = std.ToPython();
algorithm.RegisterIndicator(symbol, pySma, pyConsolidator);
algorithm.RegisterIndicator(symbol, pyStd, pyConsolidator);
Assert.AreEqual(1, GetDataConsolidatedHandlerCount(source));

// both indicators receive the bars the python consolidator emits
source.OnDataConsolidated(pyConsolidator, new TradeBar(algorithm.Time, symbol, 1, 1, 1, 1, 1, TimeSpan.FromMinutes(1)));
Assert.AreEqual(1, sma.Samples);
Assert.AreEqual(1, std.Samples);

algorithm.SubscriptionManager.RemoveConsolidator(symbol, pyConsolidator);
Assert.IsEmpty(GetConsolidators(algorithm, symbol));
Assert.AreEqual(0, GetDataConsolidatedHandlerCount(source));

source.OnDataConsolidated(pyConsolidator, new TradeBar(algorithm.Time, symbol, 1, 1, 1, 1, 1, TimeSpan.FromMinutes(1)));
Assert.AreEqual(1, sma.Samples);
Assert.AreEqual(1, std.Samples);
}

[Test]
public void DeregisteringIndicatorsSharingAPythonConsolidatorUnsubscribesFromItsEvent()
{
using var _ = Py.GIL();
using var module = PyModule.FromString(Guid.NewGuid().ToString(), @"
from AlgorithmImports import *

def register(algorithm, symbol, consolidator, sma, std):
algorithm.register_indicator(symbol, sma, consolidator)
algorithm.register_indicator(symbol, std, consolidator)

def deregister(algorithm, sma, std):
algorithm.deregister_indicator(sma)
algorithm.deregister_indicator(std)
");
var algorithm = new AlgorithmStub();
var symbol = algorithm.AddEquity("SPY").Symbol;
using var otherConsolidator = new TradeBarConsolidator(TimeSpan.FromMinutes(1));
algorithm.SubscriptionManager.AddConsolidator(symbol, otherConsolidator);
using var pyConsolidator = CreateCustomTradeBarPythonConsolidator();
var source = pyConsolidator.As<PythonConsolidator>();
using var pyAlgorithm = algorithm.ToPython();
using var pySymbol = symbol.ToPython();
using var pySma = new SimpleMovingAverage(2).ToPython();
using var pyStd = new StandardDeviation(2).ToPython();

module.GetAttr("register").Invoke(pyAlgorithm, pySymbol, pyConsolidator, pySma, pyStd).Dispose();
Assert.AreEqual(1, GetDataConsolidatedHandlerCount(source));

// the shared consolidator has not emitted a bar yet, both deregistrations remove it
module.GetAttr("deregister").Invoke(pyAlgorithm, pySma, pyStd).Dispose();

CollectionAssert.AreEqual(new[] { otherConsolidator }, GetConsolidators(algorithm, symbol));
Assert.AreEqual(0, GetDataConsolidatedHandlerCount(source));
}

private static PyObject CreateCustomTradeBarPythonConsolidator()
{
using var module = PyModule.FromString(Guid.NewGuid().ToString(), @"
from AlgorithmImports import *

class CustomTradeBarConsolidator(PythonConsolidator):
def __init__(self):
self.consolidated = None
self.working_data = None
self.input_type = TradeBar
self.output_type = TradeBar

def update(self, data):
pass

def scan(self, time):
pass
");
return module.GetAttr("CustomTradeBarConsolidator").Invoke();
}

private static int GetDataConsolidatedHandlerCount(PythonConsolidator consolidator)
{
var handlers = (Delegate)typeof(PythonConsolidator)
.GetField(nameof(PythonConsolidator.DataConsolidated), System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)
.GetValue(consolidator);
return handlers?.GetInvocationList().Length ?? 0;
}

[Test]
public void RemoveConsolidatorDisposesScanWrapperEvenWithoutSubscriptions()
{
Expand Down
Loading