From 9a3b1c18108d06809bf1b20b1050d16440079ac8 Mon Sep 17 00:00:00 2001 From: refinedheart <3102024448@qq.com> Date: Thu, 19 Nov 2026 08:19:33 +0800 Subject: [PATCH] [fix] clone indexed expert views before Gemini release --- colossalai/tensor/colo_parameter.py | 7 ++++ .../test_colo_parameter_view_lifetime.py | 33 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 tests/test_zero/test_gemini/test_colo_parameter_view_lifetime.py diff --git a/colossalai/tensor/colo_parameter.py b/colossalai/tensor/colo_parameter.py index 8992b89a3c39..6c456cee7bca 100644 --- a/colossalai/tensor/colo_parameter.py +++ b/colossalai/tensor/colo_parameter.py @@ -64,6 +64,13 @@ def __torch_function__(cls, func, types, args=..., kwargs=None): with torch._C.DisableTorchFunction(): func = ColoParamOpHookManager.rewrite_op(func) ret = super().__torch_function__(func, types, args, kwargs) + # Transformers v5 Qwen3-MoE indexes 3D expert parameters and keeps + # the resulting view for backward. Gemini may release the source + # chunk before that backward read, so detach the indexed view from + # the chunk storage while preserving its autograd history. + if func is torch.Tensor.__getitem__ and any(p.ndim >= 3 for p in params): + with torch._C.DisableTorchFunction(): + ret = ret.clone() with torch._C.DisableTorchFunction(): ret = ColoParamOpHookManager.post_op(params, ret) return _convert_output(ret, func) diff --git a/tests/test_zero/test_gemini/test_colo_parameter_view_lifetime.py b/tests/test_zero/test_gemini/test_colo_parameter_view_lifetime.py new file mode 100644 index 000000000000..e3e732e6ce69 --- /dev/null +++ b/tests/test_zero/test_gemini/test_colo_parameter_view_lifetime.py @@ -0,0 +1,33 @@ +import torch + +from colossalai.tensor.colo_parameter import ColoParameter +from colossalai.tensor.param_op_hook import ColoParamOpHook, ColoParamOpHookManager +from colossalai.utils import free_storage + + +class ReleaseParameterStorageHook(ColoParamOpHook): + def pre_forward(self, params): + pass + + def post_forward(self, params): + for param in params: + free_storage(param) + + def pre_backward(self, params): + pass + + def post_backward(self, params): + pass + + +def test_indexed_3d_parameter_survives_storage_release(): + param = ColoParameter(torch.randn(4, 3, 2, requires_grad=True)) + hook = ReleaseParameterStorageHook() + + with ColoParamOpHookManager.use_hooks(hook): + expert = param[0] + assert expert.storage().size() == 3 * 2 + expert.square().sum().backward() + + assert param.grad is not None + assert param.grad.shape == param.shape