[v1] add FSDPTurbo EP/EFSDP plugin for MoE training (#10676)

tests_v1/plugins/model_plugins/test_kernel_plugin.py

@@ -13,12 +13,32 @@
 # limitations under the License.
 
 import sys
+from functools import partial
 from unittest.mock import MagicMock, patch
 
+import pytest
 import torch.multiprocessing as mp
+from torch import nn
 from transformers import AutoModelForCausalLM
 
 
+def _original_fla_op(*args, **kwargs):
+    return args, kwargs
+
+
+class _LinearAttention(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.chunk_gated_delta_rule = _original_fla_op
+        self.recurrent_gated_delta_rule = _original_fla_op
+
+
+class _FLAModel(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.linear_attn = _LinearAttention()
+
+
 def _apply_kernel(rank) -> None:
     with patch("torch.accelerator.current_accelerator") as mock_get_accelerator:
         mock_device = MagicMock()
@@ -73,3 +93,62 @@ def test_apply_kernel():
 
 def test_apply_all_kernels():
     mp.spawn(_apply_all_kernels)
+
+
+@pytest.mark.runs_on(["npu"])
+def test_flash_linear_attention_kernels_compose_with_auto(monkeypatch):
+    import fsdp_turbo.ops.fla  # noqa: F401
+    from fsdp_turbo.ops import get_op
+
+    from llamafactory.v1.plugins.model_plugins.kernels import interface
+    from llamafactory.v1.plugins.model_plugins.kernels.ops.linear_attention.fla import (
+        FlashLinearAttentionKernel,
+    )
+
+    model = _FLAModel()
+    auto_calls = []
+
+    monkeypatch.setattr(
+        interface,
+        "_apply_auto_kernels",
+        lambda model, **kwargs: auto_calls.append((model, kwargs)) or model,
+    )
+    # FLA execution is outside this bridge test; its external runtime is not required.
+    monkeypatch.setattr(FlashLinearAttentionKernel, "check_deps", staticmethod(lambda: None))
+
+    config = {
+        "name": "auto, flash-linear-attention",
+        "include_kernels": "fused_recurrent_gated_delta_rule, chunk_gated_delta_rule",
+        "chunk_size": 32,
+    }
+    assert interface.apply_kernels(model, config) is model
+    assert auto_calls == [(model, {"config": config, "require_logits": False})]
+    assert get_op("chunk_gated_delta_rule").__module__ == "fsdp_turbo.ops.fla"
+
+    chunk_op = model.linear_attn.chunk_gated_delta_rule
+    assert isinstance(chunk_op, partial)
+    assert chunk_op.func.__module__ == "fsdp_turbo.ops.fla"
+    assert chunk_op.keywords == {"chunk_size": 32}
+    assert model.linear_attn.recurrent_gated_delta_rule.__module__ == "fsdp_turbo.ops.fla"
+
+    with pytest.raises(RuntimeError, match="did not match any model module attributes"):
+        FlashLinearAttentionKernel.apply(
+            model=nn.Linear(2, 2),
+            config={"include_kernels": "chunk_gated_delta_rule", "chunk_size": 32},
+        )
+
+
+def test_flash_linear_attention_kernel_validates_config(monkeypatch):
+    from llamafactory.v1.plugins.model_plugins.kernels.ops.linear_attention.fla import (
+        FlashLinearAttentionKernel,
+    )
+
+    model = nn.Sequential(nn.Linear(2, 2))
+    monkeypatch.setattr(FlashLinearAttentionKernel, "check_device", staticmethod(lambda: None))
+    monkeypatch.setattr(FlashLinearAttentionKernel, "check_deps", staticmethod(lambda: None))
+
+    with pytest.raises(ValueError, match="chunk_size"):
+        FlashLinearAttentionKernel.apply(model=model, config={"include_kernels": "auto", "chunk_size": 48})
+
+    with pytest.raises(ValueError, match="Unsupported Flash Linear Attention kernels"):
+        FlashLinearAttentionKernel.apply(model=model, config={"include_kernels": "not_a_kernel"})

tests_v1/plugins/model_plugins/test_ulysses_cp.py

@@ -20,6 +20,7 @@ from llamafactory.v1.accelerator.interface import DistributedInterface
 from llamafactory.v1.config.model_args import ModelArguments
 from llamafactory.v1.config.training_args import TrainingArguments
 from llamafactory.v1.core.model_engine import ModelEngine
+from llamafactory.v1.plugins.model_plugins.parallelization import ulysses
 from llamafactory.v1.plugins.model_plugins.parallelization.sequence_parallel import (
     SequenceParallelModelPlugin,
     sequence_parallel_loss,
@@ -28,6 +29,39 @@ from llamafactory.v1.utils.env import find_available_port
 from llamafactory.v1.utils.pytest import dist_env
 
 
+def test_qwen3_5_broadcast_position_ids_keep_packed_boundaries(monkeypatch: pytest.MonkeyPatch):
+    local_position_ids = torch.tensor([[0, 1, 0]])
+    remote_position_ids = torch.tensor([[1, 2, 3]])
+    mrope_position_ids = local_position_ids.unsqueeze(0).expand(3, -1, -1)
+    captured = {}
+
+    monkeypatch.setattr(ulysses.SeqAllToAll4D, "apply", lambda _, tensor, *__: tensor)
+    monkeypatch.setattr(ulysses, "get_ulysses_sequence_parallel_world_size", lambda _: 2)
+
+    def fake_all_gather(outputs, tensor, **_):
+        outputs[0].copy_(tensor)
+        outputs[1].copy_(remote_position_ids if tensor.shape == local_position_ids.shape else tensor)
+
+    def fake_attention(query, _key, _value, _attention_mask, **kwargs):
+        captured["position_ids"] = kwargs["position_ids"]
+        return query
+
+    monkeypatch.setattr(ulysses.dist, "all_gather", fake_all_gather)
+    attention = ulysses.UlyssesAttention(sequence_process_group=object(), attn_fn=fake_attention)
+    hidden_states = torch.zeros(1, 3, 2, 4)
+
+    attention(hidden_states, hidden_states, hidden_states, None, 6, position_ids=mrope_position_ids)
+
+    assert captured["position_ids"].tolist() == [[0, 1, 0, 1, 2, 3]]
+    assert captured["position_ids"].is_contiguous()
+
+
+def test_true_mrope_position_ids_are_not_used_as_packed_boundaries():
+    mrope_position_ids = torch.tensor([[[0, 1, 2]], [[0, 1, 1]], [[0, 1, 0]]])
+
+    assert ulysses._get_text_position_ids(mrope_position_ids) is None
+
+
 def _test_sequence_parallel_loss(
     local_rank: int, world_size: int, master_port: int, cp_size: int, dp_size: int, batch_size: int
 ):