[v1] add FSDPTurbo EP/EFSDP plugin for MoE training (#10676)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.github/workflows/docker.yml

@@ -29,8 +29,6 @@ jobs:
       matrix:
         include:
           - device: "cuda"
-          - device: "npu-a2"
-          - device: "npu-a3"
 
     runs-on: ubuntu-latest
 
@@ -71,14 +69,6 @@ jobs:
           username: ${{ vars.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Login to Quay
-        if: ${{ github.event_name != 'pull_request' && startsWith(matrix.device, 'npu') }}
-        uses: docker/login-action@v3
-        with:
-          registry: quay.io
-          username: ${{ vars.QUAY_ASCEND_USERNAME }}
-          password: ${{ secrets.QUAY_ASCEND_TOKEN }}
-
       - name: Build and push Docker image (CUDA)
         if: ${{ matrix.device == 'cuda' }}
         uses: docker/build-push-action@v6
@@ -88,29 +78,3 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: |
             docker.io/hiyouga/llamafactory:${{ steps.version.outputs.tag }}
-
-      - name: Build and push Docker image (NPU-A2)
-        if: ${{ matrix.device == 'npu-a2' }}
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          file: ./docker/docker-npu/Dockerfile
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: |
-            docker.io/hiyouga/llamafactory:${{ steps.version.outputs.tag }}-npu-a2
-            quay.io/ascend/llamafactory:${{ steps.version.outputs.tag }}-npu-a2
-
-      - name: Build and push Docker image (NPU-A3)
-        if: ${{ matrix.device == 'npu-a3' }}
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          file: ./docker/docker-npu/Dockerfile
-          build-args: |
-            BASE_IMAGE=quay.io/ascend/cann:9.0.0-a3-ubuntu22.04-py3.11
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: |
-            docker.io/hiyouga/llamafactory:${{ steps.version.outputs.tag }}-npu-a3
-            quay.io/ascend/llamafactory:${{ steps.version.outputs.tag }}-npu-a3

.github/workflows/docker_npu.yml

@@ -0,0 +1,132 @@
+name: docker-npu
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "17 2 * * *"
+      timezone: "Asia/Shanghai"
+  release:
+    types:
+      - published
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - device: "npu-a2"
+            os: "ubuntu"
+            base_image: "quay.io/ascend/cann:9.1.0-910b-ubuntu22.04-py3.12"
+          - device: "npu-a3"
+            os: "ubuntu"
+            base_image: "quay.io/ascend/cann:9.1.0-a3-ubuntu22.04-py3.12"
+          - device: "npu-a2"
+            os: "openeuler"
+            base_image: "quay.io/ascend/cann:9.1.0-910b-openeuler24.03-py3.12"
+          - device: "npu-a3"
+            os: "openeuler"
+            base_image: "quay.io/ascend/cann:9.1.0-a3-openeuler24.03-py3.12"
+
+    runs-on: ubuntu-latest
+
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.device }}-${{ matrix.os }}
+      cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+    environment:
+      name: docker
+      url: https://hub.docker.com/r/hiyouga/llamafactory
+
+    steps:
+      - name: Free up disk space
+        uses: jlumbroso/free-disk-space@v1.3.1
+        with:
+          tool-cache: true
+          docker-images: false
+
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Get LlamaFactory version
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "release" ]; then
+            echo "tag=$(grep -oP 'VERSION = "\K[^"]+' src/llamafactory/extras/env.py)" >> "$GITHUB_OUTPUT"
+          else
+            echo "tag=latest" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Get NPU image tag
+        id: npu_tag
+        env:
+          BASE_IMAGE: ${{ matrix.base_image }}
+          MATRIX_DEVICE: ${{ matrix.device }}
+          MATRIX_OS: ${{ matrix.os }}
+          LLAMAFACTORY_VERSION: ${{ steps.version.outputs.tag }}
+        run: |
+          base_image_tag="${BASE_IMAGE##*:}"
+          cann_version="${base_image_tag%%-*}"
+          torch_npu_version="$(sed -nE 's/^torch[-_]npu==([0-9]+(\.[0-9]+)*(\.post[0-9]+)?).*/\1/p' requirements/npu.txt)"
+          soc="$(grep -oE '(910b|a3)' <<< "${base_image_tag}" | head -n 1)"
+          operating_system="$(grep -oE '(ubuntu|openeuler)[0-9]+(\.[0-9]+)*' <<< "${base_image_tag}" | head -n 1)"
+          python_version="$(grep -oE 'py[0-9]+\.[0-9]+' <<< "${base_image_tag}" | head -n 1)"
+
+          if [[ -z "${cann_version}" || -z "${torch_npu_version}" || -z "${soc}" || -z "${operating_system}" || -z "${python_version}" ]]; then
+            echo "Failed to derive the NPU image tag from ${BASE_IMAGE} and requirements/npu.txt" >&2
+            exit 1
+          fi
+
+          if [[ "${operating_system}" != "${MATRIX_OS}"* ]]; then
+            echo "Operating system ${operating_system} derived from ${BASE_IMAGE} does not match matrix OS family ${MATRIX_OS}" >&2
+            exit 1
+          fi
+
+          case "${MATRIX_DEVICE}" in
+            npu-a2) expected_soc="910b" ;;
+            npu-a3) expected_soc="a3" ;;
+            *)
+              echo "Unsupported NPU device ${MATRIX_DEVICE}" >&2
+              exit 1
+              ;;
+          esac
+
+          if [[ "${soc}" != "${expected_soc}" ]]; then
+            echo "SoC ${soc} derived from ${BASE_IMAGE} does not match matrix device ${MATRIX_DEVICE}" >&2
+            exit 1
+          fi
+
+          if [[ "${LLAMAFACTORY_VERSION}" == "latest" ]]; then
+            echo "tag=latest-${soc}-${MATRIX_OS}" >> "$GITHUB_OUTPUT"
+          else
+            echo "tag=${LLAMAFACTORY_VERSION}-cann${cann_version}-torch_npu${torch_npu_version}-${soc}-${operating_system}-${python_version}" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to Quay
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ vars.QUAY_ASCEND_USERNAME }}
+          password: ${{ secrets.QUAY_ASCEND_TOKEN }}
+
+      - name: Build and push Docker image (${{ matrix.device }}-${{ matrix.os }})
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          file: ./docker/docker-npu/Dockerfile
+          build-args: |
+            BASE_IMAGE=${{ matrix.base_image }}
+          push: true
+          tags: |
+            docker.io/hiyouga/llamafactory:${{ steps.npu_tag.outputs.tag }}
+            quay.io/ascend/llamafactory:${{ steps.npu_tag.outputs.tag }}

.github/workflows/docs.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Install dependencies
         run: |
@@ -66,6 +66,7 @@ jobs:
           path: docs/_build/html
 
   deploy:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}

.github/workflows/tests_npu.yml

@@ -8,6 +8,7 @@ on:
     paths:
       - "**/*.py"
       - "pyproject.toml"
+      - "requirements/fsdpturbo.txt"
       - "Makefile"
       - ".github/workflows/*.yml"
   pull_request:
@@ -16,6 +17,7 @@ on:
     paths:
       - "**/*.py"
       - "pyproject.toml"
+      - "requirements/fsdpturbo.txt"
       - "Makefile"
       - ".github/workflows/*.yml"
 
@@ -25,11 +27,11 @@ jobs:
       fail-fast: false
       matrix:
         python:
-          - "3.11"
+          - "3.12"
         os:
           - "linux-aarch64-a2-4"
         pytorch_npu:
-          - "2.7.1"
+          - "2.10.0"
 
     runs-on: ${{ matrix.os }}
 
@@ -38,7 +40,7 @@ jobs:
       cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
     container:
-      image: ascendai/cann:9.0.0-910b-ubuntu22.04-py3.11
+      image: ascendai/cann:9.1.0-910b-ubuntu22.04-py3.12
       env:
         HF_ENDPOINT: https://hf-mirror.com
         HF_TOKEN: ${{ secrets.HF_TOKEN }}
@@ -68,6 +70,7 @@ jobs:
           uv pip install -e .
           uv pip install -r requirements/npu.txt
           uv pip install -r requirements/dev.txt
+          uv pip install --no-deps -r requirements/fsdpturbo.txt
 
       - name: Install node
         run: |

.pre-commit-config.yaml

@@ -21,7 +21,7 @@ repos:
         args: [--py39-plus]
 
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.13.2
+    rev: v0.15.5
     hooks:
     -   id: ruff
         args: [--fix]

Makefile

@@ -2,9 +2,12 @@
 
 check_dirs := scripts src tests tests_v1
 
+ruff_version := 0.15.5
+
 RUN := $(shell command -v uv >/dev/null 2>&1 && echo "uv run" || echo "")
 BUILD := $(shell command -v uv >/dev/null 2>&1 && echo "uv build" || echo "python -m build")
 TOOL := $(shell command -v uv >/dev/null 2>&1 && echo "uvx" || echo "")
+RUFF := $(shell command -v uv >/dev/null 2>&1 && echo "uvx ruff@$(ruff_version)" || echo "ruff")
 
 build:
 	$(BUILD)
@@ -17,12 +20,12 @@ license:
 	$(RUN) python3 tests/check_license.py $(check_dirs)
 
 quality:
-	$(TOOL) ruff check $(check_dirs)
+	$(RUFF) check $(check_dirs)
-	$(TOOL) ruff format --check $(check_dirs)
+	$(RUFF) format --check $(check_dirs)
 
 style:
-	$(TOOL) ruff check $(check_dirs) --fix
+	$(RUFF) check $(check_dirs) --fix
-	$(TOOL) ruff format $(check_dirs)
+	$(RUFF) format $(check_dirs)
 
 test:
 	WANDB_DISABLED=true $(RUN) pytest -vv --import-mode=importlib tests/ tests_v1/

README.md

@@ -1,9 +1,9 @@
-![# LLaMA Factory](assets/logo.png)
+![# LlamaFactory](assets/logo.png)
 
-[![GitHub Repo stars](https://img.shields.io/github/stars/hiyouga/LLaMA-Factory?style=social)](https://github.com/hiyouga/LLaMA-Factory/stargazers)
+[![GitHub Repo stars](https://img.shields.io/github/stars/hiyouga/LlamaFactory?style=social)](https://github.com/hiyouga/LlamaFactory/stargazers)
-[![GitHub last commit](https://img.shields.io/github/last-commit/hiyouga/LLaMA-Factory)](https://github.com/hiyouga/LLaMA-Factory/commits/main)
+[![GitHub last commit](https://img.shields.io/github/last-commit/hiyouga/LlamaFactory)](https://github.com/hiyouga/LlamaFactory/commits/main)
-[![GitHub contributors](https://img.shields.io/github/contributors/hiyouga/LLaMA-Factory?color=orange)](https://github.com/hiyouga/LLaMA-Factory/graphs/contributors)
+[![GitHub contributors](https://img.shields.io/github/contributors/hiyouga/LlamaFactory?color=orange)](https://github.com/hiyouga/LlamaFactory/graphs/contributors)
-[![GitHub workflow](https://github.com/hiyouga/LLaMA-Factory/actions/workflows/tests.yml/badge.svg)](https://github.com/hiyouga/LLaMA-Factory/actions/workflows/tests.yml)
+[![GitHub workflow](https://github.com/hiyouga/LlamaFactory/actions/workflows/tests.yml/badge.svg)](https://github.com/hiyouga/LlamaFactory/actions/workflows/tests.yml)
 [![PyPI](https://img.shields.io/pypi/v/llamafactory)](https://pypi.org/project/llamafactory/)
 [![Citation](https://img.shields.io/badge/citation-1000+-green)](https://scholar.google.com/scholar?cites=12620864006390196564)
 [![Docker Pulls](https://img.shields.io/docker/pulls/hiyouga/llamafactory)](https://hub.docker.com/r/hiyouga/llamafactory/tags)
@@ -19,7 +19,21 @@
 [![Open in Studios](https://img.shields.io/badge/ModelScope-Open%20in%20Studios-blue)](https://modelscope.cn/studios/hiyouga/LLaMA-Board)
 [![Open in Novita](https://img.shields.io/badge/Novita-Deploy%20Template-blue)](https://novita.ai/templates-library/105981?sharer=88115474-394e-4bda-968e-b88e123d0c47)
 
-### Used by [Amazon](https://aws.amazon.com/cn/blogs/machine-learning/how-apoidea-group-enhances-visual-information-extraction-from-banking-documents-with-multimodal-models-using-llama-factory-on-amazon-sagemaker-hyperpod/), [NVIDIA](https://developer.nvidia.com/rtx/ai-toolkit), [Aliyun](https://help.aliyun.com/zh/pai/use-cases/fine-tune-a-llama-3-model-with-llama-factory), etc.
+### Used by [Amazon](https://aws.amazon.com/cn/blogs/machine-learning/how-apoidea-group-enhances-visual-information-extraction-from-banking-documents-with-multimodal-models-using-llama-factory-on-amazon-sagemaker-hyperpod/), [NVIDIA](https://build.nvidia.com/spark/llama-factory), [Aliyun](https://help.aliyun.com/zh/pai/use-cases/fine-tune-a-llama-3-model-with-llama-factory), etc.
+
+----
+
+<div align="center" markdown="1">
+
+### Check our new open-source project —<br>🐧 [PenguinHarness](https://github.com/Prism-Shadow/penguin-harness): Your desktop agent that automatically builds agents for just $0.02 of tokens!
+
+Follow our project: https://github.com/Prism-Shadow/penguin-harness
+
+</div>
+
+https://github.com/user-attachments/assets/9b7033e8-f08a-4c3f-bd33-547896664e6e
+
+----
 
 <div align="center" markdown="1">
 
@@ -32,7 +46,7 @@
 
 ### Easily fine-tune 100+ large language models with zero-code [CLI](#quickstart) and [Web UI](#fine-tuning-with-llama-board-gui-powered-by-gradio)
 
-![GitHub Trend](https://trendshift.io/api/badge/repositories/4535)
+![GitHub Trend](https://trendshift.io/api/badge/repositories/17371)
 
 </div>
 
@@ -81,7 +95,7 @@ Read technical notes:
   - [Download from Modelers Hub](#download-from-modelers-hub)
   - [Use W&B Logger](#use-wb-logger)
   - [Use SwanLab Logger](#use-swanlab-logger)
-- [Projects using LLaMA Factory](#projects-using-llama-factory)
+- [Projects using LlamaFactory](#projects-using-llamafactory)
 - [License](#license)
 - [Citation](#citation)
 - [Acknowledgement](#acknowledgement)
@@ -107,35 +121,35 @@ Read technical notes:
 ## Blogs
 
 > [!TIP]
-> Now we have a dedicated blog for LLaMA Factory!
+> Now we have a dedicated blog for LlamaFactory!
 >
 > Website: https://blog.llamafactory.net/en/
 
-- 💡 [KTransformers Fine-Tuning × LLaMA Factory: Fine-tuning 1000 Billion models with 2 4090-GPU + CPU](https://blog.llamafactory.net/en/posts/ktransformers/) (English)
+- 💡 [KTransformers Fine-Tuning × LlamaFactory: Fine-tuning 1000 Billion models with 2 4090-GPU + CPU](https://blog.llamafactory.net/en/posts/ktransformers/) (English)
-- 💡 [Easy Dataset × LLaMA Factory: Enabling LLMs to Efficiently Learn Domain Knowledge](https://buaa-act.feishu.cn/wiki/GVzlwYcRFiR8OLkHbL6cQpYin7g) (English)
+- 💡 [Easy Dataset × LlamaFactory: Enabling LLMs to Efficiently Learn Domain Knowledge](https://buaa-act.feishu.cn/wiki/GVzlwYcRFiR8OLkHbL6cQpYin7g) (English)
-- 💡 [DataFlow × LLaMA Factory: Producing High-Quality Data for LLM Training with a Data Preparation Pipeline](https://wcny4qa9krto.feishu.cn/wiki/LWkkwTDBfiiRKqkDSvucG6yjnbW) (English) | [中文](https://wcny4qa9krto.feishu.cn/wiki/LlMxweUAJimrmykRD5qcGuswnHd)
+- 💡 [DataFlow × LlamaFactory: Producing High-Quality Data for LLM Training with a Data Preparation Pipeline](https://wcny4qa9krto.feishu.cn/wiki/LWkkwTDBfiiRKqkDSvucG6yjnbW) (English) | [中文](https://wcny4qa9krto.feishu.cn/wiki/LlMxweUAJimrmykRD5qcGuswnHd)
-- 💡 [DataFlex × LLaMA Factory: A Data-Centric Dynamic Training System Built on LLaMA-Factory](https://wcny4qa9krto.feishu.cn/wiki/OlREwPQWdi9K6ZkJNHIciLhtnkv) (English) | [中文](https://wcny4qa9krto.feishu.cn/wiki/H2A9wSsbCinzavkT2oyc2C5Vn0e)
+- 💡 [DataFlex × LlamaFactory: A Data-Centric Dynamic Training System Built on LlamaFactory](https://wcny4qa9krto.feishu.cn/wiki/OlREwPQWdi9K6ZkJNHIciLhtnkv) (English) | [中文](https://wcny4qa9krto.feishu.cn/wiki/H2A9wSsbCinzavkT2oyc2C5Vn0e)
-- [A One-Stop Code-Free Model Reinforcement Learning and Deployment Platform based on LLaMA-Factory and EasyR1](https://aws.amazon.com/cn/blogs/china/building-llm-model-hub-based-on-llamafactory-and-easyr1/) (Chinese)
+- [A One-Stop Code-Free Model Reinforcement Learning and Deployment Platform based on LlamaFactory and EasyR1](https://aws.amazon.com/cn/blogs/china/building-llm-model-hub-based-on-llamafactory-and-easyr1/) (Chinese)
-- [How Apoidea Group enhances visual information extraction from banking documents with multimodal models using LLaMA-Factory on Amazon SageMaker HyperPod](https://aws.amazon.com/cn/blogs/machine-learning/how-apoidea-group-enhances-visual-information-extraction-from-banking-documents-with-multimodal-models-using-llama-factory-on-amazon-sagemaker-hyperpod/) (English)
+- [How Apoidea Group enhances visual information extraction from banking documents with multimodal models using LlamaFactory on Amazon SageMaker HyperPod](https://aws.amazon.com/cn/blogs/machine-learning/how-apoidea-group-enhances-visual-information-extraction-from-banking-documents-with-multimodal-models-using-llama-factory-on-amazon-sagemaker-hyperpod/) (English)
 
 <details><summary>All Blogs</summary>
 
-- [LLaMA Factory: Fine-tuning the DeepSeek-R1-Distill-Qwen-7B Model for News Classifier](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory_deepseek_r1_distill_7b) (Chinese)
+- [LlamaFactory: Fine-tuning the DeepSeek-R1-Distill-Qwen-7B Model for News Classifier](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory_deepseek_r1_distill_7b) (Chinese)
-- [A One-Stop Code-Free Model Fine-Tuning \& Deployment Platform based on SageMaker and LLaMA-Factory](https://aws.amazon.com/cn/blogs/china/a-one-stop-code-free-model-fine-tuning-deployment-platform-based-on-sagemaker-and-llama-factory/) (Chinese)
+- [A One-Stop Code-Free Model Fine-Tuning \& Deployment Platform based on SageMaker and LlamaFactory](https://aws.amazon.com/cn/blogs/china/a-one-stop-code-free-model-fine-tuning-deployment-platform-based-on-sagemaker-and-llama-factory/) (Chinese)
-- [LLaMA Factory Multi-Modal Fine-Tuning Practice: Fine-Tuning Qwen2-VL for Personal Tourist Guide](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory_qwen2vl) (Chinese)
+- [LlamaFactory Multi-Modal Fine-Tuning Practice: Fine-Tuning Qwen2-VL for Personal Tourist Guide](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory_qwen2vl) (Chinese)
-- [LLaMA Factory: Fine-tuning Llama3 for Role-Playing](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory) (Chinese)
+- [LlamaFactory: Fine-tuning Llama3 for Role-Playing](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory) (Chinese)
 
 </details>
 
 ## Changelog
 
-[25/10/26] We support Megatron-core training backend with [**mcore_adapter**](https://github.com/alibaba/ROLL/tree/main/mcore_adapter). See [PR #9237](https://github.com/hiyouga/LLaMA-Factory/pull/9237) to get started.
+[25/10/26] We support Megatron-core training backend with [**mcore_adapter**](https://github.com/alibaba/ROLL/tree/main/mcore_adapter). See [PR #9237](https://github.com/hiyouga/LlamaFactory/pull/9237) to get started.
 
 [25/08/22] We supported **[OFT](https://arxiv.org/abs/2306.07280)** and **[OFTv2](https://arxiv.org/abs/2506.19847)**. See [examples](examples/README.md) for usage.
 
-[25/08/20] We supported fine-tuning the **[Intern-S1-mini](https://huggingface.co/internlm/Intern-S1-mini)** models. See [PR #8976](https://github.com/hiyouga/LLaMA-Factory/pull/8976) to get started.
+[25/08/20] We supported fine-tuning the **[Intern-S1-mini](https://huggingface.co/internlm/Intern-S1-mini)** models. See [PR #8976](https://github.com/hiyouga/LlamaFactory/pull/8976) to get started.
 
-[25/08/06] We supported fine-tuning the **[GPT-OSS](https://github.com/openai/gpt-oss)** models. See [PR #8826](https://github.com/hiyouga/LLaMA-Factory/pull/8826) to get started.
+[25/08/06] We supported fine-tuning the **[GPT-OSS](https://github.com/openai/gpt-oss)** models. See [PR #8826](https://github.com/hiyouga/LlamaFactory/pull/8826) to get started.
 
 <details><summary>Full Changelog</summary>
 
@@ -145,13 +159,13 @@ Read technical notes:
 
 [25/04/21] We supported the **[Muon](https://github.com/KellerJordan/Muon)** optimizer. See [examples](examples/README.md) for usage. Thank [@tianshijing](https://github.com/tianshijing)'s PR.
 
-[25/04/16] We supported fine-tuning the **[InternVL3](https://huggingface.co/OpenGVLab/InternVL3-8B)** model. See [PR #7258](https://github.com/hiyouga/LLaMA-Factory/pull/7258) to get started.
+[25/04/16] We supported fine-tuning the **[InternVL3](https://huggingface.co/OpenGVLab/InternVL3-8B)** model. See [PR #7258](https://github.com/hiyouga/LlamaFactory/pull/7258) to get started.
 
 [25/04/14] We supported fine-tuning the **[GLM-Z1](https://huggingface.co/THUDM/GLM-Z1-9B-0414)** and **[Kimi-VL](https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct)** models.
 
-[25/04/06] We supported fine-tuning the **[Llama 4](https://ai.meta.com/blog/llama-4-multimodal-intelligence/)** model. See [PR #7611](https://github.com/hiyouga/LLaMA-Factory/pull/7611) to get started.
+[25/04/06] We supported fine-tuning the **[Llama 4](https://ai.meta.com/blog/llama-4-multimodal-intelligence/)** model. See [PR #7611](https://github.com/hiyouga/LlamaFactory/pull/7611) to get started.
 
-[25/03/31] We supported fine-tuning the **[Qwen2.5 Omni](https://qwenlm.github.io/blog/qwen2.5-omni/)** model. See [PR #7537](https://github.com/hiyouga/LLaMA-Factory/pull/7537) to get started.
+[25/03/31] We supported fine-tuning the **[Qwen2.5 Omni](https://qwenlm.github.io/blog/qwen2.5-omni/)** model. See [PR #7537](https://github.com/hiyouga/LlamaFactory/pull/7537) to get started.
 
 [25/03/15] We supported **[SGLang](https://github.com/sgl-project/sglang)** as inference backend. Try `infer_backend: sglang` to accelerate inference.
 
@@ -203,13 +217,13 @@ Read technical notes:
 
 [24/04/26] We supported fine-tuning the **LLaVA-1.5** multimodal LLMs. See [examples](examples/README.md) for usage.
 
-[24/04/22] We provided a **[Colab notebook](https://colab.research.google.com/drive/1eRTPn37ltBbYsISy9Aw2NuI2Aq5CQrD9?usp=sharing)** for fine-tuning the Llama-3 model on a free T4 GPU. Two Llama-3-derived models fine-tuned using LLaMA Factory are available at Hugging Face, check [Llama3-8B-Chinese-Chat](https://huggingface.co/shenzhi-wang/Llama3-8B-Chinese-Chat) and [Llama3-Chinese](https://huggingface.co/zhichen/Llama3-Chinese) for details.
+[24/04/22] We provided a **[Colab notebook](https://colab.research.google.com/drive/1eRTPn37ltBbYsISy9Aw2NuI2Aq5CQrD9?usp=sharing)** for fine-tuning the Llama-3 model on a free T4 GPU. Two Llama-3-derived models fine-tuned using LlamaFactory are available at Hugging Face, check [Llama3-8B-Chinese-Chat](https://huggingface.co/shenzhi-wang/Llama3-8B-Chinese-Chat) and [Llama3-Chinese](https://huggingface.co/zhichen/Llama3-Chinese) for details.
 
 [24/04/21] We supported **[Mixture-of-Depths](https://arxiv.org/abs/2404.02258)** according to [AstraMindAI's implementation](https://github.com/astramind-ai/Mixture-of-depths). See [examples](examples/README.md) for usage.
 
 [24/04/16] We supported **[BAdam](https://arxiv.org/abs/2404.02827)** optimizer. See [examples](examples/README.md) for usage.
 
-[24/04/16] We supported **[unsloth](https://github.com/unslothai/unsloth)**'s long-sequence training (Llama-2-7B-56k within 24GB). It achieves **117%** speed and **50%** memory compared with FlashAttention-2, more benchmarks can be found in [this page](https://github.com/hiyouga/LLaMA-Factory/wiki/Performance-comparison).
+[24/04/16] We supported **[unsloth](https://github.com/unslothai/unsloth)**'s long-sequence training (Llama-2-7B-56k within 24GB). It achieves **117%** speed and **50%** memory compared with FlashAttention-2, more benchmarks can be found in [this page](https://github.com/hiyouga/LlamaFactory/wiki/Performance-comparison).
 
 [24/03/31] We supported **[ORPO](https://arxiv.org/abs/2403.07691)**. See [examples](examples/README.md) for usage.
 
@@ -227,11 +241,11 @@ Read technical notes:
 
 [24/02/15] We supported **block expansion** proposed by [LLaMA Pro](https://github.com/TencentARC/LLaMA-Pro). See [examples](examples/README.md) for usage.
 
-[24/02/05] Qwen1.5 (Qwen2 beta version) series models are supported in LLaMA-Factory. Check this [blog post](https://qwenlm.github.io/blog/qwen1.5/) for details.
+[24/02/05] Qwen1.5 (Qwen2 beta version) series models are supported in LlamaFactory. Check this [blog post](https://qwenlm.github.io/blog/qwen1.5/) for details.
 
 [24/01/18] We supported **agent tuning** for most models, equipping model with tool using abilities by fine-tuning with `dataset: glaive_toolcall_en`.
 
-[23/12/23] We supported **[unsloth](https://github.com/unslothai/unsloth)**'s implementation to boost LoRA tuning for the LLaMA, Mistral and Yi models. Try `use_unsloth: true` argument to activate unsloth patch. It achieves **170%** speed in our benchmark, check [this page](https://github.com/hiyouga/LLaMA-Factory/wiki/Performance-comparison) for details.
+[23/12/23] We supported **[unsloth](https://github.com/unslothai/unsloth)**'s implementation to boost LoRA tuning for the LLaMA, Mistral and Yi models. Try `use_unsloth: true` argument to activate unsloth patch. It achieves **170%** speed in our benchmark, check [this page](https://github.com/hiyouga/LlamaFactory/wiki/Performance-comparison) for details.
 
 [23/12/12] We supported fine-tuning the latest MoE model **[Mixtral 8x7B](https://huggingface.co/mistralai/Mixtral-8x7B-v0.1)** in our framework. See hardware requirement [here](#hardware-requirement).
 
@@ -266,7 +280,7 @@ Read technical notes:
 </details>
 
 > [!TIP]
-> If you cannot use the latest feature, please pull the latest code and install LLaMA-Factory again.
+> If you cannot use the latest feature, please pull the latest code and install LlamaFactory again.
 
 ## Supported Models
 
@@ -590,19 +604,23 @@ To enable FlashAttention-2 on the Windows platform, please use the script from [
 
 <details><summary>For Ascend NPU users</summary>
 
-To install LLaMA Factory on Ascend NPU devices, please upgrade Python to version 3.10 or higher: `pip install -r requirements/npu.txt`. Additionally, you need to install the **Ascend CANN Toolkit and Kernels**. Please follow the [installation tutorial](https://llamafactory.readthedocs.io/en/latest/advanced/npu_installation.html).
+To install LlamaFactory on Ascend NPU devices, please use Python 3.12 and install the extra dependencies with `pip install -r requirements/npu.txt`. Additionally, you need to install the **Ascend CANN Toolkit and Kernels**. Please follow the [installation tutorial](https://llamafactory.readthedocs.io/en/latest/multibackend/npu/npu_installation.html).
 
 
 You can also download the pre-built Docker images:
 
 ```bash
 # Docker Hub
-docker pull hiyouga/llamafactory:latest-npu-a2
+docker pull hiyouga/llamafactory:latest-910b-ubuntu
-docker pull hiyouga/llamafactory:latest-npu-a3
+docker pull hiyouga/llamafactory:latest-a3-ubuntu
+docker pull hiyouga/llamafactory:latest-910b-openeuler
+docker pull hiyouga/llamafactory:latest-a3-openeuler
 
 # quay.io
-docker pull quay.io/ascend/llamafactory:latest-npu-a2
+docker pull quay.io/ascend/llamafactory:latest-910b-ubuntu
-docker pull quay.io/ascend/llamafactory:latest-npu-a3
+docker pull quay.io/ascend/llamafactory:latest-a3-ubuntu
+docker pull quay.io/ascend/llamafactory:latest-910b-openeuler
+docker pull quay.io/ascend/llamafactory:latest-a3-openeuler
 ```
 
 #### Install BitsAndBytes
@@ -665,7 +683,7 @@ See [examples/README.md](examples/README.md) for advanced usage (including distr
 > [!TIP]
 > Use `llamafactory-cli help` to show help information.
 >
-> Read [FAQs](https://github.com/hiyouga/LLaMA-Factory/issues/4614) first if you encounter any problems.
+> Read [FAQs](https://github.com/hiyouga/LlamaFactory/issues/4614) first if you encounter any problems.
 
 ### Fine-Tuning with LLaMA Board GUI (powered by [Gradio](https://github.com/gradio-app/gradio))
 
@@ -687,8 +705,22 @@ For Ascend NPU users:
 
 ```bash
 cd docker/docker-npu/
-docker compose up -d
+
-docker compose exec llamafactory bash
+# A2 with Ubuntu
+docker compose --profile a2-ubuntu up -d
+docker compose --profile a2-ubuntu exec llamafactory-a2-ubuntu bash
+
+# A3 with Ubuntu
+docker compose --profile a3-ubuntu up -d
+docker compose --profile a3-ubuntu exec llamafactory-a3-ubuntu bash
+
+# A2 with openEuler
+docker compose --profile a2-openeuler up -d
+docker compose --profile a2-openeuler exec llamafactory-a2-openeuler bash
+
+# A3 with openEuler
+docker compose --profile a3-openeuler up -d
+docker compose --profile a3-openeuler exec llamafactory-a3-openeuler bash
 ```
 
 For AMD ROCm users:
@@ -830,7 +862,7 @@ When launching training tasks, you can log in to SwanLab in three ways:
 2. Set the environment variable `SWANLAB_API_KEY` to your [API key](https://swanlab.cn/settings).
 3. Use the `swanlab login` command to complete the login.
 
-## Projects using LLaMA Factory
+## Projects using LlamaFactory
 
 If you have a project that should be incorporated, please contact via email or create a pull request.
 
@@ -910,7 +942,7 @@ If you have a project that should be incorporated, please contact via email or c
 1. Sun et al. LAMBDA: A Large Model Based Data Agent. 2024. [[arxiv]](https://arxiv.org/abs/2407.17535)
 1. Zhu et al. CollectiveSFT: Scaling Large Language Models for Chinese Medical Benchmark with Collective Instructions in Healthcare. 2024. [[arxiv]](https://arxiv.org/abs/2407.19705)
 1. Yu et al. Correcting Negative Bias in Large Language Models through Negative Attention Score Alignment. 2024. [[arxiv]](https://arxiv.org/abs/2408.00137)
-1. Xie et al. The Power of Personalized Datasets: Advancing Chinese Composition Writing for Elementary School through Targeted Model Fine-Tuning. IALP 2024. [[paper]](https://www.asianlp.sg/conferences/ialp2024/proceedings/papers/IALP2024_P055.pdf)
+1. Xie et al. The Power of Personalized Datasets: Advancing Chinese Composition Writing for Elementary School through Targeted Model Fine-Tuning. IALP 2024. [[paper]](https://doi.org/10.1142/S2717554524500176)
 1. Liu et al. Instruct-Code-Llama: Improving Capabilities of Language Model in Competition Level Code Generation by Online Judge Feedback. ICIC 2024. [[paper]](https://link.springer.com/chapter/10.1007/978-981-97-5669-8_11)
 1. Wang et al. Cybernetic Sentinels: Unveiling the Impact of Safety Data Selection on Model Security in Supervised Fine-Tuning. ICIC 2024. [[paper]](https://link.springer.com/chapter/10.1007/978-981-97-5669-8_23)
 1. Xia et al. Understanding the Performance and Estimating the Cost of LLM Fine-Tuning. 2024. [[arxiv]](https://arxiv.org/abs/2408.04693)
@@ -928,7 +960,7 @@ If you have a project that should be incorporated, please contact via email or c
 1. **[Chinese-LLaVA-Med](https://github.com/BUAADreamer/Chinese-LLaVA-Med)**: A multimodal large language model specialized in Chinese medical domain, based on LLaVA-1.5-7B.
 1. **[AutoRE](https://github.com/THUDM/AutoRE)**: A document-level relation extraction system based on large language models.
 1. **[NVIDIA RTX AI Toolkit](https://github.com/NVIDIA/RTX-AI-Toolkit)**: SDKs for fine-tuning LLMs on Windows PC for NVIDIA RTX.
-1. **[LazyLLM](https://github.com/LazyAGI/LazyLLM)**: An easy and lazy way for building multi-agent LLMs applications and supports model fine-tuning via LLaMA Factory.
+1. **[LazyLLM](https://github.com/LazyAGI/LazyLLM)**: An easy and lazy way for building multi-agent LLMs applications and supports model fine-tuning via LlamaFactory.
 1. **[RAG-Retrieval](https://github.com/NLPJCL/RAG-Retrieval)**: A full pipeline for RAG retrieval model fine-tuning, inference, and distillation. [[blog]](https://zhuanlan.zhihu.com/p/987727357)
 1. **[360-LLaMA-Factory](https://github.com/Qihoo360/360-LLaMA-Factory)**: A modified library that supports long sequence SFT & DPO using ring attention.
 1. **[Sky-T1](https://novasky-ai.github.io/posts/sky-t1/)**: An o1-like model fine-tuned by NovaSky AI with very small cost.
@@ -940,7 +972,7 @@ If you have a project that should be incorporated, please contact via email or c
 
 This repository is licensed under the [Apache-2.0 License](LICENSE).
 
-Please follow the model licenses to use the corresponding model weights: [BLOOM](https://huggingface.co/spaces/bigscience/license) / [DeepSeek](https://github.com/deepseek-ai/DeepSeek-LLM/blob/main/LICENSE-MODEL) / [Falcon](https://huggingface.co/tiiuae/falcon-180B/blob/main/LICENSE.txt) / [Gemma](https://ai.google.dev/gemma/terms) / [GLM-4](https://huggingface.co/THUDM/glm-4-9b/blob/main/LICENSE) / [GPT-2](https://github.com/openai/gpt-2/blob/master/LICENSE) / [Granite](LICENSE) / [InternLM](https://github.com/InternLM/InternLM#license) / [Llama](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) / [Llama 2](https://ai.meta.com/llama/license/) / [Llama 3](https://llama.meta.com/llama3/license/) / [Llama 4](https://github.com/meta-llama/llama-models/blob/main/models/llama4/LICENSE) / [MiniCPM](https://github.com/OpenBMB/MiniCPM/blob/main/MiniCPM%20Model%20License.md) / [Mistral/Mixtral/Pixtral](LICENSE) / [Phi-3/Phi-4](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/blob/main/LICENSE) / [Qwen](https://github.com/QwenLM/Qwen/blob/main/Tongyi%20Qianwen%20LICENSE%20AGREEMENT) / [StarCoder 2](https://huggingface.co/spaces/bigcode/bigcode-model-license-agreement) / [TeleChat2](https://huggingface.co/Tele-AI/telechat-7B/blob/main/TeleChat%E6%A8%A1%E5%9E%8B%E7%A4%BE%E5%8C%BA%E8%AE%B8%E5%8F%AF%E5%8D%8F%E8%AE%AE.pdf) / [Yuan 2](https://github.com/IEIT-Yuan/Yuan-2.0/blob/main/LICENSE-Yuan)
+Please follow the model licenses to use the corresponding model weights: [BLOOM](https://huggingface.co/spaces/bigscience/license) / [DeepSeek](https://github.com/deepseek-ai/DeepSeek-LLM/blob/main/LICENSE-MODEL) / [Falcon](https://huggingface.co/tiiuae/falcon-180B/blob/main/LICENSE.txt) / [Gemma](https://ai.google.dev/gemma/terms) / [GLM-4](https://huggingface.co/THUDM/glm-4-9b/blob/main/LICENSE) / [GPT-2](https://github.com/openai/gpt-2/blob/master/LICENSE) / [Granite](LICENSE) / [InternLM](https://github.com/InternLM/InternLM#license) / [Llama](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) / [Llama 2](https://ai.meta.com/llama/license/) / [Llama 3](https://llama.meta.com/llama3/license/) / [Llama 4](https://github.com/meta-llama/llama-models/blob/main/models/llama4/LICENSE) / [MiniCPM](https://github.com/OpenBMB/MiniCPM/blob/main/LICENSE) / [Mistral/Mixtral/Pixtral](LICENSE) / [Phi-3/Phi-4](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/blob/main/LICENSE) / [Qwen](https://github.com/QwenLM/Qwen/blob/main/Tongyi%20Qianwen%20LICENSE%20AGREEMENT) / [StarCoder 2](https://huggingface.co/spaces/bigcode/bigcode-model-license-agreement) / [TeleChat2](https://huggingface.co/Tele-AI/telechat-7B/blob/main/TeleChat%E6%A8%A1%E5%9E%8B%E7%A4%BE%E5%8C%BA%E8%AE%B8%E5%8F%AF%E5%8D%8F%E8%AE%AE.pdf) / [Yuan 2](https://github.com/IEIT-Yuan/Yuan-2.0/blob/main/LICENSE-Yuan)
 
 ## Citation
 
@@ -961,7 +993,3 @@ If this work is helpful, please kindly cite as:
 ## Acknowledgement
 
 This repo benefits from [PEFT](https://github.com/huggingface/peft), [TRL](https://github.com/huggingface/trl), [QLoRA](https://github.com/artidoro/qlora) and [FastChat](https://github.com/lm-sys/FastChat). Thanks for their wonderful works.
-
-## Star History
-
-![Star History Chart](https://api.star-history.com/svg?repos=hiyouga/LLaMA-Factory&type=Date)

README_zh.md

@@ -1,9 +1,9 @@
-![# LLaMA Factory](assets/logo.png)
+![# LlamaFactory](assets/logo.png)
 
-[![GitHub Repo stars](https://img.shields.io/github/stars/hiyouga/LLaMA-Factory?style=social)](https://github.com/hiyouga/LLaMA-Factory/stargazers)
+[![GitHub Repo stars](https://img.shields.io/github/stars/hiyouga/LlamaFactory?style=social)](https://github.com/hiyouga/LlamaFactory/stargazers)
-[![GitHub last commit](https://img.shields.io/github/last-commit/hiyouga/LLaMA-Factory)](https://github.com/hiyouga/LLaMA-Factory/commits/main)
+[![GitHub last commit](https://img.shields.io/github/last-commit/hiyouga/LlamaFactory)](https://github.com/hiyouga/LlamaFactory/commits/main)
-[![GitHub contributors](https://img.shields.io/github/contributors/hiyouga/LLaMA-Factory?color=orange)](https://github.com/hiyouga/LLaMA-Factory/graphs/contributors)
+[![GitHub contributors](https://img.shields.io/github/contributors/hiyouga/LlamaFactory?color=orange)](https://github.com/hiyouga/LlamaFactory/graphs/contributors)
-[![GitHub workflow](https://github.com/hiyouga/LLaMA-Factory/actions/workflows/tests.yml/badge.svg)](https://github.com/hiyouga/LLaMA-Factory/actions/workflows/tests.yml)
+[![GitHub workflow](https://github.com/hiyouga/LlamaFactory/actions/workflows/tests.yml/badge.svg)](https://github.com/hiyouga/LlamaFactory/actions/workflows/tests.yml)
 [![PyPI](https://img.shields.io/pypi/v/llamafactory)](https://pypi.org/project/llamafactory/)
 [![Citation](https://img.shields.io/badge/citation-1000+-green)](https://scholar.google.com/scholar?cites=12620864006390196564)
 [![Docker Pulls](https://img.shields.io/docker/pulls/hiyouga/llamafactory)](https://hub.docker.com/r/hiyouga/llamafactory/tags)
@@ -19,7 +19,21 @@
 [![Open in Studios](https://img.shields.io/badge/ModelScope-Open%20in%20Studios-blue)](https://modelscope.cn/studios/hiyouga/LLaMA-Board)
 [![Open in Novita](https://img.shields.io/badge/Novita-Deploy%20Template-blue)](https://novita.ai/templates-library/105981?sharer=88115474-394e-4bda-968e-b88e123d0c47)
 
-### 获得[亚马逊](https://aws.amazon.com/cn/blogs/china/a-one-stop-code-free-model-fine-tuning-deployment-platform-based-on-sagemaker-and-llama-factory/)、[英伟达](https://developer.nvidia.cn/rtx/ai-toolkit)、[阿里云](https://help.aliyun.com/zh/pai/use-cases/fine-tune-a-llama-3-model-with-llama-factory)等的应用。
+### 获得[亚马逊](https://aws.amazon.com/cn/blogs/china/a-one-stop-code-free-model-fine-tuning-deployment-platform-based-on-sagemaker-and-llama-factory/)、[英伟达](https://build.nvidia.com/spark/llama-factory)、[阿里云](https://help.aliyun.com/zh/pai/use-cases/fine-tune-a-llama-3-model-with-llama-factory)等的应用。
+
+----
+
+<div align="center" markdown="1">
+
+### 欢迎关注我们全新的开源项目——<br>🐧 [PenguinHarness](https://github.com/Prism-Shadow/penguin-harness)：只需 0.2 元的 Token，即可自动构建 Agent 的桌面级 Agent！
+
+点击关注项目：https://github.com/Prism-Shadow/penguin-harness
+
+</div>
+
+https://github.com/user-attachments/assets/604eb626-0a5d-4a62-87e3-14ebade1cd5f
+
+----
 
 <div align="center" markdown="1">
 
@@ -32,7 +46,7 @@
 
 ### 使用零代码[命令行](#快速开始)与 [Web UI](#llama-board-可视化微调由-gradio-驱动) 轻松微调百余种大模型
 
-![GitHub Trend](https://trendshift.io/api/badge/repositories/4535)
+![GitHub Trend](https://trendshift.io/api/badge/repositories/17371)
 
 </div>
 
@@ -72,7 +86,7 @@ https://github.com/user-attachments/assets/43b700c6-a178-41db-b1f8-8190a5d3fcfc
 - [数据集](#数据集)
 - [软硬件依赖](#软硬件依赖)
 - [如何使用](#如何使用)
-  - [安装 LLaMA Factory](#安装-llama-factory)
+  - [安装 LlamaFactory](#安装-llamafactory)
   - [数据准备](#数据准备)
   - [快速开始](#快速开始)
   - [LLaMA Board 可视化微调](#llama-board-可视化微调由-gradio-驱动)
@@ -82,7 +96,7 @@ https://github.com/user-attachments/assets/43b700c6-a178-41db-b1f8-8190a5d3fcfc
   - [从魔乐社区下载](#从魔乐社区下载)
   - [使用 W&B 面板](#使用-wb-面板)
   - [使用 SwanLab 面板](#使用-swanlab-面板)
-- [使用了 LLaMA Factory 的项目](#使用了-llama-factory-的项目)
+- [使用了 LlamaFactory 的项目](#使用了-llamafactory-的项目)
 - [协议](#协议)
 - [引用](#引用)
 - [致谢](#致谢)
@@ -108,35 +122,35 @@ https://github.com/user-attachments/assets/43b700c6-a178-41db-b1f8-8190a5d3fcfc
 ## 官方博客
 
 > [!TIP]
-> 我们现在拥有了 LLaMA Factory 的专属博客！
+> 我们现在拥有了 LlamaFactory 的专属博客！
 >
 > 网站地址：https://blog.llamafactory.net/
 
-- 💡 [KTransformers Fine-Tuning × LLaMA Factory: 用2张4090级的GPU+CPU 微调 1000B规模的超大模型](https://swcil84qspu.feishu.cn/wiki/Z1sSwb2poijybxkyPEkcDG6enVc) (中文)
+- 💡 [KTransformers Fine-Tuning × LlamaFactory: 用2张4090级的GPU+CPU 微调 1000B规模的超大模型](https://swcil84qspu.feishu.cn/wiki/Z1sSwb2poijybxkyPEkcDG6enVc) (中文)
-- 💡 [Easy Dataset × LLaMA Factory: 让大模型高效学习领域知识](https://buaa-act.feishu.cn/wiki/KY9xwTGs1iqHrRkjXBwcZP9WnL9)（中文）
+- 💡 [Easy Dataset × LlamaFactory: 让大模型高效学习领域知识](https://buaa-act.feishu.cn/wiki/KY9xwTGs1iqHrRkjXBwcZP9WnL9)（中文）
-- 💡 [DataFlow × LLaMA Factory: 利用数据准备流水线产出高质量数据训练 LLM](https://wcny4qa9krto.feishu.cn/wiki/LlMxweUAJimrmykRD5qcGuswnHd)（中文）| [English](https://wcny4qa9krto.feishu.cn/wiki/LWkkwTDBfiiRKqkDSvucG6yjnbW)
+- 💡 [DataFlow × LlamaFactory: 利用数据准备流水线产出高质量数据训练 LLM](https://wcny4qa9krto.feishu.cn/wiki/LlMxweUAJimrmykRD5qcGuswnHd)（中文）| [English](https://wcny4qa9krto.feishu.cn/wiki/LWkkwTDBfiiRKqkDSvucG6yjnbW)
-- 💡 [DataFlex × LLaMA Factory: 构建在 LLaMA-Factory 之上的以数据为中心的动态训练系统](https://wcny4qa9krto.feishu.cn/wiki/H2A9wSsbCinzavkT2oyc2C5Vn0e)（中文）| [English](https://wcny4qa9krto.feishu.cn/wiki/OlREwPQWdi9K6ZkJNHIciLhtnkv)
+- 💡 [DataFlex × LlamaFactory: 构建在 LlamaFactory 之上的以数据为中心的动态训练系统](https://wcny4qa9krto.feishu.cn/wiki/H2A9wSsbCinzavkT2oyc2C5Vn0e)（中文）| [English](https://wcny4qa9krto.feishu.cn/wiki/OlREwPQWdi9K6ZkJNHIciLhtnkv)
-- [基于 LLaMA-Factory 和 EasyR1 打造一站式无代码大模型强化学习和部署平台 LLM Model Hub](https://aws.amazon.com/cn/blogs/china/building-llm-model-hub-based-on-llamafactory-and-easyr1/)（中文）
+- [基于 LlamaFactory 和 EasyR1 打造一站式无代码大模型强化学习和部署平台 LLM Model Hub](https://aws.amazon.com/cn/blogs/china/building-llm-model-hub-based-on-llamafactory-and-easyr1/)（中文）
-- [通过亚马逊 SageMaker HyperPod 上的 LLaMA-Factory 增强多模态模型银行文档的视觉信息提取](https://aws.amazon.com/cn/blogs/machine-learning/how-apoidea-group-enhances-visual-information-extraction-from-banking-documents-with-multimodal-models-using-llama-factory-on-amazon-sagemaker-hyperpod/)（英文）
+- [通过亚马逊 SageMaker HyperPod 上的 LlamaFactory 增强多模态模型银行文档的视觉信息提取](https://aws.amazon.com/cn/blogs/machine-learning/how-apoidea-group-enhances-visual-information-extraction-from-banking-documents-with-multimodal-models-using-llama-factory-on-amazon-sagemaker-hyperpod/)（英文）
 
 <details><summary>全部博客</summary>
 
-- [LLaMA Factory：微调 DeepSeek-R1-Distill-Qwen-7B 模型实现新闻标题分类器](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory_deepseek_r1_distill_7b)（中文）
+- [LlamaFactory：微调 DeepSeek-R1-Distill-Qwen-7B 模型实现新闻标题分类器](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory_deepseek_r1_distill_7b)（中文）
-- [基于 Amazon SageMaker 和 LLaMA-Factory 打造一站式无代码模型微调部署平台 Model Hub](https://aws.amazon.com/cn/blogs/china/a-one-stop-code-free-model-fine-tuning-deployment-platform-based-on-sagemaker-and-llama-factory/)（中文）
+- [基于 Amazon SageMaker 和 LlamaFactory 打造一站式无代码模型微调部署平台 Model Hub](https://aws.amazon.com/cn/blogs/china/a-one-stop-code-free-model-fine-tuning-deployment-platform-based-on-sagemaker-and-llama-factory/)（中文）
-- [LLaMA Factory 多模态微调实践：微调 Qwen2-VL 构建文旅大模型](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory_qwen2vl)（中文）
+- [LlamaFactory 多模态微调实践：微调 Qwen2-VL 构建文旅大模型](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory_qwen2vl)（中文）
-- [LLaMA Factory：微调 Llama3 模型实现角色扮演](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory)（中文）
+- [LlamaFactory：微调 Llama3 模型实现角色扮演](https://gallery.pai-ml.com/#/preview/deepLearning/nlp/llama_factory)（中文）
 
 </details>
 
 ## 更新日志
 
-[25/10/26] 我们支持了Megatron-core作为训练后端和适配了[**mcore_adapter**](https://github.com/alibaba/ROLL/tree/main/mcore_adapter)。查看[PR #9237](https://github.com/hiyouga/LLaMA-Factory/pull/9237)以使用。
+[25/10/26] 我们支持了Megatron-core作为训练后端和适配了[**mcore_adapter**](https://github.com/alibaba/ROLL/tree/main/mcore_adapter)。查看[PR #9237](https://github.com/hiyouga/LlamaFactory/pull/9237)以使用。
 
 [25/08/22] 我们支持了 **[OFT](https://arxiv.org/abs/2306.07280)** 和 **[OFTv2](https://arxiv.org/abs/2506.19847)** 模型的微调。查看 [examples](examples/README.md) 以使用。
 
-[25/08/20] 我们支持了 **[Intern-S1-mini](https://huggingface.co/internlm/Intern-S1-mini)** 模型的微调。查看 [PR #8976](https://github.com/hiyouga/LLaMA-Factory/pull/8976) 以使用。
+[25/08/20] 我们支持了 **[Intern-S1-mini](https://huggingface.co/internlm/Intern-S1-mini)** 模型的微调。查看 [PR #8976](https://github.com/hiyouga/LlamaFactory/pull/8976) 以使用。
 
-[25/08/06] 我们支持了 **[GPT-OSS](https://github.com/openai/gpt-oss)** 模型的微调。查看 [PR #8826](https://github.com/hiyouga/LLaMA-Factory/pull/8826) 以使用。
+[25/08/06] 我们支持了 **[GPT-OSS](https://github.com/openai/gpt-oss)** 模型的微调。查看 [PR #8826](https://github.com/hiyouga/LlamaFactory/pull/8826) 以使用。
 
 <details><summary>展开日志</summary>
 
@@ -146,13 +160,13 @@ https://github.com/user-attachments/assets/43b700c6-a178-41db-b1f8-8190a5d3fcfc
 
 [25/04/21] 我们支持了 **[Muon](https://github.com/KellerJordan/Muon)** 优化器。详细用法请参照 [examples](examples/README_zh.md)。感谢 [@tianshijing](https://github.com/tianshijing) 的 PR。
 
-[25/04/16] 我们支持了 **[InternVL3](https://huggingface.co/OpenGVLab/InternVL3-8B)** 模型的微调。查看 [PR #7258](https://github.com/hiyouga/LLaMA-Factory/pull/7258) 以使用。
+[25/04/16] 我们支持了 **[InternVL3](https://huggingface.co/OpenGVLab/InternVL3-8B)** 模型的微调。查看 [PR #7258](https://github.com/hiyouga/LlamaFactory/pull/7258) 以使用。
 
 [25/04/14] 我们支持了 **[GLM-Z1](https://huggingface.co/THUDM/GLM-Z1-9B-0414)** 和 **[Kimi-VL](https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct)** 模型的微调。
 
-[25/04/06] 我们支持了 **[Llama 4](https://ai.meta.com/blog/llama-4-multimodal-intelligence/)** 模型的微调。查看 [PR #7611](https://github.com/hiyouga/LLaMA-Factory/pull/7611) 以使用。
+[25/04/06] 我们支持了 **[Llama 4](https://ai.meta.com/blog/llama-4-multimodal-intelligence/)** 模型的微调。查看 [PR #7611](https://github.com/hiyouga/LlamaFactory/pull/7611) 以使用。
 
-[25/03/31] 我们支持了 **[Qwen2.5 Omni](https://qwenlm.github.io/blog/qwen2.5-omni/)** 模型的微调。查看 [PR #7537](https://github.com/hiyouga/LLaMA-Factory/pull/7537) 以使用。
+[25/03/31] 我们支持了 **[Qwen2.5 Omni](https://qwenlm.github.io/blog/qwen2.5-omni/)** 模型的微调。查看 [PR #7537](https://github.com/hiyouga/LlamaFactory/pull/7537) 以使用。
 
 [25/03/15] 我们支持了 **[SGLang](https://github.com/sgl-project/sglang)** 推理后端，请使用 `infer_backend: sglang` 启用。
 
@@ -200,17 +214,17 @@ https://github.com/user-attachments/assets/43b700c6-a178-41db-b1f8-8190a5d3fcfc
 
 [24/05/18] 我们支持了 **[KTO](https://arxiv.org/abs/2402.01306)** 偏好对齐算法。详细用法请参照 [examples](examples/README_zh.md)。
 
-[24/05/14] 我们支持了昇腾 NPU 设备的训练和推理。详情请查阅[安装](#安装-llama-factory)部分。
+[24/05/14] 我们支持了昇腾 NPU 设备的训练和推理。详情请查阅[安装](#安装-llamafactory)部分。
 
 [24/04/26] 我们支持了多模态模型 **LLaVA-1.5** 的微调。详细用法请参照 [examples](examples/README_zh.md)。
 
-[24/04/22] 我们提供了在免费 T4 GPU 上微调 Llama-3 模型的 **[Colab 笔记本](https://colab.research.google.com/drive/1d5KQtbemerlSDSxZIfAaWXhKr30QypiK?usp=sharing)**。Hugging Face 社区公开了两个利用 LLaMA Factory 微调的 Llama-3 模型，详情请见 [Llama3-8B-Chinese-Chat](https://huggingface.co/shenzhi-wang/Llama3-8B-Chinese-Chat) 和 [Llama3-Chinese](https://huggingface.co/zhichen/Llama3-Chinese)。
+[24/04/22] 我们提供了在免费 T4 GPU 上微调 Llama-3 模型的 **[Colab 笔记本](https://colab.research.google.com/drive/1d5KQtbemerlSDSxZIfAaWXhKr30QypiK?usp=sharing)**。Hugging Face 社区公开了两个利用 LlamaFactory 微调的 Llama-3 模型，详情请见 [Llama3-8B-Chinese-Chat](https://huggingface.co/shenzhi-wang/Llama3-8B-Chinese-Chat) 和 [Llama3-Chinese](https://huggingface.co/zhichen/Llama3-Chinese)。
 
 [24/04/21] 我们基于 [AstraMindAI 的仓库](https://github.com/astramind-ai/Mixture-of-depths)支持了 **[混合深度训练](https://arxiv.org/abs/2404.02258)**。详细用法请参照 [examples](examples/README_zh.md)。
 
 [24/04/16] 我们支持了 **[BAdam](https://arxiv.org/abs/2404.02827)** 优化器。详细用法请参照 [examples](examples/README_zh.md)。
 
-[24/04/16] 我们支持了 **[unsloth](https://github.com/unslothai/unsloth)** 的长序列训练（24GB 可训练 Llama-2-7B-56k）。该方法相比 FlashAttention-2 提供了 **117%** 的训练速度和 **50%** 的显存节约。更多数据请见[此页面](https://github.com/hiyouga/LLaMA-Factory/wiki/Performance-comparison)。
+[24/04/16] 我们支持了 **[unsloth](https://github.com/unslothai/unsloth)** 的长序列训练（24GB 可训练 Llama-2-7B-56k）。该方法相比 FlashAttention-2 提供了 **117%** 的训练速度和 **50%** 的显存节约。更多数据请见[此页面](https://github.com/hiyouga/LlamaFactory/wiki/Performance-comparison)。
 
 [24/03/31] 我们支持了 **[ORPO](https://arxiv.org/abs/2403.07691)**。详细用法请参照 [examples](examples/README_zh.md)。
 
@@ -228,11 +242,11 @@ https://github.com/user-attachments/assets/43b700c6-a178-41db-b1f8-8190a5d3fcfc
 
 [24/02/15] 我们支持了 [LLaMA Pro](https://github.com/TencentARC/LLaMA-Pro) 提出的**块扩展**方法。详细用法请参照 [examples](examples/README_zh.md)。
 
-[24/02/05] Qwen1.5（Qwen2 测试版）系列模型已在 LLaMA-Factory 中实现微调支持。详情请查阅该[博客页面](https://qwenlm.github.io/zh/blog/qwen1.5/)。
+[24/02/05] Qwen1.5（Qwen2 测试版）系列模型已在 LlamaFactory 中实现微调支持。详情请查阅该[博客页面](https://qwenlm.github.io/zh/blog/qwen1.5/)。
 
 [24/01/18] 我们针对绝大多数模型实现了 **Agent 微调**，微调时指定 `dataset: glaive_toolcall_zh` 即可使模型获得工具调用能力。
 
-[23/12/23] 我们针对 LLaMA, Mistral 和 Yi 模型支持了 **[unsloth](https://github.com/unslothai/unsloth)** 的 LoRA 训练加速。请使用 `use_unsloth: true` 参数启用 unsloth 优化。该方法可提供 **170%** 的训练速度，详情请查阅[此页面](https://github.com/hiyouga/LLaMA-Factory/wiki/Performance-comparison)。
+[23/12/23] 我们针对 LLaMA, Mistral 和 Yi 模型支持了 **[unsloth](https://github.com/unslothai/unsloth)** 的 LoRA 训练加速。请使用 `use_unsloth: true` 参数启用 unsloth 优化。该方法可提供 **170%** 的训练速度，详情请查阅[此页面](https://github.com/hiyouga/LlamaFactory/wiki/Performance-comparison)。
 
 [23/12/12] 我们支持了微调最新的混合专家模型 **[Mixtral 8x7B](https://huggingface.co/mistralai/Mixtral-8x7B-v0.1)**。硬件需求请查阅[此处](#硬件依赖)。
 
@@ -267,7 +281,7 @@ https://github.com/user-attachments/assets/43b700c6-a178-41db-b1f8-8190a5d3fcfc
 </details>
 
 > [!TIP]
-> 如果您无法使用最新的功能，请尝试重新拉取代码并再次安装 LLaMA-Factory。
+> 如果您无法使用最新的功能，请尝试重新拉取代码并再次安装 LlamaFactory。
 
 ## 模型
 
@@ -502,7 +516,7 @@ huggingface-cli login
 
 ## 如何使用
 
-### 安装 LLaMA Factory
+### 安装 LlamaFactory
 
 > [!IMPORTANT]
 > 此步骤为必需。
@@ -591,18 +605,22 @@ pip install https://github.com/jllllll/bitsandbytes-windows-webui/releases/downl
 
 <details><summary>昇腾 NPU 用户指南</summary>
 
-在昇腾 NPU 设备上安装 LLaMA Factory 时，请升级 Python 到 3.10 及以上，并需要指定额外依赖项，使用 `pip install -r requirements/npu.txt` 命令安装。此外，还需要安装 **Ascend CANN Toolkit 与 Kernels**，安装方法请参考[安装教程](https://llamafactory.readthedocs.io/zh-cn/latest/advanced/npu_installation.html)。
+在昇腾 NPU 设备上安装 LlamaFactory 时，请使用 Python 3.12，并使用 `pip install -r requirements/npu.txt` 命令安装额外依赖项。此外，还需要安装 **Ascend CANN Toolkit 与 Kernels**，安装方法请参考[安装教程](https://llamafactory.readthedocs.io/zh-cn/latest/multibackend/npu/npu_installation.html)。
 
 您可以直接下载预安装的最新docker镜像：
 
 ```bash
 # Docker Hub
-docker pull hiyouga/llamafactory:latest-npu-a2
+docker pull hiyouga/llamafactory:latest-910b-ubuntu
-docker pull hiyouga/llamafactory:latest-npu-a3
+docker pull hiyouga/llamafactory:latest-a3-ubuntu
+docker pull hiyouga/llamafactory:latest-910b-openeuler
+docker pull hiyouga/llamafactory:latest-a3-openeuler
 
 # quay.io
-docker pull quay.io/ascend/llamafactory:latest-npu-a2
+docker pull quay.io/ascend/llamafactory:latest-910b-ubuntu
-docker pull quay.io/ascend/llamafactory:latest-npu-a3
+docker pull quay.io/ascend/llamafactory:latest-a3-ubuntu
+docker pull quay.io/ascend/llamafactory:latest-910b-openeuler
+docker pull quay.io/ascend/llamafactory:latest-a3-openeuler
 ```
 
 #### 安装 BitsAndBytes
@@ -665,7 +683,7 @@ llamafactory-cli export examples/merge_lora/qwen3_lora_sft.yaml
 > [!TIP]
 > 使用 `llamafactory-cli help` 显示帮助信息。
 >
-> 遇到报错请先看[常见问题](https://github.com/hiyouga/LLaMA-Factory/issues/4614)。
+> 遇到报错请先看[常见问题](https://github.com/hiyouga/LlamaFactory/issues/4614)。
 
 ### LLaMA Board 可视化微调（由 [Gradio](https://github.com/gradio-app/gradio) 驱动）
 
@@ -687,8 +705,22 @@ docker compose exec llamafactory bash
 
 ```bash
 cd docker/docker-npu/
-docker compose up -d
+
-docker compose exec llamafactory bash
+# A2 + Ubuntu
+docker compose --profile a2-ubuntu up -d
+docker compose --profile a2-ubuntu exec llamafactory-a2-ubuntu bash
+
+# A3 + Ubuntu
+docker compose --profile a3-ubuntu up -d
+docker compose --profile a3-ubuntu exec llamafactory-a3-ubuntu bash
+
+# A2 + openEuler
+docker compose --profile a2-openeuler up -d
+docker compose --profile a2-openeuler exec llamafactory-a2-openeuler bash
+
+# A3 + openEuler
+docker compose --profile a3-openeuler up -d
+docker compose --profile a3-openeuler exec llamafactory-a3-openeuler bash
 ```
 
 AMD ROCm 用户：
@@ -723,7 +755,6 @@ docker exec -it llamafactory bash
 ```bash
 docker build -f ./docker/docker-npu/Dockerfile \
     --build-arg PIP_INDEX=https://pypi.org/simple \
-    --build-arg EXTRAS=torch-npu,metrics \
     -t llamafactory:latest .
 
 docker run -dit --ipc=host \
@@ -833,7 +864,7 @@ swanlab_run_name: test_run # 可选
 方式二：将环境变量 `SWANLAB_API_KEY` 设置为你的 [API 密钥](https://swanlab.cn/settings)。
 方式三：启动前使用 `swanlab login` 命令完成登录。
 
-## 使用了 LLaMA Factory 的项目
+## 使用了 LlamaFactory 的项目
 
 如果您有项目希望添加至下述列表，请通过邮件联系或者创建一个 PR。
 
@@ -913,7 +944,7 @@ swanlab_run_name: test_run # 可选
 1. Sun et al. LAMBDA: A Large Model Based Data Agent. 2024. [[arxiv]](https://arxiv.org/abs/2407.17535)
 1. Zhu et al. CollectiveSFT: Scaling Large Language Models for Chinese Medical Benchmark with Collective Instructions in Healthcare. 2024. [[arxiv]](https://arxiv.org/abs/2407.19705)
 1. Yu et al. Correcting Negative Bias in Large Language Models through Negative Attention Score Alignment. 2024. [[arxiv]](https://arxiv.org/abs/2408.00137)
-1. Xie et al. The Power of Personalized Datasets: Advancing Chinese Composition Writing for Elementary School through Targeted Model Fine-Tuning. IALP 2024. [[paper]](https://www.asianlp.sg/conferences/ialp2024/proceedings/papers/IALP2024_P055.pdf)
+1. Xie et al. The Power of Personalized Datasets: Advancing Chinese Composition Writing for Elementary School through Targeted Model Fine-Tuning. IALP 2024. [[paper]](https://doi.org/10.1142/S2717554524500176)
 1. Liu et al. Instruct-Code-Llama: Improving Capabilities of Language Model in Competition Level Code Generation by Online Judge Feedback. ICIC 2024. [[paper]](https://link.springer.com/chapter/10.1007/978-981-97-5669-8_11)
 1. Wang et al. Cybernetic Sentinels: Unveiling the Impact of Safety Data Selection on Model Security in Supervised Fine-Tuning. ICIC 2024. [[paper]](https://link.springer.com/chapter/10.1007/978-981-97-5669-8_23)
 1. Xia et al. Understanding the Performance and Estimating the Cost of LLM Fine-Tuning. 2024. [[arxiv]](https://arxiv.org/abs/2408.04693)
@@ -930,7 +961,7 @@ swanlab_run_name: test_run # 可选
 1. **[Chinese-LLaVA-Med](https://github.com/BUAADreamer/Chinese-LLaVA-Med)**：中文多模态医学大模型，基于 LLaVA-1.5-7B 在中文多模态医疗数据上微调而得。
 1. **[AutoRE](https://github.com/THUDM/AutoRE)**：基于大语言模型的文档级关系抽取系统。
 1. **[NVIDIA RTX AI Toolkit](https://github.com/NVIDIA/RTX-AI-Toolkit)**：在 Windows 主机上利用英伟达 RTX 设备进行大型语言模型微调的开发包。
-1. **[LazyLLM](https://github.com/LazyAGI/LazyLLM)**：一个低代码构建多 Agent 大模型应用的开发工具，支持基于 LLaMA Factory 的模型微调.
+1. **[LazyLLM](https://github.com/LazyAGI/LazyLLM)**：一个低代码构建多 Agent 大模型应用的开发工具，支持基于 LlamaFactory 的模型微调.
 1. **[RAG-Retrieval](https://github.com/NLPJCL/RAG-Retrieval)**：一个全链路 RAG 检索模型微调、推理和蒸馏代码库。[[blog]](https://zhuanlan.zhihu.com/p/987727357)
 1. **[360-LLaMA-Factory](https://github.com/Qihoo360/360-LLaMA-Factory)**：一个魔改后的代码库，通过 Ring Attention 支持长序列的 SFT 和 DPO 训练。
 1. **[Sky-T1](https://novasky-ai.github.io/posts/sky-t1/)**：由 NovaSky AI 微调的低成本类 o1 长推理模型。
@@ -942,7 +973,7 @@ swanlab_run_name: test_run # 可选
 
 本仓库的代码依照 [Apache-2.0](LICENSE) 协议开源。
 
-使用模型权重时，请遵循对应的模型协议：[BLOOM](https://huggingface.co/spaces/bigscience/license)/ [DeepSeek](https://github.com/deepseek-ai/DeepSeek-LLM/blob/main/LICENSE-MODEL) / [Falcon](https://huggingface.co/tiiuae/falcon-180B/blob/main/LICENSE.txt) / [Gemma](https://ai.google.dev/gemma/terms) / [GLM-4](https://huggingface.co/THUDM/glm-4-9b/blob/main/LICENSE) / [GPT-2](https://github.com/openai/gpt-2/blob/master/LICENSE) / [Granite](LICENSE) / [InternLM](https://github.com/InternLM/InternLM#license) / [Llama](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) / [Llama 2](https://ai.meta.com/llama/license/) / [Llama 3](https://llama.meta.com/llama3/license/) / [Llama 4](https://github.com/meta-llama/llama-models/blob/main/models/llama4/LICENSE) / [MiniCPM](https://github.com/OpenBMB/MiniCPM/blob/main/MiniCPM%20Model%20License.md) / [Mistral/Mixtral/Pixtral](LICENSE) / [Phi-3/Phi-4](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/blob/main/LICENSE) / [Qwen](https://github.com/QwenLM/Qwen/blob/main/Tongyi%20Qianwen%20LICENSE%20AGREEMENT) / [StarCoder 2](https://huggingface.co/spaces/bigcode/bigcode-model-license-agreement) / [TeleChat2](https://huggingface.co/Tele-AI/telechat-7B/blob/main/TeleChat%E6%A8%A1%E5%9E%8B%E7%A4%BE%E5%8C%BA%E8%AE%B8%E5%8F%AF%E5%8D%8F%E8%AE%AE.pdf) / [Yuan 2](https://github.com/IEIT-Yuan/Yuan-2.0/blob/main/LICENSE-Yuan)
+使用模型权重时，请遵循对应的模型协议：[BLOOM](https://huggingface.co/spaces/bigscience/license)/ [DeepSeek](https://github.com/deepseek-ai/DeepSeek-LLM/blob/main/LICENSE-MODEL) / [Falcon](https://huggingface.co/tiiuae/falcon-180B/blob/main/LICENSE.txt) / [Gemma](https://ai.google.dev/gemma/terms) / [GLM-4](https://huggingface.co/THUDM/glm-4-9b/blob/main/LICENSE) / [GPT-2](https://github.com/openai/gpt-2/blob/master/LICENSE) / [Granite](LICENSE) / [InternLM](https://github.com/InternLM/InternLM#license) / [Llama](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) / [Llama 2](https://ai.meta.com/llama/license/) / [Llama 3](https://llama.meta.com/llama3/license/) / [Llama 4](https://github.com/meta-llama/llama-models/blob/main/models/llama4/LICENSE) / [MiniCPM](https://github.com/OpenBMB/MiniCPM/blob/main/LICENSE) / [Mistral/Mixtral/Pixtral](LICENSE) / [Phi-3/Phi-4](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/blob/main/LICENSE) / [Qwen](https://github.com/QwenLM/Qwen/blob/main/Tongyi%20Qianwen%20LICENSE%20AGREEMENT) / [StarCoder 2](https://huggingface.co/spaces/bigcode/bigcode-model-license-agreement) / [TeleChat2](https://huggingface.co/Tele-AI/telechat-7B/blob/main/TeleChat%E6%A8%A1%E5%9E%8B%E7%A4%BE%E5%8C%BA%E8%AE%B8%E5%8F%AF%E5%8D%8F%E8%AE%AE.pdf) / [Yuan 2](https://github.com/IEIT-Yuan/Yuan-2.0/blob/main/LICENSE-Yuan)
 
 ## 引用
 
@@ -963,7 +994,3 @@ swanlab_run_name: test_run # 可选
 ## 致谢
 
 本项目受益于 [PEFT](https://github.com/huggingface/peft)、[TRL](https://github.com/huggingface/trl)、[QLoRA](https://github.com/artidoro/qlora) 和 [FastChat](https://github.com/lm-sys/FastChat)，感谢以上诸位作者的付出。
-
-## Star History
-
-![Star History Chart](https://api.star-history.com/svg?repos=hiyouga/LLaMA-Factory&type=Date)

data/v1_multimodal_demo.jsonl

@@ -0,0 +1,25 @@
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}, {"type": "text", "value": "Who are they?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "They're Kane and Gretzka from Bayern Munich."}]}, {"role": "user", "content": [{"type": "text", "value": "What are they doing?"}, {"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}]}, {"role": "assistant", "content": [{"type": "text", "value": "They are celebrating on the soccer field."}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/2.jpg"}, {"type": "text", "value": "Who is he?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "He's Thomas Muller from Bayern Munich."}]}, {"role": "user", "content": [{"type": "text", "value": "Why is he on the ground?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "Because he's sliding on his knees to celebrate."}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/3.jpg"}, {"type": "text", "value": "Please describe this image"}]}, {"role": "assistant", "content": [{"type": "text", "value": "Chinese astronaut Gui Haichao is giving a speech."}]}, {"role": "user", "content": [{"type": "text", "value": "What has he accomplished?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "He was appointed to be a payload specialist on Shenzhou 16 mission in June 2022, thus becoming the first Chinese civilian of Group 3 in space on 30 May 2023. He is responsible for the on-orbit operation of space science experimental payloads."}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}, {"type": "text", "value": "他们是谁？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他们是拜仁慕尼黑的凯恩和格雷茨卡。"}]}, {"role": "user", "content": [{"type": "text", "value": "他们在做什么？"}, {"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他们在足球场上庆祝。"}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/2.jpg"}, {"type": "text", "value": "他是谁？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他是来自拜仁慕尼黑的托马斯·穆勒。"}]}, {"role": "user", "content": [{"type": "text", "value": "他为什么在地上？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "因为他正在双膝跪地滑行庆祝。"}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/3.jpg"}, {"type": "text", "value": "请描述这张图片"}]}, {"role": "assistant", "content": [{"type": "text", "value": "中国宇航员桂海潮正在讲话。"}]}, {"role": "user", "content": [{"type": "text", "value": "他取得过哪些成就？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他于2022年6月被任命为神舟十六号任务的有效载荷专家，从而成为2023年5月30日进入太空的首位平民宇航员。他负责在轨操作空间科学实验有效载荷。"}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}, {"type": "text", "value": "Who are they?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "They're Kane and Gretzka from Bayern Munich."}]}, {"role": "user", "content": [{"type": "text", "value": "What are they doing?"}, {"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}]}, {"role": "assistant", "content": [{"type": "text", "value": "They are celebrating on the soccer field."}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/2.jpg"}, {"type": "text", "value": "Who is he?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "He's Thomas Muller from Bayern Munich."}]}, {"role": "user", "content": [{"type": "text", "value": "Why is he on the ground?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "Because he's sliding on his knees to celebrate."}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/3.jpg"}, {"type": "text", "value": "Please describe this image"}]}, {"role": "assistant", "content": [{"type": "text", "value": "Chinese astronaut Gui Haichao is giving a speech."}]}, {"role": "user", "content": [{"type": "text", "value": "What has he accomplished?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "He was appointed to be a payload specialist on Shenzhou 16 mission in June 2022, thus becoming the first Chinese civilian of Group 3 in space on 30 May 2023. He is responsible for the on-orbit operation of space science experimental payloads."}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}, {"type": "text", "value": "他们是谁？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他们是拜仁慕尼黑的凯恩和格雷茨卡。"}]}, {"role": "user", "content": [{"type": "text", "value": "他们在做什么？"}, {"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他们在足球场上庆祝。"}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/2.jpg"}, {"type": "text", "value": "他是谁？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他是来自拜仁慕尼黑的托马斯·穆勒。"}]}, {"role": "user", "content": [{"type": "text", "value": "他为什么在地上？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "因为他正在双膝跪地滑行庆祝。"}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/3.jpg"}, {"type": "text", "value": "请描述这张图片"}]}, {"role": "assistant", "content": [{"type": "text", "value": "中国宇航员桂海潮正在讲话。"}]}, {"role": "user", "content": [{"type": "text", "value": "他取得过哪些成就？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他于2022年6月被任命为神舟十六号任务的有效载荷专家，从而成为2023年5月30日进入太空的首位平民宇航员。他负责在轨操作空间科学实验有效载荷。"}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}, {"type": "text", "value": "Who are they?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "They're Kane and Gretzka from Bayern Munich."}]}, {"role": "user", "content": [{"type": "text", "value": "What are they doing?"}, {"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}]}, {"role": "assistant", "content": [{"type": "text", "value": "They are celebrating on the soccer field."}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/2.jpg"}, {"type": "text", "value": "Who is he?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "He's Thomas Muller from Bayern Munich."}]}, {"role": "user", "content": [{"type": "text", "value": "Why is he on the ground?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "Because he's sliding on his knees to celebrate."}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/3.jpg"}, {"type": "text", "value": "Please describe this image"}]}, {"role": "assistant", "content": [{"type": "text", "value": "Chinese astronaut Gui Haichao is giving a speech."}]}, {"role": "user", "content": [{"type": "text", "value": "What has he accomplished?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "He was appointed to be a payload specialist on Shenzhou 16 mission in June 2022, thus becoming the first Chinese civilian of Group 3 in space on 30 May 2023. He is responsible for the on-orbit operation of space science experimental payloads."}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}, {"type": "text", "value": "他们是谁？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他们是拜仁慕尼黑的凯恩和格雷茨卡。"}]}, {"role": "user", "content": [{"type": "text", "value": "他们在做什么？"}, {"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他们在足球场上庆祝。"}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/2.jpg"}, {"type": "text", "value": "他是谁？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他是来自拜仁慕尼黑的托马斯·穆勒。"}]}, {"role": "user", "content": [{"type": "text", "value": "他为什么在地上？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "因为他正在双膝跪地滑行庆祝。"}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/3.jpg"}, {"type": "text", "value": "请描述这张图片"}]}, {"role": "assistant", "content": [{"type": "text", "value": "中国宇航员桂海潮正在讲话。"}]}, {"role": "user", "content": [{"type": "text", "value": "他取得过哪些成就？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他于2022年6月被任命为神舟十六号任务的有效载荷专家，从而成为2023年5月30日进入太空的首位平民宇航员。他负责在轨操作空间科学实验有效载荷。"}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}, {"type": "text", "value": "Who are they?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "They're Kane and Gretzka from Bayern Munich."}]}, {"role": "user", "content": [{"type": "text", "value": "What are they doing?"}, {"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}]}, {"role": "assistant", "content": [{"type": "text", "value": "They are celebrating on the soccer field."}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/2.jpg"}, {"type": "text", "value": "Who is he?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "He's Thomas Muller from Bayern Munich."}]}, {"role": "user", "content": [{"type": "text", "value": "Why is he on the ground?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "Because he's sliding on his knees to celebrate."}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/3.jpg"}, {"type": "text", "value": "Please describe this image"}]}, {"role": "assistant", "content": [{"type": "text", "value": "Chinese astronaut Gui Haichao is giving a speech."}]}, {"role": "user", "content": [{"type": "text", "value": "What has he accomplished?"}]}, {"role": "assistant", "content": [{"type": "text", "value": "He was appointed to be a payload specialist on Shenzhou 16 mission in June 2022, thus becoming the first Chinese civilian of Group 3 in space on 30 May 2023. He is responsible for the on-orbit operation of space science experimental payloads."}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}, {"type": "text", "value": "他们是谁？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他们是拜仁慕尼黑的凯恩和格雷茨卡。"}]}, {"role": "user", "content": [{"type": "text", "value": "他们在做什么？"}, {"type": "image_url", "value": "data/mllm_demo_data/1.jpg"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他们在足球场上庆祝。"}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/2.jpg"}, {"type": "text", "value": "他是谁？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他是来自拜仁慕尼黑的托马斯·穆勒。"}]}, {"role": "user", "content": [{"type": "text", "value": "他为什么在地上？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "因为他正在双膝跪地滑行庆祝。"}]}]}
+{"messages": [{"role": "user", "content": [{"type": "image_url", "value": "data/mllm_demo_data/3.jpg"}, {"type": "text", "value": "请描述这张图片"}]}, {"role": "assistant", "content": [{"type": "text", "value": "中国宇航员桂海潮正在讲话。"}]}, {"role": "user", "content": [{"type": "text", "value": "他取得过哪些成就？"}]}, {"role": "assistant", "content": [{"type": "text", "value": "他于2022年6月被任命为神舟十六号任务的有效载荷专家，从而成为2023年5月30日进入太空的首位平民宇航员。他负责在轨操作空间科学实验有效载荷。"}]}]}
+

data/v1_multimodal_demo.yaml

@@ -0,0 +1,4 @@
+multimodal_demo:
+  path: data/v1_multimodal_demo.jsonl
+  source: local
+

docker/docker-cuda/Dockerfile.mbridge

@@ -0,0 +1,85 @@
+# LLaMA-Factory + Megatron Bridge (CUDA) runtime
+# Mirrors the verified host env: Python 3.12, PyTorch 2.12.1+cu126,
+# TransformerEngine 2.17, megatron-core 0.18, megatron-bridge 0.5.
+#
+# CUDA user-mode libs come from the PyTorch cu126 wheels (same as the host venv).
+# Layers are intentionally few (helps vfs / nested-docker disk usage).
+#
+# Build from repo root:
+#   bash docker/docker-cuda/build-megatron.sh
+
+ARG BASE_IMAGE=ubuntu:24.04
+FROM ${BASE_IMAGE}
+
+ARG PIP_INDEX=https://mirrors.aliyun.com/pypi/simple
+ARG PYPI_TRUSTED_HOST=mirrors.aliyun.com
+ARG TORCH_INDEX=https://download.pytorch.org/whl/cu126
+ARG APT_MIRROR=https://mirrors.tuna.tsinghua.edu.cn/ubuntu/
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    PIP_ROOT_USER_ACTION=ignore \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_BREAK_SYSTEM_PACKAGES=1 \
+    PIP_CONSTRAINT="" \
+    MAX_JOBS=8 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    VLLM_WORKER_MULTIPROC_METHOD=spawn \
+    DISABLE_VERSION_CHECK=1 \
+    USE_MEGATRON_BRIDGE=1 \
+    GRADIO_SERVER_PORT=7860 \
+    API_PORT=8000 \
+    http_proxy= \
+    https_proxy=
+
+SHELL ["/bin/bash", "-c"]
+WORKDIR /app
+
+# System deps + Ubuntu Python 3.12
+RUN if [ -f /etc/apt/sources.list.d/ubuntu.sources ]; then \
+      sed -i "s|http://archive.ubuntu.com/ubuntu|${APT_MIRROR}|g; s|http://security.ubuntu.com/ubuntu|${APT_MIRROR}|g" /etc/apt/sources.list.d/ubuntu.sources; \
+    elif [ -f /etc/apt/sources.list ]; then \
+      sed -i "s|http://archive.ubuntu.com/ubuntu/|${APT_MIRROR}|g; s|http://security.ubuntu.com/ubuntu/|${APT_MIRROR}|g" /etc/apt/sources.list; \
+    fi && \
+    apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates curl git vim wget \
+        build-essential ninja-build cmake libgomp1 zip unzip \
+        python3 python3-pip python3-dev python3-venv && \
+    ln -sf /usr/bin/python3 /usr/bin/python && \
+    rm -rf /var/lib/apt/lists/* && \
+    pip install --no-cache-dir --upgrade pip setuptools wheel packaging ninja pybind11 \
+        --trusted-host ${PYPI_TRUSTED_HOST} --index-url ${PIP_INDEX}
+
+COPY . /app
+
+# PyTorch + TE + Megatron Bridge + LLaMA-Factory in one layer
+RUN pip install --no-cache-dir \
+        torch==2.12.1 torchvision==0.27.1 torchaudio==2.11.0 \
+        --index-url ${TORCH_INDEX} && \
+    pip install --no-cache-dir --no-build-isolation \
+        "transformer-engine[pytorch]==2.17.0" \
+        --trusted-host ${PYPI_TRUSTED_HOST} --index-url ${PIP_INDEX} && \
+    pip install --no-cache-dir --no-build-isolation \
+        "megatron-bridge==0.5.0" \
+        --trusted-host ${PYPI_TRUSTED_HOST} --index-url ${PIP_INDEX} && \
+    pip install --no-cache-dir --no-build-isolation -e . \
+        --trusted-host ${PYPI_TRUSTED_HOST} --index-url ${PIP_INDEX} && \
+    pip install --no-cache-dir --no-build-isolation \
+        -r requirements/metrics.txt \
+        --trusted-host ${PYPI_TRUSTED_HOST} --index-url ${PIP_INDEX} && \
+    pip install --no-cache-dir --no-build-isolation \
+        "megatron-bridge==0.5.0" \
+        --trusted-host ${PYPI_TRUSTED_HOST} --index-url ${PIP_INDEX} && \
+    python - <<'PY'
+import torch
+import megatron.core
+import transformer_engine
+from megatron.bridge import AutoBridge  # noqa: F401
+import llamafactory
+print("torch", torch.__version__, "cuda", torch.version.cuda)
+print("te", transformer_engine.__version__)
+print("megatron-bridge import ok")
+PY
+
+EXPOSE 7860 8000
+CMD ["bash"]

docker/docker-cuda/README.md

@@ -104,6 +104,37 @@ sudo usermod -aG docker $USER
 # Log out and back in for changes to take effect
 ```
 
+## Megatron Bridge Image
+
+`Dockerfile.megatron` builds a CUDA runtime for LLaMA-Factory + [Megatron Bridge](https://docs.nvidia.com/nemo/megatron-bridge/latest/):
+
+| Component | Version |
+| --- | --- |
+| Base | `ubuntu:22.04` (Python 3.12) |
+| PyTorch | 2.12.1+cu126 (CUDA libs from wheels) |
+| TransformerEngine | 2.17.0 |
+| megatron-core | 0.18.x (via megatron-bridge) |
+| megatron-bridge | 0.5.0 |
+
+### Build
+
+From repo root:
+
+```bash
+docker build -f docker/docker-cuda/Dockerfile.megatron \
+  -t llamafactory-megatron-bridge:latest .
+```
+
+### Run training
+
+```bash
+docker run --rm -it --gpus all --ipc=host --shm-size=16g \
+  -e DISABLE_VERSION_CHECK=1 \
+  -e USE_MEGATRON_BRIDGE=1 \
+  -v "$PWD":/app -w /app \
+  llamafactory-megatron-bridge:latest
+```
+
 ## Additional Notes
 
 - The default image is built on Ubuntu 22.04 (x86_64), CUDA 12.4, Python 3.11, PyTorch 2.6.0, and Flash-attn 2.7.4

docker/docker-npu/Dockerfile

@@ -1,6 +1,6 @@
 # https://hub.docker.com/r/ascendai/cann/tags
 
-ARG BASE_IMAGE=quay.io/ascend/cann:9.0.0-910b-ubuntu22.04-py3.11
+ARG BASE_IMAGE=quay.io/ascend/cann:9.1.0-910b-ubuntu22.04-py3.12
 FROM ${BASE_IMAGE}
 
 # Installation arguments

docker/docker-npu/OVERVIEW.md

@@ -0,0 +1,183 @@
+# LlamaFactory Image for Ascend NPU
+
+LlamaFactory Ascend NPU images are designed for Huawei Ascend Atlas NPUs and provide a ready-to-use LlamaFactory environment. Built on Ascend CANN container images, they include Python, PyTorch, TorchNPU, DeepSpeed, LlamaFactory, and other components.
+
+For installation and troubleshooting details, see the [English NPU installation guide](https://llamafactory.readthedocs.io/en/latest/multibackend/npu/npu_installation.html).
+
+## Quick Reference
+
+- Image registries:
+  - `docker.io/hiyouga/llamafactory`
+  - `quay.io/ascend/llamafactory`
+- Dockerfile: `docker/docker-npu/Dockerfile`
+- Docker Compose file: `docker/docker-npu/docker-compose.yml`
+
+The following `latest` NPU image tags are available:
+
+| Hardware series | Operating system | Tag |
+| --- | --- | --- |
+| A2 | Ubuntu 22.04 | `latest-910b-ubuntu` |
+| A3 | Ubuntu 22.04 | `latest-a3-ubuntu` |
+| A2 | openEuler 24.03 | `latest-910b-openeuler` |
+| A3 | openEuler 24.03 | `latest-a3-openeuler` |
+
+## Image Overview
+
+The image includes the following core components:
+
+| Component | Version |
+| --- | --- |
+| CANN | `9.1.0` |
+| Python | `3.12` |
+| PyTorch | `2.10.0` |
+| TorchNPU | `2.10.0.post2` |
+| torchvision / torchaudio | `0.25.0` / `2.10.0` |
+| Transformers | Latest compatible version at build time |
+| Triton Ascend | `3.2.1` |
+| DeepSpeed | Latest compatible version at build time |
+| LlamaFactory | Installed from the repository build context |
+
+The image does not include model weights or datasets. Mount or download them separately and comply with their respective licenses and acceptable-use requirements.
+
+## Image Tags
+
+NPU `latest` and release tags use different formats; the following rules do not apply to CUDA images.
+
+Non-release builds reuse the following short tags. Each scheduled build updates the image referenced by the corresponding tag:
+
+```text
+latest-<chip>-<os>
+```
+
+| Field | Values | Description |
+| --- | --- | --- |
+| `chip` | `910b` or `a3` | Ascend chip model supported by the image |
+| `os` | `ubuntu` or `openeuler` | Container operating system family |
+
+Release builds use full tags:
+
+```text
+<LlamaFactory-version>-cann<CANN-version>-torch_npu<TorchNPU-version>-<chip>-<os>-<Python-version>
+```
+
+| Field | Example | Description |
+| --- | --- | --- |
+| `LlamaFactory-version` | `0.9.5` | LlamaFactory release version |
+| `CANN-version` | `9.1.0` | Parsed from the CANN base image tag |
+| `TorchNPU-version` | `2.10.0.post2` | Full TorchNPU version used by the image, including suffixes such as `.postN` |
+| `chip` | `910b` or `a3` | Ascend chip model supported by the image |
+| `os` | `ubuntu22.04` or `openeuler24.03` | Container operating system family and version |
+| `Python-version` | `py3.12` | Parsed from the CANN base image tag |
+
+For example:
+
+```text
+0.9.5-cann9.1.0-torch_npu2.10.0.post2-a3-ubuntu22.04-py3.12
+```
+
+## Quick Start
+
+### Prerequisites
+
+Before starting a container:
+
+1. Install an Ascend driver and firmware compatible with the CANN version in the image.
+2. Verify that `npu-smi info` works on the host.
+3. Install Docker with permission to access the required Ascend device nodes and driver files.
+
+Driver, firmware, CANN, TorchNPU, and the target Ascend hardware must be mutually compatible.
+
+### Pull and Run
+
+The following example starts the latest A2 Ubuntu image with one NPU. Adjust `DOCKER_IMAGE` and the `--device` options for your environment.
+
+```bash
+CONTAINER_NAME=llamafactory-npu
+DOCKER_IMAGE=hiyouga/llamafactory:latest-910b-ubuntu
+
+docker run --rm -it \
+  --net=host \
+  --device=/dev/davinci0 \
+  --device=/dev/davinci_manager \
+  --device=/dev/devmm_svm \
+  --device=/dev/hisi_hdc \
+  -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
+  -v /usr/local/dcmi:/usr/local/dcmi \
+  -v /etc/ascend_install.info:/etc/ascend_install.info \
+  -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
+  -v /data:/data \
+  --name "$CONTAINER_NAME" \
+  "$DOCKER_IMAGE" \
+  /bin/bash
+```
+
+The host path for `npu-smi` may be `/usr/local/sbin/npu-smi` on some driver installations. Adjust the mount source when necessary. Add more `--device=/dev/davinci<N>` options to expose additional NPUs.
+
+Verify the runtime inside the container:
+
+```bash
+source /usr/local/Ascend/ascend-toolkit/set_env.sh
+npu-smi info
+python -c "import torch, torch_npu; print(torch.__version__, torch_npu.__version__, torch.npu.is_available())"
+llamafactory-cli help
+```
+
+### Build Locally
+
+Run the build from the repository root. The following example builds the A2 Ubuntu variant:
+
+```bash
+docker build \
+  -f ./docker/docker-npu/Dockerfile \
+  --build-arg BASE_IMAGE=quay.io/ascend/cann:9.1.0-910b-ubuntu22.04-py3.12 \
+  --build-arg PIP_INDEX=https://pypi.org/simple \
+  -t llamafactory:npu-910b-ubuntu \
+  .
+```
+
+Available build arguments:
+
+| Argument | Default | Purpose |
+| --- | --- | --- |
+| `BASE_IMAGE` | `quay.io/ascend/cann:9.1.0-910b-ubuntu22.04-py3.12` | Selects the base image that matches the device model and container operating system |
+| `PIP_INDEX` | `https://pypi.org/simple` | Selects the Python package index |
+| `PYTORCH_INDEX` | `https://download.pytorch.org/whl/cpu` | Selects the PyTorch wheel index used with TorchNPU |
+| `HTTP_PROXY` | Empty | Provides an optional HTTP/HTTPS proxy during the build |
+
+### Start with Docker Compose
+
+The preceding `docker build` command invokes the Dockerfile directly. It builds an image but does not start a container. Docker Compose does not use a separate build implementation: it reads the presets in `docker-compose.yml`, reuses the same Dockerfile, and selects a hardware-series and operating-system combination through a profile. Each `up -d` command below starts the selected container in the background. If the image is not available locally, Docker Compose builds it first:
+
+```bash
+cd docker/docker-npu
+
+# A2 with Ubuntu
+docker compose --profile a2-ubuntu up -d
+
+# A3 with Ubuntu
+docker compose --profile a3-ubuntu up -d
+
+# A2 with openEuler
+docker compose --profile a2-openeuler up -d
+
+# A3 with openEuler
+docker compose --profile a3-openeuler up -d
+```
+
+To build an image with Docker Compose without starting a container, use `docker compose --profile <profile> build`.
+
+## Hardware Support and Compatibility Notes
+
+- A2 images use the `910b` CANN base image; A3 images use the `a3` CANN base image.
+- The image build targets both x86-64 (`linux/amd64`) and AArch64 (`linux/arm64`) hosts. The CPU architecture is independent of whether the hardware series is A2 or A3.
+- Ubuntu 22.04 and openEuler 24.03 refer to the operating system inside the container.
+- Legacy NPU tags are replaced by the `latest-<910b|a3>-<ubuntu|openeuler>` format.
+- Validate the exact driver, firmware, CANN, and SoC combination before production deployment.
+
+## License and Disclaimer
+
+LlamaFactory is distributed under the [Apache License 2.0](../../LICENSE).
+
+Ascend CANN, TorchNPU, Triton Ascend, DeepSpeed, base operating-system packages, model weights, datasets, and other third-party components are governed by their respective licenses and terms. The LlamaFactory license does not replace or override those terms.
+
+The image is provided on an "AS IS" basis, without warranties or conditions of any kind. Users are responsible for validating hardware and software compatibility, securing the container and its runtime configuration, complying with applicable licenses and laws, and reviewing model and dataset terms before training, evaluation, or deployment.

docker/docker-npu/OVERVIEW.zh.md

@@ -0,0 +1,183 @@
+# 面向昇腾 NPU 的 LlamaFactory 镜像
+
+LlamaFactory 昇腾 NPU 镜像面向华为昇腾 Atlas NPU，提供可直接使用的 LlamaFactory 环境。镜像基于昇腾 CANN 容器镜像构建，预装 Python、PyTorch、TorchNPU、DeepSpeed、LlamaFactory 等组件。
+
+安装方法和问题排查请参考 [LlamaFactory NPU 安装及配置文档](https://llamafactory.readthedocs.io/zh-cn/latest/multibackend/npu/npu_installation.html)。
+
+## 快速参考
+
+- 镜像仓库：
+  - `docker.io/hiyouga/llamafactory`
+  - `quay.io/ascend/llamafactory`
+- Dockerfile：`docker/docker-npu/Dockerfile`
+- Docker Compose 文件：`docker/docker-npu/docker-compose.yml`
+
+当前提供以下 `latest` NPU 镜像 tag：
+
+| 硬件系列 | 操作系统 | Tag |
+| --- | --- | --- |
+| A2 | Ubuntu 22.04 | `latest-910b-ubuntu` |
+| A3 | Ubuntu 22.04 | `latest-a3-ubuntu` |
+| A2 | openEuler 24.03 | `latest-910b-openeuler` |
+| A3 | openEuler 24.03 | `latest-a3-openeuler` |
+
+## 镜像介绍
+
+镜像内预装以下主要组件：
+
+| 组件 | 版本 |
+| --- | --- |
+| CANN | `9.1.0` |
+| Python | `3.12` |
+| PyTorch | `2.10.0` |
+| TorchNPU | `2.10.0.post2` |
+| torchvision / torchaudio | `0.25.0` / `2.10.0` |
+| Transformers | 构建时的最新兼容版本 |
+| Triton Ascend | `3.2.1` |
+| DeepSpeed | 构建时的最新兼容版本 |
+| LlamaFactory | 从构建上下文中的仓库源码安装 |
+
+镜像不包含模型权重和数据集。请通过目录挂载或运行时下载的方式单独提供，并遵守对应的许可证和使用要求。
+
+## 镜像 Tag 说明
+
+NPU 镜像的 `latest` 和 release tag 使用不同格式；以下规则不适用于 CUDA 镜像。
+
+非 release 构建复用以下简短 tag，每次定时构建会更新对应 tag 所指向的镜像：
+
+```text
+latest-<芯片信息>-<操作系统>
+```
+
+| 字段 | 可选值 | 说明 |
+| --- | --- | --- |
+| `芯片信息` | `910b` 或 `a3` | 镜像所适配的昇腾芯片型号 |
+| `操作系统` | `ubuntu` 或 `openeuler` | 容器操作系统类型 |
+
+Release 构建使用完整 tag：
+
+```text
+<LlamaFactory版本>-cann<CANN版本>-torch_npu<TorchNPU版本>-<芯片信息>-<操作系统>-<Python版本>
+```
+
+| 字段 | 示例 | 说明 |
+| --- | --- | --- |
+| `LlamaFactory版本` | `0.9.5` | LlamaFactory release 版本号 |
+| `CANN版本` | `9.1.0` | 从 CANN 基础镜像 tag 中提取 |
+| `TorchNPU版本` | `2.10.0.post2` | 镜像使用的 TorchNPU 完整版本，包含 `.postN` 等后缀 |
+| `芯片信息` | `910b` 或 `a3` | 镜像所适配的昇腾芯片型号 |
+| `操作系统` | `ubuntu22.04` 或 `openeuler24.03` | 容器操作系统类型和版本 |
+| `Python版本` | `py3.12` | 从 CANN 基础镜像 tag 中提取 |
+
+例如：
+
+```text
+0.9.5-cann9.1.0-torch_npu2.10.0.post2-a3-ubuntu22.04-py3.12
+```
+
+## 快速开始
+
+### 前置条件
+
+启动容器前需要：
+
+1. 在宿主机安装与镜像内 CANN 版本兼容的昇腾驱动和固件。
+2. 确认宿主机执行 `npu-smi info` 可以正常识别 NPU。
+3. 安装 Docker，并确保当前用户有权访问所需的昇腾设备节点和驱动文件。
+
+驱动、固件、CANN、TorchNPU 与目标昇腾硬件需要保持兼容。
+
+### 拉取并运行镜像
+
+以下示例使用一张 NPU 启动最新的 A2 Ubuntu 镜像。请根据实际情况修改 ``DOCKER_IMAGE`` 和 ``device``。
+
+```bash
+CONTAINER_NAME=llamafactory-npu
+DOCKER_IMAGE=hiyouga/llamafactory:latest-910b-ubuntu
+
+docker run --rm -it \
+  --net=host \
+  --device=/dev/davinci0 \
+  --device=/dev/davinci_manager \
+  --device=/dev/devmm_svm \
+  --device=/dev/hisi_hdc \
+  -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
+  -v /usr/local/dcmi:/usr/local/dcmi \
+  -v /etc/ascend_install.info:/etc/ascend_install.info \
+  -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
+  -v /data:/data \
+  --name "$CONTAINER_NAME" \
+  "$DOCKER_IMAGE" \
+  /bin/bash
+```
+
+部分驱动环境中的 `npu-smi` 位于 `/usr/local/sbin/npu-smi`，此时需要调整挂载源路径。使用多张 NPU 时，继续追加 `--device=/dev/davinci<N>` 参数。
+
+进入容器后验证运行环境：
+
+```bash
+source /usr/local/Ascend/ascend-toolkit/set_env.sh
+npu-smi info
+python -c "import torch, torch_npu; print(torch.__version__, torch_npu.__version__, torch.npu.is_available())"
+llamafactory-cli help
+```
+
+### 本地构建镜像
+
+在仓库根目录执行构建。以下示例构建 A2 Ubuntu 镜像：
+
+```bash
+docker build \
+  -f ./docker/docker-npu/Dockerfile \
+  --build-arg BASE_IMAGE=quay.io/ascend/cann:9.1.0-910b-ubuntu22.04-py3.12 \
+  --build-arg PIP_INDEX=https://pypi.org/simple \
+  -t llamafactory:npu-910b-ubuntu \
+  .
+```
+
+可用构建参数：
+
+| 参数 | 默认值 | 用途 |
+| --- | --- | --- |
+| `BASE_IMAGE` | `quay.io/ascend/cann:9.1.0-910b-ubuntu22.04-py3.12` | 根据设备型号和容器操作系统选择对应的基础镜像 |
+| `PIP_INDEX` | `https://pypi.org/simple` | 指定 Python 软件包索引 |
+| `PYTORCH_INDEX` | `https://download.pytorch.org/whl/cpu` | 指定配合 TorchNPU 使用的 PyTorch wheel 索引 |
+| `HTTP_PROXY` | 空 | 构建期间可选的 HTTP/HTTPS 代理 |
+
+### 通过 Docker Compose 启动
+
+前面的 `docker build` 命令直接调用 Dockerfile，只构建镜像，不启动容器。Docker Compose 不使用另一套构建逻辑：它读取 `docker-compose.yml` 中的预设配置，复用同一个 Dockerfile，并通过 profile 选择硬件系列和操作系统组合。下面的 `up -d` 会在后台启动容器；若本地镜像不存在，Docker Compose 会先构建镜像：
+
+```bash
+cd docker/docker-npu
+
+# A2 + Ubuntu
+docker compose --profile a2-ubuntu up -d
+
+# A3 + Ubuntu
+docker compose --profile a3-ubuntu up -d
+
+# A2 + openEuler
+docker compose --profile a2-openeuler up -d
+
+# A3 + openEuler
+docker compose --profile a3-openeuler up -d
+```
+
+如果只想通过 Docker Compose 构建镜像而不启动容器，请使用 `docker compose --profile <profile> build`。
+
+## 硬件支持与兼容性说明
+
+- A2 镜像使用标记为 `910b` 的 CANN 基础镜像，A3 镜像使用标记为 `a3` 的 CANN 基础镜像。
+- 镜像构建目标同时包含 x86-64（`linux/amd64`）和 AArch64（`linux/arm64`）宿主机。CPU 架构与硬件系列是 A2 还是 A3 无关。
+- Ubuntu 22.04 和 openEuler 24.03 指容器内部的操作系统。
+- 旧式 NPU tag 已由 `latest-<910b|a3>-<ubuntu|openeuler>` 格式取代。
+- 正式部署前，请验证具体驱动、固件、CANN 和 SoC 组合的兼容性。
+
+## 许可证与免责声明
+
+LlamaFactory 基于 [Apache License 2.0](../../LICENSE) 发布。
+
+昇腾 CANN、TorchNPU、Triton Ascend、DeepSpeed、基础操作系统软件包、模型权重、数据集和其他第三方组件分别受其自身许可证与条款约束。LlamaFactory 的许可证不会替代或覆盖这些条款。
+
+本镜像按“原样”提供，不附带任何明示或暗示的保证。用户需要自行验证软硬件兼容性、保障容器及运行配置的安全、遵守适用的许可证和法律，并在训练、评测或部署前审查模型与数据集的使用条款。

docker/docker-npu/docker-compose.yml

@@ -1,58 +1,81 @@
+x-build-args: &build-args
+  PIP_INDEX: https://pypi.org/simple
+
+x-build: &build
+  dockerfile: ./docker/docker-npu/Dockerfile
+  context: ../..
+
+x-npu-common: &npu-common
+  volumes:
+    - /usr/local/dcmi:/usr/local/dcmi
+    - /usr/local/bin/npu-smi:/usr/local/bin/npu-smi
+    - /usr/local/Ascend/driver:/usr/local/Ascend/driver
+    - /etc/ascend_install.info:/etc/ascend_install.info
+  ipc: host
+  tty: true
+  # shm_size: "16gb"  # ipc: host is set
+  stdin_open: true
+  command: bash
+  devices:
+    - /dev/davinci0
+    - /dev/davinci_manager
+    - /dev/devmm_svm
+    - /dev/hisi_hdc
+  restart: unless-stopped
+
 services:
-  llamafactory-a2:
+  llamafactory-a2-ubuntu:
+    <<: *npu-common
+    profiles: ["a2-ubuntu"]
     build:
-      dockerfile: ./docker/docker-npu/Dockerfile
+      <<: *build
-      context: ../..
       args:
-        PIP_INDEX: https://pypi.org/simple
+        <<: *build-args
-    container_name: llamafactory-a2
+        BASE_IMAGE: quay.io/ascend/cann:9.1.0-910b-ubuntu22.04-py3.12
-    image: llamafactory:npu-a2
+    container_name: llamafactory-910b-ubuntu
-    volumes:
+    image: llamafactory:npu-910b-ubuntu
-      - /usr/local/dcmi:/usr/local/dcmi
-      - /usr/local/bin/npu-smi:/usr/local/bin/npu-smi
-      - /usr/local/Ascend/driver:/usr/local/Ascend/driver
-      - /etc/ascend_install.info:/etc/ascend_install.info
     ports:
       - "7860:7860"
       - "8000:8000"
-    ipc: host
-    tty: true
-    # shm_size: "16gb"  # ipc: host is set
-    stdin_open: true
-    command: bash
-    devices:
-      - /dev/davinci0
-      - /dev/davinci_manager
-      - /dev/devmm_svm
-      - /dev/hisi_hdc
-    restart: unless-stopped
 
-  llamafactory-a3:
+  llamafactory-a3-ubuntu:
-    profiles: ["a3"]
+    <<: *npu-common
+    profiles: ["a3-ubuntu"]
     build:
-      dockerfile: ./docker/docker-npu/Dockerfile
+      <<: *build
-      context: ../..
       args:
-        BASE_IMAGE: quay.io/ascend/cann:9.0.0-a3-ubuntu22.04-py3.11
+        <<: *build-args
-        PIP_INDEX: https://pypi.org/simple
+        BASE_IMAGE: quay.io/ascend/cann:9.1.0-a3-ubuntu22.04-py3.12
-    container_name: llamafactory-a3
+    container_name: llamafactory-a3-ubuntu
-    image: llamafactory:npu-a3
+    image: llamafactory:npu-a3-ubuntu
-    volumes:
-      - /usr/local/dcmi:/usr/local/dcmi
-      - /usr/local/bin/npu-smi:/usr/local/bin/npu-smi
-      - /usr/local/Ascend/driver:/usr/local/Ascend/driver
-      - /etc/ascend_install.info:/etc/ascend_install.info
     ports:
       - "7861:7860"
       - "8001:8000"
-    ipc: host
+
-    tty: true
+  llamafactory-a2-openeuler:
-    # shm_size: "16gb"  # ipc: host is set
+    <<: *npu-common
-    stdin_open: true
+    profiles: ["a2-openeuler"]
-    command: bash
+    build:
-    devices:
+      <<: *build
-      - /dev/davinci0
+      args:
-      - /dev/davinci_manager
+        <<: *build-args
-      - /dev/devmm_svm
+        BASE_IMAGE: quay.io/ascend/cann:9.1.0-910b-openeuler24.03-py3.12
-      - /dev/hisi_hdc
+    container_name: llamafactory-910b-openeuler
-    restart: unless-stopped
+    image: llamafactory:npu-910b-openeuler
+    ports:
+      - "7862:7860"
+      - "8002:8000"
+
+  llamafactory-a3-openeuler:
+    <<: *npu-common
+    profiles: ["a3-openeuler"]
+    build:
+      <<: *build
+      args:
+        <<: *build-args
+        BASE_IMAGE: quay.io/ascend/cann:9.1.0-a3-openeuler24.03-py3.12
+    container_name: llamafactory-a3-openeuler
+    image: llamafactory:npu-a3-openeuler
+    ports:
+      - "7863:7860"
+      - "8003:8000"

docs/en/advanced/distributed/fsdpturbo-ep-efsdp.md

@@ -0,0 +1,173 @@
+# FSDPTurbo EP/EFSDP and LlamaFactory FSDP2/CP Design
+
+Chinese version: [FSDPTurbo EP/EFSDP 与 LlamaFactory FSDP2/CP 设计说明](../../../zh/advanced/distributed/fsdpturbo-ep-efsdp.md)
+
+This document describes the current implementation of the `fsdpturbo` distributed plugin. Its core principle is a clear separation of responsibilities:
+
+- FSDPTurbo owns expert parallelism (EP), expert parameter sharding (EFSDP), and device operator registration.
+- LlamaFactory owns process initialization, the base DeviceMesh, outer FSDP2, CP, model initialization, and weight loading.
+- The LlamaFactory integration layer combines the two parameter layouts and handles gradient norms across meshes.
+
+## 1. Configuration Boundaries
+
+Common parallel topology belongs to `TrainingArguments`, while FSDPTurbo-only settings remain in `dist_config`:
+
+```yaml
+cp_size: 1
+
+dist_config:
+  name: fsdpturbo
+  ep_size: 16
+  ep_dispatcher: eager
+```
+
+The fields used by the minimal example have the following responsibilities:
+
+- `ep_size`: expert-parallel group size.
+- `ep_dispatcher`: FSDPTurbo EP dispatcher, which defaults to `eager`.
+
+`dp_size`, `cp_size`, `cp_mode`, `mp_replicate_size`, `mp_shard_size`, and `dist_timeout` are common topology fields and therefore remain at the top level. `dist_config` is parsed strictly as `FSDPTurboParams`; putting a common topology field inside it is rejected instead of being silently ignored.
+
+The top-level training option `bf16` controls FSDPTurbo parameter storage and compute dtype. The backend casts the model before FSDP materialization, so `ModelEngine` does not need to read distributed-backend configuration.
+
+The following advanced fields are optional and are therefore omitted from the minimal YAML example above:
+
+- `fsdp_ignored_modules`: additional modules excluded from the outer LlamaFactory FSDP2 path. Expert parameters selected by the model spec are automatically added to the ignored set by the integration layer, so normal configurations do not need to repeat them here.
+- `hook_modules`: optional module patterns for FSDPTurbo EFSDP hooks. The default is an empty list.
+- `fsdp_implementation`: the FSDPTurbo EFSDP implementation, either `native` or `custom`. The default is `native`.
+
+The model spec determines the EFSDP targets. Non-expert parameters such as attention, embeddings, and the LM head do not enter the FSDPTurbo EFSDP plan. They remain managed by the outer LlamaFactory FSDP2 layer.
+
+Model-specific module paths and preparation logic are managed exclusively by the `FSDPTurboEPModelSpec` registry. Built-in specs currently cover `qwen3_moe` and `qwen3_5_moe`; unregistered models fail with an explicit error. `ep_modules` and `ep_fsdp_modules` are not YAML options, and strict parameter parsing rejects them to prevent configuration from drifting away from the actual model structure.
+
+## 2. Mesh Initialization
+
+LlamaFactory's `DistributedInterface` initializes only its existing model and data meshes. It is unaware of EP and EFSDP and does not expose an extra mesh registration interface for distributed plugins. The FSDPTurbo expert topology is independently created and owned by `FSDPTurboParallelState` in the plugin module:
+
+```text
+run_sft / run_dpo / run_rm
+  -> DistributedInterface(training_args)
+     -> initialize LlamaFactory model/data meshes
+  -> DistributedPlugin("fsdpturbo").shard_model(...)
+     -> FSDPTurboFSDP2Engine.__init__()
+        -> FSDPTurboParallelState.initialize()
+           -> initialize and retain the expert parent mesh and submeshes
+```
+
+`FSDPTurboParallelState` creates a four-dimensional expert parent mesh:
+
+```text
+(edp, efsdp, ep, expert_cp)
+```
+
+Its current dimensions are calculated as follows:
+
+```text
+dp_size       = world_size / cp_size
+ep_fsdp_size  = dp_size / ep_size
+edp_size      = dp_size / (ep_size * ep_fsdp_size)
+mesh_shape    = (edp_size, ep_fsdp_size, ep_size, cp_size)
+```
+
+The state object retains `edp_mesh`, `efsdp_mesh`, `ep_mesh`, and `expert_cp_mesh`. Model sharding and gradient norm logic inside the plugin read expert communication domains from this state, while other LlamaFactory backends do not need to implement or know about these interfaces. Initialization validates that `ep_size` is positive and divides `dp_size`; repeated initialization also rejects topology changes.
+
+## 3. Model Sharding Order
+
+The wrapping order must remain "expert side first, outer FSDP2 second":
+
+```text
+DistributedPlugin("fsdpturbo")
+  -> FSDPTurboFSDP2Engine.shard_model(model)
+     -> prepare_model_ep(model)
+        -> expert_parallelize_modules(model, ep_mesh, ep_plan)
+        -> expert_fully_shard_modules(model, efsdp_mesh, ep_plan, fsdp_plan)
+        -> collect expert parameters as ignored_params
+     -> FSDP2Engine.prepare_model(model, ignored_params=...)
+        -> apply outer fully_shard to the remaining Transformer Layers and root module
+```
+
+This prevents the same expert parameter from being managed by both EFSDP and outer FSDP2. Outer FSDP2 continues to reuse LlamaFactory's model initialization, checkpoint, and save flows.
+
+The LlamaFactory integration layer accepts `eager`, `fused`, `mc2`, and `domino` and forwards the selected value unchanged to FSDPTurbo. Their implementation boundaries and current validation status differ:
+
+| Dispatcher | Main path | Additional requirements | Validation in this PR |
+| --- | --- | --- | --- |
+| `eager` | Uses PyTorch implementations of permute, unpermute, and grouped matmul while tensors remain on the current accelerator, with standard AllToAll for token dispatch and combine | Minimal dependencies; serves as the reference implementation | End-to-end numerical and performance validation completed on Ascend A3 |
+| `fused` | Keeps the same AllToAll topology while replacing permute, unpermute, and grouped matmul with device-fused operators | Requires matching device operators, dtypes, and layouts; local operators may fall back to eager when an expert receives no tokens | End-to-end numerical and performance validation completed on Ascend A3 |
+| `mc2` | Uses dedicated operators that fuse AllToAllV with grouped matmul to reduce intermediate communication-computation overhead | Requires the MC2 NPU operators, an HCCL communicator, and their shape and dtype constraints | Implemented by FSDPTurbo but not validated end to end in this PR |
+| `domino` | Splits the first dimension of the expert-module input into two slices and uses a separate communication stream and events to overlap AllToAll with expert computation | Requires asynchronous stream/event support and enough token work in both slices to amortize scheduling overhead | Implemented by FSDPTurbo but not validated end to end in this PR |
+
+Only `eager` and `fused` are validated here because they cover the reference path and the commonly used A3 device-fused path, respectively, and therefore isolate and establish the correctness of the EP/EFSDP integration between LlamaFactory and FSDPTurbo. The current experiment matrix was not extended to `mc2` and `domino`: they add operator, communication-scheduling, and input-shape constraints that require separate numerical comparisons, long-run stability tests, and profiler analysis. They are accepted configuration choices, but the results in this PR should not be interpreted as evidence that they have reached the same stability, numerical, or performance level.
+
+## 4. FSDPTurbo Dependency Entry Points
+
+LlamaFactory imports each required object directly from the module that defines it:
+
+```python
+from fsdp_turbo.distributed.expert_parallel.expert_fully_shard_parallel import (
+    expert_fully_shard_modules,
+)
+from fsdp_turbo.distributed.expert_parallel.expert_parallel import expert_parallelize_modules
+from fsdp_turbo.fsdp_turbo_config import EPPlanConfig, FSDPPlanConfig
+from fsdp_turbo.utils.str_match import module_name_match
+```
+
+The imports occur inside `prepare_model_ep()`, so other distributed backends remain importable when FSDPTurbo is not installed. They intentionally bypass aggregate exports from `fsdp_turbo.distributed.__init__` to avoid extra dependencies and potential import cycles during package initialization.
+
+## 5. Gradient Norms
+
+Outer and expert parameters can belong to different DTensor meshes and therefore cannot be passed together to a single standard `clip_grad_norm_()` call. The `fsdpturbo` plugin groups parameters by their owning mesh and computes local p-power sums:
+
+- Non-expert parameters are reduced over the DP and CP groups.
+- Expert parameters are reduced over the EFSDP, EP, and expert-CP groups retained by `FSDPTurboParallelState`.
+- After the global norm is assembled, the same clipping coefficient is applied to every local gradient.
+
+A zero-gradient warmup runs during startup so that the required collectives are initialized before training begins. This is currently a backend-specific implementation for `fsdpturbo`; other backends retain their existing gradient norm paths until the upstream distributed plugin interface is decoupled.
+
+## 6. Weight Loading
+
+LlamaFactory retains the `init_on_meta` and safetensors loading flow. The parent `FSDP2Engine` loader dynamically invokes the FSDPTurbo engine override through `self._copy_weights(...)`, so the method is not dead code. It supports DTensors with multiple `Shard` placements by calculating the local slice for the current rank along each mesh dimension in sequence. Model save and checkpoint interfaces continue to reuse the LlamaFactory FSDP2 implementation.
+
+## 7. Kernel Plugin
+
+FLA operators do not belong in the distributed configuration. Operator selection is handled through an independent `kernel_config`:
+
+```yaml
+kernel_config:
+  name: auto, flash-linear-attention
+  include_kernels: chunk_gated_delta_rule, fused_recurrent_gated_delta_rule
+  chunk_size: 32
+```
+
+The call path is:
+
+```text
+ModelEngine
+  -> apply_kernels("auto, flash-linear-attention")
+     -> accelerator-specific LlamaFactory auto kernels
+     -> KernelPlugin("flash-linear-attention").apply(...)
+        -> fsdp_turbo.ops.get_op()
+           -> FSDPTurbo device operator registry
+        -> fsdp_turbo.utils.patch.patch_model_members()
+           -> FLA backend implementation
+```
+
+`chunk_size` accepts `16`, `32`, and `64`, with a default of `64`. The kernel plugin and distributed plugin are independent. `name: flash-linear-attention` installs only the selected FLA operators. The comma-separated `name: auto, flash-linear-attention` form composes LlamaFactory's accelerator-specific automatic kernels with the FLA plugin before distributed sharding. LlamaFactory owns the operator-to-model-attribute mapping and `chunk_size` binding; FSDPTurbo owns device operator registration, selection, and generic callable patching. FLA stays explicit because it has optional external dependencies and is not part of the built-in `auto` set. FSDPTurbo subsequently replaces the target expert module's `forward`, so the final expert execution path is selected by `ep_dispatcher`; an MoE kernel applied during the auto stage is not retained as a separate second expert execution path.
+
+## 8. CP Runtime Constraints and Validation Scope
+
+When `init_on_meta` constructs the model, it must propagate `attn_implementation` in the same way as the `from_pretrained` path. Otherwise, the model falls back to a non-FlashAttention implementation and Ulysses CP cannot start. Before calling Hugging Face FlashAttention, Ulysses reconstructs the global attention mask. Only two-dimensional position IDs participate in packed-sequence detection. Multi-axis position IDs such as Qwen3.5 mRoPE have already been consumed by rotary embedding and must not be passed to the FlashAttention packed-sequence detection logic.
+
+The current implementation has completed the following BF16 AdamW full SFT validations with Qwen3.5-35B-A3B on Atlas 900 A3 SuperPoD and Atlas 950 SuperPoD systems. This revalidation used FSDPTurbo `0e96fbc`. The A3 environment used CANN 9.0.0, PyTorch 2.7.1, and torch-npu 2.7.1.post4; the A5 environment used CANN 9.1.0-beta.3, PyTorch 2.10.0, and torch-npu 2.10.0.post2. Performance is calculated from the step 1 and step 100 log timestamps and excludes initialization and compilation before the first step as well as model saving after training:
+
+| Machine | CP | EP | EFSDP | Checkpoint | Kernel / Dispatcher | Steps | Loss (first -> last) | Performance | Result |
+| --- | ---: | ---: | ---: | --- | --- | ---: | --- | ---: | --- |
+| Atlas 900 A3 SuperPoD | 1 | 16 | 1 | Off | FLA (chunk size 16) / eager | 100 | 1.3361 -> 0.0793 | 2.51 s/it | Passed and saved |
+| Atlas 900 A3 SuperPoD | 1 | 16 | 1 | Off | FLA (chunk size 16) / fused | 100 | 1.3354 -> 0.1179 | 2.17 s/it | Passed and saved |
+| Atlas 900 A3 SuperPoD | 2 | 4 | 2 | Off | auto + FLA (chunk size 64) / fused | 100 | 1.8114 -> 0.5260 | 7.65 s/it | Passed and saved |
+| Atlas 900 A3 SuperPoD | 2 | 4 | 2 | Off | auto + FLA (chunk size 64) / eager | 100 | 1.8095 -> 0.5596 | 5.88 s/it | Passed and saved |
+| Atlas 950 SuperPoD | 1 | 8 | 1 | Off | no kernel plugin configured / eager | 100 | 1.3575 -> 0.4439 | 2.68 s/it | Passed and saved |
+
+Loss and gradient norm remained finite in all five runs, and every run completed 100 steps and model saving. With the same partition, the per-step loss correlation between eager and fused was 0.997 for EP16 and 0.977 for CP2/EP4/EFSDP2, which indicates consistent optimization trajectories. The performance effect depends on the partition: fused was about 13% faster than eager with EP16, but about 30% slower after adding CP and EFSDP. Fused therefore should not be treated as the default optimum for every mesh.
+
+The EP16 runs used global batch 16 and cutoff length 256. The CP2 runs used global batch 8 and cutoff length 128. The A5 run used global batch 8 and cutoff length 256. The first-to-last loss validates convergence within each run; absolute loss values across different partition groups should not be used directly as a numerical-equivalence conclusion.

docs/en/advanced/ktransformers.md

@@ -0,0 +1,83 @@
+# KTransformers LoRA SFT
+
+KTransformers (KT) executes routed MoE experts on CPU while LLaMA-Factory remains responsible for data,
+LoRA arguments, and the training entry point. The production scope is routed-BF16 and routed-INT8 LoRA.
+
+KT has one user configuration source: the training YAML. Accelerate YAML contains FSDP2 settings only.
+LLaMA-Factory derives LoRA rank, alpha, dropout, activation policy, and local runtime capacity.
+
+```yaml
+finetuning_type: lora
+lora_rank: 8
+lora_alpha: 16
+lora_target: all
+
+use_kt: true
+disable_gradient_checkpointing: false
+kt_cpu_activation: retain
+kt_config:
+  kt_expert_weight_format: bf16
+  kt_backend: AMXBF16
+  kt_num_threads: 96
+  kt_tp_enabled: true
+  kt_threadpool_count: 2
+  kt_max_cache_depth: 2
+```
+
+Routed INT8 additionally requires matching expert and BF16 non-expert artifacts:
+
+```yaml
+kt_weight_path: /abs/path/to/routed-int8-experts
+kt_non_expert_weight_path: /abs/path/to/bf16-non-expert-cache
+kt_config:
+  kt_expert_weight_format: int8
+  kt_backend: auto
+  kt_weight_lifecycle: persistent
+```
+
+Launch the standard training entry point through Accelerate:
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1 accelerate launch \
+  --config_file examples/ktransformers/accelerate/fsdp2_kt_bf16.yaml \
+  src/train.py examples/ktransformers/train_lora/qwen3_5moe_lora_sft_kt.yaml
+```
+
+## Load a saved adapter
+
+Use a local, complete KT adapter directory for chat or evaluation. Repeat the training LoRA shape (`finetuning_type`,
+`lora_rank`, `lora_alpha`, and `lora_dropout`) and the KT base-weight settings. In particular, routed INT8 loading
+must use the same `kt_weight_path` and `kt_non_expert_weight_path` as training.
+
+```yaml
+model_name_or_path: /abs/path/to/base-model
+adapter_name_or_path: /abs/path/to/output/checkpoint-300
+finetuning_type: lora
+lora_rank: 8
+lora_alpha: 16
+lora_dropout: 0.0
+
+use_kt: true
+kt_cpu_activation: retain
+kt_config:
+  kt_expert_weight_format: bf16
+  kt_backend: AMXBF16
+  kt_num_threads: 96
+```
+
+```bash
+llamafactory-cli chat path/to/kt_adapter_infer.yaml
+llamafactory-cli eval path/to/kt_adapter_eval.yaml
+```
+
+The directory must contain the standard PEFT adapter files and, when fused routed-expert LoRA is used,
+`fused_expert_lora.safetensors` plus `kt_adapter_manifest.json`. LLaMA-Factory first loads the standard PEFT
+adapter, then KT validates and restores the fused artifact. `adapter_folder` may select a local subdirectory;
+paths outside the adapter root and Hub adapter IDs fail before model loading. Download a Hub bundle locally first.
+
+For training resume, keep the original training YAML and use `resume_from_checkpoint`. The optimizer checkpoint
+currently requires the same distributed world size. Missing, tampered, or mismatched artifacts fail closed instead
+of falling back to the source checkpoint.
+
+Do not combine KT with a second Transformers/FSDP checkpoint wrapper or Unsloth GC, and do not put `kt_config`
+in the Accelerate YAML. See the BF16 and INT8 examples under `examples/ktransformers/train_lora/`.

docs/en/index.rst

@@ -34,9 +34,11 @@ LlamaFactory Docs
 
    advanced/lora-and-quantization/lora
    advanced/lora-and-quantization/quantization
+   advanced/ktransformers
    advanced/distributed/fsdp
    advanced/distributed/deepspeed
    advanced/distributed/parallel-dp-tp-ep-sp-cp
+   advanced/distributed/fsdpturbo-ep-efsdp
    advanced/custom-kernels/triton
    advanced/custom-kernels/fused-operators
 

docs/zh/advanced/distributed/fsdpturbo-ep-efsdp.md

@@ -0,0 +1,215 @@
+# FSDPTurbo EP/EFSDP 与 LlamaFactory FSDP2/CP 设计说明
+
+English version: [FSDPTurbo EP/EFSDP and LlamaFactory FSDP2/CP Design](../../../en/advanced/distributed/fsdpturbo-ep-efsdp.md)
+
+本文描述 `fsdpturbo` distributed plugin 的当前实现。核心原则是保持两侧职责清晰：
+
+- FSDPTurbo 负责专家并行（EP）、专家参数分片（EFSDP）和设备算子注册。
+- LlamaFactory 负责进程初始化、基础 DeviceMesh、外层 FSDP2、CP、模型初始化与权重加载。
+- LlamaFactory 的集成层负责把两套参数布局组合起来，并处理跨 Mesh 的梯度范数。
+
+## 1. 配置边界
+
+公共并行拓扑放在 `TrainingArguments` 顶层，FSDPTurbo 私有参数保留在 `dist_config`：
+
+```yaml
+cp_size: 1
+
+dist_config:
+  name: fsdpturbo
+  ep_size: 16
+  ep_dispatcher: eager
+```
+
+最小示例中的字段职责如下：
+
+- `ep_size`：专家并行组大小。
+- `ep_dispatcher`：FSDPTurbo EP dispatcher，默认为 `eager`。
+
+`dp_size`、`cp_size`、`cp_mode`、`mp_replicate_size`、`mp_shard_size` 和 `dist_timeout`
+属于公共拓扑字段，继续放在顶层。`dist_config` 会被严格解析为 `FSDPTurboParams`；如果把公共拓扑
+字段误放进去，会直接报错，而不是静默忽略。
+
+顶层训练参数 `bf16` 同时控制 FSDPTurbo 的参数存储和计算 dtype。backend 会在 FSDP materialization
+前完成模型 dtype 转换，因此 `ModelEngine` 不需要读取 distributed backend 配置。
+
+以下高级字段为可选项，因此没有写入上面的最小 YAML 示例：
+
+- `fsdp_ignored_modules`：额外排除在 LlamaFactory 外层 FSDP2 之外的模块。模型规格选中的专家参数
+  会被集成层自动加入忽略集合，普通配置无需重复填写。
+- `hook_modules`：FSDPTurbo EFSDP hook 的可选模块模式，默认为空列表。
+- `fsdp_implementation`：FSDPTurbo EFSDP 实现，可选 `native` 或 `custom`，默认为 `native`。
+
+EFSDP 的目标由模型规格决定。Attention、Embedding、LM Head 等非专家参数不进入 FSDPTurbo
+EFSDP plan，而是继续由 LlamaFactory 外层 FSDP2 管理。
+
+模型相关的模块路径和准备逻辑统一由 `FSDPTurboEPModelSpec` 注册表管理。当前内置 `qwen3_moe`
+和 `qwen3_5_moe`；未注册的模型会明确报错。`ep_modules` 和 `ep_fsdp_modules` 不属于 YAML
+接口，严格参数解析会拒绝这两个字段，避免用户配置与模型实际结构失配。
+
+## 2. Mesh 初始化
+
+LlamaFactory 的 `DistributedInterface` 只初始化自身原有的 model/data mesh。它不感知 EP、EFSDP，
+也不为 distributed plugin 提供额外 mesh 注册接口。FSDPTurbo 的专家拓扑由插件文件内的
+`FSDPTurboParallelState` 独立创建和持有：
+
+```text
+run_sft / run_dpo / run_rm
+  -> DistributedInterface(training_args)
+     -> 初始化 LlamaFactory model/data mesh
+  -> DistributedPlugin("fsdpturbo").shard_model(...)
+     -> FSDPTurboFSDP2Engine.__init__()
+        -> FSDPTurboParallelState.initialize()
+           -> 初始化并保存 expert parent mesh 及其子 mesh
+```
+
+`FSDPTurboParallelState` 创建专家侧四维父 Mesh：
+
+```text
+(edp, efsdp, ep, expert_cp)
+```
+
+当前尺寸计算为：
+
+```text
+dp_size       = world_size / cp_size
+ep_fsdp_size  = dp_size / ep_size
+edp_size      = dp_size / (ep_size * ep_fsdp_size)
+mesh_shape    = (edp_size, ep_fsdp_size, ep_size, cp_size)
+```
+
+状态对象保存 `edp_mesh`、`efsdp_mesh`、`ep_mesh` 和 `expert_cp_mesh`。插件内部模型切分和梯度范数
+都从这个状态对象读取专家通信域；LlamaFactory 其他 backend 不需要实现或感知这些接口。状态初始化
+会校验 `ep_size` 为正数且能够整除 `dp_size`，重复初始化时也会拒绝拓扑发生变化。
+
+## 3. 模型切分顺序
+
+模型包装顺序必须保持为“专家侧优先，外层 FSDP2 随后”：
+
+```text
+DistributedPlugin("fsdpturbo")
+  -> FSDPTurboFSDP2Engine.shard_model(model)
+     -> prepare_model_ep(model)
+        -> expert_parallelize_modules(model, ep_mesh, ep_plan)
+        -> expert_fully_shard_modules(model, efsdp_mesh, ep_plan, fsdp_plan)
+        -> 收集专家参数作为 ignored_params
+     -> FSDP2Engine.prepare_model(model, ignored_params=...)
+        -> 对剩余 Transformer Layer 和根模块执行 outer fully_shard
+```
+
+这样可以避免同一专家参数同时被 EFSDP 和外层 FSDP2 管理。外层 FSDP2 仍复用 LlamaFactory
+原有的初始化、checkpoint 和保存流程。
+
+LlamaFactory 集成层接受 `eager`、`fused`、`mc2` 和 `domino`，并将选项原样传给
+FSDPTurbo。这四种模式的实现边界和当前验证状态不同：
+
+| Dispatcher | 主要路径 | 额外要求 | 本 PR 验证状态 |
+| --- | --- | --- | --- |
+| `eager` | 使用 PyTorch 实现 permute、unpermute 和 grouped matmul，张量仍在当前加速设备上，通过标准 AllToAll 完成 token dispatch/combine | 依赖最少，用作参考实现 | 已在 A3 上完成精度和性能验证 |
+| `fused` | 保持相同的 AllToAll 拓扑，将 permute、unpermute 和 grouped matmul 切换为设备融合算子 | 需要对应的设备算子、dtype 和 layout 支持；存在空专家时可回退到 eager 局部算子 | 已在 A3 上完成精度和性能验证 |
+| `mc2` | 使用专用算子融合 AllToAllV 和 grouped matmul，减少通信与计算之间的中间开销 | 依赖 MC2 NPU 算子、HCCL communicator 以及对应的 shape/dtype 约束 | FSDPTurbo 提供实现，本 PR 未做端到端验证 |
+| `domino` | 将专家模块输入的第一维分成两片，使用独立通信流和 event 重叠 AllToAll 与专家计算 | 需要异步 stream/event 支持，且两个分片都要有足够的 token 工作量才能覆盖调度开销 | FSDPTurbo 提供实现，本 PR 未做端到端验证 |
+
+当前只验证 `eager` 和 `fused`，是因为它们分别覆盖参考实现和 A3 常用设备融合路径，可用于隔离并验证
+LlamaFactory 与 FSDPTurbo 之间的 EP/EFSDP 集成正确性。本次实验矩阵没有继续扩展到 `mc2` 和
+`domino`：它们还引入了额外的算子、通信调度和输入形状约束，需要独立比较数值、长步稳定性和 profiler 结果。
+因此，它们在配置接口上可选，但不应从本 PR 的实验结果推断为已达到相同的稳定性、精度或性能水平。
+
+## 4. FSDPTurbo 依赖入口
+
+LlamaFactory 从各功能的定义模块直接导入所需对象：
+
+```python
+from fsdp_turbo.distributed.expert_parallel.expert_fully_shard_parallel import (
+    expert_fully_shard_modules,
+)
+from fsdp_turbo.distributed.expert_parallel.expert_parallel import expert_parallelize_modules
+from fsdp_turbo.fsdp_turbo_config import EPPlanConfig, FSDPPlanConfig
+from fsdp_turbo.utils.str_match import module_name_match
+```
+
+导入发生在 `prepare_model_ep()` 内，因此没有安装 FSDPTurbo 时，其他 distributed backend 仍可正常导入。
+这里不通过 `fsdp_turbo.distributed.__init__` 聚合导出，避免 package 初始化期间的额外依赖和潜在循环导入。
+
+## 5. 梯度范数
+
+外层参数和专家参数可能属于不同 DTensor Mesh，不能直接放入一次标准 `clip_grad_norm_()`。
+`fsdpturbo` plugin 按参数所属 Mesh 分组计算局部 p 次方和：
+
+- 非专家参数沿 DP 和 CP group 汇总。
+- 专家参数沿 `FSDPTurboParallelState` 保存的 EFSDP、EP 和 expert-CP group 汇总。
+- 汇总得到全局范数后，对所有本地梯度应用同一个 clipping coefficient。
+
+启动阶段会执行一次零梯度 warmup，使相关 collective 在正式训练前完成初始化。
+当前这是 `fsdpturbo` backend 的专用实现；其他 backend 继续保留原有梯度范数路径，等待上游
+distributed plugin 解耦后再统一公共接口。
+
+## 6. 权重加载
+
+LlamaFactory 保留 `init_on_meta` 和 safetensors 加载流程。父类 `FSDP2Engine` 的加载器通过
+`self._copy_weights(...)` 动态调用 FSDPTurbo engine 的覆写实现，因此该方法不是未使用代码。
+它支持包含多个 `Shard` placement 的 DTensor，按各 Mesh 维度依次计算当前 rank 对应的本地切片。
+模型保存和 checkpoint 接口继续复用 LlamaFactory FSDP2 实现。
+
+## 7. Kernel plugin
+
+FLA 算子不属于 distributed config。算子选择通过独立的 `kernel_config` 完成：
+
+```yaml
+kernel_config:
+  name: auto, flash-linear-attention
+  include_kernels: chunk_gated_delta_rule, fused_recurrent_gated_delta_rule
+  chunk_size: 32
+```
+
+调用链如下：
+
+```text
+ModelEngine
+  -> apply_kernels("auto, flash-linear-attention")
+     -> LlamaFactory 当前加速器对应的 auto kernels
+     -> KernelPlugin("flash-linear-attention").apply(...)
+        -> fsdp_turbo.ops.get_op()
+           -> FSDPTurbo device operator registry
+        -> fsdp_turbo.utils.patch.patch_model_members()
+           -> FLA backend implementation
+```
+
+`chunk_size` 当前支持 `16`、`32` 和 `64`，默认值为 `64`。Kernel plugin 与 distributed plugin
+彼此独立。`name: flash-linear-attention` 只安装所选 FLA 算子；逗号分隔的
+`name: auto, flash-linear-attention` 会在分布式切分前组合 LlamaFactory 当前加速器的 auto kernels
+与 FLA plugin。LlamaFactory 负责算子名到模型属性的映射和 `chunk_size` 参数绑定；FSDPTurbo 负责设备
+算子注册、选择和通用 callable patch。FLA 依赖可选的外部三方件，因此保持显式选择，不属于内置
+`auto` 集合。FSDPTurbo
+随后会替换目标专家模块的 `forward`，所以专家计算的最终路径由 `ep_dispatcher` 决定；auto 阶段
+应用的 MoE kernel 不会作为独立的第二条专家执行路径保留下来。
+
+## 8. CP 运行约束与验证范围
+
+`init_on_meta` 构造模型时必须与 `from_pretrained` 路径一样传递 `attn_implementation`，否则模型会退回
+非 FlashAttention 实现，Ulysses CP 无法启动。Ulysses 在调用 Hugging Face FlashAttention 前重建全局
+attention mask；只有二维 position IDs 才参与 packed-sequence 检测。Qwen3.5 mRoPE 等多轴 position IDs
+已经在 rotary embedding 中消费，不应传入 FlashAttention 的 packed-sequence 检测逻辑。
+
+当前实现已在 Atlas 900 A3 SuperPoD 和 Atlas 950 SuperPoD 上用 Qwen3.5-35B-A3B 完成以下
+BF16、AdamW full SFT 验证。本次重验证使用 FSDPTurbo `0e96fbc`；A3 环境为 CANN 9.0.0、
+PyTorch 2.7.1 和 torch-npu 2.7.1.post4，A5 环境为 CANN 9.1.0-beta.3、PyTorch 2.10.0 和
+torch-npu 2.10.0.post2。表中性能按第 1 步至第 100 步的日志时间戳计算，不包含首步前的初始化、
+编译和训练后的模型保存时间：
+
+| 机器型号 | CP | EP | EFSDP | Checkpoint | Kernel / Dispatcher | 步数 | Loss（首步 -> 末步） | 性能 | 结果 |
+| --- | ---: | ---: | ---: | --- | --- | ---: | --- | ---: | --- |
+| Atlas 900 A3 SuperPoD | 1 | 16 | 1 | 关闭 | FLA（chunk size 16）/ eager | 100 | 1.3361 -> 0.0793 | 2.51 s/it | 通过并完成保存 |
+| Atlas 900 A3 SuperPoD | 1 | 16 | 1 | 关闭 | FLA（chunk size 16）/ fused | 100 | 1.3354 -> 0.1179 | 2.17 s/it | 通过并完成保存 |
+| Atlas 900 A3 SuperPoD | 2 | 4 | 2 | 关闭 | auto + FLA（chunk size 64）/ fused | 100 | 1.8114 -> 0.5260 | 7.65 s/it | 通过并完成保存 |
+| Atlas 900 A3 SuperPoD | 2 | 4 | 2 | 关闭 | auto + FLA（chunk size 64）/ eager | 100 | 1.8095 -> 0.5596 | 5.88 s/it | 通过并完成保存 |
+| Atlas 950 SuperPoD | 1 | 8 | 1 | 关闭 | 未配置 kernel plugin / eager | 100 | 1.3575 -> 0.4439 | 2.68 s/it | 通过并完成保存 |
+
+五组训练的 loss 和 grad norm 均保持有限，并完成 100 步及模型保存。同一切分下，EP16 eager/fused
+的逐步 loss 相关系数为 0.997，CP2/EP4/EFSDP2 eager/fused 为 0.977，说明两种 dispatcher 的
+优化轨迹一致。性能收益与切分有关：EP16 下 fused 比 eager 快约 13%，而加入 CP 和 EFSDP 后 fused
+比 eager 慢约 30%，因此不能把 fused 视为所有 mesh 的默认最优选择。
+
+EP16 两组使用 global batch 16 和 cutoff length 256；CP2 两组使用 global batch 8 和 cutoff length
+128；A5 组使用 global batch 8 和 cutoff length 256。因此，首末 loss 用于验证各组自身的收敛趋势，
+不同切分组之间的绝对 loss 不应直接作为精度等价结论。

docs/zh/advanced/ktransformers.md

@@ -0,0 +1,119 @@
+# KTransformers LoRA SFT
+
+KTransformers（KT）将 MoE routed experts 放在 CPU 执行，LLaMA-Factory 继续负责数据、LoRA 参数和训练入口。
+当前生产范围是 routed-BF16 LoRA 与 routed-INT8 LoRA；Accelerate 配置只负责 FSDP2，不再保存 KT 参数。
+
+## 安装检查
+
+必须同时安装带 KT 公共接口的 `ktransformers`、`transformers-kt` 和 `accelerate-kt`。启动前可检查：
+
+```bash
+python - <<'PY'
+from accelerate import Accelerator
+from kt_kernel.sft import resolve_kt_pretrained_artifacts
+from transformers import TrainingArguments
+
+assert hasattr(TrainingArguments, "update_kt_config")
+assert "adapter_only" in __import__("inspect").signature(Accelerator.get_state_dict).parameters
+print(resolve_kt_pretrained_artifacts)
+PY
+```
+
+## 配置
+
+KT 只有一个用户配置源：训练 YAML。LoRA rank、alpha、dropout 和 runtime capacity 由 LLaMA-Factory
+标准字段派生；不要在 `kt_config` 中重复填写。
+
+BF16 示例：
+
+```yaml
+finetuning_type: lora
+lora_rank: 8
+lora_alpha: 16
+lora_target: all
+
+use_kt: true
+disable_gradient_checkpointing: false
+kt_cpu_activation: retain
+kt_config:
+  kt_expert_weight_format: bf16
+  kt_backend: AMXBF16
+  kt_num_threads: 96
+  kt_tp_enabled: true
+  kt_threadpool_count: 2
+  kt_max_cache_depth: 2
+```
+
+INT8 还需要相互匹配的 routed expert 与 BF16 non-expert cache：
+
+```yaml
+kt_weight_path: /abs/path/to/routed-int8-experts
+kt_non_expert_weight_path: /abs/path/to/bf16-non-expert-cache
+kt_config:
+  kt_expert_weight_format: int8
+  kt_backend: auto
+  kt_weight_lifecycle: persistent
+```
+
+完整配置见：
+
+- `examples/ktransformers/train_lora/qwen3_5moe_lora_sft_kt.yaml`
+- `examples/ktransformers/train_lora/deepseek_v3_int8_lora_sft_kt.yaml`
+
+Activation 策略：
+
+| `disable_gradient_checkpointing` | `kt_cpu_activation` | CPU / GPU |
+| --- | --- | --- |
+| `false` | `recompute` 或省略 | recompute / recompute |
+| `false` | `retain` | retain / recompute |
+| `true` | `retain` 或省略 | retain / retain |
+| `true` | `recompute` | 不支持，启动前报错 |
+
+## 启动与复用
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1 accelerate launch \
+  --config_file examples/ktransformers/accelerate/fsdp2_kt_bf16.yaml \
+  src/train.py examples/ktransformers/train_lora/qwen3_5moe_lora_sft_kt.yaml
+```
+
+输出 adapter 同时包含 standard PEFT 与 fused expert LoRA。
+
+## 新进程加载
+
+对话或评测必须使用本地的完整 KT adapter 目录，并重复训练时的 LoRA 形状配置：`finetuning_type`、
+`lora_rank`、`lora_alpha`、`lora_dropout`，以及相同的 KT base weight 配置。routed INT8 尤其要沿用训练时
+的 `kt_weight_path` 和 `kt_non_expert_weight_path`。
+
+```yaml
+model_name_or_path: /abs/path/to/base-model
+adapter_name_or_path: /abs/path/to/output/checkpoint-300
+finetuning_type: lora
+lora_rank: 8
+lora_alpha: 16
+lora_dropout: 0.0
+
+use_kt: true
+kt_cpu_activation: retain
+kt_config:
+  kt_expert_weight_format: bf16
+  kt_backend: AMXBF16
+  kt_num_threads: 96
+```
+
+```bash
+llamafactory-cli chat path/to/kt_adapter_infer.yaml
+llamafactory-cli eval path/to/kt_adapter_eval.yaml
+```
+
+目录必须包含 standard PEFT adapter 文件；使用 fused routed-expert LoRA 时，还必须包含
+`fused_expert_lora.safetensors` 和 `kt_adapter_manifest.json`。LLaMA-Factory 先加载 standard PEFT，随后由
+KT 校验并恢复 fused artifact。`adapter_folder` 可以选择本地子目录；越出 adapter 根目录的路径和 Hub
+adapter ID 会在加载模型前报错，Hub bundle 需要先完整下载到本地。
+
+续训应保留原训练 YAML，并使用 `resume_from_checkpoint`。分布式 optimizer checkpoint 暂要求相同 world
+size。artifact 缺失、hash 不匹配或来源模型不一致时会直接失败，不会退回源 checkpoint。
+
+不要同时启用 Transformers/FSDP activation checkpointing、Unsloth GC，也不要把 `kt_config` 放入
+Accelerate YAML。每次训练都应确认 loss/grad finite、base model 未修改，并验证 standard/router/fused LoRA
+均包含非零更新。

docs/zh/index.rst

@@ -34,9 +34,11 @@ LlamaFactory 文档
 
    advanced/lora-and-quantization/lora
    advanced/lora-and-quantization/quantization
+   advanced/ktransformers
    advanced/distributed/fsdp
    advanced/distributed/deepspeed
    advanced/distributed/parallel-dp-tp-ep-sp-cp
+   advanced/distributed/fsdpturbo-ep-efsdp
    advanced/custom-kernels/triton
    advanced/custom-kernels/fused-operators
 

examples/inference/mossvl.yaml

@@ -0,0 +1,6 @@
+### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt`
+
+model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708
+template: moss_vl
+infer_backend: huggingface  # choices: [huggingface, vllm, sglang, ktransformers]
+trust_remote_code: true

examples/ktransformers/accelerate/fsdp2_kt_bf16.yaml

@@ -13,13 +13,3 @@ num_processes: 4 # Adjust based on your GPU count; 4 is suitable for 4 GPUs
 rdzv_backend: static
 same_network: true
 use_cpu: false
-
-kt_config:
-  enabled: true
-  kt_backend: AMXBF16  # Use with original BF16 expert weights.
-  kt_num_threads: 96
-  kt_tp_enabled: true
-  kt_threadpool_count: 2
-  kt_max_cache_depth: 2
-  kt_share_backward_bb: true
-  lora_rank: 8

examples/ktransformers/accelerate/fsdp2_kt_int4.yaml

@@ -13,13 +13,3 @@ num_processes: 4 # Adjust based on your GPU count; 4 is suitable for 4 GPUs
 rdzv_backend: static
 same_network: true
 use_cpu: false
-
-kt_config:
-  enabled: true
-  kt_backend: AMXINT4  # Use with online-converted INT4 expert weights
-  kt_num_threads: 96
-  kt_tp_enabled: true
-  kt_threadpool_count: 2
-  kt_max_cache_depth: 2
-  kt_share_backward_bb: true
-  lora_rank: 8

examples/ktransformers/accelerate/fsdp2_kt_int8.yaml

@@ -13,13 +13,3 @@ num_processes: 4 # Adjust based on your GPU count; 4 is suitable for 4 GPUs
 rdzv_backend: static
 same_network: true
 use_cpu: false
-
-kt_config:
-  enabled: true
-  kt_backend: AMXINT8  # Use with online-converted INT8 expert weights
-  kt_num_threads: 96
-  kt_tp_enabled: true
-  kt_threadpool_count: 2
-  kt_max_cache_depth: 2
-  kt_share_backward_bb: true
-  lora_rank: 8

examples/ktransformers/accelerate/fsdp2_kt_int8_1gpu.yaml

@@ -13,13 +13,3 @@ num_processes: 1 # Adjust based on your GPU count; 1 is suitable for 1 GPU
 rdzv_backend: static
 same_network: true
 use_cpu: false
-
-kt_config:
-  enabled: true
-  kt_backend: AMXINT8  # Use with online-converted INT8 expert weights
-  kt_num_threads: 96
-  kt_tp_enabled: true
-  kt_threadpool_count: 2
-  kt_max_cache_depth: 2
-  kt_share_backward_bb: true
-  lora_rank: 8

examples/ktransformers/accelerate/fsdp2_kt_int8_8gpu.yaml

@@ -13,13 +13,3 @@ num_processes: 8 # Adjust based on your GPU count; 8 is suitable for 8 GPUs
 rdzv_backend: static
 same_network: true
 use_cpu: false
-
-kt_config:
-  enabled: true
-  kt_backend: AMXINT8  # Use with online-converted INT8 expert weights
-  kt_num_threads: 96
-  kt_tp_enabled: true
-  kt_threadpool_count: 2
-  kt_max_cache_depth: 2
-  kt_share_backward_bb: true
-  lora_rank: 8

examples/ktransformers/train_lora/deepseek_v3_int8_lora_sft_kt.yaml

@@ -0,0 +1,54 @@
+### model
+model_name_or_path: /path/to/DeepSeek-V3.1-source
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_rank: 8
+lora_alpha: 16
+lora_target: all
+
+### dataset
+dataset: identity, alpaca_en_demo
+template: deepseek3
+cutoff_len: 2048
+max_samples: 100000
+overwrite_cache: true
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+
+### output
+output_dir: saves/KT_FT_deepseekV3_int8
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+learning_rate: 1.0e-4
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+
+### ktransformers
+use_kt: true
+kt_cpu_activation: retain
+kt_weight_path: /path/to/routed-int8-experts
+kt_non_expert_weight_path: /path/to/bf16-non-expert-cache
+kt_config:
+  kt_expert_weight_format: int8
+  kt_backend: auto
+  kt_weight_lifecycle: persistent
+  kt_num_threads: 96
+  kt_tp_enabled: true
+  kt_threadpool_count: 2
+  kt_max_cache_depth: 2
+  kt_share_backward_bb: true

examples/ktransformers/train_lora/deepseek_v3_lora_sft_kt.yaml

@@ -40,6 +40,13 @@ resume_from_checkpoint: null
 
 ### ktransformers
 use_kt: true
-# Pair with fsdp2_kt_bf16.yaml for original BF16 checkpoints.
+kt_cpu_activation: retain
-# For pre-converted expert weights, uncomment kt_weight_path and use fsdp2_kt_int8.yaml or fsdp2_kt_int4.yaml.
+kt_config:
-# kt_weight_path: /path/to/DeepSeek-V3-AMXINT8
+  kt_expert_weight_format: bf16
+  kt_backend: AMXBF16
+  kt_num_threads: 96
+  kt_tp_enabled: true
+  kt_threadpool_count: 2
+  kt_max_cache_depth: 2
+  kt_share_backward_bb: true
+# The Accelerate YAML contains FSDP settings only. KT has a single configuration owner here.

examples/ktransformers/train_lora/qwen3_5moe_lora_sft_kt.yaml

@@ -40,7 +40,14 @@ resume_from_checkpoint: null
 
 ### ktransformers
 use_kt: true
-# For original BF16 checkpoints, start with examples/ktransformers/accelerate/fsdp2_kt_bf16.yaml.
+kt_cpu_activation: retain
-# For pre-converted expert weights, uncomment kt_weight_path and use fsdp2_kt_int8.yaml or fsdp2_kt_int4.yaml.
+kt_config:
-# Pair the 397B path with fsdp2_kt_int8.yaml, tune cutoff_len to prepared weights and GPU memory.
+  kt_expert_weight_format: bf16
-# kt_weight_path: /path/to/Qwen3.5-MoE-AMXINT8
+  kt_backend: AMXBF16
+  kt_num_threads: 96
+  kt_tp_enabled: true
+  kt_threadpool_count: 2
+  kt_max_cache_depth: 2
+  kt_model_max_length: 2176  # Includes the text-only template's dummy-image tokens.
+  kt_share_backward_bb: true
+# The Accelerate YAML contains FSDP settings only. KT has a single configuration owner here.

examples/megatron_bridge/llama3_sft.yaml

@@ -0,0 +1,67 @@
+### model
+model_name_or_path: meta-llama/Llama-3.2-1B-Instruct
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora  # full or lora
+dataset: alpaca_en_demo
+template: llama3
+cutoff_len: 2048
+preprocessing_num_workers: 8
+# disable_shuffling: true  # keep sample order aligned with HF baseline
+
+### output
+output_dir: saves/mbridge/llama3_sft
+logging_steps: 1
+overwrite_output_dir: true
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+num_train_epochs: 3
+max_steps: 1000  # when set, overrides num_train_epochs for Megatron Bridge schedule
+save_steps: 3000
+learning_rate: 5.0e-6
+lr_scheduler_type: cosine
+warmup_steps: 10
+adam_beta1: 0.9
+adam_beta2: 0.999
+weight_decay: 0.0
+max_grad_norm: 1.0
+bf16: true
+
+### megatron bridge parallelism
+tensor_model_parallel_size: 1
+pipeline_model_parallel_size: 1
+context_parallel_size: 1
+expert_model_parallel_size: 1
+# virtual_pipeline_model_parallel_size: 2
+sequence_parallel: false
+
+### megatron bridge optimizer / overlap
+use_distributed_optimizer: true
+overlap_param_gather: true
+overlap_grad_reduce: true
+mixed_precision: bf16_mixed
+
+### megatron bridge activation recompute (optional)
+# recompute_granularity: full
+# recompute_method: uniform
+# recompute_num_layers: 1
+
+### megatron bridge model kernels (optional; None keeps provider defaults)
+# bias_activation_fusion: true
+# apply_rope_fusion: true
+# masked_softmax_fusion: true
+# cross_entropy_loss_fusion: true
+
+### megatron bridge MoE (optional)
+# moe_grouped_gemm: true
+# moe_token_dispatcher_type: alltoall
+
+### megatron bridge data / checkpoint
+use_packed_sequences: false
+# megatron_pretrained_checkpoint: /path/to/megatron_ckpt
+export_hf_on_finish: false  # disable for short loss-comparison runs (avoids checkpoint OOM)
+# extra_config: '{"train.train_iters": 5, "logger.log_interval": 1}'

examples/merge_lora/mossvl_lora_sft.yaml

@@ -0,0 +1,14 @@
+### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt`
+### Note: DO NOT use quantized model or quantization_bit when merging lora adapters
+
+### model
+model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708
+adapter_name_or_path: saves/moss-vl-11b/lora/sft
+template: moss_vl
+trust_remote_code: true
+
+### export
+export_dir: saves/moss_vl_sft_merged
+export_size: 5
+export_device: cpu  # choices: [cpu, auto]
+export_legacy_format: false

examples/train_full/mossvl_full_sft.yaml

@@ -0,0 +1,57 @@
+### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt`
+
+### model
+model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708
+image_max_pixels: 262144
+video_max_pixels: 16384
+video_fps: 1.0
+video_maxlen: 256
+use_reentrant_gc: false
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: full
+freeze_vision_tower: true
+freeze_multi_modal_projector: true
+freeze_language_model: false
+deepspeed: examples/deepspeed/ds_z3_config.json
+
+### dataset
+dataset: mllm_demo,identity,alpaca_en_demo  # video: mllm_video_demo
+template: moss_vl
+cutoff_len: 4096
+max_samples: 1000
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+packing: false
+
+### output
+output_dir: saves/moss-vl-11b/full/sft
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+  use_reentrant: false
+learning_rate: 1.0e-5
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+resume_from_checkpoint: null
+
+### eval
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/train_lora/mossvl_lora_sft.yaml

@@ -0,0 +1,54 @@
+### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt`
+
+### model
+model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708
+image_max_pixels: 262144
+video_max_pixels: 16384
+video_fps: 1.0
+video_maxlen: 256
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_rank: 8
+lora_target: all
+freeze_vision_tower: true
+freeze_multi_modal_projector: true
+freeze_language_model: false
+
+### dataset
+dataset: mllm_demo,identity,alpaca_en_demo  # video: mllm_video_demo
+template: moss_vl
+cutoff_len: 4096
+max_samples: 1000
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+packing: false
+
+### output
+output_dir: saves/moss-vl-11b/lora/sft
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 2
+gradient_accumulation_steps: 1
+learning_rate: 1.0e-4
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+resume_from_checkpoint: null
+
+### eval
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

examples/v1/train_batching_strategy/train_full_fsdp2_batching_normal.yaml

@@ -1,12 +1,8 @@
 model: Qwen/Qwen3-0.6B
 model_class: llm
 
-template: qwen3_nothink
-
-
 kernel_config:
   name: auto
-  include_kernels: auto # choice: null/true/false/auto/kernel_id1,kernel_id2,kernel_id3, default is null
 
 quant_config: null
 

examples/v1/train_batching_strategy/train_full_fsdp2_dynamic_batching.yaml

@@ -1,11 +1,8 @@
 model: Qwen/Qwen3-0.6B
 model_class: llm
 
-template: qwen3_nothink
-
 kernel_config:
   name: auto
-  include_kernels: auto # choice: null/true/false/auto/kernel_id1,kernel_id2,kernel_id3, default is null
 
 quant_config: null
 

examples/v1/train_batching_strategy/train_full_fsdp2_dynamic_padding_free.yaml

@@ -1,11 +1,8 @@
 model: Qwen/Qwen3-0.6B
 model_class: llm
 
-template: qwen3_nothink
-
 kernel_config:
   name: auto
-  include_kernels: auto # choice: null/true/false/auto/kernel_id1,kernel_id2,kernel_id3, default is null
 
 quant_config: null
 

examples/v1/train_batching_strategy/train_full_fsdp2_padding_free.yaml

@@ -1,11 +1,8 @@
 model: Qwen/Qwen3-0.6B
 model_class: llm
 
-template: qwen3_nothink
-
 kernel_config:
   name: auto
-  include_kernels: auto # choice: null/true/false/auto/kernel_id1,kernel_id2,kernel_id3, default is null
 
 quant_config: null
 

examples/v1/train_freeze/train_freeze_sft.yaml

@@ -1,7 +1,6 @@
 model: Qwen/Qwen3-4B
 model_class: llm
 
-
 # Freeze Configuration
 peft_config:
   name: freeze
@@ -12,7 +11,6 @@ peft_config:
 # Kernel Config
 kernel_config:
   name: auto
-  include_kernels: auto
 
 # FSDP Config
 dist_config:
@@ -25,7 +23,6 @@ train_dataset: data/v1_sft_demo.yaml
 ### training
 output_dir: ./outputs/test_freeze
 micro_batch_size: 1
-global_batch_size: 4
 cutoff_len: 2048
 learning_rate: 2.0e-5
 max_steps: 10

examples/v1/train_full/train_full_deepspeed.yaml

@@ -1,10 +1,8 @@
 model: Qwen/Qwen3-0.6B
 model_class: llm
 
-
 kernel_config:
     name: auto
-    include_kernels: auto
 
 dist_config:
     name: deepspeed

examples/v1/train_full/train_full_fsdp2.yaml

@@ -1,10 +1,8 @@
 model: Qwen/Qwen3-0.6B
 model_class: llm
 
-
 kernel_config:
   name: auto
-  include_kernels: auto # choice: null/true/false/auto/kernel_id1,kernel_id2,kernel_id3, default is null
 
 quant_config: null
 

examples/v1/train_full/train_full_liger_kernel.yaml

@@ -1,11 +1,8 @@
 model: Qwen/Qwen3-0.6B
 model_class: llm
 
-template: qwen3_nothink
-
 kernel_config:
   name: liger_kernel
-  include_kernels: auto # choice: null/true/false/auto/kernel_id1,kernel_id2,kernel_id3, default is null
 
 quant_config: null
 

examples/v1/train_full/train_full_qwen3_moe_fsdpturbo_ep_fsdp.yaml

@@ -0,0 +1,32 @@
+model: Qwen/Qwen3.5-35B-A3B
+model_class: llm
+
+kernel_config:
+  name: auto, flash-linear-attention
+  include_kernels: chunk_gated_delta_rule, fused_recurrent_gated_delta_rule
+  chunk_size: 64
+
+dist_config:
+  name: fsdpturbo
+  ep_size: 16
+  ep_dispatcher: eager
+
+cp_size: 1
+
+init_config:
+  name: init_on_meta
+
+### data
+train_dataset: data/v1_sft_demo.yaml
+
+### training
+output_dir: outputs/Qwen3.5-35B-A3B/full/sft
+micro_batch_size: 1
+cutoff_len: 256
+learning_rate: 1.0e-4
+bf16: true
+max_steps: 5
+
+### sample
+sample_backend: hf
+max_new_tokens: 128

examples/v1/train_full/train_full_ulysses_cp.yaml

@@ -8,8 +8,9 @@ flash_attn: flash_attention_2
 dist_config:
   name: fsdp2
   dcp_path: null
-  cp_mode: ulysses
+
-  cp_size: 2
+cp_mode: ulysses
+cp_size: 2
 
 ### data
 train_dataset: data/v1_sft_demo.yaml

examples/v1/train_full/train_multimodal.yaml

@@ -0,0 +1,27 @@
+model: Qwen/Qwen3.5-0.8B
+model_class: llm
+
+
+kernel_config:
+  name: auto
+
+quant_config: null
+
+dist_config:
+  name: fsdp2
+  dcp_path: null
+
+### data
+train_dataset: data/v1_multimodal_demo.yaml
+
+### training
+output_dir: outputs/test_multimodal
+micro_batch_size: 1
+cutoff_len: 2048
+learning_rate: 1.0e-4
+max_steps: 5
+
+### sample
+sample_backend: hf
+max_new_tokens: 128
+

examples/v1/train_lora/train_lora_dpo.yaml

@@ -1,8 +1,6 @@
 model: Qwen/Qwen3-4B
 model_class: llm
 
-template: qwen3_nothink
-
 # PEFT Configuration
 peft_config:
   name: lora
@@ -14,7 +12,6 @@ peft_config:
 # Kernel Config
 kernel_config:
   name: auto
-  include_kernels: auto
 
 # FSDP Config
 dist_config:

examples/v1/train_lora/train_lora_sft.yaml

@@ -1,7 +1,6 @@
 model: Qwen/Qwen3-4B
 model_class: llm
 
-
 # PEFT Configuration
 peft_config:
   name: lora
@@ -13,7 +12,6 @@ peft_config:
 # Kernel Config
 kernel_config:
   name: auto
-  include_kernels: auto
 
 # FSDP Config
 dist_config:

examples/v1/train_lora/train_lora_sft_rank0.yaml

@@ -1,7 +1,6 @@
 model: Qwen/Qwen3-4B
 model_class: llm
 
-
 # PEFT Configuration
 peft_config:
   name: lora
@@ -13,7 +12,6 @@ peft_config:
 # Kernel Config
 kernel_config:
   name: auto
-  include_kernels: auto
 
 # FSDP Config
 dist_config:

examples/v1/train_qlora/quantization.yaml

@@ -1,7 +1,6 @@
 model: Qwen/Qwen3-0.6B
 model_class: llm
 
-
 # PEFT Configuration
 peft_config:
   name: lora
@@ -13,7 +12,6 @@ peft_config:
 # Kernel Config
 kernel_config:
   name: auto
-  include_kernels: auto
 
 # FSDP Config
 dist_config:

pyproject.toml

@@ -112,7 +112,7 @@ ignore = [
     "D105",  # no doc magic method
     "D107",  # no doc __init__
 ]
-extend-select = [
+select = [
     "C",      # complexity
     "E",      # error
     "F",      # pyflakes

requirements/fsdpturbo.txt

@@ -0,0 +1,2 @@
+# Pin the FSDPTurbo API to a reproducible upstream main revision.
+fsdp-turbo @ git+https://gitcode.com/Ascend/FSDPTurbo.git@d878dffdb1e0312dc098599f2b56810d6b592ee2

requirements/moss-vl.txt

@@ -0,0 +1,3 @@
+transformers==4.57.1
+torchcodec==0.7.0
+joblib

requirements/npu.txt

@@ -1,5 +1,5 @@
-torch==2.7.1
+torch==2.10.0
-torch-npu==2.7.1.post4
+torch-npu==2.10.0.post2
-torchvision==0.22.1
+torchvision==0.25.0
-torchaudio==2.7.1
+torchaudio==2.10.0
 decorator

src/llamafactory/data/collator.py

@@ -150,7 +150,9 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
         if isinstance(self.model, PeftModel):
             self.model = self.model.base_model.model
 
-        if self.model is not None and hasattr(self.model, "get_rope_index"):  # for qwen2vl mrope
+        if getattr(getattr(self.model, "config", None), "model_type", None) == "moss_vl":
+            self.get_rope_func = None  # MOSS-VL computes its own XRoPE positions in model.forward.
+        elif self.model is not None and hasattr(self.model, "get_rope_index"):  # for qwen2vl mrope
             self.get_rope_func = self.model.get_rope_index  # transformers < 4.52.0 or qwen2.5 omni
         elif self.model is not None and hasattr(self.model, "model") and hasattr(self.model.model, "get_rope_index"):
             self.get_rope_func = self.model.model.get_rope_index  # transformers >= 4.52.0
@@ -322,6 +324,8 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
             )
 
     def __call__(self, features: list[dict[str, Any]]) -> dict[str, "torch.Tensor"]:
+        model_type = getattr(getattr(self.model, "config", None), "model_type", None)
+        is_moss_vl = model_type == "moss_vl"
         batch_images, batch_videos, batch_audios = [], [], []
         batch_imglens, batch_vidlens, batch_audlens, batch_input_ids = [], [], [], []
         packing_params_list: list[dict[str, Any] | None] = []
@@ -341,7 +345,10 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
         fake_input_ids = []
         has_dummy_image = False
         if (
-            self.template.mm_plugin.image_token is not None and sum(batch_imglens) == 0 and sum(batch_vidlens) == 0
+            self.template.mm_plugin.image_token is not None
+            and sum(batch_imglens) == 0
+            and sum(batch_vidlens) == 0
+            and not is_moss_vl  # MOSS-VL builds one native zero-valued dummy per text-only sample in its plugin.
         ):  # avoid process hanging in zero3/fsdp case
             fake_messages = [{"role": "user", "content": IMAGE_PLACEHOLDER}]
             fake_images = [Image.new("RGB", (64, 64), (255, 255, 255))]
@@ -416,7 +423,6 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
         features: dict[str, torch.Tensor] = super().__call__(features)
 
         bsz, seq_len = features["input_ids"].shape[:2]
-        model_type = getattr(self.model.config, "model_type", None) if self.model is not None else None
         is_omni = model_type in [
             "qwen2_5_omni_thinker",
             "qwen3_omni_moe_thinker",
@@ -461,12 +467,17 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
         ):
             raise ValueError(f"{self.model.config.model_type} requires 3D position ids for mrope.")
 
-        if "cross_attention_mask" in mm_inputs:  # for mllama inputs when pad_to_multiple_of is enabled
+        if (
+            "cross_attention_mask" in mm_inputs and mm_inputs["cross_attention_mask"].dtype != torch.bool
+        ):  # for mllama inputs when pad_to_multiple_of is enabled
             cross_attention_mask = mm_inputs.pop("cross_attention_mask")
             seq_len = features["input_ids"].size(1)
             orig_len = cross_attention_mask.size(1)
             mm_inputs["cross_attention_mask"] = F.pad(cross_attention_mask, (0, 0, 0, 0, 0, seq_len - orig_len))
 
+        if is_moss_vl:
+            mm_inputs = self.template.mm_plugin.post_process_mossvl_inputs(features, mm_inputs, self.processor)
+
         features.update(mm_inputs)
 
         if "image_bound" in features:  # for minicpmv inputs
@@ -530,6 +541,17 @@ class SFTDataCollatorWith4DAttentionMask(MultiModalDataCollatorForSeq2Seq):
                 self._unpad_packed_features(features)
 
             features["attention_mask"] = None  # let transformers handle causal packed mask.
+        else:
+            # `DataCollatorForSeq2Seq(pad_to_multiple_of=...)` pads `input_ids`/`attention_mask`
+            # but leaves `position_ids` untouched (it is not in `model_input_names`). On the
+            # non-FA2 packing path we do not unpad, so `position_ids` stays shorter than
+            # `input_ids`, which makes cos/sin shorter than query and crashes
+            # `apply_rotary_pos_emb`. Right-pad `position_ids` to the padded length to match.
+            position_ids = features.get("position_ids")
+            if torch.is_tensor(position_ids):
+                pad_len = features["input_ids"].shape[-1] - position_ids.shape[-1]
+                if pad_len > 0:
+                    features["position_ids"] = F.pad(position_ids, (0, pad_len), value=0)
 
         for key, value in features.items():  # cast data dtype for paligemma
             if torch.is_tensor(value) and torch.is_floating_point(value):

src/llamafactory/data/mm_plugin.py

@@ -472,6 +472,354 @@ class BasePlugin(MMPluginMixin):
         return self._get_mm_inputs(images, videos, audios, processor)
 
 
+@dataclass
+class MossVLPlugin(BasePlugin):
+    vision_bos_token: str = "<|vision_start|>"
+    vision_eos_token: str = "<|vision_end|>"
+    time_bos_token: str = "<|time_start|>"
+    time_eos_token: str = "<|time_end|>"
+
+    @staticmethod
+    def _split_pixel_values(
+        pixel_values: "torch.Tensor",
+        grid_thw: "torch.Tensor",
+    ) -> list["torch.Tensor"]:
+        patch_counts = [int(grid.prod().item()) for grid in grid_thw]
+        return list(torch.split(pixel_values, patch_counts))
+
+    @staticmethod
+    def _create_cross_attention_mask(
+        input_ids: Union[list[list[int]], "torch.Tensor"],
+        grid_thw: "torch.Tensor",
+        media_nums_per_sample: list[int],
+        image_token_id: int,
+        attention_mask: Optional["torch.Tensor"] = None,
+        padding_side: Literal["left", "right"] = "right",
+    ) -> "torch.Tensor":
+        r"""Create the native MOSS-VL frame-level causal cross-attention mask."""
+        if isinstance(input_ids, list):
+            max_text_len = max(len(token_ids) for token_ids in input_ids)
+            input_ids_tensor = torch.full((len(input_ids), max_text_len), -1, dtype=torch.long)
+            attention_mask_tensor = torch.zeros_like(input_ids_tensor, dtype=torch.bool)
+            for batch_index, token_ids in enumerate(input_ids):
+                seq_len = len(token_ids)
+                start = max_text_len - seq_len if padding_side == "left" else 0
+                input_ids_tensor[batch_index, start : start + seq_len] = torch.tensor(token_ids, dtype=torch.long)
+                attention_mask_tensor[batch_index, start : start + seq_len] = True
+        else:
+            input_ids_tensor = input_ids
+            attention_mask_tensor = (
+                torch.ones_like(input_ids_tensor, dtype=torch.bool)
+                if attention_mask is None
+                else attention_mask.bool()
+            )
+
+        total_frames_per_sample = []
+        media_index = 0
+        for num_media in media_nums_per_sample:
+            sample_grid = grid_thw[media_index : media_index + num_media]
+            total_frames_per_sample.append(int(sample_grid[:, 0].sum().item()))
+            media_index += num_media
+
+        max_num_frames = max(total_frames_per_sample)
+        frame_indices = torch.arange(max_num_frames, device=input_ids_tensor.device).view(1, 1, -1)
+        visible_mask = (input_ids_tensor == image_token_id).cumsum(dim=1).unsqueeze(-1) > frame_indices
+        visible_mask &= attention_mask_tensor.unsqueeze(-1)
+        valid_frames = frame_indices < torch.tensor(
+            total_frames_per_sample,
+            device=input_ids_tensor.device,
+        ).view(-1, 1, 1)
+        visible_mask &= valid_frames
+        return (~visible_mask).unsqueeze(1)
+
+    def _get_video_inputs(
+        self,
+        videos: list["VideoInput"],
+        processor: "MMProcessor",
+        return_metadata: bool,
+    ) -> dict[str, Any]:
+        video_kwargs = {"return_tensors": "pt", "return_metadata": return_metadata}
+        if getattr(processor, "video_fps", None) is not None:
+            video_kwargs["video_fps"] = processor.video_fps
+        if getattr(processor, "video_maxlen", None) is not None:
+            video_kwargs["max_frames"] = processor.video_maxlen
+
+        video_min_pixels = getattr(processor, "video_min_pixels", None)
+        video_max_pixels = getattr(processor, "video_max_pixels", None)
+        if video_min_pixels is not None and video_max_pixels is not None:
+            video_kwargs["size"] = {
+                "shortest_edge": video_min_pixels,
+                "longest_edge": video_max_pixels,
+            }
+
+        return dict(processor.video_processor(videos=videos, **video_kwargs))
+
+    def _get_media_order_from_ids(
+        self,
+        input_ids: list[int],
+        processor: "MMProcessor",
+        num_images: int,
+        num_videos: int,
+        expected_video_frames: Optional[list[int]] = None,
+    ) -> list[str]:
+        media_order = []
+        video_frame_counts = []
+        in_video = False
+        current_video_frames = 0
+        for token_id in input_ids:
+            if token_id == processor.vision_start_token_id:
+                if in_video:
+                    raise ValueError(
+                        "MOSS-VL encountered nested video token blocks after tokenization. "
+                        "Please increase `cutoff_len` if a video placeholder was truncated."
+                    )
+
+                media_order.append("video")
+                in_video = True
+                current_video_frames = 0
+            elif token_id == processor.vision_end_token_id:
+                if not in_video:
+                    raise ValueError(
+                        "MOSS-VL encountered a video end token without a matching start token after tokenization. "
+                        "Please increase `cutoff_len` if a video placeholder was truncated."
+                    )
+
+                video_frame_counts.append(current_video_frames)
+                in_video = False
+            elif token_id == processor.image_token_id:
+                if in_video:
+                    current_video_frames += 1
+                else:
+                    media_order.append("image")
+
+        if in_video:
+            raise ValueError(
+                "MOSS-VL encountered an incomplete video token block after tokenization. "
+                "Please increase `cutoff_len` or reduce `video_maxlen`."
+            )
+
+        if media_order.count("image") != num_images or media_order.count("video") != num_videos:
+            raise ValueError(
+                "MOSS-VL media tokens do not match the provided media after tokenization: "
+                f"order={media_order}, images={num_images}, videos={num_videos}. "
+                "Please increase `cutoff_len` if a visual placeholder was truncated."
+            )
+
+        if expected_video_frames is not None and video_frame_counts != expected_video_frames:
+            raise ValueError(
+                "MOSS-VL video frame tokens do not match the processed video after tokenization: "
+                f"tokens={video_frame_counts}, frames={expected_video_frames}. "
+                "Please increase `cutoff_len` or reduce `video_maxlen`."
+            )
+
+        return media_order
+
+    @override
+    def process_messages(
+        self,
+        messages: list[dict[str, str]],
+        images: list["ImageInput"],
+        videos: list["VideoInput"],
+        audios: list["AudioInput"],
+        processor: Optional["MMProcessor"],
+    ) -> list[dict[str, str]]:
+        self._validate_input(processor, images, videos, audios)
+        self._validate_messages(messages, images, videos, audios)
+        messages = deepcopy(messages)
+
+        video_inputs = self._get_video_inputs(videos, processor, return_metadata=True) if videos else {}
+        video_grid_thw = video_inputs.get("video_grid_thw", [])
+        video_metadata = video_inputs.get("video_metadata", [])
+        video_index = 0
+        for message in messages:
+            content = message["content"]
+            content = content.replace(IMAGE_PLACEHOLDER, self.image_token)
+            while VIDEO_PLACEHOLDER in content:
+                metadata = video_metadata[video_index]
+                if metadata.fps is None:
+                    metadata.fps = 24
+
+                timestamps = processor._calculate_timestamps(
+                    metadata.frames_indices,
+                    metadata.total_num_frames,
+                    metadata.fps,
+                    metadata.duration,
+                    processor.video_processor.temporal_patch_size,
+                    actual_timestamps=getattr(metadata, "actual_timestamps", None),
+                )
+                num_frames = int(video_grid_thw[video_index][0].item())
+                frame_tokens = [
+                    f"{self.time_bos_token}{timestamps[frame_idx]:.1f} seconds{self.time_eos_token}{self.image_token}"
+                    for frame_idx in range(num_frames)
+                ]
+                video_tokens = f"{self.vision_bos_token}{''.join(frame_tokens)}{self.vision_eos_token}"
+                content = content.replace(VIDEO_PLACEHOLDER, video_tokens, 1)
+                video_index += 1
+
+            message["content"] = content
+
+        return messages
+
+    @override
+    def get_mm_inputs(
+        self,
+        images: list["ImageInput"],
+        videos: list["VideoInput"],
+        audios: list["AudioInput"],
+        imglens: list[int],
+        vidlens: list[int],
+        audlens: list[int],
+        batch_ids: list[list[int]],
+        processor: Optional["MMProcessor"],
+    ) -> dict[str, Union[list[int], "torch.Tensor"]]:
+        self._validate_input(processor, images, videos, audios)
+        if audios:
+            raise ValueError("MOSS-VL does not support audio inputs.")
+
+        if not (len(imglens) == len(vidlens) == len(batch_ids)):
+            raise ValueError("MOSS-VL batch metadata must have one entry per sample.")
+        final_pixel_values = []
+        final_grid_thw = []
+        media_nums_per_sample = []
+        image_offset = 0
+        video_offset = 0
+        for imglen, vidlen, input_ids in zip(imglens, vidlens, batch_ids):
+            sample_images = images[image_offset : image_offset + imglen]
+            sample_videos = videos[video_offset : video_offset + vidlen]
+            image_offset += imglen
+            video_offset += vidlen
+            image_chunks, image_grids = [], []
+            if sample_images:
+                regularized_images = self._regularize_images(
+                    sample_images,
+                    image_max_pixels=2**63 - 1,
+                    image_min_pixels=1,
+                )["images"]
+                image_kwargs = {"return_tensors": "pt"}
+                if getattr(processor, "image_min_pixels", None) is not None:
+                    image_kwargs["min_pixels"] = processor.image_min_pixels
+                if getattr(processor, "image_max_pixels", None) is not None:
+                    image_kwargs["max_pixels"] = processor.image_max_pixels
+
+                image_inputs = processor.image_processor(images=regularized_images, **image_kwargs)
+                image_grids = list(image_inputs["image_grid_thw"])
+                image_chunks = self._split_pixel_values(image_inputs["pixel_values"], image_inputs["image_grid_thw"])
+
+            video_chunks, video_grids = [], []
+            if sample_videos:
+                video_inputs = self._get_video_inputs(sample_videos, processor, return_metadata=False)
+                video_grids = list(video_inputs["video_grid_thw"])
+                video_chunks = self._split_pixel_values(
+                    video_inputs["pixel_values_videos"],
+                    video_inputs["video_grid_thw"],
+                )
+
+            media_order = self._get_media_order_from_ids(
+                input_ids,
+                processor,
+                imglen,
+                vidlen,
+                expected_video_frames=[int(grid[0].item()) for grid in video_grids],
+            )
+
+            if not media_order:
+                patch_size = getattr(processor.image_processor, "patch_size", None)
+                if patch_size is None:  # lightweight/test processors without the native MOSS-VL contract
+                    blank_image = Image.new("RGB", (128, 128), (255, 255, 255))
+                    blank_inputs = processor.image_processor(images=[blank_image], return_tensors="pt")
+                    final_pixel_values.append(blank_inputs["pixel_values"])
+                    final_grid_thw.append(blank_inputs["image_grid_thw"][0])
+                else:
+                    temporal_patch_size = getattr(processor.image_processor, "temporal_patch_size", None) or 1
+                    merge_size = getattr(processor.image_processor, "merge_size", None) or 2
+                    factor = patch_size * merge_size
+                    side = math.ceil(128 / factor) * factor
+                    grid_thw = torch.tensor([1, side // patch_size, side // patch_size])
+                    feature_dim = 3 * temporal_patch_size * patch_size * patch_size
+                    final_pixel_values.append(torch.zeros((int(grid_thw.prod()), feature_dim), dtype=torch.float32))
+                    final_grid_thw.append(grid_thw)
+                media_nums_per_sample.append(1)
+                continue
+
+            image_index = 0
+            video_index = 0
+            for modality in media_order:
+                if modality == "image":
+                    final_pixel_values.append(image_chunks[image_index])
+                    final_grid_thw.append(image_grids[image_index])
+                    image_index += 1
+                else:
+                    final_pixel_values.append(video_chunks[video_index])
+                    final_grid_thw.append(video_grids[video_index])
+                    video_index += 1
+
+            media_nums_per_sample.append(len(media_order))
+
+        if image_offset != len(images) or video_offset != len(videos):
+            raise ValueError("MOSS-VL media lengths do not consume all provided inputs.")
+
+        mm_inputs = {
+            "pixel_values": torch.cat(final_pixel_values, dim=0),
+            "grid_thw": torch.stack(final_grid_thw),
+            "media_nums_per_sample": media_nums_per_sample,
+        }
+        mm_inputs["cross_attention_mask"] = self._create_cross_attention_mask(
+            batch_ids,
+            mm_inputs["grid_thw"],
+            media_nums_per_sample,
+            processor.image_token_id,
+            padding_side=processor.tokenizer.padding_side,
+        )
+        return mm_inputs
+
+    def post_process_mossvl_inputs(
+        self,
+        features: dict[str, "torch.Tensor"],
+        mm_inputs: dict[str, Any],
+        processor: "MMProcessor",
+    ) -> dict[str, Any]:
+        r"""Create MOSS-VL batch-only inputs after the text batch has been padded."""
+        input_ids = features["input_ids"]
+        attention_mask = features["attention_mask"].bool()
+        mm_inputs["cross_attention_mask"] = self._create_cross_attention_mask(
+            input_ids,
+            mm_inputs["grid_thw"],
+            mm_inputs["media_nums_per_sample"],
+            processor.image_token_id,
+            attention_mask,
+        )
+
+        dummy_image_tokens = (input_ids == processor.image_token_id) & ~attention_mask
+        input_ids.masked_fill_(dummy_image_tokens, processor.tokenizer.pad_token_id)
+
+        labels = features.get("labels")
+        if labels is not None:
+            control_token_ids = {
+                processor.image_token_id,
+                processor.video_token_id,
+                processor.vision_start_token_id,
+                processor.vision_end_token_id,
+                processor.tokenizer.convert_tokens_to_ids(self.time_bos_token),
+                processor.tokenizer.convert_tokens_to_ids(self.time_eos_token),
+            }
+            for batch_index, token_ids in enumerate(input_ids):
+                in_vision = False
+                for token_index, token_id in enumerate(token_ids.tolist()):
+                    if token_id == processor.vision_start_token_id:
+                        in_vision = True
+                    if in_vision or token_id in control_token_ids:
+                        labels[batch_index, token_index] = IGNORE_INDEX
+                    if token_id == processor.vision_end_token_id:
+                        in_vision = False
+
+            # Native MOSS-VL labels_spans supervise through <|im_end|>, but not its trailing newline.
+            im_end_token_id = processor.tokenizer.convert_tokens_to_ids("<|im_end|>")
+            labels[:, 1:].masked_fill_(input_ids[:, :-1] == im_end_token_id, IGNORE_INDEX)
+
+        features.pop("position_ids", None)
+        return mm_inputs
+
+
 @dataclass
 class ErnieVLPlugin(BasePlugin):
     @override
@@ -2911,6 +3259,7 @@ PLUGINS = {
     "minicpm_v": MiniCPMVPlugin,
     "minicpm_v_4_6": MiniCPMV4_6Plugin,
     "mllama": MllamaPlugin,
+    "moss_vl": MossVLPlugin,
     "paligemma": PaliGemmaPlugin,
     "pixtral": PixtralPlugin,
     "qwen2_audio": Qwen2AudioPlugin,

src/llamafactory/data/template.py

@@ -333,6 +333,50 @@ class Template:
         return modelfile
 
 
+@dataclass
+class MossVLTemplate(Template):
+    @override
+    def _encode(
+        self,
+        tokenizer: "PreTrainedTokenizer",
+        messages: list[dict[str, str]],
+        system: Optional[str],
+        tools: Optional[str],
+    ) -> list[list[int]]:
+        system = system or self.default_system
+        encoded_messages = []
+        for i, message in enumerate(messages):
+            elements = []
+
+            if i == 0:
+                elements += self.format_prefix.apply()
+                if system or tools:
+                    tool_text = self.format_tools.apply(content=tools)[0] if tools else ""
+                    if tools and not system:
+                        tool_text = tool_text.lstrip("\n")
+
+                    elements += self.format_system.apply(content=(system + tool_text))
+
+            if message["role"] == Role.USER:
+                elements += self.format_user.apply(content=message["content"], idx=str(i // 2))
+            elif message["role"] == Role.ASSISTANT:
+                elements += self.format_assistant.apply(content=message["content"])
+            elif message["role"] == Role.OBSERVATION:
+                elements += self.format_observation.apply(content=message["content"])
+            elif message["role"] == Role.FUNCTION:
+                elements += self.format_function.apply(
+                    content=message["content"],
+                    thought_words=self.thought_words,
+                    tool_call_words=self.tool_call_words,
+                )
+            else:
+                raise NotImplementedError("Unexpected role: {}".format(message["role"]))
+
+            encoded_messages.append(self._convert_elements_to_ids(tokenizer, elements))
+
+        return encoded_messages
+
+
 @dataclass
 class Llama2Template(Template):
     r"""A template that fuse the system message to first user message."""
@@ -1526,6 +1570,32 @@ register_template(
 )
 
 
+# copied from qwen template
+register_template(
+    name="moss_vl",
+    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
+    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
+    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
+    format_function=FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen"),
+    format_observation=StringFormatter(
+        slots=["<|im_start|>user\n<tool_response>\n{{content}}\n</tool_response><|im_end|>\n<|im_start|>assistant\n"]
+    ),
+    format_tools=ToolFormatter(tool_format="qwen"),
+    stop_words=["<|im_end|>"],
+    replace_eos=True,
+    mm_plugin=get_mm_plugin(
+        name="moss_vl",
+        image_token="<|image_pad|>",
+        video_token="<|video_pad|>",
+        vision_bos_token="<|vision_start|>",
+        vision_eos_token="<|vision_end|>",
+        time_bos_token="<|time_start|>",
+        time_eos_token="<|time_end|>",
+    ),
+    template_class=MossVLTemplate,
+)
+
+
 # copied from vicuna template
 register_template(
     name="llava",

src/llamafactory/extras/constants.py

@@ -73,6 +73,23 @@ MCA_SUPPORTED_MODELS = {
     "qwen3_5_moe",
 }
 
+# Text LLM model_types supported by the Megatron Bridge PT/SFT path (gpt_step).
+# Multimodal / audio / omni architectures are excluded in v0.
+MEGATRON_BRIDGE_SUPPORTED_MODELS = {
+    "deepseek_v3",
+    "deepseek_v4",
+    "llama",
+    "mistral",
+    "qwen2",
+    "qwen3",
+    "qwen3_5",
+    "qwen3_5_moe",
+    "qwen3_5_moe_text",
+    "qwen3_5_text",
+    "qwen3_moe",
+    "qwen3_next",
+}
+
 METHODS = ["full", "freeze", "lora", "oft"]
 
 MOD_SUPPORTED_MODELS = {"bloom", "falcon", "gemma", "llama", "mistral", "mixtral", "phi", "starcoder2"}
@@ -2181,6 +2198,17 @@ register_model_group(
 )
 
 
+register_model_group(
+    models={
+        "MOSS-VL-Instruct-0708": {
+            DownloadSource.DEFAULT: "OpenMOSS-Team/MOSS-VL-Instruct-0708",
+        },
+    },
+    template="moss_vl",
+    multimodal=True,
+)
+
+
 register_model_group(
     models={
         "OLMo-1B": {

src/llamafactory/extras/packages.py

@@ -79,6 +79,13 @@ def is_mcore_adapter_available():
     return _is_package_available("mcore_adapter")
 
 
+def is_megatron_bridge_available():
+    try:
+        return _is_package_available("megatron.bridge")
+    except ModuleNotFoundError:
+        return False
+
+
 def is_pillow_available():
     return _is_package_available("PIL")
 

src/llamafactory/hparams/__init__.py

@@ -16,6 +16,7 @@ from .data_args import DataArguments
 from .evaluation_args import EvaluationArguments
 from .finetuning_args import FinetuningArguments
 from .generating_args import GeneratingArguments
+from .megatron_bridge_args import MegatronBridgeArguments
 from .model_args import ModelArguments
 from .parser import get_eval_args, get_infer_args, get_ray_args, get_train_args, read_args
 from .training_args import RayArguments, TrainingArguments
@@ -26,6 +27,7 @@ __all__ = [
     "EvaluationArguments",
     "FinetuningArguments",
     "GeneratingArguments",
+    "MegatronBridgeArguments",
     "ModelArguments",
     "RayArguments",
     "TrainingArguments",

src/llamafactory/hparams/finetuning_args.py

@@ -482,6 +482,21 @@ class FinetuningArguments(
             )
         },
     )
+    use_megatron_bridge: bool = field(
+        default=False,
+        metadata={
+            "help": (
+                "Whether or not to use Megatron Bridge training backend. "
+                "Controlled by USE_MEGATRON_BRIDGE environment variable."
+            )
+        },
+    )
+    megatron_bridge_args: Any = field(
+        default=None,
+        init=False,
+        repr=False,
+        metadata={"help": "Megatron Bridge specific arguments, set when USE_MEGATRON_BRIDGE=1."},
+    )
     use_hyper_parallel: bool = field(
         default=False,
         metadata={

src/llamafactory/hparams/megatron_bridge_args.py

@@ -0,0 +1,193 @@
+# Copyright 2025 the LlamaFactory team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+from dataclasses import dataclass, field
+from typing import Literal, Optional
+
+from transformers.training_args import _convert_str_dict
+
+
+@dataclass
+class MegatronBridgeArguments:
+    r"""Arguments for Megatron Bridge distributed training backend.
+
+    Parallelism, optimizer overlap, checkpoint conversion, and selected Megatron
+    model-provider knobs are exposed here because Megatron Bridge uses a
+    standalone workflow outside the Hugging Face Trainer.
+    """
+
+    tensor_model_parallel_size: int = field(
+        default=1,
+        metadata={"help": "Tensor model parallel size for Megatron Bridge."},
+    )
+    pipeline_model_parallel_size: int = field(
+        default=1,
+        metadata={"help": "Pipeline model parallel size for Megatron Bridge."},
+    )
+    expert_model_parallel_size: int = field(
+        default=1,
+        metadata={"help": "Expert model parallel size for MoE models."},
+    )
+    context_parallel_size: int = field(
+        default=1,
+        metadata={"help": "Context parallel size for Megatron Bridge."},
+    )
+    virtual_pipeline_model_parallel_size: Optional[int] = field(
+        default=None,
+        metadata={"help": "Virtual pipeline (interleaved) parallel size. None keeps provider default."},
+    )
+    sequence_parallel: bool = field(
+        default=False,
+        metadata={"help": "Whether to enable sequence parallelism."},
+    )
+    recompute_granularity: Optional[str] = field(
+        default=None,
+        metadata={"help": "Activation recomputation granularity: 'full' or 'selective'."},
+    )
+    recompute_method: Optional[Literal["uniform", "block"]] = field(
+        default=None,
+        metadata={"help": "Activation recomputation method: 'uniform' or 'block'."},
+    )
+    recompute_num_layers: Optional[int] = field(
+        default=None,
+        metadata={"help": "Number of layers per recompute unit when recompute_method is set."},
+    )
+    account_for_embedding_in_pipeline_split: Optional[bool] = field(
+        default=None,
+        metadata={"help": "Whether pipeline split accounts for the embedding layer."},
+    )
+    account_for_loss_in_pipeline_split: Optional[bool] = field(
+        default=None,
+        metadata={"help": "Whether pipeline split accounts for the loss layer."},
+    )
+    bias_activation_fusion: Optional[bool] = field(
+        default=None,
+        metadata={"help": "Enable bias+activation fusion. None keeps Megatron provider default."},
+    )
+    apply_rope_fusion: Optional[bool] = field(
+        default=None,
+        metadata={"help": "Enable RoPE fusion kernel. None keeps Megatron provider default."},
+    )
+    masked_softmax_fusion: Optional[bool] = field(
+        default=None,
+        metadata={"help": "Enable masked softmax fusion. None keeps Megatron provider default."},
+    )
+    cross_entropy_loss_fusion: Optional[bool] = field(
+        default=None,
+        metadata={"help": "Enable cross-entropy loss fusion. None keeps Megatron provider default."},
+    )
+    moe_grouped_gemm: Optional[bool] = field(
+        default=None,
+        metadata={"help": "Enable grouped GEMM for MoE experts. None keeps provider default."},
+    )
+    moe_token_dispatcher_type: Optional[Literal["allgather", "alltoall", "flex"]] = field(
+        default=None,
+        metadata={"help": "MoE token dispatcher type: allgather, alltoall, or flex."},
+    )
+    calculate_per_token_loss: Optional[bool] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Whether to compute per-token loss. When context_parallel_size > 1, "
+                "this is forced to True regardless of this setting."
+            )
+        },
+    )
+    use_distributed_optimizer: bool = field(
+        default=True,
+        metadata={"help": "Whether to use Megatron distributed optimizer."},
+    )
+    overlap_param_gather: bool = field(
+        default=True,
+        metadata={"help": "Whether to overlap parameter all-gather with forward compute."},
+    )
+    overlap_grad_reduce: bool = field(
+        default=True,
+        metadata={"help": "Whether to overlap gradient all-reduce with backward compute."},
+    )
+    use_packed_sequences: bool = field(
+        default=False,
+        metadata={"help": "Whether to use packed sequences for SFT efficiency."},
+    )
+    mixed_precision: str = field(
+        default="bf16_mixed",
+        metadata={"help": "Mixed precision mode for Megatron Bridge, e.g. bf16_mixed or fp8."},
+    )
+    megatron_pretrained_checkpoint: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Path to a Megatron-format pretrained checkpoint. "
+                "If unset, HF weights are converted automatically before training."
+            )
+        },
+    )
+    export_hf_on_finish: bool = field(
+        default=False,
+        metadata={"help": "Whether to export the final checkpoint to Hugging Face format after training."},
+    )
+    extra_config: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Optional JSON string or path to a JSON file with extra Megatron Bridge model/training overrides. "
+                "Dot-paths are supported (e.g. train.train_iters or checkpoint.save_interval)."
+            )
+        },
+    )
+
+    def __post_init__(self) -> None:
+        if self.tensor_model_parallel_size < 1:
+            raise ValueError("`tensor_model_parallel_size` must be >= 1.")
+        if self.pipeline_model_parallel_size < 1:
+            raise ValueError("`pipeline_model_parallel_size` must be >= 1.")
+        if self.expert_model_parallel_size < 1:
+            raise ValueError("`expert_model_parallel_size` must be >= 1.")
+        if self.context_parallel_size < 1:
+            raise ValueError("`context_parallel_size` must be >= 1.")
+        if self.virtual_pipeline_model_parallel_size is not None and self.virtual_pipeline_model_parallel_size < 1:
+            raise ValueError("`virtual_pipeline_model_parallel_size` must be >= 1 when set.")
+        if self.sequence_parallel and self.tensor_model_parallel_size <= 1:
+            raise ValueError("`sequence_parallel` requires `tensor_model_parallel_size` > 1.")
+        if self.recompute_granularity is not None and self.recompute_granularity not in ("full", "selective"):
+            raise ValueError("`recompute_granularity` must be 'full' or 'selective'.")
+        if self.recompute_method is not None and self.recompute_method not in ("uniform", "block"):
+            raise ValueError("`recompute_method` must be 'uniform' or 'block'.")
+        if self.recompute_num_layers is not None and self.recompute_num_layers < 1:
+            raise ValueError("`recompute_num_layers` must be >= 1 when set.")
+        if self.moe_token_dispatcher_type is not None and self.moe_token_dispatcher_type not in (
+            "allgather",
+            "alltoall",
+            "flex",
+        ):
+            raise ValueError("`moe_token_dispatcher_type` must be 'allgather', 'alltoall', or 'flex'.")
+
+        if isinstance(self.extra_config, str):
+            config_str = self.extra_config.strip()
+            if config_str.startswith("{"):
+                self.extra_config = _convert_str_dict(json.loads(config_str))
+            else:
+                self.extra_config = config_str
+
+    def load_extra_config(self) -> dict:
+        if self.extra_config is None:
+            return {}
+        if isinstance(self.extra_config, dict):
+            return self.extra_config
+        if not os.path.isfile(self.extra_config):
+            raise ValueError(f"`extra_config` file not found: {self.extra_config}")
+        with open(self.extra_config, encoding="utf-8") as f:
+            return json.load(f)

src/llamafactory/hparams/model_args.py

@@ -470,10 +470,23 @@ class KTransformersArguments:
         default=False,
         metadata={"help": "Whether to use KTransformers AMX MoE backend for SFT training."},
     )
+    kt_cpu_activation: Literal["retain", "recompute"] | None = field(
+        default=None,
+        metadata={
+            "help": (
+                "Whether KTransformers retains CPU expert activations. Defaults to recompute while GPU "
+                "gradient checkpointing is enabled and retain otherwise."
+            )
+        },
+    )
     kt_weight_path: str | None = field(
         default=None,
         metadata={"help": "Path to pre-quantized INT8 expert weights (.kt files)."},
     )
+    kt_non_expert_weight_path: str | None = field(
+        default=None,
+        metadata={"help": "Path to the KT BF16 non-expert weight cache used with routed INT8 experts."},
+    )
     kt_expert_checkpoint_path: str | None = field(
         default=None,
         metadata={"help": "Path to expert checkpoint (safetensors) for online conversion."},
@@ -490,52 +503,202 @@ class KTransformersArguments:
         default=None,
         metadata={"help": "Intermediate size for GPU-side LoRA Experts."},
     )
+    _kt_inference_config: dict[str, Any] | None = field(default=None, init=False, repr=False)
+    _kt_config_handle: Any = field(default=None, init=False, repr=False)
+    _kt_adapter_artifact_path: str | None = field(default=None, init=False, repr=False)
 
-    def get_kt_config_dict(self, finetuning_args: Any, model_max_length: int | None) -> dict[str, Any]:
+    _KT_DERIVED_KEYS = frozenset(
-        r"""Build KT config values from LLaMA-Factory model and LoRA arguments."""
+        {
-        kt_config = {
+            "enabled",
-            "kt_lora_rank": getattr(finetuning_args, "lora_rank", None),
+            "kt_activation_policy",
-            "kt_lora_alpha": getattr(finetuning_args, "lora_alpha", None),
+            "kt_expert_checkpoint_path",
-            "kt_weight_path": self.kt_weight_path,
+            "kt_full_weight_grad",
-            "kt_expert_checkpoint_path": self.kt_expert_checkpoint_path,
+            "kt_lora_alpha",
-            "kt_model_max_length": model_max_length,
+            "kt_lora_dropout",
-            "kt_use_lora_experts": self.kt_use_lora_experts,
+            "kt_lora_expert_intermediate_size",
-            "kt_lora_expert_num": self.kt_lora_expert_num,
+            "kt_lora_expert_num",
-            "kt_lora_expert_intermediate_size": self.kt_lora_expert_intermediate_size,
+            "kt_lora_rank",
+            "kt_non_expert_weight_path",
+            "kt_skip_expert_loading",
+            "kt_train_mode",
+            "kt_use_lora_experts",
+            "kt_weight_path",
         }
+    )
+
+    def __post_init__(self) -> None:
+        if self.kt_cpu_activation not in {None, "retain", "recompute"}:
+            raise ValueError("`kt_cpu_activation` must be `retain` or `recompute`.")
+        if not self.use_kt and self.kt_cpu_activation is not None:
+            raise ValueError("`kt_cpu_activation` is only valid when `use_kt: true`.")
+
+    def get_kt_activation_policy(self) -> dict[str, str]:
+        r"""Resolve LF's GPU checkpoint switch and KT's CPU activation setting."""
+        gpu_activation = "retain" if self.disable_gradient_checkpointing else "recompute"
+        cpu_activation = self.kt_cpu_activation or gpu_activation
+        if cpu_activation == "recompute" and gpu_activation == "retain":
+            raise ValueError(
+                "`kt_cpu_activation: recompute` requires GPU gradient checkpointing. "
+                "Set `disable_gradient_checkpointing: false` or use `kt_cpu_activation: retain`."
+            )
+
+        return {"cpu": cpu_activation, "gpu": gpu_activation}
+
+    @staticmethod
+    def _get_accelerator_kt_config(training_args: Any) -> Any:
+        accelerator_config = getattr(training_args, "accelerator_config", None)
+        if isinstance(accelerator_config, dict):
+            return accelerator_config.get("kt_config")
+        return getattr(accelerator_config, "kt_config", None)
+
+    def _normalize_advanced_kt_config(self, raw_config: Any) -> dict[str, Any]:
+        if raw_config is None:
+            return {}
+        if not isinstance(raw_config, dict):
+            raise TypeError("LLaMA-Factory `kt_config` must be a flat mapping.")
+
+        config = dict(raw_config)
+        conflicts = sorted(set(config) & self._KT_DERIVED_KEYS)
+        if conflicts:
+            raise ValueError(f"These `kt_config` values are derived from LLaMA-Factory arguments: {conflicts}.")
+        return config
+
+    def _get_advanced_kt_config(self, training_args: Any) -> dict[str, Any]:
+        raw_config = getattr(training_args, "kt_config", None)
+        accelerator_config = self._get_accelerator_kt_config(training_args)
+        if raw_config is None:
+            if accelerator_config is not None:
+                raise ValueError(
+                    "Put KTransformers settings in the LLaMA-Factory training YAML `kt_config`; "
+                    "remove `kt_config` from the Accelerate config."
+                )
+            return {}
+        if accelerator_config is not None and accelerator_config != raw_config:
+            raise ValueError("LLaMA-Factory YAML and Accelerate config cannot define different KT settings.")
+        return self._normalize_advanced_kt_config(raw_config)
+
+    def configure_kt_checkpointing(self, training_args: Any) -> None:
+        r"""Keep LLaMA-Factory as the single gradient-checkpointing entry point."""
+        if self.use_unsloth or self.use_unsloth_gc:
+            raise ValueError("KTransformers cannot be combined with Unsloth checkpoint wrapping.")
+        if getattr(training_args, "gradient_checkpointing", False):
+            raise ValueError(
+                "KTransformers uses LLaMA-Factory's `disable_gradient_checkpointing`; "
+                "remove `gradient_checkpointing: true`."
+            )
+        if getattr(training_args, "gradient_checkpointing_kwargs", None) is not None:
+            raise ValueError("KTransformers supplies its checkpoint context; remove `gradient_checkpointing_kwargs`.")
+
+        fsdp_config = getattr(training_args, "fsdp_config", None)
+        if isinstance(fsdp_config, dict) and fsdp_config.get("activation_checkpointing"):
+            raise ValueError("Disable FSDP activation checkpointing when using KTransformers.")
+        if os.environ.get("FSDP_ACTIVATION_CHECKPOINTING", "false").lower() in {"1", "true", "yes"}:
+            raise ValueError("Disable FSDP activation checkpointing when using KTransformers.")
+
+        self.get_kt_activation_policy()
+        if not self.disable_gradient_checkpointing:
+            self.use_reentrant_gc = False
+        training_args.gradient_checkpointing = False
+        training_args.gradient_checkpointing_kwargs = None
+
+    def get_kt_config_dict(
+        self,
+        finetuning_args: Any,
+        model_max_length: int | None,
+        advanced_config: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        r"""Map LLaMA-Factory-owned training values to the public KT configuration."""
+        if getattr(finetuning_args, "finetuning_type", None) != "lora":
+            raise ValueError("KTransformers thin integration currently supports LoRA finetuning only.")
+
+        kt_config = dict(advanced_config or {})
+        configured_capacity = kt_config.pop("kt_model_max_length", None)
+        if configured_capacity is not None:
+            try:
+                configured_capacity = int(configured_capacity)
+            except (TypeError, ValueError) as exc:
+                raise ValueError("`kt_model_max_length` must be a positive integer.") from exc
+            if configured_capacity <= 0:
+                raise ValueError("`kt_model_max_length` must be a positive integer.")
+
+        kt_config.update(
+            {
+                "kt_lora_rank": getattr(finetuning_args, "lora_rank", None),
+                "kt_lora_alpha": getattr(finetuning_args, "lora_alpha", None),
+                "kt_lora_dropout": getattr(finetuning_args, "lora_dropout", None),
+                "kt_weight_path": self.kt_weight_path,
+                "kt_non_expert_weight_path": self.kt_non_expert_weight_path,
+                "kt_expert_checkpoint_path": self.kt_expert_checkpoint_path,
+                "kt_model_max_length": max(model_max_length or 0, configured_capacity or 0) or None,
+                "kt_use_lora_experts": self.kt_use_lora_experts,
+                "kt_lora_expert_num": self.kt_lora_expert_num,
+                "kt_lora_expert_intermediate_size": self.kt_lora_expert_intermediate_size,
+                "kt_activation_policy": self.get_kt_activation_policy(),
+                "kt_train_mode": "lora",
+                "kt_full_weight_grad": False,
+            }
+        )
         return {key: value for key, value in kt_config.items() if value is not None}
 
+    def _resolve_kt_adapter_artifact_dir(self, operation: str) -> str | None:
+        if not self.adapter_name_or_path:
+            return None
+        if len(self.adapter_name_or_path) != 1:
+            raise ValueError("KTransformers accepts a single `adapter_name_or_path`.")
+
+        adapter_root = os.path.realpath(os.path.expanduser(self.adapter_name_or_path[0]))
+        adapter_dir = adapter_root
+        if self.adapter_folder:
+            adapter_dir = os.path.realpath(os.path.join(adapter_root, self.adapter_folder))
+            if os.path.commonpath((adapter_root, adapter_dir)) != adapter_root:
+                raise ValueError("`adapter_folder` must stay inside the KT adapter directory.")
+        if not os.path.isdir(adapter_dir):
+            raise ValueError(f"KTransformers {operation} requires a local adapter directory.")
+        return adapter_dir
+
     def apply_kt_config(self, finetuning_args: Any, training_args: Any, model_max_length: int | None) -> None:
         r"""Apply LLaMA-Factory KT args to transformers/accelerate KT integration points."""
         if not self.use_kt:
             return
 
-        kt_config = self.get_kt_config_dict(finetuning_args, model_max_length)
+        self.configure_kt_checkpointing(training_args)
-        env_mapping = {
+        kt_config = self.get_kt_config_dict(
-            "kt_weight_path": "ACCELERATE_KT_WEIGHT_PATH",
+            finetuning_args,
-            "kt_expert_checkpoint_path": "ACCELERATE_KT_EXPERT_CHECKPOINT_PATH",
+            model_max_length,
-            "kt_model_max_length": "ACCELERATE_KT_MODEL_MAX_LENGTH",
+            self._get_advanced_kt_config(training_args),
-            "kt_lora_rank": "ACCELERATE_KT_LORA_RANK",
-            "kt_lora_alpha": "ACCELERATE_KT_LORA_ALPHA",
-            "kt_use_lora_experts": "ACCELERATE_KT_USE_LORA_EXPERTS",
-            "kt_lora_expert_num": "ACCELERATE_KT_LORA_EXPERT_NUM",
-            "kt_lora_expert_intermediate_size": "ACCELERATE_KT_LORA_EXPERT_INTERMEDIATE_SIZE",
-        }
-        for key, env_key in env_mapping.items():
-            value = kt_config.get(key)
-            if value is not None:
-                os.environ[env_key] = str(value)
-
-        hf_kt = getattr(training_args, "hf_kt_config", None)
-        if hf_kt is None or not hasattr(hf_kt, "_kt_config") or not isinstance(hf_kt._kt_config, dict):
-            return
-
-        hf_kt._kt_config.update(kt_config)
-        gc_enabled = getattr(training_args, "gradient_checkpointing", False) or not getattr(
-            self, "disable_gradient_checkpointing", True
         )
-        if gc_enabled:
+        update_kt_config = getattr(training_args, "update_kt_config", None)
-            hf_kt._kt_config.setdefault("kt_share_cache_pool", True)
+        if not callable(update_kt_config):
+            raise RuntimeError(
+                "The installed Transformers-KT does not provide `TrainingArguments.update_kt_config()`."
+            )
+
+        adapter_dir = self._resolve_kt_adapter_artifact_dir("training")
+        update_kt_config(kt_config, adapter_name_or_path=adapter_dir)
+
+    def configure_kt_loading(self, finetuning_args: Any, model_max_length: int | None) -> None:
+        r"""Configure KT model loading for inference and evaluation."""
+        if not self.use_kt:
+            if self._kt_inference_config is not None:
+                raise ValueError("`kt_config` requires `use_kt: true`.")
+            return
+        if self.infer_backend != EngineName.HF:
+            raise ValueError("KTransformers inference requires `infer_backend: huggingface`.")
+
+        adapter_dir = self._resolve_kt_adapter_artifact_dir("inference")
+
+        try:
+            from transformers.integrations.kt import configure_kt
+        except (ImportError, ModuleNotFoundError) as exc:
+            raise RuntimeError("The installed Transformers-KT does not provide `configure_kt()`.") from exc
+
+        kt_config = self.get_kt_config_dict(
+            finetuning_args,
+            model_max_length,
+            self._normalize_advanced_kt_config(self._kt_inference_config),
+        )
+        self._kt_adapter_artifact_path = adapter_dir
+        self._kt_config_handle = configure_kt(kt_config)
 
 
 @dataclass
@@ -580,6 +743,7 @@ class ModelArguments(
         ExportArguments.__post_init__(self)
         VllmArguments.__post_init__(self)
         SGLangArguments.__post_init__(self)
+        KTransformersArguments.__post_init__(self)
 
     @classmethod
     def copyfrom(cls, source: "Self", **kwargs) -> "Self":

src/llamafactory/hparams/parser.py

@@ -18,6 +18,7 @@
 import json
 import os
 import sys
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Optional
 
@@ -33,11 +34,12 @@ from transformers.utils import is_torch_bf16_gpu_available, is_torch_npu_availab
 from ..extras import logging
 from ..extras.constants import CHECKPOINT_NAMES, EngineName
 from ..extras.misc import check_dependencies, check_version, get_current_device, is_env_enabled
-from ..extras.packages import is_mcore_adapter_available
+from ..extras.packages import is_mcore_adapter_available, is_megatron_bridge_available
 from .data_args import DataArguments
 from .evaluation_args import EvaluationArguments
 from .finetuning_args import FinetuningArguments
 from .generating_args import GeneratingArguments
+from .megatron_bridge_args import MegatronBridgeArguments
 from .model_args import ModelArguments
 from .training_args import RayArguments, TrainingArguments
 
@@ -47,6 +49,14 @@ logger = logging.get_logger(__name__)
 check_dependencies()
 
 
+@dataclass
+class _KTransformersRuntimeArguments:
+    kt_config: dict[str, Any] | None = field(
+        default=None,
+        metadata={"help": "Advanced KTransformers settings used during inference or evaluation."},
+    )
+
+
 _TRAIN_ARGS = [
     ModelArguments,
     DataArguments,
@@ -55,9 +65,9 @@ _TRAIN_ARGS = [
     GeneratingArguments,
 ]
 _TRAIN_CLS = tuple[ModelArguments, DataArguments, TrainingArguments, FinetuningArguments, GeneratingArguments]
-_INFER_ARGS = [ModelArguments, DataArguments, FinetuningArguments, GeneratingArguments]
+_INFER_ARGS = [ModelArguments, DataArguments, FinetuningArguments, GeneratingArguments, _KTransformersRuntimeArguments]
 _INFER_CLS = tuple[ModelArguments, DataArguments, FinetuningArguments, GeneratingArguments]
-_EVAL_ARGS = [ModelArguments, DataArguments, EvaluationArguments, FinetuningArguments]
+_EVAL_ARGS = [ModelArguments, DataArguments, EvaluationArguments, FinetuningArguments, _KTransformersRuntimeArguments]
 _EVAL_CLS = tuple[ModelArguments, DataArguments, EvaluationArguments, FinetuningArguments]
 
 if is_mcore_adapter_available() and is_env_enabled("USE_MCA"):
@@ -81,6 +91,23 @@ else:
     _TRAIN_MCA_ARGS = []
     _TRAIN_MCA_CLS = tuple()
 
+_TRAIN_MBRIDGE_ARGS = [
+    ModelArguments,
+    DataArguments,
+    TrainingArguments,
+    FinetuningArguments,
+    MegatronBridgeArguments,
+    GeneratingArguments,
+]
+_TRAIN_MBRIDGE_CLS = tuple[
+    ModelArguments,
+    DataArguments,
+    TrainingArguments,
+    FinetuningArguments,
+    MegatronBridgeArguments,
+    GeneratingArguments,
+]
+
 
 def read_args(args: dict[str, Any] | list[str] | None = None) -> dict[str, Any] | list[str]:
     r"""Get arguments from the command line or a config file."""
@@ -99,6 +126,26 @@ def read_args(args: dict[str, Any] | list[str] | None = None) -> dict[str, Any]
         return sys.argv[1:]
 
 
+def _get_kt_runtime_capacity(
+    data_args: "DataArguments",
+    training_args: "TrainingArguments",
+    finetuning_args: "FinetuningArguments",
+) -> int:
+    r"""Return the largest local token batch submitted to a KT expert."""
+    tokens_per_sample = data_args.cutoff_len
+    if finetuning_args.stage == "sft" and data_args.packing:
+        tokens_per_sample += 1
+    if finetuning_args.stage == "sft" and training_args.do_train:
+        tokens_per_sample = ((tokens_per_sample + 7) // 8) * 8
+
+    local_batch_sizes = [1]
+    if training_args.do_train:
+        local_batch_sizes.append(training_args.per_device_train_batch_size)
+    if training_args.do_eval or training_args.do_predict:
+        local_batch_sizes.append(training_args.per_device_eval_batch_size)
+    return tokens_per_sample * max(local_batch_sizes)
+
+
 def _parse_args(
     parser: "HfArgumentParser", args: dict[str, Any] | list[str] | None = None, allow_extra_keys: bool = False
 ) -> tuple[Any]:
@@ -246,6 +293,9 @@ def _check_extra_dependencies(
     if finetuning_args.plot_loss:
         check_version("matplotlib", mandatory=True)
 
+    if finetuning_args.use_megatron_bridge:
+        check_version("megatron-bridge", mandatory=True)
+
     if training_args is not None:
         if training_args.deepspeed:
             check_version("deepspeed", mandatory=True)
@@ -283,16 +333,57 @@ def _configure_mca_training_args(training_args, data_args, finetuning_args) -> N
     finetuning_args.use_mca = True
 
 
+def _validate_megatron_bridge_parallel_args(mb_args: MegatronBridgeArguments, world_size: int) -> None:
+    parallel_size = (
+        mb_args.tensor_model_parallel_size
+        * mb_args.pipeline_model_parallel_size
+        * mb_args.context_parallel_size
+        * mb_args.expert_model_parallel_size
+    )
+    if parallel_size > world_size:
+        raise ValueError(f"Total Megatron Bridge parallel size ({parallel_size}) exceeds `world_size` ({world_size}).")
+    if world_size % parallel_size != 0:
+        raise ValueError(
+            f"Total Megatron Bridge parallel size ({parallel_size}) must divide `world_size` ({world_size})."
+        )
+
+
+def _parse_train_mbridge_args(args: dict[str, Any] | list[str] | None = None) -> _TRAIN_MBRIDGE_CLS:
+    parser = HfArgumentParser(_TRAIN_MBRIDGE_ARGS)
+    allow_extra_keys = is_env_enabled("ALLOW_EXTRA_ARGS")
+    model_args, data_args, training_args, finetuning_args, mb_args, generating_args = _parse_args(
+        parser, args, allow_extra_keys=allow_extra_keys
+    )
+    _configure_mbridge_training_args(training_args, data_args, finetuning_args)
+    return model_args, data_args, training_args, finetuning_args, mb_args, generating_args
+
+
+def _configure_mbridge_training_args(training_args, data_args, finetuning_args) -> None:
+    """Patch training args to avoid args checking errors and sync Megatron Bridge settings."""
+    training_args.predict_with_generate = False
+    training_args.generation_max_length = data_args.cutoff_len
+    training_args.generation_num_beams = 1
+    finetuning_args.use_megatron_bridge = True
+
+
 def _parse_infer_args(args: dict[str, Any] | list[str] | None = None) -> _INFER_CLS:
     parser = HfArgumentParser(_INFER_ARGS)
     allow_extra_keys = is_env_enabled("ALLOW_EXTRA_ARGS")
-    return _parse_args(parser, args, allow_extra_keys=allow_extra_keys)
+    model_args, data_args, finetuning_args, generating_args, kt_args = _parse_args(
+        parser, args, allow_extra_keys=allow_extra_keys
+    )
+    model_args._kt_inference_config = kt_args.kt_config
+    return model_args, data_args, finetuning_args, generating_args
 
 
 def _parse_eval_args(args: dict[str, Any] | list[str] | None = None) -> _EVAL_CLS:
     parser = HfArgumentParser(_EVAL_ARGS)
     allow_extra_keys = is_env_enabled("ALLOW_EXTRA_ARGS")
-    return _parse_args(parser, args, allow_extra_keys=allow_extra_keys)
+    model_args, data_args, eval_args, finetuning_args, kt_args = _parse_args(
+        parser, args, allow_extra_keys=allow_extra_keys
+    )
+    model_args._kt_inference_config = kt_args.kt_config
+    return model_args, data_args, eval_args, finetuning_args
 
 
 def get_ray_args(args: dict[str, Any] | list[str] | None = None) -> RayArguments:
@@ -302,11 +393,22 @@ def get_ray_args(args: dict[str, Any] | list[str] | None = None) -> RayArguments
 
 
 def get_train_args(args: dict[str, Any] | list[str] | None = None) -> _TRAIN_CLS:
+    mb_args = None
     if is_env_enabled("USE_MCA"):
         model_args, data_args, training_args, finetuning_args, generating_args = _parse_train_mca_args(args)
+    elif is_env_enabled("USE_MEGATRON_BRIDGE"):
+        if not is_megatron_bridge_available():
+            raise ImportError(
+                "megatron-bridge is required when USE_MEGATRON_BRIDGE=1. "
+                "Please install `megatron-bridge` and its dependencies."
+            )
+        model_args, data_args, training_args, finetuning_args, mb_args, generating_args = _parse_train_mbridge_args(
+            args
+        )
     else:
         model_args, data_args, training_args, finetuning_args, generating_args = _parse_train_args(args)
         finetuning_args.use_mca = False
+        finetuning_args.use_megatron_bridge = False
 
     # Setup logging
     if training_args.should_log:
@@ -326,6 +428,22 @@ def get_train_args(args: dict[str, Any] | list[str] | None = None) -> _TRAIN_CLS
     if finetuning_args.stage == "sft" and training_args.do_predict and not training_args.predict_with_generate:
         raise ValueError("Please enable `predict_with_generate` to save model predictions.")
 
+    if finetuning_args.use_megatron_bridge:
+        if finetuning_args.use_mca or finetuning_args.use_hyper_parallel:
+            raise ValueError("Megatron Bridge cannot be used together with MCA or HyperParallel.")
+        if finetuning_args.stage not in ["pt", "sft"]:
+            raise ValueError("Megatron Bridge only supports the `pt` and `sft` stages.")
+        if finetuning_args.finetuning_type not in ["full", "lora"]:
+            raise ValueError("Megatron Bridge only supports `full` and `lora` finetuning.")
+        if model_args.quantization_bit is not None:
+            raise ValueError("Quantized models are not supported with Megatron Bridge.")
+        if training_args.deepspeed is not None:
+            raise ValueError("Megatron Bridge is incompatible with DeepSpeed.")
+        if mb_args is None:
+            raise ValueError("Megatron Bridge arguments are missing. Please set USE_MEGATRON_BRIDGE=1.")
+        _validate_megatron_bridge_parallel_args(mb_args, training_args.world_size)
+        finetuning_args.megatron_bridge_args = mb_args
+
     if finetuning_args.stage in ["rm", "ppo"] and training_args.load_best_model_at_end:
         raise ValueError("RM and PPO stages do not support `load_best_model_at_end`.")
 
@@ -400,7 +518,12 @@ def get_train_args(args: dict[str, Any] | list[str] | None = None) -> _TRAIN_CLS
     if training_args.deepspeed is not None and (finetuning_args.use_galore or finetuning_args.use_apollo):
         raise ValueError("GaLore and APOLLO are incompatible with DeepSpeed yet.")
 
-    if not finetuning_args.use_mca and training_args.fp8 and model_args.quantization_bit is not None:
+    if (
+        not finetuning_args.use_mca
+        and not finetuning_args.use_megatron_bridge
+        and training_args.fp8
+        and model_args.quantization_bit is not None
+    ):
         raise ValueError("FP8 training is not compatible with quantization. Please disable one of them.")
 
     if model_args.infer_backend != EngineName.HF:
@@ -417,7 +540,12 @@ def get_train_args(args: dict[str, Any] | list[str] | None = None) -> _TRAIN_CLS
     _check_extra_dependencies(model_args, finetuning_args, training_args)
     _verify_trackio_args(training_args)
 
-    if not finetuning_args.use_mca and training_args.fp8_enable_fsdp_float8_all_gather and not training_args.fp8:
+    if (
+        not finetuning_args.use_mca
+        and not finetuning_args.use_megatron_bridge
+        and training_args.fp8_enable_fsdp_float8_all_gather
+        and not training_args.fp8
+    ):
         logger.warning_rank0("fp8_enable_fsdp_float8_all_gather requires fp8=True. Setting fp8=True.")
         model_args.fp8 = True
 
@@ -514,10 +642,10 @@ def get_train_args(args: dict[str, Any] | list[str] | None = None) -> _TRAIN_CLS
     elif training_args.fp16:
         model_args.compute_dtype = torch.float16
 
+    data_args.packing = data_args.packing if data_args.packing is not None else finetuning_args.stage == "pt"
     model_args.device_map = {"": get_current_device()}
     model_args.model_max_length = data_args.cutoff_len
     model_args.block_diag_attn = data_args.neat_packing
-    data_args.packing = data_args.packing if data_args.packing is not None else finetuning_args.stage == "pt"
 
     # Log on each process the small summary
     logger.info(
@@ -529,7 +657,11 @@ def get_train_args(args: dict[str, Any] | list[str] | None = None) -> _TRAIN_CLS
     transformers.set_seed(training_args.seed)
 
     if model_args.use_kt:
-        model_args.apply_kt_config(finetuning_args, training_args, model_args.model_max_length)
+        model_args.apply_kt_config(
+            finetuning_args,
+            training_args,
+            _get_kt_runtime_capacity(data_args, training_args, finetuning_args),
+        )
 
     return model_args, data_args, training_args, finetuning_args, generating_args
 
@@ -566,6 +698,8 @@ def get_infer_args(args: dict[str, Any] | list[str] | None = None) -> _INFER_CLS
     else:
         model_args.device_map = "auto"
 
+    model_args.configure_kt_loading(finetuning_args, data_args.cutoff_len)
+
     return model_args, data_args, finetuning_args, generating_args
 
 
@@ -584,6 +718,7 @@ def get_eval_args(args: dict[str, Any] | list[str] | None = None) -> _EVAL_CLS:
     _check_extra_dependencies(model_args, finetuning_args)
 
     model_args.device_map = "auto"
+    model_args.configure_kt_loading(finetuning_args, data_args.cutoff_len)
 
     transformers.set_seed(eval_args.seed)
 

src/llamafactory/launcher.py

@@ -54,7 +54,7 @@ def launch():
     )
 
     command = sys.argv.pop(1) if len(sys.argv) > 1 else "help"
-    if is_env_enabled("USE_MCA"):  # force use torchrun
+    if is_env_enabled("USE_MCA") or is_env_enabled("USE_MEGATRON_BRIDGE"):  # force use torchrun
         os.environ["FORCE_TORCHRUN"] = "1"
 
     if command == "train" and (

src/llamafactory/model/adapter.py

@@ -138,6 +138,12 @@ def _setup_freeze_tuning(
     logger.info_rank0("Set trainable layers: {}".format(",".join(trainable_layers)))
 
 
+def _load_kt_inference_adapter_artifacts(model: "PreTrainedModel", adapter_path: str) -> None:
+    from kt_kernel.sft import load_kt_adapter_artifacts
+
+    load_kt_adapter_artifacts(model, adapter_path)
+
+
 def _setup_lora_tuning(
     config: "PretrainedConfig",
     model: "PreTrainedModel",
@@ -185,6 +191,8 @@ def _setup_lora_tuning(
             "revision": model_args.model_revision,
             "token": model_args.hf_hub_token,
         }
+        if model_args.use_kt:
+            init_kwargs["autocast_adapter_dtype"] = cast_trainable_params_to_fp32
 
         for adapter in adapter_to_merge:
             model: LoraModel = PeftModel.from_pretrained(model, adapter, **init_kwargs)
@@ -198,12 +206,22 @@ def _setup_lora_tuning(
                 pass  # already loaded via load_unsloth_peft_model in loader.py
             else:
                 if model_args.use_unsloth:
-                    peft_model = load_unsloth_peft_model(config, model_args, finetuning_args, is_trainable=is_trainable)
+                    peft_model = load_unsloth_peft_model(
+                        config, model_args, finetuning_args, is_trainable=is_trainable
+                    )
                     if peft_model is not None:
                         model = peft_model
 
                 if not model_args.use_unsloth:  # unsloth was disabled or fell back
-                    model = PeftModel.from_pretrained(model, adapter_to_resume, is_trainable=is_trainable, **init_kwargs)
+                    model = PeftModel.from_pretrained(
+                        model, adapter_to_resume, is_trainable=is_trainable, **init_kwargs
+                    )
+
+            if model_args.use_kt and not is_trainable:
+                adapter_path = model_args._kt_adapter_artifact_path
+                if adapter_path is None:
+                    raise RuntimeError("KT adapter artifacts were not resolved before model loading.")
+                _load_kt_inference_adapter_artifacts(model, adapter_path)
 
         logger.info_rank0("Loaded adapter(s): {}".format(",".join(model_args.adapter_name_or_path)))
 
@@ -260,7 +278,7 @@ def _setup_lora_tuning(
                 raise ValueError("KTransformers only supports LoRA finetuning.")
 
             peft_config = LoraConfig(task_type=TaskType.CAUSAL_LM, inference_mode=False, **peft_kwargs)
-            model = get_peft_model(model, peft_config)
+            model = get_peft_model(model, peft_config, autocast_adapter_dtype=cast_trainable_params_to_fp32)
         elif model_args.use_unsloth:
             if finetuning_args.finetuning_type == "oft":
                 raise ValueError("Unsloth is currently not supported for OFT.")

src/llamafactory/model/loader.py

@@ -217,9 +217,9 @@ def load_model(
             "You are try to using future feature about kernels, please note that this feature "
             "is not supported for all models. If get any error, please disable this feature, or report the issue."
         )
-        from ..v1.plugins.model_plugins.kernels.interface import apply_default_kernels
+        from ..v1.plugins.model_plugins.kernels.interface import apply_v1_kernels
 
-        model = apply_default_kernels(model, include_kernels=model_args.use_v1_kernels)
+        model = apply_v1_kernels(model, use_v1_kernels=model_args.use_v1_kernels)
 
     trainable_params, all_param = count_parameters(model)
     if is_trainable:

src/llamafactory/model/model_utils/attention.py

@@ -82,6 +82,14 @@ def configure_attn_implementation(config: "PretrainedConfig", model_args: "Model
             return
 
         requested_attn_implementation = "flash_attention_2"
+    elif model_args.flash_attn == AttentionFunction.FA3:
+        from transformers.utils import is_flash_attn_3_available
+
+        if not is_flash_attn_3_available():
+            logger.warning_rank0("FlashAttention-3 is not installed.")
+            return
+
+        requested_attn_implementation = "flash_attention_3"
     else:
         raise NotImplementedError(f"Unknown attention type: {model_args.flash_attn}")
 
@@ -109,6 +117,8 @@ def print_attn_implementation(config: "PretrainedConfig") -> None:
 
     if attn_implementation == "flash_attention_2":
         logger.info_rank0("Using FlashAttention-2 for faster training and inference.")
+    elif attn_implementation == "flash_attention_3":
+        logger.info_rank0("Using FlashAttention-3 for faster training and inference.")
     elif attn_implementation == "sdpa":
         logger.info_rank0("Using torch SDPA for faster training and inference.")
     else:

src/llamafactory/model/model_utils/checkpointing.py

@@ -40,6 +40,23 @@ if TYPE_CHECKING:
 logger = logging.get_logger(__name__)
 
 
+def _get_gradient_checkpointing_kwargs(model_args: "ModelArguments") -> dict[str, Any]:
+    r"""Build checkpoint kwargs through KT's public activation-context provider."""
+    if not model_args.use_kt:
+        return {"use_reentrant": model_args.use_reentrant_gc}
+
+    policy = model_args.get_kt_activation_policy()
+    if policy["gpu"] != "recompute":
+        return {"use_reentrant": False}
+
+    try:
+        from kt_kernel.sft import get_activation_checkpoint_context_fn
+    except (ImportError, ModuleNotFoundError) as exc:
+        raise RuntimeError("The installed kt-kernel does not provide the activation checkpoint context API.") from exc
+
+    return {"use_reentrant": False, "context_fn": get_activation_checkpoint_context_fn()}
+
+
 def get_unsloth_gradient_checkpointing_func() -> Callable:
     class UnslothGradientCheckpointing(torch.autograd.Function):
         r"""Saves VRAM by smartly offloading to RAM."""
@@ -172,7 +189,7 @@ def prepare_model_for_training(model: "PreTrainedModel", model_args: "ModelArgum
             )
             model.gradient_checkpointing_enable = MethodType(gradient_checkpointing_enable, model)
             model.gradient_checkpointing_enable(
-                gradient_checkpointing_kwargs={"use_reentrant": model_args.use_reentrant_gc}
+                gradient_checkpointing_kwargs=_get_gradient_checkpointing_kwargs(model_args)
             )
             setattr(model.config, "use_cache", False)  # turn off when gradient checkpointing is enabled
             logger.info_rank0("Gradient checkpointing enabled.")

src/llamafactory/model/model_utils/embedding.py

@@ -120,9 +120,7 @@ def _noisy_mean_initialization(
     avg_weight = _existing_embeddings(embed_weight, num_new_tokens, token_ids).mean(dim=0, keepdim=True)
 
     if token_ids:
-        noise_weight = torch.empty(
+        noise_weight = torch.empty(len(token_ids), embedding_dim, device=embed_weight.device, dtype=embed_weight.dtype)
-            len(token_ids), embedding_dim, device=embed_weight.device, dtype=embed_weight.dtype
-        )
         noise_weight.normal_(mean=0, std=(1.0 / math.sqrt(embedding_dim)))
         embed_weight[token_ids] = avg_weight + noise_weight
     else:
@@ -202,8 +200,7 @@ def _description_based_initialization(
             if len(valid_token_ids) == 0:
                 # Fallback: use mean of all existing embeddings
                 logger.warning_rank0(
-                    f"Description for token '{token_str}' contains no valid tokens. "
+                    f"Description for token '{token_str}' contains no valid tokens. Using mean of existing embeddings."
-                    "Using mean of existing embeddings."
                 )
                 base_embedding = fallback_embedding
             else:

src/llamafactory/model/model_utils/quantization.py

@@ -40,6 +40,10 @@ if TYPE_CHECKING:
 logger = logging.get_logger(__name__)
 
 
+def _uses_kt_non_expert_cache(model_args: "ModelArguments") -> bool:
+    return model_args.use_kt and bool(model_args.kt_non_expert_weight_path)
+
+
 def _get_quantization_dataset(tokenizer: "PreTrainedTokenizer", model_args: "ModelArguments") -> list[dict[str, Any]]:
     r"""Prepare the tokenized dataset to perform AutoGPTQ. Do not use tensor output for JSON serialization."""
     if os.path.isfile(model_args.export_quantization_dataset):
@@ -108,6 +112,13 @@ def configure_quantization(
             init_kwargs["ignore_mismatched_sizes"] = True
 
         if quant_method == QuantizationMethod.FP8:
+            if _uses_kt_non_expert_cache(model_args):
+                if model_args.quantization_bit is not None:
+                    raise ValueError("`quantization_bit` cannot be combined with KT weight caches.")
+
+                logger.info_rank0("Skipping source FP8 dequantization because KT weight caches are configured.")
+                return
+
             from transformers import FineGrainedFP8Config
 
             quant_config = FineGrainedFP8Config(dequantize=True)

src/llamafactory/model/model_utils/visual.py

@@ -56,7 +56,7 @@ class CompositeModel:
                     )
                     break
 
-            if project_module is not None:
+            if isinstance(project_module, torch.nn.Module):
                 mm_projectors.append(project_module)
 
         return mm_projectors
@@ -344,6 +344,15 @@ _register_composite_model(
 )
 
 
+_register_composite_model(
+    model_type="moss_vl",
+    projector_keys=["model.visual.merger", "model.separator_token"],
+    vision_model_keys=["model.visual.pos_embed", "model.visual.patch_embed", "model.visual.blocks"],
+    language_model_keys=["model.language_model", "lm_head"],
+    lora_conflict_keys=["patch_embed"],
+)
+
+
 _register_composite_model(
     model_type="mllama",
     vision_model_keys=["vision_model"],

src/llamafactory/model/patcher.py

@@ -23,7 +23,7 @@ from transformers.modeling_utils import is_fsdp_enabled
 from transformers.utils import is_torch_cuda_available, is_torch_npu_available
 
 from ..extras import logging
-from ..extras.misc import infer_optim_dtype
+from ..extras.misc import check_version, infer_optim_dtype
 from ..extras.packages import is_transformers_version_greater_than
 from .model_utils.attention import configure_attn_implementation, print_attn_implementation
 from .model_utils.checkpointing import prepare_model_for_training
@@ -418,6 +418,10 @@ def patch_config(
             "pip install git+https://github.com/huggingface/transformers.git@3c2517727ce28a30f5044e01663ee204deb1cdbe"
         )
 
+    if getattr(config, "model_type", None) == "moss_vl":
+        check_version("transformers==4.57.1", mandatory=True)
+        check_version("torchcodec==0.7.0", mandatory=True)
+
     if getattr(config, "model_type", None) == "qwen3_omni_moe":
         patch_qwen3_omni_moe_thinker_text_sparse_moe_block()
 

src/llamafactory/train/hyper_parallel/trainer.py

@@ -278,7 +278,9 @@ class HyperParallelTrainer(CustomSeq2SeqTrainer):
         )
         logical_batches = len(batch_sampler) // self._cp_size
         dp_size = max(1, get_platform().get_world_size() // self._cp_size)
-        logical_length = logical_batches // dp_size if self.args.dataloader_drop_last else _ceil_div(logical_batches, dp_size)
+        logical_length = (
+            logical_batches // dp_size if self.args.dataloader_drop_last else _ceil_div(logical_batches, dp_size)
+        )
 
         dataloader_params = {
             "batch_sampler": batch_sampler,
@@ -359,7 +361,12 @@ class HyperParallelTrainer(CustomSeq2SeqTrainer):
             loss = loss.mean()
 
         if not getattr(self, "model_accepts_loss_kwargs", False) and getattr(self, "compute_loss_func", None) is None:
-            loss = loss / self.args.gradient_accumulation_steps
+            accumulation_steps = getattr(
+                self,
+                "current_gradient_accumulation_steps",
+                self.args.gradient_accumulation_steps,
+            )
+            loss = loss / accumulation_steps
 
         self.accelerator.backward(loss)
 

src/llamafactory/train/megatron_bridge/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2025 the LlamaFactory team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .workflow import run_pt, run_sft
+
+
+__all__ = ["run_pt", "run_sft"]

src/llamafactory/train/megatron_bridge/config_builder.py

@@ -0,0 +1,708 @@
+# Copyright 2025 the LlamaFactory team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+import os
+from typing import TYPE_CHECKING, Any
+
+from ...extras.logging import get_logger
+
+
+if TYPE_CHECKING:
+    from ...hparams import (
+        DataArguments,
+        FinetuningArguments,
+        MegatronBridgeArguments,
+        ModelArguments,
+        TrainingArguments,
+    )
+
+
+logger = get_logger(__name__)
+
+_LR_SCHEDULER_MAP = {
+    "cosine": "cosine",
+    "linear": "linear",
+    "constant": "constant",
+    "constant_with_warmup": "constant",
+}
+
+
+def _map_lr_scheduler_type(lr_scheduler_type: str) -> str:
+    mapped = _LR_SCHEDULER_MAP.get(lr_scheduler_type)
+    if mapped is None:
+        logger.warning_rank0(
+            f"lr_scheduler_type '{lr_scheduler_type}' is not supported by Megatron Bridge; using cosine."
+        )
+        return "cosine"
+    return mapped
+
+
+def _resolve_warmup_steps(training_args: "TrainingArguments", train_iters: int) -> int:
+    r"""Resolve warmup steps with Hugging Face Trainer semantics.
+
+    Absolute ``warmup_steps`` are kept as-is (even when larger than ``train_iters``)
+    so short debugging runs with ``max_steps < warmup_steps`` match HF LR values.
+    ``lr_decay_iters`` is expanded separately to avoid Megatron capping warmup.
+    """
+    warmup_steps = getattr(training_args, "warmup_steps", 0) or 0
+    if warmup_steps > 0:
+        return warmup_steps
+
+    warmup_ratio = getattr(training_args, "warmup_ratio", 0.0) or 0.0
+    if warmup_ratio > 0:
+        return min(int(train_iters * warmup_ratio), train_iters)
+    return 0
+
+
+def _resolve_decay_iters(train_iters: int, warmup_steps: int) -> int:
+    r"""Ensure decay span is longer than warmup so Megatron does not shrink warmup.
+
+    Megatron Bridge caps ``lr_warmup_steps`` whenever it is ``>= lr_decay_steps``,
+    so keep decay strictly larger than warmup on short comparison runs.
+    """
+    if warmup_steps <= 0:
+        return train_iters
+    return max(train_iters, warmup_steps + 1)
+
+
+def _import_training_config():
+    from megatron.bridge.training.config import (
+        CheckpointConfig,
+        ConfigContainer,
+        FinetuningDatasetConfig,
+        GPTDatasetConfig,
+        LoggerConfig,
+        RNGConfig,
+        TrainingConfig,
+    )
+
+    try:
+        from megatron.bridge.training.config import DistributedInitConfig
+    except ImportError:
+        DistributedInitConfig = None
+
+    try:
+        from megatron.bridge.training.config import ValidationConfig
+    except ImportError:
+        ValidationConfig = None
+
+    try:
+        from megatron.bridge.training.tokenizers.config import TokenizerConfig
+    except ImportError:
+        from megatron.bridge.training.config import TokenizerConfig
+
+    return (
+        CheckpointConfig,
+        ConfigContainer,
+        DistributedInitConfig,
+        FinetuningDatasetConfig,
+        GPTDatasetConfig,
+        LoggerConfig,
+        RNGConfig,
+        TokenizerConfig,
+        TrainingConfig,
+        ValidationConfig,
+    )
+
+
+def _create_optimizer_scheduler(
+    training_args: "TrainingArguments",
+    warmup_steps: int,
+    train_iters: int,
+    finetuning_args: "FinetuningArguments",
+    use_distributed_optimizer: bool,
+):
+    from megatron.bridge.training.config import OptimizerConfig, SchedulerConfig
+
+    finetuning_type = finetuning_args.finetuning_type
+    learning_rate = training_args.learning_rate
+    if finetuning_type in ("lora", "full"):
+        max_lr, min_lr, default_beta2 = learning_rate, 0.0, 0.999
+    else:
+        max_lr, min_lr, default_beta2 = learning_rate, learning_rate * 0.1, 0.95
+
+    # Match Hugging Face Trainer defaults unless the user overrides them.
+    adam_beta1 = getattr(training_args, "adam_beta1", 0.9)
+    adam_beta2 = getattr(training_args, "adam_beta2", default_beta2)
+    adam_eps = getattr(training_args, "adam_epsilon", 1e-8)
+    weight_decay = getattr(training_args, "weight_decay", 0.0)
+    max_grad_norm = getattr(training_args, "max_grad_norm", 1.0)
+    decay_iters = _resolve_decay_iters(train_iters, warmup_steps)
+    optimizer = OptimizerConfig(
+        optimizer="adam",
+        lr=max_lr,
+        min_lr=min_lr,
+        weight_decay=weight_decay,
+        bf16=getattr(training_args, "bf16", True),
+        fp16=getattr(training_args, "fp16", False),
+        adam_beta1=adam_beta1,
+        adam_beta2=adam_beta2,
+        adam_eps=adam_eps,
+        use_distributed_optimizer=use_distributed_optimizer,
+        clip_grad=max_grad_norm,
+    )
+    scheduler = SchedulerConfig(
+        start_weight_decay=weight_decay,
+        end_weight_decay=weight_decay,
+        weight_decay_incr_style="constant",
+        lr_decay_style=_map_lr_scheduler_type(getattr(training_args, "lr_scheduler_type", "cosine")),
+        lr_wsd_decay_style="minus_sqrt",
+        lr_wsd_decay_iters=decay_iters,
+        lr_warmup_iters=warmup_steps,
+        lr_warmup_init=0.0,
+        lr_decay_iters=decay_iters,
+        override_opt_param_scheduler=True,
+    )
+    return optimizer, scheduler
+
+
+def ensure_create_sft_dataset_applies_chat_template() -> None:
+    r"""Apply ``dataset_kwargs['chat_template']`` before Megatron builds SFT datasets.
+
+    Megatron Bridge only pops ``chat_template`` on the packed-sequence path. The
+    non-packed finetuning path would otherwise forward it as an unexpected kwarg.
+    Patch both the defining module and the builder import binding.
+    """
+    from megatron.bridge.data.builders import finetuning_dataset as finetuning_module
+    from megatron.bridge.data.datasets import sft as sft_module
+
+    if getattr(sft_module.create_sft_dataset, "_llamafactory_chat_template_patched", False):
+        return
+
+    original = sft_module.create_sft_dataset
+
+    def create_sft_dataset(*args, **kwargs):
+        chat_template = kwargs.pop("chat_template", None)
+        tokenizer = kwargs.get("tokenizer")
+        if tokenizer is None and len(args) >= 2:
+            tokenizer = args[1]
+        if chat_template is not None and tokenizer is not None:
+            # Megatron `_chat_preprocess` may read either the wrapper or the
+            # inner HuggingFace tokenizer depending on `legacy`.
+            if hasattr(tokenizer, "chat_template"):
+                tokenizer.chat_template = chat_template
+            hf_tokenizer = getattr(tokenizer, "_tokenizer", None)
+            if hf_tokenizer is not None and hasattr(hf_tokenizer, "chat_template"):
+                hf_tokenizer.chat_template = chat_template
+        return original(*args, **kwargs)
+
+    create_sft_dataset._llamafactory_chat_template_patched = True  # type: ignore[attr-defined]
+    sft_module.create_sft_dataset = create_sft_dataset
+    finetuning_module.create_sft_dataset = create_sft_dataset
+    logger.info_rank0("Patched Megatron create_sft_dataset to apply chat_template overrides.")
+
+
+def _create_peft_config(finetuning_args: "FinetuningArguments"):
+    if finetuning_args.finetuning_type != "lora":
+        return None
+
+    from megatron.bridge.peft.lora import LoRA
+
+    default_targets = ["linear_qkv", "linear_proj", "linear_fc1", "linear_fc2"]
+    if list(finetuning_args.lora_target) != ["all"]:
+        logger.warning_rank0(
+            f"Custom lora_target {finetuning_args.lora_target} is not supported by Megatron Bridge. "
+            f"Using default Megatron target modules: {default_targets}."
+        )
+
+    return LoRA(
+        target_modules=default_targets,
+        dim=finetuning_args.lora_rank,
+        alpha=finetuning_args.lora_alpha,
+    )
+
+
+def _build_gpt_dataset_config(
+    GPTDatasetConfig,
+    dataset_path: str,
+    seq_length: int,
+    seed: int,
+    num_workers: int,
+):
+    kwargs: dict[str, Any] = {
+        "random_seed": seed,
+        "reset_attention_mask": False,
+        "reset_position_ids": False,
+        "eod_mask_loss": False,
+        "blend": ([dataset_path], 1.0),
+        "split": "100,0,0",
+        "num_workers": num_workers,
+        "data_sharding": True,
+        "dataloader_type": "single",
+    }
+    if "sequence_length" in GPTDatasetConfig.__dataclass_fields__:
+        kwargs["sequence_length"] = seq_length
+    else:
+        kwargs["seq_length"] = seq_length
+    if "num_dataset_builder_threads" in GPTDatasetConfig.__dataclass_fields__:
+        kwargs["num_dataset_builder_threads"] = 1
+    return GPTDatasetConfig(**kwargs)
+
+
+def _build_finetuning_dataset_config(
+    FinetuningDatasetConfig,
+    dataset_root: str,
+    seq_length: int,
+    seed: int,
+    num_workers: int,
+    do_validation: bool,
+    dataset_kwargs: dict[str, Any],
+    packed_sequence_specs,
+    disable_shuffling: bool = False,
+):
+    kwargs: dict[str, Any] = {
+        "dataset_root": dataset_root,
+        "seq_length": seq_length,
+        "seed": seed,
+        "num_workers": num_workers,
+        "do_validation": do_validation,
+        "do_test": False,
+        "dataset_kwargs": dataset_kwargs,
+        "packed_sequence_specs": packed_sequence_specs,
+    }
+    if "dataloader_type" in FinetuningDatasetConfig.__dataclass_fields__:
+        from .dataset_export import get_finetuning_dataloader_type
+
+        kwargs["dataloader_type"] = get_finetuning_dataloader_type(disable_shuffling=disable_shuffling)
+    return FinetuningDatasetConfig(**kwargs)
+
+
+def _has_megatron_checkpoint(output_dir: str) -> bool:
+    r"""Return whether ``output_dir`` contains a resumable Megatron checkpoint."""
+    return any(
+        os.path.isfile(os.path.join(output_dir, name))
+        for name in ("latest_checkpointed_iteration.txt", "latest_train_state.pt")
+    )
+
+
+def _should_resume_checkpoint(training_args: "TrainingArguments") -> bool:
+    r"""Resume only when a tracker exists and ``overwrite_output_dir`` is false."""
+    if getattr(training_args, "overwrite_output_dir", False):
+        return False
+    return _has_megatron_checkpoint(training_args.output_dir)
+
+
+def _create_base_config(
+    *,
+    training_args: "TrainingArguments",
+    finetuning_args: "FinetuningArguments",
+    train_iters: int,
+    micro_batch_size: int,
+    global_batch_size: int,
+    mb_args: "MegatronBridgeArguments",
+    is_sft: bool,
+):
+    from megatron.core.distributed import DistributedDataParallelConfig
+
+    (
+        CheckpointConfig,
+        ConfigContainer,
+        DistributedInitConfig,
+        _FinetuningDatasetConfig,
+        _GPTDatasetConfig,
+        LoggerConfig,
+        RNGConfig,
+        TokenizerConfig,
+        TrainingConfig,
+        ValidationConfig,
+    ) = _import_training_config()
+
+    warmup_steps = _resolve_warmup_steps(training_args, train_iters)
+    opt_cfg, scheduler_cfg = _create_optimizer_scheduler(
+        training_args=training_args,
+        warmup_steps=warmup_steps,
+        train_iters=train_iters,
+        finetuning_args=finetuning_args,
+        use_distributed_optimizer=mb_args.use_distributed_optimizer,
+    )
+
+    train_kwargs: dict[str, Any] = {
+        "train_iters": train_iters,
+        "global_batch_size": global_batch_size,
+        "micro_batch_size": micro_batch_size,
+    }
+    eval_steps = training_args.eval_steps
+    if ValidationConfig is not None:
+        validation = ValidationConfig(eval_interval=eval_steps or 100, eval_iters=32)
+    else:
+        train_kwargs["eval_interval"] = eval_steps or 100
+        train_kwargs["eval_iters"] = 32
+        validation = None
+
+    output_dir = training_args.output_dir
+    resume_checkpoint = _should_resume_checkpoint(training_args)
+    # mcore >= 0.14 removed ShardedTensor.flattened_range. The legacy default
+    # sharding type ``fully_sharded_model_space`` still depends on it, so prefer
+    # the fully_reshardable distributed-optimizer format when dist opt is on.
+    dist_ckpt_optim_fully_reshardable = mb_args.use_distributed_optimizer
+    dist_cfg = DistributedInitConfig() if DistributedInitConfig is not None else None
+    container_kwargs: dict[str, Any] = {
+        "model": None,
+        "train": TrainingConfig(**train_kwargs),
+        "optimizer": opt_cfg,
+        "scheduler": scheduler_cfg,
+        "ddp": DistributedDataParallelConfig(
+            check_for_nan_in_grad=True,
+            grad_reduce_in_fp32=True,
+            overlap_grad_reduce=mb_args.overlap_grad_reduce,
+            overlap_param_gather=mb_args.overlap_param_gather,
+            use_distributed_optimizer=mb_args.use_distributed_optimizer,
+        ),
+        "dataset": None,
+        "logger": LoggerConfig(
+            log_interval=training_args.logging_steps,
+            tensorboard_dir=os.path.join(output_dir, "tb_logs"),
+        ),
+        "tokenizer": TokenizerConfig(
+            tokenizer_type="HuggingFaceTokenizer",
+            tokenizer_model=None,
+        ),
+        "checkpoint": CheckpointConfig(
+            save_interval=training_args.save_steps,
+            save=output_dir,
+            load=output_dir if resume_checkpoint else None,
+            # SFT from pretrained should not load optimizer/RNG from a partial ckpt.
+            finetune=is_sft and not resume_checkpoint,
+            ckpt_format="torch_dist",
+            fully_parallel_save=False,
+            use_persistent_ckpt_worker=False,
+            save_optim=True,
+            dist_ckpt_optim_fully_reshardable=dist_ckpt_optim_fully_reshardable,
+        ),
+        "rng": RNGConfig(seed=training_args.seed),
+        "mixed_precision": mb_args.mixed_precision,
+        "peft": _create_peft_config(finetuning_args) if is_sft and finetuning_args.finetuning_type == "lora" else None,
+    }
+    if validation is not None:
+        container_kwargs["validation"] = validation
+    if dist_cfg is not None:
+        container_kwargs["dist"] = dist_cfg
+
+    return ConfigContainer(**container_kwargs)
+
+
+def _compute_train_schedule(
+    training_args: "TrainingArguments",
+    mb_args: "MegatronBridgeArguments",
+    num_train_samples: int,
+) -> tuple[int, int, int]:
+    micro_batch_size = training_args.per_device_train_batch_size
+    global_batch_size = micro_batch_size * training_args.gradient_accumulation_steps * training_args.world_size
+    parallel_size = (
+        mb_args.tensor_model_parallel_size
+        * mb_args.pipeline_model_parallel_size
+        * mb_args.context_parallel_size
+        * mb_args.expert_model_parallel_size
+    )
+    global_batch_size //= parallel_size
+    global_batch_size = max(global_batch_size, micro_batch_size)
+
+    max_steps = getattr(training_args, "max_steps", -1)
+    if max_steps is not None and max_steps > 0:
+        train_iters = max_steps
+    else:
+        train_iters = max(1, math.ceil(num_train_samples / global_batch_size * training_args.num_train_epochs))
+    return micro_batch_size, global_batch_size, train_iters
+
+
+def _is_apex_grad_accum_fusion_available() -> bool:
+    try:
+        import fused_weight_gradient_mlp_cuda  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
+
+
+def _apply_fusion_safety(model_provider) -> None:
+    r"""Disable gradient_accumulation_fusion when the APEX CUDA extension is missing.
+
+    Megatron Bridge enables this fusion when TransformerEngine is installed, but
+    ColumnParallelLinear (e.g. the output layer) still requires the APEX
+    fused_weight_gradient_mlp_cuda extension at model construction time.
+    """
+    if getattr(model_provider, "gradient_accumulation_fusion", False) and not _is_apex_grad_accum_fusion_available():
+        logger.warning_rank0(
+            "Disabling gradient_accumulation_fusion because the APEX CUDA extension "
+            "fused_weight_gradient_mlp_cuda is not installed."
+        )
+        model_provider.gradient_accumulation_fusion = False
+
+
+def _apply_optional_provider_attr(model_provider, name: str, value) -> None:
+    if value is None or not hasattr(model_provider, name):
+        return
+    setattr(model_provider, name, value)
+
+
+def _apply_model_parallelism(model_provider, mb_args: "MegatronBridgeArguments") -> None:
+    model_provider.tensor_model_parallel_size = mb_args.tensor_model_parallel_size
+    model_provider.pipeline_model_parallel_size = mb_args.pipeline_model_parallel_size
+    if hasattr(model_provider, "expert_model_parallel_size"):
+        model_provider.expert_model_parallel_size = mb_args.expert_model_parallel_size
+    model_provider.context_parallel_size = mb_args.context_parallel_size
+    model_provider.sequence_parallel = mb_args.sequence_parallel
+    _apply_optional_provider_attr(
+        model_provider, "virtual_pipeline_model_parallel_size", mb_args.virtual_pipeline_model_parallel_size
+    )
+    _apply_optional_provider_attr(model_provider, "recompute_granularity", mb_args.recompute_granularity)
+    _apply_optional_provider_attr(model_provider, "recompute_method", mb_args.recompute_method)
+    _apply_optional_provider_attr(model_provider, "recompute_num_layers", mb_args.recompute_num_layers)
+    _apply_optional_provider_attr(
+        model_provider, "account_for_embedding_in_pipeline_split", mb_args.account_for_embedding_in_pipeline_split
+    )
+    _apply_optional_provider_attr(
+        model_provider, "account_for_loss_in_pipeline_split", mb_args.account_for_loss_in_pipeline_split
+    )
+    _apply_optional_provider_attr(model_provider, "bias_activation_fusion", mb_args.bias_activation_fusion)
+    _apply_optional_provider_attr(model_provider, "apply_rope_fusion", mb_args.apply_rope_fusion)
+    _apply_optional_provider_attr(model_provider, "masked_softmax_fusion", mb_args.masked_softmax_fusion)
+    _apply_optional_provider_attr(model_provider, "cross_entropy_loss_fusion", mb_args.cross_entropy_loss_fusion)
+    _apply_optional_provider_attr(model_provider, "moe_grouped_gemm", mb_args.moe_grouped_gemm)
+    _apply_optional_provider_attr(model_provider, "moe_token_dispatcher_type", mb_args.moe_token_dispatcher_type)
+    _apply_optional_provider_attr(model_provider, "calculate_per_token_loss", mb_args.calculate_per_token_loss)
+
+
+def _apply_context_parallel_finetuning_requirements(cfg, mb_args: "MegatronBridgeArguments") -> None:
+    r"""Apply Megatron Bridge SFT requirements when context parallelism is enabled."""
+    if mb_args.context_parallel_size <= 1:
+        return
+    cfg.model.calculate_per_token_loss = True
+    cfg.ddp.average_in_collective = False
+
+
+def _apply_extra_overrides(cfg, extra: dict) -> None:
+    for key, value in extra.items():
+        parts = key.split(".")
+        obj = cfg
+        for part in parts[:-1]:
+            obj = getattr(obj, part)
+        setattr(obj, parts[-1], value)
+
+
+def _reset_megatron_bridge_global_state_after_checkpoint_conversion() -> None:
+    r"""Clear Megatron globals left over from HF-to-Megatron conversion.
+
+    save_megatron_model() can implicitly initialize the rerun state machine while
+    writing checkpoints. provide_distributed_model() also initializes model parallel
+    groups. Training later calls initialize_megatron(), which expects a fresh global
+    state and fails with "Rerun state machine is already initialized" or keeps stale
+    parallel groups that mismatch the configured tensor/pipeline/context parallel sizes.
+    """
+    from megatron.core import parallel_state
+    from megatron.core.rerun_state_machine import destroy_rerun_state_machine
+
+    destroy_rerun_state_machine()
+    parallel_state.destroy_model_parallel()
+
+
+def ensure_megatron_pretrained_checkpoint(
+    model_args: "ModelArguments",
+    mb_args: "MegatronBridgeArguments",
+    output_dir: str,
+) -> str:
+    r"""Convert Hugging Face weights to Megatron format when needed."""
+    from megatron.bridge import AutoBridge
+
+    if mb_args.megatron_pretrained_checkpoint and os.path.isdir(mb_args.megatron_pretrained_checkpoint):
+        return mb_args.megatron_pretrained_checkpoint
+
+    ckpt_dir = os.path.join(output_dir, "megatron_pretrained")
+    if os.path.isdir(ckpt_dir) and os.listdir(ckpt_dir):
+        logger.info_rank0(f"Reusing existing Megatron checkpoint at {ckpt_dir}.")
+        return ckpt_dir
+
+    os.makedirs(ckpt_dir, exist_ok=True)
+    logger.info_rank0(f"Converting Hugging Face weights to Megatron format at {ckpt_dir}...")
+    bridge = AutoBridge.from_hf_pretrained(
+        model_args.model_name_or_path,
+        trust_remote_code=model_args.trust_remote_code,
+    )
+    provider = bridge.to_megatron_provider()
+    _apply_model_parallelism(provider, mb_args)
+    _apply_fusion_safety(provider)
+    if hasattr(provider, "finalize"):
+        provider.finalize()
+    # TP/PP/CP weight scatter uses NCCL, which cannot operate on CPU tensors.
+    use_cpu_initialization = (
+        mb_args.tensor_model_parallel_size == 1
+        and mb_args.pipeline_model_parallel_size == 1
+        and mb_args.context_parallel_size == 1
+    )
+    if not use_cpu_initialization:
+        logger.info_rank0(
+            "Using GPU initialization for Megatron checkpoint conversion because model parallelism "
+            "requires NCCL scatter/gather on CUDA tensors."
+        )
+    try:
+        megatron_model = provider.provide_distributed_model(
+            wrap_with_ddp=False,
+            use_cpu_initialization=use_cpu_initialization,
+        )
+        hf_tokenizer_kwargs = {"trust_remote_code": True} if model_args.trust_remote_code else None
+        bridge.save_megatron_model(
+            megatron_model,
+            ckpt_dir,
+            hf_tokenizer_path=model_args.model_name_or_path,
+            hf_tokenizer_kwargs=hf_tokenizer_kwargs,
+            low_memory_save=True,
+        )
+    finally:
+        _reset_megatron_bridge_global_state_after_checkpoint_conversion()
+    return ckpt_dir
+
+
+def build_pretrain_config(
+    model_args: "ModelArguments",
+    data_args: "DataArguments",
+    training_args: "TrainingArguments",
+    finetuning_args: "FinetuningArguments",
+    mb_args: "MegatronBridgeArguments",
+    dataset_path: str,
+    num_train_samples: int,
+):
+    from megatron.bridge import AutoBridge
+
+    micro_batch_size, global_batch_size, train_iters = _compute_train_schedule(
+        training_args, mb_args, num_train_samples
+    )
+    cfg = _create_base_config(
+        training_args=training_args,
+        finetuning_args=finetuning_args,
+        train_iters=train_iters,
+        micro_batch_size=micro_batch_size,
+        global_batch_size=global_batch_size,
+        mb_args=mb_args,
+        is_sft=False,
+    )
+
+    (
+        _CheckpointConfig,
+        _ConfigContainer,
+        _DistributedInitConfig,
+        _FinetuningDatasetConfig,
+        GPTDatasetConfig,
+        _LoggerConfig,
+        _RNGConfig,
+        _TokenizerConfig,
+        _TrainingConfig,
+        _ValidationConfig,
+    ) = _import_training_config()
+    bridge = AutoBridge.from_hf_pretrained(
+        model_args.model_name_or_path,
+        trust_remote_code=model_args.trust_remote_code,
+    )
+    cfg.model = bridge.to_megatron_provider(load_weights=False)
+    _apply_model_parallelism(cfg.model, mb_args)
+    _apply_fusion_safety(cfg.model)
+    if hasattr(cfg.model, "seq_length"):
+        cfg.model.seq_length = data_args.cutoff_len
+
+    cfg.tokenizer.tokenizer_model = model_args.model_name_or_path
+    cfg.dataset = _build_gpt_dataset_config(
+        GPTDatasetConfig,
+        dataset_path=dataset_path,
+        seq_length=data_args.cutoff_len,
+        seed=training_args.seed,
+        num_workers=data_args.preprocessing_num_workers,
+    )
+
+    _apply_extra_overrides(cfg, mb_args.load_extra_config())
+    return cfg
+
+
+def build_sft_config(
+    model_args: "ModelArguments",
+    data_args: "DataArguments",
+    training_args: "TrainingArguments",
+    finetuning_args: "FinetuningArguments",
+    mb_args: "MegatronBridgeArguments",
+    dataset_root: str,
+    pretrained_checkpoint: str,
+    num_train_samples: int,
+):
+    from megatron.bridge import AutoBridge
+    from megatron.bridge.data.datasets.packed_sequence import PackedSequenceSpecs
+
+    micro_batch_size, global_batch_size, train_iters = _compute_train_schedule(
+        training_args, mb_args, num_train_samples
+    )
+    cfg = _create_base_config(
+        training_args=training_args,
+        finetuning_args=finetuning_args,
+        train_iters=train_iters,
+        micro_batch_size=micro_batch_size,
+        global_batch_size=global_batch_size,
+        mb_args=mb_args,
+        is_sft=True,
+    )
+
+    (
+        _CheckpointConfig,
+        _ConfigContainer,
+        _DistributedInitConfig,
+        FinetuningDatasetConfig,
+        _GPTDatasetConfig,
+        _LoggerConfig,
+        _RNGConfig,
+        _TokenizerConfig,
+        _TrainingConfig,
+        _ValidationConfig,
+    ) = _import_training_config()
+    bridge = AutoBridge.from_hf_pretrained(
+        model_args.model_name_or_path,
+        trust_remote_code=model_args.trust_remote_code,
+    )
+    cfg.model = bridge.to_megatron_provider(load_weights=False)
+    _apply_model_parallelism(cfg.model, mb_args)
+    _apply_fusion_safety(cfg.model)
+    if hasattr(cfg.model, "seq_length"):
+        cfg.model.seq_length = data_args.cutoff_len
+
+    cfg.tokenizer.tokenizer_model = model_args.model_name_or_path
+
+    from .dataset_export import get_sft_dataset_kwargs
+
+    ensure_create_sft_dataset_applies_chat_template()
+    dataset_kwargs = get_sft_dataset_kwargs(
+        tokenizer_path=model_args.model_name_or_path,
+        trust_remote_code=model_args.trust_remote_code,
+        template_name=data_args.template,
+    )
+    packed_sequence_specs = None
+    if mb_args.use_packed_sequences:
+        pad_seq_to_mult = mb_args.context_parallel_size * 2 if mb_args.context_parallel_size > 1 else 1
+        packed_sequence_specs = PackedSequenceSpecs(
+            packed_sequence_size=data_args.cutoff_len,
+            pad_seq_to_mult=pad_seq_to_mult,
+        )
+        dataset_kwargs["pad_to_max_length"] = True
+
+    cfg.dataset = _build_finetuning_dataset_config(
+        FinetuningDatasetConfig,
+        dataset_root=dataset_root,
+        seq_length=data_args.cutoff_len,
+        seed=training_args.seed,
+        num_workers=data_args.preprocessing_num_workers,
+        do_validation=data_args.val_size > 0 or data_args.eval_dataset is not None,
+        dataset_kwargs=dataset_kwargs,
+        packed_sequence_specs=packed_sequence_specs,
+        disable_shuffling=getattr(finetuning_args, "disable_shuffling", False),
+    )
+    cfg.checkpoint.pretrained_checkpoint = pretrained_checkpoint
+
+    _apply_context_parallel_finetuning_requirements(cfg, mb_args)
+    _apply_extra_overrides(cfg, mb_args.load_extra_config())
+    return cfg

src/llamafactory/train/megatron_bridge/dataset_export.py

@@ -0,0 +1,344 @@
+# Copyright 2025 the LlamaFactory team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+import json
+import os
+import re
+import typing
+from typing import TYPE_CHECKING, Any, Optional
+
+from ...extras.logging import get_logger
+
+
+if TYPE_CHECKING:
+    from datasets import Dataset, IterableDataset
+
+
+logger = get_logger(__name__)
+_GENERATION_REGEX = re.compile(r"\{%-?\s+generation\s+-?%\}")
+_END_GENERATION_REGEX = re.compile(r"\{%-?\s+endgeneration\s+-?%\}")
+_ASSISTANT_ELIF_REGEX = re.compile(
+    r"(\{%\s*elif\s+message\['role'\]\s*==\s*'assistant'\s*%\})"
+    r"(.*?)"
+    r"(\{%\s*endif\s*%\})",
+    flags=re.DOTALL,
+)
+
+
+def supports_hf_chat_template() -> bool:
+    r"""Return whether the installed Megatron Bridge supports HF chat templates."""
+    try:
+        from megatron.bridge.data.datasets.sft import GPTSFTChatDataset
+
+        return "use_hf_tokenizer_chat_template" in inspect.signature(GPTSFTChatDataset.__init__).parameters
+    except Exception:
+        return False
+
+
+def _inject_generation_block(chat_template: str) -> str:
+    r"""Wrap assistant content with ``{% generation %}`` when missing."""
+    if _GENERATION_REGEX.search(chat_template):
+        return chat_template
+
+    match = _ASSISTANT_ELIF_REGEX.search(chat_template)
+    if match is None:
+        raise ValueError(
+            "Cannot inject {% generation %} into chat template: "
+            "no `{% elif message['role'] == 'assistant' %}` block found."
+        )
+
+    body = match.group(2)
+    if _END_GENERATION_REGEX.search(body):
+        return chat_template
+
+    patched = (
+        chat_template[: match.start()]
+        + match.group(1)
+        + "{% generation %}"
+        + body
+        + "{% endgeneration %}"
+        + match.group(3)
+        + chat_template[match.end() :]
+    )
+    if not _GENERATION_REGEX.search(patched):
+        raise ValueError("Failed to inject {% generation %} into chat template.")
+    return patched
+
+
+def build_chat_template_with_generation(
+    tokenizer_path: str | None,
+    trust_remote_code: bool = False,
+    template_name: str | None = None,
+) -> str | None:
+    r"""Build a chat template that supports assistant-only loss masks.
+
+    Prefers the LLaMA-Factory registered template (same formatting as HF Trainer),
+    then falls back to patching the tokenizer's native chat template.
+    """
+    if not tokenizer_path:
+        return None
+
+    try:
+        from transformers import AutoTokenizer
+
+        tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=trust_remote_code)
+    except Exception as exc:
+        logger.warning_rank0(f"Failed to load tokenizer for chat template patching: {exc}")
+        return None
+
+    if template_name:
+        try:
+            from ...data.template import TEMPLATES
+
+            template = TEMPLATES.get(template_name)
+            if template is not None:
+                return _inject_generation_block(template._get_jinja_template(tokenizer))
+        except Exception as exc:
+            logger.warning_rank0(f"Failed to build LLaMA-Factory chat template '{template_name}': {exc}")
+
+    native = tokenizer.chat_template
+    if not isinstance(native, str) or not native:
+        return None
+
+    try:
+        return _inject_generation_block(native)
+    except Exception as exc:
+        logger.warning_rank0(f"Failed to inject {{% generation %}} into native chat template: {exc}")
+        return None
+
+
+def tokenizer_supports_hf_chat_template(
+    tokenizer_path: str | None,
+    trust_remote_code: bool = False,
+    template_name: str | None = None,
+) -> bool:
+    r"""Return whether Megatron Bridge can use HF chat templates for this tokenizer."""
+    if not supports_hf_chat_template() or not tokenizer_path:
+        return False
+
+    try:
+        from transformers import AutoTokenizer
+
+        tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=trust_remote_code)
+        if _GENERATION_REGEX.search(tokenizer.chat_template or ""):
+            return True
+    except Exception as exc:
+        logger.warning_rank0(f"Failed to inspect tokenizer chat template: {exc}")
+        return False
+
+    return build_chat_template_with_generation(tokenizer_path, trust_remote_code, template_name) is not None
+
+
+def get_sft_dataset_kwargs(
+    tokenizer_path: str | None = None,
+    trust_remote_code: bool = False,
+    template_name: str | None = None,
+) -> dict[str, Any]:
+    r"""Return dataset kwargs compatible with the installed Megatron Bridge version."""
+    kwargs: dict[str, Any] = {"chat": True}
+    if not tokenizer_supports_hf_chat_template(tokenizer_path, trust_remote_code, template_name):
+        return kwargs
+
+    chat_template = build_chat_template_with_generation(tokenizer_path, trust_remote_code, template_name)
+    if chat_template is None:
+        return kwargs
+
+    kwargs["use_hf_tokenizer_chat_template"] = True
+    kwargs["chat_template"] = chat_template
+    # Chat templates already include BOS/EOS / turn markers.
+    kwargs["add_bos"] = False
+    kwargs["add_eos"] = False
+    logger.info_rank0(
+        "Using HuggingFace chat template with {% generation %} for Megatron Bridge SFT "
+        f"(template={template_name or 'native'})."
+    )
+    return kwargs
+
+
+def get_finetuning_dataloader_type(disable_shuffling: bool = False) -> str:
+    r"""Return a finetuning dataloader type supported by the installed Megatron Bridge.
+
+    Prefer sequential ``batch``/``single`` samplers. When shuffling is disabled for
+    HF loss comparison, avoid the random ``cyclic`` sampler.
+    """
+    try:
+        from megatron.bridge.training.config import FinetuningDatasetConfig
+
+        field = FinetuningDatasetConfig.__dataclass_fields__.get("dataloader_type")
+        if field is None:
+            return "single"
+
+        choices: set[str] = set()
+        for arg in typing.get_args(field.type):
+            for choice in typing.get_args(arg):
+                if isinstance(choice, str):
+                    choices.add(choice)
+
+        if disable_shuffling:
+            if "batch" in choices:
+                return "batch"
+            if "single" in choices:
+                return "single"
+            return next(iter(choices), "single")
+
+        if "batch" in choices:
+            return "batch"
+        if "single" in choices:
+            return "single"
+        return next(iter(choices), "single")
+    except Exception:
+        return "single"
+
+
+def _role_to_sharegpt(role: str) -> str:
+    mapping = {"user": "User", "assistant": "Assistant", "system": "System"}
+    return mapping.get(role, role.capitalize())
+
+
+def _example_to_record(
+    example: dict[str, Any], stage: str = "sft", use_messages_format: bool = True
+) -> dict[str, Any] | None:
+    r"""Convert an aligned LLaMA-Factory example to Megatron Bridge JSONL format."""
+    if example.get("text") is not None:
+        return {"text": example["text"]}
+
+    prompt = example.get("_prompt")
+    if stage == "pt":
+        if not prompt:
+            return None
+        return {"text": prompt[0]["content"]}
+
+    response = example.get("_response")
+    if not prompt or not response:
+        return None
+
+    if use_messages_format:
+        messages = []
+        system = example.get("_system")
+        if system:
+            messages.append({"role": "system", "content": system})
+        for message in prompt:
+            messages.append({"role": message["role"], "content": message["content"]})
+        for message in response:
+            messages.append({"role": message["role"], "content": message["content"]})
+        record: dict[str, Any] = {"messages": messages}
+        tools = example.get("_tools")
+        if tools:
+            record["tools"] = tools
+        return record
+
+    conversations = []
+    for message in prompt:
+        conversations.append({"from": _role_to_sharegpt(message["role"]), "value": message["content"]})
+    for message in response:
+        conversations.append({"from": _role_to_sharegpt(message["role"]), "value": message["content"]})
+
+    return {
+        "system": example.get("_system") or "",
+        "conversations": conversations,
+        "mask": "User",
+    }
+
+
+def _remove_stale_memmap_index(path: str) -> None:
+    r"""Remove cached memmap index files after rewriting a JSONL dataset."""
+    for suffix in (".idx.npy", ".idx.info"):
+        index_path = path + suffix
+        if os.path.exists(index_path):
+            os.remove(index_path)
+            logger.info_rank0(f"Removed stale Megatron dataset index: {index_path}")
+
+
+def _write_jsonl(
+    path: str,
+    dataset: "Dataset | IterableDataset",
+    stage: str = "sft",
+    use_messages_format: bool = True,
+) -> int:
+    count = 0
+    parent = os.path.dirname(path)
+    if parent:
+        os.makedirs(parent, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        for example in dataset:
+            record = _example_to_record(example, stage=stage, use_messages_format=use_messages_format)
+            if record is None:
+                continue
+            f.write(json.dumps(record, ensure_ascii=False) + "\n")
+            count += 1
+    _remove_stale_memmap_index(path)
+    return count
+
+
+def export_dataset_for_megatron_bridge(
+    train_dataset: "Dataset | IterableDataset",
+    output_dir: str,
+    eval_dataset: Optional["Dataset | IterableDataset | dict[str, Dataset]"] = None,
+    val_size: float = 0.0,
+    seed: int = 42,
+    stage: str = "sft",
+    model_name_or_path: str | None = None,
+    trust_remote_code: bool = False,
+    template_name: str | None = None,
+) -> str:
+    r"""Export aligned LLaMA-Factory datasets to Megatron Bridge JSONL files."""
+    os.makedirs(output_dir, exist_ok=True)
+    use_messages_format = stage != "sft" or tokenizer_supports_hf_chat_template(
+        model_name_or_path,
+        trust_remote_code=trust_remote_code,
+        template_name=template_name,
+    )
+    if stage == "sft" and not use_messages_format:
+        logger.info_rank0(
+            "Cannot enable HuggingFace chat template for Megatron Bridge; "
+            "exporting ShareGPT conversations for legacy preprocessing."
+        )
+
+    if val_size > 0 and eval_dataset is None:
+        split = train_dataset.train_test_split(test_size=val_size, seed=seed)
+        train_dataset = split["train"]
+        eval_dataset = split["test"]
+
+    train_path = os.path.join(output_dir, "training.jsonl")
+    train_count = _write_jsonl(
+        train_path,
+        train_dataset,
+        stage=stage,
+        use_messages_format=use_messages_format,
+    )
+    logger.info_rank0(f"Exported {train_count} training samples to {train_path}.")
+
+    if isinstance(eval_dataset, dict):
+        for name, dataset in eval_dataset.items():
+            split_name = "validation" if name == "validation" else name
+            eval_path = os.path.join(output_dir, f"{split_name}.jsonl")
+            eval_count = _write_jsonl(
+                eval_path,
+                dataset,
+                stage=stage,
+                use_messages_format=use_messages_format,
+            )
+            logger.info_rank0(f"Exported {eval_count} {split_name} samples to {eval_path}.")
+    elif eval_dataset is not None:
+        eval_path = os.path.join(output_dir, "validation.jsonl")
+        eval_count = _write_jsonl(
+            eval_path,
+            eval_dataset,
+            stage=stage,
+            use_messages_format=use_messages_format,
+        )
+        logger.info_rank0(f"Exported {eval_count} validation samples to {eval_path}.")
+
+    return output_dir

src/llamafactory/train/megatron_bridge/workflow.py

@@ -0,0 +1,379 @@
+# Copyright 2025 the LlamaFactory team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Optional
+
+from transformers import AutoConfig as HfAutoConfig
+
+from ...data.data_utils import split_dataset
+from ...data.loader import _get_merged_dataset
+from ...extras.constants import MEGATRON_BRIDGE_SUPPORTED_MODELS
+from ...extras.logging import get_logger
+from ...extras.packages import is_megatron_bridge_available
+from .config_builder import (
+    _apply_fusion_safety,
+    build_pretrain_config,
+    build_sft_config,
+    ensure_megatron_pretrained_checkpoint,
+)
+from .dataset_export import export_dataset_for_megatron_bridge
+
+
+if TYPE_CHECKING:
+    from transformers import TrainerCallback
+
+    from ...hparams import (
+        DataArguments,
+        FinetuningArguments,
+        MegatronBridgeArguments,
+        ModelArguments,
+        TrainingArguments,
+    )
+
+
+logger = get_logger(__name__)
+
+
+def _check_model_support(model_args: "ModelArguments") -> None:
+    r"""Ensure the HF ``model_type`` is covered by the Megatron Bridge PT/SFT path."""
+    config = HfAutoConfig.from_pretrained(
+        model_args.model_name_or_path, trust_remote_code=model_args.trust_remote_code
+    )
+    model_type = getattr(config, "model_type", None)
+    if model_type not in MEGATRON_BRIDGE_SUPPORTED_MODELS:
+        raise ValueError(
+            f"Model type `{model_type}` is not supported by the Megatron Bridge PT/SFT path. "
+            f"Supported model types: {sorted(MEGATRON_BRIDGE_SUPPORTED_MODELS)}. "
+            "Multimodal / audio / omni models are not enabled in v0."
+        )
+
+
+def _run_on_main_process(training_args: "TrainingArguments", work: Callable[[], None], sync_dir: str) -> None:
+    r"""Run ``work`` only on global rank 0, then synchronize other ranks.
+
+    Prefer ``torch.distributed.barrier`` when the process group is already initialized;
+    otherwise fall back to a file flag under ``sync_dir`` so non-main ranks wait for
+    shared filesystem writes (e.g. dataset export) to finish.
+    """
+    done_file = os.path.join(sync_dir, ".main_process_done")
+    is_main = getattr(training_args, "process_index", 0) == 0
+    wait_start = time.time()
+
+    import torch.distributed as dist
+
+    dist_ready = dist.is_available() and dist.is_initialized()
+    if is_main:
+        os.makedirs(sync_dir, exist_ok=True)
+        if os.path.isfile(done_file):
+            os.remove(done_file)
+        work()
+        with open(done_file, "w", encoding="utf-8") as f:
+            f.write("done")
+
+    if dist_ready:
+        dist.barrier()
+    elif not is_main:
+        while True:
+            if os.path.isfile(done_file) and os.path.getmtime(done_file) >= wait_start - 1.0:
+                break
+            time.sleep(0.5)
+
+
+def _check_backend_available() -> None:
+    if not is_megatron_bridge_available():
+        raise ImportError(
+            "megatron-bridge is not installed. "
+            "Please install it with `pip install --no-build-isolation megatron-bridge` "
+            "or use the NeMo Framework container."
+        )
+    _patch_dataset_helper_compilation()
+    _patch_dist_checkpoint_preload()
+
+
+def _patch_dist_checkpoint_preload() -> None:
+    r"""Use blocking GPU->CPU copies when saving distributed checkpoints.
+
+    Megatron's default ``non_blocking=True`` preload can raise ``cudaErrorInvalidValue``
+    on some GPUs (e.g. V100) when saving distributed optimizer shards, because pinned
+    host memory allocation or async D2H transfer may fail under memory pressure.
+    """
+    from megatron.core.dist_checkpointing.strategies import filesystem_async
+
+    if getattr(filesystem_async.FileSystemWriterAsync.preload_tensors, "_llamafactory_patched", False):
+        return
+
+    original_preload = filesystem_async.FileSystemWriterAsync.preload_tensors
+
+    @staticmethod
+    def preload_tensors(write_buckets, non_blocking=True):
+        return original_preload(write_buckets, non_blocking=False)
+
+    preload_tensors._llamafactory_patched = True
+    filesystem_async.FileSystemWriterAsync.preload_tensors = preload_tensors
+    logger.info_rank0("Patched Megatron dist checkpoint preload to use blocking GPU->CPU copies.")
+
+
+def _patch_dataset_helper_compilation() -> None:
+    r"""Skip make-based helper compilation when the pybind extension is prebuilt.
+
+    Pip-installed megatron-core already ships helpers_cpp, but compile_helpers()
+    still invokes make and fails when no Makefile is present.
+    """
+    from megatron.core.datasets import utils as dataset_utils
+
+    if getattr(dataset_utils.compile_helpers, "_llamafactory_patched", False):
+        return
+
+    try:
+        import megatron.core.datasets.helpers_cpp  # noqa: F401
+    except ImportError:
+        return
+
+    def compile_helpers():
+        import megatron.core.datasets.helpers_cpp  # noqa: F401
+
+    compile_helpers._llamafactory_patched = True
+    dataset_utils.compile_helpers = compile_helpers
+    logger.info_rank0("Using prebuilt megatron.core.datasets.helpers_cpp; skipping make compilation.")
+
+
+def _load_aligned_datasets(
+    model_args: "ModelArguments",
+    data_args: "DataArguments",
+    training_args: "TrainingArguments",
+    stage: str,
+):
+    dataset = _get_merged_dataset(data_args.dataset, model_args, data_args, training_args, stage)
+    eval_dataset = _get_merged_dataset(
+        data_args.eval_dataset,
+        model_args,
+        data_args,
+        training_args,
+        stage,
+        return_dict=data_args.eval_on_each_dataset,
+    )
+    train_dict, eval_dict = split_dataset(dataset, eval_dataset, data_args, seed=training_args.seed)
+    return train_dict.get("train"), eval_dict
+
+
+def _latest_iter_checkpoint_dir(output_dir: str) -> Optional[str]:
+    r"""Return the latest ``iter_*`` directory under ``output_dir``, if any.
+
+    ``export_adapter_ckpt`` needs the iteration directory that holds the
+    distributed checkpoint payload (``.distcp`` / ``run_config.yaml``), not the
+    parent run directory.
+    """
+    if not os.path.isdir(output_dir):
+        return None
+
+    # Already pointing at an iteration directory.
+    if os.path.isfile(os.path.join(output_dir, "run_config.yaml")) or os.path.exists(
+        os.path.join(output_dir, ".metadata")
+    ):
+        return output_dir
+
+    iter_dirs = [
+        name
+        for name in os.listdir(output_dir)
+        if name.startswith("iter_") and os.path.isdir(os.path.join(output_dir, name))
+    ]
+    if not iter_dirs:
+        return None
+
+    def _iter_number(name: str) -> int:
+        try:
+            return int(name.replace("iter_", ""))
+        except ValueError:
+            return -1
+
+    latest = max(iter_dirs, key=_iter_number)
+    return os.path.join(output_dir, latest)
+
+
+def _checkpoint_uses_peft(checkpoint_dir: str) -> bool:
+    r"""Whether the Megatron checkpoint was saved with a PEFT (e.g. LoRA) config."""
+    cfg_path = os.path.join(checkpoint_dir, "run_config.yaml")
+    if not os.path.isfile(cfg_path):
+        return False
+
+    try:
+        import yaml
+
+        with open(cfg_path, encoding="utf-8") as f:
+            cfg = yaml.safe_load(f) or {}
+        return isinstance(cfg, dict) and bool(cfg.get("peft"))
+    except Exception:
+        return False
+
+
+def _with_fusion_safe_provider(bridge):
+    r"""Wrap ``to_megatron_provider`` so export paths disable missing APEX fusion.
+
+    Training already applies ``_apply_fusion_safety``, but AutoBridge export helpers
+    (e.g. ``export_adapter_ckpt``) rebuild a provider with the default
+    ``gradient_accumulation_fusion=True`` whenever TransformerEngine is importable,
+    even if ``fused_weight_gradient_mlp_cuda`` is absent.
+    """
+    original = bridge.to_megatron_provider
+
+    def _to_megatron_provider(*args, **kwargs):
+        provider = original(*args, **kwargs)
+        _apply_fusion_safety(provider)
+        return provider
+
+    bridge.to_megatron_provider = _to_megatron_provider  # type: ignore[method-assign]
+    return bridge
+
+
+def _maybe_export_hf_checkpoint(
+    model_args: "ModelArguments",
+    mb_args: "MegatronBridgeArguments",
+    output_dir: str,
+) -> None:
+    if not mb_args.export_hf_on_finish or not training_args_should_save(output_dir):
+        return
+
+    import torch.distributed as dist
+    from megatron.bridge import AutoBridge
+
+    checkpoint_dir = _latest_iter_checkpoint_dir(output_dir)
+    if checkpoint_dir is None:
+        logger.warning_rank0(f"No Megatron iteration checkpoint found under {output_dir}; skip HF export.")
+        return
+
+    export_dir = os.path.join(output_dir, "hf_export")
+    bridge = _with_fusion_safe_provider(
+        AutoBridge.from_hf_pretrained(
+            model_args.model_name_or_path,
+            trust_remote_code=model_args.trust_remote_code,
+        )
+    )
+
+    # LoRA / PEFT checkpoints only store adapter weights. Loading them as a full
+    # model raises KeyError for base tensors such as linear_proj.weight.
+    if _checkpoint_uses_peft(checkpoint_dir):
+        logger.info_rank0(f"Exporting LoRA adapter to Hugging Face PEFT format at {export_dir}...")
+        bridge.export_adapter_ckpt(peft_checkpoint=checkpoint_dir, output_path=export_dir)
+        return
+
+    logger.info_rank0(f"Exporting Megatron checkpoint to Hugging Face format at {export_dir}...")
+    if dist.is_initialized():
+        # export_ckpt() always creates a fresh single-process gloo group, which fails
+        # when torchrun has already initialized NCCL for training.
+        megatron_model = bridge.load_megatron_model(output_dir)
+        bridge.save_hf_pretrained(megatron_model, export_dir)
+    else:
+        bridge.export_ckpt(megatron_path=output_dir, hf_path=export_dir)
+
+
+def training_args_should_save(output_dir: str) -> bool:
+    return os.path.isdir(output_dir) and bool(os.listdir(output_dir))
+
+
+def run_pt(
+    model_args: "ModelArguments",
+    data_args: "DataArguments",
+    training_args: "TrainingArguments",
+    finetuning_args: "FinetuningArguments",
+    mb_args: "MegatronBridgeArguments",
+    callbacks: Optional[list["TrainerCallback"]] = None,
+):
+    if callbacks:
+        logger.warning_rank0("Megatron Bridge does not support Trainer callbacks yet; ignoring provided callbacks.")
+    _check_backend_available()
+    _check_model_support(model_args)
+    from megatron.bridge.training.gpt_step import forward_step
+    from megatron.bridge.training.pretrain import pretrain
+
+    train_dataset, eval_dict = _load_aligned_datasets(model_args, data_args, training_args, "pt")
+    dataset_dir = os.path.join(training_args.output_dir, "mb_dataset")
+
+    def _export_pt_dataset() -> None:
+        export_dataset_for_megatron_bridge(
+            train_dataset=train_dataset,
+            output_dir=dataset_dir,
+            eval_dataset=eval_dict.get("validation") if eval_dict else None,
+            val_size=data_args.val_size,
+            seed=training_args.seed,
+            stage="pt",
+        )
+
+    _run_on_main_process(training_args, _export_pt_dataset, dataset_dir)
+
+    cfg = build_pretrain_config(
+        model_args=model_args,
+        data_args=data_args,
+        training_args=training_args,
+        finetuning_args=finetuning_args,
+        mb_args=mb_args,
+        dataset_path=os.path.join(dataset_dir, "training.jsonl"),
+        num_train_samples=len(train_dataset),
+    )
+    pretrain(cfg, forward_step)
+    _maybe_export_hf_checkpoint(model_args, mb_args, training_args.output_dir)
+
+
+def run_sft(
+    model_args: "ModelArguments",
+    data_args: "DataArguments",
+    training_args: "TrainingArguments",
+    finetuning_args: "FinetuningArguments",
+    mb_args: "MegatronBridgeArguments",
+    callbacks: Optional[list["TrainerCallback"]] = None,
+):
+    if callbacks:
+        logger.warning_rank0("Megatron Bridge does not support Trainer callbacks yet; ignoring provided callbacks.")
+    _check_backend_available()
+    _check_model_support(model_args)
+    from megatron.bridge.training.finetune import finetune
+    from megatron.bridge.training.gpt_step import forward_step
+
+    train_dataset, eval_dict = _load_aligned_datasets(model_args, data_args, training_args, "sft")
+    dataset_dir = os.path.join(training_args.output_dir, "mb_dataset")
+
+    def _export_sft_dataset() -> None:
+        export_dataset_for_megatron_bridge(
+            train_dataset=train_dataset,
+            output_dir=dataset_dir,
+            eval_dataset=eval_dict or None,
+            val_size=data_args.val_size if not eval_dict else 0.0,
+            seed=training_args.seed,
+            stage="sft",
+            model_name_or_path=model_args.model_name_or_path,
+            trust_remote_code=model_args.trust_remote_code,
+            template_name=data_args.template,
+        )
+
+    _run_on_main_process(training_args, _export_sft_dataset, dataset_dir)
+
+    pretrained_checkpoint = ensure_megatron_pretrained_checkpoint(
+        model_args=model_args,
+        mb_args=mb_args,
+        output_dir=training_args.output_dir,
+    )
+    cfg = build_sft_config(
+        model_args=model_args,
+        data_args=data_args,
+        training_args=training_args,
+        finetuning_args=finetuning_args,
+        mb_args=mb_args,
+        dataset_root=dataset_dir,
+        pretrained_checkpoint=pretrained_checkpoint,
+        num_train_samples=len(train_dataset),
+    )
+    finetune(cfg, forward_step_func=forward_step)
+    _maybe_export_hf_checkpoint(model_args, mb_args, training_args.output_dir)

src/llamafactory/train/tuner.py

@@ -27,6 +27,7 @@ from ..extras.misc import find_available_port, get_device_name, get_torch_device
 from ..extras.packages import (
     is_hyper_parallel_available,
     is_mcore_adapter_available,
+    is_megatron_bridge_available,
     is_ray_available,
     is_transformers_version_greater_than,
 )
@@ -90,9 +91,7 @@ def _training_function(config: dict[str, Any]) -> None:
 
     if finetuning_args.stage in ["pt", "sft"] and finetuning_args.use_hyper_parallel:
         if not is_hyper_parallel_available():
-            raise ImportError(
+            raise ImportError("hyper_parallel is not installed. Please install it with `pip install hyper_parallel`.")
-                "hyper_parallel is not installed. Please install it with `pip install hyper_parallel`."
-            )
         if finetuning_args.stage == "pt":
             from .hyper_parallel import run_pt as run_pt_hp
 
@@ -102,6 +101,24 @@ def _training_function(config: dict[str, Any]) -> None:
 
             run_sft_hp(model_args, data_args, training_args, finetuning_args, generating_args, callbacks)
 
+    elif finetuning_args.stage in ["pt", "sft"] and finetuning_args.use_megatron_bridge:
+        if not is_megatron_bridge_available():
+            raise ImportError(
+                "megatron-bridge is not installed. "
+                "Please install it with `pip install --no-build-isolation megatron-bridge`."
+            )
+        mb_args = finetuning_args.megatron_bridge_args
+        if mb_args is None:
+            raise ValueError("Megatron Bridge arguments are missing. Please set USE_MEGATRON_BRIDGE=1.")
+        if finetuning_args.stage == "pt":
+            from .megatron_bridge import run_pt as run_pt_mb
+
+            run_pt_mb(model_args, data_args, training_args, finetuning_args, mb_args, callbacks)
+        else:
+            from .megatron_bridge import run_sft as run_sft_mb
+
+            run_sft_mb(model_args, data_args, training_args, finetuning_args, mb_args, callbacks)
+
     elif finetuning_args.stage in ["pt", "sft", "dpo"] and finetuning_args.use_mca:
         if not is_mcore_adapter_available():
             raise ImportError("mcore_adapter is not installed. Please install it with `pip install mcore-adapter`.")

src/llamafactory/v1/accelerator/interface.py

@@ -29,16 +29,20 @@ And data parallelism types:
 from dataclasses import dataclass
 from datetime import timedelta
 from enum import StrEnum
-from typing import Any, Optional
+from typing import TYPE_CHECKING, Any, Optional
 
 from torch.distributed import barrier, destroy_process_group, init_process_group
 from torch.distributed.device_mesh import DeviceMesh, init_device_mesh
 
 from ..utils import logging
-from ..utils.types import DistributedConfig, ProcessGroup, TensorLike
+from ..utils.types import ProcessGroup, TensorLike
 from . import helper
 
 
+if TYPE_CHECKING:
+    from ..config.training_args import TrainingArguments
+
+
 logger = logging.get_logger(__name__)
 
 
@@ -128,12 +132,13 @@ class DistributedInterface:
 
         return cls._instance
 
-    def __init__(self, config: DistributedConfig | None = None) -> None:
+    def __init__(
+        self,
+        training_args: "TrainingArguments | None" = None,
+    ) -> None:
         if self._initialized:
             return
 
-        self.dist_config = config
-
         helper.set_device_index()
         self._is_distributed = helper.is_distributed()
         self._rank = helper.get_rank()
@@ -143,17 +148,17 @@ class DistributedInterface:
         self.current_device = helper.get_current_device()
         self.device_count = helper.get_device_count()
 
-        if config is None:
+        if training_args is None:
             self.strategy = DistributedStrategy()
             timeout = 18000
         else:
             self.strategy = DistributedStrategy(
-                mp_replicate_size=config.get("mp_replicate_size", 1),
+                mp_replicate_size=training_args.mp_replicate_size,
-                mp_shard_size=config.get("mp_shard_size", None),
+                mp_shard_size=training_args.mp_shard_size,
-                dp_size=config.get("dp_size", None),
+                dp_size=training_args.dp_size,
-                cp_size=config.get("cp_size", 1),
+                cp_size=training_args.cp_size,
             )
-            timeout = config.get("timeout", 18000)
+            timeout = training_args.dist_timeout
 
         if self._is_distributed:
             init_process_group(timeout=timedelta(seconds=timeout), backend=helper.get_process_group_backend())

src/llamafactory/v1/config/training_args.py

@@ -76,7 +76,31 @@ class TrainingArguments:
     )
     dist_config: PluginConfig | None = field(
         default=None,
-        metadata={"help": "Distribution configuration for training."},
+        metadata={"help": "Distributed backend plugin configuration."},
+    )
+    dp_size: int | None = field(
+        default=None,
+        metadata={"help": "Data parallel size, default to world_size // cp_size."},
+    )
+    cp_size: int = field(
+        default=1,
+        metadata={"help": "Context parallel size."},
+    )
+    cp_mode: str = field(
+        default="ulysses",
+        metadata={"help": "Context parallel implementation."},
+    )
+    mp_replicate_size: int = field(
+        default=1,
+        metadata={"help": "Model parallel replicate size."},
+    )
+    mp_shard_size: int | None = field(
+        default=None,
+        metadata={"help": "Model parallel shard size, default to world_size // mp_replicate_size."},
+    )
+    dist_timeout: int = field(
+        default=18000,
+        metadata={"help": "Distributed process group initialization timeout in seconds."},
     )
     optim_config: PluginConfig | None = field(
         default=None,
@@ -149,6 +173,12 @@ class TrainingArguments:
         self.dist_config = get_plugin_config(self.dist_config)
         self.optim_config = get_plugin_config(self.optim_config)
         self.lr_scheduler_config = get_plugin_config(self.lr_scheduler_config)
+        try:
+            from ..plugins.model_plugins.deepspeed_utils import register_deepspeed_dist_config
+
+            register_deepspeed_dist_config(self.dist_config)
+        except ImportError:
+            pass
 
         # The optimizer learning rate has a single source of truth: ``learning_rate``.
         # Propagate it into ``optim_config["lr"]`` so optimizer plugins (e.g. Muon) pick it up

src/llamafactory/v1/core/base_trainer.py

@@ -43,7 +43,7 @@ from ..utils.callbacks import (
     TrainerCallback,
     TrainerState,
 )
-from ..utils.helper import compute_valid_tokens
+from ..utils.helper import compute_valid_tokens, is_tokenizer, model_uses_mrope
 from ..utils.types import BatchInput, HFModel, ModelOutput, Tensor, TorchDataset
 from .rendering import Renderer
 from .utils.batching import BatchGenerator
@@ -75,6 +75,7 @@ class BaseTrainer:
         self.dp_size = DistributedInterface().get_world_size(Dim.DP)
         self.cp_size = DistributedInterface().get_world_size(Dim.CP)
         self.model_input_names = self.renderer.processor.model_input_names
+        self._uses_mrope = model_uses_mrope(self.model.config)
 
         self._create_batch_generator()
         # Calculate num_training_steps: max_steps takes priority if set
@@ -89,14 +90,20 @@ class BaseTrainer:
 
         if self.args.enable_activation_checkpointing:
             self.model.gradient_checkpointing_enable({"use_reentrant": False})
+            # Note: under FSDP2 bf16, encoder-tower nn.LayerNorms are made dtype-safe for the
+            # checkpoint recompute inside the FSDP2 engine (see fsdp2.py prepare_model), so the
+            # tower keeps activation checkpointing too.
 
         self._deepspeed_engine = None
         dist_name = self.args.dist_config.name if self.args.dist_config is not None else None
 
         if dist_name == "deepspeed":
-            from ..plugins.trainer_plugins.distributed.hub import DistributedPlugin
+            if self.args.cp_size > 1:
+                raise ValueError("Context parallelism currently requires `dist_config.name: fsdp2`.")
 
-            self._deepspeed_engine = DistributedPlugin("deepspeed")(
+            from ..plugins.trainer_plugins.distributed.interface import DistributedPlugin
+
+            self._deepspeed_engine = DistributedPlugin("deepspeed").shard_model(
                 self.model,
                 self.args.dist_config,
                 num_micro_batch=self.train_batch_generator.num_micro_batch,
@@ -139,7 +146,7 @@ class BaseTrainer:
         self.state.global_step = self.global_step
         self.state.epoch = self._resume_epoch
 
-        if self.args.dist_config is not None and self.args.dist_config.get("cp_size", 1) > 1:
+        if self.args.cp_size > 1:
             # qwen3.5 is not supported because of the different attention implementation, which will be supported in the future.
             if model.config.model_type == "qwen3_5":
                 raise RuntimeError(
@@ -152,7 +159,7 @@ class BaseTrainer:
                     "Sequence parallelism requires flash attention. Please set `flash_attn: flash_attention_2`."
                 )
 
-            SequenceParallelModelPlugin(self.args.dist_config.get("cp_mode", "ulysses"))(model, self.args.dist_config)
+            SequenceParallelModelPlugin(self.args.cp_mode)(model, self.args.cp_size)
 
     def _create_batch_generator(self) -> None:
         if (
@@ -181,11 +188,15 @@ class BaseTrainer:
                     "dist_config is None but distributed training is enabled; falling back to DistributedDataParallel."
                 )
                 device_ids = None if self.device.type == "cpu" else [self.device.index]
-                self.model = DDP(self.model, device_ids=device_ids)
+                # Multimodal models invoke the vision tower only when a step carries media; a
+                # globally media-less step leaves vision params unused, which trips DDP's default
+                # all-params-used assertion. (FSDP tolerates a uniform skip; DDP does not.)
+                find_unused = not is_tokenizer(self.renderer.processor)
+                self.model = DDP(self.model, device_ids=device_ids, find_unused_parameters=find_unused)
         else:
-            from ..plugins.trainer_plugins.distributed.hub import DistributedPlugin
+            from ..plugins.trainer_plugins.distributed.interface import DistributedPlugin
 
-            self.model = DistributedPlugin(self.args.dist_config.name)(
+            self.model = DistributedPlugin(self.args.dist_config.name).shard_model(
                 self.model,
                 self.args.dist_config,
                 bf16=self.args.bf16,
@@ -221,6 +232,9 @@ class BaseTrainer:
         model_inputs = {
             k: v.to(self.device, non_blocking=True) for k, v in batch.items() if isinstance(v, torch.Tensor)
         }
+        # Let mRoPE models build their own multimodal 3D position ids (see _uses_mrope in __init__).
+        if self._uses_mrope:
+            model_inputs.pop("position_ids", None)
         labels = batch["labels"].to(self.device, non_blocking=True)
         outputs: ModelOutput = model(**model_inputs)
         logits = outputs.logits.float()
@@ -256,7 +270,7 @@ class BaseTrainer:
                 step_valid_tokens = DistributedInterface().all_reduce(step_valid_tokens, op=ReduceOp.SUM)
                 num_micro = len(micro_batches)
                 for i, micro_batch in enumerate(micro_batches):
-                    if self.args.dist_config and self.args.dist_config.get("cp_size", 1) > 1:
+                    if self.args.cp_size > 1:
                         from ..plugins.model_plugins.parallelization.sequence_parallel import (
                             SequenceParallelLossPlugin,
                         )
@@ -280,21 +294,24 @@ class BaseTrainer:
                     # deepspeed: engine.step() already ran inside backward at the sync boundary
                     grad_norm = self._deepspeed_engine.get_grad_norm()
                 else:
-                    # FSDP2 shards params/grads across the fsdp mesh, so clip_grad_norm_ returns a
+                    dist_name = self.args.dist_config.name if self.args.dist_config else None
-                    # per-rank local shard norm (global / sqrt(shard_size)): reported grad_norm then
+                    if dist_name == "fsdpturbo":
-                    # scales as 1/sqrt(dp_size) and the clip coefficient is applied per-shard. Reduce
+                        from ..plugins.trainer_plugins.distributed.interface import DistributedPlugin
-                    # to the true global norm first, then clip with it.
+
-                    grads = [p.grad for p in self.model.parameters() if p.grad is not None]
+                        grad_norm = DistributedPlugin(dist_name).clip_grad_norm(self.model, self.args.max_grad_norm)
-                    total_norm = torch.nn.utils.get_total_norm(grads)
+                    else:
-                    if isinstance(total_norm, DTensor):
+                        # FSDP2 shards params/grads across the fsdp mesh, so clip_grad_norm_ returns a
-                        # full_tensor all-reduces across the fsdp mesh (spans CP under default
+                        # per-rank local shard norm. Materialize the true global norm before clipping.
-                        # mp_shard=world); a separate CP reduce would over-count by sqrt(cp_size).
+                        grads = [p.grad for p in self.model.parameters() if p.grad is not None]
-                        total_norm = total_norm.full_tensor()
+                        total_norm = torch.nn.utils.get_total_norm(grads)
-                    # pass a Tensor: clip_grads_with_norm_ clamps max_norm / (total_norm + 1e-6).
+                        if isinstance(total_norm, DTensor):
-                    torch.nn.utils.clip_grads_with_norm_(
+                            # full_tensor all-reduces across the fsdp mesh (spans CP under default
-                        self.model.parameters(), self.args.max_grad_norm, total_norm
+                            # mp_shard=world); a separate CP reduce would over-count by sqrt(cp_size).
-                    )
+                            total_norm = total_norm.full_tensor()
-                    grad_norm = total_norm.item()
+                        torch.nn.utils.clip_grads_with_norm_(
+                            self.model.parameters(), self.args.max_grad_norm, total_norm
+                        )
+                        grad_norm = total_norm.item()
 
                     if not torch.isfinite(torch.tensor(grad_norm)):  # type: ignore # pyright: ignore [reportUnknownReturnType]
                         logger.warning_rank0(f"Gradient norm is not finite: {grad_norm}")
@@ -351,8 +368,8 @@ class BaseTrainer:
 
     def save_model(self) -> None:
         """Save the model."""
-        if self.args.dist_config is not None and self.args.dist_config.name in ("deepspeed", "fsdp2"):
+        if self.args.dist_config is not None and self.args.dist_config.name in ("deepspeed", "fsdp2", "fsdpturbo"):
-            from ..plugins.trainer_plugins.distributed.hub import DistributedPlugin
+            from ..plugins.trainer_plugins.distributed.interface import DistributedPlugin
 
             DistributedPlugin(self.args.dist_config.name).save_model(
                 self.model, self.args.output_dir, self.renderer.processor

src/llamafactory/v1/core/data_engine.py

@@ -137,9 +137,7 @@ class DataEngine(Dataset):
         messages = sample.get("messages")
         if not messages:
             return [None]
-        cuts = [
+        cuts = [i + 1 for i, m in enumerate(messages) if m["role"] == "assistant" and m.get("loss_weight", 1.0) > 1e-6]
-            i + 1 for i, m in enumerate(messages) if m["role"] == "assistant" and m.get("loss_weight", 1.0) > 1e-6
-        ]
         return cuts or [None]
 
     def _convert_data_sample(self, raw_sample: dict[str, Any], dataset_name: str) -> Sample:

src/llamafactory/v1/core/model_engine.py

@@ -69,24 +69,25 @@ class ModelEngine:
         """Model configuration."""
         self.renderer = Renderer(self.processor)
         """Renderer."""
-        self._dist_config = DistributedInterface().dist_config
-        self._deepspeed_zero3_plugin = None
         self._deepspeed_zero3_enabled = False
 
-        if self.is_train and self._dist_config is not None and self._dist_config.get("name") == "deepspeed":
+        try:
             from ..plugins.model_plugins.deepspeed_utils import (
+                is_deepspeed_zero3_enabled,
                 setup_deepspeed_zero3_model_loading,
                 teardown_deepspeed_zero3_model_loading,
             )
 
+            self._deepspeed_zero3_enabled = self.is_train and is_deepspeed_zero3_enabled()
+        except ImportError:
+            pass
+
+        if self._deepspeed_zero3_enabled:
+            plugin = setup_deepspeed_zero3_model_loading()
             try:
-                self._deepspeed_zero3_plugin = setup_deepspeed_zero3_model_loading(self.is_train, self._dist_config)
-                self._deepspeed_zero3_enabled = self._deepspeed_zero3_plugin is not None
                 self.model = self._init_model()
             finally:
-                teardown_deepspeed_zero3_model_loading(self._deepspeed_zero3_plugin)
+                teardown_deepspeed_zero3_model_loading(plugin)
-                self._deepspeed_zero3_plugin = None
-                self._deepspeed_zero3_enabled = False
         else:
             self.model = self._init_model()
 
@@ -111,8 +112,7 @@ class ModelEngine:
         if self.args.custom_chat_template:
             if not is_tokenizer(self.processor):
                 self.processor.chat_template = self.args.custom_chat_template
-            else:
+            tokenizer.chat_template = self.args.custom_chat_template
-                tokenizer.chat_template = self.args.custom_chat_template
 
     def _init_model_config(self) -> HFConfig:
         """Init model config."""
@@ -142,17 +142,26 @@ class ModelEngine:
 
             init_kwargs = QuantizationPlugin(self.args.quant_config.name)(
                 init_kwargs=init_kwargs,
-                config=self.model_config,
+                quant_config=self.args.quant_config,
-                tokenizer=self.processor,
-                model_args=self.args,
                 is_trainable=self.is_train,
             )
 
         if self.args.model_class == ModelClass.LLM:
             from transformers import AutoModelForCausalLM, AutoModelForImageTextToText
 
-            if type(self.model_config) in AutoModelForImageTextToText._model_mapping.keys():
+            # AutoModelForMultimodalLM (audio / other multimodal LMs, e.g. Qwen2-Audio) was added in
+            # a newer transformers; fall back gracefully when it is absent (e.g. 4.57.1).
+            try:
+                from transformers import AutoModelForMultimodalLM
+            except ImportError:
+                AutoModelForMultimodalLM = None
+
+            cfg_type = type(self.model_config)
+            if cfg_type in AutoModelForImageTextToText._model_mapping.keys():
                 AutoClass = AutoModelForImageTextToText
+            elif AutoModelForMultimodalLM is not None and cfg_type in AutoModelForMultimodalLM._model_mapping.keys():
+                # Audio / other multimodal LMs (e.g. Qwen2-Audio) live here, not in CausalLM.
+                AutoClass = AutoModelForMultimodalLM
             else:
                 AutoClass = AutoModelForCausalLM
 
@@ -174,7 +183,7 @@ class ModelEngine:
         if init_device.type == DeviceType.META:
             assert self.args.quant_config is None, "Quantization is not supported with meta device."
             with init_empty_weights():
-                model = AutoClass.from_config(self.model_config)
+                model = AutoClass.from_config(self.model_config, attn_implementation=self.args.flash_attn)
         else:
             model = AutoClass.from_pretrained(
                 self.args.model,
@@ -188,6 +197,10 @@ class ModelEngine:
         init_mode = self.args.init_config.name if self.args.init_config is not None else "init_on_default"
         model._init_mode = init_mode
 
+        if hasattr(model, "thinker"):
+            model = model.thinker
+            model._init_mode = init_mode
+
         if self.args.peft_config is None:
             if self.is_train:
                 logger.info_rank0("Fine-tuning mode: full tuning")
@@ -200,17 +213,16 @@ class ModelEngine:
 
             from ..plugins.model_plugins.peft import PeftPlugin
 
-            model = PeftPlugin(self.args.peft_config.name)(model, self.args.peft_config, self.is_train)
+            model = PeftPlugin(self.args.peft_config.name)(
+                model,
+                peft_config=self.args.peft_config,
+                is_train=self.is_train,
+            )
 
         if self.args.kernel_config is not None:
-            from ..plugins.model_plugins.kernels.interface import KernelPlugin
+            from ..plugins.model_plugins.kernels.interface import apply_kernels
 
-            kernel_config = self.args.kernel_config
+            model = apply_kernels(model, self.args.kernel_config, require_logits=self.is_train)
-            kernel_kwargs: dict = {"model": model, "include_kernels": kernel_config.get("include_kernels")}
-            if kernel_config.name == "liger_kernel":
-                # Fused linear CE omits logits; SFT stage needs logits for loss_weights.
-                kernel_kwargs["require_logits"] = self.is_train
-            model = KernelPlugin(kernel_config.name)(**kernel_kwargs)
 
         return model
 

src/llamafactory/v1/core/rendering/format.py

@@ -1,4 +1,4 @@
-# Copyright 2025 the LlamaFactory team.
+# Copyright 2026 the LlamaFactory team.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,13 +14,15 @@
 
 """Message <-> HF-template plumbing for rendering.
 
-Pure, stateless helpers: convert v1 ``Message`` to HF chat-template format. No tokenization policy
+Pure, stateless helpers: convert v1 ``Message`` to HF chat-template format, extract/count media, and
-decisions live here -- only mechanical conversion used by ``rendering.py``.
+guard media placeholder counts. No tokenization policy decisions live here -- only mechanical
+conversion used by ``rendering.py``.
 """
 
 import json
 
-from ...utils.types import Message
+from ...utils.helper import get_tokenizer
+from ...utils.types import Message, Processor
 
 
 _FALLBACK_CHATML_JINJA = (
@@ -33,32 +35,59 @@ _FALLBACK_CHATML_JINJA = (
 )
 
 
-def _to_hf_messages(messages: list[Message]) -> list[dict]:
+def _to_hf_messages(messages: list[Message], is_multimodal: bool = False) -> list[dict]:
     """Convert v1 Message format to HF format for apply_chat_template."""
     hf_messages = []
     for message in messages:
         tool_calls: list[dict] = []
         reasoning_content = ""
 
-        text = ""
+        if is_multimodal:
-        for content in message["content"]:
+            hf_content = []
-            if content["type"] == "text":
+            for content in message["content"]:
-                text += content["value"]
+                if content["type"] == "text":
-            elif content["type"] == "reasoning":
+                    hf_content.append({"type": "text", "text": content["value"]})
-                reasoning_content += content["value"]
+                elif content["type"] == "reasoning":
-            elif content["type"] == "tool_call":
+                    reasoning_content += content["value"]
-                try:
+                elif content["type"] == "tool_call":
-                    tc = json.loads(content["value"])
+                    try:
-                except json.JSONDecodeError as e:
+                        tc = json.loads(content["value"])
-                    raise ValueError(f"tool_call value is not valid JSON: {content['value']!r}") from e
+                    except json.JSONDecodeError as e:
-                if not isinstance(tc, dict) or "name" not in tc or "arguments" not in tc:
+                        raise ValueError(f"tool_call value is not valid JSON: {content['value']!r}") from e
-                    raise ValueError(
+                    if not isinstance(tc, dict) or "name" not in tc or "arguments" not in tc:
-                        f"tool_call must be a JSON object with 'name' and 'arguments' keys, got {tc!r}"
+                        raise ValueError(
+                            f"tool_call must be a JSON object with 'name' and 'arguments' keys, got {tc!r}"
+                        )
+                    tool_calls.append(
+                        {"type": "function", "function": {"name": tc["name"], "arguments": tc["arguments"]}}
                     )
-                tool_calls.append(
+                elif content["type"] == "image_url":
-                    {"type": "function", "function": {"name": tc["name"], "arguments": tc["arguments"]}}
+                    hf_content.append({"type": "image", "image": content["value"]})
-                )
+                elif content["type"] == "video_url":
-        hf_msg = {"role": message["role"], "content": text}
+                    hf_content.append({"type": "video", "video": content["value"]})
+                elif content["type"] == "audio_url":
+                    hf_content.append({"type": "audio", "audio": content["value"]})
+            hf_msg = {"role": message["role"], "content": hf_content}
+        else:
+            text = ""
+            for content in message["content"]:
+                if content["type"] == "text":
+                    text += content["value"]
+                elif content["type"] == "reasoning":
+                    reasoning_content += content["value"]
+                elif content["type"] == "tool_call":
+                    try:
+                        tc = json.loads(content["value"])
+                    except json.JSONDecodeError as e:
+                        raise ValueError(f"tool_call value is not valid JSON: {content['value']!r}") from e
+                    if not isinstance(tc, dict) or "name" not in tc or "arguments" not in tc:
+                        raise ValueError(
+                            f"tool_call must be a JSON object with 'name' and 'arguments' keys, got {tc!r}"
+                        )
+                    tool_calls.append(
+                        {"type": "function", "function": {"name": tc["name"], "arguments": tc["arguments"]}}
+                    )
+            hf_msg = {"role": message["role"], "content": text}
 
         if tool_calls:
             hf_msg["tool_calls"] = tool_calls
@@ -68,3 +97,74 @@ def _to_hf_messages(messages: list[Message]) -> list[dict]:
         hf_messages.append(hf_msg)
     return hf_messages
 
+
+def _extract_media_from_messages(messages: list[Message]) -> tuple[list, list, list]:
+    """Extract image, video and audio paths/values from messages in order."""
+    images, videos, audios = [], [], []
+    for message in messages:
+        for content in message["content"]:
+            if content["type"] == "image_url":
+                images.append(content["value"])
+            elif content["type"] == "video_url":
+                videos.append(content["value"])
+            elif content["type"] == "audio_url":
+                audios.append(content["value"])
+    return images, videos, audios
+
+
+def _count_media_in_messages(messages: list[Message]) -> tuple[int, int, int]:
+    """Count total images, videos and audios in messages."""
+    n_images, n_videos, n_audios = 0, 0, 0
+    for message in messages:
+        for content in message["content"]:
+            if content["type"] == "image_url":
+                n_images += 1
+            elif content["type"] == "video_url":
+                n_videos += 1
+            elif content["type"] == "audio_url":
+                n_audios += 1
+    return n_images, n_videos, n_audios
+
+
+def _load_audios(values: list, sampling_rate: int) -> list:
+    """Load audio inputs into mono waveforms resampled to ``sampling_rate``."""
+    import numpy as np
+    import torchaudio
+
+    results = []
+    for value in values:
+        if isinstance(value, np.ndarray):
+            results.append(value)
+            continue
+
+        waveform, sr = torchaudio.load(value)
+        if waveform.shape[0] > 1:  # downmix to mono
+            waveform = waveform.mean(dim=0, keepdim=True)
+        if sr != sampling_rate:
+            waveform = torchaudio.functional.resample(waveform, sr, sampling_rate)
+        results.append(waveform.squeeze(0).numpy())
+    return results
+
+
+def _check_placeholder_counts(
+    processor: "Processor", full_text: str, n_images: int, n_videos: int, n_audios: int = 0
+) -> None:
+    """Guard: every media placeholder in the rendered text must originate from a media block."""
+    tokenizer = get_tokenizer(processor)
+    for attr, count, kind in (
+        ("image_token_id", n_images, "image"),
+        ("video_token_id", n_videos, "video"),
+        ("audio_token_id", n_audios, "audio"),
+    ):
+        tid = getattr(processor, attr, None)
+        if tid is None:
+            tid = getattr(tokenizer, attr, None)
+        if tid is None:
+            continue
+        placeholder = tokenizer.convert_ids_to_tokens(tid)
+        seen = full_text.count(placeholder)
+        if seen != count:
+            raise ValueError(
+                f"{kind} placeholder count ({seen}) != number of {kind} blocks ({count}); "
+                "media must be provided via image_url/video_url content blocks."
+            )

src/llamafactory/v1/core/rendering/rendering.py

@@ -1,4 +1,4 @@
-# Copyright 2025 the LlamaFactory team.
+# Copyright 2026 the LlamaFactory team.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,24 +19,29 @@ sibling modules:
   - ``format``  -- v1<->HF message conversion
   - ``escape``  -- special-token escaping (prompt-injection hardening)
 
-Assistant supervision is located WITHOUT a per-model marker table: a training sample is rendered
+Note: ``position_ids`` are assigned by ``process_samples`` (1-based); multimodal (mrope) position
-so that its last message is the supervised assistant turn, and that turn's token span is recovered
+ids are expected to be recomputed by the model/trainer.
-by a single prompt/full difference -- encode the prompt (everything up to and including the
-assistant role header, via ``add_generation_prompt=True``) and the full sequence, then the tail of
-the full sequence that the prompt does not cover is exactly this turn. Multi-turn conversations are
-split into one sample per supervised turn (see ``process_samples``) so the supervised turn is always
-the last one; this keeps the diff on the only boundary that is prefix-stable across chat templates
-(appending the final assistant turn never restripts earlier turns), so models with reasoning-history
-stripping (e.g. Qwen3 ``<think>``) are handled correctly without hard-coding role markers.
 """
 
+
 import json
 
+import numpy as np
+import torch
+
 from ...utils.constants import IGNORE_INDEX
-from ...utils.helper import get_tokenizer
+from ...utils.helper import get_tokenizer, is_tokenizer
 from ...utils.types import Message, ModelInput, Processor, Sample
+from ..utils.collation import _MULTIMODAL_PASSTHROUGH_KEYS
 from .escape import _escape_special, _escape_special_in_messages, _special_token_strings
-from .format import _FALLBACK_CHATML_JINJA, _to_hf_messages
+from .format import (
+    _FALLBACK_CHATML_JINJA,
+    _check_placeholder_counts,
+    _count_media_in_messages,
+    _extract_media_from_messages,
+    _load_audios,
+    _to_hf_messages,
+)
 
 
 def _render_messages(
@@ -46,21 +51,23 @@ def _render_messages(
     is_generate: bool = False,
     **kwargs,
 ) -> ModelInput:
-    r"""Render messages using the model's own chat template.
+    r"""Render messages using the model's own chat template, locating supervision by a prompt/full diff.
 
     Note: ``position_ids`` are not produced here; ``process_samples`` assigns a 1-based range.
     """
-
     tokenizer = get_tokenizer(processor)
-    if not getattr(tokenizer, "chat_template", None):
+    is_multimodal = not is_tokenizer(processor)
-        tokenizer.chat_template = _FALLBACK_CHATML_JINJA
+
+    template_caller = processor if is_multimodal else tokenizer
+    if not getattr(template_caller, "chat_template", None):
+        template_caller.chat_template = _FALLBACK_CHATML_JINJA
 
     # 0. Neutralize special-token strings in user-controlled text (no-op for normal data).
     specials = _special_token_strings(tokenizer)
     special_ids = {tid for tid, t in tokenizer.added_tokens_decoder.items() if getattr(t, "special", False)}
     messages = _escape_special_in_messages(messages, specials, special_ids, tokenizer)
 
-    hf_messages = _to_hf_messages(messages)
+    hf_messages = _to_hf_messages(messages, is_multimodal=is_multimodal)
 
     tools_parsed = None
     if tools:
@@ -71,35 +78,76 @@ def _render_messages(
             raise ValueError(f"tools is not valid JSON: {tools!r}") from e
         if not isinstance(tools_parsed, list):
             tools_parsed = [tools_parsed]
-    if not is_generate and hf_messages and hf_messages[-1].get("reasoning_content"):
-        kwargs["enable_thinking"] = True
-    def _encode(msgs: list[dict], add_generation_prompt: bool) -> list[int]:
-        text = tokenizer.apply_chat_template(
-            msgs, tokenize=False, add_generation_prompt=add_generation_prompt, tools=tools_parsed, **kwargs
-        )
-        return tokenizer(text, add_special_tokens=False)["input_ids"]
 
-    # 1. Full sequence, used verbatim.
+    if not is_generate and hf_messages and hf_messages[-1]["role"] == "assistant":
-    input_ids = _encode(hf_messages, add_generation_prompt=is_generate)
+        kwargs["enable_thinking"] = bool(hf_messages[-1].get("reasoning_content"))
+
+    def _encode(hf_msgs: list[dict], src_msgs: list[Message], add_generation_prompt: bool):
+        """Render + tokenize, expanding media via the processor. Returns (input_ids, mm_outputs)."""
+        text = template_caller.apply_chat_template(
+            hf_msgs, tokenize=False, add_generation_prompt=add_generation_prompt, tools=tools_parsed, **kwargs
+        )
+        if is_multimodal and _count_media_in_messages(src_msgs) != (0, 0, 0):
+            images, videos, audios = _extract_media_from_messages(src_msgs)
+            # Every placeholder must come from a media block (escaping broke any literal ones).
+            _check_placeholder_counts(processor, text, len(images), len(videos), len(audios))
+            proc_kwargs = {"return_tensors": "pt"}
+            if images:
+                proc_kwargs["images"] = images
+            if videos:
+                proc_kwargs["videos"] = videos
+            if audios:
+                # Audio processors want decoded waveforms at the model's sampling rate, not paths.
+                proc_kwargs["audio"] = _load_audios(audios, processor.feature_extractor.sampling_rate)
+            mm_outputs = processor(text=text, **proc_kwargs)
+            return mm_outputs["input_ids"][0].tolist(), mm_outputs
+        return tokenizer(text, add_special_tokens=False)["input_ids"], None
+
+    # 1. Full sequence (used verbatim), plus its multimodal feature outputs.
+    input_ids, outputs = _encode(hf_messages, messages, add_generation_prompt=is_generate)
     n = len(input_ids)
 
+    def _attach_multimodal(result: ModelInput) -> None:
+        if outputs is None:
+            return
+        for key in _MULTIMODAL_PASSTHROUGH_KEYS:
+            if key in outputs:
+                result[key] = outputs[key]
+        mm_type_ids = outputs["mm_token_type_ids"][0].tolist() if "mm_token_type_ids" in outputs else None
+
+        for attr, marker in (("image_token_id", 1), ("video_token_id", 2), ("audio_token_id", 3)):
+            token_id = getattr(processor, attr, None)
+            if token_id is None:
+                token_id = getattr(tokenizer, attr, None)
+            if token_id is None or token_id not in input_ids:
+                continue
+            if mm_type_ids is not None and marker in mm_type_ids:
+                continue
+            if mm_type_ids is None:
+                mm_type_ids = [0] * len(input_ids)
+            mm_type_ids = [marker if tid == token_id else t for t, tid in zip(mm_type_ids, input_ids)]
+
+        if mm_type_ids is not None:
+            result["mm_token_type_ids"] = mm_type_ids
+
     if is_generate:
         # Generation prompt only -- nothing is supervised.
-        return ModelInput(
+        result = ModelInput(
             input_ids=input_ids,
             attention_mask=[1] * n,
             labels=[IGNORE_INDEX] * n,
             loss_weights=[0.0] * n,
         )
+        _attach_multimodal(result)
+        return result
 
-    # 2. Locate the supervised (last) assistant turn by a prompt/full diff (no marker table).
     if not messages or messages[-1]["role"] != "assistant":
         raise ValueError(
             "training render expects the last message to be the supervised assistant turn; "
             "multi-turn conversations are split per turn in process_samples."
         )
 
-    prompt_ids = _encode(hf_messages[:-1], add_generation_prompt=True)
+    prompt_ids, _ = _encode(hf_messages[:-1], messages[:-1], add_generation_prompt=True)
     if input_ids[: len(prompt_ids)] != prompt_ids:
         # The prompt must be a token-prefix of the full sequence for the diff to be valid. If a
         # template re-renders earlier turns when the final turn is appended, fail loud rather than
@@ -117,16 +165,21 @@ def _render_messages(
         labels.append(tid if supervised else IGNORE_INDEX)
         loss_weights.append(weight)
 
-    return ModelInput(
+    result = ModelInput(
         input_ids=input_ids,
         attention_mask=[1] * n,
         labels=labels,
         loss_weights=loss_weights,
     )
+    _attach_multimodal(result)
+    return result
 
 
 class Renderer:
-    def __init__(self, processor: Processor) -> None:
+    def __init__(self, processor: Processor, config=None):
+        # ``config`` is accepted for call-site compatibility (ModelEngine passes the model config)
+        # but is no longer needed: supervision is located by a prompt/full diff, not a per-model
+        # marker table, so the renderer is model-agnostic.
         self.processor = processor
 
     def render_messages(
@@ -150,13 +203,62 @@ class Renderer:
         Returns:
             ModelInput with input_ids, attention_mask, labels, and loss_weights.
         """
-        return _render_messages(
+        return _render_messages(self.processor, messages, tools, is_generate, **kwargs)
-            self.processor,
+
-            messages,
+    def get_dummy_media_fragment(self, modality: str) -> dict:
-            tools,
+        """Build (and cache) a minimal valid media fragment for ``modality`` ("image"|"video"|"audio")."""
-            is_generate,
+        if modality not in ("image", "video", "audio"):
-            **kwargs
+            raise ValueError(f"Unsupported dummy media modality: {modality!r} (expected image/video/audio).")
-        )
+        if is_tokenizer(self.processor):
+            raise RuntimeError("Cannot build a dummy media fragment for a text-only processor.")
+
+        if not hasattr(self, "_dummy_fragments"):
+            self._dummy_fragments: dict[str, dict] = {}
+        if modality in self._dummy_fragments:
+            return self._dummy_fragments[modality]
+
+        from PIL import Image as _PILImage
+
+        if modality == "image":
+            media_block = {"type": "image_url", "value": _PILImage.new("RGB", (64, 64))}
+            target, presence_key = 1, "pixel_values"
+        elif modality == "video":
+            # A minimal clip: the temporal patch size is typically 2, so provide two frames.
+            media_block = {"type": "video_url", "value": np.zeros((2, 64, 64, 3), dtype=np.uint8)}
+            target, presence_key = 2, "pixel_values_videos"
+        else:
+            # A short synthetic waveform at the model's sampling rate; the feature extractor pads it.
+            sr = self.processor.feature_extractor.sampling_rate
+            media_block = {"type": "audio_url", "value": np.zeros(sr // 10, dtype=np.float32)}
+            target, presence_key = 3, "input_features"
+
+        messages: list[Message] = [
+            {"role": "user", "content": [media_block]},
+            {"role": "assistant", "content": [{"type": "text", "value": "ok"}]},
+        ]
+        rendered = self.render_messages(messages)
+
+        mm_type_ids = rendered.get("mm_token_type_ids")
+        if not mm_type_ids or target not in mm_type_ids or presence_key not in rendered:
+            raise RuntimeError(f"Processor did not emit {modality} placeholder tokens for the dummy sample.")
+
+        positions = [i for i, t in enumerate(mm_type_ids) if t == target]
+        # Include the surrounding start/end delimiters (vision_start/end or audio_bos/eos) so the
+        # fragment matches exactly what the template emits around real media.
+        lo = max(positions[0] - 1, 0)
+        hi = min(positions[-1] + 2, len(rendered["input_ids"]))
+
+        fragment: dict = {
+            "input_ids": list(rendered["input_ids"][lo:hi]),
+            "mm_token_type_ids": list(mm_type_ids[lo:hi]),
+        }
+
+        for key in _MULTIMODAL_PASSTHROUGH_KEYS:
+            if key in rendered:
+                fragment[key] = rendered[key]
+
+        self._dummy_fragments[modality] = fragment
+        return fragment
 
     def process_samples(self, samples: list[Sample]) -> list[ModelInput]:
         """Process samples to model input.
@@ -195,6 +297,17 @@ class Renderer:
                 model_input["position_ids"] = list(range(1, len(chosen_input["input_ids"]) + 1)) + list(
                     range(1, len(rejected_input["input_ids"]) + 1)
                 )
+
+                for key in _MULTIMODAL_PASSTHROUGH_KEYS:
+                    tensors = [inp[key] for inp in (chosen_input, rejected_input) if key in inp]
+                    if tensors:
+                        model_input[key] = torch.cat(tensors, dim=0)
+
+                if "mm_token_type_ids" in chosen_input or "mm_token_type_ids" in rejected_input:
+                    chosen_mm = chosen_input.get("mm_token_type_ids", [0] * len(chosen_input["input_ids"]))
+                    rejected_mm = rejected_input.get("mm_token_type_ids", [0] * len(rejected_input["input_ids"]))
+                    model_input["mm_token_type_ids"] = chosen_mm + rejected_mm
+
                 rendered.append(model_input)
             else:
                 raise ValueError("No valid messages or chosen_messages/rejected_messages found in sample.")

src/llamafactory/v1/core/utils/batching.py

@@ -1,4 +1,4 @@
-# Copyright 2025 the LlamaFactory team.
+# Copyright 2026 the LlamaFactory team.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -31,13 +31,16 @@ from torch.utils.data import default_collate
 from torchdata.stateful_dataloader import StatefulDataLoader
 from torchdata.stateful_dataloader.sampler import StatefulDistributedSampler
 
+from ...accelerator.helper import ReduceOp
 from ...accelerator.interface import Dim, DistributedInterface
 from ...config import BatchingStrategy
 from ...utils import logging
-from ...utils.helper import pad_and_truncate
+from ...utils.constants import IGNORE_INDEX
+from ...utils.helper import is_tokenizer
 from ...utils.objects import StatefulBuffer
-from ...utils.types import BatchInfo, BatchInput, ModelInput, TorchDataset
+from ...utils.types import BatchInfo, BatchInput, ModelInput, Tensor, TorchDataset
 from ..rendering import Renderer
+from .collation import _MULTIMODAL_PASSTHROUGH_KEYS, pad_and_truncate
 
 
 logger = logging.get_logger(__name__)
@@ -45,7 +48,82 @@ logger = logging.get_logger(__name__)
 __all__ = ["BatchGenerator"]
 
 
-def default_collate_fn(buffer: StatefulBuffer, batch_info: BatchInfo) -> list[BatchInput] | None:
+# (modality, presence/feature key, grid key, mm_token_type_ids marker) for encoder-tower alignment.
+# The presence key is what survives collation when the modality is present; the grid key is unused
+# here (kept for parity with the collation specs). Audio carries no grid -- feature_attention_mask
+# rides along as a passthrough feature.
+_ALIGN_MODALITIES = (
+    ("image", "pixel_values", "image_grid_thw", 1),
+    ("video", "pixel_values_videos", "video_grid_thw", 2),
+    ("audio", "input_features", "feature_attention_mask", 3),
+)
+
+
+def _collate_micro_batch(micro_batch: list[ModelInput], cutoff_len: int) -> BatchInput:
+    """Pad/truncate then collate one micro batch (text fields stacked, MM features dim-0 concat)."""
+    padded = pad_and_truncate(micro_batch, cutoff_len)
+    standard_samples = [{k: v for k, v in s.items() if k not in _MULTIMODAL_PASSTHROUGH_KEYS} for s in padded]
+    collated = default_collate(standard_samples)
+    for key in _MULTIMODAL_PASSTHROUGH_KEYS:
+        tensors = [s[key] for s in padded if key in s]
+        if tensors:
+            collated[key] = torch.cat(tensors, dim=0)
+    return collated
+
+
+def _inject_dummy_into_collated(collated: BatchInput, fragment: dict, marker: int) -> None:
+    """Append a zero-loss dummy media fragment to an already-collated micro batch, in place.
+
+    Operates *after* pad_and_truncate so it reflects post-truncation presence: an image whose
+    placeholder tokens were partially cut is deleted by ``_align_multimodal_on_truncation``,
+    turning that sample text-only -- which must be detected here (not before truncation) or the
+    vision-tower call count still desyncs across ranks.
+
+    The dummy tokens are appended (extra columns) into row 0 only; other rows get padding there.
+    Causal attention keeps every real token's logits unchanged; the dummy carries IGNORE_INDEX
+    labels and zero loss weight, so it contributes nothing to the loss while forcing the (FSDP-
+    sharded) vision tower to run.
+    """
+    bsz, seqlen = collated["input_ids"].shape
+    frag_ids = torch.tensor(fragment["input_ids"], dtype=collated["input_ids"].dtype)
+    frag_len = frag_ids.numel()
+    frag_mm = torch.tensor(fragment["mm_token_type_ids"], dtype=torch.long)
+    new_len = seqlen + frag_len
+
+    def _grow(tensor: Tensor, pad_value, row0_tail=None) -> Tensor:
+        out = torch.full((bsz, new_len), pad_value, dtype=tensor.dtype)
+        out[:, :seqlen] = tensor
+        if row0_tail is not None:
+            out[0, seqlen:] = row0_tail.to(tensor.dtype)
+        return out
+
+    collated["input_ids"] = _grow(collated["input_ids"], 0, frag_ids)
+    collated["attention_mask"] = _grow(collated["attention_mask"], 0)
+    collated["attention_mask"][0, seqlen:] = 1
+    collated["labels"] = _grow(collated["labels"], IGNORE_INDEX)  # dummy region stays ignored
+    collated["loss_weights"] = _grow(collated["loss_weights"], 0.0)
+    if "position_ids" in collated:
+        pos = _grow(collated["position_ids"], 0)
+        pos[0, seqlen:] = torch.arange(seqlen + 1, new_len + 1, dtype=pos.dtype)
+        collated["position_ids"] = pos
+
+    mm = collated.get("mm_token_type_ids")
+    if mm is not None:
+        collated["mm_token_type_ids"] = _grow(mm, 0, frag_mm)
+    else:
+        mm = torch.zeros((bsz, new_len), dtype=torch.long)
+        mm[0, seqlen:] = frag_mm
+        collated["mm_token_type_ids"] = mm
+
+    for key, value in fragment.items():
+        if key in ("input_ids", "mm_token_type_ids"):
+            continue
+        collated[key] = torch.cat([collated[key], value], dim=0) if key in collated else value
+
+
+def default_collate_fn(
+    buffer: StatefulBuffer, batch_info: BatchInfo, renderer: Renderer | None = None
+) -> list[BatchInput] | None:
     micro_batch_size = batch_info["micro_batch_size"]
     num_micro_batch = batch_info["num_micro_batch"]
     cutoff_len = batch_info["cutoff_len"]
@@ -54,10 +132,24 @@ def default_collate_fn(buffer: StatefulBuffer, batch_info: BatchInfo) -> list[Ba
         return None
 
     samples = buffer.get(batch_size)
-    batch = []
+    micro_batches = [samples[i * micro_batch_size : (i + 1) * micro_batch_size] for i in range(num_micro_batch)]
-    for i in range(num_micro_batch):
+
-        micro_batch = samples[i * micro_batch_size : (i + 1) * micro_batch_size]
+    # Collate first; presence is judged on the *post-truncation* result, since truncation can
-        batch.append(default_collate(pad_and_truncate(micro_batch, cutoff_len)))
+    # delete a partially-cut image and turn a sample text-only (see _inject_dummy_into_collated).
+    batch = [_collate_micro_batch(mb, cutoff_len) for mb in micro_batches]
+
+    if renderer is not None and not is_tokenizer(renderer.processor):
+        present = torch.zeros((num_micro_batch, len(_ALIGN_MODALITIES)), dtype=torch.int64)
+        for i, collated in enumerate(batch):
+            for m, (_, pixel_key, _, _) in enumerate(_ALIGN_MODALITIES):
+                present[i, m] = int(pixel_key in collated)
+
+        present = DistributedInterface().all_reduce(present, op=ReduceOp.MAX, dim=Dim.DP)
+
+        for i, collated in enumerate(batch):
+            for m, (modality, pixel_key, _, marker) in enumerate(_ALIGN_MODALITIES):
+                if present[i, m] and pixel_key not in collated:
+                    _inject_dummy_into_collated(collated, renderer.get_dummy_media_fragment(modality), marker)
 
     return batch
 
@@ -227,8 +319,17 @@ class BatchGenerator(Iterator):
 
     def _generate_batch(self) -> list[BatchInput] | None:
         if self.batching_strategy == BatchingStrategy.NORMAL:
-            return default_collate_fn(self._buffer, self._batch_info)
+            return default_collate_fn(self._buffer, self._batch_info, self.renderer)
         else:
+            # Non-NORMAL strategies (dynamic / padding_free) collate ragged pixel tensors with a
+            # bare default_collate and have no vision-tower alignment, so multimodal data would
+            # crash or hang. Fail loud instead of silently mishandling it.
+            if any(k in s for s in self._buffer.samples for k in _MULTIMODAL_PASSTHROUGH_KEYS):
+                raise NotImplementedError(
+                    f"batching_strategy={self.batching_strategy.value!r} does not support multimodal data; "
+                    "use the NORMAL strategy for image/video training."
+                )
+
             from ...plugins.trainer_plugins.batching import BatchingPlugin
 
             return BatchingPlugin(self.batching_strategy).generate_batch(self._buffer, self._batch_info)

src/llamafactory/v1/core/utils/checkpoint.py

@@ -250,8 +250,8 @@ class TrainingCheckpointCoordinator:
                 num_training_steps=self._t.num_training_steps,
             )
 
-        if self._dist_name in ("fsdp2", "deepspeed"):
+        if self._dist_name in ("fsdp2", "fsdpturbo", "deepspeed"):
-            from ...plugins.trainer_plugins.distributed.hub import DistributedPlugin
+            from ...plugins.trainer_plugins.distributed.interface import DistributedPlugin
 
             DistributedPlugin(self._dist_name).save_checkpoint(
                 self._t.model,
@@ -306,8 +306,8 @@ class TrainingCheckpointCoordinator:
         self._t.global_step = metadata["global_step"]
         self._t._resume_epoch = metadata["epoch"]
 
-        if self._dist_name in ("fsdp2", "deepspeed"):
+        if self._dist_name in ("fsdp2", "fsdpturbo", "deepspeed"):
-            from ...plugins.trainer_plugins.distributed.hub import DistributedPlugin
+            from ...plugins.trainer_plugins.distributed.interface import DistributedPlugin
 
             DistributedPlugin(self._dist_name).load_checkpoint(
                 self._t.model,

src/llamafactory/v1/core/utils/collation.py

@@ -0,0 +1,277 @@
+# Copyright 2026 the LlamaFactory team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Batch collation utils: padding/truncation and multimodal-feature alignment.
+
+These operate on already-rendered ``ModelInput`` dicts (token lists + pixel tensors) and produce
+padded ``BatchInput`` tensors. They are pure batching concerns -- independent of how a sample was
+rendered -- and are consumed by the batch generators in ``core/utils/batching.py`` and
+``plugins/trainer_plugins/batching.py``. Kept out of ``rendering.py`` so that file is only about
+turning messages into a single tokenized sample.
+"""
+
+import torch
+
+from ...utils.constants import IGNORE_INDEX
+from ...utils.types import BatchInput, ModelInput, Tensor
+
+
+# Multimodal feature keys the processor emits per sample. They are NOT padded/stacked like text
+# fields: pixel/audio-feature tensors are ragged (variable patch / frame counts), so the collators
+# concatenate them along dim 0 instead. Shared by rendering (which copies them verbatim from the
+# processor) and the collators (which merge them across a micro batch).
+_MULTIMODAL_PASSTHROUGH_KEYS = frozenset(
+    {
+        "pixel_values",
+        "image_grid_thw",
+        "pixel_values_videos",
+        "video_grid_thw",
+        "second_per_grid_ts",  # Qwen2.5-VL name for the video temporal grid spacing
+        "video_second_per_grid",  # Qwen2.5-Omni name for the same (fed to get_rope_index)
+        "input_features",
+        "feature_attention_mask",
+    }
+)
+
+
+def _pad_and_truncate(tensor: Tensor, max_seqlen: int, pad_value: int = 0) -> Tensor:
+    if tensor.shape[-1] >= max_seqlen:
+        return tensor[..., :max_seqlen]
+
+    pad_shape = list(tensor.shape)
+    pad_shape[-1] = max_seqlen - tensor.shape[-1]
+    pad_tensor = torch.full(pad_shape, pad_value, dtype=tensor.dtype, device=tensor.device)
+    return torch.cat([tensor, pad_tensor], dim=-1)
+
+
+def _align_grid_media(
+    sample: ModelInput,
+    mm_type_ids: list[int],
+    max_length: int,
+    *,
+    target: int,
+    grid_key: str,
+    pixel_key: str,
+) -> list[int]:
+    """Trim and zero one modality's orphaned tokens for a single sample.
+
+    Layout-agnostic: a media item's placeholder tokens may be a single contiguous run or split
+    into per-frame sub-runs; completeness is decided per token *position*, so both are handled
+    identically.
+
+    Returns the (possibly updated) ``mm_token_type_ids`` so chained calls see earlier zeroing.
+    """
+    if grid_key not in sample or pixel_key not in sample:
+        return mm_type_ids
+
+    grid = sample[grid_key]
+    n_items = len(grid)
+    if n_items == 0:
+        return mm_type_ids
+
+    positions = [i for i, t in enumerate(mm_type_ids) if t == target]
+    patches_per_item = [int(grid[i].prod()) for i in range(n_items)]
+    total_patches = sum(patches_per_item)
+    total_tokens = len(positions)
+
+    # merge_size**2 = pixel patches per placeholder token, derived from the data. Bail out
+    # untouched if the sample is inconsistent.
+    if total_tokens == 0 or total_patches % total_tokens != 0:
+        return mm_type_ids
+    merge_sq = total_patches // total_tokens
+    tokens_per_item = [p // merge_sq for p in patches_per_item]
+    if sum(tokens_per_item) != total_tokens:
+        return mm_type_ids
+
+    # Each item owns a contiguous slice of `positions`; it is complete iff its last
+    # placeholder token lands inside the kept window [0, max_length).
+    n_complete = 0
+    cum = 0
+    for n_i in tokens_per_item:
+        if positions[cum + n_i - 1] < max_length:
+            n_complete += 1
+            cum += n_i
+        else:
+            break
+
+    if n_complete >= n_items:
+        return mm_type_ids
+
+    # Trim pixel features and grid to the complete prefix.
+    keep_patches = sum(patches_per_item[:n_complete])
+    sample[pixel_key] = sample[pixel_key][:keep_patches]
+    sample[grid_key] = grid[:n_complete]
+
+    # Zero out orphaned placeholder tokens that fall inside the kept window; tokens
+    # beyond max_length are removed by truncation anyway (positions are sorted).
+    input_ids = list(sample["input_ids"])
+    mm_type_ids = list(mm_type_ids)
+    labels = list(sample["labels"]) if "labels" in sample else None
+    loss_weights = list(sample["loss_weights"]) if "loss_weights" in sample else None
+
+    for pos in positions[cum:]:
+        if pos >= max_length:
+            break
+        input_ids[pos] = 0
+        mm_type_ids[pos] = 0
+        if labels is not None:
+            labels[pos] = IGNORE_INDEX
+        if loss_weights is not None:
+            loss_weights[pos] = 0.0
+
+    sample["input_ids"] = input_ids
+    sample["mm_token_type_ids"] = mm_type_ids
+    if labels is not None:
+        sample["labels"] = labels
+    if loss_weights is not None:
+        sample["loss_weights"] = loss_weights
+    return mm_type_ids
+
+
+def _align_audio(sample: ModelInput, mm_type_ids: list[int], max_length: int, *, target: int = 3) -> list[int]:
+    """Trim and zero orphaned audio tokens for a single sample on truncation.
+
+    Returns the (possibly updated) ``mm_token_type_ids``.
+    """
+    if "input_features" not in sample or "feature_attention_mask" not in sample:
+        return mm_type_ids
+
+    n_items = sample["input_features"].shape[0]
+    if n_items == 0:
+        return mm_type_ids
+
+    positions = [i for i, t in enumerate(mm_type_ids) if t == target]
+    if not positions:
+        return mm_type_ids
+
+    # Group the marked positions into maximal contiguous runs; each run is one audio's token span.
+    runs: list[tuple[int, int]] = []
+    run_start = prev = positions[0]
+    for pos in positions[1:]:
+        if pos != prev + 1:
+            runs.append((run_start, prev))
+            run_start = pos
+        prev = pos
+    runs.append((run_start, prev))
+
+    # Layout must match the feature rows one-to-one, else bail rather than corrupt the mapping.
+    if len(runs) != n_items:
+        return mm_type_ids
+
+    # An audio is complete iff its last placeholder token lands inside the kept window.
+    n_complete = 0
+    for _start, end in runs:
+        if end < max_length:
+            n_complete += 1
+        else:
+            break
+
+    if n_complete >= n_items:
+        return mm_type_ids
+
+    # Trim feature rows to the complete prefix.
+    sample["input_features"] = sample["input_features"][:n_complete]
+    sample["feature_attention_mask"] = sample["feature_attention_mask"][:n_complete]
+
+    # Zero out orphaned placeholder tokens that fall inside the kept window; tokens beyond
+    # max_length are removed by truncation anyway.
+    input_ids = list(sample["input_ids"])
+    mm_type_ids = list(mm_type_ids)
+    labels = list(sample["labels"]) if "labels" in sample else None
+    loss_weights = list(sample["loss_weights"]) if "loss_weights" in sample else None
+
+    for start, end in runs[n_complete:]:
+        for pos in range(start, end + 1):
+            if pos >= max_length:
+                break
+            input_ids[pos] = 0
+            mm_type_ids[pos] = 0
+            if labels is not None:
+                labels[pos] = IGNORE_INDEX
+            if loss_weights is not None:
+                loss_weights[pos] = 0.0
+
+    sample["input_ids"] = input_ids
+    sample["mm_token_type_ids"] = mm_type_ids
+    if labels is not None:
+        sample["labels"] = labels
+    if loss_weights is not None:
+        sample["loss_weights"] = loss_weights
+    return mm_type_ids
+
+
+def _align_multimodal_on_truncation(sample: ModelInput, max_length: int) -> ModelInput:
+    """Remove orphaned multimodal data when the sequence will be truncated.
+
+    When cutoff_len truncates input_ids, media whose placeholder tokens are partially cut lose
+    their token<->feature correspondence. Trims pixel_values/grid_thw (vision) and
+    input_features/feature_attention_mask (audio) to the complete items and zeros out orphaned
+    placeholder tokens so the model ignores them.
+    """
+    mm_type_ids = sample.get("mm_token_type_ids")
+    if mm_type_ids is None:
+        return sample
+
+    sample = dict(sample)
+
+    mm_type_ids = _align_grid_media(
+        sample, mm_type_ids, max_length, target=1, grid_key="image_grid_thw", pixel_key="pixel_values"
+    )
+    mm_type_ids = _align_grid_media(
+        sample, mm_type_ids, max_length, target=2, grid_key="video_grid_thw", pixel_key="pixel_values_videos"
+    )
+    mm_type_ids = _align_audio(sample, mm_type_ids, max_length, target=3)
+
+    # Remove empty multimodal fields entirely
+    if "image_grid_thw" in sample and len(sample["image_grid_thw"]) == 0:
+        del sample["pixel_values"]
+        del sample["image_grid_thw"]
+    if "video_grid_thw" in sample and len(sample["video_grid_thw"]) == 0:
+        del sample["pixel_values_videos"]
+        del sample["video_grid_thw"]
+    if "input_features" in sample and sample["input_features"].shape[0] == 0:
+        del sample["input_features"]
+        del sample["feature_attention_mask"]
+
+    return sample
+
+
+def pad_and_truncate(samples: list[ModelInput], max_seqlen: int) -> list[BatchInput]:
+    max_length = min(max(len(sample["input_ids"]) for sample in samples), max_seqlen)
+    padded_samples = []
+    for sample in samples:
+        # Align multimodal fields before truncation: remove images/videos whose
+        # placeholder tokens would be partially cut, preventing pixel<->token mismatch.
+        if len(sample["input_ids"]) > max_length and any(k in sample for k in _MULTIMODAL_PASSTHROUGH_KEYS):
+            sample = _align_multimodal_on_truncation(sample, max_length)
+
+        padded_sample = {}
+        for key, value in sample.items():
+            if key in _MULTIMODAL_PASSTHROUGH_KEYS:
+                padded_sample[key] = value
+                continue
+
+            if "label" in key:
+                pad_value = IGNORE_INDEX
+            else:
+                pad_value = 0
+
+            if not isinstance(value, str):
+                padded_sample[key] = _pad_and_truncate(torch.tensor(value), max_length, pad_value)
+            else:
+                padded_sample[key] = value
+
+        padded_samples.append(padded_sample)
+
+    return padded_samples

src/llamafactory/v1/plugins/data_plugins/converter.py

@@ -14,11 +14,13 @@
 
 
 import json
+import re
 from typing import Any, Literal, NotRequired, TypedDict
 
 from ...utils import logging
+from ...utils.constants import AUDIO_PLACEHOLDER, IMAGE_PLACEHOLDER, VIDEO_PLACEHOLDER
 from ...utils.plugin import BasePlugin
-from ...utils.types import DPOSample, Sample, SFTSample, ToolCall
+from ...utils.types import Content, DPOSample, Sample, SFTSample, ToolCall
 
 
 logger = logging.get_logger(__name__)
@@ -29,6 +31,9 @@ class AlpacaSample(TypedDict, total=False):
     instruction: str
     input: NotRequired[str]
     output: str
+    images: NotRequired[list[str] | str]
+    videos: NotRequired[list[str] | str]
+    audios: NotRequired[list[str] | str]
 
 
 SharegptMessage = TypedDict(
@@ -40,6 +45,9 @@ SharegptMessage = TypedDict(
 class SharegptSample(TypedDict, total=False):
     conversations: list[SharegptMessage]
     tools: NotRequired[str]
+    images: NotRequired[list[str] | str]
+    videos: NotRequired[list[str] | str]
+    audios: NotRequired[list[str] | str]
 
 
 class OpenaiMessage(TypedDict, total=False):
@@ -54,6 +62,65 @@ class OpenaiSample(TypedDict, total=False):
 class PairSample(TypedDict, total=False):
     chosen: list[OpenaiMessage]
     rejected: list[OpenaiMessage]
+    images: NotRequired[list[str] | str]
+    videos: NotRequired[list[str] | str]
+    audios: NotRequired[list[str] | str]
+
+
+# Inline media tag -> v1 content block type, and the raw-sample column holding the paths.
+_MEDIA_SPECS: tuple[tuple[str, str, str], ...] = (
+    (IMAGE_PLACEHOLDER, "image_url", "images"),
+    (VIDEO_PLACEHOLDER, "video_url", "videos"),
+    (AUDIO_PLACEHOLDER, "audio_url", "audios"),
+)
+_TAG_TO_BLOCK = {tag: block_type for tag, block_type, _col in _MEDIA_SPECS}
+_TAG_PATTERN = re.compile("(" + "|".join(re.escape(tag) for tag, _b, _c in _MEDIA_SPECS) + ")")
+
+
+def _as_media_list(value: Any) -> list:
+    """Normalize a media column value into a list of paths/URLs (None -> [], scalar -> [scalar])."""
+    if value is None:
+        return []
+    if isinstance(value, (list, tuple)):
+        return list(value)
+    return [value]
+
+
+def _build_media_iters(raw_sample: dict[str, Any]) -> dict[str, Any]:
+    """Build per-modality path iterators from a raw sample's media columns."""
+    return {tag: iter(_as_media_list(raw_sample.get(col))) for tag, _block_type, col in _MEDIA_SPECS}
+
+
+def _to_content_blocks(text: str, media_iters: dict[str, Any]) -> list[Content]:
+    """Split ``text`` on inline media placeholders, interleaving media-url content blocks.
+
+    Each placeholder consumes the next path from its modality iterator (in document order). Plain
+    text with no placeholders yields a single text block (byte-identical to the legacy behavior).
+    Raises on an unmatched placeholder (more tags than media files).
+    """
+    if not _TAG_PATTERN.search(text):
+        return [{"type": "text", "value": text}]
+
+    blocks: list[Content] = []
+    for segment in _TAG_PATTERN.split(text):
+        block_type = _TAG_TO_BLOCK.get(segment)
+        if block_type is not None:
+            try:
+                path = next(media_iters[segment])
+            except StopIteration:
+                raise ValueError(f"More {segment} tags than provided media files.") from None
+            blocks.append({"type": block_type, "value": path})
+        elif segment:
+            blocks.append({"type": "text", "value": segment})
+    return blocks
+
+
+def _assert_media_consumed(media_iters: dict[str, Any]) -> None:
+    """Ensure every media file was referenced by a tag (fewer tags than media -> error)."""
+    for tag, media_iter in media_iters.items():
+        unused = len(list(media_iter))
+        if unused:
+            raise ValueError(f"Fewer {tag} tags than provided media files ({unused} unused).")
 
 
 class DataConverterPlugin(BasePlugin):
@@ -76,6 +143,7 @@ def alpaca_converter(raw_sample: AlpacaSample) -> SFTSample:
         SFTSample: SFT sample.
     """
     messages = []
+    media_iters = _build_media_iters(raw_sample)
     if "system" in raw_sample:
         messages.append(
             {"role": "system", "content": [{"type": "text", "value": raw_sample["system"]}], "loss_weight": 0.0}
@@ -85,9 +153,9 @@ def alpaca_converter(raw_sample: AlpacaSample) -> SFTSample:
         messages.append(
             {
                 "role": "user",
-                "content": [
+                "content": _to_content_blocks(
-                    {"type": "text", "value": raw_sample.get("instruction", "") + raw_sample.get("input", "")}
+                    raw_sample.get("instruction", "") + raw_sample.get("input", ""), media_iters
-                ],
+                ),
                 "loss_weight": 0.0,
             }
         )
@@ -97,6 +165,7 @@ def alpaca_converter(raw_sample: AlpacaSample) -> SFTSample:
             {"role": "assistant", "content": [{"type": "text", "value": raw_sample["output"]}], "loss_weight": 1.0}
         )
 
+    _assert_media_consumed(media_iters)
     return {"messages": messages}
 
 
@@ -121,6 +190,7 @@ def sharegpt_converter(raw_sample: SharegptSample) -> SFTSample:
     }
     sample = {}
     messages = []
+    media_iters = _build_media_iters(raw_sample)
     for message in raw_sample.get("conversations", []):
         tag = message["from"]
         if tag not in tag_mapping:
@@ -146,11 +216,12 @@ def sharegpt_converter(raw_sample: SharegptSample) -> SFTSample:
             messages.append(
                 {
                     "role": tag_mapping[tag],
-                    "content": [{"type": "text", "value": message["value"]}],
+                    "content": _to_content_blocks(message["value"], media_iters),
                     "loss_weight": 1.0 if tag == "gpt" else 0.0,
                 }
             )
 
+    _assert_media_consumed(media_iters)
     sample["messages"] = messages
 
     tools = raw_sample.get("tools")
@@ -178,6 +249,8 @@ def pair_converter(raw_sample: PairSample) -> DPOSample:
     """
 
     def process_message(raw_messages: list[OpenaiMessage]):
+        # chosen and rejected share the sample's media; each side consumes its own iterators.
+        media_iters = _build_media_iters(raw_sample)
         messages = []
         for message in raw_messages:
             if message["role"] == "tool":
@@ -201,11 +274,12 @@ def pair_converter(raw_sample: PairSample) -> DPOSample:
                 messages.append(
                     {
                         "role": message["role"],
-                        "content": [{"type": "text", "value": message["content"]}],
+                        "content": _to_content_blocks(message["content"], media_iters),
                         "loss_weight": 1.0 if message["role"] == "assistant" else 0.0,
                     }
                 )
 
+        _assert_media_consumed(media_iters)
         return messages
 
     sample = {}
@@ -221,3 +295,4 @@ def pair_converter(raw_sample: PairSample) -> DPOSample:
             logger.warning_rank0(f"Invalid tools format: {str(tools)}")
 
     return sample
+

src/llamafactory/v1/plugins/model_plugins/deepspeed_utils.py

@@ -14,9 +14,32 @@
 
 import json
 from copy import deepcopy
+from functools import lru_cache
 from typing import Any
 
 
+_registered_dist_config: Any | None = None
+
+
+def register_deepspeed_dist_config(dist_config: Any | None) -> None:
+    """Register backend config before model loading without involving the accelerator."""
+    global _registered_dist_config
+    _registered_dist_config = dist_config
+    is_deepspeed_zero3_enabled.cache_clear()
+
+
+@lru_cache(maxsize=1)
+def is_deepspeed_zero3_enabled() -> bool:
+    dist_config = _registered_dist_config
+    if dist_config is None or getattr(dist_config, "name", None) != "deepspeed":
+        return False
+
+    config_file = dist_config.get("config_file")
+    if not config_file:
+        return False
+    return _load_deepspeed_config(config_file).get("zero_optimization", {}).get("stage") == 3
+
+
 def _normalize_precision_enabled(value: Any) -> bool | str:
     if isinstance(value, str):
         value_lower = value.lower()
@@ -69,18 +92,19 @@ def _load_deepspeed_config(config_file: str) -> dict[str, Any]:
         return json.load(f)
 
 
-def setup_deepspeed_zero3_model_loading(is_train: bool, dist_config: dict[str, Any] | None):
+def setup_deepspeed_zero3_model_loading():
-    """Enable transformers' ZeRO-3-aware model loading for the current thread."""
+    """Enable ZeRO-3-aware model loading for the registered backend config."""
-    config_file = dist_config.get("config_file")
+    dist_config = _registered_dist_config
+    config_file = dist_config.get("config_file") if dist_config is not None else None
     if not config_file:
         raise ValueError("DeepSpeed config_file is required in dist_config")
 
     from accelerate.utils import DeepSpeedPlugin
 
     try:
-        from transformers.integrations import is_deepspeed_zero3_enabled
+        from transformers.integrations import is_deepspeed_zero3_enabled as _hf_is_deepspeed_zero3_enabled
     except ImportError:
-        from transformers.deepspeed import is_deepspeed_zero3_enabled
+        from transformers.deepspeed import is_deepspeed_zero3_enabled as _hf_is_deepspeed_zero3_enabled
 
     # DeepSpeed configs often use "auto" placeholders that only make sense once
     # we know the current runtime batch settings and precision mode.
@@ -109,7 +133,7 @@ def setup_deepspeed_zero3_model_loading(is_train: bool, dist_config: dict[str, A
     plugin.set_mixed_precision(mixed_precision)
     plugin.set_deepspeed_weakref()
 
-    if not is_deepspeed_zero3_enabled():
+    if not _hf_is_deepspeed_zero3_enabled():
         raise RuntimeError(
             "DeepSpeed ZeRO-3 model-loading bootstrap failed: transformers still reports zero3 disabled "
             "after constructing HfDeepSpeedConfig. This usually means the runtime is using a different transformers "

src/llamafactory/v1/plugins/model_plugins/kernels/base.py

@@ -12,76 +12,40 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""The definition of base kernel class.
-
-Init Phase:
-1. Define base kernel class.
-2. Define abstract methods.
-
-"""
-
 from abc import ABC, abstractmethod
-from typing import Any
 
-from ....accelerator.helper import DeviceType, get_current_accelerator
+from ....utils.plugin import BasePlugin, ensure_methods_implemented
 from ....utils.types import HFModel
 
 
+class KernelPlugin(BasePlugin):
+    """Plugin family for model kernel optimization classes."""
+
+
 class BaseKernel(ABC):
-    r"""Base class for all kernel implementations.
+    """Template base for concrete kernel implementations."""
 
-    Subclasses must implement the abstract methods and define the required class attributes.
+    def __init_subclass__(cls, **kwargs) -> None:
-    """
+        super().__init_subclass__(**kwargs)
+        ensure_methods_implemented(cls)
 
-    _kernel_id: Any = ""  # kernel ID, any hashable value to identify a kernel implementation
+    @staticmethod
-    _device: list[DeviceType] = [DeviceType.CPU]  # "cuda", "npu", "cpu", etc.
-
-    @classmethod
-    def get_kernel_id(cls) -> str:
-        """Returns the unique identifier for the kernel."""
-        return cls._kernel_id
-
-    @classmethod
-    def get_device(cls) -> list[DeviceType]:
-        """Returns the device type list associated with the kernel (e.g., ["cuda", "npu", "cpu"])."""
-        return cls._device
-
-    @classmethod
-    def check_deps(cls) -> bool:
-        """Checks if the required dependencies for the kernel are available.
-
-        Returns:
-            bool: ``True`` if dependencies are met, ``False`` otherwise.
-
-        .. note::
-            In explicit mode, if a user specifies an implementation but this check fails,
-            it should raise an error instead of silently switching.
-            Kernels can override this method to implement custom dependency checks.
-        """
-        if get_current_accelerator().type not in cls._device:
-            return False
-        return True
-
-    @classmethod
     @abstractmethod
+    def check_device() -> None: ...
+
+    @staticmethod
+    def check_deps() -> None:
+        pass
+
+    @classmethod
     def apply(cls, **kwargs) -> HFModel:
-        """Applies the kernel optimization to the model.
+        cls.check_device()
+        cls.check_deps()
+        if kwargs.get("model") is None:
+            raise ValueError(f"HFModel instance is required for {cls.__name__}.")
 
-        Args:
+        return cls._apply(**kwargs)
-            **kwargs: Arbitrary keyword arguments, usually containing the model instance and the kernel configuration.
 
-        Returns:
+    @staticmethod
-            HFModel: The model with the kernel applied.
+    @abstractmethod
-
+    def _apply(**kwargs) -> HFModel: ...
-        Raises:
-            RuntimeError: If the kernel dependencies are not met.
-            NotImplementedError: If the method is not implemented by the subclass.
-
-        Example:
-            >>> from llamafactory.v1.plugins.model_plugins.kernels.interface import apply_kernel
-            >>> model = HFModel(config=config)
-            >>> model = apply_kernel(model=model, kernel_id="npu_fused_moe")
-        """
-        if not cls.check_deps():
-            raise RuntimeError(f"{cls.__name__} is not available but {cls.__name__} kernel was called.")
-        raise NotImplementedError

src/llamafactory/v1/plugins/model_plugins/kernels/interface.py

@@ -12,174 +12,64 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""The definition of kernel interface.
+from typing import Any
 
-Init Phase:
+from ....accelerator.helper import DeviceType, get_current_accelerator
-1. Scan all kernels.
-2. Register default kernels.
-3. Define kernel plugin.
-
-"""
-
-import importlib
-from pathlib import Path
-
-from ....utils import logging
-from ....utils.plugin import BasePlugin
 from ....utils.types import HFModel
-from .registry import Registry
+from .base import KernelPlugin
+
+# Import built-in implementations so their class decorators populate the registry.
+from .liger_kernel_ops import LigerKernel  # noqa: F401
+from .ops.linear_attention.fla import FlashLinearAttentionKernel  # noqa: F401
+from .ops.mlp.cuda_fused_moe import CudaFusedMoEKernel  # noqa: F401
+from .ops.mlp.npu_fused_moe import NpuFusedMoEKernel  # noqa: F401
+from .ops.mlp.npu_swiglu import NpuSwiGluKernel  # noqa: F401
+from .ops.rms_norm.npu_rms_norm import NpuRMSNormKernel  # noqa: F401
+from .ops.rope.npu_rope import NpuRoPEKernel  # noqa: F401
 
 
-logger = logging.get_logger(__name__)
+_AUTO_KERNELS = {
+    DeviceType.NPU: ("npu_fused_moe", "npu_fused_rmsnorm", "npu_fused_rope", "npu_fused_swiglu"),
+}
 
 
-def scan_all_kernels():
+def _apply_auto_kernels(model: HFModel, **kwargs) -> HFModel:
-    """Scan all kernels in the ``ops`` directory.
+    device_type = get_current_accelerator().type
-
+    for kernel_name in _AUTO_KERNELS.get(device_type, ()):
-    Scans the ``ops`` directory for all ``.py`` files and attempts to import them.
+        model = KernelPlugin(kernel_name).apply(model=model, **kwargs)
-    Importing triggers the :func:`~registry.register_kernel` decorator, which automatically registers the kernels.
-
-    Returns:
-        dict[str, type[BaseKernel]]: A dictionary of registered kernels.
-
-    .. note::
-        This function assumes that the ``ops`` directory is located in the same directory as this file.
-        It recursively searches for ``.py`` files and constructs the module path for import.
-    """
-    ops_path = Path(__file__).parent / "ops"
-
-    if not ops_path.exists():
-        return
-
-    base_package = __package__
-
-    for file_path in ops_path.rglob("*.py"):
-        if file_path.name == "__init__.py":
-            continue
-
-        # calculate the relative path:
-        # file_path = .../kernels_v2/ops/mlp/npu_swiglu.py
-        # rel_path  = ops/mlp/npu_swiglu.py
-        rel_path = file_path.relative_to(Path(__file__).parent)
-
-        # build module path:
-        module_name = ".".join(rel_path.parts)[:-3]
-        full_module_name = f"{base_package}.{module_name}"
-
-        try:
-            importlib.import_module(full_module_name)
-        except Exception as e:
-            logger.warning(f"[Kernel Registry] Failed to import {full_module_name} when loading kernels: {e}")
-
-    return Registry.get_registered_kernels()
-
-
-default_kernels = scan_all_kernels()
-
-
-def get_default_kernels():
-    """Get a list of default registered kernel IDs.
-
-    Returns:
-        list[str]: List of kernel IDs.
-    """
-    return list(default_kernels.keys())
-
-
-def apply_kernel(kernel_id: str, **kwargs):
-    """Applies a specific kernel to the model.
-
-    Args:
-        kernel_id (str): The ID of the kernel to apply.
-        **kwargs: Keyword arguments passed to the kernel application function.
-                  Typically includes the model instance.
-
-    Returns:
-        HFModel: The model with applied kernel.
-    """
-    kernel = default_kernels.get(kernel_id)
-    if kernel is None:
-        raise ValueError(f"Kernel {kernel_id} not found")
-
-    kernel.apply(**kwargs)
-
-
-class KernelPlugin(BasePlugin):
-    """Plugin for managing kernel optimizations."""
-
-    pass
-
-
-@KernelPlugin("auto").register()
-def apply_default_kernels(model: HFModel, include_kernels: str = None) -> HFModel:
-    """Applies all default registered kernels to the model.
-
-    Args:
-        model (HFModel): The model instance to apply kernels to.
-        include_kernels (str, optional): Comma-separated list of kernel IDs to apply.
-                                         If "auto" or True, applies all default kernels.
-                                         If None or False, no kernels are applied.
-                                         Defaults to None.
-
-    Returns:
-        HFModel: The model with applied kernels.
-    """
-    if not include_kernels:
-        return model
-    elif include_kernels == "auto" or include_kernels is True:
-        use_kernels = default_kernels.keys()
-    else:
-        use_kernels = include_kernels.split(",")  # "kernel_id1,kernel_id2,kernel_id3"
-
-    for kernel in use_kernels:
-        if kernel not in default_kernels:
-            raise ValueError(f"Kernel {kernel} not found")
-
-        apply_kernel(kernel, model=model)
 
     return model
 
 
-@KernelPlugin("liger_kernel").register()
+def apply_kernels(model: HFModel, config: dict[str, Any], require_logits: bool = False) -> HFModel:
-def apply_liger_kernels(
+    """Apply the comma-separated kernel names selected by ``kernel_config.name``."""
-    model: HFModel,
+    kernel_names = config.get("name")
-    include_kernels: str = None,
+    if not isinstance(kernel_names, str):
-    require_logits: bool = False,
+        raise TypeError("kernel_config.name must be a string.")
-) -> HFModel:
-    """Applies Liger kernel to the model.
 
-    Args:
+    names = [name.strip() for name in kernel_names.split(",") if name.strip()]
-        model (HFModel): The model instance to apply kernels to.
+    if not names:
-        include_kernels (str, optional): If ``"auto"`` or ``True``, apply Liger with
+        raise ValueError("kernel_config.name must contain at least one kernel name.")
-                                         library defaults. If a comma-separated list (e.g.
-                                         ``rope,rms_norm``), enable only those ops; names match
-                                         ``apply_liger_kernel_to_*`` kwargs: ``rope``, ``rms_norm``,
-                                         ``swiglu``, ``cross_entropy``, ``fused_linear_cross_entropy``.
-                                         If ``None`` or ``False``, do nothing. Defaults to ``None``.
-        require_logits (bool, optional): When true, disables ``fused_linear_cross_entropy`` in favor
-                                         of non-fused CE so the forward pass returns ``logits``. Needed
-                                         for trainers that compute weighted loss from logits (e.g. v1
-                                         SFT with ``loss_weights``). Defaults to ``False`` (fused CE
-                                         when supported). The v1 ``run_sft`` entrypoint sets
-                                         ``require_logits`` to true for ``liger_kernel`` when the key
-                                         is omitted so SFT weighted loss keeps working.
 
-    Returns:
+    for name in names:
-        HFModel: The model with Liger kernel applied.
+        if name == "auto":
-    """
+            model = _apply_auto_kernels(model=model, config=config, require_logits=require_logits)
-    if not include_kernels:
+        else:
-        return model
+            model = KernelPlugin(name).apply(model=model, config=config, require_logits=require_logits)
-    if include_kernels == "auto" or include_kernels is True:
-        use_kernels = "auto"
-    else:
-        use_kernels = [k.strip() for k in include_kernels.split(",") if k.strip()]
-        if not use_kernels:
-            return model
 
-    try:
+    return model
-        from .liger_kernel_ops import LigerKernel
+
-    except ImportError as e:
+
-        logger.warning_rank0(f"[Kernel] Failed to import liger_kernel ops, skip. Error: {e}")
+def apply_v1_kernels(model: HFModel, use_v1_kernels: bool) -> HFModel:
+    """Apply v1 automatic kernels for the transitional v0 ``use_v1_kernels`` option."""
+    if not use_v1_kernels:
         return model
 
-    return LigerKernel.apply(use_kernels=use_kernels, model=model, require_logits=require_logits)
+    return apply_kernels(model, {"name": "auto"})
+
+
+def apply_kernel(kernel_id: str, **kwargs) -> HFModel:
+    if kernel_id == "auto":
+        return _apply_auto_kernels(**kwargs)
+
+    return KernelPlugin(kernel_id).apply(**kwargs)

src/llamafactory/v1/plugins/model_plugins/kernels/liger_kernel_ops.py

@@ -25,7 +25,7 @@ import inspect
 from ....accelerator.helper import DeviceType, get_current_accelerator
 from ....utils.logging import get_logger
 from ....utils.types import HFModel
-from .base import BaseKernel
+from .base import BaseKernel, KernelPlugin
 
 
 logger = get_logger(__name__)
@@ -41,26 +41,26 @@ _LIGER_FN_BY_MODEL_TYPE: dict[str, str] = {
 }
 
 
+@KernelPlugin("liger_kernel").register()
 class LigerKernel(BaseKernel):
     """Liger Kernel for optimized model training."""
 
-    _device = [DeviceType.CUDA, DeviceType.NPU]
+    @staticmethod
+    def check_device() -> None:
+        current = get_current_accelerator().type
+        if current not in (DeviceType.CUDA, DeviceType.NPU):
+            raise RuntimeError(f"LigerKernel requires CUDA or NPU, current accelerator is {current}.")
 
-    @classmethod
+    @staticmethod
-    def check_deps(cls) -> bool:
+    def check_deps() -> None:
         """Checks if the required dependencies for the kernel are available."""
         try:
             import liger_kernel  # noqa: F401
-
-            return super().check_deps()
         except ImportError:
-            logger.warning_rank0(
+            raise RuntimeError("Liger kernel is not installed.") from None
-                "Liger kernel is not installed, the kernel_config liger_kernel will be ignored. Please install it from https://github.com/linkedin/Liger-Kernel."
-            )
-            return False
 
-    @classmethod
+    @staticmethod
-    def apply(cls, **kwargs) -> "HFModel":
+    def _apply(**kwargs) -> "HFModel":
         """Applies the Liger kernel to the model.
 
         Args:
@@ -78,16 +78,12 @@ class LigerKernel(BaseKernel):
             RuntimeError: If dependencies are not met.
         """
         model = kwargs.get("model")
-        use_kernels = kwargs.get("use_kernels", None)
+        config = kwargs.get("config")
-        if model is None:
+        use_kernels = kwargs.get("use_kernels", "auto")
-            raise ValueError(f"HFModel instance is required for {cls.__name__}.")
-
-        if not cls.check_deps():
-            raise RuntimeError(
-                f"current device is not supported by liger_kernel. Current device is {get_current_accelerator().type}, supported devices are {cls.get_device()}"
-            )
 
         require_logits = kwargs.get("require_logits", False)
+        if config is not None:
+            require_logits = config.get("require_logits", require_logits)
 
         model_type = getattr(model.config, "model_type", None)
 

src/llamafactory/v1/plugins/model_plugins/kernels/ops/linear_attention/fla.py

@@ -0,0 +1,103 @@
+# Copyright 2025 the LlamaFactory team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Flash Linear Attention kernel plugin backed by FSDPTurbo's operator registry."""
+
+from functools import partial
+
+from ......accelerator.helper import DeviceType, get_current_accelerator
+from ......utils import logging
+from ......utils.types import HFModel
+from ...base import BaseKernel, KernelPlugin
+
+
+logger = logging.get_logger(__name__)
+
+CHUNK_GATED_DELTA_RULE = "chunk_gated_delta_rule"
+FUSED_RECURRENT_GATED_DELTA_RULE = "fused_recurrent_gated_delta_rule"
+FLASH_LINEAR_ATTENTION_KERNELS = (
+    CHUNK_GATED_DELTA_RULE,
+    FUSED_RECURRENT_GATED_DELTA_RULE,
+)
+FLA_MODULE_ATTRIBUTES = {
+    CHUNK_GATED_DELTA_RULE: "chunk_gated_delta_rule",
+    FUSED_RECURRENT_GATED_DELTA_RULE: "recurrent_gated_delta_rule",
+}
+SUPPORTED_CHUNK_SIZES = (16, 32, 64)
+
+
+@KernelPlugin("flash-linear-attention").register()
+class FlashLinearAttentionKernel(BaseKernel):
+    """Install selected FLA callables through FSDPTurbo's device operator registry."""
+
+    @staticmethod
+    def check_device() -> None:
+        current = get_current_accelerator().type
+        if current not in (DeviceType.CUDA, DeviceType.NPU):
+            raise RuntimeError(f"FlashLinearAttentionKernel requires CUDA or NPU, current accelerator is {current}.")
+
+    @staticmethod
+    def check_deps() -> None:
+        try:
+            import fla.ops.gated_delta_rule  # noqa: F401
+            import fsdp_turbo.ops.fla  # noqa: F401
+            from fsdp_turbo.ops.registry import get_op  # noqa: F401
+            from fsdp_turbo.utils.patch import patch_model_members  # noqa: F401
+        except ImportError as exc:
+            raise RuntimeError("Flash Linear Attention and FSDPTurbo are required for this kernel.") from exc
+
+    @staticmethod
+    def _apply(**kwargs) -> HFModel:
+        model = kwargs["model"]
+        config = kwargs.get("config") or {}
+        include_kernels = config.get("include_kernels", "auto")
+        chunk_size = config.get("chunk_size", 64)
+
+        if include_kernels == "auto" or include_kernels is True:
+            selected = list(FLASH_LINEAR_ATTENTION_KERNELS)
+        elif isinstance(include_kernels, str):
+            selected = [name.strip() for name in include_kernels.split(",") if name.strip()]
+        else:
+            raise TypeError("kernel_config.include_kernels must be 'auto' or a comma-separated string.")
+
+        if not selected:
+            raise ValueError("kernel_config.include_kernels must select at least one FLA kernel.")
+
+        unsupported = set(selected).difference(FLASH_LINEAR_ATTENTION_KERNELS)
+        if unsupported:
+            raise ValueError(f"Unsupported Flash Linear Attention kernels: {sorted(unsupported)}")
+        if isinstance(chunk_size, bool) or not isinstance(chunk_size, int) or chunk_size not in SUPPORTED_CHUNK_SIZES:
+            raise ValueError(f"chunk_size must be one of {SUPPORTED_CHUNK_SIZES}, got {chunk_size!r}.")
+
+        from fsdp_turbo.ops.registry import get_op
+        from fsdp_turbo.utils.patch import patch_model_members
+
+        patched = 0
+        named_modules = tuple(model.named_modules())
+        for op_name in selected:
+            module_attribute = FLA_MODULE_ATTRIBUTES[op_name]
+            op = get_op(op_name)
+            configured_op = partial(op, chunk_size=chunk_size) if op_name == CHUNK_GATED_DELTA_RULE else op
+            targets = {
+                f"{type(module).__module__}.{type(module).__name__}.{module_attribute}"
+                for _, module in named_modules
+                if callable(getattr(module, module_attribute, None))
+            }
+            matched = patch_model_members(model, sorted(targets), configured_op) if targets else 0
+            if matched == 0:
+                raise RuntimeError(f"FLA operator `{op_name}` did not match any model module attributes.")
+            patched += matched
+
+        logger.info_rank0(f"Flash Linear Attention kernels updated {patched} module callables: {selected}.")
+        return model

src/llamafactory/v1/plugins/model_plugins/kernels/ops/mlp/cuda_fused_moe.py

@@ -27,16 +27,22 @@ import types
 import torch
 import torch.nn.functional as F
 
-from ......accelerator.helper import DeviceType
+from ......accelerator.helper import DeviceType, get_current_accelerator
 from ......utils.types import HFModel
-from ...base import BaseKernel
+from ...base import BaseKernel, KernelPlugin
-from ...registry import register_kernel
+
-from .triton_grouped_gemm import (
+
-    group_gemm_same_mn,
+try:
-    group_gemm_same_nk,
+    from .triton_grouped_gemm import (
-    moe_gather,
+        group_gemm_same_mn,
-    moe_scatter,
+        group_gemm_same_nk,
-)
+        moe_gather,
+        moe_scatter,
+    )
+except ImportError as exc:
+    _TRITON_IMPORT_ERROR = exc
+else:
+    _TRITON_IMPORT_ERROR = None
 
 
 logger = logging.getLogger(__name__)
@@ -351,7 +357,7 @@ _TRITON_MOE_MAPPING: dict[str, dict[str, object]] = {
 # ---------------------------------------------------------------------------
 
 
-@register_kernel
+@KernelPlugin("cuda_fused_moe").register()
 class CudaFusedMoEKernel(BaseKernel):
     """Pure-Triton fused MoE kernel for NVIDIA CUDA GPUs.
 
@@ -362,30 +368,20 @@ class CudaFusedMoEKernel(BaseKernel):
     Requires: CUDA GPU + Triton
     """
 
-    _kernel_id = "cuda_fused_moe"
+    @staticmethod
-    _device = DeviceType.CUDA
+    def check_device() -> None:
+        current = get_current_accelerator().type
+        if current != DeviceType.CUDA:
+            raise RuntimeError(f"CudaFusedMoEKernel requires CUDA, current accelerator is {current}.")
 
-    @classmethod
+    @staticmethod
-    def check_deps(cls) -> bool:
+    def check_deps() -> None:
-        if not super().check_deps():
+        if _TRITON_IMPORT_ERROR is not None:
-            return False
+            raise RuntimeError("cuda_fused_moe requires Triton.") from _TRITON_IMPORT_ERROR
-        try:
-            import triton  # noqa: F401
 
-            return True
+    @staticmethod
-        except ImportError:
+    def _apply(**kwargs) -> HFModel:
-            logger.info("cuda_fused_moe: Triton not available, kernel disabled.")
-            return False
-
-    @classmethod
-    def apply(cls, **kwargs) -> HFModel:
         model = kwargs.get("model")
-        if model is None:
-            raise ValueError(f"HFModel instance is required for {cls.__name__}.")
-
-        if not cls.check_deps():
-            logger.warning("cuda_fused_moe: Dependencies not met. Skipping kernel application.")
-            return model
 
         archs = getattr(model.config, "architectures", None) or []
         target_mapping = None

src/llamafactory/v1/plugins/model_plugins/kernels/ops/mlp/npu_fused_moe.py

@@ -29,14 +29,19 @@ import torch.nn.functional as F
 
 try:
     import torch_npu
-except ImportError:
+except ImportError as exc:
-    pass
+    _TORCH_NPU_IMPORT_ERROR = exc
+else:
+    _TORCH_NPU_IMPORT_ERROR = None
 
-from ......accelerator.helper import DeviceType
+from ......accelerator.helper import DeviceType, get_current_accelerator
+from ......utils.logging import get_logger
 from ......utils.packages import is_transformers_version_greater_than
 from ......utils.types import HFModel
-from ...base import BaseKernel
+from ...base import BaseKernel, KernelPlugin
-from ...registry import register_kernel
+
+
+logger = get_logger(__name__)
 
 
 class GmmFunction(torch.autograd.Function):
@@ -50,7 +55,7 @@ class GmmFunction(torch.autograd.Function):
             ctx: Context object to save tensors for backward pass.
             x (Tensor): Input tensor.
             weight (Tensor): Weight tensor.
-            group_list (list): List of group sizes.
+            group_list (Tensor): Number of tokens assigned to each expert.
 
         Returns:
             Tensor: The result of the grouped matrix multiplication.
@@ -175,14 +180,14 @@ class HybridGmmFunction(torch.autograd.Function):
         return (None, *grad_x_list, *grad_w_list)
 
 
-class NpuMoeFused:
+class NpuMoeFusedV4:
-    """Container for NPU fused MoE forward functions."""
+    """Container for Transformers v4 NPU fused MoE forward functions."""
 
     @staticmethod
-    def npu_moe_experts_forward(
+    def stacked_experts_forward(
         self, hidden_states: torch.Tensor, routing_weights: torch.Tensor, router_indices: torch.Tensor
     ) -> torch.Tensor:
-        """Forward pass for MoE experts using NPU fused operations.
+        """Forward pass for Transformers v4 MoE experts using NPU fused operations.
 
         Args:
             self: The MoE layer instance.
@@ -198,7 +203,9 @@ class NpuMoeFused:
         permuted_hidden_states, row_ids_map = torch_npu.npu_moe_token_permute(
             hidden_states, router_indices.to(torch.int32)
         )
-        tokens_per_expert = torch.histc(router_indices, bins=self.num_experts, min=0, max=self.num_experts)
+        tokens_per_expert = torch.histc(
+            router_indices.float(), bins=self.num_experts, min=0, max=self.num_experts
+        ).long()
         intermediate_hidden_states = GmmFunction.apply(permuted_hidden_states, self.gate_up_proj, tokens_per_expert)
         intermediate_activations = torch_npu.npu_swiglu(intermediate_hidden_states, dim=-1)
         output = GmmFunction.apply(intermediate_activations, self.down_proj, tokens_per_expert)
@@ -207,61 +214,33 @@ class NpuMoeFused:
         return next_states
 
     @staticmethod
-    def npu_moe_sparse_block_forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+    def stacked_sparse_block_forward(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
-        r"""Forward pass for sparse MoE block using NPU optimization.
+        r"""Forward pass for Transformers v4 sparse MoE block using NPU optimization.
 
         Args:
             self: The MoE sparse block instance.
             hidden_states (Tensor): Input hidden states.
 
         Returns:
-            Tensor: The routed output.
+            tuple: A tuple containing the routed output and router logits.
         """
         batch_size = hidden_states.shape[0]
         hidden_states = hidden_states.reshape(-1, self.hidden_size)
         router_logits = self.gate(hidden_states)
-        routing_weights = torch.nn.functional.softmax(router_logits, dim=-1, dtype=torch.float)
+        routing_weights = F.softmax(router_logits, dim=-1, dtype=torch.float)
         routing_weights, router_indices = torch.topk(routing_weights, self.top_k, dim=-1)
         routing_weights = routing_weights / routing_weights.sum(dim=-1, keepdim=True)
         routing_weights = routing_weights.to(hidden_states.dtype)
         hidden_states = hidden_states.reshape(batch_size, -1, self.hidden_size)
         routed_out = self.experts(hidden_states, routing_weights, router_indices)
-        return routed_out
+        return routed_out, router_logits
 
     @staticmethod
-    def npu_moe_experts_v5_forward(
+    def sparse_block_forward(self, hidden_states: torch.Tensor):
-        self, hidden_states: torch.Tensor, top_k_index: torch.Tensor, top_k_weights: torch.Tensor
+        """Forward pass for a Transformers v4 list-backed sparse MoE block using NPU fused operations.
-    ) -> torch.Tensor:
-        """Forward pass for Transformers v5+ MoE experts using NPU fused operations.
-
-        Transformers v5 stores expert weights in F.linear layout:
-        gate_up_proj: [num_experts, 2 * intermediate_dim, hidden_dim]
-        down_proj: [num_experts, hidden_dim, intermediate_dim]
-        The NPU grouped matmul path expects matmul layout, so both weights are transposed.
-        """
-        hidden_states = hidden_states.reshape(-1, self.hidden_dim)
-        permuted_hidden_states, row_ids_map = torch_npu.npu_moe_token_permute(
-            hidden_states, top_k_index.to(torch.int32)
-        )
-        tokens_per_expert = torch.histc(top_k_index.float(), bins=self.num_experts, min=0, max=self.num_experts).long()
-
-        gate_up_proj = self.gate_up_proj.transpose(1, 2)
-        down_proj = self.down_proj.transpose(1, 2)
-        intermediate_hidden_states = GmmFunction.apply(permuted_hidden_states, gate_up_proj, tokens_per_expert)
-        intermediate_activations = torch_npu.npu_swiglu(intermediate_hidden_states, dim=-1)
-        output = GmmFunction.apply(intermediate_activations, down_proj, tokens_per_expert)
-        return torch_npu.npu_moe_token_unpermute(output, row_ids_map, probs=top_k_weights)
-
-
-class Qwen3NpuMoeFused:
-    """Container for Qwen3 NPU fused MoE forward functions."""
-
-    @staticmethod
-    def qwen3moe_sparse_moe_block_forward(self, hidden_states: torch.Tensor):
-        """Forward pass for Qwen3 sparse MoE block using NPU fused operations.
 
         Args:
-            self: The Qwen3 MoE block instance.
+            self: The sparse MoE block instance.
             hidden_states (Tensor): Input hidden states.
 
         Returns:
@@ -305,44 +284,115 @@ class Qwen3NpuMoeFused:
         next_states = next_states.view(batch_size, sequence_length, -1)
         return next_states, router_logits
 
+    @staticmethod
+    def shared_sparse_block_forward(self, hidden_states: torch.Tensor):
+        """Forward pass for a Transformers v4 sparse MoE block with a shared expert."""
+        next_states, router_logits = NpuMoeFusedV4.sparse_block_forward(self, hidden_states)
 
-# moe patch config mapping
+        shared_expert_output = self.shared_expert(hidden_states)
-if is_transformers_version_greater_than("5.0.0"):
+        shared_expert_output = F.sigmoid(self.shared_expert_gate(hidden_states)) * shared_expert_output
-    kernel_moe_mapping = {
+        next_states = next_states + shared_expert_output
-        "Qwen3MoeForCausalLM": {
+        return next_states, router_logits
-            "Qwen3MoeExperts": NpuMoeFused.npu_moe_experts_v5_forward,
-        },
-        "Qwen3VLMoeForConditionalGeneration": {
-            "Qwen3VLMoeTextExperts": NpuMoeFused.npu_moe_experts_v5_forward,
-        },
-        "Qwen3_5MoeForCausalLM": {
-            "Qwen3_5MoeExperts": NpuMoeFused.npu_moe_experts_v5_forward,
-        },
-        "Qwen3_5MoeForConditionalGeneration": {
-            "Qwen3_5MoeExperts": NpuMoeFused.npu_moe_experts_v5_forward,
-        },
-    }
-else:
-    kernel_moe_mapping = {
-        "Qwen3MoeForCausalLM": {
-            "Qwen3MoeSparseMoeBlock": Qwen3NpuMoeFused.qwen3moe_sparse_moe_block_forward,
-        },
-        "Qwen3VLMoeForConditionalGeneration": {
-            "Qwen3VLMoeTextExperts": NpuMoeFused.npu_moe_experts_forward,
-            "Qwen3VLMoeTextSparseMoeBlock": NpuMoeFused.npu_moe_sparse_block_forward,
-        },
-    }
 
 
-@register_kernel
+class NpuMoeFusedV5:
+    """Container for Transformers v5 NPU fused MoE forward functions."""
+
+    @staticmethod
+    def experts_forward(
+        self, hidden_states: torch.Tensor, top_k_index: torch.Tensor, top_k_weights: torch.Tensor
+    ) -> torch.Tensor:
+        """Forward pass for Transformers v5+ MoE experts using NPU fused operations.
+
+        Transformers v5 stores expert weights in F.linear layout:
+        gate_up_proj: [num_experts, 2 * intermediate_dim, hidden_dim]
+        down_proj: [num_experts, hidden_dim, intermediate_dim]
+        The NPU grouped matmul path expects matmul layout, so both weights are transposed.
+        """
+        hidden_states = hidden_states.reshape(-1, self.hidden_dim)
+        permuted_hidden_states, row_ids_map = torch_npu.npu_moe_token_permute(
+            hidden_states, top_k_index.to(torch.int32)
+        )
+        tokens_per_expert = torch.histc(top_k_index.float(), bins=self.num_experts, min=0, max=self.num_experts).long()
+
+        gate_up_proj = self.gate_up_proj.transpose(1, 2)
+        down_proj = self.down_proj.transpose(1, 2)
+        intermediate_hidden_states = GmmFunction.apply(permuted_hidden_states, gate_up_proj, tokens_per_expert)
+        intermediate_activations = torch_npu.npu_swiglu(intermediate_hidden_states, dim=-1)
+        output = GmmFunction.apply(intermediate_activations, down_proj, tokens_per_expert)
+        return torch_npu.npu_moe_token_unpermute(output, row_ids_map, probs=top_k_weights)
+
+
+_V4_MODEL_TYPE_TO_PATCHES = {
+    "qwen3_moe": {
+        "Qwen3MoeSparseMoeBlock": NpuMoeFusedV4.sparse_block_forward,
+    },
+    "qwen3_next": {
+        "Qwen3NextSparseMoeBlock": NpuMoeFusedV4.shared_sparse_block_forward,
+    },
+    "qwen3_omni_moe": {
+        "Qwen3OmniMoeThinkerTextSparseMoeBlock": NpuMoeFusedV4.sparse_block_forward,
+        "Qwen3OmniMoeTalkerTextSparseMoeBlock": NpuMoeFusedV4.shared_sparse_block_forward,
+    },
+    "qwen3_omni_moe_thinker": {
+        "Qwen3OmniMoeThinkerTextSparseMoeBlock": NpuMoeFusedV4.sparse_block_forward,
+    },
+    "qwen3_vl_moe": {
+        "Qwen3VLMoeTextExperts": NpuMoeFusedV4.stacked_experts_forward,
+        "Qwen3VLMoeTextSparseMoeBlock": NpuMoeFusedV4.stacked_sparse_block_forward,
+    },
+}
+
+_V5_MODEL_TYPE_TO_PATCHES = {
+    "qwen3_moe": {
+        "Qwen3MoeExperts": NpuMoeFusedV5.experts_forward,
+    },
+    "qwen3_next": {
+        "Qwen3NextExperts": NpuMoeFusedV5.experts_forward,
+    },
+    "qwen3_omni_moe": {
+        "Qwen3OmniMoeThinkerTextExperts": NpuMoeFusedV5.experts_forward,
+        "Qwen3OmniMoeTalkerTextExperts": NpuMoeFusedV5.experts_forward,
+    },
+    "qwen3_omni_moe_thinker": {
+        "Qwen3OmniMoeThinkerTextExperts": NpuMoeFusedV5.experts_forward,
+    },
+    "qwen3_vl_moe": {
+        "Qwen3VLMoeTextExperts": NpuMoeFusedV5.experts_forward,
+    },
+    "qwen3_5_moe": {
+        "Qwen3_5MoeExperts": NpuMoeFusedV5.experts_forward,
+    },
+}
+
+_MODEL_TYPE_TO_PATCHES = (
+    _V5_MODEL_TYPE_TO_PATCHES if is_transformers_version_greater_than("5.0.0") else _V4_MODEL_TYPE_TO_PATCHES
+)
+
+
+@KernelPlugin("npu_fused_moe").register()
 class NpuFusedMoEKernel(BaseKernel):
     """NPU Fused MoE Kernel implementation."""
 
-    _kernel_id = "npu_fused_moe"
+    @staticmethod
-    _device = DeviceType.NPU
+    def check_device() -> None:
+        current = get_current_accelerator().type
+        if current != DeviceType.NPU:
+            raise RuntimeError(f"NpuFusedMoEKernel requires NPU, current accelerator is {current}.")
 
-    @classmethod
+    @staticmethod
-    def apply(cls, **kwargs) -> HFModel:
+    def check_deps() -> None:
+        if _TORCH_NPU_IMPORT_ERROR is not None:
+            raise RuntimeError("NpuFusedMoEKernel requires torch_npu.") from _TORCH_NPU_IMPORT_ERROR
+
+    @staticmethod
+    def _get_patch_forward(model_type: str, module: torch.nn.Module):
+        """Return the version-specific NPU forward function for a matched MoE module."""
+        model_patches = _MODEL_TYPE_TO_PATCHES.get(model_type, {})
+        return model_patches.get(module.__class__.__name__)
+
+    @staticmethod
+    def _apply(**kwargs) -> HFModel:
         """Applies the NPU fused MoE kernel to the model.
 
         Args:
@@ -350,32 +400,21 @@ class NpuFusedMoEKernel(BaseKernel):
 
         Returns:
             HFModel: The model with patched MoE forward functions.
-
-        Raises:
-            ValueError: If the model is not provided.
-            RuntimeError: If dependencies are not met.
         """
-        model = kwargs.get("model", None)
+        model = kwargs["model"]
-        if model is None:
-            raise ValueError(f"HFModel instance is required for {cls.__name__}.")
 
-        if not cls.check_deps():
+        model_type = getattr(model.config, "model_type", None)
-            raise RuntimeError("torch_npu is not available but NpuMoEFusedMoEKernel was called.")
+        if model_type not in _MODEL_TYPE_TO_PATCHES:
-
-        archs = getattr(model.config, "architectures", None) or []
-        target_moe_mapping = None
-        for arch in archs:
-            if arch in kernel_moe_mapping:
-                target_moe_mapping = kernel_moe_mapping[arch]
-                break
-
-        if target_moe_mapping is None:
             return model
 
+        patched_count = 0
         for module in model.modules():
-            class_name = module.__class__.__name__
+            patch_forward = NpuFusedMoEKernel._get_patch_forward(model_type, module)
-            if class_name in target_moe_mapping:
+            if patch_forward is not None:
-                new_forward_func = target_moe_mapping[class_name]
+                module.forward = types.MethodType(patch_forward, module)
-                module.forward = types.MethodType(new_forward_func, module)
+                patched_count += 1
+
+        if patched_count:
+            logger.info_rank0(f"Applied NPU fused MoE kernel to {patched_count} modules for model type: {model_type}.")
 
         return model

src/llamafactory/v1/plugins/model_plugins/kernels/ops/mlp/npu_swiglu.py

@@ -20,21 +20,24 @@ Init Phase:
 
 """
 
-import re
 import types
 
 import torch
 
-from ......accelerator.helper import DeviceType
+from ......accelerator.helper import DeviceType, get_current_accelerator
+from ......utils.logging import get_logger
 from ......utils.types import HFModel
-from ...base import BaseKernel
+from ...base import BaseKernel, KernelPlugin
-from ...registry import register_kernel
 
 
+logger = get_logger(__name__)
+
 try:
     import torch_npu
-except ImportError:
+except ImportError as exc:
-    pass
+    _TORCH_NPU_IMPORT_ERROR = exc
+else:
+    _TORCH_NPU_IMPORT_ERROR = None
 
 
 def npu_swiglu_forward(self, hidden_state):
@@ -52,80 +55,71 @@ def npu_swiglu_forward(self, hidden_state):
     )
 
 
-def _npu_swiglu_glm4_forward(self, hidden_states):
+_MODEL_TYPE_TO_PATCHES = {
-    """SwiGLU forward pass for GLM4 on NPU.
+    "qwen3": {
-
+        "Qwen3MLP": npu_swiglu_forward,
-    Args:
+    },
-        self: The GLM4 MLP layer instance.
+    "qwen3_moe": {
-        hidden_states (Tensor): Input hidden states.
+        "Qwen3MoeMLP": npu_swiglu_forward,
-
+    },
-    Returns:
+    "qwen3_next": {
-        Tensor: Output of SwiGLU.
+        "Qwen3NextMLP": npu_swiglu_forward,
-    """
+    },
-    up_states = self.gate_up_proj(hidden_states)
+    "qwen3_omni_moe": {
-    gate, up_states = up_states.chunk(2, dim=-1)
+        "Qwen3OmniMoeThinkerTextMLP": npu_swiglu_forward,
-    return self.down_proj(torch_npu.npu_swiglu(torch.cat((gate, up_states), dim=-1), dim=-1))
+        "Qwen3OmniMoeMLP": npu_swiglu_forward,
+        "Qwen3OmniMoeTalkerTextMLP": npu_swiglu_forward,
+        "Qwen3OmniMoeCode2WavMlp": npu_swiglu_forward,
+    },
+    "qwen3_omni_moe_thinker": {
+        "Qwen3OmniMoeThinkerTextMLP": npu_swiglu_forward,
+    },
+    "qwen3_vl": {
+        "Qwen3VLTextMLP": npu_swiglu_forward,
+    },
+    "qwen3_vl_moe": {
+        "Qwen3VLMoeTextMLP": npu_swiglu_forward,
+    },
+    "qwen3_5": {
+        "Qwen3_5MLP": npu_swiglu_forward,
+    },
+    "qwen3_5_moe": {
+        "Qwen3_5MoeMLP": npu_swiglu_forward,
+    },
+}
 
 
-def _npu_swiglu_gemma3ntext_forward(self, hidden_states):
+@KernelPlugin("npu_fused_swiglu").register()
-    """SwiGLU forward pass for Gemma3nText on NPU.
-
-    Args:
-        self: The Gemma3nText MLP layer instance.
-        hidden_states (Tensor): Input hidden states.
-
-    Returns:
-        Tensor: Output of SwiGLU.
-    """
-    gate_proj = self.gate_proj(hidden_states)
-    if self.activation_sparsity > 0.0:
-        gate_proj = self._gaussian_topk(gate_proj)
-    down_proj = self.down_proj(
-        torch_npu.npu_swiglu(torch.cat((gate_proj, self.up_proj(hidden_states)), dim=-1), dim=-1)
-    )
-    return down_proj
-
-
-@register_kernel
 class NpuSwiGluKernel(BaseKernel):
     """NPU Kernel for fused SwiGLU activation."""
 
-    # just support apply to the following module layers
+    @staticmethod
-    expect_modules = frozenset(
+    def check_device() -> None:
-        {
+        current = get_current_accelerator().type
-            "Qwen3VLMoeTextMLP",
+        if current != DeviceType.NPU:
-            "Qwen3VLTextMLP",
+            raise RuntimeError(f"NpuSwiGluKernel requires NPU, current accelerator is {current}.")
-            "Qwen3OmniMoeThinkerTextMLP",
-            "Qwen3OmniMoeMLP",
-            "Qwen3OmniMoeTalkerTextMLP",
-            "Qwen3OmniMoeCode2WavMlp",
-            "Qwen3NextMLP",
-            "Qwen3MoeMLP",
-            "Qwen3MLP",
-            "Qwen2MLP",
-            "Qwen2MoeMLP",
-            "Qwen2_5_VLMLP",
-            "Qwen2_5OmniMLP",
-            "Llama4TextMLP",
-            "LlamaMLP",
-            "Glm4MLP",
-            "Glm4MoeMLP",
-            "Glm4vMoeTextMLP",
-            "Gemma3MLP",
-            "Gemma2MLP",
-            "Gemma3nTextMLP",
-            "Phi3MLP",
-            "DeepseekV2MLP",
-            "DeepseekV3MLP",
-            "SeedOssMLP",
-        }
-    )
 
-    _kernel_id = "npu_fused_swiglu"
+    @staticmethod
-    _device = DeviceType.NPU
+    def check_deps() -> None:
+        if _TORCH_NPU_IMPORT_ERROR is not None:
+            raise RuntimeError("NpuSwiGluKernel requires torch_npu.") from _TORCH_NPU_IMPORT_ERROR
 
-    @classmethod
+    @staticmethod
-    def apply(cls, **kwargs) -> "HFModel":
+    def _get_patch_forward(model_type: str, module: torch.nn.Module):
+        """Return the NPU forward function for a matched SwiGLU MLP module."""
+        model_patches = _MODEL_TYPE_TO_PATCHES.get(model_type, {})
+        patch_forward = model_patches.get(module.__class__.__name__)
+        if patch_forward is None:
+            return None
+
+        config = getattr(module, "config", None)
+        if getattr(config, "hidden_act", None) != "silu":
+            return None
+
+        return patch_forward
+
+    @staticmethod
+    def _apply(**kwargs) -> "HFModel":
         """Applies the NPU fused SwiGLU kernel to the model.
 
         Args:
@@ -133,36 +127,21 @@ class NpuSwiGluKernel(BaseKernel):
 
         Returns:
             HFModel: The model with patched SwiGLU forward functions.
-
-        Raises:
-            ValueError: If the model is not provided.
-            RuntimeError: If dependencies are not met.
         """
-        model = kwargs.get("model", None)
+        model = kwargs["model"]
-        if model is None:
-            raise ValueError(f"HFModel instance is required for {cls.__name__}.")
 
-        if not cls.check_deps():
+        model_type = getattr(model.config, "model_type", None)
-            raise RuntimeError("torch_npu is not available but NpuSwiGluKernel was called.")
+        if model_type not in _MODEL_TYPE_TO_PATCHES:
+            return model
 
-        # Mapping of specific mlp modules to their corresponding kernel implementations
+        patched_count = 0
-        kernel_mapping = {
+        for module in model.modules():
-            "Glm4MLP": _npu_swiglu_glm4_forward,
+            patch_forward = NpuSwiGluKernel._get_patch_forward(model_type, module)
-            "Glm4vTextMLP": _npu_swiglu_glm4_forward,
+            if patch_forward is not None:
-            "Phi3MLP": _npu_swiglu_glm4_forward,
+                module.forward = types.MethodType(patch_forward, module)
-            "Gemma3nTextMLP": _npu_swiglu_gemma3ntext_forward,
+                patched_count += 1
-        }
 
-        swiglu_pattern = re.compile("MLP", re.IGNORECASE)
+        if patched_count:
-        for name, module in model.named_modules():
+            logger.info_rank0(f"Applied NPU SwiGLU kernel to {patched_count} modules for model type: {model_type}.")
-            # Match any module whose class name contains "MLP"
-            if (
-                re.search(swiglu_pattern, module.__class__.__name__)
-                and module.__class__.__name__ in cls.expect_modules
-            ):
-                # Bind function as an instance method to preserve `self` semantics
-                # and replace the original forward
-                kernel_func = kernel_mapping.get(module.__class__.__name__, npu_swiglu_forward)
-                module.forward = types.MethodType(kernel_func, module)
 
         return model

src/llamafactory/v1/plugins/model_plugins/kernels/ops/rms_norm/npu_rms_norm.py

@@ -20,55 +20,32 @@ Init Phase:
 
 """
 
-import re
 import types
 
 import torch
 import torch.nn.functional as F
 
-from ......accelerator.helper import DeviceType
+from ......accelerator.helper import DeviceType, get_current_accelerator
+from ......utils.logging import get_logger
 from ......utils.types import HFModel
-from ...base import BaseKernel
+from ...base import BaseKernel, KernelPlugin
-from ...registry import register_kernel
 
 
+logger = get_logger(__name__)
+
 try:
     import torch_npu
-except ImportError:
+except ImportError as exc:
-    pass
+    _TORCH_NPU_IMPORT_ERROR = exc
-
+else:
-
+    _TORCH_NPU_IMPORT_ERROR = None
-def _should_use_residual_rmsnorm(module):
-    """Detect whether the module uses residual RMSNorm parameterization.
-
-    Residual RMSNorm uses ``scale = 1.0 + weight`` where weight is initialized to 0,
-    while standard RMSNorm uses ``scale = weight`` where weight is initialized to 1.
-
-    Args:
-        module (nn.Module): The RMSNorm module to check.
-
-    Returns:
-        bool: ``True`` if the module uses residual parameterization, ``False`` otherwise.
-
-    .. note::
-        This must follow the module's forward semantics. Do not infer it from trained
-        weight values because standard RMSNorm weights can also be close to zero.
-    """
-    residual_rmsnorm_classes = {
-        "Qwen3_5RMSNorm",
-        "Qwen3_5MoeRMSNorm",
-        "Qwen3NextRMSNorm",
-    }
-
-    class_name = module.__class__.__name__
-    return class_name in residual_rmsnorm_classes
 
 
 def npu_rms_norm_forward(self, hidden_states):
     """NPU forward implementation for standard RMSNorm.
 
     Args:
-        self (nn.Module): The RMSNorm module instance with ``weight`` and ``variance_epsilon``.
+        self (nn.Module): The RMSNorm module instance with ``weight`` and either ``variance_epsilon`` or ``eps``.
         hidden_states (Tensor): Input hidden states tensor.
 
     Returns:
@@ -76,88 +53,157 @@ def npu_rms_norm_forward(self, hidden_states):
     """
     _eps = getattr(self, "variance_epsilon", None) or getattr(self, "eps", 1e-6)
 
-    if hasattr(self, "weight") and self.weight is not None:
+    weight = getattr(self, "weight", None)
-        if getattr(self, "_npu_use_residual_rmsnorm", False):
+    if weight is None:
-            effective_weight = 1.0 + self.weight.float()
+        raise RuntimeError(f"{self.__class__.__name__} has no RMSNorm weight for NPU RMSNorm kernel.")
-        else:
-            effective_weight = self.weight.float()
-    else:
-        effective_weight = None
 
-    if effective_weight is not None:
+    effective_weight = weight.float()
-        return torch_npu.npu_rms_norm(hidden_states, effective_weight.to(hidden_states.dtype), epsilon=_eps)[0]
+
-    else:
+    return torch_npu.npu_rms_norm(hidden_states, effective_weight.to(hidden_states.dtype), epsilon=_eps)[0]
-        return torch_npu.npu_rms_norm(hidden_states, self.weight, epsilon=_eps)[0]
+
+
+def npu_residual_rms_norm_forward(self, hidden_states):
+    """NPU forward implementation for residual RMSNorm.
+
+    Residual RMSNorm uses ``scale = 1.0 + weight`` where ``weight`` is initialized
+    to 0 in the original transformers implementation.
+
+    Args:
+        self (nn.Module): The residual RMSNorm module with ``weight`` and either ``variance_epsilon`` or ``eps``.
+        hidden_states (Tensor): Input hidden states tensor.
+
+    Returns:
+        Tensor: Normalized tensor consistent with residual RMSNorm behavior.
+    """
+    _eps = getattr(self, "variance_epsilon", None) or getattr(self, "eps", 1e-6)
+
+    weight = getattr(self, "weight", None)
+    if weight is None:
+        raise RuntimeError(f"{self.__class__.__name__} has no RMSNorm weight for NPU RMSNorm kernel.")
+
+    effective_weight = 1.0 + weight.float()
+
+    return torch_npu.npu_rms_norm(hidden_states, effective_weight.to(hidden_states.dtype), epsilon=_eps)[0]
 
 
 def npu_gated_rms_norm_forward(self, hidden_states, gate=None):
     """NPU forward implementation for Gated RMSNorm with high-precision FP32 computation.
 
     This function performs RMSNorm and gated SiLU multiplication in FP32 for numerical
-    stability. Unlike standard RMSNorm, Gated RMSNorm in Qwen3.5 uses standard
+    stability. The supported gated RMSNorm modules use ``scale = weight`` with weight
-    parameterization (``scale = weight`` where weight is initialized to 1), so the
+    initialized to 1, unlike the residual RMSNorm variants that use ``1.0 + weight``.
-    residual weight adjustment (``1.0 + weight``) is not applied here.
 
     Args:
         self (nn.Module): The Gated RMSNorm module instance.
         hidden_states (Tensor): Input hidden states tensor.
-        gate (Tensor, optional): Gate tensor for SiLU activation. Defaults to ``None``.
+        gate (Tensor): Gate tensor for SiLU activation.
 
     Returns:
         Tensor: Output tensor cast back to the original input dtype.
+
+    Raises:
+        ValueError: If the gate tensor is not provided.
     """
+    if gate is None:
+        raise ValueError(f"{self.__class__.__name__} requires a gate tensor for NPU Gated RMSNorm.")
+
     input_dtype = hidden_states.dtype
     hidden_states = hidden_states.to(torch.float32)
     _eps = getattr(self, "variance_epsilon", None) or getattr(self, "eps", 1e-6)
 
     hidden_states = torch_npu.npu_rms_norm(hidden_states, self.weight.float(), epsilon=_eps)[0]
-
+    hidden_states = hidden_states * F.silu(gate.to(torch.float32))
-    if gate is not None:
-        hidden_states = hidden_states * F.silu(gate.to(torch.float32))
 
     return hidden_states.to(input_dtype)
 
 
-@register_kernel
+_MODEL_TYPE_TO_PATCHES = {
+    "qwen3": {
+        "Qwen3RMSNorm": npu_rms_norm_forward,
+    },
+    "qwen3_moe": {
+        "Qwen3MoeRMSNorm": npu_rms_norm_forward,
+    },
+    "qwen3_next": {
+        "Qwen3NextRMSNorm": npu_residual_rms_norm_forward,
+        "Qwen3NextRMSNormGated": npu_gated_rms_norm_forward,
+    },
+    "qwen3_omni_moe": {
+        "Qwen3OmniMoeThinkerTextRMSNorm": npu_rms_norm_forward,
+        "Qwen3OmniMoeTextRMSNorm": npu_rms_norm_forward,
+        "Qwen3OmniMoeRMSNorm": npu_rms_norm_forward,
+        "Qwen3OmniMoeCode2WavRMSNorm": npu_rms_norm_forward,
+    },
+    "qwen3_omni_moe_thinker": {
+        "Qwen3OmniMoeThinkerTextRMSNorm": npu_rms_norm_forward,
+        "Qwen3OmniMoeTextRMSNorm": npu_rms_norm_forward,
+    },
+    "qwen3_vl": {
+        "Qwen3VLTextRMSNorm": npu_rms_norm_forward,
+    },
+    "qwen3_vl_moe": {
+        "Qwen3VLMoeTextRMSNorm": npu_rms_norm_forward,
+    },
+    "qwen3_5": {
+        "Qwen3_5RMSNorm": npu_residual_rms_norm_forward,
+        "Qwen3_5RMSNormGated": npu_gated_rms_norm_forward,
+    },
+    "qwen3_5_moe": {
+        "Qwen3_5MoeRMSNorm": npu_residual_rms_norm_forward,
+        "Qwen3_5MoeRMSNormGated": npu_gated_rms_norm_forward,
+    },
+}
+
+
+@KernelPlugin("npu_fused_rmsnorm").register()
 class NpuRMSNormKernel(BaseKernel):
     """NPU kernel wrapper for RMSNorm that applies the replacement within a model."""
 
-    _kernel_id = "npu_fused_rmsnorm"
+    @staticmethod
-    _device = DeviceType.NPU
+    def check_device() -> None:
+        current = get_current_accelerator().type
+        if current != DeviceType.NPU:
+            raise RuntimeError(f"NpuRMSNormKernel requires NPU, current accelerator is {current}.")
 
-    @classmethod
+    @staticmethod
-    def apply(cls, **kwargs) -> "HFModel":
+    def check_deps() -> None:
+        if _TORCH_NPU_IMPORT_ERROR is not None:
+            raise RuntimeError("NpuRMSNormKernel requires torch_npu.") from _TORCH_NPU_IMPORT_ERROR
+
+    @staticmethod
+    def _get_patch_forward(model_type: str, module: torch.nn.Module):
+        """Return the NPU forward function for a matched RMSNorm module."""
+        model_patches = _MODEL_TYPE_TO_PATCHES.get(model_type, {})
+        return model_patches.get(module.__class__.__name__)
+
+    @staticmethod
+    def _apply(**kwargs) -> "HFModel":
         """Iterate the model and apply NPU-optimized forward to matched RMSNorm modules.
 
-        Matches modules whose class name contains "RMSNorm" (case-insensitive) and binds
+        Matches modules configured for the current model type, then binds the corresponding
-        the appropriate NPU-optimized forward function as an instance method via
+        NPU-optimized forward function as an instance method via ``types.MethodType`` to
-        ``types.MethodType`` to replace the original ``forward``.
+        replace the original ``forward``.
 
         Args:
             **kwargs: Keyword arguments containing the model.
 
         Returns:
             HFModel: The model with NPU fused RMSNorm.
-
-        Raises:
-            RuntimeError: If ``torch_npu`` is not available.
-            ValueError: If the model is not provided.
         """
-        model = kwargs.get("model")
+        model = kwargs["model"]
-        if model is None:
-            raise ValueError(f"HFModel instance is required for {cls.__name__}.")
 
-        if not cls.check_deps():
+        model_type = getattr(model.config, "model_type", None)
-            raise RuntimeError(f"torch_npu is not available but {cls.__name__} was called.")
+        if model_type not in _MODEL_TYPE_TO_PATCHES:
+            return model
 
-        rms_norm_pattern = re.compile("RMSNorm", re.IGNORECASE)
+        patched_count = 0
+        for module in model.modules():
+            patch_forward = NpuRMSNormKernel._get_patch_forward(model_type, module)
+            if patch_forward is not None:
+                module.forward = types.MethodType(patch_forward, module)
+                patched_count += 1
 
-        for _, module in model.named_modules():
+        if patched_count:
-            if re.search(rms_norm_pattern, module.__class__.__name__):
+            logger.info_rank0(f"Applied NPU RMSNorm kernel to {patched_count} modules for model type: {model_type}.")
-                if "Gated" in module.__class__.__name__:
-                    module.forward = types.MethodType(npu_gated_rms_norm_forward, module)
-                else:
-                    module._npu_use_residual_rmsnorm = _should_use_residual_rmsnorm(module)
-                    module.forward = types.MethodType(npu_rms_norm_forward, module)
 
         return model

src/llamafactory/v1/plugins/model_plugins/kernels/ops/rope/npu_rope.py

@@ -20,31 +20,32 @@ Init Phase:
 
 """
 
-import sys
+import importlib
 
 import torch
 
-from ......accelerator.helper import DeviceType
+from ......accelerator.helper import DeviceType, get_current_accelerator
 from ......utils.logging import get_logger
 from ......utils.types import HFModel
-from ...base import BaseKernel
+from ...base import BaseKernel, KernelPlugin
-from ...registry import register_kernel
 
 
 logger = get_logger(__name__)
 
 try:
     import torch_npu
-except ImportError:
+except ImportError as exc:
-    pass
+    _TORCH_NPU_IMPORT_ERROR = exc
+else:
+    _TORCH_NPU_IMPORT_ERROR = None
 
 
 def _apply_npu_rotary_emb(q, k, cos, sin):
     """Apply NPU-accelerated rotary embedding with automatic Partial RoPE detection.
 
-    This function automatically detects whether to use Partial RoPE or Full RoPE
+    Partial RoPE is detected when the ``cos/sin`` width is smaller than the ``q/k``
-    based on the dimension ratio between ``cos/sin`` and ``q/k`` tensors, ensuring
+    head dimension. The leading rotary dimensions are transformed and any trailing
-    compatibility with future model versions without hardcoding.
+    dimensions are passed through unchanged.
 
     Args:
         q (Tensor): Query tensor.
@@ -62,14 +63,14 @@ def _apply_npu_rotary_emb(q, k, cos, sin):
         q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:]
         k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:]
 
-        q_embed = torch_npu.npu_rotary_mul(q_rot, cos, sin).to(q.dtype)
+        q_embed = torch_npu.npu_rotary_mul(q_rot, cos, sin, "half").to(q.dtype)
-        k_embed = torch_npu.npu_rotary_mul(k_rot, cos, sin).to(k.dtype)
+        k_embed = torch_npu.npu_rotary_mul(k_rot, cos, sin, "half").to(k.dtype)
 
         q_embed = torch.cat([q_embed, q_pass], dim=-1)
         k_embed = torch.cat([k_embed, k_pass], dim=-1)
     else:
-        q_embed = torch_npu.npu_rotary_mul(q, cos, sin).to(q.dtype)
+        q_embed = torch_npu.npu_rotary_mul(q, cos, sin, "half").to(q.dtype)
-        k_embed = torch_npu.npu_rotary_mul(k, cos, sin).to(k.dtype)
+        k_embed = torch_npu.npu_rotary_mul(k, cos, sin, "half").to(k.dtype)
 
     return q_embed, k_embed
 
@@ -85,103 +86,108 @@ def _apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
         k (Tensor): Key tensor.
         cos (Tensor): Cosine part of embedding.
         sin (Tensor): Sine part of embedding.
-        position_ids (Tensor, optional): Position IDs. Defaults to ``None``.
+        position_ids (Tensor | int, optional): Ignored Transformers v4 position IDs, or the Transformers v5
+            ``unsqueeze_dim`` when supplied as the fifth positional argument.
         unsqueeze_dim (int): Dimension to unsqueeze cos and sin. Defaults to 1.
 
     Returns:
         tuple[Tensor, Tensor]: The embedded query and key tensors ``(q_embed, k_embed)``.
     """
+    # In transformers v5, the fifth positional argument is ``unsqueeze_dim``.
+    if isinstance(position_ids, int):
+        unsqueeze_dim = position_ids
+
     cos = cos.unsqueeze(unsqueeze_dim)
     sin = sin.unsqueeze(unsqueeze_dim)
 
     return _apply_npu_rotary_emb(q, k, cos, sin)
 
 
-def _apply_multimodal_rotary_pos_emb_qwen25_vl(q, k, cos, sin, mrope_section, unsqueeze_dim=1):
+def _default_rope_patch(module_type: str):
-    """Apply Rotary Position Embedding with multimodal sections (Qwen2-VL) on NPU.
+    return (
-
+        (
-    This function supports Partial RoPE for multimodal inputs with automatic dimension
+            f"transformers.models.{module_type}.modeling_{module_type}",
-    detection, ensuring compatibility with future model versions.
+            (("apply_rotary_pos_emb", _apply_rotary_pos_emb),),
-
+        ),
-    Args:
-        q (Tensor): Query tensor.
-        k (Tensor): Key tensor.
-        cos (Tensor): Cosine part of embedding.
-        sin (Tensor): Sine part of embedding.
-        mrope_section (list[int]): Multimodal RoPE section sizes.
-        unsqueeze_dim (int): Dimension to unsqueeze cos and sin. Defaults to 1.
-
-    Returns:
-        tuple[Tensor, Tensor]: The embedded query and key tensors ``(q_embed, k_embed)``.
-    """
-    mrope_section = mrope_section * 2
-    cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(mrope_section, dim=-1))], dim=-1).unsqueeze(
-        unsqueeze_dim
-    )
-    sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(mrope_section, dim=-1))], dim=-1).unsqueeze(
-        unsqueeze_dim
     )
 
-    return _apply_npu_rotary_emb(q, k, cos, sin)
+
+_MODEL_TYPE_TO_PATCHES = {
+    "qwen3": _default_rope_patch("qwen3"),
+    "qwen3_moe": _default_rope_patch("qwen3_moe"),
+    "qwen3_next": _default_rope_patch("qwen3_next"),
+    "qwen3_omni_moe": _default_rope_patch("qwen3_omni_moe"),
+    "qwen3_omni_moe_thinker": _default_rope_patch("qwen3_omni_moe"),
+    "qwen3_vl": _default_rope_patch("qwen3_vl"),
+    "qwen3_vl_moe": _default_rope_patch("qwen3_vl_moe"),
+    "qwen3_5": _default_rope_patch("qwen3_5"),
+    "qwen3_5_moe": _default_rope_patch("qwen3_5_moe"),
+}
 
 
-@register_kernel
+@KernelPlugin("npu_fused_rope").register()
 class NpuRoPEKernel(BaseKernel):
     """NPU Kernel for Rotary Position Embedding."""
 
-    _kernel_id = "npu_fused_rope"
+    @staticmethod
-    _device = DeviceType.NPU
+    def check_device() -> None:
+        current = get_current_accelerator().type
+        if current != DeviceType.NPU:
+            raise RuntimeError(f"NpuRoPEKernel requires NPU, current accelerator is {current}.")
 
-    @classmethod
+    @staticmethod
-    def apply(cls, **kwargs) -> "HFModel":
+    def check_deps() -> None:
-        """Apply RoPE acceleration by monkey-patching ``apply_rotary_pos_emb``.
+        if _TORCH_NPU_IMPORT_ERROR is not None:
+            raise RuntimeError("NpuRoPEKernel requires torch_npu.") from _TORCH_NPU_IMPORT_ERROR
 
-        Iterates through the model's modules to find attention layers, identifies
+    @staticmethod
-        the module where they are defined, and replaces the original
+    def _apply_model_patches(model_type: str) -> int:
-        ``apply_rotary_pos_emb`` function in that module's namespace with the
+        patches = _MODEL_TYPE_TO_PATCHES.get(model_type)
-        NPU-accelerated version.
+        if patches is None:
+            return 0
+
+        patched_count = 0
+        for module_name, replacements in patches:
+            try:
+                target_module = importlib.import_module(module_name)
+            except Exception as e:
+                logger.warning_rank0_once(f"Failed to import {module_name} for NPU RoPE kernel: {e}")
+                continue
+
+            for target_function_name, replacement in replacements:
+                if not hasattr(target_module, target_function_name):
+                    logger.warning_rank0_once(f"{module_name} has no {target_function_name}, skip NPU RoPE patch.")
+                    continue
+
+                if getattr(target_module, target_function_name) is replacement:
+                    continue
+
+                setattr(target_module, target_function_name, replacement)
+                patched_count += 1
+
+        return patched_count
+
+    @staticmethod
+    def _apply(**kwargs) -> "HFModel":
+        """Apply RoPE acceleration by monkey-patching rotary embedding functions.
+
+        Selects the target transformers modeling module from ``model.config.model_type``
+        and replaces its rotary embedding helper with the NPU-accelerated version.
 
         Args:
             **kwargs: Keyword arguments containing the model.
 
         Returns:
             HFModel: The model with patched RoPE functions.
-
-        Raises:
-            RuntimeError: If ``torch_npu`` is not available.
-            ValueError: If the model is not provided.
         """
-        if not cls.check_deps():
+        model = kwargs["model"]
-            raise RuntimeError(f"torch_npu is not available but {cls.__name__} was called.")
 
-        model = kwargs.get("model", None)
+        model_type = getattr(model.config, "model_type", None)
-        if model is None:
+        if model_type not in _MODEL_TYPE_TO_PATCHES:
-            raise ValueError(f"HFModel instance is required for {cls.__name__}.")
+            return model
 
-        _modules = set()
+        patched_count = NpuRoPEKernel._apply_model_patches(model_type)
-        for module in model.modules():
+        if patched_count:
-            if "Attention" in module.__class__.__name__:
+            logger.info_rank0(f"Applied NPU RoPE kernel to {patched_count} functions for model type: {model_type}.")
-                module_name = module.__class__.__module__
-                if module_name in _modules:
-                    continue
-                try:
-                    target_module = sys.modules[module_name]
-                    if hasattr(target_module, "apply_rotary_pos_emb"):
-                        if getattr(target_module, "apply_rotary_pos_emb") is not _apply_rotary_pos_emb:
-                            setattr(target_module, "apply_rotary_pos_emb", _apply_rotary_pos_emb)
-                            _modules.add(module_name)
-                    if hasattr(target_module, "apply_multimodal_rotary_pos_emb"):
-                        if (
-                            getattr(target_module, "apply_multimodal_rotary_pos_emb")
-                            is not _apply_multimodal_rotary_pos_emb_qwen25_vl
-                        ):
-                            setattr(
-                                target_module,
-                                "apply_multimodal_rotary_pos_emb",
-                                _apply_multimodal_rotary_pos_emb_qwen25_vl,
-                            )
-                            _modules.add(module_name)
-                except Exception as e:
-                    logger.warning_rank0_once(f"Failed to apply RoPE kernel to module {module_name}: {e}")
 
         return model