[feat] add biren supa accelerator support (#10844)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.github/workflows/docker_npu.yml

@@ -27,6 +27,12 @@ jobs:
           - device: "npu-a3"
             os: "openeuler"
             base_image: "quay.io/ascend/cann:9.1.0-a3-openeuler24.03-py3.12"
+          - device: "npu-a5"
+            os: "ubuntu"
+            base_image: "quay.io/ascend/cann:9.1.0-950-ubuntu22.04-py3.12"
+          - device: "npu-a5"
+            os: "openeuler"
+            base_image: "quay.io/ascend/cann:9.1.0-950-openeuler24.03-py3.12"
 
     runs-on: ubuntu-latest
 
@@ -68,7 +74,7 @@ jobs:
           base_image_tag="${BASE_IMAGE##*:}"
           cann_version="${base_image_tag%%-*}"
           torch_npu_version="$(sed -nE 's/^torch[-_]npu==([0-9]+(\.[0-9]+)*(\.post[0-9]+)?).*/\1/p' requirements/npu.txt)"
-          soc="$(grep -oE '(910b|a3)' <<< "${base_image_tag}" | head -n 1)"
+          soc="$(grep -oE '(910b|a3|950)' <<< "${base_image_tag}" | head -n 1)"
           operating_system="$(grep -oE '(ubuntu|openeuler)[0-9]+(\.[0-9]+)*' <<< "${base_image_tag}" | head -n 1)"
           python_version="$(grep -oE 'py[0-9]+\.[0-9]+' <<< "${base_image_tag}" | head -n 1)"
 
@@ -85,6 +91,7 @@ jobs:
           case "${MATRIX_DEVICE}" in
             npu-a2) expected_soc="910b" ;;
             npu-a3) expected_soc="a3" ;;
+            npu-a5) expected_soc="950" ;;
             *)
               echo "Unsupported NPU device ${MATRIX_DEVICE}" >&2
               exit 1

README.md

@@ -613,14 +613,18 @@ You can also download the pre-built Docker images:
 # Docker Hub
 docker pull hiyouga/llamafactory:latest-910b-ubuntu
 docker pull hiyouga/llamafactory:latest-a3-ubuntu
+docker pull hiyouga/llamafactory:latest-950-ubuntu
 docker pull hiyouga/llamafactory:latest-910b-openeuler
 docker pull hiyouga/llamafactory:latest-a3-openeuler
+docker pull hiyouga/llamafactory:latest-950-openeuler
 
 # quay.io
 docker pull quay.io/ascend/llamafactory:latest-910b-ubuntu
 docker pull quay.io/ascend/llamafactory:latest-a3-ubuntu
+docker pull quay.io/ascend/llamafactory:latest-950-ubuntu
 docker pull quay.io/ascend/llamafactory:latest-910b-openeuler
 docker pull quay.io/ascend/llamafactory:latest-a3-openeuler
+docker pull quay.io/ascend/llamafactory:latest-950-openeuler
 ```
 
 #### Install BitsAndBytes

README_zh.md

@@ -613,14 +613,18 @@ pip install https://github.com/jllllll/bitsandbytes-windows-webui/releases/downl
 # Docker Hub
 docker pull hiyouga/llamafactory:latest-910b-ubuntu
 docker pull hiyouga/llamafactory:latest-a3-ubuntu
+docker pull hiyouga/llamafactory:latest-950-ubuntu
 docker pull hiyouga/llamafactory:latest-910b-openeuler
 docker pull hiyouga/llamafactory:latest-a3-openeuler
+docker pull hiyouga/llamafactory:latest-950-openeuler
 
 # quay.io
 docker pull quay.io/ascend/llamafactory:latest-910b-ubuntu
 docker pull quay.io/ascend/llamafactory:latest-a3-ubuntu
+docker pull quay.io/ascend/llamafactory:latest-950-ubuntu
 docker pull quay.io/ascend/llamafactory:latest-910b-openeuler
 docker pull quay.io/ascend/llamafactory:latest-a3-openeuler
+docker pull quay.io/ascend/llamafactory:latest-950-openeuler
 ```
 
 #### 安装 BitsAndBytes

docker/docker-npu/OVERVIEW.md

@@ -14,12 +14,14 @@ For installation and troubleshooting details, see the [English NPU installation
 
 The following `latest` NPU image tags are available:
 
-| Hardware series | Operating system | Tag |
+| Product series | Operating system | Tag |
 | --- | --- | --- |
-| A2 | Ubuntu 22.04 | `latest-910b-ubuntu` |
-| A3 | Ubuntu 22.04 | `latest-a3-ubuntu` |
-| A2 | openEuler 24.03 | `latest-910b-openeuler` |
-| A3 | openEuler 24.03 | `latest-a3-openeuler` |
+| Atlas A2 products | Ubuntu 22.04 | `latest-910b-ubuntu` |
+| Atlas A3 products | Ubuntu 22.04 | `latest-a3-ubuntu` |
+| Atlas A2 products | openEuler 24.03 | `latest-910b-openeuler` |
+| Atlas A3 products | openEuler 24.03 | `latest-a3-openeuler` |
+| Ascend 950PR&950DT products | Ubuntu 22.04 | `latest-950-ubuntu` |
+| Ascend 950PR&950DT products | openEuler 24.03 | `latest-950-openeuler` |
 
 For historical NPU image tags and links to their corresponding archived Dockerfiles, see [Supported Tags](./supported_tags.md).
 
@@ -53,7 +55,7 @@ latest-<chip>-<os>
 
 | Field | Values | Description |
 | --- | --- | --- |
-| `chip` | `910b` or `a3` | Ascend chip model supported by the image |
+| `chip` | `910b`, `a3`, or `950` | Ascend chip model supported by the image |
 | `os` | `ubuntu` or `openeuler` | Container operating system family |
 
 Future NPU release images will use the full tag format below. Historical release images used earlier naming formats and do not follow this rule.
@@ -67,7 +69,7 @@ Future NPU release images will use the full tag format below. Historical release
 | `LlamaFactory-version` | `0.9.5` | LlamaFactory release version |
 | `CANN-version` | `9.1.0` | Parsed from the CANN base image tag |
 | `TorchNPU-version` | `2.10.0.post2` | Full TorchNPU version used by the image, including suffixes such as `.postN` |
-| `chip` | `910b` or `a3` | Ascend chip model supported by the image |
+| `chip` | `910b`, `a3`, or `950` | Ascend chip model supported by the image |
 | `os` | `ubuntu22.04` or `openeuler24.03` | Container operating system family and version |
 | `Python-version` | `py3.12` | Parsed from the CANN base image tag |
 
@@ -148,7 +150,7 @@ Available build arguments:
 
 ### Start with Docker Compose
 
-The preceding `docker build` command invokes the Dockerfile directly. It builds an image but does not start a container. Docker Compose does not use a separate build implementation: it reads the presets in `docker-compose.yml`, reuses the same Dockerfile, and selects a hardware-series and operating-system combination through a profile. Each `up -d` command below starts the selected container in the background. If the image is not available locally, Docker Compose builds it first:
+The preceding `docker build` command invokes the Dockerfile directly. It builds an image but does not start a container. Docker Compose does not use a separate build implementation: it reads the presets in `docker-compose.yml`, reuses the same Dockerfile, and selects a product-series and operating-system combination through a profile. Each `up -d` command below starts the selected container in the background. If the image is not available locally, Docker Compose builds it first:
 
 ```bash
 cd docker/docker-npu
@@ -164,18 +166,33 @@ docker compose --profile a2-openeuler up -d
 
 # A3 with openEuler
 docker compose --profile a3-openeuler up -d
+
+# 950PR&950DT with Ubuntu
+docker compose --profile 950-ubuntu up -d
+
+# 950PR&950DT with openEuler
+docker compose --profile 950-openeuler up -d
 ```
 
 To build an image with Docker Compose without starting a container, use `docker compose --profile <profile> build`.
 
 ## Hardware Support and Compatibility Notes
 
-- A2 images use the `910b` CANN base image; A3 images use the `a3` CANN base image.
-- The image build targets both x86-64 (`linux/amd64`) and AArch64 (`linux/arm64`) hosts. The CPU architecture is independent of whether the hardware series is A2 or A3.
+- A2 images use the `910b` CANN base image; A3 images use the `a3` CANN base image; 950PR&950DT images use the `950` CANN base image.
+- The image build targets both x86-64 (`linux/amd64`) and AArch64 (`linux/arm64`) hosts. The CPU architecture is independent of whether the product series is A2, A3, or 950PR&950DT.
 - Ubuntu 22.04 and openEuler 24.03 refer to the operating system inside the container.
-- Legacy NPU tags are replaced by the `latest-<910b|a3>-<ubuntu|openeuler>` format.
+- Legacy NPU tags are replaced by the `latest-<910b|a3|950>-<ubuntu|openeuler>` format.
 - Validate the exact driver, firmware, CANN, and SoC combination before production deployment.
 
+## Security Risks
+When running containers with these images, be aware of the following security risks:
+
+- Running as the `root` user: Containers run as the `root` user by default, which can introduce security risks. In production environments, it is recommended to create a non-privileged user to run the application.
+
+- Lack of CPU and memory resource limits: Not setting resource limits may cause a container to consume excessive system resources and affect host performance. It is recommended to use the `--cpus` and `--memory` parameters to limit resource usage.
+
+- Device use with `rwm` permissions: The NPU device is assigned read, write, and `mknod` permissions. Although these are required for functionality, the scope of permissions should be carefully evaluated in security-sensitive environments.
+
 ## License and Disclaimer
 
 LlamaFactory is distributed under the [Apache License 2.0](../../LICENSE).

docker/docker-npu/OVERVIEW.zh.md

@@ -14,12 +14,14 @@ LlamaFactory 昇腾 NPU 镜像面向华为昇腾 Atlas NPU，提供可直接使
 
 当前提供以下 `latest` NPU 镜像 tag：
 
-| 硬件系列 | 操作系统 | Tag |
+| 产品系列 | 操作系统 | Tag |
 | --- | --- | --- |
-| A2 | Ubuntu 22.04 | `latest-910b-ubuntu` |
-| A3 | Ubuntu 22.04 | `latest-a3-ubuntu` |
-| A2 | openEuler 24.03 | `latest-910b-openeuler` |
-| A3 | openEuler 24.03 | `latest-a3-openeuler` |
+| Atlas A2 系列产品 | Ubuntu 22.04 | `latest-910b-ubuntu` |
+| Atlas A3 系列产品 | Ubuntu 22.04 | `latest-a3-ubuntu` |
+| Atlas A2 系列产品 | openEuler 24.03 | `latest-910b-openeuler` |
+| Atlas A3 系列产品 | openEuler 24.03 | `latest-a3-openeuler` |
+| Ascend 950PR&950DT 系列产品 | Ubuntu 22.04 | `latest-950-ubuntu` |
+| Ascend 950PR&950DT 系列产品 | openEuler 24.03 | `latest-950-openeuler` |
 
 历史版本的 NPU 镜像 Tag 及其对应的 Dockerfile 归档链接，请参阅 [Supported Tags](./supported_tags.md)。
 
@@ -53,7 +55,7 @@ latest-<芯片信息>-<操作系统>
 
 | 字段 | 可选值 | 说明 |
 | --- | --- | --- |
-| `芯片信息` | `910b` 或 `a3` | 镜像所适配的昇腾芯片型号 |
+| `芯片信息` | `910b`、`a3` 或 `950` | 镜像所适配的昇腾芯片型号 |
 | `操作系统` | `ubuntu` 或 `openeuler` | 容器操作系统类型 |
 
 后续发布的 NPU release 镜像将使用以下完整 tag 格式。历史 release 镜像使用旧的命名格式，不适用以下规则。
@@ -67,7 +69,7 @@ latest-<芯片信息>-<操作系统>
 | `LlamaFactory版本` | `0.9.5` | LlamaFactory release 版本号 |
 | `CANN版本` | `9.1.0` | 从 CANN 基础镜像 tag 中提取 |
 | `TorchNPU版本` | `2.10.0.post2` | 镜像使用的 TorchNPU 完整版本，包含 `.postN` 等后缀 |
-| `芯片信息` | `910b` 或 `a3` | 镜像所适配的昇腾芯片型号 |
+| `芯片信息` | `910b`、`a3` 或 `950` | 镜像所适配的昇腾芯片型号 |
 | `操作系统` | `ubuntu22.04` 或 `openeuler24.03` | 容器操作系统类型和版本 |
 | `Python版本` | `py3.12` | 从 CANN 基础镜像 tag 中提取 |
 
@@ -148,7 +150,7 @@ docker build \
 
 ### 通过 Docker Compose 启动
 
-前面的 `docker build` 命令直接调用 Dockerfile，只构建镜像，不启动容器。Docker Compose 不使用另一套构建逻辑：它读取 `docker-compose.yml` 中的预设配置，复用同一个 Dockerfile，并通过 profile 选择硬件系列和操作系统组合。下面的 `up -d` 会在后台启动容器；若本地镜像不存在，Docker Compose 会先构建镜像：
+前面的 `docker build` 命令直接调用 Dockerfile，只构建镜像，不启动容器。Docker Compose 不使用另一套构建逻辑：它读取 `docker-compose.yml` 中的预设配置，复用同一个 Dockerfile，并通过 profile 选择产品系列和操作系统组合。下面的 `up -d` 会在后台启动容器；若本地镜像不存在，Docker Compose 会先构建镜像：
 
 ```bash
 cd docker/docker-npu
@@ -164,18 +166,34 @@ docker compose --profile a2-openeuler up -d
 
 # A3 + openEuler
 docker compose --profile a3-openeuler up -d
+
+# 950PR&950DT + Ubuntu
+docker compose --profile 950-ubuntu up -d
+
+# 950PR&950DT + openEuler
+docker compose --profile 950-openeuler up -d
 ```
 
 如果只想通过 Docker Compose 构建镜像而不启动容器，请使用 `docker compose --profile <profile> build`。
 
 ## 硬件支持与兼容性说明
 
-- A2 镜像使用标记为 `910b` 的 CANN 基础镜像，A3 镜像使用标记为 `a3` 的 CANN 基础镜像。
-- 镜像构建目标同时包含 x86-64（`linux/amd64`）和 AArch64（`linux/arm64`）宿主机。CPU 架构与硬件系列是 A2 还是 A3 无关。
+- A2 镜像使用标记为 `910b` 的 CANN 基础镜像，A3 镜像使用标记为 `a3` 的 CANN 基础镜像，950PR&950DT 镜像使用标记为 `950` 的 CANN 基础镜像。
+- 镜像构建目标同时包含 x86-64（`linux/amd64`）和 AArch64（`linux/arm64`）宿主机。CPU 架构与产品系列是 A2、A3 还是 950PR&950DT 无关。
 - Ubuntu 22.04 和 openEuler 24.03 指容器内部的操作系统。
-- 旧式 NPU tag 已由 `latest-<910b|a3>-<ubuntu|openeuler>` 格式取代。
+- 旧式 NPU tag 已由 `latest-<910b|a3|950>-<ubuntu|openeuler>` 格式取代。
 - 正式部署前，请验证具体驱动、固件、CANN 和 SoC 组合的兼容性。
 
+## 安全风险
+
+在运行容器时，需要注意以下安全风险：
+
+- 使用 `root` 用户运行：容器默认以 `root` 用户身份运行，可能带来安全隐患。建议在生产环境中创建非特权用户来运行应用程序。
+
+- 缺少 CPU 和内存资源限制：未设置资源限制可能导致容器消耗过多系统资源，影响宿主机性能。建议使用 `--cpus` 和 `--memory` 参数限制资源使用。
+
+- 设备使用 `rwm` 权限：为 NPU 设备分配了读写和 `mknod` 权限，虽然功能运行需要，但在安全敏感环境中应谨慎评估权限范围。
+
 ## 许可证与免责声明
 
 LlamaFactory 基于 [Apache License 2.0](../../LICENSE) 发布。

docker/docker-npu/docker-compose.yml

@@ -23,6 +23,13 @@ x-npu-common: &npu-common
     - /dev/hisi_hdc
   restart: unless-stopped
 
+x-npu-950: &npu-950
+  <<: *npu-common
+  devices:
+    - /dev/davinci0
+    - /dev/davinci_manager
+    - /dev/hisi_hdc
+
 services:
   llamafactory-a2-ubuntu:
     <<: *npu-common
@@ -79,3 +86,31 @@ services:
     ports:
       - "7863:7860"
       - "8003:8000"
+
+  llamafactory-950-ubuntu:
+    <<: *npu-950
+    profiles: ["950-ubuntu"]
+    build:
+      <<: *build
+      args:
+        <<: *build-args
+        BASE_IMAGE: quay.io/ascend/cann:9.1.0-950-ubuntu22.04-py3.12
+    container_name: llamafactory-950-ubuntu
+    image: llamafactory:npu-950-ubuntu
+    ports:
+      - "7864:7860"
+      - "8004:8000"
+
+  llamafactory-950-openeuler:
+    <<: *npu-950
+    profiles: ["950-openeuler"]
+    build:
+      <<: *build
+      args:
+        <<: *build-args
+        BASE_IMAGE: quay.io/ascend/cann:9.1.0-950-openeuler24.03-py3.12
+    container_name: llamafactory-950-openeuler
+    image: llamafactory:npu-950-openeuler
+    ports:
+      - "7865:7860"
+      - "8005:8000"

docker/docker-xpu/Dockerfile

@@ -0,0 +1,88 @@
+# https://hub.docker.com/r/intel/deep-learning-essentials/tags
+# Unlike the CUDA/ROCm bases this image ships no PyTorch, and its version must match the
+# host Intel GPU driver — see README.md.
+
+ARG BASE_IMAGE=intel/deep-learning-essentials:2026.1.0-devel-ubuntu24.04
+FROM ${BASE_IMAGE}
+
+# Installation arguments
+ARG PIP_INDEX=https://pypi.org/simple
+ARG PYTORCH_INDEX=https://download.pytorch.org/whl/xpu
+# Must stay in the same compute-runtime series as the base image — see README.md ("ocloc").
+ARG OCLOC_VERSION=26.18.38308.1
+
+# Define environments
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PIP_ROOT_USER_ACTION=ignore
+# The base image's Python is distro-managed (PEP 668); nothing to protect in a container.
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+# expandable_segments trips a Level-Zero bug on XPU — see README.md ("Multi-GPU and OPTIM_TORCH").
+ENV OPTIM_TORCH=0
+
+# Use Bash instead of default /bin/sh
+SHELL ["/bin/bash", "-c"]
+
+# Set the working directory
+WORKDIR /app
+
+# Install pip, Python dev headers and a C/C++ toolchain — see README.md ("Python.h: No such file or directory").
+RUN PY_MM="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')" && \
+    apt-get update -y && \
+    apt-get install -y --no-install-recommends \
+        "python${PY_MM}-dev" python3-pip build-essential && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Install ocloc — pinned .deb rather than apt, see README.md ("ocloc and torch.compile").
+RUN wget -q "https://github.com/intel/compute-runtime/releases/download/${OCLOC_VERSION}/intel-ocloc_${OCLOC_VERSION}-0_amd64.deb" \
+        -O /tmp/intel-ocloc.deb && \
+    dpkg -i /tmp/intel-ocloc.deb && \
+    rm -f /tmp/intel-ocloc.deb
+
+# Change pip source — pip is deliberately not upgraded, see README.md ("How pip is installed").
+RUN pip config set global.index-url "${PIP_INDEX}" && \
+    pip config set global.extra-index-url "${PIP_INDEX}" && \
+    pip install --no-cache-dir packaging wheel setuptools editables "hatchling>=1.18.0"
+
+# Copy the application into the image
+COPY . /app
+
+# Install PyTorch for XPU — all three wheels pinned as a set, see README.md ("Pinned torch stack")
+RUN pip install --no-cache-dir -r requirements/xpu.txt --index-url "${PYTORCH_INDEX}"
+
+# Install LLaMA Factory
+# metrics.txt is explicit because this project defines no extras, so `.[metrics]` would no-op.
+RUN pip install --no-cache-dir -e . --no-build-isolation && \
+    pip install --no-cache-dir -r requirements/metrics.txt
+
+# Optional accelerators — see README.md ("Optional accelerators").
+# py-cpuinfo must land first: deepspeed's own setup.py imports deepspeed/ops/adam/cpu_adam.py
+# during metadata generation (before pip installs deepspeed's declared deps), which needs it.
+# deepspeed is pinned separately from requirements/deepspeed.txt's <=0.18.4: that range's XPU
+# accelerator imports IPEX's DpcppBuildExtension unconditionally, which no longer exists on a
+# post-IPEX torch. Fixed upstream in 0.18.7 (plain torch.utils.cpp_extension.BuildExtension).
+RUN pip install --no-cache-dir py-cpuinfo && \
+    (DS_BUILD_OPS=0 pip install --no-cache-dir "deepspeed==0.19.6" \
+        || echo "WARNING: deepspeed install failed - deepspeed training unavailable in this image")
+RUN pip install --no-cache-dir -r requirements/bitsandbytes.txt \
+        || echo "WARNING: bitsandbytes install failed - 4-bit quantization (QLoRA) unavailable in this image"
+
+# Source the oneAPI environment in every interactive shell
+RUN echo "source /opt/intel/oneapi/setvars.sh --force" >> /root/.bashrc
+
+# Set up volumes
+# VOLUME [ "/root/.cache/huggingface", "/app/shared_data", "/app/output" ]
+
+# Expose port 7860 for LLaMA Board
+ENV GRADIO_SERVER_PORT=7860
+EXPOSE 7860
+
+# Expose port 8000 for API service
+ENV API_PORT=8000
+EXPOSE 8000
+
+# Reset pip config
+RUN pip config unset global.index-url && \
+    pip config unset global.extra-index-url
+
+# oneAPI must be on the loader path before torch can open its XPU backend
+CMD ["bash", "-c", "source /opt/intel/oneapi/setvars.sh --force && exec llamafactory-cli webui"]

docker/docker-xpu/README.md

@@ -0,0 +1,206 @@
+# Docker Setup for Intel GPUs
+
+This directory contains Docker configuration files for running LLaMA Factory with Intel GPU (XPU) support.
+
+## Image Details
+
+| Component | Version |
+|---|---|
+| Base OS | Ubuntu 24.04 LTS (x86_64) |
+| Intel DLE base | [intel/deep-learning-essentials:2026.1.0-devel-ubuntu24.04](https://hub.docker.com/r/intel/deep-learning-essentials) |
+| Python | 3.12 |
+| PyTorch | 2.13.0+xpu |
+| Intel GPU runtime | Bundled inside DLE (libze-intel-gpu 26.18.x, oneAPI 2026.1) |
+
+The Intel compute runtime is bundled inside the DLE base image — no GPU compute packages are needed on the host, only the kernel driver. The bundled runtime must be compatible with the host kernel driver; verify with `dpkg -l libze-intel-gpu1 | grep -oP '\d+\.\d+\.\d+'`.
+
+## Prerequisites
+
+### 1. Docker & Docker Compose
+
+```bash
+# Ubuntu/Debian
+sudo apt-get update && sudo apt-get install docker.io docker-compose-v2
+```
+
+See the [official Docker install docs](https://docs.docker.com/engine/install/) for other distros or newer versions.
+
+### 2. Intel GPU Kernel Driver (host only)
+
+```bash
+# Add the Intel GPU PPA
+sudo apt-get install -y gpg-agent wget
+wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | \
+    sudo gpg --dearmor -o /usr/share/keyrings/intel-graphics.gpg
+echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] \
+    https://repositories.intel.com/gpu/ubuntu noble client" | \
+    sudo tee /etc/apt/sources.list.d/intel-graphics.list
+sudo apt-get update
+
+# Kernel driver only — no compute runtime packages needed on the host
+sudo apt-get install -y intel-i915-dkms intel-fw-gpu
+sudo reboot
+```
+
+After reboot, verify `/dev/dri` is populated: `ls /dev/dri/` (expect `card0`, `renderD128`, etc).
+
+See the [Intel GPU Installation Guide](https://dgpu-docs.intel.com/installation-guides/installing-packages-from-the-intel-ppa.html) for details.
+
+> [!IMPORTANT]
+> Enable **Resizable BAR** in your system BIOS before proceeding, or you may see `Bus error (core dumped)` or degraded performance. See [Intel's guide](https://www.intel.com/content/www/us/en/support/articles/000090831/graphics.html).
+
+### 3. Add your user to the GPU groups
+
+```bash
+sudo usermod -aG render,video $USER
+# Log out and back in for group membership to take effect
+```
+
+(Optional) sanity-check the host side with `sudo apt-get install -y clinfo && clinfo --list | grep Device`. The check that actually matters — `torch.xpu.device_count()` — runs inside the container, in Usage below.
+
+## Usage
+
+### Using Docker Compose (Recommended)
+
+```bash
+cd docker/docker-xpu/
+docker compose up -d
+docker compose exec llamafactory bash
+```
+
+Verify GPU access inside the container:
+
+```bash
+python3 -c "import torch; print(torch.xpu.device_count(), 'XPU device(s) found')"
+```
+
+### Using Docker Run
+
+```bash
+# Build the image (from the repo root)
+docker build -t llamafactory:xpu -f docker/docker-xpu/Dockerfile .
+
+# Run the container
+docker run -it --rm \
+    --device /dev/dri \
+    -v /dev/dri/by-path:/dev/dri/by-path \
+    --group-add $(getent group render | cut -d: -f3) \
+    --group-add $(getent group video  | cut -d: -f3) \
+    --ipc=host \
+    -p 7860:7860 \
+    -p 8000:8000 \
+    -v ~/.cache/huggingface:/root/.cache/huggingface \
+    --name llamafactory \
+    llamafactory:xpu bash
+```
+
+## Build arguments
+
+| Argument | Default | Purpose |
+|---|---|---|
+| `BASE_IMAGE` | `intel/deep-learning-essentials:2026.1.0-devel-ubuntu24.04` | oneAPI / GPU runtime version |
+| `PIP_INDEX` | `https://pypi.org/simple` | PyPI mirror for everything except the torch wheels |
+| `PYTORCH_INDEX` | `https://download.pytorch.org/whl/xpu` | where the `+xpu` torch wheels come from |
+| `OCLOC_VERSION` | `26.18.38308.1` | pinned `intel-ocloc` build — must match `BASE_IMAGE`'s compute-runtime series |
+
+```bash
+docker build -t llamafactory:xpu -f docker/docker-xpu/Dockerfile . \
+    --build-arg PIP_INDEX=https://pypi.org/simple
+```
+
+## Design notes
+
+Why the image is built the way it is — skip to [Troubleshooting](#troubleshooting) if you just want to run it.
+
+### Pinned torch stack
+
+`torch`, `torchvision` and `torchaudio` install together from `requirements/xpu.txt` as `+xpu` builds, via `--index-url https://download.pytorch.org/whl/xpu` (not `--extra-index-url`, which would let a plain PyPI build win). Pinning all three avoids `RuntimeError: operator torchvision::nms does not exist` from an ABI-mismatched pair.
+
+No pip constraint file is needed: every other `torch` requirement in the dependency graph is a lower bound that `2.13.0+xpu` already satisfies, so nothing later swaps it out.
+
+If the XPU wheels ever get replaced with plain ones, `torch.xpu.device_count()` returns `0`. Restore with:
+
+```bash
+pip install --force-reinstall -r requirements/xpu.txt \
+    --index-url https://download.pytorch.org/whl/xpu
+```
+
+### How pip is installed
+
+The DLE base ships Python 3.12 with no pip and no `ensurepip` (Ubuntu strips it). pip comes from apt (`python3-pip`) and is **left at the distro version** rather than upgraded — apt's pip has no `RECORD` file, so `pip install --upgrade pip` fails with:
+
+```
+ERROR: Cannot uninstall pip 24.0, RECORD file not found. Hint: The package was installed by debian.
+```
+
+— taking `setuptools`/`wheel`/`hatchling` down with it in the same command.
+
+### Multi-GPU and `OPTIM_TORCH`
+
+The image sets `ENV OPTIM_TORCH=0`. The launcher's default (`OPTIM_TORCH=1`) sets `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True`, whose allocator path trips a Level-Zero driver bug on 2-GPU XPU runs (DDP segfault, FSDP2 hang). Disabling it costs <1% throughput. Re-enable per-run with `OPTIM_TORCH=1 llamafactory-cli train ...` to test it. It is tracked in intel, will be update or remove this ENV when it is resolve with PyTorch or level zero etc fixes.
+
+### Optional accelerators
+
+`deepspeed` and `bitsandbytes` install on a best-effort basis — neither is required to train on XPU, so a build failure only prints a warning instead of failing the image:
+
+```
+WARNING: deepspeed install failed - deepspeed training unavailable in this image
+WARNING: bitsandbytes install failed - 4-bit quantization (QLoRA) unavailable in this image
+```
+
+Both install and work on XPU today (verified: `torch.compile`, LoRA + DeepSpeed ZeRO-2 SFT, and bitsandbytes 4-bit all pass on real Arc Pro B60 hardware). `deepspeed` is pinned to `==0.19.6` here, separately from `requirements/deepspeed.txt`'s shared `<=0.18.4` cap: any version in that range detects XPU as the active accelerator and imports `torch.utils.cpp_extension.DpcppBuildExtension` — a class that used to be provided by `intel_extension_for_pytorch` (IPEX). IPEX is no longer part of this stack, so that import fails with `ImportError: cannot import name 'DpcppBuildExtension'`. deepspeed dropped the IPEX dependency and switched to the stock `BuildExtension` in `0.18.7`, so anything from there onward works; `0.19.6` is the latest release at time of writing.
+
+Check the build log, or verify inside the container with `python3 -c "import deepspeed"` / `import bitsandbytes`. Install manually if missing:
+
+```bash
+DS_BUILD_OPS=0 pip install "deepspeed==0.19.6"
+pip install -r requirements/bitsandbytes.txt
+```
+
+### `ocloc` and `torch.compile`
+
+`torch.compile` lowers to Intel Triton, which shells out to `ocloc` (the Intel offline GPU compiler) — not bundled in the DLE base, so compiled paths fail with `FileNotFoundError: 'ocloc'` without it. The Dockerfile installs a single pinned `.deb` from Intel's GitHub releases (`ARG OCLOC_VERSION`) rather than `apt-get install intel-ocloc`, because the Intel graphics PPA keeps only its newest revision and would drag `libze-intel-gpu1` (the host-driver-facing package) up with it.
+
+> [!IMPORTANT]
+> If you override `ARG BASE_IMAGE`, move `OCLOC_VERSION` to match — check `docker run --rm <base-image> dpkg -l libze-intel-gpu1 | tail -1` and pick the nearest [compute-runtime release](https://github.com/intel/compute-runtime/releases).
+
+## Troubleshooting
+
+### GPU Not Detected (`torch.xpu.device_count()` returns 0)
+
+1. **Kernel driver too old for the DLE runtime** — compare `dpkg -l libze-intel-gpu1` on the host vs. `docker run --rm llamafactory:xpu dpkg -l libze-intel-gpu1`. Update the host driver (`sudo apt-get install -y intel-i915-dkms intel-fw-gpu && sudo reboot`) or use an older `BASE_IMAGE`.
+2. **Missing `/dev/dri` device** — pass `--device /dev/dri` (done automatically by `docker compose`).
+3. **Missing group membership** — the container process needs the `render` and `video` groups; `docker compose` sets these via `group_add`. For manual `docker run`, pass `--group-add $(getent group render | cut -d: -f3) --group-add $(getent group video | cut -d: -f3)`.
+4. **`by-path` mount missing** — required for multi-GPU Level-Zero IPC: `-v /dev/dri/by-path:/dev/dri/by-path` (included in `docker-compose.yml`).
+
+### `fatal error: Python.h: No such file or directory`
+
+Intel Triton JIT-compiles a C driver at the first XPU kernel launch, which needs matching `pythonX.Y-dev` headers and a C compiler present in the image — installed at build time even though the error would happen at runtime. If you've swapped the interpreter, reinstall matching headers:
+
+```bash
+PY_MM="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
+apt-get update && apt-get install -y "python${PY_MM}-dev" build-essential
+```
+
+### Permission Denied on `/dev/dri`
+
+```bash
+sudo usermod -aG render,video $USER
+newgrp render   # apply without logout
+```
+
+### `SYCL Backends mismatch` / `libsycl.so.N: cannot open shared object file`
+
+Two SYCL runtimes exist in this image by design — torch's own `intel-sycl-rt` (pip) and the DLE base's oneAPI — and normally share the same `libsycl.so.9` SONAME, so both coexist fine. A mismatch shows up only after changing `torch` or `BASE_IMAGE` independently. Compare versions:
+
+```bash
+pip list | grep -E "intel-sycl-rt|dpcpp-cpp-rt"
+ls /opt/intel/oneapi/*/lib/libsycl.so.*
+```
+
+The `libsycl.so.N` numbers on both sides must match. `ldconfig -p | grep libsycl` returning nothing is expected — neither runtime is in the ldconfig cache.
+
+## Additional Notes
+
+- The container automatically sources `/opt/intel/oneapi/setvars.sh` in every interactive shell (`~/.bashrc`). For non-interactive scripts, source it explicitly.
+- For training, `llamafactory-cli train` dispatches automatically via `torchrun` for multi-GPU.

docker/docker-xpu/docker-compose.yml

@@ -0,0 +1,48 @@
+services:
+  llamafactory:
+    build:
+      dockerfile: ./docker/docker-xpu/Dockerfile
+      context: ../..
+      args:
+        PIP_INDEX: https://pypi.org/simple
+        # The DLE base image determines which Intel GPU runtime (libze-intel-gpu)
+        # is shipped. The runtime version must be >= the host driver version,
+        # otherwise torch.xpu.device_count() returns 0:
+        #   DLE 2025.3 → libze-intel-gpu 25.18.x  (works on driver ≤26.09)
+        #   DLE 2026.1 → libze-intel-gpu 26.18.x+ (works on driver 26.18/26.22)
+        # Verify host driver: dpkg -l libze-intel-gpu1 | grep -oP '\d+\.\d+\.\d+'
+        BASE_IMAGE: intel/deep-learning-essentials:2026.1.0-devel-ubuntu24.04
+    container_name: llamafactory
+    image: llamafactory:xpu
+    ports:
+      - "7860:7860"
+      - "8000:8000"
+    ipc: host
+    tty: true
+    # shm_size: "16gb"  # ipc: host is set
+    stdin_open: true
+    command: bash
+    devices:
+      # Intel GPU character devices (renderD* and card* under /dev/dri).
+      # renderD* nodes are owned by group `render`; card* by group `video`.
+      - /dev/dri:/dev/dri
+    volumes:
+      # by-path symlinks are required for oneCCL's ze_fd_manager (2-GPU IPC).
+      # `devices:` alone does not carry sub-directories — this volume does.
+      - /dev/dri/by-path:/dev/dri/by-path
+      - ~/.cache/huggingface:/root/.cache/huggingface
+    group_add:
+      # Grant access to the Intel GPU device nodes.
+      # renderD* nodes are owned by group `render`; card* nodes by `video`.
+      # Use numeric GIDs here — Docker compose resolves group names against
+      # the CONTAINER's /etc/group (not the host), and the DLE base image
+      # does not carry a `render` group entry. Standard Linux GIDs:
+      # If your host uses different GIDs, run `getent group render video` and
+      # update these values accordingly.
+      # example output.
+      # getent group render video
+      # render:x:992:root
+      # video:x:44:support
+      - "992"   # render — owns /dev/dri/renderD* nodes
+      - "44"    # video  — owns /dev/dri/card* nodes
+    restart: unless-stopped

requirements/xpu.txt

@@ -0,0 +1,3 @@
+torch==2.13.0+xpu
+torchvision==0.28.0+xpu
+torchaudio==2.11.0+xpu

src/llamafactory/data/mm_plugin.py

@@ -19,7 +19,7 @@ import inspect
 import math
 import os
 import re
-from copy import deepcopy
+from copy import copy, deepcopy
 from dataclasses import dataclass
 from io import BytesIO
 from types import SimpleNamespace
@@ -56,6 +56,7 @@ if TYPE_CHECKING:
     from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor
     from transformers.image_processing_utils import BaseImageProcessor
     from transformers.video_processing_utils import BaseVideoProcessor
+    from transformers.video_utils import VideoMetadata
 
     class EncodedImage(TypedDict):
         path: str | None
@@ -2776,6 +2777,213 @@ class Qwen3VLPlugin(Qwen2VLPlugin):
         return messages
 
 
+@dataclass
+class Glm5NextPlugin(BasePlugin):
+    r"""GLM5-Next images and timestamped video frames using the native processors."""
+
+    @override
+    def _validate_input(
+        self,
+        processor: Optional["MMProcessor"],
+        images: list["ImageInput"],
+        videos: list["VideoInput"],
+        audios: list["AudioInput"],
+    ) -> None:
+        if processor is None and not (images or videos or audios):
+            return  # Preserve tokenizer-only text preprocessing.
+        super()._validate_input(processor, images, videos, audios)
+
+    def _decode_video(
+        self, video: "VideoInput", processor: "MMProcessor"
+    ) -> tuple[list["ImageObject"], "VideoMetadata"]:
+        r"""Decode sampled frames and preserve their source timing for the native processor."""
+        from transformers.video_utils import VideoMetadata
+
+        fps = getattr(processor, "video_fps", 2.0)
+        maxlen = getattr(processor, "video_maxlen", 128)
+        temporal = processor.video_processor.temporal_patch_size
+        if not math.isfinite(fps) or fps <= 0 or maxlen < temporal:
+            raise ValueError("glm5_next requires video_fps > 0 and video_maxlen >= temporal_patch_size.")
+        # Native sampling can duplicate the last frame for temporal patching.
+        # Use a private sampler so frame limits never mutate the shared processor.
+        sampler = copy(processor.video_processor)
+        sampler.max_frames = min(sampler.max_frames, maxlen // temporal * temporal)
+        if _check_video_is_nested_images(video):
+            if not video:
+                raise ValueError("glm5_next received an empty video frame list.")
+            total = len(video)
+            indices = np.linspace(0, total - 1, min(total, sampler.max_frames), dtype=int).tolist()
+            frames = self._regularize_images(
+                [video[i] for i in indices], image_max_pixels=float("inf"), image_min_pixels=0
+            )["images"]
+            # Frame lists have no source timing; video_fps describes their spacing.
+            metadata = VideoMetadata(total_num_frames=total, fps=fps, frames_indices=indices)
+        else:
+            with av.open(video, "r") as container:
+                stream = next((s for s in container.streams if s.type == "video"), None)
+                if stream is None or not stream.average_rate or stream.average_rate <= 0:
+                    raise ValueError("glm5_next requires a video stream with a valid source FPS.")
+                source_fps = float(stream.average_rate)
+                duration = float(stream.duration * stream.time_base) if stream.duration is not None else None
+                metadata = VideoMetadata(total_num_frames=stream.frames, fps=source_fps, duration=duration)
+                # Some containers omit frame counts. Count without retaining all
+                # decoded pixels, then seek back and retain only selected frames.
+                if not metadata.total_num_frames:
+                    metadata.total_num_frames = sum(1 for _ in container.decode(stream))
+                    container.seek(0)
+                if metadata.total_num_frames <= 0:
+                    raise ValueError("glm5_next received a video with no decodable frames.")
+                indices = sampler.sample_frames(metadata, fps=fps).tolist()
+                if not indices:  # Sub-second clips can round to zero in the native sampler.
+                    indices = [0]
+                indices = indices[: sampler.max_frames]
+                selected = set(indices)
+                decoded = {}
+                for i, frame in enumerate(container.decode(stream)):
+                    if i in selected:
+                        decoded[i] = frame.to_image()
+                    if i >= max(selected):
+                        break
+                if selected - decoded.keys():
+                    raise ValueError("glm5_next could not decode all sampled video frames.")
+                frames = [decoded[i] for i in indices]
+                metadata.frames_indices = indices
+        # Keep both pixels and timestamps aligned when repeating an odd last frame.
+        while len(frames) % temporal:
+            frames.append(frames[-1])
+            metadata.frames_indices.append(metadata.frames_indices[-1])
+        return frames, metadata
+
+    @override
+    def _get_mm_inputs(
+        self,
+        images: list["ImageInput"],
+        videos: list["VideoInput"],
+        audios: list["AudioInput"],
+        processor: "MMProcessor",
+    ) -> dict[str, Any]:
+        self._validate_input(processor, images, videos, audios)
+        mm_inputs = {}
+        if images:
+            # Only decode/convert here: resizing twice changes the native pixels.
+            images = self._regularize_images(images, image_max_pixels=float("inf"), image_min_pixels=0)["images"]
+            image_processor = processor.image_processor
+            pixels_per_token = (image_processor.patch_size * image_processor.merge_size) ** 2
+            kwargs = {}
+            if hasattr(processor, "image_max_pixels"):
+                kwargs["max_image_tokens"] = max(1, processor.image_max_pixels // pixels_per_token)
+            if hasattr(processor, "image_min_pixels"):
+                kwargs["min_image_tokens"] = max(1, math.ceil(processor.image_min_pixels / pixels_per_token))
+            mm_inputs.update(image_processor(images=images, return_tensors="pt", **kwargs))
+        if videos:
+            video_processor = processor.video_processor
+            pixels_per_token = (video_processor.patch_size * video_processor.merge_size) ** 2
+            processed = []
+            for video in videos:
+                frames, metadata = self._decode_video(video, processor)
+                temporal_tokens = len(frames) // video_processor.temporal_patch_size
+                # LlamaFactory limits pixels per frame; native GLM5 budgets tokens
+                # across the whole video (after temporal merging).
+                kwargs = {
+                    "min_image_tokens": max(
+                        1, math.ceil(getattr(processor, "video_min_pixels", 256) / pixels_per_token)
+                    )
+                    * temporal_tokens,
+                    "max_image_tokens": max(1, getattr(processor, "video_max_pixels", 65536) // pixels_per_token)
+                    * temporal_tokens,
+                }
+                processed.append(
+                    video_processor(
+                        videos=[np.stack([np.asarray(frame) for frame in frames])],
+                        video_metadata=[metadata],
+                        do_sample_frames=False,
+                        return_metadata=True,
+                        return_tensors="pt",
+                        **kwargs,
+                    )
+                )
+            for key in ("pixel_values_videos", "video_grid_thw"):
+                mm_inputs[key] = torch.cat([item[key] for item in processed])
+            mm_inputs["video_metadata"] = [item["video_metadata"][0] for item in processed]
+        return mm_inputs
+
+    @override
+    def process_messages(
+        self,
+        messages: list[dict[str, str]],
+        images: list["ImageInput"],
+        videos: list["VideoInput"],
+        audios: list["AudioInput"],
+        processor: Optional["MMProcessor"],
+    ) -> list[dict[str, str]]:
+        self._validate_input(processor, images, videos, audios)
+        self._validate_messages(messages, images, videos, audios)
+        messages = deepcopy(messages)
+        mm_inputs = self._get_mm_inputs(images, videos, audios, processor) if self.expand_mm_tokens else {}
+        image_idx, video_idx = 0, 0
+        for message in messages:
+            if message["role"] != "user" and any(
+                p in message["content"] for p in (IMAGE_PLACEHOLDER, VIDEO_PLACEHOLDER)
+            ):
+                raise ValueError("glm5_next supports images and videos in user messages only.")
+            while IMAGE_PLACEHOLDER in message["content"]:
+                tokens = (
+                    processor.replace_image_token(mm_inputs, image_idx) if self.expand_mm_tokens else self.image_token
+                )
+                message["content"] = message["content"].replace(
+                    IMAGE_PLACEHOLDER, f"<|begin_of_image|>{tokens}<|end_of_image|>", 1
+                )
+                image_idx += 1
+            while VIDEO_PLACEHOLDER in message["content"]:
+                tokens = (
+                    processor.replace_video_token(mm_inputs, video_idx) if self.expand_mm_tokens else self.video_token
+                )
+                message["content"] = message["content"].replace(
+                    VIDEO_PLACEHOLDER, f"<|begin_of_video|>{tokens}<|end_of_video|>", 1
+                )
+                video_idx += 1
+        return messages
+
+    @override
+    def get_mm_inputs(
+        self,
+        images: list["ImageInput"],
+        videos: list["VideoInput"],
+        audios: list["AudioInput"],
+        imglens: list[int],
+        vidlens: list[int],
+        audlens: list[int],
+        batch_ids: list[list[int]],
+        processor: Optional["MMProcessor"],
+    ) -> dict[str, Union[list[list[int]], "torch.Tensor"]]:
+        mm_inputs = self._get_mm_inputs(images, videos, audios, processor)
+        token_types = processor.create_mm_token_type_ids(batch_ids)
+        for count, ids in zip(vidlens, batch_ids):
+            boundaries = [token for token in ids if token in (processor.video_start_id, processor.video_end_id)]
+            if boundaries != [processor.video_start_id, processor.video_end_id] * count:
+                raise ValueError("glm5_next video boundaries do not match inputs; check cutoff_len.")
+        # Images and video frames share image_token_id: validate by modality,
+        # otherwise a truncated image could be hidden by extra video tokens.
+        for modality, lens, key, subprocessor in (
+            (1, imglens, "image_grid_thw", processor.image_processor),
+            (2, vidlens, "video_grid_thw", processor.video_processor),
+        ):
+            grids = mm_inputs.get(key, [])
+            offset = 0
+            for count, types in zip(lens, token_types):
+                expected = sum(
+                    int(grid.prod()) // subprocessor.merge_size**2 for grid in grids[offset : offset + count]
+                )
+                if types.count(modality) != expected:
+                    raise ValueError(
+                        "glm5_next image/video tokens do not match features; check cutoff_len and media limits."
+                    )
+                offset += count
+        mm_inputs.pop("video_metadata", None)  # Used for prompt timestamps, never forwarded to model.forward.
+        mm_inputs["mm_token_type_ids"] = token_types
+        return mm_inputs
+
+
 @dataclass
 class GLM4VPlugin(Qwen2VLPlugin):
     @override
@@ -3249,6 +3457,7 @@ PLUGINS = {
     "gemma3n": Gemma3nPlugin,
     "gemma4": Gemma4Plugin,
     "glm4v": GLM4VPlugin,
+    "glm5_next": Glm5NextPlugin,
     "intern_vl": InternVLPlugin,
     "kimi_vl": KimiVLPlugin,
     "llama4": Llama4Plugin,

src/llamafactory/data/template.py

@@ -339,6 +339,73 @@ class Template:
         return modelfile
 
 
+@dataclass
+class Glm5NextTemplate(Template):
+    r"""GLM-5.3-Flash template with preserved thinking and EOS only on the final assistant response."""
+
+    @override
+    def _encode(
+        self,
+        tokenizer: "PreTrainedTokenizer",
+        messages: list[dict[str, str]],
+        system: Optional[str],
+        tools: Optional[str],
+    ) -> list[list[int]]:
+        if not messages or len(messages) % 2 or messages[0]["role"] != Role.USER:
+            raise ValueError(
+                "glm5_next expects alternating user/observation and assistant/function pairs, starting with user."
+            )
+
+        system = system or self.default_system
+        encoded = []
+        for i, message in enumerate(messages):
+            role, content = message["role"], message["content"]
+            expected = (Role.USER, Role.OBSERVATION) if i % 2 == 0 else (Role.ASSISTANT, Role.FUNCTION)
+            if role not in expected or not isinstance(content, str):
+                raise ValueError(
+                    "glm5_next expects alternating user/observation and assistant/function text messages."
+                )
+            if message.get("tool_calls"):
+                raise ValueError("glm5_next expects tool calls as JSON in role=function content (v0 ShareGPT format).")
+            if i % 2 == 0 and i > 0:
+                after_function = messages[i - 1]["role"] == Role.FUNCTION
+                if (role == Role.OBSERVATION) != after_function:
+                    raise ValueError("glm5_next requires an observation after each function turn.")
+
+            elements = []
+            if i == 0:
+                elements += self.format_prefix.apply()
+                if tools:
+                    elements += self.format_tools.apply(content=tools)
+                if system:
+                    elements += self.format_system.apply(content=system)
+
+            if role == Role.USER:
+                elements += self.format_user.apply(content=content)
+            elif role == Role.OBSERVATION:
+                elements += self.format_observation.apply(content=content)
+            else:
+                if role == Role.FUNCTION:
+                    content = self.format_function.apply(
+                        content=content,
+                        thought_words=self.thought_words,
+                        tool_call_words=self.tool_call_words,
+                    )[0]
+                if "</think>" in content:
+                    reasoning = content.split("</think>")[0].split("<think>")[-1]
+                    content = content.split("</think>")[-1]
+                else:
+                    reasoning = ""
+                elements += self.format_assistant.apply(content=reasoning + "</think>" + content.strip())
+                if role == Role.FUNCTION:
+                    # Supervise the observation marker so the model learns to hand control to tools.
+                    elements += ["<|observation|>"]
+                elif i == len(messages) - 1:
+                    elements += [{"eos_token"}]
+            encoded.append(self._convert_elements_to_ids(tokenizer, elements))
+        return encoded
+
+
 @dataclass
 class MossVLTemplate(Template):
     @override
@@ -1211,6 +1278,23 @@ register_template(
 )
 
 
+register_template(
+    name="glm5_next",
+    format_user=StringFormatter(slots=["<|user|>{{content}}<|assistant|><think>"]),
+    format_assistant=StringFormatter(slots=["{{content}}"]),
+    format_system=StringFormatter(slots=["<|system|>{{content}}"]),
+    format_function=FunctionFormatter(slots=["{{content}}"], tool_format="glm5_next"),
+    format_observation=StringFormatter(slots=["<tool_response>{{content}}</tool_response><|assistant|><think>"]),
+    format_tools=ToolFormatter(tool_format="glm5_next"),
+    format_prefix=EmptyFormatter(slots=["[gMASK]<sop><|system|>Reasoning Effort: Max"]),
+    stop_words=["<|user|>", "<|observation|>"],
+    thought_words=("<think>", "</think>"),
+    preserve_thinking=True,
+    mm_plugin=get_mm_plugin(name="glm5_next", image_token="<|image|>", video_token="<|video|>"),
+    template_class=Glm5NextTemplate,
+)
+
+
 register_template(
     name="glm4",
     format_user=StringFormatter(slots=["<|user|>\n{{content}}<|assistant|>"]),

src/llamafactory/data/tool_utils.py

@@ -55,6 +55,14 @@ GLM4_MOE_TOOL_PROMPT = (
     "\n...\n</tool_call>\n"
 )
 
+GLM5_NEXT_TOOL_PROMPT = (
+    "<|system|>\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\n"
+    "You are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{tool_text}"
+    "</tools>\n\nFor each function call, output the function name and arguments within the following XML format:\n"
+    "<tool_call>{{function-name}}<arg_key>{{arg-key-1}}</arg_key><arg_value>{{arg-value-1}}</arg_value>"
+    "<arg_key>{{arg-key-2}}</arg_key><arg_value>{{arg-value-2}}</arg_value>...</tool_call>"
+)
+
 LLAMA3_TOOL_PROMPT = (
     "Cutting Knowledge Date: December 2023\nToday Date: {date}\n\n"
     "You have access to the following functions. To call a function, please respond with JSON for a function call. "
@@ -804,6 +812,44 @@ class GLM4MOEToolUtils(QwenToolUtils):
         return "\n".join(function_texts)
 
 
+class GLM5NextToolUtils(GLM4MOEToolUtils):
+    r"""GLM5-Next tool using template."""
+
+    @override
+    @staticmethod
+    def tool_formatter(tools: list[dict[str, Any]]) -> str:
+        if not isinstance(tools, list):
+            raise ValueError("glm5_next tools must be a JSON list.")
+        tool_text = ""
+        for tool in tools:
+            tool = tool.get("function", tool)
+            if tool.get("defer_loading", False):
+                continue
+            tool = {key: value for key, value in tool.items() if key not in {"strict", "defer_loading"}}
+            tool_text += json.dumps(tool, ensure_ascii=False) + "\n"
+
+        return GLM5_NEXT_TOOL_PROMPT.format(tool_text=tool_text)
+
+    @override
+    @staticmethod
+    def function_formatter(functions: list["FunctionCall"]) -> str:
+        if not functions:
+            raise ValueError("glm5_next function messages must contain at least one call.")
+        calls = []
+        for name, arguments in functions:
+            if not isinstance(name, str) or not re.fullmatch(r"[^\s<>]+", name):
+                raise ValueError("Invalid glm5_next function name.")
+            arguments = json.loads(arguments)
+            if not isinstance(arguments, dict):
+                raise ValueError("glm5_next function arguments must be a JSON object.")
+            text = "<tool_call>" + name
+            for key, value in arguments.items():
+                value = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
+                text += f"<arg_key>{key}</arg_key><arg_value>{value}</arg_value>"
+            calls.append(text + "</tool_call>")
+        return "".join(calls)
+
+
 class SeedToolUtils(ToolUtils):
     r"""Seed tool using template."""
 
@@ -986,6 +1032,7 @@ TOOLS = {
     "qwen3_5": Qwen35ToolUtils(),
     "qwen3_8": Qwen38ToolUtils(),
     "glm4_moe": GLM4MOEToolUtils(),
+    "glm5_next": GLM5NextToolUtils(),
     "seed_oss": SeedToolUtils(),
     "ling": LingToolUtils(),
 }

src/llamafactory/extras/env.py

@@ -33,6 +33,8 @@ def print_env() -> None:
     import transformers
     from transformers.utils import is_torch_cuda_available, is_torch_npu_available
 
+    from .misc import is_torch_supa_available
+
     info = OrderedDict(
         {
             "`llamafactory` version": VERSION,
@@ -46,7 +48,15 @@ def print_env() -> None:
         }
     )
 
-    if is_torch_cuda_available():
+    if is_torch_supa_available():
+        info["PyTorch version"] += " (SUPA)"
+        info["SUPA type"] = torch.supa.get_device_name()
+        info["SUPA number"] = torch.supa.device_count()
+        try:
+            info["SUPA memory"] = f"{torch.supa.mem_get_info()[1] / (1024**3):.2f}GB"
+        except Exception:
+            pass
+    elif is_torch_cuda_available():
         info["PyTorch version"] += " (GPU)"
         info["GPU type"] = torch.cuda.get_device_name()
         info["GPU number"] = torch.cuda.device_count()

src/llamafactory/extras/misc.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
 import gc
 import os
 import socket
@@ -38,9 +39,25 @@ from transformers.utils.versions import require_version
 from . import logging
 
 
-_is_fp16_available = is_torch_npu_available() or is_torch_cuda_available()
+@functools.lru_cache
+def is_torch_supa_available() -> bool:
+    """Check if the supa (SUPA PrivateUse1) accelerator is available."""
+    try:
+        import torch_supa  # noqa: F401  # registers the "supa" device
+
+        return hasattr(torch, "supa") and torch.supa.is_available()
+    except Exception:
+        return False
+
+
+_is_fp16_available = is_torch_npu_available() or is_torch_supa_available() or is_torch_cuda_available()
 try:
-    _is_bf16_available = is_torch_bf16_gpu_available() or (is_torch_npu_available() and torch.npu.is_bf16_supported())
+    if is_torch_supa_available():
+        _is_bf16_available = torch.supa.is_bf16_supported()
+    else:
+        _is_bf16_available = is_torch_bf16_gpu_available() or (
+            is_torch_npu_available() and torch.npu.is_bf16_supported()
+        )
 except Exception:
     _is_bf16_available = False
 
@@ -147,6 +164,8 @@ def get_current_device() -> "torch.device":
         device = "xpu:{}".format(os.getenv("LOCAL_RANK", "0"))
     elif is_torch_npu_available():
         device = "npu:{}".format(os.getenv("LOCAL_RANK", "0"))
+    elif is_torch_supa_available():
+        device = "supa:{}".format(os.getenv("LOCAL_RANK", "0"))
     elif is_torch_mps_available():
         device = "mps:{}".format(os.getenv("LOCAL_RANK", "0"))
     elif is_torch_cuda_available():
@@ -163,6 +182,8 @@ def get_device_name() -> str:
         device = "xpu"
     elif is_torch_npu_available():
         device = "npu"
+    elif is_torch_supa_available():
+        device = "supa"
     elif is_torch_mps_available():
         device = "mps"
     elif is_torch_cuda_available():
@@ -190,6 +211,8 @@ def get_device_count() -> int:
         return torch.xpu.device_count()
     elif is_torch_npu_available():
         return torch.npu.device_count()
+    elif is_torch_supa_available():
+        return torch.supa.device_count()
     elif is_torch_mps_available():
         return torch.mps.device_count()
     elif is_torch_cuda_available():
@@ -211,6 +234,8 @@ def get_current_memory() -> tuple[int, int]:
         return torch.xpu.mem_get_info()
     elif is_torch_npu_available():
         return torch.npu.mem_get_info()
+    elif is_torch_supa_available():
+        return torch.supa.mem_get_info()
     elif is_torch_mps_available():
         return torch.mps.current_allocated_memory(), torch.mps.recommended_max_memory()
     elif is_torch_cuda_available():
@@ -225,6 +250,8 @@ def get_peak_memory() -> tuple[int, int]:
         return torch.xpu.max_memory_allocated(), torch.xpu.max_memory_reserved()
     elif is_torch_npu_available():
         return torch.npu.max_memory_allocated(), torch.npu.max_memory_reserved()
+    elif is_torch_supa_available():
+        return torch.supa.max_memory_allocated(), torch.supa.max_memory_reserved()
     elif is_torch_mps_available():
         return torch.mps.current_allocated_memory(), -1
     elif is_torch_cuda_available():
@@ -251,7 +278,11 @@ def infer_optim_dtype(model_dtype: Optional["torch.dtype"]) -> "torch.dtype":
 def is_accelerator_available() -> bool:
     r"""Check if the accelerator is available."""
     return (
-        is_torch_xpu_available() or is_torch_npu_available() or is_torch_mps_available() or is_torch_cuda_available()
+        is_torch_xpu_available()
+        or is_torch_npu_available()
+        or is_torch_supa_available()
+        or is_torch_mps_available()
+        or is_torch_cuda_available()
     )
 
 
@@ -285,6 +316,8 @@ def torch_gc() -> None:
         torch.xpu.empty_cache()
     elif is_torch_npu_available():
         torch.npu.empty_cache()
+    elif is_torch_supa_available():
+        torch.supa.empty_cache()
     elif is_torch_mps_available():
         torch.mps.empty_cache()
     elif is_torch_cuda_available():

src/llamafactory/hparams/parser.py

@@ -33,7 +33,13 @@ from transformers.utils import is_torch_bf16_gpu_available, is_torch_npu_availab
 
 from ..extras import logging
 from ..extras.constants import CHECKPOINT_NAMES, EngineName
-from ..extras.misc import check_dependencies, check_version, get_current_device, is_env_enabled
+from ..extras.misc import (
+    check_dependencies,
+    check_version,
+    get_current_device,
+    is_env_enabled,
+    is_torch_supa_available,
+)
 from ..extras.packages import is_mcore_adapter_available, is_megatron_bridge_available
 from .data_args import DataArguments
 from .evaluation_args import EvaluationArguments
@@ -223,6 +229,20 @@ def _set_env_vars() -> None:
         # avoid use fork method on NPU devices, see https://github.com/hiyouga/LLaMA-Factory/issues/7447
         os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 
+    if is_torch_supa_available():
+        # register the fused SUDNN operators (torch.ops.sudnn.*) used by the supa kernel path.
+        # torch_supa itself is auto-loaded on `import torch`, but torch_supa_ext is not.
+        try:
+            import torch_supa_ext  # noqa: F401
+        except Exception:
+            logger.warning_rank0_once(
+                "Failed to import torch_supa_ext; fused supa operators (e.g. sudnn.rms_norm_func) "
+                "will be unavailable and the eager path may be extremely slow."
+            )
+
+        # supa uses fork-unsafe device handles; align with the NPU workaround.
+        os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+
 
 def _verify_model_args(
     model_args: "ModelArguments",
@@ -260,6 +280,9 @@ def _check_extra_dependencies(
         check_version("accelerate-kt", mandatory=True)
 
     if model_args.use_unsloth:
+        if is_torch_supa_available():
+            raise ValueError("`use_unsloth` is not supported on supa devices.")
+
         check_version("unsloth", mandatory=True)
 
     if model_args.enable_liger_kernel:
@@ -496,7 +519,11 @@ def get_train_args(args: dict[str, Any] | list[str] | None = None) -> _TRAIN_CLS
         raise ValueError("Please use scripts/pissa_init.py to initialize PiSSA in DeepSpeed ZeRO-3.")
 
     if finetuning_args.pure_bf16:
-        if not (is_torch_bf16_gpu_available() or (is_torch_npu_available() and torch.npu.is_bf16_supported())):
+        if not (
+            is_torch_bf16_gpu_available()
+            or (is_torch_npu_available() and torch.npu.is_bf16_supported())
+            or (is_torch_supa_available() and torch.supa.is_bf16_supported())
+        ):
             raise ValueError("This device does not support `pure_bf16`.")
 
         if is_deepspeed_zero3_enabled():

src/llamafactory/model/model_utils/attention.py

@@ -16,6 +16,7 @@ from typing import TYPE_CHECKING
 
 from ...extras import logging
 from ...extras.constants import AttentionFunction
+from ...extras.misc import is_torch_supa_available
 from ...extras.packages import is_torch_version_greater_than
 
 
@@ -63,7 +64,12 @@ def configure_attn_implementation(config: "PretrainedConfig", model_args: "Model
             model_args.flash_attn = AttentionFunction.DISABLED
 
     if model_args.flash_attn == AttentionFunction.AUTO:
-        return
+        if is_torch_supa_available():
+            # supa has no FlashAttention build, and AUTO would otherwise be resolved against the
+            # cuda-masqueraded device (which may pick FA2); pin SDPA, the fastest supported backend.
+            requested_attn_implementation = "sdpa"
+        else:
+            return
 
     elif model_args.flash_attn == AttentionFunction.DISABLED:
         requested_attn_implementation = "eager"

src/llamafactory/model/model_utils/liger_kernel.py

@@ -37,6 +37,10 @@ def apply_liger_kernel(
     if not is_trainable or not model_args.enable_liger_kernel:
         return
 
+    if get_device_name() == "supa":
+        # liger kernels can not run on supa currently.
+        return
+
     model_type = getattr(config, "model_type", None)
     if model_type == "gemma":
         from liger_kernel.transformers import apply_liger_kernel_to_gemma as apply_liger_kernel

src/llamafactory/model/model_utils/visual.py

@@ -261,6 +261,15 @@ _register_composite_model(
 )
 
 
+_register_composite_model(
+    model_type="glm5_next",
+    projector_keys=["model.visual.merger", "model.visual.downsample"],
+    vision_model_keys=["model.visual.patch_embed", "model.visual.blocks", "model.visual.post_layernorm"],
+    language_model_keys=["model.language_model", "lm_head"],
+    lora_conflict_keys=["patch_embed"],
+)
+
+
 _register_composite_model(
     model_type="glm_ocr",
     projector_keys=["visual.merger"],

src/llamafactory/model/patcher.py

@@ -86,12 +86,9 @@ def _check_fla_dependencies() -> None:
 
 
 def patch_qwen3_5_forward_npu(model: "PreTrainedModel") -> None:
-    """Patch for Qwen3.5 models on NPU by importing torch_npu to enable torch.cuda compatibility.
+    """Replace the Qwen3.5 gated delta rule kernel with the NPU-compatible implementation.
 
-    On NPU, torch.cuda operations will fail unless torch_npu is imported.
-    torch_npu provides compatibility layer that maps torch.cuda calls to NPU operations.
-
-    Also replaces chunk_gated_delta_rule with NPU-compatible implementation.
+    Skip the patch unless an Ascend910-series device is detected and triton_ascend is installed.
     """
     import importlib.metadata
 
@@ -483,7 +480,7 @@ def patch_model(
         prepare_model_for_training(model, model_args)
         autocast_projector_dtype(model, model_args)
         add_z3_leaf_module(model)
-
+        
         if getattr(model.config, "model_type", None) in [
             "qwen3_5",
             "qwen3_5_moe",

src/llamafactory/train/callbacks.py

@@ -33,7 +33,14 @@ from typing_extensions import override
 
 from ..extras import logging
 from ..extras.constants import TRAINER_LOG, V_HEAD_SAFE_WEIGHTS_NAME, V_HEAD_WEIGHTS_NAME
-from ..extras.misc import get_peak_memory, is_env_enabled, is_torch_cuda_available, is_torch_npu_available, use_ray
+from ..extras.misc import (
+    get_peak_memory,
+    is_env_enabled,
+    is_torch_cuda_available,
+    is_torch_npu_available,
+    is_torch_supa_available,
+    use_ray,
+)
 from ..extras.packages import is_safetensors_available
 
 
@@ -504,6 +511,8 @@ class ModuleProfilerCallback(TrainerCallback):
     @staticmethod
     def _get_accelerator():
         """Detect available accelerator and return (event_factory, synchronize_fn)."""
+        if is_torch_supa_available():
+            return torch.supa.Event, torch.supa.synchronize
         if is_torch_cuda_available():
             return torch.cuda.Event, torch.cuda.synchronize
         if is_torch_npu_available():

src/llamafactory/train/hyper_parallel/loader.py

@@ -0,0 +1,162 @@
+# Copyright 2026 the LlamaFactory team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""HyperParallel-specific meta model construction and sharded weight loading."""
+
+import os
+from typing import TYPE_CHECKING
+
+import torch
+from hyper_parallel import init_empty_weights
+from hyper_parallel.integration.llamafactory import parallelize_model
+from transformers import (
+    AutoModelForCausalLM,
+    AutoModelForImageTextToText,
+    AutoModelForSeq2SeqLM,
+    AutoModelForTextToWaveform,
+)
+from transformers.modeling_utils import ContextManagers
+
+from ...extras import logging
+from ...extras.misc import count_parameters
+from ...extras.packages import is_torch_version_greater_than
+from ...model.adapter import init_adapter
+from ...model.loader import _get_init_kwargs, load_config
+from ...model.model_utils.liger_kernel import apply_liger_kernel
+from ...model.model_utils.misc import register_autoclass
+from ...model.patcher import patch_config, patch_model
+
+
+if TYPE_CHECKING:
+    from transformers import PreTrainedModel, PreTrainedTokenizer
+
+    from ...hparams import FinetuningArguments, ModelArguments
+
+
+logger = logging.get_logger(__name__)
+
+
+def _get_no_init_weights():
+    """Resolve the Transformers no-init context across supported versions."""
+    try:
+        from transformers.modeling_utils import no_init_weights
+    except ImportError:
+        from transformers.initialization import no_init_weights
+
+    return no_init_weights
+
+
+def _get_model_class(config):
+    """Select the same Transformers AutoModel class as the native loader."""
+    if type(config) in AutoModelForImageTextToText._model_mapping.keys():
+        return AutoModelForImageTextToText
+    if type(config) in AutoModelForSeq2SeqLM._model_mapping.keys():
+        return AutoModelForSeq2SeqLM
+    if type(config) in AutoModelForTextToWaveform._model_mapping.keys():
+        return AutoModelForTextToWaveform
+    return AutoModelForCausalLM
+
+
+def _validate_conv3d_compatibility(model: "PreTrainedModel", model_args: "ModelArguments") -> None:
+    """Retain the native loader's torch 2.9 Conv3D compatibility guard."""
+    if not is_torch_version_greater_than("2.9.0") or is_torch_version_greater_than("2.10.0"):
+        return
+
+    conv3d_modules = [module for module in model.modules() if isinstance(module, torch.nn.Conv3d)]
+    kt_conv3d_ready = (
+        model_args.use_kt
+        and bool(conv3d_modules)
+        and all(getattr(module, "_kt_conv3d_compatible", False) for module in conv3d_modules)
+    )
+    if conv3d_modules and not kt_conv3d_ready:
+        raise ValueError(
+            "Unsupported torch version detected: torch 2.9.x with Conv3D. "
+            "This combination is known to cause severe performance regression. "
+            "Please downgrade torch to <2.9 or remove Conv3D. "
+            "See https://github.com/pytorch/pytorch/issues/166122"
+        )
+    if kt_conv3d_ready:
+        logger.info_rank0("Using KTransformers instance-scoped Conv3D fallback for torch 2.9.x VLM training.")
+
+
+def load_hyper_parallel_model(
+    tokenizer: "PreTrainedTokenizer",
+    model_args: "ModelArguments",
+    finetuning_args: "FinetuningArguments",
+    distributed_setup,
+    hp_args,
+    is_trainable: bool = False,
+) -> "PreTrainedModel":
+    """Build on meta, apply HyperParallel layouts, then load local weight shards."""
+    init_kwargs = _get_init_kwargs(model_args)
+    config = load_config(model_args)
+    patch_config(config, tokenizer, model_args, init_kwargs, is_trainable)
+    apply_liger_kernel(config, model_args, is_trainable, require_logits=False)
+
+    load_class = _get_model_class(config)
+    no_init_weights = _get_no_init_weights()
+    with ContextManagers([no_init_weights(), init_empty_weights()]):
+        model = load_class.from_config(
+            config,
+            trust_remote_code=model_args.trust_remote_code,
+            dtype=model_args.compute_dtype,
+        )
+
+    if getattr(model.config, "model_type", None) in ["qwen2_5_omni", "qwen3_omni_moe"]:
+        model = getattr(model, "thinker")
+
+    patch_model(model, tokenizer, model_args, is_trainable, add_valuehead=False)
+    register_autoclass(config, model, tokenizer)
+    model = init_adapter(config, model, model_args, finetuning_args, is_trainable)
+
+    if model_args.use_v1_kernels and is_trainable:
+        logger.warning_rank0(
+            "You are try to using future feature about kernels, please note that this feature "
+            "is not supported for all models. If get any error, please disable this feature, or report the issue."
+        )
+        from ...v1.plugins.model_plugins.kernels.interface import apply_v1_kernels
+
+        model = apply_v1_kernels(model, use_v1_kernels=model_args.use_v1_kernels)
+
+    activation_mode = hp_args.activation_mode
+    model = parallelize_model(
+        model,
+        distributed_setup,
+        pretrained_path=None if model_args.train_from_scratch else model_args.model_name_or_path,
+        activation_checkpoint="full" if activation_mode in {"recompute", "swap"} else None,
+        swap_inputs=activation_mode == "swap" and hp_args.activation_swap_inputs,
+    )
+
+    _validate_conv3d_compatibility(model, model_args)
+    if not is_trainable:
+        model.requires_grad_(False)
+        model.eval()
+    else:
+        model.train()
+
+    trainable_params, all_param = count_parameters(model)
+    if is_trainable:
+        param_stats = (
+            f"trainable params: {trainable_params:,} || "
+            f"all params: {all_param:,} || trainable%: {100 * trainable_params / all_param:.4f}"
+        )
+    else:
+        param_stats = f"all params: {all_param:,}"
+    logger.info_rank0(param_stats)
+
+    if model_args.print_param_status and int(os.getenv("LOCAL_RANK", "0")) == 0:
+        for name, param in model.named_parameters():
+            print(f"name: {name}, dtype: {param.dtype}, device: {param.device}, trainable: {param.requires_grad}")
+
+    return model

src/llamafactory/train/hyper_parallel/model_registry.py

@@ -0,0 +1,125 @@
+# Copyright 2026 the LlamaFactory team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Model-specific HyperParallel plans owned by the LlamaFactory backend."""
+
+from collections.abc import Callable
+from typing import Any
+
+
+ParallelPlan = list[dict[str, Any]]
+ParallelPlanProvider = Callable[[], ParallelPlan]
+
+
+def _get_qwen3_moe_parallel_plan() -> ParallelPlan:
+    return [
+        {
+            "match": "*.self_attn",
+            "when": "cp",
+            "inner_target": "self",
+            "region_dispatch": False,
+            "inner_wrapper": {
+                "_target_": (
+                    "hyper_parallel.models.qwen3_moe.adapter.distributed.context_parallel_async."
+                    "qwen3_moe_async_ulysses_cp_wrapper"
+                )
+            },
+        },
+        {
+            "match": "*.mlp",
+            "when": "ep",
+            "region_dispatch": False,
+            "local_compute_fn": {
+                "_target_": (
+                    "hyper_parallel.models.qwen3_moe.adapter.distributed.expert_parallel.qwen3moe_ep_compute_fn"
+                ),
+                "use_grouped_gemm": True,
+            },
+        },
+    ]
+
+
+def _get_qwen3_vl_moe_parallel_plan() -> ParallelPlan:
+    replicated_vision_wrapper = (
+        "hyper_parallel.models.qwen3_vl_moe.adapter.distributed.context_parallel."
+        "qwen3_vl_moe_replicated_vision_cp_wrapper"
+    )
+    return [
+        *[
+            {
+                "match": match,
+                "when": "cp",
+                "params": {},
+                "inner_target": "self",
+                "region_dispatch": False,
+                "inner_wrapper": {"_target_": replicated_vision_wrapper},
+            }
+            for match in (
+                "model.visual.blocks.*.attn",
+                "model.visual.merger",
+                "model.visual.deepstack_merger_list",
+            )
+        ],
+        {
+            "match": "model.language_model",
+            "when": "cp",
+            "params": {},
+            "inner_target": "self",
+            "region_dispatch": False,
+            "inner_wrapper": {
+                "_target_": (
+                    "hyper_parallel.models.qwen3_vl_moe.adapter.distributed.context_parallel."
+                    "qwen3_vl_moe_text_input_cp_wrapper"
+                )
+            },
+        },
+        {
+            "match": "model.language_model.layers.*.self_attn",
+            "when": "cp",
+            "inner_target": "self",
+            "region_dispatch": False,
+            "inner_wrapper": {
+                "_target_": (
+                    "hyper_parallel.models.qwen3_vl_moe.adapter.distributed.context_parallel."
+                    "qwen3_vl_moe_async_ulysses_cp_wrapper"
+                )
+            },
+        },
+        {
+            "match": "model.language_model.layers.*.mlp",
+            "when": "ep",
+            "region_dispatch": False,
+            "local_compute_fn": {
+                "_target_": (
+                    "hyper_parallel.models.qwen3_vl_moe.adapter.distributed.expert_parallel.qwen3_vl_moe_ep_compute_fn"
+                ),
+                "use_grouped_gemm": True,
+            },
+        },
+    ]
+
+
+MODEL_PARALLEL_PLAN_REGISTRY: dict[str, ParallelPlanProvider] = {
+    "qwen3_moe": _get_qwen3_moe_parallel_plan,
+    "qwen3_vl_moe": _get_qwen3_vl_moe_parallel_plan,
+}
+
+
+def apply_model_parallel_plan(hp_args: Any, model_type: str | None) -> None:
+    """Prepend LlamaFactory defaults while preserving explicit user overrides."""
+    provider = MODEL_PARALLEL_PLAN_REGISTRY.get(model_type or "")
+    if provider is None:
+        return
+
+    hp_args.plan_overrides = [*provider(), *(hp_args.plan_overrides or [])]

src/llamafactory/train/hyper_parallel/trainer.py

@@ -18,15 +18,18 @@ import logging
 import os
 import types
 from contextlib import nullcontext
-from functools import partial
+from functools import partial, wraps
 from typing import Any, Optional
 
 import torch
+import torch.distributed as dist
+import torch.nn.functional as F
+from hyper_parallel.core.optimizer import ChainedOptimizer, get_hyper_optimizer
+from hyper_parallel.data.parallel import shard_batch_for_cp
 from hyper_parallel.integration.llamafactory import (
     HSDPModule,
     HyperParallelArguments,
     export_to_hf_format,
-    fsdp2_prepare_model,
     hsdp_sync_stream,
     load_hsdp_model,
     load_hsdp_optimizer_and_scheduler,
@@ -36,14 +39,6 @@ from hyper_parallel.integration.llamafactory import (
 from hyper_parallel.integration.llamafactory import (
     clip_grad_norm_ as hp_clip_grad_norm_,
 )
-from hyper_parallel.integration.llamafactory.context_parallel import (
-    cp_prepare_model,
-    get_cp_rank,
-    get_dp_rank,
-    shard_inputs_for_cp,
-)
-from hyper_parallel.integration.llamafactory.expert_parallel import ep_prepare_model
-from hyper_parallel.platform import get_platform
 from torch import nn
 
 from ..sft.trainer import CustomSeq2SeqTrainer
@@ -52,6 +47,220 @@ from ..sft.trainer import CustomSeq2SeqTrainer
 logger = logging.getLogger(__name__)
 
 
+class _TrainerOptimizerAdapter(torch.optim.Optimizer):
+    """Expose a Hyper optimizer chain through PyTorch's Trainer protocol."""
+
+    def __init__(self, optimizer: ChainedOptimizer):
+        self.optimizer = optimizer
+        super().__init__(optimizer.param_groups, optimizer.defaults)
+        optimizer.param_groups = self.param_groups
+        self.optimizers_dict = optimizer.optimizers_dict
+
+    def step(self, closure=None):
+        return self.optimizer.step(closure=closure)
+
+    def zero_grad(self, set_to_none: bool = True):
+        return self.optimizer.zero_grad(set_to_none=set_to_none)
+
+    def state_dict(self):
+        return self.optimizer.state_dict()
+
+    def load_state_dict(self, state_dict):
+        return self.optimizer.load_state_dict(state_dict)
+
+
+def _create_hyper_muon_optimizer(model: nn.Module, training_args) -> torch.optim.Optimizer:
+    """Create HyperParallel's distributed Muon and AdamW optimizer chain."""
+    muon_params = []
+    adamw_params = []
+    for name, param in model.named_parameters():
+        if not param.requires_grad:
+            continue
+
+        if param.ndim == 2 and "embed" not in name and "lm_head" not in name:
+            muon_params.append(param)
+        else:
+            adamw_params.append(param)
+
+    optimizer = get_hyper_optimizer(
+        model=model,
+        muon_params=[{"params": muon_params}] if muon_params else [],
+        adamw_params=[{"params": adamw_params}] if adamw_params else [],
+        muon_kwargs={
+            "muon_lr": training_args.learning_rate,
+            "muon_weight_decay": training_args.weight_decay,
+        },
+        adamw_kwargs={
+            "adamw_lr": training_args.learning_rate,
+            "adamw_weight_decay": training_args.weight_decay,
+            "adamw_betas": (training_args.adam_beta1, training_args.adam_beta2),
+            "adamw_eps": training_args.adam_epsilon,
+        },
+    )
+    logger.info(
+        "Using HyperParallel Muon optimizer with %d Muon params and %d AdamW params.",
+        len(muon_params),
+        len(adamw_params),
+    )
+    return _TrainerOptimizerAdapter(optimizer)
+
+
+def _shard_inputs_for_cp(inputs: dict[str, Any], cp_mesh) -> dict[str, Any]:
+    """Adapt HuggingFace batches to Hyper's existing CP batch contract."""
+    inputs = dict(inputs)
+    seq_len = inputs["input_ids"].shape[1]
+    if "position_ids" not in inputs:
+        position_ids = torch.arange(
+            seq_len,
+            device=inputs["input_ids"].device,
+            dtype=torch.long,
+        ).unsqueeze(0)
+        inputs["position_ids"] = position_ids.expand(inputs["input_ids"].shape[0], -1)
+
+    global_attention_mask = inputs.get("attention_mask")
+    global_labels = inputs.get("labels")
+    preserve_attention_mask = (
+        isinstance(global_attention_mask, torch.Tensor)
+        and global_attention_mask.ndim == 2
+        and global_attention_mask.shape[-1] == seq_len
+    )
+    sharded = shard_batch_for_cp(inputs, cp_mesh)
+    if preserve_attention_mask:
+        pad_len = (-seq_len) % (cp_mesh.size() * 2)
+        if pad_len > 0:
+            global_attention_mask = torch.nn.functional.pad(global_attention_mask, (0, pad_len), value=0)
+        sharded["attention_mask"] = global_attention_mask
+    if isinstance(global_labels, torch.Tensor) and global_labels.shape[-1] == seq_len:
+        sharded["labels"] = global_labels
+    return sharded
+
+
+def _pad_inputs_for_model_cp(inputs: dict[str, Any], cp_mesh) -> dict[str, Any]:
+    """Pad only token-aligned fields when a multimodal model shards internally."""
+    inputs = dict(inputs)
+    seq_len = inputs["input_ids"].shape[1]
+    pad_len = (-seq_len) % (cp_mesh.size() * 2)
+    if pad_len == 0:
+        return inputs
+
+    pad_values = {
+        "input_ids": 0,
+        "labels": -100,
+        "attention_mask": 0,
+        "mm_token_type_ids": 0,
+    }
+    for name in ("input_ids", "labels", "attention_mask", "mm_token_type_ids", "position_ids"):
+        value = inputs.get(name)
+        if not isinstance(value, torch.Tensor) or value.shape[-1] != seq_len:
+            continue
+        if name == "position_ids":
+            increment = torch.arange(1, pad_len + 1, device=value.device, dtype=value.dtype)
+            increment = increment.reshape(*([1] * (value.ndim - 1)), pad_len)
+            pad = value[..., -1:] + increment
+            pad = pad.expand(*value.shape[:-1], pad_len)
+        else:
+            pad = value.new_full((*value.shape[:-1], pad_len), pad_values[name])
+        inputs[name] = torch.cat((value, pad), dim=-1)
+    return inputs
+
+
+def _build_cp_shift_labels(
+    labels: torch.Tensor,
+    local_seq_len: int,
+    cp_rank: int,
+    ignore_index: int,
+) -> torch.Tensor:
+    """Align full-sequence causal labels with one rank's local logits."""
+    start = cp_rank * local_seq_len
+    end = start + local_seq_len
+    shift_labels = labels[..., start + 1 : min(end + 1, labels.shape[-1])]
+    if shift_labels.shape[-1] < local_seq_len:
+        pad_shape = (*shift_labels.shape[:-1], local_seq_len - shift_labels.shape[-1])
+        shift_labels = torch.cat(
+            (shift_labels, shift_labels.new_full(pad_shape, ignore_index)),
+            dim=-1,
+        )
+    return shift_labels.contiguous()
+
+
+def _enable_cp_causal_lm_loss(model: nn.Module, cp_mesh) -> None:
+    """Use full labels to preserve causal boundaries and CP-global normalization."""
+    cp_size = cp_mesh.size()
+    cp_rank = cp_mesh.get_local_rank()
+    cp_group = cp_mesh.get_group()
+    original_loss_function = model.loss_function
+
+    @wraps(original_loss_function)
+    def cp_loss_function(
+        logits,
+        labels,
+        vocab_size: int,
+        num_items_in_batch=None,
+        ignore_index: int = -100,
+        shift_labels=None,
+        **kwargs,
+    ):
+        if (
+            shift_labels is not None
+            or not isinstance(logits, torch.Tensor)
+            or not isinstance(labels, torch.Tensor)
+            or logits.ndim < 3
+            or labels.ndim < 2
+            or logits.shape[-2] * cp_size != labels.shape[-1]
+        ):
+            return original_loss_function(
+                logits=logits,
+                labels=labels,
+                vocab_size=vocab_size,
+                num_items_in_batch=num_items_in_batch,
+                ignore_index=ignore_index,
+                shift_labels=shift_labels,
+                **kwargs,
+            )
+
+        local_shift_labels = _build_cp_shift_labels(
+            labels,
+            local_seq_len=logits.shape[-2],
+            cp_rank=cp_rank,
+            ignore_index=ignore_index,
+        ).to(logits.device)
+        local_logits = logits[..., :vocab_size].contiguous().float()
+        local_loss_sum = F.cross_entropy(
+            local_logits.view(-1, vocab_size),
+            local_shift_labels.view(-1),
+            ignore_index=ignore_index,
+            reduction="sum",
+        )
+        if num_items_in_batch is not None:
+            denominator = num_items_in_batch
+            if isinstance(denominator, torch.Tensor):
+                denominator = denominator.to(device=local_loss_sum.device, dtype=local_loss_sum.dtype)
+            # Trainer counts the repeated full labels on every CP rank. Remove
+            # that duplication here; Trainer applies its normal world-size
+            # compensation after this loss function returns.
+            return (
+                local_loss_sum
+                * cp_size
+                / torch.clamp(
+                    torch.as_tensor(denominator, device=local_loss_sum.device),
+                    min=1,
+                )
+            )
+
+        cp_num_items = local_shift_labels.ne(ignore_index).sum()
+        dist.all_reduce(cp_num_items, group=cp_group)
+        denominator = torch.clamp(cp_num_items.to(local_loss_sum.dtype), min=1)
+
+        # Hyper FSDP averages gradients over the flattened DP x CP mesh.
+        backward_loss = local_loss_sum * cp_size / denominator
+        group_loss_sum = local_loss_sum.detach().clone()
+        dist.all_reduce(group_loss_sum, group=cp_group)
+        logging_loss = group_loss_sum / denominator
+        return backward_loss + (logging_loss - backward_loss).detach()
+
+    model.loss_function = cp_loss_function
+
+
 class _CPBatchRepeatedBatchSampler(torch.utils.data.BatchSampler):
     """Repeat logical batches so Accelerate shards CP peers onto the same samples."""
 
@@ -143,16 +352,13 @@ class HyperParallelTrainer(CustomSeq2SeqTrainer):
     def __init__(
         self,
         hp_args: HyperParallelArguments,
+        distributed_setup,
         finetuning_args=None,
         processor=None,
-        ref_model: Optional[nn.Module] = None,
         **kwargs,
     ):
-        self._hp_args = hp_args
-
-        # Let CustomSeq2SeqTrainer handle everything except ref_model —
-        # Custom would prepare it with accelerate's fsdp2_prepare_model,
-        # but we need HP's version instead.
+        # Keep LlamaFactory's Trainer lifecycle; Hyper owns only the already
+        # constructed model's distributed layout and checkpoint state.
         super().__init__(
             finetuning_args=finetuning_args,
             processor=processor,
@@ -164,40 +370,48 @@ class HyperParallelTrainer(CustomSeq2SeqTrainer):
             raise ValueError("HyperParallel trainer requires Accelerate FSDP2 mode to be enabled.")
 
         self._cp_size = hp_args.cp_size
-        self._ep_size = hp_args.ep_size
-        self._cp_rank = get_cp_rank(hp_args) if self._cp_size > 1 else 0
-        self._dp_rank = get_dp_rank(hp_args) if self._cp_size > 1 else get_platform().get_rank()
-
-        # Prepare ref_model with the same CP + EP + HSDP path as the train model.
-        self.ref_model = ref_model
-        if self.ref_model is not None:
-            self.ref_model = self._prepare_model_for_hyper_parallel(self.ref_model)
+        self._cp_mesh = distributed_setup.mesh_context.cp_mesh
+        if self._cp_size > 1 and self._cp_mesh is None:
+            raise RuntimeError("HyperParallel CP requires a cp_mesh from DistributedSetup.")
+        if self._cp_size > 1:
+            _enable_cp_causal_lm_loss(self.model, self._cp_mesh)
+        self._model_shards_cp_inputs = any(
+            getattr(module, "_hp_cp_shards_inputs", False) for module in self.model.modules()
+        )
 
         self._orig_accelerator_clip_grad_norm = self.accelerator.clip_grad_norm_
         self._orig_fsdp2_prepare_model = None
+        self._orig_fsdp2_set_auto_wrap_policy = None
         self._accelerator_patches_active = False
 
     def _prepare_model_for_hyper_parallel(self, model: nn.Module) -> nn.Module:
-        """Apply CP/EP preparation before delegating to HyperParallel FSDP2."""
-        if self._cp_size > 1:
-            model = cp_prepare_model(model, self.accelerator, self._hp_args)
-        if self._ep_size > 1:
-            model = ep_prepare_model(model, self.accelerator, self._hp_args)
-        return fsdp2_prepare_model(self.accelerator, model, self._hp_args)
+        """Return a model already prepared by HyperParallel's unified model path."""
+        if not isinstance(model, HSDPModule):
+            raise RuntimeError(
+                "HyperParallel models must be parallelized during meta initialization before Trainer preparation."
+            )
+        return model
 
     def _activate_accelerator_patches(self) -> None:
-        """Patch Accelerate to use HyperParallel fsdp2_prepare_model and clip_grad_norm_."""
+        """Keep Accelerate from rewrapping the prepared model and patch gradient clipping."""
         if self._accelerator_patches_active:
             return
 
         import accelerate.accelerator as acc_module  # pylint: disable=C0415
 
         self._orig_fsdp2_prepare_model = acc_module.fsdp2_prepare_model
+        fsdp_plugin = self.accelerator.state.fsdp_plugin
+        self._orig_fsdp2_set_auto_wrap_policy = fsdp_plugin.set_auto_wrap_policy
 
         def _hp_fsdp2_prepare_model(accelerator, model):
             return self._prepare_model_for_hyper_parallel(model)
 
+        def _hp_set_auto_wrap_policy(plugin, model):
+            del plugin
+            self._prepare_model_for_hyper_parallel(model)
+
         acc_module.fsdp2_prepare_model = _hp_fsdp2_prepare_model
+        fsdp_plugin.set_auto_wrap_policy = types.MethodType(_hp_set_auto_wrap_policy, fsdp_plugin)
 
         def _hp_clip_grad_norm(accelerator, parameters, max_norm, norm_type=2):
             if getattr(accelerator, "is_fsdp2", False):
@@ -224,16 +438,16 @@ class HyperParallelTrainer(CustomSeq2SeqTrainer):
 
         if self._orig_fsdp2_prepare_model is not None:
             acc_module.fsdp2_prepare_model = self._orig_fsdp2_prepare_model
+        if self._orig_fsdp2_set_auto_wrap_policy is not None:
+            self.accelerator.state.fsdp_plugin.set_auto_wrap_policy = self._orig_fsdp2_set_auto_wrap_policy
         self.accelerator.clip_grad_norm_ = self._orig_accelerator_clip_grad_norm
         self._accelerator_patches_active = False
 
     def _wrap_model(self, model: nn.Module, training: bool = True, dataloader=None) -> nn.Module:
-        """Let Accelerate own FSDP2/HSDP wrapping so optimizer remapping stays correct."""
+        """Keep Trainer from wrapping a model already prepared by HyperParallel."""
         del dataloader
         if isinstance(model, HSDPModule):
             return model
-        if training and getattr(self.accelerator, "is_fsdp2", False):
-            return model
         return super()._wrap_model(model, training=training)
 
     def _get_train_sampler(self, train_dataset=None):
@@ -252,7 +466,7 @@ class HyperParallelTrainer(CustomSeq2SeqTrainer):
             batch_size=batch_size,
             drop_last=drop_last,
             repeat_factor=self._cp_size,
-            logical_group_size=max(1, get_platform().get_world_size() // self._cp_size),
+            logical_group_size=max(1, dist.get_world_size() // self._cp_size),
         )
 
     def _get_cp_dataloader(self, dataset, batch_size: int, shuffle: bool):
@@ -281,7 +495,7 @@ class HyperParallelTrainer(CustomSeq2SeqTrainer):
             drop_last=self.args.dataloader_drop_last,
         )
         logical_batches = len(batch_sampler) // self._cp_size
-        dp_size = max(1, get_platform().get_world_size() // self._cp_size)
+        dp_size = max(1, dist.get_world_size() // self._cp_size)
         logical_length = (
             logical_batches // dp_size if self.args.dataloader_drop_last else _ceil_div(logical_batches, dp_size)
         )
@@ -350,7 +564,10 @@ class HyperParallelTrainer(CustomSeq2SeqTrainer):
         inputs = self._prepare_inputs(inputs)
 
         if self._cp_size > 1:
-            inputs = shard_inputs_for_cp(inputs, self._cp_rank, self._cp_size)
+            if self._model_shards_cp_inputs:
+                inputs = _pad_inputs_for_model_cp(inputs, self._cp_mesh)
+            else:
+                inputs = _shard_inputs_for_cp(inputs, self._cp_mesh)
 
         sync_gradients = getattr(self.accelerator, "sync_gradients", True)
         if isinstance(model, HSDPModule):
@@ -372,16 +589,25 @@ class HyperParallelTrainer(CustomSeq2SeqTrainer):
             )
             loss = loss / accumulation_steps
 
-        self.accelerator.backward(loss)
+        # The loss already accounts for the accumulation window, either via
+        # num_items_in_batch or the explicit division above. Compensate the
+        # additional scaling inside Accelerator.backward() so it happens once.
+        backward_loss = loss * self.accelerator.gradient_accumulation_steps
+        self.accelerator.backward(backward_loss)
 
         if isinstance(model, HSDPModule) and sync_gradients:
             hsdp_sync_stream()
 
         return loss.detach()
 
-    def create_optimizer(self):
+    def create_optimizer(self, *args, **kwargs):
         """Create optimizer and wrap step with SkipDTensorDispatch."""
-        optimizer = super().create_optimizer()
+        if self.optimizer is None and getattr(self.finetuning_args, "use_muon", False):
+            model = args[0] if args else kwargs.get("model", self.model)
+            self.optimizer = _create_hyper_muon_optimizer(model, self.args)
+            optimizer = self.optimizer
+        else:
+            optimizer = super().create_optimizer(*args, **kwargs)
         wrap_optimizer_with_skip_dtensor_dispatch(optimizer)
         return optimizer
 

src/llamafactory/train/hyper_parallel/workflow.py

@@ -23,9 +23,12 @@ from ...extras.logging import get_logger
 from ...extras.misc import calculate_tps
 from ...extras.packages import is_hyper_parallel_available, is_transformers_version_greater_than
 from ...extras.ploting import plot_loss
-from ...model import load_model, load_tokenizer
+from ...model import load_tokenizer
+from ...model.loader import load_config
 from ..sft.metric import ComputeAccuracy, ComputeSimilarity, eval_logit_processor
-from ..trainer_utils import create_modelcard_and_push, create_ref_model
+from ..trainer_utils import create_modelcard_and_push
+from .loader import load_hyper_parallel_model
+from .model_registry import apply_model_parallel_plan
 from .trainer import HyperParallelTrainer
 
 
@@ -38,19 +41,38 @@ if TYPE_CHECKING:
 logger = get_logger(__name__)
 
 
-def _prepare_hp_args(finetuning_args: "FinetuningArguments", model_args: "ModelArguments"):
+def _prepare_hp_args(
+    finetuning_args: "FinetuningArguments",
+    model_args: "ModelArguments",
+    training_args: "Seq2SeqTrainingArguments",
+):
     r"""Load HyperParallel arguments and apply LlamaFactory-side overrides.
 
     When activation optimization is enabled, skip native gradient checkpointing
-    so HP can install its own via ``setup_activation_optimization``.
+    so Hyper's unified model builder can install it before FSDP wrapping.
     """
     if not is_hyper_parallel_available():
         raise ImportError("hyper_parallel is not installed. Please install it with `pip install hyper_parallel`.")
 
+    if finetuning_args.finetuning_type != "full":
+        raise ValueError("HyperParallel currently requires full fine-tuning in LlamaFactory.")
+    if model_args.quantization_bit is not None:
+        raise ValueError("HyperParallel meta initialization does not support quantized LlamaFactory models yet.")
+    if model_args.use_unsloth or model_args.mixture_of_depths is not None:
+        raise ValueError("HyperParallel meta initialization is incompatible with Unsloth and mixture-of-depths.")
+    if finetuning_args.use_asft_loss:
+        raise ValueError("HyperParallel does not yet support the additional ASFT reference model.")
+
     from hyper_parallel.integration.llamafactory import HyperParallelArguments  # pylint: disable=C0415
 
     hp_args = HyperParallelArguments.from_finetuning_args(finetuning_args)
 
+    if hp_args.param_dtype is None:
+        if training_args.bf16:
+            hp_args.param_dtype = "bfloat16"
+        elif training_args.fp16:
+            hp_args.param_dtype = "float16"
+
     if getattr(hp_args, "cp_size", None) != finetuning_args.hyper_parallel_cp_size:
         setattr(hp_args, "cp_size", finetuning_args.hyper_parallel_cp_size)
 
@@ -77,17 +99,28 @@ def run_pt(
     finetuning_args: "FinetuningArguments",
     callbacks: Optional[list["TrainerCallback"]] = None,
 ):
-    hp_args = _prepare_hp_args(finetuning_args, model_args)
+    hp_args = _prepare_hp_args(finetuning_args, model_args, training_args)
+    model_type = getattr(load_config(model_args), "model_type", None)
+    apply_model_parallel_plan(hp_args, model_type)
+    distributed_setup = hp_args.build_distributed_setup()
 
     tokenizer_module = load_tokenizer(model_args)
     tokenizer = tokenizer_module["tokenizer"]
     template = get_template_and_fix_tokenizer(tokenizer, data_args)
     dataset_module = get_dataset(template, model_args, data_args, training_args, stage="pt", **tokenizer_module)
-    model = load_model(tokenizer, model_args, finetuning_args, training_args.do_train)
+    model = load_hyper_parallel_model(
+        tokenizer,
+        model_args,
+        finetuning_args,
+        distributed_setup,
+        hp_args,
+        is_trainable=training_args.do_train,
+    )
     data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
 
     trainer = HyperParallelTrainer(
         hp_args=hp_args,
+        distributed_setup=distributed_setup,
         model=model,
         args=training_args,
         finetuning_args=finetuning_args,
@@ -145,17 +178,23 @@ def run_sft(
     generating_args: "GeneratingArguments",
     callbacks: Optional[list["TrainerCallback"]] = None,
 ):
-    hp_args = _prepare_hp_args(finetuning_args, model_args)
+    hp_args = _prepare_hp_args(finetuning_args, model_args, training_args)
+    model_type = getattr(load_config(model_args), "model_type", None)
+    apply_model_parallel_plan(hp_args, model_type)
+    distributed_setup = hp_args.build_distributed_setup()
 
     tokenizer_module = load_tokenizer(model_args)
     tokenizer = tokenizer_module["tokenizer"]
     template = get_template_and_fix_tokenizer(tokenizer, data_args)
     dataset_module = get_dataset(template, model_args, data_args, training_args, stage="sft", **tokenizer_module)
-    model = load_model(tokenizer, model_args, finetuning_args, training_args.do_train)
-
-    ref_model = None
-    if finetuning_args.use_asft_loss:
-        ref_model = create_ref_model(model_args, finetuning_args)
+    model = load_hyper_parallel_model(
+        tokenizer,
+        model_args,
+        finetuning_args,
+        distributed_setup,
+        hp_args,
+        is_trainable=training_args.do_train,
+    )
 
     data_collator = SFTDataCollatorWith4DAttentionMask(
         template=template,
@@ -192,13 +231,13 @@ def run_sft(
 
     trainer = HyperParallelTrainer(
         hp_args=hp_args,
+        distributed_setup=distributed_setup,
         model=model,
         args=training_args,
         finetuning_args=finetuning_args,
         data_collator=data_collator,
         callbacks=callbacks,
         gen_kwargs=gen_kwargs,
-        ref_model=ref_model,
         **dataset_module,
         **tokenizer_module,
         **metric_module,

src/llamafactory/train/tuner.py

@@ -291,6 +291,7 @@ class Worker:
         RAY_NOSET_VISIBLE_DEVICES_LIST = [
             "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES",
             "RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES",
+            "RAY_EXPERIMENTAL_NOSET_SUPA_VISIBLE_DEVICES",
         ]
         is_ray_noset_visible_devices = any(os.environ.get(env_var, None) for env_var in RAY_NOSET_VISIBLE_DEVICES_LIST)
         if is_ray_noset_visible_devices:

src/llamafactory/v1/accelerator/helper.py

@@ -45,6 +45,7 @@ class DeviceType(StrEnum):
     META = "meta"
     MPS = "mps"
     NPU = "npu"
+    SUPA = "supa"
     XPU = "xpu"
 
 
@@ -186,6 +187,8 @@ def get_process_group_backend() -> str:
         return "hccl"
     elif get_current_accelerator().type == DeviceType.CUDA:
         return "nccl"
+    elif get_current_accelerator().type == DeviceType.XPU:
+        return "xccl"
     else:
         return "gloo"
 

src/llamafactory/v1/plugins/model_plugins/kernels/interface.py

@@ -25,11 +25,13 @@ from .ops.mlp.cuda_fused_moe import CudaFusedMoEKernel  # noqa: F401
 from .ops.mlp.npu_fused_moe import NpuFusedMoEKernel  # noqa: F401
 from .ops.mlp.npu_swiglu import NpuSwiGluKernel  # noqa: F401
 from .ops.rms_norm.npu_rms_norm import NpuRMSNormKernel  # noqa: F401
+from .ops.rms_norm.supa_rms_norm import SupaRMSNormKernel  # noqa: F401
 from .ops.rope.npu_rope import NpuRoPEKernel  # noqa: F401
 
 
 _AUTO_KERNELS = {
     DeviceType.NPU: ("npu_fused_moe", "npu_fused_rmsnorm", "npu_fused_rope", "npu_fused_swiglu"),
+    DeviceType.SUPA: ("supa_fused_rmsnorm",),
 }
 
 

src/llamafactory/v1/plugins/model_plugins/kernels/ops/rms_norm/supa_rms_norm.py

@@ -0,0 +1,158 @@
+# Copyright 2025 the LlamaFactory team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The definition of the SUPA fused RMSNorm kernel."""
+
+import types
+from functools import lru_cache
+
+import torch
+
+from ......accelerator.helper import DeviceType, get_current_accelerator
+from ......utils.logging import get_logger
+from ......utils.types import HFModel
+from ...base import BaseKernel, KernelPlugin
+
+
+logger = get_logger(__name__)
+
+
+# Standard-convention RMSNorm classes: ``y = x * rsqrt(mean(x^2) + eps) * weight``.
+# The fused ``sudnn.rms_norm_func`` matches this convention, so only these classes are patched.
+# Variants with a different weight convention (e.g. Gemma's ``1 + weight``, or gated/residual
+# RMSNorm) are intentionally excluded and fall back to the eager path. Add new class names here
+# after verifying they follow the standard convention.
+_SUPPORTED_RMSNORM_CLASSES = frozenset(
+    {
+        "LlamaRMSNorm",
+        "MistralRMSNorm",
+        "MixtralRMSNorm",
+        "Phi3RMSNorm",
+        "InternLM2RMSNorm",
+        "GlmRMSNorm",
+        "DeepseekV2RMSNorm",
+        "DeepseekV3RMSNorm",
+        "Qwen2RMSNorm",
+        "Qwen2MoeRMSNorm",
+        "Qwen2VLRMSNorm",
+        "Qwen2_5_VLRMSNorm",
+        "Qwen2_5_VLTextRMSNorm",
+        "Qwen3RMSNorm",
+        "Qwen3MoeRMSNorm",
+        "Qwen3VLTextRMSNorm",
+        "Qwen3VLMoeTextRMSNorm",
+    }
+)
+
+
+
+@lru_cache
+def _is_sudnn_rms_norm_available() -> bool:
+    """Return whether the fused SUPA RMSNorm operator is registered."""
+    try:
+        import torch_supa  # noqa: F401
+        import torch_supa_ext  # noqa: F401
+    except Exception:
+        return False
+
+    return hasattr(torch.ops, "sudnn") and hasattr(torch.ops.sudnn, "rms_norm_func")
+
+
+class SupaRMSNormFunction(torch.autograd.Function):
+    """Autograd wrapper around the fused SUPA RMSNorm forward operator."""
+
+    @staticmethod
+    def forward(ctx, hidden_states: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
+        weight = weight.to(dtype=hidden_states.dtype)
+        bias = torch.zeros_like(weight)
+        output = torch.ops.sudnn.rms_norm_func(hidden_states, weight, bias, eps, False, False)
+        ctx.save_for_backward(hidden_states, weight)
+        ctx.eps = eps
+        return output
+
+    @staticmethod
+    def backward(ctx, grad_output: torch.Tensor):
+        hidden_states, weight = ctx.saved_tensors
+        eps = ctx.eps
+        hidden_size = hidden_states.shape[-1]
+
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        rrms = torch.rsqrt(variance + eps)
+        x_norm = hidden_states * rrms
+
+        grad_input = grad_weight = None
+        if ctx.needs_input_grad[1]:
+            grad_weight = (grad_output * x_norm).reshape(-1, hidden_size).sum(0).to(weight.dtype)
+
+        if ctx.needs_input_grad[0]:
+            grad_norm = grad_output * weight
+            dot = (grad_norm * hidden_states).sum(-1, keepdim=True)
+            grad_input = rrms * grad_norm - (rrms.pow(3) / hidden_size) * hidden_states * dot
+            grad_input = grad_input.to(hidden_states.dtype)
+
+        return grad_input, grad_weight, None
+
+
+def supa_rms_norm_forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+    """Run the fused kernel for standard RMSNorm modules."""
+    if hidden_states.device.type != DeviceType.SUPA:
+        return self._supa_rmsnorm_original_forward(hidden_states)
+
+    return SupaRMSNormFunction.apply(hidden_states, self.weight, self.variance_epsilon)
+
+
+@KernelPlugin("supa_fused_rmsnorm").register()
+class SupaRMSNormKernel(BaseKernel):
+    """SUPA fused RMSNorm kernel for trainable models."""
+
+    @staticmethod
+    def check_device() -> None:
+        current = get_current_accelerator().type
+        if current != DeviceType.SUPA:
+            raise RuntimeError(f"SupaRMSNormKernel requires SUPA, current accelerator is {current}.")
+
+    @staticmethod
+    def _apply(**kwargs) -> HFModel:
+        model = kwargs["model"]
+
+        if not _is_sudnn_rms_norm_available():
+            logger.warning_rank0(
+                "Fused SUPA RMSNorm (torch.ops.sudnn.rms_norm_func) is unavailable; "
+                "the eager RMSNorm path will be used. Ensure torch_supa_ext is installed."
+            )
+            return model
+
+        patched_count = 0
+        patched_classes = set()
+        for module in model.modules():
+            cls = module.__class__
+            if getattr(module, "_supa_rmsnorm_patched", False):
+                continue
+            if cls.__name__ not in _SUPPORTED_RMSNORM_CLASSES:
+                continue
+
+            module._supa_rmsnorm_original_forward = module.forward
+            module.forward = types.MethodType(supa_rms_norm_forward, module)
+            module._supa_rmsnorm_patched = True
+            patched_count += 1
+            patched_classes.add(cls)
+
+        if patched_count:
+            logger.info_rank0(
+                "Applied fused SUPA RMSNorm to {} modules: {}.".format(
+                    patched_count, ", ".join(sorted(cls.__name__ for cls in patched_classes))
+                )
+            )
+
+        return model

tests/conftest.py

@@ -78,6 +78,8 @@ def _get_visible_devices_env() -> str | None:
         return "CUDA_VISIBLE_DEVICES"
     elif CURRENT_DEVICE == "npu":
         return "ASCEND_RT_VISIBLE_DEVICES"
+    elif CURRENT_DEVICE == "xpu":
+        return "ZE_AFFINITY_MASK"
     else:
         return None
 
@@ -160,6 +162,8 @@ def _manage_distributed_env(request: FixtureRequest, monkeypatch: MonkeyPatch) -
             monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
         elif CURRENT_DEVICE == "npu":
             monkeypatch.setattr(torch.npu, "device_count", lambda: 1)
+        elif CURRENT_DEVICE == "xpu":
+            monkeypatch.setattr(torch.xpu, "device_count", lambda: 1)
 
 
 @pytest.fixture

tests/data/processor/test_feedback.py

@@ -43,7 +43,7 @@ TRAIN_ARGS = {
 }
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.parametrize("num_samples", [16])
 def test_feedback_data(num_samples: int):
     train_dataset = load_dataset_module(**TRAIN_ARGS)["train_dataset"]

tests/data/processor/test_pairwise.py

@@ -52,7 +52,7 @@ def _convert_sharegpt_to_openai(messages: list[dict[str, str]]) -> list[dict[str
     return new_messages
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.parametrize("num_samples", [16])
 def test_pairwise_data(num_samples: int):
     train_dataset = load_dataset_module(**TRAIN_ARGS)["train_dataset"]

tests/data/processor/test_processor_utils.py

@@ -18,7 +18,7 @@ import pytest
 from llamafactory.data.processor.processor_utils import infer_seqlen
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.parametrize(
     "test_input,test_output",
     [

tests/data/processor/test_supervised.py

@@ -43,7 +43,7 @@ TRAIN_ARGS = {
 }
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.parametrize("num_samples", [16])
 def test_supervised_single_turn(num_samples: int):
     train_dataset = load_dataset_module(dataset_dir="ONLINE", dataset=TINY_DATA, **TRAIN_ARGS)["train_dataset"]
@@ -72,7 +72,7 @@ def test_supervised_single_turn(num_samples: int):
         assert train_dataset["labels"][index] == ref_label_ids
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.parametrize("num_samples", [8])
 def test_supervised_multi_turn(num_samples: int):
     train_dataset = load_dataset_module(dataset_dir="REMOTE:" + DEMO_DATA, dataset="system_chat", **TRAIN_ARGS)[
@@ -90,7 +90,7 @@ def test_supervised_multi_turn(num_samples: int):
         assert train_dataset["input_ids"][index] == ref_input_ids
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.parametrize("num_samples", [4])
 def test_supervised_train_on_prompt(num_samples: int):
     train_dataset = load_dataset_module(
@@ -108,7 +108,7 @@ def test_supervised_train_on_prompt(num_samples: int):
         assert train_dataset["labels"][index] == ref_input_ids
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.parametrize("num_samples", [4])
 def test_supervised_mask_history(num_samples: int):
     train_dataset = load_dataset_module(

tests/data/processor/test_unsupervised.py

@@ -47,7 +47,7 @@ TRAIN_ARGS = {
 }
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.parametrize("num_samples", [16])
 def test_unsupervised_data(num_samples: int):
     train_dataset = load_dataset_module(**TRAIN_ARGS)["train_dataset"]

tests/data/test_collator.py

@@ -32,7 +32,7 @@ from llamafactory.model import load_tokenizer
 TINY_LLAMA3 = os.getenv("TINY_LLAMA3", "llamafactory/tiny-random-Llama-3")
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_base_collator():
     model_args, data_args, *_ = get_infer_args({"model_name_or_path": TINY_LLAMA3, "template": "default"})
     tokenizer_module = load_tokenizer(model_args)
@@ -76,7 +76,7 @@ def test_base_collator():
         assert batch_input[k].eq(torch.tensor(expected_input[k])).all()
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_multimodal_collator():
     model_args, data_args, *_ = get_infer_args(
         {"model_name_or_path": "Qwen/Qwen2-VL-2B-Instruct", "template": "qwen2_vl"}
@@ -270,7 +270,7 @@ def _get_expected_position_ids(
     return torch.cat(all_position_ids, dim=-1)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_multimodal_collator_with_packing():
     model_args, data_args, *_ = get_infer_args(
         {"model_name_or_path": "Qwen/Qwen2-VL-2B-Instruct", "template": "qwen2_vl"}
@@ -322,7 +322,7 @@ def test_multimodal_collator_with_packing():
     assert batch_input["position_ids"][1:, :, :valid_len].eq(expected_position_ids).all()
 
 
-@pytest.mark.runs_on(["cpu"])
+@pytest.mark.runs_on(["cpu", "xpu"])
 def test_4d_attention_mask():
     o = 0.0
     x = torch.finfo(torch.float16).min

tests/data/test_converter.py

@@ -20,7 +20,7 @@ from llamafactory.data.parser import DatasetAttr
 from llamafactory.hparams import DataArguments
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_alpaca_converter():
     dataset_attr = DatasetAttr("hf_hub", "llamafactory/tiny-supervised-dataset")
     data_args = DataArguments()
@@ -41,7 +41,7 @@ def test_alpaca_converter():
     }
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_sharegpt_converter():
     dataset_attr = DatasetAttr("hf_hub", "llamafactory/tiny-supervised-dataset")
     data_args = DataArguments()

tests/data/test_formatter.py

@@ -38,19 +38,19 @@ TOOLS = [
 ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_empty_formatter():
     formatter = EmptyFormatter(slots=["\n"])
     assert formatter.apply() == ["\n"]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_string_formatter():
     formatter = StringFormatter(slots=["<s>", "Human: {{content}}\nAssistant:"])
     assert formatter.apply(content="Hi") == ["<s>", "Human: Hi\nAssistant:"]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_function_formatter():
     formatter = FunctionFormatter(slots=["{{content}}", "</s>"], tool_format="default")
     tool_calls = json.dumps(FUNCTION)
@@ -60,7 +60,7 @@ def test_function_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_multi_function_formatter():
     formatter = FunctionFormatter(slots=["{{content}}", "</s>"], tool_format="default")
     tool_calls = json.dumps([FUNCTION] * 2)
@@ -71,7 +71,7 @@ def test_multi_function_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_default_tool_formatter():
     formatter = ToolFormatter(tool_format="default")
     assert formatter.apply(content=json.dumps(TOOLS)) == [
@@ -90,14 +90,14 @@ def test_default_tool_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_default_tool_extractor():
     formatter = ToolFormatter(tool_format="default")
     result = """Action: test_tool\nAction Input: {"foo": "bar", "size": 10}"""
     assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_default_multi_tool_extractor():
     formatter = ToolFormatter(tool_format="default")
     result = (
@@ -110,14 +110,14 @@ def test_default_multi_tool_extractor():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_glm4_function_formatter():
     formatter = FunctionFormatter(slots=["{{content}}"], tool_format="glm4")
     tool_calls = json.dumps(FUNCTION)
     assert formatter.apply(content=tool_calls) == ["""tool_name\n{"foo": "bar", "size": 10}"""]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_glm4_tool_formatter():
     formatter = ToolFormatter(tool_format="glm4")
     assert formatter.apply(content=json.dumps(TOOLS)) == [
@@ -128,14 +128,14 @@ def test_glm4_tool_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_glm4_tool_extractor():
     formatter = ToolFormatter(tool_format="glm4")
     result = """test_tool\n{"foo": "bar", "size": 10}\n"""
     assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_llama3_function_formatter():
     formatter = FunctionFormatter(slots=["{{content}}<|eot_id|>"], tool_format="llama3")
     tool_calls = json.dumps(FUNCTION)
@@ -144,7 +144,7 @@ def test_llama3_function_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_llama3_multi_function_formatter():
     formatter = FunctionFormatter(slots=["{{content}}<|eot_id|>"], tool_format="llama3")
     tool_calls = json.dumps([FUNCTION] * 2)
@@ -155,7 +155,7 @@ def test_llama3_multi_function_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_llama3_tool_formatter():
     formatter = ToolFormatter(tool_format="llama3")
     date = datetime.now().strftime("%d %b %Y")
@@ -169,14 +169,14 @@ def test_llama3_tool_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_llama3_tool_extractor():
     formatter = ToolFormatter(tool_format="llama3")
     result = """{"name": "test_tool", "parameters": {"foo": "bar", "size": 10}}\n"""
     assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_llama3_multi_tool_extractor():
     formatter = ToolFormatter(tool_format="llama3")
     result = (
@@ -189,7 +189,7 @@ def test_llama3_multi_tool_extractor():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_mistral_function_formatter():
     formatter = FunctionFormatter(slots=["[TOOL_CALLS] {{content}}", "</s>"], tool_format="mistral")
     tool_calls = json.dumps(FUNCTION)
@@ -199,7 +199,7 @@ def test_mistral_function_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_mistral_multi_function_formatter():
     formatter = FunctionFormatter(slots=["[TOOL_CALLS] {{content}}", "</s>"], tool_format="mistral")
     tool_calls = json.dumps([FUNCTION] * 2)
@@ -211,7 +211,7 @@ def test_mistral_multi_function_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_mistral_tool_formatter():
     formatter = ToolFormatter(tool_format="mistral")
     wrapped_tool = {"type": "function", "function": TOOLS[0]}
@@ -220,14 +220,14 @@ def test_mistral_tool_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_mistral_tool_extractor():
     formatter = ToolFormatter(tool_format="mistral")
     result = """{"name": "test_tool", "arguments": {"foo": "bar", "size": 10}}"""
     assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_mistral_multi_tool_extractor():
     formatter = ToolFormatter(tool_format="mistral")
     result = (
@@ -240,7 +240,7 @@ def test_mistral_multi_tool_extractor():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_qwen_function_formatter():
     formatter = FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen")
     tool_calls = json.dumps(FUNCTION)
@@ -249,7 +249,7 @@ def test_qwen_function_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_qwen_multi_function_formatter():
     formatter = FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen")
     tool_calls = json.dumps([FUNCTION] * 2)
@@ -260,7 +260,7 @@ def test_qwen_multi_function_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_qwen_tool_formatter():
     formatter = ToolFormatter(tool_format="qwen")
     wrapped_tool = {"type": "function", "function": TOOLS[0]}
@@ -274,14 +274,14 @@ def test_qwen_tool_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_qwen_tool_extractor():
     formatter = ToolFormatter(tool_format="qwen")
     result = """<tool_call>\n{"name": "test_tool", "arguments": {"foo": "bar", "size": 10}}\n</tool_call>"""
     assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_qwen38_tool_formatter():
     formatter = ToolFormatter(tool_format="qwen3_8")
     wrapped_tool = {"type": "function", "function": TOOLS[0]}
@@ -289,7 +289,7 @@ def test_qwen38_tool_formatter():
     assert json.dumps(wrapped_tool, ensure_ascii=False) in output
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_qwen_multi_tool_extractor():
     formatter = ToolFormatter(tool_format="qwen")
     result = (
@@ -302,7 +302,7 @@ def test_qwen_multi_tool_extractor():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_lfm2_function_formatter():
     formatter = FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="lfm2")
     tool_calls = json.dumps(FUNCTION)
@@ -311,7 +311,7 @@ def test_lfm2_function_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_lfm2_multi_function_formatter():
     formatter = FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="lfm2")
     tool_calls = json.dumps([FUNCTION] * 2)
@@ -321,7 +321,7 @@ def test_lfm2_multi_function_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_lfm2_tool_formatter():
     formatter = ToolFormatter(tool_format="lfm2")
     assert formatter.apply(content=json.dumps(TOOLS)) == [
@@ -329,14 +329,14 @@ def test_lfm2_tool_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_lfm2_tool_extractor():
     formatter = ToolFormatter(tool_format="lfm2")
     result = """<|tool_call_start|>[test_tool(foo="bar", size=10)]<|tool_call_end|>"""
     assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_lfm2_multi_tool_extractor():
     formatter = ToolFormatter(tool_format="lfm2")
     result = """<|tool_call_start|>[test_tool(foo="bar", size=10), another_tool(foo="job", size=2)]<|tool_call_end|>"""
@@ -346,7 +346,7 @@ def test_lfm2_multi_tool_extractor():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_lfm2_tool_extractor_with_nested_dict():
     formatter = ToolFormatter(tool_format="lfm2")
     result = """<|tool_call_start|>[search(query="test", options={"limit": 10, "offset": 0})]<|tool_call_end|>"""
@@ -358,7 +358,7 @@ def test_lfm2_tool_extractor_with_nested_dict():
     assert args["options"] == {"limit": 10, "offset": 0}
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_lfm2_tool_extractor_with_list_arg():
     formatter = ToolFormatter(tool_format="lfm2")
     result = """<|tool_call_start|>[batch_process(items=[1, 2, 3], enabled=True)]<|tool_call_end|>"""
@@ -370,7 +370,7 @@ def test_lfm2_tool_extractor_with_list_arg():
     assert args["enabled"] is True
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_lfm2_tool_extractor_no_match():
     formatter = ToolFormatter(tool_format="lfm2")
     result = "This is a regular response without tool calls."
@@ -378,7 +378,7 @@ def test_lfm2_tool_extractor_no_match():
     assert extracted == result
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_lfm2_tool_round_trip():
     formatter = FunctionFormatter(slots=["{{content}}"], tool_format="lfm2")
     tool_formatter = ToolFormatter(tool_format="lfm2")
@@ -390,7 +390,7 @@ def test_lfm2_tool_round_trip():
     assert json.loads(extracted[0][1]) == original["arguments"]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_minicpm5_function_formatter():
     formatter = FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="minicpm5")
     tool_calls = json.dumps(FUNCTION)
@@ -399,7 +399,7 @@ def test_minicpm5_function_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_minicpm5_multi_function_formatter():
     formatter = FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="minicpm5")
     tool_calls = json.dumps([FUNCTION] * 2)
@@ -411,7 +411,7 @@ def test_minicpm5_multi_function_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_minicpm5_tool_formatter():
     formatter = ToolFormatter(tool_format="minicpm5")
     wrapped = json.dumps({"type": "function", "function": TOOLS[0]}, ensure_ascii=False)
@@ -427,28 +427,28 @@ def test_minicpm5_tool_formatter():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_minicpm5_tool_extractor():
     formatter = ToolFormatter(tool_format="minicpm5")
     result = '<function name="test_tool"><param name="foo">bar</param><param name="size">10</param></function>'
     assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_minicpm5_tool_extractor_cdata():
     formatter = ToolFormatter(tool_format="minicpm5")
     result = '<function name="test_tool"><param name="foo"><![CDATA[a < b\nsecond line]]></param></function>'
     assert formatter.extract(result) == [("test_tool", json.dumps({"foo": "a < b\nsecond line"}))]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_minicpm5_tool_extractor_malformed_value():
     formatter = ToolFormatter(tool_format="minicpm5")
     result = '<function name="test_tool"><param name="foo">{[1, 2], [3, 4]}</param></function>'
     assert formatter.extract(result) == [("test_tool", json.dumps({"foo": "{[1, 2], [3, 4]}"}))]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.parametrize(
     "arguments",
     [

tests/data/test_loader.py

@@ -40,21 +40,21 @@ TRAIN_ARGS = {
 }
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_load_train_only():
     dataset_module = load_dataset_module(**TRAIN_ARGS)
     assert dataset_module.get("train_dataset") is not None
     assert dataset_module.get("eval_dataset") is None
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_load_val_size():
     dataset_module = load_dataset_module(val_size=0.1, **TRAIN_ARGS)
     assert dataset_module.get("train_dataset") is not None
     assert dataset_module.get("eval_dataset") is not None
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_load_eval_data():
     dataset_module = load_dataset_module(eval_dataset=TINY_DATA, **TRAIN_ARGS)
     assert dataset_module.get("train_dataset") is not None

tests/data/test_mm_plugin.py

@@ -180,7 +180,7 @@ def _check_plugin(
     )
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_base_plugin():
     tokenizer_module = _load_tokenizer_module(model_name_or_path=TINY_LLAMA3)
     base_plugin = get_mm_plugin(name="base")
@@ -188,7 +188,7 @@ def test_base_plugin():
     _check_plugin(**check_inputs)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.skipif(not HF_TOKEN, reason="Gated model.")
 @pytest.mark.skipif(not is_transformers_version_greater_than("4.50.0"), reason="Requires transformers>=4.50.0")
 def test_gemma3_plugin():
@@ -211,7 +211,7 @@ def test_gemma3_plugin():
     _check_plugin(**check_inputs)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.skipif(not is_transformers_version_greater_than("5.6.0"), reason="Requires transformers>=5.6.0")
 def test_gemma4_plugin():
     tokenizer_module = _load_tokenizer_module(model_name_or_path="google/gemma-4-31B-it")
@@ -244,7 +244,7 @@ def test_gemma4_plugin():
     _check_plugin(**check_inputs)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.skipif(not is_transformers_version_greater_than("4.52.0"), reason="Requires transformers>=4.52.0")
 def test_internvl_plugin():
     image_seqlen = 256
@@ -263,7 +263,7 @@ def test_internvl_plugin():
     _check_plugin(**check_inputs)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.skipif(not is_transformers_version_greater_than("4.51.0"), reason="Requires transformers>=4.51.0")
 def test_llama4_plugin():
     tokenizer_module = _load_tokenizer_module(model_name_or_path=TINY_LLAMA4)
@@ -285,7 +285,7 @@ def test_llama4_plugin():
     _check_plugin(**check_inputs)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_llava_plugin():
     image_seqlen = 576
     tokenizer_module = _load_tokenizer_module(model_name_or_path="llava-hf/llava-1.5-7b-hf")
@@ -299,7 +299,7 @@ def test_llava_plugin():
     _check_plugin(**check_inputs)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_llava_next_plugin():
     image_seqlen = 1176
     tokenizer_module = _load_tokenizer_module(model_name_or_path="llava-hf/llava-v1.6-vicuna-7b-hf")
@@ -313,7 +313,7 @@ def test_llava_next_plugin():
     _check_plugin(**check_inputs)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_llava_next_video_plugin():
     image_seqlen = 1176
     tokenizer_module = _load_tokenizer_module(model_name_or_path="llava-hf/LLaVA-NeXT-Video-7B-hf")
@@ -327,7 +327,7 @@ def test_llava_next_video_plugin():
     _check_plugin(**check_inputs)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.skipif(not HF_TOKEN, reason="Gated model.")
 def test_paligemma_plugin():
     image_seqlen = 256
@@ -347,7 +347,7 @@ def test_paligemma_plugin():
     _check_plugin(**check_inputs)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.skipif(not is_transformers_version_greater_than("4.50.0"), reason="Requires transformers>=4.50.0")
 def test_pixtral_plugin():
     image_slice_height, image_slice_width = 2, 2
@@ -370,7 +370,7 @@ def test_pixtral_plugin():
     _check_plugin(**check_inputs)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.skipif(not is_transformers_version_greater_than("4.52.0"), reason="Requires transformers>=4.52.0")
 def test_qwen2_omni_plugin():
     image_seqlen, audio_seqlen = 4, 2
@@ -401,7 +401,7 @@ def test_qwen2_omni_plugin():
     _check_plugin(**check_inputs)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_qwen2_vl_plugin():
     image_seqlen = 4
     tokenizer_module = _load_tokenizer_module(model_name_or_path="Qwen/Qwen2-VL-7B-Instruct")
@@ -436,7 +436,7 @@ def test_moss_vl_plugin():
     assert messages[0]["content"] == "First <image>, finally <image>."
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.skipif(not is_transformers_version_greater_than("4.57.0"), reason="Requires transformers>=4.57.0")
 def test_qwen3_vl_plugin():
     frame_seqlen = 1
@@ -470,7 +470,7 @@ def test_qwen3_vl_plugin():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.skipif(not is_transformers_version_greater_than("4.57.0"), reason="Requires transformers>=4.57.0")
 @pytest.mark.skipif(not is_pyav_available(), reason="Requires pyav")
 def test_qwen3_vl_plugin_video_path():
@@ -504,7 +504,7 @@ def test_qwen3_vl_plugin_video_path():
     )
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.skipif(not is_transformers_version_greater_than("4.47.0"), reason="Requires transformers>=4.47.0")
 def test_video_llava_plugin():
     image_seqlen = 256
@@ -519,7 +519,7 @@ def test_video_llava_plugin():
     _check_plugin(**check_inputs)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_lfm2_vl_plugin():
     """Test LFM2.5-VL plugin instantiation."""
     # Test plugin can be instantiated with correct tokens

tests/data/test_template.py

@@ -139,7 +139,7 @@ def test_qwen38_registration():
     assert {"qwen3_5_moe_text", "qwen3_5_text"} <= MCA_SUPPORTED_MODELS
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_encode_oneturn():
     tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)
     template = get_template_and_fix_tokenizer(tokenizer, DataArguments(template="llama3"))
@@ -154,7 +154,7 @@ def test_encode_oneturn():
     _check_tokenization(tokenizer, (prompt_ids, answer_ids), (prompt_str, answer_str))
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_encode_multiturn():
     tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)
     template = get_template_and_fix_tokenizer(tokenizer, DataArguments(template="llama3"))
@@ -175,7 +175,7 @@ def test_encode_multiturn():
     )
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.parametrize("cot_messages", [True, False])
 @pytest.mark.parametrize("enable_thinking", [True, False, None])
 def test_reasoning_encode_oneturn(cot_messages: bool, enable_thinking: bool):
@@ -201,7 +201,7 @@ def test_reasoning_encode_oneturn(cot_messages: bool, enable_thinking: bool):
     _check_tokenization(tokenizer, (prompt_ids, answer_ids), (prompt_str, answer_str))
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.parametrize("cot_messages", [True, False])
 @pytest.mark.parametrize("enable_thinking", [True, False, None])
 def test_reasoning_encode_multiturn(cot_messages: bool, enable_thinking: bool):
@@ -230,7 +230,7 @@ def test_reasoning_encode_multiturn(cot_messages: bool, enable_thinking: bool):
     )
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.parametrize("enable_thinking", [True, False, None])
 @pytest.mark.parametrize("discarding_history_cot", [True, False])
 def test_reasoning_encode_multiturn_discarding_history_cot(enable_thinking: bool, discarding_history_cot: bool):
@@ -266,7 +266,7 @@ def test_reasoning_encode_multiturn_discarding_history_cot(enable_thinking: bool
     )
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_qwen38_reasoning_instruction():
     tokenizer = CharTokenizer()
     template = deepcopy(TEMPLATES["qwen3_8"])
@@ -284,7 +284,7 @@ def test_qwen38_reasoning_instruction():
     assert answer_text == "<think>\n\n</think>\n\nI am fine!<|im_end|>\n"
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_qwen38_tool_system_order():
     tokenizer = CharTokenizer()
     template = deepcopy(TEMPLATES["qwen3_8"])
@@ -308,7 +308,7 @@ def test_qwen38_tool_system_order():
     )
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_jinja_template():
     tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)
     ref_tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)
@@ -318,7 +318,7 @@ def test_jinja_template():
     assert tokenizer.apply_chat_template(MESSAGES) == ref_tokenizer.apply_chat_template(MESSAGES)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_ollama_modelfile():
     tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)
     template = get_template_and_fix_tokenizer(tokenizer, DataArguments(template="llama3"))
@@ -336,14 +336,14 @@ def test_ollama_modelfile():
     )
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_get_stop_token_ids():
     tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)
     template = get_template_and_fix_tokenizer(tokenizer, DataArguments(template="llama3"))
     assert set(template.get_stop_token_ids(tokenizer)) == {128008, 128009}
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.skipif(not HF_TOKEN, reason="Gated model.")
 def test_gemma_template():
     prompt_str = (
@@ -356,7 +356,7 @@ def test_gemma_template():
     _check_template("google/gemma-3-4b-it", "gemma", prompt_str, answer_str)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.skipif(not HF_TOKEN, reason="Gated model.")
 def test_gemma2_template():
     prompt_str = (
@@ -369,7 +369,7 @@ def test_gemma2_template():
     _check_template("google/gemma-2-2b-it", "gemma2", prompt_str, answer_str)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.skipif(not HF_TOKEN, reason="Gated model.")
 def test_llama3_template():
     prompt_str = (
@@ -382,7 +382,7 @@ def test_llama3_template():
     _check_template("meta-llama/Meta-Llama-3-8B-Instruct", "llama3", prompt_str, answer_str)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_llama4_template():
     prompt_str = (
         f"<|begin_of_text|><|header_start|>user<|header_end|>\n\n{MESSAGES[0]['content']}<|eot|>"
@@ -394,7 +394,7 @@ def test_llama4_template():
     _check_template(TINY_LLAMA4, "llama4", prompt_str, answer_str)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_phi4_template():
     prompt_str = (
         f"<|im_start|>user<|im_sep|>{MESSAGES[0]['content']}<|im_end|>"
@@ -406,7 +406,7 @@ def test_phi4_template():
     _check_template("microsoft/phi-4", "phi4", prompt_str, answer_str)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.xfail(not HF_TOKEN, reason="Authorization.")
 def test_qwen2_5_template():
     prompt_str = (
@@ -420,7 +420,7 @@ def test_qwen2_5_template():
     _check_template("Qwen/Qwen2.5-7B-Instruct", "qwen", prompt_str, answer_str)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.parametrize("cot_messages", [True, False])
 def test_qwen3_template(cot_messages: bool):
     prompt_str = (
@@ -439,7 +439,7 @@ def test_qwen3_template(cot_messages: bool):
     _check_template("Qwen/Qwen3-8B", "qwen3", prompt_str, answer_str, messages=messages)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_parse_llama3_template():
     tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)
     template = parse_template(tokenizer)
@@ -453,7 +453,7 @@ def test_parse_llama3_template():
     assert template.default_system == ""
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.xfail(not HF_TOKEN, reason="Authorization.")
 def test_parse_qwen_template():
     tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B-Instruct")
@@ -466,7 +466,7 @@ def test_parse_qwen_template():
     assert template.default_system == "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.xfail(not HF_TOKEN, reason="Authorization.")
 def test_parse_qwen3_template():
     tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-8B")

tests/e2e/test_chat.py

@@ -37,13 +37,13 @@ MESSAGES = [
 EXPECTED_RESPONSE = "_rho"
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_chat():
     chat_model = ChatModel(INFER_ARGS)
     assert chat_model.chat(MESSAGES)[0].response_text == EXPECTED_RESPONSE
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_stream_chat():
     chat_model = ChatModel(INFER_ARGS)
     response = ""

tests/e2e/test_train.py

@@ -49,7 +49,7 @@ INFER_ARGS = {
 OS_NAME = os.getenv("OS_NAME", "")
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 @pytest.mark.parametrize(
     "stage,dataset",
     [
@@ -66,7 +66,7 @@ def test_run_exp(stage: str, dataset: str):
     assert os.path.exists(output_dir)
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_export():
     export_dir = os.path.join("output", "llama3_export")
     export_model({"export_dir": export_dir, **INFER_ARGS})

tests/eval/test_eval_template.py

@@ -17,7 +17,7 @@ import pytest
 from llamafactory.eval.template import get_eval_template
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_eval_template_en():
     support_set = [
         {
@@ -56,7 +56,7 @@ def test_eval_template_en():
     ]
 
 
-@pytest.mark.runs_on(["cpu", "mps"])
+@pytest.mark.runs_on(["cpu", "mps", "xpu"])
 def test_eval_template_zh():
     support_set = [
         {

tests/train/test_hyper_parallel.py

@@ -0,0 +1,375 @@
+# Copyright 2026 the LlamaFactory team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from contextlib import nullcontext
+from importlib.util import find_spec
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+
+
+if find_spec("hyper_parallel") is None:
+    pytest.skip("HyperParallel is an optional dependency and is not installed.", allow_module_level=True)
+else:
+    import torch
+    from hyper_parallel.core.optimizer import ChainedOptimizer
+
+    from llamafactory.train.hyper_parallel.loader import load_hyper_parallel_model
+    from llamafactory.train.hyper_parallel.model_registry import apply_model_parallel_plan
+    from llamafactory.train.hyper_parallel.trainer import (
+        HyperParallelTrainer,
+        _build_cp_shift_labels,
+        _create_hyper_muon_optimizer,
+        _pad_inputs_for_model_cp,
+        _shard_inputs_for_cp,
+    )
+    from llamafactory.train.hyper_parallel.workflow import _prepare_hp_args
+
+
+def test_create_hyper_muon_optimizer_preserves_llamafactory_parameter_split():
+    class _Model(torch.nn.Module):
+        def __init__(self):
+            super().__init__()
+            self.embed_tokens = torch.nn.Embedding(8, 4)
+            self.proj = torch.nn.Linear(4, 4)
+            self.norm = torch.nn.LayerNorm(4)
+            self.lm_head = torch.nn.Linear(4, 8, bias=False)
+
+    model = _Model()
+    training_args = SimpleNamespace(
+        learning_rate=3.0e-4,
+        weight_decay=0.1,
+        adam_beta1=0.8,
+        adam_beta2=0.95,
+        adam_epsilon=1.0e-8,
+    )
+    expected_optimizer = ChainedOptimizer(
+        model,
+        {"adamw": torch.optim.AdamW(model.parameters(), lr=training_args.learning_rate)},
+    )
+
+    with patch(
+        "llamafactory.train.hyper_parallel.trainer.get_hyper_optimizer",
+        return_value=expected_optimizer,
+    ) as get_optimizer:
+        optimizer = _create_hyper_muon_optimizer(model, training_args)
+
+    assert isinstance(optimizer, torch.optim.Optimizer)
+    assert optimizer.optimizer is expected_optimizer
+    call_kwargs = get_optimizer.call_args.kwargs
+    assert call_kwargs["muon_params"][0]["params"] == [model.proj.weight]
+    assert set(call_kwargs["adamw_params"][0]["params"]) == {
+        model.embed_tokens.weight,
+        model.proj.bias,
+        model.norm.weight,
+        model.norm.bias,
+        model.lm_head.weight,
+    }
+    assert call_kwargs["muon_kwargs"] == {"muon_lr": 3.0e-4, "muon_weight_decay": 0.1}
+    assert call_kwargs["adamw_kwargs"] == {
+        "adamw_lr": 3.0e-4,
+        "adamw_weight_decay": 0.1,
+        "adamw_betas": (0.8, 0.95),
+        "adamw_eps": 1.0e-8,
+    }
+    optimizer.param_groups[0]["lr"] = 1.0e-4
+    assert expected_optimizer.chained_optimizers[0].param_groups[0]["lr"] == 1.0e-4
+
+
+def test_cp_batch_adapter_keeps_global_mask_and_generates_local_positions():
+    class _CPMesh:
+        @staticmethod
+        def size():
+            return 2
+
+        @staticmethod
+        def get_local_rank():
+            return 1
+
+    inputs = {
+        "input_ids": torch.arange(8).unsqueeze(0),
+        "labels": torch.arange(8).unsqueeze(0),
+        "attention_mask": torch.ones(1, 8, dtype=torch.long),
+    }
+
+    sharded = _shard_inputs_for_cp(inputs, _CPMesh())
+
+    torch.testing.assert_close(sharded["input_ids"], torch.arange(4, 8).unsqueeze(0))
+    torch.testing.assert_close(sharded["labels"], inputs["labels"])
+    torch.testing.assert_close(sharded["position_ids"], torch.arange(4, 8).unsqueeze(0))
+    torch.testing.assert_close(sharded["attention_mask"], inputs["attention_mask"])
+
+
+def test_cp_shift_labels_preserve_cross_rank_causal_boundary():
+    labels = torch.tensor([[10, 11, 12, 13, 14, 15, 16, 17]])
+
+    rank0_labels = _build_cp_shift_labels(labels, local_seq_len=4, cp_rank=0, ignore_index=-100)
+    rank1_labels = _build_cp_shift_labels(labels, local_seq_len=4, cp_rank=1, ignore_index=-100)
+
+    torch.testing.assert_close(rank0_labels, torch.tensor([[11, 12, 13, 14]]))
+    torch.testing.assert_close(rank1_labels, torch.tensor([[15, 16, 17, -100]]))
+
+
+def test_model_cp_padding_preserves_multimodal_fields():
+    class _CPMesh:
+        @staticmethod
+        def size():
+            return 2
+
+    pixel_values = torch.randn(6, 12)
+    image_grid_thw = torch.tensor([[1, 2, 3]])
+    inputs = {
+        "input_ids": torch.arange(6).unsqueeze(0),
+        "labels": torch.arange(6).unsqueeze(0),
+        "attention_mask": torch.ones(1, 6, dtype=torch.long),
+        "mm_token_type_ids": torch.zeros(1, 6, dtype=torch.long),
+        "pixel_values": pixel_values,
+        "image_grid_thw": image_grid_thw,
+    }
+
+    padded = _pad_inputs_for_model_cp(inputs, _CPMesh())
+
+    assert padded["input_ids"].shape[-1] == 8
+    torch.testing.assert_close(padded["labels"][..., -2:], torch.full((1, 2), -100))
+    torch.testing.assert_close(padded["attention_mask"][..., -2:], torch.zeros(1, 2, dtype=torch.long))
+    assert padded["pixel_values"] is pixel_values
+    assert padded["image_grid_thw"] is image_grid_thw
+
+
+def test_accelerate_patches_skip_torch_auto_wrap_for_prepared_hyper_model():
+    class _FSDPPlugin:
+        def __init__(self):
+            self.calls = []
+
+        def set_auto_wrap_policy(self, model):
+            self.calls.append(model)
+
+    plugin = _FSDPPlugin()
+    original_clip = object()
+    trainer = object.__new__(HyperParallelTrainer)
+    trainer.accelerator = SimpleNamespace(
+        state=SimpleNamespace(fsdp_plugin=plugin),
+        clip_grad_norm_=original_clip,
+    )
+    trainer._orig_accelerator_clip_grad_norm = original_clip
+    trainer._orig_fsdp2_prepare_model = None
+    trainer._orig_fsdp2_set_auto_wrap_policy = None
+    trainer._accelerator_patches_active = False
+    prepared_models = []
+    trainer._prepare_model_for_hyper_parallel = lambda model: prepared_models.append(model) or model
+
+    trainer._activate_accelerator_patches()
+    model = object()
+    plugin.set_auto_wrap_policy(model)
+    trainer._restore_accelerator_patches()
+
+    assert prepared_models == [model]
+    assert plugin.calls == []
+    assert plugin.set_auto_wrap_policy == trainer._orig_fsdp2_set_auto_wrap_policy
+
+
+def test_training_step_compensates_accelerate_gradient_accumulation_scaling():
+    for model_accepts_loss_kwargs, expected_loss in ((True, 8.0), (False, 4.0)):
+        model = torch.nn.Linear(1, 1, bias=False)
+        torch.nn.init.ones_(model.weight)
+        trainer = object.__new__(HyperParallelTrainer)
+        trainer._cp_size = 1
+        trainer.args = SimpleNamespace(n_gpu=1, gradient_accumulation_steps=2)
+        trainer.current_gradient_accumulation_steps = 2
+        trainer.model_accepts_loss_kwargs = model_accepts_loss_kwargs
+        trainer.compute_loss_func = None
+        trainer._prepare_inputs = lambda inputs: inputs
+        trainer.compute_loss_context_manager = nullcontext
+        trainer.compute_loss = lambda model, inputs, num_items_in_batch=None: model.weight.sum() * 8
+        trainer.accelerator = SimpleNamespace(
+            sync_gradients=True,
+            gradient_accumulation_steps=2,
+            backward=lambda loss: (loss / 2).backward(),
+        )
+
+        logged_loss = trainer.training_step(model, {}, num_items_in_batch=1)
+
+        torch.testing.assert_close(logged_loss, torch.tensor(expected_loss))
+        torch.testing.assert_close(model.weight.grad, torch.full_like(model.weight, expected_loss))
+
+
+def test_hyper_model_loader_uses_meta_then_parallelize_path():
+    config = SimpleNamespace(model_type="test")
+    model = torch.nn.Linear(2, 2, device="meta")
+    model.config = config
+    calls = []
+
+    def _apply_v1_kernels(input_model, use_v1_kernels):
+        assert input_model is model
+        assert use_v1_kernels is True
+        calls.append("v1")
+        return input_model
+
+    def _parallelize_model(input_model, *args, **kwargs):
+        assert input_model is model
+        calls.append("parallelize")
+        return input_model
+
+    class _AutoModel:
+        @staticmethod
+        def from_config(input_config, trust_remote_code=False, dtype=None):
+            assert input_config is config
+            assert trust_remote_code is False
+            assert dtype is torch.bfloat16
+            return model
+
+    model_args = SimpleNamespace(
+        model_name_or_path="checkpoint",
+        train_from_scratch=False,
+        trust_remote_code=False,
+        use_kt=False,
+        use_v1_kernels=True,
+        print_param_status=False,
+        compute_dtype=torch.bfloat16,
+    )
+    finetuning_args = SimpleNamespace(stage="sft")
+    hp_args = SimpleNamespace(activation_mode="none", activation_swap_inputs=True)
+    distributed_setup = object()
+
+    with (
+        patch("llamafactory.train.hyper_parallel.loader._get_init_kwargs", return_value={}),
+        patch("llamafactory.train.hyper_parallel.loader.load_config", return_value=config),
+        patch("llamafactory.train.hyper_parallel.loader.patch_config"),
+        patch("llamafactory.train.hyper_parallel.loader.apply_liger_kernel"),
+        patch("llamafactory.train.hyper_parallel.loader._get_model_class", return_value=_AutoModel),
+        patch("llamafactory.train.hyper_parallel.loader._get_no_init_weights", return_value=nullcontext),
+        patch("llamafactory.train.hyper_parallel.loader.init_empty_weights", return_value=nullcontext()),
+        patch("llamafactory.train.hyper_parallel.loader.patch_model"),
+        patch("llamafactory.train.hyper_parallel.loader.register_autoclass"),
+        patch("llamafactory.train.hyper_parallel.loader.init_adapter", return_value=model),
+        patch(
+            "llamafactory.v1.plugins.model_plugins.kernels.interface.apply_v1_kernels",
+            side_effect=_apply_v1_kernels,
+        ) as apply_v1_kernels,
+        patch(
+            "llamafactory.train.hyper_parallel.loader.parallelize_model",
+            side_effect=_parallelize_model,
+        ) as parallelize,
+        patch("llamafactory.train.hyper_parallel.loader._validate_conv3d_compatibility"),
+    ):
+        result = load_hyper_parallel_model(
+            tokenizer=object(),
+            model_args=model_args,
+            finetuning_args=finetuning_args,
+            distributed_setup=distributed_setup,
+            hp_args=hp_args,
+            is_trainable=True,
+        )
+
+    assert result is model
+    assert calls == ["v1", "parallelize"]
+    apply_v1_kernels.assert_called_once_with(model, use_v1_kernels=True)
+    parallelize.assert_called_once_with(
+        model,
+        distributed_setup,
+        pretrained_path="checkpoint",
+        activation_checkpoint=None,
+        swap_inputs=False,
+    )
+
+
+def _make_hyper_workflow_args(finetuning_type="full"):
+    finetuning_args = SimpleNamespace(
+        hyper_parallel_args=None,
+        hyper_parallel_cp_size=1,
+        hyper_parallel_ep_size=1,
+        hyper_parallel_efsdp_size=1,
+        hyper_parallel_token_dispatcher="all_to_all",
+        finetuning_type=finetuning_type,
+        use_asft_loss=False,
+    )
+    model_args = SimpleNamespace(
+        disable_gradient_checkpointing=False,
+        quantization_bit=None,
+        use_unsloth=False,
+        mixture_of_depths=None,
+    )
+    return finetuning_args, model_args
+
+
+def test_prepare_hp_args_accepts_supported_full_tuning():
+    finetuning_args, model_args = _make_hyper_workflow_args()
+    training_args = SimpleNamespace(bf16=True, fp16=False)
+
+    with patch("hyper_parallel.integration.llamafactory.utils.dist.get_world_size", return_value=1):
+        hp_args = _prepare_hp_args(finetuning_args, model_args, training_args)
+
+    assert hp_args.cp_size == 1
+    assert hp_args.ep_size == 1
+    assert hp_args.param_dtype == "bfloat16"
+
+
+def test_prepare_hp_args_rejects_parameter_efficient_tuning():
+    finetuning_args, model_args = _make_hyper_workflow_args(finetuning_type="lora")
+    training_args = SimpleNamespace(bf16=True, fp16=False)
+
+    try:
+        _prepare_hp_args(finetuning_args, model_args, training_args)
+    except ValueError as exc:
+        assert "requires full fine-tuning" in str(exc)
+    else:
+        raise AssertionError("Expected HyperParallel to reject LoRA tuning.")
+
+
+def test_qwen3_moe_parallel_plan_resolves_through_hyper_builder():
+    from hyper_parallel.integration.llamafactory import HyperParallelArguments
+
+    hp_args = HyperParallelArguments(
+        cp_size=2,
+        ep_size=4,
+        device_type="npu",
+        plan_overrides=[{"match": "*.mlp", "region_dispatch": True}],
+    )
+    apply_model_parallel_plan(hp_args, "qwen3_moe")
+
+    with (
+        patch("hyper_parallel.integration.llamafactory.utils.dist.get_world_size", return_value=8),
+        patch("hyper_parallel.integration.llamafactory.utils.MeshContext.build_meshs"),
+    ):
+        setup = hp_args.build_distributed_setup()
+
+    cp_spec = setup.plan_overrides["*.self_attn"]
+    ep_spec = setup.plan_overrides["*.mlp"]
+    assert cp_spec.inner_wrapper.callable.__name__ == "qwen3_moe_async_ulysses_cp_wrapper"
+    assert ep_spec.local_compute_fn.callable.__name__ == "qwen3moe_ep_compute_fn"
+    assert ep_spec.local_compute_fn.use_grouped_gemm is True
+    assert ep_spec.region_dispatch is True
+    assert not setup.module_replacements
+
+
+def test_qwen3_vl_moe_parallel_plan_only_uses_its_model_adapter():
+    hp_args = SimpleNamespace(plan_overrides=None)
+
+    apply_model_parallel_plan(hp_args, "qwen3_vl_moe")
+
+    targets = str(hp_args.plan_overrides)
+    assert "hyper_parallel.models.qwen3_vl_moe.adapter" in targets
+    assert "hyper_parallel.models.qwen3_moe.adapter" not in targets
+    assert {entry["match"] for entry in hp_args.plan_overrides} == {
+        "model.visual.blocks.*.attn",
+        "model.visual.merger",
+        "model.visual.deepstack_merger_list",
+        "model.language_model",
+        "model.language_model.layers.*.self_attn",
+        "model.language_model.layers.*.mlp",
+    }
+    for entry in hp_args.plan_overrides:
+        if entry["match"] in {"model.visual.merger", "model.visual.deepstack_merger_list"}:
+            assert entry["params"] == {}

tests_v1/accelerator/test_interface.py

@@ -52,7 +52,7 @@ def test_all_device():
     assert DistributedInterface().get_local_world_size() == int(os.getenv("LOCAL_WORLD_SIZE", "1"))
 
 
-@pytest.mark.runs_on(["cuda", "npu"])
+@pytest.mark.runs_on(["cuda", "npu", "xpu"])
 @pytest.mark.require_distributed(2)
 def test_multi_device():
     master_port = find_available_port()

tests_v1/conftest.py

@@ -79,6 +79,8 @@ def _get_visible_devices_env() -> str | None:
         return "CUDA_VISIBLE_DEVICES"
     elif CURRENT_DEVICE == "npu":
         return "ASCEND_RT_VISIBLE_DEVICES"
+    elif CURRENT_DEVICE == "xpu":
+        return "ZE_AFFINITY_MASK"
     else:
         return None
 
@@ -172,6 +174,8 @@ def _manage_distributed_env(request: FixtureRequest, monkeypatch: MonkeyPatch) -
             monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
         elif CURRENT_DEVICE == "npu":
             monkeypatch.setattr(torch.npu, "device_count", lambda: 1)
+        elif CURRENT_DEVICE == "xpu":
+            monkeypatch.setattr(torch.xpu, "device_count", lambda: 1)
 
 
 @pytest.fixture(scope="session", autouse=True)

tests_v1/plugins/model_plugins/test_ulysses_cp.py

@@ -96,7 +96,7 @@ def _test_sequence_parallel_loss(
         assert loss is not None
 
 
-@pytest.mark.runs_on(["cuda", "npu"])
+@pytest.mark.runs_on(["cuda", "npu", "xpu"])
 @pytest.mark.require_distributed(2)
 @pytest.mark.parametrize(("cp_size", "dp_size", "batch_size"), [(2, 1, 1), (2, 1, 2)])
 def test_sequence_parallel_loss(cp_size, dp_size, batch_size):

tests_v1/sampler/test_cli_sampler.py

@@ -19,7 +19,7 @@ from llamafactory.v1.core.model_engine import ModelEngine
 from llamafactory.v1.samplers.cli_sampler import SyncSampler
 
 
-@pytest.mark.runs_on(["cuda", "npu"])
+@pytest.mark.runs_on(["cuda", "npu", "xpu"])
 def test_sync_sampler():
     model_args = ModelArguments(model="Qwen/Qwen3-4B-Instruct-2507")
     sample_args = SampleArguments()

tests_v1/trainers/test_fsdp2_dpo_trainer.py

@@ -21,7 +21,7 @@ import pytest
 
 
 @pytest.mark.xfail(reason="CI machines may OOM when heavily loaded.")
-@pytest.mark.runs_on(["cuda", "npu"])
+@pytest.mark.runs_on(["cuda", "npu", "xpu"])
 def test_fsdp2_dpo_trainer(tmp_path: Path):
     """Test FSDP2 DPO trainer with sigmoid loss by simulating `llamafactory-cli dpo config.yaml`."""
     config_yaml = """\

tests_v1/trainers/test_fsdp2_sft_trainer.py

@@ -20,7 +20,7 @@ import pytest
 
 
 @pytest.mark.xfail(reason="CI machines may OOM when heavily loaded.")
-@pytest.mark.runs_on(["cuda", "npu"])
+@pytest.mark.runs_on(["cuda", "npu", "xpu"])
 def test_fsdp2_sft_trainer(tmp_path: Path):
     """Test FSDP2 SFT trainer by simulating `llamafactory-cli sft config.yaml` behavior."""
     config_yaml = """\