mirror of
https://github.com/hiyouga/LLaMA-Factory.git
synced 2026-10-17 12:25:45 +08:00
Compare commits
4 Commits
97b32d3133
...
ce9dc9e072
| Author | SHA1 | Date | |
|---|---|---|---|
|
ce9dc9e072 | ||
|
|
c1dc0f51ab | ||
|
4d6c7cf03b | ||
|
d4823eb10e |
9
.github/workflows/docker_npu.yml
vendored
9
.github/workflows/docker_npu.yml
vendored
@@ -27,6 +27,12 @@ jobs:
|
||||
- device: "npu-a3"
|
||||
os: "openeuler"
|
||||
base_image: "quay.io/ascend/cann:9.1.0-a3-openeuler24.03-py3.12"
|
||||
- device: "npu-a5"
|
||||
os: "ubuntu"
|
||||
base_image: "quay.io/ascend/cann:9.1.0-950-ubuntu22.04-py3.12"
|
||||
- device: "npu-a5"
|
||||
os: "openeuler"
|
||||
base_image: "quay.io/ascend/cann:9.1.0-950-openeuler24.03-py3.12"
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -68,7 +74,7 @@ jobs:
|
||||
base_image_tag="${BASE_IMAGE##*:}"
|
||||
cann_version="${base_image_tag%%-*}"
|
||||
torch_npu_version="$(sed -nE 's/^torch[-_]npu==([0-9]+(\.[0-9]+)*(\.post[0-9]+)?).*/\1/p' requirements/npu.txt)"
|
||||
soc="$(grep -oE '(910b|a3)' <<< "${base_image_tag}" | head -n 1)"
|
||||
soc="$(grep -oE '(910b|a3|950)' <<< "${base_image_tag}" | head -n 1)"
|
||||
operating_system="$(grep -oE '(ubuntu|openeuler)[0-9]+(\.[0-9]+)*' <<< "${base_image_tag}" | head -n 1)"
|
||||
python_version="$(grep -oE 'py[0-9]+\.[0-9]+' <<< "${base_image_tag}" | head -n 1)"
|
||||
|
||||
@@ -85,6 +91,7 @@ jobs:
|
||||
case "${MATRIX_DEVICE}" in
|
||||
npu-a2) expected_soc="910b" ;;
|
||||
npu-a3) expected_soc="a3" ;;
|
||||
npu-a5) expected_soc="950" ;;
|
||||
*)
|
||||
echo "Unsupported NPU device ${MATRIX_DEVICE}" >&2
|
||||
exit 1
|
||||
|
||||
@@ -613,14 +613,18 @@ You can also download the pre-built Docker images:
|
||||
# Docker Hub
|
||||
docker pull hiyouga/llamafactory:latest-910b-ubuntu
|
||||
docker pull hiyouga/llamafactory:latest-a3-ubuntu
|
||||
docker pull hiyouga/llamafactory:latest-950-ubuntu
|
||||
docker pull hiyouga/llamafactory:latest-910b-openeuler
|
||||
docker pull hiyouga/llamafactory:latest-a3-openeuler
|
||||
docker pull hiyouga/llamafactory:latest-950-openeuler
|
||||
|
||||
# quay.io
|
||||
docker pull quay.io/ascend/llamafactory:latest-910b-ubuntu
|
||||
docker pull quay.io/ascend/llamafactory:latest-a3-ubuntu
|
||||
docker pull quay.io/ascend/llamafactory:latest-950-ubuntu
|
||||
docker pull quay.io/ascend/llamafactory:latest-910b-openeuler
|
||||
docker pull quay.io/ascend/llamafactory:latest-a3-openeuler
|
||||
docker pull quay.io/ascend/llamafactory:latest-950-openeuler
|
||||
```
|
||||
|
||||
#### Install BitsAndBytes
|
||||
|
||||
@@ -613,14 +613,18 @@ pip install https://github.com/jllllll/bitsandbytes-windows-webui/releases/downl
|
||||
# Docker Hub
|
||||
docker pull hiyouga/llamafactory:latest-910b-ubuntu
|
||||
docker pull hiyouga/llamafactory:latest-a3-ubuntu
|
||||
docker pull hiyouga/llamafactory:latest-950-ubuntu
|
||||
docker pull hiyouga/llamafactory:latest-910b-openeuler
|
||||
docker pull hiyouga/llamafactory:latest-a3-openeuler
|
||||
docker pull hiyouga/llamafactory:latest-950-openeuler
|
||||
|
||||
# quay.io
|
||||
docker pull quay.io/ascend/llamafactory:latest-910b-ubuntu
|
||||
docker pull quay.io/ascend/llamafactory:latest-a3-ubuntu
|
||||
docker pull quay.io/ascend/llamafactory:latest-950-ubuntu
|
||||
docker pull quay.io/ascend/llamafactory:latest-910b-openeuler
|
||||
docker pull quay.io/ascend/llamafactory:latest-a3-openeuler
|
||||
docker pull quay.io/ascend/llamafactory:latest-950-openeuler
|
||||
```
|
||||
|
||||
#### 安装 BitsAndBytes
|
||||
|
||||
@@ -14,12 +14,14 @@ For installation and troubleshooting details, see the [English NPU installation
|
||||
|
||||
The following `latest` NPU image tags are available:
|
||||
|
||||
| Hardware series | Operating system | Tag |
|
||||
| Product series | Operating system | Tag |
|
||||
| --- | --- | --- |
|
||||
| A2 | Ubuntu 22.04 | `latest-910b-ubuntu` |
|
||||
| A3 | Ubuntu 22.04 | `latest-a3-ubuntu` |
|
||||
| A2 | openEuler 24.03 | `latest-910b-openeuler` |
|
||||
| A3 | openEuler 24.03 | `latest-a3-openeuler` |
|
||||
| Atlas A2 products | Ubuntu 22.04 | `latest-910b-ubuntu` |
|
||||
| Atlas A3 products | Ubuntu 22.04 | `latest-a3-ubuntu` |
|
||||
| Atlas A2 products | openEuler 24.03 | `latest-910b-openeuler` |
|
||||
| Atlas A3 products | openEuler 24.03 | `latest-a3-openeuler` |
|
||||
| Ascend 950PR&950DT products | Ubuntu 22.04 | `latest-950-ubuntu` |
|
||||
| Ascend 950PR&950DT products | openEuler 24.03 | `latest-950-openeuler` |
|
||||
|
||||
For historical NPU image tags and links to their corresponding archived Dockerfiles, see [Supported Tags](./supported_tags.md).
|
||||
|
||||
@@ -53,7 +55,7 @@ latest-<chip>-<os>
|
||||
|
||||
| Field | Values | Description |
|
||||
| --- | --- | --- |
|
||||
| `chip` | `910b` or `a3` | Ascend chip model supported by the image |
|
||||
| `chip` | `910b`, `a3`, or `950` | Ascend chip model supported by the image |
|
||||
| `os` | `ubuntu` or `openeuler` | Container operating system family |
|
||||
|
||||
Future NPU release images will use the full tag format below. Historical release images used earlier naming formats and do not follow this rule.
|
||||
@@ -67,7 +69,7 @@ Future NPU release images will use the full tag format below. Historical release
|
||||
| `LlamaFactory-version` | `0.9.5` | LlamaFactory release version |
|
||||
| `CANN-version` | `9.1.0` | Parsed from the CANN base image tag |
|
||||
| `TorchNPU-version` | `2.10.0.post2` | Full TorchNPU version used by the image, including suffixes such as `.postN` |
|
||||
| `chip` | `910b` or `a3` | Ascend chip model supported by the image |
|
||||
| `chip` | `910b`, `a3`, or `950` | Ascend chip model supported by the image |
|
||||
| `os` | `ubuntu22.04` or `openeuler24.03` | Container operating system family and version |
|
||||
| `Python-version` | `py3.12` | Parsed from the CANN base image tag |
|
||||
|
||||
@@ -148,7 +150,7 @@ Available build arguments:
|
||||
|
||||
### Start with Docker Compose
|
||||
|
||||
The preceding `docker build` command invokes the Dockerfile directly. It builds an image but does not start a container. Docker Compose does not use a separate build implementation: it reads the presets in `docker-compose.yml`, reuses the same Dockerfile, and selects a hardware-series and operating-system combination through a profile. Each `up -d` command below starts the selected container in the background. If the image is not available locally, Docker Compose builds it first:
|
||||
The preceding `docker build` command invokes the Dockerfile directly. It builds an image but does not start a container. Docker Compose does not use a separate build implementation: it reads the presets in `docker-compose.yml`, reuses the same Dockerfile, and selects a product-series and operating-system combination through a profile. Each `up -d` command below starts the selected container in the background. If the image is not available locally, Docker Compose builds it first:
|
||||
|
||||
```bash
|
||||
cd docker/docker-npu
|
||||
@@ -164,16 +166,22 @@ docker compose --profile a2-openeuler up -d
|
||||
|
||||
# A3 with openEuler
|
||||
docker compose --profile a3-openeuler up -d
|
||||
|
||||
# 950PR&950DT with Ubuntu
|
||||
docker compose --profile 950-ubuntu up -d
|
||||
|
||||
# 950PR&950DT with openEuler
|
||||
docker compose --profile 950-openeuler up -d
|
||||
```
|
||||
|
||||
To build an image with Docker Compose without starting a container, use `docker compose --profile <profile> build`.
|
||||
|
||||
## Hardware Support and Compatibility Notes
|
||||
|
||||
- A2 images use the `910b` CANN base image; A3 images use the `a3` CANN base image.
|
||||
- The image build targets both x86-64 (`linux/amd64`) and AArch64 (`linux/arm64`) hosts. The CPU architecture is independent of whether the hardware series is A2 or A3.
|
||||
- A2 images use the `910b` CANN base image; A3 images use the `a3` CANN base image; 950PR&950DT images use the `950` CANN base image.
|
||||
- The image build targets both x86-64 (`linux/amd64`) and AArch64 (`linux/arm64`) hosts. The CPU architecture is independent of whether the product series is A2, A3, or 950PR&950DT.
|
||||
- Ubuntu 22.04 and openEuler 24.03 refer to the operating system inside the container.
|
||||
- Legacy NPU tags are replaced by the `latest-<910b|a3>-<ubuntu|openeuler>` format.
|
||||
- Legacy NPU tags are replaced by the `latest-<910b|a3|950>-<ubuntu|openeuler>` format.
|
||||
- Validate the exact driver, firmware, CANN, and SoC combination before production deployment.
|
||||
|
||||
## License and Disclaimer
|
||||
|
||||
@@ -14,12 +14,14 @@ LlamaFactory 昇腾 NPU 镜像面向华为昇腾 Atlas NPU，提供可直接使
|
||||
|
||||
当前提供以下 `latest` NPU 镜像 tag：
|
||||
|
||||
| 硬件系列 | 操作系统 | Tag |
|
||||
| 产品系列 | 操作系统 | Tag |
|
||||
| --- | --- | --- |
|
||||
| A2 | Ubuntu 22.04 | `latest-910b-ubuntu` |
|
||||
| A3 | Ubuntu 22.04 | `latest-a3-ubuntu` |
|
||||
| A2 | openEuler 24.03 | `latest-910b-openeuler` |
|
||||
| A3 | openEuler 24.03 | `latest-a3-openeuler` |
|
||||
| Atlas A2 系列产品 | Ubuntu 22.04 | `latest-910b-ubuntu` |
|
||||
| Atlas A3 系列产品 | Ubuntu 22.04 | `latest-a3-ubuntu` |
|
||||
| Atlas A2 系列产品 | openEuler 24.03 | `latest-910b-openeuler` |
|
||||
| Atlas A3 系列产品 | openEuler 24.03 | `latest-a3-openeuler` |
|
||||
| Ascend 950PR&950DT 系列产品 | Ubuntu 22.04 | `latest-950-ubuntu` |
|
||||
| Ascend 950PR&950DT 系列产品 | openEuler 24.03 | `latest-950-openeuler` |
|
||||
|
||||
历史版本的 NPU 镜像 Tag 及其对应的 Dockerfile 归档链接，请参阅 [Supported Tags](./supported_tags.md)。
|
||||
|
||||
@@ -53,7 +55,7 @@ latest-<芯片信息>-<操作系统>
|
||||
|
||||
| 字段 | 可选值 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `芯片信息` | `910b` 或 `a3` | 镜像所适配的昇腾芯片型号 |
|
||||
| `芯片信息` | `910b`、`a3` 或 `950` | 镜像所适配的昇腾芯片型号 |
|
||||
| `操作系统` | `ubuntu` 或 `openeuler` | 容器操作系统类型 |
|
||||
|
||||
后续发布的 NPU release 镜像将使用以下完整 tag 格式。历史 release 镜像使用旧的命名格式，不适用以下规则。
|
||||
@@ -67,7 +69,7 @@ latest-<芯片信息>-<操作系统>
|
||||
| `LlamaFactory版本` | `0.9.5` | LlamaFactory release 版本号 |
|
||||
| `CANN版本` | `9.1.0` | 从 CANN 基础镜像 tag 中提取 |
|
||||
| `TorchNPU版本` | `2.10.0.post2` | 镜像使用的 TorchNPU 完整版本，包含 `.postN` 等后缀 |
|
||||
| `芯片信息` | `910b` 或 `a3` | 镜像所适配的昇腾芯片型号 |
|
||||
| `芯片信息` | `910b`、`a3` 或 `950` | 镜像所适配的昇腾芯片型号 |
|
||||
| `操作系统` | `ubuntu22.04` 或 `openeuler24.03` | 容器操作系统类型和版本 |
|
||||
| `Python版本` | `py3.12` | 从 CANN 基础镜像 tag 中提取 |
|
||||
|
||||
@@ -148,7 +150,7 @@ docker build \
|
||||
|
||||
### 通过 Docker Compose 启动
|
||||
|
||||
前面的 `docker build` 命令直接调用 Dockerfile，只构建镜像，不启动容器。Docker Compose 不使用另一套构建逻辑：它读取 `docker-compose.yml` 中的预设配置，复用同一个 Dockerfile，并通过 profile 选择硬件系列和操作系统组合。下面的 `up -d` 会在后台启动容器；若本地镜像不存在，Docker Compose 会先构建镜像：
|
||||
前面的 `docker build` 命令直接调用 Dockerfile，只构建镜像，不启动容器。Docker Compose 不使用另一套构建逻辑：它读取 `docker-compose.yml` 中的预设配置，复用同一个 Dockerfile，并通过 profile 选择产品系列和操作系统组合。下面的 `up -d` 会在后台启动容器；若本地镜像不存在，Docker Compose 会先构建镜像：
|
||||
|
||||
```bash
|
||||
cd docker/docker-npu
|
||||
@@ -164,16 +166,22 @@ docker compose --profile a2-openeuler up -d
|
||||
|
||||
# A3 + openEuler
|
||||
docker compose --profile a3-openeuler up -d
|
||||
|
||||
# 950PR&950DT + Ubuntu
|
||||
docker compose --profile 950-ubuntu up -d
|
||||
|
||||
# 950PR&950DT + openEuler
|
||||
docker compose --profile 950-openeuler up -d
|
||||
```
|
||||
|
||||
如果只想通过 Docker Compose 构建镜像而不启动容器，请使用 `docker compose --profile <profile> build`。
|
||||
|
||||
## 硬件支持与兼容性说明
|
||||
|
||||
- A2 镜像使用标记为 `910b` 的 CANN 基础镜像，A3 镜像使用标记为 `a3` 的 CANN 基础镜像。
|
||||
- 镜像构建目标同时包含 x86-64（`linux/amd64`）和 AArch64（`linux/arm64`）宿主机。CPU 架构与硬件系列是 A2 还是 A3 无关。
|
||||
- A2 镜像使用标记为 `910b` 的 CANN 基础镜像，A3 镜像使用标记为 `a3` 的 CANN 基础镜像，950PR&950DT 镜像使用标记为 `950` 的 CANN 基础镜像。
|
||||
- 镜像构建目标同时包含 x86-64（`linux/amd64`）和 AArch64（`linux/arm64`）宿主机。CPU 架构与产品系列是 A2、A3 还是 950PR&950DT 无关。
|
||||
- Ubuntu 22.04 和 openEuler 24.03 指容器内部的操作系统。
|
||||
- 旧式 NPU tag 已由 `latest-<910b|a3>-<ubuntu|openeuler>` 格式取代。
|
||||
- 旧式 NPU tag 已由 `latest-<910b|a3|950>-<ubuntu|openeuler>` 格式取代。
|
||||
- 正式部署前，请验证具体驱动、固件、CANN 和 SoC 组合的兼容性。
|
||||
|
||||
## 许可证与免责声明
|
||||
|
||||
@@ -23,6 +23,13 @@ x-npu-common: &npu-common
|
||||
- /dev/hisi_hdc
|
||||
restart: unless-stopped
|
||||
|
||||
x-npu-950: &npu-950
|
||||
<<: *npu-common
|
||||
devices:
|
||||
- /dev/davinci0
|
||||
- /dev/davinci_manager
|
||||
- /dev/hisi_hdc
|
||||
|
||||
services:
|
||||
llamafactory-a2-ubuntu:
|
||||
<<: *npu-common
|
||||
@@ -79,3 +86,31 @@ services:
|
||||
ports:
|
||||
- "7863:7860"
|
||||
- "8003:8000"
|
||||
|
||||
llamafactory-950-ubuntu:
|
||||
<<: *npu-950
|
||||
profiles: ["950-ubuntu"]
|
||||
build:
|
||||
<<: *build
|
||||
args:
|
||||
<<: *build-args
|
||||
BASE_IMAGE: quay.io/ascend/cann:9.1.0-950-ubuntu22.04-py3.12
|
||||
container_name: llamafactory-950-ubuntu
|
||||
image: llamafactory:npu-950-ubuntu
|
||||
ports:
|
||||
- "7864:7860"
|
||||
- "8004:8000"
|
||||
|
||||
llamafactory-950-openeuler:
|
||||
<<: *npu-950
|
||||
profiles: ["950-openeuler"]
|
||||
build:
|
||||
<<: *build
|
||||
args:
|
||||
<<: *build-args
|
||||
BASE_IMAGE: quay.io/ascend/cann:9.1.0-950-openeuler24.03-py3.12
|
||||
container_name: llamafactory-950-openeuler
|
||||
image: llamafactory:npu-950-openeuler
|
||||
ports:
|
||||
- "7865:7860"
|
||||
- "8005:8000"
|
||||
|
||||
88
docker/docker-xpu/Dockerfile
Normal file
88
docker/docker-xpu/Dockerfile
Normal file
@@ -0,0 +1,88 @@
|
||||
# https://hub.docker.com/r/intel/deep-learning-essentials/tags
|
||||
# Unlike the CUDA/ROCm bases this image ships no PyTorch, and its version must match the
|
||||
# host Intel GPU driver — see README.md.
|
||||
|
||||
ARG BASE_IMAGE=intel/deep-learning-essentials:2026.1.0-devel-ubuntu24.04
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
# Installation arguments
|
||||
ARG PIP_INDEX=https://pypi.org/simple
|
||||
ARG PYTORCH_INDEX=https://download.pytorch.org/whl/xpu
|
||||
# Must stay in the same compute-runtime series as the base image — see README.md ("ocloc").
|
||||
ARG OCLOC_VERSION=26.18.38308.1
|
||||
|
||||
# Define environments
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV PIP_ROOT_USER_ACTION=ignore
|
||||
# The base image's Python is distro-managed (PEP 668); nothing to protect in a container.
|
||||
ENV PIP_BREAK_SYSTEM_PACKAGES=1
|
||||
# expandable_segments trips a Level-Zero bug on XPU — see README.md ("Multi-GPU and OPTIM_TORCH").
|
||||
ENV OPTIM_TORCH=0
|
||||
|
||||
# Use Bash instead of default /bin/sh
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
|
||||
# Set the working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Install pip, Python dev headers and a C/C++ toolchain — see README.md ("Python.h: No such file or directory").
|
||||
RUN PY_MM="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')" && \
|
||||
apt-get update -y && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
"python${PY_MM}-dev" python3-pip build-essential && \
|
||||
apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install ocloc — pinned .deb rather than apt, see README.md ("ocloc and torch.compile").
|
||||
RUN wget -q "https://github.com/intel/compute-runtime/releases/download/${OCLOC_VERSION}/intel-ocloc_${OCLOC_VERSION}-0_amd64.deb" \
|
||||
-O /tmp/intel-ocloc.deb && \
|
||||
dpkg -i /tmp/intel-ocloc.deb && \
|
||||
rm -f /tmp/intel-ocloc.deb
|
||||
|
||||
# Change pip source — pip is deliberately not upgraded, see README.md ("How pip is installed").
|
||||
RUN pip config set global.index-url "${PIP_INDEX}" && \
|
||||
pip config set global.extra-index-url "${PIP_INDEX}" && \
|
||||
pip install --no-cache-dir packaging wheel setuptools editables "hatchling>=1.18.0"
|
||||
|
||||
# Copy the application into the image
|
||||
COPY . /app
|
||||
|
||||
# Install PyTorch for XPU — all three wheels pinned as a set, see README.md ("Pinned torch stack")
|
||||
RUN pip install --no-cache-dir -r requirements/xpu.txt --index-url "${PYTORCH_INDEX}"
|
||||
|
||||
# Install LLaMA Factory
|
||||
# metrics.txt is explicit because this project defines no extras, so `.[metrics]` would no-op.
|
||||
RUN pip install --no-cache-dir -e . --no-build-isolation && \
|
||||
pip install --no-cache-dir -r requirements/metrics.txt
|
||||
|
||||
# Optional accelerators — see README.md ("Optional accelerators").
|
||||
# py-cpuinfo must land first: deepspeed's own setup.py imports deepspeed/ops/adam/cpu_adam.py
|
||||
# during metadata generation (before pip installs deepspeed's declared deps), which needs it.
|
||||
# deepspeed is pinned separately from requirements/deepspeed.txt's <=0.18.4: that range's XPU
|
||||
# accelerator imports IPEX's DpcppBuildExtension unconditionally, which no longer exists on a
|
||||
# post-IPEX torch. Fixed upstream in 0.18.7 (plain torch.utils.cpp_extension.BuildExtension).
|
||||
RUN pip install --no-cache-dir py-cpuinfo && \
|
||||
(DS_BUILD_OPS=0 pip install --no-cache-dir "deepspeed==0.19.6" \
|
||||
|| echo "WARNING: deepspeed install failed - deepspeed training unavailable in this image")
|
||||
RUN pip install --no-cache-dir -r requirements/bitsandbytes.txt \
|
||||
|| echo "WARNING: bitsandbytes install failed - 4-bit quantization (QLoRA) unavailable in this image"
|
||||
|
||||
# Source the oneAPI environment in every interactive shell
|
||||
RUN echo "source /opt/intel/oneapi/setvars.sh --force" >> /root/.bashrc
|
||||
|
||||
# Set up volumes
|
||||
# VOLUME [ "/root/.cache/huggingface", "/app/shared_data", "/app/output" ]
|
||||
|
||||
# Expose port 7860 for LLaMA Board
|
||||
ENV GRADIO_SERVER_PORT=7860
|
||||
EXPOSE 7860
|
||||
|
||||
# Expose port 8000 for API service
|
||||
ENV API_PORT=8000
|
||||
EXPOSE 8000
|
||||
|
||||
# Reset pip config
|
||||
RUN pip config unset global.index-url && \
|
||||
pip config unset global.extra-index-url
|
||||
|
||||
# oneAPI must be on the loader path before torch can open its XPU backend
|
||||
CMD ["bash", "-c", "source /opt/intel/oneapi/setvars.sh --force && exec llamafactory-cli webui"]
|
||||
206
docker/docker-xpu/README.md
Normal file
206
docker/docker-xpu/README.md
Normal file
@@ -0,0 +1,206 @@
|
||||
# Docker Setup for Intel GPUs
|
||||
|
||||
This directory contains Docker configuration files for running LLaMA Factory with Intel GPU (XPU) support.
|
||||
|
||||
## Image Details
|
||||
|
||||
| Component | Version |
|
||||
|---|---|
|
||||
| Base OS | Ubuntu 24.04 LTS (x86_64) |
|
||||
| Intel DLE base | [intel/deep-learning-essentials:2026.1.0-devel-ubuntu24.04](https://hub.docker.com/r/intel/deep-learning-essentials) |
|
||||
| Python | 3.12 |
|
||||
| PyTorch | 2.13.0+xpu |
|
||||
| Intel GPU runtime | Bundled inside DLE (libze-intel-gpu 26.18.x, oneAPI 2026.1) |
|
||||
|
||||
The Intel compute runtime is bundled inside the DLE base image — no GPU compute packages are needed on the host, only the kernel driver. The bundled runtime must be compatible with the host kernel driver; verify with `dpkg -l libze-intel-gpu1 | grep -oP '\d+\.\d+\.\d+'`.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
### 1. Docker & Docker Compose
|
||||
|
||||
```bash
|
||||
# Ubuntu/Debian
|
||||
sudo apt-get update && sudo apt-get install docker.io docker-compose-v2
|
||||
```
|
||||
|
||||
See the [official Docker install docs](https://docs.docker.com/engine/install/) for other distros or newer versions.
|
||||
|
||||
### 2. Intel GPU Kernel Driver (host only)
|
||||
|
||||
```bash
|
||||
# Add the Intel GPU PPA
|
||||
sudo apt-get install -y gpg-agent wget
|
||||
wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | \
|
||||
sudo gpg --dearmor -o /usr/share/keyrings/intel-graphics.gpg
|
||||
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] \
|
||||
https://repositories.intel.com/gpu/ubuntu noble client" | \
|
||||
sudo tee /etc/apt/sources.list.d/intel-graphics.list
|
||||
sudo apt-get update
|
||||
|
||||
# Kernel driver only — no compute runtime packages needed on the host
|
||||
sudo apt-get install -y intel-i915-dkms intel-fw-gpu
|
||||
sudo reboot
|
||||
```
|
||||
|
||||
After reboot, verify `/dev/dri` is populated: `ls /dev/dri/` (expect `card0`, `renderD128`, etc).
|
||||
|
||||
See the [Intel GPU Installation Guide](https://dgpu-docs.intel.com/installation-guides/installing-packages-from-the-intel-ppa.html) for details.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> Enable **Resizable BAR** in your system BIOS before proceeding, or you may see `Bus error (core dumped)` or degraded performance. See [Intel's guide](https://www.intel.com/content/www/us/en/support/articles/000090831/graphics.html).
|
||||
|
||||
### 3. Add your user to the GPU groups
|
||||
|
||||
```bash
|
||||
sudo usermod -aG render,video $USER
|
||||
# Log out and back in for group membership to take effect
|
||||
```
|
||||
|
||||
(Optional) sanity-check the host side with `sudo apt-get install -y clinfo && clinfo --list | grep Device`. The check that actually matters — `torch.xpu.device_count()` — runs inside the container, in Usage below.
|
||||
|
||||
## Usage
|
||||
|
||||
### Using Docker Compose (Recommended)
|
||||
|
||||
```bash
|
||||
cd docker/docker-xpu/
|
||||
docker compose up -d
|
||||
docker compose exec llamafactory bash
|
||||
```
|
||||
|
||||
Verify GPU access inside the container:
|
||||
|
||||
```bash
|
||||
python3 -c "import torch; print(torch.xpu.device_count(), 'XPU device(s) found')"
|
||||
```
|
||||
|
||||
### Using Docker Run
|
||||
|
||||
```bash
|
||||
# Build the image (from the repo root)
|
||||
docker build -t llamafactory:xpu -f docker/docker-xpu/Dockerfile .
|
||||
|
||||
# Run the container
|
||||
docker run -it --rm \
|
||||
--device /dev/dri \
|
||||
-v /dev/dri/by-path:/dev/dri/by-path \
|
||||
--group-add $(getent group render | cut -d: -f3) \
|
||||
--group-add $(getent group video | cut -d: -f3) \
|
||||
--ipc=host \
|
||||
-p 7860:7860 \
|
||||
-p 8000:8000 \
|
||||
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
||||
--name llamafactory \
|
||||
llamafactory:xpu bash
|
||||
```
|
||||
|
||||
## Build arguments
|
||||
|
||||
| Argument | Default | Purpose |
|
||||
|---|---|---|
|
||||
| `BASE_IMAGE` | `intel/deep-learning-essentials:2026.1.0-devel-ubuntu24.04` | oneAPI / GPU runtime version |
|
||||
| `PIP_INDEX` | `https://pypi.org/simple` | PyPI mirror for everything except the torch wheels |
|
||||
| `PYTORCH_INDEX` | `https://download.pytorch.org/whl/xpu` | where the `+xpu` torch wheels come from |
|
||||
| `OCLOC_VERSION` | `26.18.38308.1` | pinned `intel-ocloc` build — must match `BASE_IMAGE`'s compute-runtime series |
|
||||
|
||||
```bash
|
||||
docker build -t llamafactory:xpu -f docker/docker-xpu/Dockerfile . \
|
||||
--build-arg PIP_INDEX=https://pypi.org/simple
|
||||
```
|
||||
|
||||
## Design notes
|
||||
|
||||
Why the image is built the way it is — skip to [Troubleshooting](#troubleshooting) if you just want to run it.
|
||||
|
||||
### Pinned torch stack
|
||||
|
||||
`torch`, `torchvision` and `torchaudio` install together from `requirements/xpu.txt` as `+xpu` builds, via `--index-url https://download.pytorch.org/whl/xpu` (not `--extra-index-url`, which would let a plain PyPI build win). Pinning all three avoids `RuntimeError: operator torchvision::nms does not exist` from an ABI-mismatched pair.
|
||||
|
||||
No pip constraint file is needed: every other `torch` requirement in the dependency graph is a lower bound that `2.13.0+xpu` already satisfies, so nothing later swaps it out.
|
||||
|
||||
If the XPU wheels ever get replaced with plain ones, `torch.xpu.device_count()` returns `0`. Restore with:
|
||||
|
||||
```bash
|
||||
pip install --force-reinstall -r requirements/xpu.txt \
|
||||
--index-url https://download.pytorch.org/whl/xpu
|
||||
```
|
||||
|
||||
### How pip is installed
|
||||
|
||||
The DLE base ships Python 3.12 with no pip and no `ensurepip` (Ubuntu strips it). pip comes from apt (`python3-pip`) and is **left at the distro version** rather than upgraded — apt's pip has no `RECORD` file, so `pip install --upgrade pip` fails with:
|
||||
|
||||
```
|
||||
ERROR: Cannot uninstall pip 24.0, RECORD file not found. Hint: The package was installed by debian.
|
||||
```
|
||||
|
||||
— taking `setuptools`/`wheel`/`hatchling` down with it in the same command.
|
||||
|
||||
### Multi-GPU and `OPTIM_TORCH`
|
||||
|
||||
The image sets `ENV OPTIM_TORCH=0`. The launcher's default (`OPTIM_TORCH=1`) sets `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True`, whose allocator path trips a Level-Zero driver bug on 2-GPU XPU runs (DDP segfault, FSDP2 hang). Disabling it costs <1% throughput. Re-enable per-run with `OPTIM_TORCH=1 llamafactory-cli train ...` to test it. It is tracked in intel, will be update or remove this ENV when it is resolve with PyTorch or level zero etc fixes.
|
||||
|
||||
### Optional accelerators
|
||||
|
||||
`deepspeed` and `bitsandbytes` install on a best-effort basis — neither is required to train on XPU, so a build failure only prints a warning instead of failing the image:
|
||||
|
||||
```
|
||||
WARNING: deepspeed install failed - deepspeed training unavailable in this image
|
||||
WARNING: bitsandbytes install failed - 4-bit quantization (QLoRA) unavailable in this image
|
||||
```
|
||||
|
||||
Both install and work on XPU today (verified: `torch.compile`, LoRA + DeepSpeed ZeRO-2 SFT, and bitsandbytes 4-bit all pass on real Arc Pro B60 hardware). `deepspeed` is pinned to `==0.19.6` here, separately from `requirements/deepspeed.txt`'s shared `<=0.18.4` cap: any version in that range detects XPU as the active accelerator and imports `torch.utils.cpp_extension.DpcppBuildExtension` — a class that used to be provided by `intel_extension_for_pytorch` (IPEX). IPEX is no longer part of this stack, so that import fails with `ImportError: cannot import name 'DpcppBuildExtension'`. deepspeed dropped the IPEX dependency and switched to the stock `BuildExtension` in `0.18.7`, so anything from there onward works; `0.19.6` is the latest release at time of writing.
|
||||
|
||||
Check the build log, or verify inside the container with `python3 -c "import deepspeed"` / `import bitsandbytes`. Install manually if missing:
|
||||
|
||||
```bash
|
||||
DS_BUILD_OPS=0 pip install "deepspeed==0.19.6"
|
||||
pip install -r requirements/bitsandbytes.txt
|
||||
```
|
||||
|
||||
### `ocloc` and `torch.compile`
|
||||
|
||||
`torch.compile` lowers to Intel Triton, which shells out to `ocloc` (the Intel offline GPU compiler) — not bundled in the DLE base, so compiled paths fail with `FileNotFoundError: 'ocloc'` without it. The Dockerfile installs a single pinned `.deb` from Intel's GitHub releases (`ARG OCLOC_VERSION`) rather than `apt-get install intel-ocloc`, because the Intel graphics PPA keeps only its newest revision and would drag `libze-intel-gpu1` (the host-driver-facing package) up with it.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> If you override `ARG BASE_IMAGE`, move `OCLOC_VERSION` to match — check `docker run --rm <base-image> dpkg -l libze-intel-gpu1 | tail -1` and pick the nearest [compute-runtime release](https://github.com/intel/compute-runtime/releases).
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### GPU Not Detected (`torch.xpu.device_count()` returns 0)
|
||||
|
||||
1. **Kernel driver too old for the DLE runtime** — compare `dpkg -l libze-intel-gpu1` on the host vs. `docker run --rm llamafactory:xpu dpkg -l libze-intel-gpu1`. Update the host driver (`sudo apt-get install -y intel-i915-dkms intel-fw-gpu && sudo reboot`) or use an older `BASE_IMAGE`.
|
||||
2. **Missing `/dev/dri` device** — pass `--device /dev/dri` (done automatically by `docker compose`).
|
||||
3. **Missing group membership** — the container process needs the `render` and `video` groups; `docker compose` sets these via `group_add`. For manual `docker run`, pass `--group-add $(getent group render | cut -d: -f3) --group-add $(getent group video | cut -d: -f3)`.
|
||||
4. **`by-path` mount missing** — required for multi-GPU Level-Zero IPC: `-v /dev/dri/by-path:/dev/dri/by-path` (included in `docker-compose.yml`).
|
||||
|
||||
### `fatal error: Python.h: No such file or directory`
|
||||
|
||||
Intel Triton JIT-compiles a C driver at the first XPU kernel launch, which needs matching `pythonX.Y-dev` headers and a C compiler present in the image — installed at build time even though the error would happen at runtime. If you've swapped the interpreter, reinstall matching headers:
|
||||
|
||||
```bash
|
||||
PY_MM="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
|
||||
apt-get update && apt-get install -y "python${PY_MM}-dev" build-essential
|
||||
```
|
||||
|
||||
### Permission Denied on `/dev/dri`
|
||||
|
||||
```bash
|
||||
sudo usermod -aG render,video $USER
|
||||
newgrp render # apply without logout
|
||||
```
|
||||
|
||||
### `SYCL Backends mismatch` / `libsycl.so.N: cannot open shared object file`
|
||||
|
||||
Two SYCL runtimes exist in this image by design — torch's own `intel-sycl-rt` (pip) and the DLE base's oneAPI — and normally share the same `libsycl.so.9` SONAME, so both coexist fine. A mismatch shows up only after changing `torch` or `BASE_IMAGE` independently. Compare versions:
|
||||
|
||||
```bash
|
||||
pip list | grep -E "intel-sycl-rt|dpcpp-cpp-rt"
|
||||
ls /opt/intel/oneapi/*/lib/libsycl.so.*
|
||||
```
|
||||
|
||||
The `libsycl.so.N` numbers on both sides must match. `ldconfig -p | grep libsycl` returning nothing is expected — neither runtime is in the ldconfig cache.
|
||||
|
||||
## Additional Notes
|
||||
|
||||
- The container automatically sources `/opt/intel/oneapi/setvars.sh` in every interactive shell (`~/.bashrc`). For non-interactive scripts, source it explicitly.
|
||||
- For training, `llamafactory-cli train` dispatches automatically via `torchrun` for multi-GPU.
|
||||
48
docker/docker-xpu/docker-compose.yml
Normal file
48
docker/docker-xpu/docker-compose.yml
Normal file
@@ -0,0 +1,48 @@
|
||||
services:
|
||||
llamafactory:
|
||||
build:
|
||||
dockerfile: ./docker/docker-xpu/Dockerfile
|
||||
context: ../..
|
||||
args:
|
||||
PIP_INDEX: https://pypi.org/simple
|
||||
# The DLE base image determines which Intel GPU runtime (libze-intel-gpu)
|
||||
# is shipped. The runtime version must be >= the host driver version,
|
||||
# otherwise torch.xpu.device_count() returns 0:
|
||||
# DLE 2025.3 → libze-intel-gpu 25.18.x (works on driver ≤26.09)
|
||||
# DLE 2026.1 → libze-intel-gpu 26.18.x+ (works on driver 26.18/26.22)
|
||||
# Verify host driver: dpkg -l libze-intel-gpu1 | grep -oP '\d+\.\d+\.\d+'
|
||||
BASE_IMAGE: intel/deep-learning-essentials:2026.1.0-devel-ubuntu24.04
|
||||
container_name: llamafactory
|
||||
image: llamafactory:xpu
|
||||
ports:
|
||||
- "7860:7860"
|
||||
- "8000:8000"
|
||||
ipc: host
|
||||
tty: true
|
||||
# shm_size: "16gb" # ipc: host is set
|
||||
stdin_open: true
|
||||
command: bash
|
||||
devices:
|
||||
# Intel GPU character devices (renderD* and card* under /dev/dri).
|
||||
# renderD* nodes are owned by group `render`; card* by group `video`.
|
||||
- /dev/dri:/dev/dri
|
||||
volumes:
|
||||
# by-path symlinks are required for oneCCL's ze_fd_manager (2-GPU IPC).
|
||||
# `devices:` alone does not carry sub-directories — this volume does.
|
||||
- /dev/dri/by-path:/dev/dri/by-path
|
||||
- ~/.cache/huggingface:/root/.cache/huggingface
|
||||
group_add:
|
||||
# Grant access to the Intel GPU device nodes.
|
||||
# renderD* nodes are owned by group `render`; card* nodes by `video`.
|
||||
# Use numeric GIDs here — Docker compose resolves group names against
|
||||
# the CONTAINER's /etc/group (not the host), and the DLE base image
|
||||
# does not carry a `render` group entry. Standard Linux GIDs:
|
||||
# If your host uses different GIDs, run `getent group render video` and
|
||||
# update these values accordingly.
|
||||
# example output.
|
||||
# getent group render video
|
||||
# render:x:992:root
|
||||
# video:x:44:support
|
||||
- "992" # render — owns /dev/dri/renderD* nodes
|
||||
- "44" # video — owns /dev/dri/card* nodes
|
||||
restart: unless-stopped
|
||||
3
requirements/xpu.txt
Normal file
3
requirements/xpu.txt
Normal file
@@ -0,0 +1,3 @@
|
||||
torch==2.13.0+xpu
|
||||
torchvision==0.28.0+xpu
|
||||
torchaudio==2.11.0+xpu
|
||||
@@ -19,7 +19,7 @@ import inspect
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
from copy import deepcopy
|
||||
from copy import copy, deepcopy
|
||||
from dataclasses import dataclass
|
||||
from io import BytesIO
|
||||
from types import SimpleNamespace
|
||||
@@ -56,6 +56,7 @@ if TYPE_CHECKING:
|
||||
from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor
|
||||
from transformers.image_processing_utils import BaseImageProcessor
|
||||
from transformers.video_processing_utils import BaseVideoProcessor
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
class EncodedImage(TypedDict):
|
||||
path: str | None
|
||||
@@ -2776,6 +2777,213 @@ class Qwen3VLPlugin(Qwen2VLPlugin):
|
||||
return messages
|
||||
|
||||
|
||||
@dataclass
|
||||
class Glm5NextPlugin(BasePlugin):
|
||||
r"""GLM5-Next images and timestamped video frames using the native processors."""
|
||||
|
||||
@override
|
||||
def _validate_input(
|
||||
self,
|
||||
processor: Optional["MMProcessor"],
|
||||
images: list["ImageInput"],
|
||||
videos: list["VideoInput"],
|
||||
audios: list["AudioInput"],
|
||||
) -> None:
|
||||
if processor is None and not (images or videos or audios):
|
||||
return # Preserve tokenizer-only text preprocessing.
|
||||
super()._validate_input(processor, images, videos, audios)
|
||||
|
||||
def _decode_video(
|
||||
self, video: "VideoInput", processor: "MMProcessor"
|
||||
) -> tuple[list["ImageObject"], "VideoMetadata"]:
|
||||
r"""Decode sampled frames and preserve their source timing for the native processor."""
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
fps = getattr(processor, "video_fps", 2.0)
|
||||
maxlen = getattr(processor, "video_maxlen", 128)
|
||||
temporal = processor.video_processor.temporal_patch_size
|
||||
if not math.isfinite(fps) or fps <= 0 or maxlen < temporal:
|
||||
raise ValueError("glm5_next requires video_fps > 0 and video_maxlen >= temporal_patch_size.")
|
||||
# Native sampling can duplicate the last frame for temporal patching.
|
||||
# Use a private sampler so frame limits never mutate the shared processor.
|
||||
sampler = copy(processor.video_processor)
|
||||
sampler.max_frames = min(sampler.max_frames, maxlen // temporal * temporal)
|
||||
if _check_video_is_nested_images(video):
|
||||
if not video:
|
||||
raise ValueError("glm5_next received an empty video frame list.")
|
||||
total = len(video)
|
||||
indices = np.linspace(0, total - 1, min(total, sampler.max_frames), dtype=int).tolist()
|
||||
frames = self._regularize_images(
|
||||
[video[i] for i in indices], image_max_pixels=float("inf"), image_min_pixels=0
|
||||
)["images"]
|
||||
# Frame lists have no source timing; video_fps describes their spacing.
|
||||
metadata = VideoMetadata(total_num_frames=total, fps=fps, frames_indices=indices)
|
||||
else:
|
||||
with av.open(video, "r") as container:
|
||||
stream = next((s for s in container.streams if s.type == "video"), None)
|
||||
if stream is None or not stream.average_rate or stream.average_rate <= 0:
|
||||
raise ValueError("glm5_next requires a video stream with a valid source FPS.")
|
||||
source_fps = float(stream.average_rate)
|
||||
duration = float(stream.duration * stream.time_base) if stream.duration is not None else None
|
||||
metadata = VideoMetadata(total_num_frames=stream.frames, fps=source_fps, duration=duration)
|
||||
# Some containers omit frame counts. Count without retaining all
|
||||
# decoded pixels, then seek back and retain only selected frames.
|
||||
if not metadata.total_num_frames:
|
||||
metadata.total_num_frames = sum(1 for _ in container.decode(stream))
|
||||
container.seek(0)
|
||||
if metadata.total_num_frames <= 0:
|
||||
raise ValueError("glm5_next received a video with no decodable frames.")
|
||||
indices = sampler.sample_frames(metadata, fps=fps).tolist()
|
||||
if not indices: # Sub-second clips can round to zero in the native sampler.
|
||||
indices = [0]
|
||||
indices = indices[: sampler.max_frames]
|
||||
selected = set(indices)
|
||||
decoded = {}
|
||||
for i, frame in enumerate(container.decode(stream)):
|
||||
if i in selected:
|
||||
decoded[i] = frame.to_image()
|
||||
if i >= max(selected):
|
||||
break
|
||||
if selected - decoded.keys():
|
||||
raise ValueError("glm5_next could not decode all sampled video frames.")
|
||||
frames = [decoded[i] for i in indices]
|
||||
metadata.frames_indices = indices
|
||||
# Keep both pixels and timestamps aligned when repeating an odd last frame.
|
||||
while len(frames) % temporal:
|
||||
frames.append(frames[-1])
|
||||
metadata.frames_indices.append(metadata.frames_indices[-1])
|
||||
return frames, metadata
|
||||
|
||||
@override
|
||||
def _get_mm_inputs(
|
||||
self,
|
||||
images: list["ImageInput"],
|
||||
videos: list["VideoInput"],
|
||||
audios: list["AudioInput"],
|
||||
processor: "MMProcessor",
|
||||
) -> dict[str, Any]:
|
||||
self._validate_input(processor, images, videos, audios)
|
||||
mm_inputs = {}
|
||||
if images:
|
||||
# Only decode/convert here: resizing twice changes the native pixels.
|
||||
images = self._regularize_images(images, image_max_pixels=float("inf"), image_min_pixels=0)["images"]
|
||||
image_processor = processor.image_processor
|
||||
pixels_per_token = (image_processor.patch_size * image_processor.merge_size) ** 2
|
||||
kwargs = {}
|
||||
if hasattr(processor, "image_max_pixels"):
|
||||
kwargs["max_image_tokens"] = max(1, processor.image_max_pixels // pixels_per_token)
|
||||
if hasattr(processor, "image_min_pixels"):
|
||||
kwargs["min_image_tokens"] = max(1, math.ceil(processor.image_min_pixels / pixels_per_token))
|
||||
mm_inputs.update(image_processor(images=images, return_tensors="pt", **kwargs))
|
||||
if videos:
|
||||
video_processor = processor.video_processor
|
||||
pixels_per_token = (video_processor.patch_size * video_processor.merge_size) ** 2
|
||||
processed = []
|
||||
for video in videos:
|
||||
frames, metadata = self._decode_video(video, processor)
|
||||
temporal_tokens = len(frames) // video_processor.temporal_patch_size
|
||||
# LlamaFactory limits pixels per frame; native GLM5 budgets tokens
|
||||
# across the whole video (after temporal merging).
|
||||
kwargs = {
|
||||
"min_image_tokens": max(
|
||||
1, math.ceil(getattr(processor, "video_min_pixels", 256) / pixels_per_token)
|
||||
)
|
||||
* temporal_tokens,
|
||||
"max_image_tokens": max(1, getattr(processor, "video_max_pixels", 65536) // pixels_per_token)
|
||||
* temporal_tokens,
|
||||
}
|
||||
processed.append(
|
||||
video_processor(
|
||||
videos=[np.stack([np.asarray(frame) for frame in frames])],
|
||||
video_metadata=[metadata],
|
||||
do_sample_frames=False,
|
||||
return_metadata=True,
|
||||
return_tensors="pt",
|
||||
**kwargs,
|
||||
)
|
||||
)
|
||||
for key in ("pixel_values_videos", "video_grid_thw"):
|
||||
mm_inputs[key] = torch.cat([item[key] for item in processed])
|
||||
mm_inputs["video_metadata"] = [item["video_metadata"][0] for item in processed]
|
||||
return mm_inputs
|
||||
|
||||
@override
|
||||
def process_messages(
|
||||
self,
|
||||
messages: list[dict[str, str]],
|
||||
images: list["ImageInput"],
|
||||
videos: list["VideoInput"],
|
||||
audios: list["AudioInput"],
|
||||
processor: Optional["MMProcessor"],
|
||||
) -> list[dict[str, str]]:
|
||||
self._validate_input(processor, images, videos, audios)
|
||||
self._validate_messages(messages, images, videos, audios)
|
||||
messages = deepcopy(messages)
|
||||
mm_inputs = self._get_mm_inputs(images, videos, audios, processor) if self.expand_mm_tokens else {}
|
||||
image_idx, video_idx = 0, 0
|
||||
for message in messages:
|
||||
if message["role"] != "user" and any(
|
||||
p in message["content"] for p in (IMAGE_PLACEHOLDER, VIDEO_PLACEHOLDER)
|
||||
):
|
||||
raise ValueError("glm5_next supports images and videos in user messages only.")
|
||||
while IMAGE_PLACEHOLDER in message["content"]:
|
||||
tokens = (
|
||||
processor.replace_image_token(mm_inputs, image_idx) if self.expand_mm_tokens else self.image_token
|
||||
)
|
||||
message["content"] = message["content"].replace(
|
||||
IMAGE_PLACEHOLDER, f"<|begin_of_image|>{tokens}<|end_of_image|>", 1
|
||||
)
|
||||
image_idx += 1
|
||||
while VIDEO_PLACEHOLDER in message["content"]:
|
||||
tokens = (
|
||||
processor.replace_video_token(mm_inputs, video_idx) if self.expand_mm_tokens else self.video_token
|
||||
)
|
||||
message["content"] = message["content"].replace(
|
||||
VIDEO_PLACEHOLDER, f"<|begin_of_video|>{tokens}<|end_of_video|>", 1
|
||||
)
|
||||
video_idx += 1
|
||||
return messages
|
||||
|
||||
@override
|
||||
def get_mm_inputs(
|
||||
self,
|
||||
images: list["ImageInput"],
|
||||
videos: list["VideoInput"],
|
||||
audios: list["AudioInput"],
|
||||
imglens: list[int],
|
||||
vidlens: list[int],
|
||||
audlens: list[int],
|
||||
batch_ids: list[list[int]],
|
||||
processor: Optional["MMProcessor"],
|
||||
) -> dict[str, Union[list[list[int]], "torch.Tensor"]]:
|
||||
mm_inputs = self._get_mm_inputs(images, videos, audios, processor)
|
||||
token_types = processor.create_mm_token_type_ids(batch_ids)
|
||||
for count, ids in zip(vidlens, batch_ids):
|
||||
boundaries = [token for token in ids if token in (processor.video_start_id, processor.video_end_id)]
|
||||
if boundaries != [processor.video_start_id, processor.video_end_id] * count:
|
||||
raise ValueError("glm5_next video boundaries do not match inputs; check cutoff_len.")
|
||||
# Images and video frames share image_token_id: validate by modality,
|
||||
# otherwise a truncated image could be hidden by extra video tokens.
|
||||
for modality, lens, key, subprocessor in (
|
||||
(1, imglens, "image_grid_thw", processor.image_processor),
|
||||
(2, vidlens, "video_grid_thw", processor.video_processor),
|
||||
):
|
||||
grids = mm_inputs.get(key, [])
|
||||
offset = 0
|
||||
for count, types in zip(lens, token_types):
|
||||
expected = sum(
|
||||
int(grid.prod()) // subprocessor.merge_size**2 for grid in grids[offset : offset + count]
|
||||
)
|
||||
if types.count(modality) != expected:
|
||||
raise ValueError(
|
||||
"glm5_next image/video tokens do not match features; check cutoff_len and media limits."
|
||||
)
|
||||
offset += count
|
||||
mm_inputs.pop("video_metadata", None) # Used for prompt timestamps, never forwarded to model.forward.
|
||||
mm_inputs["mm_token_type_ids"] = token_types
|
||||
return mm_inputs
|
||||
|
||||
|
||||
@dataclass
|
||||
class GLM4VPlugin(Qwen2VLPlugin):
|
||||
@override
|
||||
@@ -3249,6 +3457,7 @@ PLUGINS = {
|
||||
"gemma3n": Gemma3nPlugin,
|
||||
"gemma4": Gemma4Plugin,
|
||||
"glm4v": GLM4VPlugin,
|
||||
"glm5_next": Glm5NextPlugin,
|
||||
"intern_vl": InternVLPlugin,
|
||||
"kimi_vl": KimiVLPlugin,
|
||||
"llama4": Llama4Plugin,
|
||||
|
||||
@@ -339,6 +339,73 @@ class Template:
|
||||
return modelfile
|
||||
|
||||
|
||||
@dataclass
|
||||
class Glm5NextTemplate(Template):
|
||||
r"""GLM-5.3-Flash template with preserved thinking and EOS only on the final assistant response."""
|
||||
|
||||
@override
|
||||
def _encode(
|
||||
self,
|
||||
tokenizer: "PreTrainedTokenizer",
|
||||
messages: list[dict[str, str]],
|
||||
system: Optional[str],
|
||||
tools: Optional[str],
|
||||
) -> list[list[int]]:
|
||||
if not messages or len(messages) % 2 or messages[0]["role"] != Role.USER:
|
||||
raise ValueError(
|
||||
"glm5_next expects alternating user/observation and assistant/function pairs, starting with user."
|
||||
)
|
||||
|
||||
system = system or self.default_system
|
||||
encoded = []
|
||||
for i, message in enumerate(messages):
|
||||
role, content = message["role"], message["content"]
|
||||
expected = (Role.USER, Role.OBSERVATION) if i % 2 == 0 else (Role.ASSISTANT, Role.FUNCTION)
|
||||
if role not in expected or not isinstance(content, str):
|
||||
raise ValueError(
|
||||
"glm5_next expects alternating user/observation and assistant/function text messages."
|
||||
)
|
||||
if message.get("tool_calls"):
|
||||
raise ValueError("glm5_next expects tool calls as JSON in role=function content (v0 ShareGPT format).")
|
||||
if i % 2 == 0 and i > 0:
|
||||
after_function = messages[i - 1]["role"] == Role.FUNCTION
|
||||
if (role == Role.OBSERVATION) != after_function:
|
||||
raise ValueError("glm5_next requires an observation after each function turn.")
|
||||
|
||||
elements = []
|
||||
if i == 0:
|
||||
elements += self.format_prefix.apply()
|
||||
if tools:
|
||||
elements += self.format_tools.apply(content=tools)
|
||||
if system:
|
||||
elements += self.format_system.apply(content=system)
|
||||
|
||||
if role == Role.USER:
|
||||
elements += self.format_user.apply(content=content)
|
||||
elif role == Role.OBSERVATION:
|
||||
elements += self.format_observation.apply(content=content)
|
||||
else:
|
||||
if role == Role.FUNCTION:
|
||||
content = self.format_function.apply(
|
||||
content=content,
|
||||
thought_words=self.thought_words,
|
||||
tool_call_words=self.tool_call_words,
|
||||
)[0]
|
||||
if "</think>" in content:
|
||||
reasoning = content.split("</think>")[0].split("<think>")[-1]
|
||||
content = content.split("</think>")[-1]
|
||||
else:
|
||||
reasoning = ""
|
||||
elements += self.format_assistant.apply(content=reasoning + "</think>" + content.strip())
|
||||
if role == Role.FUNCTION:
|
||||
# Supervise the observation marker so the model learns to hand control to tools.
|
||||
elements += ["<|observation|>"]
|
||||
elif i == len(messages) - 1:
|
||||
elements += [{"eos_token"}]
|
||||
encoded.append(self._convert_elements_to_ids(tokenizer, elements))
|
||||
return encoded
|
||||
|
||||
|
||||
@dataclass
|
||||
class MossVLTemplate(Template):
|
||||
@override
|
||||
@@ -1211,6 +1278,23 @@ register_template(
|
||||
)
|
||||
|
||||
|
||||
register_template(
|
||||
name="glm5_next",
|
||||
format_user=StringFormatter(slots=["<|user|>{{content}}<|assistant|><think>"]),
|
||||
format_assistant=StringFormatter(slots=["{{content}}"]),
|
||||
format_system=StringFormatter(slots=["<|system|>{{content}}"]),
|
||||
format_function=FunctionFormatter(slots=["{{content}}"], tool_format="glm5_next"),
|
||||
format_observation=StringFormatter(slots=["<tool_response>{{content}}</tool_response><|assistant|><think>"]),
|
||||
format_tools=ToolFormatter(tool_format="glm5_next"),
|
||||
format_prefix=EmptyFormatter(slots=["[gMASK]<sop><|system|>Reasoning Effort: Max"]),
|
||||
stop_words=["<|user|>", "<|observation|>"],
|
||||
thought_words=("<think>", "</think>"),
|
||||
preserve_thinking=True,
|
||||
mm_plugin=get_mm_plugin(name="glm5_next", image_token="<|image|>", video_token="<|video|>"),
|
||||
template_class=Glm5NextTemplate,
|
||||
)
|
||||
|
||||
|
||||
register_template(
|
||||
name="glm4",
|
||||
format_user=StringFormatter(slots=["<|user|>\n{{content}}<|assistant|>"]),
|
||||
|
||||
@@ -55,6 +55,14 @@ GLM4_MOE_TOOL_PROMPT = (
|
||||
"\n...\n</tool_call>\n"
|
||||
)
|
||||
|
||||
GLM5_NEXT_TOOL_PROMPT = (
|
||||
"<|system|>\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\n"
|
||||
"You are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{tool_text}"
|
||||
"</tools>\n\nFor each function call, output the function name and arguments within the following XML format:\n"
|
||||
"<tool_call>{{function-name}}<arg_key>{{arg-key-1}}</arg_key><arg_value>{{arg-value-1}}</arg_value>"
|
||||
"<arg_key>{{arg-key-2}}</arg_key><arg_value>{{arg-value-2}}</arg_value>...</tool_call>"
|
||||
)
|
||||
|
||||
LLAMA3_TOOL_PROMPT = (
|
||||
"Cutting Knowledge Date: December 2023\nToday Date: {date}\n\n"
|
||||
"You have access to the following functions. To call a function, please respond with JSON for a function call. "
|
||||
@@ -804,6 +812,44 @@ class GLM4MOEToolUtils(QwenToolUtils):
|
||||
return "\n".join(function_texts)
|
||||
|
||||
|
||||
class GLM5NextToolUtils(GLM4MOEToolUtils):
|
||||
r"""GLM5-Next tool using template."""
|
||||
|
||||
@override
|
||||
@staticmethod
|
||||
def tool_formatter(tools: list[dict[str, Any]]) -> str:
|
||||
if not isinstance(tools, list):
|
||||
raise ValueError("glm5_next tools must be a JSON list.")
|
||||
tool_text = ""
|
||||
for tool in tools:
|
||||
tool = tool.get("function", tool)
|
||||
if tool.get("defer_loading", False):
|
||||
continue
|
||||
tool = {key: value for key, value in tool.items() if key not in {"strict", "defer_loading"}}
|
||||
tool_text += json.dumps(tool, ensure_ascii=False) + "\n"
|
||||
|
||||
return GLM5_NEXT_TOOL_PROMPT.format(tool_text=tool_text)
|
||||
|
||||
@override
|
||||
@staticmethod
|
||||
def function_formatter(functions: list["FunctionCall"]) -> str:
|
||||
if not functions:
|
||||
raise ValueError("glm5_next function messages must contain at least one call.")
|
||||
calls = []
|
||||
for name, arguments in functions:
|
||||
if not isinstance(name, str) or not re.fullmatch(r"[^\s<>]+", name):
|
||||
raise ValueError("Invalid glm5_next function name.")
|
||||
arguments = json.loads(arguments)
|
||||
if not isinstance(arguments, dict):
|
||||
raise ValueError("glm5_next function arguments must be a JSON object.")
|
||||
text = "<tool_call>" + name
|
||||
for key, value in arguments.items():
|
||||
value = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
|
||||
text += f"<arg_key>{key}</arg_key><arg_value>{value}</arg_value>"
|
||||
calls.append(text + "</tool_call>")
|
||||
return "".join(calls)
|
||||
|
||||
|
||||
class SeedToolUtils(ToolUtils):
|
||||
r"""Seed tool using template."""
|
||||
|
||||
@@ -986,6 +1032,7 @@ TOOLS = {
|
||||
"qwen3_5": Qwen35ToolUtils(),
|
||||
"qwen3_8": Qwen38ToolUtils(),
|
||||
"glm4_moe": GLM4MOEToolUtils(),
|
||||
"glm5_next": GLM5NextToolUtils(),
|
||||
"seed_oss": SeedToolUtils(),
|
||||
"ling": LingToolUtils(),
|
||||
}
|
||||
|
||||
@@ -261,6 +261,15 @@ _register_composite_model(
|
||||
)
|
||||
|
||||
|
||||
_register_composite_model(
|
||||
model_type="glm5_next",
|
||||
projector_keys=["model.visual.merger", "model.visual.downsample"],
|
||||
vision_model_keys=["model.visual.patch_embed", "model.visual.blocks", "model.visual.post_layernorm"],
|
||||
language_model_keys=["model.language_model", "lm_head"],
|
||||
lora_conflict_keys=["patch_embed"],
|
||||
)
|
||||
|
||||
|
||||
_register_composite_model(
|
||||
model_type="glm_ocr",
|
||||
projector_keys=["visual.merger"],
|
||||
|
||||
@@ -186,6 +186,8 @@ def get_process_group_backend() -> str:
|
||||
return "hccl"
|
||||
elif get_current_accelerator().type == DeviceType.CUDA:
|
||||
return "nccl"
|
||||
elif get_current_accelerator().type == DeviceType.XPU:
|
||||
return "xccl"
|
||||
else:
|
||||
return "gloo"
|
||||
|
||||
|
||||
@@ -78,6 +78,8 @@ def _get_visible_devices_env() -> str | None:
|
||||
return "CUDA_VISIBLE_DEVICES"
|
||||
elif CURRENT_DEVICE == "npu":
|
||||
return "ASCEND_RT_VISIBLE_DEVICES"
|
||||
elif CURRENT_DEVICE == "xpu":
|
||||
return "ZE_AFFINITY_MASK"
|
||||
else:
|
||||
return None
|
||||
|
||||
@@ -160,6 +162,8 @@ def _manage_distributed_env(request: FixtureRequest, monkeypatch: MonkeyPatch) -
|
||||
monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
|
||||
elif CURRENT_DEVICE == "npu":
|
||||
monkeypatch.setattr(torch.npu, "device_count", lambda: 1)
|
||||
elif CURRENT_DEVICE == "xpu":
|
||||
monkeypatch.setattr(torch.xpu, "device_count", lambda: 1)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -43,7 +43,7 @@ TRAIN_ARGS = {
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.parametrize("num_samples", [16])
|
||||
def test_feedback_data(num_samples: int):
|
||||
train_dataset = load_dataset_module(**TRAIN_ARGS)["train_dataset"]
|
||||
|
||||
@@ -52,7 +52,7 @@ def _convert_sharegpt_to_openai(messages: list[dict[str, str]]) -> list[dict[str
|
||||
return new_messages
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.parametrize("num_samples", [16])
|
||||
def test_pairwise_data(num_samples: int):
|
||||
train_dataset = load_dataset_module(**TRAIN_ARGS)["train_dataset"]
|
||||
|
||||
@@ -18,7 +18,7 @@ import pytest
|
||||
from llamafactory.data.processor.processor_utils import infer_seqlen
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.parametrize(
|
||||
"test_input,test_output",
|
||||
[
|
||||
|
||||
@@ -43,7 +43,7 @@ TRAIN_ARGS = {
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.parametrize("num_samples", [16])
|
||||
def test_supervised_single_turn(num_samples: int):
|
||||
train_dataset = load_dataset_module(dataset_dir="ONLINE", dataset=TINY_DATA, **TRAIN_ARGS)["train_dataset"]
|
||||
@@ -72,7 +72,7 @@ def test_supervised_single_turn(num_samples: int):
|
||||
assert train_dataset["labels"][index] == ref_label_ids
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.parametrize("num_samples", [8])
|
||||
def test_supervised_multi_turn(num_samples: int):
|
||||
train_dataset = load_dataset_module(dataset_dir="REMOTE:" + DEMO_DATA, dataset="system_chat", **TRAIN_ARGS)[
|
||||
@@ -90,7 +90,7 @@ def test_supervised_multi_turn(num_samples: int):
|
||||
assert train_dataset["input_ids"][index] == ref_input_ids
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.parametrize("num_samples", [4])
|
||||
def test_supervised_train_on_prompt(num_samples: int):
|
||||
train_dataset = load_dataset_module(
|
||||
@@ -108,7 +108,7 @@ def test_supervised_train_on_prompt(num_samples: int):
|
||||
assert train_dataset["labels"][index] == ref_input_ids
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.parametrize("num_samples", [4])
|
||||
def test_supervised_mask_history(num_samples: int):
|
||||
train_dataset = load_dataset_module(
|
||||
|
||||
@@ -47,7 +47,7 @@ TRAIN_ARGS = {
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.parametrize("num_samples", [16])
|
||||
def test_unsupervised_data(num_samples: int):
|
||||
train_dataset = load_dataset_module(**TRAIN_ARGS)["train_dataset"]
|
||||
|
||||
@@ -32,7 +32,7 @@ from llamafactory.model import load_tokenizer
|
||||
TINY_LLAMA3 = os.getenv("TINY_LLAMA3", "llamafactory/tiny-random-Llama-3")
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_base_collator():
|
||||
model_args, data_args, *_ = get_infer_args({"model_name_or_path": TINY_LLAMA3, "template": "default"})
|
||||
tokenizer_module = load_tokenizer(model_args)
|
||||
@@ -76,7 +76,7 @@ def test_base_collator():
|
||||
assert batch_input[k].eq(torch.tensor(expected_input[k])).all()
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_multimodal_collator():
|
||||
model_args, data_args, *_ = get_infer_args(
|
||||
{"model_name_or_path": "Qwen/Qwen2-VL-2B-Instruct", "template": "qwen2_vl"}
|
||||
@@ -270,7 +270,7 @@ def _get_expected_position_ids(
|
||||
return torch.cat(all_position_ids, dim=-1)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_multimodal_collator_with_packing():
|
||||
model_args, data_args, *_ = get_infer_args(
|
||||
{"model_name_or_path": "Qwen/Qwen2-VL-2B-Instruct", "template": "qwen2_vl"}
|
||||
@@ -322,7 +322,7 @@ def test_multimodal_collator_with_packing():
|
||||
assert batch_input["position_ids"][1:, :, :valid_len].eq(expected_position_ids).all()
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu"])
|
||||
@pytest.mark.runs_on(["cpu", "xpu"])
|
||||
def test_4d_attention_mask():
|
||||
o = 0.0
|
||||
x = torch.finfo(torch.float16).min
|
||||
|
||||
@@ -20,7 +20,7 @@ from llamafactory.data.parser import DatasetAttr
|
||||
from llamafactory.hparams import DataArguments
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_alpaca_converter():
|
||||
dataset_attr = DatasetAttr("hf_hub", "llamafactory/tiny-supervised-dataset")
|
||||
data_args = DataArguments()
|
||||
@@ -41,7 +41,7 @@ def test_alpaca_converter():
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_sharegpt_converter():
|
||||
dataset_attr = DatasetAttr("hf_hub", "llamafactory/tiny-supervised-dataset")
|
||||
data_args = DataArguments()
|
||||
|
||||
@@ -38,19 +38,19 @@ TOOLS = [
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_empty_formatter():
|
||||
formatter = EmptyFormatter(slots=["\n"])
|
||||
assert formatter.apply() == ["\n"]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_string_formatter():
|
||||
formatter = StringFormatter(slots=["<s>", "Human: {{content}}\nAssistant:"])
|
||||
assert formatter.apply(content="Hi") == ["<s>", "Human: Hi\nAssistant:"]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_function_formatter():
|
||||
formatter = FunctionFormatter(slots=["{{content}}", "</s>"], tool_format="default")
|
||||
tool_calls = json.dumps(FUNCTION)
|
||||
@@ -60,7 +60,7 @@ def test_function_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_multi_function_formatter():
|
||||
formatter = FunctionFormatter(slots=["{{content}}", "</s>"], tool_format="default")
|
||||
tool_calls = json.dumps([FUNCTION] * 2)
|
||||
@@ -71,7 +71,7 @@ def test_multi_function_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_default_tool_formatter():
|
||||
formatter = ToolFormatter(tool_format="default")
|
||||
assert formatter.apply(content=json.dumps(TOOLS)) == [
|
||||
@@ -90,14 +90,14 @@ def test_default_tool_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_default_tool_extractor():
|
||||
formatter = ToolFormatter(tool_format="default")
|
||||
result = """Action: test_tool\nAction Input: {"foo": "bar", "size": 10}"""
|
||||
assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_default_multi_tool_extractor():
|
||||
formatter = ToolFormatter(tool_format="default")
|
||||
result = (
|
||||
@@ -110,14 +110,14 @@ def test_default_multi_tool_extractor():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_glm4_function_formatter():
|
||||
formatter = FunctionFormatter(slots=["{{content}}"], tool_format="glm4")
|
||||
tool_calls = json.dumps(FUNCTION)
|
||||
assert formatter.apply(content=tool_calls) == ["""tool_name\n{"foo": "bar", "size": 10}"""]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_glm4_tool_formatter():
|
||||
formatter = ToolFormatter(tool_format="glm4")
|
||||
assert formatter.apply(content=json.dumps(TOOLS)) == [
|
||||
@@ -128,14 +128,14 @@ def test_glm4_tool_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_glm4_tool_extractor():
|
||||
formatter = ToolFormatter(tool_format="glm4")
|
||||
result = """test_tool\n{"foo": "bar", "size": 10}\n"""
|
||||
assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_llama3_function_formatter():
|
||||
formatter = FunctionFormatter(slots=["{{content}}<|eot_id|>"], tool_format="llama3")
|
||||
tool_calls = json.dumps(FUNCTION)
|
||||
@@ -144,7 +144,7 @@ def test_llama3_function_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_llama3_multi_function_formatter():
|
||||
formatter = FunctionFormatter(slots=["{{content}}<|eot_id|>"], tool_format="llama3")
|
||||
tool_calls = json.dumps([FUNCTION] * 2)
|
||||
@@ -155,7 +155,7 @@ def test_llama3_multi_function_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_llama3_tool_formatter():
|
||||
formatter = ToolFormatter(tool_format="llama3")
|
||||
date = datetime.now().strftime("%d %b %Y")
|
||||
@@ -169,14 +169,14 @@ def test_llama3_tool_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_llama3_tool_extractor():
|
||||
formatter = ToolFormatter(tool_format="llama3")
|
||||
result = """{"name": "test_tool", "parameters": {"foo": "bar", "size": 10}}\n"""
|
||||
assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_llama3_multi_tool_extractor():
|
||||
formatter = ToolFormatter(tool_format="llama3")
|
||||
result = (
|
||||
@@ -189,7 +189,7 @@ def test_llama3_multi_tool_extractor():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_mistral_function_formatter():
|
||||
formatter = FunctionFormatter(slots=["[TOOL_CALLS] {{content}}", "</s>"], tool_format="mistral")
|
||||
tool_calls = json.dumps(FUNCTION)
|
||||
@@ -199,7 +199,7 @@ def test_mistral_function_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_mistral_multi_function_formatter():
|
||||
formatter = FunctionFormatter(slots=["[TOOL_CALLS] {{content}}", "</s>"], tool_format="mistral")
|
||||
tool_calls = json.dumps([FUNCTION] * 2)
|
||||
@@ -211,7 +211,7 @@ def test_mistral_multi_function_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_mistral_tool_formatter():
|
||||
formatter = ToolFormatter(tool_format="mistral")
|
||||
wrapped_tool = {"type": "function", "function": TOOLS[0]}
|
||||
@@ -220,14 +220,14 @@ def test_mistral_tool_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_mistral_tool_extractor():
|
||||
formatter = ToolFormatter(tool_format="mistral")
|
||||
result = """{"name": "test_tool", "arguments": {"foo": "bar", "size": 10}}"""
|
||||
assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_mistral_multi_tool_extractor():
|
||||
formatter = ToolFormatter(tool_format="mistral")
|
||||
result = (
|
||||
@@ -240,7 +240,7 @@ def test_mistral_multi_tool_extractor():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_qwen_function_formatter():
|
||||
formatter = FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen")
|
||||
tool_calls = json.dumps(FUNCTION)
|
||||
@@ -249,7 +249,7 @@ def test_qwen_function_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_qwen_multi_function_formatter():
|
||||
formatter = FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen")
|
||||
tool_calls = json.dumps([FUNCTION] * 2)
|
||||
@@ -260,7 +260,7 @@ def test_qwen_multi_function_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_qwen_tool_formatter():
|
||||
formatter = ToolFormatter(tool_format="qwen")
|
||||
wrapped_tool = {"type": "function", "function": TOOLS[0]}
|
||||
@@ -274,14 +274,14 @@ def test_qwen_tool_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_qwen_tool_extractor():
|
||||
formatter = ToolFormatter(tool_format="qwen")
|
||||
result = """<tool_call>\n{"name": "test_tool", "arguments": {"foo": "bar", "size": 10}}\n</tool_call>"""
|
||||
assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_qwen38_tool_formatter():
|
||||
formatter = ToolFormatter(tool_format="qwen3_8")
|
||||
wrapped_tool = {"type": "function", "function": TOOLS[0]}
|
||||
@@ -289,7 +289,7 @@ def test_qwen38_tool_formatter():
|
||||
assert json.dumps(wrapped_tool, ensure_ascii=False) in output
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_qwen_multi_tool_extractor():
|
||||
formatter = ToolFormatter(tool_format="qwen")
|
||||
result = (
|
||||
@@ -302,7 +302,7 @@ def test_qwen_multi_tool_extractor():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_lfm2_function_formatter():
|
||||
formatter = FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="lfm2")
|
||||
tool_calls = json.dumps(FUNCTION)
|
||||
@@ -311,7 +311,7 @@ def test_lfm2_function_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_lfm2_multi_function_formatter():
|
||||
formatter = FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="lfm2")
|
||||
tool_calls = json.dumps([FUNCTION] * 2)
|
||||
@@ -321,7 +321,7 @@ def test_lfm2_multi_function_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_lfm2_tool_formatter():
|
||||
formatter = ToolFormatter(tool_format="lfm2")
|
||||
assert formatter.apply(content=json.dumps(TOOLS)) == [
|
||||
@@ -329,14 +329,14 @@ def test_lfm2_tool_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_lfm2_tool_extractor():
|
||||
formatter = ToolFormatter(tool_format="lfm2")
|
||||
result = """<|tool_call_start|>[test_tool(foo="bar", size=10)]<|tool_call_end|>"""
|
||||
assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_lfm2_multi_tool_extractor():
|
||||
formatter = ToolFormatter(tool_format="lfm2")
|
||||
result = """<|tool_call_start|>[test_tool(foo="bar", size=10), another_tool(foo="job", size=2)]<|tool_call_end|>"""
|
||||
@@ -346,7 +346,7 @@ def test_lfm2_multi_tool_extractor():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_lfm2_tool_extractor_with_nested_dict():
|
||||
formatter = ToolFormatter(tool_format="lfm2")
|
||||
result = """<|tool_call_start|>[search(query="test", options={"limit": 10, "offset": 0})]<|tool_call_end|>"""
|
||||
@@ -358,7 +358,7 @@ def test_lfm2_tool_extractor_with_nested_dict():
|
||||
assert args["options"] == {"limit": 10, "offset": 0}
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_lfm2_tool_extractor_with_list_arg():
|
||||
formatter = ToolFormatter(tool_format="lfm2")
|
||||
result = """<|tool_call_start|>[batch_process(items=[1, 2, 3], enabled=True)]<|tool_call_end|>"""
|
||||
@@ -370,7 +370,7 @@ def test_lfm2_tool_extractor_with_list_arg():
|
||||
assert args["enabled"] is True
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_lfm2_tool_extractor_no_match():
|
||||
formatter = ToolFormatter(tool_format="lfm2")
|
||||
result = "This is a regular response without tool calls."
|
||||
@@ -378,7 +378,7 @@ def test_lfm2_tool_extractor_no_match():
|
||||
assert extracted == result
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_lfm2_tool_round_trip():
|
||||
formatter = FunctionFormatter(slots=["{{content}}"], tool_format="lfm2")
|
||||
tool_formatter = ToolFormatter(tool_format="lfm2")
|
||||
@@ -390,7 +390,7 @@ def test_lfm2_tool_round_trip():
|
||||
assert json.loads(extracted[0][1]) == original["arguments"]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_minicpm5_function_formatter():
|
||||
formatter = FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="minicpm5")
|
||||
tool_calls = json.dumps(FUNCTION)
|
||||
@@ -399,7 +399,7 @@ def test_minicpm5_function_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_minicpm5_multi_function_formatter():
|
||||
formatter = FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="minicpm5")
|
||||
tool_calls = json.dumps([FUNCTION] * 2)
|
||||
@@ -411,7 +411,7 @@ def test_minicpm5_multi_function_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_minicpm5_tool_formatter():
|
||||
formatter = ToolFormatter(tool_format="minicpm5")
|
||||
wrapped = json.dumps({"type": "function", "function": TOOLS[0]}, ensure_ascii=False)
|
||||
@@ -427,28 +427,28 @@ def test_minicpm5_tool_formatter():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_minicpm5_tool_extractor():
|
||||
formatter = ToolFormatter(tool_format="minicpm5")
|
||||
result = '<function name="test_tool"><param name="foo">bar</param><param name="size">10</param></function>'
|
||||
assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_minicpm5_tool_extractor_cdata():
|
||||
formatter = ToolFormatter(tool_format="minicpm5")
|
||||
result = '<function name="test_tool"><param name="foo"><![CDATA[a < b\nsecond line]]></param></function>'
|
||||
assert formatter.extract(result) == [("test_tool", json.dumps({"foo": "a < b\nsecond line"}))]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_minicpm5_tool_extractor_malformed_value():
|
||||
formatter = ToolFormatter(tool_format="minicpm5")
|
||||
result = '<function name="test_tool"><param name="foo">{[1, 2], [3, 4]}</param></function>'
|
||||
assert formatter.extract(result) == [("test_tool", json.dumps({"foo": "{[1, 2], [3, 4]}"}))]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.parametrize(
|
||||
"arguments",
|
||||
[
|
||||
|
||||
@@ -40,21 +40,21 @@ TRAIN_ARGS = {
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_load_train_only():
|
||||
dataset_module = load_dataset_module(**TRAIN_ARGS)
|
||||
assert dataset_module.get("train_dataset") is not None
|
||||
assert dataset_module.get("eval_dataset") is None
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_load_val_size():
|
||||
dataset_module = load_dataset_module(val_size=0.1, **TRAIN_ARGS)
|
||||
assert dataset_module.get("train_dataset") is not None
|
||||
assert dataset_module.get("eval_dataset") is not None
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_load_eval_data():
|
||||
dataset_module = load_dataset_module(eval_dataset=TINY_DATA, **TRAIN_ARGS)
|
||||
assert dataset_module.get("train_dataset") is not None
|
||||
|
||||
@@ -180,7 +180,7 @@ def _check_plugin(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_base_plugin():
|
||||
tokenizer_module = _load_tokenizer_module(model_name_or_path=TINY_LLAMA3)
|
||||
base_plugin = get_mm_plugin(name="base")
|
||||
@@ -188,7 +188,7 @@ def test_base_plugin():
|
||||
_check_plugin(**check_inputs)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.skipif(not HF_TOKEN, reason="Gated model.")
|
||||
@pytest.mark.skipif(not is_transformers_version_greater_than("4.50.0"), reason="Requires transformers>=4.50.0")
|
||||
def test_gemma3_plugin():
|
||||
@@ -211,7 +211,7 @@ def test_gemma3_plugin():
|
||||
_check_plugin(**check_inputs)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.skipif(not is_transformers_version_greater_than("5.6.0"), reason="Requires transformers>=5.6.0")
|
||||
def test_gemma4_plugin():
|
||||
tokenizer_module = _load_tokenizer_module(model_name_or_path="google/gemma-4-31B-it")
|
||||
@@ -244,7 +244,7 @@ def test_gemma4_plugin():
|
||||
_check_plugin(**check_inputs)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.skipif(not is_transformers_version_greater_than("4.52.0"), reason="Requires transformers>=4.52.0")
|
||||
def test_internvl_plugin():
|
||||
image_seqlen = 256
|
||||
@@ -263,7 +263,7 @@ def test_internvl_plugin():
|
||||
_check_plugin(**check_inputs)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.skipif(not is_transformers_version_greater_than("4.51.0"), reason="Requires transformers>=4.51.0")
|
||||
def test_llama4_plugin():
|
||||
tokenizer_module = _load_tokenizer_module(model_name_or_path=TINY_LLAMA4)
|
||||
@@ -285,7 +285,7 @@ def test_llama4_plugin():
|
||||
_check_plugin(**check_inputs)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_llava_plugin():
|
||||
image_seqlen = 576
|
||||
tokenizer_module = _load_tokenizer_module(model_name_or_path="llava-hf/llava-1.5-7b-hf")
|
||||
@@ -299,7 +299,7 @@ def test_llava_plugin():
|
||||
_check_plugin(**check_inputs)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_llava_next_plugin():
|
||||
image_seqlen = 1176
|
||||
tokenizer_module = _load_tokenizer_module(model_name_or_path="llava-hf/llava-v1.6-vicuna-7b-hf")
|
||||
@@ -313,7 +313,7 @@ def test_llava_next_plugin():
|
||||
_check_plugin(**check_inputs)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_llava_next_video_plugin():
|
||||
image_seqlen = 1176
|
||||
tokenizer_module = _load_tokenizer_module(model_name_or_path="llava-hf/LLaVA-NeXT-Video-7B-hf")
|
||||
@@ -327,7 +327,7 @@ def test_llava_next_video_plugin():
|
||||
_check_plugin(**check_inputs)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.skipif(not HF_TOKEN, reason="Gated model.")
|
||||
def test_paligemma_plugin():
|
||||
image_seqlen = 256
|
||||
@@ -347,7 +347,7 @@ def test_paligemma_plugin():
|
||||
_check_plugin(**check_inputs)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.skipif(not is_transformers_version_greater_than("4.50.0"), reason="Requires transformers>=4.50.0")
|
||||
def test_pixtral_plugin():
|
||||
image_slice_height, image_slice_width = 2, 2
|
||||
@@ -370,7 +370,7 @@ def test_pixtral_plugin():
|
||||
_check_plugin(**check_inputs)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.skipif(not is_transformers_version_greater_than("4.52.0"), reason="Requires transformers>=4.52.0")
|
||||
def test_qwen2_omni_plugin():
|
||||
image_seqlen, audio_seqlen = 4, 2
|
||||
@@ -401,7 +401,7 @@ def test_qwen2_omni_plugin():
|
||||
_check_plugin(**check_inputs)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_qwen2_vl_plugin():
|
||||
image_seqlen = 4
|
||||
tokenizer_module = _load_tokenizer_module(model_name_or_path="Qwen/Qwen2-VL-7B-Instruct")
|
||||
@@ -436,7 +436,7 @@ def test_moss_vl_plugin():
|
||||
assert messages[0]["content"] == "First <image>, finally <image>."
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.skipif(not is_transformers_version_greater_than("4.57.0"), reason="Requires transformers>=4.57.0")
|
||||
def test_qwen3_vl_plugin():
|
||||
frame_seqlen = 1
|
||||
@@ -470,7 +470,7 @@ def test_qwen3_vl_plugin():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.skipif(not is_transformers_version_greater_than("4.57.0"), reason="Requires transformers>=4.57.0")
|
||||
@pytest.mark.skipif(not is_pyav_available(), reason="Requires pyav")
|
||||
def test_qwen3_vl_plugin_video_path():
|
||||
@@ -504,7 +504,7 @@ def test_qwen3_vl_plugin_video_path():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.skipif(not is_transformers_version_greater_than("4.47.0"), reason="Requires transformers>=4.47.0")
|
||||
def test_video_llava_plugin():
|
||||
image_seqlen = 256
|
||||
@@ -519,7 +519,7 @@ def test_video_llava_plugin():
|
||||
_check_plugin(**check_inputs)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_lfm2_vl_plugin():
|
||||
"""Test LFM2.5-VL plugin instantiation."""
|
||||
# Test plugin can be instantiated with correct tokens
|
||||
|
||||
@@ -139,7 +139,7 @@ def test_qwen38_registration():
|
||||
assert {"qwen3_5_moe_text", "qwen3_5_text"} <= MCA_SUPPORTED_MODELS
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_encode_oneturn():
|
||||
tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)
|
||||
template = get_template_and_fix_tokenizer(tokenizer, DataArguments(template="llama3"))
|
||||
@@ -154,7 +154,7 @@ def test_encode_oneturn():
|
||||
_check_tokenization(tokenizer, (prompt_ids, answer_ids), (prompt_str, answer_str))
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_encode_multiturn():
|
||||
tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)
|
||||
template = get_template_and_fix_tokenizer(tokenizer, DataArguments(template="llama3"))
|
||||
@@ -175,7 +175,7 @@ def test_encode_multiturn():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.parametrize("cot_messages", [True, False])
|
||||
@pytest.mark.parametrize("enable_thinking", [True, False, None])
|
||||
def test_reasoning_encode_oneturn(cot_messages: bool, enable_thinking: bool):
|
||||
@@ -201,7 +201,7 @@ def test_reasoning_encode_oneturn(cot_messages: bool, enable_thinking: bool):
|
||||
_check_tokenization(tokenizer, (prompt_ids, answer_ids), (prompt_str, answer_str))
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.parametrize("cot_messages", [True, False])
|
||||
@pytest.mark.parametrize("enable_thinking", [True, False, None])
|
||||
def test_reasoning_encode_multiturn(cot_messages: bool, enable_thinking: bool):
|
||||
@@ -230,7 +230,7 @@ def test_reasoning_encode_multiturn(cot_messages: bool, enable_thinking: bool):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.parametrize("enable_thinking", [True, False, None])
|
||||
@pytest.mark.parametrize("discarding_history_cot", [True, False])
|
||||
def test_reasoning_encode_multiturn_discarding_history_cot(enable_thinking: bool, discarding_history_cot: bool):
|
||||
@@ -266,7 +266,7 @@ def test_reasoning_encode_multiturn_discarding_history_cot(enable_thinking: bool
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_qwen38_reasoning_instruction():
|
||||
tokenizer = CharTokenizer()
|
||||
template = deepcopy(TEMPLATES["qwen3_8"])
|
||||
@@ -284,7 +284,7 @@ def test_qwen38_reasoning_instruction():
|
||||
assert answer_text == "<think>\n\n</think>\n\nI am fine!<|im_end|>\n"
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_qwen38_tool_system_order():
|
||||
tokenizer = CharTokenizer()
|
||||
template = deepcopy(TEMPLATES["qwen3_8"])
|
||||
@@ -308,7 +308,7 @@ def test_qwen38_tool_system_order():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_jinja_template():
|
||||
tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)
|
||||
ref_tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)
|
||||
@@ -318,7 +318,7 @@ def test_jinja_template():
|
||||
assert tokenizer.apply_chat_template(MESSAGES) == ref_tokenizer.apply_chat_template(MESSAGES)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_ollama_modelfile():
|
||||
tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)
|
||||
template = get_template_and_fix_tokenizer(tokenizer, DataArguments(template="llama3"))
|
||||
@@ -336,14 +336,14 @@ def test_ollama_modelfile():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_get_stop_token_ids():
|
||||
tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)
|
||||
template = get_template_and_fix_tokenizer(tokenizer, DataArguments(template="llama3"))
|
||||
assert set(template.get_stop_token_ids(tokenizer)) == {128008, 128009}
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.skipif(not HF_TOKEN, reason="Gated model.")
|
||||
def test_gemma_template():
|
||||
prompt_str = (
|
||||
@@ -356,7 +356,7 @@ def test_gemma_template():
|
||||
_check_template("google/gemma-3-4b-it", "gemma", prompt_str, answer_str)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.skipif(not HF_TOKEN, reason="Gated model.")
|
||||
def test_gemma2_template():
|
||||
prompt_str = (
|
||||
@@ -369,7 +369,7 @@ def test_gemma2_template():
|
||||
_check_template("google/gemma-2-2b-it", "gemma2", prompt_str, answer_str)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.skipif(not HF_TOKEN, reason="Gated model.")
|
||||
def test_llama3_template():
|
||||
prompt_str = (
|
||||
@@ -382,7 +382,7 @@ def test_llama3_template():
|
||||
_check_template("meta-llama/Meta-Llama-3-8B-Instruct", "llama3", prompt_str, answer_str)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_llama4_template():
|
||||
prompt_str = (
|
||||
f"<|begin_of_text|><|header_start|>user<|header_end|>\n\n{MESSAGES[0]['content']}<|eot|>"
|
||||
@@ -394,7 +394,7 @@ def test_llama4_template():
|
||||
_check_template(TINY_LLAMA4, "llama4", prompt_str, answer_str)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_phi4_template():
|
||||
prompt_str = (
|
||||
f"<|im_start|>user<|im_sep|>{MESSAGES[0]['content']}<|im_end|>"
|
||||
@@ -406,7 +406,7 @@ def test_phi4_template():
|
||||
_check_template("microsoft/phi-4", "phi4", prompt_str, answer_str)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.xfail(not HF_TOKEN, reason="Authorization.")
|
||||
def test_qwen2_5_template():
|
||||
prompt_str = (
|
||||
@@ -420,7 +420,7 @@ def test_qwen2_5_template():
|
||||
_check_template("Qwen/Qwen2.5-7B-Instruct", "qwen", prompt_str, answer_str)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.parametrize("cot_messages", [True, False])
|
||||
def test_qwen3_template(cot_messages: bool):
|
||||
prompt_str = (
|
||||
@@ -439,7 +439,7 @@ def test_qwen3_template(cot_messages: bool):
|
||||
_check_template("Qwen/Qwen3-8B", "qwen3", prompt_str, answer_str, messages=messages)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_parse_llama3_template():
|
||||
tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)
|
||||
template = parse_template(tokenizer)
|
||||
@@ -453,7 +453,7 @@ def test_parse_llama3_template():
|
||||
assert template.default_system == ""
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.xfail(not HF_TOKEN, reason="Authorization.")
|
||||
def test_parse_qwen_template():
|
||||
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B-Instruct")
|
||||
@@ -466,7 +466,7 @@ def test_parse_qwen_template():
|
||||
assert template.default_system == "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.xfail(not HF_TOKEN, reason="Authorization.")
|
||||
def test_parse_qwen3_template():
|
||||
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-8B")
|
||||
|
||||
@@ -37,13 +37,13 @@ MESSAGES = [
|
||||
EXPECTED_RESPONSE = "_rho"
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_chat():
|
||||
chat_model = ChatModel(INFER_ARGS)
|
||||
assert chat_model.chat(MESSAGES)[0].response_text == EXPECTED_RESPONSE
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_stream_chat():
|
||||
chat_model = ChatModel(INFER_ARGS)
|
||||
response = ""
|
||||
|
||||
@@ -49,7 +49,7 @@ INFER_ARGS = {
|
||||
OS_NAME = os.getenv("OS_NAME", "")
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
@pytest.mark.parametrize(
|
||||
"stage,dataset",
|
||||
[
|
||||
@@ -66,7 +66,7 @@ def test_run_exp(stage: str, dataset: str):
|
||||
assert os.path.exists(output_dir)
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_export():
|
||||
export_dir = os.path.join("output", "llama3_export")
|
||||
export_model({"export_dir": export_dir, **INFER_ARGS})
|
||||
|
||||
@@ -17,7 +17,7 @@ import pytest
|
||||
from llamafactory.eval.template import get_eval_template
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_eval_template_en():
|
||||
support_set = [
|
||||
{
|
||||
@@ -56,7 +56,7 @@ def test_eval_template_en():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cpu", "mps"])
|
||||
@pytest.mark.runs_on(["cpu", "mps", "xpu"])
|
||||
def test_eval_template_zh():
|
||||
support_set = [
|
||||
{
|
||||
|
||||
@@ -52,7 +52,7 @@ def test_all_device():
|
||||
assert DistributedInterface().get_local_world_size() == int(os.getenv("LOCAL_WORLD_SIZE", "1"))
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cuda", "npu"])
|
||||
@pytest.mark.runs_on(["cuda", "npu", "xpu"])
|
||||
@pytest.mark.require_distributed(2)
|
||||
def test_multi_device():
|
||||
master_port = find_available_port()
|
||||
|
||||
@@ -79,6 +79,8 @@ def _get_visible_devices_env() -> str | None:
|
||||
return "CUDA_VISIBLE_DEVICES"
|
||||
elif CURRENT_DEVICE == "npu":
|
||||
return "ASCEND_RT_VISIBLE_DEVICES"
|
||||
elif CURRENT_DEVICE == "xpu":
|
||||
return "ZE_AFFINITY_MASK"
|
||||
else:
|
||||
return None
|
||||
|
||||
@@ -172,6 +174,8 @@ def _manage_distributed_env(request: FixtureRequest, monkeypatch: MonkeyPatch) -
|
||||
monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
|
||||
elif CURRENT_DEVICE == "npu":
|
||||
monkeypatch.setattr(torch.npu, "device_count", lambda: 1)
|
||||
elif CURRENT_DEVICE == "xpu":
|
||||
monkeypatch.setattr(torch.xpu, "device_count", lambda: 1)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
|
||||
@@ -96,7 +96,7 @@ def _test_sequence_parallel_loss(
|
||||
assert loss is not None
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cuda", "npu"])
|
||||
@pytest.mark.runs_on(["cuda", "npu", "xpu"])
|
||||
@pytest.mark.require_distributed(2)
|
||||
@pytest.mark.parametrize(("cp_size", "dp_size", "batch_size"), [(2, 1, 1), (2, 1, 2)])
|
||||
def test_sequence_parallel_loss(cp_size, dp_size, batch_size):
|
||||
|
||||
@@ -19,7 +19,7 @@ from llamafactory.v1.core.model_engine import ModelEngine
|
||||
from llamafactory.v1.samplers.cli_sampler import SyncSampler
|
||||
|
||||
|
||||
@pytest.mark.runs_on(["cuda", "npu"])
|
||||
@pytest.mark.runs_on(["cuda", "npu", "xpu"])
|
||||
def test_sync_sampler():
|
||||
model_args = ModelArguments(model="Qwen/Qwen3-4B-Instruct-2507")
|
||||
sample_args = SampleArguments()
|
||||
|
||||
@@ -21,7 +21,7 @@ import pytest
|
||||
|
||||
|
||||
@pytest.mark.xfail(reason="CI machines may OOM when heavily loaded.")
|
||||
@pytest.mark.runs_on(["cuda", "npu"])
|
||||
@pytest.mark.runs_on(["cuda", "npu", "xpu"])
|
||||
def test_fsdp2_dpo_trainer(tmp_path: Path):
|
||||
"""Test FSDP2 DPO trainer with sigmoid loss by simulating `llamafactory-cli dpo config.yaml`."""
|
||||
config_yaml = """\
|
||||
|
||||
@@ -20,7 +20,7 @@ import pytest
|
||||
|
||||
|
||||
@pytest.mark.xfail(reason="CI machines may OOM when heavily loaded.")
|
||||
@pytest.mark.runs_on(["cuda", "npu"])
|
||||
@pytest.mark.runs_on(["cuda", "npu", "xpu"])
|
||||
def test_fsdp2_sft_trainer(tmp_path: Path):
|
||||
"""Test FSDP2 SFT trainer by simulating `llamafactory-cli sft config.yaml` behavior."""
|
||||
config_yaml = """\
|
||||
|
||||
Reference in New Issue
Block a user