From 5817583630ef6540df79ce91b7cfaa6f84a3cfc1 Mon Sep 17 00:00:00 2001 From: Yaowei Zheng Date: Mon, 7 Jul 2025 20:31:50 +0800 Subject: [PATCH] [deps] bump transformers to 4.49.0 (#8564) --- .github/workflows/tests.yml | 3 --- README.md | 4 ++-- README_zh.md | 4 ++-- docker/docker-cuda/README.md | 8 ++++---- requirements.txt | 31 +++++++++++++++++++----------- setup.py | 1 - src/llamafactory/data/mm_plugin.py | 13 +++++-------- src/llamafactory/extras/misc.py | 7 +------ src/llamafactory/hparams/parser.py | 2 +- src/llamafactory/model/loader.py | 11 ++--------- tests/data/test_mm_plugin.py | 1 - 11 files changed, 37 insertions(+), 48 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 382a837b..1dc6230f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,9 +34,6 @@ jobs: transformers: - null include: # test backward compatibility - - python: "3.9" - os: "ubuntu-latest" - transformers: "4.45.0" - python: "3.9" os: "ubuntu-latest" transformers: "4.49.0" diff --git a/README.md b/README.md index 3ba18238..d7f8ae85 100644 --- a/README.md +++ b/README.md @@ -449,7 +449,7 @@ huggingface-cli login | python | 3.9 | 3.10 | | torch | 2.0.0 | 2.6.0 | | torchvision | 0.15.0 | 0.21.0 | -| transformers | 4.45.0 | 4.50.0 | +| transformers | 4.49.0 | 4.50.0 | | datasets | 2.16.0 | 3.2.0 | | accelerate | 0.34.0 | 1.2.1 | | peft | 0.14.0 | 0.15.1 | @@ -491,7 +491,7 @@ cd LLaMA-Factory pip install -e ".[torch,metrics]" --no-build-isolation ``` -Extra dependencies available: torch, torch-npu, metrics, deepspeed, liger-kernel, bitsandbytes, hqq, eetq, gptq, aqlm, vllm, sglang, galore, apollo, badam, adam-mini, qwen, minicpm_v, modelscope, openmind, swanlab, dev +Extra dependencies available: torch, torch-npu, metrics, deepspeed, liger-kernel, bitsandbytes, hqq, eetq, gptq, aqlm, vllm, sglang, galore, apollo, badam, adam-mini, qwen, minicpm_v, openmind, swanlab, dev #### Install from Docker Image diff --git a/README_zh.md b/README_zh.md index 35d63d1e..466f57d5 100644 --- a/README_zh.md +++ b/README_zh.md @@ -451,7 +451,7 @@ huggingface-cli login | python | 3.9 | 3.10 | | torch | 2.0.0 | 2.6.0 | | torchvision | 0.15.0 | 0.21.0 | -| transformers | 4.45.0 | 4.50.0 | +| transformers | 4.49.0 | 4.50.0 | | datasets | 2.16.0 | 3.2.0 | | accelerate | 0.34.0 | 1.2.1 | | peft | 0.14.0 | 0.15.1 | @@ -493,7 +493,7 @@ cd LLaMA-Factory pip install -e ".[torch,metrics]" --no-build-isolation ``` -可选的额外依赖项：torch、torch-npu、metrics、deepspeed、liger-kernel、bitsandbytes、hqq、eetq、gptq、aqlm、vllm、sglang、galore、apollo、badam、adam-mini、qwen、minicpm_v、modelscope、openmind、swanlab、dev +可选的额外依赖项：torch、torch-npu、metrics、deepspeed、liger-kernel、bitsandbytes、hqq、eetq、gptq、aqlm、vllm、sglang、galore、apollo、badam、adam-mini、qwen、minicpm_v、openmind、swanlab、dev #### 从镜像安装 diff --git a/docker/docker-cuda/README.md b/docker/docker-cuda/README.md index e8c73be1..86707ce9 100644 --- a/docker/docker-cuda/README.md +++ b/docker/docker-cuda/README.md @@ -13,7 +13,7 @@ Before running the Docker container with GPU support, you need to install the fo # Ubuntu/Debian sudo apt-get update sudo apt-get install docker.io - + # Or install Docker Engine from the official repository: # https://docs.docker.com/engine/install/ ``` @@ -22,7 +22,7 @@ Before running the Docker container with GPU support, you need to install the fo ```bash # Ubuntu/Debian sudo apt-get install docker-compose - + # Or install the latest version: # https://docs.docker.com/compose/install/ ``` @@ -33,11 +33,11 @@ Before running the Docker container with GPU support, you need to install the fo distribution=$(. /etc/os-release;echo $ID$VERSION_ID) curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - + # Install nvidia-container-toolkit sudo apt-get update sudo apt-get install -y nvidia-container-toolkit - + # Restart Docker to apply changes sudo systemctl restart docker ``` diff --git a/requirements.txt b/requirements.txt index 917a51d1..486cc604 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,27 +1,36 @@ -transformers>=4.45.0,<=4.52.4,!=4.46.*,!=4.47.*,!=4.48.0,!=4.52.0; sys_platform != 'darwin' -transformers>=4.45.0,<=4.51.3,!=4.46.*,!=4.47.*,!=4.48.0,!=4.52.0; sys_platform == 'darwin' +# core deps +transformers>=4.49.0,<=4.52.4,!=4.52.0; sys_platform != 'darwin' +transformers>=4.49.0,<=4.51.3,!=4.52.0; sys_platform == 'darwin' datasets>=2.16.0,<=3.6.0 accelerate>=1.3.0,<=1.7.0 peft>=0.14.0,<=0.15.2 trl>=0.8.6,<=0.9.6 tokenizers>=0.19.0,<=0.21.1 +# gui gradio>=4.38.0,<=5.31.0 -scipy +matplotlib>=3.7.0 +tyro<0.9.0 +# ops einops +numpy<2.0.0 +pandas>=2.0.0 +scipy +# model and tokenizer sentencepiece tiktoken -protobuf -uvicorn -fastapi -sse-starlette -matplotlib>=3.7.0 +modelscope>=1.23 +hf-transfer +# python fire omegaconf packaging +protobuf pyyaml -numpy<2.0.0 pydantic<=2.10.6 -pandas>=2.0.0 +# api +uvicorn +fastapi +sse-starlette +# media av librosa -tyro<0.9.0 diff --git a/setup.py b/setup.py index 73f9424e..0d604103 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,6 @@ extra_require = { "referencing", "jsonschema_specifications", ], - "modelscope": ["modelscope"], "openmind": ["openmind"], "swanlab": ["swanlab"], "dev": ["pre-commit", "ruff", "pytest", "build"], diff --git a/src/llamafactory/data/mm_plugin.py b/src/llamafactory/data/mm_plugin.py index 9da81682..2cb3e320 100644 --- a/src/llamafactory/data/mm_plugin.py +++ b/src/llamafactory/data/mm_plugin.py @@ -27,6 +27,10 @@ from typing import TYPE_CHECKING, BinaryIO, Literal, Optional, TypedDict, Union import numpy as np import torch from transformers.image_utils import get_image_size, is_valid_image, to_numpy_array +from transformers.models.mllama.processing_mllama import ( + convert_sparse_cross_attention_mask_to_dense, + get_cross_attention_token_mask, +) from typing_extensions import override from ..extras.constants import AUDIO_PLACEHOLDER, IGNORE_INDEX, IMAGE_PLACEHOLDER, VIDEO_PLACEHOLDER @@ -51,17 +55,10 @@ if is_pyav_available(): import av -if is_transformers_version_greater_than("4.45.0"): - from transformers.models.mllama.processing_mllama import ( - convert_sparse_cross_attention_mask_to_dense, - get_cross_attention_token_mask, - ) - - if is_transformers_version_greater_than("4.52.0"): from transformers.image_utils import make_flat_list_of_images from transformers.video_utils import make_batched_videos -elif is_transformers_version_greater_than("4.49.0"): +else: from transformers.image_utils import make_batched_videos, make_flat_list_of_images diff --git a/src/llamafactory/extras/misc.py b/src/llamafactory/extras/misc.py index 7a86c23d..a5b777e9 100644 --- a/src/llamafactory/extras/misc.py +++ b/src/llamafactory/extras/misc.py @@ -35,7 +35,6 @@ from transformers.utils import ( from transformers.utils.versions import require_version from . import logging -from .packages import is_transformers_version_greater_than _is_fp16_available = is_torch_npu_available() or is_torch_cuda_available() @@ -94,15 +93,11 @@ def check_version(requirement: str, mandatory: bool = False) -> None: def check_dependencies() -> None: r"""Check the version of the required packages.""" - check_version( - "transformers>=4.45.0,<=4.52.4,!=4.46.0,!=4.46.1,!=4.46.2,!=4.46.3,!=4.47.0,!=4.47.1,!=4.48.0,!=4.52.0" - ) + check_version("transformers>=4.49.0,<=4.52.4,!=4.52.0") check_version("datasets>=2.16.0,<=3.6.0") check_version("accelerate>=1.3.0,<=1.7.0") check_version("peft>=0.14.0,<=0.15.2") check_version("trl>=0.8.6,<=0.9.6") - if is_transformers_version_greater_than("4.46.0") and not is_transformers_version_greater_than("4.48.1"): - logger.warning_rank0_once("There are known bugs in transformers v4.46.0-v4.48.0, please use other versions.") def calculate_tps(dataset: list[dict[str, Any]], metrics: dict[str, float], stage: Literal["sft", "rm"]) -> float: diff --git a/src/llamafactory/hparams/parser.py b/src/llamafactory/hparams/parser.py index d49e4e1d..26f09ca4 100644 --- a/src/llamafactory/hparams/parser.py +++ b/src/llamafactory/hparams/parser.py @@ -348,7 +348,7 @@ def get_train_args(args: Optional[Union[dict[str, Any], list[str]]] = None) -> _ if finetuning_args.finetuning_type == "lora": # https://github.com/huggingface/transformers/blob/v4.50.0/src/transformers/trainer.py#L782 training_args.label_names = training_args.label_names or ["labels"] - + if "swanlab" in training_args.report_to and finetuning_args.use_swanlab: training_args.report_to.remove("swanlab") diff --git a/src/llamafactory/model/loader.py b/src/llamafactory/model/loader.py index 7c0ce2f4..7d7ee2e0 100644 --- a/src/llamafactory/model/loader.py +++ b/src/llamafactory/model/loader.py @@ -19,6 +19,7 @@ import torch from transformers import ( AutoConfig, AutoModelForCausalLM, + AutoModelForImageTextToText, AutoModelForSeq2SeqLM, AutoModelForTextToWaveform, AutoModelForVision2Seq, @@ -29,7 +30,6 @@ from trl import AutoModelForCausalLMWithValueHead from ..extras import logging from ..extras.misc import count_parameters, skip_check_imports, try_download_model_from_other_hub -from ..extras.packages import is_transformers_version_greater_than from .adapter import init_adapter from .model_utils.liger_kernel import apply_liger_kernel from .model_utils.misc import register_autoclass @@ -39,10 +39,6 @@ from .model_utils.valuehead import load_valuehead_params from .patcher import patch_config, patch_model, patch_processor, patch_tokenizer, patch_valuehead_model -if is_transformers_version_greater_than("4.46.0"): - from transformers import AutoModelForImageTextToText - - if TYPE_CHECKING: from transformers import PretrainedConfig, PreTrainedModel, PreTrainedTokenizer, ProcessorMixin @@ -162,10 +158,7 @@ def load_model( else: if type(config) in AutoModelForVision2Seq._model_mapping.keys(): # image-text load_class = AutoModelForVision2Seq - elif ( - is_transformers_version_greater_than("4.46.0") - and type(config) in AutoModelForImageTextToText._model_mapping.keys() - ): # image-text + elif type(config) in AutoModelForImageTextToText._model_mapping.keys(): # image-text load_class = AutoModelForImageTextToText elif type(config) in AutoModelForSeq2SeqLM._model_mapping.keys(): # audio-text load_class = AutoModelForSeq2SeqLM diff --git a/tests/data/test_mm_plugin.py b/tests/data/test_mm_plugin.py index 6910cbad..7f05e563 100644 --- a/tests/data/test_mm_plugin.py +++ b/tests/data/test_mm_plugin.py @@ -238,7 +238,6 @@ def test_llama4_plugin(): _check_plugin(**check_inputs) -@pytest.mark.skipif(not is_transformers_version_greater_than("4.47.0"), reason="Requires transformers>=4.47.0") def test_llava_plugin(): image_seqlen = 576 tokenizer_module = _load_tokenizer_module(model_name_or_path="llava-hf/llava-1.5-7b-hf")