423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-09-12 16:12:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

tiny fix

Browse Source 
Former-commit-id: 76f2e5950483c669a15a961f0554442b6eb5c4a6
This commit is contained in:
hiyouga 2024-09-05 23:41:16 +08:00
parent 52d3c42265
commit 995491594d
5 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/llamafactory/__init__.py
View File

@ -28,9 +28,9 @@ Dependency graph:
attention: attention:
transformers>=4.42.4 (gemma+fa2) transformers>=4.42.4 (gemma+fa2)
longlora: longlora:
transformers>=4.41.2,<=4.44.3 transformers>=4.41.2,<=4.45.0
packing: packing:
transformers>=4.41.2,<=4.44.3 transformers>=4.41.2,<=4.45.0
Disable version checking: DISABLE_VERSION_CHECK=1 Disable version checking: DISABLE_VERSION_CHECK=1
Enable VRAM recording: RECORD_VRAM=1 Enable VRAM recording: RECORD_VRAM=1

4
src/llamafactory/data/collator.py
View File

@ -81,8 +81,8 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
def __call__(self, features: Sequence[Dict[str, Any]]) -> Dict[str, "torch.Tensor"]: def __call__(self, features: Sequence[Dict[str, Any]]) -> Dict[str, "torch.Tensor"]:
batch_images, batch_videos, batch_imglens, batch_vidlens, batch_seqlens = [], [], [], [], [] batch_images, batch_videos, batch_imglens, batch_vidlens, batch_seqlens = [], [], [], [], []
for feature in features: for feature in features:
images = feature.pop("images") or [] # avoid NoneType images = feature.pop("images", None) or []
videos = feature.pop("videos") or [] videos = feature.pop("videos", None) or []
batch_images.extend(images) batch_images.extend(images)
batch_videos.extend(videos) batch_videos.extend(videos)
batch_imglens.append(len(images)) batch_imglens.append(len(images))

2
src/llamafactory/data/processors/pretrain.py
View File

@ -36,7 +36,7 @@ def preprocess_pretrain_dataset(
if data_args.template == "gemma": if data_args.template == "gemma":
text_examples = [tokenizer.bos_token + example for example in text_examples] text_examples = [tokenizer.bos_token + example for example in text_examples]
result = tokenizer(text_examples, add_special_tokens=False, max_length=data_args.cutoff_len, truncation=True) result = tokenizer(text_examples, add_special_tokens=False, truncation=True, max_length=data_args.cutoff_len)
else: else:
tokenized_examples = tokenizer(text_examples, add_special_tokens=False) tokenized_examples = tokenizer(text_examples, add_special_tokens=False)
concatenated_examples = {k: list(chain(*tokenized_examples[k])) for k in tokenized_examples.keys()} concatenated_examples = {k: list(chain(*tokenized_examples[k])) for k in tokenized_examples.keys()}

2
src/llamafactory/model/model_utils/longlora.py
View File

@ -353,7 +353,7 @@ def llama_sdpa_attention_forward(
def _apply_llama_patch() -> None: def _apply_llama_patch() -> None:
require_version("transformers>=4.41.2,<=4.44.3", "To fix: pip install transformers>=4.41.2,<=4.44.3") require_version("transformers>=4.41.2,<=4.45.0", "To fix: pip install transformers>=4.41.2,<=4.45.0")
LlamaAttention.forward = llama_attention_forward LlamaAttention.forward = llama_attention_forward
LlamaFlashAttention2.forward = llama_flash_attention_2_forward LlamaFlashAttention2.forward = llama_flash_attention_2_forward
LlamaSdpaAttention.forward = llama_sdpa_attention_forward LlamaSdpaAttention.forward = llama_sdpa_attention_forward

2
src/llamafactory/model/model_utils/packing.py
View File

@ -114,7 +114,7 @@ def get_unpad_data(attention_mask: "torch.Tensor") -> Tuple["torch.Tensor", "tor
def _patch_for_block_diag_attn(model_type: str) -> None: def _patch_for_block_diag_attn(model_type: str) -> None:
require_version("transformers>=4.41.2,<=4.44.3", "To fix: pip install transformers>=4.41.2,<=4.44.3") require_version("transformers>=4.41.2,<=4.45.0", "To fix: pip install transformers>=4.41.2,<=4.45.0")
if is_transformers_version_greater_than_4_43(): if is_transformers_version_greater_than_4_43():
import transformers.modeling_flash_attention_utils import transformers.modeling_flash_attention_utils