423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-18 04:40:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

support mllm hf inference

Browse Source
This commit is contained in:
hiyouga
2024-04-26 05:34:58 +08:00
parent c20f750d11
commit e057c8de48
27 changed files with 130 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/llmtuner/data/preprocess.py
View File

@@ -8,7 +8,7 @@ from .utils import Role
if TYPE_CHECKING:
from PIL import Image
from PIL.Image import Image
from transformers import ProcessorMixin, Seq2SeqTrainingArguments
from transformers.image_processing_utils import BaseImageProcessor
from transformers.tokenization_utils import PreTrainedTokenizer
@@ -271,7 +271,11 @@ def get_preprocess_and_print_func(
processor: Optional["ProcessorMixin"],
) -> Tuple[Callable, Callable]:
if stage == "pt":
preprocess_func = partial(preprocess_pretrain_dataset, tokenizer=tokenizer, data_args=data_args)
preprocess_func = partial(
preprocess_pretrain_dataset,
tokenizer=tokenizer,
data_args=data_args,
)
print_function = partial(print_unsupervised_dataset_example, tokenizer=tokenizer)
elif stage == "sft" and not training_args.predict_with_generate:
if data_args.packing: