423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-15 08:08:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

tiny fix

Browse Source 
Former-commit-id: 935703b46d2871ce1014832da067dfe4a50c0610
This commit is contained in:
hiyouga 2024-07-04 03:02:23 +08:00
parent 5bc0b9b31c
commit 0517d7bee5
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/llamafactory/data/preprocess.py
View File

@ -15,8 +15,6 @@
from functools import partial from functools import partial
from typing import TYPE_CHECKING, Callable, Literal, Optional, Tuple from typing import TYPE_CHECKING, Callable, Literal, Optional, Tuple
from datasets.arrow_writer import OptimizedTypedSequence, TypedSequence
from .processors.feedback import preprocess_feedback_dataset from .processors.feedback import preprocess_feedback_dataset
from .processors.pairwise import preprocess_pairwise_dataset, print_pairwise_dataset_example from .processors.pairwise import preprocess_pairwise_dataset, print_pairwise_dataset_example
from .processors.pretrain import preprocess_pretrain_dataset from .processors.pretrain import preprocess_pretrain_dataset
@ -53,6 +51,7 @@ def get_preprocess_and_print_func(
elif stage == "sft" and not training_args.predict_with_generate: elif stage == "sft" and not training_args.predict_with_generate:
if data_args.packing: if data_args.packing:
if data_args.neat_packing: if data_args.neat_packing:
from datasets.arrow_writer import OptimizedTypedSequence, TypedSequence
def __init__(self, data, **kwargs): def __init__(self, data, **kwargs):
return TypedSequence.__init__( return TypedSequence.__init__(