mirror of
https://github.com/hiyouga/LLaMA-Factory.git
synced 2025-12-16 20:00:36 +08:00
feat: add batch size of map function in the preprocessed dataset
This commit is contained in:
@@ -109,6 +109,10 @@ class DataArguments:
|
||||
default=None,
|
||||
metadata={"help": "Path to save or load the tokenized datasets."},
|
||||
)
|
||||
dataset_map_batch_size: Optional[int] = field(
|
||||
default=None,
|
||||
metadata={"help": "Batch size for dataset mapping."},
|
||||
)
|
||||
|
||||
def __post_init__(self):
|
||||
def split_arg(arg):
|
||||
|
||||
Reference in New Issue
Block a user