423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-20 13:50:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

imporve log

Browse Source
This commit is contained in:
hiyouga
2025-01-08 09:56:10 +00:00
parent d23a98825b
commit 47e17dd689
16 changed files with 78 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/llamafactory/data/data_utils.py
View File

@@ -56,12 +56,12 @@ def merge_dataset(
return all_datasets[0]
elif data_args.mix_strategy == "concat":
if data_args.streaming:
logger.warning_once("The samples between different datasets will not be mixed in streaming mode.")
logger.warning_rank0_once("The samples between different datasets will not be mixed in streaming mode.")
return concatenate_datasets(all_datasets)
elif data_args.mix_strategy.startswith("interleave"):
if not data_args.streaming:
logger.warning_once("We recommend using `mix_strategy=concat` in non-streaming mode.")
logger.warning_rank0_once("We recommend using `mix_strategy=concat` in non-streaming mode.")
return interleave_datasets(
datasets=all_datasets,