423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-04 04:32:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix bug in pretraining

Browse Source 
Former-commit-id: b3fbba57eb8be33a018b5904bdf08d1c95412005
This commit is contained in:
hiyouga 2023-09-28 00:45:20 +08:00
parent deb17942ab
commit d2ce9b879b
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/dsets/preprocess.py
View File

@ -22,7 +22,7 @@ def preprocess_dataset(
column_names = list(next(iter(dataset)).keys()) column_names = list(next(iter(dataset)).keys())
template = get_template_and_fix_tokenizer(data_args.template, tokenizer) template = get_template_and_fix_tokenizer(data_args.template, tokenizer)
if template.efficient_eos and data_args.sft_packing: if template is not None and template.efficient_eos and data_args.sft_packing:
raise ValueError("Current template is incompatible with packing.") raise ValueError("Current template is incompatible with packing.")
def construct_example(examples: Dict[str, List[Any]]) -> Generator[Any, None, None]: def construct_example(examples: Dict[str, List[Any]]) -> Generator[Any, None, None]: