423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-14 10:56:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

1. add custom eval dataset support

Browse Source 
2. merge load dataset and split dataset function


Former-commit-id: 76f3bbcfc0
This commit is contained in:
codingma
2024-07-05 15:52:10 +08:00
parent 8379a39776
commit 74f0d02eb8
16 changed files with 104 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/data/test_supervised.py
View File

@@ -47,7 +47,7 @@ def test_supervised(num_samples: int):
model_args, data_args, training_args, _, _ = get_train_args(TRAIN_ARGS)
tokenizer_module = load_tokenizer(model_args)
tokenizer = tokenizer_module["tokenizer"]
tokenized_data = get_dataset(model_args, data_args, training_args, stage="sft", **tokenizer_module)
dataset_module = get_dataset(model_args, data_args, training_args, stage="sft", **tokenizer_module)
ref_tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA)
@@ -63,5 +63,5 @@ def test_supervised(num_samples: int):
{"role": "assistant", "content": original_data[index]["output"]},
]
templated_result = ref_tokenizer.apply_chat_template(messages, tokenize=False)
decoded_result = tokenizer.decode(tokenized_data["input_ids"][index])
decoded_result = tokenizer.decode(dataset_module["train_dataset"]["input_ids"][index])
assert templated_result == decoded_result