refactor dataset_attr, add eos in pt, fix #757

This commit is contained in:
hiyouga
2023-09-01 19:00:45 +08:00
parent cf106e9d65
commit a9d1fb72f7
20 changed files with 160 additions and 198 deletions

View File

@@ -3,7 +3,7 @@ transformers>=4.29.1
datasets>=2.12.0
accelerate>=0.21.0
peft>=0.4.0
trl>=0.5.0
trl>=0.7.1
scipy
sentencepiece
tiktoken