423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-23 22:32:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

remove redundant code

Browse Source 
Former-commit-id: 53d95725c588a9858e699e3e591cb0d3c1441208
This commit is contained in:
hiyouga 2023-08-05 00:27:27 +08:00
parent ab95f569a4
commit 1a1caf2116
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/llmtuner/extras/template.py
View File

@ -25,8 +25,8 @@ class Template:
r""" r"""
Returns a single pair of token ids representing prompt and response respectively. Returns a single pair of token ids representing prompt and response respectively.
""" """
prefix, history = self._format(query=query, resp=resp, history=history, prefix=prefix) prefix, history = self._format(query, resp, history, prefix)
encoded_pairs = self._encode(tokenizer=tokenizer, prefix=prefix, history=history) encoded_pairs = self._encode(tokenizer, prefix, history)
prompt_ids = [] prompt_ids = []
for query_ids, resp_ids in encoded_pairs[:-1]: for query_ids, resp_ids in encoded_pairs[:-1]:
prompt_ids = prompt_ids + query_ids + resp_ids prompt_ids = prompt_ids + query_ids + resp_ids
@ -44,8 +44,8 @@ class Template:
r""" r"""
Returns multiple pairs of token ids representing prompts and responses respectively. Returns multiple pairs of token ids representing prompts and responses respectively.
""" """
prefix, history = self._format(query=query, resp=resp, history=history, prefix=prefix) prefix, history = self._format(query, resp, history, prefix)
encoded_pairs = self._encode(tokenizer=tokenizer, prefix=prefix, history=history) encoded_pairs = self._encode(tokenizer, prefix, history)
return encoded_pairs return encoded_pairs
def _format( def _format(