diff --git a/src/llmtuner/extras/template.py b/src/llmtuner/extras/template.py
index dd88782d..d94d5b37 100644
--- a/src/llmtuner/extras/template.py
+++ b/src/llmtuner/extras/template.py
@@ -101,6 +101,9 @@ class Template:
                 prefix_ids = sep_ids
             elif prefix:
                 prefix_ids = self._convert_inputs_to_ids(tokenizer, context=prefix) + eos_ids + sep_ids
+            else:
+                prefix_ids = []
+
             query_ids = self._convert_inputs_to_ids(tokenizer, context=self.prompt, query=query)
             resp_ids = self._convert_inputs_to_ids(tokenizer, context=[resp])
             encoded_pairs.append((bos_ids + prefix_ids + query_ids, resp_ids + eos_ids))