cahce

9 months ago · be7fb52cf7
parent 3fec014d0a
commit be7fb52cf7
1 changed files with 1 additions and 0 deletions
--- a/programming_runs/generators/model.py
+++ b/programming_runs/generators/model.py
@ -132,6 +132,7 @@ class HFModelBase(ModelBase):
            prompt,
            max_new_tokens=min(
                max_tokens, self.model.config.max_position_embeddings),
+            use_cache=True,
            do_sample=True,
            temperature=temperature,
            top_p=0.95,