Hotfix for ExLlamaV2
See:
https://github.com/oobabooga/text-generation-webui/issues/4002
and
ec5164b8a8
?diff=split
pull/39/head
parent
13c7bad5cd
commit
e704efd2fa
@ -0,0 +1,23 @@
|
||||
From ec5164b8a8e282b91aedb2af94dfeb89887656b7 Mon Sep 17 00:00:00 2001
|
||||
From: turboderp <11859846+turboderp@users.noreply.github.com>
|
||||
Date: Tue, 19 Sep 2023 11:56:52 +0200
|
||||
Subject: [PATCH] Fix for non-numerical version number in flash-attn
|
||||
|
||||
---
|
||||
exllamav2/attn.py | 2 +-
|
||||
1 file changed, 1 insertion(+), 1 deletion(-)
|
||||
|
||||
diff --git a/exllamav2/attn.py b/exllamav2/attn.py
|
||||
index b8b1752..09ef6d6 100644
|
||||
--- a/exllamav2/attn.py
|
||||
+++ b/exllamav2/attn.py
|
||||
@@ -16,7 +16,7 @@
|
||||
has_flash_attn = False
|
||||
try:
|
||||
import flash_attn
|
||||
- flash_attn_ver = [int(t) for t in flash_attn.__version__.split(".")]
|
||||
+ flash_attn_ver = [int(t) for t in flash_attn.__version__.split(".") if t.isdigit()]
|
||||
if flash_attn_ver >= [2, 2, 1]:
|
||||
from flash_attn import flash_attn_func
|
||||
has_flash_attn = True
|
||||
|
@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -x
|
||||
|
||||
# Get current directory
|
||||
cur_dir=$(pwd)
|
||||
exlv2_dir="/venv/lib/python3.10/site-packages/exllamav2/"
|
||||
|
||||
# Go to exl directory
|
||||
cd $exlv2_dir
|
||||
|
||||
# Apply patch
|
||||
patch -i /scripts/exllama_version_fix.patch
|
||||
|
||||
# Go back to the original directory
|
||||
cd $cur_dir
|
Loading…
Reference in New Issue