From d6d905b24270e97e489ac14f6a416cbe710bf1e3 Mon Sep 17 00:00:00 2001
From: Nikolay Borisov <nik.borisov@suse.com>
Date: Mon, 6 Nov 2023 13:01:00 +0200
Subject: [PATCH] convert: Fix detection of LLAMA2

In recent downloads of LLAMA2 dataset the norm_eps is set to 1e-06, this
leads to convert.py erroneously considering the model to be LLAMA1 and
setting the context to 2k tokens.

Fix it by extending the existing hack to also check for the 1e-06 value.
---
 convert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/convert.py b/convert.py
index 9110f1580..d88dd096d 100755
--- a/convert.py
+++ b/convert.py
@@ -250,7 +250,7 @@ class Params:
         if config.get("rope_theta") == 1000000:
             # CodeLlama
             n_ctx = 16384
-        elif config["norm_eps"] == 1e-05:
+        elif config["norm_eps"] in (1e-05, 1e-06):
             # LLaMA v2
             n_ctx = 4096
         else: