From ddce116ec96ddb8d1e4491fcaa78b93abdbd6a0f Mon Sep 17 00:00:00 2001
From: kalomaze <66376113+kalomaze@users.noreply.github.com>
Date: Thu, 19 Oct 2023 10:20:44 -0500
Subject: [PATCH] Fix for Top K disabling (#480)

* Update gpttype_adapter.cpp

* use n_vocab instead of 32000 for when top k is off
---
 gpttype_adapter.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp
index c6e3846c6..1e50f516a 100644
--- a/gpttype_adapter.cpp
+++ b/gpttype_adapter.cpp
@@ -1312,7 +1312,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
     }
     if (params.top_k < 1)
     {
-        params.top_k = 120; //to disable top_k we actually need to increase this value to a very high number
+        params.top_k = n_vocab; // all tokens in the vocabulary should be considered if top k is disabled
     }
     if (params.seed <= 0 || params.seed==0xFFFFFFFF)
     {