From ce89d8a49cb69e77cb02dd38c202a74e0287fe8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= <sigbjorn.skjaeret@scala.com>
Date: Wed, 19 Jun 2024 21:57:36 +0200
Subject: [PATCH] add --spm-infill option

---
 common/common.cpp | 6 ++++++
 common/common.h   | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/common/common.cpp b/common/common.cpp
index 73ff0e85b..7564b7080 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1285,6 +1285,10 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
         params.input_suffix = argv[i];
         return true;
     }
+    if (arg == "--spm-infill") {
+        params.spm_infill = true;
+        return true;
+    }
     if (arg == "--grammar") {
         if (++i >= argc) {
             invalid_param = true;
@@ -1781,6 +1785,8 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param
     options.push_back({ "main infill", "       --in-prefix-bos",        "prefix BOS to user inputs, preceding the `--in-prefix` string" });
     options.push_back({ "main infill", "       --in-prefix STRING",     "string to prefix user inputs with (default: empty)" });
     options.push_back({ "main infill", "       --in-suffix STRING",     "string to suffix after user inputs with (default: empty)" });
+    options.push_back({ "server infill",
+                                       "       --spm-infill",           "use Suffix/Prefix/Middle pattern for infill (instead of Prefix/Suffix/Middle) as some models prefer this. (default: %s)", params.spm_infill ? "enabled" : "disabled" });
 
     options.push_back({ "sampling" });
     options.push_back({ "*",           "       --samplers SAMPLERS",    "samplers that will be used for generation in the order, separated by \';\'\n"
diff --git a/common/common.h b/common/common.h
index 9a1dc4a2f..c8d23a383 100644
--- a/common/common.h
+++ b/common/common.h
@@ -240,6 +240,8 @@ struct gpt_params {
     std::string cvector_completions_file = "examples/cvector-generator/completions.txt";
     std::string cvector_positive_file    = "examples/cvector-generator/positive.txt";
     std::string cvector_negative_file    = "examples/cvector-generator/negative.txt";
+
+    bool spm_infill = false; // suffix/prefix/middle pattern for infill
 };
 
 void gpt_params_handle_model_default(gpt_params & params);