diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index d178ca0f7..1a2e07f5f 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -690,7 +690,9 @@ struct server_context {
         n_ctx = llama_n_ctx(ctx);
 
         add_bos_token = llama_should_add_bos_token(model);
-        GGML_ASSERT(llama_add_eos_token(model) != 1);
+        if (!llama_model_has_encoder(model)) {
+            GGML_ASSERT(llama_add_eos_token(model) != 1);
+        }
 
         return true;
     }