From 198daa4e3434958a56473a23c7d72906bbf7d352 Mon Sep 17 00:00:00 2001 From: Mathijs Henquet Date: Thu, 22 Aug 2024 00:04:21 +0200 Subject: [PATCH] server : Add tokenize with pieces tests to server.feature --- examples/server/tests/features/server.feature | 10 +++++++++- examples/server/tests/features/steps/steps.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index b55971454..6ba6f39f3 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -104,7 +104,15 @@ Feature: llama.cpp server Then tokens begin with BOS Given first token is removed Then tokens can be detokenized - + + Scenario: Tokenize with pieces + When tokenizing with pieces: + """ + What is the capital of Germany? + 媽 + """ + Then tokens are given with pieces + Scenario: Models available Given available models Then 1 models are supported diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index fec6bcae5..da24f47d9 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -702,7 +702,7 @@ async def step_tokenize_with_pieces(context): context.tokens_with_pieces = tokenize_json["tokens"] -@step("tokens with pieces are complete") +@step("tokens are given with pieces") @async_run_until_complete async def step_tokenize_with_pieces(context): # Verify that the response contains both token IDs and pieces