diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index b55971454..6ba6f39f3 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -104,7 +104,15 @@ Feature: llama.cpp server Then tokens begin with BOS Given first token is removed Then tokens can be detokenized - + + Scenario: Tokenize with pieces + When tokenizing with pieces: + """ + What is the capital of Germany? + 媽 + """ + Then tokens are given with pieces + Scenario: Models available Given available models Then 1 models are supported diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index fec6bcae5..da24f47d9 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -702,7 +702,7 @@ async def step_tokenize_with_pieces(context): context.tokens_with_pieces = tokenize_json["tokens"] -@step("tokens with pieces are complete") +@step("tokens are given with pieces") @async_run_until_complete async def step_tokenize_with_pieces(context): # Verify that the response contains both token IDs and pieces