Fix review remarks.
This commit is contained in:
parent
39c9a3b553
commit
fe7508c408
3 changed files with 1 additions and 4 deletions
|
@ -1866,9 +1866,6 @@ struct llama_tokenizer {
|
|||
while (offs < text.size()) {
|
||||
llama_sp_symbol sym;
|
||||
size_t len = utf8_len(text[offs]);
|
||||
if (len == 0) {
|
||||
len = utf8_len(text[offs]);
|
||||
}
|
||||
sym.text = text.c_str() + offs;
|
||||
sym.n = len;
|
||||
offs += len;
|
||||
|
|
|
@ -4,6 +4,7 @@
|
|||
#include "llama.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <string>
|
||||
|
||||
static std::string llama_token_to_str(
|
||||
const struct llama_context * ctx,
|
||||
|
|
1
llama.h
1
llama.h
|
@ -11,7 +11,6 @@
|
|||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <string>
|
||||
|
||||
#ifdef LLAMA_SHARED
|
||||
# if defined(_WIN32) && !defined(__MINGW32__)
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue