vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remove unused fileds to avoid unused filed build error

Browse source
This commit is contained in:
zhenweijin 2024-09-23 10:24:34 +08:00
parent 25d4599e19
commit 768c43f852
1 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

12
src/llama-vocab.cpp
View file

@ -201,8 +201,7 @@ struct llm_tokenizer_spm : llm_tokenizer {
};
struct llm_tokenizer_spm_session {
llm_tokenizer_spm_session(const llama_vocab & vocab) : vocab(vocab),
spm_tokenizer(static_cast<const llm_tokenizer_spm *>(vocab.tokenizer)) {}
llm_tokenizer_spm_session(const llama_vocab & vocab) : vocab(vocab) {}
void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
@ -320,7 +319,8 @@ private:
}
const llama_vocab & vocab;
const llm_tokenizer_spm * spm_tokenizer; // currently unused
// currently unused
// const llm_tokenizer_spm * spm_tokenizer;
std::vector<llm_symbol> symbols;
llm_bigram_spm::queue work_queue;
@ -668,8 +668,7 @@ struct llm_tokenizer_wpm : llm_tokenizer {
};
struct llm_tokenizer_wpm_session {
llm_tokenizer_wpm_session(const llama_vocab & vocab) : vocab(vocab),
wpm_tokenizer(static_cast<const llm_tokenizer_wpm *>(vocab.tokenizer)) {}
llm_tokenizer_wpm_session(const llama_vocab & vocab) : vocab(vocab) {}
void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
const auto & token_map = vocab.token_to_id;
@ -773,7 +772,8 @@ struct llm_tokenizer_wpm_session {
private:
const llama_vocab & vocab;
const llm_tokenizer_wpm * wpm_tokenizer;
// currently unused
// const llm_tokenizer_wpm * wpm_tokenizer;
};
//