vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: Apply fix to generate-vocab.sh script

Browse source 
Signed-off-by: teleprint-me <77757836+teleprint-me@users.noreply.github.com>
This commit is contained in:
teleprint-me 2024-05-08 13:48:22 -04:00
parent 69efb59853
commit 906c3f73d5
No known key found for this signature in database
GPG key ID: B0D11345E65C4D48
1 changed files with 18 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

36
generate-vocab.sh
View file

@ -1,21 +1,21 @@
#!/usr/bin/env bash
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-llama-spm.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-llama-bpe.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-phi-3.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-deepseek-llm.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-deepseek-coder.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-falcon.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-bert-bge.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-mpt.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-starcoder.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-gpt-2.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-phi.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-stablelm.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-qwen.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-mistral-bpe.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-mistral-spm.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-mixtral-bpe.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-mixtral-spm.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-refact.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/llama-spm/ --outfile models/ggml-vocab-llama-spm.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/llama-bpe/ --outfile models/ggml-vocab-llama-bpe.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/phi-3/ --outfile models/ggml-vocab-phi-3.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/deepseek-llm/ --outfile models/ggml-vocab-deepseek-llm.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/deepseek-coder/ --outfile models/ggml-vocab-deepseek-coder.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/falcon/ --outfile models/ggml-vocab-falcon.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/bert-bge/ --outfile models/ggml-vocab-bert-bge.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/mpt/ --outfile models/ggml-vocab-mpt.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/starcoder/ --outfile models/ggml-vocab-starcoder.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/gpt-2/ --outfile models/ggml-vocab-gpt-2.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/phi/ --outfile models/ggml-vocab-phi.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/stablelm/ --outfile models/ggml-vocab-stablelm.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/qwen/ --outfile models/ggml-vocab-qwen.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/mistral-bpe/ --outfile models/ggml-vocab-mistral-bpe.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/mistral-spm/ --outfile models/ggml-vocab-mistral-spm.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/mixtral-bpe/ --outfile models/ggml-vocab-mixtral-bpe.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/mixtral-spm/ --outfile models/ggml-vocab-mixtral-spm.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/refact/ --outfile models/ggml-vocab-refact.gguf --vocab-only
python3 convert-hf-to-gguf.py models/tokenizers/command-r/ --outfile models/ggml-vocab-command-r.gguf --vocab-only