vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : add pre-tokenizer regexes for BLOOM and gpt3-finnish (#8850)

Browse source
This commit is contained in:
Esko Toivonen 2024-08-15 10:17:12 +03:00 • committed by GitHub
parent d5492f0525
commit 6bda7ce6c3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 19 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

8
convert_hf_to_gguf.py
View file

@ -590,6 +590,12 @@ class Model:
if chkhsh == "855059429035d75a914d1eda9f10a876752e281a054a7a3d421ef0533e5b6249":
# ref: https://huggingface.co/HuggingFaceTB/SmolLM-135M
res = "smollm"
if chkhsh == "3c30d3ad1d6b64202cd222813e7736c2db6e1bd6d67197090fc1211fbc612ae7":
# ref: https://huggingface.co/bigscience/bloom
res = "bloom"
if chkhsh == "bc01ce58980e1db43859146dc51b1758b3b88729b217a74792e9f8d43e479d21":
# ref: https://huggingface.co/TurkuNLP/gpt3-finnish-small
res = "gpt3-finnish"
if res is None:
logger.warning("\n")
@ -893,7 +899,7 @@ class GPTNeoXModel(Model):
return tensors
@Model.register("BloomForCausalLM")
@Model.register("BloomForCausalLM", "BloomModel")
class BloomModel(Model):
model_arch = gguf.MODEL_ARCH.BLOOM