Add custom kq scaling from Gemma2Attention

This commit is contained in:
Andrei Betlen 2024-06-29 10:17:33 -04:00
parent 6f2464e3dd
commit a89427908d
4 changed files with 12 additions and 1 deletions

View file

@ -2369,6 +2369,9 @@ class Gemma2Model(Model):
self.gguf_writer.add_final_logit_softcapping(
self.hparams["final_logit_softcapping"]
)
self.gguf_writer.add_query_pre_attn_scalar(
self.hparams["query_pre_attn_scalar"]
)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
del bid # unusem