This commit is contained in:
Eddie-Wang1120 2024-06-10 00:00:52 +08:00
parent 97d22be58c
commit 344467f2b8
2 changed files with 6 additions and 2 deletions

View file

@ -1417,7 +1417,7 @@ class BitnetModel(Model):
def weight_quant(self, weight):
dtype = weight.dtype
weight = weight.float()
s = 1 / weight.abs().mean().clamp(min=1e-5)
s = 1 / weight.abs().mean().clamp(min=1e-5)
result = (weight * s).round().clamp(-1, 1) / s
return result.type(dtype)