fix phi 3 conversion (#8262)
This commit is contained in:
parent
f8d6a23804
commit
916248af1f
1 changed files with 1 additions and 1 deletions
|
@ -1942,7 +1942,7 @@ class Phi3MiniModel(Model):
|
|||
if len(rope_scaling_type) == 0:
|
||||
raise KeyError('Missing the required key rope_scaling.type')
|
||||
|
||||
if rope_scaling_type == 'su':
|
||||
if rope_scaling_type == 'su' or rope_scaling_type == 'longrope':
|
||||
attn_factor = math.sqrt(1 + math.log(scale) / math.log(orig_max_pos_embds)) if scale > 1.0 else 1.0
|
||||
elif rope_scaling_type == 'yarn':
|
||||
attn_factor = 0.1 * math.log(scale) + 1.0 if scale > 1.0 else 1.0
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue