Merge pull request #33 from NexaAI/weili/dev

fix clip allocation size error for 81-series vlm models
This commit is contained in:
Zack Li 2024-12-02 23:11:24 -08:00 committed by GitHub
commit b2958b33dd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -1927,16 +1927,20 @@ int clip_n_patches(const struct clip_ctx * ctx) {
int n_patches = (params.image_size / params.patch_size) * (params.image_size / params.patch_size);
if (ctx->proj_type == PROJECTOR_TYPE_LDP || ctx->proj_type == PROJECTOR_TYPE_LDPV2) {
n_patches /= 4;
} else if (ctx->proj_type == PROJECTOR_TYPE_RESAMPLER) {
if (ctx->minicpmv_version == 2) {
n_patches = 96;
}
else if (ctx->minicpmv_version == 3) {
n_patches = 64;
}
if(ctx->omni_vlm_ver_type == omni_vlm_version_type::VLM_81_OCR
|| ctx->omni_vlm_ver_type == omni_vlm_version_type::VLM_81_INSTRUCT) {
n_patches /= 9;
}
// if (ctx->proj_type == PROJECTOR_TYPE_LDP || ctx->proj_type == PROJECTOR_TYPE_LDPV2) {
// n_patches /= 4;
// } else if (ctx->proj_type == PROJECTOR_TYPE_RESAMPLER) {
// if (ctx->minicpmv_version == 2) {
// n_patches = 96;
// }
// else if (ctx->minicpmv_version == 3) {
// n_patches = 64;
// }
// }
return n_patches;
}