vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

move llava into its own subdir

Browse source
This commit is contained in:
Damian Stewart 2023-10-14 17:11:32 +02:00
parent f8eddcf8e8
commit b9f533b997
11 changed files with 36 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -44,7 +44,7 @@ models-mnt
/infill
/libllama.so
/llama-bench
/llava
/llava-cli
/main
/metal
/perplexity

1
CMakeLists.txt
View file

@ -793,6 +793,7 @@ endif()
#
add_subdirectory(common)
add_subdirectory(llava)
if (LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)
include(CTest)

1
common/CMakeLists.txt
View file

@ -3,6 +3,7 @@
set(TARGET common)
add_library(${TARGET} OBJECT
base64.hpp
common.h
common.cpp
sampling.h

16
examples/llava/CMakeLists.txt
View file

@ -1,16 +1,3 @@
set(TARGET llava)
add_library(${TARGET} llava.cpp llava.h clip.cpp clip.h)
install(TARGETS ${TARGET} LIBRARY)
target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
target_compile_features(${TARGET} PRIVATE cxx_std_11)
if (NOT MSVC)
target_compile_options(${TARGET} PRIVATE -Wno-cast-qual) # stb_image.h
endif()
if(TARGET BUILD_INFO)
add_dependencies(${TARGET} BUILD_INFO)
endif()
set(TARGET llava-cli)
add_executable(${TARGET} llava-cli.cpp)
install(TARGETS ${TARGET} RUNTIME)
@ -19,6 +6,3 @@ target_compile_features(${TARGET} PRIVATE cxx_std_11)
if(TARGET BUILD_INFO)
add_dependencies(${TARGET} BUILD_INFO)
endif()
unset(TARGET)
llama_build_and_test_executable(test-llava.cpp)

6
examples/llava/llava-cli.cpp
View file

@ -32,7 +32,11 @@ static bool load_image(llava_context * ctx_llava, gpt_params * params, float **i
return false;
}
}
llava_build_img_embed(ctx_llava, params->n_threads, &img, image_embd, n_image_pos);
bool image_embed_result = llava_build_img_embed(ctx_llava->ctx_llama, ctx_llava->ctx_clip, params->n_threads, &img, image_embd, n_image_pos);
if (!image_embed_result) {
fprintf(stderr, "%s: coulnd't embed the image\n", __func__);
return false;
}
return true;
}

13
llava/CMakeLists.txt Normal file
View file

@ -0,0 +1,13 @@
set(TARGET llava)
add_library(${TARGET} llava.cpp llava.h clip.cpp clip.h)
target_include_directories(${TARGET} PUBLIC .)
target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
target_compile_features(${TARGET} PRIVATE cxx_std_11)
if (NOT MSVC)
target_compile_options(${TARGET} PRIVATE -Wno-cast-qual) # stb_image.h
endif()
if(TARGET BUILD_INFO)
add_dependencies(${TARGET} BUILD_INFO)
endif()

0
examples/llava/clip.cpp → llava/clip.cpp
View file

0
examples/llava/clip.h → llava/clip.h
View file

0
examples/llava/llava-utils.h → llava/llava-utils.h
View file

28
examples/llava/llava.cpp → llava/llava.cpp
View file

@ -10,8 +10,7 @@
#include "base64.hpp"
static bool encode_image_with_clip(llava_context * ctx_llava, int n_threads, const clip_image_u8 * img, float * image_embd, int * n_img_embd, int * n_img_pos) {
auto ctx_clip = ctx_llava->ctx_clip;
static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float * image_embd, int * n_img_embd, int * n_img_pos) {
clip_image_f32 img_res;
if (!clip_image_preprocess(ctx_clip, img, &img_res, /*pad2square =*/ true)) {
fprintf(stderr, "%s: unable to preprocess image\n", __func__);
@ -22,14 +21,6 @@ static bool encode_image_with_clip(llava_context * ctx_llava, int n_threads, con
*n_img_pos = clip_n_patches(ctx_clip);
*n_img_embd = clip_n_mmproj_embd(ctx_clip);
// make sure that the correct mmproj was used, i.e., compare apples to apples
int n_llama_embd = llama_n_embd(llama_get_model(ctx_llava->ctx_llama));
if (*n_img_embd != n_llama_embd) {
printf("%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, *n_img_embd, n_llama_embd);
return false;
}
const int64_t t_img_enc_start_us = ggml_time_us();
if (!clip_image_encode(ctx_clip, n_threads, &img_res, image_embd)) {
fprintf(stderr, "Unable to encode image\n");
@ -46,9 +37,8 @@ static bool encode_image_with_clip(llava_context * ctx_llava, int n_threads, con
return true;
}
bool llava_build_img_embed(struct llava_context * ctx_llava, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_image_pos_out) {
bool llava_build_img_embed(const llama_context * ctx_llama, clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_image_pos_out) {
auto ctx_clip = ctx_llava->ctx_clip;
float * image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip));
if (!image_embd) {
fprintf(stderr, "Unable to allocate memory for image embeddings\n");
@ -58,13 +48,22 @@ bool llava_build_img_embed(struct llava_context * ctx_llava, int n_threads, cons
int n_image_pos;
int n_img_embd;
if (!encode_image_with_clip(ctx_llava, n_threads, img, image_embd, &n_img_embd, &n_image_pos)) {
if (!encode_image_with_clip(ctx_clip, n_threads, img, image_embd, &n_img_embd, &n_image_pos)) {
fprintf(stderr, "%s: cannot encode image, aborting\n", __func__);
free(image_embd);
return false;
}
// make sure that the correct mmproj was used, i.e., compare apples to apples
int n_llama_embd = llama_n_embd(llama_get_model(ctx_llama));
if (n_img_embd != n_llama_embd) {
printf("%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, n_img_embd, n_llama_embd);
free(image_embd);
return false;
}
*image_embd_out = image_embd;
*n_image_pos_out = n_image_pos;
return true;
}
@ -102,16 +101,15 @@ struct llava_context * llava_init(gpt_params * params) {
return NULL;
}
auto ctx_llava = (struct llava_context *)malloc(sizeof(llava_context));
ctx_llava->ctx_llama = ctx_llama;
ctx_llava->ctx_clip = ctx_clip;
ctx_llava->model = model;
return ctx_llava;
}
void llava_free(struct llava_context * ctx_llava) {
if (ctx_llava->ctx_clip) {
clip_free(ctx_llava->ctx_clip);

3
examples/llava/llava.h → llava/llava.h
View file

@ -19,7 +19,8 @@ struct llava_context {
struct llava_context * llava_init(gpt_params * params);
void llava_free(struct llava_context * ctx_llava);
bool llava_build_img_embed(struct llava_context * ctx_llava, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_image_pos_out);
/** build a llava image embedding from the passed-in clip image `img`. result is returned as image_embd_out, size n_image_pos_out */
bool llava_build_img_embed(const struct llama_context * ctx_llama, struct clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_image_pos_out);
#ifdef __cplusplus