vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

clang-format the llama.* files

Browse source
This commit is contained in:
Will Findley 2023-12-07 17:07:29 -05:00
parent bcc0eb4591
commit 92887c4684
2 changed files with 10010 additions and 9127 deletions
Download patch file Download diff file Expand all files Collapse all files

4256
llama.cpp
View file

File diff suppressed because it is too large Load diff

509
llama.h
View file

@ -8,10 +8,10 @@
#else #else
#define LLAMA_MAX_DEVICES 1 #define LLAMA_MAX_DEVICES 1
#endif // GGML_USE_CUBLAS #endif // GGML_USE_CUBLAS
#include <stdbool.h>
#include <stddef.h> #include <stddef.h>
#include <stdint.h> #include <stdint.h>
#include <stdio.h> #include <stdio.h>
#include <stdbool.h>
#ifdef LLAMA_SHARED #ifdef LLAMA_SHARED
#if defined(_WIN32) && !defined(__MINGW32__) #if defined(_WIN32) && !defined(__MINGW32__)
@ -44,8 +44,10 @@
#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN #define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN
#define LLAMA_SESSION_VERSION 3 #define LLAMA_SESSION_VERSION 3
#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) || defined(GGML_USE_METAL) #if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) || \
// Defined when llama.cpp is compiled with support for offloading model layers to GPU. defined(GGML_USE_METAL)
// Defined when llama.cpp is compiled with support for offloading model layers
// to GPU.
#define LLAMA_SUPPORTS_GPU_OFFLOAD #define LLAMA_SUPPORTS_GPU_OFFLOAD
#endif #endif
@ -87,7 +89,8 @@ extern "C" {
LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors
LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors
LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors
LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 =
4, // tok_embeddings.weight and output.weight are F16
// LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed
// LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed
LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors
@ -133,7 +136,8 @@ extern "C" {
// The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens
// //
// - token : the token ids of the input (used when embd is NULL) // - token : the token ids of the input (used when embd is NULL)
// - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL) // - embd : token embeddings (i.e. float vector of size n_embd) (used when
// token is NULL)
// - pos : the positions of the respective token in the sequence // - pos : the positions of the respective token in the sequence
// - seq_id : the sequence to which the respective token belongs // - seq_id : the sequence to which the respective token belongs
// - logits : if zero, the logits for the respective token will not be output // - logits : if zero, the logits for the respective token will not be output
@ -149,7 +153,8 @@ extern "C" {
int8_t *logits; int8_t *logits;
// NOTE: helpers for smooth API transition - can be deprecated in the future // NOTE: helpers for smooth API transition - can be deprecated in the future
// for future-proof code, use the above fields instead and ignore everything below // for future-proof code, use the above fields instead and ignore
// everything below
// //
// pos[i] = all_pos_0 + i*all_pos_1 // pos[i] = all_pos_0 + i*all_pos_1
// //
@ -177,7 +182,8 @@ extern "C" {
struct llama_model_params { struct llama_model_params {
int32_t n_gpu_layers; // number of layers to store in VRAM int32_t n_gpu_layers; // number of layers to store in VRAM
int32_t main_gpu; // the GPU that is used for scratch and small tensors int32_t main_gpu; // the GPU that is used for scratch and small tensors
const float * tensor_split; // how to split layers across multiple GPUs (size: LLAMA_MAX_DEVICES) const float *tensor_split; // how to split layers across multiple GPUs (size:
// LLAMA_MAX_DEVICES)
// called with a progress value between 0 and 1, pass NULL to disable // called with a progress value between 0 and 1, pass NULL to disable
llama_progress_callback progress_callback; llama_progress_callback progress_callback;
@ -200,7 +206,8 @@ extern "C" {
uint32_t n_batch; // prompt processing maximum batch size uint32_t n_batch; // prompt processing maximum batch size
uint32_t n_threads; // number of threads to use for generation uint32_t n_threads; // number of threads to use for generation
uint32_t n_threads_batch; // number of threads to use for batch processing uint32_t n_threads_batch; // number of threads to use for batch processing
int8_t rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` int8_t rope_scaling_type; // RoPE scaling type, from `enum
// llama_rope_scaling_type`
// ref: https://github.com/ggerganov/llama.cpp/pull/2054 // ref: https://github.com/ggerganov/llama.cpp/pull/2054
float rope_freq_base; // RoPE base frequency, 0 = from model float rope_freq_base; // RoPE base frequency, 0 = from model
@ -215,20 +222,26 @@ extern "C" {
enum ggml_type type_v; // data type for V cache enum ggml_type type_v; // data type for V cache
// Keep the booleans together to avoid misalignment during copy-by-value. // Keep the booleans together to avoid misalignment during copy-by-value.
bool mul_mat_q; // if true, use experimental mul_mat_q kernels (DEPRECATED - always true) bool mul_mat_q; // if true, use experimental mul_mat_q kernels (DEPRECATED -
bool logits_all; // the llama_eval() call computes all logits, not just the last one // always true)
bool logits_all; // the llama_eval() call computes all logits, not just the
// last one
bool embedding; // embedding mode only bool embedding; // embedding mode only
bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU bool offload_kqv; // whether to offload the KQV ops (including the KV cache)
// to GPU
}; };
// model quantization parameters // model quantization parameters
typedef struct llama_model_quantize_params { typedef struct llama_model_quantize_params {
int nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() int nthread; // number of threads to use for quantizing, if <=0 will use
// std::thread::hardware_concurrency()
enum llama_ftype ftype; // quantize to this llama_ftype enum llama_ftype ftype; // quantize to this llama_ftype
bool allow_requantize; // allow quantizing non-f32/f16 tensors bool allow_requantize; // allow quantizing non-f32/f16 tensors
bool quantize_output_tensor; // quantize output.weight bool quantize_output_tensor; // quantize output.weight
bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored bool only_copy; // only copy tensors - ftype, allow_requantize and
bool pure; // disable k-quant mixtures and quantize all tensors to the same type // quantize_output_tensor are ignored
bool pure; // disable k-quant mixtures and quantize all tensors to the same
// type
} llama_model_quantize_params; } llama_model_quantize_params;
// grammar types // grammar types
@ -256,7 +269,8 @@ extern "C" {
LLAMA_GRETYPE_CHAR_RNG_UPPER = 5, LLAMA_GRETYPE_CHAR_RNG_UPPER = 5,
// modifies a preceding LLAMA_GRETYPE_CHAR or // modifies a preceding LLAMA_GRETYPE_CHAR or
// LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA]) // LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab],
// [a-zA])
LLAMA_GRETYPE_CHAR_ALT = 6, LLAMA_GRETYPE_CHAR_ALT = 6,
}; };
@ -282,7 +296,8 @@ extern "C" {
// Helpers for getting default parameters // Helpers for getting default parameters
LLAMA_API struct llama_model_params llama_model_default_params(void); LLAMA_API struct llama_model_params llama_model_default_params(void);
LLAMA_API struct llama_context_params llama_context_default_params(void); LLAMA_API struct llama_context_params llama_context_default_params(void);
LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void); LLAMA_API struct llama_model_quantize_params
llama_model_quantize_default_params(void);
// Initialize the llama + ggml backend // Initialize the llama + ggml backend
// If numa is true, use NUMA optimizations // If numa is true, use NUMA optimizations
@ -292,14 +307,14 @@ extern "C" {
// Call once at the end of the program - currently only used for MPI // Call once at the end of the program - currently only used for MPI
LLAMA_API void llama_backend_free(void); LLAMA_API void llama_backend_free(void);
LLAMA_API struct llama_model * llama_load_model_from_file( LLAMA_API struct llama_model *
const char * path_model, llama_load_model_from_file(const char *path_model,
struct llama_model_params params); struct llama_model_params params);
LLAMA_API void llama_free_model(struct llama_model *model); LLAMA_API void llama_free_model(struct llama_model *model);
LLAMA_API struct llama_context * llama_new_context_with_model( LLAMA_API struct llama_context *
struct llama_model * model, llama_new_context_with_model(struct llama_model *model,
struct llama_context_params params); struct llama_context_params params);
// Frees all allocated memory // Frees all allocated memory
@ -311,11 +326,13 @@ extern "C" {
LLAMA_API bool llama_mmap_supported(void); LLAMA_API bool llama_mmap_supported(void);
LLAMA_API bool llama_mlock_supported(void); LLAMA_API bool llama_mlock_supported(void);
LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); LLAMA_API const struct llama_model *
llama_get_model(const struct llama_context *ctx);
LLAMA_API int llama_n_ctx(const struct llama_context *ctx); LLAMA_API int llama_n_ctx(const struct llama_context *ctx);
LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_model * model); LLAMA_API enum llama_vocab_type
llama_vocab_type(const struct llama_model *model);
LLAMA_API int llama_n_vocab(const struct llama_model *model); LLAMA_API int llama_n_vocab(const struct llama_model *model);
LLAMA_API int llama_n_ctx_train(const struct llama_model *model); LLAMA_API int llama_n_ctx_train(const struct llama_model *model);
@ -330,19 +347,25 @@ extern "C" {
// - GGUF array values are not supported by these functions // - GGUF array values are not supported by these functions
// Get metadata value as a string by key name // Get metadata value as a string by key name
LLAMA_API int llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size); LLAMA_API int llama_model_meta_val_str(const struct llama_model *model,
const char *key, char *buf,
size_t buf_size);
// Get the number of metadata key/value pairs // Get the number of metadata key/value pairs
LLAMA_API int llama_model_meta_count(const struct llama_model *model); LLAMA_API int llama_model_meta_count(const struct llama_model *model);
// Get metadata key name by index // Get metadata key name by index
LLAMA_API int llama_model_meta_key_by_index(const struct llama_model * model, int i, char * buf, size_t buf_size); LLAMA_API int llama_model_meta_key_by_index(const struct llama_model *model,
int i, char *buf, size_t buf_size);
// Get metadata value as a string by index // Get metadata value as a string by index
LLAMA_API int llama_model_meta_val_str_by_index(const struct llama_model * model, int i, char * buf, size_t buf_size); LLAMA_API int llama_model_meta_val_str_by_index(const struct llama_model *model,
int i, char *buf,
size_t buf_size);
// Get a string describing the model type // Get a string describing the model type
LLAMA_API int llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size); LLAMA_API int llama_model_desc(const struct llama_model *model, char *buf,
size_t buf_size);
// Returns the total size of all the tensors in the model in bytes // Returns the total size of all the tensors in the model in bytes
LLAMA_API uint64_t llama_model_size(const struct llama_model *model); LLAMA_API uint64_t llama_model_size(const struct llama_model *model);
@ -351,30 +374,27 @@ extern "C" {
LLAMA_API uint64_t llama_model_n_params(const struct llama_model *model); LLAMA_API uint64_t llama_model_n_params(const struct llama_model *model);
// Get a llama model tensor // Get a llama model tensor
LLAMA_API struct ggml_tensor * llama_get_model_tensor(struct llama_model * model, const char * name); LLAMA_API struct ggml_tensor *llama_get_model_tensor(struct llama_model *model,
const char *name);
// Returns 0 on success // Returns 0 on success
LLAMA_API int llama_model_quantize( LLAMA_API int llama_model_quantize(const char *fname_inp, const char *fname_out,
const char * fname_inp,
const char * fname_out,
const llama_model_quantize_params *params); const llama_model_quantize_params *params);
// Apply a LoRA adapter to a loaded model // Apply a LoRA adapter to a loaded model
// path_base_model is the path to a higher quality model to use as a base for // path_base_model is the path to a higher quality model to use as a base for
// the layers modified by the adapter. Can be NULL to use the current loaded model. // the layers modified by the adapter. Can be NULL to use the current loaded
// The model needs to be reloaded before applying a new adapter, otherwise the adapter // model. The model needs to be reloaded before applying a new adapter,
// will be applied on top of the previous one // otherwise the adapter will be applied on top of the previous one Returns 0 on
// Returns 0 on success // success
LLAMA_API DEPRECATED(int llama_apply_lora_from_file( LLAMA_API DEPRECATED(int llama_apply_lora_from_file(struct llama_context *ctx,
struct llama_context * ctx,
const char *path_lora, const char *path_lora,
float scale, float scale,
const char *path_base_model, const char *path_base_model,
int n_threads), int n_threads),
"use llama_model_apply_lora_from_file instead"); "use llama_model_apply_lora_from_file instead");
LLAMA_API int llama_model_apply_lora_from_file( LLAMA_API int llama_model_apply_lora_from_file(const struct llama_model *model,
const struct llama_model * model,
const char *path_lora, const char *path_lora,
float scale, float scale,
const char *path_base_model, const char *path_base_model,
@ -424,61 +444,54 @@ extern "C" {
}; };
// Create an empty KV cache view. (use only for debugging purposes) // Create an empty KV cache view. (use only for debugging purposes)
LLAMA_API struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_max_seq); LLAMA_API struct llama_kv_cache_view
llama_kv_cache_view_init(const struct llama_context *ctx, int32_t n_max_seq);
// Free a KV cache view. (use only for debugging purposes) // Free a KV cache view. (use only for debugging purposes)
LLAMA_API void llama_kv_cache_view_free(struct llama_kv_cache_view *view); LLAMA_API void llama_kv_cache_view_free(struct llama_kv_cache_view *view);
// Update the KV cache view structure with the current state of the KV cache. (use only for debugging purposes) // Update the KV cache view structure with the current state of the KV cache.
LLAMA_API void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_kv_cache_view * view); // (use only for debugging purposes)
LLAMA_API void llama_kv_cache_view_update(const struct llama_context *ctx,
struct llama_kv_cache_view *view);
// Returns the number of tokens in the KV cache (slow, use only for debug) // Returns the number of tokens in the KV cache (slow, use only for debug)
// If a KV cell has multiple sequences assigned to it, it will be counted multiple times // If a KV cell has multiple sequences assigned to it, it will be counted
// multiple times
LLAMA_API int llama_get_kv_cache_token_count(const struct llama_context *ctx); LLAMA_API int llama_get_kv_cache_token_count(const struct llama_context *ctx);
// Returns the number of used KV cells (i.e. have at least one sequence assigned to them) // Returns the number of used KV cells (i.e. have at least one sequence assigned
// to them)
LLAMA_API int llama_get_kv_cache_used_cells(const struct llama_context *ctx); LLAMA_API int llama_get_kv_cache_used_cells(const struct llama_context *ctx);
// Clear the KV cache // Clear the KV cache
LLAMA_API void llama_kv_cache_clear( LLAMA_API void llama_kv_cache_clear(struct llama_context *ctx);
struct llama_context * ctx);
// Removes all tokens that belong to the specified sequence and have positions in [p0, p1) // Removes all tokens that belong to the specified sequence and have positions
// seq_id < 0 : match any sequence // in [p0, p1) seq_id < 0 : match any sequence p0 < 0 : [0, p1] p1 < 0 :
// p0 < 0 : [0, p1] // [p0, inf)
// p1 < 0 : [p0, inf) LLAMA_API void llama_kv_cache_seq_rm(struct llama_context *ctx,
LLAMA_API void llama_kv_cache_seq_rm( llama_seq_id seq_id, llama_pos p0,
struct llama_context * ctx,
llama_seq_id seq_id,
llama_pos p0,
llama_pos p1); llama_pos p1);
// Copy all tokens that belong to the specified sequence to another sequence // Copy all tokens that belong to the specified sequence to another sequence
// Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence // Note that this does not allocate extra KV cache memory - it simply assigns
// p0 < 0 : [0, p1] // the tokens to the new sequence p0 < 0 : [0, p1] p1 < 0 : [p0, inf)
// p1 < 0 : [p0, inf) LLAMA_API void llama_kv_cache_seq_cp(struct llama_context *ctx,
LLAMA_API void llama_kv_cache_seq_cp(
struct llama_context * ctx,
llama_seq_id seq_id_src, llama_seq_id seq_id_src,
llama_seq_id seq_id_dst, llama_seq_id seq_id_dst, llama_pos p0,
llama_pos p0,
llama_pos p1); llama_pos p1);
// Removes all tokens that do not belong to the specified sequence // Removes all tokens that do not belong to the specified sequence
LLAMA_API void llama_kv_cache_seq_keep( LLAMA_API void llama_kv_cache_seq_keep(struct llama_context *ctx,
struct llama_context * ctx,
llama_seq_id seq_id); llama_seq_id seq_id);
// Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) // Adds relative position "delta" to all tokens that belong to the specified
// If the KV cache is RoPEd, the KV data is updated accordingly // sequence and have positions in [p0, p1) If the KV cache is RoPEd, the KV data
// p0 < 0 : [0, p1] // is updated accordingly p0 < 0 : [0, p1] p1 < 0 : [p0, inf)
// p1 < 0 : [p0, inf) LLAMA_API void llama_kv_cache_seq_shift(struct llama_context *ctx,
LLAMA_API void llama_kv_cache_seq_shift( llama_seq_id seq_id, llama_pos p0,
struct llama_context * ctx, llama_pos p1, llama_pos delta);
llama_seq_id seq_id,
llama_pos p0,
llama_pos p1,
llama_pos delta);
// //
// State / sessions // State / sessions
@ -491,26 +504,20 @@ extern "C" {
// Copies the state to the specified destination address. // Copies the state to the specified destination address.
// Destination needs to have allocated enough memory. // Destination needs to have allocated enough memory.
// Returns the number of bytes copied // Returns the number of bytes copied
LLAMA_API size_t llama_copy_state_data( LLAMA_API size_t llama_copy_state_data(struct llama_context *ctx, uint8_t *dst);
struct llama_context * ctx,
uint8_t * dst);
// Set the state reading from the specified address // Set the state reading from the specified address
// Returns the number of bytes read // Returns the number of bytes read
LLAMA_API size_t llama_set_state_data( LLAMA_API size_t llama_set_state_data(struct llama_context *ctx, uint8_t *src);
struct llama_context * ctx,
uint8_t * src);
// Save/load session file // Save/load session file
LLAMA_API bool llama_load_session_file( LLAMA_API bool llama_load_session_file(struct llama_context *ctx,
struct llama_context * ctx,
const char *path_session, const char *path_session,
llama_token *tokens_out, llama_token *tokens_out,
size_t n_token_capacity, size_t n_token_capacity,
size_t *n_token_count_out); size_t *n_token_count_out);
LLAMA_API bool llama_save_session_file( LLAMA_API bool llama_save_session_file(struct llama_context *ctx,
struct llama_context * ctx,
const char *path_session, const char *path_session,
const llama_token *tokens, const llama_token *tokens,
size_t n_token_count); size_t n_token_count);
@ -519,33 +526,28 @@ extern "C" {
// Decoding // Decoding
// //
// Run the llama inference to obtain the logits and probabilities for the next token(s). // Run the llama inference to obtain the logits and probabilities for the next
// tokens + n_tokens is the provided batch of new tokens to process // token(s). tokens + n_tokens is the provided batch of new tokens to process
// n_past is the number of tokens to use from previous eval calls // n_past is the number of tokens to use from previous eval calls
// Returns 0 on success // Returns 0 on success
// DEPRECATED: use llama_decode() instead // DEPRECATED: use llama_decode() instead
LLAMA_API DEPRECATED(int llama_eval( LLAMA_API DEPRECATED(int llama_eval(struct llama_context *ctx,
struct llama_context * ctx, llama_token *tokens, int32_t n_tokens,
llama_token * tokens,
int32_t n_tokens,
int n_past), int n_past),
"use llama_decode() instead"); "use llama_decode() instead");
// Same as llama_eval, but use float matrix input directly. // Same as llama_eval, but use float matrix input directly.
// DEPRECATED: use llama_decode() instead // DEPRECATED: use llama_decode() instead
LLAMA_API DEPRECATED(int llama_eval_embd( LLAMA_API DEPRECATED(int llama_eval_embd(struct llama_context *ctx, float *embd,
struct llama_context * ctx, int32_t n_tokens, int n_past),
float * embd,
int32_t n_tokens,
int n_past),
"use llama_decode() instead"); "use llama_decode() instead");
// Return batch for single sequence of tokens starting at pos_0 // Return batch for single sequence of tokens starting at pos_0
// //
// NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it // NOTE: this is a helper function to facilitate transition to the new batch API
// - avoid using it
// //
LLAMA_API struct llama_batch llama_batch_get_one( LLAMA_API struct llama_batch llama_batch_get_one(llama_token *tokens,
llama_token * tokens,
int32_t n_tokens, int32_t n_tokens,
llama_pos pos_0, llama_pos pos_0,
llama_seq_id seq_id); llama_seq_id seq_id);
@ -553,13 +555,11 @@ extern "C" {
// Allocates a batch of tokens on the heap that can hold a maximum of n_tokens // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
// Each token can be assigned up to n_seq_max sequence ids // Each token can be assigned up to n_seq_max sequence ids
// The batch has to be freed with llama_batch_free() // The batch has to be freed with llama_batch_free()
// If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float) // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd
// Otherwise, llama_batch.token will be allocated to store n_tokens llama_token // * sizeof(float) Otherwise, llama_batch.token will be allocated to store
// The rest of the llama_batch members are allocated with size n_tokens // n_tokens llama_token The rest of the llama_batch members are allocated with
// All members are left uninitialized // size n_tokens All members are left uninitialized
LLAMA_API struct llama_batch llama_batch_init( LLAMA_API struct llama_batch llama_batch_init(int32_t n_tokens, int32_t embd,
int32_t n_tokens,
int32_t embd,
int32_t n_seq_max); int32_t n_seq_max);
// Frees a batch of tokens allocated with llama_batch_init() // Frees a batch of tokens allocated with llama_batch_init()
@ -567,16 +567,18 @@ extern "C" {
// Positive return values does not mean a fatal error, but rather a warning. // Positive return values does not mean a fatal error, but rather a warning.
// 0 - success // 0 - success
// 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) // 1 - could not find a KV slot for the batch (try reducing the size of the
// batch or increase the context)
// < 0 - error // < 0 - error
LLAMA_API int llama_decode( LLAMA_API int llama_decode(struct llama_context *ctx, struct llama_batch batch);
struct llama_context * ctx,
struct llama_batch batch);
// Set the number of threads used for decoding // Set the number of threads used for decoding
// n_threads is the number of threads used for generation (single token) // n_threads is the number of threads used for generation (single token)
// n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) // n_threads_batch is the number of threads used for prompt and batch processing
LLAMA_API void llama_set_n_threads(struct llama_context * ctx, uint32_t n_threads, uint32_t n_threads_batch); // (multiple tokens)
LLAMA_API void llama_set_n_threads(struct llama_context *ctx,
uint32_t n_threads,
uint32_t n_threads_batch);
// Token logits obtained from the last call to llama_eval() // Token logits obtained from the last call to llama_eval()
// The logits for the last token are stored in the last row // The logits for the last token are stored in the last row
@ -597,16 +599,22 @@ extern "C" {
// Vocab // Vocab
// //
LLAMA_API const char * llama_token_get_text(const struct llama_model * model, llama_token token); LLAMA_API const char *llama_token_get_text(const struct llama_model *model,
llama_token token);
LLAMA_API float llama_token_get_score(const struct llama_model * model, llama_token token); LLAMA_API float llama_token_get_score(const struct llama_model *model,
llama_token token);
LLAMA_API enum llama_token_type llama_token_get_type(const struct llama_model * model, llama_token token); LLAMA_API enum llama_token_type
llama_token_get_type(const struct llama_model *model, llama_token token);
// Special tokens // Special tokens
LLAMA_API llama_token llama_token_bos(const struct llama_model * model); // beginning-of-sentence LLAMA_API llama_token
LLAMA_API llama_token llama_token_eos(const struct llama_model * model); // end-of-sentence llama_token_bos(const struct llama_model *model); // beginning-of-sentence
LLAMA_API llama_token llama_token_nl (const struct llama_model * model); // next-line LLAMA_API llama_token
llama_token_eos(const struct llama_model *model); // end-of-sentence
LLAMA_API llama_token
llama_token_nl(const struct llama_model *model); // next-line
// Returns -1 if unknown, 1 for true or 0 for false. // Returns -1 if unknown, 1 for true or 0 for false.
LLAMA_API int llama_add_bos_token(const struct llama_model *model); LLAMA_API int llama_add_bos_token(const struct llama_model *model);
@ -615,52 +623,52 @@ extern "C" {
LLAMA_API int llama_add_eos_token(const struct llama_model *model); LLAMA_API int llama_add_eos_token(const struct llama_model *model);
// codellama infill tokens // codellama infill tokens
LLAMA_API llama_token llama_token_prefix(const struct llama_model * model); // Beginning of infill prefix LLAMA_API llama_token llama_token_prefix(
LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle const struct llama_model *model); // Beginning of infill prefix
LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix LLAMA_API llama_token llama_token_middle(
LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle const struct llama_model *model); // Beginning of infill middle
LLAMA_API llama_token llama_token_suffix(
const struct llama_model *model); // Beginning of infill suffix
LLAMA_API llama_token
llama_token_eot(const struct llama_model *model); // End of infill middle
// //
// Tokenization // Tokenization
// //
/// @details Convert the provided text into tokens. /// @details Convert the provided text into tokens.
/// @param tokens The tokens pointer must be large enough to hold the resulting tokens. /// @param tokens The tokens pointer must be large enough to hold the resulting
/// tokens.
/// @return Returns the number of tokens on success, no more than n_max_tokens /// @return Returns the number of tokens on success, no more than n_max_tokens
/// @return Returns a negative number on failure - the number of tokens that would have been returned /// @return Returns a negative number on failure - the number of tokens that
/// @param special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. /// would have been returned
/// @param special Allow tokenizing special and/or control tokens which
/// otherwise are not exposed and treated as plaintext.
/// Does not insert a leading space. /// Does not insert a leading space.
LLAMA_API int llama_tokenize( LLAMA_API int llama_tokenize(const struct llama_model *model, const char *text,
const struct llama_model * model, int text_len, llama_token *tokens,
const char * text, int n_max_tokens, bool add_bos, bool special);
int text_len,
llama_token * tokens,
int n_max_tokens,
bool add_bos,
bool special);
// Token Id -> Piece. // Token Id -> Piece.
// Uses the vocabulary in the provided context. // Uses the vocabulary in the provided context.
// Does not write null terminator to the buffer. // Does not write null terminator to the buffer.
// User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens. // User code is responsible to remove the leading whitespace of the first
LLAMA_API int llama_token_to_piece( // non-BOS token when decoding multiple tokens.
const struct llama_model * model, LLAMA_API int llama_token_to_piece(const struct llama_model *model,
llama_token token, llama_token token, char *buf, int length);
char * buf,
int length);
// //
// Grammar // Grammar
// //
LLAMA_API struct llama_grammar * llama_grammar_init( LLAMA_API struct llama_grammar *
const llama_grammar_element ** rules, llama_grammar_init(const llama_grammar_element **rules, size_t n_rules,
size_t n_rules,
size_t start_rule_index); size_t start_rule_index);
LLAMA_API void llama_grammar_free(struct llama_grammar *grammar); LLAMA_API void llama_grammar_free(struct llama_grammar *grammar);
LLAMA_API struct llama_grammar * llama_grammar_copy(const struct llama_grammar * grammar); LLAMA_API struct llama_grammar *
llama_grammar_copy(const struct llama_grammar *grammar);
// //
// Sampling functions // Sampling functions
@ -669,124 +677,138 @@ extern "C" {
// Sets the current rng seed. // Sets the current rng seed.
LLAMA_API void llama_set_rng_seed(struct llama_context *ctx, uint32_t seed); LLAMA_API void llama_set_rng_seed(struct llama_context *ctx, uint32_t seed);
/// @details Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix. /// @details Repetition penalty described in CTRL academic paper
/// @details Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details. /// https://arxiv.org/abs/1909.05858, with negative logit fix.
/// @details Frequency and presence penalties described in OpenAI API
/// https://platform.openai.com/docs/api-reference/parameter-details.
LLAMA_API void llama_sample_repetition_penalties( LLAMA_API void llama_sample_repetition_penalties(
struct llama_context * ctx, struct llama_context *ctx, llama_token_data_array *candidates,
llama_token_data_array * candidates, const llama_token *last_tokens, size_t penalty_last_n, float penalty_repeat,
const llama_token * last_tokens, float penalty_freq, float penalty_present);
size_t penalty_last_n,
float penalty_repeat,
float penalty_freq,
float penalty_present);
/// @details Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806 /// @details Apply classifier-free guidance to the logits as described in
/// @param candidates A vector of `llama_token_data` containing the candidate tokens, the logits must be directly extracted from the original generation context without being sorted. /// academic paper "Stay on topic with Classifier-Free Guidance"
/// @params guidance_ctx A separate context from the same model. Other than a negative prompt at the beginning, it should have all generated and user input tokens copied from the main context. /// https://arxiv.org/abs/2306.17806
/// @params scale Guidance strength. 1.0f means no guidance. Higher values mean stronger guidance. /// @param candidates A vector of `llama_token_data` containing the candidate
/// tokens, the logits must be directly extracted from the original generation
/// context without being sorted.
/// @params guidance_ctx A separate context from the same model. Other than a
/// negative prompt at the beginning, it should have all generated and user
/// input tokens copied from the main context.
/// @params scale Guidance strength. 1.0f means no guidance. Higher values mean
/// stronger guidance.
LLAMA_API void llama_sample_classifier_free_guidance( LLAMA_API void llama_sample_classifier_free_guidance(
struct llama_context * ctx, struct llama_context *ctx, llama_token_data_array *candidates,
llama_token_data_array * candidates, struct llama_context *guidance_ctx, float scale);
struct llama_context * guidance_ctx,
float scale);
/// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. /// @details Sorts candidate tokens by their logits in descending order and
LLAMA_API void llama_sample_softmax( /// calculate probabilities based on logits.
struct llama_context * ctx, LLAMA_API void llama_sample_softmax(struct llama_context *ctx,
llama_token_data_array *candidates); llama_token_data_array *candidates);
/// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 /// @details Top-K sampling described in academic paper "The Curious Case of
LLAMA_API void llama_sample_top_k( /// Neural Text Degeneration" https://arxiv.org/abs/1904.09751
struct llama_context * ctx, LLAMA_API void llama_sample_top_k(struct llama_context *ctx,
llama_token_data_array * candidates, llama_token_data_array *candidates, int k,
int k,
size_t min_keep); size_t min_keep);
/// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 /// @details Nucleus sampling described in academic paper "The Curious Case of
LLAMA_API void llama_sample_top_p( /// Neural Text Degeneration" https://arxiv.org/abs/1904.09751
struct llama_context * ctx, LLAMA_API void llama_sample_top_p(struct llama_context *ctx,
llama_token_data_array * candidates, llama_token_data_array *candidates, float p,
float p,
size_t min_keep); size_t min_keep);
/// @details Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841 /// @details Minimum P sampling as described in
LLAMA_API void llama_sample_min_p( /// https://github.com/ggerganov/llama.cpp/pull/3841
struct llama_context * ctx, LLAMA_API void llama_sample_min_p(struct llama_context *ctx,
llama_token_data_array * candidates, llama_token_data_array *candidates, float p,
float p,
size_t min_keep); size_t min_keep);
/// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/. /// @details Tail Free Sampling described in
LLAMA_API void llama_sample_tail_free( /// https://www.trentonbricken.com/Tail-Free-Sampling/.
struct llama_context * ctx, LLAMA_API void llama_sample_tail_free(struct llama_context *ctx,
llama_token_data_array *candidates, llama_token_data_array *candidates,
float z, float z, size_t min_keep);
/// @details Locally Typical Sampling implementation described in the paper
/// https://arxiv.org/abs/2202.00666.
LLAMA_API void llama_sample_typical(struct llama_context *ctx,
llama_token_data_array *candidates, float p,
size_t min_keep); size_t min_keep);
/// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. LLAMA_API void llama_sample_temp(struct llama_context *ctx,
LLAMA_API void llama_sample_typical(
struct llama_context * ctx,
llama_token_data_array * candidates,
float p,
size_t min_keep);
LLAMA_API void llama_sample_temp(
struct llama_context * ctx,
llama_token_data_array *candidates, llama_token_data_array *candidates,
float temp); float temp);
LLAMA_API DEPRECATED(void llama_sample_temperature( LLAMA_API
struct llama_context * ctx, DEPRECATED(void llama_sample_temperature(struct llama_context *ctx,
llama_token_data_array *candidates, llama_token_data_array *candidates,
float temp), float temp),
"use llama_sample_temp instead"); "use llama_sample_temp instead");
/// @details Apply constraints from grammar /// @details Apply constraints from grammar
LLAMA_API void llama_sample_grammar( LLAMA_API void llama_sample_grammar(struct llama_context *ctx,
struct llama_context * ctx,
llama_token_data_array *candidates, llama_token_data_array *candidates,
const struct llama_grammar *grammar); const struct llama_grammar *grammar);
/// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. /// @details Mirostat 1.0 algorithm described in the paper
/// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. /// https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
/// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. /// @param candidates A vector of `llama_token_data` containing the candidate
/// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. /// tokens, their probabilities (p), and log-odds (logit) for the current
/// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. /// position in the generated text.
/// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. /// @param tau The target cross-entropy (or surprise) value you want to achieve
/// for the generated text. A higher value corresponds to more surprising or
/// less predictable text, while a lower value corresponds to less surprising or
/// more predictable text.
/// @param eta The learning rate used to update `mu` based on the error between
/// the target and observed surprisal of the sampled word. A larger learning
/// rate will cause `mu` to be updated more quickly, while a smaller learning
/// rate will result in slower updates.
/// @param m The number of tokens considered in the estimation of `s_hat`. This
/// is an arbitrary value that is used to calculate `s_hat`, which in turn helps
/// to calculate the value of `k`. In the paper, they use `m = 100`, but you can
/// experiment with different values to see how it affects the performance of
/// the algorithm.
/// @param mu Maximum cross-entropy. This value is initialized to be twice the
/// target cross-entropy (`2 * tau`) and is updated in the algorithm based on
/// the error between the target and observed surprisal.
LLAMA_API llama_token llama_sample_token_mirostat( LLAMA_API llama_token llama_sample_token_mirostat(
struct llama_context * ctx, struct llama_context *ctx, llama_token_data_array *candidates, float tau,
llama_token_data_array * candidates, float eta, int m, float *mu);
float tau,
float eta,
int m,
float * mu);
/// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. /// @details Mirostat 2.0 algorithm described in the paper
/// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. /// https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
/// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. /// @param candidates A vector of `llama_token_data` containing the candidate
/// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. /// tokens, their probabilities (p), and log-odds (logit) for the current
/// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. /// position in the generated text.
/// @param tau The target cross-entropy (or surprise) value you want to achieve
/// for the generated text. A higher value corresponds to more surprising or
/// less predictable text, while a lower value corresponds to less surprising or
/// more predictable text.
/// @param eta The learning rate used to update `mu` based on the error between
/// the target and observed surprisal of the sampled word. A larger learning
/// rate will cause `mu` to be updated more quickly, while a smaller learning
/// rate will result in slower updates.
/// @param mu Maximum cross-entropy. This value is initialized to be twice the
/// target cross-entropy (`2 * tau`) and is updated in the algorithm based on
/// the error between the target and observed surprisal.
LLAMA_API llama_token llama_sample_token_mirostat_v2( LLAMA_API llama_token llama_sample_token_mirostat_v2(
struct llama_context * ctx, struct llama_context *ctx, llama_token_data_array *candidates, float tau,
llama_token_data_array * candidates, float eta, float *mu);
float tau,
float eta,
float * mu);
/// @details Selects the token with the highest probability. /// @details Selects the token with the highest probability.
/// Does not compute the token probabilities. Use llama_sample_softmax() instead. /// Does not compute the token probabilities. Use
/// llama_sample_softmax() instead.
LLAMA_API llama_token llama_sample_token_greedy( LLAMA_API llama_token llama_sample_token_greedy(
struct llama_context * ctx, struct llama_context *ctx, llama_token_data_array *candidates);
llama_token_data_array * candidates);
/// @details Randomly selects a token from the candidates based on their probabilities. /// @details Randomly selects a token from the candidates based on their
LLAMA_API llama_token llama_sample_token( /// probabilities.
struct llama_context * ctx, LLAMA_API llama_token llama_sample_token(struct llama_context *ctx,
llama_token_data_array *candidates); llama_token_data_array *candidates);
/// @details Accepts the sampled token into the grammar /// @details Accepts the sampled token into the grammar
LLAMA_API void llama_grammar_accept_token( LLAMA_API void llama_grammar_accept_token(struct llama_context *ctx,
struct llama_context * ctx,
struct llama_grammar *grammar, struct llama_grammar *grammar,
llama_token token); llama_token token);
@ -803,36 +825,39 @@ extern "C" {
}; };
// Passed to beam_search_callback function. // Passed to beam_search_callback function.
// Whenever 0 < common_prefix_length, this number of tokens should be copied from any of the beams // Whenever 0 < common_prefix_length, this number of tokens should be copied
// (e.g. beams[0]) as they will be removed (shifted) from all beams in all subsequent callbacks. // from any of the beams (e.g. beams[0]) as they will be removed (shifted) from
// These pointers are valid only during the synchronous callback, so should not be saved. // all beams in all subsequent callbacks. These pointers are valid only during
// the synchronous callback, so should not be saved.
struct llama_beams_state { struct llama_beams_state {
struct llama_beam_view *beam_views; struct llama_beam_view *beam_views;
size_t n_beams; // Number of elements in beam_views[]. size_t n_beams; // Number of elements in beam_views[].
size_t common_prefix_length; // Current max length of prefix tokens shared by all beams. size_t common_prefix_length; // Current max length of prefix tokens shared by
// all beams.
bool last_call; // True iff this is the last callback invocation. bool last_call; // True iff this is the last callback invocation.
}; };
// Type of pointer to the beam_search_callback function. // Type of pointer to the beam_search_callback function.
// void* callback_data is any custom data passed to llama_beam_search, that is subsequently // void* callback_data is any custom data passed to llama_beam_search, that is
// passed back to beam_search_callback. This avoids having to use global variables in the callback. // subsequently passed back to beam_search_callback. This avoids having to use
typedef void (*llama_beam_search_callback_fn_t)(void * callback_data, struct llama_beams_state); // global variables in the callback.
typedef void (*llama_beam_search_callback_fn_t)(void *callback_data,
struct llama_beams_state);
/// @details Deterministically returns entire sentence constructed by a beam search. /// @details Deterministically returns entire sentence constructed by a beam
/// search.
/// @param ctx Pointer to the llama_context. /// @param ctx Pointer to the llama_context.
/// @param callback Invoked for each iteration of the beam_search loop, passing in beams_state. /// @param callback Invoked for each iteration of the beam_search loop, passing
/// in beams_state.
/// @param callback_data A pointer that is simply passed back to callback. /// @param callback_data A pointer that is simply passed back to callback.
/// @param n_beams Number of beams to use. /// @param n_beams Number of beams to use.
/// @param n_past Number of tokens already evaluated. /// @param n_past Number of tokens already evaluated.
/// @param n_predict Maximum number of tokens to predict. EOS may occur earlier. /// @param n_predict Maximum number of tokens to predict. EOS may occur earlier.
LLAMA_API void llama_beam_search( LLAMA_API void llama_beam_search(struct llama_context *ctx,
struct llama_context * ctx,
llama_beam_search_callback_fn_t callback, llama_beam_search_callback_fn_t callback,
void * callback_data, void *callback_data, size_t n_beams,
size_t n_beams, int n_past, int n_predict);
int n_past,
int n_predict);
// Performance information // Performance information
LLAMA_API struct llama_timings llama_get_timings(struct llama_context *ctx); LLAMA_API struct llama_timings llama_get_timings(struct llama_context *ctx);
@ -847,7 +872,8 @@ extern "C" {
// If this is not called, or NULL is supplied, everything is output on stderr. // If this is not called, or NULL is supplied, everything is output on stderr.
LLAMA_API void llama_log_set(ggml_log_callback log_callback, void *user_data); LLAMA_API void llama_log_set(ggml_log_callback log_callback, void *user_data);
LLAMA_API void llama_dump_timing_info_yaml(FILE * stream, const struct llama_context * ctx); LLAMA_API void llama_dump_timing_info_yaml(FILE *stream,
const struct llama_context *ctx);
#ifdef __cplusplus #ifdef __cplusplus
} }
@ -856,14 +882,13 @@ extern "C" {
// Internal API to be implemented by llama.cpp and used by tests/benchmarks only // Internal API to be implemented by llama.cpp and used by tests/benchmarks only
#ifdef LLAMA_API_INTERNAL #ifdef LLAMA_API_INTERNAL
#include <vector>
#include <string> #include <string>
#include <vector>
struct ggml_tensor; struct ggml_tensor;
const std::vector<std::pair<std::string, struct ggml_tensor *>> & llama_internal_get_tensor_map( const std::vector<std::pair<std::string, struct ggml_tensor *>> &
struct llama_context * ctx llama_internal_get_tensor_map(struct llama_context *ctx);
);
#endif // LLAMA_API_INTERNAL #endif // LLAMA_API_INTERNAL