diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index 28584e14b..f90f1c145 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -101,9 +101,9 @@ static void usage(const char * executable) { printf(" --exclude-weights tensor_name: use importance matrix for this/these tensor(s)\n"); printf(" --output-tensor-type ggml_type: use this ggml_type for the output.weight tensor\n"); printf(" --token-embedding-type ggml_type: use this ggml_type for the token embeddings tensor\n"); - printf(" --keep-split: will generate quatized model in the same shards as input"); + printf(" --keep-split: will generate quatized model in the same shards as input\n"); printf(" --override-kv KEY=TYPE:VALUE\n"); - printf(" Advanced option to override model metadata by key in the quantized model. May be specified multiple times.\n"); + printf(" : Advanced option to override model metadata by key in the quantized model. May be specified multiple times.\n\n"); printf("Note: --include-weights and --exclude-weights cannot be used together\n"); printf("\nAllowed quantization types:\n"); for (auto & it : QUANT_OPTIONS) { diff --git a/llama.cpp b/llama.cpp index 8818c6928..51f1444ac 100644 --- a/llama.cpp +++ b/llama.cpp @@ -15691,15 +15691,15 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s new_type = default_type; // get more optimal quantization type based on the tensor shape, layer, etc. - if (!params->pure && ggml_is_quantized(default_type)) { - new_type = llama_tensor_get_type(qs, new_type, tensor, ftype); - } if (params->token_embedding_type < GGML_TYPE_COUNT && strcmp(tensor->name, "token_embd.weight") == 0) { new_type = params->token_embedding_type; } if (params->output_tensor_type < GGML_TYPE_COUNT && strcmp(tensor->name, "output.weight") == 0) { new_type = params->output_tensor_type; } + if (!params->pure && ggml_is_quantized(default_type)) { + new_type = llama_tensor_get_type(qs, new_type, tensor, ftype); + } // If we've decided to quantize to the same type the tensor is already // in then there's nothing to do. diff --git "a/\177\177" "b/\177\177" new file mode 100644 index 000000000..f8c08c23b --- /dev/null +++ "b/\177\177" @@ -0,0 +1,28 @@ +export PATH=/usr/local/cuda-12.2/bin${PATH:+:${PATH}} +export LD_LIBRARY_PATH=/usr/local/cuda-12.2/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} +export PATH=/local/mnt/workspace/miniconda3/bin:$PATH +export CUDA_VISIBLE_DEVICES=0,1,2,3ii: +: +: +# >>> conda initialize >>> +# !! Contents within this block are managed by 'conda init' !! +__conda_setup="$('/local/mnt/workspace/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +if [ $? -eq 0 ]; then + eval "$__conda_setup" +else + if [ -f "/local/mnt/workspace/miniconda3/etc/profile.d/conda.sh" ]; then + . "/local/mnt/workspace/miniconda3/etc/profile.d/conda.sh" + else + export PATH="/local/mnt/workspace/miniconda3/bin:$PATH" + fi +fi +unset __conda_setup +# <<< conda initialize <<< + +# proxy setting +export http_proxy=http://secure-proxy-aprdc2-1.qualcomm.com:9090/ +export https_proxy=http://secure-proxy-aprdc2-1.qualcomm.com:9090/ +export ftp_proxy=http://secure-proxy-aprdc2-1.qualcomm.com:9090/ + +# HF cache +export HF_HOME=/local/mnt/zzx/.cache/huggingface