vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

improve error handling

Browse source
This commit is contained in:
Yazan Agha-Schrader 2023-11-28 09:43:01 +01:00
parent 57f8edd016
commit 5ac0f300a9
3 changed files with 169 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

20
examples/server/public/index.html
View file

@ -32,11 +32,11 @@
const params = signal({
n_predict: 358,
temperature: 0.7,
temperature: 1.0, // adapt to optimized min-p requierements
repeat_last_n: 256, // 0 = disable penalty, -1 = context size
repeat_penalty: 1.18, // 1.0 = disabled
top_k: 1, // <= 0 to use vocab size
top_p: 0.5, // 1.0 = disabled
top_k: 0, // <= 0 to use vocab size
top_p: 1.0, // 1.0 = disabled
min_p: 0.05, // 0 = disabled
tfs_z: 1.0, // 1.0 = disabled
typical_p: 1.0, // 1.0 = disabled
@ -658,12 +658,14 @@ async function updateSystemLanguage(event) {
</optgroup>
<optgroup label="More Prompt-Styles">
<option value="airoboros180">Airoboros L2</option>
<option value="bakllava">BakLLaVA-1</option>
<option value="codeCherryPop">Code Cherry Pop</option>
<option value="deepseekCoder">Deepseek Coder</option>
<option value="dolphinMistral">Dolphin Mistral</option>
<option value="evolvedSeeker">evolvedSeeker 1.3B</option>
<option value="goliath120b">Goliath 120B</option>
<option value="jordan">Jordan</option>
<option value="llava">LLaVA</option>
<option value="leoHessianai">Leo Hessianai</option>
<option value="leoMistral">Leo Mistral</option>
<option value="marx">Marx</option>
@ -773,21 +775,21 @@ async function updateSystemLanguage(event) {
<fieldset class="two">
${IntField({ label: "Prediction", max: 2048, min: -1, step: 16, name: "n_predict", value: params.value.n_predict, })}
${FloatField({ label: "Temperature", max: 1.5, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature })}
${IntField({ label: "Top-K", max: 100, min: -1, step: 1, name: "top_k", value: params.value.top_k })}
${FloatField({ label: "Repetition Penalty", max: 2.0, min: 0.0, name: "repeat_penalty", step: 0.01, value: params.value.repeat_penalty })}
${FloatField({ label: "Min-P sampling", max: 1.0, min: 0.0, name: "min_p", step: 0.01, value: params.value.min_p })}
${FloatField({ label: "Repetition Penalty", max: 2.0, min: 0.0, name: "repeat_penalty", step: 0.01, value: params.value.repeat_penalty })}
${FloatField({ label: "Temperature", max: 1.5, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature })}
</fieldset>
<details>
<summary><span class="summary-title">Further Options</span></summary>
<fieldset class="two">
${FloatField({ label: "Top-P", max: 1.0, min: 0.0, name: "top_p", step: 0.01, value: params.value.top_p })}
${IntField({ label: "Top-K", max: 100, min: -1, step: 1, name: "top_k", value: params.value.top_k })}
${IntField({ label: "Penalize Last N", max: 2048, min: 0, step: 16, name: "repeat_last_n", value: params.value.repeat_last_n })}
${FloatField({ label: "TFS-Z", max: 1.0, min: 0.0, name: "tfs_z", step: 0.01, value: params.value.tfs_z })}
${FloatField({ label: "Top-P", max: 1.0, min: 0.0, name: "top_p", step: 0.01, value: params.value.top_p })}
${FloatField({ label: "Presence Penalty", max: 1.0, min: 0.0, name: "presence_penalty", step: 0.01, value: params.value.presence_penalty })}
${FloatField({ label: "Typical-P", max: 1.0, min: 0.0, name: "typical_p", step: 0.01, value: params.value.typical_p })}
${FloatField({ label: "TFS-Z", max: 1.0, min: 0.0, name: "tfs_z", step: 0.01, value: params.value.tfs_z })}
${FloatField({ label: "Frequency Penalty", max: 1.0, min: 0.0, name: "frequency_penalty", step: 0.01, value: params.value.frequency_penalty })}
${FloatField({ label: "Typical-P", max: 1.0, min: 0.0, name: "typical_p", step: 0.01, value: params.value.typical_p })}
</fieldset>
<hr style="height: 1px; background-color: #ececf1; border: none;" />

24
examples/server/public/promptFormats.js
View file

@ -12,6 +12,12 @@ export const promptFormats = {
char: "Response",
user: "Instruction"
},
"bakllava": {
template: "{{history}}{{char}}:",
historyTemplate: "{{name}}: {{message}}\n",
char: "ASSISTANT",
user: "USER"
},
"chatml": {
template: "<|im_start|>system\n{{prompt}}<|im_end|>\n{{history}}\n<|im_start|>{{char}}",
historyTemplate: "<|im_start|>{{user}}\n{{message}}<|im_end|>",
@ -54,6 +60,18 @@ export const promptFormats = {
char: "ASSISTANT",
user: "USER"
},
"llama2": {
template: "<s>[INST] <<SYS>>\n{{prompt}}\n<</SYS>>\n\n{{history}} [/INST] {{char}} </s><s>[INST] ",
historyTemplate: "{{name}}: {{message}} [/INST]",
char: "llama",
user: "user"
},
"llava": {
template: "{{history}}{{char}}:",
historyTemplate: "{{name}}: {{message}}\n",
char: "ASSISTANT",
user: "USER"
},
"leoHessianai": {
template: "<|im_start|>system\n{{prompt}}<|im_end|>\n{{history}}\n<|im_start|>{{char}}",
historyTemplate: "<|im_start|>{{user}}\n{{message}}<|im_end|>",
@ -66,12 +84,6 @@ export const promptFormats = {
char: "ASSISTANT",
user: "USER"
},
"llama2": {
template: "<s>[INST] <<SYS>>\n{{prompt}}\n<</SYS>>\n\n{{history}} [/INST] {{char}} </s><s>[INST] ",
historyTemplate: "{{name}}: {{message}} [/INST]",
char: "llama",
user: "user"
},
"marx": {
template: "{{history}}\n{{char}}:",
historyTemplate: "{{name}}: {{message}}",

142
examples/start-server.sh
View file

@ -1,5 +1,13 @@
#!/bin/bash
# Determine the size of the terminal window
TERMINAL_HEIGHT=$(tput lines)
TERMINAL_WIDTH=$(tput cols)
# Calculate a size for the dialog box as a percentage of the terminal to make sure it fits
DIALOG_HEIGHT=$((TERMINAL_HEIGHT * 5 / 8)) # approx. golden ratio
DIALOG_WIDTH=$((TERMINAL_WIDTH * 5 / 8)) # approx. golden ratio
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
# Set default values
@ -71,7 +79,7 @@ fi
model_selection_warning() {
dialog --title "Hinweis" --msgbox "\n\n\nPlease note: To navigate to a folder, please press the space bar twice. To return to a higher-level folder, press the Backspace key.\n\n\nAlternatively, you can also enter the desired path manually in the lower address field. \n\n\nOnly confirm your selection with the Enter key once you have selected the file – or the desired folder to be searched." 23 65
dialog --title "Hinweis" --msgbox "\n\n\nPlease note!\n\nTo navigate to a folder, please press the space bar twice. To return to a higher-level folder, press the Backspace key.\n\n\nAlternatively, you can also enter the desired path manually in the lower path field. \n\n\nOnly confirm your selection with the Enter key once you have selected the file – or the desired folder to be searched recursively." $DIALOG_HEIGHT $DIALOG_WIDTH
}
@ -85,7 +93,7 @@ model_selection() {
model_path=$(dialog --backtitle "Model Selection" \
--title "Select Model File or Folder" \
--fselect "$INITIAL_DIR" 23 65 \
--fselect "$INITIAL_DIR" $DIALOG_HEIGHT $DIALOG_WIDTH \
2>&1 1>&3)
exit_status=$?
exec 3>&-
@ -98,12 +106,19 @@ model_selection() {
# If a folder has been selected, search for *.gguf files
if [ -d "$model_path" ]; then
model_files=($(find "$model_path" -name "*.gguf" 2>/dev/null))
# Check whether files have been found
if [ ${#model_files[@]} -eq 0 ]; then
dialog --backtitle "Model Selection" \
--title "No Models Found" \
--msgbox "\n\n\nNo model files (*.gguf) were found in the selected directory." $DIALOG_HEIGHT $DIALOG_WIDTH
return
fi
elif [ -f "$model_path" ]; then
model_files=("$model_path")
else
dialog --backtitle "Model Selection" \
--title "Invalid Selection" \
--msgbox "The selected path is not valid." 23 65
--msgbox "\n\n\nThe selected path is not valid." $DIALOG_HEIGHT $DIALOG_WIDTH
return
fi
@ -111,7 +126,7 @@ model_selection() {
exec 3>&1
model_choice=$(dialog --backtitle "Model Selection" \
--title "Select a Model File" \
--menu "Choose one of the found models:" 23 65 4 \
--menu "Choose one of the found models:" $DIALOG_HEIGHT $DIALOG_WIDTH \
$(for i in "${!model_files[@]}"; do echo "$((i+1))" "$(basename "${model_files[$i]}")"; done) \
2>&1 1>&3)
exit_status=$?
@ -135,7 +150,7 @@ multimodal_model_selection() {
mmproj_path=$(dialog --backtitle "Multimodal Model Selection" \
--title "Select Multimodal Model File or Folder" \
--fselect "$INITIAL_DIR" 23 65 \
--fselect "$INITIAL_DIR" $DIALOG_HEIGHT $DIALOG_WIDTH \
2>&1 1>&3)
exit_status=$?
exec 3>&-
@ -148,12 +163,19 @@ multimodal_model_selection() {
# If a folder has been selected, search for *.bin files
if [ -d "$mmproj_path" ]; then
multi_modal_files=($(find "$mmproj_path" -name "*.bin" 2>/dev/null))
# Check whether files have been found
if [ ${#multi_modal_files[@]} -eq 0 ]; then
dialog --backtitle "Multimodal Model Selection" \
--title "No Multimodal Models Found" \
--msgbox "\n\n\nNo multimodal model files (*.bin) were found in the selected directory." $DIALOG_HEIGHT $DIALOG_WIDTH
return
fi
elif [ -f "$mmproj_path" ]; then
multi_modal_files=("$mmproj_path")
else
dialog --backtitle "Multimodal Model" \
dialog --backtitle "Multimodal Model Selection" \
--title "Invalid Selection" \
--msgbox "The selected path is not valid." 7 50
--msgbox "\n\n\nThe selected path is not valid." $DIALOG_HEIGHT $DIALOG_WIDTH
return
fi
@ -161,7 +183,7 @@ multimodal_model_selection() {
exec 3>&1
multi_modal_choice=$(dialog --backtitle "Multimodal Model" \
--title "Select a Model File" \
--menu "Choose one of the found models:" 23 65 4 \
--menu "Choose one of the found models:" $DIALOG_HEIGHT $DIALOG_WIDTH 8 \
$(for i in "${!multi_modal_files[@]}"; do echo "$((i+1))" "$(basename "${multi_modal_files[$i]}")"; done) \
2>&1 1>&3)
exit_status=$?
@ -184,7 +206,7 @@ options() {
form_values=$(dialog --backtitle "Options Configuration" \
--title "Set Options" \
--form "Enter the values for the following options:" \
23 65 0 \
$DIALOG_HEIGHT $DIALOG_WIDTH 0 \
"Number of Threads (-t):" 1 1 "$threads" 1 25 25 5 \
"Context Size (-c):" 2 1 "$ctx_size" 2 25 25 5 \
"Batch Size (-b):" 3 1 "$batch_size" 3 25 25 5 \
@ -214,7 +236,7 @@ further_options() {
exec 3>&1
choices=$(dialog --backtitle "Further Options" \
--title "Boolean Options" \
--checklist "Select options:" 23 65 3 \
--checklist "Select options:" $DIALOG_HEIGHT $DIALOG_WIDTH 3 \
"1" "Continuous Batching (-cb)" $cb_value \
"2" "Memory Lock (--mlock)" $mlock_value \
"3" "No Memory Map (--no-mmap)" $no_mmap_value \
@ -248,10 +270,10 @@ advanced_options() {
advanced_values=$(dialog --backtitle "Advanced Options" \
--title "Advanced Server Configuration" \
--form "Enter the advanced configuration options:" \
23 65 0 \
"Host IP:" 1 1 "$host" 1 15 15 0 \
"Port:" 2 1 "$port" 2 15 5 0 \
"Additional Options:" 3 1 "$advanced_options" 3 15 30 0 \
$DIALOG_HEIGHT $DIALOG_WIDTH 0 \
"Host IP:" 1 1 "$host" 1 25 15 0 \
"Port:" 2 1 "$port" 2 25 5 0 \
"Additional Options:" 3 1 "$advanced_options" 3 25 30 0 \
2>&1 1>&3)
exit_status=$?
exec 3>&-
@ -272,7 +294,7 @@ save_config() {
exec 3>&1
config_file=$(dialog --backtitle "Save Configuration" \
--title "Save Configuration File" \
--fselect "$SCRIPT_DIR/" 23 65 \
--fselect "$SCRIPT_DIR/" $DIALOG_HEIGHT $DIALOG_WIDTH \
2>&1 1>&3)
exit_status=$?
exec 3>&-
@ -282,10 +304,13 @@ save_config() {
return
fi
absolute_model_path=$(get_absolute_path "$model_path")
absolute_mmproj_path=$(get_absolute_path "$mmproj_path")
# Saving the configuration to the file with absolute paths using custom function
cat > "$config_file" << EOF
model_path=$(get_absolute_path "$model_path")
mmproj_path=$(get_absolute_path "$mmproj_path")
model_path=$absolute_model_path
mmproj_path=$absolute_mmproj_path
threads=$threads
ctx_size=$ctx_size
batch_size=$batch_size
@ -300,7 +325,7 @@ EOF
dialog --backtitle "Save Configuration" \
--title "Configuration Saved" \
--msgbox "Configuration has been saved to $config_file" 7 50
--msgbox "\n\n\nYour configuration has been saved to\n\n$config_file" $DIALOG_HEIGHT $DIALOG_WIDTH
}
@ -310,7 +335,7 @@ load_config() {
exec 3>&1
config_file=$(dialog --backtitle "Load Configuration" \
--title "Load Configuration File" \
--fselect "$SCRIPT_DIR/" 23 65 \
--fselect "$SCRIPT_DIR/" $DIALOG_HEIGHT $DIALOG_WIDTH \
2>&1 1>&3)
exit_status=$?
exec 3>&-
@ -324,23 +349,73 @@ load_config() {
if [ ! -f "$config_file" ]; then
dialog --backtitle "Load Configuration" \
--title "File Not Found" \
--msgbox "The file $config_file was not found." 7 50
--msgbox "\n\n\nThe file $config_file was not found." $DIALOG_HEIGHT $DIALOG_WIDTH
return
fi
# Load configuration from the file
source "$config_file"
# Convert model paths to absolute paths
model_path=$(get_absolute_path "$model_path")
mmproj_path=$(get_absolute_path "$mmproj_path")
dialog --backtitle "Load Configuration" \
--title "Configuration Loaded" \
--msgbox "Configuration has been loaded from $config_file" 7 50
--msgbox "\n\n\nConfiguration has been loaded successfully from\n\n$config_file" $DIALOG_HEIGHT $DIALOG_WIDTH
}
# Checking the existence of the server executable
check_server_executable() {
local server_executable="$SCRIPT_DIR/../server"
local makefile_path="$SCRIPT_DIR/../Makefile"
local cmake_lists_path="$SCRIPT_DIR/../CMakeLists.txt"
if [ ! -f "$server_executable" ]; then
# Server executable does not exist, check Makefile and CMakeLists.txt
if [ -f "$makefile_path" ] && [ -f "$cmake_lists_path" ]; then
# Offer the user to build the server now
exec 3>&1
response=$(dialog --title "Server Executable Missing" \
--yesno "\n\n\nThe server executable does not exist. Would you like to run 'make' to build it? Note: This will build a basic server without GPU acceleration. Please read the documentation if you need more options and run the build process manually." $DIALOG_HEIGHT $DIALOG_WIDTH \
2>&1 1>&3)
exit_status=$?
exec 3>&-
if [ $exit_status = 0 ]; then
# User has agreed, run 'make'
(cd "$SCRIPT_DIR/.." && make)
# Check if 'make' was successful
if [ ! -f "$server_executable" ]; then
dialog --title "Build Failed" --msgbox "\n\n\nThe server could not be built. Please check the build process manually." $DIALOG_HEIGHT $DIALOG_WIDTH
exit 1
fi
else
# User has rejected or pressed ESC
dialog --title "Build Canceled" --msgbox "\n\n\nServer build was canceled. Cannot start the server without the executable." $DIALOG_HEIGHT $DIALOG_WIDTH
exit 1
fi
else
# Makefile and CMakeLists.txt do not exist
dialog --title "Critical Error" --msgbox "\n\n\nMakefile and CMakeLists.txt are missing. This script may not be in the correct directory. Please make sure this script is in its correct directory." $DIALOG_HEIGHT $DIALOG_WIDTH
exit 1
fi
fi
}
confirm_and_start_server() {
# Check whether model_path refers to a valid .gguf file
if [[ ! "$model_path" =~ \.gguf$ ]] || [ ! -f "$model_path" ]; then
dialog --title "Invalid Model File" --msgbox "\n\n\nThe selected model file ($model_path) is not valid or does not end with .gguf or a model file was not selected yet.\n\n\nPlease select a valid .gguf model file." $DIALOG_HEIGHT $DIALOG_WIDTH
return 1
fi
# Show the compiled command in a dialog box
dialog --title "Server Start Confirmation" --yesno "The server will be started with the following command:\n\n$cmd\n\nDo not forget to close the server with Ctrl+C as soon as you are finished.\n\nWould you like to continue?" 23 65
dialog --title "Server Start Confirmation" --yesno "\n\n\nThe server will be started with the following command:\n\n$cmd\n\nDo not forget to close the server with Ctrl+C as soon as you are finished.\n\nWould you like to continue?" $DIALOG_HEIGHT $DIALOG_WIDTH
# Check exit status of dialog
response=$?
@ -377,6 +452,21 @@ start_server() {
# Function to confirm exit
confirm_exit() {
exec 3>&1
selection=$(dialog \
--backtitle "Confirm Exit" \
--title "Are you sure?" \
--yesno "Are you sure you want to exit?" 7 60 \
2>&1 1>&3)
exit_status=$?
exec 3>&-
return $exit_status
}
# Function to show the main menu
show_main_menu() {
while true; do
@ -386,7 +476,7 @@ show_main_menu() {
--title "Main Menu" \
--clear \
--cancel-label "Exit" \
--menu "Welcome to llama.cpp Dialog" 23 65 6 \
--menu "Welcome to llama.cpp Dialog" $DIALOG_HEIGHT $DIALOG_WIDTH 8 \
"1" "Model Selection" \
"2" "Multimodal Model Selection" \
"3" "Options" \
@ -399,11 +489,14 @@ show_main_menu() {
exit_status=$?
exec 3>&-
# Check whether user has selected 'Exit'
# Check whether user has unintentionally selected 'Exit'
if [ $exit_status = 1 ]; then
confirm_exit
if [ $? = 0 ]; then
clear
exit
fi
else
# Call up the corresponding function based on the selection
case $selection in
@ -417,6 +510,7 @@ show_main_menu() {
8) start_server ;;
*) clear ;;
esac
fi
done
}