diff --git a/examples/server/public/index.html b/examples/server/public/index.html index 94284d659..84faa1e4d 100644 --- a/examples/server/public/index.html +++ b/examples/server/public/index.html @@ -32,11 +32,11 @@ const params = signal({ n_predict: 358, - temperature: 0.7, + temperature: 1.0, // adapt to optimized min-p requierements repeat_last_n: 256, // 0 = disable penalty, -1 = context size repeat_penalty: 1.18, // 1.0 = disabled - top_k: 1, // <= 0 to use vocab size - top_p: 0.5, // 1.0 = disabled + top_k: 0, // <= 0 to use vocab size + top_p: 1.0, // 1.0 = disabled min_p: 0.05, // 0 = disabled tfs_z: 1.0, // 1.0 = disabled typical_p: 1.0, // 1.0 = disabled @@ -658,12 +658,14 @@ async function updateSystemLanguage(event) { + + @@ -773,21 +775,21 @@ async function updateSystemLanguage(event) {
${IntField({ label: "Prediction", max: 2048, min: -1, step: 16, name: "n_predict", value: params.value.n_predict, })} - ${FloatField({ label: "Temperature", max: 1.5, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature })} - ${IntField({ label: "Top-K", max: 100, min: -1, step: 1, name: "top_k", value: params.value.top_k })} - ${FloatField({ label: "Repetition Penalty", max: 2.0, min: 0.0, name: "repeat_penalty", step: 0.01, value: params.value.repeat_penalty })} ${FloatField({ label: "Min-P sampling", max: 1.0, min: 0.0, name: "min_p", step: 0.01, value: params.value.min_p })} + ${FloatField({ label: "Repetition Penalty", max: 2.0, min: 0.0, name: "repeat_penalty", step: 0.01, value: params.value.repeat_penalty })} + ${FloatField({ label: "Temperature", max: 1.5, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature })}
Further Options
- ${FloatField({ label: "Top-P", max: 1.0, min: 0.0, name: "top_p", step: 0.01, value: params.value.top_p })} + ${IntField({ label: "Top-K", max: 100, min: -1, step: 1, name: "top_k", value: params.value.top_k })} ${IntField({ label: "Penalize Last N", max: 2048, min: 0, step: 16, name: "repeat_last_n", value: params.value.repeat_last_n })} - ${FloatField({ label: "TFS-Z", max: 1.0, min: 0.0, name: "tfs_z", step: 0.01, value: params.value.tfs_z })} + ${FloatField({ label: "Top-P", max: 1.0, min: 0.0, name: "top_p", step: 0.01, value: params.value.top_p })} ${FloatField({ label: "Presence Penalty", max: 1.0, min: 0.0, name: "presence_penalty", step: 0.01, value: params.value.presence_penalty })} - ${FloatField({ label: "Typical-P", max: 1.0, min: 0.0, name: "typical_p", step: 0.01, value: params.value.typical_p })} + ${FloatField({ label: "TFS-Z", max: 1.0, min: 0.0, name: "tfs_z", step: 0.01, value: params.value.tfs_z })} ${FloatField({ label: "Frequency Penalty", max: 1.0, min: 0.0, name: "frequency_penalty", step: 0.01, value: params.value.frequency_penalty })} + ${FloatField({ label: "Typical-P", max: 1.0, min: 0.0, name: "typical_p", step: 0.01, value: params.value.typical_p })}

diff --git a/examples/server/public/promptFormats.js b/examples/server/public/promptFormats.js index 6bbecd60f..f6b179a81 100644 --- a/examples/server/public/promptFormats.js +++ b/examples/server/public/promptFormats.js @@ -12,6 +12,12 @@ export const promptFormats = { char: "Response", user: "Instruction" }, +"bakllava": { + template: "{{history}}{{char}}:", + historyTemplate: "{{name}}: {{message}}\n", + char: "ASSISTANT", + user: "USER" +}, "chatml": { template: "<|im_start|>system\n{{prompt}}<|im_end|>\n{{history}}\n<|im_start|>{{char}}", historyTemplate: "<|im_start|>{{user}}\n{{message}}<|im_end|>", @@ -54,6 +60,18 @@ export const promptFormats = { char: "ASSISTANT", user: "USER" }, +"llama2": { + template: "[INST] <>\n{{prompt}}\n<>\n\n{{history}} [/INST] {{char}} [INST] ", + historyTemplate: "{{name}}: {{message}} [/INST]", + char: "llama", + user: "user" +}, +"llava": { + template: "{{history}}{{char}}:", + historyTemplate: "{{name}}: {{message}}\n", + char: "ASSISTANT", + user: "USER" +}, "leoHessianai": { template: "<|im_start|>system\n{{prompt}}<|im_end|>\n{{history}}\n<|im_start|>{{char}}", historyTemplate: "<|im_start|>{{user}}\n{{message}}<|im_end|>", @@ -66,12 +84,6 @@ export const promptFormats = { char: "ASSISTANT", user: "USER" }, -"llama2": { - template: "[INST] <>\n{{prompt}}\n<>\n\n{{history}} [/INST] {{char}} [INST] ", - historyTemplate: "{{name}}: {{message}} [/INST]", - char: "llama", - user: "user" -}, "marx": { template: "{{history}}\n{{char}}:", historyTemplate: "{{name}}: {{message}}", diff --git a/examples/start-server.sh b/examples/start-server.sh index 2cd7f4807..e775f96b4 100755 --- a/examples/start-server.sh +++ b/examples/start-server.sh @@ -1,5 +1,13 @@ #!/bin/bash +# Determine the size of the terminal window +TERMINAL_HEIGHT=$(tput lines) +TERMINAL_WIDTH=$(tput cols) + +# Calculate a size for the dialog box as a percentage of the terminal to make sure it fits +DIALOG_HEIGHT=$((TERMINAL_HEIGHT * 5 / 8)) # approx. golden ratio +DIALOG_WIDTH=$((TERMINAL_WIDTH * 5 / 8)) # approx. golden ratio + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" # Set default values @@ -71,7 +79,7 @@ fi model_selection_warning() { - dialog --title "Hinweis" --msgbox "\n\n\nPlease note: To navigate to a folder, please press the space bar twice. To return to a higher-level folder, press the Backspace key.\n\n\nAlternatively, you can also enter the desired path manually in the lower address field. \n\n\nOnly confirm your selection with the Enter key once you have selected the file – or the desired folder to be searched." 23 65 + dialog --title "Hinweis" --msgbox "\n\n\nPlease note!\n\nTo navigate to a folder, please press the space bar twice. To return to a higher-level folder, press the Backspace key.\n\n\nAlternatively, you can also enter the desired path manually in the lower path field. \n\n\nOnly confirm your selection with the Enter key once you have selected the file – or the desired folder to be searched recursively." $DIALOG_HEIGHT $DIALOG_WIDTH } @@ -85,7 +93,7 @@ model_selection() { model_path=$(dialog --backtitle "Model Selection" \ --title "Select Model File or Folder" \ - --fselect "$INITIAL_DIR" 23 65 \ + --fselect "$INITIAL_DIR" $DIALOG_HEIGHT $DIALOG_WIDTH \ 2>&1 1>&3) exit_status=$? exec 3>&- @@ -98,12 +106,19 @@ model_selection() { # If a folder has been selected, search for *.gguf files if [ -d "$model_path" ]; then model_files=($(find "$model_path" -name "*.gguf" 2>/dev/null)) + # Check whether files have been found + if [ ${#model_files[@]} -eq 0 ]; then + dialog --backtitle "Model Selection" \ + --title "No Models Found" \ + --msgbox "\n\n\nNo model files (*.gguf) were found in the selected directory." $DIALOG_HEIGHT $DIALOG_WIDTH + return + fi elif [ -f "$model_path" ]; then model_files=("$model_path") else dialog --backtitle "Model Selection" \ --title "Invalid Selection" \ - --msgbox "The selected path is not valid." 23 65 + --msgbox "\n\n\nThe selected path is not valid." $DIALOG_HEIGHT $DIALOG_WIDTH return fi @@ -111,7 +126,7 @@ model_selection() { exec 3>&1 model_choice=$(dialog --backtitle "Model Selection" \ --title "Select a Model File" \ - --menu "Choose one of the found models:" 23 65 4 \ + --menu "Choose one of the found models:" $DIALOG_HEIGHT $DIALOG_WIDTH \ $(for i in "${!model_files[@]}"; do echo "$((i+1))" "$(basename "${model_files[$i]}")"; done) \ 2>&1 1>&3) exit_status=$? @@ -135,7 +150,7 @@ multimodal_model_selection() { mmproj_path=$(dialog --backtitle "Multimodal Model Selection" \ --title "Select Multimodal Model File or Folder" \ - --fselect "$INITIAL_DIR" 23 65 \ + --fselect "$INITIAL_DIR" $DIALOG_HEIGHT $DIALOG_WIDTH \ 2>&1 1>&3) exit_status=$? exec 3>&- @@ -145,23 +160,30 @@ multimodal_model_selection() { return fi - # If a folder has been selected, search for *.bin files - if [ -d "$mmproj_path" ]; then - multi_modal_files=($(find "$mmproj_path" -name "*.bin" 2>/dev/null)) - elif [ -f "$mmproj_path" ]; then - multi_modal_files=("$mmproj_path") - else - dialog --backtitle "Multimodal Model" \ - --title "Invalid Selection" \ - --msgbox "The selected path is not valid." 7 50 +# If a folder has been selected, search for *.bin files +if [ -d "$mmproj_path" ]; then + multi_modal_files=($(find "$mmproj_path" -name "*.bin" 2>/dev/null)) + # Check whether files have been found + if [ ${#multi_modal_files[@]} -eq 0 ]; then + dialog --backtitle "Multimodal Model Selection" \ + --title "No Multimodal Models Found" \ + --msgbox "\n\n\nNo multimodal model files (*.bin) were found in the selected directory." $DIALOG_HEIGHT $DIALOG_WIDTH return fi +elif [ -f "$mmproj_path" ]; then + multi_modal_files=("$mmproj_path") +else + dialog --backtitle "Multimodal Model Selection" \ + --title "Invalid Selection" \ + --msgbox "\n\n\nThe selected path is not valid." $DIALOG_HEIGHT $DIALOG_WIDTH + return +fi # Selection menu for models found exec 3>&1 multi_modal_choice=$(dialog --backtitle "Multimodal Model" \ --title "Select a Model File" \ - --menu "Choose one of the found models:" 23 65 4 \ + --menu "Choose one of the found models:" $DIALOG_HEIGHT $DIALOG_WIDTH 8 \ $(for i in "${!multi_modal_files[@]}"; do echo "$((i+1))" "$(basename "${multi_modal_files[$i]}")"; done) \ 2>&1 1>&3) exit_status=$? @@ -184,7 +206,7 @@ options() { form_values=$(dialog --backtitle "Options Configuration" \ --title "Set Options" \ --form "Enter the values for the following options:" \ - 23 65 0 \ + $DIALOG_HEIGHT $DIALOG_WIDTH 0 \ "Number of Threads (-t):" 1 1 "$threads" 1 25 25 5 \ "Context Size (-c):" 2 1 "$ctx_size" 2 25 25 5 \ "Batch Size (-b):" 3 1 "$batch_size" 3 25 25 5 \ @@ -214,7 +236,7 @@ further_options() { exec 3>&1 choices=$(dialog --backtitle "Further Options" \ --title "Boolean Options" \ - --checklist "Select options:" 23 65 3 \ + --checklist "Select options:" $DIALOG_HEIGHT $DIALOG_WIDTH 3 \ "1" "Continuous Batching (-cb)" $cb_value \ "2" "Memory Lock (--mlock)" $mlock_value \ "3" "No Memory Map (--no-mmap)" $no_mmap_value \ @@ -248,10 +270,10 @@ advanced_options() { advanced_values=$(dialog --backtitle "Advanced Options" \ --title "Advanced Server Configuration" \ --form "Enter the advanced configuration options:" \ - 23 65 0 \ - "Host IP:" 1 1 "$host" 1 15 15 0 \ - "Port:" 2 1 "$port" 2 15 5 0 \ - "Additional Options:" 3 1 "$advanced_options" 3 15 30 0 \ + $DIALOG_HEIGHT $DIALOG_WIDTH 0 \ + "Host IP:" 1 1 "$host" 1 25 15 0 \ + "Port:" 2 1 "$port" 2 25 5 0 \ + "Additional Options:" 3 1 "$advanced_options" 3 25 30 0 \ 2>&1 1>&3) exit_status=$? exec 3>&- @@ -272,7 +294,7 @@ save_config() { exec 3>&1 config_file=$(dialog --backtitle "Save Configuration" \ --title "Save Configuration File" \ - --fselect "$SCRIPT_DIR/" 23 65 \ + --fselect "$SCRIPT_DIR/" $DIALOG_HEIGHT $DIALOG_WIDTH \ 2>&1 1>&3) exit_status=$? exec 3>&- @@ -282,10 +304,13 @@ save_config() { return fi +absolute_model_path=$(get_absolute_path "$model_path") +absolute_mmproj_path=$(get_absolute_path "$mmproj_path") + # Saving the configuration to the file with absolute paths using custom function cat > "$config_file" << EOF -model_path=$(get_absolute_path "$model_path") -mmproj_path=$(get_absolute_path "$mmproj_path") +model_path=$absolute_model_path +mmproj_path=$absolute_mmproj_path threads=$threads ctx_size=$ctx_size batch_size=$batch_size @@ -300,7 +325,7 @@ EOF dialog --backtitle "Save Configuration" \ --title "Configuration Saved" \ - --msgbox "Configuration has been saved to $config_file" 7 50 + --msgbox "\n\n\nYour configuration has been saved to\n\n$config_file" $DIALOG_HEIGHT $DIALOG_WIDTH } @@ -310,7 +335,7 @@ load_config() { exec 3>&1 config_file=$(dialog --backtitle "Load Configuration" \ --title "Load Configuration File" \ - --fselect "$SCRIPT_DIR/" 23 65 \ + --fselect "$SCRIPT_DIR/" $DIALOG_HEIGHT $DIALOG_WIDTH \ 2>&1 1>&3) exit_status=$? exec 3>&- @@ -324,23 +349,73 @@ load_config() { if [ ! -f "$config_file" ]; then dialog --backtitle "Load Configuration" \ --title "File Not Found" \ - --msgbox "The file $config_file was not found." 7 50 + --msgbox "\n\n\nThe file $config_file was not found." $DIALOG_HEIGHT $DIALOG_WIDTH return fi # Load configuration from the file source "$config_file" + # Convert model paths to absolute paths + model_path=$(get_absolute_path "$model_path") + mmproj_path=$(get_absolute_path "$mmproj_path") + dialog --backtitle "Load Configuration" \ --title "Configuration Loaded" \ - --msgbox "Configuration has been loaded from $config_file" 7 50 + --msgbox "\n\n\nConfiguration has been loaded successfully from\n\n$config_file" $DIALOG_HEIGHT $DIALOG_WIDTH +} + + + +# Checking the existence of the server executable +check_server_executable() { + local server_executable="$SCRIPT_DIR/../server" + local makefile_path="$SCRIPT_DIR/../Makefile" + local cmake_lists_path="$SCRIPT_DIR/../CMakeLists.txt" + + if [ ! -f "$server_executable" ]; then + # Server executable does not exist, check Makefile and CMakeLists.txt + if [ -f "$makefile_path" ] && [ -f "$cmake_lists_path" ]; then + # Offer the user to build the server now + exec 3>&1 + response=$(dialog --title "Server Executable Missing" \ + --yesno "\n\n\nThe server executable does not exist. Would you like to run 'make' to build it? Note: This will build a basic server without GPU acceleration. Please read the documentation if you need more options and run the build process manually." $DIALOG_HEIGHT $DIALOG_WIDTH \ + 2>&1 1>&3) + exit_status=$? + exec 3>&- + + if [ $exit_status = 0 ]; then + # User has agreed, run 'make' + (cd "$SCRIPT_DIR/.." && make) + # Check if 'make' was successful + if [ ! -f "$server_executable" ]; then + dialog --title "Build Failed" --msgbox "\n\n\nThe server could not be built. Please check the build process manually." $DIALOG_HEIGHT $DIALOG_WIDTH + exit 1 + fi + else + # User has rejected or pressed ESC + dialog --title "Build Canceled" --msgbox "\n\n\nServer build was canceled. Cannot start the server without the executable." $DIALOG_HEIGHT $DIALOG_WIDTH + exit 1 + fi + else + # Makefile and CMakeLists.txt do not exist + dialog --title "Critical Error" --msgbox "\n\n\nMakefile and CMakeLists.txt are missing. This script may not be in the correct directory. Please make sure this script is in its correct directory." $DIALOG_HEIGHT $DIALOG_WIDTH + exit 1 + fi + fi } confirm_and_start_server() { + # Check whether model_path refers to a valid .gguf file + if [[ ! "$model_path" =~ \.gguf$ ]] || [ ! -f "$model_path" ]; then + dialog --title "Invalid Model File" --msgbox "\n\n\nThe selected model file ($model_path) is not valid or does not end with .gguf or a model file was not selected yet.\n\n\nPlease select a valid .gguf model file." $DIALOG_HEIGHT $DIALOG_WIDTH + return 1 + fi + # Show the compiled command in a dialog box - dialog --title "Server Start Confirmation" --yesno "The server will be started with the following command:\n\n$cmd\n\nDo not forget to close the server with Ctrl+C as soon as you are finished.\n\nWould you like to continue?" 23 65 + dialog --title "Server Start Confirmation" --yesno "\n\n\nThe server will be started with the following command:\n\n$cmd\n\nDo not forget to close the server with Ctrl+C as soon as you are finished.\n\nWould you like to continue?" $DIALOG_HEIGHT $DIALOG_WIDTH # Check exit status of dialog response=$? @@ -377,6 +452,21 @@ start_server() { +# Function to confirm exit +confirm_exit() { + exec 3>&1 + selection=$(dialog \ + --backtitle "Confirm Exit" \ + --title "Are you sure?" \ + --yesno "Are you sure you want to exit?" 7 60 \ + 2>&1 1>&3) + exit_status=$? + exec 3>&- + return $exit_status +} + + + # Function to show the main menu show_main_menu() { while true; do @@ -386,7 +476,7 @@ show_main_menu() { --title "Main Menu" \ --clear \ --cancel-label "Exit" \ - --menu "Welcome to llama.cpp Dialog" 23 65 6 \ + --menu "Welcome to llama.cpp Dialog" $DIALOG_HEIGHT $DIALOG_WIDTH 8 \ "1" "Model Selection" \ "2" "Multimodal Model Selection" \ "3" "Options" \ @@ -399,24 +489,28 @@ show_main_menu() { exit_status=$? exec 3>&- - # Check whether user has selected 'Exit' + # Check whether user has unintentionally selected 'Exit' if [ $exit_status = 1 ]; then - clear - exit - fi + confirm_exit + if [ $? = 0 ]; then + clear + exit + fi + else - # Call up the corresponding function based on the selection - case $selection in - 1) model_selection_warning; model_selection ;; - 2) model_selection_warning; multimodal_model_selection ;; - 3) options ;; - 4) further_options ;; - 5) advanced_options ;; - 6) save_config ;; - 7) load_config ;; - 8) start_server ;; - *) clear ;; - esac + # Call up the corresponding function based on the selection + case $selection in + 1) model_selection_warning; model_selection ;; + 2) model_selection_warning; multimodal_model_selection ;; + 3) options ;; + 4) further_options ;; + 5) advanced_options ;; + 6) save_config ;; + 7) load_config ;; + 8) start_server ;; + *) clear ;; + esac + fi done }