diff --git a/inference_test/chat.sh b/inference_test/chat.sh new file mode 100755 index 000000000..c535f4f4b --- /dev/null +++ b/inference_test/chat.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# +# Temporary script - will be removed in the future +# + +cd `dirname $0` +cd .. + +./main -m ./models/llama-2-7b-pubmed-qa-211k_q8_0.gguf\ + --color \ + -f ./prompts/alpaca.txt \ + --ctx_size 2048 \ + -n -1 \ + -ins -b 256 \ + --top_k 10000 \ + --temp 0.2 \ + --repeat_penalty 1.1 \ + -t 7