Move requirements into ./requirements

check-requirements.sh

@@ -32,6 +32,10 @@ log() {
     >&2 printf "$level: $format\n" "$@"
 }
 
+debug () {
+    log 'DEBUG' "$@"
+}
+
 info() {
     log 'INFO' "$@"
 }
@@ -72,12 +76,13 @@ else
 fi
 
 set -eu -o pipefail
-this="$(realpath "$0")"
+this="$(realpath "$0")"; readonly this
-readonly this
 cd "$(dirname "$this")"
 
 shellcheck "$this"
 
+readonly reqs_dir='./requirements'
+
 workdir=
 if [[ -n ${1+x} ]]; then
     arg_dir="$(realpath "$1")"
@@ -115,12 +120,13 @@ check_requirements() {
 
 check_convert_script() {
     assert_arg_count 1 "$@"
-    local py="$1"
+    local py="$1"; shift                     # e.g. ./convert-hf-to-gguf.py
-    local pyname="${py%.py}"
+    local pyname; pyname="$(basename "$py")" # e.g. convert-hf-to-gguf.py
+    pyname="${pyname%.py}"                   # e.g. convert-hf-to-gguf
 
     info "$py: beginning check"
 
-    local reqs="requirements-$pyname.txt"
+    local reqs="$reqs_dir/requirements-$pyname.txt"
     if [[ ! -r "$reqs" ]]; then
         fatal "$py missing requirements. Expected: $reqs"
     fi
@@ -144,13 +150,19 @@ check_convert_script() {
     info "$py: imports OK"
 }
 
-# Check requirements.txt
+# Check that all sub-requirements are added to top-level requirements.txt
+for req in "$reqs_dir"/*; do
+    if ! grep -qFe "$req" ./requirements.txt; then
+        fatal "$req needs to be added to ./requirements.txt"
+    fi
+done
+
 all_venv="$workdir/all-venv"
 python3 -m venv "$all_venv"
-check_requirements "$all_venv" 'requirements.txt'
+check_requirements "$all_venv" './requirements.txt'
 
-check_convert_script 'convert.py'
+check_convert_script './convert.py'
-for py in convert-*.py; do
+for py in ./convert-*.py;do
     check_convert_script "$py"
 done
 

requirements-convert-llama-ggml-to-gguf.txt

@@ -1 +0,0 @@
--r requirements-convert.txt

requirements-convert-lora-to-ggml.txt

@@ -1,2 +0,0 @@
--r requirements-convert.txt
-torch==2.1.1

requirements-convert-persimmon-to-gguf.txt

@@ -1,2 +0,0 @@
--r requirements-convert.txt
-torch==2.1.1

requirements-hf-to-gguf.txt

@@ -1,3 +0,0 @@
--r requirements.txt
-torch==2.1.1
-transformers==4.35.2

requirements.txt

@@ -4,8 +4,9 @@
 # Package versions must stay compatible across all top-level python scripts.
 #
 
--r requirements-convert.txt
+-r ./requirements/requirements-convert.txt
 
--r requirements-convert-hf-to-gguf.txt
+-r ./requirements/requirements-convert-hf-to-gguf.txt
--r requirements-convert-lora-to-ggml.txt
+-r ./requirements/requirements-convert-llama-ggml-to-gguf.txt
--r requirements-convert-persimmon-to-gguf.txt
+-r ./requirements/requirements-convert-lora-to-ggml.txt
+-r ./requirements/requirements-convert-persimmon-to-gguf.txt

requirements/requirements-convert-hf-to-gguf.txt

@@ -1,3 +1,3 @@
--r requirements-convert.txt
+-r ./requirements-convert.txt
 torch==2.1.1
 transformers==4.35.2

requirements/requirements-convert-llama-ggml-to-gguf.txt

@@ -0,0 +1 @@
+-r ./requirements-convert.txt

requirements/requirements-convert-lora-to-ggml.txt

@@ -0,0 +1,2 @@
+-r ./requirements-convert.txt
+torch==2.1.1

requirements/requirements-convert-persimmon-to-gguf.txt

@@ -0,0 +1,2 @@
+-r ./requirements-convert.txt
+torch==2.1.1