64 lines
No EOL
1.9 KiB
Bash
Executable file
64 lines
No EOL
1.9 KiB
Bash
Executable file
#! /usr/bin/env bash
|
|
# Compare multiple models by running them with the same questions
|
|
|
|
NUMBEROFCHOICES=4
|
|
SELECTIONS=()
|
|
declare -a SUMS=()
|
|
|
|
# Get the list of models
|
|
CHOICES=$(ollama list | awk '{print $1}')
|
|
|
|
# Select which models to run as a comparison
|
|
echo "Select $NUMBEROFCHOICES models to compare:"
|
|
select ITEM in $CHOICES; do
|
|
if [[ -n $ITEM ]]; then
|
|
echo "You have selected $ITEM"
|
|
SELECTIONS+=("$ITEM")
|
|
((COUNT++))
|
|
if [[ $COUNT -eq $NUMBEROFCHOICES ]]; then
|
|
break
|
|
fi
|
|
else
|
|
echo "Invalid selection"
|
|
fi
|
|
done
|
|
|
|
# Loop through each of the selected models
|
|
for ITEM in "${SELECTIONS[@]}"; do
|
|
echo "--------------------------------------------------------------"
|
|
echo "Loading the model $ITEM into memory"
|
|
ollama run "$ITEM" ""
|
|
echo "--------------------------------------------------------------"
|
|
echo "Running the questions through the model $ITEM"
|
|
COMMAND_OUTPUT=$(ollama run "$ITEM" --verbose < sourcequestions 2>&1| tee /dev/stderr)
|
|
|
|
# eval duration is sometimes listed in seconds and sometimes in milliseconds.
|
|
# Add up the values for each model
|
|
SUM=$(echo "$COMMAND_OUTPUT" | awk '
|
|
/eval duration:/ {
|
|
value = $3
|
|
if (index(value, "ms") > 0) {
|
|
gsub("ms", "", value)
|
|
value /= 1000
|
|
} else {
|
|
gsub("s", "", value)
|
|
}
|
|
sum += value
|
|
}
|
|
END { print sum }')
|
|
|
|
|
|
SUMS+=("All questions for $ITEM completed in $SUM seconds")
|
|
done
|
|
|
|
echo ""
|
|
echo "--------------------------------------------------------------"
|
|
echo -e "Sums of eval durations for each run:"
|
|
for val in "${SUMS[@]}"; do
|
|
echo "$val"
|
|
done
|
|
|
|
echo "--------------------------------------------------------------"
|
|
echo "Comparison complete. Now you can decide"
|
|
echo "which model is best."
|
|
echo "--------------------------------------------------------------" |