matthias/local_ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding README and logo

Browse Source
This commit is contained in:
ml
2026-10-03 07:05:07 +02:00
parent 899a6cd9cd
commit cf604c5927
3 changed files with 95 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
setup.sh
+18 -6
View File
@@ -144,6 +144,12 @@ elif awk -v b="$budget" 'BEGIN{exit !(b>=40)}'; then CONTEXT=32768
elif awk -v b="$budget" 'BEGIN{exit !(b>=20)}'; then CONTEXT=16384
else CONTEXT=8192; fi
# Rough share (%) of a model's layers that fit in VRAM when a GPU is used and the
# model is bigger than the VRAM budget (Ollama then offloads part of the layers).
gpu_share() {
awk -v m="$1" -v b="$gpu_budget" -v u="$gpu_usable" 'BEGIN{ if(!u||m<=b){print 0; exit} s=int(b/m*100/5)*5; print (s<10?0:s) }'
}
# ---------------------------------------------------------------- catalogue
mapfile -t rows < <(grep -v '^#' models.tsv | grep -v '^[[:space:]]*$' | sort -t$'\t' -k5,5n)
tags=(); status=(); speed=(); rec=-1
@@ -154,11 +160,13 @@ for i in "${!rows[@]}"; do
fits_hard_ram=$(awk -v m="$mem" -v b="$hard_budget" 'BEGIN{print (m<=b)}')
fits_gpu=$(awk -v m="$mem" -v b="$gpu_budget" -v u="$gpu_usable" 'BEGIN{print (u && m<=b)}')
fits_hard_gpu=$(awk -v m="$mem" -v b="$gpu_hard" -v u="$gpu_usable" 'BEGIN{print (u && m<=b)}')
share=$(gpu_share "$mem")
if [ "$share" -gt 0 ]; then hyb="hybrid ~${share}% GPU"; else hyb=""; fi
if [ "$fits_gpu" = 1 ]; then status[i]="fits"; speed[i]="fast (GPU)"
elif [ "$fits_hard_gpu" = 1 ]; then status[i]="tight"; speed[i]="tight (GPU)"
elif [ "$fits_ram" = 1 ] && [ "$cpu_ok" = yes ]; then status[i]="fits"; speed[i]="ok (CPU)"
elif [ "$fits_ram" = 1 ]; then status[i]="slow"; speed[i]="slow (CPU)"
elif [ "$fits_hard_ram" = 1 ]; then status[i]="tight"; speed[i]="tight (CPU)"
elif [ "$fits_ram" = 1 ] && [ "$cpu_ok" = yes ]; then status[i]="fits"; speed[i]="${hyb:-ok (CPU)}"
elif [ "$fits_ram" = 1 ]; then status[i]="slow"; speed[i]="slow ${hyb:-(CPU)}"
elif [ "$fits_hard_ram" = 1 ]; then status[i]="tight"; speed[i]="tight ${hyb:-(CPU)}"
else status[i]="too big"; speed[i]="-"; fi
[ "${status[$i]}" = fits ] && rec=$i # sorted by rank, so last "fits" wins
done
@@ -204,14 +212,15 @@ fi
# --------------------------------------------------------------------- menu
if [ -z "$FORCE_MODEL" ]; then
echo "=== Models (budget: ${budget} GB RAM$([ "$gpu_usable" = 1 ] && echo ", ${gpu_budget} GB VRAM")) ==="
printf " %-3s %-24s %7s %7s %-9s %-11s %s\n" "#" "model" "disk" "memory" "fit" "speed" ""
printf " %-3s %-24s %7s %7s %-9s %-21s %s\n" "#" "model" "disk" "memory" "fit" "speed" ""
for i in "${!rows[@]}"; do
IFS=$'\t' read -r tag disk mem cpu_ok rank desc <<<"${rows[$i]}"
mark=" "; [ "$i" = "$rec" ] && mark="*"
printf " %s%-3s %-24s %5s GB %5s GB %-9s %-11s %s\n" "$mark" "$((i+1))" "$tag" "$disk" "$mem" "${status[$i]}" "${speed[$i]}" "$desc"
printf " %s%-3s %-24s %5s GB %5s GB %-9s %-21s %s\n" "$mark" "$((i+1))" "$tag" "$disk" "$mem" "${status[$i]}" "${speed[$i]}" "$desc"
done
echo
echo " * = recommended for this machine: ${tags[$rec]}"
[ "$gpu_usable" = 1 ] && echo " hybrid ~N% GPU = model is bigger than VRAM; Ollama puts about N% of it on the GPU, the rest runs on the CPU (rough estimate)."
echo
if [ "$ASSUME_YES" = 1 ]; then choice=""; else read -r -p "Select model number [$((rec+1))]: " choice; fi
choice=${choice:-$((rec+1))}
@@ -265,7 +274,10 @@ case "$MODEL" in
*:1.5b|*:3b) EDIT_HINT=$'\nSmall models struggle with diff-style edits; add `--edit-format whole`.\n' ;;
*) EDIT_HINT="" ;;
esac
if [ "$USE_GPU" = 1 ]; then SPEED="Expect fast responses."
sel_mem=0; for r in "${rows[@]}"; do [ "$(cut -f1 <<<"$r")" = "$MODEL" ] && sel_mem=$(cut -f3 <<<"$r"); done
sel_share=$(gpu_share "$sel_mem")
if [ "$USE_GPU" = 1 ] && [ "$sel_share" -gt 0 ]; then SPEED="Hybrid mode: $MODEL is bigger than your VRAM, so roughly ${sel_share}% of it runs on the GPU and the rest on the CPU. Faster than CPU-only, slower than a model that fits entirely in VRAM. Check with: docker exec $CONTAINER_NAME ollama ps"
elif [ "$USE_GPU" = 1 ]; then SPEED="The model fits in VRAM: expect fast responses."
else SPEED="CPU-only: expect a few tokens/second, and a slow first response while the model loads into RAM. Keep prompts and added files small."; fi
{