diff --git a/models.tsv b/models.tsv index 669fd25..47b71fb 100644 --- a/models.tsv +++ b/models.tsv @@ -3,10 +3,15 @@ # cpu_ok: yes = usable speed without a GPU (small dense or MoE with few active params). # rank: higher = better quality for coding; the recommender picks the highest rank that fits. qwen2.5-coder:1.5b 1.0 2.0 yes 10 Tiny; very fast, weak edits. For very small machines. +granite-code:3b 2.0 3.0 yes 15 IBM, Apache-2.0 (fully permissive); a bit weaker than Qwen at this size. qwen2.5-coder:3b 1.9 3.0 yes 20 Small and fast; OK for simple edits. +granite-code:8b 4.6 6.2 yes 27 IBM, Apache-2.0 (fully permissive); solid mid-size coder. qwen2.5-coder:7b 4.7 6.5 yes 30 Best all-round choice for laptops / CPU-only. -deepseek-coder-v2:16b 8.9 11.5 yes 35 MoE (2.4B active): fast on CPU, needs ~12 GB RAM. +granite-code:20b 12 15 no 33 IBM, Apache-2.0 (fully permissive); dense, wants a GPU (~15 GB). +deepseek-coder-v2:16b 8.9 11.5 yes 35 MoE (Mixture of Experts, 2.4B active): fast on CPU, needs ~12 GB RAM. qwen2.5-coder:14b 9.0 11.5 no 40 Noticeably smarter than 7B; wants a GPU (~12 GB VRAM). -gpt-oss:20b 14 17 yes 50 MoE reasoning model; needs ~17 GB RAM or VRAM. -qwen3-coder:30b 19 23 yes 60 MoE (3B active): strong coder, fast on CPU with 24+ GB RAM. +gpt-oss:20b 14 17 yes 50 MoE (Mixture of Experts) reasoning model; needs ~17 GB RAM or VRAM. +granite-code:34b 19 23 no 55 IBM, Apache-2.0 (fully permissive); dense, needs a 24 GB GPU. +devstral:24b 14 17 no 58 Mistral+All Hands, Apache-2.0; built for agentic multi-file edits, needs ~17 GB RAM or VRAM. +qwen3-coder:30b 19 23 yes 60 MoE (Mixture of Experts, 3B active): strong coder, fast on CPU with 24+ GB RAM. qwen2.5-coder:32b 20 24 no 70 Top-quality dense coder; needs a 24 GB GPU. diff --git a/run-ai-image.sh b/run-ai-image.sh index 2b59137..e349575 100755 --- a/run-ai-image.sh +++ b/run-ai-image.sh @@ -10,6 +10,7 @@ cd "$(dirname "$0")" [ -f .ai-config ] || { echo "No .ai-config yet - run ./setup.sh first." >&2; exit 1; } # shellcheck disable=SC1091 source .ai-config +PORT=${AI_PORT:-$PORT} # override for one run: AI_PORT=11500 ./run-ai-image.sh running() { [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null)" = "true" ]; } exists() { docker inspect "$CONTAINER_NAME" >/dev/null 2>&1; } @@ -28,6 +29,12 @@ if running; then echo "Already running on http://127.0.0.1:$PORT" else exists && docker rm -f "$CONTAINER_NAME" >/dev/null + if (exec 3<>"/dev/tcp/127.0.0.1/$PORT") 2>/dev/null; then + echo "Port $PORT is already in use by something else (a host Ollama, another container?):" >&2 + docker ps --filter "publish=$PORT" --format ' container {{.Names}}' >&2 || true + echo "Re-run ./setup.sh to pick a free port, or use: AI_PORT= ./run-ai-image.sh" >&2 + exit 1 + fi args=(-d --name "$CONTAINER_NAME" -p "127.0.0.1:$PORT:11434" --restart unless-stopped) [ "$USE_GPU" = "1" ] && args+=(--gpus all) docker run "${args[@]}" "$IMAGE_NAME" >/dev/null diff --git a/setup.sh b/setup.sh index ac2c394..3e5adbe 100755 --- a/setup.sh +++ b/setup.sh @@ -4,24 +4,29 @@ # .ai-config (settings used by build-ai-image.sh / run-ai-image.sh) # RUN_AIDER.md (how to run aider against the container on this machine) # -# Usage: ./setup.sh [--model ] [--gpu|--no-gpu] [--yes] +# Usage: ./setup.sh [--model ] [--gpu|--no-gpu] [--port ] [--yes] set -euo pipefail cd "$(dirname "$0")" +# Numeric formatting/parsing below assumes '.' as the decimal point (awk, sort -n). +# A locale like de_DE (comma decimal separator) would corrupt those comparisons. +export LC_ALL=C -FORCE_MODEL=""; FORCE_GPU=""; ASSUME_YES=0 +FORCE_MODEL=""; FORCE_GPU=""; ASSUME_YES=0; FORCE_PORT="" while [ $# -gt 0 ]; do case "$1" in --model) FORCE_MODEL="${2:?--model needs a value}"; shift ;; --gpu) FORCE_GPU=1 ;; --no-gpu) FORCE_GPU=0 ;; + --port) FORCE_PORT="${2:?--port needs a value}"; shift ;; --yes|-y) ASSUME_YES=1 ;; - -h|--help) sed -n '2,8p' "$0" | sed 's/^# \?//'; exit 0 ;; + -h|--help) sed -n '2,9p' "$0" | sed 's/^# \?//'; exit 0 ;; *) echo "Unknown option: $1" >&2; exit 1 ;; esac shift done -IMAGE_NAME="local-ai"; CONTAINER_NAME="local-ai"; PORT=11434 +IMAGE_NAME="local-ai"; CONTAINER_NAME="local-ai"; PORT="" +port_busy() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; } NOTES=() # machine-specific remarks, echoed to the terminal and RUN_AIDER.md # Distro-specific commands to make an NVIDIA GPU usable from Docker. @@ -127,9 +132,12 @@ echo # ------------------------------------------------ memory budget / context # What a model must fit into: VRAM when a GPU is used, else RAM minus headroom -# for the OS, aider and the editor. +# for the OS, aider and the editor. hard_budget/gpu_hard have almost no headroom - +# a model only fitting there is "tight" (usable, but close other apps first). budget=$(awk -v r="$ram_gb" 'BEGIN {b=r-4; if (b<1) b=1; printf "%.1f", b}') +hard_budget=$(awk -v r="$ram_gb" 'BEGIN {b=r-1; if (b<1) b=1; printf "%.1f", b}') gpu_budget=$(awk -v v="$vram_gb" 'BEGIN {printf "%.1f", v*0.9}') +gpu_hard=$(awk -v v="$vram_gb" 'BEGIN {printf "%.1f", v*0.98}') if awk -v b="$budget" 'BEGIN{exit !(b<6)}'; then CONTEXT=4096 elif awk -v b="$budget" 'BEGIN{exit !(b>=40)}'; then CONTEXT=32768 @@ -143,10 +151,14 @@ for i in "${!rows[@]}"; do IFS=$'\t' read -r tag disk mem cpu_ok rank desc <<<"${rows[$i]}" tags[i]=$tag fits_ram=$(awk -v m="$mem" -v b="$budget" 'BEGIN{print (m<=b)}') + fits_hard_ram=$(awk -v m="$mem" -v b="$hard_budget" 'BEGIN{print (m<=b)}') fits_gpu=$(awk -v m="$mem" -v b="$gpu_budget" -v u="$gpu_usable" 'BEGIN{print (u && m<=b)}') + fits_hard_gpu=$(awk -v m="$mem" -v b="$gpu_hard" -v u="$gpu_usable" 'BEGIN{print (u && m<=b)}') if [ "$fits_gpu" = 1 ]; then status[i]="fits"; speed[i]="fast (GPU)" + elif [ "$fits_hard_gpu" = 1 ]; then status[i]="tight"; speed[i]="tight (GPU)" elif [ "$fits_ram" = 1 ] && [ "$cpu_ok" = yes ]; then status[i]="fits"; speed[i]="ok (CPU)" elif [ "$fits_ram" = 1 ]; then status[i]="slow"; speed[i]="slow (CPU)" + elif [ "$fits_hard_ram" = 1 ]; then status[i]="tight"; speed[i]="tight (CPU)" else status[i]="too big"; speed[i]="-"; fi [ "${status[$i]}" = fits ] && rec=$i # sorted by rank, so last "fits" wins done @@ -206,7 +218,10 @@ if [ -z "$FORCE_MODEL" ]; then [[ "$choice" =~ ^[0-9]+$ ]] && [ "$choice" -ge 1 ] && [ "$choice" -le "${#rows[@]}" ] || { echo "Invalid choice." >&2; exit 1; } sel=$((choice-1)) MODEL=${tags[$sel]} - if [ "${status[$sel]}" = "too big" ] || [ "${status[$sel]}" = "slow" ]; then + if [ "${status[$sel]}" = "tight" ]; then + echo "Warning: $MODEL is 'tight' on this machine (fits, but leaves little headroom - close other apps first)." + if [ "$ASSUME_YES" != 1 ]; then read -r -p "Continue anyway? [y/N] " a; [[ "$a" =~ ^[Yy] ]] || exit 1; fi + elif [ "${status[$sel]}" = "too big" ] || [ "${status[$sel]}" = "slow" ]; then echo "Warning: $MODEL is '${status[$sel]}' on this machine (may swap, crawl, or crash)." if [ "$ASSUME_YES" != 1 ]; then read -r -p "Continue anyway? [y/N] " a; [[ "$a" =~ ^[Yy] ]] || exit 1; fi fi @@ -219,6 +234,19 @@ if awk -v d="$sel_disk" -v f="$disk_gb" 'BEGIN{exit !(d*2>f)}'; then NOTES+=("Low disk space: ${disk_gb} GB free, the build needs roughly 2x the model size (${sel_disk} GB).") fi +# ------------------------------------------------------------ host port +# Default 11434 (Ollama's standard); if something else (e.g. a host Ollama) +# already uses it, take the next free port. +if [ -n "$FORCE_PORT" ]; then PORT=$FORCE_PORT +else + for PORT in $(seq 11434 11534); do + # our own container from an earlier run doesn't count as busy + [ "$(docker port "$CONTAINER_NAME" 11434/tcp 2>/dev/null | head -1 | sed 's/.*://')" = "$PORT" ] && break + port_busy "$PORT" || break + done + [ "$PORT" != 11434 ] && NOTES+=("Port 11434 is already in use on this machine, so the AI container will use port $PORT.") +fi + # ------------------------------------------------------------------- output USE_GPU=$gpu_usable sed -e "s|@@MODEL@@|$MODEL|g" -e "s|@@CONTEXT@@|$CONTEXT|g" Dockerfile.template > Dockerfile