diff --git a/README.md b/README.md index 892149e..0ea9127 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,91 @@ # local_ai +

+ local_ai: a laptop running Qwen, DeepSeek and Granite coding models +

+ Local coding AI for [aider](https://aider.chat): an Ollama container with a model chosen for *this* machine. Aider runs on the host and talks to the container. ```bash ./setup.sh # analyse machine, pick model, generate Dockerfile + RUN_AIDER.md ./build-ai-image.sh # build image (downloads the model) -./run-ai-image.sh # start container on 127.0.0.1:11434 +./run-ai-image.sh # start container (port 11434, or the next free one) # then follow RUN_AIDER.md to start aider ``` - `models.tsv` – model catalogue (edit to add models) - `Dockerfile.template` – container template; `setup.sh` fills in the model - `Dockerfile`, `.ai-config`, `RUN_AIDER.md` – generated per machine (git-ignored) -- `./setup.sh --model --yes` for non-interactive use +- `./setup.sh --model --port --gpu|--no-gpu --yes` for non-interactive use + +## What decides which model fits + +- **Memory is the limit.** A model must fit in VRAM (GPU) or RAM (CPU). `setup.sh` keeps + about 4 GB of RAM free for the OS, aider and your editor, and about 10% of VRAM. +- **Fits in VRAM = fast.** The whole model on the GPU is by far the quickest option + (`fast (GPU)` in the menu). +- **Bigger than VRAM = hybrid.** Ollama puts as many layers as fit on the GPU and runs the + rest on the CPU (`hybrid ~N% GPU`, a rough estimate). Faster than CPU-only, but the CPU + part sets the pace. Check the real split with `docker exec local-ai ollama ps`. +- **CPU-only** works, at a few tokens/second for a 7B model. Mixture-of-Experts (MoE) models + such as `qwen3-coder:30b` only activate a few billion parameters per token, so they are + much faster on CPU than dense models of the same size, but still need the RAM for all of it. +- **Context window** (4k to 32k tokens) is chosen from the memory budget and costs RAM/VRAM + too. Aider needs room for the repo map plus your files, so 8k is a practical minimum. +- **Status in the menu:** `fits`, `tight` (fits with almost no headroom: close other apps), + `slow` (dense model without a GPU), `too big` (will swap or crash). +- **Port:** the container uses 11434 like a normal Ollama. If that is taken (for example by a + host Ollama), `setup.sh` picks the next free port and writes it into `RUN_AIDER.md`. + +### Using an NVIDIA GPU + +Docker needs both the **NVIDIA driver** and **nvidia-container-toolkit**. `setup.sh` detects +a GPU that is not usable yet and prints distro-specific install commands (also in +`RUN_AIDER.md`), but only when a driver would actually help. Be careful with driver installs: + +- Take a snapshot first (Timeshift) and know how to reach the GRUB menu. +- Prefer the distro's recommended driver (`ubuntu-drivers devices`), not an arbitrary version. +- With Secure Boot on you must enrol the MOK key at the first reboot (`mokutil --sb-state`). +- If the machine stops booting: boot an older kernel or add `nomodeset` in GRUB, then + remove the packages (`dpkg -l | awk '/^ii/ && /nvidia/ && !/linux-firmware-nvidia/ {print $2}' | xargs sudo dpkg --purge --force-depends`, + then `sudo update-initramfs -u`). No network is needed for that. +- After a successful install, verify with `docker run --rm --gpus all ubuntu nvidia-smi` and + re-run `./setup.sh`. + +## Example configurations + +Speeds are rough orders of magnitude, not benchmarks. Your hardware will differ. + +### 1. CPU-only laptop, 16 GB RAM, no usable GPU + +`qwen2.5-coder:7b`, context 8k (this is what `setup.sh` picks on such a machine). + +| Pros | Cons | +| --- | --- | +| Works anywhere with Docker, no driver risk | A few tokens/second; long answers take a while | +| Good quality for its size on everyday edits | First response is slow while the model loads into RAM | +| Leaves ~9 GB RAM for the rest of the system | Keep prompts small and add only the files aider needs | + +### 2. Small GPU (about 4 GB VRAM, e.g. GTX 1650) with 16 GB RAM + +Either `qwen2.5-coder:3b` (fully on the GPU) or `qwen2.5-coder:7b` (hybrid, ~55% on the GPU). + +| Pros | Cons | +| --- | --- | +| 3B: fully in VRAM, fast and responsive | 3B: noticeably weaker; use `--edit-format whole` in aider | +| 7B hybrid: better answers, still sped up by the GPU | 7B hybrid: speedup is modest, the CPU part dominates | +| Both fit comfortably in 16 GB RAM | Needs the NVIDIA driver + container toolkit (driver install can break boot) | + +A reasonable approach: start with the 7B hybrid for quality and switch to the 3B for quick, +simple edits. Re-run `./setup.sh` and `./build-ai-image.sh` to switch. + +### 3. Workstation with a 24 GB GPU and 32+ GB RAM + +`qwen2.5-coder:32b` (dense, best quality) or `qwen3-coder:30b` (MoE, faster). + +| Pros | Cons | +| --- | --- | +| Close to cloud-assistant quality for code | Large image (~20 GB) and long first build | +| Fully in VRAM: fast, with room for a 16k+ context | The GPU is busy while the model is loaded | +| MoE variant stays usable even if it spills into RAM | Needs the most expensive hardware; dense 32B is slow if it does not fit | diff --git a/logo.png b/logo.png new file mode 100644 index 0000000..a9cd62a Binary files /dev/null and b/logo.png differ diff --git a/setup.sh b/setup.sh index 3e5adbe..0bca9bd 100755 --- a/setup.sh +++ b/setup.sh @@ -144,6 +144,12 @@ elif awk -v b="$budget" 'BEGIN{exit !(b>=40)}'; then CONTEXT=32768 elif awk -v b="$budget" 'BEGIN{exit !(b>=20)}'; then CONTEXT=16384 else CONTEXT=8192; fi +# Rough share (%) of a model's layers that fit in VRAM when a GPU is used and the +# model is bigger than the VRAM budget (Ollama then offloads part of the layers). +gpu_share() { + awk -v m="$1" -v b="$gpu_budget" -v u="$gpu_usable" 'BEGIN{ if(!u||m<=b){print 0; exit} s=int(b/m*100/5)*5; print (s<10?0:s) }' +} + # ---------------------------------------------------------------- catalogue mapfile -t rows < <(grep -v '^#' models.tsv | grep -v '^[[:space:]]*$' | sort -t$'\t' -k5,5n) tags=(); status=(); speed=(); rec=-1 @@ -154,11 +160,13 @@ for i in "${!rows[@]}"; do fits_hard_ram=$(awk -v m="$mem" -v b="$hard_budget" 'BEGIN{print (m<=b)}') fits_gpu=$(awk -v m="$mem" -v b="$gpu_budget" -v u="$gpu_usable" 'BEGIN{print (u && m<=b)}') fits_hard_gpu=$(awk -v m="$mem" -v b="$gpu_hard" -v u="$gpu_usable" 'BEGIN{print (u && m<=b)}') + share=$(gpu_share "$mem") + if [ "$share" -gt 0 ]; then hyb="hybrid ~${share}% GPU"; else hyb=""; fi if [ "$fits_gpu" = 1 ]; then status[i]="fits"; speed[i]="fast (GPU)" elif [ "$fits_hard_gpu" = 1 ]; then status[i]="tight"; speed[i]="tight (GPU)" - elif [ "$fits_ram" = 1 ] && [ "$cpu_ok" = yes ]; then status[i]="fits"; speed[i]="ok (CPU)" - elif [ "$fits_ram" = 1 ]; then status[i]="slow"; speed[i]="slow (CPU)" - elif [ "$fits_hard_ram" = 1 ]; then status[i]="tight"; speed[i]="tight (CPU)" + elif [ "$fits_ram" = 1 ] && [ "$cpu_ok" = yes ]; then status[i]="fits"; speed[i]="${hyb:-ok (CPU)}" + elif [ "$fits_ram" = 1 ]; then status[i]="slow"; speed[i]="slow ${hyb:-(CPU)}" + elif [ "$fits_hard_ram" = 1 ]; then status[i]="tight"; speed[i]="tight ${hyb:-(CPU)}" else status[i]="too big"; speed[i]="-"; fi [ "${status[$i]}" = fits ] && rec=$i # sorted by rank, so last "fits" wins done @@ -204,14 +212,15 @@ fi # --------------------------------------------------------------------- menu if [ -z "$FORCE_MODEL" ]; then echo "=== Models (budget: ${budget} GB RAM$([ "$gpu_usable" = 1 ] && echo ", ${gpu_budget} GB VRAM")) ===" - printf " %-3s %-24s %7s %7s %-9s %-11s %s\n" "#" "model" "disk" "memory" "fit" "speed" "" + printf " %-3s %-24s %7s %7s %-9s %-21s %s\n" "#" "model" "disk" "memory" "fit" "speed" "" for i in "${!rows[@]}"; do IFS=$'\t' read -r tag disk mem cpu_ok rank desc <<<"${rows[$i]}" mark=" "; [ "$i" = "$rec" ] && mark="*" - printf " %s%-3s %-24s %5s GB %5s GB %-9s %-11s %s\n" "$mark" "$((i+1))" "$tag" "$disk" "$mem" "${status[$i]}" "${speed[$i]}" "$desc" + printf " %s%-3s %-24s %5s GB %5s GB %-9s %-21s %s\n" "$mark" "$((i+1))" "$tag" "$disk" "$mem" "${status[$i]}" "${speed[$i]}" "$desc" done echo echo " * = recommended for this machine: ${tags[$rec]}" + [ "$gpu_usable" = 1 ] && echo " hybrid ~N% GPU = model is bigger than VRAM; Ollama puts about N% of it on the GPU, the rest runs on the CPU (rough estimate)." echo if [ "$ASSUME_YES" = 1 ]; then choice=""; else read -r -p "Select model number [$((rec+1))]: " choice; fi choice=${choice:-$((rec+1))} @@ -265,7 +274,10 @@ case "$MODEL" in *:1.5b|*:3b) EDIT_HINT=$'\nSmall models struggle with diff-style edits; add `--edit-format whole`.\n' ;; *) EDIT_HINT="" ;; esac -if [ "$USE_GPU" = 1 ]; then SPEED="Expect fast responses." +sel_mem=0; for r in "${rows[@]}"; do [ "$(cut -f1 <<<"$r")" = "$MODEL" ] && sel_mem=$(cut -f3 <<<"$r"); done +sel_share=$(gpu_share "$sel_mem") +if [ "$USE_GPU" = 1 ] && [ "$sel_share" -gt 0 ]; then SPEED="Hybrid mode: $MODEL is bigger than your VRAM, so roughly ${sel_share}% of it runs on the GPU and the rest on the CPU. Faster than CPU-only, slower than a model that fits entirely in VRAM. Check with: docker exec $CONTAINER_NAME ollama ps" +elif [ "$USE_GPU" = 1 ]; then SPEED="The model fits in VRAM: expect fast responses." else SPEED="CPU-only: expect a few tokens/second, and a slow first response while the model loads into RAM. Keep prompts and added files small."; fi {