Add engine-debug-tools and fix critter animation rendering bugs
build / build (push) Successful in 2m52s

- engine-debug-tools/: a debug dump hotkey + dump2png to inspect
  decoded critter frames independent of the renderer (see root
  README.md's "Debugging" section).
- engine-patches/04,05: fix critter sprite scale and an OpenGL
  texture cache collision, found using that tooling.
- mdlp: fix encodeFrames RLE-diffing upscaled frames against a
  previous-frame reference the real RSD8 decoder never implements
  (verified against Night Dive's original Mac source down to the
  68K assembly) - was rendering as scattered static in-game.
- Upscale pipeline: better color quantization (dithering, pure-ramp
  exclusion) and per-resource model selection.
This commit is contained in:
ml
2026-10-18 06:59:47 +02:00
parent ab2b778f44
commit 9d0a0d5798
21 changed files with 1185 additions and 82 deletions
+65
View File
@@ -0,0 +1,65 @@
#!/usr/bin/env bash
# Builds dump2png (see dump2png/main.go) inside modding/mdlp-image's Go
# toolchain - dump2png has no cgo dependencies of its own, so it reuses that
# image rather than needing one of its own, same reasoning as
# modding/gen_animgallery.sh.
#
# Usage: ./build-dump2png.sh <output-path>
# <output-path> is relative to the repository root (typically
# dist/dump2png - see the root Makefile's dist_debug target and
# README.md's "Debugging" section) - it's written on both sides of the
# bind mount below, so an absolute or ../-escaping path won't resolve to
# the same file on the host afterward.
set -euo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
REPO_ROOT="$(cd .. && pwd)"
[ "$#" -eq 1 ] || { echo "usage: $0 <output-path-relative-to-repo-root>" >&2; exit 1; }
OUT_PATH="$1"
case "$OUT_PATH" in
/*) echo "PREFLIGHT FAIL: <output-path> must be relative to the repo root, not absolute: $OUT_PATH" >&2; exit 1 ;;
esac
fail() { echo "PREFLIGHT FAIL: $*" >&2; exit 1; }
command -v docker >/dev/null 2>&1 \
|| fail "docker not found in PATH"
IMAGE="${MDLP_REGISTRY_IMAGE:-questshock/mdlp}"
if [ -f "$REPO_ROOT/registry.env" ]; then
# shellcheck disable=SC1091
source "$REPO_ROOT/registry.env"
IMAGE="${MDLP_REGISTRY_IMAGE:-$IMAGE}"
fi
IMAGE_TAG="$(<"$REPO_ROOT/modding/mdlp-image/VERSION")"
[ -n "$IMAGE_TAG" ] || fail "modding/mdlp-image/VERSION is empty"
IMAGE="$IMAGE:$IMAGE_TAG"
docker image inspect "$IMAGE" >/dev/null 2>&1 \
|| fail "$IMAGE not found locally - run modding/mdlp-image.sh first"
docker run --rm \
-v "$REPO_ROOT:/workspace" \
-w /workspace \
-e HOST_UID="$(id -u)" \
-e HOST_GID="$(id -g)" \
"$IMAGE" \
bash -c '
set -euo pipefail
# dump2png builds against ../../modding/src/hacked via its own
# go.mod replace (matching that relative depth exactly, since
# engine-debug-tools/ sits next to modding/ at the repo root) -
# stage a patched copy (see modding/src/hacked-patches/) at that
# same relative position rather than building against the
# unpatched vendored source.
SCRATCH_DIR="engine-debug-tools/build/dump2png-src"
rm -rf "$SCRATCH_DIR"
mkdir -p "$SCRATCH_DIR/engine-debug-tools"
bash modding/mdlp-image/stage-hacked.sh modding "$SCRATCH_DIR/modding/src/hacked"
cp -a engine-debug-tools/dump2png "$SCRATCH_DIR/engine-debug-tools/dump2png"
mkdir -p "$(dirname "/workspace/$1")"
(cd "$SCRATCH_DIR/engine-debug-tools/dump2png" && go build -o "/workspace/$1" .)
' _ "$OUT_PATH"
echo "Built: $OUT_PATH"
+12
View File
@@ -0,0 +1,12 @@
module questshock/dump2png
go 1.25.0
// hacked/ is vendored under modding/src/ (see ../../modding/README.md);
// dump2png reuses its resource/palette reading code (ss1/world,
// ss1/content/bitmap) instead of duplicating System Shock's file formats.
require github.com/inkyblackness/hacked v0.0.0
require github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
replace github.com/inkyblackness/hacked => ../../modding/src/hacked
+10
View File
@@ -0,0 +1,10 @@
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+105
View File
@@ -0,0 +1,105 @@
// dump2png converts the raw indexed-pixel frame dumps written by the
// engine's debug dump hotkey (see ../engine-patches/ and the root
// README.md's "Debugging" section) into viewable PNGs, using the real game
// palette - see that same README section for the dump file format.
package main
import (
"encoding/binary"
"fmt"
"image"
"image/color"
"image/png"
"os"
"path/filepath"
"sort"
"strings"
"github.com/inkyblackness/hacked/ss1/content/bitmap"
"github.com/inkyblackness/hacked/ss1/resource"
"github.com/inkyblackness/hacked/ss1/world"
)
func main() {
if len(os.Args) != 4 {
fmt.Fprintln(os.Stderr, "usage: dump2png <game-data-dir> <dumps-dir> <out-dir>")
os.Exit(1)
}
dataDir, inDir, outDir := os.Args[1], os.Args[2], os.Args[3]
entries, err := os.ReadDir(dataDir)
if err != nil {
panic(err)
}
merged := &world.ManifestEntry{Origin: []string{dataDir}, ID: dataDir}
for _, e := range entries {
if e.IsDir() {
continue
}
ext := strings.ToLower(filepath.Ext(e.Name()))
if ext != ".res" && ext != ".dat" {
continue
}
fe, err := world.NewManifestEntryFrom([]string{filepath.Join(dataDir, e.Name())})
if err != nil {
continue
}
merged.Resources = append(merged.Resources, fe.Resources...)
}
sort.Slice(merged.Resources, func(i, j int) bool { return merged.Resources[i].ID < merged.Resources[j].ID })
mod := world.NewMod(func([]resource.ID, []resource.ID) {}, func() {})
if err := mod.World().InsertEntry(0, merged); err != nil {
panic(err)
}
pal, err := bitmap.NewPaletteCache(mod).Palette(resource.KeyOf(0x02BC, resource.LangDefault, 0))
if err != nil {
panic(err)
}
colors := pal.ColorPalette(false)
if err := os.MkdirAll(outDir, 0o755); err != nil {
panic(err)
}
files, err := os.ReadDir(inDir)
if err != nil {
panic(err)
}
for _, fi := range files {
if !strings.HasSuffix(fi.Name(), ".raw") {
continue
}
raw, err := os.ReadFile(filepath.Join(inDir, fi.Name()))
if err != nil {
fmt.Println("skip", fi.Name(), err)
continue
}
if len(raw) < 4 {
continue
}
w := int(binary.LittleEndian.Uint16(raw[0:2]))
h := int(binary.LittleEndian.Uint16(raw[2:4]))
pixels := raw[4:]
if len(pixels) < w*h {
fmt.Println("truncated", fi.Name())
continue
}
img := image.NewRGBA(image.Rect(0, 0, w, h))
for i := 0; i < w*h; i++ {
r, g, b, _ := colors[pixels[i]].RGBA()
img.Set(i%w, i/w, color.RGBA{R: byte(r >> 8), G: byte(g >> 8), B: byte(b >> 8), A: 0xff})
}
outPath := filepath.Join(outDir, strings.TrimSuffix(fi.Name(), ".raw")+".png")
of, err := os.Create(outPath)
if err != nil {
fmt.Println("create failed", outPath, err)
continue
}
err = png.Encode(of, img)
of.Close()
if err != nil {
fmt.Println("encode failed", outPath, err)
continue
}
fmt.Println("wrote", outPath, w, h)
}
}
@@ -0,0 +1,153 @@
--- a/src/GameSrc/objsim.c
+++ b/src/GameSrc/objsim.c
@@ -26,8 +26,10 @@
// Object simulator code for Citadel
#define __OBJSIM_SRC
-#include <string.h>
+#include <stdio.h>
#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
#include "Shock.h"
@@ -1924,6 +1926,93 @@
*h = s_critterNativeH[id][frame];
}
+// Resource id of the last-drawn critter animation, updated every frame from
+// gameobj.c's FAUBJ_CRIT case - read by debug_dump_critter_animation() on
+// the debug dump hotkey (see engine-debug-tools/, root README.md's
+// "Debugging" section). Debug-build only: see
+// engine-debug-tools/engine-patches/.
+static Id s_lastCritterId = 0;
+
+void debug_track_critter_draw(Id id) { s_lastCritterId = id; }
+
+// Dumps every frame of the last-drawn critter's current animation (the
+// whole resource id, not just the currently-visible frame) as raw
+// palette-index bytes, one file per frame, independent of the renderer's
+// own texture/GL state - lets a corrupted frame be told apart from
+// corruption introduced later, in rendering. Format: 2-byte LE width,
+// 2-byte LE height, then width*height index bytes - converted to a viewable
+// image by engine-debug-tools/dump2png, not here, to avoid this debug path
+// depending on the renderer's own palette/SDL state.
+void debug_dump_critter_animation(void) {
+ RefTable *prt;
+ int numFrames, frame;
+ char path[256];
+ FILE *f;
+
+ if (s_lastCritterId == 0) {
+ fprintf(stderr, "[debug dump] no critter drawn yet\n");
+ return;
+ }
+
+ mkdir("dumps", 0755); // ignore EEXIST - fopen below reports any real failure
+
+ prt = (RefTable *)ResLock(s_lastCritterId);
+ if (prt == NULL) {
+ fprintf(stderr, "[debug dump] ResLock(%d) failed\n", (int)s_lastCritterId);
+ return;
+ }
+ numFrames = prt->numRefs;
+ ResUnlock(s_lastCritterId);
+
+ fprintf(stderr, "[debug dump] id=%d numFrames=%d\n", (int)s_lastCritterId, numFrames);
+
+ for (frame = 0; frame < numFrames; frame++) {
+ FrameDesc *fd;
+ grs_bitmap decoded;
+ uchar *bits;
+ short w, h;
+
+ fd = (FrameDesc *)RefLock(MKREF(s_lastCritterId, frame));
+ if (fd == NULL) {
+ fprintf(stderr, "[debug dump] RefLock frame %d failed\n", frame);
+ continue;
+ }
+
+ if (fd->bm.type == BMT_RSD8) {
+ if (gr_rsd8_convert(&fd->bm, &decoded) != 0 /* GR_UNPACK_RSD8_OK */) {
+ fprintf(stderr, "[debug dump] gr_rsd8_convert frame %d failed\n", frame);
+ RefUnlock(MKREF(s_lastCritterId, frame));
+ continue;
+ }
+ bits = decoded.bits;
+ w = decoded.w;
+ h = decoded.h;
+ } else {
+ bits = fd->bm.bits;
+ w = fd->bm.w;
+ h = fd->bm.h;
+ }
+
+ snprintf(path, sizeof(path), "dumps/id%d_frame%d.raw", (int)s_lastCritterId, frame);
+ f = fopen(path, "wb");
+ if (f != NULL) {
+ uchar hdr[4];
+ hdr[0] = (uchar)(w & 0xFF);
+ hdr[1] = (uchar)((w >> 8) & 0xFF);
+ hdr[2] = (uchar)(h & 0xFF);
+ hdr[3] = (uchar)((h >> 8) & 0xFF);
+ fwrite(hdr, 1, 4, f);
+ fwrite(bits, 1, (size_t)w * (size_t)h, f);
+ fclose(f);
+ fprintf(stderr, "[debug dump] wrote %s (%dx%d)\n", path, (int)w, (int)h);
+ } else {
+ fprintf(stderr, "[debug dump] failed to open %s for writing\n", path);
+ }
+
+ RefUnlock(MKREF(s_lastCritterId, frame));
+ }
+}
+
errtype obj_set_secondary_properties() {
char i, j;
RefTable *prt;
--- a/src/GameSrc/Headers/objsim.h
+++ b/src/GameSrc/Headers/objsim.h
@@ -109,6 +109,10 @@
errtype obj_create_player(ObjLoc *plr_loc);
Ref ref_from_critter_data(ObjID oid, int triple, byte posture, short frame, short view);
void get_critter_native_size(Id id, short frame, short *w, short *h);
+// Debug-build only - see engine-debug-tools/, root README.md's "Debugging"
+// section.
+void debug_track_critter_draw(Id id);
+void debug_dump_critter_animation(void);
errtype obj_zero_unused();
grs_bitmap *obj_get_model_data(ObjID id, fix *x, fix *y, fix *z, grs_bitmap *bm2, Ref *ref1, Ref *ref2);
errtype obj_model_hack(ObjID id, uchar *hack_x, uchar *hack_y, uchar *hack_z, uchar *hack_type);
--- a/src/GameSrc/gameobj.c
+++ b/src/GameSrc/gameobj.c
@@ -1019,6 +1019,7 @@
{
short native_w = 0, native_h = 0;
get_critter_native_size(REFID(ref), REFINDEX(ref), &native_w, &native_h);
+ debug_track_critter_draw(REFID(ref));
_fr_draw_bitmap(tpdata, _fdt_dist, 0, anch.ul.x, anch.ul.y, native_w, native_h);
}
release_critter_bitmap_fast(ref);
--- a/src/Libraries/INPUT/Source/sdl_events.c
+++ b/src/Libraries/INPUT/Source/sdl_events.c
@@ -481,6 +481,19 @@
SDL_Event ev;
while (SDL_PollEvent(&ev)) {
+ // Debug-build only (engine-debug-tools/, root README.md's
+ // "Debugging" section): K dumps every frame of the last-drawn
+ // critter's current animation to dumps/. Checked here, before the
+ // switch below dispatches on event type at all, so it can never
+ // collide with an existing game keybinding - a plain letter rather
+ // than Insert, which some keyboards don't send at all (compact
+ // layouts, or routed through Fn).
+ if (ev.type == SDL_KEYDOWN && ev.key.keysym.sym == SDLK_k) {
+ extern void debug_dump_critter_animation(void);
+ debug_dump_critter_animation();
+ continue;
+ }
+
switch (ev.type) {
case SDL_QUIT:
// a bit hacky at this place, but this would allow exiting the game via the window's [x] button
@@ -0,0 +1,166 @@
--- a/src/Libraries/2D/Source/RSD/rsdcvt.c
+++ b/src/Libraries/2D/Source/RSD/rsdcvt.c
@@ -37,6 +37,7 @@
*
*/
#include <assert.h>
+#include <stdio.h>
#include <string.h>
#include "grs.h"
#include "bitmap.h"
@@ -47,6 +48,14 @@
uchar *grd_unpack_buf=NULL;
+// Set by debug_dump_critter_animation() (engine-debug-tools/, root
+// README.md's "Debugging" section) around its own gr_rsd8_convert() calls,
+// so a trace of this decode's internals - which path ran, whether it wrote
+// fewer bytes than the bitmap's own area, and (gr_rsd8_unpack(),
+// RSDUnpack.c) each opcode consumed - prints only for those calls, not
+// every RSD8 decode in the game. Debug-build only.
+int qs_debug_rsd8_trace = 0;
+
/*************************************************/
/* Puts 0's in place of skips and pads with 0's. */
/* i.e., grd_unpack_buf is entirely overwritten. */
@@ -138,6 +147,15 @@
}
}
rsd_done:
+ if (qs_debug_rsd8_trace) {
+ long written = (long)(p_dst - dbm->bits);
+ long expected = (long)dbm->row * (long)dbm->h;
+ fprintf(stderr,
+ "[debug rsd8] path=%s sbm(w=%d h=%d row=%d) dbm(row=%d h=%d) "
+ "written=%ld expected=%ld short_by=%ld\n",
+ (dbm->w == dbm->row) ? "fast" : "slow", sbm->w, sbm->h, sbm->row,
+ dbm->row, dbm->h, written, expected, expected - written);
+ }
if ((over_run = (dbm->bits+dbm->row*dbm->h)-p_dst)) {
assert(over_run > 0);
LG_memset (p_dst, kSkipColor, over_run);
--- a/src/Libraries/2D/Source/RSD/RSDUnpack.c
+++ b/src/Libraries/2D/Source/RSD/RSDUnpack.c
@@ -22,6 +22,7 @@
//
#include "grs.h"
+#include <stdio.h>
#include <stdbool.h>
//----------------------------------------------------------------------------
@@ -34,6 +35,10 @@
short count,count2;
ushort longcode;
uint32_t longval, *longdest, *longsrc;
+
+ extern int qs_debug_rsd8_trace; // rsdcvt.c - engine-debug-tools/, root README.md's "Debugging" section
+ uchar *src_start = src, *dest_start = dest;
+ long op_index = 0;
do
{
@@ -42,6 +47,7 @@
{
count = *(src++); // get count
val = *(src++); // get val
+ if (qs_debug_rsd8_trace) fprintf(stderr, "[debug rsd8 op] #%ld src_off=%ld dest_off=%ld type=run count=%d val=%d\n", op_index++, (long)(src-src_start), (long)(dest-dest_start), (int)count, (int)val);
if (count>=kMinLongLoop) // if at least kMinLongLoop bytes, do long word stuff
{
@@ -63,6 +69,7 @@
else if (code<0x80) // dump (copy) bytes
{
count = code;
+ if (qs_debug_rsd8_trace) fprintf(stderr, "[debug rsd8 op] #%ld src_off=%ld dest_off=%ld type=dump count=%d\n", op_index++, (long)(src-src_start), (long)(dest-dest_start), (int)count);
if (code>=kMinLongLoop) // if at least kMinLongLoop bytes, do long word stuff
{
count2 = count>>2;
@@ -84,6 +91,7 @@
{
count = code & 0x007f; // clear high byte
val = longval = 0L;
+ if (qs_debug_rsd8_trace) fprintf(stderr, "[debug rsd8 op] #%ld src_off=%ld dest_off=%ld type=skip count=%d\n", op_index++, (long)(src-src_start), (long)(dest-dest_start), (int)count);
if (count>=kMinLongLoop) // if at least kMinLongLoop bytes, do long word stuff
{
@@ -105,11 +113,15 @@
longcode = * (ushort *) src;
src += 2L;
- if (!longcode) break; // done?
+ if (!longcode) {
+ if (qs_debug_rsd8_trace) fprintf(stderr, "[debug rsd8 op] #%ld src_off=%ld dest_off=%ld type=terminator\n", op_index++, (long)(src-src_start), (long)(dest-dest_start));
+ break; // done?
+ }
else if (longcode<0x8000) // skip (zero)
{
count = longcode;
val = longval = 0L;
+ if (qs_debug_rsd8_trace) fprintf(stderr, "[debug rsd8 op] #%ld src_off=%ld dest_off=%ld type=longskip count=%d\n", op_index++, (long)(src-src_start), (long)(dest-dest_start), (int)count);
if (count>=kMinLongLoop) // if at least kMinLongLoop bytes, do long word stuff
{
@@ -129,6 +141,7 @@
else if (longcode<0xC000) // dump (copy)
{
count = longcode & 0x7fff; // clear high bit
+ if (qs_debug_rsd8_trace) fprintf(stderr, "[debug rsd8 op] #%ld src_off=%ld dest_off=%ld type=longdump count=%d\n", op_index++, (long)(src-src_start), (long)(dest-dest_start), (int)count);
if (count>=kMinLongLoop) // if at least kMinLongLoop bytes, do long word stuff
{
@@ -151,6 +164,7 @@
{
count = longcode & 0x3fff;
val = *(src++); // get val
+ if (qs_debug_rsd8_trace) fprintf(stderr, "[debug rsd8 op] #%ld src_off=%ld dest_off=%ld type=longrun count=%d val=%d\n", op_index++, (long)(src-src_start), (long)(dest-dest_start), (int)count, (int)val);
if (count>=kMinLongLoop) // if at least kMinLongLoop bytes, do long word stuff
{
--- a/src/GameSrc/objsim.c
+++ b/src/GameSrc/objsim.c
@@ -1979,11 +1979,43 @@
}
if (fd->bm.type == BMT_RSD8) {
- if (gr_rsd8_convert(&fd->bm, &decoded) != 0 /* GR_UNPACK_RSD8_OK */) {
+ extern int qs_debug_rsd8_trace; // rsdcvt.c - engine-debug-tools/, root README.md's "Debugging" section
+ uchar *first_copy;
+ size_t nbytes;
+ int rsd8_ok;
+
+ qs_debug_rsd8_trace = 1;
+ rsd8_ok = gr_rsd8_convert(&fd->bm, &decoded) == 0 /* GR_UNPACK_RSD8_OK */;
+ qs_debug_rsd8_trace = 0;
+ if (!rsd8_ok) {
fprintf(stderr, "[debug dump] gr_rsd8_convert frame %d failed\n", frame);
RefUnlock(MKREF(s_lastCritterId, frame));
continue;
}
+
+ // Re-decode the same compressed source (fd->bm.bits, unchanged
+ // by the first call - only grd_unpack_buf, the shared
+ // decode-output buffer, gets overwritten) a second time and
+ // compare: decode runs back to back on this single thread, with
+ // nothing else able to touch fd->bm.bits or grd_unpack_buf in
+ // between, so a byte-for-byte match here means any garbled
+ // pixels are a deterministic property of this exact compressed
+ // stream (bad source data, or a decoder bug it triggers), not a
+ // one-off from a resource-cache slot being reused mid-decode.
+ nbytes = (size_t)decoded.w * (size_t)decoded.h;
+ first_copy = malloc(nbytes);
+ if (first_copy != NULL) {
+ memcpy(first_copy, decoded.bits, nbytes);
+
+ qs_debug_rsd8_trace = 1;
+ gr_rsd8_convert(&fd->bm, &decoded);
+ qs_debug_rsd8_trace = 0;
+
+ fprintf(stderr, "[debug dump] frame %d repeat-decode of bm.bits=%p %s\n", frame,
+ (void *)fd->bm.bits, memcmp(first_copy, decoded.bits, nbytes) == 0 ? "identical" : "DIFFERS");
+ free(first_copy);
+ }
+
bits = decoded.bits;
w = decoded.w;
h = decoded.h;