Give the game quad the same laser-beam pointer as the overlays

Aiming looked and behaved differently depending on whether the
keyboard/menu was open: xr_input_try_overlay() claimed a hand's ray for
its overlay whenever the overlay was merely visible, regardless of
whether the ray was actually pointed at it - so with the keyboard open,
aiming at the main game screen still showed a beam terminating at the
keyboard's fixed depth, while closing the keyboard swapped in the old
flat cross-shaped reticle at whatever the ray was really pointing at.

Only claim an overlay when the ray is actually relevant this frame - a
real hit, or a touch/drag begun on a previous frame still pending
release - so a visible-but-unaimed-at overlay now falls through to the
next-priority target. Give the game quad its own hit-test
(xr_input_try_game_quad()) and the same billboarded laser-beam
treatment as the overlays instead of the old flat reticle, so aiming
looks and behaves identically everywhere - overlay open or not. The
old reticle GL program/shaders are now fully dead and removed. No
click/touch dispatch is added to the game quad itself; that's future
work.

android/app/src/main/cpp/xr_input.c

@@ -70,9 +70,6 @@ static int g_keyboard_drag_hand = -1;
 static float g_keyboard_drag_offset_x = 0.0f;
 static float g_keyboard_drag_offset_y = 0.0f;
 
-static GLuint g_reticle_program = 0;
-static GLint g_reticle_color_loc = -1;
-
 // A "view" reference space, located once per frame (not per hand) to get
 // an approximate head position for the laser-beam billboard math below -
 // the only consumer of a head pose in this file. Both LOCAL (g_local_space,
@@ -82,9 +79,9 @@ static XrSpace g_view_space = XR_NULL_HANDLE;
 
 // Per-hand laser-beam state: a thin, billboarded quad spanning from the
 // controller to wherever that hand's ray currently crosses the plane of
-// whichever overlay (keyboard/menu) claimed it this frame - see
+// whichever target (keyboard, menu, or the game quad) claimed it this
-// xr_input_build_beam()/xr_input_get_beam_layer(). Each hand gets its own
+// frame - see xr_input_build_beam()/xr_input_get_beam_layer(). Each hand
-// tiny solid-color swapchain (no shared tint on XrCompositionLayerQuad,
+// gets its own tiny solid-color swapchain (no shared tint on XrCompositionLayerQuad,
 // so two separately-colored textures is simplest). g_beam_quad_valid is
 // reset to false at the top of every xr_input_sync_and_draw() call and
 // only set back to true if that hand actually claims a beam this frame.
@@ -183,63 +180,6 @@ static void mat3_to_quat(const float x[3], const float y[3], const float z[3], X
     }
 }
 
-static GLuint compile_shader(GLenum type, const char *src) {
-    GLuint shader = glCreateShader(type);
-    glShaderSource(shader, 1, &src, NULL);
-    glCompileShader(shader);
-    GLint compiled = GL_FALSE;
-    glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled);
-    if (!compiled) {
-        char log[512];
-        glGetShaderInfoLog(shader, sizeof(log), NULL, log);
-        LOGE("XR: reticle shader compile failed: %s", log);
-    }
-    return shader;
-}
-
-// A standalone flat-color shader, independent of the engine's own
-// textureShaderProgram (OpenGL.cc is a separate, C++-only translation
-// unit, and shader state isn't shared across programs anyway) - position
-// is emitted directly in clip space, matching the same [-1,1] local quad
-// coordinates android_draw_surface_as_quad() (see android/engine-patches/
-// 11-android-openxr-present.patch) already uses for its own vertex
-// positions, so no view/projection matrix is needed here either.
-static const char *kVertexSrc = "attribute vec3 position;\n"
-                                 "void main() { gl_Position = vec4(position, 1.0); }\n";
-static const char *kFragmentSrc = "precision mediump float;\n"
-                                   "uniform vec4 color;\n"
-                                   "void main() { gl_FragColor = color; }\n";
-
-// Attribute 0 to match kVertexSrc's single "position" attribute - fine to
-// reuse the same numeric index the engine's own immediate-mode drawing
-// treats specially, since that's a per-program binding and this program is
-// never current at the same time as gl4es's immediate-mode emulation runs;
-// xr_input_sync_and_draw() disables the array again right after drawing,
-// the same discipline android_draw_surface_as_quad() already established.
-#define RETICLE_POSITION_LOC 0
-
-static bool xr_input_init_reticle_program(void) {
-    GLuint vs = compile_shader(GL_VERTEX_SHADER, kVertexSrc);
-    GLuint fs = compile_shader(GL_FRAGMENT_SHADER, kFragmentSrc);
-    g_reticle_program = glCreateProgram();
-    glAttachShader(g_reticle_program, vs);
-    glAttachShader(g_reticle_program, fs);
-    glBindAttribLocation(g_reticle_program, RETICLE_POSITION_LOC, "position");
-    glLinkProgram(g_reticle_program);
-    GLint linked = GL_FALSE;
-    glGetProgramiv(g_reticle_program, GL_LINK_STATUS, &linked);
-    glDeleteShader(vs);
-    glDeleteShader(fs);
-    if (!linked) {
-        char log[512];
-        glGetProgramInfoLog(g_reticle_program, sizeof(log), NULL, log);
-        LOGE("XR: reticle program link failed: %s", log);
-        return false;
-    }
-    g_reticle_color_loc = glGetUniformLocation(g_reticle_program, "color");
-    return true;
-}
-
 bool xr_input_init(XrInstance instance, XrSession session, int64_t swapchain_format) {
     g_instance = instance;
     g_session = session;
@@ -349,9 +289,6 @@ bool xr_input_init(XrInstance instance, XrSession session, int64_t swapchain_for
         }
     }
 
-    if (!xr_input_init_reticle_program())
-        return false;
-
     LOGI("XR: input action set ready (aim pose + trigger + menu-toggle)");
     return true;
 }
@@ -361,10 +298,13 @@ bool xr_input_init(XrInstance instance, XrSession session, int64_t swapchain_for
 // below. dragCapable enables the keyboard-only title-bar/drag-handle
 // handling (see TITLE_BAR_V_FRACTION/CLOSE_BUTTON_U_FRACTION); the menu
 // launcher has no title bar, so it's always false there and every hit is a
-// plain click. Returns true if this overlay is visible - claiming the
+// plain click. Returns true only when this hand's ray is actually
-// hand's ray processing for this frame, regardless of whether the ray
+// relevant to this overlay this frame - landing on it now, or continuing
-// actually hits it - so the caller should stop trying other targets (menu/
+// a touch/drag begun on a previous frame for this hand that hasn't been
-// keyboard/game quad are mutually exclusive per hand, per frame).
+// released yet - not merely because the overlay is visible. The caller
+// should only stop trying other targets (menu/keyboard/game quad) when
+// this returns true; a visible-but-unclaimed overlay falls through so a
+// lower-priority target (ultimately the game quad) can still be aimed at.
 static bool xr_input_try_overlay(XrOverlay *overlay, bool dragCapable, int hand,
                                   const XrSpaceLocation *location, float fx, float fy, float fz,
                                   bool selectDownEdge, bool selectUpEdge, bool *touchActive,
@@ -375,6 +315,14 @@ static bool xr_input_try_overlay(XrOverlay *overlay, bool dragCapable, int hand,
     if (!xr_overlay_is_visible(overlay))
         return false;
 
+    // Captured before this call's own down/up-edge handling below can
+    // mutate them, so a hand with a pending down-touch (waiting for its
+    // matching up) or an in-progress drag on *this* overlay still claims
+    // the ray this frame even if it has strayed off the panel's current
+    // bounds - see `claims` below.
+    bool hadTouch = *touchActive;
+    bool hadDrag = dragCapable && g_keyboard_dragging && g_keyboard_drag_hand == hand;
+
     float quadCenterX, quadCenterY, distance, halfWidth, halfHeight;
     xr_overlay_get_quad_extent(overlay, &quadCenterX, &quadCenterY, &distance, &halfWidth,
                                 &halfHeight);
@@ -416,6 +364,12 @@ static bool xr_input_try_overlay(XrOverlay *overlay, bool dragCapable, int hand,
         *outCursorV = v;
     }
 
+    // Only claim (and thus route/beam) this hand's ray to this overlay
+    // when it's actually relevant this frame: landing on it now, or
+    // continuing a touch/drag that started on it - never merely because
+    // it's visible.
+    bool claims = hit || hadTouch || hadDrag;
+
     if (!dragCapable) {
         if (selectDownEdge && hit) {
             xr_overlay_touch(overlay, u, v, true);
@@ -424,7 +378,7 @@ static bool xr_input_try_overlay(XrOverlay *overlay, bool dragCapable, int hand,
             xr_overlay_touch(overlay, u, v, false);
             *touchActive = false;
         }
-        return true;
+        return claims;
     }
 
     bool onTitleBar = hit && v < TITLE_BAR_V_FRACTION;
@@ -457,7 +411,7 @@ static bool xr_input_try_overlay(XrOverlay *overlay, bool dragCapable, int hand,
         xr_overlay_set_position(overlay, worldX - g_keyboard_drag_offset_x,
                                  worldY - g_keyboard_drag_offset_y);
     }
-    return true;
+    return claims;
 }
 
 // Builds this hand's laser-beam quad - a thin, billboarded ribbon from the
@@ -568,6 +522,41 @@ static void xr_input_build_beam(int hand, const XrSpaceLocation *location, float
     g_beam_quad_valid[hand] = true;
 }
 
+// Hit-tests this hand's ray against the game quad (xr_get_game_quad_extent())
+// - the lowest-priority target, tried only once neither the keyboard nor
+// menu overlay claimed the ray this frame. There's no touch/drag state
+// for the game quad yet (see the file comment above) - this purely drives
+// the laser-beam visual, symmetric with xr_input_try_overlay()'s `hit`
+// term but without a touch/drag component. Returns plain `hit`.
+static bool xr_input_try_game_quad(int hand, const XrSpaceLocation *location, float fx, float fy,
+                                    float fz, bool *outPlaneHit, float *outPlaneDistance) {
+    float distance, halfWidth, halfHeight;
+    xr_get_game_quad_extent(&distance, &halfWidth, &halfHeight);
+
+    bool planeHit = false, hit = false;
+    float u = 0.0f, v = 0.0f, t = 0.0f;
+    if (fabsf(fz) > 1e-5f) {
+        t = (-distance - location->pose.position.z) / fz;
+        if (t > 0.0f) {
+            planeHit = true;
+            float cx = (location->pose.position.x + t * fx) / halfWidth;
+            float cy = (location->pose.position.y + t * fy) / halfHeight;
+            hit = fabsf(cx) <= 1.0f && fabsf(cy) <= 1.0f;
+            u = (cx + 1.0f) * 0.5f;
+            v = (1.0f - cy) * 0.5f;
+        }
+    }
+    *outPlaneHit = planeHit;
+    *outPlaneDistance = t;
+
+    if (hit != g_game_prev_hit[hand]) {
+        LOGI("XR: %s aim ray %s game quad (u=%.2f v=%.2f)", kHandName[hand],
+             hit ? "entered" : "left", u, v);
+        g_game_prev_hit[hand] = hit;
+    }
+    return hit;
+}
+
 void xr_input_sync_and_draw(XrSpace baseSpace, XrTime time, bool draw) {
     if (g_action_set == XR_NULL_HANDLE)
         return;
@@ -597,8 +586,10 @@ void xr_input_sync_and_draw(XrSpace baseSpace, XrTime time, bool draw) {
     g_beam_quad_valid[LEFT] = g_beam_quad_valid[RIGHT] = false;
 
     // The billboard math needs an approximate head position - only bother
-    // locating it on frames where a beam could possibly be drawn at all.
+    // locating it on frames where a beam could possibly be drawn at all
-    bool needBeams = draw && (menuVisible || keyboardVisible);
+    // (i.e. any drawn frame - the game quad can claim a beam on its own
+    // even with both overlays closed).
+    bool needBeams = draw;
     XrSpaceLocation headLoc = {XR_TYPE_SPACE_LOCATION};
     bool haveHead = false;
     if (needBeams) {
@@ -609,17 +600,6 @@ void xr_input_sync_and_draw(XrSpace baseSpace, XrTime time, bool draw) {
                    (headLoc.locationFlags & neededHead) == neededHead;
     }
 
-    // The GL reticle is drawn directly into whatever framebuffer is
-    // currently bound - the game quad's swapchain image (see xr_frame_end(),
-    // which calls this while that image is still bound). That only makes
-    // sense while aiming at the game quad; the menu/keyboard overlays' own
-    // cursors are drawn by their Java views instead (see
-    // xr_overlay_update_cursor() below), composited into their Bitmaps the
-    // same way their other content is.
-    bool drawReticle = draw && !menuVisible && !keyboardVisible;
-    if (drawReticle)
-        glUseProgram(g_reticle_program);
-
     // Fed to xr_overlay_update_cursor() after the loop below - whichever
     // hand's ray hits a given overlay last wins if both do, good enough
     // for a single on-quad cursor per overlay (only ever updated when hit
@@ -686,12 +666,12 @@ void xr_input_sync_and_draw(XrSpace baseSpace, XrTime time, bool draw) {
         quat_rotate_vec(&location.pose.orientation, 0.0f, 0.0f, -1.0f, &fx, &fy, &fz);
 
         // Keyboard first (it's the more likely target while it's up), then
-        // the menu launcher, then - only if neither is visible - the game
+        // the menu launcher, then - only once neither actually claims the
-        // quad's own reticle below. A single ray only ever interacts with
+        // ray this frame (not merely "isn't visible" - see
-        // one target per hand per frame. Whichever overlay claims the ray
+        // xr_input_try_overlay()'s doc comment) - the game quad itself.
-        // also gets a laser-beam quad built for it (if a head pose is
+        // Whichever target claims the ray gets a laser-beam quad built for
-        // available and the ray actually crosses that overlay's plane) -
+        // it (if a head pose is available and the ray actually crosses
-        // see xr_input_build_beam().
+        // that target's plane) - see xr_input_build_beam().
         bool keyboardPlaneHit = false;
         float keyboardPlaneDistance = 0.0f;
         if (xr_input_try_overlay(keyboardOverlay, true, hand, &location, fx, fy, fz,
@@ -716,40 +696,19 @@ void xr_input_sync_and_draw(XrSpace baseSpace, XrTime time, bool draw) {
             continue;
         }
 
-        float distance, halfWidth, halfHeight;
+        // Lowest priority: the game quad itself - purely visual (beam
-        xr_get_game_quad_extent(&distance, &halfWidth, &halfHeight);
+        // only, same as the other two targets), no click/touch dispatch
-
+        // here yet (see the file comment above).
-        bool hit = false;
+        bool gamePlaneHit = false;
-        float u = 0.0f, v = 0.0f, cx = 0.0f, cy = 0.0f;
+        float gamePlaneDistance = 0.0f;
-        if (fabsf(fz) > 1e-5f) {
+        bool gameHit =
-            float t = (-distance - location.pose.position.z) / fz;
+            xr_input_try_game_quad(hand, &location, fx, fy, fz, &gamePlaneHit, &gamePlaneDistance);
-            if (t > 0.0f) {
+        // gameHit implies gamePlaneHit (both only ever set together above),
-                cx = (location.pose.position.x + t * fx) / halfWidth;
+        // kept as a separate out-param for symmetry with
-                cy = (location.pose.position.y + t * fy) / halfHeight;
+        // xr_input_try_overlay()'s outPlaneHit/outPlaneDistance pair.
-                hit = fabsf(cx) <= 1.0f && fabsf(cy) <= 1.0f;
+        if (haveHead && gameHit)
-                u = (cx + 1.0f) * 0.5f;
+            xr_input_build_beam(hand, &location, fx, fy, fz,
-                v = (1.0f - cy) * 0.5f;
+                                 fminf(gamePlaneDistance, MAX_BEAM_LENGTH_METERS), &headLoc);
-            }
-        }
-
-        if (hit != g_game_prev_hit[hand]) {
-            LOGI("XR: %s aim ray %s game quad (u=%.2f v=%.2f)", kHandName[hand],
-                 hit ? "entered" : "left", u, v);
-            g_game_prev_hit[hand] = hit;
-        }
-
-        if (!hit || !drawReticle)
-            continue;
-
-        const float kSize = 0.03f;
-        const float verts[] = {
-            cx - kSize, cy, 0.0f, cx + kSize, cy, 0.0f, cx, cy - kSize, 0.0f, cx, cy + kSize, 0.0f,
-        };
-        glUniform4fv(g_reticle_color_loc, 1, kHandColor[hand]);
-        glEnableVertexAttribArray(RETICLE_POSITION_LOC);
-        glVertexAttribPointer(RETICLE_POSITION_LOC, 3, GL_FLOAT, GL_FALSE, 0, verts);
-        glDrawArrays(GL_LINES, 0, 4);
-        glDisableVertexAttribArray(RETICLE_POSITION_LOC);
     }
 
     if (menuVisible)
@@ -783,10 +742,6 @@ void xr_input_shutdown(void) {
     g_select_click_action = XR_NULL_HANDLE;
     g_menu_toggle_action = XR_NULL_HANDLE;
 
-    if (g_reticle_program != 0)
-        glDeleteProgram(g_reticle_program);
-    g_reticle_program = 0;
-
     g_instance = XR_NULL_HANDLE;
     g_session = XR_NULL_HANDLE;
     memset(g_prev_select, 0, sizeof(g_prev_select));

android/app/src/main/cpp/xr_input.h

@@ -1,16 +1,20 @@
 // Controller input for questshock's immersive Quest build: one OpenXR
 // action set (aim pose + trigger click per hand, a menu-toggle button on
-// the left controller) plus ray/quad hit-testing. While neither the menu
+// the left controller) plus ray/quad hit-testing. Each hand's ray is
-// launcher nor keyboard overlay (see xr_overlay.h, xr_session.c) is
+// tried against the keyboard overlay, then the menu overlay (see
-// visible, this tests against the game quad xr_session.c submits and draws
+// xr_overlay.h, xr_session.c), then - only if neither actually claims it
-// a small reticle where each hand's aim ray crosses it; while either
+// this frame (landing on it now, or continuing a touch/drag begun on a
-// overlay is visible, it tests against that overlay's quad instead (the
+// previous frame - not merely because that overlay happens to be visible)
-// keyboard is tried first), forwards trigger edges as synthetic touches,
+// - the game quad xr_session.c submits, forwarding trigger edges to
-// and - since neither overlay's own on-quad cursor gives any feedback
+// whichever overlay claims the ray as synthetic touches. Since none of
-// while the ray is short of actually landing on it - builds a thin,
+// the three targets' own feedback (the overlays' on-quad cursor, drawn by
-// billboarded laser-beam quad per hand from the controller to wherever
+// their Java views; nothing at all for the game quad, which has no
-// the ray currently crosses that overlay's plane (see
+// click/touch dispatch here yet) gives any indication while the ray is
-// xr_input_get_beam_layer()).
+// short of actually landing on something, every claimed target also
+// builds a thin, billboarded laser-beam quad per hand from the controller
+// to wherever the ray currently crosses that target's plane (see
+// xr_input_get_beam_layer()) - so aiming looks and behaves the same
+// whether the target is an overlay or the game quad itself.
 #ifndef QUESTSHOCK_XR_INPUT_H
 #define QUESTSHOCK_XR_INPUT_H
 
@@ -38,17 +42,18 @@ bool xr_input_init(XrInstance instance, XrSession session, int64_t swapchain_for
 // Call once per frame from xr_frame_end(), before releasing the acquired
 // swapchain image - syncs this frame's action states (always, so edge
 // detection stays correct even on frames with nothing to draw), handles
-// the menu_toggle button's edge, and either draws a game-quad reticle or
+// the menu_toggle button's edge, and hit-tests/dispatches each hand's ray
-// forwards menu-quad touches, per the menu's current visibility (see the
+// against keyboard/menu/game quad in that priority order (see the file
-// file comment above). draw gates only the reticle - if false (nothing to
+// comment above). draw gates only the laser-beam visuals (and the
-// draw into this frame, e.g. no swapchain image was acquired), hit-testing
+// head-pose locate that feeds them) - if false (nothing to draw into this
-// and touch-forwarding still run. baseSpace/time must match whatever
+// frame, e.g. no swapchain image was acquired), hit-testing and
+// touch-forwarding still run. baseSpace/time must match whatever
 // xr_frame_begin() used to predict this frame.
 void xr_input_sync_and_draw(XrSpace baseSpace, XrTime time, bool draw);
 
 // Call once per hand (hand: 0=left, 1=right) after xr_input_sync_and_draw()
-// in the same frame - if that hand's ray claimed the keyboard or menu
+// in the same frame - if that hand's ray claimed the keyboard, menu, or
-// overlay this frame (see the file comment above), fills *out_quad's
+// game quad this frame (see the file comment above), fills *out_quad's
 // subImage/pose/size for its laser-beam ribbon and returns true.
 // space/eyeVisibility/layerFlags are left for the caller to set, same
 // convention as xr_overlay_render_and_build_layer(). Returns false

android/app/src/main/cpp/xr_session.c

@@ -350,14 +350,13 @@ void xr_frame_end(void) {
     if (g_session == XR_NULL_HANDLE)
         return;
 
-    // Draw the laser-pointer reticle into the still-bound game swapchain
+    // Syncs actions and hit-tests/dispatches each hand's ray against
-    // framebuffer before releasing it, so it composites on top of whatever
+    // keyboard/menu/game quad (see xr_input.c) before the game swapchain
-    // this frame's game content already drew there. Syncing actions
+    // image is released below - this always runs, even when there's
-    // happens even when there's nothing to draw (no acquired image this
+    // nothing to draw (no acquired image this frame), so edge detection
-    // frame), so edge detection (trigger/menu-button clicks) doesn't miss a
+    // (trigger/menu-button clicks) doesn't miss a frame. Any resulting
-    // frame. Only ever draws while neither overlay is visible (see
+    // laser-beam quads are submitted as their own composition layers
-    // xr_input.c) - the menu/keyboard render into their own independent
+    // further down, not drawn into the game swapchain itself.
-    // swapchain images below.
     if (xr_is_session_running())
         xr_input_sync_and_draw(g_local_space, g_predicted_display_time,
                                 g_have_acquired_game_image);
@@ -374,8 +373,8 @@ void xr_frame_end(void) {
     // xr_overlay_render_and_build_layer()) any time before xrEndFrame,
     // unlike the game quad there's no per-frame engine rendering to wrap
     // around here, just each overlay's own blit - and, per hand, a
-    // laser-beam quad while that hand's ray is aimed at one of those two
+    // laser-beam quad while that hand's ray is aimed at the keyboard,
-    // overlays (see xr_input_get_beam_layer()).
+    // menu, or the game quad itself (see xr_input_get_beam_layer()).
     XrCompositionLayerQuad gameQuad = {XR_TYPE_COMPOSITION_LAYER_QUAD};
     gameQuad.space = g_local_space;
     gameQuad.eyeVisibility = XR_EYE_VISIBILITY_BOTH;

android/app/src/main/cpp/xr_swapchain.c

@@ -146,8 +146,7 @@ bool xr_swapchain_acquire(XrInstance instance, XrSwapchainState *state) {
     // gl4es's own (linked, not dlsym'd) bind - this targets an FBO id gl4es
     // itself created (see xr_swapchain_create()), so its own "current FBO"
     // bookkeeping updates correctly and its immediate-mode draw calls
-    // (android_draw_surface_as_quad(), the laser reticle) land in the right
+    // (android_draw_surface_as_quad()) land in the right place.
-    // place.
     glBindFramebuffer(GL_FRAMEBUFFER, state->fbos[imageIndex]);
     glViewport(0, 0, state->width, state->height);
     return true;