matthias/questshock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Split the keyboard onto its own translucent OpenXR quad
build / build (push) Successful in 1m52s
Details

Browse Source 
Expand the on-screen keyboard to a full US layout (letters, digits,
punctuation, Shift layer, Tab, arrows, F1-F12, one-shot Ctrl/Alt
modifiers), make it a persistent, movable, closeable panel via a
title-bar drag handle, and lower its opacity.

Previously the keyboard was just a second panel swapped into the same
quad as the menu launcher, which made the launcher translucent too and
made the keyboard and menu mutually exclusive. Extract the shared
swapchain/JNI/touch-dispatch plumbing into two generic, reusable
modules - xr_swapchain.c (swapchain + per-image FBO setup) and
xr_overlay.c (an off-screen Android View rendered into its own OpenXR
quad, hit-tested via a laser pointer) - and make the menu launcher and
keyboard two independent XrOverlay instances instead of one. The
controller's menu button now only opens/closes the launcher; the
launcher's "Keyboard" button only opens the keyboard - so the keyboard
can stay up while picking something from the menu, and only the
keyboard's own Close button hides it.

xr_menu.c/.h are gone, fully absorbed into xr_overlay.c. On the Java
side, OverlayPanel.java carries the shared off-screen-render/touch/
cursor plumbing that MenuOverlay and the new KeyboardOverlay both
subclass.
This commit is contained in:
ml
2026-08-14 06:43:44 +02:00
parent 2d8fb49bf5
commit 3be310ade6
16 changed files with 1897 additions and 1102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
android/app/src/main/cpp/xr_input.c
+200 -68
View File
@@ -7,7 +7,7 @@
#include <GLES3/gl3.h>
#include "xr_menu.h"
#include "xr_overlay.h"
#include "xr_session.h"
#define TAG "QuestShock"
@@ -17,6 +17,24 @@
#define LEFT 0
#define RIGHT 1
static const char *kHandName[2] = {"left", "right"};
static const float kHandColor[2][4] = {
{0.2f, 1.0f, 1.0f, 1.0f}, // left: cyan
{1.0f, 0.85f, 0.1f, 1.0f}, // right: amber
};
// The keyboard overlay's title bar (drag handle + Close button) is handled
// entirely here, before a hit would normally be dispatched to
// KeyboardOverlay as a touch - these fractions must agree with
// KeyboardOverlay.java's TITLE_BAR_HEIGHT/CLOSE_BUTTON_WIDTH constants (in
// pixels, out of KeyboardOverlay.WIDTH/HEIGHT there). A hit in the top
// TITLE_BAR_V_FRACTION of the quad, at or left of CLOSE_BUTTON_U_FRACTION,
// is the drag handle; right of that (still within the title bar) is the
// Close button, which is dispatched as an ordinary click instead. The menu
// launcher overlay has no title bar - it's a plain click-only panel.
#define TITLE_BAR_V_FRACTION (90.0f / 768.0f)
#define CLOSE_BUTTON_U_FRACTION (1.0f - 160.0f / 1024.0f)
static XrInstance g_instance = XR_NULL_HANDLE;
static XrSession g_session = XR_NULL_HANDLE;
static XrActionSet g_action_set = XR_NULL_HANDLE;
@@ -27,14 +45,29 @@ static XrPath g_hand_path[2];
static XrSpace g_aim_space[2] = {XR_NULL_HANDLE, XR_NULL_HANDLE};
// Edge-detection state, so touch dispatch and logging only fire on actual
// state changes, not every frame.
// state changes, not every frame. Each interaction target (game quad, menu
// launcher, keyboard) tracks its own "was this hand's ray hitting it last
// frame" independently, since the menu and keyboard overlays can now both
// be visible at once.
static bool g_prev_select[2] = {false, false};
static bool g_prev_menu = false;
static bool g_prev_hit[2] = {false, false};
// Tracks which hand currently has an in-flight synthetic touch down on the
// menu (so a later trigger-up edge only dispatches a matching touch-up if
// a touch-down was actually sent for that hand).
static bool g_game_prev_hit[2] = {false, false};
static bool g_menu_prev_hit[2] = {false, false};
static bool g_menu_touch_active[2] = {false, false};
static bool g_keyboard_prev_hit[2] = {false, false};
static bool g_keyboard_touch_active[2] = {false, false};
// Drag state for repositioning the keyboard overlay via its title bar (see
// TITLE_BAR_V_FRACTION/CLOSE_BUTTON_U_FRACTION above) - the menu launcher
// has no title bar, so it's never draggable. g_keyboard_drag_hand is -1
// when g_keyboard_dragging is false. The offset is the vector from the
// quad's center to the grabbed point at drag-start, kept constant for the
// whole gesture so the quad follows the hand rigidly rather than
// re-centering under the ray each frame.
static bool g_keyboard_dragging = false;
static int g_keyboard_drag_hand = -1;
static float g_keyboard_drag_offset_x = 0.0f;
static float g_keyboard_drag_offset_y = 0.0f;
static GLuint g_reticle_program = 0;
static GLint g_reticle_color_loc = -1;
@@ -220,6 +253,106 @@ bool xr_input_init(XrInstance instance, XrSession session) {
return true;
}
// Hit-tests/dispatches this hand's ray against one overlay quad - shared
// by the menu launcher and keyboard overlays in xr_input_sync_and_draw()
// below. dragCapable enables the keyboard-only title-bar/drag-handle
// handling (see TITLE_BAR_V_FRACTION/CLOSE_BUTTON_U_FRACTION); the menu
// launcher has no title bar, so it's always false there and every hit is a
// plain click. Returns true if this overlay is visible - claiming the
// hand's ray processing for this frame, regardless of whether the ray
// actually hits it - so the caller should stop trying other targets (menu/
// keyboard/game quad are mutually exclusive per hand, per frame).
static bool xr_input_try_overlay(XrOverlay *overlay, bool dragCapable, int hand,
const XrSpaceLocation *location, float fx, float fy, float fz,
bool selectDownEdge, bool selectUpEdge, bool *touchActive,
bool *prevHit, const char *quadName, float *outCursorU,
float *outCursorV, bool *outCursorHit) {
if (!xr_overlay_is_visible(overlay))
return false;
float quadCenterX, quadCenterY, distance, halfWidth, halfHeight;
xr_overlay_get_quad_extent(overlay, &quadCenterX, &quadCenterY, &distance, &halfWidth,
&halfHeight);
// The quad's plane is z = -distance (facing the viewer along -Z) - a
// ray parallel to it (fz ~ 0) never crosses. worldX/Y is the raw
// world-space intersection point, needed as-is for dragging (which can
// legitimately move the quad to where the ray isn't currently hitting
// it); cx/cy are that same point in the quad's own [-1,1] local space;
// hit/u/v (the equivalent 0..1, top-left-origin coordinates the Java
// view's pixel grid and the on-quad hit logs use) are only meaningful
// within the quad's current bounds, unlike planeHit.
bool planeHit = false, hit = false;
float u = 0.0f, v = 0.0f, worldX = 0.0f, worldY = 0.0f;
if (fabsf(fz) > 1e-5f) {
float t = (-distance - location->pose.position.z) / fz;
if (t > 0.0f) {
planeHit = true;
worldX = location->pose.position.x + t * fx;
worldY = location->pose.position.y + t * fy;
float cx = (worldX - quadCenterX) / halfWidth;
float cy = (worldY - quadCenterY) / halfHeight;
hit = fabsf(cx) <= 1.0f && fabsf(cy) <= 1.0f;
u = (cx + 1.0f) * 0.5f;
v = (1.0f - cy) * 0.5f;
}
}
if (hit != *prevHit) {
LOGI("XR: %s aim ray %s %s quad (u=%.2f v=%.2f)", kHandName[hand],
hit ? "entered" : "left", quadName, u, v);
*prevHit = hit;
}
if (hit) {
*outCursorHit = true;
*outCursorU = u;
*outCursorV = v;
}
if (!dragCapable) {
if (selectDownEdge && hit) {
xr_overlay_touch(overlay, u, v, true);
*touchActive = true;
} else if (selectUpEdge && *touchActive) {
xr_overlay_touch(overlay, u, v, false);
*touchActive = false;
}
return true;
}
bool onTitleBar = hit && v < TITLE_BAR_V_FRACTION;
bool onDragHandle = onTitleBar && u <= CLOSE_BUTTON_U_FRACTION;
if (selectDownEdge) {
if (onDragHandle) {
g_keyboard_dragging = true;
g_keyboard_drag_hand = hand;
g_keyboard_drag_offset_x = worldX - quadCenterX;
g_keyboard_drag_offset_y = worldY - quadCenterY;
} else if (hit) {
xr_overlay_touch(overlay, u, v, true);
*touchActive = true;
}
} else if (selectUpEdge) {
if (g_keyboard_dragging && g_keyboard_drag_hand == hand) {
g_keyboard_dragging = false;
g_keyboard_drag_hand = -1;
} else if (*touchActive) {
xr_overlay_touch(overlay, u, v, false);
*touchActive = false;
}
}
// Re-intersects against the same fixed-distance plane every frame the
// drag continues, so the quad tracks the hand in real time rather than
// only jumping on the down/up edges above.
if (g_keyboard_dragging && g_keyboard_drag_hand == hand && planeHit) {
xr_overlay_set_position(overlay, worldX - g_keyboard_drag_offset_x,
worldY - g_keyboard_drag_offset_y);
}
return true;
}
void xr_input_sync_and_draw(XrSpace baseSpace, XrTime time, bool draw) {
if (g_action_set == XR_NULL_HANDLE)
return;
@@ -230,36 +363,37 @@ void xr_input_sync_and_draw(XrSpace baseSpace, XrTime time, bool draw) {
syncInfo.activeActionSets = &activeSet;
xr_check(xrSyncActions(g_session, &syncInfo), "xrSyncActions");
bool menuVisible = xr_menu_is_visible();
XrOverlay *menuOverlay = xr_session_get_menu_overlay();
XrOverlay *keyboardOverlay = xr_session_get_keyboard_overlay();
bool menuVisible = xr_overlay_is_visible(menuOverlay);
bool keyboardVisible = xr_overlay_is_visible(keyboardOverlay);
float quadCenterX = 0.0f, quadCenterY = 0.0f, distance, halfWidth, halfHeight;
if (menuVisible)
xr_menu_get_quad_extent(&quadCenterX, &quadCenterY, &distance, &halfWidth, &halfHeight);
else
xr_get_game_quad_extent(&distance, &halfWidth, &halfHeight);
static const char *kHandName[2] = {"left", "right"};
static const float kHandColor[2][4] = {
{0.2f, 1.0f, 1.0f, 1.0f}, // left: cyan
{1.0f, 0.85f, 0.1f, 1.0f}, // right: amber
};
// A drag can't get stuck active across a hide/reopen - e.g. Close
// being hit on one hand mid-drag on the other, or the session ending.
if (!keyboardVisible) {
g_keyboard_dragging = false;
g_keyboard_drag_hand = -1;
}
// The GL reticle is drawn directly into whatever framebuffer is
// currently bound - the game quad's swapchain image (see xr_frame_end(),
// which calls this while that image is still bound). That only makes
// sense while aiming at the game quad; the menu quad's own cursor is
// drawn by MenuOverlay itself instead (see xr_menu_update_cursor()
// below), composited into its Bitmap the same way its other content is.
bool drawReticle = draw && !menuVisible;
// sense while aiming at the game quad; the menu/keyboard overlays' own
// cursors are drawn by their Java views instead (see
// xr_overlay_update_cursor() below), composited into their Bitmaps the
// same way their other content is.
bool drawReticle = draw && !menuVisible && !keyboardVisible;
if (drawReticle)
glUseProgram(g_reticle_program);
// Fed to xr_menu_update_cursor() after the loop below - whichever hand's
// ray hits the menu quad last wins if both do, good enough for a single
// on-quad cursor (only ever updated when hit is true, so a later hand
// that misses doesn't hide an earlier hand's hit).
bool menuCursorHit = false;
// Fed to xr_overlay_update_cursor() after the loop below - whichever
// hand's ray hits a given overlay last wins if both do, good enough
// for a single on-quad cursor per overlay (only ever updated when hit
// is true, so a later hand that misses doesn't hide an earlier hand's
// hit).
bool menuCursorHit = false, keyboardCursorHit = false;
float menuCursorU = 0.0f, menuCursorV = 0.0f;
float keyboardCursorU = 0.0f, keyboardCursorV = 0.0f;
for (int hand = 0; hand < 2; hand++) {
XrActionStateGetInfo selectInfo = {XR_TYPE_ACTION_STATE_GET_INFO};
@@ -284,13 +418,14 @@ void xr_input_sync_and_draw(XrSpace baseSpace, XrTime time, bool draw) {
bool menuDown = menuState.isActive && menuState.currentState;
if (menuDown && !g_prev_menu) {
LOGI("XR: menu-toggle button DOWN");
xr_menu_toggle_visible();
// Avoid a stale hit/touch-active carrying over from
// whichever quad was being tested against before the
// toggle - the next frame re-evaluates against the new
// target from a clean state.
g_prev_hit[LEFT] = g_prev_hit[RIGHT] = false;
g_menu_touch_active[LEFT] = g_menu_touch_active[RIGHT] = false;
// Only ever affects the menu launcher - the keyboard is
// fully independent (its own Close button is the only way
// to hide it, see KeyboardOverlay.java).
if (menuOverlay != NULL) {
xr_overlay_toggle_visible(menuOverlay);
g_menu_prev_hit[LEFT] = g_menu_prev_hit[RIGHT] = false;
g_menu_touch_active[LEFT] = g_menu_touch_active[RIGHT] = false;
}
} else if (!menuDown && g_prev_menu) {
LOGI("XR: menu-toggle button UP");
}
@@ -316,51 +451,40 @@ void xr_input_sync_and_draw(XrSpace baseSpace, XrTime time, bool draw) {
float fx, fy, fz;
quat_rotate_vec(&location.pose.orientation, 0.0f, 0.0f, -1.0f, &fx, &fy, &fz);
// The quad's plane is z = -distance (facing the viewer along -Z,
// see QUAD_DISTANCE_METERS/xr_menu.c's MENU_DISTANCE_METERS) - a
// ray parallel to it (fz ~ 0) never crosses. cx/cy are the hit
// point in the quad's own [-1,1] local space (the same space
// android_draw_surface_as_quad() draws its own vertex positions
// in); u/v are the equivalent 0..1, top-left-origin coordinates
// MenuOverlay's pixel grid and the on-quad hit logs both use.
// Keyboard first (it's the more likely target while it's up), then
// the menu launcher, then - only if neither is visible - the game
// quad's own reticle below. A single ray only ever interacts with
// one target per hand per frame.
if (xr_input_try_overlay(keyboardOverlay, true, hand, &location, fx, fy, fz,
selectDownEdge, selectUpEdge, &g_keyboard_touch_active[hand],
&g_keyboard_prev_hit[hand], "keyboard", &keyboardCursorU,
&keyboardCursorV, &keyboardCursorHit))
continue;
if (xr_input_try_overlay(menuOverlay, false, hand, &location, fx, fy, fz, selectDownEdge,
selectUpEdge, &g_menu_touch_active[hand], &g_menu_prev_hit[hand],
"menu", &menuCursorU, &menuCursorV, &menuCursorHit))
continue;
float distance, halfWidth, halfHeight;
xr_get_game_quad_extent(&distance, &halfWidth, &halfHeight);
bool hit = false;
float u = 0.0f, v = 0.0f, cx = 0.0f, cy = 0.0f;
if (fabsf(fz) > 1e-5f) {
float t = (-distance - location.pose.position.z) / fz;
if (t > 0.0f) {
cx = (location.pose.position.x + t * fx - quadCenterX) / halfWidth;
cy = (location.pose.position.y + t * fy - quadCenterY) / halfHeight;
cx = (location.pose.position.x + t * fx) / halfWidth;
cy = (location.pose.position.y + t * fy) / halfHeight;
hit = fabsf(cx) <= 1.0f && fabsf(cy) <= 1.0f;
u = (cx + 1.0f) * 0.5f;
v = (1.0f - cy) * 0.5f;
}
}
if (menuVisible) {
if (hit != g_prev_hit[hand]) {
LOGI("XR: %s aim ray %s menu quad (u=%.2f v=%.2f)", kHandName[hand],
hit ? "entered" : "left", u, v);
g_prev_hit[hand] = hit;
}
if (hit) {
menuCursorHit = true;
menuCursorU = u;
menuCursorV = v;
}
if (selectDownEdge && hit) {
xr_menu_touch(u, v, true);
g_menu_touch_active[hand] = true;
} else if (selectUpEdge && g_menu_touch_active[hand]) {
xr_menu_touch(u, v, false);
g_menu_touch_active[hand] = false;
}
continue;
}
if (hit != g_prev_hit[hand]) {
if (hit != g_game_prev_hit[hand]) {
LOGI("XR: %s aim ray %s game quad (u=%.2f v=%.2f)", kHandName[hand],
hit ? "entered" : "left", u, v);
g_prev_hit[hand] = hit;
g_game_prev_hit[hand] = hit;
}
if (!hit || !drawReticle)
@@ -378,7 +502,10 @@ void xr_input_sync_and_draw(XrSpace baseSpace, XrTime time, bool draw) {
}
if (menuVisible)
xr_menu_update_cursor(menuCursorU, menuCursorV, menuCursorHit);
xr_overlay_update_cursor(menuOverlay, menuCursorU, menuCursorV, menuCursorHit);
if (keyboardVisible)
xr_overlay_update_cursor(keyboardOverlay, keyboardCursorU, keyboardCursorV,
keyboardCursorHit);
}
void xr_input_shutdown(void) {
@@ -401,7 +528,12 @@ void xr_input_shutdown(void) {
g_instance = XR_NULL_HANDLE;
g_session = XR_NULL_HANDLE;
memset(g_prev_select, 0, sizeof(g_prev_select));
memset(g_prev_hit, 0, sizeof(g_prev_hit));
memset(g_game_prev_hit, 0, sizeof(g_game_prev_hit));
memset(g_menu_prev_hit, 0, sizeof(g_menu_prev_hit));
memset(g_menu_touch_active, 0, sizeof(g_menu_touch_active));
memset(g_keyboard_prev_hit, 0, sizeof(g_keyboard_prev_hit));
memset(g_keyboard_touch_active, 0, sizeof(g_keyboard_touch_active));
g_prev_menu = false;
g_keyboard_dragging = false;
g_keyboard_drag_hand = -1;
}