Add hand-built on-screen keyboard to the OpenXR menu quad

MenuOverlay's key grid forwards every press to the game as real input:
non-printable keys (Esc/Enter/Backspace) as an SDLActivity.onNativeKeyDown()/
onNativeKeyUp() pair, and printable keys (letters, Space) as a synthesized
SDL_TEXTINPUT event pushed directly from native code, since the engine's
pump_events() only picks up printable ASCII from that event type, not from
SDL_KEYDOWN. Confirmed on-device, including that Space now correctly skips
the intro cutscene alongside Esc/Enter.

README.md

@@ -140,15 +140,25 @@ can be sideloaded onto any Android-based VR headset with OpenXR support
 vendor's store. The game's own rendering is unchanged (still a flat, 2D
 render, no stereo 3D scene), but instead of running as a Home-hosted 2D
 panel it's now shown as a single head-tracked quad floating in front of
-the viewer, with a separate menu quad (currently just one "Keyboard"
+the viewer, with a separate menu quad toggled by a controller button and
-button - the actual on-screen keyboard behind it is still ongoing work)
+driven by a laser-pointer-style aim ray from each hand
-toggled by a controller button and driven by a laser-pointer-style aim
+(`android/app/src/main/cpp/xr_input.c`) - point and pull the trigger to
-ray from each hand (`android/app/src/main/cpp/xr_input.c`) - point and
+interact with it, same as the game's own Bluetooth mouse/keyboard input
-pull the trigger to interact with the menu, same as the game's own
+otherwise works unchanged. The menu's "Keyboard" button swaps to a
-Bluetooth mouse/keyboard input otherwise works unchanged. The laser
+hand-built on-screen key grid (`MenuOverlay.java` - there's no system IME
-pointer/cursor is currently only visible while actually aiming at the
+to borrow once immersive) that forwards each key press straight to the
-game or menu quad respectively - there's no visual feedback yet while
+game as real input - non-printable keys (Esc/Enter/Backspace) as a
-aiming at empty space between them. (Only tested on Meta Quest so far -
+`SDLActivity.onNativeKeyDown()`/`onNativeKeyUp()` pair, the same one a
+physical Bluetooth keyboard's presses already go through, and printable
+keys (letters, Space) via a synthesized `SDL_TEXTINPUT` event pushed
+directly from native code (`questshock_native.c`) - meant as a general
+stand-in for keyboard-driven functionality that isn't (yet, or ever)
+mapped onto the controllers, not just a future config screen's text
+entry. The laser
+pointer/cursor is currently only visible while
+actually aiming at the game or menu quad respectively - there's no visual
+feedback yet while aiming at empty space between them. (Only tested on
+Meta Quest so far -
 the steps below use Quest-specific tool names where relevant, but the
 same `adb install` flow applies to any Android headset with USB
 debugging enabled.)

android/app/src/main/cpp/questshock_native.c

@@ -7,9 +7,32 @@
 #include <jni.h>
 #include <unistd.h>
 
+#include <SDL.h>
+
 JNIEXPORT void JNICALL
 Java_de_ladkau_questshock_QuestShockActivity_nativeChdir(JNIEnv *env, jclass clazz, jstring path) {
     const char *cpath = (*env)->GetStringUTFChars(env, path, NULL);
     chdir(cpath);
     (*env)->ReleaseStringUTFChars(env, path, cpath);
 }
+
+// MenuOverlay's hand-built on-screen keyboard (see
+// MenuOverlay.commitPrintableChar()) has no real Android IME session behind
+// it, so printable characters can't reach the game via Android's usual IME
+// path - instead this builds and pushes an SDL_TEXTINPUT event directly,
+// the same event type/shape SDL's own Android backend pushes for typed
+// text, using only the public SDL_Event/SDL_PushEvent API. The engine's
+// pump_events() (sdl_events.c) only picks up printable characters from
+// this event, not from SDL_KEYDOWN (see handleKeyPress() in MenuOverlay.java
+// for why).
+JNIEXPORT void JNICALL
+Java_de_ladkau_questshock_MenuOverlay_nativeSendPrintableChar(JNIEnv *env, jclass clazz, jchar c) {
+    SDL_Event event;
+    SDL_zero(event);
+    event.type = SDL_TEXTINPUT;
+    event.text.timestamp = SDL_GetTicks();
+    event.text.windowID = 0;
+    event.text.text[0] = (char)c;
+    event.text.text[1] = '\0';
+    SDL_PushEvent(&event);
+}

android/app/src/main/cpp/xr_menu.h

@@ -1,8 +1,8 @@
 // Menu quad for questshock's immersive Quest build: MenuOverlay.java's
-// off-screen-rendered View tree (currently just a "Keyboard" button - see
+// off-screen-rendered View tree (a "Keyboard" button that swaps to a
-// the plan's step D for the on-screen keyboard that goes behind it),
+// hand-built on-screen key grid - there's no system IME to borrow once
-// toggled by xr_input.c's menu_toggle action and hit-tested by the same ray
+// immersive), toggled by xr_input.c's menu_toggle action and hit-tested by
-// xr_input.c already computes per hand.
+// the same ray xr_input.c already computes per hand.
 //
 // The menu has its own independent OpenXR swapchain, sized to exactly
 // xr_menu_get_content_size()'s dimensions (see xr_session.c, which creates

android/app/src/main/java/de/ladkau/questshock/MenuOverlay.java

@@ -5,13 +5,18 @@ import android.graphics.Bitmap;
 import android.graphics.Canvas;
 import android.graphics.Paint;
 import android.os.SystemClock;
-import android.util.Log;
+import android.util.TypedValue;
 import android.view.Gravity;
+import android.view.KeyEvent;
 import android.view.MotionEvent;
 import android.view.View;
+import android.view.ViewGroup;
 import android.widget.Button;
 import android.widget.FrameLayout;
+import android.widget.LinearLayout;
+import android.widget.TextView;
 import java.nio.ByteBuffer;
+import org.libsdl.app.SDLActivity;
 
 /**
  * An off-screen, never-attached-to-a-window View tree for questshock's
@@ -21,10 +26,22 @@ import java.nio.ByteBuffer;
  * synthetic MotionEvents into an unattached hierarchy both work the same
  * way they would for an attached View - draw(Canvas)/dispatchTouchEvent()
  * don't require a ViewRootImpl/window, just a measured+laid-out tree.
+ *
+ * Holds two same-size panels inside root, only one of which is visible at a
+ * time (see showPanel()): mainPanel (just the "Keyboard" button so far) and
+ * keyboardPanel (a hand-built key grid - there's no system IME to borrow
+ * once immersive - plus a text field echoing what's been sent). Each key
+ * press is forwarded straight to the game as real input (see
+ * handleKeyPress()) - non-printable keys as an SDLActivity.onNativeKeyDown()/
+ * onNativeKeyUp() pair, the same one a physical Bluetooth keyboard's presses
+ * already drive, and printable keys via a synthesized SDL_TEXTINPUT event
+ * pushed from native code (see commitPrintableChar()/
+ * nativeSendPrintableChar() in questshock_native.c) - this keyboard is
+ * meant as a general stand-in for whatever keyboard-driven functionality
+ * isn't (yet, or ever) mapped onto the controllers, not just a text-entry
+ * widget.
  */
 public class MenuOverlay {
-    private static final String TAG = "QuestShock";
-
     // Matches xr_menu.c's MENU_WIDTH/MENU_HEIGHT swapchain size - fixed,
     // not tied to any real display metric, since this tree is never shown
     // in a real window.
@@ -32,12 +49,60 @@ public class MenuOverlay {
     static final int HEIGHT = 768;
 
     private static final float CURSOR_RADIUS = 10f;
+    private static final int TYPED_TEXT_HEIGHT = 100;
+
+    // Button's own click handling (View.onTouchEvent()'s ACTION_UP case)
+    // calls View.post(mPerformClick) rather than invoking performClick()
+    // directly; post() queues the runnable to run once the view is attached
+    // to a window and returns true immediately even when unattached, so on
+    // this permanently-unattached tree that queued click silently never
+    // fires - dispatchTouchEvent() still delivers the down/up events
+    // correctly, only the click callback is swallowed. Registering this as
+    // each button's OnTouchListener bypasses that path entirely: returning
+    // true here skips View's internal onTouchEvent() (see
+    // ViewGroup/View#dispatchTouchEvent), so performClick() (which still
+    // runs any OnClickListener set via setOnClickListener()) is called
+    // directly instead.
+    private static final View.OnTouchListener CLICK_ON_TOUCH_UP = (v, event) -> {
+        switch (event.getAction()) {
+            case MotionEvent.ACTION_DOWN:
+                v.setPressed(true);
+                break;
+            case MotionEvent.ACTION_UP:
+                v.setPressed(false);
+                v.performClick();
+                break;
+            case MotionEvent.ACTION_CANCEL:
+                v.setPressed(false);
+                break;
+        }
+        return true;
+    };
 
     private static MenuOverlay instance;
 
     private final Activity activity;
     private final FrameLayout root;
-    private final Button keyboardButton;
+    private final FrameLayout mainPanel;
+    private final LinearLayout keyboardPanel;
+    private final TextView typedTextView;
+    private final StringBuilder typedText = new StringBuilder();
+    // contentBitmap holds just the panel's own rendered content (no cursor),
+    // redrawn only when that content actually changes (showPanel()/
+    // handleTouch()/handleKeyPress()) via root.draw() - a full off-screen
+    // View-tree traversal that gets noticeably more expensive on the
+    // busier keyboardPanel. bitmap is what's actually published to native
+    // (see nativeTakePixelsIfDirty()): compositeAndPublish() cheaply blits
+    // contentBitmap plus the cursor circle into it. Splitting these two
+    // apart matters because nativeUpdateCursor() below fires every single
+    // frame (~90Hz) while aiming at the menu - if it called the expensive
+    // root.draw() path every time (as an earlier version did), the
+    // keyboardPanel's larger view tree made each redraw slow enough that
+    // runOnUiThread() posts piled up faster than the UI thread could drain
+    // them, so the cursor visibly lagged minutes behind the controller's
+    // actual aim instead of tracking it.
+    private final Bitmap contentBitmap;
+    private final Canvas contentCanvas;
     private final Bitmap bitmap;
     private final Canvas canvas;
     private final Paint cursorPaint;
@@ -46,7 +111,7 @@ public class MenuOverlay {
     private byte[] pendingPixels;
 
     // Only ever touched on the UI thread (both nativeUpdateCursor() and
-    // forceRedraw() run/are posted there) - no lock needed, unlike
+    // compositeAndPublish() run/are posted there) - no lock needed, unlike
     // pendingPixels above.
     private boolean cursorVisible = false;
     private float cursorX = 0f;
@@ -58,13 +123,16 @@ public class MenuOverlay {
         root = new FrameLayout(activity);
         root.setBackgroundColor(0xFF202020);
 
-        keyboardButton = new Button(activity);
+        mainPanel = buildMainPanel();
-        keyboardButton.setText("Keyboard");
+        keyboardPanel = buildKeyboardPanel();
-        keyboardButton.setOnClickListener(v -> Log.i(TAG, "MenuOverlay: Keyboard button clicked"));
+        // buildKeyboardPanel() adds the text field as keyboardPanel's first
-        FrameLayout.LayoutParams lp = new FrameLayout.LayoutParams(420, 140);
+        // child, before any key rows.
-        lp.gravity = Gravity.CENTER;
+        typedTextView = (TextView) keyboardPanel.getChildAt(0);
-        root.addView(keyboardButton, lp);
+        root.addView(mainPanel, new FrameLayout.LayoutParams(WIDTH, HEIGHT));
+        root.addView(keyboardPanel, new FrameLayout.LayoutParams(WIDTH, HEIGHT));
 
+        contentBitmap = Bitmap.createBitmap(WIDTH, HEIGHT, Bitmap.Config.ARGB_8888);
+        contentCanvas = new Canvas(contentBitmap);
         bitmap = Bitmap.createBitmap(WIDTH, HEIGHT, Bitmap.Config.ARGB_8888);
         canvas = new Canvas(bitmap);
 
@@ -72,12 +140,138 @@ public class MenuOverlay {
         cursorPaint.setColor(0xFFFFFFFF);
         cursorPaint.setAntiAlias(true);
 
+        showPanel(mainPanel);
+        redrawContent();
+    }
+
+    private FrameLayout buildMainPanel() {
+        FrameLayout panel = new FrameLayout(activity);
+        Button keyboardButton = new Button(activity);
+        keyboardButton.setText("Keyboard");
+        keyboardButton.setOnClickListener(v -> showPanel(keyboardPanel));
+        keyboardButton.setOnTouchListener(CLICK_ON_TOUCH_UP);
+        FrameLayout.LayoutParams lp = new FrameLayout.LayoutParams(420, 140);
+        lp.gravity = Gravity.CENTER;
+        panel.addView(keyboardButton, lp);
+        return panel;
+    }
+
+    private LinearLayout buildKeyboardPanel() {
+        LinearLayout panel = new LinearLayout(activity);
+        panel.setOrientation(LinearLayout.VERTICAL);
+
+        TextView textView = new TextView(activity);
+        textView.setTextColor(0xFFFFFFFF);
+        textView.setTextSize(TypedValue.COMPLEX_UNIT_SP, 28);
+        textView.setBackgroundColor(0xFF303030);
+        textView.setPadding(20, 20, 20, 20);
+        panel.addView(textView, new LinearLayout.LayoutParams(
+            ViewGroup.LayoutParams.MATCH_PARENT, TYPED_TEXT_HEIGHT));
+
+        addKeyRow(panel, "Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P");
+        addKeyRow(panel, "A", "S", "D", "F", "G", "H", "J", "K", "L");
+        addKeyRow(panel, "Z", "X", "C", "V", "B", "N", "M");
+        addKeyRow(panel, "Esc", "Space", "Back", "Enter", "Done");
+        return panel;
+    }
+
+    // Each row fills the panel's remaining height evenly (weight 1 on the
+    // row itself); within a row, most keys share width equally except
+    // "Space" (wider, like a real keyboard) and "Esc"/"Back"/"Enter"/"Done"
+    // (narrower).
+    private void addKeyRow(LinearLayout panel, String... labels) {
+        LinearLayout row = new LinearLayout(activity);
+        row.setOrientation(LinearLayout.HORIZONTAL);
+        for (String label : labels) {
+            Button key = new Button(activity);
+            key.setText(label);
+            key.setOnClickListener(v -> handleKeyPress(label));
+            key.setOnTouchListener(CLICK_ON_TOUCH_UP);
+            row.addView(key, new LinearLayout.LayoutParams(
+                0, ViewGroup.LayoutParams.MATCH_PARENT, keyWeight(label)));
+        }
+        panel.addView(row, new LinearLayout.LayoutParams(
+            ViewGroup.LayoutParams.MATCH_PARENT, 0, 1f));
+    }
+
+    private static float keyWeight(String label) {
+        if (label.equals("Space")) {
+            return 4f;
+        }
+        if (label.equals("Back") || label.equals("Done") || label.equals("Esc")
+                || label.equals("Enter")) {
+            return 1.5f;
+        }
+        return 1f;
+    }
+
+    // "Done" is this overlay's own UI navigation (closes the key grid, no
+    // game-visible effect). Esc/Back/Enter are non-printable, so they're
+    // forwarded as real Android key events via SDLActivity.onNativeKeyDown()/
+    // onNativeKeyUp() - the same pair a physical Bluetooth keyboard's key
+    // events already drive (see SDLActivity.handleKeyEvent()). Every other
+    // key (letters, Space) instead goes through commitPrintableChar() below:
+    // the engine's own pump_events() (sdl_events.c) deliberately ignores
+    // SDL_KEYDOWN for printable ASCII and only reacts to SDL_TEXTINPUT for
+    // those, to avoid double-counting a typed character against a system
+    // IME - so that event has to be synthesized directly (see
+    // commitPrintableChar()). This all matters beyond just the visible
+    // typedTextView below: the engine's cutscene skip handler (cutsloop.c's
+    // cutscene_key_handler()) only reacts to Esc/Enter/Space, so those need
+    // to reach the game as actual game input, not just be echoed locally.
+    private void handleKeyPress(String label) {
+        if (label.equals("Done")) {
+            showPanel(mainPanel);
+            return;
+        }
+
+        if (label.equals("Back")) {
+            SDLActivity.onNativeKeyDown(KeyEvent.KEYCODE_DEL);
+            SDLActivity.onNativeKeyUp(KeyEvent.KEYCODE_DEL);
+            if (typedText.length() > 0) {
+                typedText.setLength(typedText.length() - 1);
+            }
+        } else if (label.equals("Esc") || label.equals("Enter")) {
+            int keyCode = label.equals("Esc") ? KeyEvent.KEYCODE_ESCAPE : KeyEvent.KEYCODE_ENTER;
+            SDLActivity.onNativeKeyDown(keyCode);
+            SDLActivity.onNativeKeyUp(keyCode);
+            typedText.append('[').append(label).append(']');
+        } else if (label.equals("Space")) {
+            commitPrintableChar(' ');
+            typedText.append(' ');
+        } else {
+            commitPrintableChar(Character.toLowerCase(label.charAt(0)));
+            typedText.append(label);
+        }
+        typedTextView.setText(typedText.toString());
+    }
+
+    // Synthesizes the SDL_TEXTINPUT event pump_events() (sdl_events.c)
+    // requires for printable characters (see handleKeyPress() above), by
+    // building and pushing it directly in native code (see
+    // nativeSendPrintableChar() in questshock_native.c) rather than through
+    // Android's IME (there's no real IME session behind this off-screen,
+    // never-attached grid for that plumbing to hook into).
+    private static void commitPrintableChar(char c) {
+        nativeSendPrintableChar(c);
+    }
+
+    private static native void nativeSendPrintableChar(char c);
+
+    // Only one of mainPanel/keyboardPanel is ever visible at a time. Both
+    // were already measured/laid out once in the constructor's initial
+    // showPanel() call - re-running measure()/layout() here (rather than
+    // just flipping visibility) is required every time regardless, since
+    // FrameLayout skips GONE children during measure/layout, so a panel
+    // switching from GONE to VISIBLE needs a fresh pass to get valid bounds
+    // before its content can be drawn or hit-tested.
+    private void showPanel(View panel) {
+        mainPanel.setVisibility(panel == mainPanel ? View.VISIBLE : View.GONE);
+        keyboardPanel.setVisibility(panel == keyboardPanel ? View.VISIBLE : View.GONE);
         int widthSpec = View.MeasureSpec.makeMeasureSpec(WIDTH, View.MeasureSpec.EXACTLY);
         int heightSpec = View.MeasureSpec.makeMeasureSpec(HEIGHT, View.MeasureSpec.EXACTLY);
         root.measure(widthSpec, heightSpec);
         root.layout(0, 0, WIDTH, HEIGHT);
-
-        forceRedraw();
     }
 
     // Called from xr_menu.c's xr_menu_init(), on the render thread, right
@@ -140,7 +334,7 @@ public class MenuOverlay {
         cursorVisible = visible;
         cursorX = x;
         cursorY = y;
-        forceRedraw();
+        compositeAndPublish();
     }
 
     private void handleTouch(float u, float v, boolean down) {
@@ -154,12 +348,23 @@ public class MenuOverlay {
         } finally {
             event.recycle();
         }
-        forceRedraw();
+        redrawContent();
     }
 
-    private void forceRedraw() {
+    // Re-runs the full off-screen View-tree draw (expensive - see the
-        canvas.drawColor(0xFF202020);
+    // contentBitmap field comment above) - call only when the panel's
-        root.draw(canvas);
+    // actual content changed, not for the cursor-only updates
+    // compositeAndPublish() below handles on its own.
+    private void redrawContent() {
+        contentCanvas.drawColor(0xFF202020);
+        root.draw(contentCanvas);
+        compositeAndPublish();
+    }
+
+    // Cheap per-frame path: blits the last-rendered contentBitmap (no View
+    // traversal) plus the cursor circle into bitmap and publishes it.
+    private void compositeAndPublish() {
+        canvas.drawBitmap(contentBitmap, 0, 0, null);
         if (cursorVisible) {
             canvas.drawCircle(cursorX, cursorY, CURSOR_RADIUS, cursorPaint);
         }