diff --git a/README.md b/README.md index 323d2a7..d964827 100644 --- a/README.md +++ b/README.md @@ -140,15 +140,25 @@ can be sideloaded onto any Android-based VR headset with OpenXR support vendor's store. The game's own rendering is unchanged (still a flat, 2D render, no stereo 3D scene), but instead of running as a Home-hosted 2D panel it's now shown as a single head-tracked quad floating in front of -the viewer, with a separate menu quad (currently just one "Keyboard" -button - the actual on-screen keyboard behind it is still ongoing work) -toggled by a controller button and driven by a laser-pointer-style aim -ray from each hand (`android/app/src/main/cpp/xr_input.c`) - point and -pull the trigger to interact with the menu, same as the game's own -Bluetooth mouse/keyboard input otherwise works unchanged. The laser -pointer/cursor is currently only visible while actually aiming at the -game or menu quad respectively - there's no visual feedback yet while -aiming at empty space between them. (Only tested on Meta Quest so far - +the viewer, with a separate menu quad toggled by a controller button and +driven by a laser-pointer-style aim ray from each hand +(`android/app/src/main/cpp/xr_input.c`) - point and pull the trigger to +interact with it, same as the game's own Bluetooth mouse/keyboard input +otherwise works unchanged. The menu's "Keyboard" button swaps to a +hand-built on-screen key grid (`MenuOverlay.java` - there's no system IME +to borrow once immersive) that forwards each key press straight to the +game as real input - non-printable keys (Esc/Enter/Backspace) as a +`SDLActivity.onNativeKeyDown()`/`onNativeKeyUp()` pair, the same one a +physical Bluetooth keyboard's presses already go through, and printable +keys (letters, Space) via a synthesized `SDL_TEXTINPUT` event pushed +directly from native code (`questshock_native.c`) - meant as a general +stand-in for keyboard-driven functionality that isn't (yet, or ever) +mapped onto the controllers, not just a future config screen's text +entry. The laser +pointer/cursor is currently only visible while +actually aiming at the game or menu quad respectively - there's no visual +feedback yet while aiming at empty space between them. (Only tested on +Meta Quest so far - the steps below use Quest-specific tool names where relevant, but the same `adb install` flow applies to any Android headset with USB debugging enabled.) diff --git a/android/app/src/main/cpp/questshock_native.c b/android/app/src/main/cpp/questshock_native.c index 26dd92a..93aa706 100644 --- a/android/app/src/main/cpp/questshock_native.c +++ b/android/app/src/main/cpp/questshock_native.c @@ -7,9 +7,32 @@ #include #include +#include + JNIEXPORT void JNICALL Java_de_ladkau_questshock_QuestShockActivity_nativeChdir(JNIEnv *env, jclass clazz, jstring path) { const char *cpath = (*env)->GetStringUTFChars(env, path, NULL); chdir(cpath); (*env)->ReleaseStringUTFChars(env, path, cpath); } + +// MenuOverlay's hand-built on-screen keyboard (see +// MenuOverlay.commitPrintableChar()) has no real Android IME session behind +// it, so printable characters can't reach the game via Android's usual IME +// path - instead this builds and pushes an SDL_TEXTINPUT event directly, +// the same event type/shape SDL's own Android backend pushes for typed +// text, using only the public SDL_Event/SDL_PushEvent API. The engine's +// pump_events() (sdl_events.c) only picks up printable characters from +// this event, not from SDL_KEYDOWN (see handleKeyPress() in MenuOverlay.java +// for why). +JNIEXPORT void JNICALL +Java_de_ladkau_questshock_MenuOverlay_nativeSendPrintableChar(JNIEnv *env, jclass clazz, jchar c) { + SDL_Event event; + SDL_zero(event); + event.type = SDL_TEXTINPUT; + event.text.timestamp = SDL_GetTicks(); + event.text.windowID = 0; + event.text.text[0] = (char)c; + event.text.text[1] = '\0'; + SDL_PushEvent(&event); +} diff --git a/android/app/src/main/cpp/xr_menu.h b/android/app/src/main/cpp/xr_menu.h index 7e49068..8c2ff2f 100644 --- a/android/app/src/main/cpp/xr_menu.h +++ b/android/app/src/main/cpp/xr_menu.h @@ -1,8 +1,8 @@ // Menu quad for questshock's immersive Quest build: MenuOverlay.java's -// off-screen-rendered View tree (currently just a "Keyboard" button - see -// the plan's step D for the on-screen keyboard that goes behind it), -// toggled by xr_input.c's menu_toggle action and hit-tested by the same ray -// xr_input.c already computes per hand. +// off-screen-rendered View tree (a "Keyboard" button that swaps to a +// hand-built on-screen key grid - there's no system IME to borrow once +// immersive), toggled by xr_input.c's menu_toggle action and hit-tested by +// the same ray xr_input.c already computes per hand. // // The menu has its own independent OpenXR swapchain, sized to exactly // xr_menu_get_content_size()'s dimensions (see xr_session.c, which creates diff --git a/android/app/src/main/java/de/ladkau/questshock/MenuOverlay.java b/android/app/src/main/java/de/ladkau/questshock/MenuOverlay.java index b76dbeb..1ad046d 100644 --- a/android/app/src/main/java/de/ladkau/questshock/MenuOverlay.java +++ b/android/app/src/main/java/de/ladkau/questshock/MenuOverlay.java @@ -5,13 +5,18 @@ import android.graphics.Bitmap; import android.graphics.Canvas; import android.graphics.Paint; import android.os.SystemClock; -import android.util.Log; +import android.util.TypedValue; import android.view.Gravity; +import android.view.KeyEvent; import android.view.MotionEvent; import android.view.View; +import android.view.ViewGroup; import android.widget.Button; import android.widget.FrameLayout; +import android.widget.LinearLayout; +import android.widget.TextView; import java.nio.ByteBuffer; +import org.libsdl.app.SDLActivity; /** * An off-screen, never-attached-to-a-window View tree for questshock's @@ -21,10 +26,22 @@ import java.nio.ByteBuffer; * synthetic MotionEvents into an unattached hierarchy both work the same * way they would for an attached View - draw(Canvas)/dispatchTouchEvent() * don't require a ViewRootImpl/window, just a measured+laid-out tree. + * + * Holds two same-size panels inside root, only one of which is visible at a + * time (see showPanel()): mainPanel (just the "Keyboard" button so far) and + * keyboardPanel (a hand-built key grid - there's no system IME to borrow + * once immersive - plus a text field echoing what's been sent). Each key + * press is forwarded straight to the game as real input (see + * handleKeyPress()) - non-printable keys as an SDLActivity.onNativeKeyDown()/ + * onNativeKeyUp() pair, the same one a physical Bluetooth keyboard's presses + * already drive, and printable keys via a synthesized SDL_TEXTINPUT event + * pushed from native code (see commitPrintableChar()/ + * nativeSendPrintableChar() in questshock_native.c) - this keyboard is + * meant as a general stand-in for whatever keyboard-driven functionality + * isn't (yet, or ever) mapped onto the controllers, not just a text-entry + * widget. */ public class MenuOverlay { - private static final String TAG = "QuestShock"; - // Matches xr_menu.c's MENU_WIDTH/MENU_HEIGHT swapchain size - fixed, // not tied to any real display metric, since this tree is never shown // in a real window. @@ -32,12 +49,60 @@ public class MenuOverlay { static final int HEIGHT = 768; private static final float CURSOR_RADIUS = 10f; + private static final int TYPED_TEXT_HEIGHT = 100; + + // Button's own click handling (View.onTouchEvent()'s ACTION_UP case) + // calls View.post(mPerformClick) rather than invoking performClick() + // directly; post() queues the runnable to run once the view is attached + // to a window and returns true immediately even when unattached, so on + // this permanently-unattached tree that queued click silently never + // fires - dispatchTouchEvent() still delivers the down/up events + // correctly, only the click callback is swallowed. Registering this as + // each button's OnTouchListener bypasses that path entirely: returning + // true here skips View's internal onTouchEvent() (see + // ViewGroup/View#dispatchTouchEvent), so performClick() (which still + // runs any OnClickListener set via setOnClickListener()) is called + // directly instead. + private static final View.OnTouchListener CLICK_ON_TOUCH_UP = (v, event) -> { + switch (event.getAction()) { + case MotionEvent.ACTION_DOWN: + v.setPressed(true); + break; + case MotionEvent.ACTION_UP: + v.setPressed(false); + v.performClick(); + break; + case MotionEvent.ACTION_CANCEL: + v.setPressed(false); + break; + } + return true; + }; private static MenuOverlay instance; private final Activity activity; private final FrameLayout root; - private final Button keyboardButton; + private final FrameLayout mainPanel; + private final LinearLayout keyboardPanel; + private final TextView typedTextView; + private final StringBuilder typedText = new StringBuilder(); + // contentBitmap holds just the panel's own rendered content (no cursor), + // redrawn only when that content actually changes (showPanel()/ + // handleTouch()/handleKeyPress()) via root.draw() - a full off-screen + // View-tree traversal that gets noticeably more expensive on the + // busier keyboardPanel. bitmap is what's actually published to native + // (see nativeTakePixelsIfDirty()): compositeAndPublish() cheaply blits + // contentBitmap plus the cursor circle into it. Splitting these two + // apart matters because nativeUpdateCursor() below fires every single + // frame (~90Hz) while aiming at the menu - if it called the expensive + // root.draw() path every time (as an earlier version did), the + // keyboardPanel's larger view tree made each redraw slow enough that + // runOnUiThread() posts piled up faster than the UI thread could drain + // them, so the cursor visibly lagged minutes behind the controller's + // actual aim instead of tracking it. + private final Bitmap contentBitmap; + private final Canvas contentCanvas; private final Bitmap bitmap; private final Canvas canvas; private final Paint cursorPaint; @@ -46,7 +111,7 @@ public class MenuOverlay { private byte[] pendingPixels; // Only ever touched on the UI thread (both nativeUpdateCursor() and - // forceRedraw() run/are posted there) - no lock needed, unlike + // compositeAndPublish() run/are posted there) - no lock needed, unlike // pendingPixels above. private boolean cursorVisible = false; private float cursorX = 0f; @@ -58,13 +123,16 @@ public class MenuOverlay { root = new FrameLayout(activity); root.setBackgroundColor(0xFF202020); - keyboardButton = new Button(activity); - keyboardButton.setText("Keyboard"); - keyboardButton.setOnClickListener(v -> Log.i(TAG, "MenuOverlay: Keyboard button clicked")); - FrameLayout.LayoutParams lp = new FrameLayout.LayoutParams(420, 140); - lp.gravity = Gravity.CENTER; - root.addView(keyboardButton, lp); + mainPanel = buildMainPanel(); + keyboardPanel = buildKeyboardPanel(); + // buildKeyboardPanel() adds the text field as keyboardPanel's first + // child, before any key rows. + typedTextView = (TextView) keyboardPanel.getChildAt(0); + root.addView(mainPanel, new FrameLayout.LayoutParams(WIDTH, HEIGHT)); + root.addView(keyboardPanel, new FrameLayout.LayoutParams(WIDTH, HEIGHT)); + contentBitmap = Bitmap.createBitmap(WIDTH, HEIGHT, Bitmap.Config.ARGB_8888); + contentCanvas = new Canvas(contentBitmap); bitmap = Bitmap.createBitmap(WIDTH, HEIGHT, Bitmap.Config.ARGB_8888); canvas = new Canvas(bitmap); @@ -72,12 +140,138 @@ public class MenuOverlay { cursorPaint.setColor(0xFFFFFFFF); cursorPaint.setAntiAlias(true); + showPanel(mainPanel); + redrawContent(); + } + + private FrameLayout buildMainPanel() { + FrameLayout panel = new FrameLayout(activity); + Button keyboardButton = new Button(activity); + keyboardButton.setText("Keyboard"); + keyboardButton.setOnClickListener(v -> showPanel(keyboardPanel)); + keyboardButton.setOnTouchListener(CLICK_ON_TOUCH_UP); + FrameLayout.LayoutParams lp = new FrameLayout.LayoutParams(420, 140); + lp.gravity = Gravity.CENTER; + panel.addView(keyboardButton, lp); + return panel; + } + + private LinearLayout buildKeyboardPanel() { + LinearLayout panel = new LinearLayout(activity); + panel.setOrientation(LinearLayout.VERTICAL); + + TextView textView = new TextView(activity); + textView.setTextColor(0xFFFFFFFF); + textView.setTextSize(TypedValue.COMPLEX_UNIT_SP, 28); + textView.setBackgroundColor(0xFF303030); + textView.setPadding(20, 20, 20, 20); + panel.addView(textView, new LinearLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, TYPED_TEXT_HEIGHT)); + + addKeyRow(panel, "Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P"); + addKeyRow(panel, "A", "S", "D", "F", "G", "H", "J", "K", "L"); + addKeyRow(panel, "Z", "X", "C", "V", "B", "N", "M"); + addKeyRow(panel, "Esc", "Space", "Back", "Enter", "Done"); + return panel; + } + + // Each row fills the panel's remaining height evenly (weight 1 on the + // row itself); within a row, most keys share width equally except + // "Space" (wider, like a real keyboard) and "Esc"/"Back"/"Enter"/"Done" + // (narrower). + private void addKeyRow(LinearLayout panel, String... labels) { + LinearLayout row = new LinearLayout(activity); + row.setOrientation(LinearLayout.HORIZONTAL); + for (String label : labels) { + Button key = new Button(activity); + key.setText(label); + key.setOnClickListener(v -> handleKeyPress(label)); + key.setOnTouchListener(CLICK_ON_TOUCH_UP); + row.addView(key, new LinearLayout.LayoutParams( + 0, ViewGroup.LayoutParams.MATCH_PARENT, keyWeight(label))); + } + panel.addView(row, new LinearLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, 0, 1f)); + } + + private static float keyWeight(String label) { + if (label.equals("Space")) { + return 4f; + } + if (label.equals("Back") || label.equals("Done") || label.equals("Esc") + || label.equals("Enter")) { + return 1.5f; + } + return 1f; + } + + // "Done" is this overlay's own UI navigation (closes the key grid, no + // game-visible effect). Esc/Back/Enter are non-printable, so they're + // forwarded as real Android key events via SDLActivity.onNativeKeyDown()/ + // onNativeKeyUp() - the same pair a physical Bluetooth keyboard's key + // events already drive (see SDLActivity.handleKeyEvent()). Every other + // key (letters, Space) instead goes through commitPrintableChar() below: + // the engine's own pump_events() (sdl_events.c) deliberately ignores + // SDL_KEYDOWN for printable ASCII and only reacts to SDL_TEXTINPUT for + // those, to avoid double-counting a typed character against a system + // IME - so that event has to be synthesized directly (see + // commitPrintableChar()). This all matters beyond just the visible + // typedTextView below: the engine's cutscene skip handler (cutsloop.c's + // cutscene_key_handler()) only reacts to Esc/Enter/Space, so those need + // to reach the game as actual game input, not just be echoed locally. + private void handleKeyPress(String label) { + if (label.equals("Done")) { + showPanel(mainPanel); + return; + } + + if (label.equals("Back")) { + SDLActivity.onNativeKeyDown(KeyEvent.KEYCODE_DEL); + SDLActivity.onNativeKeyUp(KeyEvent.KEYCODE_DEL); + if (typedText.length() > 0) { + typedText.setLength(typedText.length() - 1); + } + } else if (label.equals("Esc") || label.equals("Enter")) { + int keyCode = label.equals("Esc") ? KeyEvent.KEYCODE_ESCAPE : KeyEvent.KEYCODE_ENTER; + SDLActivity.onNativeKeyDown(keyCode); + SDLActivity.onNativeKeyUp(keyCode); + typedText.append('[').append(label).append(']'); + } else if (label.equals("Space")) { + commitPrintableChar(' '); + typedText.append(' '); + } else { + commitPrintableChar(Character.toLowerCase(label.charAt(0))); + typedText.append(label); + } + typedTextView.setText(typedText.toString()); + } + + // Synthesizes the SDL_TEXTINPUT event pump_events() (sdl_events.c) + // requires for printable characters (see handleKeyPress() above), by + // building and pushing it directly in native code (see + // nativeSendPrintableChar() in questshock_native.c) rather than through + // Android's IME (there's no real IME session behind this off-screen, + // never-attached grid for that plumbing to hook into). + private static void commitPrintableChar(char c) { + nativeSendPrintableChar(c); + } + + private static native void nativeSendPrintableChar(char c); + + // Only one of mainPanel/keyboardPanel is ever visible at a time. Both + // were already measured/laid out once in the constructor's initial + // showPanel() call - re-running measure()/layout() here (rather than + // just flipping visibility) is required every time regardless, since + // FrameLayout skips GONE children during measure/layout, so a panel + // switching from GONE to VISIBLE needs a fresh pass to get valid bounds + // before its content can be drawn or hit-tested. + private void showPanel(View panel) { + mainPanel.setVisibility(panel == mainPanel ? View.VISIBLE : View.GONE); + keyboardPanel.setVisibility(panel == keyboardPanel ? View.VISIBLE : View.GONE); int widthSpec = View.MeasureSpec.makeMeasureSpec(WIDTH, View.MeasureSpec.EXACTLY); int heightSpec = View.MeasureSpec.makeMeasureSpec(HEIGHT, View.MeasureSpec.EXACTLY); root.measure(widthSpec, heightSpec); root.layout(0, 0, WIDTH, HEIGHT); - - forceRedraw(); } // Called from xr_menu.c's xr_menu_init(), on the render thread, right @@ -140,7 +334,7 @@ public class MenuOverlay { cursorVisible = visible; cursorX = x; cursorY = y; - forceRedraw(); + compositeAndPublish(); } private void handleTouch(float u, float v, boolean down) { @@ -154,12 +348,23 @@ public class MenuOverlay { } finally { event.recycle(); } - forceRedraw(); + redrawContent(); } - private void forceRedraw() { - canvas.drawColor(0xFF202020); - root.draw(canvas); + // Re-runs the full off-screen View-tree draw (expensive - see the + // contentBitmap field comment above) - call only when the panel's + // actual content changed, not for the cursor-only updates + // compositeAndPublish() below handles on its own. + private void redrawContent() { + contentCanvas.drawColor(0xFF202020); + root.draw(contentCanvas); + compositeAndPublish(); + } + + // Cheap per-frame path: blits the last-rendered contentBitmap (no View + // traversal) plus the cursor circle into bitmap and publishes it. + private void compositeAndPublish() { + canvas.drawBitmap(contentBitmap, 0, 0, null); if (cursorVisible) { canvas.drawCircle(cursorX, cursorY, CURSOR_RADIUS, cursorPaint); }