rework interactions

README.md

@@ -34,13 +34,21 @@ cargo run --release
 
 You'll see a screen and keyboard. You can turn these on and off using the watch on your left wrist.
 
-Right click: Touch (do not push down) the B/Y button on your controller to get a YELLOW laser, and pull the trigger.
+Right click: Touch (do not push down) the B/Y button on your controller to get a ORANGE laser, and pull the trigger.
 
 Middle click: Same as right click, but A/X to get a PURPLE laser.
 
 Move screen: Point your laser on the screen and then grab using grip. Adjust distance using stick up/down while gripping.
 
-Resize screen: While grabbing, touch B/Y to get a YELLOW laser and use stick up/down (this is wonky, I'll come up with a better one!)
+Resize screen: While grabbing, pull trigger to get a RED laser and use stick up/down
+
+Reset size/position: Click the button corresponding to the screen or keyboard on your watch, hold for 3s, then release.
+
+Show/hide: Quickly hide or show your selection of screens by double-tapping B/Y on your left controller.
+
+Lock a screen in place: On your non-watch hand, touch B/Y to get an ORANGE laser and click the screen's button on your watch. You will no longer be able to grab the screen, it will not re-center in front of you when show, nor it will react to the show/hide shortcut.
+
+Make a screen non-clickable: On your non-watch hand, touch A/X to get a PURPLE laser and click the screen's button on your watch. You will no longer get a laser when pointing to that screen. Repeat to toggle back off.
 
 # Known Issues
 

src/backend/common.rs

@@ -11,6 +11,7 @@ use crate::{
     overlays::{
         keyboard::create_keyboard,
         screen::{get_screens_wayland, get_screens_x11},
+        toast::Toast,
         watch::create_watch,
     },
     state::AppState,
@@ -106,7 +107,7 @@ where
         self.overlays.values_mut().for_each(|o| {
             if o.state.show_hide {
                 o.state.want_visible = !any_shown;
-                if o.state.want_visible && app.session.recenter_on_show {
+                if o.state.want_visible && o.state.recenter {
                     o.state.reset(app, false);
                 }
             }
@@ -114,6 +115,7 @@ where
     }
 }
 
+#[derive(Clone)]
 pub enum OverlaySelector {
     Id(usize),
     Name(Arc<str>),
@@ -147,6 +149,7 @@ pub enum TaskType {
         OverlaySelector,
         Box<dyn FnOnce(&mut AppState, &mut OverlayState) + Send>,
     ),
+    Toast(Toast),
 }
 
 pub struct TaskContainer {

src/backend/input.rs

@@ -239,6 +239,7 @@ pub enum PointerMode {
     Left,
     Right,
     Middle,
+    Special,
 }
 
 pub fn interact<O>(
@@ -371,7 +372,7 @@ impl Pointer {
         let mut hits: SmallVec<[RayHit; 8]> = smallvec!();
 
         for overlay in overlays.iter() {
-            if !overlay.state.want_visible {
+            if !overlay.state.want_visible || !overlay.state.interactable {
                 continue;
             }
 
@@ -435,30 +436,32 @@ impl Pointer {
         O: Default,
     {
         if self.now.grab {
-            if self.now.click && !self.before.click {
-                log::warn!("todo: click-while-grabbed");
-            }
-
             let grab_data = self.interaction.grabbed.as_mut().unwrap();
-            match self.interaction.mode {
+            if self.now.click {
-                PointerMode::Left => {
+                self.interaction.mode = PointerMode::Special;
-                    grab_data.offset.z -= self.now.scroll * 0.05;
+                let cur_scale = overlay.state.transform.x_axis.length();
-                }
+                if cur_scale < 0.1 && self.now.scroll > 0.0 {
-                _ => {
+                    return;
-                    overlay.state.transform.matrix3 = overlay
+                } else if cur_scale > 20. && self.now.scroll < 0.0 {
-                        .state
+                    return;
-                        .transform
-                        .matrix3
-                        .mul_scalar(1.0 + 0.01 * self.now.scroll);
                 }
+
+                overlay.state.transform.matrix3 = overlay
+                    .state
+                    .transform
+                    .matrix3
+                    .mul_scalar(1.0 - 0.025 * self.now.scroll);
+            } else {
+                grab_data.offset.z -= self.now.scroll * 0.05;
             }
             overlay.state.transform.translation = self.pose.transform_point3a(grab_data.offset);
             overlay.state.realign(hmd);
             overlay.state.dirty = true;
         } else {
-            overlay.state.spawn_point = hmd
+            overlay.state.saved_point = Some(
-                .inverse()
+                hmd.inverse()
-                .transform_point3a(overlay.state.transform.translation);
+                    .transform_point3a(overlay.state.transform.translation),
+            );
             self.interaction.grabbed = None;
             log::info!("Hand {}: dropped {}", self.idx, overlay.state.name);
         }

src/backend/openvr/lines.rs

@@ -21,7 +21,7 @@ static AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(1);
 pub(super) struct LinePool {
     lines: IdMap<usize, OverlayData<OpenVrOverlayData>>,
     view: Arc<ImageView>,
-    colors: [Vec4; 4],
+    colors: [Vec4; 5],
 }
 
 impl LinePool {
@@ -52,6 +52,7 @@ impl LinePool {
                 Vec4::new(0., 0.375, 0.5, 1.),
                 Vec4::new(0.69, 0.188, 0., 1.),
                 Vec4::new(0.375, 0., 0.5, 1.),
+                Vec4::new(1., 0., 0., 1.),
             ],
         }
     }

src/backend/openvr/mod.rs

@@ -140,6 +140,10 @@ pub fn openvr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
                         f(&mut state, &mut o.state);
                     }
                 }
+                TaskType::Toast(t) => {
+                    // TODO toasts
+                    log::info!("Toast: {} {}", t.title, t.body);
+                }
             }
         }
 

src/backend/openxr/lines.rs

@@ -36,6 +36,7 @@ impl LinePool {
             [0x00, 0x60, 0x80, 0xff],
             [0xB0, 0x30, 0x00, 0xff],
             [0x60, 0x00, 0x80, 0xff],
+            [0xff, 0x00, 0x00, 0xff],
         ];
 
         let views = colors

src/backend/openxr/mod.rs

@@ -186,6 +186,10 @@ pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
                         f(&mut app_state, &mut o.state);
                     }
                 }
+                TaskType::Toast(t) => {
+                    // TODO toasts
+                    log::info!("Toast: {} {}", t.title, t.body);
+                }
             }
         }
 

src/backend/overlay.rs

@@ -24,8 +24,11 @@ pub struct OverlayState {
     pub want_visible: bool,
     pub show_hide: bool,
     pub grabbable: bool,
+    pub interactable: bool,
+    pub recenter: bool,
     pub dirty: bool,
     pub transform: Affine3A,
+    pub saved_point: Option<Vec3A>,
     pub spawn_scale: f32, // aka width
     pub spawn_point: Vec3A,
     pub spawn_rotation: Quat,
@@ -43,8 +46,11 @@ impl Default for OverlayState {
             want_visible: false,
             show_hide: false,
             grabbable: false,
+            recenter: false,
+            interactable: false,
             dirty: true,
             relative_to: RelativeTo::None,
+            saved_point: None,
             spawn_scale: 1.0,
             spawn_point: Vec3A::NEG_Z,
             spawn_rotation: Quat::IDENTITY,
@@ -90,26 +96,31 @@ impl OverlayState {
 
     pub fn auto_movement(&mut self, app: &mut AppState) {
         if let Some(parent) = self.parent_transform(app) {
+            let point = self.saved_point.unwrap_or(self.spawn_point);
             self.transform = parent
                 * Affine3A::from_scale_rotation_translation(
                     Vec3::ONE * self.spawn_scale,
                     self.spawn_rotation
                         * Quat::from_rotation_x(f32::to_radians(-180.0))
                         * Quat::from_rotation_z(f32::to_radians(180.0)),
-                    self.spawn_point.into(),
+                    point.into(),
                 );
 
             self.dirty = true;
         }
     }
 
-    pub fn reset(&mut self, app: &mut AppState, reset_scale: bool) {
+    pub fn reset(&mut self, app: &mut AppState, hard_reset: bool) {
-        let translation = app.input_state.hmd.transform_point3a(self.spawn_point);
+        let scale = if hard_reset {
-        let scale = if reset_scale {
+            self.saved_point = None;
             self.spawn_scale
         } else {
             self.transform.x_axis.length()
         };
+
+        let point = self.saved_point.unwrap_or(self.spawn_point);
+
+        let translation = app.input_state.hmd.transform_point3a(point);
         self.transform = Affine3A::from_scale_rotation_translation(
             Vec3::ONE * scale,
             Quat::IDENTITY,

src/overlays/keyboard.rs

@@ -3,7 +3,6 @@ use std::{
     io::Cursor,
     process::{Child, Command},
     str::FromStr,
-    sync::Arc,
 };
 
 use crate::{
@@ -26,6 +25,8 @@ const PIXELS_PER_UNIT: f32 = 80.;
 const BUTTON_PADDING: f32 = 4.;
 const AUTO_RELEASE_MODS: [KeyModifier; 5] = [SHIFT, CTRL, ALT, SUPER, META];
 
+pub const KEYBOARD_NAME: &str = "kbd";
+
 pub fn create_keyboard<O>(app: &AppState) -> OverlayData<O>
 where
     O: Default,
@@ -116,9 +117,11 @@ where
 
     OverlayData {
         state: OverlayState {
-            name: Arc::from("kbd"),
+            name: KEYBOARD_NAME.into(),
             size: (size.x as _, size.y as _),
             grabbable: true,
+            recenter: true,
+            interactable: true,
             spawn_scale: width,
             spawn_point: vec3a(0., -0.5, -1.),
             interaction_transform,

src/overlays/mod.rs

@@ -1,3 +1,4 @@
 pub mod keyboard;
 pub mod screen;
+pub mod toast;
 pub mod watch;

src/overlays/screen.rs

@@ -378,6 +378,8 @@ where
                 size,
                 want_visible: session.show_screens.iter().any(|s| s == &*output.name),
                 grabbable: true,
+                recenter: true,
+                interactable: true,
                 spawn_scale: 1.5 * session.config.desktop_view_scale,
                 spawn_point: vec3a(0., 0.5, -1.),
                 spawn_rotation: Quat::from_axis_angle(axis, angle),

src/overlays/toast.rs

@@ -0,0 +1,34 @@
+use std::sync::Arc;
+
+pub struct Toast {
+    pub title: Arc<str>,
+    pub body: Arc<str>,
+    pub opacity: f32,
+    pub timeout: f32,
+    pub sound: bool,
+}
+
+#[allow(dead_code)]
+impl Toast {
+    pub fn new(title: Arc<str>, body: Arc<str>) -> Self {
+        Toast {
+            title,
+            body,
+            opacity: 1.0,
+            timeout: 3.0,
+            sound: false,
+        }
+    }
+    pub fn with_timeout(mut self, timeout: f32) -> Self {
+        self.timeout = timeout;
+        self
+    }
+    pub fn with_opacity(mut self, opacity: f32) -> Self {
+        self.opacity = opacity;
+        self
+    }
+    pub fn with_sound(mut self) -> Self {
+        self.sound = true;
+        self
+    }
+}

src/overlays/watch.rs

@@ -6,12 +6,15 @@ use glam::{vec2, Affine2};
 use crate::{
     backend::{
         common::{OverlaySelector, TaskType},
+        input::PointerMode,
         overlay::{OverlayData, OverlayState, RelativeTo},
     },
-    gui::{color_parse, CanvasBuilder},
+    gui::{color_parse, CanvasBuilder, Control},
     state::AppState,
 };
 
+use super::{keyboard::KEYBOARD_NAME, toast::Toast};
+
 pub const WATCH_NAME: &str = "watch";
 
 pub fn create_watch<O>(state: &AppState, screens: &[OverlayData<O>]) -> OverlayData<O>
@@ -79,44 +82,21 @@ where
     let button_width = 360. / num_buttons as f32;
     let mut button_x = 40.;
 
-    let keyboard = canvas.button(button_x + 2., 162., button_width - 4., 36., "kbd".into());
+    let keyboard = canvas.button(
+        button_x + 2.,
+        162.,
+        button_width - 4.,
+        36.,
+        KEYBOARD_NAME.into(),
+    );
     keyboard.state = Some(WatchButtonState {
         pressed_at: Instant::now(),
-        scr_idx: 0,
+        overlay: OverlaySelector::Name(KEYBOARD_NAME.into()),
+        mode: PointerMode::Left,
     });
 
-    keyboard.on_press = Some(|control, _data, _app, _| {
+    keyboard.on_press = Some(overlay_button_dn);
-        if let Some(state) = control.state.as_mut() {
+    keyboard.on_release = Some(overlay_button_up);
-            state.pressed_at = Instant::now();
-        }
-    });
-    keyboard.on_release = Some(|control, _data, app| {
-        if let Some(state) = control.state.as_ref() {
-            if Instant::now()
-                .saturating_duration_since(state.pressed_at)
-                .as_millis()
-                < 2000
-            {
-                app.tasks.enqueue(TaskType::Overlay(
-                    OverlaySelector::Name("kbd".into()),
-                    Box::new(|app, o| {
-                        o.show_hide = !o.show_hide;
-                        o.want_visible = o.show_hide;
-                        if app.session.recenter_on_show {
-                            o.reset(app, false);
-                        }
-                    }),
-                ));
-            } else {
-                app.tasks.enqueue(TaskType::Overlay(
-                    OverlaySelector::Name("kbd".into()),
-                    Box::new(|app, o| {
-                        o.reset(app, true);
-                    }),
-                ));
-            }
-        }
-    });
     button_x += button_width;
 
     canvas.bg_color = color_parse("#405060");
@@ -131,42 +111,12 @@ where
         );
         button.state = Some(WatchButtonState {
             pressed_at: Instant::now(),
-            scr_idx: screen.state.id,
+            overlay: OverlaySelector::Id(screen.state.id),
+            mode: PointerMode::Left,
         });
 
-        button.on_press = Some(|control, _data, _app, _| {
+        button.on_press = Some(overlay_button_dn);
-            if let Some(state) = control.state.as_mut() {
+        button.on_release = Some(overlay_button_up);
-                state.pressed_at = Instant::now();
-            }
-        });
-        button.on_release = Some(|control, _data, app| {
-            if let Some(state) = control.state.as_ref() {
-                let scr_idx = state.scr_idx;
-                if Instant::now()
-                    .saturating_duration_since(state.pressed_at)
-                    .as_millis()
-                    < 2000
-                {
-                    app.tasks.enqueue(TaskType::Overlay(
-                        OverlaySelector::Id(scr_idx),
-                        Box::new(|app, o| {
-                            o.show_hide = !o.show_hide;
-                            o.want_visible = o.show_hide;
-                            if app.session.recenter_on_show {
-                                o.reset(app, false);
-                            }
-                        }),
-                    ));
-                } else {
-                    app.tasks.enqueue(TaskType::Overlay(
-                        OverlaySelector::Id(scr_idx),
-                        Box::new(|app, o| {
-                            o.reset(app, true);
-                        }),
-                    ));
-                }
-            }
-        });
         button_x += button_width;
     }
     let interaction_transform =
@@ -179,6 +129,7 @@ where
             name: WATCH_NAME.into(),
             size: (400, 200),
             want_visible: true,
+            interactable: true,
             spawn_scale: 0.11 * state.session.config.watch_scale,
             spawn_point: state.session.watch_pos.into(),
             spawn_rotation: state.session.watch_rot,
@@ -193,5 +144,82 @@ where
 
 struct WatchButtonState {
     pressed_at: Instant,
-    scr_idx: usize,
+    mode: PointerMode,
+    overlay: OverlaySelector,
+}
+
+fn overlay_button_dn(
+    control: &mut Control<(), WatchButtonState>,
+    _: &mut (),
+    _: &mut AppState,
+    mode: PointerMode,
+) {
+    if let Some(state) = control.state.as_mut() {
+        state.pressed_at = Instant::now();
+        state.mode = mode;
+    }
+}
+
+fn overlay_button_up(control: &mut Control<(), WatchButtonState>, _: &mut (), app: &mut AppState) {
+    if let Some(state) = control.state.as_ref() {
+        let selector = state.overlay.clone();
+        if Instant::now()
+            .saturating_duration_since(state.pressed_at)
+            .as_millis()
+            < 2000
+        {
+            match state.mode {
+                PointerMode::Left => {
+                    app.tasks.enqueue(TaskType::Overlay(
+                        selector,
+                        Box::new(|app, o| {
+                            o.want_visible = !o.want_visible;
+                            if o.recenter {
+                                o.show_hide = o.want_visible;
+                                o.reset(app, false);
+                            }
+                        }),
+                    ));
+                }
+                PointerMode::Right => {
+                    app.tasks.enqueue(TaskType::Overlay(
+                        selector,
+                        Box::new(|app, o| {
+                            o.recenter = !o.recenter;
+                            o.grabbable = o.recenter;
+                            o.show_hide = o.recenter;
+                            if !o.recenter {
+                                app.tasks.enqueue(TaskType::Toast(Toast::new(
+                                    format!("{} is now locked in place!", o.name).into(),
+                                    "Right-click again to toggle.".into(),
+                                )))
+                            }
+                        }),
+                    ));
+                }
+                PointerMode::Middle => {
+                    app.tasks.enqueue(TaskType::Overlay(
+                        selector,
+                        Box::new(|app, o| {
+                            o.interactable = !o.interactable;
+                            if !o.interactable {
+                                app.tasks.enqueue(TaskType::Toast(Toast::new(
+                                    format!("{} is now non-interactable!", o.name).into(),
+                                    "Middle-click again to toggle.".into(),
+                                )))
+                            }
+                        }),
+                    ));
+                }
+                _ => {}
+            }
+        } else {
+            app.tasks.enqueue(TaskType::Overlay(
+                selector,
+                Box::new(|app, o| {
+                    o.reset(app, true);
+                }),
+            ));
+        }
+    }
 }

src/res/config.yaml

@@ -3,8 +3,6 @@
 # Default: 300
 click_freeze_time_ms: 300
 
-recenter_on_show: true
-
 # Default: true
 keyboard_sound_enabled: true
 

src/state.rs

@@ -77,7 +77,6 @@ pub struct AppSession {
     pub config: GeneralConfig,
 
     pub show_screens: Vec<String>,
-    pub recenter_on_show: bool,
 
     pub watch_hand: usize,
     pub watch_pos: Vec3,
@@ -103,7 +102,6 @@ impl AppSession {
             config_root_path,
             config,
             show_screens: vec!["DP-3".to_string()],
-            recenter_on_show: true,
             capture_method: "auto".to_string(),
             primary_hand: 1,
             watch_hand: 0,