not gonna comback to this

Author: Exidex

rust/client/build.rs

@@ -13,7 +13,7 @@ use gauntlet_component_model::PropertyType;
 use gauntlet_component_model::create_component_model;
 
 fn main() -> anyhow::Result<()> {
-    #[cfg(target_os = "macos")] // needed for window focus stuff, specifically SkyLight framework
+    #[cfg(target_os = "macos")]
     println!(
         "cargo:rustc-link-search=framework={}",
         "/System/Library/PrivateFrameworks"

rust/client/src/ui/window_tracking/macos/apps.rs

@@ -7,7 +7,12 @@ use accessibility::AXUIElement;
 use accessibility_sys::kAXRaiseAction;
 use accessibility_sys::pid_t;
 use anyhow::Context;
+use core_foundation::base::CFType;
+use core_foundation::base::FromVoid;
+use core_foundation::dictionary::CFDictionary;
 use core_foundation::string::CFString;
+use core_graphics::display::CGDisplay;
+use core_graphics::window::kCGWindowListExcludeDesktopElements;
 use gauntlet_server::plugins::ApplicationManager;
 use objc2::AnyThread;
 use objc2::DefinedClass;
@@ -24,7 +29,8 @@ use objc2_app_kit::NSWorkspaceDidLaunchApplicationNotification;
 use objc2_app_kit::NSWorkspaceDidTerminateApplicationNotification;
 use objc2_foundation::NSNotification;
 
-use super::window::WindowNotificationDelegate;
+use super::window::{WindowNotificationDelegate, WindowType};
+use crate::ui::window_tracking::macos::sys::ax_window_id;
 use crate::ui::window_tracking::macos::sys::make_key_window;
 
 pub struct MacosWindowTracker {
@@ -84,7 +90,6 @@ impl ApplicationNotificationDelegate {
         let state = ApplicationNotificationDelegateState {
             application_manager,
             applications: RefCell::new(HashMap::new()),
-            windows: Rc::new(RefCell::new(vec![])),
         };
 
         let delegate = ApplicationNotificationDelegate::alloc().set_ivars(state);
@@ -117,9 +122,8 @@ impl ApplicationNotificationDelegate {
 
     fn create_window_notification_delegate(&self, pid: pid_t) -> anyhow::Result<()> {
         let application_manager = self.ivars().application_manager.clone();
-        let windows = self.ivars().windows.clone();
 
-        let delegate = WindowNotificationDelegate::new(pid, application_manager, windows)
+        let delegate = WindowNotificationDelegate::new(pid, application_manager)
             .context("Error creating window notification delegate")?;
 
         delegate
@@ -153,7 +157,16 @@ impl ApplicationNotificationDelegate {
 
         println!("Focusing window: {}, {:?}, {}", window_uuid, window, pid);
 
-        make_key_window(*pid, window).context("Failed to make window key")?;
+        if let Some(windows) = CGDisplay::window_list_info(kCGWindowListExcludeDesktopElements, None) {
+            for item in windows.into_iter() {
+                let item: CFDictionary<CFString, CFType> = unsafe { CFDictionary::from_void(item.clone()) }.clone();
+                println!("CFDictionary: {:?}", item);
+            }
+        };
+
+        let window_id = ax_window_id(window).context("Failed to get window id")?;
+
+        make_key_window(*pid, window_id).context("Failed to make window key")?;
 
         // some apps seem to also require additional raise action
         window
@@ -167,7 +180,6 @@ impl ApplicationNotificationDelegate {
 struct ApplicationNotificationDelegateState {
     application_manager: Arc<ApplicationManager>,
     applications: RefCell<HashMap<pid_t, WindowNotificationDelegate>>,
-    windows: Rc<RefCell<Vec<(String, pid_t, AXUIElement)>>>,
 }
 
 define_class!(

rust/client/src/ui/window_tracking/macos/mod.rs

@@ -21,35 +21,29 @@ pub fn request_macos_accessibility_permissions() -> bool {
     unsafe { AXIsProcessTrustedWithOptions(options.as_concrete_TypeRef()) }
 }
 
-// todo
-//  on each ax notification + non-ax destroy event
-//  get app pid using AXUIElementGetPid
-//  run bruteforce search (+ regular) for axuielement using _AXUIElementCreateWithRemoteToken
-//  from each found window axuielement
-//    filter based on window role/subrole
-//    get window using _AXUIElementGetWindow
-//    get title
-//  to focus use window id and private api
+// only active space(s) and fullscreen apps (only single item per app will be shown) are supported
+// tabs are supported only on active "desktop" space, not supported in fullscreen window
+// show warning when:
+//  there are non-active spaces
+//    except fullscreen windows
+//  there multiple fullscreen windows for specific app
 
+// warning should say:
+// gauntlet doesn't show windows on non-active spaces except fullscreen applications
+// gauntlet doesn't support fullscreen applications on multiple spaces
+// gauntlet doesn't support native tabs for fullscreen applications
 
-// do not support hidden windows
-// do not support multiple "desktop" spaces unless they are all visible
-// what about multiple fullscreen windows of the same app?
-// what about tabs in visible apps in non-visible spaces?
-// i.e., only non-hidden windows in visible spaces and maybe(?) fullscreen apps
-// support minimized windows but only on visible spaces
+// refresh window list when space switches
+// if current space is fullscreen do not scan for tabs? show warning?
 
-// support fullscreen applications?
-// support tabs on the visible windows in visible spaces only
+// CGSSpaceGetType to get type of given space
+// CGSGetWindowWorkspace to get list of spaces for specific window
+// ? to get space for given
+// ? to get current space
 
-// I think ignoring existence of spaces is fine???
+//  todo what if there are 2 monitors. is it same space or multiple? what does "separate spaces" setting do?
+//  todo what if gauntlet started on fullscreen space?
 
-// is the private function for focusing a window needed?
-
-// todo support for windows on separate spaces
-// todo multiple desktop spaces ("Desktop" vs. "Desktop 1" and "Desktop 2")
-// todo sometimes the window state seems to be lost, clearing the list of windows
-// todo when starting the gauntlet, tabbed windows only show single one
 // todo implement this https://github.com/glide-wm/glide/issues/10
 // todo how to handle system apps and settings wrt window tracking?
 // todo add all github issue links and appreciations to the commit message
@@ -63,5 +57,6 @@ pub fn request_macos_accessibility_permissions() -> bool {
 //   https://github.com/koekeishiya/yabai/issues/68
 //   https://github.com/koekeishiya/yabai/issues/199#issuecomment-519152388
 //   https://github.com/lwouis/alt-tab-macos/issues/1324#issuecomment-2631035482
+//   https://github.com/glide-wm/glide/issues/10
 
 

rust/client/src/ui/window_tracking/macos/sys.rs

@@ -57,12 +57,10 @@ impl ProcessSerialNumber {
     }
 }
 
-pub fn make_key_window(pid: pid_t, window: &AXUIElement) -> anyhow::Result<()> {
+pub fn make_key_window(pid: pid_t, window_id: CGWindowID) -> anyhow::Result<()> {
     // See https://github.com/Hammerspoon/hammerspoon/issues/370#issuecomment-545545468.
     // god bless all the wizards in that thread that worked on it, thank you
 
-    let window_id = ax_window_id(window)?;
-
     println!("window id: {}", window_id);
 
     #[allow(non_upper_case_globals)]
@@ -95,7 +93,9 @@ pub fn make_key_window(pid: pid_t, window: &AXUIElement) -> anyhow::Result<()> {
     Ok(())
 }
 
+#[allow(unused)]
 pub fn bruteforce_windows_for_app(app_pid: pid_t) -> Vec<AXUIElement> {
+    // this whole thing can take more than a second, do not run on the main thread
     unsafe {
         let mut result = vec![];
         let mut data = [0; 0x14];

rust/client/src/ui/window_tracking/macos/window.rs

@@ -1,9 +1,11 @@
 use std::cell::RefCell;
+use std::collections::HashMap;
 use std::ffi::c_void;
 use std::mem::MaybeUninit;
 use std::rc::Rc;
 use std::sync::Arc;
 
+use accessibility::AXAttribute;
 use accessibility::AXUIElement;
 use accessibility::AXUIElementAttributes;
 use accessibility::Error;
@@ -15,12 +17,17 @@ use accessibility_sys::AXUIElementRef;
 use accessibility_sys::kAXDialogSubrole;
 use accessibility_sys::kAXErrorSuccess;
 use accessibility_sys::kAXStandardWindowSubrole;
+use accessibility_sys::kAXTabGroupRole;
+use accessibility_sys::kAXTabsAttribute;
 use accessibility_sys::kAXTitleChangedNotification;
 use accessibility_sys::kAXUIElementDestroyedNotification;
 use accessibility_sys::kAXWindowCreatedNotification;
 use accessibility_sys::kAXWindowRole;
 use accessibility_sys::pid_t;
 use anyhow::Context;
+use core_foundation::array::CFArray;
+use core_foundation::base::CFType;
+use core_foundation::base::FromVoid;
 use core_foundation::base::TCFType;
 use core_foundation::runloop::CFRunLoop;
 use core_foundation::runloop::kCFRunLoopDefaultMode;
@@ -32,17 +39,28 @@ use objc2_app_kit::NSRunningApplication;
 use uuid::Uuid;
 
 use super::sys::AXObserver;
-use super::sys::bruteforce_windows_for_app;
+use super::sys::ax_window_id;
 
 pub struct WindowNotificationDelegate {
     app_element: AXUIElement,
     observer: AXObserver,
     inner: Rc<WindowNotificationDelegateInner>,
 }
 
+pub enum WindowType {
+    Window,
+    Tab,
+}
+
+struct WindowData {
+    app_pid: pid_t,
+    window_type: WindowType,
+    element: AXUIElement,
+}
+
 struct WindowNotificationDelegateInner {
     app_pid: pid_t,
-    windows: Rc<RefCell<Vec<(String, pid_t, AXUIElement)>>>,
+    windows: Rc<RefCell<HashMap<String, WindowData>>>,
     application_manager: Arc<ApplicationManager>,
 }
 
@@ -52,14 +70,8 @@ const WINDOW_EVENTS: [&str; 3] = [
     kAXTitleChangedNotification,
 ];
 
-const MESSAGING_TIMEOUT_SEC: f32 = 1.0;
-
 impl WindowNotificationDelegate {
-    pub fn new(
-        pid: pid_t,
-        application_manager: Arc<ApplicationManager>,
-        windows: Rc<RefCell<Vec<(String, pid_t, AXUIElement)>>>,
-    ) -> anyhow::Result<Self> {
+    pub fn new(pid: pid_t, application_manager: Arc<ApplicationManager>) -> anyhow::Result<Self> {
         let observer = unsafe {
             let mut result = MaybeUninit::uninit();
 
@@ -76,7 +88,7 @@ impl WindowNotificationDelegate {
             let element = AXUIElement::application(pid);
 
             element
-                .set_messaging_timeout(MESSAGING_TIMEOUT_SEC)
+                .set_messaging_timeout(1.0)
                 .context("Failed to set messaging timeout")?;
 
             element
@@ -87,7 +99,7 @@ impl WindowNotificationDelegate {
             observer,
             inner: Rc::new(WindowNotificationDelegateInner {
                 app_pid: pid,
-                windows,
+                windows: Rc::new(RefCell::new(HashMap::new())),
                 application_manager,
             }),
         })
@@ -142,7 +154,7 @@ impl WindowNotificationDelegate {
 
         let windows = self.inner.windows.borrow();
 
-        for (window_id, _, _) in windows.iter() {
+        for (window_id, _) in windows.iter() {
             let event = MacosWindowTrackingEvent::WindowClosed {
                 window_id: window_id.clone(),
             };
@@ -205,40 +217,51 @@ fn get_bundle_path(pid: pid_t) -> Option<String> {
 impl WindowNotificationDelegateInner {
     fn refresh_windows(&self) -> anyhow::Result<()> {
         tracing::debug!("Refreshing windows for app: {}", self.app_pid);
+
         let mut retrieved_windows: Vec<_> = AXUIElement::application(self.app_pid)
             .windows()?
             .into_iter()
-            .map(|item| item.clone())
-            .collect();
+            .map(|window| window.clone())
+            .flat_map(|window| {
+                let tabs = list_tabs(window.clone());
 
-        for window in bruteforce_windows_for_app(self.app_pid) {
-            if !retrieved_windows.contains(&window) {
-                retrieved_windows.push(window);
-            };
-        }
+                if !tabs.is_empty() {
+                    return tabs.into_iter().map(|tab| (WindowType::Tab, tab)).collect::<Vec<_>>();
+                }
+
+                return vec![(WindowType::Window, window)];
+            })
+            .collect();
 
-        tracing::debug!("Retrieved {} windows", retrieved_windows.len());
+        tracing::debug!("Retrieved windows: {}", retrieved_windows.len());
 
         let stored_windows = self
             .windows
             .borrow()
             .iter()
-            .map(|(_, _, window)| window.clone())
+            .map(|(_, window)| (window.app_pid.clone(), window.clone()))
             .collect::<Vec<_>>();
 
-        tracing::debug!("Stored {} windows", stored_windows.len());
+        tracing::debug!("Stored windows: {}", stored_windows.len());
+
+        let mut destroyed_windows = 0;
+        for (pid, window) in stored_windows.iter() {
+            if pid != &self.app_pid {
+                continue;
+            }
 
-        for window in stored_windows.into_iter() {
-            let Some(index) = retrieved_windows.iter().position(|el| el == &window) else {
+            let Some(index) = retrieved_windows.iter().position(|el| el == window) else {
                 // doesn't exist anymore, destroy it
-                self.window_destroyed(window);
+                self.window_destroyed(window.clone());
                 continue;
             };
 
+            destroyed_windows += 1;
             retrieved_windows.swap_remove(index);
         }
 
-        tracing::debug!("left retrieved {} windows", retrieved_windows.len());
+        tracing::debug!("Destroyed windows: {}", retrieved_windows.len());
+        tracing::debug!("New windows: {}", retrieved_windows.len());
 
         for window in retrieved_windows.into_iter() {
             self.window_opened(window);
@@ -264,7 +287,7 @@ impl WindowNotificationDelegateInner {
         }
 
         let window_id = Uuid::new_v4().to_string();
-        windows.push((window_id.clone(), self.app_pid, window.clone()));
+        windows.insert(window_id.clone(), (self.app_pid, window.clone()));
 
         let title = window.title().map(|title| title.to_string()).ok();
         let bundle_path = get_bundle_path(self.app_pid);
@@ -285,7 +308,7 @@ impl WindowNotificationDelegateInner {
             return;
         };
 
-        let (window_id, _, _) = windows.swap_remove(index);
+        let (window_id, _, _) = windows.remove(index);
 
         let event = MacosWindowTrackingEvent::WindowClosed { window_id };
 
@@ -309,3 +332,40 @@ impl WindowNotificationDelegateInner {
         self.application_manager.send_macos_window_tracking_event(event);
     }
 }
+
+fn list_tabs(window: AXUIElement) -> Vec<AXUIElement> {
+    let Ok(children) = window.children() else {
+        return vec![];
+    };
+
+    let tab_group = children.into_iter().find(|child| {
+        let role = child.role().map(|val| val.to_string()).ok();
+        if role.as_deref() != Some(kAXTabGroupRole) {
+            return false;
+        }
+
+        let title = child.title().map(|val| val.to_string()).ok();
+        if title.as_deref() != Some("tab bar") {
+            return false;
+        }
+
+        return true;
+    });
+
+    let Some(tab_group) = tab_group else {
+        return vec![];
+    };
+
+    let tabs_attribute = AXAttribute::<CFType>::new(&CFString::from_static_string(kAXTabsAttribute));
+    let Some(tabs) = tab_group.attribute(&tabs_attribute).ok() else {
+        return vec![];
+    };
+
+    let Some(tabs) = tabs.downcast::<CFArray>() else {
+        return vec![];
+    };
+
+    tabs.into_iter()
+        .map(|item| unsafe { AXUIElement::from_void(item.clone()) }.clone())
+        .collect()
+}