Popups inside a XAML island do not respect DesktopWindowXamlSource.OverrideScale and can be clipped

[mslukebo]

Describe the bug

If a popup (e.g. a tool tip of flyout) opens inside a XAML island with a custom OverrideScale, the popup does not respect the OverrideScale value. Instead, the content renders with the scale of the containing window.

If the OverrideScale is less than the scale of the containing window, this causes the content to be clipped. because it appears the popup container does render at the correct scale.

Why is this important?

Anyone using XAML islands that can expect their content to have popups cannot reliably use the OverrideScale feature if their scale could ever be less than the containing window.

Steps to reproduce the bug

Place a XAML island on a window and set its OverrideScale to a value lower than the window's scale. Have a popup appear on this XAML island, e.g. by adding a context flyout.

Here is the code that produced the attached screenshots:

public class Win32XamlWindow
{
    private IntPtr _hwnd;
    private DesktopWindowXamlSource? _xamlSource;

    public Win32XamlWindow()
    {
        RegisterWindowClass();
        CreateNativeWindow();
        InitializeXamlIsland();
    }

    private void RegisterWindowClass()
    {
        var wc = new WNDCLASSEX
        {
            cbSize = (uint)Marshal.SizeOf<WNDCLASSEX>(),
            style = 0,
            lpfnWndProc = Marshal.GetFunctionPointerForDelegate(_wndProcDelegate),
            cbClsExtra = 0,
            cbWndExtra = 0,
            hInstance = GetModuleHandle(null),
            hIcon = IntPtr.Zero,
            hCursor = LoadCursor(IntPtr.Zero, IDC_ARROW),
            hbrBackground = COLOR_WINDOW + 1,
            lpszMenuName = null,
            lpszClassName = "Win32XamlHostWindow",
            hIconSm = IntPtr.Zero
        };

        if (RegisterClassEx(ref wc) == 0)
        {
            throw new InvalidOperationException("Failed to register window class");
        }
    }

    private void CreateNativeWindow()
    {
        _hwnd = CreateWindowEx(
            0,
            "Win32XamlHostWindow",
            "WinUI3 XAML Island Window",
            WS_OVERLAPPEDWINDOW,
            CW_USEDEFAULT,
            CW_USEDEFAULT,
            1000,
            800,
            IntPtr.Zero,
            IntPtr.Zero,
            GetModuleHandle(null),
            IntPtr.Zero
        );

        if (_hwnd == IntPtr.Zero)
        {
            throw new InvalidOperationException("Failed to create window");
        }

        SetWindowLongPtr(_hwnd, GWLP_USERDATA, GCHandle.ToIntPtr(GCHandle.Alloc(this)));
    }

    private void InitializeXamlIsland()
    {
        // Create the DesktopWindowXamlSource
        _xamlSource = new DesktopWindowXamlSource();
        var hostWindowId = Win32Interop.GetWindowIdFromWindow(_hwnd);
        _xamlSource.Initialize(hostWindowId);

        RECT rect;
        GetClientRect(_hwnd, out rect);

        _xamlSource.SiteBridge.MoveAndResize(new RectInt32(rect.left, rect.top, rect.right - rect.left,
            rect.bottom - rect.top));

        // Get DPI and set OverrideScale
        uint windowDpi = GetDpiForWindow(_hwnd);
        float windowScale = windowDpi / 96.0f;
        float xamlIslandScale = windowScale / 2.0f;

        _xamlSource.SiteBridge.OverrideScale = xamlIslandScale;

        var foregroundColor = new SolidColorBrush(Colors.Black);
        var fontSize = 36;

        var stackPanel = new StackPanel
        {
            VerticalAlignment = VerticalAlignment.Center,
            HorizontalAlignment = HorizontalAlignment.Center,
            Spacing = 40,
        };

        var windowScaleTextBlock = new TextBlock
        {
            Text = $"Window Scale: {windowScale} ({windowDpi} DPI)",
            Foreground = foregroundColor,
            FontSize = fontSize
        };

        var dpiTextBlock = new TextBlock
        {
            Text = $"XAML Island Scale: {xamlIslandScale}",
            Foreground = foregroundColor,
            FontSize = fontSize
        };

        stackPanel.Children.Add(windowScaleTextBlock);
        stackPanel.Children.Add(dpiTextBlock);

        var button = new Button
        {
            Content = "Context Menu Button (With Tooltip)",
            Foreground = foregroundColor,
            FontSize = fontSize,
            Padding = new Thickness(25.0),
            Background = new SolidColorBrush(Colors.Beige),
        };

        ToolTipService.SetToolTip(button, "Test tooltip");

        var flyout = new MenuFlyout();
        flyout.Items.Add(new MenuFlyoutItem { Text = "Dummy Item 1", Foreground = foregroundColor, FontSize = fontSize });
        flyout.Items.Add(new MenuFlyoutItem { Text = "Dummy Item 2", Foreground = foregroundColor, FontSize = fontSize });
        flyout.Items.Add(new MenuFlyoutItem { Text = "Dummy Item 3", Foreground = foregroundColor, FontSize = fontSize });

        button.Flyout = flyout;

        stackPanel.Children.Add(button);

        var border = new Border()
        {
            Background = new SolidColorBrush(Colors.LightSkyBlue),
            BorderThickness = new Thickness(2),
            HorizontalAlignment = HorizontalAlignment.Stretch,
            VerticalAlignment = VerticalAlignment.Stretch,
            Child = stackPanel
        };

        _xamlSource.Content = border;
    }

    public void Show()
    {
        ShowWindow(_hwnd, SW_SHOW);
        UpdateWindow(_hwnd);
    }

    private readonly WndProc _wndProcDelegate = WindowProc;

    private static IntPtr WindowProc(IntPtr hwnd, uint msg, IntPtr wParam, IntPtr lParam)
    {
        switch (msg)
        {
            case WM_SIZE:
            {
                var handle = GetWindowLongPtr(hwnd, GWLP_USERDATA);
                if (handle != IntPtr.Zero)
                {
                    var gcHandle = GCHandle.FromIntPtr(handle);
                    if (gcHandle.Target is Win32XamlWindow window && window._xamlSource != null)
                    {
                        IntPtr xamlIslandHwnd =
                            Win32Interop.GetWindowFromWindowId(window._xamlSource.SiteBridge.WindowId);

                        int width = LOWORD(lParam.ToInt32());
                        int height = HIWORD(lParam.ToInt32());

                        SetWindowPos(xamlIslandHwnd, IntPtr.Zero, 0, 0, width, height, SWP_SHOWWINDOW);
                    }
                }

                return IntPtr.Zero;
            }

            case WM_DESTROY:
            {
                var handle = GetWindowLongPtr(hwnd, GWLP_USERDATA);
                if (handle != IntPtr.Zero)
                {
                    var gcHandle = GCHandle.FromIntPtr(handle);
                    if (gcHandle.IsAllocated)
                    {
                        gcHandle.Free();
                    }
                }

                PostQuitMessage(0);
                return IntPtr.Zero;
            }
        }

        return DefWindowProc(hwnd, msg, wParam, lParam);
    }

    private static int LOWORD(int l) => l & 0xFFFF;
    private static int HIWORD(int l) => (l >> 16) & 0xFFFF;

    #region Win32 Interop

    private delegate IntPtr WndProc(IntPtr hWnd, uint msg, IntPtr wParam, IntPtr lParam);

    [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
    private struct WNDCLASSEX
    {
        public uint cbSize;
        public uint style;
        public IntPtr lpfnWndProc;
        public int cbClsExtra;
        public int cbWndExtra;
        public IntPtr hInstance;
        public IntPtr hIcon;
        public IntPtr hCursor;
        public IntPtr hbrBackground;
        [MarshalAs(UnmanagedType.LPWStr)] public string? lpszMenuName;
        [MarshalAs(UnmanagedType.LPWStr)] public string lpszClassName;
        public IntPtr hIconSm;
    }

    [StructLayout(LayoutKind.Sequential)]
    private struct RECT
    {
        public int left;
        public int top;
        public int right;
        public int bottom;
    }

    private const uint WS_OVERLAPPEDWINDOW = 0x00CF0000;
    private const int CW_USEDEFAULT = unchecked((int)0x80000000);
    private const int GWLP_USERDATA = -21;
    private const int COLOR_WINDOW = 5;
    private const uint WM_DESTROY = 0x0002;
    private const uint WM_SIZE = 0x0005;
    private const int SW_SHOW = 5;
    private const uint SWP_SHOWWINDOW = 0x0040;
    private const int IDC_ARROW = 32512;

    [DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
    private static extern ushort RegisterClassEx(ref WNDCLASSEX lpwcx);

    [DllImport("user32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
    private static extern IntPtr CreateWindowEx(
        uint dwExStyle,
        string lpClassName,
        string lpWindowName,
        uint dwStyle,
        int x,
        int y,
        int nWidth,
        int nHeight,
        IntPtr hWndParent,
        IntPtr hMenu,
        IntPtr hInstance,
        IntPtr lpParam);

    [DllImport("user32.dll")]
    private static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);

    [DllImport("user32.dll")]
    private static extern bool UpdateWindow(IntPtr hWnd);

    [DllImport("user32.dll")]
    private static extern IntPtr DefWindowProc(IntPtr hWnd, uint uMsg, IntPtr wParam, IntPtr lParam);

    [DllImport("user32.dll")]
    private static extern void PostQuitMessage(int nExitCode);

    [DllImport("kernel32.dll", CharSet = CharSet.Unicode)]
    private static extern IntPtr GetModuleHandle(string? lpModuleName);

    [DllImport("user32.dll")]
    private static extern IntPtr LoadCursor(IntPtr hInstance, int lpCursorName);

    [DllImport("user32.dll", EntryPoint = "SetWindowLongPtr")]
    private static extern IntPtr SetWindowLongPtr(IntPtr hWnd, int nIndex, IntPtr dwNewLong);

    [DllImport("user32.dll", EntryPoint = "GetWindowLongPtr")]
    private static extern IntPtr GetWindowLongPtr(IntPtr hWnd, int nIndex);

    [DllImport("user32.dll")]
    private static extern bool SetWindowPos(IntPtr hWnd, IntPtr hWndInsertAfter, int X, int Y, int cx, int cy,
        uint uFlags);

    [DllImport("user32.dll")]
    private static extern bool GetClientRect(IntPtr hWnd, out RECT lpRect);

    [DllImport("user32.dll")]
    private static extern uint GetDpiForWindow(IntPtr hwnd);

    #endregion
}

Actual behavior

The popup renders bigger than it should and gets clipped to a small container size.

Expected behavior

The popup renders at the correct scale.

Screenshots

NuGet package version

WinUI 3 - Windows App SDK 1.8.3: 1.8.251106002

Windows version

Windows 11 (24H2): Build 26100

Additional context

No response