From b3bf440d628840fbcf3410cc9f836e55b44dadb9 Mon Sep 17 00:00:00 2001 From: Sebastian Godelet Date: Fri, 6 Mar 2026 12:08:32 +1100 Subject: [PATCH 1/2] Add SGR-Pixels mouse mode (1016) for sub-cell precision mouse tracking Implements DECSET 1016 (sgr-pixels), which uses the same CSI sequence format as SGR mode (1006) but reports pixel coordinates instead of character-cell coordinates. This enables sub-cell mouse precision for sixel-aware programs and smooth scrolling in TUI frameworks. - Add SgrPixelMouseEncoding mode, mutually exclusive with 1005/1006 - Thread pixel coordinates through the input pipeline - Add DECSET/DECRST/DECRPM support for mode 1016 - Add SgrPixelModeTests covering coords, buttons, modifiers, exclusivity Co-Authored-By: Claude Opus 4.6 --- src/cascadia/TerminalControl/ControlCore.cpp | 5 +- src/cascadia/TerminalControl/ControlCore.h | 3 +- .../TerminalControl/ControlInteractivity.cpp | 14 ++-- .../TerminalControl/ControlInteractivity.h | 3 +- src/cascadia/TerminalControl/HwndTerminal.cpp | 2 +- src/cascadia/TerminalCore/ITerminalInput.hpp | 2 +- src/cascadia/TerminalCore/Terminal.cpp | 4 +- src/cascadia/TerminalCore/Terminal.hpp | 2 +- src/host/inputBuffer.cpp | 6 +- src/terminal/adapter/DispatchTypes.hpp | 1 + src/terminal/adapter/adaptDispatch.cpp | 6 ++ .../adapter/ut_adapter/MouseInputTest.cpp | 71 +++++++++++++++++++ src/terminal/input/mouseInput.cpp | 9 ++- src/terminal/input/terminalInput.cpp | 4 +- src/terminal/input/terminalInput.hpp | 3 +- 15 files changed, 115 insertions(+), 20 deletions(-) diff --git a/src/cascadia/TerminalControl/ControlCore.cpp b/src/cascadia/TerminalControl/ControlCore.cpp index 86747d0f8ff..97cfdd48064 100644 --- a/src/cascadia/TerminalControl/ControlCore.cpp +++ b/src/cascadia/TerminalControl/ControlCore.cpp @@ -716,12 +716,13 @@ namespace winrt::Microsoft::Terminal::Control::implementation const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, - const TerminalInput::MouseButtonState state) + const TerminalInput::MouseButtonState state, + const til::point pixelPosition) { TerminalInput::OutputType out; { const auto lock = _terminal->LockForReading(); - out = _terminal->SendMouseEvent(viewportPos, uiButton, states, wheelDelta, state); + out = _terminal->SendMouseEvent(viewportPos, uiButton, states, wheelDelta, state, pixelPosition); } if (out) { diff --git a/src/cascadia/TerminalControl/ControlCore.h b/src/cascadia/TerminalControl/ControlCore.h index 690f6fb465b..dd505f8ec6c 100644 --- a/src/cascadia/TerminalControl/ControlCore.h +++ b/src/cascadia/TerminalControl/ControlCore.h @@ -207,7 +207,8 @@ namespace winrt::Microsoft::Terminal::Control::implementation const unsigned int uiButton, const ::Microsoft::Terminal::Core::ControlKeyStates states, const short wheelDelta, - const ::Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state); + const ::Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state, + const til::point pixelPosition = {}); void UserScrollViewport(const int viewTop); void ClearBuffer(Control::ClearBufferType clearType); diff --git a/src/cascadia/TerminalControl/ControlInteractivity.cpp b/src/cascadia/TerminalControl/ControlInteractivity.cpp index 0c4e788961d..32bff80fd8e 100644 --- a/src/cascadia/TerminalControl/ControlInteractivity.cpp +++ b/src/cascadia/TerminalControl/ControlInteractivity.cpp @@ -262,7 +262,7 @@ namespace winrt::Microsoft::Terminal::Control::implementation } else if (_canSendVTMouseInput(modifiers)) { - _sendMouseEventHelper(terminalPosition, pointerUpdateKind, modifiers, 0, buttonState); + _sendMouseEventHelper(terminalPosition, pointerUpdateKind, modifiers, 0, buttonState, til::point{ pixelPosition }); } else if (WI_IsFlagSet(buttonState, MouseButtonState::IsLeftButtonDown)) { @@ -346,7 +346,7 @@ namespace winrt::Microsoft::Terminal::Control::implementation // Short-circuit isReadOnly check to avoid warning dialog if (focused && !_core->IsInReadOnlyMode() && _canSendVTMouseInput(modifiers)) { - _sendMouseEventHelper(terminalPosition, pointerUpdateKind, modifiers, 0, buttonState); + _sendMouseEventHelper(terminalPosition, pointerUpdateKind, modifiers, 0, buttonState, til::point{ pixelPosition }); handledCompletely = true; } // GH#4603 - don't modify the selection if the pointer press didn't @@ -445,7 +445,7 @@ namespace winrt::Microsoft::Terminal::Control::implementation // Short-circuit isReadOnly check to avoid warning dialog if (!_core->IsInReadOnlyMode() && _canSendVTMouseInput(modifiers)) { - _sendMouseEventHelper(terminalPosition, pointerUpdateKind, modifiers, 0, buttonState); + _sendMouseEventHelper(terminalPosition, pointerUpdateKind, modifiers, 0, buttonState, til::point{ pixelPosition }); return; } @@ -508,7 +508,8 @@ namespace winrt::Microsoft::Terminal::Control::implementation delta.Y != 0 ? WM_MOUSEWHEEL : WM_MOUSEHWHEEL, modifiers, ::base::saturated_cast(delta.Y != 0 ? delta.Y : delta.X), - buttonState); + buttonState, + til::point{ pixelPosition }); } const auto ctrlPressed = modifiers.IsCtrlPressed(); @@ -716,12 +717,13 @@ namespace winrt::Microsoft::Terminal::Control::implementation const unsigned int pointerUpdateKind, const ::Microsoft::Terminal::Core::ControlKeyStates modifiers, const SHORT wheelDelta, - Control::MouseButtonState buttonState) + Control::MouseButtonState buttonState, + const til::point pixelPosition) { const auto adjustment = _core->BufferHeight() - _core->ScrollOffset() - _core->ViewHeight(); // If the click happened outside the active region, core should get a chance to filter it out or clamp it. const auto adjustedY = terminalPosition.y - adjustment; - return _core->SendMouseEvent({ terminalPosition.x, adjustedY }, pointerUpdateKind, modifiers, wheelDelta, toInternalMouseState(buttonState)); + return _core->SendMouseEvent({ terminalPosition.x, adjustedY }, pointerUpdateKind, modifiers, wheelDelta, toInternalMouseState(buttonState), pixelPosition); } // Method Description: diff --git a/src/cascadia/TerminalControl/ControlInteractivity.h b/src/cascadia/TerminalControl/ControlInteractivity.h index b800fbb25a1..ee295aeb474 100644 --- a/src/cascadia/TerminalControl/ControlInteractivity.h +++ b/src/cascadia/TerminalControl/ControlInteractivity.h @@ -161,7 +161,8 @@ namespace winrt::Microsoft::Terminal::Control::implementation const unsigned int pointerUpdateKind, const ::Microsoft::Terminal::Core::ControlKeyStates modifiers, const SHORT wheelDelta, - Control::MouseButtonState buttonState); + Control::MouseButtonState buttonState, + const til::point pixelPosition = {}); friend class ControlUnitTests::ControlCoreTests; friend class ControlUnitTests::ControlInteractivityTests; diff --git a/src/cascadia/TerminalControl/HwndTerminal.cpp b/src/cascadia/TerminalControl/HwndTerminal.cpp index 2c792e81c7e..d3701cdf881 100644 --- a/src/cascadia/TerminalControl/HwndTerminal.cpp +++ b/src/cascadia/TerminalControl/HwndTerminal.cpp @@ -826,7 +826,7 @@ try TerminalInput::OutputType out; { const auto lock = _terminal->LockForReading(); - out = _terminal->SendMouseEvent(cursorPosition / fontSize, uMsg, getControlKeyState(), wheelDelta, state); + out = _terminal->SendMouseEvent(cursorPosition / fontSize, uMsg, getControlKeyState(), wheelDelta, state, cursorPosition); } if (out) { diff --git a/src/cascadia/TerminalCore/ITerminalInput.hpp b/src/cascadia/TerminalCore/ITerminalInput.hpp index e882004d289..d296a1e854a 100644 --- a/src/cascadia/TerminalCore/ITerminalInput.hpp +++ b/src/cascadia/TerminalCore/ITerminalInput.hpp @@ -17,7 +17,7 @@ namespace Microsoft::Terminal::Core ITerminalInput& operator=(ITerminalInput&&) = default; [[nodiscard]] virtual ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendKeyEvent(const WORD vkey, const WORD scanCode, const ControlKeyStates states, const bool keyDown) = 0; - [[nodiscard]] virtual ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendMouseEvent(const til::point viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state) = 0; + [[nodiscard]] virtual ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendMouseEvent(const til::point viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state, const til::point pixelPosition = {}) = 0; [[nodiscard]] virtual ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendCharEvent(const wchar_t ch, const WORD scanCode, const ControlKeyStates states) = 0; [[nodiscard]] virtual ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType FocusChanged(const bool focused) = 0; diff --git a/src/cascadia/TerminalCore/Terminal.cpp b/src/cascadia/TerminalCore/Terminal.cpp index e43180e9aaf..850f79b08a6 100644 --- a/src/cascadia/TerminalCore/Terminal.cpp +++ b/src/cascadia/TerminalCore/Terminal.cpp @@ -679,14 +679,14 @@ TerminalInput::OutputType Terminal::SendKeyEvent(const WORD vkey, // Return Value: // - true if we translated the key event, and it should not be processed any further. // - false if we did not translate the key, and it should be processed into a character. -TerminalInput::OutputType Terminal::SendMouseEvent(til::point viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const TerminalInput::MouseButtonState state) +TerminalInput::OutputType Terminal::SendMouseEvent(til::point viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const TerminalInput::MouseButtonState state, const til::point pixelPosition) { // GH#6401: VT applications should be able to receive mouse events from outside the // terminal buffer. This is likely to happen when the user drags the cursor offscreen. // We shouldn't throw away perfectly good events when they're offscreen, so we just // clamp them to be within the range [(0, 0), (W, H)]. _GetMutableViewport().ToOrigin().Clamp(viewportPos); - return _getTerminalInput().HandleMouse(viewportPos, uiButton, GET_KEYSTATE_WPARAM(states.Value()), wheelDelta, state); + return _getTerminalInput().HandleMouse(viewportPos, uiButton, GET_KEYSTATE_WPARAM(states.Value()), wheelDelta, state, pixelPosition); } // Method Description: diff --git a/src/cascadia/TerminalCore/Terminal.hpp b/src/cascadia/TerminalCore/Terminal.hpp index e69b42e373b..d6b2281430f 100644 --- a/src/cascadia/TerminalCore/Terminal.hpp +++ b/src/cascadia/TerminalCore/Terminal.hpp @@ -171,7 +171,7 @@ class Microsoft::Terminal::Core::Terminal final : #pragma region ITerminalInput // These methods are defined in Terminal.cpp [[nodiscard]] ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendKeyEvent(const WORD vkey, const WORD scanCode, const Microsoft::Terminal::Core::ControlKeyStates states, const bool keyDown) override; - [[nodiscard]] ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendMouseEvent(const til::point viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state) override; + [[nodiscard]] ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendMouseEvent(const til::point viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state, const til::point pixelPosition = {}) override; [[nodiscard]] ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendCharEvent(const wchar_t ch, const WORD scanCode, const ControlKeyStates states) override; [[nodiscard]] ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType FocusChanged(const bool focused) override; diff --git a/src/host/inputBuffer.cpp b/src/host/inputBuffer.cpp index 240b3ee3291..9ea62363cbf 100644 --- a/src/host/inputBuffer.cpp +++ b/src/host/inputBuffer.cpp @@ -623,7 +623,11 @@ bool InputBuffer::WriteMouseEvent(til::point position, const unsigned int button const auto& gci = ServiceLocator::LocateGlobals().getConsoleInformation(); gci.GetActiveOutputBuffer().GetViewport().ToOrigin().Clamp(position); - if (const auto out = _termInput.HandleMouse(position, button, keyState, wheelDelta, state)) + // Approximate pixel position from cell coordinates for SGR-Pixel mode (1016). + const auto fontSize = gci.GetActiveOutputBuffer().GetCurrentFont().GetSize(); + const til::point pixelPosition{ position.x * fontSize.width, position.y * fontSize.height }; + + if (const auto out = _termInput.HandleMouse(position, button, keyState, wheelDelta, state, pixelPosition)) { _writeString(*out); return true; diff --git a/src/terminal/adapter/DispatchTypes.hpp b/src/terminal/adapter/DispatchTypes.hpp index b99dd997dde..a44c912ad8d 100644 --- a/src/terminal/adapter/DispatchTypes.hpp +++ b/src/terminal/adapter/DispatchTypes.hpp @@ -542,6 +542,7 @@ namespace Microsoft::Console::VirtualTerminal::DispatchTypes UTF8_EXTENDED_MODE = DECPrivateMode(1005), SGR_EXTENDED_MODE = DECPrivateMode(1006), ALTERNATE_SCROLL = DECPrivateMode(1007), + SGR_PIXEL_MODE = DECPrivateMode(1016), ASB_AlternateScreenBuffer = DECPrivateMode(1049), XTERM_BracketedPasteMode = DECPrivateMode(2004), SO_SynchronizedOutput = DECPrivateMode(2026), diff --git a/src/terminal/adapter/adaptDispatch.cpp b/src/terminal/adapter/adaptDispatch.cpp index d49bbfad8fa..b66adb0e6b9 100644 --- a/src/terminal/adapter/adaptDispatch.cpp +++ b/src/terminal/adapter/adaptDispatch.cpp @@ -1859,6 +1859,9 @@ void AdaptDispatch::_ModeParamsHelper(const DispatchTypes::ModeParams param, con case DispatchTypes::ModeParams::SGR_EXTENDED_MODE: _terminalInput.SetInputMode(TerminalInput::Mode::SgrMouseEncoding, enable); break; + case DispatchTypes::ModeParams::SGR_PIXEL_MODE: + _terminalInput.SetInputMode(TerminalInput::Mode::SgrPixelMouseEncoding, enable); + break; case DispatchTypes::ModeParams::FOCUS_EVENT_MODE: _terminalInput.SetInputMode(TerminalInput::Mode::FocusEvent, enable); // ConPTY always wants to know about focus events, so let it know that it needs to re-enable this mode. @@ -2009,6 +2012,9 @@ void AdaptDispatch::RequestMode(const DispatchTypes::ModeParams param) case DispatchTypes::ModeParams::SGR_EXTENDED_MODE: state = mapTemp(_terminalInput.GetInputMode(TerminalInput::Mode::SgrMouseEncoding)); break; + case DispatchTypes::ModeParams::SGR_PIXEL_MODE: + state = mapTemp(_terminalInput.GetInputMode(TerminalInput::Mode::SgrPixelMouseEncoding)); + break; case DispatchTypes::ModeParams::FOCUS_EVENT_MODE: state = mapTemp(_terminalInput.GetInputMode(TerminalInput::Mode::FocusEvent)); break; diff --git a/src/terminal/adapter/ut_adapter/MouseInputTest.cpp b/src/terminal/adapter/ut_adapter/MouseInputTest.cpp index bcfb6fe023e..bdcecde5dff 100644 --- a/src/terminal/adapter/ut_adapter/MouseInputTest.cpp +++ b/src/terminal/adapter/ut_adapter/MouseInputTest.cpp @@ -485,6 +485,77 @@ class MouseInputTest } } + TEST_METHOD(SgrPixelModeTests) + { + BEGIN_TEST_METHOD_PROPERTIES() + // TEST_METHOD_PROPERTY(L"Data:uiButton", L"{WM_LBUTTONDOWN, WM_LBUTTONUP, WM_MBUTTONDOWN, WM_MBUTTONUP, WM_RBUTTONDOWN, WM_RBUTTONUP, WM_MOUSEMOVE}") + TEST_METHOD_PROPERTY(L"Data:uiButton", L"{0x0201, 0x0202, 0x0207, 0x0208, 0x0204, 0x0205, 0x0200}") + // None, SHIFT, LEFT_CONTROL, RIGHT_ALT, RIGHT_ALT | LEFT_CONTROL + TEST_METHOD_PROPERTY(L"Data:uiModifierKeystate", L"{0x0000, 0x0010, 0x0008, 0x0001, 0x0009}") + END_TEST_METHOD_PROPERTIES() + + Log::Comment(L"Starting test..."); + + TerminalInput mouseInput; + unsigned int uiModifierKeystate = 0; + VERIFY_SUCCEEDED_RETURN(TestData::TryGetValue(L"uiModifierKeystate", uiModifierKeystate)); + auto sModifierKeystate = (SHORT)uiModifierKeystate; + short sScrollDelta = 0; + + unsigned int uiButton; + VERIFY_SUCCEEDED_RETURN(TestData::TryGetValue(L"uiButton", uiButton)); + + VERIFY_ARE_EQUAL(TerminalInput::MakeUnhandled(), mouseInput.HandleMouse({ 0, 0 }, uiButton, sModifierKeystate, sScrollDelta, {})); + + mouseInput.SetInputMode(TerminalInput::Mode::SgrPixelMouseEncoding, true); + + // SGR-Pixel mode uses pixel coordinates instead of cell coordinates. + // The format is identical to SGR: ESC [ < button ; px ; py M/m + // where px and py are 1-based pixel positions. + static const til::point testPixelCoords[] = { + { 0, 0 }, + { 5, 10 }, + { 100, 200 }, + { 1920, 1080 }, + }; + static const wchar_t* testPixelOutput[] = { + L"\x1b[<%d;1;1M", + L"\x1b[<%d;6;11M", + L"\x1b[<%d;101;201M", + L"\x1b[<%d;1921;1081M", + }; + + // Test with AnyEventMouseTracking to cover all button types including hovers + mouseInput.SetInputMode(TerminalInput::Mode::AnyEventMouseTracking, true); + for (auto i = 0; i < ARRAYSIZE(testPixelCoords); i++) + { + const auto pixelCoord = testPixelCoords[i]; + // Cell position doesn't matter for SGR-Pixel output; pixel position is used. + const til::point cellCoord{ pixelCoord.x / 8, pixelCoord.y / 16 }; // approximate cell coords + + const auto expected = BuildSGRTestOutput(testPixelOutput[i], uiButton, sModifierKeystate, sScrollDelta); + + VERIFY_ARE_EQUAL(expected, + mouseInput.HandleMouse(cellCoord, + uiButton, + sModifierKeystate, + sScrollDelta, + {}, + pixelCoord), + NoThrowString().Format(L"pixel(x,y)=(%d,%d)", pixelCoord.x, pixelCoord.y)); + } + + // Verify mutual exclusivity: enabling SgrPixelMouseEncoding should disable SgrMouseEncoding + mouseInput.SetInputMode(TerminalInput::Mode::SgrMouseEncoding, true); + VERIFY_IS_FALSE(mouseInput.GetInputMode(TerminalInput::Mode::SgrPixelMouseEncoding)); + VERIFY_IS_TRUE(mouseInput.GetInputMode(TerminalInput::Mode::SgrMouseEncoding)); + + // And vice versa + mouseInput.SetInputMode(TerminalInput::Mode::SgrPixelMouseEncoding, true); + VERIFY_IS_FALSE(mouseInput.GetInputMode(TerminalInput::Mode::SgrMouseEncoding)); + VERIFY_IS_TRUE(mouseInput.GetInputMode(TerminalInput::Mode::SgrPixelMouseEncoding)); + } + TEST_METHOD(ScrollWheelTests) { BEGIN_TEST_METHOD_PROPERTIES() diff --git a/src/terminal/input/mouseInput.cpp b/src/terminal/input/mouseInput.cpp index 99ba22eccba..b53e5937dfb 100644 --- a/src/terminal/input/mouseInput.cpp +++ b/src/terminal/input/mouseInput.cpp @@ -292,7 +292,7 @@ bool TerminalInput::IsTrackingMouseInput() const noexcept // Return value: // - Returns an empty optional if we didn't handle the mouse event and the caller can opt to handle it in some other way. // - Returns a string if we successfully translated it into a VT input sequence. -TerminalInput::OutputType TerminalInput::HandleMouse(const til::point position, const unsigned int button, const short modifierKeyState, const short delta, const MouseButtonState state) +TerminalInput::OutputType TerminalInput::HandleMouse(const til::point position, const unsigned int button, const short modifierKeyState, const short delta, const MouseButtonState state, const til::point pixelPosition) { if (Utils::Sign(delta) != Utils::Sign(_mouseInputState.accumulatedDelta)) { @@ -365,6 +365,13 @@ TerminalInput::OutputType TerminalInput::HandleMouse(const til::point position, { return _GenerateUtf8Sequence(position, realButton, isHover, modifierKeyState, delta); } + else if (_inputMode.test(Mode::SgrPixelMouseEncoding)) + { + // SGR-Pixel mode (1016) uses the same format as SGR (1006), + // but with pixel coordinates instead of cell coordinates. + const auto effectiveButton = physicalButtonPressed ? realButton : button; + return _GenerateSGRSequence(pixelPosition, effectiveButton, _isButtonUp(button), isHover, modifierKeyState, delta); + } else if (_inputMode.test(Mode::SgrMouseEncoding)) { // For SGR encoding, if no physical buttons were pressed, diff --git a/src/terminal/input/terminalInput.cpp b/src/terminal/input/terminalInput.cpp index 9755b0da8c1..4fba75aad0d 100644 --- a/src/terminal/input/terminalInput.cpp +++ b/src/terminal/input/terminalInput.cpp @@ -71,9 +71,9 @@ void TerminalInput::SetInputMode(const Mode mode, const bool enabled) noexcept // But if we're changing the encoding, we only clear out the other encoding modes // when enabling a new encoding - not when disabling. - if ((mode == Mode::Utf8MouseEncoding || mode == Mode::SgrMouseEncoding) && enabled) + if ((mode == Mode::Utf8MouseEncoding || mode == Mode::SgrMouseEncoding || mode == Mode::SgrPixelMouseEncoding) && enabled) { - _inputMode.reset(Mode::Utf8MouseEncoding, Mode::SgrMouseEncoding); + _inputMode.reset(Mode::Utf8MouseEncoding, Mode::SgrMouseEncoding, Mode::SgrPixelMouseEncoding); } _inputMode.set(mode, enabled); diff --git a/src/terminal/input/terminalInput.hpp b/src/terminal/input/terminalInput.hpp index 80c4be509f8..d1bb65ce424 100644 --- a/src/terminal/input/terminalInput.hpp +++ b/src/terminal/input/terminalInput.hpp @@ -22,7 +22,7 @@ namespace Microsoft::Console::VirtualTerminal [[nodiscard]] static OutputType MakeOutput(const std::wstring_view& str); [[nodiscard]] OutputType HandleKey(const INPUT_RECORD& pInEvent); [[nodiscard]] OutputType HandleFocus(bool focused) const; - [[nodiscard]] OutputType HandleMouse(til::point position, unsigned int button, short modifierKeyState, short delta, MouseButtonState state); + [[nodiscard]] OutputType HandleMouse(til::point position, unsigned int button, short modifierKeyState, short delta, MouseButtonState state, til::point pixelPosition = {}); enum class Mode : size_t { @@ -37,6 +37,7 @@ namespace Microsoft::Console::VirtualTerminal Utf8MouseEncoding, SgrMouseEncoding, + SgrPixelMouseEncoding, DefaultMouseTracking, ButtonEventMouseTracking, From a46bc1f31ac5cd576a74ad0c8c00ccdbc10227c1 Mon Sep 17 00:00:00 2001 From: Sebastian Godelet Date: Tue, 10 Mar 2026 09:44:37 +1100 Subject: [PATCH 2/2] Address PR review feedback for SGR-Pixels mouse mode (1016) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rework the coordinate pipeline per reviewer feedback: 1. Use virtual pixel resolution (10x20 per cell) instead of raw display pixels. The DEC VT340 convention defines character cells as 10x20 virtual pixels, so coordinates are computed as viewportX*10 and viewportY*20, then reported 1-based in SGR format. 2. Pass fractional cell coordinates (float) through the mouse event chain instead of a separate pixelPosition parameter. The UI layer already computes pixel/cellSize — switching from integer to float division preserves sub-cell precision. TerminalInput truncates to int for cell-based modes or multiplies by 10/20 for SGR-Pixels. 3. Scroll offset is now applied to the float Y coordinate in _sendMouseEventHelper, preserving sub-cell precision through the adjustment. 4. Float-based clamping in Terminal::SendMouseEvent ensures coordinates stay within the VT display area [0, viewport dimensions). 5. Disable SGR-Pixel mode in direct conhost (no ConPTY) since the Win32 console input API only provides cell coordinates. Add ITerminalApi::IsConhost() to detect this case. DECRQM reports PermanentlyDisabled for mode 1016 in direct conhost. Remove the fake pixel coordinate approximation from inputBuffer.cpp. 6. Update tests to use the float API and verify virtual pixel math (e.g., center of cell (0,0) at float pos (0.5, 0.5) produces virtual pixel output (6, 11) in 1-based SGR format). Co-Authored-By: Claude Opus 4.6 --- src/cascadia/TerminalControl/ControlCore.cpp | 8 +- src/cascadia/TerminalControl/ControlCore.h | 6 +- .../TerminalControl/ControlInteractivity.cpp | 37 ++++-- .../TerminalControl/ControlInteractivity.h | 7 +- src/cascadia/TerminalControl/HwndTerminal.cpp | 4 +- src/cascadia/TerminalCore/ITerminalInput.hpp | 2 +- src/cascadia/TerminalCore/Terminal.cpp | 8 +- src/cascadia/TerminalCore/Terminal.hpp | 3 +- src/cascadia/TerminalCore/TerminalApi.cpp | 5 + src/host/inputBuffer.cpp | 6 +- src/host/outputStream.cpp | 5 + src/host/outputStream.hpp | 1 + src/terminal/adapter/ITerminalApi.hpp | 1 + src/terminal/adapter/adaptDispatch.cpp | 18 ++- .../adapter/ut_adapter/MouseInputTest.cpp | 113 ++++++++++-------- .../adapter/ut_adapter/adapterTest.cpp | 5 + src/terminal/input/mouseInput.cpp | 18 ++- src/terminal/input/terminalInput.hpp | 2 +- 18 files changed, 161 insertions(+), 88 deletions(-) diff --git a/src/cascadia/TerminalControl/ControlCore.cpp b/src/cascadia/TerminalControl/ControlCore.cpp index 97cfdd48064..ee3c5c65cb4 100644 --- a/src/cascadia/TerminalControl/ControlCore.cpp +++ b/src/cascadia/TerminalControl/ControlCore.cpp @@ -712,17 +712,17 @@ namespace winrt::Microsoft::Terminal::Control::implementation return false; } - bool ControlCore::SendMouseEvent(const til::point viewportPos, + bool ControlCore::SendMouseEvent(const float viewportX, + const float viewportY, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, - const TerminalInput::MouseButtonState state, - const til::point pixelPosition) + const TerminalInput::MouseButtonState state) { TerminalInput::OutputType out; { const auto lock = _terminal->LockForReading(); - out = _terminal->SendMouseEvent(viewportPos, uiButton, states, wheelDelta, state, pixelPosition); + out = _terminal->SendMouseEvent(viewportX, viewportY, uiButton, states, wheelDelta, state); } if (out) { diff --git a/src/cascadia/TerminalControl/ControlCore.h b/src/cascadia/TerminalControl/ControlCore.h index dd505f8ec6c..36135649375 100644 --- a/src/cascadia/TerminalControl/ControlCore.h +++ b/src/cascadia/TerminalControl/ControlCore.h @@ -203,12 +203,12 @@ namespace winrt::Microsoft::Terminal::Control::implementation bool SendCharEvent(const wchar_t ch, const WORD scanCode, const ::Microsoft::Terminal::Core::ControlKeyStates modifiers); - bool SendMouseEvent(const til::point viewportPos, + bool SendMouseEvent(float viewportX, + float viewportY, const unsigned int uiButton, const ::Microsoft::Terminal::Core::ControlKeyStates states, const short wheelDelta, - const ::Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state, - const til::point pixelPosition = {}); + const ::Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state); void UserScrollViewport(const int viewTop); void ClearBuffer(Control::ClearBufferType clearType); diff --git a/src/cascadia/TerminalControl/ControlInteractivity.cpp b/src/cascadia/TerminalControl/ControlInteractivity.cpp index 32bff80fd8e..6b44c954465 100644 --- a/src/cascadia/TerminalControl/ControlInteractivity.cpp +++ b/src/cascadia/TerminalControl/ControlInteractivity.cpp @@ -262,7 +262,8 @@ namespace winrt::Microsoft::Terminal::Control::implementation } else if (_canSendVTMouseInput(modifiers)) { - _sendMouseEventHelper(terminalPosition, pointerUpdateKind, modifiers, 0, buttonState, til::point{ pixelPosition }); + const auto [termX, termY] = _getTerminalPositionF(til::point{ pixelPosition }); + _sendMouseEventHelper(termX, termY, pointerUpdateKind, modifiers, 0, buttonState); } else if (WI_IsFlagSet(buttonState, MouseButtonState::IsLeftButtonDown)) { @@ -346,7 +347,8 @@ namespace winrt::Microsoft::Terminal::Control::implementation // Short-circuit isReadOnly check to avoid warning dialog if (focused && !_core->IsInReadOnlyMode() && _canSendVTMouseInput(modifiers)) { - _sendMouseEventHelper(terminalPosition, pointerUpdateKind, modifiers, 0, buttonState, til::point{ pixelPosition }); + const auto [termX, termY] = _getTerminalPositionF(til::point{ pixelPosition }); + _sendMouseEventHelper(termX, termY, pointerUpdateKind, modifiers, 0, buttonState); handledCompletely = true; } // GH#4603 - don't modify the selection if the pointer press didn't @@ -445,7 +447,8 @@ namespace winrt::Microsoft::Terminal::Control::implementation // Short-circuit isReadOnly check to avoid warning dialog if (!_core->IsInReadOnlyMode() && _canSendVTMouseInput(modifiers)) { - _sendMouseEventHelper(terminalPosition, pointerUpdateKind, modifiers, 0, buttonState, til::point{ pixelPosition }); + const auto [termX, termY] = _getTerminalPositionF(til::point{ pixelPosition }); + _sendMouseEventHelper(termX, termY, pointerUpdateKind, modifiers, 0, buttonState); return; } @@ -504,12 +507,13 @@ namespace winrt::Microsoft::Terminal::Control::implementation // here with a PointerPoint. However, as of #979, we don't have a // PointerPoint to work with. So, we're just going to do a // mousewheel event manually - return _sendMouseEventHelper(terminalPosition, + const auto [termX, termY] = _getTerminalPositionF(til::point{ pixelPosition }); + return _sendMouseEventHelper(termX, + termY, delta.Y != 0 ? WM_MOUSEWHEEL : WM_MOUSEHWHEEL, modifiers, ::base::saturated_cast(delta.Y != 0 ? delta.Y : delta.X), - buttonState, - til::point{ pixelPosition }); + buttonState); } const auto ctrlPressed = modifiers.IsCtrlPressed(); @@ -713,17 +717,28 @@ namespace winrt::Microsoft::Terminal::Control::implementation return pixelPosition / fontSize; } - bool ControlInteractivity::_sendMouseEventHelper(const til::point terminalPosition, + std::pair ControlInteractivity::_getTerminalPositionF(const til::point pixelPosition) + { + // Get the size of the font, which is in pixels. + // Use float division to preserve sub-cell precision for SGR-Pixels mode. + const auto fontSize{ _core->GetFont().GetSize() }; + return { + static_cast(pixelPosition.x) / fontSize.width, + static_cast(pixelPosition.y) / fontSize.height + }; + } + + bool ControlInteractivity::_sendMouseEventHelper(const float terminalX, + const float terminalY, const unsigned int pointerUpdateKind, const ::Microsoft::Terminal::Core::ControlKeyStates modifiers, const SHORT wheelDelta, - Control::MouseButtonState buttonState, - const til::point pixelPosition) + Control::MouseButtonState buttonState) { const auto adjustment = _core->BufferHeight() - _core->ScrollOffset() - _core->ViewHeight(); // If the click happened outside the active region, core should get a chance to filter it out or clamp it. - const auto adjustedY = terminalPosition.y - adjustment; - return _core->SendMouseEvent({ terminalPosition.x, adjustedY }, pointerUpdateKind, modifiers, wheelDelta, toInternalMouseState(buttonState), pixelPosition); + const auto adjustedY = terminalY - static_cast(adjustment); + return _core->SendMouseEvent(terminalX, adjustedY, pointerUpdateKind, modifiers, wheelDelta, toInternalMouseState(buttonState)); } // Method Description: diff --git a/src/cascadia/TerminalControl/ControlInteractivity.h b/src/cascadia/TerminalControl/ControlInteractivity.h index ee295aeb474..0be7b0e7ccd 100644 --- a/src/cascadia/TerminalControl/ControlInteractivity.h +++ b/src/cascadia/TerminalControl/ControlInteractivity.h @@ -156,13 +156,14 @@ namespace winrt::Microsoft::Terminal::Control::implementation bool _shouldSendAlternateScroll(const ::Microsoft::Terminal::Core::ControlKeyStates modifiers, const Core::Point delta); til::point _getTerminalPosition(const til::point pixelPosition, bool roundToNearestCell); + std::pair _getTerminalPositionF(const til::point pixelPosition); - bool _sendMouseEventHelper(const til::point terminalPosition, + bool _sendMouseEventHelper(float terminalX, + float terminalY, const unsigned int pointerUpdateKind, const ::Microsoft::Terminal::Core::ControlKeyStates modifiers, const SHORT wheelDelta, - Control::MouseButtonState buttonState, - const til::point pixelPosition = {}); + Control::MouseButtonState buttonState); friend class ControlUnitTests::ControlCoreTests; friend class ControlUnitTests::ControlInteractivityTests; diff --git a/src/cascadia/TerminalControl/HwndTerminal.cpp b/src/cascadia/TerminalControl/HwndTerminal.cpp index d3701cdf881..3ae00200939 100644 --- a/src/cascadia/TerminalControl/HwndTerminal.cpp +++ b/src/cascadia/TerminalControl/HwndTerminal.cpp @@ -826,7 +826,9 @@ try TerminalInput::OutputType out; { const auto lock = _terminal->LockForReading(); - out = _terminal->SendMouseEvent(cursorPosition / fontSize, uMsg, getControlKeyState(), wheelDelta, state, cursorPosition); + out = _terminal->SendMouseEvent(static_cast(cursorPosition.x) / fontSize.width, + static_cast(cursorPosition.y) / fontSize.height, + uMsg, getControlKeyState(), wheelDelta, state); } if (out) { diff --git a/src/cascadia/TerminalCore/ITerminalInput.hpp b/src/cascadia/TerminalCore/ITerminalInput.hpp index d296a1e854a..2beae8c2bce 100644 --- a/src/cascadia/TerminalCore/ITerminalInput.hpp +++ b/src/cascadia/TerminalCore/ITerminalInput.hpp @@ -17,7 +17,7 @@ namespace Microsoft::Terminal::Core ITerminalInput& operator=(ITerminalInput&&) = default; [[nodiscard]] virtual ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendKeyEvent(const WORD vkey, const WORD scanCode, const ControlKeyStates states, const bool keyDown) = 0; - [[nodiscard]] virtual ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendMouseEvent(const til::point viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state, const til::point pixelPosition = {}) = 0; + [[nodiscard]] virtual ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendMouseEvent(float viewportX, float viewportY, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state) = 0; [[nodiscard]] virtual ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendCharEvent(const wchar_t ch, const WORD scanCode, const ControlKeyStates states) = 0; [[nodiscard]] virtual ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType FocusChanged(const bool focused) = 0; diff --git a/src/cascadia/TerminalCore/Terminal.cpp b/src/cascadia/TerminalCore/Terminal.cpp index 850f79b08a6..bf579f9979e 100644 --- a/src/cascadia/TerminalCore/Terminal.cpp +++ b/src/cascadia/TerminalCore/Terminal.cpp @@ -679,14 +679,16 @@ TerminalInput::OutputType Terminal::SendKeyEvent(const WORD vkey, // Return Value: // - true if we translated the key event, and it should not be processed any further. // - false if we did not translate the key, and it should be processed into a character. -TerminalInput::OutputType Terminal::SendMouseEvent(til::point viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const TerminalInput::MouseButtonState state, const til::point pixelPosition) +TerminalInput::OutputType Terminal::SendMouseEvent(float viewportX, float viewportY, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const TerminalInput::MouseButtonState state) { // GH#6401: VT applications should be able to receive mouse events from outside the // terminal buffer. This is likely to happen when the user drags the cursor offscreen. // We shouldn't throw away perfectly good events when they're offscreen, so we just // clamp them to be within the range [(0, 0), (W, H)]. - _GetMutableViewport().ToOrigin().Clamp(viewportPos); - return _getTerminalInput().HandleMouse(viewportPos, uiButton, GET_KEYSTATE_WPARAM(states.Value()), wheelDelta, state, pixelPosition); + const auto viewport = _GetMutableViewport().ToOrigin(); + viewportX = std::clamp(viewportX, 0.0f, static_cast(viewport.Width() - 1)); + viewportY = std::clamp(viewportY, 0.0f, static_cast(viewport.Height() - 1)); + return _getTerminalInput().HandleMouse(viewportX, viewportY, uiButton, GET_KEYSTATE_WPARAM(states.Value()), wheelDelta, state); } // Method Description: diff --git a/src/cascadia/TerminalCore/Terminal.hpp b/src/cascadia/TerminalCore/Terminal.hpp index d6b2281430f..e522951a937 100644 --- a/src/cascadia/TerminalCore/Terminal.hpp +++ b/src/cascadia/TerminalCore/Terminal.hpp @@ -155,6 +155,7 @@ class Microsoft::Terminal::Core::Terminal final : void UseMainScreenBuffer() override; bool IsVtInputEnabled() const noexcept override; + bool IsConhost() const noexcept override; void NotifyBufferRotation(const int delta) override; void NotifyShellIntegrationMark() override; @@ -171,7 +172,7 @@ class Microsoft::Terminal::Core::Terminal final : #pragma region ITerminalInput // These methods are defined in Terminal.cpp [[nodiscard]] ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendKeyEvent(const WORD vkey, const WORD scanCode, const Microsoft::Terminal::Core::ControlKeyStates states, const bool keyDown) override; - [[nodiscard]] ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendMouseEvent(const til::point viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state, const til::point pixelPosition = {}) override; + [[nodiscard]] ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendMouseEvent(float viewportX, float viewportY, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state) override; [[nodiscard]] ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendCharEvent(const wchar_t ch, const WORD scanCode, const ControlKeyStates states) override; [[nodiscard]] ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType FocusChanged(const bool focused) override; diff --git a/src/cascadia/TerminalCore/TerminalApi.cpp b/src/cascadia/TerminalCore/TerminalApi.cpp index e87dc765f24..5d809a9fb22 100644 --- a/src/cascadia/TerminalCore/TerminalApi.cpp +++ b/src/cascadia/TerminalCore/TerminalApi.cpp @@ -347,6 +347,11 @@ bool Terminal::IsVtInputEnabled() const noexcept return false; } +bool Terminal::IsConhost() const noexcept +{ + return false; +} + void Terminal::InvokeCompletions(std::wstring_view menuJson, unsigned int replaceLength) { if (_pfnCompletionsChanged) diff --git a/src/host/inputBuffer.cpp b/src/host/inputBuffer.cpp index 9ea62363cbf..9502ce5028a 100644 --- a/src/host/inputBuffer.cpp +++ b/src/host/inputBuffer.cpp @@ -623,11 +623,7 @@ bool InputBuffer::WriteMouseEvent(til::point position, const unsigned int button const auto& gci = ServiceLocator::LocateGlobals().getConsoleInformation(); gci.GetActiveOutputBuffer().GetViewport().ToOrigin().Clamp(position); - // Approximate pixel position from cell coordinates for SGR-Pixel mode (1016). - const auto fontSize = gci.GetActiveOutputBuffer().GetCurrentFont().GetSize(); - const til::point pixelPosition{ position.x * fontSize.width, position.y * fontSize.height }; - - if (const auto out = _termInput.HandleMouse(position, button, keyState, wheelDelta, state, pixelPosition)) + if (const auto out = _termInput.HandleMouse(static_cast(position.x), static_cast(position.y), button, keyState, wheelDelta, state)) { _writeString(*out); return true; diff --git a/src/host/outputStream.cpp b/src/host/outputStream.cpp index 4f359dba13c..2f653370156 100644 --- a/src/host/outputStream.cpp +++ b/src/host/outputStream.cpp @@ -414,6 +414,11 @@ bool ConhostInternalGetSet::IsVtInputEnabled() const return _io.GetActiveInputBuffer()->IsInVirtualTerminalInputMode(); } +bool ConhostInternalGetSet::IsConhost() const +{ + return true; +} + // Routine Description: // - Implements conhost-specific behavior when the buffer is rotated. // Arguments: diff --git a/src/host/outputStream.hpp b/src/host/outputStream.hpp index c80df20ffea..54f09af377c 100644 --- a/src/host/outputStream.hpp +++ b/src/host/outputStream.hpp @@ -64,6 +64,7 @@ class ConhostInternalGetSet final : public Microsoft::Console::VirtualTerminal:: void PlayMidiNote(const int noteNumber, const int velocity, const std::chrono::microseconds duration) override; bool IsVtInputEnabled() const override; + bool IsConhost() const override; void NotifyBufferRotation(const int delta) override; void NotifyShellIntegrationMark() override; diff --git a/src/terminal/adapter/ITerminalApi.hpp b/src/terminal/adapter/ITerminalApi.hpp index fb613c4e121..b8318e741d8 100644 --- a/src/terminal/adapter/ITerminalApi.hpp +++ b/src/terminal/adapter/ITerminalApi.hpp @@ -51,6 +51,7 @@ namespace Microsoft::Console::VirtualTerminal virtual void SetViewportPosition(const til::point position) = 0; virtual bool IsVtInputEnabled() const = 0; + virtual bool IsConhost() const = 0; enum class Mode : size_t { diff --git a/src/terminal/adapter/adaptDispatch.cpp b/src/terminal/adapter/adaptDispatch.cpp index b66adb0e6b9..90461e0caf3 100644 --- a/src/terminal/adapter/adaptDispatch.cpp +++ b/src/terminal/adapter/adaptDispatch.cpp @@ -1860,7 +1860,14 @@ void AdaptDispatch::_ModeParamsHelper(const DispatchTypes::ModeParams param, con _terminalInput.SetInputMode(TerminalInput::Mode::SgrMouseEncoding, enable); break; case DispatchTypes::ModeParams::SGR_PIXEL_MODE: - _terminalInput.SetInputMode(TerminalInput::Mode::SgrPixelMouseEncoding, enable); + // SGR-Pixel mode requires real sub-cell pixel coordinates. It's not + // supported in direct conhost because the Win32 console input API only + // provides character-cell coordinates. In ConPTY mode (IsVtInputEnabled), + // the request is passed through to the Terminal frontend which has pixel data. + if (!_api.IsConhost() || _api.IsVtInputEnabled()) + { + _terminalInput.SetInputMode(TerminalInput::Mode::SgrPixelMouseEncoding, enable); + } break; case DispatchTypes::ModeParams::FOCUS_EVENT_MODE: _terminalInput.SetInputMode(TerminalInput::Mode::FocusEvent, enable); @@ -2013,7 +2020,14 @@ void AdaptDispatch::RequestMode(const DispatchTypes::ModeParams param) state = mapTemp(_terminalInput.GetInputMode(TerminalInput::Mode::SgrMouseEncoding)); break; case DispatchTypes::ModeParams::SGR_PIXEL_MODE: - state = mapTemp(_terminalInput.GetInputMode(TerminalInput::Mode::SgrPixelMouseEncoding)); + if (_api.IsConhost() && !_api.IsVtInputEnabled()) + { + state = DispatchTypes::DECRPM_PermanentlyDisabled; + } + else + { + state = mapTemp(_terminalInput.GetInputMode(TerminalInput::Mode::SgrPixelMouseEncoding)); + } break; case DispatchTypes::ModeParams::FOCUS_EVENT_MODE: state = mapTemp(_terminalInput.GetInputMode(TerminalInput::Mode::FocusEvent)); diff --git a/src/terminal/adapter/ut_adapter/MouseInputTest.cpp b/src/terminal/adapter/ut_adapter/MouseInputTest.cpp index bdcecde5dff..39cc0a1170e 100644 --- a/src/terminal/adapter/ut_adapter/MouseInputTest.cpp +++ b/src/terminal/adapter/ut_adapter/MouseInputTest.cpp @@ -245,7 +245,7 @@ class MouseInputTest unsigned int uiButton; VERIFY_SUCCEEDED_RETURN(TestData::TryGetValue(L"uiButton", uiButton)); - VERIFY_ARE_EQUAL(TerminalInput::MakeUnhandled(), mouseInput.HandleMouse({ 0, 0 }, uiButton, sModifierKeystate, sScrollDelta, {})); + VERIFY_ARE_EQUAL(TerminalInput::MakeUnhandled(), mouseInput.HandleMouse(0, 0, uiButton, sModifierKeystate, sScrollDelta, {})); mouseInput.SetInputMode(TerminalInput::Mode::DefaultMouseTracking, true); @@ -261,7 +261,8 @@ class MouseInputTest // validate translation VERIFY_ARE_EQUAL(expected, - mouseInput.HandleMouse(Coord, + mouseInput.HandleMouse(static_cast(Coord.x), + static_cast(Coord.y), uiButton, sModifierKeystate, sScrollDelta, @@ -282,7 +283,8 @@ class MouseInputTest // validate translation VERIFY_ARE_EQUAL(expected, - mouseInput.HandleMouse(Coord, + mouseInput.HandleMouse(static_cast(Coord.x), + static_cast(Coord.y), uiButton, sModifierKeystate, sScrollDelta, @@ -303,7 +305,8 @@ class MouseInputTest // validate translation VERIFY_ARE_EQUAL(expected, - mouseInput.HandleMouse(Coord, + mouseInput.HandleMouse(static_cast(Coord.x), + static_cast(Coord.y), uiButton, sModifierKeystate, sScrollDelta, @@ -332,7 +335,7 @@ class MouseInputTest unsigned int uiButton; VERIFY_SUCCEEDED_RETURN(TestData::TryGetValue(L"uiButton", uiButton)); - VERIFY_ARE_EQUAL(TerminalInput::MakeUnhandled(), mouseInput.HandleMouse({ 0, 0 }, uiButton, sModifierKeystate, sScrollDelta, {})); + VERIFY_ARE_EQUAL(TerminalInput::MakeUnhandled(), mouseInput.HandleMouse(0, 0, uiButton, sModifierKeystate, sScrollDelta, {})); mouseInput.SetInputMode(TerminalInput::Mode::Utf8MouseEncoding, true); @@ -351,7 +354,8 @@ class MouseInputTest // validate translation VERIFY_ARE_EQUAL(expected, - mouseInput.HandleMouse(Coord, + mouseInput.HandleMouse(static_cast(Coord.x), + static_cast(Coord.y), uiButton, sModifierKeystate, sScrollDelta, @@ -372,7 +376,8 @@ class MouseInputTest // validate translation VERIFY_ARE_EQUAL(expected, - mouseInput.HandleMouse(Coord, + mouseInput.HandleMouse(static_cast(Coord.x), + static_cast(Coord.y), uiButton, sModifierKeystate, sScrollDelta, @@ -393,7 +398,8 @@ class MouseInputTest // validate translation VERIFY_ARE_EQUAL(expected, - mouseInput.HandleMouse(Coord, + mouseInput.HandleMouse(static_cast(Coord.x), + static_cast(Coord.y), uiButton, sModifierKeystate, sScrollDelta, @@ -422,7 +428,7 @@ class MouseInputTest unsigned int uiButton; VERIFY_SUCCEEDED_RETURN(TestData::TryGetValue(L"uiButton", uiButton)); - VERIFY_ARE_EQUAL(TerminalInput::MakeUnhandled(), mouseInput.HandleMouse({ 0, 0 }, uiButton, sModifierKeystate, sScrollDelta, {})); + VERIFY_ARE_EQUAL(TerminalInput::MakeUnhandled(), mouseInput.HandleMouse(0, 0, uiButton, sModifierKeystate, sScrollDelta, {})); mouseInput.SetInputMode(TerminalInput::Mode::SgrMouseEncoding, true); @@ -441,7 +447,9 @@ class MouseInputTest // validate translation VERIFY_ARE_EQUAL(expected, - mouseInput.HandleMouse(Coord, uiButton, sModifierKeystate, sScrollDelta, {}), + mouseInput.HandleMouse(static_cast(Coord.x), + static_cast(Coord.y), + uiButton, sModifierKeystate, sScrollDelta, {}), NoThrowString().Format(L"(x,y)=(%d,%d)", Coord.x, Coord.y)); } @@ -460,7 +468,8 @@ class MouseInputTest // validate translation VERIFY_ARE_EQUAL(expected, - mouseInput.HandleMouse(Coord, + mouseInput.HandleMouse(static_cast(Coord.x), + static_cast(Coord.y), uiButton, sModifierKeystate, sScrollDelta, @@ -476,7 +485,8 @@ class MouseInputTest // validate translation VERIFY_ARE_EQUAL(expected, - mouseInput.HandleMouse(Coord, + mouseInput.HandleMouse(static_cast(Coord.x), + static_cast(Coord.y), uiButton, sModifierKeystate, sScrollDelta, @@ -505,44 +515,44 @@ class MouseInputTest unsigned int uiButton; VERIFY_SUCCEEDED_RETURN(TestData::TryGetValue(L"uiButton", uiButton)); - VERIFY_ARE_EQUAL(TerminalInput::MakeUnhandled(), mouseInput.HandleMouse({ 0, 0 }, uiButton, sModifierKeystate, sScrollDelta, {})); + VERIFY_ARE_EQUAL(TerminalInput::MakeUnhandled(), mouseInput.HandleMouse(0, 0, uiButton, sModifierKeystate, sScrollDelta, {})); mouseInput.SetInputMode(TerminalInput::Mode::SgrPixelMouseEncoding, true); - // SGR-Pixel mode uses pixel coordinates instead of cell coordinates. - // The format is identical to SGR: ESC [ < button ; px ; py M/m - // where px and py are 1-based pixel positions. - static const til::point testPixelCoords[] = { - { 0, 0 }, - { 5, 10 }, - { 100, 200 }, - { 1920, 1080 }, + // SGR-Pixel mode (1016) uses virtual pixel coordinates with a 10x20 + // cell size (matching the DEC VT340 convention). The float cell position + // is multiplied by 10 (x) or 20 (y) and reported 1-based in SGR format. + struct TestCase + { + float viewportX; + float viewportY; + const wchar_t* expectedOutput; }; - static const wchar_t* testPixelOutput[] = { - L"\x1b[<%d;1;1M", - L"\x1b[<%d;6;11M", - L"\x1b[<%d;101;201M", - L"\x1b[<%d;1921;1081M", + static const TestCase testCases[] = { + // Top-left corner of cell (0,0): virtual pixel (0,0) -> 1-based (1,1) + { 0.0f, 0.0f, L"\x1b[<%d;1;1M" }, + // Center of cell (0,0): virtual pixel (5,10) -> 1-based (6,11) + { 0.5f, 0.5f, L"\x1b[<%d;6;11M" }, + // Cell (1,1) top-left: virtual pixel (10,20) -> 1-based (11,21) + { 1.0f, 1.0f, L"\x1b[<%d;11;21M" }, + // Cell (10,5) with sub-cell offset: virtual pixel (103,105) -> 1-based (104,106) + { 10.3f, 5.25f, L"\x1b[<%d;104;106M" }, }; // Test with AnyEventMouseTracking to cover all button types including hovers mouseInput.SetInputMode(TerminalInput::Mode::AnyEventMouseTracking, true); - for (auto i = 0; i < ARRAYSIZE(testPixelCoords); i++) + for (const auto& tc : testCases) { - const auto pixelCoord = testPixelCoords[i]; - // Cell position doesn't matter for SGR-Pixel output; pixel position is used. - const til::point cellCoord{ pixelCoord.x / 8, pixelCoord.y / 16 }; // approximate cell coords - - const auto expected = BuildSGRTestOutput(testPixelOutput[i], uiButton, sModifierKeystate, sScrollDelta); + const auto expected = BuildSGRTestOutput(tc.expectedOutput, uiButton, sModifierKeystate, sScrollDelta); VERIFY_ARE_EQUAL(expected, - mouseInput.HandleMouse(cellCoord, + mouseInput.HandleMouse(tc.viewportX, + tc.viewportY, uiButton, sModifierKeystate, sScrollDelta, - {}, - pixelCoord), - NoThrowString().Format(L"pixel(x,y)=(%d,%d)", pixelCoord.x, pixelCoord.y)); + {}), + NoThrowString().Format(L"viewport(%.1f,%.1f)", tc.viewportX, tc.viewportY)); } // Verify mutual exclusivity: enabling SgrPixelMouseEncoding should disable SgrMouseEncoding @@ -576,7 +586,7 @@ class MouseInputTest VERIFY_SUCCEEDED_RETURN(TestData::TryGetValue(L"sScrollDelta", iScrollDelta)); auto sScrollDelta = (short)(iScrollDelta); - VERIFY_ARE_EQUAL(TerminalInput::MakeUnhandled(), mouseInput.HandleMouse({ 0, 0 }, uiButton, sModifierKeystate, sScrollDelta, {})); + VERIFY_ARE_EQUAL(TerminalInput::MakeUnhandled(), mouseInput.HandleMouse(0, 0, uiButton, sModifierKeystate, sScrollDelta, {})); // Default Tracking, Default Encoding mouseInput.SetInputMode(TerminalInput::Mode::DefaultMouseTracking, true); @@ -593,7 +603,8 @@ class MouseInputTest // validate translation VERIFY_ARE_EQUAL(expected, - mouseInput.HandleMouse(Coord, + mouseInput.HandleMouse(static_cast(Coord.x), + static_cast(Coord.y), uiButton, sModifierKeystate, sScrollDelta, @@ -616,7 +627,8 @@ class MouseInputTest // validate translation VERIFY_ARE_EQUAL(expected, - mouseInput.HandleMouse(Coord, + mouseInput.HandleMouse(static_cast(Coord.x), + static_cast(Coord.y), uiButton, sModifierKeystate, sScrollDelta, @@ -633,7 +645,8 @@ class MouseInputTest // validate translation VERIFY_ARE_EQUAL(expected, - mouseInput.HandleMouse(Coord, + mouseInput.HandleMouse(static_cast(Coord.x), + static_cast(Coord.y), uiButton, sModifierKeystate, sScrollDelta, @@ -653,40 +666,40 @@ class MouseInputTest mouseInput.SetInputMode(TerminalInput::Mode::AlternateScroll, true); Log::Comment(L"Test mouse wheel scrolling up"); - VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1B[A"), mouseInput.HandleMouse({ 0, 0 }, WM_MOUSEWHEEL, noModifierKeys, WHEEL_DELTA, {})); + VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1B[A"), mouseInput.HandleMouse(0, 0, WM_MOUSEWHEEL, noModifierKeys, WHEEL_DELTA, {})); Log::Comment(L"Test mouse wheel scrolling down"); - VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1B[B"), mouseInput.HandleMouse({ 0, 0 }, WM_MOUSEWHEEL, noModifierKeys, -WHEEL_DELTA, {})); + VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1B[B"), mouseInput.HandleMouse(0, 0, WM_MOUSEWHEEL, noModifierKeys, -WHEEL_DELTA, {})); Log::Comment(L"Test mouse wheel scrolling right"); - VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1B[C"), mouseInput.HandleMouse({ 0, 0 }, WM_MOUSEHWHEEL, noModifierKeys, WHEEL_DELTA, {})); + VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1B[C"), mouseInput.HandleMouse(0, 0, WM_MOUSEHWHEEL, noModifierKeys, WHEEL_DELTA, {})); Log::Comment(L"Test mouse wheel scrolling left"); - VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1B[D"), mouseInput.HandleMouse({ 0, 0 }, WM_MOUSEHWHEEL, noModifierKeys, -WHEEL_DELTA, {})); + VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1B[D"), mouseInput.HandleMouse(0, 0, WM_MOUSEHWHEEL, noModifierKeys, -WHEEL_DELTA, {})); Log::Comment(L"Enable cursor keys mode"); mouseInput.SetInputMode(TerminalInput::Mode::CursorKey, true); Log::Comment(L"Test mouse wheel scrolling up"); - VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1BOA"), mouseInput.HandleMouse({ 0, 0 }, WM_MOUSEWHEEL, noModifierKeys, WHEEL_DELTA, {})); + VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1BOA"), mouseInput.HandleMouse(0, 0, WM_MOUSEWHEEL, noModifierKeys, WHEEL_DELTA, {})); Log::Comment(L"Test mouse wheel scrolling down"); - VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1BOB"), mouseInput.HandleMouse({ 0, 0 }, WM_MOUSEWHEEL, noModifierKeys, -WHEEL_DELTA, {})); + VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1BOB"), mouseInput.HandleMouse(0, 0, WM_MOUSEWHEEL, noModifierKeys, -WHEEL_DELTA, {})); Log::Comment(L"Test mouse wheel scrolling right"); - VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1BOC"), mouseInput.HandleMouse({ 0, 0 }, WM_MOUSEHWHEEL, noModifierKeys, WHEEL_DELTA, {})); + VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1BOC"), mouseInput.HandleMouse(0, 0, WM_MOUSEHWHEEL, noModifierKeys, WHEEL_DELTA, {})); Log::Comment(L"Test mouse wheel scrolling left"); - VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1BOD"), mouseInput.HandleMouse({ 0, 0 }, WM_MOUSEHWHEEL, noModifierKeys, -WHEEL_DELTA, {})); + VERIFY_ARE_EQUAL(TerminalInput::MakeOutput(L"\x1BOD"), mouseInput.HandleMouse(0, 0, WM_MOUSEHWHEEL, noModifierKeys, -WHEEL_DELTA, {})); Log::Comment(L"Confirm no effect when scroll mode is disabled"); mouseInput.UseAlternateScreenBuffer(); mouseInput.SetInputMode(TerminalInput::Mode::AlternateScroll, false); - VERIFY_ARE_EQUAL(TerminalInput::MakeUnhandled(), mouseInput.HandleMouse({ 0, 0 }, WM_MOUSEWHEEL, noModifierKeys, WHEEL_DELTA, {})); + VERIFY_ARE_EQUAL(TerminalInput::MakeUnhandled(), mouseInput.HandleMouse(0, 0, WM_MOUSEWHEEL, noModifierKeys, WHEEL_DELTA, {})); Log::Comment(L"Confirm no effect when using the main buffer"); mouseInput.UseMainScreenBuffer(); mouseInput.SetInputMode(TerminalInput::Mode::AlternateScroll, true); - VERIFY_ARE_EQUAL(TerminalInput::MakeUnhandled(), mouseInput.HandleMouse({ 0, 0 }, WM_MOUSEWHEEL, noModifierKeys, WHEEL_DELTA, {})); + VERIFY_ARE_EQUAL(TerminalInput::MakeUnhandled(), mouseInput.HandleMouse(0, 0, WM_MOUSEWHEEL, noModifierKeys, WHEEL_DELTA, {})); } }; diff --git a/src/terminal/adapter/ut_adapter/adapterTest.cpp b/src/terminal/adapter/ut_adapter/adapterTest.cpp index 9edc155e999..b742697f8d7 100644 --- a/src/terminal/adapter/ut_adapter/adapterTest.cpp +++ b/src/terminal/adapter/ut_adapter/adapterTest.cpp @@ -97,6 +97,11 @@ class TestGetSet final : public ITerminalApi return false; } + bool IsConhost() const override + { + return false; + } + void SetSystemMode(const Mode mode, const bool enabled) { Log::Comment(L"SetSystemMode MOCK called..."); diff --git a/src/terminal/input/mouseInput.cpp b/src/terminal/input/mouseInput.cpp index b53e5937dfb..37f89cabc9e 100644 --- a/src/terminal/input/mouseInput.cpp +++ b/src/terminal/input/mouseInput.cpp @@ -292,8 +292,15 @@ bool TerminalInput::IsTrackingMouseInput() const noexcept // Return value: // - Returns an empty optional if we didn't handle the mouse event and the caller can opt to handle it in some other way. // - Returns a string if we successfully translated it into a VT input sequence. -TerminalInput::OutputType TerminalInput::HandleMouse(const til::point position, const unsigned int button, const short modifierKeyState, const short delta, const MouseButtonState state, const til::point pixelPosition) +TerminalInput::OutputType TerminalInput::HandleMouse(const float viewportX, const float viewportY, const unsigned int button, const short modifierKeyState, const short delta, const MouseButtonState state) { + // Truncate float cell coordinates to integer for cell-based modes and + // for tracking the last position (used to detect same-cell moves). + const til::point position{ + static_cast(viewportX), + static_cast(viewportY) + }; + if (Utils::Sign(delta) != Utils::Sign(_mouseInputState.accumulatedDelta)) { // This works for wheel and non-wheel events and transitioning between wheel/non-wheel. @@ -368,9 +375,14 @@ TerminalInput::OutputType TerminalInput::HandleMouse(const til::point position, else if (_inputMode.test(Mode::SgrPixelMouseEncoding)) { // SGR-Pixel mode (1016) uses the same format as SGR (1006), - // but with pixel coordinates instead of cell coordinates. + // but with virtual pixel coordinates (10x20 per cell) instead of cell coordinates. + // This matches the DEC VT340 character cell size convention. + const til::point virtualPixelPos{ + static_cast(viewportX * 10), + static_cast(viewportY * 20) + }; const auto effectiveButton = physicalButtonPressed ? realButton : button; - return _GenerateSGRSequence(pixelPosition, effectiveButton, _isButtonUp(button), isHover, modifierKeyState, delta); + return _GenerateSGRSequence(virtualPixelPos, effectiveButton, _isButtonUp(button), isHover, modifierKeyState, delta); } else if (_inputMode.test(Mode::SgrMouseEncoding)) { diff --git a/src/terminal/input/terminalInput.hpp b/src/terminal/input/terminalInput.hpp index d1bb65ce424..e612eba1828 100644 --- a/src/terminal/input/terminalInput.hpp +++ b/src/terminal/input/terminalInput.hpp @@ -22,7 +22,7 @@ namespace Microsoft::Console::VirtualTerminal [[nodiscard]] static OutputType MakeOutput(const std::wstring_view& str); [[nodiscard]] OutputType HandleKey(const INPUT_RECORD& pInEvent); [[nodiscard]] OutputType HandleFocus(bool focused) const; - [[nodiscard]] OutputType HandleMouse(til::point position, unsigned int button, short modifierKeyState, short delta, MouseButtonState state, til::point pixelPosition = {}); + [[nodiscard]] OutputType HandleMouse(float viewportX, float viewportY, unsigned int button, short modifierKeyState, short delta, MouseButtonState state); enum class Mode : size_t {