Skip to content

Add SGR-Pixels mouse mode (1016) #19949

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sebgod wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Add SGR-Pixels mouse mode (1016) #19949

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b3bf440
Add SGR-Pixels mouse mode (1016) for sub-cell precision mouse tracking
sebgod Mar 6, 2026
a46bc1f
Address PR review feedback for SGR-Pixels mouse mode (1016)
sebgod Mar 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/cascadia/TerminalControl/ControlCore.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -712,7 +712,8 @@ namespace winrt::Microsoft::Terminal::Control::implementation
return false;
}

bool ControlCore::SendMouseEvent(const til::point viewportPos,
bool ControlCore::SendMouseEvent(const float viewportX,
const float viewportY,
const unsigned int uiButton,
const ControlKeyStates states,
const short wheelDelta,
Expand All @@ -721,7 +722,7 @@ namespace winrt::Microsoft::Terminal::Control::implementation
TerminalInput::OutputType out;
{
const auto lock = _terminal->LockForReading();
out = _terminal->SendMouseEvent(viewportPos, uiButton, states, wheelDelta, state);
out = _terminal->SendMouseEvent(viewportX, viewportY, uiButton, states, wheelDelta, state);
}
if (out)
{
Expand Down
3 changes: 2 additions & 1 deletion src/cascadia/TerminalControl/ControlCore.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,8 @@ namespace winrt::Microsoft::Terminal::Control::implementation
bool SendCharEvent(const wchar_t ch,
const WORD scanCode,
const ::Microsoft::Terminal::Core::ControlKeyStates modifiers);
bool SendMouseEvent(const til::point viewportPos,
bool SendMouseEvent(float viewportX,
float viewportY,
const unsigned int uiButton,
const ::Microsoft::Terminal::Core::ControlKeyStates states,
const short wheelDelta,
Expand Down
31 changes: 24 additions & 7 deletions src/cascadia/TerminalControl/ControlInteractivity.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,8 @@ namespace winrt::Microsoft::Terminal::Control::implementation
}
else if (_canSendVTMouseInput(modifiers))
{
_sendMouseEventHelper(terminalPosition, pointerUpdateKind, modifiers, 0, buttonState);
const auto [termX, termY] = _getTerminalPositionF(til::point{ pixelPosition });
_sendMouseEventHelper(termX, termY, pointerUpdateKind, modifiers, 0, buttonState);
}
else if (WI_IsFlagSet(buttonState, MouseButtonState::IsLeftButtonDown))
{
Expand Down Expand Up @@ -346,7 +347,8 @@ namespace winrt::Microsoft::Terminal::Control::implementation
// Short-circuit isReadOnly check to avoid warning dialog
if (focused && !_core->IsInReadOnlyMode() && _canSendVTMouseInput(modifiers))
{
_sendMouseEventHelper(terminalPosition, pointerUpdateKind, modifiers, 0, buttonState);
const auto [termX, termY] = _getTerminalPositionF(til::point{ pixelPosition });
_sendMouseEventHelper(termX, termY, pointerUpdateKind, modifiers, 0, buttonState);
handledCompletely = true;
}
// GH#4603 - don't modify the selection if the pointer press didn't
Expand Down Expand Up @@ -445,7 +447,8 @@ namespace winrt::Microsoft::Terminal::Control::implementation
// Short-circuit isReadOnly check to avoid warning dialog
if (!_core->IsInReadOnlyMode() && _canSendVTMouseInput(modifiers))
{
_sendMouseEventHelper(terminalPosition, pointerUpdateKind, modifiers, 0, buttonState);
const auto [termX, termY] = _getTerminalPositionF(til::point{ pixelPosition });
_sendMouseEventHelper(termX, termY, pointerUpdateKind, modifiers, 0, buttonState);
return;
}

Expand Down Expand Up @@ -504,7 +507,9 @@ namespace winrt::Microsoft::Terminal::Control::implementation
// here with a PointerPoint. However, as of #979, we don't have a
// PointerPoint to work with. So, we're just going to do a
// mousewheel event manually
return _sendMouseEventHelper(terminalPosition,
const auto [termX, termY] = _getTerminalPositionF(til::point{ pixelPosition });
return _sendMouseEventHelper(termX,
termY,
delta.Y != 0 ? WM_MOUSEWHEEL : WM_MOUSEHWHEEL,
modifiers,
::base::saturated_cast<short>(delta.Y != 0 ? delta.Y : delta.X),
Expand Down Expand Up @@ -712,16 +717,28 @@ namespace winrt::Microsoft::Terminal::Control::implementation
return pixelPosition / fontSize;
}

bool ControlInteractivity::_sendMouseEventHelper(const til::point terminalPosition,
std::pair<float, float> ControlInteractivity::_getTerminalPositionF(const til::point pixelPosition)
{
// Get the size of the font, which is in pixels.
// Use float division to preserve sub-cell precision for SGR-Pixels mode.
const auto fontSize{ _core->GetFont().GetSize() };
return {
static_cast<float>(pixelPosition.x) / fontSize.width,
static_cast<float>(pixelPosition.y) / fontSize.height
};
}

bool ControlInteractivity::_sendMouseEventHelper(const float terminalX,
const float terminalY,
const unsigned int pointerUpdateKind,
const ::Microsoft::Terminal::Core::ControlKeyStates modifiers,
const SHORT wheelDelta,
Control::MouseButtonState buttonState)
{
const auto adjustment = _core->BufferHeight() - _core->ScrollOffset() - _core->ViewHeight();
// If the click happened outside the active region, core should get a chance to filter it out or clamp it.
const auto adjustedY = terminalPosition.y - adjustment;
return _core->SendMouseEvent({ terminalPosition.x, adjustedY }, pointerUpdateKind, modifiers, wheelDelta, toInternalMouseState(buttonState));
const auto adjustedY = terminalY - static_cast<float>(adjustment);
return _core->SendMouseEvent(terminalX, adjustedY, pointerUpdateKind, modifiers, wheelDelta, toInternalMouseState(buttonState));
}

// Method Description:
Expand Down
4 changes: 3 additions & 1 deletion src/cascadia/TerminalControl/ControlInteractivity.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,8 +156,10 @@ namespace winrt::Microsoft::Terminal::Control::implementation
bool _shouldSendAlternateScroll(const ::Microsoft::Terminal::Core::ControlKeyStates modifiers, const Core::Point delta);

til::point _getTerminalPosition(const til::point pixelPosition, bool roundToNearestCell);
std::pair<float, float> _getTerminalPositionF(const til::point pixelPosition);

bool _sendMouseEventHelper(const til::point terminalPosition,
bool _sendMouseEventHelper(float terminalX,
float terminalY,
const unsigned int pointerUpdateKind,
const ::Microsoft::Terminal::Core::ControlKeyStates modifiers,
const SHORT wheelDelta,
Expand Down
4 changes: 3 additions & 1 deletion src/cascadia/TerminalControl/HwndTerminal.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -826,7 +826,9 @@ try
TerminalInput::OutputType out;
{
const auto lock = _terminal->LockForReading();
out = _terminal->SendMouseEvent(cursorPosition / fontSize, uMsg, getControlKeyState(), wheelDelta, state);
out = _terminal->SendMouseEvent(static_cast<float>(cursorPosition.x) / fontSize.width,
static_cast<float>(cursorPosition.y) / fontSize.height,
uMsg, getControlKeyState(), wheelDelta, state);
}
if (out)
{
Expand Down
2 changes: 1 addition & 1 deletion src/cascadia/TerminalCore/ITerminalInput.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ namespace Microsoft::Terminal::Core
ITerminalInput& operator=(ITerminalInput&&) = default;

[[nodiscard]] virtual ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendKeyEvent(const WORD vkey, const WORD scanCode, const ControlKeyStates states, const bool keyDown) = 0;
[[nodiscard]] virtual ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendMouseEvent(const til::point viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state) = 0;
[[nodiscard]] virtual ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendMouseEvent(float viewportX, float viewportY, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state) = 0;
[[nodiscard]] virtual ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendCharEvent(const wchar_t ch, const WORD scanCode, const ControlKeyStates states) = 0;
[[nodiscard]] virtual ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType FocusChanged(const bool focused) = 0;

Expand Down
8 changes: 5 additions & 3 deletions src/cascadia/TerminalCore/Terminal.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -679,14 +679,16 @@ TerminalInput::OutputType Terminal::SendKeyEvent(const WORD vkey,
// Return Value:
// - true if we translated the key event, and it should not be processed any further.
// - false if we did not translate the key, and it should be processed into a character.
TerminalInput::OutputType Terminal::SendMouseEvent(til::point viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const TerminalInput::MouseButtonState state)
TerminalInput::OutputType Terminal::SendMouseEvent(float viewportX, float viewportY, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const TerminalInput::MouseButtonState state)
{
// GH#6401: VT applications should be able to receive mouse events from outside the
// terminal buffer. This is likely to happen when the user drags the cursor offscreen.
// We shouldn't throw away perfectly good events when they're offscreen, so we just
// clamp them to be within the range [(0, 0), (W, H)].
_GetMutableViewport().ToOrigin().Clamp(viewportPos);
return _getTerminalInput().HandleMouse(viewportPos, uiButton, GET_KEYSTATE_WPARAM(states.Value()), wheelDelta, state);
const auto viewport = _GetMutableViewport().ToOrigin();
viewportX = std::clamp(viewportX, 0.0f, static_cast<float>(viewport.Width() - 1));
viewportY = std::clamp(viewportY, 0.0f, static_cast<float>(viewport.Height() - 1));
Copy link
Member

@lhecker lhecker Mar 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe this is wrong. Let's say the viewport is 1*1 cell large. This code would clamp the float value to between 0 and 0, permitting no fractional positions within that one cell.

Additionally, the clamping isn't quite identical to what it was before. The ToOrigin().Clamp() would essentially intersect the viewport-relative mouse coordinates with the scrollbar-relative buffer viewport. Thus, the coordinates would get correctly clamped.

When I'm writing this, I realize, however, that the old code seemingly doesn't shift the cursor-coordinates according to the scroll offset (= when the user has scrolled slightly upward, clicking into the partially visible VT viewport should translate the mouse coordinates back down the scroll offset). I think I'll have to check out this PR locally.

Copy link
Author

@sebgod sebgod Mar 12, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

w.r.t to scrolling and just to clarify the coordinate system: What this whole change really means in terms of VT command output:
Cell size: 10x20
SGR 1006 reports mouse at 7,11 (row, col)
SGR 1016 would report something like 75,230 (pixels), in case when the mouse pointer is exactly in the centre, correct?

Screenshot 2026-03-12 105243

This is what my small test program spits out at least: https://gist.github.com/sebgod/9b9a38a7d934907584978c9bafdd0489

return _getTerminalInput().HandleMouse(viewportX, viewportY, uiButton, GET_KEYSTATE_WPARAM(states.Value()), wheelDelta, state);
}

// Method Description:
Expand Down
3 changes: 2 additions & 1 deletion src/cascadia/TerminalCore/Terminal.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ class Microsoft::Terminal::Core::Terminal final :
void UseMainScreenBuffer() override;

bool IsVtInputEnabled() const noexcept override;
bool IsConhost() const noexcept override;
void NotifyBufferRotation(const int delta) override;
void NotifyShellIntegrationMark() override;

Expand All @@ -171,7 +172,7 @@ class Microsoft::Terminal::Core::Terminal final :
#pragma region ITerminalInput
// These methods are defined in Terminal.cpp
[[nodiscard]] ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendKeyEvent(const WORD vkey, const WORD scanCode, const Microsoft::Terminal::Core::ControlKeyStates states, const bool keyDown) override;
[[nodiscard]] ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendMouseEvent(const til::point viewportPos, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state) override;
[[nodiscard]] ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendMouseEvent(float viewportX, float viewportY, const unsigned int uiButton, const ControlKeyStates states, const short wheelDelta, const Microsoft::Console::VirtualTerminal::TerminalInput::MouseButtonState state) override;
[[nodiscard]] ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType SendCharEvent(const wchar_t ch, const WORD scanCode, const ControlKeyStates states) override;
[[nodiscard]] ::Microsoft::Console::VirtualTerminal::TerminalInput::OutputType FocusChanged(const bool focused) override;

Expand Down
5 changes: 5 additions & 0 deletions src/cascadia/TerminalCore/TerminalApi.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,11 @@ bool Terminal::IsVtInputEnabled() const noexcept
return false;
}

bool Terminal::IsConhost() const noexcept
{
return false;
}

void Terminal::InvokeCompletions(std::wstring_view menuJson, unsigned int replaceLength)
{
if (_pfnCompletionsChanged)
Expand Down
2 changes: 1 addition & 1 deletion src/host/inputBuffer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -623,7 +623,7 @@ bool InputBuffer::WriteMouseEvent(til::point position, const unsigned int button
const auto& gci = ServiceLocator::LocateGlobals().getConsoleInformation();
gci.GetActiveOutputBuffer().GetViewport().ToOrigin().Clamp(position);

if (const auto out = _termInput.HandleMouse(position, button, keyState, wheelDelta, state))
if (const auto out = _termInput.HandleMouse(static_cast<float>(position.x), static_cast<float>(position.y), button, keyState, wheelDelta, state))
{
_writeString(*out);
return true;
Expand Down
5 changes: 5 additions & 0 deletions src/host/outputStream.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,11 @@ bool ConhostInternalGetSet::IsVtInputEnabled() const
return _io.GetActiveInputBuffer()->IsInVirtualTerminalInputMode();
}

bool ConhostInternalGetSet::IsConhost() const
{
return true;
}

// Routine Description:
// - Implements conhost-specific behavior when the buffer is rotated.
// Arguments:
Expand Down
1 change: 1 addition & 0 deletions src/host/outputStream.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ class ConhostInternalGetSet final : public Microsoft::Console::VirtualTerminal::
void PlayMidiNote(const int noteNumber, const int velocity, const std::chrono::microseconds duration) override;

bool IsVtInputEnabled() const override;
bool IsConhost() const override;

void NotifyBufferRotation(const int delta) override;
void NotifyShellIntegrationMark() override;
Expand Down
1 change: 1 addition & 0 deletions src/terminal/adapter/DispatchTypes.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -542,6 +542,7 @@ namespace Microsoft::Console::VirtualTerminal::DispatchTypes
UTF8_EXTENDED_MODE = DECPrivateMode(1005),
SGR_EXTENDED_MODE = DECPrivateMode(1006),
ALTERNATE_SCROLL = DECPrivateMode(1007),
SGR_PIXEL_MODE = DECPrivateMode(1016),
ASB_AlternateScreenBuffer = DECPrivateMode(1049),
XTERM_BracketedPasteMode = DECPrivateMode(2004),
SO_SynchronizedOutput = DECPrivateMode(2026),
Expand Down
1 change: 1 addition & 0 deletions src/terminal/adapter/ITerminalApi.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ namespace Microsoft::Console::VirtualTerminal
virtual void SetViewportPosition(const til::point position) = 0;

virtual bool IsVtInputEnabled() const = 0;
virtual bool IsConhost() const = 0;

enum class Mode : size_t
{
Expand Down
20 changes: 20 additions & 0 deletions src/terminal/adapter/adaptDispatch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1859,6 +1859,16 @@ void AdaptDispatch::_ModeParamsHelper(const DispatchTypes::ModeParams param, con
case DispatchTypes::ModeParams::SGR_EXTENDED_MODE:
_terminalInput.SetInputMode(TerminalInput::Mode::SgrMouseEncoding, enable);
break;
case DispatchTypes::ModeParams::SGR_PIXEL_MODE:
// SGR-Pixel mode requires real sub-cell pixel coordinates. It's not
// supported in direct conhost because the Win32 console input API only
// provides character-cell coordinates. In ConPTY mode (IsVtInputEnabled),
// the request is passed through to the Terminal frontend which has pixel data.
if (!_api.IsConhost() || _api.IsVtInputEnabled())
Comment on lines +1863 to +1867
Copy link
Member

@lhecker lhecker Mar 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should be relatively easy to add conhost support in this PR actually, even if we have to refactor some code to achieve this. conhost is not limited by what the console API supports, after all - those are separate concerns (internal VS external API).

{
_terminalInput.SetInputMode(TerminalInput::Mode::SgrPixelMouseEncoding, enable);
}
break;
case DispatchTypes::ModeParams::FOCUS_EVENT_MODE:
_terminalInput.SetInputMode(TerminalInput::Mode::FocusEvent, enable);
// ConPTY always wants to know about focus events, so let it know that it needs to re-enable this mode.
Expand Down Expand Up @@ -2009,6 +2019,16 @@ void AdaptDispatch::RequestMode(const DispatchTypes::ModeParams param)
case DispatchTypes::ModeParams::SGR_EXTENDED_MODE:
state = mapTemp(_terminalInput.GetInputMode(TerminalInput::Mode::SgrMouseEncoding));
break;
case DispatchTypes::ModeParams::SGR_PIXEL_MODE:
if (_api.IsConhost() && !_api.IsVtInputEnabled())
{
state = DispatchTypes::DECRPM_PermanentlyDisabled;
}
else
{
state = mapTemp(_terminalInput.GetInputMode(TerminalInput::Mode::SgrPixelMouseEncoding));
}
break;
case DispatchTypes::ModeParams::FOCUS_EVENT_MODE:
state = mapTemp(_terminalInput.GetInputMode(TerminalInput::Mode::FocusEvent));
break;
Expand Down
Loading