From d9e847930504f2ff96f1b061e56c6c1950b2c389 Mon Sep 17 00:00:00 2001 From: Paul Kocialkowski Date: Mon, 24 Nov 2025 19:24:52 +0100 Subject: [PATCH 1/6] slides: graphics: Rework section/subsection splitting Signed-off-by: Paul Kocialkowski --- slides/graphics-hardware/graphics-hardware.tex | 10 +++++++--- slides/graphics-software/graphics-software.tex | 18 +++++++----------- slides/graphics-theory/graphics-theory.tex | 6 +++++- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/slides/graphics-hardware/graphics-hardware.tex b/slides/graphics-hardware/graphics-hardware.tex index 7ccd1fe075..4aa0368ec7 100644 --- a/slides/graphics-hardware/graphics-hardware.tex +++ b/slides/graphics-hardware/graphics-hardware.tex @@ -1,6 +1,6 @@ \section{Hardware Aspects} -\subsection{Pipeline Components Overview and Generalities} +\subsection{Overview} \begin{frame}{Technological types of graphics hardware implementations} \begin{itemize} @@ -137,6 +137,8 @@ \subsection{Pipeline Components Overview and Generalities} \end{itemize}~ \end{frame} +\subsection{Pipelines} + \begin{frame}{I/O with graphics hardware, pipelines} \begin{itemize} \item Graphics hardware is \textbf{I/O-based} and interacts with pixel data @@ -196,7 +198,7 @@ \subsection{Pipeline Components Overview and Generalities} \end{itemize} \end{frame} -\subsection{Display Hardware Specifics} +\subsection{Display Devices} \begin{frame}{Visual display technologies generalities} \begin{itemize} @@ -349,6 +351,8 @@ \subsection{Display Hardware Specifics} \end{center} \end{frame} +\subsection{Display Interfaces} + \begin{frame}{Display panels refreshing} \begin{itemize} \item Most display panel technologies need frequent pixel updates: \textbf{refreshing} @@ -691,7 +695,7 @@ \subsection{Display Hardware Specifics} \end{center} \end{frame} -\subsection{Rendering Hardware Specifics} +\subsection{Render Accelerators} \begin{frame}{Digital Signal Processors} \begin{itemize} diff --git a/slides/graphics-software/graphics-software.tex b/slides/graphics-software/graphics-software.tex index e3a948ffc2..65a58ff653 100644 --- a/slides/graphics-software/graphics-software.tex +++ b/slides/graphics-software/graphics-software.tex @@ -1,6 +1,6 @@ \section{Software Aspects} -\subsection{Display Stack Overview} +\subsection{Stack Overview} \begin{frame}{System-agnostic overview: kernel} \begin{itemize} @@ -243,7 +243,7 @@ \subsection{Display Stack Overview} \end{minipage} \end{frame} -\subsection{TTY Kernel Aspects} +\subsection{Linux Kernel: TTY} \begin{frame}{Linux TTY subsystem introduction} \begin{itemize} @@ -319,7 +319,7 @@ \subsection{TTY Kernel Aspects} \end{itemize} \end{frame} -\subsection{Framebuffer Device Kernel Aspects} +\subsection{Linux Kernel: Framebuffer Device} \begin{frame}{Fbdev overview} \begin{itemize} @@ -391,7 +391,7 @@ \subsection{Framebuffer Device Kernel Aspects} \end{center} \end{frame} -\subsection{DRM Kernel Aspects} +\subsection{Linux Kernel: DRM} \begin{frame}{DRM devices} \begin{itemize} @@ -950,8 +950,6 @@ \subsection{DRM Kernel Aspects} \end{itemize} \end{frame} -\subsection{DRM Userspace Aspects} - \begin{frame}{libdrm wrapper} \begin{itemize} \item Userspace access to DRM devices is wrapped with \textbf{libdrm} @@ -969,7 +967,7 @@ \subsection{DRM Userspace Aspects} \end{center} \end{frame} -\subsection{X Window Userspace Aspects} +\subsection{Userspace: X Window} \begin{frame}{X11 protocol and architecture} \begin{itemize} @@ -1219,9 +1217,7 @@ \subsection{X Window Userspace Aspects} \end{itemize} \end{frame} -\subsection{Wayland Userspace Aspects} - -\subsubsection{Wayland} +\subsection{Userspace: Wayland} \begin{frame}{Wayland overview and paradigm} \begin{itemize} @@ -1454,7 +1450,7 @@ \subsubsection{Wayland} \end{itemize} \end{frame} -\subsection{Mesa 3D Userspace Aspects} +\subsection{Userspace: Mesa 3D} \begin{frame}[t]{Standardized 3D rendering APIs: OpenGL} \begin{center} diff --git a/slides/graphics-theory/graphics-theory.tex b/slides/graphics-theory/graphics-theory.tex index e97e9ea1a5..83bb269e21 100644 --- a/slides/graphics-theory/graphics-theory.tex +++ b/slides/graphics-theory/graphics-theory.tex @@ -1,6 +1,6 @@ \section{Base Theory and Concepts About Graphics} -\subsection{Image and Color Representation} +\subsection{Image Representation} \begin{frame}{Light, pixels and pictures} \begin{minipage}{0.75\textwidth} @@ -163,6 +163,8 @@ \subsection{Image and Color Representation} \end{minipage} \end{frame} +\subsection{Color Representation} + \begin{frame}{Light and color representation} \begin{minipage}[c]{0.75\textwidth} \begin{itemize} @@ -363,6 +365,8 @@ \subsection{Image and Color Representation} \end{center} \end{frame} +\subsection{Layout and Formats} + \begin{frame}{Frame size and chroma sub-sampling} \begin{itemize} \item Digital pictures easily take up a lot of space (more so for videos) From f6a98d66696af48293fe35d04d6ce5b958d4461e Mon Sep 17 00:00:00 2001 From: Paul Kocialkowski Date: Mon, 24 Nov 2025 21:03:37 +0100 Subject: [PATCH 2/6] slides: graphics: Split files into subsections Signed-off-by: Paul Kocialkowski --- mk/graphics.mk | 21 +- .../crt-color.png | Bin .../display-panel.jpg | Bin .../e-reader.jpg | Bin .../epd-detail.jpg | Bin .../graphics-hardware-display-devices.tex | 152 ++ .../lcd-ips-shape.jpg | Bin .../oled-display.jpg | Bin .../pixel-array-text.jpg | Bin .../pixel-array.jpg | Bin .../a13-olinuxino-vga.png | Bin .../display-interface-encoders.dia | Bin .../display-timings.dia | Bin .../dp-dvi-bridge.jpg | Bin .../dp-pinout.svg | 0 .../dvi-pinout.svg | 0 .../graphics-hardware-display-interfaces.tex | 343 +++ .../hdmi-pinout.svg | 0 .../timings-table.png | Bin .../vga-pinout.svg | 0 .../ati-hercules-1986.png | Bin .../display-pipe.dia | Bin .../graphics-hardware-overview.tex | 138 ++ .../intel-skylake.jpg | Bin .../graphics-hardware-pipelines.tex | 60 + .../pipeline-complex.dia | Bin .../pipeline-sink-source.dia | Bin .../anisotropic-filtering.png | Bin .../g2d-block.png | Bin .../gpu-pipeline.png | Bin .../graphics-hardware-render.tex | 162 ++ .../mip-map.jpg | Bin .../graphics-hardware-system.tex | 183 ++ .../sunxi-tiled-format.png | Bin .../sunxi-tiled-linear.png | Bin .../sunxi-tiled.png | Bin .../tearing-glitch.jpg | Bin .../graphics-hardware/graphics-hardware.tex | 1043 --------- .../graphics-software-linux-drm.tex | 575 +++++ .../libdrm-userspace.svg | 0 .../graphics-software-linux-fbdev.tex | 71 + .../getty.jpg | Bin .../graphics-software-linux-tty.tex | 75 + .../agnostic-display-stack-overview.dia | Bin .../compositing-gnome-shell.jpg | Bin .../compositing-gnome-terminal.jpg | Bin .../compositing-lollipop.jpg | Bin .../compositing-result.jpg | Bin .../enlightenment-logo.svg | 0 .../gnome-logo.svg | 0 .../graphics-software-stack-overview.tex | 244 +++ .../gtk-logo.svg | 0 .../kde-logo.svg | 0 .../qt-logo.svg | 0 .../sdl-logo.png | Bin .../wayland-logo.svg | 0 .../x11-logo.svg | 0 .../xfce-logo.svg | 0 .../egl-logo.svg | 0 .../graphics-software-userspace-mesa.tex | 452 ++++ .../mesa-dri-gallium.svg | 0 .../mesa-ir.dia | Bin .../opengl-es-logo.svg | 0 .../opengl-logo.svg | 0 .../vulkan-logo.svg | 0 .../graphics-software-userspace-wayland.tex | 232 ++ .../wayland-architecture-roundtrip.png | Bin .../wayland-egl.svg | 0 .../wayland-stack.svg | 0 .../dri-data-flow.png | Bin .../graphics-software-userspace-x.tex | 249 +++ .../x-architecture-roundtrip.png | Bin .../graphics-software/graphics-software.tex | 1904 ----------------- .../barn-yuv.jpg | Bin .../barn.jpg | Bin .../gamut.png | Bin .../graphics-theory-color.tex | 201 ++ .../hsv-diagram.svg | 0 .../pair-of-merops-16-colors-range.jpg | Bin .../pair-of-merops-16-colors.jpg | Bin .../pair-of-merops.jpg | Bin .../rgb-cube.svg | 0 .../aliasing-1d.svg | 0 .../bricks-alias.jpg | Bin .../bricks-fft-rotation.jpg | Bin .../bricks-fft.jpg | Bin .../bricks-rich.jpg | Bin .../bricks-rotation.jpg | Bin .../bricks.jpg | Bin .../ccc-rocket.jpg | Bin .../dot-matrix-display.jpg | Bin .../first-photo.jpg | Bin .../graphics-theory-image.tex | 164 ++ .../moire.jpg | Bin .../graphics-theory-layout-formats.tex | 111 + .../raster-order.svg | 0 .../yuv-sub-sampling.html | 0 .../yuv-sub-sampling.svg | 0 .../diamond-sub-pixel.png | Bin .../graphics-theory-pixel-drawing.tex | 244 +++ .../line-sub-pixel-rendering.svg | 0 .../line-sub-pixel.png | Bin .../polar-coordinates.svg | 0 .../raster-destination.jpg | Bin .../raster-source.jpg | Bin .../box-blur.svg | 0 .../cat-depth-dither.png | Bin .../cat-depth-initial.png | Bin .../cat-depth-low.png | Bin .../chroma-key-blender.jpg | Bin .../gaussian-blur.svg | 0 .../graphics-theory-pixel-operations.tex | 239 +++ .../interpolation.svg | 0 .../linear-filtering.svg | 0 .../porter-duff-compositing.svg | 0 slides/graphics-theory/graphics-theory.tex | 963 --------- slides/graphics-theory/resolution.dia | Bin 1674 -> 0 bytes 117 files changed, 3913 insertions(+), 3913 deletions(-) rename slides/{graphics-hardware => graphics-hardware-display-devices}/crt-color.png (100%) rename slides/{graphics-hardware => graphics-hardware-display-devices}/display-panel.jpg (100%) rename slides/{graphics-hardware => graphics-hardware-display-devices}/e-reader.jpg (100%) rename slides/{graphics-hardware => graphics-hardware-display-devices}/epd-detail.jpg (100%) create mode 100644 slides/graphics-hardware-display-devices/graphics-hardware-display-devices.tex rename slides/{graphics-hardware => graphics-hardware-display-devices}/lcd-ips-shape.jpg (100%) rename slides/{graphics-hardware => graphics-hardware-display-devices}/oled-display.jpg (100%) rename slides/{graphics-hardware => graphics-hardware-display-devices}/pixel-array-text.jpg (100%) rename slides/{graphics-hardware => graphics-hardware-display-devices}/pixel-array.jpg (100%) rename slides/{graphics-hardware => graphics-hardware-display-interfaces}/a13-olinuxino-vga.png (100%) rename slides/{graphics-hardware => graphics-hardware-display-interfaces}/display-interface-encoders.dia (100%) rename slides/{graphics-hardware => graphics-hardware-display-interfaces}/display-timings.dia (100%) rename slides/{graphics-hardware => graphics-hardware-display-interfaces}/dp-dvi-bridge.jpg (100%) rename slides/{graphics-hardware => graphics-hardware-display-interfaces}/dp-pinout.svg (100%) rename slides/{graphics-hardware => graphics-hardware-display-interfaces}/dvi-pinout.svg (100%) create mode 100644 slides/graphics-hardware-display-interfaces/graphics-hardware-display-interfaces.tex rename slides/{graphics-hardware => graphics-hardware-display-interfaces}/hdmi-pinout.svg (100%) rename slides/{graphics-hardware => graphics-hardware-display-interfaces}/timings-table.png (100%) rename slides/{graphics-hardware => graphics-hardware-display-interfaces}/vga-pinout.svg (100%) rename slides/{graphics-hardware => graphics-hardware-overview}/ati-hercules-1986.png (100%) rename slides/{graphics-hardware => graphics-hardware-overview}/display-pipe.dia (100%) create mode 100644 slides/graphics-hardware-overview/graphics-hardware-overview.tex rename slides/{graphics-hardware => graphics-hardware-overview}/intel-skylake.jpg (100%) create mode 100644 slides/graphics-hardware-pipelines/graphics-hardware-pipelines.tex rename slides/{graphics-hardware => graphics-hardware-pipelines}/pipeline-complex.dia (100%) rename slides/{graphics-hardware => graphics-hardware-pipelines}/pipeline-sink-source.dia (100%) rename slides/{graphics-hardware => graphics-hardware-render}/anisotropic-filtering.png (100%) rename slides/{graphics-hardware => graphics-hardware-render}/g2d-block.png (100%) rename slides/{graphics-hardware => graphics-hardware-render}/gpu-pipeline.png (100%) create mode 100644 slides/graphics-hardware-render/graphics-hardware-render.tex rename slides/{graphics-hardware => graphics-hardware-render}/mip-map.jpg (100%) create mode 100644 slides/graphics-hardware-system/graphics-hardware-system.tex rename slides/{graphics-hardware => graphics-hardware-system}/sunxi-tiled-format.png (100%) rename slides/{graphics-hardware => graphics-hardware-system}/sunxi-tiled-linear.png (100%) rename slides/{graphics-hardware => graphics-hardware-system}/sunxi-tiled.png (100%) rename slides/{graphics-hardware => graphics-hardware-system}/tearing-glitch.jpg (100%) delete mode 100644 slides/graphics-hardware/graphics-hardware.tex create mode 100644 slides/graphics-software-linux-drm/graphics-software-linux-drm.tex rename slides/{graphics-software => graphics-software-linux-drm}/libdrm-userspace.svg (100%) create mode 100644 slides/graphics-software-linux-fbdev/graphics-software-linux-fbdev.tex rename slides/{graphics-software => graphics-software-linux-tty}/getty.jpg (100%) create mode 100644 slides/graphics-software-linux-tty/graphics-software-linux-tty.tex rename slides/{graphics-software => graphics-software-stack-overview}/agnostic-display-stack-overview.dia (100%) rename slides/{graphics-software => graphics-software-stack-overview}/compositing-gnome-shell.jpg (100%) rename slides/{graphics-software => graphics-software-stack-overview}/compositing-gnome-terminal.jpg (100%) rename slides/{graphics-software => graphics-software-stack-overview}/compositing-lollipop.jpg (100%) rename slides/{graphics-software => graphics-software-stack-overview}/compositing-result.jpg (100%) rename slides/{graphics-software => graphics-software-stack-overview}/enlightenment-logo.svg (100%) rename slides/{graphics-software => graphics-software-stack-overview}/gnome-logo.svg (100%) create mode 100644 slides/graphics-software-stack-overview/graphics-software-stack-overview.tex rename slides/{graphics-software => graphics-software-stack-overview}/gtk-logo.svg (100%) rename slides/{graphics-software => graphics-software-stack-overview}/kde-logo.svg (100%) rename slides/{graphics-software => graphics-software-stack-overview}/qt-logo.svg (100%) rename slides/{graphics-software => graphics-software-stack-overview}/sdl-logo.png (100%) rename slides/{graphics-software => graphics-software-stack-overview}/wayland-logo.svg (100%) rename slides/{graphics-software => graphics-software-stack-overview}/x11-logo.svg (100%) rename slides/{graphics-software => graphics-software-stack-overview}/xfce-logo.svg (100%) rename slides/{graphics-software => graphics-software-userspace-mesa}/egl-logo.svg (100%) create mode 100644 slides/graphics-software-userspace-mesa/graphics-software-userspace-mesa.tex rename slides/{graphics-software => graphics-software-userspace-mesa}/mesa-dri-gallium.svg (100%) rename slides/{graphics-software => graphics-software-userspace-mesa}/mesa-ir.dia (100%) rename slides/{graphics-software => graphics-software-userspace-mesa}/opengl-es-logo.svg (100%) rename slides/{graphics-software => graphics-software-userspace-mesa}/opengl-logo.svg (100%) rename slides/{graphics-software => graphics-software-userspace-mesa}/vulkan-logo.svg (100%) create mode 100644 slides/graphics-software-userspace-wayland/graphics-software-userspace-wayland.tex rename slides/{graphics-software => graphics-software-userspace-wayland}/wayland-architecture-roundtrip.png (100%) rename slides/{graphics-software => graphics-software-userspace-wayland}/wayland-egl.svg (100%) rename slides/{graphics-software => graphics-software-userspace-wayland}/wayland-stack.svg (100%) rename slides/{graphics-software => graphics-software-userspace-x}/dri-data-flow.png (100%) create mode 100644 slides/graphics-software-userspace-x/graphics-software-userspace-x.tex rename slides/{graphics-software => graphics-software-userspace-x}/x-architecture-roundtrip.png (100%) delete mode 100644 slides/graphics-software/graphics-software.tex rename slides/{graphics-theory => graphics-theory-color}/barn-yuv.jpg (100%) rename slides/{graphics-theory => graphics-theory-color}/barn.jpg (100%) rename slides/{graphics-theory => graphics-theory-color}/gamut.png (100%) create mode 100644 slides/graphics-theory-color/graphics-theory-color.tex rename slides/{graphics-theory => graphics-theory-color}/hsv-diagram.svg (100%) rename slides/{graphics-theory => graphics-theory-color}/pair-of-merops-16-colors-range.jpg (100%) rename slides/{graphics-theory => graphics-theory-color}/pair-of-merops-16-colors.jpg (100%) rename slides/{graphics-theory => graphics-theory-color}/pair-of-merops.jpg (100%) rename slides/{graphics-theory => graphics-theory-color}/rgb-cube.svg (100%) rename slides/{graphics-theory => graphics-theory-image}/aliasing-1d.svg (100%) rename slides/{graphics-theory => graphics-theory-image}/bricks-alias.jpg (100%) rename slides/{graphics-theory => graphics-theory-image}/bricks-fft-rotation.jpg (100%) rename slides/{graphics-theory => graphics-theory-image}/bricks-fft.jpg (100%) rename slides/{graphics-theory => graphics-theory-image}/bricks-rich.jpg (100%) rename slides/{graphics-theory => graphics-theory-image}/bricks-rotation.jpg (100%) rename slides/{graphics-theory => graphics-theory-image}/bricks.jpg (100%) rename slides/{graphics-theory => graphics-theory-image}/ccc-rocket.jpg (100%) rename slides/{graphics-theory => graphics-theory-image}/dot-matrix-display.jpg (100%) rename slides/{graphics-theory => graphics-theory-image}/first-photo.jpg (100%) create mode 100644 slides/graphics-theory-image/graphics-theory-image.tex rename slides/{graphics-theory => graphics-theory-image}/moire.jpg (100%) create mode 100644 slides/graphics-theory-layout-formats/graphics-theory-layout-formats.tex rename slides/{graphics-theory => graphics-theory-layout-formats}/raster-order.svg (100%) rename slides/{graphics-theory => graphics-theory-layout-formats}/yuv-sub-sampling.html (100%) rename slides/{graphics-theory => graphics-theory-layout-formats}/yuv-sub-sampling.svg (100%) rename slides/{graphics-theory => graphics-theory-pixel-drawing}/diamond-sub-pixel.png (100%) create mode 100644 slides/graphics-theory-pixel-drawing/graphics-theory-pixel-drawing.tex rename slides/{graphics-theory => graphics-theory-pixel-drawing}/line-sub-pixel-rendering.svg (100%) rename slides/{graphics-theory => graphics-theory-pixel-drawing}/line-sub-pixel.png (100%) rename slides/{graphics-theory => graphics-theory-pixel-drawing}/polar-coordinates.svg (100%) rename slides/{graphics-theory => graphics-theory-pixel-drawing}/raster-destination.jpg (100%) rename slides/{graphics-theory => graphics-theory-pixel-drawing}/raster-source.jpg (100%) rename slides/{graphics-theory => graphics-theory-pixel-operations}/box-blur.svg (100%) rename slides/{graphics-theory => graphics-theory-pixel-operations}/cat-depth-dither.png (100%) rename slides/{graphics-theory => graphics-theory-pixel-operations}/cat-depth-initial.png (100%) rename slides/{graphics-theory => graphics-theory-pixel-operations}/cat-depth-low.png (100%) rename slides/{graphics-theory => graphics-theory-pixel-operations}/chroma-key-blender.jpg (100%) rename slides/{graphics-theory => graphics-theory-pixel-operations}/gaussian-blur.svg (100%) create mode 100644 slides/graphics-theory-pixel-operations/graphics-theory-pixel-operations.tex rename slides/{graphics-theory => graphics-theory-pixel-operations}/interpolation.svg (100%) rename slides/{graphics-theory => graphics-theory-pixel-operations}/linear-filtering.svg (100%) rename slides/{graphics-theory => graphics-theory-pixel-operations}/porter-duff-compositing.svg (100%) delete mode 100644 slides/graphics-theory/graphics-theory.tex delete mode 100644 slides/graphics-theory/resolution.dia diff --git a/mk/graphics.mk b/mk/graphics.mk index 18298467cc..faf1804632 100644 --- a/mk/graphics.mk +++ b/mk/graphics.mk @@ -2,8 +2,23 @@ GRAPHICS_SLIDES = \ first-slides \ about-us \ course-information \ - graphics-theory \ - graphics-hardware \ - graphics-software \ + graphics-theory-image \ + graphics-theory-color \ + graphics-theory-layout-formats \ + graphics-theory-pixel-drawing \ + graphics-theory-pixel-operations \ + graphics-hardware-overview \ + graphics-hardware-pipelines \ + graphics-hardware-display-devices \ + graphics-hardware-display-interfaces \ + graphics-hardware-render \ + graphics-hardware-system \ + graphics-software-stack-overview \ + graphics-software-linux-tty \ + graphics-software-linux-fbdev \ + graphics-software-linux-drm \ + graphics-software-userspace-x \ + graphics-software-userspace-wayland \ + graphics-software-userspace-mesa \ last-slides \ graphics-extra diff --git a/slides/graphics-hardware/crt-color.png b/slides/graphics-hardware-display-devices/crt-color.png similarity index 100% rename from slides/graphics-hardware/crt-color.png rename to slides/graphics-hardware-display-devices/crt-color.png diff --git a/slides/graphics-hardware/display-panel.jpg b/slides/graphics-hardware-display-devices/display-panel.jpg similarity index 100% rename from slides/graphics-hardware/display-panel.jpg rename to slides/graphics-hardware-display-devices/display-panel.jpg diff --git a/slides/graphics-hardware/e-reader.jpg b/slides/graphics-hardware-display-devices/e-reader.jpg similarity index 100% rename from slides/graphics-hardware/e-reader.jpg rename to slides/graphics-hardware-display-devices/e-reader.jpg diff --git a/slides/graphics-hardware/epd-detail.jpg b/slides/graphics-hardware-display-devices/epd-detail.jpg similarity index 100% rename from slides/graphics-hardware/epd-detail.jpg rename to slides/graphics-hardware-display-devices/epd-detail.jpg diff --git a/slides/graphics-hardware-display-devices/graphics-hardware-display-devices.tex b/slides/graphics-hardware-display-devices/graphics-hardware-display-devices.tex new file mode 100644 index 0000000000..d72b3bb2d2 --- /dev/null +++ b/slides/graphics-hardware-display-devices/graphics-hardware-display-devices.tex @@ -0,0 +1,152 @@ +\subsection{Display Devices} + +\begin{frame}{Visual display technologies generalities} + \begin{itemize} + \item Pixel data is pushed from the display interface to a visible surface\\ + \textit{using a dedicated controller on the display device} + \item Pixels are split into 3 color cells (R-G-B) + \begin{itemize} + \item The human eye naturally merges light from the 3 cells + \end{itemize} + \item Pixel frames are displayed as (physical) arrays of color cells + \item Smooth sequences require at least 24 fps, more is usually best + \end{itemize}~ + + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[height=6.5em]{slides/graphics-hardware-display-devices/pixel-array.jpg}\\ + \textit{\small Pixel color cells on a LCD TN panel} + \end{minipage} + \hfill + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[height=6.5em]{slides/graphics-hardware-display-devices/pixel-array-text.jpg}\\ + \textit{\small A pixel array displaying text} + \end{minipage} +\end{frame} + +\begin{frame}{CRT display technology} + \begin{itemize} + \item Color \textbf{cathode-ray tubes} (CRTs), since the 1950s: + \begin{itemize} + \item Using electron beams to excite a phosphorescent screen + \item Beams are guided by magnetic deflection + \item One beam for each color with increased intensity for increased luminosity + \item High energy consumption + \item High contrast, low response time (\(1-10~\mu s\)) + \item Other issues: monitor size, burn-in (screensavers), remnant magnetic field (degaussing), high voltages and magnetic fields + \end{itemize} + \end{itemize} + + \begin{center} + \includegraphics[height=8em]{slides/graphics-hardware-display-devices/crt-color.png} + \end{center} +\end{frame} + +\begin{frame}{Plasma display panels technology} + \begin{itemize} + \item \textbf{Plasma display panels} (PDPs), since the 1990s-2000s: + \begin{itemize} + \item Using gas cells brought to plasma state to strike light-emitting phosphor + \item Flat array of cells, scales to large surfaces + \item Medium energy consumption (depends on luminance) + \item High contrast, low response time (\(\leq 1~\mu s\)) + \item Other issues: burn-in + \item Gradually being \textbf{deprecated} in favor of other flat-panel technologies + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{LCD display technology} + \begin{itemize} + \item \textbf{Liquid crystal displays} (LCDs) using \textbf{Thin-film-transistors} (TFT): + \begin{itemize} + \item Using the electrically-controlled alignment of crystal structures to block light + \item Does not emit light: needs an always-on backlight source (usually LEDs) + \item Low energy consumption (depends on backlight) + \item Medium to low contrast, medium response time (\(1-10~ms\)) + \item \textbf{Twisted nematic} (TN): limited color quality and viewing angles, since the 1980s + \item \textbf{In-plane switching} (IPS): improved color and viewing angles, since the 2000s + \end{itemize} + \end{itemize} + + \begin{center} + \includegraphics[height=6em]{slides/graphics-hardware-display-devices/lcd-ips-shape.jpg}\\ + \textit{\small Chevron shapes that improve the viewing angle on IPS LCDs} + \end{center} +\end{frame} + +\begin{frame}{OLED display technology} + \begin{itemize} + \item \textbf{Organic light-emitting diodes} (OLEDs), since 2010: + \begin{itemize} + \item Using organic compounds (carbon-based) to emit light as R-G-B LEDs + \item Allows flat and flexible surfaces, with a large viewing angle + \item Low energy consumption + \item Very high contrast, low response time (\(1-10~\mu s\)) + \item Issues: burn-in, independent cells aging, affected by UV light + \item Rapidly becoming \textbf{very popular} and used + \end{itemize} + \end{itemize} + + \begin{center} + \includegraphics[height=8em]{slides/graphics-hardware-display-devices/oled-display.jpg}\\ + \textit{\small A flexible OLED display panel} + \end{center} +\end{frame} + +\begin{frame}{EPD display technology} + \begin{itemize} + \item \textbf{Electrophoretic displays} (EPDs), since the 2000s: + \begin{itemize} + \item Using black and white electrically-charged ink-like particles in oil\\ + \textit{e.g. positive charge for black and negative for white} + \item Electric fields attract one or the other color with current flow\\ + \textit{the particles stay in place after they were moved} + \item Using incident light, does not emit light itself + \item Very low consumption (only for changes) + \item Very high response time (\(100~ms\)) and ghosting + \end{itemize} + \end{itemize}~ + + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[height=6.5em]{slides/graphics-hardware-display-devices/e-reader.jpg}\\ + \textit{\small An e-reader with an EPD display} + \end{minipage} + \hfill + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[height=6.5em]{slides/graphics-hardware-display-devices/epd-detail.jpg}\\ + \textit{\small Detail of an EPD display} + \end{minipage} +\end{frame} + +\begin{frame}{Display panels integration and monitors} + \begin{itemize} + \item Panels come with a dedicated \textbf{controller} to: + \begin{itemize} + \item Decode pixels from the display interface + \item Electrically control the color cells of the panel + \end{itemize} + \item Panels can be used \textbf{standalone}, usually with: + \begin{itemize} + \item A single simple display interface + \item No standardized connector, weak and short cables + \item Only native dimensions supported and little to no configuration + \end{itemize} + \item Or they can be \textbf{integrated in monitors}, usually with: + \begin{itemize} + \item More complex and multiple display interfaces + \item Standardized connectors, external cables + \item Configuration (buttons and UI overlay), multiple dimensions + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Display panels integration and monitors (illustrated)} + \begin{center} + \includegraphics[width=0.5\textwidth]{slides/graphics-hardware-display-devices/display-panel.jpg}\\ + \textit{\small A display panel used standalone with an embedded board} + \end{center} +\end{frame} diff --git a/slides/graphics-hardware/lcd-ips-shape.jpg b/slides/graphics-hardware-display-devices/lcd-ips-shape.jpg similarity index 100% rename from slides/graphics-hardware/lcd-ips-shape.jpg rename to slides/graphics-hardware-display-devices/lcd-ips-shape.jpg diff --git a/slides/graphics-hardware/oled-display.jpg b/slides/graphics-hardware-display-devices/oled-display.jpg similarity index 100% rename from slides/graphics-hardware/oled-display.jpg rename to slides/graphics-hardware-display-devices/oled-display.jpg diff --git a/slides/graphics-hardware/pixel-array-text.jpg b/slides/graphics-hardware-display-devices/pixel-array-text.jpg similarity index 100% rename from slides/graphics-hardware/pixel-array-text.jpg rename to slides/graphics-hardware-display-devices/pixel-array-text.jpg diff --git a/slides/graphics-hardware/pixel-array.jpg b/slides/graphics-hardware-display-devices/pixel-array.jpg similarity index 100% rename from slides/graphics-hardware/pixel-array.jpg rename to slides/graphics-hardware-display-devices/pixel-array.jpg diff --git a/slides/graphics-hardware/a13-olinuxino-vga.png b/slides/graphics-hardware-display-interfaces/a13-olinuxino-vga.png similarity index 100% rename from slides/graphics-hardware/a13-olinuxino-vga.png rename to slides/graphics-hardware-display-interfaces/a13-olinuxino-vga.png diff --git a/slides/graphics-hardware/display-interface-encoders.dia b/slides/graphics-hardware-display-interfaces/display-interface-encoders.dia similarity index 100% rename from slides/graphics-hardware/display-interface-encoders.dia rename to slides/graphics-hardware-display-interfaces/display-interface-encoders.dia diff --git a/slides/graphics-hardware/display-timings.dia b/slides/graphics-hardware-display-interfaces/display-timings.dia similarity index 100% rename from slides/graphics-hardware/display-timings.dia rename to slides/graphics-hardware-display-interfaces/display-timings.dia diff --git a/slides/graphics-hardware/dp-dvi-bridge.jpg b/slides/graphics-hardware-display-interfaces/dp-dvi-bridge.jpg similarity index 100% rename from slides/graphics-hardware/dp-dvi-bridge.jpg rename to slides/graphics-hardware-display-interfaces/dp-dvi-bridge.jpg diff --git a/slides/graphics-hardware/dp-pinout.svg b/slides/graphics-hardware-display-interfaces/dp-pinout.svg similarity index 100% rename from slides/graphics-hardware/dp-pinout.svg rename to slides/graphics-hardware-display-interfaces/dp-pinout.svg diff --git a/slides/graphics-hardware/dvi-pinout.svg b/slides/graphics-hardware-display-interfaces/dvi-pinout.svg similarity index 100% rename from slides/graphics-hardware/dvi-pinout.svg rename to slides/graphics-hardware-display-interfaces/dvi-pinout.svg diff --git a/slides/graphics-hardware-display-interfaces/graphics-hardware-display-interfaces.tex b/slides/graphics-hardware-display-interfaces/graphics-hardware-display-interfaces.tex new file mode 100644 index 0000000000..9b51f491d7 --- /dev/null +++ b/slides/graphics-hardware-display-interfaces/graphics-hardware-display-interfaces.tex @@ -0,0 +1,343 @@ +\subsection{Display Interfaces} + +\begin{frame}{Display panels refreshing} + \begin{itemize} + \item Most display panel technologies need frequent pixel updates: \textbf{refreshing} + \begin{itemize} + \item Colors would fade out without refresh + \item Panels usually lack internal memory to self-refresh + \end{itemize} + \item Requires a \textbf{fixed refresh rate}: + \begin{itemize} + \item Capped by the display technology or display interface + \item Directly impacts smoothness: minimum is usually \(30~Hz\) + \item The whole frame must be sent over each time + \end{itemize} + \item Requires \textbf{synchronization points}: + \begin{itemize} + \item Vertical: beginning of a new frame + \item Horizontal: beginning of a new line + \end{itemize} + \item Requires \textbf{blank times}: + \begin{itemize} + \item Account for line/frame preparation time + \item Initially for CRT electron gun repositioning + \item More or less useful depending on the technology + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Display timings and modes} + \begin{itemize} + \item Timings coordinate how a frame is \textbf{transmitted to a display over time} + \item Required signals are synced to a \textbf{pixel clock} (time base) + \item Display timings are split in a few stages (horizontal and vertical): + \begin{enumerate} + \item Sync pulse (vsync/hsync) + \item Back porch (vbp/hbp): blanking + \item Active region (vactive/hactive) + \item Front porch (vfp/hfp): blanking + \end{enumerate} + \item Pixels are transmitted during the \textbf{horizontal active region} (one line) + \item A \textbf{display mode} groups all timing and signal characterization information + \begin{itemize} + \item Signals are \textbf{generated by the CRTC} according to the display mode + \end{itemize} + \item Monitors usually support \textbf{multiple modes} (and dimensions): + \begin{itemize} + \item Standard modes are defined by VESA + \item Extra specific modes for the monitor can be supported + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Display timings and modes (illustrated)} + \begin{center} + \includegraphics[width=0.8\textwidth]{slides/graphics-hardware-display-interfaces/display-timings.pdf} + \end{center} + + \begin{itemize} + \item The unit for horizontal stages is \textbf{one pixel clock period} + \item The unit for vertical stages is \textbf{one line's duration} + \end{itemize} +\end{frame} + +\begin{frame}{Display timings and modes (panel example)} + + \begin{minipage}[b]{0.35\textwidth} + \centering + \vspace{2em} + \includegraphics[width=\textwidth]{slides/graphics-hardware-display-interfaces/timings-table.png} + \textit{\small AT070TN94 panel datasheet} + \vfill~ + \end{minipage} + \hfill + \begin{minipage}[b]{0.6\textwidth} + \small + \begin{itemize} + \item \(hsync = thpw = 20 \in \llbracket 1;40 \rrbracket\)\\ + \(hbp = thb - thpw = 46 - 20 = 26\) (from diagram)\\ + \(hactive = thd = 800\)\\ + \(hfp = thfp = 210 \in \llbracket 16;354 \rrbracket\)\\ + \(htotal = hsync + hbp + hactive + hfp = 1056\) + + \item \(vsync = tvpw = 10 \in \llbracket 1;20 \rrbracket\)\\ + \(vbp = tvb - tvpw = 23 - 10 = 13\) (from diagram)\\ + \(vactive = tvd = 480\)\\ + \(vfp = tvfp = 22 \in \llbracket 7;147 \rrbracket\)\\ + \(vtotal = vsync + vbp + vactive + vfp = 525\) + \item 1 frame takes: \(vtotal \times htotal = 554400~t_{clk}\)\\ + 60 frames take: \(vtotal \times htotal \times 60 = 33264000~t_{clk}\)\\ + 60 fps requires: \(f_{clk} \geq 33.264~MHz\) + \end{itemize} + \end{minipage} + \vspace{0.5em} + + \begin{itemize} + \item Panels usually support a \textbf{range of timings} + \item The pixel clock rate is often \textbf{rounded} (refresh rate not always strictly respected) + \end{itemize} + +\end{frame} + +\begin{frame}{Side-channel and identification} + \begin{itemize} + \item Monitor display connectors often come with a \textbf{Display Data Channel} (DDC) + \begin{itemize} + \item Side-bus to allow communication between host and display + \item Usually based on I2C, quite slow (\(\approx 100~kHz\)) + \end{itemize} + \item DDC provides access to the \textbf{Extended Display Identification Data} (EDID) + \begin{itemize} + \item Contains the list of supported modes in a standard format + \item Usually stored in an EEPROM at I2C address \(0x50\) + \end{itemize} + \item Another common monitor signal is \textbf{Hotplug Detect} (HPD) + \begin{itemize} + \item Connected to a pin of the connector, asserted with a cable plugged + \item Can be wired to an interrupt pin to detect connection changes + \end{itemize} + \item Direct panel interfaces (not monitors) usually lack DDC, EDID and HPD + \begin{itemize} + \item Panel is always considered connected + \item Modes need to be known in advance + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Extra display interface features and EDID extensions} + \begin{itemize} + \item The EDID standard keeps evolving and exposes new features through extensions + \item Configuration data for each feature is embedded in the EDID + \item More or fewer features are supported depending on the display interface + \item Common extra display interface features: + \begin{itemize} + \item \textbf{Interlaced}: Every other pixel line is sent at a time, alternating between top-fields and bottom-fields; Allows faster refreshing for CRTs, needs deinterlacing for progressive panels; + \item \textbf{Audio}: Send audio in addition to pixels, during blanking periods; + \item \textbf{Stereoscopy}: Pixel data is split between two screens that show a different geometrical perspective, providing 3D perception; + \item \textbf{Variable Refresh Rate} (VRR): Pixel data can be sent at any point and does not need to conform to a given refresh rate; + \item \textbf{Consumer Electronic Control} (CEC): Remote control features on a dedicated bus; + \item \textbf{High-Bandwidth Digital Content Protection} (HDCP): Anti-copy protection + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Types of display interfaces} + \begin{itemize} + \item Legacy display interfaces are usually \textbf{analog}: + \begin{itemize} + \item Transmission through a DAC-based chain + \item Lack of precision, noise and chain error: not pixel-perfect, capped + \item Requires few signal pins (1 per color channel and sync or less) + \end{itemize} + \item Recent interfaces are usually \textbf{digital}: + \begin{itemize} + \item Encoded binary transmission, usually with dedicated clock + \item Encoders contain a controller (logic) and a PHY (signal) + \item Pixel data is expected to be bit-perfect \textit{(but noise still exists)} + \end{itemize} + \item Digital interfaces can be \textbf{parallelized}: + \begin{itemize} + \item One signal per color bit (e.g. 24 signals for 24-bit RGB), clock and sync + \item One clock cycle for one pixel (low clock rate) + \end{itemize} + \item Or they can be \textbf{serialized}: + \begin{itemize} + \item Pixel data is sent over physical lanes (one or more) + \item One clock cycle for one bit on each lane (high clock rate) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Types of display interfaces (illustrated)} + \begin{center} + \includegraphics[width=0.7\textwidth]{slides/graphics-hardware-display-interfaces/display-interface-encoders.pdf} + \end{center} +\end{frame} + +\begin{frame}{VGA display interface} + \begin{itemize} + \item \textbf{Video Graphics Array} (VGA), since 1987 (IBM) + \begin{itemize} + \item \textbf{Analog} pixel data on 3 pins (R-G-B), DAC encoder to \(0.7~V\) peak-to-peak + \item \textbf{Per-channel pixel streaming} (voltage change), following mode timings + \item \textbf{Hsync} and \textbf{vsync} signals, I2C SDA and SDL \textbf{DDC} signals + \item Hotplug detection with R/G/B pins \textbf{current sensing} + \item Using a DB-15 connector for signals: + \end{itemize} + \begin{center} + \includegraphics[height=2em]{slides/graphics-hardware-display-interfaces/vga-pinout.pdf} + \end{center} + \begin{itemize} + \item \textbf{Pixel}: Red, Green, Blue {\footnotesize(1, 2, 3)}, Ground returns {\footnotesize(6, 7, 8)} + \item \textbf{Sync}: Hsync, Vsync {\footnotesize(13, 14)} + \item \textbf{Side}: DDC SDA, DDC SCL {\footnotesize(12, 15)} + \item \textbf{Power}: \(+5~V\) {\footnotesize(9)}, Ground {\footnotesize(10)} + \end{itemize} + \end{itemize} + +\end{frame} + +\begin{frame}{VGA display interface (illustrated)} + \begin{center} + \includegraphics[width=\textwidth]{slides/graphics-hardware-display-interfaces/a13-olinuxino-vga.png} + \end{center} + + \begin{itemize} + \item Very basic VGA encoder from parallel signals, DDC excluded + \item Resistor ladder for digital-to-analog conversion\\ + \textit{using SN74ALVC244 voltage level shifters (\(1.8~V\) to \(3.3~V\)), clamping diodes} + \item 6 most-significant bits only, 2 least-significant bits set to 0\\ + \textit{D0-D1, D8-D9 and D16-D17 are not routed} + \end{itemize} +\end{frame} + +\begin{frame}{DVI display interface} + \begin{itemize} + \item \textbf{Digital Visual Interface} (DVI), since 1999 (DDWG) + \begin{itemize} + \item \textbf{DVI-A}: Analog only, comparable to VGA + \item \textbf{DVI-D}: Digital only, single-link (3 data lanes) or dual-link (6 data lanes) + \item \textbf{DVI-I}: Both analog and digital supported, single-link or dual-link + \item Digital serial link using \textbf{Transition-Minimized Differential Signaling} (TMDS) + \item Dedicated \textbf{DDC} and \textbf{HPD} signals + \item Using a subset or variation of the full \textbf{DVI-I connector} for signals: + \end{itemize} + \begin{center} + \includegraphics[height=2em]{slides/graphics-hardware-display-interfaces/dvi-pinout.pdf} + \end{center} + \begin{itemize} + \item \textbf{TMDS}: Data+ {\footnotesize(2, 5, 10, 13, 18, 21)}, Data- {\footnotesize(1, 4, 9, 12, 17, 20)}, Clock {\footnotesize(23, 24)} + \item \textbf{Analog pixel}: Red, Green, Blue {\footnotesize(C1, C2, C3)}, Ground {\footnotesize(C5)} + \item \textbf{Analog sync}: Hsync, Vsync {\footnotesize(C4, 8)} + \item \textbf{Side}: DDC SDA, DDC SCL {\footnotesize(7, 6)}, HPD {\footnotesize(16)} + \item \textbf{Power}: \(+5~V\) {\footnotesize(14)}, Ground {\footnotesize(15)} + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{HDMI display interface} + \begin{itemize} + \item \textbf{High-Definition Multimedia Interface} (HDMI), since 2002 (HDMI Forum) + \begin{itemize} + \item Similar to \textbf{DVI-D}: no analog, 3 TMDS data lanes (R-G-B) + \item Adding the use of AVI infoframes for meta-data and audio + \item \textbf{High bandwidth} \((\leq 48~Gbit/s)\) (2.1) and clock speeds \((\leq 340~MHz)\) + \item \textbf{Extra features}: Audio, CEC (1.2), HDR (1.3), 4K (1.4), Stereoscopy (1.4),\\8K-10K (2.1), DSC (2.1), HFR (\(120~Hz\)), per-frame HDR (2.1) + \item Using a dedicated (and proprietary) \textbf{HDMI connector} for signals: + \end{itemize} + \begin{center} + \includegraphics[height=2em]{slides/graphics-hardware-display-interfaces/hdmi-pinout.pdf} + \end{center} + \begin{itemize} + \item \textbf{TMDS}: Data+ {\footnotesize(1, 4, 7)}, Data- {\footnotesize(3, 6, 9)}, Clock {\footnotesize(10, 12)} + \item \textbf{Side}: SDA, SCL {\footnotesize(16, 15)}, HPD {\footnotesize(19)}, CEC {\footnotesize(13)} + \item \textbf{Power}: \(+5~V\) {\footnotesize(18)}, Ground {\footnotesize(17)} + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{DP/eDP display interface} + \begin{itemize} + \item \textbf{DisplayPort} (DP), since 2008 (VESA) + \begin{itemize} + \item Digital serial link with 4 data lanes using \textbf{Low-Voltage Differential Signaling} (LVDS) + or TMDS for DP Dual-Mode (DP++), compatible with DVI-D and HDMI + \item Using \textbf{packets} for video/audio data and meta-data + \item Auxiliary channel encapsulating I2C DDC, CEC and more (e.g. USB) + \item \textbf{High bandwidth} \((\leq 77.37~Gbit/s)\) (2.0) + \item \textbf{Extra features}: Audio, CEC, HDR (1.4), 4K (1.3), Stereoscopy (1.2),\\8K (1.3-1.4), 10K-16K (2.0) + \item \textbf{Multi-Stream Transport} (MST) to chain displays + \item Using a dedicated (and proprietary) \textbf{DisplayPort connector} for signals: + \end{itemize} + \begin{center} + \includegraphics[height=2em]{slides/graphics-hardware-display-interfaces/dp-pinout.pdf} + \end{center} + \begin{itemize} + \item \textbf{LVDS/TMDS}: ML+ {\footnotesize(1, 4, 7, 10)}, ML- {\footnotesize(3, 6, 9, 12)} + \item \textbf{Side}: AUX+ (15), AUX- (17), HPD (18) + \item \textbf{Power}: \(+3.3~V\) {\footnotesize(20)}, Ground {\footnotesize(2, 5, 8, 11, 16)} + \end{itemize} + \item \textbf{Embedded DisplayPort} (eDP) for internal panels (without connector) + \end{itemize} +\end{frame} + +\begin{frame}{LVDS and DSI display interfaces} + +% TODO split and add photo of custom LVDS connector as well as a DSI panel (librem 5 dvt?) + + \begin{itemize} + \item \textbf{Low Voltage Differential Signaling} (LVDS) + \begin{itemize} + \item Generic digital serial link with clock and data signals (3-4 lanes) using LVDS + \item Pixel data and control (vsync, hsync, display enable) sent as bits on lanes + \item Uses a fixed mode, no DDC, no packets + \item Specified with the JEIDA, LDI, DSIM and VESA specifications + \item For internal panels, exposed with specific connectors + \item Common for laptop panels + \end{itemize} + \end{itemize} + \begin{itemize} + \item \textbf{Display Serial Interface} (DSI), since 2006 (MIPI) + \begin{itemize} + \item Digital serial link with clock and up to 4 data lanes using LVDS + \item Using \textbf{packets} for video data and meta-data + \item Commands for configuration can be issued with the \textbf{DSI Command Set} (DCS)\\ + \textit{Generic base with proprietary vendor-specific extensions} + \item For internal panels, exposed with specific connectors + \item Common for mobile devices' panels + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{DPI display interface} + \begin{itemize} + \item \textbf{Display Parallel Interface} (DPI) + \begin{itemize} + \item Generic parallel digital interface, with 1 signal per color bit, clock and sync + \item Exists with different numbers of bits: 24 (8-8-8), 18 (6-6-6) or 16 (5-6-5)\\ + \textit{Dithering is required when using 16 or 18 bits} + \item Sends pixel data bits following mode timings + \item Base signals: color data bits, vsync, hsync + \item Extra signals: display enable (DE) + \item Beware: sync and DE signals can be active-high or active-low + \item For internal panels, requires many signals + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Bridges/transcoders} + \begin{itemize} + \item Not every display interface is supported by the hardware at hand + \item Bridges or transcoders are used to translate from one interface to another + \item They are composed of a decoder and an encoder (in a single package) + \item Usually standalone and transparent, often only replicate timings\\ + \textit{but some can have a side-bus for configuration and fine-tuning} + \item Example: VGA interfaces are usually bridged from digital interfaces nowadays + \end{itemize} + + \begin{center} + \includegraphics[height=6em]{slides/graphics-hardware-display-interfaces/dp-dvi-bridge.jpg}\\ + \textit{\small A DP to DVI bridge} + \end{center} +\end{frame} diff --git a/slides/graphics-hardware/hdmi-pinout.svg b/slides/graphics-hardware-display-interfaces/hdmi-pinout.svg similarity index 100% rename from slides/graphics-hardware/hdmi-pinout.svg rename to slides/graphics-hardware-display-interfaces/hdmi-pinout.svg diff --git a/slides/graphics-hardware/timings-table.png b/slides/graphics-hardware-display-interfaces/timings-table.png similarity index 100% rename from slides/graphics-hardware/timings-table.png rename to slides/graphics-hardware-display-interfaces/timings-table.png diff --git a/slides/graphics-hardware/vga-pinout.svg b/slides/graphics-hardware-display-interfaces/vga-pinout.svg similarity index 100% rename from slides/graphics-hardware/vga-pinout.svg rename to slides/graphics-hardware-display-interfaces/vga-pinout.svg diff --git a/slides/graphics-hardware/ati-hercules-1986.png b/slides/graphics-hardware-overview/ati-hercules-1986.png similarity index 100% rename from slides/graphics-hardware/ati-hercules-1986.png rename to slides/graphics-hardware-overview/ati-hercules-1986.png diff --git a/slides/graphics-hardware/display-pipe.dia b/slides/graphics-hardware-overview/display-pipe.dia similarity index 100% rename from slides/graphics-hardware/display-pipe.dia rename to slides/graphics-hardware-overview/display-pipe.dia diff --git a/slides/graphics-hardware-overview/graphics-hardware-overview.tex b/slides/graphics-hardware-overview/graphics-hardware-overview.tex new file mode 100644 index 0000000000..c1a46c6836 --- /dev/null +++ b/slides/graphics-hardware-overview/graphics-hardware-overview.tex @@ -0,0 +1,138 @@ +\section{Hardware Aspects} + +\subsection{Overview} + +\begin{frame}{Technological types of graphics hardware implementations} + \begin{itemize} + \item Dedicated graphics hardware is often used along a general-purpose CPU + \item Two commonly-used technologies, with typical pros/cons: + \end{itemize} + + \begin{center} + \small + \def\arraystretch{1.2} + \begin{tabular}{l|c|c} + & \textbf{Fixed-function} & \textbf{Programmable} \\ + \hline + \textbf{Technology} & Circuit & Software \\ + \textbf{Source form} & HDL & Source code \\ + \textbf{Product form} & Silicon, bitstream & Firmware binaries \\ + \textbf{Implementation} & FPGA, ASIC, SoC block & DSP, custom ISA \\ + \textbf{Arithmetic} & Fixed-point & Fixed-point, floating point \\ + \textbf{Clock rate / power} & Low & High \\ + \textbf{Pixel data access} & Queue (FIFO) & Memory \\ + \textbf{CPU control} & Direct registers & Mailbox \\ + \textbf{Die surface} & High & Low \\ + \textbf{Reusability} & Low & High \\ + \hline + \textbf{Example} & Allwinner SoCs Display Engine & TI TMS340 DSP \\ + \end{tabular} + \end{center} +\end{frame} + +\begin{frame}{Graphics memory and buffers} + \begin{itemize} + \item Pixel data is stored in memory buffers, called \textbf{framebuffers} + \item Framebuffers live either in: + \begin{itemize} + \item \textbf{System memory}: shared with the rest of the system (e.g. SDRAM or SRAM) + \item \textbf{Dedicated memory}: only for graphics (e.g. SGRAM) + \end{itemize} + \item Framebuffers that can be displayed are called \textbf{scanout framebuffers}\\ + \textit{hardware constraints don't always allow any framebuffer to be scanned out} + \item CPU access to pixel data in dedicated memory is neither always granted nor easy! + \item Graphics hardware \textbf{needs configuration} to interpret framebuffer pixel data\\ + \textit{pixel meta-data is rarely to never stored aside of the pixel data} + \end{itemize} +\end{frame} + +\begin{frame}{Display hardware overview} + \begin{itemize} + \item \textbf{Stream pixel data} to a display device, via a display interface + \item Internal pipeline with \textbf{multiple components} + \item Generally \textbf{fixed-function} hardware, pipeline sink only + \item Either \textbf{discrete} (video card) or \textbf{integrated} + \item Connected to the CPU (and RAM) via a \textbf{high-speed bus}:\\ + \textit{e.g. AXI with ARM, ISA, PCI, AGP, PCI-e with x86} + \end{itemize}~ + + \begin{minipage}[t]{0.45\textwidth} + \centering + \includegraphics[height=7em]{slides/graphics-hardware-overview/ati-hercules-1986.png}\\ + \textit{\small A 1986 Hercules discrete video card} + \end{minipage} + \hfill + \begin{minipage}[t]{0.45\textwidth} + \centering + \includegraphics[height=7em]{slides/graphics-hardware-overview/intel-skylake.jpg}\\ + \textit{\small An Intel processor with integrated graphics} + \end{minipage} +\end{frame} + +\begin{frame}{Common components of a display pipeline overview} + \begin{center} + \includegraphics[width=0.7\textwidth]{slides/graphics-hardware-overview/display-pipe.pdf} + \end{center} + + \begin{enumerate} + \item \textbf{Framebuffers} for storing the pixel data\\ + \textit{streamed using a DMA engine} + \item \textbf{Planes} for associating a framebuffer with its dimensions and position\\ + \textit{composited into a single result on-the-fly} + \item \textbf{CRTC} for streaming resulting pixels with specific timings\\ + \textit{terminology comes from the legacy Cathode-Ray Tube Controller} + \item \textbf{Encoder} for meta-data addition and physical signal conversion + \item \textbf{Connector} for video signal, display data channel (DDC), hotplug detection + \item \textbf{Display} for decoding and displaying pixels (panel or monitor) + \end{enumerate} +\end{frame} + +\begin{frame}{Render hardware overview} + \textbf{Rendering} hardware includes a wide range of aspects (usual cases below): + \begin{itemize} + \item \textbf{Basic} pixel processing: + \begin{itemize} + \item Common operations: pixel format conversion, dithering, scaling, blitting and blending + \item Fixed-function hardware, pipeline sink and source + \end{itemize} + \item \textbf{Complex} pixel processing: + \begin{itemize} + \item Defined by the application: any computable operation + \item Programmable hardware (DSP), pipeline sink and source + \end{itemize} + \item \textbf{2D vector} drawing: + \begin{itemize} + \item Rasterization from equations, parameters and data (e.g. points) + \item Either fixed-function or programmable hardware (custom), pipeline source + \end{itemize} + \item \textbf{3D scene} rendering: + \begin{itemize} + \item Rasterization from programs (shaders) and data (e.g. vertices, lines, triangles textures) + \item Programmable hardware (GPU), pipeline source + \end{itemize} + \item Rendering can \textbf{always fallback} to general-purpose CPU operations + \end{itemize} +\end{frame} + +\begin{frame}{Video hardware overview} + \textbf{Video-oriented} hardware comes in different forms (usual cases below): + + \begin{itemize} + \item \textbf{Hardware video decoder} (VPU/video codec decoder) + \begin{itemize} + \item Decodes a video from compressed data (bitstream) to pixel frames + \item Fixed-function hardware, pipeline source + \end{itemize} + \item \textbf{Hardware video encoder} (VPU/video codec encoder) + \begin{itemize} + \item Encodes a video from pixel frames to compressed data (bitstream) + \item Fixed-function hardware, pipeline sink + \end{itemize} + \item \textbf{Camera sensors, video input, video broadcasting} (DVB) + \begin{itemize} + \item Receives/sends data in a given configuration from/to \textit{the outside} + \item Can be compressed data (bitstream) or raw pixel data + \item Fixed-function hardware, pipeline source + \end{itemize} + \end{itemize}~ +\end{frame} diff --git a/slides/graphics-hardware/intel-skylake.jpg b/slides/graphics-hardware-overview/intel-skylake.jpg similarity index 100% rename from slides/graphics-hardware/intel-skylake.jpg rename to slides/graphics-hardware-overview/intel-skylake.jpg diff --git a/slides/graphics-hardware-pipelines/graphics-hardware-pipelines.tex b/slides/graphics-hardware-pipelines/graphics-hardware-pipelines.tex new file mode 100644 index 0000000000..e9d7487a3d --- /dev/null +++ b/slides/graphics-hardware-pipelines/graphics-hardware-pipelines.tex @@ -0,0 +1,60 @@ +\subsection{Pipelines} + +\begin{frame}{I/O with graphics hardware, pipelines} + \begin{itemize} + \item Graphics hardware is \textbf{I/O-based} and interacts with pixel data + \item Pipeline elements have input-output abilities: + \begin{itemize} + \item \textbf{Source} components: \textbf{feed pixel data}: \textit{e.g. camera} + \item \textbf{Sink} components: \textbf{grab pixel data}: \textit{e.g. display} + \end{itemize} + \item Some components are \textbf{both a source and a sink}: \textit{e.g. converters} + \item Graphics components can be chained in \textbf{pipelines}\\ + \textit{Usually from a source-only element to a sink-only element} + \end{itemize} + + \vspace{-2em} + \begin{center} + \includegraphics[width=0.7\textwidth]{slides/graphics-hardware-pipelines/pipeline-sink-source.pdf} + \end{center} +\end{frame} + +\begin{frame}{Building complex pipelines} + \begin{itemize} + \item Display, rendering and video elements are chained from source(s) to sink(s) + \item On source-sink boundaries: + \begin{itemize} + \item Mutually-supported pixel format (or conversion) + \item Mutually-accessible memory (or copy) or internal FIFO + \end{itemize} + \item Target frame rate (\(fps\)) gives a time budget for pipeline traversal: + \( t_0 + t_2 + t_4 + t_5 < fps^{-1},~ t_0 + t_3 < fps^{-1},~ t_1 + t_4 + t_5 < fps^{-1} \) + \end{itemize} + + \begin{center} + \includegraphics[width=0.7\textwidth]{slides/graphics-hardware-pipelines/pipeline-complex.pdf} + \end{center} +\end{frame} + +\begin{frame}{Debugging pipelines} + \begin{itemize} + \item Debugging a complex pipeline can become \textbf{hard}: + \begin{itemize} + \item Many elements in the chain means many possibilities for trouble + \item A single point of failure is enough + \item Can be difficult to identify + \end{itemize} + \item Debugging \textbf{tips}: + \begin{itemize} + \item Validating each element separately + \item Dumping (intermediary) framebuffers to memory/storage for validation + \item Using hardware pattern generators + \end{itemize} + \item Common \textbf{pitfalls}: + \begin{itemize} + \item Mismatch between advertized format and reality + \item Lack of output-input sync between elements + \item Input data updated too fast + \end{itemize} + \end{itemize} +\end{frame} diff --git a/slides/graphics-hardware/pipeline-complex.dia b/slides/graphics-hardware-pipelines/pipeline-complex.dia similarity index 100% rename from slides/graphics-hardware/pipeline-complex.dia rename to slides/graphics-hardware-pipelines/pipeline-complex.dia diff --git a/slides/graphics-hardware/pipeline-sink-source.dia b/slides/graphics-hardware-pipelines/pipeline-sink-source.dia similarity index 100% rename from slides/graphics-hardware/pipeline-sink-source.dia rename to slides/graphics-hardware-pipelines/pipeline-sink-source.dia diff --git a/slides/graphics-hardware/anisotropic-filtering.png b/slides/graphics-hardware-render/anisotropic-filtering.png similarity index 100% rename from slides/graphics-hardware/anisotropic-filtering.png rename to slides/graphics-hardware-render/anisotropic-filtering.png diff --git a/slides/graphics-hardware/g2d-block.png b/slides/graphics-hardware-render/g2d-block.png similarity index 100% rename from slides/graphics-hardware/g2d-block.png rename to slides/graphics-hardware-render/g2d-block.png diff --git a/slides/graphics-hardware/gpu-pipeline.png b/slides/graphics-hardware-render/gpu-pipeline.png similarity index 100% rename from slides/graphics-hardware/gpu-pipeline.png rename to slides/graphics-hardware-render/gpu-pipeline.png diff --git a/slides/graphics-hardware-render/graphics-hardware-render.tex b/slides/graphics-hardware-render/graphics-hardware-render.tex new file mode 100644 index 0000000000..f840d1f054 --- /dev/null +++ b/slides/graphics-hardware-render/graphics-hardware-render.tex @@ -0,0 +1,162 @@ +\subsection{Render Accelerators} + +\begin{frame}{Digital Signal Processors} + \begin{itemize} + \item Digital Signal Processors (DSPs) allow programmable image signal processing\\ + \textit{can also be used for implementing 2D rendering primitives} + \item Using a dedicated Instruction Set Architectures (ISA) + \item Arithmetic implementations are either: + \begin{itemize} + \item \textbf{fixed-point}: simple hardware implementation, fixed range (usually 16.16)\\ + \textit{16 bits for the integer part and 16 bits for the decimal part} + \item \textbf{floating-point}: complex implementations, trade-off between range and precision + \end{itemize} + \item Usually more power-efficient than general-purpose CPUs + \item Depending on the DSP, the software can be: + \begin{itemize} + \item A \textbf{standalone firmware}, usually developed from vendor libraries (C/C++/ASM) + \item A \textbf{real-time operating system} (RTOS) application (C/C++/...) + \end{itemize} + \item Can be used standalone in a video pipeline or to offload a CPU + \item Modern DSPs can be multi-core and feature various I/O controllers + \end{itemize} +\end{frame} + +\begin{frame}{Dedicated hardware accelerators} + \begin{itemize} + \item Fixed-function hardware can be used for accelerating specific operations + \begin{itemize} + \item Implemented as hardware circuits in Systems on a Chip (SoCs) or DSPs + \item Implemented as logic configuration bitstream in FPGAs + \end{itemize} + \item Implement a configurable fixed pipeline for image operations + \item Accessed and configured through specific registers exposed via a bus + \begin{itemize} + \item Global configuration registers to build the pipeline between blocks + \item Configuration registers for each block + \item Kick and status registers + \end{itemize} + \item Usually very power-efficient and very fast + \end{itemize} +\end{frame} + +\begin{frame}{Dedicated hardware accelerators (illustrated)} + \begin{center} + \includegraphics[width=0.7\textwidth]{slides/graphics-hardware-render/g2d-block.png}\\ + \textit{An example hardware pipeline for a 2D graphics block}\\ + \end{center} +\end{frame} + +\begin{frame}{Graphics Processing Unit} + \begin{itemize} + \item Graphics Processing Units (GPUs) are \textbf{3D rendering hardware} implementations\\ + \textit{the term no longer designates all graphics-processing hardware} + \item Operate on 3D graphics primitives: \textbf{points (vertices), lines and triangles} + \item Generate a 2D view (viewport) from a given perspective + \begin{itemize} + \item Objects are formed of interconnected triangles + \item A color can be applied to each vertex and interpolated + \item Textures can be applied to objects with texture element to vertex mapping + \item Lighting is applied from various sources + \end{itemize} + \item Expected to render at \textbf{display scanout rate} (pseudo real-time) + \begin{itemize} + \item Usually not photo-realistic methods as ray tracing or photon mapping + \item Extremely efficient compared to any general-purpose CPU + \end{itemize} + \item GPUs are also used for general-purpose computing with GPGPU + \end{itemize} +\end{frame} + +\begin{frame}{Graphics Processing Unit architectures} + \begin{itemize} + \item GPUs implement a pipeline of hacks accumulated since the 1970s + \item GPU hardware architectures evolved over time: + \begin{itemize} + \item From \textbf{fixed-function} configurable hardware block pipelines + \item To pipelines with both fixed blocks and specialized \textbf{programmable} processing units + \end{itemize} + \item Shaders are programs that run at different steps of the pipeline: + \begin{itemize} + \item \textbf{vertex shaders}: define the position, texture coordinates and lighting of each vertex + \item \textbf{geometry shaders}: generate new primitives from the provided ones + \item \textbf{tesselation shaders}: perform vertex sub-division (e.g. Catmull-Clark) + \item \textbf{fragment/pixel shaders}: perform rasterization for each output pixel + \end{itemize} + \item Scenes can be rendered with multiples passes and multiple shaders + \end{itemize} +\end{frame} + +\begin{frame}{Graphics Processing Unit architectures (illustrated)} + \begin{center} + \includegraphics[width=0.5\textwidth]{slides/graphics-hardware-render/gpu-pipeline.png}\\ + \textit{\small A programmable GPU pipeline} + \end{center} +\end{frame} + +\begin{frame}{Graphics Processing Unit techniques} + \begin{itemize} + \item GPUs are optimized for \textbf{performance} and \textbf{good-looking results} + \item \textbf{Texture sampling} can easily cause aliasing (at a distance) + \begin{itemize} + \item \textbf{Bilinear and trilinear filtering} is often used + \item \textbf{Anisotropic filtering} provides best visual results + \item \textbf{Mip-maps} provide the same texture at different sizes + \item \textbf{Multi-Sample Anti-Aliasing} (MSAA) averages colors from multiple points + \end{itemize} + \item \textbf{Texture compression} reduces memory pressure (e.g. S3TC, ASTC) + \item \textbf{Normal mapping/bump maps} provide increased details with low vertex count\\ + \textit{affects light path calculation} + \item Avoiding \textbf{useless rendering} operations: + \begin{itemize} + \item Occluded surfaces are not rendered (visible surface determination) + \item Using a \textbf{depth/z-buffer} to keep track of the z-order for each output pixel + \item Culling for early vertex removal: back-face, view frustum, z-buffer + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Graphics Processing Unit techniques (illustrated)} + \begin{minipage}{0.45\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-hardware-render/anisotropic-filtering.png}\\ + \textit{Comparison of tri-linear and anisotropic anti-aliasing filtering} + \end{minipage} + \hfill + \begin{minipage}{0.45\textwidth} + \centering + \includegraphics[height=7em]{slides/graphics-hardware-render/mip-map.jpg}\\ + \textit{A mip-mapped texture representation} + \end{minipage} +\end{frame} + +\begin{frame}{Graphics Processing Unit internals} + \begin{itemize} + \item A \textbf{command stream} is parsed and used to configure the pipeline + \item Shader cores have highly-specialized ISAs adapted for geometry: + \begin{minipage}[b]{0.45\textwidth} + \begin{itemize} + \item Vector operations, SIMD + \item Trigonometric operations + \end{itemize} + \end{minipage} + \begin{minipage}[b]{0.45\textwidth} + \begin{itemize} + \item Interpolation operations + \item Usually few conditionals + \end{itemize} + \end{minipage} + \item Texture access is provided by a \textbf{Texture Mapping Unit} (TMU) + \begin{itemize} + \item Caching is used to reduce memory pressure + \end{itemize} + \item Modern GPUs sometimes have a \textbf{unified shader core}\\ + \textit{allows efficient hardware resources usage, with complex scheduling} + \item Shading cores are \textbf{duplicated} and work in parallel (especially rasterization) + \item Some architectures implement tiled processing: + \begin{itemize} + \item Output is divided in tiles (clipping areas) and distributed to cores + \item Each rasterized tile is written to the output framebuffer separately + \end{itemize} + \end{itemize} +\end{frame} diff --git a/slides/graphics-hardware/mip-map.jpg b/slides/graphics-hardware-render/mip-map.jpg similarity index 100% rename from slides/graphics-hardware/mip-map.jpg rename to slides/graphics-hardware-render/mip-map.jpg diff --git a/slides/graphics-hardware-system/graphics-hardware-system.tex b/slides/graphics-hardware-system/graphics-hardware-system.tex new file mode 100644 index 0000000000..42827835ba --- /dev/null +++ b/slides/graphics-hardware-system/graphics-hardware-system.tex @@ -0,0 +1,183 @@ +\subsection{System Integration, Memory and Performance} + +\begin{frame}{Graphics integration and memory} + \begin{itemize} + \item Graphics devices integrated in larger systems need two main interfaces: + \begin{itemize} + \item \textbf{Control interface} (low speed): to program the device from the main CPU + \item \textbf{Memory interface} (high speed): to read the source data and write their framebuffer + \end{itemize} + \item Other usual required elements: clocks, interrupts, reset + \item Both the graphics device and the CPU need to access the memory + \item Different types of memory used by graphics hardware: + \begin{itemize} + \item \textbf{graphics memory}: dedicated memory attached to the graphics device\\ + \textit{the memory is made available to the CPU through the memory interface} + \item \textbf{dedicated system memory}: a reserved contiguous area of system memory\\ + \textit{required when the device has no mapping unit} + \item \textbf{system memory pages}: any system memory page can be mapped for access\\ + \textit{for devices with a dedicated IOMMU and graphics address remapping table (GART)} + \end{itemize} + \item Since the two parties access the same memory, cache can become incoherent + \item Cache must be \textbf{synchronized} before reading and after writing, or \textbf{disabled} + \end{itemize} +\end{frame} + +\begin{frame}{Shared graphics memory access} + \begin{itemize} + \item Concurrent access to memory can lead to trouble: + \begin{itemize} + \item Concurrent read-write accesses result in partially-updated data + \item Concurrent write-write accesses result in incoherent data + \end{itemize} + \item Common issue with display hardware: tearing + \begin{itemize} + \item The framebuffer is scanned out at a fixed rate (e.g. \(60~fps\)) + \item Any modification during scan out will result in a \textbf{partial update} + \item Causes an unpleasant \textbf{visual glitch} effect + \end{itemize} + \item Solved using (at least) \textbf{double-buffering}: + \begin{itemize} + \item The displayed buffer (\textbf{front buffer}) is kept intact + \item Another buffer (\textbf{back buffer}) is used for drawing the next contents + \item Front and back buffer are exchanged with \textbf{page flipping}, during vertical blanking + \item Using more buffers is possible if rendering can be done in advance + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Graphics shared memory access (illustrated tearing)} + \begin{center} + \includegraphics[width=0.7\textwidth]{slides/graphics-hardware-system/tearing-glitch.jpg}\\ + \textit{\small Tearing example: data is updated during scanout} + \end{center} +\end{frame} + +\begin{frame}{Graphics memory constraints and performance} + \begin{itemize} + \item Fixed-pipeline 2D graphics hardware usually streams pixels through FIFOs + \item A DMA engine fetches pixel data from framebuffer memory + \item To simplify the logic and optimize memory access, memory constraints may apply:\\ + \begin{itemize} + \item The start address of each line needs to be aligned to \(2^n\) + \item The byte size of each line needs to be aligned to \(2^n\) + \end{itemize} + \item The \textbf{stride} or \textbf{pitch} describes the line byte size + \begin{itemize} + \item Calculated as: \(ALIGN(width \times bpp / 8,~2^n)\) + \item The size of a framebuffer becomes (single-planar): \(stride \times height\)\\ + \textit{memory may be over-allocated to satisfy alignment constraints} + \end{itemize} + \item Pixel order in memory may not follow raster order: + \begin{itemize} + \item Optimized depending on the hardware architecture + \item Optimized for efficient memory access + \item Tiled orders are frequent for parallel hardware + \item Framebuffer sizes are calculated with a specific formula + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Tiled framebuffer format example} + \begin{minipage}{0.45\textwidth} + \centering + \textit{\small The Allwinner VPU tiled format} + \end{minipage} + \hfill + \begin{minipage}{0.45\textwidth} + \centering + \vspace{1em} + \includegraphics[width=0.35\textwidth]{slides/graphics-hardware-system/sunxi-tiled-format.png}\\ + \end{minipage} + \vspace{2em} + \begin{minipage}[b]{0.45\textwidth} + \centering + \vspace{1em} + \includegraphics[width=\textwidth]{slides/graphics-hardware-system/sunxi-tiled-linear.png} + \textit{\small A tiled framebuffer read in raster order} + \end{minipage} + \hfill + \begin{minipage}[b]{0.45\textwidth} + \centering + \vspace{1em} + \includegraphics[width=\textwidth]{slides/graphics-hardware-system/sunxi-tiled.png} + \textit{\small The same framebuffer read properly} + \end{minipage} +\end{frame} + +\begin{frame}{Offloading graphics to hardware} + \begin{itemize} + \item Offloading graphics to hardware frees up significant CPU time + \item For many use cases, it is crucially needed: + \begin{itemize} + \item Video presentation at a given frame-rate, with format conversion and scaling + \item 3D scene rendering at display refresh rate + \item Windows and cursor composition at display refresh rate + \end{itemize} + \item Offloading is not (always) a magical solution: + \begin{itemize} + \item Fixed setup costs must be significantly lower than CPU processing time\\ + \textit{small operations are sometimes more efficient on-CPU} + \item Asynchronous interrupts can introduce latency compared to active polling\\ + \textit{but blocking the whole system is not always an option} + \end{itemize} + \item 2D hardware is usually more efficient and adapted than bringing up the GPU\\ + \textit{the GPU is a power-hungry war machine that solves problems at a price} + \end{itemize} +\end{frame} + +\begin{frame}{Graphics performance tips} + \begin{itemize} + \item Making the most of hardware can help a lot\\ + \textit{e.g. camera controllers can often provide format conversion and scaling} + \item Generally reducing the number of devices in the graphics pipeline + \item One major bottleneck in graphics pipelines is memory access: + \begin{itemize} + \item Memory buffer copies must be avoided at all costs (zero-copy) + \item Chained elements in a pipeline should share the same buffer\\ + \textit{hardware constraints are usually compatible in SoCs} + \end{itemize} + \item Redrawing full frames should be avoided when possible + \begin{itemize} + \item Local operations should be clipped + \item Buffer damage has to be accumulated in the multi-buffering case + \end{itemize} + \item Graphics in operating systems is usually best-effort + \item DSPs can usually provide real-time guarantees + \end{itemize} +\end{frame} + +\begin{frame}{Graphics hardware online references} + \small + \begin{itemize} + \item Wikipedia (\url{https://en.wikipedia.org/}): + \begin{itemize} + \item \href{https://en.wikipedia.org/wiki/Graphics_pipeline}{Graphics pipeline} + \item \href{https://en.wikipedia.org/wiki/Comparison_of_display_technology}{Comparison of display technology} + \item \href{https://en.wikipedia.org/wiki/List_of_video_connectors}{List of video connectors} + \item \href{https://en.wikipedia.org/wiki/Digital_signal_processor}{Digital signal processor} + \item \href{https://en.wikipedia.org/wiki/Graphics_processing_unit}{Graphics processing unit} + \item \href{https://en.wikipedia.org/wiki/Tiled_rendering}{Tiled rendering} + \item \href{https://en.wikipedia.org/wiki/Multiple_buffering}{Multiple buffering} + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Graphics hardware illustrations attributions} + \small + \begin{itemize} + \item \href{https://commons.wikimedia.org/wiki/File:ATI_Hercules_Card_1986.xcf}{ATI Hercules Card 1986: Jörgen Nixdorf, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:Intel@14nm@@Skylake@Skylake-X(LCC)@i7-7820X@SR3L5_DSC00646_(25952474218).jpg}{Intel Skylake 14nm: Fritzchens Fritz, CC0 1.0} + \item \href{https://commons.wikimedia.org/wiki/File:TN_display_closeup_300X.jpg}{TN display closeup 300X: Akpch, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:LCDmonitorscreenimage.jpg}{LCD monitor screen image: Koperczak, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:CRT_color_enhanced_unlabeled.png}{CRT color enhanced unlabeled: Grm\_wnr, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:Wiki_dell_lcd.jpg}{Dell TFT LCD: Szasz-Revai Endre, CC BY 2.5} + \item \href{https://commons.wikimedia.org/wiki/File:CeBIT_2011_Samstag_PD_115.JPG}{CeBIT 2011 3D Passport system: Bin im Garten, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:Bouquin_\%C3\%A9lectronique_iLiad_en_plein_soleil.jpg}{Bouquin électronique iLiad en plein soleil: Mathieu Despont, public domain} + \item \href{https://commons.wikimedia.org/wiki/File:Kindle_3_texture_(crop).jpg}{Kindle 3 E Ink screen: HorsePunchKid, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:DP_to_DVI_converter_unmounted.jpg}{DP to DVI converter unmounted: Antifumo, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:Anisotropic_filtering_en.png}{Anisotropic filtering: Thomas, Lampak, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:MipMap_Example_STS101.jpg}{MipMap Example STS101: Mike Hicks, NASA, CC BY-SA 3.0} + \item \href{https://peach.blender.org/}{Big Bug Bunny: Blender Foundation, CC BY 3.0} + \end{itemize} +\end{frame} diff --git a/slides/graphics-hardware/sunxi-tiled-format.png b/slides/graphics-hardware-system/sunxi-tiled-format.png similarity index 100% rename from slides/graphics-hardware/sunxi-tiled-format.png rename to slides/graphics-hardware-system/sunxi-tiled-format.png diff --git a/slides/graphics-hardware/sunxi-tiled-linear.png b/slides/graphics-hardware-system/sunxi-tiled-linear.png similarity index 100% rename from slides/graphics-hardware/sunxi-tiled-linear.png rename to slides/graphics-hardware-system/sunxi-tiled-linear.png diff --git a/slides/graphics-hardware/sunxi-tiled.png b/slides/graphics-hardware-system/sunxi-tiled.png similarity index 100% rename from slides/graphics-hardware/sunxi-tiled.png rename to slides/graphics-hardware-system/sunxi-tiled.png diff --git a/slides/graphics-hardware/tearing-glitch.jpg b/slides/graphics-hardware-system/tearing-glitch.jpg similarity index 100% rename from slides/graphics-hardware/tearing-glitch.jpg rename to slides/graphics-hardware-system/tearing-glitch.jpg diff --git a/slides/graphics-hardware/graphics-hardware.tex b/slides/graphics-hardware/graphics-hardware.tex deleted file mode 100644 index 4aa0368ec7..0000000000 --- a/slides/graphics-hardware/graphics-hardware.tex +++ /dev/null @@ -1,1043 +0,0 @@ -\section{Hardware Aspects} - -\subsection{Overview} - -\begin{frame}{Technological types of graphics hardware implementations} - \begin{itemize} - \item Dedicated graphics hardware is often used along a general-purpose CPU - \item Two commonly-used technologies, with typical pros/cons: - \end{itemize} - - \begin{center} - \small - \def\arraystretch{1.2} - \begin{tabular}{l|c|c} - & \textbf{Fixed-function} & \textbf{Programmable} \\ - \hline - \textbf{Technology} & Circuit & Software \\ - \textbf{Source form} & HDL & Source code \\ - \textbf{Product form} & Silicon, bitstream & Firmware binaries \\ - \textbf{Implementation} & FPGA, ASIC, SoC block & DSP, custom ISA \\ - \textbf{Arithmetic} & Fixed-point & Fixed-point, floating point \\ - \textbf{Clock rate / power} & Low & High \\ - \textbf{Pixel data access} & Queue (FIFO) & Memory \\ - \textbf{CPU control} & Direct registers & Mailbox \\ - \textbf{Die surface} & High & Low \\ - \textbf{Reusability} & Low & High \\ - \hline - \textbf{Example} & Allwinner SoCs Display Engine & TI TMS340 DSP \\ - \end{tabular} - \end{center} -\end{frame} - -\begin{frame}{Graphics memory and buffers} - \begin{itemize} - \item Pixel data is stored in memory buffers, called \textbf{framebuffers} - \item Framebuffers live either in: - \begin{itemize} - \item \textbf{System memory}: shared with the rest of the system (e.g. SDRAM or SRAM) - \item \textbf{Dedicated memory}: only for graphics (e.g. SGRAM) - \end{itemize} - \item Framebuffers that can be displayed are called \textbf{scanout framebuffers}\\ - \textit{hardware constraints don't always allow any framebuffer to be scanned out} - \item CPU access to pixel data in dedicated memory is neither always granted nor easy! - \item Graphics hardware \textbf{needs configuration} to interpret framebuffer pixel data\\ - \textit{pixel meta-data is rarely to never stored aside of the pixel data} - \end{itemize} -\end{frame} - -\begin{frame}{Display hardware overview} - \begin{itemize} - \item \textbf{Stream pixel data} to a display device, via a display interface - \item Internal pipeline with \textbf{multiple components} - \item Generally \textbf{fixed-function} hardware, pipeline sink only - \item Either \textbf{discrete} (video card) or \textbf{integrated} - \item Connected to the CPU (and RAM) via a \textbf{high-speed bus}:\\ - \textit{e.g. AXI with ARM, ISA, PCI, AGP, PCI-e with x86} - \end{itemize}~ - - \begin{minipage}[t]{0.45\textwidth} - \centering - \includegraphics[height=7em]{slides/graphics-hardware/ati-hercules-1986.png}\\ - \textit{\small A 1986 Hercules discrete video card} - \end{minipage} - \hfill - \begin{minipage}[t]{0.45\textwidth} - \centering - \includegraphics[height=7em]{slides/graphics-hardware/intel-skylake.jpg}\\ - \textit{\small An Intel processor with integrated graphics} - \end{minipage} -\end{frame} - -\begin{frame}{Common components of a display pipeline overview} - \begin{center} - \includegraphics[width=0.7\textwidth]{slides/graphics-hardware/display-pipe.pdf} - \end{center} - - \begin{enumerate} - \item \textbf{Framebuffers} for storing the pixel data\\ - \textit{streamed using a DMA engine} - \item \textbf{Planes} for associating a framebuffer with its dimensions and position\\ - \textit{composited into a single result on-the-fly} - \item \textbf{CRTC} for streaming resulting pixels with specific timings\\ - \textit{terminology comes from the legacy Cathode-Ray Tube Controller} - \item \textbf{Encoder} for meta-data addition and physical signal conversion - \item \textbf{Connector} for video signal, display data channel (DDC), hotplug detection - \item \textbf{Display} for decoding and displaying pixels (panel or monitor) - \end{enumerate} -\end{frame} - -\begin{frame}{Render hardware overview} - \textbf{Rendering} hardware includes a wide range of aspects (usual cases below): - \begin{itemize} - \item \textbf{Basic} pixel processing: - \begin{itemize} - \item Common operations: pixel format conversion, dithering, scaling, blitting and blending - \item Fixed-function hardware, pipeline sink and source - \end{itemize} - \item \textbf{Complex} pixel processing: - \begin{itemize} - \item Defined by the application: any computable operation - \item Programmable hardware (DSP), pipeline sink and source - \end{itemize} - \item \textbf{2D vector} drawing: - \begin{itemize} - \item Rasterization from equations, parameters and data (e.g. points) - \item Either fixed-function or programmable hardware (custom), pipeline source - \end{itemize} - \item \textbf{3D scene} rendering: - \begin{itemize} - \item Rasterization from programs (shaders) and data (e.g. vertices, lines, triangles textures) - \item Programmable hardware (GPU), pipeline source - \end{itemize} - \item Rendering can \textbf{always fallback} to general-purpose CPU operations - \end{itemize} -\end{frame} - -\begin{frame}{Video hardware overview} - \textbf{Video-oriented} hardware comes in different forms (usual cases below): - - \begin{itemize} - \item \textbf{Hardware video decoder} (VPU/video codec decoder) - \begin{itemize} - \item Decodes a video from compressed data (bitstream) to pixel frames - \item Fixed-function hardware, pipeline source - \end{itemize} - \item \textbf{Hardware video encoder} (VPU/video codec encoder) - \begin{itemize} - \item Encodes a video from pixel frames to compressed data (bitstream) - \item Fixed-function hardware, pipeline sink - \end{itemize} - \item \textbf{Camera sensors, video input, video broadcasting} (DVB) - \begin{itemize} - \item Receives/sends data in a given configuration from/to \textit{the outside} - \item Can be compressed data (bitstream) or raw pixel data - \item Fixed-function hardware, pipeline source - \end{itemize} - \end{itemize}~ -\end{frame} - -\subsection{Pipelines} - -\begin{frame}{I/O with graphics hardware, pipelines} - \begin{itemize} - \item Graphics hardware is \textbf{I/O-based} and interacts with pixel data - \item Pipeline elements have input-output abilities: - \begin{itemize} - \item \textbf{Source} components: \textbf{feed pixel data}: \textit{e.g. camera} - \item \textbf{Sink} components: \textbf{grab pixel data}: \textit{e.g. display} - \end{itemize} - \item Some components are \textbf{both a source and a sink}: \textit{e.g. converters} - \item Graphics components can be chained in \textbf{pipelines}\\ - \textit{Usually from a source-only element to a sink-only element} - \end{itemize} - - \vspace{-2em} - \begin{center} - \includegraphics[width=0.7\textwidth]{slides/graphics-hardware/pipeline-sink-source.pdf} - \end{center} -\end{frame} - -\begin{frame}{Building complex pipelines} - \begin{itemize} - \item Display, rendering and video elements are chained from source(s) to sink(s) - \item On source-sink boundaries: - \begin{itemize} - \item Mutually-supported pixel format (or conversion) - \item Mutually-accessible memory (or copy) or internal FIFO - \end{itemize} - \item Target frame rate (\(fps\)) gives a time budget for pipeline traversal: - \( t_0 + t_2 + t_4 + t_5 < fps^{-1},~ t_0 + t_3 < fps^{-1},~ t_1 + t_4 + t_5 < fps^{-1} \) - \end{itemize} - - \begin{center} - \includegraphics[width=0.7\textwidth]{slides/graphics-hardware/pipeline-complex.pdf} - \end{center} -\end{frame} - -\begin{frame}{Debugging pipelines} - \begin{itemize} - \item Debugging a complex pipeline can become \textbf{hard}: - \begin{itemize} - \item Many elements in the chain means many possibilities for trouble - \item A single point of failure is enough - \item Can be difficult to identify - \end{itemize} - \item Debugging \textbf{tips}: - \begin{itemize} - \item Validating each element separately - \item Dumping (intermediary) framebuffers to memory/storage for validation - \item Using hardware pattern generators - \end{itemize} - \item Common \textbf{pitfalls}: - \begin{itemize} - \item Mismatch between advertized format and reality - \item Lack of output-input sync between elements - \item Input data updated too fast - \end{itemize} - \end{itemize} -\end{frame} - -\subsection{Display Devices} - -\begin{frame}{Visual display technologies generalities} - \begin{itemize} - \item Pixel data is pushed from the display interface to a visible surface\\ - \textit{using a dedicated controller on the display device} - \item Pixels are split into 3 color cells (R-G-B) - \begin{itemize} - \item The human eye naturally merges light from the 3 cells - \end{itemize} - \item Pixel frames are displayed as (physical) arrays of color cells - \item Smooth sequences require at least 24 fps, more is usually best - \end{itemize}~ - - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[height=6.5em]{slides/graphics-hardware/pixel-array.jpg}\\ - \textit{\small Pixel color cells on a LCD TN panel} - \end{minipage} - \hfill - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[height=6.5em]{slides/graphics-hardware/pixel-array-text.jpg}\\ - \textit{\small A pixel array displaying text} - \end{minipage} -\end{frame} - -\begin{frame}{CRT display technology} - \begin{itemize} - \item Color \textbf{cathode-ray tubes} (CRTs), since the 1950s: - \begin{itemize} - \item Using electron beams to excite a phosphorescent screen - \item Beams are guided by magnetic deflection - \item One beam for each color with increased intensity for increased luminosity - \item High energy consumption - \item High contrast, low response time (\(1-10~\mu s\)) - \item Other issues: monitor size, burn-in (screensavers), remnant magnetic field (degaussing), high voltages and magnetic fields - \end{itemize} - \end{itemize} - - \begin{center} - \includegraphics[height=8em]{slides/graphics-hardware/crt-color.png} - \end{center} -\end{frame} - -\begin{frame}{Plasma display panels technology} - \begin{itemize} - \item \textbf{Plasma display panels} (PDPs), since the 1990s-2000s: - \begin{itemize} - \item Using gas cells brought to plasma state to strike light-emitting phosphor - \item Flat array of cells, scales to large surfaces - \item Medium energy consumption (depends on luminance) - \item High contrast, low response time (\(\leq 1~\mu s\)) - \item Other issues: burn-in - \item Gradually being \textbf{deprecated} in favor of other flat-panel technologies - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{LCD display technology} - \begin{itemize} - \item \textbf{Liquid crystal displays} (LCDs) using \textbf{Thin-film-transistors} (TFT): - \begin{itemize} - \item Using the electrically-controlled alignment of crystal structures to block light - \item Does not emit light: needs an always-on backlight source (usually LEDs) - \item Low energy consumption (depends on backlight) - \item Medium to low contrast, medium response time (\(1-10~ms\)) - \item \textbf{Twisted nematic} (TN): limited color quality and viewing angles, since the 1980s - \item \textbf{In-plane switching} (IPS): improved color and viewing angles, since the 2000s - \end{itemize} - \end{itemize} - - \begin{center} - \includegraphics[height=6em]{slides/graphics-hardware/lcd-ips-shape.jpg}\\ - \textit{\small Chevron shapes that improve the viewing angle on IPS LCDs} - \end{center} -\end{frame} - -\begin{frame}{OLED display technology} - \begin{itemize} - \item \textbf{Organic light-emitting diodes} (OLEDs), since 2010: - \begin{itemize} - \item Using organic compounds (carbon-based) to emit light as R-G-B LEDs - \item Allows flat and flexible surfaces, with a large viewing angle - \item Low energy consumption - \item Very high contrast, low response time (\(1-10~\mu s\)) - \item Issues: burn-in, independent cells aging, affected by UV light - \item Rapidly becoming \textbf{very popular} and used - \end{itemize} - \end{itemize} - - \begin{center} - \includegraphics[height=8em]{slides/graphics-hardware/oled-display.jpg}\\ - \textit{\small A flexible OLED display panel} - \end{center} -\end{frame} - -\begin{frame}{EPD display technology} - \begin{itemize} - \item \textbf{Electrophoretic displays} (EPDs), since the 2000s: - \begin{itemize} - \item Using black and white electrically-charged ink-like particles in oil\\ - \textit{e.g. positive charge for black and negative for white} - \item Electric fields attract one or the other color with current flow\\ - \textit{the particles stay in place after they were moved} - \item Using incident light, does not emit light itself - \item Very low consumption (only for changes) - \item Very high response time (\(100~ms\)) and ghosting - \end{itemize} - \end{itemize}~ - - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[height=6.5em]{slides/graphics-hardware/e-reader.jpg}\\ - \textit{\small An e-reader with an EPD display} - \end{minipage} - \hfill - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[height=6.5em]{slides/graphics-hardware/epd-detail.jpg}\\ - \textit{\small Detail of an EPD display} - \end{minipage} -\end{frame} - -\begin{frame}{Display panels integration and monitors} - \begin{itemize} - \item Panels come with a dedicated \textbf{controller} to: - \begin{itemize} - \item Decode pixels from the display interface - \item Electrically control the color cells of the panel - \end{itemize} - \item Panels can be used \textbf{standalone}, usually with: - \begin{itemize} - \item A single simple display interface - \item No standardized connector, weak and short cables - \item Only native dimensions supported and little to no configuration - \end{itemize} - \item Or they can be \textbf{integrated in monitors}, usually with: - \begin{itemize} - \item More complex and multiple display interfaces - \item Standardized connectors, external cables - \item Configuration (buttons and UI overlay), multiple dimensions - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Display panels integration and monitors (illustrated)} - \begin{center} - \includegraphics[width=0.5\textwidth]{slides/graphics-hardware/display-panel.jpg}\\ - \textit{\small A display panel used standalone with an embedded board} - \end{center} -\end{frame} - -\subsection{Display Interfaces} - -\begin{frame}{Display panels refreshing} - \begin{itemize} - \item Most display panel technologies need frequent pixel updates: \textbf{refreshing} - \begin{itemize} - \item Colors would fade out without refresh - \item Panels usually lack internal memory to self-refresh - \end{itemize} - \item Requires a \textbf{fixed refresh rate}: - \begin{itemize} - \item Capped by the display technology or display interface - \item Directly impacts smoothness: minimum is usually \(30~Hz\) - \item The whole frame must be sent over each time - \end{itemize} - \item Requires \textbf{synchronization points}: - \begin{itemize} - \item Vertical: beginning of a new frame - \item Horizontal: beginning of a new line - \end{itemize} - \item Requires \textbf{blank times}: - \begin{itemize} - \item Account for line/frame preparation time - \item Initially for CRT electron gun repositioning - \item More or less useful depending on the technology - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Display timings and modes} - \begin{itemize} - \item Timings coordinate how a frame is \textbf{transmitted to a display over time} - \item Required signals are synced to a \textbf{pixel clock} (time base) - \item Display timings are split in a few stages (horizontal and vertical): - \begin{enumerate} - \item Sync pulse (vsync/hsync) - \item Back porch (vbp/hbp): blanking - \item Active region (vactive/hactive) - \item Front porch (vfp/hfp): blanking - \end{enumerate} - \item Pixels are transmitted during the \textbf{horizontal active region} (one line) - \item A \textbf{display mode} groups all timing and signal characterization information - \begin{itemize} - \item Signals are \textbf{generated by the CRTC} according to the display mode - \end{itemize} - \item Monitors usually support \textbf{multiple modes} (and dimensions): - \begin{itemize} - \item Standard modes are defined by VESA - \item Extra specific modes for the monitor can be supported - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Display timings and modes (illustrated)} - \begin{center} - \includegraphics[width=0.8\textwidth]{slides/graphics-hardware/display-timings.pdf} - \end{center} - - \begin{itemize} - \item The unit for horizontal stages is \textbf{one pixel clock period} - \item The unit for vertical stages is \textbf{one line's duration} - \end{itemize} -\end{frame} - -\begin{frame}{Display timings and modes (panel example)} - - \begin{minipage}[b]{0.35\textwidth} - \centering - \vspace{2em} - \includegraphics[width=\textwidth]{slides/graphics-hardware/timings-table.png} - \textit{\small AT070TN94 panel datasheet} - \vfill~ - \end{minipage} - \hfill - \begin{minipage}[b]{0.6\textwidth} - \small - \begin{itemize} - \item \(hsync = thpw = 20 \in \llbracket 1;40 \rrbracket\)\\ - \(hbp = thb - thpw = 46 - 20 = 26\) (from diagram)\\ - \(hactive = thd = 800\)\\ - \(hfp = thfp = 210 \in \llbracket 16;354 \rrbracket\)\\ - \(htotal = hsync + hbp + hactive + hfp = 1056\) - - \item \(vsync = tvpw = 10 \in \llbracket 1;20 \rrbracket\)\\ - \(vbp = tvb - tvpw = 23 - 10 = 13\) (from diagram)\\ - \(vactive = tvd = 480\)\\ - \(vfp = tvfp = 22 \in \llbracket 7;147 \rrbracket\)\\ - \(vtotal = vsync + vbp + vactive + vfp = 525\) - \item 1 frame takes: \(vtotal \times htotal = 554400~t_{clk}\)\\ - 60 frames take: \(vtotal \times htotal \times 60 = 33264000~t_{clk}\)\\ - 60 fps requires: \(f_{clk} \geq 33.264~MHz\) - \end{itemize} - \end{minipage} - \vspace{0.5em} - - \begin{itemize} - \item Panels usually support a \textbf{range of timings} - \item The pixel clock rate is often \textbf{rounded} (refresh rate not always strictly respected) - \end{itemize} - -\end{frame} - -\begin{frame}{Side-channel and identification} - \begin{itemize} - \item Monitor display connectors often come with a \textbf{Display Data Channel} (DDC) - \begin{itemize} - \item Side-bus to allow communication between host and display - \item Usually based on I2C, quite slow (\(\approx 100~kHz\)) - \end{itemize} - \item DDC provides access to the \textbf{Extended Display Identification Data} (EDID) - \begin{itemize} - \item Contains the list of supported modes in a standard format - \item Usually stored in an EEPROM at I2C address \(0x50\) - \end{itemize} - \item Another common monitor signal is \textbf{Hotplug Detect} (HPD) - \begin{itemize} - \item Connected to a pin of the connector, asserted with a cable plugged - \item Can be wired to an interrupt pin to detect connection changes - \end{itemize} - \item Direct panel interfaces (not monitors) usually lack DDC, EDID and HPD - \begin{itemize} - \item Panel is always considered connected - \item Modes need to be known in advance - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Extra display interface features and EDID extensions} - \begin{itemize} - \item The EDID standard keeps evolving and exposes new features through extensions - \item Configuration data for each feature is embedded in the EDID - \item More or fewer features are supported depending on the display interface - \item Common extra display interface features: - \begin{itemize} - \item \textbf{Interlaced}: Every other pixel line is sent at a time, alternating between top-fields and bottom-fields; Allows faster refreshing for CRTs, needs deinterlacing for progressive panels; - \item \textbf{Audio}: Send audio in addition to pixels, during blanking periods; - \item \textbf{Stereoscopy}: Pixel data is split between two screens that show a different geometrical perspective, providing 3D perception; - \item \textbf{Variable Refresh Rate} (VRR): Pixel data can be sent at any point and does not need to conform to a given refresh rate; - \item \textbf{Consumer Electronic Control} (CEC): Remote control features on a dedicated bus; - \item \textbf{High-Bandwidth Digital Content Protection} (HDCP): Anti-copy protection - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Types of display interfaces} - \begin{itemize} - \item Legacy display interfaces are usually \textbf{analog}: - \begin{itemize} - \item Transmission through a DAC-based chain - \item Lack of precision, noise and chain error: not pixel-perfect, capped - \item Requires few signal pins (1 per color channel and sync or less) - \end{itemize} - \item Recent interfaces are usually \textbf{digital}: - \begin{itemize} - \item Encoded binary transmission, usually with dedicated clock - \item Encoders contain a controller (logic) and a PHY (signal) - \item Pixel data is expected to be bit-perfect \textit{(but noise still exists)} - \end{itemize} - \item Digital interfaces can be \textbf{parallelized}: - \begin{itemize} - \item One signal per color bit (e.g. 24 signals for 24-bit RGB), clock and sync - \item One clock cycle for one pixel (low clock rate) - \end{itemize} - \item Or they can be \textbf{serialized}: - \begin{itemize} - \item Pixel data is sent over physical lanes (one or more) - \item One clock cycle for one bit on each lane (high clock rate) - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Types of display interfaces (illustrated)} - \begin{center} - \includegraphics[width=0.7\textwidth]{slides/graphics-hardware/display-interface-encoders.pdf} - \end{center} -\end{frame} - -\begin{frame}{VGA display interface} - \begin{itemize} - \item \textbf{Video Graphics Array} (VGA), since 1987 (IBM) - \begin{itemize} - \item \textbf{Analog} pixel data on 3 pins (R-G-B), DAC encoder to \(0.7~V\) peak-to-peak - \item \textbf{Per-channel pixel streaming} (voltage change), following mode timings - \item \textbf{Hsync} and \textbf{vsync} signals, I2C SDA and SDL \textbf{DDC} signals - \item Hotplug detection with R/G/B pins \textbf{current sensing} - \item Using a DB-15 connector for signals: - \end{itemize} - \begin{center} - \includegraphics[height=2em]{slides/graphics-hardware/vga-pinout.pdf} - \end{center} - \begin{itemize} - \item \textbf{Pixel}: Red, Green, Blue {\footnotesize(1, 2, 3)}, Ground returns {\footnotesize(6, 7, 8)} - \item \textbf{Sync}: Hsync, Vsync {\footnotesize(13, 14)} - \item \textbf{Side}: DDC SDA, DDC SCL {\footnotesize(12, 15)} - \item \textbf{Power}: \(+5~V\) {\footnotesize(9)}, Ground {\footnotesize(10)} - \end{itemize} - \end{itemize} - -\end{frame} - -\begin{frame}{VGA display interface (illustrated)} - \begin{center} - \includegraphics[width=\textwidth]{slides/graphics-hardware/a13-olinuxino-vga.png} - \end{center} - - \begin{itemize} - \item Very basic VGA encoder from parallel signals, DDC excluded - \item Resistor ladder for digital-to-analog conversion\\ - \textit{using SN74ALVC244 voltage level shifters (\(1.8~V\) to \(3.3~V\)), clamping diodes} - \item 6 most-significant bits only, 2 least-significant bits set to 0\\ - \textit{D0-D1, D8-D9 and D16-D17 are not routed} - \end{itemize} -\end{frame} - -\begin{frame}{DVI display interface} - \begin{itemize} - \item \textbf{Digital Visual Interface} (DVI), since 1999 (DDWG) - \begin{itemize} - \item \textbf{DVI-A}: Analog only, comparable to VGA - \item \textbf{DVI-D}: Digital only, single-link (3 data lanes) or dual-link (6 data lanes) - \item \textbf{DVI-I}: Both analog and digital supported, single-link or dual-link - \item Digital serial link using \textbf{Transition-Minimized Differential Signaling} (TMDS) - \item Dedicated \textbf{DDC} and \textbf{HPD} signals - \item Using a subset or variation of the full \textbf{DVI-I connector} for signals: - \end{itemize} - \begin{center} - \includegraphics[height=2em]{slides/graphics-hardware/dvi-pinout.pdf} - \end{center} - \begin{itemize} - \item \textbf{TMDS}: Data+ {\footnotesize(2, 5, 10, 13, 18, 21)}, Data- {\footnotesize(1, 4, 9, 12, 17, 20)}, Clock {\footnotesize(23, 24)} - \item \textbf{Analog pixel}: Red, Green, Blue {\footnotesize(C1, C2, C3)}, Ground {\footnotesize(C5)} - \item \textbf{Analog sync}: Hsync, Vsync {\footnotesize(C4, 8)} - \item \textbf{Side}: DDC SDA, DDC SCL {\footnotesize(7, 6)}, HPD {\footnotesize(16)} - \item \textbf{Power}: \(+5~V\) {\footnotesize(14)}, Ground {\footnotesize(15)} - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{HDMI display interface} - \begin{itemize} - \item \textbf{High-Definition Multimedia Interface} (HDMI), since 2002 (HDMI Forum) - \begin{itemize} - \item Similar to \textbf{DVI-D}: no analog, 3 TMDS data lanes (R-G-B) - \item Adding the use of AVI infoframes for meta-data and audio - \item \textbf{High bandwidth} \((\leq 48~Gbit/s)\) (2.1) and clock speeds \((\leq 340~MHz)\) - \item \textbf{Extra features}: Audio, CEC (1.2), HDR (1.3), 4K (1.4), Stereoscopy (1.4),\\8K-10K (2.1), DSC (2.1), HFR (\(120~Hz\)), per-frame HDR (2.1) - \item Using a dedicated (and proprietary) \textbf{HDMI connector} for signals: - \end{itemize} - \begin{center} - \includegraphics[height=2em]{slides/graphics-hardware/hdmi-pinout.pdf} - \end{center} - \begin{itemize} - \item \textbf{TMDS}: Data+ {\footnotesize(1, 4, 7)}, Data- {\footnotesize(3, 6, 9)}, Clock {\footnotesize(10, 12)} - \item \textbf{Side}: SDA, SCL {\footnotesize(16, 15)}, HPD {\footnotesize(19)}, CEC {\footnotesize(13)} - \item \textbf{Power}: \(+5~V\) {\footnotesize(18)}, Ground {\footnotesize(17)} - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{DP/eDP display interface} - \begin{itemize} - \item \textbf{DisplayPort} (DP), since 2008 (VESA) - \begin{itemize} - \item Digital serial link with 4 data lanes using \textbf{Low-Voltage Differential Signaling} (LVDS) - or TMDS for DP Dual-Mode (DP++), compatible with DVI-D and HDMI - \item Using \textbf{packets} for video/audio data and meta-data - \item Auxiliary channel encapsulating I2C DDC, CEC and more (e.g. USB) - \item \textbf{High bandwidth} \((\leq 77.37~Gbit/s)\) (2.0) - \item \textbf{Extra features}: Audio, CEC, HDR (1.4), 4K (1.3), Stereoscopy (1.2),\\8K (1.3-1.4), 10K-16K (2.0) - \item \textbf{Multi-Stream Transport} (MST) to chain displays - \item Using a dedicated (and proprietary) \textbf{DisplayPort connector} for signals: - \end{itemize} - \begin{center} - \includegraphics[height=2em]{slides/graphics-hardware/dp-pinout.pdf} - \end{center} - \begin{itemize} - \item \textbf{LVDS/TMDS}: ML+ {\footnotesize(1, 4, 7, 10)}, ML- {\footnotesize(3, 6, 9, 12)} - \item \textbf{Side}: AUX+ (15), AUX- (17), HPD (18) - \item \textbf{Power}: \(+3.3~V\) {\footnotesize(20)}, Ground {\footnotesize(2, 5, 8, 11, 16)} - \end{itemize} - \item \textbf{Embedded DisplayPort} (eDP) for internal panels (without connector) - \end{itemize} -\end{frame} - -\begin{frame}{LVDS and DSI display interfaces} - -% TODO split and add photo of custom LVDS connector as well as a DSI panel (librem 5 dvt?) - - \begin{itemize} - \item \textbf{Low Voltage Differential Signaling} (LVDS) - \begin{itemize} - \item Generic digital serial link with clock and data signals (3-4 lanes) using LVDS - \item Pixel data and control (vsync, hsync, display enable) sent as bits on lanes - \item Uses a fixed mode, no DDC, no packets - \item Specified with the JEIDA, LDI, DSIM and VESA specifications - \item For internal panels, exposed with specific connectors - \item Common for laptop panels - \end{itemize} - \end{itemize} - \begin{itemize} - \item \textbf{Display Serial Interface} (DSI), since 2006 (MIPI) - \begin{itemize} - \item Digital serial link with clock and up to 4 data lanes using LVDS - \item Using \textbf{packets} for video data and meta-data - \item Commands for configuration can be issued with the \textbf{DSI Command Set} (DCS)\\ - \textit{Generic base with proprietary vendor-specific extensions} - \item For internal panels, exposed with specific connectors - \item Common for mobile devices' panels - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{DPI display interface} - \begin{itemize} - \item \textbf{Display Parallel Interface} (DPI) - \begin{itemize} - \item Generic parallel digital interface, with 1 signal per color bit, clock and sync - \item Exists with different numbers of bits: 24 (8-8-8), 18 (6-6-6) or 16 (5-6-5)\\ - \textit{Dithering is required when using 16 or 18 bits} - \item Sends pixel data bits following mode timings - \item Base signals: color data bits, vsync, hsync - \item Extra signals: display enable (DE) - \item Beware: sync and DE signals can be active-high or active-low - \item For internal panels, requires many signals - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Bridges/transcoders} - \begin{itemize} - \item Not every display interface is supported by the hardware at hand - \item Bridges or transcoders are used to translate from one interface to another - \item They are composed of a decoder and an encoder (in a single package) - \item Usually standalone and transparent, often only replicate timings\\ - \textit{but some can have a side-bus for configuration and fine-tuning} - \item Example: VGA interfaces are usually bridged from digital interfaces nowadays - \end{itemize} - - \begin{center} - \includegraphics[height=6em]{slides/graphics-hardware/dp-dvi-bridge.jpg}\\ - \textit{\small A DP to DVI bridge} - \end{center} -\end{frame} - -\subsection{Render Accelerators} - -\begin{frame}{Digital Signal Processors} - \begin{itemize} - \item Digital Signal Processors (DSPs) allow programmable image signal processing\\ - \textit{can also be used for implementing 2D rendering primitives} - \item Using a dedicated Instruction Set Architectures (ISA) - \item Arithmetic implementations are either: - \begin{itemize} - \item \textbf{fixed-point}: simple hardware implementation, fixed range (usually 16.16)\\ - \textit{16 bits for the integer part and 16 bits for the decimal part} - \item \textbf{floating-point}: complex implementations, trade-off between range and precision - \end{itemize} - \item Usually more power-efficient than general-purpose CPUs - \item Depending on the DSP, the software can be: - \begin{itemize} - \item A \textbf{standalone firmware}, usually developed from vendor libraries (C/C++/ASM) - \item A \textbf{real-time operating system} (RTOS) application (C/C++/...) - \end{itemize} - \item Can be used standalone in a video pipeline or to offload a CPU - \item Modern DSPs can be multi-core and feature various I/O controllers - \end{itemize} -\end{frame} - -\begin{frame}{Dedicated hardware accelerators} - \begin{itemize} - \item Fixed-function hardware can be used for accelerating specific operations - \begin{itemize} - \item Implemented as hardware circuits in Systems on a Chip (SoCs) or DSPs - \item Implemented as logic configuration bitstream in FPGAs - \end{itemize} - \item Implement a configurable fixed pipeline for image operations - \item Accessed and configured through specific registers exposed via a bus - \begin{itemize} - \item Global configuration registers to build the pipeline between blocks - \item Configuration registers for each block - \item Kick and status registers - \end{itemize} - \item Usually very power-efficient and very fast - \end{itemize} -\end{frame} - -\begin{frame}{Dedicated hardware accelerators (illustrated)} - \begin{center} - \includegraphics[width=0.7\textwidth]{slides/graphics-hardware/g2d-block.png}\\ - \textit{An example hardware pipeline for a 2D graphics block}\\ - \end{center} -\end{frame} - -\begin{frame}{Graphics Processing Unit} - \begin{itemize} - \item Graphics Processing Units (GPUs) are \textbf{3D rendering hardware} implementations\\ - \textit{the term no longer designates all graphics-processing hardware} - \item Operate on 3D graphics primitives: \textbf{points (vertices), lines and triangles} - \item Generate a 2D view (viewport) from a given perspective - \begin{itemize} - \item Objects are formed of interconnected triangles - \item A color can be applied to each vertex and interpolated - \item Textures can be applied to objects with texture element to vertex mapping - \item Lighting is applied from various sources - \end{itemize} - \item Expected to render at \textbf{display scanout rate} (pseudo real-time) - \begin{itemize} - \item Usually not photo-realistic methods as ray tracing or photon mapping - \item Extremely efficient compared to any general-purpose CPU - \end{itemize} - \item GPUs are also used for general-purpose computing with GPGPU - \end{itemize} -\end{frame} - -\begin{frame}{Graphics Processing Unit architectures} - \begin{itemize} - \item GPUs implement a pipeline of hacks accumulated since the 1970s - \item GPU hardware architectures evolved over time: - \begin{itemize} - \item From \textbf{fixed-function} configurable hardware block pipelines - \item To pipelines with both fixed blocks and specialized \textbf{programmable} processing units - \end{itemize} - \item Shaders are programs that run at different steps of the pipeline: - \begin{itemize} - \item \textbf{vertex shaders}: define the position, texture coordinates and lighting of each vertex - \item \textbf{geometry shaders}: generate new primitives from the provided ones - \item \textbf{tesselation shaders}: perform vertex sub-division (e.g. Catmull-Clark) - \item \textbf{fragment/pixel shaders}: perform rasterization for each output pixel - \end{itemize} - \item Scenes can be rendered with multiples passes and multiple shaders - \end{itemize} -\end{frame} - -\begin{frame}{Graphics Processing Unit architectures (illustrated)} - \begin{center} - \includegraphics[width=0.5\textwidth]{slides/graphics-hardware/gpu-pipeline.png}\\ - \textit{\small A programmable GPU pipeline} - \end{center} -\end{frame} - -\begin{frame}{Graphics Processing Unit techniques} - \begin{itemize} - \item GPUs are optimized for \textbf{performance} and \textbf{good-looking results} - \item \textbf{Texture sampling} can easily cause aliasing (at a distance) - \begin{itemize} - \item \textbf{Bilinear and trilinear filtering} is often used - \item \textbf{Anisotropic filtering} provides best visual results - \item \textbf{Mip-maps} provide the same texture at different sizes - \item \textbf{Multi-Sample Anti-Aliasing} (MSAA) averages colors from multiple points - \end{itemize} - \item \textbf{Texture compression} reduces memory pressure (e.g. S3TC, ASTC) - \item \textbf{Normal mapping/bump maps} provide increased details with low vertex count\\ - \textit{affects light path calculation} - \item Avoiding \textbf{useless rendering} operations: - \begin{itemize} - \item Occluded surfaces are not rendered (visible surface determination) - \item Using a \textbf{depth/z-buffer} to keep track of the z-order for each output pixel - \item Culling for early vertex removal: back-face, view frustum, z-buffer - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Graphics Processing Unit techniques (illustrated)} - \begin{minipage}{0.45\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-hardware/anisotropic-filtering.png}\\ - \textit{Comparison of tri-linear and anisotropic anti-aliasing filtering} - \end{minipage} - \hfill - \begin{minipage}{0.45\textwidth} - \centering - \includegraphics[height=7em]{slides/graphics-hardware/mip-map.jpg}\\ - \textit{A mip-mapped texture representation} - \end{minipage} -\end{frame} - -\begin{frame}{Graphics Processing Unit internals} - \begin{itemize} - \item A \textbf{command stream} is parsed and used to configure the pipeline - \item Shader cores have highly-specialized ISAs adapted for geometry: - \begin{minipage}[b]{0.45\textwidth} - \begin{itemize} - \item Vector operations, SIMD - \item Trigonometric operations - \end{itemize} - \end{minipage} - \begin{minipage}[b]{0.45\textwidth} - \begin{itemize} - \item Interpolation operations - \item Usually few conditionals - \end{itemize} - \end{minipage} - \item Texture access is provided by a \textbf{Texture Mapping Unit} (TMU) - \begin{itemize} - \item Caching is used to reduce memory pressure - \end{itemize} - \item Modern GPUs sometimes have a \textbf{unified shader core}\\ - \textit{allows efficient hardware resources usage, with complex scheduling} - \item Shading cores are \textbf{duplicated} and work in parallel (especially rasterization) - \item Some architectures implement tiled processing: - \begin{itemize} - \item Output is divided in tiles (clipping areas) and distributed to cores - \item Each rasterized tile is written to the output framebuffer separately - \end{itemize} - \end{itemize} -\end{frame} - -\subsection{System Integration, Memory and Performance} - -\begin{frame}{Graphics integration and memory} - \begin{itemize} - \item Graphics devices integrated in larger systems need two main interfaces: - \begin{itemize} - \item \textbf{Control interface} (low speed): to program the device from the main CPU - \item \textbf{Memory interface} (high speed): to read the source data and write their framebuffer - \end{itemize} - \item Other usual required elements: clocks, interrupts, reset - \item Both the graphics device and the CPU need to access the memory - \item Different types of memory used by graphics hardware: - \begin{itemize} - \item \textbf{graphics memory}: dedicated memory attached to the graphics device\\ - \textit{the memory is made available to the CPU through the memory interface} - \item \textbf{dedicated system memory}: a reserved contiguous area of system memory\\ - \textit{required when the device has no mapping unit} - \item \textbf{system memory pages}: any system memory page can be mapped for access\\ - \textit{for devices with a dedicated IOMMU and graphics address remapping table (GART)} - \end{itemize} - \item Since the two parties access the same memory, cache can become incoherent - \item Cache must be \textbf{synchronized} before reading and after writing, or \textbf{disabled} - \end{itemize} -\end{frame} - -\begin{frame}{Shared graphics memory access} - \begin{itemize} - \item Concurrent access to memory can lead to trouble: - \begin{itemize} - \item Concurrent read-write accesses result in partially-updated data - \item Concurrent write-write accesses result in incoherent data - \end{itemize} - \item Common issue with display hardware: tearing - \begin{itemize} - \item The framebuffer is scanned out at a fixed rate (e.g. \(60~fps\)) - \item Any modification during scan out will result in a \textbf{partial update} - \item Causes an unpleasant \textbf{visual glitch} effect - \end{itemize} - \item Solved using (at least) \textbf{double-buffering}: - \begin{itemize} - \item The displayed buffer (\textbf{front buffer}) is kept intact - \item Another buffer (\textbf{back buffer}) is used for drawing the next contents - \item Front and back buffer are exchanged with \textbf{page flipping}, during vertical blanking - \item Using more buffers is possible if rendering can be done in advance - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Graphics shared memory access (illustrated tearing)} - \begin{center} - \includegraphics[width=0.7\textwidth]{slides/graphics-hardware/tearing-glitch.jpg}\\ - \textit{\small Tearing example: data is updated during scanout} - \end{center} -\end{frame} - -\begin{frame}{Graphics memory constraints and performance} - \begin{itemize} - \item Fixed-pipeline 2D graphics hardware usually streams pixels through FIFOs - \item A DMA engine fetches pixel data from framebuffer memory - \item To simplify the logic and optimize memory access, memory constraints may apply:\\ - \begin{itemize} - \item The start address of each line needs to be aligned to \(2^n\) - \item The byte size of each line needs to be aligned to \(2^n\) - \end{itemize} - \item The \textbf{stride} or \textbf{pitch} describes the line byte size - \begin{itemize} - \item Calculated as: \(ALIGN(width \times bpp / 8,~2^n)\) - \item The size of a framebuffer becomes (single-planar): \(stride \times height\)\\ - \textit{memory may be over-allocated to satisfy alignment constraints} - \end{itemize} - \item Pixel order in memory may not follow raster order: - \begin{itemize} - \item Optimized depending on the hardware architecture - \item Optimized for efficient memory access - \item Tiled orders are frequent for parallel hardware - \item Framebuffer sizes are calculated with a specific formula - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Tiled framebuffer format example} - \begin{minipage}{0.45\textwidth} - \centering - \textit{\small The Allwinner VPU tiled format} - \end{minipage} - \hfill - \begin{minipage}{0.45\textwidth} - \centering - \vspace{1em} - \includegraphics[width=0.35\textwidth]{slides/graphics-hardware/sunxi-tiled-format.png}\\ - \end{minipage} - \vspace{2em} - \begin{minipage}[b]{0.45\textwidth} - \centering - \vspace{1em} - \includegraphics[width=\textwidth]{slides/graphics-hardware/sunxi-tiled-linear.png} - \textit{\small A tiled framebuffer read in raster order} - \end{minipage} - \hfill - \begin{minipage}[b]{0.45\textwidth} - \centering - \vspace{1em} - \includegraphics[width=\textwidth]{slides/graphics-hardware/sunxi-tiled.png} - \textit{\small The same framebuffer read properly} - \end{minipage} -\end{frame} - -\begin{frame}{Offloading graphics to hardware} - \begin{itemize} - \item Offloading graphics to hardware frees up significant CPU time - \item For many use cases, it is crucially needed: - \begin{itemize} - \item Video presentation at a given frame-rate, with format conversion and scaling - \item 3D scene rendering at display refresh rate - \item Windows and cursor composition at display refresh rate - \end{itemize} - \item Offloading is not (always) a magical solution: - \begin{itemize} - \item Fixed setup costs must be significantly lower than CPU processing time\\ - \textit{small operations are sometimes more efficient on-CPU} - \item Asynchronous interrupts can introduce latency compared to active polling\\ - \textit{but blocking the whole system is not always an option} - \end{itemize} - \item 2D hardware is usually more efficient and adapted than bringing up the GPU\\ - \textit{the GPU is a power-hungry war machine that solves problems at a price} - \end{itemize} -\end{frame} - -\begin{frame}{Graphics performance tips} - \begin{itemize} - \item Making the most of hardware can help a lot\\ - \textit{e.g. camera controllers can often provide format conversion and scaling} - \item Generally reducing the number of devices in the graphics pipeline - \item One major bottleneck in graphics pipelines is memory access: - \begin{itemize} - \item Memory buffer copies must be avoided at all costs (zero-copy) - \item Chained elements in a pipeline should share the same buffer\\ - \textit{hardware constraints are usually compatible in SoCs} - \end{itemize} - \item Redrawing full frames should be avoided when possible - \begin{itemize} - \item Local operations should be clipped - \item Buffer damage has to be accumulated in the multi-buffering case - \end{itemize} - \item Graphics in operating systems is usually best-effort - \item DSPs can usually provide real-time guarantees - \end{itemize} -\end{frame} - -\begin{frame}{Graphics hardware online references} - \small - \begin{itemize} - \item Wikipedia (\url{https://en.wikipedia.org/}): - \begin{itemize} - \item \href{https://en.wikipedia.org/wiki/Graphics_pipeline}{Graphics pipeline} - \item \href{https://en.wikipedia.org/wiki/Comparison_of_display_technology}{Comparison of display technology} - \item \href{https://en.wikipedia.org/wiki/List_of_video_connectors}{List of video connectors} - \item \href{https://en.wikipedia.org/wiki/Digital_signal_processor}{Digital signal processor} - \item \href{https://en.wikipedia.org/wiki/Graphics_processing_unit}{Graphics processing unit} - \item \href{https://en.wikipedia.org/wiki/Tiled_rendering}{Tiled rendering} - \item \href{https://en.wikipedia.org/wiki/Multiple_buffering}{Multiple buffering} - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Graphics hardware illustrations attributions} - \small - \begin{itemize} - \item \href{https://commons.wikimedia.org/wiki/File:ATI_Hercules_Card_1986.xcf}{ATI Hercules Card 1986: Jörgen Nixdorf, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:Intel@14nm@@Skylake@Skylake-X(LCC)@i7-7820X@SR3L5_DSC00646_(25952474218).jpg}{Intel Skylake 14nm: Fritzchens Fritz, CC0 1.0} - \item \href{https://commons.wikimedia.org/wiki/File:TN_display_closeup_300X.jpg}{TN display closeup 300X: Akpch, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:LCDmonitorscreenimage.jpg}{LCD monitor screen image: Koperczak, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:CRT_color_enhanced_unlabeled.png}{CRT color enhanced unlabeled: Grm\_wnr, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:Wiki_dell_lcd.jpg}{Dell TFT LCD: Szasz-Revai Endre, CC BY 2.5} - \item \href{https://commons.wikimedia.org/wiki/File:CeBIT_2011_Samstag_PD_115.JPG}{CeBIT 2011 3D Passport system: Bin im Garten, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:Bouquin_\%C3\%A9lectronique_iLiad_en_plein_soleil.jpg}{Bouquin électronique iLiad en plein soleil: Mathieu Despont, public domain} - \item \href{https://commons.wikimedia.org/wiki/File:Kindle_3_texture_(crop).jpg}{Kindle 3 E Ink screen: HorsePunchKid, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:DP_to_DVI_converter_unmounted.jpg}{DP to DVI converter unmounted: Antifumo, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:Anisotropic_filtering_en.png}{Anisotropic filtering: Thomas, Lampak, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:MipMap_Example_STS101.jpg}{MipMap Example STS101: Mike Hicks, NASA, CC BY-SA 3.0} - \item \href{https://peach.blender.org/}{Big Bug Bunny: Blender Foundation, CC BY 3.0} - \end{itemize} -\end{frame} diff --git a/slides/graphics-software-linux-drm/graphics-software-linux-drm.tex b/slides/graphics-software-linux-drm/graphics-software-linux-drm.tex new file mode 100644 index 0000000000..23fdda7a38 --- /dev/null +++ b/slides/graphics-software-linux-drm/graphics-software-linux-drm.tex @@ -0,0 +1,575 @@ +\subsection{Linux Kernel: DRM} + +\begin{frame}{DRM devices} + \begin{itemize} + \item UNIX-style devices are identified with major/minor numbers + \begin{itemize} + \item More details in the \code{makedev} manpage, using \code{dev_t} type + \item Minor/major can be retrieved with \code{stat/fstat} + \item DRM major in Linux is \textbf{226} + \end{itemize} + \item Two types of DRM devices exist: + \begin{itemize} + \item \textbf{Primary} nodes at \code{/dev/dri/card*} with minor \(< 128\)\\ + Used for display operations with the KMS (mode) interface + \item \textbf{Render} nodes at \code{/dev/dri/renderD*} with minor \(\geq 128\)\\ + Used for render operations with a driver-specific interface + \end{itemize} + \item DRM devices can also be used by the kernel directly (internal clients): + \begin{itemize} + \item \textbf{fbdev} compatibility layer to provide \code{/dev/fb*} nodes + \item Used by \textbf{fbcon} to provide virtual consoles + \end{itemize} + \item Userspace needs rights to open device nodes: + \begin{itemize} + \item Usually allowed via the \code{video} \textbf{group} or \textbf{Access Control Lists} (ACLs) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM driver identification and capabilities} + \begin{itemize} + \item Driver-specific \textbf{name and version} (major/minor/patchlevel) can be queried:% + \begin{minted}[fontsize=\small]{console} +struct drm_version version = { ... }; +ret = ioctl(drm_fd, DRM_IOCTL_VERSION, &version); + \end{minted} + \item Drivers expose specific \textbf{capabilities}, that can be queried: + \begin{minted}[fontsize=\small]{console} +struct drm_get_cap get_cap = { 0 }; +get_cap.capability = DRM_CAP_DUMB_BUFFER; +ret = ioctl(drm_fd, DRM_IOCTL_GET_CAP, &get_cap); + \end{minted} + \item The kernel \textbf{must} be informed of client support for some features: + \begin{minted}[fontsize=\small]{console} +struct drm_set_client_cap client_cap = { 0 }; +client_cap.capability = DRM_CLIENT_CAP_UNIVERSAL_PLANES; +client_cap.value = 1; +ret = ioctl(drm_fd, DRM_IOCTL_SET_CLIENT_CAP, &client_cap); + \end{minted} + \item Driver and client capabilities defined in Linux's \kfile{include/uapi/drm/drm.h} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM master, magic and authentication} + \begin{itemize} + \item Multiple userspace clients can open the same primary device node + \item Only the \textbf{master} client is allowed to configure display (KMS) + \item Master is exclusive and can be \textbf{acquired} and \textbf{dropped} (VT switching): + \begin{minted}[fontsize=\small]{console} +ret = ioctl(drm_fd, DRM_IOCTL_SET_MASTER, NULL); +ret = ioctl(drm_fd, DRM_IOCTL_DROP_MASTER, NULL); + \end{minted} + \item Requires \code{CAP_SYS_ADMIN} Linux capability, see \code{capabilities} man page\\ + \textit{usually reserved to the root super user} + \item Some operations can be allowed on trusted clients with \textbf{magic authentication}: + \begin{itemize} + \item Mostly used before render nodes or for allocating buffers on another process + \end{itemize} + \begin{enumerate} + \item Client \textit{foo} gets its client-specific magic: + \begin{minted}[fontsize=\small]{console} +struct drm_auth auth = { 0 }; +ret = ioctl(drm_fd, DRM_IOCTL_GET_MAGIC, &auth); + \end{minted} + \item Client \textit{foo} sends \code{auth.magic} to master client \textit{bar} (via IPC) + \item Master client \textit{bar} authenticates client \textit{foo}: + \begin{minted}[fontsize=\small]{console} +ret = ioctl(drm_fd, DRM_IOCTL_AUTH_MAGIC, &auth); + \end{minted} + \end{enumerate} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM memory management} + \begin{itemize} + \item The \textbf{Graphics Execution Manager} (GEM) handles memory in DRM + \item Used both by KMS and render drivers, with specific backends: + \begin{itemize} + \item \textbf{CMA}: Contiguous Memory Allocator (reserved area at boot) + \item \textbf{Shmem}: Shared system memory (anonymous pages) + \item \textbf{Vram}: Video RAM, using the \textbf{Translation Table Manager} (TTM) + \end{itemize} + \item Ensures buffers \textbf{coherency} on access (cache management) + \item Allocated buffers are identified with a unique \textbf{handle} number + \item In KMS, the \textbf{dumb buffer} API exposes memory operations: + \begin{itemize} + \item For memory used for \textbf{scanout framebuffers} + \item Drivers calculate aligned pitch/stride and size based on dimensions and bpp + \item Sometimes too limiting (e.g. multi-planar formats) + \end{itemize} + \item More details in the \textit{drm-memory} man page + \item Drivers sometimes expose extra \code{ioctls} for more advanced needs + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM KMS dumb buffer API} + \begin{itemize} + \item \textbf{Allocating} from \code{width}, \code{height} and \code{bpp}, returning \code{handle}, \code{pitch} and \code{size}:\\ + \begin{minted}[fontsize=\small]{console} +struct drm_mode_create_dumb create_dumb = { ... }; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_CREATE_DUMB, &create_dumb); + \end{minted} + \item \textbf{Destroying} an allocated buffer: + \begin{minted}[fontsize=\small]{console} +struct drm_mode_destroy_dumb destroy_dumb = { .handle = ..., }; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_DESTROY_DUMB, &destroy_dumb); + \end{minted} + \item \textbf{Preparing a mapping} in user memory for a buffer, returning an \code{offset}: + \begin{minted}[fontsize=\small]{console} +struct drm_mode_map_dumb map_dumb = { .handle = ..., }; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_MAP_DUMB, &map_dumb); + \end{minted} + \item \textbf{Mapping memory} to userspace using the \code{offset}: + \begin{minted}[fontsize=\small]{console} +map = mmap(NULL, create_dumb.size, PROT_READ | PROT_WRITE, MAP_SHARED, + drm_fd, map_dumb.offset); + \end{minted} + \item \textbf{Unmapping} memory after use: + \begin{minted}[fontsize=\small]{console} +munmap(map, create_dumb.size); + \end{minted} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM FourCCs and modifiers} + \begin{itemize} + \item DRM has its own representation of \textbf{pixel formats}, with FourCC codes (on 32 bits) + \item Defined in the \kfile{include/uapi/drm/drm_fourcc.h} header + \item They can specify up to 4 distinct \textbf{data planes} for color components + \item Pixel formats are named "MSB-to-LSB" and specified in \textbf{little-endian} order\\ + \textit{LSB comes first in memory in little-endian} + \item For instance, \code{DRM_FORMAT_XRGB8888} has the \code{B} byte first in memory\\ + \textit{Memory order is independent from the CPU or hardware endianness} + \item A format \textbf{modifier} (on 64 bits) indicates the pixel order in memory + \item \code{DRM_FORMAT_MOD_LINEAR} indicates raster order\\ + \textit{line-major left-to-right, top-to-bottom} + \item Other modifiers are usually hardware-specific, often tiled\\ + (e.g. \code{DRM_FORMAT_MOD_VIVANTE_TILED}) + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM KMS resources probing} + \begin{itemize} + \item KMS hardware resources are exposed through the following entities: + \begin{itemize} + \item Connectors + \item Encoders + \item CRTCs + \item Planes: primary, overlay and cursor + \item Framebuffers + \end{itemize} + \item Each resource instance is identified with a unique \textbf{identification number} + \item The list of resource ids is retrieved with: + \begin{minted}[fontsize=\small]{console} +struct drm_mode_card_res res = { ... }; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_GETRESOURCES, &res); + \end{minted} + \item Plane ids (that were introduced later) are retrieved with: + \begin{minted}[fontsize=\small]{console} +struct drm_mode_get_plane_res res = { ... }; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_GETPLANERESOURCES, &res); + \end{minted} + \item Resource ids are used with subsequent resource-specific calls + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM KMS connector probing} + \begin{itemize} + \item The starting point to configure a KMS pipeline is the connector + \item Current connector state is probed with: + \begin{minted}[fontsize=\small]{console} +struct drm_mode_get_connector get_connector = { .connector_id = ... }; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_GETCONNECTOR, &get_connector); + \end{minted} + \item \code{struct drm_mode_get_connector} exposes various information: + \begin{itemize} + \item Connector type and connection state + \item Possible encoders, currently-attached encoder + \item Available modes and physical monitor size + \end{itemize} + \item Probing modes triggers EDID read: optional and usually quite slow + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM KMS modes} + \begin{itemize} + \item A display mode is represented as a \code{struct drm_mode_modeinfo} in DRM + \item Members: \code{clock}, \code{[hv]display}, \code{[hv]sync_start}, \code{[hv]sync_end}, \code{[hv]total} and \code{flags} for signal-specific details (polarities) + \item Diagram from \kfile{include/drm/drm_modes.h}: + \begin{minted}[fontsize=\small]{console} + Active Front Sync Back + Region Porch Porch +<-----------------------><----------------><-------------><--------------> + //////////////////////| + ////////////////////// | +////////////////////// |.................. ................ + _______________ +<----- [hv]display -----> +<------------- [hv]sync_start ------------> +<--------------------- [hv]sync_end ---------------------> +<-------------------------------- [hv]total ----------------------------->* + \end{minted} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM KMS encoder probing} + \begin{itemize} + \item The next step is to find which CRTC id can be used with the connector + \item The encoder is the link between the connector and CRTC + \item Current encoder state can be probed with: + \begin{minted}[fontsize=\small]{console} +struct drm_mode_get_encoder get_encoder = { .encoder_id = ... }; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_GETENCODER, &get_encoder); + \end{minted} + \item \code{struct drm_mode_get_connector} exposes some information: + \begin{itemize} + \item Encoder type + \item Possible CRTCs, currently-attached CRTC + \end{itemize} + \item This allows selecting the CRTC to use for the connector! + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM KMS framebuffer management} + \begin{itemize} + \item Framebuffers in DRM are described with a number of parameters: + \begin{itemize} + \item Picture-wide: \code{width}, \code{height}, \code{pixel_format} + \item Plane-specific: GEM \code{handle}, \code{pitch}, \code{offset} and \code{modifier} + \end{itemize} + \item Up to 4 memory planes are supported (depending on the format) + \item Allows supporting a wide range of possible configurations + \item Flags are passed to indicate that modifiers or interlaced scan are used + \item Framebuffers are registered from their parameters, returning a \code{fb_id}: + \begin{minted}[fontsize=\small]{console} +struct drm_mode_fb_cmd2 fb_cmd2 = { ... }; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_ADDFB2, &fb_cmd2); + \end{minted} + \item They are destroyed using the \code{fb_id}: + \begin{minted}[fontsize=\small]{console} +unsigned int fb_id = fb_cmd2.fb_id; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_RMFB, &fb_id); + \end{minted} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM KMS CRTC configuration (legacy)} + \begin{itemize} + \item The pipeline can then be configured with the connector and the CRTC + \item The current CRTC configuration can be retrieved with: + \begin{minted}[fontsize=\small]{console} +struct drm_mode_crtc crtc = { .crtc_id = ... }; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_GETCRTC, &crtc); + \end{minted} + \item The CRTC is configured with the connector id + \begin{minted}[fontsize=\small]{console} +struct drm_mode_crtc crtc = { .crtc_id = ... }; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_SETCRTC, &crtc); + \end{minted} + \item A mode and a framebuffer can be set (previous setup used otherwise)\\ + \textit{mandatory if the CRTC was unused before} + \item The kernel will automatically select the best encoder for the connector and CRTC + \item \textbf{Legacy and deprecated} way to do modesetting: only concerns the primary plane + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM KMS page flipping (legacy)} + \begin{itemize} + \item Page flipping is the action of switching the CRTC to another framebuffer\\ + \textit{only concerns the primary plane} + \item An event can be requested when the flip happens + \item Can be scheduled at different times (specified with \code{flags}): + \begin{itemize} + \item At a specified vblank target (absolute or relative) to avoid tearing + \item As soon as possible (asynchronously) if supported + \end{itemize} + \begin{minted}[fontsize=\small]{console} +struct drm_mode_crtc_page_flip page_flip = { .crtc_id = ..., .fb_id = ... }; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_PAGE_FLIP, &page_flip); + \end{minted} + \item \textbf{Legacy and deprecated}: limited to the primary plane + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM KMS overlay plane configuration (legacy)} + \begin{itemize} + \item Overlay planes are configured separately from the CRTC main plane + \item The current state of a plane can be retrieved with: + \begin{minted}[fontsize=\small]{console} +struct drm_mode_get_plane get_plane = { .plane_id = ... }; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_GETPLANE, &get_plane); + \end{minted} + \item Provides possible CRTCs, current framebuffer and supported formats + \item Planes are configured with source and destination parameters: + \begin{itemize} + \item \code{crtc_[xywh]}: On-CRTC position and dimensions + \item \code{src_[xywh]}: In-framebuffer position and dimensions (source clipping area) + \end{itemize} + \item Configuration takes place with: + \begin{minted}[fontsize=\small]{console} +struct drm_mode_set_plane set_plane = { .plane_id = ... }; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_SETPLANE, &set_plane); + \end{minted} + \item \textbf{Legacy and deprecated}: not synchronized to vblank or page flip + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM KMS cursor configuration and position (legacy)} + \begin{itemize} + \item Cursor planes have a separate dedicated legacy API + \item Configured per-CRTC with a GEM \code{handle} and dimensions (\code{width}, \code{height})\\ + \textit{a zero GEM \code{handle} deconfigures and removes the cursor} + \item Only supports the \code{DRM_FORMAT_ARGB8888} format (not configurable) + \item Using a single \code{ioctl} with the \code{flags} field for the operation + \begin{minted}[fontsize=\small]{console} +struct drm_mode_cursor cursor = { .flags = DRM_MODE_CURSOR_BO, + .crtc_id = ...}; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_CURSOR, &cursor); + \end{minted} + \item Once configured, the cursor can be moved to \code{x}, \code{y} on-CRTC coordinates + \begin{minted}[fontsize=\small]{console} +struct drm_mode_cursor cursor = { .flags = DRM_MODE_CURSOR_MOVE, + .crtc_id = ... }; +ret = ioctl(drm_fd, DRM_IOCTL_MODE_CURSOR, &cursor); + \end{minted} + \item \code{DRM_IOCTL_MODE_CURSOR2} variant provides cursor hotspot for virtual machines + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM event notification and wait} + \begin{itemize} + \item DRM provides an event notification mechanism for \textbf{vblank} and \textbf{page flip done} + \item Available through the primary (KMS) file descriptor + \item Can be used with \code{poll} and \code{select} (integrated in main loop) + \item Events with a \code{struct drm_event} base are read using \code{read} + \item Expand to \code{struct drm_event_vblank} for vblank and page flip done events\\ + \textit{only complete events are returned, so the buffer must be large enough} + \item Events can be requested at page flip time or explicitly: + \begin{minted}[fontsize=\small]{console} +union drm_wait_vblank wait_vblank = { .request = ... }; +ret = ioctl(drm_fd, DRM_IOCTL_WAIT_VBLANK, &wait_vblank); + \end{minted} + \item A blocking wait for an absolute or relative vblank sequence can also be requested\\ + \textit{using the same \code{ioctl} and dedicated \code{request.type} values} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM KMS object properties} + \begin{itemize} + \item KMS objects expose generic (or driver-specific) properties with names and values + \textit{concerns \textbf{connectors}, \textbf{CRTCs} and \textbf{planes}} + \begin{itemize} + \item \textbf{Range} properties: limits for the value (signed or unsigned) + \item \textbf{Enum} properties: fixed values with associated names for the values + \item \textbf{Blob} properties: raw data with a given length + \end{itemize} + \item Properties have a \textbf{unique identifier across objects}, details can be queried: + \begin{minted}[fontsize=\small]{console} +struct drm_mode_obj_get_property get_property = { .prop_id = ... } +ret = ioctl(drm_fd, DRM_IOCTL_MODE_GETPROPERTY, &get_property); + \end{minted} + \item Registered properties of an object can be retrieved using: + \begin{minted}[fontsize=\small]{console} +struct drm_mode_obj_get_properties get_properties = { .obj_id = ... } +ret = ioctl(drm_fd, DRM_IOCTL_MODE_OBJ_GETPROPERTIES, &get_properties); + \end{minted} + \item The \code{value} of a property can be assigned with: + \begin{minted}[fontsize=\small]{console} +struct drm_mode_obj_set_property set_property = { .obj_id = ..., .prop_id = ... } +ret = ioctl(drm_fd, DRM_IOCTL_MODE_OBJ_SETPROPERTY, &set_property); + \end{minted} + \item Blob properties need to be created and destroyed (with their own identifier) + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM KMS atomic} + \begin{itemize} + \item The legacy API comes with major design issues: + \begin{itemize} + \item Overlay and cursor plane updates are applied instantly (tearing) + \item Plane updates cannot be synchronized together (intermediate states) + \item No way to check that setup is valid before applying it + \end{itemize} + \item The atomic API lifts these restrictions with a new paradigm: + \begin{itemize} + \item Objects are configured based on their KMS properties\\ + \textit{values are affected to each changed property} + \item Property changes of different objects are grouped in an \textbf{atomic commit} + \item Planes are handled regardless of their type (primary, overlay, cursor) + \item Commits can be marked for test only: checked but not applied + \item Changes are applied at next vblank, unless marked asynchronous + \end{itemize} + \begin{minted}[fontsize=\small]{console} +struct drm_mode_atomic atomic = { ... } +ret = ioctl(drm_fd, DRM_IOCTL_MODE_ATOMIC, &atomic); + \end{minted} + \item Unless marked non-blocking, the \code{ioctl} returns when changes are applied + \item A page flip event can also be requested + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM KMS atomic common properties} + \begin{itemize} + \item Common properties used to configure \textbf{connectors}: + \begin{itemize} + \item \code{CRTC_ID}: id of the CRTC to bind with the connector + \end{itemize} + \item Common properties used to configure \textbf{CRTCs}: + \begin{itemize} + \item \code{ACTIVE}: whether the CRTC is in use + \item \code{MODE_ID}: id of the property blob with the \code{struct drm_mode_modeinfo} mode + \end{itemize} + \item Common properties used to configure \textbf{planes}: + \begin{itemize} + \item \code{FB_ID}: id of the framebuffer to bind with the plane + \item \code{CRTC_ID}: id of the CRTC to bind with the plane + \item \code{CRTC_[XYWH]}: on-CRTC position and dimensions of the plane + \item \code{SRC_[XYWH]}: in-framebuffer position and dimensions (source clipping area) + \end{itemize} + \item Common properties used to probe \textbf{planes}: + \begin{itemize} + \item \code{TYPE}: type of the plane (primary/overlay/cursor) + \item \code{IN_FORMATS}: list of supported formats/modifiers + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM KMS atomic driver walkthrough} + \begin{itemize} + \item A state-of-the-art DRM KMS driver: \code{vc4} at \kdir{drivers/gpu/drm/vc4}\\ + \textit{integrates both DRM KMS and render} + \item Entry point at \kfile{drivers/gpu/drm/vc4/vc4_drv.c} + \item Dedicated documentation: \url{https://dri.freedesktop.org/docs/drm/gpu/vc4.html} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM render generalities} + \begin{itemize} + \item DRM render drivers have their own \textbf{driver-specific API}\\ + \textit{unlike KMS, render hardware abstraction is done in userspace} + \item Their API is exposed through custom \code{ioctls} + \item Can be associated with a KMS driver (e.g. \code{vc4}) or separate (e.g. \code{v3d}) + \item Drivers handle memory, job submission and scheduling, interrupts + \item DRM has a common scheduler (from AMD) in \kdir{drivers/gpu/drm/scheduler} + \item Usual operations: + \begin{itemize} + \item Managing buffer objects (BOs) of different types (create, destroy, mmap)\\ + \textit{using GEM under the hood} + \item Submitting job data structures for programming the GPU (command lists)\\ + \textit{with a validation step to ensure its validity} + \item Waiting for operations to complete + \item Exposing performance-related information + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM render driver walkthrough} + \begin{itemize} + \item A state-of-the-art DRM render driver: \code{v3d} at \kdir{drivers/gpu/drm/v3d} + \item Entry point at \kfile{drivers/gpu/drm/v3d/v3d_drv.c} + \item Dedicated documentation: \url{https://dri.freedesktop.org/docs/drm/gpu/v3d.html} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM Prime zero-copy memory sharing (dma-buf)} + \begin{itemize} + \item Memory buffers often need to be shared between different devices\\ + \textit{e.g. DRM KMS and DRM render but also concerns V4L2 for media devices} + \item The kernel-wide \code{dma-buf} API allows exporting and importing buffers + \item Buffers are represented as \textbf{file descriptors} in userspace\\ + \textit{file descriptors can be shared between programs via IPC} + \item DRM exposes dma-buf via the \textbf{DRM Prime} API + \item DRM prime exports a GEM \code{handle} to a returned \code{fd}: + \begin{minted}[fontsize=\small]{console} +struct drm_prime_handle prime_handle = { .handle = ... } +ret = ioctl(drm_fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &prime_handle); + \end{minted} + \item And vice-versa: + \begin{minted}[fontsize=\small]{console} +struct drm_prime_handle prime_handle = { .fd = ... } +ret = ioctl(drm_fd, DRM_IOCTL_PRIME_FD_TO_HANDLE, &prime_handle); + \end{minted} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM sync object fencing} + \begin{itemize} + \item In a multi-device pipeline with zero-copy, only scheduling is left to userspace\\ + \textit{each device signals completion and userspace moves on to the next} + \item Fences were introduced to avoid the extra roundtrip in userspace: + \begin{itemize} + \item The flow of buffers between devices is usually known in advance + \item The kernel can coordinate internally and trigger the next device + \item Requires submitting all commands in advance with fences attached + \end{itemize} + \item DRM exposes fences via the \textbf{Sync object} API + \item Sync objects contain one fence, exposed as a file descriptor + \item The KMS atomic API and some render driver APIs take input fence fds + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM sync object fencing} + \begin{itemize} + \item Sync objects are created and destroyed with a \code{handle}: + \begin{minted}[fontsize=\small]{console} +struct drm_syncobj_create syncobj_create = { 0 } +ret = ioctl(drm_fd, DRM_IOCTL_SYNCOBJ_CREATE, &syncobj_create); + \end{minted} + \begin{minted}[fontsize=\small]{console} +struct drm_syncobj_destroy syncobj_destroy = { .handle = syncobj_create.handle } +ret = ioctl(drm_fd, DRM_IOCTL_SYNCOBJ_DESTROY, &syncobj_destroy); + \end{minted} + \item An output fence's \code{fd} is exported from a device's sync object with: + \begin{minted}[fontsize=\small]{console} +struct drm_syncobj_handle syncobj_handle = { .handle = handle, ... } +ret = ioctl(drm_fd, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &syncobj_handle); + \end{minted} + \item An input fence's \code{fd} is imported to a device's sync object with: + \begin{minted}[fontsize=\small]{console} +struct drm_syncobj_handle syncobj_handle = { .handle = handle, .fd = fd } +ret = ioctl(drm_fd, DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE, &syncobj_handle); + \end{minted} + \item Quite a recent feature, not yet available in V4L2 (media) + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{DRM debug and documentation} + \begin{itemize} + \item \textbf{Debug message} using the \code{drm.debug} kernel cmdline argument: + \begin{itemize} + \item Detailed in the \kfile{include/drm/drm_print.h} header + \item \code{drm.debug=0x17} for core, KMS, driver and atomic debug messages + \end{itemize} + \item Current \textbf{state debug} in debugfs: \code{cat /sys/kernel/debug/dri/0/state} + \item Drivers expose \textbf{specific debugfs entries} + \item \textbf{Debug utility}: \code{modetest} from \code{libdrm} + \item \textbf{Community} contact: + \begin{itemize} + \item Mailing list: \code{dri-devel@lists.freedesktop.org} + \item IRC channel: \code{#dri-devel} on the OFTC network + \end{itemize} + \item \textbf{Documentation} resources: + \begin{itemize} + \item Linux GPU Driver Developer’s Guide: \url{https://www.kernel.org/doc/html/latest/gpu/index.html} + \item Man pages about userspace aspects: \code{drm}, \code{drm-kms}, \code{drm-memory} + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{libdrm wrapper} + \begin{itemize} + \item Userspace access to DRM devices is wrapped with \textbf{libdrm} + \item Exposes convenience wrappers, helpers and some data structures around \code{ioctls} + \begin{itemize} + \item For KMS support in the \code{libdrm.so} library + \item For hardware-specific render drivers in dedicated libraries (e.g. \code{libdrm_nouveau.so}) + \end{itemize} + \item Used by almost every userspace project dealing with DRM:\\ + \textit{weston, mutter, Xorg, mesa, etc} + \end{itemize} + + \begin{center} + \includegraphics[width=0.5\textwidth]{slides/graphics-software-linux-drm/libdrm-userspace.pdf} + \end{center} +\end{frame} diff --git a/slides/graphics-software/libdrm-userspace.svg b/slides/graphics-software-linux-drm/libdrm-userspace.svg similarity index 100% rename from slides/graphics-software/libdrm-userspace.svg rename to slides/graphics-software-linux-drm/libdrm-userspace.svg diff --git a/slides/graphics-software-linux-fbdev/graphics-software-linux-fbdev.tex b/slides/graphics-software-linux-fbdev/graphics-software-linux-fbdev.tex new file mode 100644 index 0000000000..3bf232d82b --- /dev/null +++ b/slides/graphics-software-linux-fbdev/graphics-software-linux-fbdev.tex @@ -0,0 +1,71 @@ +\subsection{Linux Kernel: Framebuffer Device} + +\begin{frame}{Fbdev overview} + \begin{itemize} + \item Fbdev is the \textbf{historical and legacy} display subsystem in Linux + \item Exposes a number of display-related features: + \begin{itemize} + \item Framebuffer access (pre-allocated for one or two frames) + \item Operation primitives (bit blit, solid fill, area copy, etc) + \item Cursor drawing + \item Power management (blank/unblank, power down) + \end{itemize} + \item Initial state can be configured with the \code{video} option of the \code{cmdline} + \item Available to userspace via \code{/dev/fb*} nodes: + \begin{itemize} + \item Generic base \code{ioctls} with driver-specific additions + \item Direct framebuffer memory mapping using \code{mmap} + \item Relatively simple and minimalistic interface + \end{itemize} + \item Used by the kernel to provide a graphical console with \textbf{fbcon} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{Fbdev basic operations} + \begin{itemize} + \item \textbf{Fixed information} about the display is retrieved with: + \begin{minted}[fontsize=\small]{console} +struct fb_fix_screeninfo fix_screeninfo = { 0 }; +ret = ioctl(fb_fd, FBIOGET_FSCREENINFO, &fix_screeninfo); + \end{minted} + \item \textbf{Variable information} (including mode) is retrieved and configured with: + \begin{minted}[fontsize=\small]{console} +struct fb_var_screeninfo var_screeninfo = { 0 }; +ret = ioctl(fb_fd, FBIOGET_VSCREENINFO, &var_screeninfo); +... +ret = ioctl(fb_fd, FBIOPUT_VSCREENINFO, &var_screeninfo); + \end{minted} + \item \textbf{Power management} is operated with: + \begin{minted}[fontsize=\small]{console} +ret = ioctl(fb_fd, FBIOBLANK, FB_BLANK_POWERDOWN); + \end{minted} + \item \textbf{Double-buffering} is sometimes supported (within the same buffer): + \begin{minted}[fontsize=\small]{console} +var_screeninfo.yoffset = height; +ret = ioctl(fb_fd, FBIOPAN_DISPLAY, &var_screeninfo); + \end{minted} + \item Blocking until the next \textbf{vblank} is possible with: + \begin{minted}[fontsize=\small]{console} +int vsync = 0; +ret = ioctl(fb_fd, FBIO_WAITFORVSYNC, &vsync); + \end{minted} + \end{itemize} +\end{frame} + +\begin{frame}{Fbdev limitations} + \begin{itemize} + \item Fbdev does not expose or allow configuration of the display pipeline + \item Output setup is mostly static (provided through the \code{cmdline}) + \item Designed for simple cases (with a single output) + \item Buffer allocation and management is not available + \item No possibility of zero-copy import from other devices + \item Limited page flipping with no associated synchronization mechanism + \item Insufficient external synchronization interface (blocking wait) + \item Mixes display, operation primitives and power management + \end{itemize} + + \begin{center} + \textbf{Fbdev is mostly adapted to display from the 1990s and 2000s}\\ + \small please consider avoiding it at all costs! + \end{center} +\end{frame} diff --git a/slides/graphics-software/getty.jpg b/slides/graphics-software-linux-tty/getty.jpg similarity index 100% rename from slides/graphics-software/getty.jpg rename to slides/graphics-software-linux-tty/getty.jpg diff --git a/slides/graphics-software-linux-tty/graphics-software-linux-tty.tex b/slides/graphics-software-linux-tty/graphics-software-linux-tty.tex new file mode 100644 index 0000000000..3feab0a359 --- /dev/null +++ b/slides/graphics-software-linux-tty/graphics-software-linux-tty.tex @@ -0,0 +1,75 @@ +\subsection{Linux Kernel: TTY} + +\begin{frame}{Linux TTY subsystem introduction} + \begin{itemize} + \item The \textbf{TTY} subsystem handles teletypewriters to send/receive characters + \item Source code located at \code{drivers/tty} in Linux + \item Supports physical instances (e.g. UART, RS-232) and virtual ones + \item Virtual terminals/consoles (VTs/VCs) associate a distinct keyboard and display + \begin{itemize} + \item Many VTs are created by Linux, available under: \code{/dev/tty*} + \item Only a single VT is active at a time, switched with \code{Ctrl + Alt + Fi} + \item Display grabbed using \textbf{fbcon} from the \textbf{fbdev} subsystem + \item Keyboard grabbed using the \textbf{input} subsystem + \item Can be used to show kernel messages (\code{console=tty1} in the cmdline) + \item Every program runs under a controlling tty (given by the \code{tty} command) + \end{itemize} + \item Pseudo-terminals also exist, for software-based I/O only + \begin{itemize} + \item Created by programs (e.g. terminal emulator) under: \code{/dev/pts/*} + \item Unrelated to graphics topics + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Linux TTY (illustrated)} + \begin{center} + \includegraphics[width=0.6\textwidth]{slides/graphics-software-linux-tty/getty.jpg}\\ + \textit{Getty running on \code{tty1} of a GNU/Linux system}\\ + \end{center} +\end{frame} + +\begin{frame}[fragile]{Virtual terminals and graphics} + \begin{itemize} + \item With VTs, the kernel is \textbf{already using} the display and keyboard! + \item Display servers need to switch to \textbf{graphics mode} to release the display: + \begin{minted}[fontsize=\small]{console} +ret = ioctl(tty_fd, KDSETMODE, KD_GRAPHICS); + \end{minted} + \item And disable \textbf{keyboard support} on the standard input: + \begin{minted}[fontsize=\small]{console} +ret = ioctl(tty_fd, KDSKBMODE, K_OFF); + \end{minted} + \item The display device can then be used \textbf{exclusively} + \item Input is no longer interpreted (e.g. \code{Ctrl-C} is ignored) + \item Graphics and keyboard mode must be restored when leaving to keep the VT usable + \item Current modes can be queried with: + \begin{minted}[fontsize=\small]{console} +short mode, kbmode; +ret = ioctl(tty_fd, KDGETMODE, &mode); +ret = ioctl(tty_fd, KDGKBMODE, &kbmode); + \end{minted} + \item More details in the \code{console_ioctl} man page + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{Virtual terminals switching and graphics} + \begin{itemize} + \item However, the user might still want to switch VTs! + \item So the display device must be \textbf{released/reacquired} for VT switching + \item UNIX signals are used to notify the application, configured with: + \begin{minted}[fontsize=\small]{console} +struct vt_mode vt_mode = { 0 }; +vt_mode.mode = VT_PROCESS; +vt_mode.relsig = SIGUSR1; +vt_mode.acqsig = SIGUSR2; +ret = ioctl(tty_fd, VT_SETMODE, &vt_mode); + \end{minted} + \item VT switching must be acknowledged for the other VT to take over: + \begin{minted}[fontsize=\small]{console} +ret = ioctl(tty_fd, VT_RELDISP, VT_ACKACQ); /* when entering VT */ +ret = ioctl(tty_fd, VT_RELDISP, 1); /* when leaving VT */ + \end{minted} + \item Failure to acknowledge will cause a \textbf{system hang} + \end{itemize} +\end{frame} diff --git a/slides/graphics-software/agnostic-display-stack-overview.dia b/slides/graphics-software-stack-overview/agnostic-display-stack-overview.dia similarity index 100% rename from slides/graphics-software/agnostic-display-stack-overview.dia rename to slides/graphics-software-stack-overview/agnostic-display-stack-overview.dia diff --git a/slides/graphics-software/compositing-gnome-shell.jpg b/slides/graphics-software-stack-overview/compositing-gnome-shell.jpg similarity index 100% rename from slides/graphics-software/compositing-gnome-shell.jpg rename to slides/graphics-software-stack-overview/compositing-gnome-shell.jpg diff --git a/slides/graphics-software/compositing-gnome-terminal.jpg b/slides/graphics-software-stack-overview/compositing-gnome-terminal.jpg similarity index 100% rename from slides/graphics-software/compositing-gnome-terminal.jpg rename to slides/graphics-software-stack-overview/compositing-gnome-terminal.jpg diff --git a/slides/graphics-software/compositing-lollipop.jpg b/slides/graphics-software-stack-overview/compositing-lollipop.jpg similarity index 100% rename from slides/graphics-software/compositing-lollipop.jpg rename to slides/graphics-software-stack-overview/compositing-lollipop.jpg diff --git a/slides/graphics-software/compositing-result.jpg b/slides/graphics-software-stack-overview/compositing-result.jpg similarity index 100% rename from slides/graphics-software/compositing-result.jpg rename to slides/graphics-software-stack-overview/compositing-result.jpg diff --git a/slides/graphics-software/enlightenment-logo.svg b/slides/graphics-software-stack-overview/enlightenment-logo.svg similarity index 100% rename from slides/graphics-software/enlightenment-logo.svg rename to slides/graphics-software-stack-overview/enlightenment-logo.svg diff --git a/slides/graphics-software/gnome-logo.svg b/slides/graphics-software-stack-overview/gnome-logo.svg similarity index 100% rename from slides/graphics-software/gnome-logo.svg rename to slides/graphics-software-stack-overview/gnome-logo.svg diff --git a/slides/graphics-software-stack-overview/graphics-software-stack-overview.tex b/slides/graphics-software-stack-overview/graphics-software-stack-overview.tex new file mode 100644 index 0000000000..b6d6bfb968 --- /dev/null +++ b/slides/graphics-software-stack-overview/graphics-software-stack-overview.tex @@ -0,0 +1,244 @@ +\section{Software Aspects} + +\subsection{Stack Overview} + +\begin{frame}{System-agnostic overview: kernel} + \begin{itemize} + \item The kernel provides access to the hardware from userspace + \item Handles clocks, power, register access, interrupts, etc + \item Coordinates memory management with the rest of the system + \item Exposes features to userspace through hardware-agnostic interfaces\\ + \textit{or at least, as much as possible} + \item Three aspects are usually involved: + \begin{itemize} + \item \textbf{display}: from framebuffer to encoder + \item \textbf{render}: GPU and/or 2D accelerators + \item \textbf{input}: keyboard, mouse and other devices + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{System-agnostic overview: display userspace} + \begin{itemize} + \item The kernel provides \textbf{exclusive access} to display hardware + \item Many applications need to show their buffers concurrently + \item The \textbf{display server} is in charge of coordinating between applications: + \begin{itemize} + \item Part of the core of the system, privileged + \item Applications (via libraries) contact the server to display pixel buffers + \item Dispatches input events to the concerned applications + \item Only the display server deals with the kernel display and input APIs + \end{itemize} + \item The \textbf{compositor} merges pixel buffers from applications into the final buffer + \item The \textbf{window manager} defines stacking order, focus, decorations, etc + \item Both can be part of the display server or distinct components + \item Serious security concerns: I/O isolation for applications, server privileges + \end{itemize} +\end{frame} + +\begin{frame}{System-agnostic overview: display userspace (illustrated)} + + \begin{minipage}[t]{0.49\textwidth} + \centering + \includegraphics[height=7em]{slides/graphics-software-stack-overview/compositing-gnome-shell.jpg}\\ + \textit{\small GNOME-Shell (green) displaying the\\ top-bar and background}\\ + \vspace{0.5em} + \includegraphics[height=7em]{slides/graphics-software-stack-overview/compositing-gnome-terminal.jpg}\\ + \textit{\small GNOME-Terminal (green) with\\ window decorations (red)} + \end{minipage} + \hfill + \begin{minipage}[t]{0.49\textwidth} + \centering + \includegraphics[height=7em]{slides/graphics-software-stack-overview/compositing-lollipop.jpg}\\ + \textit{\small Lollipop (green) with\\ window decorations (red)}\\ + \vspace{0.5em} + \includegraphics[height=7em]{slides/graphics-software-stack-overview/compositing-result.jpg}\\ + \textit{\small The composited result} + \end{minipage} +\end{frame} + +\begin{frame}{System-agnostic overview: render userspace} + \begin{itemize} + \item Graphics applications and libraries need to render visual elements + \item Rendering can be a major \textbf{performance bottleneck} + \item The system often provides \textbf{accelerated 2D primitives}: + \begin{itemize} + \item Either in the display server + \item Either in dedicated libraries + \end{itemize} + \item Their implementation can take different forms: + \begin{itemize} + \item Using dedicated 2D hardware + \item Using 3D hardware in 2D setups (\(z = 0\)) + \item Using specific efficient CPU instructions (SIMD) + \item Using optimized generic algorithms + \end{itemize} + \item \textbf{3D rendering} comes with its own interfaces and libraries + \item Usually with generic interfaces and hardware-specific implementations + \end{itemize} +\end{frame} + +\begin{frame}{System-agnostic overview (illustrated)} + \begin{center} + \includegraphics[width=\textwidth]{slides/graphics-software-stack-overview/agnostic-display-stack-overview.pdf} + \end{center} +\end{frame} + +\begin{frame}{Linux kernel overview} + \begin{itemize} + \item \textbf{Input} subsystem + \begin{itemize} + \item Supports devices such as mice, keyboards, joysticks, touchscreens + \item Legacy (unused) interfaces for keyboard, mice + \item Unified \textbf{evdev} event interface to simplify userspace + \end{itemize} + \item \textbf{Framebuffer device} (fbdev) subsystem + \begin{itemize} + \item \textbf{Legacy} interface for displaying pixel buffers on-screen + \item Very limited pipeline configuration, no hotplug support + \item Extended features added through driver-specific interfaces + \end{itemize} + \item \textbf{Direct Rendering Manager} (DRM) subsystem + \begin{itemize} + \item Unified display configuration interface: \textbf{Kernel Mode Setting} (KMS or DRM mode) + \item Allows synchronizing changes together (DRM atomic) + \item Exposes render devices through driver-specific interfaces (DRM render)\\ + \textit{Mostly for 3D rendering with GPUs, but a few 2D devices too} + \item Provides memory management mechanisms (DRM GEM) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Linux-compatible low-level userspace overview} + \begin{itemize} + \item \textbf{Input} low-level libraries + \begin{itemize} + \item \textbf{libevdev} (C): Wrapper for evdev interface system calls + \item \textbf{libinput} (C): Input device management, abstraction and quirks, using libevdev + \end{itemize} + \item \textbf{Display/render} low-level interface library + \begin{itemize} + \item \textbf{libdrm} (C): Wrapper for DRM system calls + \end{itemize} + \item \textbf{2D render} low-level libraries + \begin{itemize} + \item \textbf{Pixman} (C): Optimized pixel-level operations + \item \textbf{Cairo} (C): Optimized vector drawing (can use 3D) + \item \textbf{Skia} (C): Optimized vector drawing from Google (can use 3D) + \item \textbf{Clutter} (C++): Accelerated UI animation (using 3D) + \end{itemize} + \item \textbf{3D render} low-level libraries + \begin{itemize} + \item \textbf{Mesa 3D} (C): Reference free software OpenGL implementation + \item \textbf{Proprietary vendor implementations} for specific hardware + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{X Window overview} + \begin{itemize} + \item \textbf{X Window} overview + \begin{itemize} + \item \textbf{X Window/X11} is the historical (and \textbf{legacy}) \textbf{display protocol} + \item Complemented by numerous protocol \textbf{extensions} for extra features + \item \textbf{X.org} is the reference X11 server \textbf{implementation} + \item Needs an external \textbf{window manager} to handle multiple applications + \item \textbf{Composition} by the server or the window manager (Composite extension) + \end{itemize} + \end{itemize} + \begin{minipage}[b]{0.8\textwidth} + \begin{itemize} + \item \textbf{Window manager} implementations examples + \begin{itemize} + \item \textbf{Mutter}: GNOME accelerated compositing window manager + \item \textbf{i3}: Popular tiling window manager + \item \textbf{Compiz}: Popular 3D-enabled compositing window manager + \end{itemize} + \item \textbf{Display client} libraries + \begin{itemize} + \item \textbf{Xlib} (C): The legacy X11 client-side protocol library helper + \item \textbf{XCB} (C): The updated X11 client-side protocol library helper + \item Integrated in most higher-level graphics-oriented libraries + \end{itemize} + \end{itemize} + \end{minipage} + \begin{minipage}[b]{0.15\textwidth} + \includegraphics[width=\textwidth]{slides/graphics-software-stack-overview/x11-logo.pdf} + \end{minipage} +\end{frame} + +\begin{frame}{Wayland overview} + \begin{itemize} + \item \textbf{Wayland} overview + \begin{itemize} + \item Wayland is a \textbf{display protocol} (with a core and extensions), not an implementation + \item Replaces X11 with a \textbf{less intrusive, more modern and minimal} paradigm + \item Compositors (server-side) handle \textbf{input, windows, composition and display} + \end{itemize} + \end{itemize} + \begin{minipage}[b]{0.8\textwidth} + \begin{itemize} + \item \textbf{Wayland compositor} implementations examples + \begin{itemize} + \item Using the \textbf{libwayland-server} base protocol library + \item \textbf{Weston/libweston}: Reference implementation + \item \textbf{Sway/wlroots}: Tiling window manager and base library + \item \textbf{Mutter}: GNOME compositor + \end{itemize} + \item \textbf{Display client} libraries + \begin{itemize} + \item Using the \textbf{libwayland-client} base protocol library + \item Integrated in many higher-level graphics-oriented libraries + \end{itemize} + \end{itemize} + \end{minipage} + \begin{minipage}[b]{0.15\textwidth} + \includegraphics[width=\textwidth]{slides/graphics-software-stack-overview/wayland-logo.pdf} + \end{minipage} +\end{frame} + +\begin{frame}{High-level graphics libraries and desktop environments overview} + \begin{minipage}[b]{0.09\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-software-stack-overview/gtk-logo.pdf}\\ + \vspace{3em} + \includegraphics[width=\textwidth]{slides/graphics-software-stack-overview/qt-logo.pdf}\\ + \vspace{3em} + \includegraphics[width=\textwidth]{slides/graphics-software-stack-overview/sdl-logo.png}\\ + \vspace{2em} + \end{minipage} + \hfill + \begin{minipage}[b]{0.8\textwidth} + \begin{itemize} + \item Applications rarely to never use Wayland or X11 directly + \item Drawing and managing a user interface is complex + \item Widely-used high-level graphics libraries (aka toolkits) + \begin{itemize} + \item \textbf{GTK} (C): Widget-based UI toolkit, drawing helpers (GDK) + \item \textbf{Qt} (C++): Widget-based UI toolkit, wide framework + \item \textbf{EFL} (C): Lightweight UI and application library + \item \textbf{SDL} (C): Drawing-oriented graphics library (used in games) + \end{itemize} + \item A desktop environment groups related libraries and components\\ + \textit{gives a consistent look and feel across the system} + \item \textbf{Desktop environment} examples + \begin{itemize} + \item \textbf{GNOME}: Using GTK, GNOME-Shell desktop + \item \textbf{KDE}: Using Qt, Plasma desktop + \item \textbf{Xfce}: Using GTK, lightweight + \item \textbf{Enlightenment}: Using EFL + \end{itemize} + \end{itemize} + \vfill~ + \end{minipage} + \begin{minipage}[b]{0.09\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-software-stack-overview/gnome-logo.pdf}\\ + \vspace{1em} + \includegraphics[width=\textwidth]{slides/graphics-software-stack-overview/kde-logo.pdf}\\ + \vspace{1em} + \includegraphics[width=\textwidth]{slides/graphics-software-stack-overview/xfce-logo.pdf}\\ + \vspace{0.5em} + \includegraphics[width=\textwidth]{slides/graphics-software-stack-overview/enlightenment-logo.pdf} + \end{minipage} +\end{frame} diff --git a/slides/graphics-software/gtk-logo.svg b/slides/graphics-software-stack-overview/gtk-logo.svg similarity index 100% rename from slides/graphics-software/gtk-logo.svg rename to slides/graphics-software-stack-overview/gtk-logo.svg diff --git a/slides/graphics-software/kde-logo.svg b/slides/graphics-software-stack-overview/kde-logo.svg similarity index 100% rename from slides/graphics-software/kde-logo.svg rename to slides/graphics-software-stack-overview/kde-logo.svg diff --git a/slides/graphics-software/qt-logo.svg b/slides/graphics-software-stack-overview/qt-logo.svg similarity index 100% rename from slides/graphics-software/qt-logo.svg rename to slides/graphics-software-stack-overview/qt-logo.svg diff --git a/slides/graphics-software/sdl-logo.png b/slides/graphics-software-stack-overview/sdl-logo.png similarity index 100% rename from slides/graphics-software/sdl-logo.png rename to slides/graphics-software-stack-overview/sdl-logo.png diff --git a/slides/graphics-software/wayland-logo.svg b/slides/graphics-software-stack-overview/wayland-logo.svg similarity index 100% rename from slides/graphics-software/wayland-logo.svg rename to slides/graphics-software-stack-overview/wayland-logo.svg diff --git a/slides/graphics-software/x11-logo.svg b/slides/graphics-software-stack-overview/x11-logo.svg similarity index 100% rename from slides/graphics-software/x11-logo.svg rename to slides/graphics-software-stack-overview/x11-logo.svg diff --git a/slides/graphics-software/xfce-logo.svg b/slides/graphics-software-stack-overview/xfce-logo.svg similarity index 100% rename from slides/graphics-software/xfce-logo.svg rename to slides/graphics-software-stack-overview/xfce-logo.svg diff --git a/slides/graphics-software/egl-logo.svg b/slides/graphics-software-userspace-mesa/egl-logo.svg similarity index 100% rename from slides/graphics-software/egl-logo.svg rename to slides/graphics-software-userspace-mesa/egl-logo.svg diff --git a/slides/graphics-software-userspace-mesa/graphics-software-userspace-mesa.tex b/slides/graphics-software-userspace-mesa/graphics-software-userspace-mesa.tex new file mode 100644 index 0000000000..aa1f2e3442 --- /dev/null +++ b/slides/graphics-software-userspace-mesa/graphics-software-userspace-mesa.tex @@ -0,0 +1,452 @@ +\subsection{Userspace: Mesa 3D} + +\begin{frame}[t]{Standardized 3D rendering APIs: OpenGL} + \begin{center} + \includegraphics[height=4em]{slides/graphics-software-userspace-mesa/opengl-logo.pdf} + \end{center} + + \begin{itemize} + \item 3D rendering API, designed for GPU \textbf{hardware acceleration} + \begin{itemize} + \item Generic API but \textbf{hardware-specific implementations} + \item Started by \textbf{Silicon Graphics} in 1992, now managed by the \textbf{Khronos Group} + \end{itemize} + \item OpenGL provides a \textbf{high-level approach} to 3D graphics + \begin{itemize} + \item Compromise between complexity and fine-grained control + \item Efficient abstraction, adapted to the hardware + \item Leaves most memory management to the implementation + \end{itemize} + \item \textbf{Stateful} and \textbf{context-based} programming model + \end{itemize} +\end{frame} + +\begin{frame}[t]{Standardized 3D rendering APIs: OpenGL} + \begin{center} + \includegraphics[height=4em]{slides/graphics-software-userspace-mesa/opengl-logo.pdf} + \end{center} + + \begin{itemize} + \item OpenGL versions evolved with hardware features + \begin{itemize} + \item Version 1 targeted fixed-function pipeline GPUs + \item Version 2 and up allow programming \textbf{vertex and fragment shaders} + \item More shaders supported with new versions (\textit{geometry, tesselation}) + \end{itemize} + \item OpenGL comes with the \textbf{GL Shading Language} (GLSL) + \begin{itemize} + \item Source code language for OpenGL shaders + \item C-like syntax with intrinsic functions (e.g. texture access) + \item Compiled on-the-fly by the GL implementation + \end{itemize} + \item Supports \textbf{extensions} that can be queried, for extra features + \end{itemize} +\end{frame} + +\begin{frame}[t]{Standardized 3D rendering APIs: OpenGL ES and EGL} + \begin{minipage}[t]{0.49\textwidth} + \centering + \includegraphics[height=4em]{slides/graphics-software-userspace-mesa/opengl-es-logo.pdf} + \end{minipage} + \hfill + \begin{minipage}[t]{0.49\textwidth} + \centering + \includegraphics[height=4em]{slides/graphics-software-userspace-mesa/egl-logo.pdf} + \end{minipage} + \vspace{0.5em} + \begin{itemize} + \item \textbf{OpenGL ES} was introduced as a simplified version for embedded devices + \item OpenGL ES versions are loosely following OpenGL versions: + \begin{itemize} + \item Version 1 targets \textbf{fixed-function} GPUs + \item Version 2 and up target \textbf{programmable} GPUs + \end{itemize} + \item Uses GLSL shaders and the same programming model as OpenGL + \item \textbf{EGL} was introduced as standardized window integration API + \begin{itemize} + \item Connects with the native system display server + \item Replaces GLX for X11 and adopted as default by Wayland + \end{itemize} + \item Supports \textbf{extensions} that can be queried, for extra platform-specific features + \end{itemize} +\end{frame} + +\begin{frame}[t]{Standardized 3D rendering APIs: Vulkan} + \begin{center} + \includegraphics[height=4em]{slides/graphics-software-userspace-mesa/vulkan-logo.pdf} + \end{center} + + \begin{itemize} + \item \textbf{Vulkan} is a \textbf{low-level} generic API for GPU access + \begin{itemize} + \item Started by the Khronos group in 2016 and widely adopted + \end{itemize} + \item Suitable for both \textbf{3D rendering} and \textbf{compute} + \item Uses \textbf{Standard Portable Intermediate Representation} (SPIR-V) shaders + \begin{itemize} + \item Unified intermediate representation from (adapted) GLSL/HLSL sources + \item Compiled with the program instead of on-the-fly (less overhead) + \item Translated to native GPU operations by implementations + \end{itemize} + \item \textbf{Direct programming} model, with low-level \textbf{memory management} + \item The API provides \textbf{window system integration} (WSI) for many platforms\\ + \textit{e.g. for Wayland: \code{vkCreateWaylandSurfaceKHR}} + \end{itemize} +\end{frame} + +\begin{frame}{Mesa 3D overview} + \begin{itemize} + \item Mesa is the reference free software \textbf{3D graphics implementation} + \begin{itemize} + \item Started back in 1993, evolved with GPU implementations + \item Project works with the Khronos Group and develops extensions + \end{itemize} + \item Implements support for \textbf{rendering} APIs: + \begin{itemize} + \item \textbf{OpenGL} (up to 4.6) and \textbf{OpenGL ES} (up to 3.2) + \item \textbf{Vulkan} (up to 1.1) with translation to OpenGL via \textbf{Zink} + \item \textbf{Direct 3D} (version 9 only) + \end{itemize} + \item Implements \textbf{windowing system} integration: + \begin{itemize} + \item \textbf{EGL} for Wayland, X11, Android, native DRM (GBM) and surface-less + \item \textbf{Vulkan WSI} for Wayland and X11 (XCB/Xlib) + \item \textbf{GLX} for X11 + \end{itemize} + \item Also supports other \textbf{GPU-related features}: + \begin{itemize} + \item \textbf{Video decoding} acceleration via VDPAU, VAAPI, OMX + \item \textbf{Compute} (GPGPU) support via OpenCL (\code{clover}) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Mesa 3D implementation highlights} + \begin{itemize} + \item Unlike other devices, 3D hardware is \textbf{abstracted in userspace} + \begin{itemize} + \item 3D rendering is a very bad fit for in-kernel abstraction + \item Kernel drivers are much less complicated than GL implementations + \end{itemize} + \item Mesa implements driver-specific \textbf{DRM render} support\\ + \begin{itemize} + \item Manages memory with the GEM and Prime DRM APIs + \item Manages DRI2 to allow direct rendering + \end{itemize} + \item \textbf{Virtual} drivers are also supported (for virtual machines): + \begin{itemize} + \item \textbf{vmwgfx}: VMware bridge (proprietary virtual hardware implementation) + \item \textbf{virgl}: Virtio bridge (standard for Linux and QEMU) + \end{itemize} + \item Also provides \textbf{software backends}: + \begin{itemize} + \item \textbf{softpipe}: Generic reference software renderer + \item \textbf{swr}: OpenSWR renderer (for x86 by Intel) + \item \textbf{llvmpipe}: LLVM-based renderer (high-performance) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Mesa 3D internals: Gallium 3D} + \begin{itemize} + \item Classic mesa drivers have significant \textbf{code duplication}: + \begin{itemize} + \item API state tracking + \item Compiler implementation + \end{itemize} + \item The Gallium 3D interface splits things up instead: + \begin{itemize} + \item \textbf{API State trackers}: maintain the current state for the API in use + \item \textbf{Drivers}: implement shader compilation and hardware configuration + \item \textbf{Winsys}: implement low-level kernel interfaces + \end{itemize} + \item Gallium drivers implement a pipe interface: + \begin{itemize} + \item \code{struct pipe_screen}: textures, buffers and sync management + \item \code{struct pipe_state}: pipeline configuration and resources state + \item \code{struct pipe_context}: rendering operation functions + \end{itemize} + \item Pipe loaders (DRM or software) select the right pipe driver + \end{itemize} +\end{frame} + +\begin{frame}{Mesa 3D internals: Gallium 3D (illustrated)} + \begin{center} + \includegraphics[width=0.5\textwidth]{slides/graphics-software-userspace-mesa/mesa-dri-gallium.pdf}\\ + \textit{\small Driver integration in Mesa 3D} + \end{center} +\end{frame} + +\begin{frame}{Mesa 3D internals: intermediate representations} + \begin{itemize} + \item Mesa is in charge of \textbf{compiling shaders} to the native shading ISA + \item \textbf{Intermediate representations} (IRs) are used for translation + \item \textbf{Input-level} IRs: + \begin{itemize} + \item \textbf{GLSL IR}: Internal GLSL shader representation + \item \textbf{SPIR-V}: Khronos' Standard Portable Intermediate Representation + \end{itemize} + \item \textbf{Internal} IRs: + \begin{itemize} + \item \textbf{TGSI IR}: Tungsten Graphics Shader Interface representation + \item \textbf{NIR}: New efficient internal representation for Mesa + \item \textbf{Mesa}: Historical implementation (deprecated) + \end{itemize} + \item \textbf{External} IR: + \begin{itemize} + \item \textbf{LLVM IR}: Used for LLVM interaction (e.g. with llvmpipe) + \end{itemize} + \item Drivers emit \textbf{native instructions} from an internal IR and lowering + \item Once ready, compiled shaders are \textbf{submitted to the GPU} + \end{itemize} +\end{frame} + +\begin{frame}{Mesa 3D internals: intermediate representations (illustrated)} + \begin{center} + \includegraphics[width=0.4\textwidth]{slides/graphics-software-userspace-mesa/mesa-ir.pdf}\\ + \textit{\small The Mesa 3D internal representation pipeline} + \end{center} +\end{frame} + +\begin{frame}[fragile]{Mesa 3D Generic Buffer Management} + \begin{itemize} + \item Mesa provides a Generic Buffer Management (GBM) interface: + \begin{itemize} + \item Buffer creation/destruction (supporting modifiers) + \item Buffer information (bpp, dimensions, planes, stride, modifiers) + \item Buffer mapping/unmapping + \item Buffer dma-buf import/unimport + \end{itemize} + \item Compatible with the EGL API: + \begin{itemize} + \item \code{struct gbm_device} as \code{EGLNativeDisplayType} + \item \code{struct gbm_surface} as \code{EGLNativeWindowType} + \item \code{struct gbm_bo} as \code{EGLNativePixmapType} + \end{itemize} + \item Provided with DRM KMS fd for DRI2 (used internally for most operations)\\ + \begin{minted}[fontsize=\small]{console} +struct gbm_device *device = gbm_create_device(drm_fd); + \end{minted} + \item Useful when using bare-metal DRM KMS + \end{itemize} +\end{frame} + +\begin{frame}{Mesa 3D hardware support status: desktop} + \begin{itemize} + \item Updated Mesa per-driver support at \url{https://mesamatrix.net} + \item \textbf{Intel HD/Iris Graphics} + \begin{itemize} + \item Platforms: Intel only + \item Mesa driver: i965 (classic), iris (Gallium) + \item DRM driver: i915 + \item Status: state-of-the art (i965/iris) + \end{itemize} + \item \textbf{Nvidia pre-NV110} + \begin{itemize} + \item Platforms: Tegra, any PCI-e compatible + \item Mesa driver: nouveau (Gallium) + \item DRM driver: nouveau + \item Status: reverse engineered, advanced + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Mesa 3D hardware support status: desktop} + \begin{itemize} + \item \textbf{AMD Radeon GCN-ish} + \begin{itemize} + \item Platforms: AMD, any PCI-e compatible + \item Mesa driver: radeonsi (Gallium) + \item DRM driver: amdgpu + \item Status: state-of-the art + \end{itemize} + \item \textbf{AMD Radeon R600+} + \begin{itemize} + \item Platform: AMD, any PCI-e compatible + \item Driver: r600 (Gallium) + \item DRM driver: radeon + \item Status: advanced + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Mesa 3D hardware support status: embedded} + \begin{itemize} + \item \textbf{Qualcomm Adreno} + \begin{itemize} + \item Platforms: Qualcomm Snapdragon + \item Mesa driver: freedreno (Gallium) + \item DRM driver: freedreno + \item Status: reverse engineered, advanced + \end{itemize} + \item \textbf{Vivante GCx000} + \begin{itemize} + \item Platforms: i.MX6, i.MX8, i.MX8M + \item Driver: etnaviv (Gallium) + \item DRM driver: etnaviv + \item Status: vastly usable + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Mesa 3D hardware support status: embedded} + \begin{itemize} + \item \textbf{ARM Mali Utgard} + \begin{itemize} + \item Platforms: Exynos, Allwinner, Amlogic + \item Mesa driver: lima (Gallium) + \item DRM driver: lima + \item Status: reverse engineered, usable + \end{itemize} + \item \textbf{ARM Mali Midgard/Bifrost} + \begin{itemize} + \item Platforms: Rockchip, Exynos, Mediatek, Allwinner + \item Mesa driver: panfrost (Gallium) / PanVK (Vulkan) + \item DRM driver: panfrost + \item Status: advanced + \end{itemize} + \item \textbf{Imagination PowerVR Rogue} + \begin{itemize} + \item Platforms: Mediatek + \item Mesa driver: imagination + \item DRM driver: imagination + \item Status: work in progress + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Mesa 3D versus proprietary implementations} + \begin{itemize} + \item 3D support is one of the most challenging parts of hardware integration + \item Proprietary implementations easily lead to various practical issues: + \begin{itemize} + \item Lack of support outside of prescribed environments + \item Lack of specific features or APIs + \item Lack of maintenance and updates + \item No adaptation possibility + \end{itemize} + \item Mesa provides a collectively-maintained base + \begin{itemize} + \item Constantly updated and improved by the community + \item Easier to manage: works out of the box with distributions + \end{itemize} + \item Mesa support is complex and often takes some time to bring performance\\ + \textit{especially for drivers based on reverse-engineering} + \end{itemize} +\end{frame} + +\begin{frame}{Mesa 3D code structure and walkthrough} + \begin{itemize} + \item Mesa source code available at: \url{https://gitlab.freedesktop.org/mesa/mesa} + \item \textbf{Gallium 3D} components: + \begin{itemize} + \item Drivers under: \code{src/gallium/drivers/} + \item Winsys under: \code{src/gallium/winsys/} + \item API state trackers under: \code{src/gallium/state_trackers/} + \item Pipe loaders under: \code{src/gallium/auxiliary/pipe-loader/} + \end{itemize} + \item \textbf{Compilation and IR} components: + \begin{itemize} + \item IR compiler support under: \code{src/compiler/{glsl,nir,spirv}} + \item TGSI support under: \code{src/gallium/auxiliary/tgsi/} + \item State tracking between IRs under: \code{src/mesa/state_tracker} + \end{itemize} + \item \textbf{Windowing and DRI2} components: + \begin{itemize} + \item EGL support under: \code{src/egl/drivers/dri2/} + \item Vulkan WSI support under: \code{src/vulkan/wsi/} + \end{itemize} + \item \textbf{Classic drivers} (DRI-1-style) under: \code{src/mesa/drivers/dri/} + \item \textbf{GBM} support under: \code{src/gbm/} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{Mesa 3D hardware support: debug and documentation} + \begin{itemize} + \item Mesa is debugged with numerous \textbf{environment variables} + \begin{itemize} + \item Generic and per-driver, see \url{https://www.mesa3d.org/envvars.html} + \item Shader-related, see \url{https://www.mesa3d.org/shading.html} + \item \code{LIBGL_DEBUG=verbose} for OpenGL, \code{EGL_LOG_LEVEL=debug} for EGL + \end{itemize} + \item \code{eglinfo} and \code{glxinfo} show information about the implementation + \item \textbf{Community} contact: + \begin{itemize} + \item Mailing list: \code{dri-devel@lists.freedesktop.org} + \item IRC channel: \code{#dri-devel} on the OFTC network + \end{itemize} + \item \textbf{Documentation} resources: + \begin{itemize} + \item Online website: \url{https://www.mesa3d.org/} + \item Gallium 3D wiki: \url{https://www.freedesktop.org/wiki/Software/gallium/} + \item Gallium 3D documentation: \url{https://gallium.readthedocs.io/} + \item Source code reference: \url{https://elixir.bootlin.com/mesa/latest/source} + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Graphics software online references} + \small + \begin{minipage}[t]{0.45\textwidth} + \begin{itemize} + \item Linux man pages + \item Wikipedia (\url{https://en.wikipedia.org/}): + \begin{itemize} + \item \href{https://en.wikipedia.org/wiki/X_Window_System}{X Window System} + \item \href{https://en.wikipedia.org/wiki/X.Org_Server}{X.Org Server} + \item \href{https://en.wikipedia.org/wiki/Wayland_(display_server_protocol)}{Wayland (display server protocol)} + \item \href{https://en.wikipedia.org/wiki/Mesa_(computer_graphics)}{Mesa (computer graphics)} + \item \href{https://en.wikipedia.org/wiki/OpenGL}{OpenGL} + \item \href{https://en.wikipedia.org/wiki/Vulkan_(API)}{Vulkan (API)} + \end{itemize} + \end{itemize} + \end{minipage} + \hfill + \begin{minipage}[t]{0.5\textwidth} + \begin{itemize} + \item Freedesktop.org (\url{https://freedesktop.org/}): + \begin{itemize} + \item \href{https://dri.freedesktop.org/wiki/}{Direct Rendering Infrastructure} + \item \href{https://dri.freedesktop.org/wiki/DRM/}{DRM} + \item \href{https://wayland.freedesktop.org/}{Wayland} + \item \href{https://www.x.org/wiki/}{X.org wiki} + \end{itemize} + \item Khronos (\url{https://khronos.org/}): + \begin{itemize} + \item \href{https://www.khronos.org/opengl/}{OpenGL} + \item \href{https://www.khronos.org/opengl/wiki/}{OpenGL Wiki} + \item \href{https://www.khronos.org/egl/}{EGL} + \item \href{https://www.khronos.org/vulkan/}{Vulkan} + \end{itemize} + \end{itemize} + \end{minipage} +\end{frame} + +\begin{frame}{Graphics software illustrations attributions} + \small + \begin{itemize} + \item \href{https://commons.wikimedia.org/wiki/File:X11.svg}{X11 logo: public domain} + \item \href{https://commons.wikimedia.org/wiki/File:Wayland_Logo.svg}{Wayland logo: Kristian Høgsberg} + \item \href{https://commons.wikimedia.org/wiki/File:GTK_logo.svg}{GTK logo: Andreas Nilsson, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:GTK_logo.svg}{Qt logo: Qt Project, public domain} + \item \href{https://commons.wikimedia.org/wiki/File:SDL_Logo.svg}{SDL logo: Arne Claus / SDL Project, public domain} + \item \href{https://commons.wikimedia.org/wiki/File:Gnomelogo.svg}{GNOME logo: GNOME Foundation, GNU LGPL 2.1+} + \item \href{https://commons.wikimedia.org/wiki/File:KDE_logo.svg}{KDE logo: KDE e.V., GNU LGPL 2.1+} + \item \href{https://commons.wikimedia.org/wiki/File:Xfce_logo.svg}{XFCE logo: Xfce Team, GNU LGPL 2.1+} + \item \href{https://commons.wikimedia.org/wiki/File:Enlightenment_logo_black.png}{Enlightenment logo: Carsten Haitzler and various contributors, BSD} + \end{itemize} +\end{frame} + +\begin{frame}{Graphics software illustrations attributions} + \small + \begin{itemize} + \item \href{https://commons.wikimedia.org/wiki/File:Devuan_GNU-Linux_-_tty_login_-_server_rack.jpg}{Devuan GNU-Linux - tty login - server rack: Francesco Magno, CC BY-SA 4.0} + \item \href{https://commons.wikimedia.org/wiki/File:DRM_architecture.svg}{DRM architecture: Javier Cantero CC BY-SA 4.0} + \item \href{https://wayland.freedesktop.org/architecture.html}{Wayland architecture: Wayland Developers, GNU GPL 2+} + \item \href{https://wayland.freedesktop.org/architecture.html}{X architecture: Wayland Developers, GNU GPL 2+} + \item \href{https://commons.wikimedia.org/wiki/File:Wayland_display_server_protocol.svg}{Wayland display server protocol: Shmuel Csaba Otto Traian, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:The_Linux_Graphics_Stack_and_glamor.svg}{The Linux Graphics Stack and glamor: Shmuel Csaba Otto Traian, CC BY-SA 3.0} + \item \href{https://www.khronos.org/assets/utilities/retrieveFile.php?d=opengl&t=logopacks}{Khronos logo pack} + \item \href{https://commons.wikimedia.org/wiki/File:Gallium3D_vs_DRI_graphics_driver_model.svg}{Gallium3D vs DRI graphics driver model, CC BY-SA 3.0} + \end{itemize} +\end{frame} diff --git a/slides/graphics-software/mesa-dri-gallium.svg b/slides/graphics-software-userspace-mesa/mesa-dri-gallium.svg similarity index 100% rename from slides/graphics-software/mesa-dri-gallium.svg rename to slides/graphics-software-userspace-mesa/mesa-dri-gallium.svg diff --git a/slides/graphics-software/mesa-ir.dia b/slides/graphics-software-userspace-mesa/mesa-ir.dia similarity index 100% rename from slides/graphics-software/mesa-ir.dia rename to slides/graphics-software-userspace-mesa/mesa-ir.dia diff --git a/slides/graphics-software/opengl-es-logo.svg b/slides/graphics-software-userspace-mesa/opengl-es-logo.svg similarity index 100% rename from slides/graphics-software/opengl-es-logo.svg rename to slides/graphics-software-userspace-mesa/opengl-es-logo.svg diff --git a/slides/graphics-software/opengl-logo.svg b/slides/graphics-software-userspace-mesa/opengl-logo.svg similarity index 100% rename from slides/graphics-software/opengl-logo.svg rename to slides/graphics-software-userspace-mesa/opengl-logo.svg diff --git a/slides/graphics-software/vulkan-logo.svg b/slides/graphics-software-userspace-mesa/vulkan-logo.svg similarity index 100% rename from slides/graphics-software/vulkan-logo.svg rename to slides/graphics-software-userspace-mesa/vulkan-logo.svg diff --git a/slides/graphics-software-userspace-wayland/graphics-software-userspace-wayland.tex b/slides/graphics-software-userspace-wayland/graphics-software-userspace-wayland.tex new file mode 100644 index 0000000000..e9d54751ec --- /dev/null +++ b/slides/graphics-software-userspace-wayland/graphics-software-userspace-wayland.tex @@ -0,0 +1,232 @@ +\subsection{Userspace: Wayland} + +\begin{frame}{Wayland overview and paradigm} + \begin{itemize} + \item Wayland was started in 2008 as a modern replacement for the X Window system\\ + \textit{solving issues in-depth with a clean implementation from scratch} + \item Drastic \textbf{simplification} of the stack and paradigm shift: + \begin{itemize} + \item The server and compositor are \textbf{unified} as the same component + \item \textbf{Clients} are expected to do all the rendering + \item Buffers are \textbf{shared} between client and server, no transfers + \item Window decorations can be added by the client or the server + \end{itemize} + \item Improves \textbf{security} aspects: + \begin{itemize} + \item Isolates the input and output of each client + \item Only the compositor can access display buffers \textit{(and provide screenshots)} + \item Avoids running the compositor as root (using \code{systemd-logind}) + \end{itemize} + \item No network support (can be implemented by compositors) + \item \textbf{Weston} is the reference \textbf{Wayland} compositor + \end{itemize} +\end{frame} + +\begin{frame}{Wayland architecture: input to display roundtrip} + \begin{minipage}{0.49\textwidth} + \centering + \includegraphics[height=0.7\textheight]{slides/graphics-software-userspace-wayland/wayland-architecture-roundtrip.png} + \end{minipage} + \hfill + \begin{minipage}{0.49\textwidth} + \begin{enumerate} + \item An input event is read from the kernel by the compositor + \item The affected client is determined and receives the event + \item The client changes something, performs rendering and notifies the compositor + \item The compositor updates the damaged regions in the back-buffer and performs page flip + \end{enumerate} + \end{minipage} +\end{frame} + +\begin{frame}{Wayland protocol and architecture} + \begin{itemize} + \item Wayland provides a client-server \textbf{low-level API}: + \begin{itemize} + \item Wayland \textbf{display connections} happen through local IPC + \item Display is identified with the \code{WAYLAND_DISPLAY} \textbf{environment variable} + \item \textbf{Asynchronous} and \textbf{object-oriented} protocol + \item \textbf{Objects} represent \textbf{resources} from the compositor + \item Objects implement specific \textbf{interfaces}, with requests and events + \begin{itemize} + \item \textbf{Requests} are messages sent by the client, + \item \textbf{Events} are messages received from the server\\ + \textit{errors are only specific types of events} + \end{itemize} + \end{itemize} + \item Some \textbf{implementation} details: + \begin{itemize} + \item IPC is a regular UNIX socket (allows passing file descriptors for zero-copy) + \item A proxy provides client-side object representations and message translation + \item Messages are serialized (marshalling) to the wire format + \item Messages are buffered and flushed/dispatched when asked + \end{itemize} + \item Client-server protocol is implemented in \textbf{libwayland-client} and \textbf{libwayland-server} + \item These libraries do not provide any interface implementation + \end{itemize} +\end{frame} + +\begin{frame}{Wayland core protocol: global object interfaces} + \begin{itemize} + \item \textbf{Global} core object interfaces: + \begin{itemize} + \item \code{wl_display}: manages server connection, exposes the registry + \item \code{wl_registry}: exposes available global object interfaces + \item \code{wl_output}: describes the output properties (mode, geometry) + \item \code{wl_seat}: exposes input device object capabilities and interfaces + \item \code{wl_compositor}: provides surfaces and regions for composition + \item \code{wl_subcompositor}: provides sub-surfaces for in-surface compositing + \item \code{wl_shm}: exposes a shared memory interface + \item \code{wl_data_device_manager}: support for copy/paste between clients + \end{itemize} + \item Global object interfaces are bound with the registry before use + \begin{itemize} + \item Using global \code{struct wl_interface} definitions + \item \code{wl_registry_bind} returns a pointer to a proxy object + \end{itemize} + \item Global objects give access to other \textbf{specific objects} + \end{itemize} +\end{frame} + +\begin{frame}{Wayland core protocol: specific object interfaces} + \begin{itemize} + \item \textbf{Input-related} (\code{wl_seat}) specific core object interfaces: + \begin{itemize} + \item \code{wl_pointer}: exposes mice events and cursor + \item \code{wl_keyboard}: exposes keyboard events and information + \item \code{wl_touch}: exposes touchscreen events + \end{itemize} + \item \textbf{Compositor-related} (\code{wl_compositor}) specific core object interfaces: + \begin{itemize} + \item \code{wl_region}: specifies any area + \item \code{wl_surface}: rectangular on-screen pixel area + \begin{itemize} + \item contents set to a \code{wl_buffer} (can be transformed) + \item configured with a \code{wl_region} for input + \item configured with a \code{wl_region} for area-based opacity + \item updated with buffer damage regions + \end{itemize} + \item \code{wl_subsurface}: converts \code{wl_surface} objects to positioned sub-surfaces + \end{itemize} + \item \textbf{Shared memory-related} (\code{wl_shm}) specific core object interfaces: + \begin{itemize} + \item \code{wl_shm_pool}: allows creating shared-memory buffers + \end{itemize} + \item \textbf{Memory-related} specific core object interfaces: + \begin{itemize} + \item \code{wl_buffer}: generic object for a \code{wl_surface} + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Wayland extra protocols} + \begin{itemize} + \item Extra protocols (object interfaces) can be \textbf{exposed by the compositor} + \begin{itemize} + \item Protocols (including the core) are described as XML files + \item The \code{wayland-scanner} tool produces client and server C code and headers + \item Accepted additional protocol descriptions are available at: + \url{https://gitlab.freedesktop.org/wayland/wayland-protocols} + \item Some are considered stable and many unstable + \end{itemize} + \item Some widely-used \textbf{protocol extensions}: + \begin{itemize} + \item \textbf{XDG-Shell}: desktop shell integration + \begin{itemize} + \item Turns \code{wl_surfaces} to \code{xdg_surfaces} that can be resized, minimized, etc + \item Provides a popup/menu interface with \code{xdg_popup} + \end{itemize} + \item \textbf{IVI-Shell}: In-vehicle shell with surface layers + \item \textbf{Presentation time}: Precise timing and event feedback + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Wayland OpenGL integration} + \begin{itemize} + \item Wayland supports EGL for windowing integration with OpenGL + \item \code{eglGetDisplay} is called with a \code{struct wl_display}\\ + \textit{mesa's \code{_eglNativePlatformDetectNativeDisplay} figures it out} + \item Mesa 3D implements Wayland EGL interface for OpenGL integration + \begin{itemize} + \item Needs to implement DRI2 for DRM authentication + \item \code{wl_drm} interface between the wayland EGL client and the compositor + \item Both sides are actually implemented in mesa + \item The interface is bound to the compositor with \code{eglBindWaylandDisplayWL}\\ + \textit{using the compositor's EGL context as entry-point to mesa} + \item Allows sharing DRM GEM buffers with the compositor + \end{itemize} + \item Regular \code{wl_surfaces} can be bound to EGL: + \begin{itemize} + \item Converted to a \code{wl_egl_window} with \code{wl_egl_window_create} + \item Then converted to an \code{EGLSurface} with \code{eglCreateWindowSurface} + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Wayland OpenGL integration (illustrated)} + \begin{center} + \includegraphics[width=12em]{slides/graphics-software-userspace-wayland/wayland-egl.pdf}\\ + \textit{\small Wayland integration with EGL}\\ + \end{center} +\end{frame} + +\begin{frame}{Wayland status and adoption} + \begin{itemize} + \item Wayland is now quite mature, robust, efficient and widely used + \begin{itemize} + \item Most toolkits have support for it: GTK 3, Qt 5, SDL, EFL + \item Major desktop environments support it: GNOME 3, KDE Plasma 5 + \item Integrated sessions with login managers from \code{/usr/share/wayland-sessions} + \item Runs with user privileges with \code{systemd-logind} + \end{itemize} + \item X11 applications can be integrated using \textbf{XWayland} + \begin{itemize} + \item X server implementation registered as a Wayland client + \item Wayland composer acts as X compositing window manager + \item Creates a \code{wl_surface} for each window, redirects input/output + \end{itemize} + \item Diverse compositor implementations have emerged + \begin{itemize} + \item Sometimes tied to a desktop environment: mutter, kwin + \item \textbf{libinput} was created to help with input aspects + \item \textbf{libweston} emerged from the \textbf{Weston} compositor core + \item \textbf{wlroots} emerged from the \textbf{Sway} compositor core + \item Support \textbf{DRM KMS} display, \textbf{EGL} rendering (\textbf{pixman} supported by libweston) + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Wayland stack overview (illustrated)} + \begin{center} + \includegraphics[width=0.8\textwidth]{slides/graphics-software-userspace-wayland/wayland-stack.pdf}\\ + \textit{\small The graphics stack with Wayland}\\ + \end{center} +\end{frame} + +\begin{frame}[fragile]{Wayland debug and documentation} + \begin{itemize} + \item \textbf{Debugging} tips: + \begin{itemize} + \item Supported global object interfaces can be listed with \code{weston-info} + \item The \code{WAYLAND_DEBUG} environment variable enables protocol tracing + \end{itemize} + \item \textbf{Weston debugging}: + \begin{itemize} + \item Debug arguments: \code{--debug}, \code{--log=file.log} + \item Grabbing a different TTY argument: \code{--tty 1} + \item Wireframe keystroke: \code{mod + shift + space + F} + \item Timeline recording (to a JSON file) keystroke: \code{mod + shift + space + d}\\ + \textit{can produce a graph with \url{https://github.com/ppaalanen/wesgr}} + \end{itemize} + \item \textbf{Community} contact: + \begin{itemize} + \item Mailing list: \code{wayland-devel@lists.freedesktop.org} + \item IRC channel: \code{#wayland} on the OFTC network + \end{itemize} + \item \textbf{Documentation} resources: + \begin{itemize} + \item Online wiki of the project: \url{https://wayland.freedesktop.org/} + \item Online documentation: \url{https://wayland.freedesktop.org/docs/html/} + \end{itemize} + \end{itemize} +\end{frame} diff --git a/slides/graphics-software/wayland-architecture-roundtrip.png b/slides/graphics-software-userspace-wayland/wayland-architecture-roundtrip.png similarity index 100% rename from slides/graphics-software/wayland-architecture-roundtrip.png rename to slides/graphics-software-userspace-wayland/wayland-architecture-roundtrip.png diff --git a/slides/graphics-software/wayland-egl.svg b/slides/graphics-software-userspace-wayland/wayland-egl.svg similarity index 100% rename from slides/graphics-software/wayland-egl.svg rename to slides/graphics-software-userspace-wayland/wayland-egl.svg diff --git a/slides/graphics-software/wayland-stack.svg b/slides/graphics-software-userspace-wayland/wayland-stack.svg similarity index 100% rename from slides/graphics-software/wayland-stack.svg rename to slides/graphics-software-userspace-wayland/wayland-stack.svg diff --git a/slides/graphics-software/dri-data-flow.png b/slides/graphics-software-userspace-x/dri-data-flow.png similarity index 100% rename from slides/graphics-software/dri-data-flow.png rename to slides/graphics-software-userspace-x/dri-data-flow.png diff --git a/slides/graphics-software-userspace-x/graphics-software-userspace-x.tex b/slides/graphics-software-userspace-x/graphics-software-userspace-x.tex new file mode 100644 index 0000000000..13fd571751 --- /dev/null +++ b/slides/graphics-software-userspace-x/graphics-software-userspace-x.tex @@ -0,0 +1,249 @@ +\subsection{Userspace: X Window} + +\begin{frame}{X11 protocol and architecture} + \begin{itemize} + \item X11 core protocol implemented by Xorg: + \begin{itemize} + \item Asynchronous packet-based system with different types:\\ + \code{Request}, \code{Reply}, \code{Event} and \code{Error} packets + \item Can be used locally (UNIX socket) or over network (TCP/IP) + \end{itemize} + \item Exposes \textbf{drawables} for clients to transfer or draw pixel data to the server: + \begin{itemize} + \item \textbf{Windows}: area of the display buffer owned by the application\\ + \textit{without backing storage, must be redrawn when occluded} + \item \textbf{Pixmaps}: off-display backing storage that can be copied to windows + \end{itemize} + \item Windows are represented as a tree: + \begin{itemize} + \item Starting with the root window created by X + \item Top-level windows and sub-windows created by clients + \end{itemize} + \item A graphics context (GC) allows requesting basic drawing and font rendering + \item The server provides \textbf{input events} to concerned clients: + \begin{itemize} + \item Mouse movements relative to window coordinates + \item Translated key symbols a from raw keycodes + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{X11 protocol extensions} + \begin{itemize} + \item X11 has evolved over time through \textbf{extensions} to its main protocol + \begin{itemize} + \item Additional interfaces for X clients, matching \textbf{new hardware features} + \end{itemize} + \item \textbf{XKB}: complex keyboard layouts + \item \textbf{Xinput2}: touchpad, touchscreen and multi-touch support + \item \textbf{XSHM}: shared client/server memory, avoiding extra transfers/copies\\ + \textit{not\ possible to operate via the network} + \item \textbf{XRandR}: monitor configuration and hotplugging without server restart + \item \textbf{Composite}: delegates window composition to compositing window managers + \item \textbf{XRender}: 2D rendering API with with alpha composition, rasterization, transformations, filtering + \item \textbf{Xv}: video output format conversion and scaling offload in-DDX\\ + \textit{involves buffer copies and lacks synchronization with window position} + \end{itemize} +\end{frame} + +\begin{frame}{Xorg architecture and acceleration} + \begin{itemize} + \item Xorg is divided between generic and hardware-specific parts + \item \textbf{Device-Independent-X} (DIX) concerns: + \begin{itemize} + \item X11 protocol implementation, client coordination + \item Main event loop and event dispatching + \item Graphics operations logic, boilerplate and fallback implementations + \end{itemize} + \item \textbf{Device-Dependent-X} (DDX) concerns: + \begin{itemize} + \item Input drivers (\code{xf86-input-...}) to grab events from the kernel + \item Video drivers (\code{xf86-video-...}) to provide mode setting and 2D acceleration + \end{itemize} + \item \textbf{EXA} provides a 2D acceleration architecture between DIX and DDX + \begin{itemize} + \item Efficient way for drivers to expose accelerated 2D operation primitives + \item Replaced the XFree86 Acceleration Architecture (XAA) + \item Reduces driver boilerplate + \end{itemize} + \item \textbf{Glamor} provides 2D acceleration for the DDX using OpenGL 3D rendering + \end{itemize} +\end{frame} + +\begin{frame}{Xorg drivers overview} + \begin{itemize} + \item Generic Xorg \textbf{input} drivers: + \begin{itemize} + \item \code{xf86-input-libinput}: using \code{libinput} to get input events + \item \code{xf86-input-evdev}: using the \code{evdev} kernel interface directly (\textbf{deprecated}) + \end{itemize} + \item Specific Xorg \textbf{input} drivers: + \begin{itemize} + \item \code{xf86-input-synaptics}: for laptop touchpads + \item \code{xf86-input-wacom}: for Wacom drawing tablets + \item Specific drivers are \textbf{deprecated} in favor of \code{xf86-input-libinput} + \end{itemize} + \item Generic Xorg \textbf{display} drivers: + \begin{itemize} + \item \code{xf86-video-modesetting}: for \textbf{DRM KMS}, can be accelerated using \textbf{glamor} + \item \code{xf86-video-fbdev}: for the \textbf{fbdev} interface, without acceleration (legacy) + \item \code{xf86-video-vesa}: for the \textbf{Video BIOS Extension} (VBE) framebuffer (x86) + \end{itemize} + \item Specific Xorg \textbf{display} drivers: + \begin{itemize} + \item \code{xf86-video-[intel,nouveau,amdgpu]}: profiting from 2D acceleration blocks + \item Specific drivers are deprecated in favor of \code{xf86-video-modesetting} and \textbf{glamor}\\ + \textit{the trend is to accelerate everything via 3D rendering instead of 2D accelerators} + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{X11 and OpenGL acceleration: GLX and DRI2} + \begin{itemize} + \item Before DRM render nodes, there was a single device for KMS and render\\ + \textit{correlates with the idea of a graphics card mixing both aspects} + \item The X server owns the graphics device \textbf{exclusively} + \item Clients using OpenGL need to access the device for rendering + \item The \textbf{GLX} API was introduced to perform \textbf{indirect rendering}: + \begin{enumerate} + \item Integrating OpenGL with the X Window API + \item Forwarding GL calls to the GL implementation via the X server (AIGLX)\\ + \textit{introducing latency and performance issues} + \end{enumerate} + \item \textbf{The Direct Rendering Infrastructure} (DRI/DRI2) was introduced next + \begin{itemize} + \item The X server allowed access through DRM magic/auth + \item Buffers were shared via \textit{GEM flinks} + \item Now using the standalone \textbf{render node} and \textbf{dma-buf} instead + \item Still in place for coordination between render and the display server + \end{itemize} + \item GLX remained as a GL windowing API for X11 (deprecated by EGL) + \end{itemize} +\end{frame} + +\begin{frame}{X11 and OpenGL acceleration: GLX and DRI2 (illustrated)} + \begin{center} + \includegraphics[width=0.4\textwidth]{slides/graphics-software-userspace-x/dri-data-flow.png}\\ + \textit{Data flow in X11 for different types of clients} + \end{center} +\end{frame} + +\begin{frame}{Xorg usage, integration and configuration} + \begin{itemize} + \item Xorg can be started with the \code{startx} command (wrapping \code{xinit}) + \begin{itemize} + \item Executes server script from \code{/etc/X11/xinit/xserverrc} or \code{$HOME/.xserverrc} + \item Executes client script from \code{/etc/X11/xinit/xinitrc} or \code{$HOME/.xinitrc} + \end{itemize} + \item An X \textbf{display manager} offers a login interface (e.g. KDM, LightDM) + \begin{itemize} + \item Runs under a Xorg server, with its own dedicated user + \item Starts Xorg for authenticated users from session files in \code{/usr/share/xsessions/} + \end{itemize} + \item Used to require \textbf{running the server as root} to access graphics devices\\ + \textit{in particular, necessary to become DRM master} + \begin{itemize} + \item The \code{systemd-logind} login manager lifts the restriction + \item Opens the DRM KMS fd privileged and passes it to Xorg via IPC + \item Xorg can then drop privileges: details in the \code{Xorg.wrap} man page + \end{itemize} + \item Xorg is \textbf{configured} (both DIX and DDX) from \code{/etc/X11/xorg.conf} + \item The \code{DISPLAY} environment variable indicates which server connection to use\\ + \begin{itemize} + \item Already set for X client applications and inherited + \item \code{export DISPLAY=:0} useful to launch programs from other TTYs + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Xorg architecture: input to display roundtrip} + \begin{minipage}{0.49\textwidth} + \centering + \includegraphics[width=0.9\textwidth]{slides/graphics-software-userspace-x/x-architecture-roundtrip.png} + \end{minipage} + \hfill + \begin{minipage}{0.49\textwidth} + \begin{enumerate} + \item An input event is read from the kernel by the server + \item The affected client is determined and receives the event + \item The client changes something and issues a rendering request + \item The server performs rendering (DDX) and notifies the compositor + \item The compositor updates the damaged regions in the back-buffer + \item The server updates the display buffer from the compositor buffer (page flip) + \end{enumerate} + \end{minipage} +\end{frame} + +\begin{frame}{Major issues with X11} + \begin{itemize} + \item The X11 core protocol and paradigm soon caused \textbf{various issues}: + \begin{itemize} + \item Based on buffer \textbf{copies, transfers} and frequent \textbf{redraws}\\ + \textit{solved with XSHM and DRI2 extensions} + \item Immediate-mode drawing, with intermediate states scanned out\\ + \textit{solved by drawing everything client-side instead} + \item Lack of synchronization/feedback interface\\ + \textit{specified with the DRI3 and Present extensions} + \item Everything's a window with X... but not in practice (screensavers, popups)\\ + \textit{specified with the DRI3 and Present extensions} + \item Heavy packet-based protocol causing \textbf{latency issues} + \item \textbf{Security} concerns regarding client input/output isolation + \end{itemize} + \item Because the core protocol did not evolve, \textbf{extensions proliferated}: + \begin{itemize} + \item Complicated server aspects got \textbf{delegated} through extensions + \item Working around major design issues, not solving them in depth + \item In the end, the server mostly coordinates between other components + \end{itemize} + \item \textbf{Client-side rendering} became more common (raster, operations, fonts, etc) + \end{itemize} +\end{frame} + +\begin{frame}{Xorg code structure and walkthrough} + \begin{itemize} + \item Xorg source code available at: \url{https://gitlab.freedesktop.org/xorg/xserver} + \item DDX components: + \begin{itemize} + \item Code specific to the Linux kernel under: \code{hw/xfree86/os-support/linux/} + \item Modesetting DRM KMS driver under: \code{hw/xfree86/drivers/modesetting/} + \item fbdev core library under: \code{hw/xfree86/fbdevhw/} + \item Glamor implementation under: \code{glamor/} + \end{itemize} + \item DIX components: + \begin{itemize} + \item System-level helpers under: \code{os/} + \item Common framebuffer operations abstraction under: \code{fb/} + \item EXA abstraction under: \code{exa/} + \end{itemize} + \item DRI2 components: + \begin{itemize} + \item DRI2 common code under: \code{hw/xfree86/dri2} + \item Modesetting DRI2 glue under: \code{hw/xfree86/drivers/modesetting/dri2.c} + \item GLX support under: \code{glx/} + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{Xorg debug and documentation} + \begin{itemize} + \item Xorg has a \textbf{logging system} for all its components: + \begin{itemize} + \item Written to a file at \code{/var/log/Xorg.0.log}, \code{-logfile} option + \item Verbosity can be set with the \code{-logverbose} option (log level) + \item Printed on the standard output (\code{stdout}) + \end{itemize} + \item Xorg can be bound to any VT with the \code{vt} command line option\\ + \textit{useful for remote debugging, with a virtual controlling terminal} + \item \textbf{Community} contact: + \begin{itemize} + \item Mailing list: \code{xorg@lists.freedesktop.org} + \item IRC channel: \code{#xorg} and \code{#xorg-devel} on the OFTC network + \end{itemize} + \item \textbf{Documentation} resources: + \begin{itemize} + \item Online wiki of the project: \url{https://www.x.org/wiki/Documentation/} + \item Man pages: \code{X}, \code{Xserver}, \code{Xorg}, \code{xorg.conf}, \code{xinit} and more! + \item Extensions specification documents + \end{itemize} + \end{itemize} +\end{frame} diff --git a/slides/graphics-software/x-architecture-roundtrip.png b/slides/graphics-software-userspace-x/x-architecture-roundtrip.png similarity index 100% rename from slides/graphics-software/x-architecture-roundtrip.png rename to slides/graphics-software-userspace-x/x-architecture-roundtrip.png diff --git a/slides/graphics-software/graphics-software.tex b/slides/graphics-software/graphics-software.tex deleted file mode 100644 index 65a58ff653..0000000000 --- a/slides/graphics-software/graphics-software.tex +++ /dev/null @@ -1,1904 +0,0 @@ -\section{Software Aspects} - -\subsection{Stack Overview} - -\begin{frame}{System-agnostic overview: kernel} - \begin{itemize} - \item The kernel provides access to the hardware from userspace - \item Handles clocks, power, register access, interrupts, etc - \item Coordinates memory management with the rest of the system - \item Exposes features to userspace through hardware-agnostic interfaces\\ - \textit{or at least, as much as possible} - \item Three aspects are usually involved: - \begin{itemize} - \item \textbf{display}: from framebuffer to encoder - \item \textbf{render}: GPU and/or 2D accelerators - \item \textbf{input}: keyboard, mouse and other devices - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{System-agnostic overview: display userspace} - \begin{itemize} - \item The kernel provides \textbf{exclusive access} to display hardware - \item Many applications need to show their buffers concurrently - \item The \textbf{display server} is in charge of coordinating between applications: - \begin{itemize} - \item Part of the core of the system, privileged - \item Applications (via libraries) contact the server to display pixel buffers - \item Dispatches input events to the concerned applications - \item Only the display server deals with the kernel display and input APIs - \end{itemize} - \item The \textbf{compositor} merges pixel buffers from applications into the final buffer - \item The \textbf{window manager} defines stacking order, focus, decorations, etc - \item Both can be part of the display server or distinct components - \item Serious security concerns: I/O isolation for applications, server privileges - \end{itemize} -\end{frame} - -\begin{frame}{System-agnostic overview: display userspace (illustrated)} - - \begin{minipage}[t]{0.49\textwidth} - \centering - \includegraphics[height=7em]{slides/graphics-software/compositing-gnome-shell.jpg}\\ - \textit{\small GNOME-Shell (green) displaying the\\ top-bar and background}\\ - \vspace{0.5em} - \includegraphics[height=7em]{slides/graphics-software/compositing-gnome-terminal.jpg}\\ - \textit{\small GNOME-Terminal (green) with\\ window decorations (red)} - \end{minipage} - \hfill - \begin{minipage}[t]{0.49\textwidth} - \centering - \includegraphics[height=7em]{slides/graphics-software/compositing-lollipop.jpg}\\ - \textit{\small Lollipop (green) with\\ window decorations (red)}\\ - \vspace{0.5em} - \includegraphics[height=7em]{slides/graphics-software/compositing-result.jpg}\\ - \textit{\small The composited result} - \end{minipage} -\end{frame} - -\begin{frame}{System-agnostic overview: render userspace} - \begin{itemize} - \item Graphics applications and libraries need to render visual elements - \item Rendering can be a major \textbf{performance bottleneck} - \item The system often provides \textbf{accelerated 2D primitives}: - \begin{itemize} - \item Either in the display server - \item Either in dedicated libraries - \end{itemize} - \item Their implementation can take different forms: - \begin{itemize} - \item Using dedicated 2D hardware - \item Using 3D hardware in 2D setups (\(z = 0\)) - \item Using specific efficient CPU instructions (SIMD) - \item Using optimized generic algorithms - \end{itemize} - \item \textbf{3D rendering} comes with its own interfaces and libraries - \item Usually with generic interfaces and hardware-specific implementations - \end{itemize} -\end{frame} - -\begin{frame}{System-agnostic overview (illustrated)} - \begin{center} - \includegraphics[width=\textwidth]{slides/graphics-software/agnostic-display-stack-overview.pdf} - \end{center} -\end{frame} - -\begin{frame}{Linux kernel overview} - \begin{itemize} - \item \textbf{Input} subsystem - \begin{itemize} - \item Supports devices such as mice, keyboards, joysticks, touchscreens - \item Legacy (unused) interfaces for keyboard, mice - \item Unified \textbf{evdev} event interface to simplify userspace - \end{itemize} - \item \textbf{Framebuffer device} (fbdev) subsystem - \begin{itemize} - \item \textbf{Legacy} interface for displaying pixel buffers on-screen - \item Very limited pipeline configuration, no hotplug support - \item Extended features added through driver-specific interfaces - \end{itemize} - \item \textbf{Direct Rendering Manager} (DRM) subsystem - \begin{itemize} - \item Unified display configuration interface: \textbf{Kernel Mode Setting} (KMS or DRM mode) - \item Allows synchronizing changes together (DRM atomic) - \item Exposes render devices through driver-specific interfaces (DRM render)\\ - \textit{Mostly for 3D rendering with GPUs, but a few 2D devices too} - \item Provides memory management mechanisms (DRM GEM) - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Linux-compatible low-level userspace overview} - \begin{itemize} - \item \textbf{Input} low-level libraries - \begin{itemize} - \item \textbf{libevdev} (C): Wrapper for evdev interface system calls - \item \textbf{libinput} (C): Input device management, abstraction and quirks, using libevdev - \end{itemize} - \item \textbf{Display/render} low-level interface library - \begin{itemize} - \item \textbf{libdrm} (C): Wrapper for DRM system calls - \end{itemize} - \item \textbf{2D render} low-level libraries - \begin{itemize} - \item \textbf{Pixman} (C): Optimized pixel-level operations - \item \textbf{Cairo} (C): Optimized vector drawing (can use 3D) - \item \textbf{Skia} (C): Optimized vector drawing from Google (can use 3D) - \item \textbf{Clutter} (C++): Accelerated UI animation (using 3D) - \end{itemize} - \item \textbf{3D render} low-level libraries - \begin{itemize} - \item \textbf{Mesa 3D} (C): Reference free software OpenGL implementation - \item \textbf{Proprietary vendor implementations} for specific hardware - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{X Window overview} - \begin{itemize} - \item \textbf{X Window} overview - \begin{itemize} - \item \textbf{X Window/X11} is the historical (and \textbf{legacy}) \textbf{display protocol} - \item Complemented by numerous protocol \textbf{extensions} for extra features - \item \textbf{X.org} is the reference X11 server \textbf{implementation} - \item Needs an external \textbf{window manager} to handle multiple applications - \item \textbf{Composition} by the server or the window manager (Composite extension) - \end{itemize} - \end{itemize} - \begin{minipage}[b]{0.8\textwidth} - \begin{itemize} - \item \textbf{Window manager} implementations examples - \begin{itemize} - \item \textbf{Mutter}: GNOME accelerated compositing window manager - \item \textbf{i3}: Popular tiling window manager - \item \textbf{Compiz}: Popular 3D-enabled compositing window manager - \end{itemize} - \item \textbf{Display client} libraries - \begin{itemize} - \item \textbf{Xlib} (C): The legacy X11 client-side protocol library helper - \item \textbf{XCB} (C): The updated X11 client-side protocol library helper - \item Integrated in most higher-level graphics-oriented libraries - \end{itemize} - \end{itemize} - \end{minipage} - \begin{minipage}[b]{0.15\textwidth} - \includegraphics[width=\textwidth]{slides/graphics-software/x11-logo.pdf} - \end{minipage} -\end{frame} - -\begin{frame}{Wayland overview} - \begin{itemize} - \item \textbf{Wayland} overview - \begin{itemize} - \item Wayland is a \textbf{display protocol} (with a core and extensions), not an implementation - \item Replaces X11 with a \textbf{less intrusive, more modern and minimal} paradigm - \item Compositors (server-side) handle \textbf{input, windows, composition and display} - \end{itemize} - \end{itemize} - \begin{minipage}[b]{0.8\textwidth} - \begin{itemize} - \item \textbf{Wayland compositor} implementations examples - \begin{itemize} - \item Using the \textbf{libwayland-server} base protocol library - \item \textbf{Weston/libweston}: Reference implementation - \item \textbf{Sway/wlroots}: Tiling window manager and base library - \item \textbf{Mutter}: GNOME compositor - \end{itemize} - \item \textbf{Display client} libraries - \begin{itemize} - \item Using the \textbf{libwayland-client} base protocol library - \item Integrated in many higher-level graphics-oriented libraries - \end{itemize} - \end{itemize} - \end{minipage} - \begin{minipage}[b]{0.15\textwidth} - \includegraphics[width=\textwidth]{slides/graphics-software/wayland-logo.pdf} - \end{minipage} -\end{frame} - -\begin{frame}{High-level graphics libraries and desktop environments overview} - \begin{minipage}[b]{0.09\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-software/gtk-logo.pdf}\\ - \vspace{3em} - \includegraphics[width=\textwidth]{slides/graphics-software/qt-logo.pdf}\\ - \vspace{3em} - \includegraphics[width=\textwidth]{slides/graphics-software/sdl-logo.png}\\ - \vspace{2em} - \end{minipage} - \hfill - \begin{minipage}[b]{0.8\textwidth} - \begin{itemize} - \item Applications rarely to never use Wayland or X11 directly - \item Drawing and managing a user interface is complex - \item Widely-used high-level graphics libraries (aka toolkits) - \begin{itemize} - \item \textbf{GTK} (C): Widget-based UI toolkit, drawing helpers (GDK) - \item \textbf{Qt} (C++): Widget-based UI toolkit, wide framework - \item \textbf{EFL} (C): Lightweight UI and application library - \item \textbf{SDL} (C): Drawing-oriented graphics library (used in games) - \end{itemize} - \item A desktop environment groups related libraries and components\\ - \textit{gives a consistent look and feel across the system} - \item \textbf{Desktop environment} examples - \begin{itemize} - \item \textbf{GNOME}: Using GTK, GNOME-Shell desktop - \item \textbf{KDE}: Using Qt, Plasma desktop - \item \textbf{Xfce}: Using GTK, lightweight - \item \textbf{Enlightenment}: Using EFL - \end{itemize} - \end{itemize} - \vfill~ - \end{minipage} - \begin{minipage}[b]{0.09\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-software/gnome-logo.pdf}\\ - \vspace{1em} - \includegraphics[width=\textwidth]{slides/graphics-software/kde-logo.pdf}\\ - \vspace{1em} - \includegraphics[width=\textwidth]{slides/graphics-software/xfce-logo.pdf}\\ - \vspace{0.5em} - \includegraphics[width=\textwidth]{slides/graphics-software/enlightenment-logo.pdf} - \end{minipage} -\end{frame} - -\subsection{Linux Kernel: TTY} - -\begin{frame}{Linux TTY subsystem introduction} - \begin{itemize} - \item The \textbf{TTY} subsystem handles teletypewriters to send/receive characters - \item Source code located at \code{drivers/tty} in Linux - \item Supports physical instances (e.g. UART, RS-232) and virtual ones - \item Virtual terminals/consoles (VTs/VCs) associate a distinct keyboard and display - \begin{itemize} - \item Many VTs are created by Linux, available under: \code{/dev/tty*} - \item Only a single VT is active at a time, switched with \code{Ctrl + Alt + Fi} - \item Display grabbed using \textbf{fbcon} from the \textbf{fbdev} subsystem - \item Keyboard grabbed using the \textbf{input} subsystem - \item Can be used to show kernel messages (\code{console=tty1} in the cmdline) - \item Every program runs under a controlling tty (given by the \code{tty} command) - \end{itemize} - \item Pseudo-terminals also exist, for software-based I/O only - \begin{itemize} - \item Created by programs (e.g. terminal emulator) under: \code{/dev/pts/*} - \item Unrelated to graphics topics - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Linux TTY (illustrated)} - \begin{center} - \includegraphics[width=0.6\textwidth]{slides/graphics-software/getty.jpg}\\ - \textit{Getty running on \code{tty1} of a GNU/Linux system}\\ - \end{center} -\end{frame} - -\begin{frame}[fragile]{Virtual terminals and graphics} - \begin{itemize} - \item With VTs, the kernel is \textbf{already using} the display and keyboard! - \item Display servers need to switch to \textbf{graphics mode} to release the display: - \begin{minted}[fontsize=\small]{console} -ret = ioctl(tty_fd, KDSETMODE, KD_GRAPHICS); - \end{minted} - \item And disable \textbf{keyboard support} on the standard input: - \begin{minted}[fontsize=\small]{console} -ret = ioctl(tty_fd, KDSKBMODE, K_OFF); - \end{minted} - \item The display device can then be used \textbf{exclusively} - \item Input is no longer interpreted (e.g. \code{Ctrl-C} is ignored) - \item Graphics and keyboard mode must be restored when leaving to keep the VT usable - \item Current modes can be queried with: - \begin{minted}[fontsize=\small]{console} -short mode, kbmode; -ret = ioctl(tty_fd, KDGETMODE, &mode); -ret = ioctl(tty_fd, KDGKBMODE, &kbmode); - \end{minted} - \item More details in the \code{console_ioctl} man page - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{Virtual terminals switching and graphics} - \begin{itemize} - \item However, the user might still want to switch VTs! - \item So the display device must be \textbf{released/reacquired} for VT switching - \item UNIX signals are used to notify the application, configured with: - \begin{minted}[fontsize=\small]{console} -struct vt_mode vt_mode = { 0 }; -vt_mode.mode = VT_PROCESS; -vt_mode.relsig = SIGUSR1; -vt_mode.acqsig = SIGUSR2; -ret = ioctl(tty_fd, VT_SETMODE, &vt_mode); - \end{minted} - \item VT switching must be acknowledged for the other VT to take over: - \begin{minted}[fontsize=\small]{console} -ret = ioctl(tty_fd, VT_RELDISP, VT_ACKACQ); /* when entering VT */ -ret = ioctl(tty_fd, VT_RELDISP, 1); /* when leaving VT */ - \end{minted} - \item Failure to acknowledge will cause a \textbf{system hang} - \end{itemize} -\end{frame} - -\subsection{Linux Kernel: Framebuffer Device} - -\begin{frame}{Fbdev overview} - \begin{itemize} - \item Fbdev is the \textbf{historical and legacy} display subsystem in Linux - \item Exposes a number of display-related features: - \begin{itemize} - \item Framebuffer access (pre-allocated for one or two frames) - \item Operation primitives (bit blit, solid fill, area copy, etc) - \item Cursor drawing - \item Power management (blank/unblank, power down) - \end{itemize} - \item Initial state can be configured with the \code{video} option of the \code{cmdline} - \item Available to userspace via \code{/dev/fb*} nodes: - \begin{itemize} - \item Generic base \code{ioctls} with driver-specific additions - \item Direct framebuffer memory mapping using \code{mmap} - \item Relatively simple and minimalistic interface - \end{itemize} - \item Used by the kernel to provide a graphical console with \textbf{fbcon} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{Fbdev basic operations} - \begin{itemize} - \item \textbf{Fixed information} about the display is retrieved with: - \begin{minted}[fontsize=\small]{console} -struct fb_fix_screeninfo fix_screeninfo = { 0 }; -ret = ioctl(fb_fd, FBIOGET_FSCREENINFO, &fix_screeninfo); - \end{minted} - \item \textbf{Variable information} (including mode) is retrieved and configured with: - \begin{minted}[fontsize=\small]{console} -struct fb_var_screeninfo var_screeninfo = { 0 }; -ret = ioctl(fb_fd, FBIOGET_VSCREENINFO, &var_screeninfo); -... -ret = ioctl(fb_fd, FBIOPUT_VSCREENINFO, &var_screeninfo); - \end{minted} - \item \textbf{Power management} is operated with: - \begin{minted}[fontsize=\small]{console} -ret = ioctl(fb_fd, FBIOBLANK, FB_BLANK_POWERDOWN); - \end{minted} - \item \textbf{Double-buffering} is sometimes supported (within the same buffer): - \begin{minted}[fontsize=\small]{console} -var_screeninfo.yoffset = height; -ret = ioctl(fb_fd, FBIOPAN_DISPLAY, &var_screeninfo); - \end{minted} - \item Blocking until the next \textbf{vblank} is possible with: - \begin{minted}[fontsize=\small]{console} -int vsync = 0; -ret = ioctl(fb_fd, FBIO_WAITFORVSYNC, &vsync); - \end{minted} - \end{itemize} -\end{frame} - -\begin{frame}{Fbdev limitations} - \begin{itemize} - \item Fbdev does not expose or allow configuration of the display pipeline - \item Output setup is mostly static (provided through the \code{cmdline}) - \item Designed for simple cases (with a single output) - \item Buffer allocation and management is not available - \item No possibility of zero-copy import from other devices - \item Limited page flipping with no associated synchronization mechanism - \item Insufficient external synchronization interface (blocking wait) - \item Mixes display, operation primitives and power management - \end{itemize} - - \begin{center} - \textbf{Fbdev is mostly adapted to display from the 1990s and 2000s}\\ - \small please consider avoiding it at all costs! - \end{center} -\end{frame} - -\subsection{Linux Kernel: DRM} - -\begin{frame}{DRM devices} - \begin{itemize} - \item UNIX-style devices are identified with major/minor numbers - \begin{itemize} - \item More details in the \code{makedev} manpage, using \code{dev_t} type - \item Minor/major can be retrieved with \code{stat/fstat} - \item DRM major in Linux is \textbf{226} - \end{itemize} - \item Two types of DRM devices exist: - \begin{itemize} - \item \textbf{Primary} nodes at \code{/dev/dri/card*} with minor \(< 128\)\\ - Used for display operations with the KMS (mode) interface - \item \textbf{Render} nodes at \code{/dev/dri/renderD*} with minor \(\geq 128\)\\ - Used for render operations with a driver-specific interface - \end{itemize} - \item DRM devices can also be used by the kernel directly (internal clients): - \begin{itemize} - \item \textbf{fbdev} compatibility layer to provide \code{/dev/fb*} nodes - \item Used by \textbf{fbcon} to provide virtual consoles - \end{itemize} - \item Userspace needs rights to open device nodes: - \begin{itemize} - \item Usually allowed via the \code{video} \textbf{group} or \textbf{Access Control Lists} (ACLs) - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM driver identification and capabilities} - \begin{itemize} - \item Driver-specific \textbf{name and version} (major/minor/patchlevel) can be queried:% - \begin{minted}[fontsize=\small]{console} -struct drm_version version = { ... }; -ret = ioctl(drm_fd, DRM_IOCTL_VERSION, &version); - \end{minted} - \item Drivers expose specific \textbf{capabilities}, that can be queried: - \begin{minted}[fontsize=\small]{console} -struct drm_get_cap get_cap = { 0 }; -get_cap.capability = DRM_CAP_DUMB_BUFFER; -ret = ioctl(drm_fd, DRM_IOCTL_GET_CAP, &get_cap); - \end{minted} - \item The kernel \textbf{must} be informed of client support for some features: - \begin{minted}[fontsize=\small]{console} -struct drm_set_client_cap client_cap = { 0 }; -client_cap.capability = DRM_CLIENT_CAP_UNIVERSAL_PLANES; -client_cap.value = 1; -ret = ioctl(drm_fd, DRM_IOCTL_SET_CLIENT_CAP, &client_cap); - \end{minted} - \item Driver and client capabilities defined in Linux's \kfile{include/uapi/drm/drm.h} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM master, magic and authentication} - \begin{itemize} - \item Multiple userspace clients can open the same primary device node - \item Only the \textbf{master} client is allowed to configure display (KMS) - \item Master is exclusive and can be \textbf{acquired} and \textbf{dropped} (VT switching): - \begin{minted}[fontsize=\small]{console} -ret = ioctl(drm_fd, DRM_IOCTL_SET_MASTER, NULL); -ret = ioctl(drm_fd, DRM_IOCTL_DROP_MASTER, NULL); - \end{minted} - \item Requires \code{CAP_SYS_ADMIN} Linux capability, see \code{capabilities} man page\\ - \textit{usually reserved to the root super user} - \item Some operations can be allowed on trusted clients with \textbf{magic authentication}: - \begin{itemize} - \item Mostly used before render nodes or for allocating buffers on another process - \end{itemize} - \begin{enumerate} - \item Client \textit{foo} gets its client-specific magic: - \begin{minted}[fontsize=\small]{console} -struct drm_auth auth = { 0 }; -ret = ioctl(drm_fd, DRM_IOCTL_GET_MAGIC, &auth); - \end{minted} - \item Client \textit{foo} sends \code{auth.magic} to master client \textit{bar} (via IPC) - \item Master client \textit{bar} authenticates client \textit{foo}: - \begin{minted}[fontsize=\small]{console} -ret = ioctl(drm_fd, DRM_IOCTL_AUTH_MAGIC, &auth); - \end{minted} - \end{enumerate} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM memory management} - \begin{itemize} - \item The \textbf{Graphics Execution Manager} (GEM) handles memory in DRM - \item Used both by KMS and render drivers, with specific backends: - \begin{itemize} - \item \textbf{CMA}: Contiguous Memory Allocator (reserved area at boot) - \item \textbf{Shmem}: Shared system memory (anonymous pages) - \item \textbf{Vram}: Video RAM, using the \textbf{Translation Table Manager} (TTM) - \end{itemize} - \item Ensures buffers \textbf{coherency} on access (cache management) - \item Allocated buffers are identified with a unique \textbf{handle} number - \item In KMS, the \textbf{dumb buffer} API exposes memory operations: - \begin{itemize} - \item For memory used for \textbf{scanout framebuffers} - \item Drivers calculate aligned pitch/stride and size based on dimensions and bpp - \item Sometimes too limiting (e.g. multi-planar formats) - \end{itemize} - \item More details in the \textit{drm-memory} man page - \item Drivers sometimes expose extra \code{ioctls} for more advanced needs - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM KMS dumb buffer API} - \begin{itemize} - \item \textbf{Allocating} from \code{width}, \code{height} and \code{bpp}, returning \code{handle}, \code{pitch} and \code{size}:\\ - \begin{minted}[fontsize=\small]{console} -struct drm_mode_create_dumb create_dumb = { ... }; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_CREATE_DUMB, &create_dumb); - \end{minted} - \item \textbf{Destroying} an allocated buffer: - \begin{minted}[fontsize=\small]{console} -struct drm_mode_destroy_dumb destroy_dumb = { .handle = ..., }; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_DESTROY_DUMB, &destroy_dumb); - \end{minted} - \item \textbf{Preparing a mapping} in user memory for a buffer, returning an \code{offset}: - \begin{minted}[fontsize=\small]{console} -struct drm_mode_map_dumb map_dumb = { .handle = ..., }; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_MAP_DUMB, &map_dumb); - \end{minted} - \item \textbf{Mapping memory} to userspace using the \code{offset}: - \begin{minted}[fontsize=\small]{console} -map = mmap(NULL, create_dumb.size, PROT_READ | PROT_WRITE, MAP_SHARED, - drm_fd, map_dumb.offset); - \end{minted} - \item \textbf{Unmapping} memory after use: - \begin{minted}[fontsize=\small]{console} -munmap(map, create_dumb.size); - \end{minted} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM FourCCs and modifiers} - \begin{itemize} - \item DRM has its own representation of \textbf{pixel formats}, with FourCC codes (on 32 bits) - \item Defined in the \kfile{include/uapi/drm/drm_fourcc.h} header - \item They can specify up to 4 distinct \textbf{data planes} for color components - \item Pixel formats are named "MSB-to-LSB" and specified in \textbf{little-endian} order\\ - \textit{LSB comes first in memory in little-endian} - \item For instance, \code{DRM_FORMAT_XRGB8888} has the \code{B} byte first in memory\\ - \textit{Memory order is independent from the CPU or hardware endianness} - \item A format \textbf{modifier} (on 64 bits) indicates the pixel order in memory - \item \code{DRM_FORMAT_MOD_LINEAR} indicates raster order\\ - \textit{line-major left-to-right, top-to-bottom} - \item Other modifiers are usually hardware-specific, often tiled\\ - (e.g. \code{DRM_FORMAT_MOD_VIVANTE_TILED}) - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM KMS resources probing} - \begin{itemize} - \item KMS hardware resources are exposed through the following entities: - \begin{itemize} - \item Connectors - \item Encoders - \item CRTCs - \item Planes: primary, overlay and cursor - \item Framebuffers - \end{itemize} - \item Each resource instance is identified with a unique \textbf{identification number} - \item The list of resource ids is retrieved with: - \begin{minted}[fontsize=\small]{console} -struct drm_mode_card_res res = { ... }; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_GETRESOURCES, &res); - \end{minted} - \item Plane ids (that were introduced later) are retrieved with: - \begin{minted}[fontsize=\small]{console} -struct drm_mode_get_plane_res res = { ... }; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_GETPLANERESOURCES, &res); - \end{minted} - \item Resource ids are used with subsequent resource-specific calls - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM KMS connector probing} - \begin{itemize} - \item The starting point to configure a KMS pipeline is the connector - \item Current connector state is probed with: - \begin{minted}[fontsize=\small]{console} -struct drm_mode_get_connector get_connector = { .connector_id = ... }; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_GETCONNECTOR, &get_connector); - \end{minted} - \item \code{struct drm_mode_get_connector} exposes various information: - \begin{itemize} - \item Connector type and connection state - \item Possible encoders, currently-attached encoder - \item Available modes and physical monitor size - \end{itemize} - \item Probing modes triggers EDID read: optional and usually quite slow - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM KMS modes} - \begin{itemize} - \item A display mode is represented as a \code{struct drm_mode_modeinfo} in DRM - \item Members: \code{clock}, \code{[hv]display}, \code{[hv]sync_start}, \code{[hv]sync_end}, \code{[hv]total} and \code{flags} for signal-specific details (polarities) - \item Diagram from \kfile{include/drm/drm_modes.h}: - \begin{minted}[fontsize=\small]{console} - Active Front Sync Back - Region Porch Porch -<-----------------------><----------------><-------------><--------------> - //////////////////////| - ////////////////////// | -////////////////////// |.................. ................ - _______________ -<----- [hv]display -----> -<------------- [hv]sync_start ------------> -<--------------------- [hv]sync_end ---------------------> -<-------------------------------- [hv]total ----------------------------->* - \end{minted} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM KMS encoder probing} - \begin{itemize} - \item The next step is to find which CRTC id can be used with the connector - \item The encoder is the link between the connector and CRTC - \item Current encoder state can be probed with: - \begin{minted}[fontsize=\small]{console} -struct drm_mode_get_encoder get_encoder = { .encoder_id = ... }; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_GETENCODER, &get_encoder); - \end{minted} - \item \code{struct drm_mode_get_connector} exposes some information: - \begin{itemize} - \item Encoder type - \item Possible CRTCs, currently-attached CRTC - \end{itemize} - \item This allows selecting the CRTC to use for the connector! - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM KMS framebuffer management} - \begin{itemize} - \item Framebuffers in DRM are described with a number of parameters: - \begin{itemize} - \item Picture-wide: \code{width}, \code{height}, \code{pixel_format} - \item Plane-specific: GEM \code{handle}, \code{pitch}, \code{offset} and \code{modifier} - \end{itemize} - \item Up to 4 memory planes are supported (depending on the format) - \item Allows supporting a wide range of possible configurations - \item Flags are passed to indicate that modifiers or interlaced scan are used - \item Framebuffers are registered from their parameters, returning a \code{fb_id}: - \begin{minted}[fontsize=\small]{console} -struct drm_mode_fb_cmd2 fb_cmd2 = { ... }; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_ADDFB2, &fb_cmd2); - \end{minted} - \item They are destroyed using the \code{fb_id}: - \begin{minted}[fontsize=\small]{console} -unsigned int fb_id = fb_cmd2.fb_id; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_RMFB, &fb_id); - \end{minted} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM KMS CRTC configuration (legacy)} - \begin{itemize} - \item The pipeline can then be configured with the connector and the CRTC - \item The current CRTC configuration can be retrieved with: - \begin{minted}[fontsize=\small]{console} -struct drm_mode_crtc crtc = { .crtc_id = ... }; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_GETCRTC, &crtc); - \end{minted} - \item The CRTC is configured with the connector id - \begin{minted}[fontsize=\small]{console} -struct drm_mode_crtc crtc = { .crtc_id = ... }; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_SETCRTC, &crtc); - \end{minted} - \item A mode and a framebuffer can be set (previous setup used otherwise)\\ - \textit{mandatory if the CRTC was unused before} - \item The kernel will automatically select the best encoder for the connector and CRTC - \item \textbf{Legacy and deprecated} way to do modesetting: only concerns the primary plane - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM KMS page flipping (legacy)} - \begin{itemize} - \item Page flipping is the action of switching the CRTC to another framebuffer\\ - \textit{only concerns the primary plane} - \item An event can be requested when the flip happens - \item Can be scheduled at different times (specified with \code{flags}): - \begin{itemize} - \item At a specified vblank target (absolute or relative) to avoid tearing - \item As soon as possible (asynchronously) if supported - \end{itemize} - \begin{minted}[fontsize=\small]{console} -struct drm_mode_crtc_page_flip page_flip = { .crtc_id = ..., .fb_id = ... }; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_PAGE_FLIP, &page_flip); - \end{minted} - \item \textbf{Legacy and deprecated}: limited to the primary plane - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM KMS overlay plane configuration (legacy)} - \begin{itemize} - \item Overlay planes are configured separately from the CRTC main plane - \item The current state of a plane can be retrieved with: - \begin{minted}[fontsize=\small]{console} -struct drm_mode_get_plane get_plane = { .plane_id = ... }; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_GETPLANE, &get_plane); - \end{minted} - \item Provides possible CRTCs, current framebuffer and supported formats - \item Planes are configured with source and destination parameters: - \begin{itemize} - \item \code{crtc_[xywh]}: On-CRTC position and dimensions - \item \code{src_[xywh]}: In-framebuffer position and dimensions (source clipping area) - \end{itemize} - \item Configuration takes place with: - \begin{minted}[fontsize=\small]{console} -struct drm_mode_set_plane set_plane = { .plane_id = ... }; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_SETPLANE, &set_plane); - \end{minted} - \item \textbf{Legacy and deprecated}: not synchronized to vblank or page flip - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM KMS cursor configuration and position (legacy)} - \begin{itemize} - \item Cursor planes have a separate dedicated legacy API - \item Configured per-CRTC with a GEM \code{handle} and dimensions (\code{width}, \code{height})\\ - \textit{a zero GEM \code{handle} deconfigures and removes the cursor} - \item Only supports the \code{DRM_FORMAT_ARGB8888} format (not configurable) - \item Using a single \code{ioctl} with the \code{flags} field for the operation - \begin{minted}[fontsize=\small]{console} -struct drm_mode_cursor cursor = { .flags = DRM_MODE_CURSOR_BO, - .crtc_id = ...}; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_CURSOR, &cursor); - \end{minted} - \item Once configured, the cursor can be moved to \code{x}, \code{y} on-CRTC coordinates - \begin{minted}[fontsize=\small]{console} -struct drm_mode_cursor cursor = { .flags = DRM_MODE_CURSOR_MOVE, - .crtc_id = ... }; -ret = ioctl(drm_fd, DRM_IOCTL_MODE_CURSOR, &cursor); - \end{minted} - \item \code{DRM_IOCTL_MODE_CURSOR2} variant provides cursor hotspot for virtual machines - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM event notification and wait} - \begin{itemize} - \item DRM provides an event notification mechanism for \textbf{vblank} and \textbf{page flip done} - \item Available through the primary (KMS) file descriptor - \item Can be used with \code{poll} and \code{select} (integrated in main loop) - \item Events with a \code{struct drm_event} base are read using \code{read} - \item Expand to \code{struct drm_event_vblank} for vblank and page flip done events\\ - \textit{only complete events are returned, so the buffer must be large enough} - \item Events can be requested at page flip time or explicitly: - \begin{minted}[fontsize=\small]{console} -union drm_wait_vblank wait_vblank = { .request = ... }; -ret = ioctl(drm_fd, DRM_IOCTL_WAIT_VBLANK, &wait_vblank); - \end{minted} - \item A blocking wait for an absolute or relative vblank sequence can also be requested\\ - \textit{using the same \code{ioctl} and dedicated \code{request.type} values} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM KMS object properties} - \begin{itemize} - \item KMS objects expose generic (or driver-specific) properties with names and values - \textit{concerns \textbf{connectors}, \textbf{CRTCs} and \textbf{planes}} - \begin{itemize} - \item \textbf{Range} properties: limits for the value (signed or unsigned) - \item \textbf{Enum} properties: fixed values with associated names for the values - \item \textbf{Blob} properties: raw data with a given length - \end{itemize} - \item Properties have a \textbf{unique identifier across objects}, details can be queried: - \begin{minted}[fontsize=\small]{console} -struct drm_mode_obj_get_property get_property = { .prop_id = ... } -ret = ioctl(drm_fd, DRM_IOCTL_MODE_GETPROPERTY, &get_property); - \end{minted} - \item Registered properties of an object can be retrieved using: - \begin{minted}[fontsize=\small]{console} -struct drm_mode_obj_get_properties get_properties = { .obj_id = ... } -ret = ioctl(drm_fd, DRM_IOCTL_MODE_OBJ_GETPROPERTIES, &get_properties); - \end{minted} - \item The \code{value} of a property can be assigned with: - \begin{minted}[fontsize=\small]{console} -struct drm_mode_obj_set_property set_property = { .obj_id = ..., .prop_id = ... } -ret = ioctl(drm_fd, DRM_IOCTL_MODE_OBJ_SETPROPERTY, &set_property); - \end{minted} - \item Blob properties need to be created and destroyed (with their own identifier) - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM KMS atomic} - \begin{itemize} - \item The legacy API comes with major design issues: - \begin{itemize} - \item Overlay and cursor plane updates are applied instantly (tearing) - \item Plane updates cannot be synchronized together (intermediate states) - \item No way to check that setup is valid before applying it - \end{itemize} - \item The atomic API lifts these restrictions with a new paradigm: - \begin{itemize} - \item Objects are configured based on their KMS properties\\ - \textit{values are affected to each changed property} - \item Property changes of different objects are grouped in an \textbf{atomic commit} - \item Planes are handled regardless of their type (primary, overlay, cursor) - \item Commits can be marked for test only: checked but not applied - \item Changes are applied at next vblank, unless marked asynchronous - \end{itemize} - \begin{minted}[fontsize=\small]{console} -struct drm_mode_atomic atomic = { ... } -ret = ioctl(drm_fd, DRM_IOCTL_MODE_ATOMIC, &atomic); - \end{minted} - \item Unless marked non-blocking, the \code{ioctl} returns when changes are applied - \item A page flip event can also be requested - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM KMS atomic common properties} - \begin{itemize} - \item Common properties used to configure \textbf{connectors}: - \begin{itemize} - \item \code{CRTC_ID}: id of the CRTC to bind with the connector - \end{itemize} - \item Common properties used to configure \textbf{CRTCs}: - \begin{itemize} - \item \code{ACTIVE}: whether the CRTC is in use - \item \code{MODE_ID}: id of the property blob with the \code{struct drm_mode_modeinfo} mode - \end{itemize} - \item Common properties used to configure \textbf{planes}: - \begin{itemize} - \item \code{FB_ID}: id of the framebuffer to bind with the plane - \item \code{CRTC_ID}: id of the CRTC to bind with the plane - \item \code{CRTC_[XYWH]}: on-CRTC position and dimensions of the plane - \item \code{SRC_[XYWH]}: in-framebuffer position and dimensions (source clipping area) - \end{itemize} - \item Common properties used to probe \textbf{planes}: - \begin{itemize} - \item \code{TYPE}: type of the plane (primary/overlay/cursor) - \item \code{IN_FORMATS}: list of supported formats/modifiers - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM KMS atomic driver walkthrough} - \begin{itemize} - \item A state-of-the-art DRM KMS driver: \code{vc4} at \kdir{drivers/gpu/drm/vc4}\\ - \textit{integrates both DRM KMS and render} - \item Entry point at \kfile{drivers/gpu/drm/vc4/vc4_drv.c} - \item Dedicated documentation: \url{https://dri.freedesktop.org/docs/drm/gpu/vc4.html} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM render generalities} - \begin{itemize} - \item DRM render drivers have their own \textbf{driver-specific API}\\ - \textit{unlike KMS, render hardware abstraction is done in userspace} - \item Their API is exposed through custom \code{ioctls} - \item Can be associated with a KMS driver (e.g. \code{vc4}) or separate (e.g. \code{v3d}) - \item Drivers handle memory, job submission and scheduling, interrupts - \item DRM has a common scheduler (from AMD) in \kdir{drivers/gpu/drm/scheduler} - \item Usual operations: - \begin{itemize} - \item Managing buffer objects (BOs) of different types (create, destroy, mmap)\\ - \textit{using GEM under the hood} - \item Submitting job data structures for programming the GPU (command lists)\\ - \textit{with a validation step to ensure its validity} - \item Waiting for operations to complete - \item Exposing performance-related information - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM render driver walkthrough} - \begin{itemize} - \item A state-of-the-art DRM render driver: \code{v3d} at \kdir{drivers/gpu/drm/v3d} - \item Entry point at \kfile{drivers/gpu/drm/v3d/v3d_drv.c} - \item Dedicated documentation: \url{https://dri.freedesktop.org/docs/drm/gpu/v3d.html} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM Prime zero-copy memory sharing (dma-buf)} - \begin{itemize} - \item Memory buffers often need to be shared between different devices\\ - \textit{e.g. DRM KMS and DRM render but also concerns V4L2 for media devices} - \item The kernel-wide \code{dma-buf} API allows exporting and importing buffers - \item Buffers are represented as \textbf{file descriptors} in userspace\\ - \textit{file descriptors can be shared between programs via IPC} - \item DRM exposes dma-buf via the \textbf{DRM Prime} API - \item DRM prime exports a GEM \code{handle} to a returned \code{fd}: - \begin{minted}[fontsize=\small]{console} -struct drm_prime_handle prime_handle = { .handle = ... } -ret = ioctl(drm_fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &prime_handle); - \end{minted} - \item And vice-versa: - \begin{minted}[fontsize=\small]{console} -struct drm_prime_handle prime_handle = { .fd = ... } -ret = ioctl(drm_fd, DRM_IOCTL_PRIME_FD_TO_HANDLE, &prime_handle); - \end{minted} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM sync object fencing} - \begin{itemize} - \item In a multi-device pipeline with zero-copy, only scheduling is left to userspace\\ - \textit{each device signals completion and userspace moves on to the next} - \item Fences were introduced to avoid the extra roundtrip in userspace: - \begin{itemize} - \item The flow of buffers between devices is usually known in advance - \item The kernel can coordinate internally and trigger the next device - \item Requires submitting all commands in advance with fences attached - \end{itemize} - \item DRM exposes fences via the \textbf{Sync object} API - \item Sync objects contain one fence, exposed as a file descriptor - \item The KMS atomic API and some render driver APIs take input fence fds - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM sync object fencing} - \begin{itemize} - \item Sync objects are created and destroyed with a \code{handle}: - \begin{minted}[fontsize=\small]{console} -struct drm_syncobj_create syncobj_create = { 0 } -ret = ioctl(drm_fd, DRM_IOCTL_SYNCOBJ_CREATE, &syncobj_create); - \end{minted} - \begin{minted}[fontsize=\small]{console} -struct drm_syncobj_destroy syncobj_destroy = { .handle = syncobj_create.handle } -ret = ioctl(drm_fd, DRM_IOCTL_SYNCOBJ_DESTROY, &syncobj_destroy); - \end{minted} - \item An output fence's \code{fd} is exported from a device's sync object with: - \begin{minted}[fontsize=\small]{console} -struct drm_syncobj_handle syncobj_handle = { .handle = handle, ... } -ret = ioctl(drm_fd, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &syncobj_handle); - \end{minted} - \item An input fence's \code{fd} is imported to a device's sync object with: - \begin{minted}[fontsize=\small]{console} -struct drm_syncobj_handle syncobj_handle = { .handle = handle, .fd = fd } -ret = ioctl(drm_fd, DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE, &syncobj_handle); - \end{minted} - \item Quite a recent feature, not yet available in V4L2 (media) - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{DRM debug and documentation} - \begin{itemize} - \item \textbf{Debug message} using the \code{drm.debug} kernel cmdline argument: - \begin{itemize} - \item Detailed in the \kfile{include/drm/drm_print.h} header - \item \code{drm.debug=0x17} for core, KMS, driver and atomic debug messages - \end{itemize} - \item Current \textbf{state debug} in debugfs: \code{cat /sys/kernel/debug/dri/0/state} - \item Drivers expose \textbf{specific debugfs entries} - \item \textbf{Debug utility}: \code{modetest} from \code{libdrm} - \item \textbf{Community} contact: - \begin{itemize} - \item Mailing list: \code{dri-devel@lists.freedesktop.org} - \item IRC channel: \code{#dri-devel} on the OFTC network - \end{itemize} - \item \textbf{Documentation} resources: - \begin{itemize} - \item Linux GPU Driver Developer’s Guide: \url{https://www.kernel.org/doc/html/latest/gpu/index.html} - \item Man pages about userspace aspects: \code{drm}, \code{drm-kms}, \code{drm-memory} - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{libdrm wrapper} - \begin{itemize} - \item Userspace access to DRM devices is wrapped with \textbf{libdrm} - \item Exposes convenience wrappers, helpers and some data structures around \code{ioctls} - \begin{itemize} - \item For KMS support in the \code{libdrm.so} library - \item For hardware-specific render drivers in dedicated libraries (e.g. \code{libdrm_nouveau.so}) - \end{itemize} - \item Used by almost every userspace project dealing with DRM:\\ - \textit{weston, mutter, Xorg, mesa, etc} - \end{itemize} - - \begin{center} - \includegraphics[width=0.5\textwidth]{slides/graphics-software/libdrm-userspace.pdf} - \end{center} -\end{frame} - -\subsection{Userspace: X Window} - -\begin{frame}{X11 protocol and architecture} - \begin{itemize} - \item X11 core protocol implemented by Xorg: - \begin{itemize} - \item Asynchronous packet-based system with different types:\\ - \code{Request}, \code{Reply}, \code{Event} and \code{Error} packets - \item Can be used locally (UNIX socket) or over network (TCP/IP) - \end{itemize} - \item Exposes \textbf{drawables} for clients to transfer or draw pixel data to the server: - \begin{itemize} - \item \textbf{Windows}: area of the display buffer owned by the application\\ - \textit{without backing storage, must be redrawn when occluded} - \item \textbf{Pixmaps}: off-display backing storage that can be copied to windows - \end{itemize} - \item Windows are represented as a tree: - \begin{itemize} - \item Starting with the root window created by X - \item Top-level windows and sub-windows created by clients - \end{itemize} - \item A graphics context (GC) allows requesting basic drawing and font rendering - \item The server provides \textbf{input events} to concerned clients: - \begin{itemize} - \item Mouse movements relative to window coordinates - \item Translated key symbols a from raw keycodes - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{X11 protocol extensions} - \begin{itemize} - \item X11 has evolved over time through \textbf{extensions} to its main protocol - \begin{itemize} - \item Additional interfaces for X clients, matching \textbf{new hardware features} - \end{itemize} - \item \textbf{XKB}: complex keyboard layouts - \item \textbf{Xinput2}: touchpad, touchscreen and multi-touch support - \item \textbf{XSHM}: shared client/server memory, avoiding extra transfers/copies\\ - \textit{not\ possible to operate via the network} - \item \textbf{XRandR}: monitor configuration and hotplugging without server restart - \item \textbf{Composite}: delegates window composition to compositing window managers - \item \textbf{XRender}: 2D rendering API with with alpha composition, rasterization, transformations, filtering - \item \textbf{Xv}: video output format conversion and scaling offload in-DDX\\ - \textit{involves buffer copies and lacks synchronization with window position} - \end{itemize} -\end{frame} - -\begin{frame}{Xorg architecture and acceleration} - \begin{itemize} - \item Xorg is divided between generic and hardware-specific parts - \item \textbf{Device-Independent-X} (DIX) concerns: - \begin{itemize} - \item X11 protocol implementation, client coordination - \item Main event loop and event dispatching - \item Graphics operations logic, boilerplate and fallback implementations - \end{itemize} - \item \textbf{Device-Dependent-X} (DDX) concerns: - \begin{itemize} - \item Input drivers (\code{xf86-input-...}) to grab events from the kernel - \item Video drivers (\code{xf86-video-...}) to provide mode setting and 2D acceleration - \end{itemize} - \item \textbf{EXA} provides a 2D acceleration architecture between DIX and DDX - \begin{itemize} - \item Efficient way for drivers to expose accelerated 2D operation primitives - \item Replaced the XFree86 Acceleration Architecture (XAA) - \item Reduces driver boilerplate - \end{itemize} - \item \textbf{Glamor} provides 2D acceleration for the DDX using OpenGL 3D rendering - \end{itemize} -\end{frame} - -\begin{frame}{Xorg drivers overview} - \begin{itemize} - \item Generic Xorg \textbf{input} drivers: - \begin{itemize} - \item \code{xf86-input-libinput}: using \code{libinput} to get input events - \item \code{xf86-input-evdev}: using the \code{evdev} kernel interface directly (\textbf{deprecated}) - \end{itemize} - \item Specific Xorg \textbf{input} drivers: - \begin{itemize} - \item \code{xf86-input-synaptics}: for laptop touchpads - \item \code{xf86-input-wacom}: for Wacom drawing tablets - \item Specific drivers are \textbf{deprecated} in favor of \code{xf86-input-libinput} - \end{itemize} - \item Generic Xorg \textbf{display} drivers: - \begin{itemize} - \item \code{xf86-video-modesetting}: for \textbf{DRM KMS}, can be accelerated using \textbf{glamor} - \item \code{xf86-video-fbdev}: for the \textbf{fbdev} interface, without acceleration (legacy) - \item \code{xf86-video-vesa}: for the \textbf{Video BIOS Extension} (VBE) framebuffer (x86) - \end{itemize} - \item Specific Xorg \textbf{display} drivers: - \begin{itemize} - \item \code{xf86-video-[intel,nouveau,amdgpu]}: profiting from 2D acceleration blocks - \item Specific drivers are deprecated in favor of \code{xf86-video-modesetting} and \textbf{glamor}\\ - \textit{the trend is to accelerate everything via 3D rendering instead of 2D accelerators} - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{X11 and OpenGL acceleration: GLX and DRI2} - \begin{itemize} - \item Before DRM render nodes, there was a single device for KMS and render\\ - \textit{correlates with the idea of a graphics card mixing both aspects} - \item The X server owns the graphics device \textbf{exclusively} - \item Clients using OpenGL need to access the device for rendering - \item The \textbf{GLX} API was introduced to perform \textbf{indirect rendering}: - \begin{enumerate} - \item Integrating OpenGL with the X Window API - \item Forwarding GL calls to the GL implementation via the X server (AIGLX)\\ - \textit{introducing latency and performance issues} - \end{enumerate} - \item \textbf{The Direct Rendering Infrastructure} (DRI/DRI2) was introduced next - \begin{itemize} - \item The X server allowed access through DRM magic/auth - \item Buffers were shared via \textit{GEM flinks} - \item Now using the standalone \textbf{render node} and \textbf{dma-buf} instead - \item Still in place for coordination between render and the display server - \end{itemize} - \item GLX remained as a GL windowing API for X11 (deprecated by EGL) - \end{itemize} -\end{frame} - -\begin{frame}{X11 and OpenGL acceleration: GLX and DRI2 (illustrated)} - \begin{center} - \includegraphics[width=0.4\textwidth]{slides/graphics-software/dri-data-flow.png}\\ - \textit{Data flow in X11 for different types of clients} - \end{center} -\end{frame} - -\begin{frame}{Xorg usage, integration and configuration} - \begin{itemize} - \item Xorg can be started with the \code{startx} command (wrapping \code{xinit}) - \begin{itemize} - \item Executes server script from \code{/etc/X11/xinit/xserverrc} or \code{$HOME/.xserverrc} - \item Executes client script from \code{/etc/X11/xinit/xinitrc} or \code{$HOME/.xinitrc} - \end{itemize} - \item An X \textbf{display manager} offers a login interface (e.g. KDM, LightDM) - \begin{itemize} - \item Runs under a Xorg server, with its own dedicated user - \item Starts Xorg for authenticated users from session files in \code{/usr/share/xsessions/} - \end{itemize} - \item Used to require \textbf{running the server as root} to access graphics devices\\ - \textit{in particular, necessary to become DRM master} - \begin{itemize} - \item The \code{systemd-logind} login manager lifts the restriction - \item Opens the DRM KMS fd privileged and passes it to Xorg via IPC - \item Xorg can then drop privileges: details in the \code{Xorg.wrap} man page - \end{itemize} - \item Xorg is \textbf{configured} (both DIX and DDX) from \code{/etc/X11/xorg.conf} - \item The \code{DISPLAY} environment variable indicates which server connection to use\\ - \begin{itemize} - \item Already set for X client applications and inherited - \item \code{export DISPLAY=:0} useful to launch programs from other TTYs - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Xorg architecture: input to display roundtrip} - \begin{minipage}{0.49\textwidth} - \centering - \includegraphics[width=0.9\textwidth]{slides/graphics-software/x-architecture-roundtrip.png} - \end{minipage} - \hfill - \begin{minipage}{0.49\textwidth} - \begin{enumerate} - \item An input event is read from the kernel by the server - \item The affected client is determined and receives the event - \item The client changes something and issues a rendering request - \item The server performs rendering (DDX) and notifies the compositor - \item The compositor updates the damaged regions in the back-buffer - \item The server updates the display buffer from the compositor buffer (page flip) - \end{enumerate} - \end{minipage} -\end{frame} - -\begin{frame}{Major issues with X11} - \begin{itemize} - \item The X11 core protocol and paradigm soon caused \textbf{various issues}: - \begin{itemize} - \item Based on buffer \textbf{copies, transfers} and frequent \textbf{redraws}\\ - \textit{solved with XSHM and DRI2 extensions} - \item Immediate-mode drawing, with intermediate states scanned out\\ - \textit{solved by drawing everything client-side instead} - \item Lack of synchronization/feedback interface\\ - \textit{specified with the DRI3 and Present extensions} - \item Everything's a window with X... but not in practice (screensavers, popups)\\ - \textit{specified with the DRI3 and Present extensions} - \item Heavy packet-based protocol causing \textbf{latency issues} - \item \textbf{Security} concerns regarding client input/output isolation - \end{itemize} - \item Because the core protocol did not evolve, \textbf{extensions proliferated}: - \begin{itemize} - \item Complicated server aspects got \textbf{delegated} through extensions - \item Working around major design issues, not solving them in depth - \item In the end, the server mostly coordinates between other components - \end{itemize} - \item \textbf{Client-side rendering} became more common (raster, operations, fonts, etc) - \end{itemize} -\end{frame} - -\begin{frame}{Xorg code structure and walkthrough} - \begin{itemize} - \item Xorg source code available at: \url{https://gitlab.freedesktop.org/xorg/xserver} - \item DDX components: - \begin{itemize} - \item Code specific to the Linux kernel under: \code{hw/xfree86/os-support/linux/} - \item Modesetting DRM KMS driver under: \code{hw/xfree86/drivers/modesetting/} - \item fbdev core library under: \code{hw/xfree86/fbdevhw/} - \item Glamor implementation under: \code{glamor/} - \end{itemize} - \item DIX components: - \begin{itemize} - \item System-level helpers under: \code{os/} - \item Common framebuffer operations abstraction under: \code{fb/} - \item EXA abstraction under: \code{exa/} - \end{itemize} - \item DRI2 components: - \begin{itemize} - \item DRI2 common code under: \code{hw/xfree86/dri2} - \item Modesetting DRI2 glue under: \code{hw/xfree86/drivers/modesetting/dri2.c} - \item GLX support under: \code{glx/} - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{Xorg debug and documentation} - \begin{itemize} - \item Xorg has a \textbf{logging system} for all its components: - \begin{itemize} - \item Written to a file at \code{/var/log/Xorg.0.log}, \code{-logfile} option - \item Verbosity can be set with the \code{-logverbose} option (log level) - \item Printed on the standard output (\code{stdout}) - \end{itemize} - \item Xorg can be bound to any VT with the \code{vt} command line option\\ - \textit{useful for remote debugging, with a virtual controlling terminal} - \item \textbf{Community} contact: - \begin{itemize} - \item Mailing list: \code{xorg@lists.freedesktop.org} - \item IRC channel: \code{#xorg} and \code{#xorg-devel} on the OFTC network - \end{itemize} - \item \textbf{Documentation} resources: - \begin{itemize} - \item Online wiki of the project: \url{https://www.x.org/wiki/Documentation/} - \item Man pages: \code{X}, \code{Xserver}, \code{Xorg}, \code{xorg.conf}, \code{xinit} and more! - \item Extensions specification documents - \end{itemize} - \end{itemize} -\end{frame} - -\subsection{Userspace: Wayland} - -\begin{frame}{Wayland overview and paradigm} - \begin{itemize} - \item Wayland was started in 2008 as a modern replacement for the X Window system\\ - \textit{solving issues in-depth with a clean implementation from scratch} - \item Drastic \textbf{simplification} of the stack and paradigm shift: - \begin{itemize} - \item The server and compositor are \textbf{unified} as the same component - \item \textbf{Clients} are expected to do all the rendering - \item Buffers are \textbf{shared} between client and server, no transfers - \item Window decorations can be added by the client or the server - \end{itemize} - \item Improves \textbf{security} aspects: - \begin{itemize} - \item Isolates the input and output of each client - \item Only the compositor can access display buffers \textit{(and provide screenshots)} - \item Avoids running the compositor as root (using \code{systemd-logind}) - \end{itemize} - \item No network support (can be implemented by compositors) - \item \textbf{Weston} is the reference \textbf{Wayland} compositor - \end{itemize} -\end{frame} - -\begin{frame}{Wayland architecture: input to display roundtrip} - \begin{minipage}{0.49\textwidth} - \centering - \includegraphics[height=0.7\textheight]{slides/graphics-software/wayland-architecture-roundtrip.png} - \end{minipage} - \hfill - \begin{minipage}{0.49\textwidth} - \begin{enumerate} - \item An input event is read from the kernel by the compositor - \item The affected client is determined and receives the event - \item The client changes something, performs rendering and notifies the compositor - \item The compositor updates the damaged regions in the back-buffer and performs page flip - \end{enumerate} - \end{minipage} -\end{frame} - -\begin{frame}{Wayland protocol and architecture} - \begin{itemize} - \item Wayland provides a client-server \textbf{low-level API}: - \begin{itemize} - \item Wayland \textbf{display connections} happen through local IPC - \item Display is identified with the \code{WAYLAND_DISPLAY} \textbf{environment variable} - \item \textbf{Asynchronous} and \textbf{object-oriented} protocol - \item \textbf{Objects} represent \textbf{resources} from the compositor - \item Objects implement specific \textbf{interfaces}, with requests and events - \begin{itemize} - \item \textbf{Requests} are messages sent by the client, - \item \textbf{Events} are messages received from the server\\ - \textit{errors are only specific types of events} - \end{itemize} - \end{itemize} - \item Some \textbf{implementation} details: - \begin{itemize} - \item IPC is a regular UNIX socket (allows passing file descriptors for zero-copy) - \item A proxy provides client-side object representations and message translation - \item Messages are serialized (marshalling) to the wire format - \item Messages are buffered and flushed/dispatched when asked - \end{itemize} - \item Client-server protocol is implemented in \textbf{libwayland-client} and \textbf{libwayland-server} - \item These libraries do not provide any interface implementation - \end{itemize} -\end{frame} - -\begin{frame}{Wayland core protocol: global object interfaces} - \begin{itemize} - \item \textbf{Global} core object interfaces: - \begin{itemize} - \item \code{wl_display}: manages server connection, exposes the registry - \item \code{wl_registry}: exposes available global object interfaces - \item \code{wl_output}: describes the output properties (mode, geometry) - \item \code{wl_seat}: exposes input device object capabilities and interfaces - \item \code{wl_compositor}: provides surfaces and regions for composition - \item \code{wl_subcompositor}: provides sub-surfaces for in-surface compositing - \item \code{wl_shm}: exposes a shared memory interface - \item \code{wl_data_device_manager}: support for copy/paste between clients - \end{itemize} - \item Global object interfaces are bound with the registry before use - \begin{itemize} - \item Using global \code{struct wl_interface} definitions - \item \code{wl_registry_bind} returns a pointer to a proxy object - \end{itemize} - \item Global objects give access to other \textbf{specific objects} - \end{itemize} -\end{frame} - -\begin{frame}{Wayland core protocol: specific object interfaces} - \begin{itemize} - \item \textbf{Input-related} (\code{wl_seat}) specific core object interfaces: - \begin{itemize} - \item \code{wl_pointer}: exposes mice events and cursor - \item \code{wl_keyboard}: exposes keyboard events and information - \item \code{wl_touch}: exposes touchscreen events - \end{itemize} - \item \textbf{Compositor-related} (\code{wl_compositor}) specific core object interfaces: - \begin{itemize} - \item \code{wl_region}: specifies any area - \item \code{wl_surface}: rectangular on-screen pixel area - \begin{itemize} - \item contents set to a \code{wl_buffer} (can be transformed) - \item configured with a \code{wl_region} for input - \item configured with a \code{wl_region} for area-based opacity - \item updated with buffer damage regions - \end{itemize} - \item \code{wl_subsurface}: converts \code{wl_surface} objects to positioned sub-surfaces - \end{itemize} - \item \textbf{Shared memory-related} (\code{wl_shm}) specific core object interfaces: - \begin{itemize} - \item \code{wl_shm_pool}: allows creating shared-memory buffers - \end{itemize} - \item \textbf{Memory-related} specific core object interfaces: - \begin{itemize} - \item \code{wl_buffer}: generic object for a \code{wl_surface} - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Wayland extra protocols} - \begin{itemize} - \item Extra protocols (object interfaces) can be \textbf{exposed by the compositor} - \begin{itemize} - \item Protocols (including the core) are described as XML files - \item The \code{wayland-scanner} tool produces client and server C code and headers - \item Accepted additional protocol descriptions are available at: - \url{https://gitlab.freedesktop.org/wayland/wayland-protocols} - \item Some are considered stable and many unstable - \end{itemize} - \item Some widely-used \textbf{protocol extensions}: - \begin{itemize} - \item \textbf{XDG-Shell}: desktop shell integration - \begin{itemize} - \item Turns \code{wl_surfaces} to \code{xdg_surfaces} that can be resized, minimized, etc - \item Provides a popup/menu interface with \code{xdg_popup} - \end{itemize} - \item \textbf{IVI-Shell}: In-vehicle shell with surface layers - \item \textbf{Presentation time}: Precise timing and event feedback - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Wayland OpenGL integration} - \begin{itemize} - \item Wayland supports EGL for windowing integration with OpenGL - \item \code{eglGetDisplay} is called with a \code{struct wl_display}\\ - \textit{mesa's \code{_eglNativePlatformDetectNativeDisplay} figures it out} - \item Mesa 3D implements Wayland EGL interface for OpenGL integration - \begin{itemize} - \item Needs to implement DRI2 for DRM authentication - \item \code{wl_drm} interface between the wayland EGL client and the compositor - \item Both sides are actually implemented in mesa - \item The interface is bound to the compositor with \code{eglBindWaylandDisplayWL}\\ - \textit{using the compositor's EGL context as entry-point to mesa} - \item Allows sharing DRM GEM buffers with the compositor - \end{itemize} - \item Regular \code{wl_surfaces} can be bound to EGL: - \begin{itemize} - \item Converted to a \code{wl_egl_window} with \code{wl_egl_window_create} - \item Then converted to an \code{EGLSurface} with \code{eglCreateWindowSurface} - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Wayland OpenGL integration (illustrated)} - \begin{center} - \includegraphics[width=12em]{slides/graphics-software/wayland-egl.pdf}\\ - \textit{\small Wayland integration with EGL}\\ - \end{center} -\end{frame} - -\begin{frame}{Wayland status and adoption} - \begin{itemize} - \item Wayland is now quite mature, robust, efficient and widely used - \begin{itemize} - \item Most toolkits have support for it: GTK 3, Qt 5, SDL, EFL - \item Major desktop environments support it: GNOME 3, KDE Plasma 5 - \item Integrated sessions with login managers from \code{/usr/share/wayland-sessions} - \item Runs with user privileges with \code{systemd-logind} - \end{itemize} - \item X11 applications can be integrated using \textbf{XWayland} - \begin{itemize} - \item X server implementation registered as a Wayland client - \item Wayland composer acts as X compositing window manager - \item Creates a \code{wl_surface} for each window, redirects input/output - \end{itemize} - \item Diverse compositor implementations have emerged - \begin{itemize} - \item Sometimes tied to a desktop environment: mutter, kwin - \item \textbf{libinput} was created to help with input aspects - \item \textbf{libweston} emerged from the \textbf{Weston} compositor core - \item \textbf{wlroots} emerged from the \textbf{Sway} compositor core - \item Support \textbf{DRM KMS} display, \textbf{EGL} rendering (\textbf{pixman} supported by libweston) - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Wayland stack overview (illustrated)} - \begin{center} - \includegraphics[width=0.8\textwidth]{slides/graphics-software/wayland-stack.pdf}\\ - \textit{\small The graphics stack with Wayland}\\ - \end{center} -\end{frame} - -\begin{frame}[fragile]{Wayland debug and documentation} - \begin{itemize} - \item \textbf{Debugging} tips: - \begin{itemize} - \item Supported global object interfaces can be listed with \code{weston-info} - \item The \code{WAYLAND_DEBUG} environment variable enables protocol tracing - \end{itemize} - \item \textbf{Weston debugging}: - \begin{itemize} - \item Debug arguments: \code{--debug}, \code{--log=file.log} - \item Grabbing a different TTY argument: \code{--tty 1} - \item Wireframe keystroke: \code{mod + shift + space + F} - \item Timeline recording (to a JSON file) keystroke: \code{mod + shift + space + d}\\ - \textit{can produce a graph with \url{https://github.com/ppaalanen/wesgr}} - \end{itemize} - \item \textbf{Community} contact: - \begin{itemize} - \item Mailing list: \code{wayland-devel@lists.freedesktop.org} - \item IRC channel: \code{#wayland} on the OFTC network - \end{itemize} - \item \textbf{Documentation} resources: - \begin{itemize} - \item Online wiki of the project: \url{https://wayland.freedesktop.org/} - \item Online documentation: \url{https://wayland.freedesktop.org/docs/html/} - \end{itemize} - \end{itemize} -\end{frame} - -\subsection{Userspace: Mesa 3D} - -\begin{frame}[t]{Standardized 3D rendering APIs: OpenGL} - \begin{center} - \includegraphics[height=4em]{slides/graphics-software/opengl-logo.pdf} - \end{center} - - \begin{itemize} - \item 3D rendering API, designed for GPU \textbf{hardware acceleration} - \begin{itemize} - \item Generic API but \textbf{hardware-specific implementations} - \item Started by \textbf{Silicon Graphics} in 1992, now managed by the \textbf{Khronos Group} - \end{itemize} - \item OpenGL provides a \textbf{high-level approach} to 3D graphics - \begin{itemize} - \item Compromise between complexity and fine-grained control - \item Efficient abstraction, adapted to the hardware - \item Leaves most memory management to the implementation - \end{itemize} - \item \textbf{Stateful} and \textbf{context-based} programming model - \end{itemize} -\end{frame} - -\begin{frame}[t]{Standardized 3D rendering APIs: OpenGL} - \begin{center} - \includegraphics[height=4em]{slides/graphics-software/opengl-logo.pdf} - \end{center} - - \begin{itemize} - \item OpenGL versions evolved with hardware features - \begin{itemize} - \item Version 1 targeted fixed-function pipeline GPUs - \item Version 2 and up allow programming \textbf{vertex and fragment shaders} - \item More shaders supported with new versions (\textit{geometry, tesselation}) - \end{itemize} - \item OpenGL comes with the \textbf{GL Shading Language} (GLSL) - \begin{itemize} - \item Source code language for OpenGL shaders - \item C-like syntax with intrinsic functions (e.g. texture access) - \item Compiled on-the-fly by the GL implementation - \end{itemize} - \item Supports \textbf{extensions} that can be queried, for extra features - \end{itemize} -\end{frame} - -\begin{frame}[t]{Standardized 3D rendering APIs: OpenGL ES and EGL} - \begin{minipage}[t]{0.49\textwidth} - \centering - \includegraphics[height=4em]{slides/graphics-software/opengl-es-logo.pdf} - \end{minipage} - \hfill - \begin{minipage}[t]{0.49\textwidth} - \centering - \includegraphics[height=4em]{slides/graphics-software/egl-logo.pdf} - \end{minipage} - \vspace{0.5em} - \begin{itemize} - \item \textbf{OpenGL ES} was introduced as a simplified version for embedded devices - \item OpenGL ES versions are loosely following OpenGL versions: - \begin{itemize} - \item Version 1 targets \textbf{fixed-function} GPUs - \item Version 2 and up target \textbf{programmable} GPUs - \end{itemize} - \item Uses GLSL shaders and the same programming model as OpenGL - \item \textbf{EGL} was introduced as standardized window integration API - \begin{itemize} - \item Connects with the native system display server - \item Replaces GLX for X11 and adopted as default by Wayland - \end{itemize} - \item Supports \textbf{extensions} that can be queried, for extra platform-specific features - \end{itemize} -\end{frame} - -\begin{frame}[t]{Standardized 3D rendering APIs: Vulkan} - \begin{center} - \includegraphics[height=4em]{slides/graphics-software/vulkan-logo.pdf} - \end{center} - - \begin{itemize} - \item \textbf{Vulkan} is a \textbf{low-level} generic API for GPU access - \begin{itemize} - \item Started by the Khronos group in 2016 and widely adopted - \end{itemize} - \item Suitable for both \textbf{3D rendering} and \textbf{compute} - \item Uses \textbf{Standard Portable Intermediate Representation} (SPIR-V) shaders - \begin{itemize} - \item Unified intermediate representation from (adapted) GLSL/HLSL sources - \item Compiled with the program instead of on-the-fly (less overhead) - \item Translated to native GPU operations by implementations - \end{itemize} - \item \textbf{Direct programming} model, with low-level \textbf{memory management} - \item The API provides \textbf{window system integration} (WSI) for many platforms\\ - \textit{e.g. for Wayland: \code{vkCreateWaylandSurfaceKHR}} - \end{itemize} -\end{frame} - -\begin{frame}{Mesa 3D overview} - \begin{itemize} - \item Mesa is the reference free software \textbf{3D graphics implementation} - \begin{itemize} - \item Started back in 1993, evolved with GPU implementations - \item Project works with the Khronos Group and develops extensions - \end{itemize} - \item Implements support for \textbf{rendering} APIs: - \begin{itemize} - \item \textbf{OpenGL} (up to 4.6) and \textbf{OpenGL ES} (up to 3.2) - \item \textbf{Vulkan} (up to 1.1) with translation to OpenGL via \textbf{Zink} - \item \textbf{Direct 3D} (version 9 only) - \end{itemize} - \item Implements \textbf{windowing system} integration: - \begin{itemize} - \item \textbf{EGL} for Wayland, X11, Android, native DRM (GBM) and surface-less - \item \textbf{Vulkan WSI} for Wayland and X11 (XCB/Xlib) - \item \textbf{GLX} for X11 - \end{itemize} - \item Also supports other \textbf{GPU-related features}: - \begin{itemize} - \item \textbf{Video decoding} acceleration via VDPAU, VAAPI, OMX - \item \textbf{Compute} (GPGPU) support via OpenCL (\code{clover}) - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Mesa 3D implementation highlights} - \begin{itemize} - \item Unlike other devices, 3D hardware is \textbf{abstracted in userspace} - \begin{itemize} - \item 3D rendering is a very bad fit for in-kernel abstraction - \item Kernel drivers are much less complicated than GL implementations - \end{itemize} - \item Mesa implements driver-specific \textbf{DRM render} support\\ - \begin{itemize} - \item Manages memory with the GEM and Prime DRM APIs - \item Manages DRI2 to allow direct rendering - \end{itemize} - \item \textbf{Virtual} drivers are also supported (for virtual machines): - \begin{itemize} - \item \textbf{vmwgfx}: VMware bridge (proprietary virtual hardware implementation) - \item \textbf{virgl}: Virtio bridge (standard for Linux and QEMU) - \end{itemize} - \item Also provides \textbf{software backends}: - \begin{itemize} - \item \textbf{softpipe}: Generic reference software renderer - \item \textbf{swr}: OpenSWR renderer (for x86 by Intel) - \item \textbf{llvmpipe}: LLVM-based renderer (high-performance) - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Mesa 3D internals: Gallium 3D} - \begin{itemize} - \item Classic mesa drivers have significant \textbf{code duplication}: - \begin{itemize} - \item API state tracking - \item Compiler implementation - \end{itemize} - \item The Gallium 3D interface splits things up instead: - \begin{itemize} - \item \textbf{API State trackers}: maintain the current state for the API in use - \item \textbf{Drivers}: implement shader compilation and hardware configuration - \item \textbf{Winsys}: implement low-level kernel interfaces - \end{itemize} - \item Gallium drivers implement a pipe interface: - \begin{itemize} - \item \code{struct pipe_screen}: textures, buffers and sync management - \item \code{struct pipe_state}: pipeline configuration and resources state - \item \code{struct pipe_context}: rendering operation functions - \end{itemize} - \item Pipe loaders (DRM or software) select the right pipe driver - \end{itemize} -\end{frame} - -\begin{frame}{Mesa 3D internals: Gallium 3D (illustrated)} - \begin{center} - \includegraphics[width=0.5\textwidth]{slides/graphics-software/mesa-dri-gallium.pdf}\\ - \textit{\small Driver integration in Mesa 3D} - \end{center} -\end{frame} - -\begin{frame}{Mesa 3D internals: intermediate representations} - \begin{itemize} - \item Mesa is in charge of \textbf{compiling shaders} to the native shading ISA - \item \textbf{Intermediate representations} (IRs) are used for translation - \item \textbf{Input-level} IRs: - \begin{itemize} - \item \textbf{GLSL IR}: Internal GLSL shader representation - \item \textbf{SPIR-V}: Khronos' Standard Portable Intermediate Representation - \end{itemize} - \item \textbf{Internal} IRs: - \begin{itemize} - \item \textbf{TGSI IR}: Tungsten Graphics Shader Interface representation - \item \textbf{NIR}: New efficient internal representation for Mesa - \item \textbf{Mesa}: Historical implementation (deprecated) - \end{itemize} - \item \textbf{External} IR: - \begin{itemize} - \item \textbf{LLVM IR}: Used for LLVM interaction (e.g. with llvmpipe) - \end{itemize} - \item Drivers emit \textbf{native instructions} from an internal IR and lowering - \item Once ready, compiled shaders are \textbf{submitted to the GPU} - \end{itemize} -\end{frame} - -\begin{frame}{Mesa 3D internals: intermediate representations (illustrated)} - \begin{center} - \includegraphics[width=0.4\textwidth]{slides/graphics-software/mesa-ir.pdf}\\ - \textit{\small The Mesa 3D internal representation pipeline} - \end{center} -\end{frame} - -\begin{frame}[fragile]{Mesa 3D Generic Buffer Management} - \begin{itemize} - \item Mesa provides a Generic Buffer Management (GBM) interface: - \begin{itemize} - \item Buffer creation/destruction (supporting modifiers) - \item Buffer information (bpp, dimensions, planes, stride, modifiers) - \item Buffer mapping/unmapping - \item Buffer dma-buf import/unimport - \end{itemize} - \item Compatible with the EGL API: - \begin{itemize} - \item \code{struct gbm_device} as \code{EGLNativeDisplayType} - \item \code{struct gbm_surface} as \code{EGLNativeWindowType} - \item \code{struct gbm_bo} as \code{EGLNativePixmapType} - \end{itemize} - \item Provided with DRM KMS fd for DRI2 (used internally for most operations)\\ - \begin{minted}[fontsize=\small]{console} -struct gbm_device *device = gbm_create_device(drm_fd); - \end{minted} - \item Useful when using bare-metal DRM KMS - \end{itemize} -\end{frame} - -\begin{frame}{Mesa 3D hardware support status: desktop} - \begin{itemize} - \item Updated Mesa per-driver support at \url{https://mesamatrix.net} - \item \textbf{Intel HD/Iris Graphics} - \begin{itemize} - \item Platforms: Intel only - \item Mesa driver: i965 (classic), iris (Gallium) - \item DRM driver: i915 - \item Status: state-of-the art (i965/iris) - \end{itemize} - \item \textbf{Nvidia pre-NV110} - \begin{itemize} - \item Platforms: Tegra, any PCI-e compatible - \item Mesa driver: nouveau (Gallium) - \item DRM driver: nouveau - \item Status: reverse engineered, advanced - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Mesa 3D hardware support status: desktop} - \begin{itemize} - \item \textbf{AMD Radeon GCN-ish} - \begin{itemize} - \item Platforms: AMD, any PCI-e compatible - \item Mesa driver: radeonsi (Gallium) - \item DRM driver: amdgpu - \item Status: state-of-the art - \end{itemize} - \item \textbf{AMD Radeon R600+} - \begin{itemize} - \item Platform: AMD, any PCI-e compatible - \item Driver: r600 (Gallium) - \item DRM driver: radeon - \item Status: advanced - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Mesa 3D hardware support status: embedded} - \begin{itemize} - \item \textbf{Qualcomm Adreno} - \begin{itemize} - \item Platforms: Qualcomm Snapdragon - \item Mesa driver: freedreno (Gallium) - \item DRM driver: freedreno - \item Status: reverse engineered, advanced - \end{itemize} - \item \textbf{Vivante GCx000} - \begin{itemize} - \item Platforms: i.MX6, i.MX8, i.MX8M - \item Driver: etnaviv (Gallium) - \item DRM driver: etnaviv - \item Status: vastly usable - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Mesa 3D hardware support status: embedded} - \begin{itemize} - \item \textbf{ARM Mali Utgard} - \begin{itemize} - \item Platforms: Exynos, Allwinner, Amlogic - \item Mesa driver: lima (Gallium) - \item DRM driver: lima - \item Status: reverse engineered, usable - \end{itemize} - \item \textbf{ARM Mali Midgard/Bifrost} - \begin{itemize} - \item Platforms: Rockchip, Exynos, Mediatek, Allwinner - \item Mesa driver: panfrost (Gallium) / PanVK (Vulkan) - \item DRM driver: panfrost - \item Status: advanced - \end{itemize} - \item \textbf{Imagination PowerVR Rogue} - \begin{itemize} - \item Platforms: Mediatek - \item Mesa driver: imagination - \item DRM driver: imagination - \item Status: work in progress - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Mesa 3D versus proprietary implementations} - \begin{itemize} - \item 3D support is one of the most challenging parts of hardware integration - \item Proprietary implementations easily lead to various practical issues: - \begin{itemize} - \item Lack of support outside of prescribed environments - \item Lack of specific features or APIs - \item Lack of maintenance and updates - \item No adaptation possibility - \end{itemize} - \item Mesa provides a collectively-maintained base - \begin{itemize} - \item Constantly updated and improved by the community - \item Easier to manage: works out of the box with distributions - \end{itemize} - \item Mesa support is complex and often takes some time to bring performance\\ - \textit{especially for drivers based on reverse-engineering} - \end{itemize} -\end{frame} - -\begin{frame}{Mesa 3D code structure and walkthrough} - \begin{itemize} - \item Mesa source code available at: \url{https://gitlab.freedesktop.org/mesa/mesa} - \item \textbf{Gallium 3D} components: - \begin{itemize} - \item Drivers under: \code{src/gallium/drivers/} - \item Winsys under: \code{src/gallium/winsys/} - \item API state trackers under: \code{src/gallium/state_trackers/} - \item Pipe loaders under: \code{src/gallium/auxiliary/pipe-loader/} - \end{itemize} - \item \textbf{Compilation and IR} components: - \begin{itemize} - \item IR compiler support under: \code{src/compiler/{glsl,nir,spirv}} - \item TGSI support under: \code{src/gallium/auxiliary/tgsi/} - \item State tracking between IRs under: \code{src/mesa/state_tracker} - \end{itemize} - \item \textbf{Windowing and DRI2} components: - \begin{itemize} - \item EGL support under: \code{src/egl/drivers/dri2/} - \item Vulkan WSI support under: \code{src/vulkan/wsi/} - \end{itemize} - \item \textbf{Classic drivers} (DRI-1-style) under: \code{src/mesa/drivers/dri/} - \item \textbf{GBM} support under: \code{src/gbm/} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{Mesa 3D hardware support: debug and documentation} - \begin{itemize} - \item Mesa is debugged with numerous \textbf{environment variables} - \begin{itemize} - \item Generic and per-driver, see \url{https://www.mesa3d.org/envvars.html} - \item Shader-related, see \url{https://www.mesa3d.org/shading.html} - \item \code{LIBGL_DEBUG=verbose} for OpenGL, \code{EGL_LOG_LEVEL=debug} for EGL - \end{itemize} - \item \code{eglinfo} and \code{glxinfo} show information about the implementation - \item \textbf{Community} contact: - \begin{itemize} - \item Mailing list: \code{dri-devel@lists.freedesktop.org} - \item IRC channel: \code{#dri-devel} on the OFTC network - \end{itemize} - \item \textbf{Documentation} resources: - \begin{itemize} - \item Online website: \url{https://www.mesa3d.org/} - \item Gallium 3D wiki: \url{https://www.freedesktop.org/wiki/Software/gallium/} - \item Gallium 3D documentation: \url{https://gallium.readthedocs.io/} - \item Source code reference: \url{https://elixir.bootlin.com/mesa/latest/source} - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Graphics software online references} - \small - \begin{minipage}[t]{0.45\textwidth} - \begin{itemize} - \item Linux man pages - \item Wikipedia (\url{https://en.wikipedia.org/}): - \begin{itemize} - \item \href{https://en.wikipedia.org/wiki/X_Window_System}{X Window System} - \item \href{https://en.wikipedia.org/wiki/X.Org_Server}{X.Org Server} - \item \href{https://en.wikipedia.org/wiki/Wayland_(display_server_protocol)}{Wayland (display server protocol)} - \item \href{https://en.wikipedia.org/wiki/Mesa_(computer_graphics)}{Mesa (computer graphics)} - \item \href{https://en.wikipedia.org/wiki/OpenGL}{OpenGL} - \item \href{https://en.wikipedia.org/wiki/Vulkan_(API)}{Vulkan (API)} - \end{itemize} - \end{itemize} - \end{minipage} - \hfill - \begin{minipage}[t]{0.5\textwidth} - \begin{itemize} - \item Freedesktop.org (\url{https://freedesktop.org/}): - \begin{itemize} - \item \href{https://dri.freedesktop.org/wiki/}{Direct Rendering Infrastructure} - \item \href{https://dri.freedesktop.org/wiki/DRM/}{DRM} - \item \href{https://wayland.freedesktop.org/}{Wayland} - \item \href{https://www.x.org/wiki/}{X.org wiki} - \end{itemize} - \item Khronos (\url{https://khronos.org/}): - \begin{itemize} - \item \href{https://www.khronos.org/opengl/}{OpenGL} - \item \href{https://www.khronos.org/opengl/wiki/}{OpenGL Wiki} - \item \href{https://www.khronos.org/egl/}{EGL} - \item \href{https://www.khronos.org/vulkan/}{Vulkan} - \end{itemize} - \end{itemize} - \end{minipage} -\end{frame} - -\begin{frame}{Graphics software illustrations attributions} - \small - \begin{itemize} - \item \href{https://commons.wikimedia.org/wiki/File:X11.svg}{X11 logo: public domain} - \item \href{https://commons.wikimedia.org/wiki/File:Wayland_Logo.svg}{Wayland logo: Kristian Høgsberg} - \item \href{https://commons.wikimedia.org/wiki/File:GTK_logo.svg}{GTK logo: Andreas Nilsson, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:GTK_logo.svg}{Qt logo: Qt Project, public domain} - \item \href{https://commons.wikimedia.org/wiki/File:SDL_Logo.svg}{SDL logo: Arne Claus / SDL Project, public domain} - \item \href{https://commons.wikimedia.org/wiki/File:Gnomelogo.svg}{GNOME logo: GNOME Foundation, GNU LGPL 2.1+} - \item \href{https://commons.wikimedia.org/wiki/File:KDE_logo.svg}{KDE logo: KDE e.V., GNU LGPL 2.1+} - \item \href{https://commons.wikimedia.org/wiki/File:Xfce_logo.svg}{XFCE logo: Xfce Team, GNU LGPL 2.1+} - \item \href{https://commons.wikimedia.org/wiki/File:Enlightenment_logo_black.png}{Enlightenment logo: Carsten Haitzler and various contributors, BSD} - \end{itemize} -\end{frame} - -\begin{frame}{Graphics software illustrations attributions} - \small - \begin{itemize} - \item \href{https://commons.wikimedia.org/wiki/File:Devuan_GNU-Linux_-_tty_login_-_server_rack.jpg}{Devuan GNU-Linux - tty login - server rack: Francesco Magno, CC BY-SA 4.0} - \item \href{https://commons.wikimedia.org/wiki/File:DRM_architecture.svg}{DRM architecture: Javier Cantero CC BY-SA 4.0} - \item \href{https://wayland.freedesktop.org/architecture.html}{Wayland architecture: Wayland Developers, GNU GPL 2+} - \item \href{https://wayland.freedesktop.org/architecture.html}{X architecture: Wayland Developers, GNU GPL 2+} - \item \href{https://commons.wikimedia.org/wiki/File:Wayland_display_server_protocol.svg}{Wayland display server protocol: Shmuel Csaba Otto Traian, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:The_Linux_Graphics_Stack_and_glamor.svg}{The Linux Graphics Stack and glamor: Shmuel Csaba Otto Traian, CC BY-SA 3.0} - \item \href{https://www.khronos.org/assets/utilities/retrieveFile.php?d=opengl&t=logopacks}{Khronos logo pack} - \item \href{https://commons.wikimedia.org/wiki/File:Gallium3D_vs_DRI_graphics_driver_model.svg}{Gallium3D vs DRI graphics driver model, CC BY-SA 3.0} - \end{itemize} -\end{frame} diff --git a/slides/graphics-theory/barn-yuv.jpg b/slides/graphics-theory-color/barn-yuv.jpg similarity index 100% rename from slides/graphics-theory/barn-yuv.jpg rename to slides/graphics-theory-color/barn-yuv.jpg diff --git a/slides/graphics-theory/barn.jpg b/slides/graphics-theory-color/barn.jpg similarity index 100% rename from slides/graphics-theory/barn.jpg rename to slides/graphics-theory-color/barn.jpg diff --git a/slides/graphics-theory/gamut.png b/slides/graphics-theory-color/gamut.png similarity index 100% rename from slides/graphics-theory/gamut.png rename to slides/graphics-theory-color/gamut.png diff --git a/slides/graphics-theory-color/graphics-theory-color.tex b/slides/graphics-theory-color/graphics-theory-color.tex new file mode 100644 index 0000000000..1a4fd9f40b --- /dev/null +++ b/slides/graphics-theory-color/graphics-theory-color.tex @@ -0,0 +1,201 @@ +\subsection{Color Representation} + +\begin{frame}{Light and color representation} + \begin{minipage}[c]{0.75\textwidth} + \begin{itemize} + \item \textbf{Light} itself must be quantized in digital representations\\ + \textit{distinct from and unrelated to spatial quantization} + \item Perceived as \textbf{colors} based on the Human visual system + \begin{itemize} + \item Perception based on \textbf{trichromacy} (red, green, blue) + \item Not necessarily a unique frequency of the spectrum\\ + \textit{e.g. pink is not a color of the visible spectrum} + \end{itemize} + \item \textbf{Translating} light information (colors) to numbers: + \begin{itemize} + \item A \textbf{color model} defines a base of color components\\ + \textit{typically 3 components (e.g. red, green, blue)} + \item A \textbf{colorspace} is a precise translation referential\\ + \textit{unique association of a color and coordinates in the base} + \item The \textbf{color gamut} is the range of colors in the colorspace\\ + \textit{not every color can be represented in every colorspace} + \end{itemize} + \end{itemize} + \end{minipage} + \hfill + \begin{minipage}[c]{0.225\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-color/gamut.png}\\ + \textit{\small Color gamut of a given colorspace} + \end{minipage} +\end{frame} + +\begin{frame}{Color quantization approaches} + \begin{minipage}[c]{0.75\textwidth} + \begin{itemize} + \item Different approaches exist for \textbf{color quantization}: + \begin{itemize} + \item \textbf{Uniform} quantization in the color range (most common)\\ + \textit{values are attributed to colors with a regular step (resolution)} + \item \textbf{Irregular} quantization with indexed colors (palettes)\\ + \textit{values are attributed to colors as needed} + \end{itemize} + \item Uniform color coordinates are quantized with: + \begin{itemize} + \item A given \textbf{resolution}: + \textit{the smallest possible color difference} + \item A given \textbf{range}: + \textit{the span of representable colors} + \end{itemize} + \item A given number of bits are used for quantization: \textbf{bit depth} + \item A \textbf{trade-off} between range and resolution must be defined + \begin{itemize} + \item Increasing the resolution reduces the range + \item Increasing the range reduces the resolution + \end{itemize} + \end{itemize} + \end{minipage} + \hfill + \begin{minipage}[c]{0.225\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-color/rgb-cube.pdf} + \end{minipage} +\end{frame} + +\begin{frame}{Light representation, color quantization (illustrated)} + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-color/pair-of-merops.jpg} + \end{minipage} + \hfill + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-color/pair-of-merops-16-colors.jpg} + \end{minipage} + + \begin{center} + \textit{\small A pair of Merops feeding} + \end{center} + + \begin{minipage}[b]{0.45\textwidth} + \centering + \textbf{16 million colors (24 bits per pixel)} + \begin{itemize} + \item high color resolution + \item high color range + \end{itemize} + \end{minipage} + \hfill + \begin{minipage}[b]{0.45\textwidth} + \centering + \textbf{16 colors (4 bits per pixel)} + \begin{itemize} + \item medium color resolution + \item low color range + \end{itemize} + \end{minipage} +\end{frame} + +\begin{frame}{Light representation, color quantization (illustrated)} + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-color/pair-of-merops.jpg} + \end{minipage} + \hfill + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-color/pair-of-merops-16-colors-range.jpg} + \end{minipage} + + \begin{center} + \textit{\small A pair of Merops feeding} + \end{center} + + \begin{minipage}[b]{0.45\textwidth} + \centering + \textbf{16 million colors (24 bits per pixel)} + \begin{itemize} + \item low color resolution + \item high color range + \end{itemize} + \end{minipage} + \hfill + \begin{minipage}[b]{0.45\textwidth} + \centering + \textbf{16 colors (4 bits per pixel)} + \begin{itemize} + \item low color resolution + \item high color range + \end{itemize} + \end{minipage} +\end{frame} + +\begin{frame}{Colorspaces and channels} + \begin{minipage}[b]{0.7\textwidth} + \begin{itemize} + \item Each component of a color model is called a \textbf{channel} + \item Examples for usual types of color models: + \begin{itemize} + \item RGB, with 3 channels:\\ \textbf{R} (red) / \textbf{G} (green) / \textbf{B} (blue) + \item HSV, with 3 channels:\\ \textbf{H} (hue) / \textbf{S} (saturation) / \textbf{V} (value) + \item YUV or Y/Cb/Cr, with 3 channels:\\ \textbf{Y} (luminance) / \textbf{U or Cb} / \textbf{V or Cr} (chrominance) + \end{itemize} + \end{itemize} + \end{minipage} + \hfill + \begin{minipage}[b]{0.25\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-color/hsv-diagram.pdf}\\ + \vspace{-1em} + \textit{\small HSV diagram} + \vspace{0.25em} + \end{minipage} + \begin{itemize} + \item An additional channel can exist for transparency: the \textbf{alpha channel}\\ + \textit{mostly relevant for composition, not for final display} + \item Color coordinates can be \textbf{converted} between colorspaces and color models\\ + \textit{using translation formulas and associated constants} + \end{itemize} +\end{frame} + +\begin{frame}{Colorspaces and channels (illustrated with YUV)} + \begin{minipage}[t]{0.25\textwidth} + \centering + \includegraphics[height=6em]{slides/graphics-theory-color/barn.jpg}\\ + \textit{\small Original picture} + \end{minipage} + \hfill + \begin{minipage}[t]{0.7\textwidth} + \centering + \includegraphics[height=6em]{slides/graphics-theory-color/barn-yuv.jpg}\\ + \textit{\small Decomposition in Y, U and V channels} + \end{minipage} + + \vspace{1em} + + \begin{minipage}[t]{0.4\textwidth} + \small + \begin{equation*} + \begin{cases} + R = Y + 1.140 \times V\\ + G = Y - 0.395 \times U - 0.581 \times V\\ + B = Y + 2.032 \times U + \end{cases} + \end{equation*} + \end{minipage} + \hfill + \begin{minipage}[t]{0.55\textwidth} + \small + \begin{equation*} + \begin{cases} + Y = + 0.299 \times R + 0.587 \times G + 0.114 \times B\\ + U = - 0.147 \times R - 0.289 \times G + 0.436 \times B\\ + V = + 0.615 \times R - 0.515 \times G - 0.100 \times B + \end{cases} + \end{equation*} + \end{minipage} + + \begin{center} + \textit{\small Translation between BT.601 YUV and sRGB colorspaces} + \end{center} +\end{frame} diff --git a/slides/graphics-theory/hsv-diagram.svg b/slides/graphics-theory-color/hsv-diagram.svg similarity index 100% rename from slides/graphics-theory/hsv-diagram.svg rename to slides/graphics-theory-color/hsv-diagram.svg diff --git a/slides/graphics-theory/pair-of-merops-16-colors-range.jpg b/slides/graphics-theory-color/pair-of-merops-16-colors-range.jpg similarity index 100% rename from slides/graphics-theory/pair-of-merops-16-colors-range.jpg rename to slides/graphics-theory-color/pair-of-merops-16-colors-range.jpg diff --git a/slides/graphics-theory/pair-of-merops-16-colors.jpg b/slides/graphics-theory-color/pair-of-merops-16-colors.jpg similarity index 100% rename from slides/graphics-theory/pair-of-merops-16-colors.jpg rename to slides/graphics-theory-color/pair-of-merops-16-colors.jpg diff --git a/slides/graphics-theory/pair-of-merops.jpg b/slides/graphics-theory-color/pair-of-merops.jpg similarity index 100% rename from slides/graphics-theory/pair-of-merops.jpg rename to slides/graphics-theory-color/pair-of-merops.jpg diff --git a/slides/graphics-theory/rgb-cube.svg b/slides/graphics-theory-color/rgb-cube.svg similarity index 100% rename from slides/graphics-theory/rgb-cube.svg rename to slides/graphics-theory-color/rgb-cube.svg diff --git a/slides/graphics-theory/aliasing-1d.svg b/slides/graphics-theory-image/aliasing-1d.svg similarity index 100% rename from slides/graphics-theory/aliasing-1d.svg rename to slides/graphics-theory-image/aliasing-1d.svg diff --git a/slides/graphics-theory/bricks-alias.jpg b/slides/graphics-theory-image/bricks-alias.jpg similarity index 100% rename from slides/graphics-theory/bricks-alias.jpg rename to slides/graphics-theory-image/bricks-alias.jpg diff --git a/slides/graphics-theory/bricks-fft-rotation.jpg b/slides/graphics-theory-image/bricks-fft-rotation.jpg similarity index 100% rename from slides/graphics-theory/bricks-fft-rotation.jpg rename to slides/graphics-theory-image/bricks-fft-rotation.jpg diff --git a/slides/graphics-theory/bricks-fft.jpg b/slides/graphics-theory-image/bricks-fft.jpg similarity index 100% rename from slides/graphics-theory/bricks-fft.jpg rename to slides/graphics-theory-image/bricks-fft.jpg diff --git a/slides/graphics-theory/bricks-rich.jpg b/slides/graphics-theory-image/bricks-rich.jpg similarity index 100% rename from slides/graphics-theory/bricks-rich.jpg rename to slides/graphics-theory-image/bricks-rich.jpg diff --git a/slides/graphics-theory/bricks-rotation.jpg b/slides/graphics-theory-image/bricks-rotation.jpg similarity index 100% rename from slides/graphics-theory/bricks-rotation.jpg rename to slides/graphics-theory-image/bricks-rotation.jpg diff --git a/slides/graphics-theory/bricks.jpg b/slides/graphics-theory-image/bricks.jpg similarity index 100% rename from slides/graphics-theory/bricks.jpg rename to slides/graphics-theory-image/bricks.jpg diff --git a/slides/graphics-theory/ccc-rocket.jpg b/slides/graphics-theory-image/ccc-rocket.jpg similarity index 100% rename from slides/graphics-theory/ccc-rocket.jpg rename to slides/graphics-theory-image/ccc-rocket.jpg diff --git a/slides/graphics-theory/dot-matrix-display.jpg b/slides/graphics-theory-image/dot-matrix-display.jpg similarity index 100% rename from slides/graphics-theory/dot-matrix-display.jpg rename to slides/graphics-theory-image/dot-matrix-display.jpg diff --git a/slides/graphics-theory/first-photo.jpg b/slides/graphics-theory-image/first-photo.jpg similarity index 100% rename from slides/graphics-theory/first-photo.jpg rename to slides/graphics-theory-image/first-photo.jpg diff --git a/slides/graphics-theory-image/graphics-theory-image.tex b/slides/graphics-theory-image/graphics-theory-image.tex new file mode 100644 index 0000000000..8e6aeb2b3a --- /dev/null +++ b/slides/graphics-theory-image/graphics-theory-image.tex @@ -0,0 +1,164 @@ +\section{Base Theory and Concepts About Graphics} + +\subsection{Image Representation} + +\begin{frame}{Light, pixels and pictures} + \begin{minipage}{0.75\textwidth} + \begin{itemize} + \item Pictures are {\bf representations of light emissions} + \item Analog representations are spatially {\bf continuous}: + \begin{itemize} + \item With an {\bf infinite} number of elements + \item Example: photosensitive paper + \end{itemize} + \item Digital representations are spatially {\bf quantified}: + \begin{itemize} + \item With a {\bf finite} number of elements + \item Example: discrete LED-based display + \end{itemize} + \item Producing a digital representation is called \textbf{quantization} + \begin{itemize} + \item Reduction of information from the continuous world + \item Quantization requires a {\bf base element unit} or quantum + \item This quantum is called picture element or {\bf pixel} + \item Quantization is also called \textbf{sampling} in this context + \end{itemize} + \end{itemize} + \end{minipage}% + \begin{minipage}{0.25\textwidth} + \includegraphics[width=\textwidth]{slides/graphics-theory-image/ccc-rocket.jpg} + \end{minipage} +\end{frame} + +\begin{frame}{Light, pixels and pictures} + \begin{itemize} + \item \textbf{Pictures} are bi-dimensional ordered ensembles of pixels (also called \textbf{frames}): + \begin{itemize} + \item Frames have \textbf{dimensions}: \textbf{width} (horizontal) and \textbf{height} (vertical) + \item The aspect ratio is the \textbf{width:height} ratio (e.g. 16:9, 4:3) + \item Pixels are located with a \textbf{position}: \textbf{(x,y)} + \item The dimension and position unit is the number of pixels + \end{itemize} + \item Quantified pixels have a \textbf{spatial density} or spatial resolution:\\ + \begin{itemize} + \item \textit{How many pixels are found in \(n\) inches?} + \item The usual pixel resolution unit is the \textbf{dot per inch} (DPI) + \item Vertical and horizontal spatial densities are usually not distinguished\\ + \textit{pixels are assumed to have a square shape most of the time} + \end{itemize} + \end{itemize} + +\end{frame} + +\begin{frame}{Light, pixels and pictures (illustrated)} + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-image/first-photo.jpg} + \textit{\small View from the Window at Le Gras picture} + \end{minipage} + \hfill + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-image/dot-matrix-display.jpg} + \textit{\small A monochrome dot-matrix display} + \end{minipage} + + \vspace{1em} + + \begin{minipage}[b]{0.45\textwidth} + \centering + \textbf{Analog representation},\\ on a metal plate + \end{minipage} + \hfill + \begin{minipage}[b]{0.45\textwidth} + \centering + \textbf{Digital representation},\\ on a LED display + \end{minipage} +\end{frame} + +\begin{frame}{Sampling and frequency domain} + \begin{itemize} + \item Pixels are quantized/sampled representations of a \textbf{spatial domain} + \item The initial (continuous) domain has a corresponding \textbf{frequency spectrum}\\ + \textit{high frequencies provide details in pictures} + \item A 2D \textbf{Fourier transform} translates from spatial \((x,y)\) to frequency \((u,v)\) domain +\[ +F(u,v) = \int_{-\infty}^{+\infty} \int_{-\infty}^{+\infty} f(x,y)e^{-j2\pi(ux+vy)}dxdy +\] + \item The transform decomposes the domain in \textbf{periodic patterns} + \item Adapted for discrete signals as \textbf{Discrete Fourier Transform} + \item Implemented with optimized algorithms as \textbf{Fast Fourier Transform} (FFT) + \item \textbf{Frequency domain analysis} is very useful for signal processing\\ + \textit{used at the roots of image compression} + \end{itemize} +\end{frame} + +\begin{frame}{Sampling and frequency domain (illustrated)} + \begin{center} + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-image/bricks.jpg}\\ + \textit{\small A wall of bricks represented in the spatial domain} + \end{minipage} + \hfill + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-image/bricks-fft.jpg}\\ + \textit{\small The wall of bricks represented in the frequency domain} + \end{minipage} + + \end{center} +\end{frame} + +\begin{frame}{Sampling and frequency domain (illustrated)} + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-image/bricks-rotation.jpg}\\ + \textit{\small A wall of bricks rotated \(45^{\circ}\) represented in the spatial domain} + \end{minipage} + \hfill + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-image/bricks-fft-rotation.jpg}\\ + \textit{\small The wall of bricks rotated \(45^{\circ}\) represented in the frequency domain} + \end{minipage} +\end{frame} + +\begin{frame}{Sampling and aliasing} + \begin{itemize} + \item The spatial domain is quantized with a bi-dimensional \textbf{sampling resolution} + \item Matching \textbf{sampling frequencies} exist, for each axis: \((u_s,v_s)\) + \item They \textbf{limit the frequencies} that can be sampled from the initial domain + \item The \textbf{Shannon-Nyquist theorem} provides a sufficient condition for \((u_s,v_s)\): +\[ +u_s > 2 \times u_{max}, ~v_s > 2 \times v_{max} +\] + \item Frequencies such that \(u \geq \frac{u_s}{2}\) and \(v \geq \frac{v_s}{2}\) are \textbf{not correctly sampled} + \item Can result in \textbf{incorrect frequencies} being introduced: \textbf{Moiré pattern} in 2D + \end{itemize} + + \begin{center} + \includegraphics[width=0.35\textwidth]{slides/graphics-theory-image/aliasing-1d.pdf}\\ + \textit{\small Aliasing example in a uni-dimensional domain} + \end{center} +\end{frame} + +\begin{frame}{Sampling and aliasing (illustrated)} + \begin{minipage}[b]{0.29\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-image/bricks-rich.jpg}\\ + \textit{\small Another wall of bricks} + \end{minipage} + \hfill + \begin{minipage}[b]{0.29\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-image/bricks-alias.jpg}\\ + \textit{\small Moiré on the bricks} + \end{minipage} + \hfill + \begin{minipage}[b]{0.29\textwidth} + \centering + \includegraphics[width=\textwidth]{slides/graphics-theory-image/moire.jpg}\\ + \textit{\small Moiré on the garage door} + \end{minipage} +\end{frame} diff --git a/slides/graphics-theory/moire.jpg b/slides/graphics-theory-image/moire.jpg similarity index 100% rename from slides/graphics-theory/moire.jpg rename to slides/graphics-theory-image/moire.jpg diff --git a/slides/graphics-theory-layout-formats/graphics-theory-layout-formats.tex b/slides/graphics-theory-layout-formats/graphics-theory-layout-formats.tex new file mode 100644 index 0000000000..5c41d2ed7b --- /dev/null +++ b/slides/graphics-theory-layout-formats/graphics-theory-layout-formats.tex @@ -0,0 +1,111 @@ +\subsection{Layout and Formats} + +\begin{frame}{Frame size and chroma sub-sampling} + \begin{itemize} + \item Digital pictures easily take up a lot of space (more so for videos) + \item The minimal size for a picture depends on: + \begin{itemize} + \item Dimensions (\(width\) and \(height\)) + \item Number of bits per pixel (\(bpp\)): color (and alpha) depth and dead bits + \item Roughly: \(width \times height \times bpp \div 8~bytes\) + \item For 12 Mpixels with 16 Mcolors and alpha: \(4000 \times 3000 \times 32 \div 8 = 45.8 MiB\) + \end{itemize} + \item The human visual system has specificities: + \begin{itemize} + \item High sensitivity to \textbf{luminosity} (luminance) + \item Low sensitivity to \textbf{colors} (chrominance) + \end{itemize} + \item The YUV color model offers the relevant channel separation + \item Sub-sampling can be applied to the chrominance channel\\ + \textit{less data (and precision) on colors to reduce size} + \end{itemize} +\end{frame} + +\begin{frame}{Frame size and chroma sub-sampling} + \begin{itemize} + \item Chrominance samples are used for multiple luminance samples + \item With specific vertical and horizontal ratios (usually integer) + \item Usually summarized using a three-part ratio: \(J:a:b\) + \end{itemize} + + \begin{center} + \includegraphics[width=0.55\textwidth]{slides/graphics-theory-layout-formats/yuv-sub-sampling.pdf} + + YUV 4:2:0 usual example: + {\small + \begin{equation*} + \begin{gathered} + bpp_Y = 8,~bpp_U = bpp_V = 8 \div 2 \div 2 = 2 \\ + \Rightarrow~ bpp = bpp_Y + bpp_U + bpp_V = 12~bits/pixel + \end{gathered} + \end{equation*} + } + \end{center} +\end{frame} + +\begin{frame}{Pixel data distribution in memory} + \begin{itemize} + \item Pixel data can be \textbf{distributed} in different ways in memory + \item Different ways to aggregate color components in \textbf{data planes} (memory chunks): + \begin{itemize} + \item \textbf{Packed}: Components are stored in the same data plane in memory + \item \textbf{Semi-planar} (YUV): Luma and chroma are stored in distinct data planes + \item \textbf{Planar}: Each component has its own data plane in memory + \end{itemize} + \item When multiple color components are grouped, \textbf{bit order} must be specified: + \begin{itemize} + \item Which component comes first in memory? + \item Affected by endianness when read by hardware! + \end{itemize} + \item \textbf{Scan order} must also be specified: + \begin{itemize} + \item How to calculate the address for position \((x,y)\) and back? + \item Raster order (most common) specifies: row-major, left-to-right, top-to-bottom + \end{itemize} + \end{itemize} + \begin{center} + \includegraphics[height=6em]{slides/graphics-theory-layout-formats/raster-order.pdf} + \end{center} +\end{frame} + +\begin{frame}[fragile]{Pixel formats, FourCC codes} + \begin{itemize} + \item Many meta-data elements are needed to fully describe how a picture is coded + \begin{itemize} + \item Some describe \textbf{picture-level attributes} (e.g. dimensions) + \item Some describe \textbf{pixel-level attributes} (e.g. colorspace, bpp) + \end{itemize} + \item Pixel-level attributes are grouped as a \textbf{pixel format} that defines: + \begin{itemize} + \item Color model in use + \item Number of bits per channel and per pixel (bpp) + \item Bit attribution and byte order + \item Per-channel sub-sampling ratios + \item Pixel data planes distribution in memory + \end{itemize} + \item Often represented as a 4-character code called \textbf{FourCC}\\ + \item Not really standardized and implementation-specific:\\ + \textit{DRM in Linux uses \code{XR24} for \ksym{DRM_FORMAT_XRGB8888}}.\\ + \textit{Not really standardized but widely used in various forms} + \item Scan order is specified separately with a \textbf{modifier}\\ + \textit{Assumed to be raster order if unspecified} + \end{itemize} +\end{frame} + +\begin{frame}{Level of detail of quantized pictures} + Depends on a number of factors, including: + \begin{itemize} + \item Spatial density (pixel resolution) + \item Quantized dimensions (picture width and height) + \item Colorspace limits (chromaticity diagram) + \item Color depth (number of bits per pixel) + \item Color resolution and range trade-off + \end{itemize}~ + + Generally speaking: + \begin{itemize} + \item Many factors are involved + \item The major bottleneck is not always obvious + \item Implementation choices do matter + \end{itemize} +\end{frame} diff --git a/slides/graphics-theory/raster-order.svg b/slides/graphics-theory-layout-formats/raster-order.svg similarity index 100% rename from slides/graphics-theory/raster-order.svg rename to slides/graphics-theory-layout-formats/raster-order.svg diff --git a/slides/graphics-theory/yuv-sub-sampling.html b/slides/graphics-theory-layout-formats/yuv-sub-sampling.html similarity index 100% rename from slides/graphics-theory/yuv-sub-sampling.html rename to slides/graphics-theory-layout-formats/yuv-sub-sampling.html diff --git a/slides/graphics-theory/yuv-sub-sampling.svg b/slides/graphics-theory-layout-formats/yuv-sub-sampling.svg similarity index 100% rename from slides/graphics-theory/yuv-sub-sampling.svg rename to slides/graphics-theory-layout-formats/yuv-sub-sampling.svg diff --git a/slides/graphics-theory/diamond-sub-pixel.png b/slides/graphics-theory-pixel-drawing/diamond-sub-pixel.png similarity index 100% rename from slides/graphics-theory/diamond-sub-pixel.png rename to slides/graphics-theory-pixel-drawing/diamond-sub-pixel.png diff --git a/slides/graphics-theory-pixel-drawing/graphics-theory-pixel-drawing.tex b/slides/graphics-theory-pixel-drawing/graphics-theory-pixel-drawing.tex new file mode 100644 index 0000000000..b746ab5ea8 --- /dev/null +++ b/slides/graphics-theory-pixel-drawing/graphics-theory-pixel-drawing.tex @@ -0,0 +1,244 @@ +\subsection{Pixel Drawing} + +\begin{frame}{Accessing pixel data} + \begin{itemize} + \item Information required to access pixel data in memory: + \begin{itemize} + \item Pixel format (also modifier if not linear/raster order) + \item Dimensions (and total size) + \item Pointer to the base buffer address + \end{itemize} + \item The size of each line is called \textbf{stride} or \textbf{pitch} + \begin{itemize} + \item Usually equals: \(stride = width \times bpp \div 8\) + \item Can contain an extra dead zone at the end + \item Also needs to be specified explicitly + \end{itemize} + \item CPU access is either byte or word-aligned + \begin{itemize} + \item Good fit for formats with \(bpp = 32\) (very common) + \item Good fit for formats with \(bpp = 8 \times n\) + \item Not always easy to manage otherwise + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{Iterating over pixel data} + \begin{itemize} + \item Selected format (slides and demos): \textbf{XRGB8888} + \begin{itemize} + \item \(bpp = 32 = 8 \times 4\), one byte per channel, one memory plane + \end{itemize} + \item Pixel data can be access by iterating nested variables: + \begin{minted}[fontsize=\small]{c} +for (y = 0; y < height; y++) + for (x = 0; x < width; x++) + data = base + y * stride + x * 4; + \end{minted} + \item Iterating over all pixels takes numerous CPU-cycles, tips: + \begin{itemize} + \item Incrementing the address instead of re-calculating it: + \begin{minted}[fontsize=\small]{c} +data = base; +for (y = 0; y < height; y++) + for (x = 0; x < width; x++) + data += 4; + data += stride - width * 4; + \end{minted} + \item Iterating in y-major is also better for cache hits + \item Beware: C pointer arithmetic uses type size as unit + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Concepts about rasterization} + \begin{itemize} + \item Rasterization is the process of drawing vector shapes as discrete pixels + \begin{itemize} + \item Vector shapes are defined with mathematical equations + \item Converted from a continuous space domain: \(\mathbb{R}^2\) to a discrete domain + \item Results in a discretization/quantization error due to integer rounding + \item Also subject to aliasing-related trouble + \end{itemize} + \end{itemize} + \vspace{1em} + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[width=0.8\textwidth]{slides/graphics-theory-pixel-drawing/raster-source.jpg} + \textit{\small Continuous source representation} + \end{minipage} + \hfill + \begin{minipage}[b]{0.45\textwidth} + \centering + \includegraphics[width=0.8\textwidth]{slides/graphics-theory-pixel-drawing/raster-destination.jpg} + \textit{\small Rasterized destination representation} + \end{minipage} +\end{frame} + +\begin{frame}{Rectangle drawing} + \begin{itemize} + \item A rectangle is defined with two boundaries per axis: +\[ +x_{min} \leq x \leq x_{max},~ y_{min} \leq y \leq y_{max} +\] + \item Another expression involves a (top-left) start point and size: +\[ +x_{start} \leq x \leq x_{start} + x_{size},~ y_{start} \leq y \leq y_{start} + y_{size} +\] + \item Allows iterating in the rectangle area only + \end{itemize} +\end{frame} + +\begin{frame}{Linear gradient drawing} + \begin{itemize} + \item Same base as drawing a rectangle + \item A linear gradient involves interpolation between two colors + \item Following one of the two axes as major + \item Involves weighting the two colors depending on the advancement + \item Equations in x-axis major: + \end{itemize} +\begin{align*} +r &= r_{start} + (r_{stop} - r_{start}) \frac{x - x_{start}}{x_{size}}\\ +g &= g_{start} + (g_{stop} - g_{start}) \frac{x - x_{start}}{x_{size}}\\ +b &= b_{start} + (b_{stop} - b_{start}) \frac{x - x_{start}}{x_{size}} +\end{align*} +\end{frame} + +\begin{frame}{Disk drawing} + \begin{itemize} + \item A disk is delimited with a radius test (\((0,0)\)-centered): +\[ +\sqrt{x^2 + y^2} \leq radius +\] + \item Given a center point \((x_c,y_c)\): +\[ +\sqrt{(x - x_c)^2 + (y - y_c)^2} \leq radius +\] + \item Requires iterating in: +\[ +x_c - radius \leq x \leq x_c + radius,~ y_c - radius \leq y \leq y_c + radius +\] + \end{itemize} +\end{frame} + +\begin{frame}{Circular gradient drawing} + \begin{itemize} + \item Same base as drawing a disk + \item Interpolation between two colors using the radius as major: + \end{itemize} +\begin{align*} +d &= \sqrt{(x - x_c)^2 + (y - y_c)^2}\\ +r &= r_{start} + (r_{stop} - r_{start}) \frac{d}{radius}\\ +g &= g_{start} + (g_{stop} - g_{start}) \frac{d}{radius}\\ +b &= b_{start} + (b_{stop} - b_{start}) \frac{d}{radius} +\end{align*} +\end{frame} + +\begin{frame}{Line drawing} + \begin{itemize} + \item A line is defined as an affine function: +\[ +y(x) = a \times x + b +\] + \item Given start and end points, iterating in x-major: +\begin{gather*} +y(x) = y_{start} + (x - x_{start}) \times \frac{y_{end} - y_{start}}{x_{end} - x_{start}}\\ +x_{start} \leq x \leq x_{stop} +\end{gather*} + \item Axis major depends on the largest per-axis span (\(axis_{stop} - axis_{start}\)) + \begin{itemize} + \item Iterating with smaller-span axis-major results in visual holes + \item Iterating on both axes provides coherent results + \end{itemize} + \item Algorithms producing better-looking results: + \begin{itemize} + \item \textbf{Bresenham's} line algorithm, optimized for implementation + \item \textbf{Xiaolin Wu's} line algorithm, with sub-pixel rendering + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Line and shape aliasing, sub-pixel drawing} + \begin{itemize} + \item Lines are often subject to aliasing: + \begin{itemize} + \item Sampled from the continuous domain with pixel sampling resolution + \item Selecting the best axis gives a better resolution + \item Limited display resolutions still make them look pixelated + \end{itemize} + \item Any geometric shape is affected, especially fonts + \item Sub-pixel rendering is used to provide anti-aliased results: + \begin{itemize} + \item Surrounding pixels are given an intermediate value + \item Specific algorithms perform sub-pixel drawing + \item Also obtained with high-resolution rendering and anti-aliased downscaling + \end{itemize} + \end{itemize} + + \begin{center} + \includegraphics[height=4em]{slides/graphics-theory-pixel-drawing/diamond-sub-pixel.png} + \includegraphics[height=4em]{slides/graphics-theory-pixel-drawing/line-sub-pixel.png}\\ + \textit{\small Shapes rendered without and with sub-pixel anti-aliasing} + \end{center} +\end{frame} + +\begin{frame}{Line and shape aliasing, sub-pixel drawing (illustrated)} + \begin{center} + \includegraphics[width=0.4\textwidth]{slides/graphics-theory-pixel-drawing/line-sub-pixel-rendering.pdf}\\ + \textit{\small Pixel drawing versus sub-pixel drawing (x-axis major)} + \end{center} +\end{frame} + +\begin{frame}{Circles and polar coordinates} + \begin{itemize} + \item Circles centered on \((x_c,y_c)\) are defined (Pythagoras theorem) as: +\[ +(x - x_c)^2+(y - y_c)^2 = radius^2 +\] + \end{itemize} + \begin{minipage}{0.70\textwidth} + \begin{itemize} + \item Which is always verified with the expression: +\begin{gather*} +x = x_c + radius \times cos(\phi)\\ +y = y_c + radius \times sin(\phi) +\end{gather*} + \item Corresponds to a translation in polar coordinates + \begin{itemize} + \item From a \((x,y)\) base to \((r,\phi)\) + \end{itemize} + \item Iteration on \(\phi\) with a specific range: \(\phi \in [0;2\pi]\) + \end{itemize} + \end{minipage} + \hfill + \begin{minipage}{0.25\textwidth} + \centering + \includegraphics[width=0.8\textwidth]{slides/graphics-theory-pixel-drawing/polar-coordinates.pdf} + \end{minipage} +\end{frame} + +\begin{frame}{Parametric curves} + \begin{itemize} + \item Parametric curves generalize the idea of using independent parameters + \item Each curve has defining equations and ranges for parameters + \begin{itemize} + \item Equations allow calculating \((x,y)\) (or \((r,\phi)\)) + \end{itemize} + \item Drawing is achieved by iterating over parameter values + \begin{itemize} + \item Sampling is done on the range to get a finite number of points + \item X/Y coordinates are calculated for each point + \item Line interpolation is used between consecutive points + \end{itemize} + \item Ellipse: \(\phi \in [0;2\pi]\) +\begin{gather*} +x = x_c + a \times cos(\phi)\\ +y = y_c + b \times sin(\phi) +\end{gather*} + \item Many more parametric curves exist: + \begin{itemize} + \item Cycloid, Epicycloid, Epitrochoid, Hypocycloid, Hypotrochoid (spirograph) + \item Lissajous Curve, Rose curve, Butterfly curve + \end{itemize} + \end{itemize} +\end{frame} diff --git a/slides/graphics-theory/line-sub-pixel-rendering.svg b/slides/graphics-theory-pixel-drawing/line-sub-pixel-rendering.svg similarity index 100% rename from slides/graphics-theory/line-sub-pixel-rendering.svg rename to slides/graphics-theory-pixel-drawing/line-sub-pixel-rendering.svg diff --git a/slides/graphics-theory/line-sub-pixel.png b/slides/graphics-theory-pixel-drawing/line-sub-pixel.png similarity index 100% rename from slides/graphics-theory/line-sub-pixel.png rename to slides/graphics-theory-pixel-drawing/line-sub-pixel.png diff --git a/slides/graphics-theory/polar-coordinates.svg b/slides/graphics-theory-pixel-drawing/polar-coordinates.svg similarity index 100% rename from slides/graphics-theory/polar-coordinates.svg rename to slides/graphics-theory-pixel-drawing/polar-coordinates.svg diff --git a/slides/graphics-theory/raster-destination.jpg b/slides/graphics-theory-pixel-drawing/raster-destination.jpg similarity index 100% rename from slides/graphics-theory/raster-destination.jpg rename to slides/graphics-theory-pixel-drawing/raster-destination.jpg diff --git a/slides/graphics-theory/raster-source.jpg b/slides/graphics-theory-pixel-drawing/raster-source.jpg similarity index 100% rename from slides/graphics-theory/raster-source.jpg rename to slides/graphics-theory-pixel-drawing/raster-source.jpg diff --git a/slides/graphics-theory/box-blur.svg b/slides/graphics-theory-pixel-operations/box-blur.svg similarity index 100% rename from slides/graphics-theory/box-blur.svg rename to slides/graphics-theory-pixel-operations/box-blur.svg diff --git a/slides/graphics-theory/cat-depth-dither.png b/slides/graphics-theory-pixel-operations/cat-depth-dither.png similarity index 100% rename from slides/graphics-theory/cat-depth-dither.png rename to slides/graphics-theory-pixel-operations/cat-depth-dither.png diff --git a/slides/graphics-theory/cat-depth-initial.png b/slides/graphics-theory-pixel-operations/cat-depth-initial.png similarity index 100% rename from slides/graphics-theory/cat-depth-initial.png rename to slides/graphics-theory-pixel-operations/cat-depth-initial.png diff --git a/slides/graphics-theory/cat-depth-low.png b/slides/graphics-theory-pixel-operations/cat-depth-low.png similarity index 100% rename from slides/graphics-theory/cat-depth-low.png rename to slides/graphics-theory-pixel-operations/cat-depth-low.png diff --git a/slides/graphics-theory/chroma-key-blender.jpg b/slides/graphics-theory-pixel-operations/chroma-key-blender.jpg similarity index 100% rename from slides/graphics-theory/chroma-key-blender.jpg rename to slides/graphics-theory-pixel-operations/chroma-key-blender.jpg diff --git a/slides/graphics-theory/gaussian-blur.svg b/slides/graphics-theory-pixel-operations/gaussian-blur.svg similarity index 100% rename from slides/graphics-theory/gaussian-blur.svg rename to slides/graphics-theory-pixel-operations/gaussian-blur.svg diff --git a/slides/graphics-theory-pixel-operations/graphics-theory-pixel-operations.tex b/slides/graphics-theory-pixel-operations/graphics-theory-pixel-operations.tex new file mode 100644 index 0000000000..3e8e829d37 --- /dev/null +++ b/slides/graphics-theory-pixel-operations/graphics-theory-pixel-operations.tex @@ -0,0 +1,239 @@ +\subsection{Pixel Operations} + +\begin{frame}{Region copy} + \begin{itemize} + \item Requirements for all pixel operations: + \begin{itemize} + \item The source and destination must use the same pixel format + \item Must be converted before any operation otherwise + \end{itemize} + \item Most basic operation on pixels: copying a region + \begin{itemize} + \item Also known as bit blit or BITBLT (in reference to the hardware opcode) + \end{itemize} + \item Implemented as a line-per-line copy (maximum memory-contiguous block) + \item Overwriting destination memory with source memory + \begin{itemize} + \item Copies within the same image are not always safe! + \item Destination must not overlap source + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Alpha blending} + \begin{itemize} + \item Compositing multiple alpha-enabled pixel sources into a single result + \begin{itemize} + \item Simplest case: aggregating sources with z-ordered stacking + \item Equation for A over B (with \(\alpha\) the alpha and \(C\) the color component value): + \end{itemize} +\[ +C_o = \frac{C_a \alpha_a + C_b \alpha_b \left(1 - \alpha_a\right)}{\alpha_a + \alpha_b \left(1 - \alpha_a\right)} +\] + \item With alpha available, many more operations become possible + \begin{itemize} + \item Shapes can be used as masks, with logic operators + \item Formalized by Porter and Duff in 1984 + \end{itemize} + \end{itemize} + \begin{center} + \includegraphics[width=0.5\textwidth]{slides/graphics-theory-pixel-operations/porter-duff-compositing.pdf} + \end{center} +\end{frame} + +\begin{frame}{Color-keying} + \begin{itemize} + \item Color-keying (or chroma-keying): replacing given colors with alpha + \item Specified with color ranges (3 RGB ranges) + \item Pixels either within or outside of the range are made transparent + \item Used in conjunction with alpha blending + \item The famous video green-screen method uses color-keying + \end{itemize} + \begin{center} + \includegraphics[width=0.5\textwidth]{slides/graphics-theory-pixel-operations/chroma-key-blender.jpg}\\ + \textit{\small Color-keying implemented in Blender} + \end{center} +\end{frame} + +\begin{frame}{Scaling and interpolation} + \begin{itemize} + \item Scaling is a resizing operation on a pixel picture + \begin{itemize} + \item Involves a scaling factor (integer or real) + \item Values are resampled with a new resolution + \item Requires reconstructing the original signal + \end{itemize} + \item Implemented with some form of interpolation: + \begin{itemize} + \item nearest-neighbor: uses the nearest pixel value from the source +\[ +x_{source} = x_{destination} \div scale +\] + \item bilinear interpolation: sub-pixel linear weighting of neighbor colors + \item bicubic interpolation: smooth spline sub-pixel fitting with neighbor colors + \end{itemize} + \item Sub-pixel methods provide better visual results + \item Down-sampling: + \begin{itemize} + \item Reduces the maximum image frequency + \item Can cause aliasing: high frequencies need to be removed + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame}{Linear filtering and convolution} + \begin{itemize} + \item Filtering is a transformation of each pixel based on its neighbors + \item The pixel output value is a linear sum of weighted neighboring input values +\[ +o_{x,y} = \alpha_0 i_{x,y} + \alpha_1 i_{x-1,y} + \alpha_2 i_{x+1,y} + ... +\] + \item Weighting coefficients are represented in a 2D matrix: the \textbf{filter kernel} + \begin{itemize} + \item Comes with \(2n + 1\) columns and \(2m + 1\) rows + \item The coefficients are applied to each input pixel and its neighbors + \item The element at the kernel center weights the current input pixel + \end{itemize} + \item Corresponds to a \textbf{convolution} operation between pixels and the filter kernel + \item High computational cost (optimizations are implemented) + \item Allows many applications for 2D signal processing + \end{itemize} +\end{frame} + +\begin{frame}{Linear filtering and convolution (illustrated)} + \begin{center} + \includegraphics[width=0.7\textwidth]{slides/graphics-theory-pixel-operations/linear-filtering.pdf}\\ + \textit{\small Linear filtering in application} +\[ +g(x,y)= \omega *f(x,y)=\sum_{s=-n}^n{\sum_{t=-m}^m{ \omega (s,t)f(x-s,y-t)}} +\] + \textit{\small Bi-dimensional convolution operation on \(f\) with the \(\omega\) kernel} + \end{center} +\end{frame} + +\begin{frame}{Blur filters} + \begin{itemize} + \item Blurring is a common example of linear filtering + \item Corresponds to a low-pass filter + \begin{itemize} + \item Removes high frequencies from the picture (details) + \item Good fit for pre-scaling anti-aliasing + \end{itemize} +\begin{minipage}[b]{0.45\textwidth} + \item Implemented with different algorithms: + \vspace{-1.5em} + \begin{itemize} + \item Box blur: rough but easy to optimize +\[ +\frac{1}{9} +\left[ +\begin{matrix} +1 & 1 & 1 \\ +1 & 1 & 1 \\ +1 & 1 & 1 +\end{matrix} +\right] +\] + \end{itemize} +\end{minipage} +\begin{minipage}[b]{0.45\textwidth} + \begin{itemize} + \item Gaussian blur: reference smooth blur +\[ +\frac{1}{16} +\left[ +\begin{matrix} +1 & 2 & 1 \\ +2 & 4 & 2 \\ +1 & 2 & 1 +\end{matrix} +\right] +\] + \end{itemize} + \end{minipage} + \item A repeated box blur converges towards a Gaussian one (central-limit theorem) + \end{itemize} +\end{frame} + +\begin{frame}{Dithering} + \begin{itemize} + \item Reducing the color depth can lead to visually-unpleasant results + \begin{itemize} + \item Corresponds to color-space down-sampling + \item Increases color quantization error + \end{itemize} + \item Floyd–Steinberg dithering is a method for improving quality with low depth + \item Quantization error is evaluated and distributed to neighboring pixels + \item Used in hardware display engines and the GIF file format + \end{itemize}~\\ + + \begin{minipage}[t]{0.25\textwidth} + \centering + \includegraphics[width=0.9\textwidth]{slides/graphics-theory-pixel-operations/cat-depth-initial.png}\\ + \textit{\small Cat at initial depth} + \end{minipage} + \hfill + \begin{minipage}[t]{0.25\textwidth} + \centering + \includegraphics[width=0.9\textwidth]{slides/graphics-theory-pixel-operations/cat-depth-low.png}\\ + \textit{\small Cat at reduced depth without dithering} + \end{minipage} + \hfill + \begin{minipage}[t]{0.25\textwidth} + \centering + \includegraphics[width=0.9\textwidth]{slides/graphics-theory-pixel-operations/cat-depth-dither.png}\\ + \textit{\small Cat at reduced depth with dithering} + \end{minipage} +\end{frame} + +\begin{frame}{Graphics theory online references} + \small + \begin{itemize} + \item Wikipedia (\url{https://en.wikipedia.org/}): + \begin{itemize} + \item \href{https://en.wikipedia.org/wiki/Color_model}{Color model} + \item \href{https://en.wikipedia.org/wiki/Color_depth}{Color depth} + \item \href{https://en.wikipedia.org/wiki/YCbCr}{YCbCr} + \item \href{https://en.wikipedia.org/wiki/Chroma_subsampling}{Chroma subsampling} + \item \href{https://en.wikipedia.org/wiki/Nyquist–Shannon_sampling_theorem}{Nyquist–Shannon sampling theorem} + \item \href{https://en.wikipedia.org/wiki/Spatial_anti-aliasing}{Spatial anti-aliasing} + \item \href{https://en.wikipedia.org/wiki/Aliasing}{Aliasing} + \item \href{https://en.wikipedia.org/wiki/Line_drawing_algorithm}{Line drawing algorithm} + \item \href{https://en.wikipedia.org/wiki/Parametric_equation}{Parametric equation} + \item \href{https://en.wikipedia.org/wiki/Alpha_compositing}{Alpha compositing} + \item \href{https://en.wikipedia.org/wiki/Image_scaling}{Image scaling} + \item \href{https://en.wikipedia.org/wiki/Kernel_(image_processing)}{Kernel (image processing)} + \end{itemize} + \item \url{http://ssp.impulsetrain.com/porterduff.html} + \item \url{https://magcius.github.io/xplain/article/regions.html} + \item \url{https://magcius.github.io/xplain/article/rast1.html} + \end{itemize} +\end{frame} + +\begin{frame}{Graphics theory illustrations attributions} + \small + \begin{itemize} + \item \href{https://commons.wikimedia.org/wiki/File:2017-12-28_Leipzig,_34c3,_Fairy_Dust_(freddy2001).jpg}{34C3 Fairy Dust: Freddy2001, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:View_from_the_Window_at_Le_Gras,_Joseph_Nic\%C3\%A9phore_Ni\%C3\%A9pce.jpg}{Point de vue du Gras: Joseph Nicéphore Niépce, public domain} + \item \href{https://commons.wikimedia.org/wiki/File:Pinball_Dot_Matrix_Display_-_Demolition_Man.JPG}{Pinball Dot Matrix Display: ElHeineken, CC BY 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:Soderledskyrkan_brick_wall.jpg}{Soderledskyrkan brick wall: Xauxa, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:AliasingSines.svg}{Aliasing Sines: Moxfyre, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:Moire_pattern_of_bricks.jpg}{Moiré pattern of bricks: Colin M.L. Burnett, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:Moire_pattern_of_bricks_small.jpg}{Moiré Pattern at Gardham Gap: Roger Gilbertson, CC BY-SA 2.0} + \item \href{https://commons.wikimedia.org/wiki/File:RGBCube_a.svg}{RGB cube: Datumizer, CC BY-SA 4.0} + \item \href{https://commons.wikimedia.org/wiki/File:Pair_of_Merops_apiaster_feeding.jpg}{Pair of Merops apiaster feeding: Pierre Dalous, CC BY-SA 3.0} + \end{itemize} +\end{frame} + +\begin{frame}{Graphics theory illustrations attributions} + \small + \begin{itemize} + \item \href{https://commons.wikimedia.org/wiki/File:Hsl-hsv_models_b.svg}{Hsl-hsv models: Datumizer, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:Barns_grand_tetons.jpg}{Barns grand tetons: Jon Sullivan, public domain} + \item \href{https://commons.wikimedia.org/wiki/File:Top-left_triangle_rasterization_rule.gif}{Top-left triangle rasterization rule: Drummyfish, CC0 1.0} + \item \href{https://commons.wikimedia.org/wiki/File:Line_scan-conversion.svg}{Line scan-conversion: Phrood, CC BY-SA 3.0} + \item \href{https://commons.wikimedia.org/wiki/File:Alpha_compositing.svg}{Alpha compositing: Prometeusm, Wereon, public domain} + \item \href{https://commons.wikimedia.org/wiki/File:Blender3D_com_key_chroma.jpg}{Blender3D com key chroma: Toni Grappa, Blender Foundation, CC BY-2.5} + \item \href{https://en.wikipedia.org/wiki/File:Dithering_example_dithered_web_palette.png}{Dithering example: Jamelan, CC BY-SA 3.0} + \end{itemize} +\end{frame} diff --git a/slides/graphics-theory/interpolation.svg b/slides/graphics-theory-pixel-operations/interpolation.svg similarity index 100% rename from slides/graphics-theory/interpolation.svg rename to slides/graphics-theory-pixel-operations/interpolation.svg diff --git a/slides/graphics-theory/linear-filtering.svg b/slides/graphics-theory-pixel-operations/linear-filtering.svg similarity index 100% rename from slides/graphics-theory/linear-filtering.svg rename to slides/graphics-theory-pixel-operations/linear-filtering.svg diff --git a/slides/graphics-theory/porter-duff-compositing.svg b/slides/graphics-theory-pixel-operations/porter-duff-compositing.svg similarity index 100% rename from slides/graphics-theory/porter-duff-compositing.svg rename to slides/graphics-theory-pixel-operations/porter-duff-compositing.svg diff --git a/slides/graphics-theory/graphics-theory.tex b/slides/graphics-theory/graphics-theory.tex deleted file mode 100644 index 83bb269e21..0000000000 --- a/slides/graphics-theory/graphics-theory.tex +++ /dev/null @@ -1,963 +0,0 @@ -\section{Base Theory and Concepts About Graphics} - -\subsection{Image Representation} - -\begin{frame}{Light, pixels and pictures} - \begin{minipage}{0.75\textwidth} - \begin{itemize} - \item Pictures are {\bf representations of light emissions} - \item Analog representations are spatially {\bf continuous}: - \begin{itemize} - \item With an {\bf infinite} number of elements - \item Example: photosensitive paper - \end{itemize} - \item Digital representations are spatially {\bf quantified}: - \begin{itemize} - \item With a {\bf finite} number of elements - \item Example: discrete LED-based display - \end{itemize} - \item Producing a digital representation is called \textbf{quantization} - \begin{itemize} - \item Reduction of information from the continuous world - \item Quantization requires a {\bf base element unit} or quantum - \item This quantum is called picture element or {\bf pixel} - \item Quantization is also called \textbf{sampling} in this context - \end{itemize} - \end{itemize} - \end{minipage}% - \begin{minipage}{0.25\textwidth} - \includegraphics[width=\textwidth]{slides/graphics-theory/ccc-rocket.jpg} - \end{minipage} -\end{frame} - -\begin{frame}{Light, pixels and pictures} - \begin{itemize} - \item \textbf{Pictures} are bi-dimensional ordered ensembles of pixels (also called \textbf{frames}): - \begin{itemize} - \item Frames have \textbf{dimensions}: \textbf{width} (horizontal) and \textbf{height} (vertical) - \item The aspect ratio is the \textbf{width:height} ratio (e.g. 16:9, 4:3) - \item Pixels are located with a \textbf{position}: \textbf{(x,y)} - \item The dimension and position unit is the number of pixels - \end{itemize} - \item Quantified pixels have a \textbf{spatial density} or spatial resolution:\\ - \begin{itemize} - \item \textit{How many pixels are found in \(n\) inches?} - \item The usual pixel resolution unit is the \textbf{dot per inch} (DPI) - \item Vertical and horizontal spatial densities are usually not distinguished\\ - \textit{pixels are assumed to have a square shape most of the time} - \end{itemize} - \end{itemize} - -\end{frame} - -\begin{frame}{Light, pixels and pictures (illustrated)} - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/first-photo.jpg} - \textit{\small View from the Window at Le Gras picture} - \end{minipage} - \hfill - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/dot-matrix-display.jpg} - \textit{\small A monochrome dot-matrix display} - \end{minipage} - - \vspace{1em} - - \begin{minipage}[b]{0.45\textwidth} - \centering - \textbf{Analog representation},\\ on a metal plate - \end{minipage} - \hfill - \begin{minipage}[b]{0.45\textwidth} - \centering - \textbf{Digital representation},\\ on a LED display - \end{minipage} -\end{frame} - -\begin{frame}{Sampling and frequency domain} - \begin{itemize} - \item Pixels are quantized/sampled representations of a \textbf{spatial domain} - \item The initial (continuous) domain has a corresponding \textbf{frequency spectrum}\\ - \textit{high frequencies provide details in pictures} - \item A 2D \textbf{Fourier transform} translates from spatial \((x,y)\) to frequency \((u,v)\) domain -\[ -F(u,v) = \int_{-\infty}^{+\infty} \int_{-\infty}^{+\infty} f(x,y)e^{-j2\pi(ux+vy)}dxdy -\] - \item The transform decomposes the domain in \textbf{periodic patterns} - \item Adapted for discrete signals as \textbf{Discrete Fourier Transform} - \item Implemented with optimized algorithms as \textbf{Fast Fourier Transform} (FFT) - \item \textbf{Frequency domain analysis} is very useful for signal processing\\ - \textit{used at the roots of image compression} - \end{itemize} -\end{frame} - -\begin{frame}{Sampling and frequency domain (illustrated)} - \begin{center} - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/bricks.jpg}\\ - \textit{\small A wall of bricks represented in the spatial domain} - \end{minipage} - \hfill - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/bricks-fft.jpg}\\ - \textit{\small The wall of bricks represented in the frequency domain} - \end{minipage} - - \end{center} -\end{frame} - -\begin{frame}{Sampling and frequency domain (illustrated)} - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/bricks-rotation.jpg}\\ - \textit{\small A wall of bricks rotated \(45^{\circ}\) represented in the spatial domain} - \end{minipage} - \hfill - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/bricks-fft-rotation.jpg}\\ - \textit{\small The wall of bricks rotated \(45^{\circ}\) represented in the frequency domain} - \end{minipage} -\end{frame} - -\begin{frame}{Sampling and aliasing} - \begin{itemize} - \item The spatial domain is quantized with a bi-dimensional \textbf{sampling resolution} - \item Matching \textbf{sampling frequencies} exist, for each axis: \((u_s,v_s)\) - \item They \textbf{limit the frequencies} that can be sampled from the initial domain - \item The \textbf{Shannon-Nyquist theorem} provides a sufficient condition for \((u_s,v_s)\): -\[ -u_s > 2 \times u_{max}, ~v_s > 2 \times v_{max} -\] - \item Frequencies such that \(u \geq \frac{u_s}{2}\) and \(v \geq \frac{v_s}{2}\) are \textbf{not correctly sampled} - \item Can result in \textbf{incorrect frequencies} being introduced: \textbf{Moiré pattern} in 2D - \end{itemize} - - \begin{center} - \includegraphics[width=0.35\textwidth]{slides/graphics-theory/aliasing-1d.pdf}\\ - \textit{\small Aliasing example in a uni-dimensional domain} - \end{center} -\end{frame} - -\begin{frame}{Sampling and aliasing (illustrated)} - \begin{minipage}[b]{0.29\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/bricks-rich.jpg}\\ - \textit{\small Another wall of bricks} - \end{minipage} - \hfill - \begin{minipage}[b]{0.29\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/bricks-alias.jpg}\\ - \textit{\small Moiré on the bricks} - \end{minipage} - \hfill - \begin{minipage}[b]{0.29\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/moire.jpg}\\ - \textit{\small Moiré on the garage door} - \end{minipage} -\end{frame} - -\subsection{Color Representation} - -\begin{frame}{Light and color representation} - \begin{minipage}[c]{0.75\textwidth} - \begin{itemize} - \item \textbf{Light} itself must be quantized in digital representations\\ - \textit{distinct from and unrelated to spatial quantization} - \item Perceived as \textbf{colors} based on the Human visual system - \begin{itemize} - \item Perception based on \textbf{trichromacy} (red, green, blue) - \item Not necessarily a unique frequency of the spectrum\\ - \textit{e.g. pink is not a color of the visible spectrum} - \end{itemize} - \item \textbf{Translating} light information (colors) to numbers: - \begin{itemize} - \item A \textbf{color model} defines a base of color components\\ - \textit{typically 3 components (e.g. red, green, blue)} - \item A \textbf{colorspace} is a precise translation referential\\ - \textit{unique association of a color and coordinates in the base} - \item The \textbf{color gamut} is the range of colors in the colorspace\\ - \textit{not every color can be represented in every colorspace} - \end{itemize} - \end{itemize} - \end{minipage} - \hfill - \begin{minipage}[c]{0.225\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/gamut.png}\\ - \textit{\small Color gamut of a given colorspace} - \end{minipage} -\end{frame} - -\begin{frame}{Color quantization approaches} - \begin{minipage}[c]{0.75\textwidth} - \begin{itemize} - \item Different approaches exist for \textbf{color quantization}: - \begin{itemize} - \item \textbf{Uniform} quantization in the color range (most common)\\ - \textit{values are attributed to colors with a regular step (resolution)} - \item \textbf{Irregular} quantization with indexed colors (palettes)\\ - \textit{values are attributed to colors as needed} - \end{itemize} - \item Uniform color coordinates are quantized with: - \begin{itemize} - \item A given \textbf{resolution}: - \textit{the smallest possible color difference} - \item A given \textbf{range}: - \textit{the span of representable colors} - \end{itemize} - \item A given number of bits are used for quantization: \textbf{bit depth} - \item A \textbf{trade-off} between range and resolution must be defined - \begin{itemize} - \item Increasing the resolution reduces the range - \item Increasing the range reduces the resolution - \end{itemize} - \end{itemize} - \end{minipage} - \hfill - \begin{minipage}[c]{0.225\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/rgb-cube.pdf} - \end{minipage} -\end{frame} - -\begin{frame}{Light representation, color quantization (illustrated)} - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/pair-of-merops.jpg} - \end{minipage} - \hfill - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/pair-of-merops-16-colors.jpg} - \end{minipage} - - \begin{center} - \textit{\small A pair of Merops feeding} - \end{center} - - \begin{minipage}[b]{0.45\textwidth} - \centering - \textbf{16 million colors (24 bits per pixel)} - \begin{itemize} - \item high color resolution - \item high color range - \end{itemize} - \end{minipage} - \hfill - \begin{minipage}[b]{0.45\textwidth} - \centering - \textbf{16 colors (4 bits per pixel)} - \begin{itemize} - \item medium color resolution - \item low color range - \end{itemize} - \end{minipage} -\end{frame} - -\begin{frame}{Light representation, color quantization (illustrated)} - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/pair-of-merops.jpg} - \end{minipage} - \hfill - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/pair-of-merops-16-colors-range.jpg} - \end{minipage} - - \begin{center} - \textit{\small A pair of Merops feeding} - \end{center} - - \begin{minipage}[b]{0.45\textwidth} - \centering - \textbf{16 million colors (24 bits per pixel)} - \begin{itemize} - \item low color resolution - \item high color range - \end{itemize} - \end{minipage} - \hfill - \begin{minipage}[b]{0.45\textwidth} - \centering - \textbf{16 colors (4 bits per pixel)} - \begin{itemize} - \item low color resolution - \item high color range - \end{itemize} - \end{minipage} -\end{frame} - -\begin{frame}{Colorspaces and channels} - \begin{minipage}[b]{0.7\textwidth} - \begin{itemize} - \item Each component of a color model is called a \textbf{channel} - \item Examples for usual types of color models: - \begin{itemize} - \item RGB, with 3 channels:\\ \textbf{R} (red) / \textbf{G} (green) / \textbf{B} (blue) - \item HSV, with 3 channels:\\ \textbf{H} (hue) / \textbf{S} (saturation) / \textbf{V} (value) - \item YUV or Y/Cb/Cr, with 3 channels:\\ \textbf{Y} (luminance) / \textbf{U or Cb} / \textbf{V or Cr} (chrominance) - \end{itemize} - \end{itemize} - \end{minipage} - \hfill - \begin{minipage}[b]{0.25\textwidth} - \centering - \includegraphics[width=\textwidth]{slides/graphics-theory/hsv-diagram.pdf}\\ - \vspace{-1em} - \textit{\small HSV diagram} - \vspace{0.25em} - \end{minipage} - \begin{itemize} - \item An additional channel can exist for transparency: the \textbf{alpha channel}\\ - \textit{mostly relevant for composition, not for final display} - \item Color coordinates can be \textbf{converted} between colorspaces and color models\\ - \textit{using translation formulas and associated constants} - \end{itemize} -\end{frame} - -\begin{frame}{Colorspaces and channels (illustrated with YUV)} - \begin{minipage}[t]{0.25\textwidth} - \centering - \includegraphics[height=6em]{slides/graphics-theory/barn.jpg}\\ - \textit{\small Original picture} - \end{minipage} - \hfill - \begin{minipage}[t]{0.7\textwidth} - \centering - \includegraphics[height=6em]{slides/graphics-theory/barn-yuv.jpg}\\ - \textit{\small Decomposition in Y, U and V channels} - \end{minipage} - - \vspace{1em} - - \begin{minipage}[t]{0.4\textwidth} - \small - \begin{equation*} - \begin{cases} - R = Y + 1.140 \times V\\ - G = Y - 0.395 \times U - 0.581 \times V\\ - B = Y + 2.032 \times U - \end{cases} - \end{equation*} - \end{minipage} - \hfill - \begin{minipage}[t]{0.55\textwidth} - \small - \begin{equation*} - \begin{cases} - Y = + 0.299 \times R + 0.587 \times G + 0.114 \times B\\ - U = - 0.147 \times R - 0.289 \times G + 0.436 \times B\\ - V = + 0.615 \times R - 0.515 \times G - 0.100 \times B - \end{cases} - \end{equation*} - \end{minipage} - - \begin{center} - \textit{\small Translation between BT.601 YUV and sRGB colorspaces} - \end{center} -\end{frame} - -\subsection{Layout and Formats} - -\begin{frame}{Frame size and chroma sub-sampling} - \begin{itemize} - \item Digital pictures easily take up a lot of space (more so for videos) - \item The minimal size for a picture depends on: - \begin{itemize} - \item Dimensions (\(width\) and \(height\)) - \item Number of bits per pixel (\(bpp\)): color (and alpha) depth and dead bits - \item Roughly: \(width \times height \times bpp \div 8~bytes\) - \item For 12 Mpixels with 16 Mcolors and alpha: \(4000 \times 3000 \times 32 \div 8 = 45.8 MiB\) - \end{itemize} - \item The human visual system has specificities: - \begin{itemize} - \item High sensitivity to \textbf{luminosity} (luminance) - \item Low sensitivity to \textbf{colors} (chrominance) - \end{itemize} - \item The YUV color model offers the relevant channel separation - \item Sub-sampling can be applied to the chrominance channel\\ - \textit{less data (and precision) on colors to reduce size} - \end{itemize} -\end{frame} - -\begin{frame}{Frame size and chroma sub-sampling} - \begin{itemize} - \item Chrominance samples are used for multiple luminance samples - \item With specific vertical and horizontal ratios (usually integer) - \item Usually summarized using a three-part ratio: \(J:a:b\) - \end{itemize} - - \begin{center} - \includegraphics[width=0.55\textwidth]{slides/graphics-theory/yuv-sub-sampling.pdf} - - YUV 4:2:0 usual example: - {\small - \begin{equation*} - \begin{gathered} - bpp_Y = 8,~bpp_U = bpp_V = 8 \div 2 \div 2 = 2 \\ - \Rightarrow~ bpp = bpp_Y + bpp_U + bpp_V = 12~bits/pixel - \end{gathered} - \end{equation*} - } - \end{center} -\end{frame} - -\begin{frame}{Pixel data distribution in memory} - \begin{itemize} - \item Pixel data can be \textbf{distributed} in different ways in memory - \item Different ways to aggregate color components in \textbf{data planes} (memory chunks): - \begin{itemize} - \item \textbf{Packed}: Components are stored in the same data plane in memory - \item \textbf{Semi-planar} (YUV): Luma and chroma are stored in distinct data planes - \item \textbf{Planar}: Each component has its own data plane in memory - \end{itemize} - \item When multiple color components are grouped, \textbf{bit order} must be specified: - \begin{itemize} - \item Which component comes first in memory? - \item Affected by endianness when read by hardware! - \end{itemize} - \item \textbf{Scan order} must also be specified: - \begin{itemize} - \item How to calculate the address for position \((x,y)\) and back? - \item Raster order (most common) specifies: row-major, left-to-right, top-to-bottom - \end{itemize} - \end{itemize} - \begin{center} - \includegraphics[height=6em]{slides/graphics-theory/raster-order.pdf} - \end{center} -\end{frame} - -\begin{frame}[fragile]{Pixel formats, FourCC codes} - \begin{itemize} - \item Many meta-data elements are needed to fully describe how a picture is coded - \begin{itemize} - \item Some describe \textbf{picture-level attributes} (e.g. dimensions) - \item Some describe \textbf{pixel-level attributes} (e.g. colorspace, bpp) - \end{itemize} - \item Pixel-level attributes are grouped as a \textbf{pixel format} that defines: - \begin{itemize} - \item Color model in use - \item Number of bits per channel and per pixel (bpp) - \item Bit attribution and byte order - \item Per-channel sub-sampling ratios - \item Pixel data planes distribution in memory - \end{itemize} - \item Often represented as a 4-character code called \textbf{FourCC}\\ - \item Not really standardized and implementation-specific:\\ - \textit{DRM in Linux uses \code{XR24} for \ksym{DRM_FORMAT_XRGB8888}}.\\ - \textit{Not really standardized but widely used in various forms} - \item Scan order is specified separately with a \textbf{modifier}\\ - \textit{Assumed to be raster order if unspecified} - \end{itemize} -\end{frame} - -\begin{frame}{Level of detail of quantized pictures} - Depends on a number of factors, including: - \begin{itemize} - \item Spatial density (pixel resolution) - \item Quantized dimensions (picture width and height) - \item Colorspace limits (chromaticity diagram) - \item Color depth (number of bits per pixel) - \item Color resolution and range trade-off - \end{itemize}~ - - Generally speaking: - \begin{itemize} - \item Many factors are involved - \item The major bottleneck is not always obvious - \item Implementation choices do matter - \end{itemize} -\end{frame} - -\subsection{Pixel Drawing} - -\begin{frame}{Accessing pixel data} - \begin{itemize} - \item Information required to access pixel data in memory: - \begin{itemize} - \item Pixel format (also modifier if not linear/raster order) - \item Dimensions (and total size) - \item Pointer to the base buffer address - \end{itemize} - \item The size of each line is called \textbf{stride} or \textbf{pitch} - \begin{itemize} - \item Usually equals: \(stride = width \times bpp \div 8\) - \item Can contain an extra dead zone at the end - \item Also needs to be specified explicitly - \end{itemize} - \item CPU access is either byte or word-aligned - \begin{itemize} - \item Good fit for formats with \(bpp = 32\) (very common) - \item Good fit for formats with \(bpp = 8 \times n\) - \item Not always easy to manage otherwise - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}[fragile]{Iterating over pixel data} - \begin{itemize} - \item Selected format (slides and demos): \textbf{XRGB8888} - \begin{itemize} - \item \(bpp = 32 = 8 \times 4\), one byte per channel, one memory plane - \end{itemize} - \item Pixel data can be access by iterating nested variables: - \begin{minted}[fontsize=\small]{c} -for (y = 0; y < height; y++) - for (x = 0; x < width; x++) - data = base + y * stride + x * 4; - \end{minted} - \item Iterating over all pixels takes numerous CPU-cycles, tips: - \begin{itemize} - \item Incrementing the address instead of re-calculating it: - \begin{minted}[fontsize=\small]{c} -data = base; -for (y = 0; y < height; y++) - for (x = 0; x < width; x++) - data += 4; - data += stride - width * 4; - \end{minted} - \item Iterating in y-major is also better for cache hits - \item Beware: C pointer arithmetic uses type size as unit - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Concepts about rasterization} - \begin{itemize} - \item Rasterization is the process of drawing vector shapes as discrete pixels - \begin{itemize} - \item Vector shapes are defined with mathematical equations - \item Converted from a continuous space domain: \(\mathbb{R}^2\) to a discrete domain - \item Results in a discretization/quantization error due to integer rounding - \item Also subject to aliasing-related trouble - \end{itemize} - \end{itemize} - \vspace{1em} - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[width=0.8\textwidth]{slides/graphics-theory/raster-source.jpg} - \textit{\small Continuous source representation} - \end{minipage} - \hfill - \begin{minipage}[b]{0.45\textwidth} - \centering - \includegraphics[width=0.8\textwidth]{slides/graphics-theory/raster-destination.jpg} - \textit{\small Rasterized destination representation} - \end{minipage} -\end{frame} - -\begin{frame}{Rectangle drawing} - \begin{itemize} - \item A rectangle is defined with two boundaries per axis: -\[ -x_{min} \leq x \leq x_{max},~ y_{min} \leq y \leq y_{max} -\] - \item Another expression involves a (top-left) start point and size: -\[ -x_{start} \leq x \leq x_{start} + x_{size},~ y_{start} \leq y \leq y_{start} + y_{size} -\] - \item Allows iterating in the rectangle area only - \end{itemize} -\end{frame} - -\begin{frame}{Linear gradient drawing} - \begin{itemize} - \item Same base as drawing a rectangle - \item A linear gradient involves interpolation between two colors - \item Following one of the two axes as major - \item Involves weighting the two colors depending on the advancement - \item Equations in x-axis major: - \end{itemize} -\begin{align*} -r &= r_{start} + (r_{stop} - r_{start}) \frac{x - x_{start}}{x_{size}}\\ -g &= g_{start} + (g_{stop} - g_{start}) \frac{x - x_{start}}{x_{size}}\\ -b &= b_{start} + (b_{stop} - b_{start}) \frac{x - x_{start}}{x_{size}} -\end{align*} -\end{frame} - -\begin{frame}{Disk drawing} - \begin{itemize} - \item A disk is delimited with a radius test (\((0,0)\)-centered): -\[ -\sqrt{x^2 + y^2} \leq radius -\] - \item Given a center point \((x_c,y_c)\): -\[ -\sqrt{(x - x_c)^2 + (y - y_c)^2} \leq radius -\] - \item Requires iterating in: -\[ -x_c - radius \leq x \leq x_c + radius,~ y_c - radius \leq y \leq y_c + radius -\] - \end{itemize} -\end{frame} - -\begin{frame}{Circular gradient drawing} - \begin{itemize} - \item Same base as drawing a disk - \item Interpolation between two colors using the radius as major: - \end{itemize} -\begin{align*} -d &= \sqrt{(x - x_c)^2 + (y - y_c)^2}\\ -r &= r_{start} + (r_{stop} - r_{start}) \frac{d}{radius}\\ -g &= g_{start} + (g_{stop} - g_{start}) \frac{d}{radius}\\ -b &= b_{start} + (b_{stop} - b_{start}) \frac{d}{radius} -\end{align*} -\end{frame} - -\begin{frame}{Line drawing} - \begin{itemize} - \item A line is defined as an affine function: -\[ -y(x) = a \times x + b -\] - \item Given start and end points, iterating in x-major: -\begin{gather*} -y(x) = y_{start} + (x - x_{start}) \times \frac{y_{end} - y_{start}}{x_{end} - x_{start}}\\ -x_{start} \leq x \leq x_{stop} -\end{gather*} - \item Axis major depends on the largest per-axis span (\(axis_{stop} - axis_{start}\)) - \begin{itemize} - \item Iterating with smaller-span axis-major results in visual holes - \item Iterating on both axes provides coherent results - \end{itemize} - \item Algorithms producing better-looking results: - \begin{itemize} - \item \textbf{Bresenham's} line algorithm, optimized for implementation - \item \textbf{Xiaolin Wu's} line algorithm, with sub-pixel rendering - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Line and shape aliasing, sub-pixel drawing} - \begin{itemize} - \item Lines are often subject to aliasing: - \begin{itemize} - \item Sampled from the continuous domain with pixel sampling resolution - \item Selecting the best axis gives a better resolution - \item Limited display resolutions still make them look pixelated - \end{itemize} - \item Any geometric shape is affected, especially fonts - \item Sub-pixel rendering is used to provide anti-aliased results: - \begin{itemize} - \item Surrounding pixels are given an intermediate value - \item Specific algorithms perform sub-pixel drawing - \item Also obtained with high-resolution rendering and anti-aliased downscaling - \end{itemize} - \end{itemize} - - \begin{center} - \includegraphics[height=4em]{slides/graphics-theory/diamond-sub-pixel.png} - \includegraphics[height=4em]{slides/graphics-theory/line-sub-pixel.png}\\ - \textit{\small Shapes rendered without and with sub-pixel anti-aliasing} - \end{center} -\end{frame} - -\begin{frame}{Line and shape aliasing, sub-pixel drawing (illustrated)} - \begin{center} - \includegraphics[width=0.4\textwidth]{slides/graphics-theory/line-sub-pixel-rendering.pdf}\\ - \textit{\small Pixel drawing versus sub-pixel drawing (x-axis major)} - \end{center} -\end{frame} - -\begin{frame}{Circles and polar coordinates} - \begin{itemize} - \item Circles centered on \((x_c,y_c)\) are defined (Pythagoras theorem) as: -\[ -(x - x_c)^2+(y - y_c)^2 = radius^2 -\] - \end{itemize} - \begin{minipage}{0.70\textwidth} - \begin{itemize} - \item Which is always verified with the expression: -\begin{gather*} -x = x_c + radius \times cos(\phi)\\ -y = y_c + radius \times sin(\phi) -\end{gather*} - \item Corresponds to a translation in polar coordinates - \begin{itemize} - \item From a \((x,y)\) base to \((r,\phi)\) - \end{itemize} - \item Iteration on \(\phi\) with a specific range: \(\phi \in [0;2\pi]\) - \end{itemize} - \end{minipage} - \hfill - \begin{minipage}{0.25\textwidth} - \centering - \includegraphics[width=0.8\textwidth]{slides/graphics-theory/polar-coordinates.pdf} - \end{minipage} -\end{frame} - -\begin{frame}{Parametric curves} - \begin{itemize} - \item Parametric curves generalize the idea of using independent parameters - \item Each curve has defining equations and ranges for parameters - \begin{itemize} - \item Equations allow calculating \((x,y)\) (or \((r,\phi)\)) - \end{itemize} - \item Drawing is achieved by iterating over parameter values - \begin{itemize} - \item Sampling is done on the range to get a finite number of points - \item X/Y coordinates are calculated for each point - \item Line interpolation is used between consecutive points - \end{itemize} - \item Ellipse: \(\phi \in [0;2\pi]\) -\begin{gather*} -x = x_c + a \times cos(\phi)\\ -y = y_c + b \times sin(\phi) -\end{gather*} - \item Many more parametric curves exist: - \begin{itemize} - \item Cycloid, Epicycloid, Epitrochoid, Hypocycloid, Hypotrochoid (spirograph) - \item Lissajous Curve, Rose curve, Butterfly curve - \end{itemize} - \end{itemize} -\end{frame} - -\subsection{Pixel Operations} - -\begin{frame}{Region copy} - \begin{itemize} - \item Requirements for all pixel operations: - \begin{itemize} - \item The source and destination must use the same pixel format - \item Must be converted before any operation otherwise - \end{itemize} - \item Most basic operation on pixels: copying a region - \begin{itemize} - \item Also known as bit blit or BITBLT (in reference to the hardware opcode) - \end{itemize} - \item Implemented as a line-per-line copy (maximum memory-contiguous block) - \item Overwriting destination memory with source memory - \begin{itemize} - \item Copies within the same image are not always safe! - \item Destination must not overlap source - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Alpha blending} - \begin{itemize} - \item Compositing multiple alpha-enabled pixel sources into a single result - \begin{itemize} - \item Simplest case: aggregating sources with z-ordered stacking - \item Equation for A over B (with \(\alpha\) the alpha and \(C\) the color component value): - \end{itemize} -\[ -C_o = \frac{C_a \alpha_a + C_b \alpha_b \left(1 - \alpha_a\right)}{\alpha_a + \alpha_b \left(1 - \alpha_a\right)} -\] - \item With alpha available, many more operations become possible - \begin{itemize} - \item Shapes can be used as masks, with logic operators - \item Formalized by Porter and Duff in 1984 - \end{itemize} - \end{itemize} - \begin{center} - \includegraphics[width=0.5\textwidth]{slides/graphics-theory/porter-duff-compositing.pdf} - \end{center} -\end{frame} - -\begin{frame}{Color-keying} - \begin{itemize} - \item Color-keying (or chroma-keying): replacing given colors with alpha - \item Specified with color ranges (3 RGB ranges) - \item Pixels either within or outside of the range are made transparent - \item Used in conjunction with alpha blending - \item The famous video green-screen method uses color-keying - \end{itemize} - \begin{center} - \includegraphics[width=0.5\textwidth]{slides/graphics-theory/chroma-key-blender.jpg}\\ - \textit{\small Color-keying implemented in Blender} - \end{center} -\end{frame} - -\begin{frame}{Scaling and interpolation} - \begin{itemize} - \item Scaling is a resizing operation on a pixel picture - \begin{itemize} - \item Involves a scaling factor (integer or real) - \item Values are resampled with a new resolution - \item Requires reconstructing the original signal - \end{itemize} - \item Implemented with some form of interpolation: - \begin{itemize} - \item nearest-neighbor: uses the nearest pixel value from the source -\[ -x_{source} = x_{destination} \div scale -\] - \item bilinear interpolation: sub-pixel linear weighting of neighbor colors - \item bicubic interpolation: smooth spline sub-pixel fitting with neighbor colors - \end{itemize} - \item Sub-pixel methods provide better visual results - \item Down-sampling: - \begin{itemize} - \item Reduces the maximum image frequency - \item Can cause aliasing: high frequencies need to be removed - \end{itemize} - \end{itemize} -\end{frame} - -\begin{frame}{Linear filtering and convolution} - \begin{itemize} - \item Filtering is a transformation of each pixel based on its neighbors - \item The pixel output value is a linear sum of weighted neighboring input values -\[ -o_{x,y} = \alpha_0 i_{x,y} + \alpha_1 i_{x-1,y} + \alpha_2 i_{x+1,y} + ... -\] - \item Weighting coefficients are represented in a 2D matrix: the \textbf{filter kernel} - \begin{itemize} - \item Comes with \(2n + 1\) columns and \(2m + 1\) rows - \item The coefficients are applied to each input pixel and its neighbors - \item The element at the kernel center weights the current input pixel - \end{itemize} - \item Corresponds to a \textbf{convolution} operation between pixels and the filter kernel - \item High computational cost (optimizations are implemented) - \item Allows many applications for 2D signal processing - \end{itemize} -\end{frame} - -\begin{frame}{Linear filtering and convolution (illustrated)} - \begin{center} - \includegraphics[width=0.7\textwidth]{slides/graphics-theory/linear-filtering.pdf}\\ - \textit{\small Linear filtering in application} -\[ -g(x,y)= \omega *f(x,y)=\sum_{s=-n}^n{\sum_{t=-m}^m{ \omega (s,t)f(x-s,y-t)}} -\] - \textit{\small Bi-dimensional convolution operation on \(f\) with the \(\omega\) kernel} - \end{center} -\end{frame} - -\begin{frame}{Blur filters} - \begin{itemize} - \item Blurring is a common example of linear filtering - \item Corresponds to a low-pass filter - \begin{itemize} - \item Removes high frequencies from the picture (details) - \item Good fit for pre-scaling anti-aliasing - \end{itemize} -\begin{minipage}[b]{0.45\textwidth} - \item Implemented with different algorithms: - \vspace{-1.5em} - \begin{itemize} - \item Box blur: rough but easy to optimize -\[ -\frac{1}{9} -\left[ -\begin{matrix} -1 & 1 & 1 \\ -1 & 1 & 1 \\ -1 & 1 & 1 -\end{matrix} -\right] -\] - \end{itemize} -\end{minipage} -\begin{minipage}[b]{0.45\textwidth} - \begin{itemize} - \item Gaussian blur: reference smooth blur -\[ -\frac{1}{16} -\left[ -\begin{matrix} -1 & 2 & 1 \\ -2 & 4 & 2 \\ -1 & 2 & 1 -\end{matrix} -\right] -\] - \end{itemize} - \end{minipage} - \item A repeated box blur converges towards a Gaussian one (central-limit theorem) - \end{itemize} -\end{frame} - -\begin{frame}{Dithering} - \begin{itemize} - \item Reducing the color depth can lead to visually-unpleasant results - \begin{itemize} - \item Corresponds to color-space down-sampling - \item Increases color quantization error - \end{itemize} - \item Floyd–Steinberg dithering is a method for improving quality with low depth - \item Quantization error is evaluated and distributed to neighboring pixels - \item Used in hardware display engines and the GIF file format - \end{itemize}~\\ - - \begin{minipage}[t]{0.25\textwidth} - \centering - \includegraphics[width=0.9\textwidth]{slides/graphics-theory/cat-depth-initial.png}\\ - \textit{\small Cat at initial depth} - \end{minipage} - \hfill - \begin{minipage}[t]{0.25\textwidth} - \centering - \includegraphics[width=0.9\textwidth]{slides/graphics-theory/cat-depth-low.png}\\ - \textit{\small Cat at reduced depth without dithering} - \end{minipage} - \hfill - \begin{minipage}[t]{0.25\textwidth} - \centering - \includegraphics[width=0.9\textwidth]{slides/graphics-theory/cat-depth-dither.png}\\ - \textit{\small Cat at reduced depth with dithering} - \end{minipage} -\end{frame} - -\begin{frame}{Graphics theory online references} - \small - \begin{itemize} - \item Wikipedia (\url{https://en.wikipedia.org/}): - \begin{itemize} - \item \href{https://en.wikipedia.org/wiki/Color_model}{Color model} - \item \href{https://en.wikipedia.org/wiki/Color_depth}{Color depth} - \item \href{https://en.wikipedia.org/wiki/YCbCr}{YCbCr} - \item \href{https://en.wikipedia.org/wiki/Chroma_subsampling}{Chroma subsampling} - \item \href{https://en.wikipedia.org/wiki/Nyquist–Shannon_sampling_theorem}{Nyquist–Shannon sampling theorem} - \item \href{https://en.wikipedia.org/wiki/Spatial_anti-aliasing}{Spatial anti-aliasing} - \item \href{https://en.wikipedia.org/wiki/Aliasing}{Aliasing} - \item \href{https://en.wikipedia.org/wiki/Line_drawing_algorithm}{Line drawing algorithm} - \item \href{https://en.wikipedia.org/wiki/Parametric_equation}{Parametric equation} - \item \href{https://en.wikipedia.org/wiki/Alpha_compositing}{Alpha compositing} - \item \href{https://en.wikipedia.org/wiki/Image_scaling}{Image scaling} - \item \href{https://en.wikipedia.org/wiki/Kernel_(image_processing)}{Kernel (image processing)} - \end{itemize} - \item \url{http://ssp.impulsetrain.com/porterduff.html} - \item \url{https://magcius.github.io/xplain/article/regions.html} - \item \url{https://magcius.github.io/xplain/article/rast1.html} - \end{itemize} -\end{frame} - -\begin{frame}{Graphics theory illustrations attributions} - \small - \begin{itemize} - \item \href{https://commons.wikimedia.org/wiki/File:2017-12-28_Leipzig,_34c3,_Fairy_Dust_(freddy2001).jpg}{34C3 Fairy Dust: Freddy2001, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:View_from_the_Window_at_Le_Gras,_Joseph_Nic\%C3\%A9phore_Ni\%C3\%A9pce.jpg}{Point de vue du Gras: Joseph Nicéphore Niépce, public domain} - \item \href{https://commons.wikimedia.org/wiki/File:Pinball_Dot_Matrix_Display_-_Demolition_Man.JPG}{Pinball Dot Matrix Display: ElHeineken, CC BY 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:Soderledskyrkan_brick_wall.jpg}{Soderledskyrkan brick wall: Xauxa, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:AliasingSines.svg}{Aliasing Sines: Moxfyre, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:Moire_pattern_of_bricks.jpg}{Moiré pattern of bricks: Colin M.L. Burnett, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:Moire_pattern_of_bricks_small.jpg}{Moiré Pattern at Gardham Gap: Roger Gilbertson, CC BY-SA 2.0} - \item \href{https://commons.wikimedia.org/wiki/File:RGBCube_a.svg}{RGB cube: Datumizer, CC BY-SA 4.0} - \item \href{https://commons.wikimedia.org/wiki/File:Pair_of_Merops_apiaster_feeding.jpg}{Pair of Merops apiaster feeding: Pierre Dalous, CC BY-SA 3.0} - \end{itemize} -\end{frame} - -\begin{frame}{Graphics theory illustrations attributions} - \small - \begin{itemize} - \item \href{https://commons.wikimedia.org/wiki/File:Hsl-hsv_models_b.svg}{Hsl-hsv models: Datumizer, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:Barns_grand_tetons.jpg}{Barns grand tetons: Jon Sullivan, public domain} - \item \href{https://commons.wikimedia.org/wiki/File:Top-left_triangle_rasterization_rule.gif}{Top-left triangle rasterization rule: Drummyfish, CC0 1.0} - \item \href{https://commons.wikimedia.org/wiki/File:Line_scan-conversion.svg}{Line scan-conversion: Phrood, CC BY-SA 3.0} - \item \href{https://commons.wikimedia.org/wiki/File:Alpha_compositing.svg}{Alpha compositing: Prometeusm, Wereon, public domain} - \item \href{https://commons.wikimedia.org/wiki/File:Blender3D_com_key_chroma.jpg}{Blender3D com key chroma: Toni Grappa, Blender Foundation, CC BY-2.5} - \item \href{https://en.wikipedia.org/wiki/File:Dithering_example_dithered_web_palette.png}{Dithering example: Jamelan, CC BY-SA 3.0} - \end{itemize} -\end{frame} diff --git a/slides/graphics-theory/resolution.dia b/slides/graphics-theory/resolution.dia deleted file mode 100644 index dc4e0de11d608f3994fcfa61597974fbce40d1d1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1674 zcmV;526g!#iwFP!000021MOW~Z=*OAe($e9^tC0vK!T^!j&>ea`>@hVJKg8VN!)}T z0ui{EyzFmZb7|ucQkr0!G9pqdg&fYY{e9=N4-n_;x0iKjJ+UMW;^@|f4zMj2&ErK7 zEpP3A?tj1b?QeG%Ul#%WBL0>MU0dReD9zs8+AGdCUq++n=VvF}rj*Bt69$h?%0~au zFr*`4G_vn5EK7EBhjC&((eT#3 ze8^w+sB|;h^Yf1H4PCNX!syTAu7P+Fa*pc_OR8?yn>Y=G3EyrGOpf@;{!2EcRVs|4 z<=y3P?q$)W^1#klT|KgXl1h7@M#d2;}$(Au$*p@d=o?6ZkqNr-z%F4HudX zmzWI~q|;5D@Pr2ZFyt(bLq?+_Do-BS`Ek-Y4F$#C>sE_iJOrG_4efuRVfv8{6o=ok z?bOU&Cc)zU#ND$v;^jG5@YVE1({L5ux0<{^1!*t~SzYUch%0lyRpY*jPOjd&(_6Bk z@OE^1Soq9i-s#iB@-bMjw1H;(Hb*?IO0!YZs8{ViIxxkd%4D5~29d^gKedJo2@N`_7&&e2WysI{~fPsBxm{Qjaac@xvFi3 zOj;l8zua~~_o?>*`kfPGe-Zo6l@0*f1rjCVGFr2GFn99x(I~Y`)YJ0eDR0toCB;TG z2)ETBNUk8zIzeBNT!O?*jm69idLltv!xBWJ^iYBRVBr(v!JPh*`ct9apg*5kf4;{r zHT8kpx;_v~WTa7kM6I0F2T=UKL0HtwS*>tx)yzV+p3dVWI=s5Brb~H%hShBw*9StN z#&^0Bi}IS=Db3f4^)b_H%}lXg#yXkl<;+AmGbd)4d488Z!c3@048G{L~k zu9@kn5KW+6G>N2{(12zR%nxCf2{wM3QQ{22^vPX6MvIDpV8cS4}9XCN`j&!y`wXQxmIDO{8Bn zp`@DFfNBoZG)K)N8wMaH-w1LYPycsuX>YC_6tyiefyg&NIUjW z!X6pJ9yMc+`i=cq#Uovsu}2d2*ckS>8GGDs?AzZjaJ6HPCG1^e*t^ZxyZy$#{RMeX zJNB-Gy=M%2uNix<-`EoskNs%I-jlE&8^eCwjQzOR*khpL!5Yojk0tCU#;~6>V?XIP z_E5#8G0oUdBrlw65Y*5z!`Uhz9i&4M;?Triccchz5HsGQJw@fp+YP1U)bXJ!pa+d Date: Mon, 24 Nov 2025 21:10:54 +0100 Subject: [PATCH 3/6] slides: first-slides: Update my personal introduction Signed-off-by: Paul Kocialkowski --- slides/first-slides/paul-kocialkowski.tex | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/slides/first-slides/paul-kocialkowski.tex b/slides/first-slides/paul-kocialkowski.tex index aeb5987711..613b296c6b 100644 --- a/slides/first-slides/paul-kocialkowski.tex +++ b/slides/first-slides/paul-kocialkowski.tex @@ -1,12 +1,18 @@ \begin{frame}{Paul Kocialkowski} + \begin{itemize} + \item Embedded Linux engineer and trainer at {\bf Bootlin} from 2018 to 2023 + \item Contracting trainer for {\bf Bootlin} since 2024 + \item Open-source contributor \begin{itemize} - \item Embedded Linux engineer and trainer at {\bf Bootlin} - since 2018 - \item {\bf Linux kernel}: multiple contributions to the graphics - and video pipelines in the mainline kernel, in particular - on Allwinner and Broadcom ARM SoCs. - \item Living in {\bf Toulouse}, France - \item \code{paul@bootlin.com} + \item Contributor to the \textbf{cedrus} Allwinner Video Engine V4L2 driver + \item Contributor to the \textbf{sun6i-csi} Allwinner video capture V4L2 driver + \item Author of the \textbf{sun6i-isp} and \textbf{sun6i-mipi-csi2} Allwinner video capture V4L2 drivers + \item Author of the \textbf{ov5648} and \textbf{ov8865} camera sensor V4L2 drivers + \item Contributor to the \textbf{sun4i-drm} and \textbf{vc4} DRM drivers + \item Author of the \textbf{logicvc-drm} DRM driver + \item Author of the \textbf{Understanding the Linux Graphics Stack} training \end{itemize} - {\small \url{https://bootlin.com/company/staff/paul-kocialkowski/}} + \item Living in {\bf Toulouse}, south-west of France + \end{itemize} + {\small \url{https://bootlin.com/company/staff/paul-kocialkowski/}} \end{frame} From 906d5e0abe83fab220ac6e13912a48c1aedc9768 Mon Sep 17 00:00:00 2001 From: Paul Kocialkowski Date: Tue, 25 Nov 2025 17:57:26 +0100 Subject: [PATCH 4/6] slides: graphics-hardware-pipelines: Cleanup pipeline diagram layout Signed-off-by: Paul Kocialkowski --- .../graphics-hardware-pipelines.tex | 3 ++- .../pipeline-sink-source.dia | Bin 1666 -> 1743 bytes 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/slides/graphics-hardware-pipelines/graphics-hardware-pipelines.tex b/slides/graphics-hardware-pipelines/graphics-hardware-pipelines.tex index e9d7487a3d..7dc9240eaf 100644 --- a/slides/graphics-hardware-pipelines/graphics-hardware-pipelines.tex +++ b/slides/graphics-hardware-pipelines/graphics-hardware-pipelines.tex @@ -13,7 +13,7 @@ \subsection{Pipelines} \textit{Usually from a source-only element to a sink-only element} \end{itemize} - \vspace{-2em} + \vspace{1em} \begin{center} \includegraphics[width=0.7\textwidth]{slides/graphics-hardware-pipelines/pipeline-sink-source.pdf} \end{center} @@ -31,6 +31,7 @@ \subsection{Pipelines} \( t_0 + t_2 + t_4 + t_5 < fps^{-1},~ t_0 + t_3 < fps^{-1},~ t_1 + t_4 + t_5 < fps^{-1} \) \end{itemize} + \vspace{1em} \begin{center} \includegraphics[width=0.7\textwidth]{slides/graphics-hardware-pipelines/pipeline-complex.pdf} \end{center} diff --git a/slides/graphics-hardware-pipelines/pipeline-sink-source.dia b/slides/graphics-hardware-pipelines/pipeline-sink-source.dia index 701387e64928ba90107bbc3b198fcda0504f7e3d..3cad7f07064aa290bd320a578cfeaa19132ca71e 100644 GIT binary patch literal 1743 zcmV;=1~B;_iwFn+0000218`|@Wo&6~Wi4}QZfh-bZ*_8GWiDiCVF2x1%WmT~6y4`5 z1ZSJbBqdo?GRdIRqV1wUfp(@qR|ah{jwUi?NOTf!=G#j>{L;e@(R4>18c2dD)IB71 z?zxv&3_pB)pN7UOjguf^4?Spton-kme7ySL2jotCO=2=N#2qY2 ze|zXXb3VK4_ut;$%y5wq9>r!D%*}-MzmYH`ebK1jd%QA?O#?pRB)wK#C7j2>XwIp@ z$do?xMr8aliK995dwHv(-8c%P*mx!3L+|>j`1JaPq5hVo9oA=LLPs$rFU`h*_;_AB z<1-qU4NqrL5{M>zF{?Cbu#}(Rty*^TNgFg$t75EQ)zd0$zn2MNvoz%Taker|of) zF$o35b`48n6i)%?(VqT4kuW)A0Qu#Ww;k)blQ{5qJMLDC1}pwT{0zoQAIV$b^XK8a zqDPWjUnsKv8YICeq;xYMOe}^GUdxeD8Kc08op8JLtWCL`i$L{l_6*= z&ex%pE&{WB9f5P?>oOxcBwHWoT>Zcp>By7QRGqD75P7~12PU?#D^wrXY9(Q2x}p_03mXvg=nB!hz6^Ss3X~s z&V!VT&p?aM1&9xp79Z7ROK2^{$I{}X#iy2ZT70@#tMsIItRTg;N(5K6N=qd8>T8uU zBRZs3iS^EcgF_>f-%+K$T#44@N+fvdq0ZDRBPM2?(^yAK)zK2^lI3M2>ri?>9>VGs z31^CwZ8QKT&~-~)x79deUk2>8AQfZ@EO4`fSKr zug`KctwCCY>ZqqxsGBNe%jZBT6_n;cnFd)|7L?HrY7NpFR3V;Lpl&LVBmW*neIA5X zRUr}jZ2j*ix+t`dgIbQhZ#il>hX6`%tVKW&0V&;6;^pskF~;a#FfT7sLx@$MMNl4D z?Hl0sgI97Iff(43#Buc2%&bj$Xk~gBQa0K3;u^GmalL}r^J@HHg!K_5NhJa;|6)7~ zH!0McN7?^n8SB4po!-!VFD&l$w(qs7hd-4%bRtKV-k6&0nE;k*Z%$zVu$KOup3iV5 zf36>3!;ul%tuN25ORkBJacij9t(%q$_yqA80>%INBK*q2H(p*ig{Ex}P^pRat{}qT-9iM+;C5?R52` zlLCZ*{E>I61xTzqEEl3eg)GY)AnbM}JNoOlPli~g;VbAN@D2gi*^&Aq#*ogAE<&8- z_y0Po#R)mfRhy%;BP~D;wA0y9Hw8%kv{h(2wr8hwuv|ywu9qc}Co~A)`wd`c&=y#$ l(nY03O6d@!#g9J1HzU#wfpBxW2cw4yW zZQ;_lg$uL(EJ;PmLa_}wNRo)NxI`7{oS%-94Ot{9wyRkRt#}B9NcPnKfkoLN4Ja?a zj<#b*?l=ucy9e$@iIynO;YdvSFPgfmjK0tm{VB}CLByMCAI9Qr;TPu#U;lIp^+~?9oN#jd^e~=>BcAP{*=CuRNRz5qZ(pz1%N^P=<)zwWT|h)^!P6@I{c4)F zst>qQ9E%M___Tbt)+fo}A3hXSWB(K^9%-#AwlcBE2_x-hdF!t{dg3A+ zvUOe`UxNt`$CJi7tb`Ic6l}g+HPB5K9>c^_&^85Xhl>S34MVUt2so`8@gY}O;h3P(;PQSUMeU?=&r7QNIl4rS66A#Ax_;hKK;N6(o-j=quZ$$5T#d# zz5^g4opI|w%5o_)W?g&Abc=;%rtF|y_m$O%>eeEsas#HWRJMtr)Ls)WkB z=n&GJt3>cMSBa&+*Pp9YY0)9MN@{i$96UEd_1#qZ)0JqIu0+1$1>H$jCb8^s!BaC? z>Ygl-$ya^^uMU-W;vuSEk4lJgj{$IiNm`nuKO&?(v$z`K7yd$w#J9Qr#D~AAq28veU_%lb33=z<>92z{fNioUC*q+$}hw{!+l zKUmS%E>25bDxu6H9V0-E#4|aJE(#D-KkZJp0Lf*CT;T+A!Q)|=ALK(Z^$T}ISEs|j zUn%TlPT(uG7j*I}c63@7!E((zP zL95UqZs6u*z&_Er@nwk=I6ehb`lJ;yK@=NNGNRN Date: Fri, 5 Dec 2025 22:00:53 +0100 Subject: [PATCH 5/6] slides: graphics: Update various outdated or incomplete parts Signed-off-by: Paul Kocialkowski --- .../graphics-hardware-pipelines.tex | 9 +++-- .../graphics-software-linux-drm.tex | 1 + .../graphics-software-linux-tty.tex | 3 +- .../graphics-software-userspace-mesa.tex | 6 +++- .../graphics-software-userspace-wayland.tex | 10 +++--- .../graphics-theory-color.tex | 33 +++++++++++-------- .../graphics-theory-image.tex | 4 +-- 7 files changed, 41 insertions(+), 25 deletions(-) diff --git a/slides/graphics-hardware-pipelines/graphics-hardware-pipelines.tex b/slides/graphics-hardware-pipelines/graphics-hardware-pipelines.tex index 7dc9240eaf..5137dc7962 100644 --- a/slides/graphics-hardware-pipelines/graphics-hardware-pipelines.tex +++ b/slides/graphics-hardware-pipelines/graphics-hardware-pipelines.tex @@ -25,10 +25,13 @@ \subsection{Pipelines} \item On source-sink boundaries: \begin{itemize} \item Mutually-supported pixel format (or conversion) - \item Mutually-accessible memory (or copy) or internal FIFO + \item Mutually-accessible memory (or copy), or internal FIFO + \item Non-concurrent memory access \end{itemize} - \item Target frame rate (\(fps\)) gives a time budget for pipeline traversal: - \( t_0 + t_2 + t_4 + t_5 < fps^{-1},~ t_0 + t_3 < fps^{-1},~ t_1 + t_4 + t_5 < fps^{-1} \) + \item Latency is the time for an event seen on a first source to propagate to a final sink\\ + \( latency_{max} = max(t_0 + t_2 + t_4 + t_5,~ t_0 + t_3,~ t_1 + t_4 + t_5) \) + \item Target frame rate (\(fps^{-1}\)) is the output rate of a final sink \\ + \textit{latency in the worst case, better if multiple blocks are running in parallel} \end{itemize} \vspace{1em} diff --git a/slides/graphics-software-linux-drm/graphics-software-linux-drm.tex b/slides/graphics-software-linux-drm/graphics-software-linux-drm.tex index 23fdda7a38..4f4f25604c 100644 --- a/slides/graphics-software-linux-drm/graphics-software-linux-drm.tex +++ b/slides/graphics-software-linux-drm/graphics-software-linux-drm.tex @@ -553,6 +553,7 @@ \subsection{Linux Kernel: DRM} \begin{itemize} \item Linux GPU Driver Developer’s Guide: \url{https://www.kernel.org/doc/html/latest/gpu/index.html} \item Man pages about userspace aspects: \code{drm}, \code{drm-kms}, \code{drm-memory} + \item \textit{A Current Overview of the DRM KMS Driver-Side APIs} (EOSS 2023) \end{itemize} \end{itemize} \end{frame} diff --git a/slides/graphics-software-linux-tty/graphics-software-linux-tty.tex b/slides/graphics-software-linux-tty/graphics-software-linux-tty.tex index 3feab0a359..904c642b6f 100644 --- a/slides/graphics-software-linux-tty/graphics-software-linux-tty.tex +++ b/slides/graphics-software-linux-tty/graphics-software-linux-tty.tex @@ -31,7 +31,7 @@ \subsection{Linux Kernel: TTY} \begin{frame}[fragile]{Virtual terminals and graphics} \begin{itemize} - \item With VTs, the kernel is \textbf{already using} the display and keyboard! + \item With VTs, the kernel is \textbf{already using} the display and keyboard (seat)! \item Display servers need to switch to \textbf{graphics mode} to release the display: \begin{minted}[fontsize=\small]{console} ret = ioctl(tty_fd, KDSETMODE, KD_GRAPHICS); @@ -71,5 +71,6 @@ \subsection{Linux Kernel: TTY} ret = ioctl(tty_fd, VT_RELDISP, 1); /* when leaving VT */ \end{minted} \item Failure to acknowledge will cause a \textbf{system hang} + \item May be delegated to a deamon such as \code{seatd} \end{itemize} \end{frame} diff --git a/slides/graphics-software-userspace-mesa/graphics-software-userspace-mesa.tex b/slides/graphics-software-userspace-mesa/graphics-software-userspace-mesa.tex index aa1f2e3442..f3b75827b4 100644 --- a/slides/graphics-software-userspace-mesa/graphics-software-userspace-mesa.tex +++ b/slides/graphics-software-userspace-mesa/graphics-software-userspace-mesa.tex @@ -369,7 +369,11 @@ \subsection{Userspace: Mesa 3D} \item Shader-related, see \url{https://www.mesa3d.org/shading.html} \item \code{LIBGL_DEBUG=verbose} for OpenGL, \code{EGL_LOG_LEVEL=debug} for EGL \end{itemize} - \item \code{eglinfo} and \code{glxinfo} show information about the implementation + \item \textbf{Test programs} can be used to debug and test: + \begin{itemize} + \item \code{eglinfo} and \code{glxinfo} show information about the implementation + \item \code{kmscube} implements EGL via GBM and DRM KMS + \end{itemize} \item \textbf{Community} contact: \begin{itemize} \item Mailing list: \code{dri-devel@lists.freedesktop.org} diff --git a/slides/graphics-software-userspace-wayland/graphics-software-userspace-wayland.tex b/slides/graphics-software-userspace-wayland/graphics-software-userspace-wayland.tex index e9d54751ec..c26f1142da 100644 --- a/slides/graphics-software-userspace-wayland/graphics-software-userspace-wayland.tex +++ b/slides/graphics-software-userspace-wayland/graphics-software-userspace-wayland.tex @@ -15,7 +15,7 @@ \subsection{Userspace: Wayland} \begin{itemize} \item Isolates the input and output of each client \item Only the compositor can access display buffers \textit{(and provide screenshots)} - \item Avoids running the compositor as root (using \code{systemd-logind}) + \item Avoids running the compositor as root (using \code{systemd-logind} or \code{seatd}) \end{itemize} \item No network support (can be implemented by compositors) \item \textbf{Weston} is the reference \textbf{Wayland} compositor @@ -177,7 +177,7 @@ \subsection{Userspace: Wayland} \item Most toolkits have support for it: GTK 3, Qt 5, SDL, EFL \item Major desktop environments support it: GNOME 3, KDE Plasma 5 \item Integrated sessions with login managers from \code{/usr/share/wayland-sessions} - \item Runs with user privileges with \code{systemd-logind} + \item Runs with user privileges with \code{systemd-logind} or \code{seatd} \end{itemize} \item X11 applications can be integrated using \textbf{XWayland} \begin{itemize} @@ -207,13 +207,11 @@ \subsection{Userspace: Wayland} \begin{itemize} \item \textbf{Debugging} tips: \begin{itemize} - \item Supported global object interfaces can be listed with \code{weston-info} \item The \code{WAYLAND_DEBUG} environment variable enables protocol tracing \end{itemize} \item \textbf{Weston debugging}: \begin{itemize} - \item Debug arguments: \code{--debug}, \code{--log=file.log} - \item Grabbing a different TTY argument: \code{--tty 1} + \item Debug arguments: \code{--debug}, \code{--log=file.log}, \code{-l log,drm-backend} \item Wireframe keystroke: \code{mod + shift + space + F} \item Timeline recording (to a JSON file) keystroke: \code{mod + shift + space + d}\\ \textit{can produce a graph with \url{https://github.com/ppaalanen/wesgr}} @@ -227,6 +225,8 @@ \subsection{Userspace: Wayland} \begin{itemize} \item Online wiki of the project: \url{https://wayland.freedesktop.org/} \item Online documentation: \url{https://wayland.freedesktop.org/docs/html/} + \item Wayland book: \url{https://wayland-book.com} + \item Wayland protocols: \url{https://wayland.app/protocols/} \end{itemize} \end{itemize} \end{frame} diff --git a/slides/graphics-theory-color/graphics-theory-color.tex b/slides/graphics-theory-color/graphics-theory-color.tex index 1a4fd9f40b..c512407c69 100644 --- a/slides/graphics-theory-color/graphics-theory-color.tex +++ b/slides/graphics-theory-color/graphics-theory-color.tex @@ -13,10 +13,11 @@ \subsection{Color Representation} \end{itemize} \item \textbf{Translating} light information (colors) to numbers: \begin{itemize} - \item A \textbf{color model} defines a base of color components\\ - \textit{typically 3 components (e.g. red, green, blue)} - \item A \textbf{colorspace} is a precise translation referential\\ - \textit{unique association of a color and coordinates in the base} + \item Colors are represented in a \textbf{tridimensional linear space}\\ + \textit{Grassmann’s laws approximation, generally using RGB} + \item A \textbf{colorspace} defines a complete translation referential\\ + \textit{with specific primaries as a base and a white point} + \item A triplet of values represents a \textbf{unique color} in a colorspace \item The \textbf{color gamut} is the range of colors in the colorspace\\ \textit{not every color can be represented in every colorspace} \end{itemize} @@ -53,6 +54,7 @@ \subsection{Color Representation} \item Increasing the resolution reduces the range \item Increasing the range reduces the resolution \end{itemize} + \item A \textbf{transfer function} (EOTF/gamma) allows non-linear encoding \end{itemize} \end{minipage} \hfill @@ -130,15 +132,17 @@ \subsection{Color Representation} \end{minipage} \end{frame} -\begin{frame}{Colorspaces and channels} +\begin{frame}{Color transforms and channels} \begin{minipage}[b]{0.7\textwidth} \begin{itemize} - \item Each component of a color model is called a \textbf{channel} - \item Examples for usual types of color models: + \item Each component of a color triplet is called a \textbf{channel} \begin{itemize} - \item RGB, with 3 channels:\\ \textbf{R} (red) / \textbf{G} (green) / \textbf{B} (blue) - \item HSV, with 3 channels:\\ \textbf{H} (hue) / \textbf{S} (saturation) / \textbf{V} (value) - \item YUV or Y/Cb/Cr, with 3 channels:\\ \textbf{Y} (luminance) / \textbf{U or Cb} / \textbf{V or Cr} (chrominance) + \item RGB: \textbf{R} (red) / \textbf{G} (green) / \textbf{B} (blue) + \end{itemize} + \item Color transforms introduce different bases : + \begin{itemize} + \item HSV: \textbf{H} (hue) / \textbf{S} (saturation) / \textbf{V} (value) + \item YCbCr/YUV: \textbf{Y} (luminance) / \textbf{Cb} / \textbf{Cr} (chrominance) \end{itemize} \end{itemize} \end{minipage} @@ -153,8 +157,11 @@ \subsection{Color Representation} \begin{itemize} \item An additional channel can exist for transparency: the \textbf{alpha channel}\\ \textit{mostly relevant for composition, not for final display} - \item Color coordinates can be \textbf{converted} between colorspaces and color models\\ - \textit{using translation formulas and associated constants} + \item Color triplets can be converted between representations: + \begin{itemize} + \item Between \textbf{color transforms} using given formulas (colorspace is preserved) + \item Between \textbf{colorspaces} using constants and adaptation for limited gamuts + \end{itemize} \end{itemize} \end{frame} @@ -196,6 +203,6 @@ \subsection{Color Representation} \end{minipage} \begin{center} - \textit{\small Translation between BT.601 YUV and sRGB colorspaces} + \textit{\small Translation between BT.601 in YUV and sRGB in RGB} \end{center} \end{frame} diff --git a/slides/graphics-theory-image/graphics-theory-image.tex b/slides/graphics-theory-image/graphics-theory-image.tex index 8e6aeb2b3a..9d88ee291d 100644 --- a/slides/graphics-theory-image/graphics-theory-image.tex +++ b/slides/graphics-theory-image/graphics-theory-image.tex @@ -32,9 +32,9 @@ \subsection{Image Representation} \begin{frame}{Light, pixels and pictures} \begin{itemize} - \item \textbf{Pictures} are bi-dimensional ordered ensembles of pixels (also called \textbf{frames}): + \item \textbf{Pictures} (or frames) are bi-dimensional ordered ensembles of pixels: \begin{itemize} - \item Frames have \textbf{dimensions}: \textbf{width} (horizontal) and \textbf{height} (vertical) + \item Frames have \textbf{dimensions} (or size): \textbf{width} (horizontal) and \textbf{height} (vertical) \item The aspect ratio is the \textbf{width:height} ratio (e.g. 16:9, 4:3) \item Pixels are located with a \textbf{position}: \textbf{(x,y)} \item The dimension and position unit is the number of pixels From 2cf140d480e761800ae9ecb77948b2da7ead2107 Mon Sep 17 00:00:00 2001 From: Paul Kocialkowski Date: Mon, 24 Nov 2025 21:09:44 +0100 Subject: [PATCH 6/6] slides: graphics: Add introduction slides Signed-off-by: Paul Kocialkowski --- .../graphics-introduction.tex | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 slides/graphics-introduction/graphics-introduction.tex diff --git a/slides/graphics-introduction/graphics-introduction.tex b/slides/graphics-introduction/graphics-introduction.tex new file mode 100644 index 0000000000..1ca09d445a --- /dev/null +++ b/slides/graphics-introduction/graphics-introduction.tex @@ -0,0 +1,80 @@ +\input{../common/graphics-title.tex} + +\titleframe + +\begin{frame}{\trainingtitle{} training overview} + +Goals: +\begin{itemize} +\item Understand how \textbf{pictures and colors} are represented in digital systems +\item Understand the role and operation of different \textbf{graphics hardware devices} +\item Know about the good practices for \textbf{performance and system integration} +\item Distinguish the different \textbf{parts of the graphics stack} and their individual role +\item Know about the various \textbf{implementations} used in Linux-based stacks +\item Know about the \textbf{modern replacements} to legacy technologies +\end{itemize} +~\\ + +Prerequisites: +\begin{itemize} +\item Familiarity with basic concepts about digital hardware +\item Familiarity with the C programming language and low-level programming +\end{itemize} + +\end{frame} + +\begin{frame}{Base Theory and Concepts About Graphics} + +Lecture: +\begin{itemize} +\item \textbf{Image Representation} \textit{how pictures are represented spatially} +\item \textbf{Color Representation} \textit{how colors are represented} +\item \textbf{Layouts and Formats} \textit{how pictures are stored in memory} +\item \textbf{Pixel Drawing} \textit{basic methods for drawing pixels} +\item \textbf{Pixel Operations} \textit{basic operations on pixels} +\end{itemize} +~\\ + +Demos: +\begin{itemize} +\item Frequency domain visualization example +\item Pixel drawing and operations implementation walk-through +\end{itemize} + +\end{frame} + +\begin{frame}{Hardware Aspects} +Lecture: +\begin{itemize} +\item \textbf{Overview} \textit{different types of hardware components} +\item \textbf{Pipelines} \textit{connecting hardware components together} +\item \textbf{Display Devices} \textit{how display technologies work} +\item \textbf{Display Interfaces} \textit{transmitting pixels over to displays} +\item \textbf{Render Accelerators} \textit{hardware architectures for rendering pixels} +\item \textbf{System Integration, Memory and Performance}\\\textit{interactions between graphics devices, the CPU and the system} +\end{itemize} +\end{frame} + + +\begin{frame}{Software Aspects} +Lecture: +\begin{itemize} +\item \textbf{Stack Overview} \textit{the software components and their relationships} +\item \textbf{Linux Kernel: TTY} \textit{dealing with graphics/input seats} +\item \textbf{Linux Kernel: Framebuffer Device} \textit{the deprecated display uAPI} +\item \textbf{Linux Kernel: DRM} \textit{the modern display uAPI} +\item \textbf{Userspace: X Window} \textit{the deprecated display server API} +\item \textbf{Userspace: Wayland} \textit{the modern display server API} +\item \textbf{Userspace: Mesa 3D} \textit{the reference 3D stack implementation} +\end{itemize} +~\\ +Demos: +\begin{itemize} +\item DRM uAPI implementation walk-through +\item Wayland implementation walk-through +\item Other illustration examples +\end{itemize} +\end{frame} + +\input{../slides/first-slides/paul-kocialkowski.tex} +\input{../slides/about-us/about-us.tex}