fix(linux): export all Wayland DMA-BUF planes

Modifier-aware GBM allocation can return multi-plane buffers, but the wlroots capture path only described plane zero to linux-dmabuf. Query and export each plane fd, stride, and offset, then add every plane to the Wayland parameter object.

Allocate modifier-backed buffers with rendering usage and close exported descriptors when export or asynchronous buffer creation fails.

Add unit coverage for multi-plane metadata, unsupported plane counts, and partial-export cleanup.

src/platform/linux/wayland.cpp

@@ -34,6 +34,16 @@ using namespace std::literals;
 
 namespace wl {
 
+  namespace {
+    const gbm_bo_accessors_t gbm_bo_accessors {
+      .get_plane_count = gbm_bo_get_plane_count,
+      .get_fd_for_plane = gbm_bo_get_fd_for_plane,
+      .get_stride_for_plane = gbm_bo_get_stride_for_plane,
+      .get_offset = gbm_bo_get_offset,
+      .get_modifier = gbm_bo_get_modifier,
+    };
+  }  // namespace
+
   // Helper to call C++ method from wayland C callback
   template<class T, class Method, Method m, class... Params>
   static auto classCall(void *data, Params... params) -> decltype(((*reinterpret_cast<T *>(data)).*m)(params...)) {
@@ -202,8 +212,10 @@ namespace wl {
   }
 
   void interface_t::dmabuf_modifier(zwp_linux_dmabuf_v1 *zwp_linux_dmabuf, uint32_t format, uint32_t modifier_hi, uint32_t modifier_lo) {
-    uint64_t modifier = ((uint64_t) modifier_hi << 32) | modifier_lo;
-    supported_modifiers[format].push_back(modifier);
+    const auto modifier = (static_cast<std::uint64_t>(modifier_hi) << 32) | modifier_lo;
+    if (modifier != DRM_FORMAT_MOD_INVALID) {
+      supported_modifiers[format].push_back(modifier);
+    }
   }
 
   void interface_t::add_interface(
@@ -373,6 +385,31 @@ namespace wl {
     BOOST_LOG(verbose) << "Frame flags: "sv << flags << (y_invert ? " (y_invert)" : "");
   }
 
+  std::optional<std::uint32_t> export_gbm_bo_planes(gbm_bo *bo, frame_t &frame, const gbm_bo_accessors_t &accessors) {
+    frame.destroy();
+
+    const auto plane_count = accessors.get_plane_count(bo);
+    if (plane_count <= 0 || plane_count > static_cast<int>(std::size(frame.sd.fds))) {
+      BOOST_LOG(error) << "[wayland] GBM buffer has unsupported plane count ["sv << plane_count << ']';
+      return std::nullopt;
+    }
+
+    frame.sd.modifier = accessors.get_modifier(bo);
+    for (auto plane = 0; plane < plane_count; ++plane) {
+      frame.sd.fds[plane] = accessors.get_fd_for_plane(bo, plane);
+      if (frame.sd.fds[plane] < 0) {
+        BOOST_LOG(error) << "[wayland] Failed to export DMA-BUF plane ["sv << plane << ']';
+        frame.destroy();
+        return std::nullopt;
+      }
+
+      frame.sd.pitches[plane] = accessors.get_stride_for_plane(bo, plane);
+      frame.sd.offsets[plane] = accessors.get_offset(bo, plane);
+    }
+
+    return static_cast<std::uint32_t>(plane_count);
+  }
+
   // DMA-BUF creation helper
   void dmabuf_t::create_and_copy_dmabuf(zwlr_screencopy_frame_v1 *frame) {
     if (!init_gbm()) {
@@ -386,7 +423,7 @@ namespace wl {
     if (supported_modifiers) {
       auto it = supported_modifiers->find(dmabuf_info.format);
       if (it != supported_modifiers->end() && !it->second.empty()) {
-        current_bo = gbm_bo_create_with_modifiers(gbm_device, dmabuf_info.width, dmabuf_info.height, dmabuf_info.format, it->second.data(), it->second.size());
+        current_bo = gbm_bo_create_with_modifiers2(gbm_device, dmabuf_info.width, dmabuf_info.height, dmabuf_info.format, it->second.data(), it->second.size(), GBM_BO_USE_RENDERING);
       }
     }
 
@@ -401,10 +438,10 @@ namespace wl {
       return;
     }
 
-    // Get buffer info
-    int fd = gbm_bo_get_fd(current_bo);
-    if (fd < 0) {
-      BOOST_LOG(error) << "Failed to get buffer FD"sv;
+    // Export every memory plane into the surface descriptor
+    auto next_frame = get_next_frame();
+    const auto plane_count = export_gbm_bo_planes(current_bo, *next_frame, gbm_bo_accessors);
+    if (!plane_count) {
       gbm_bo_destroy(current_bo);
       current_bo = nullptr;
       zwlr_screencopy_frame_v1_destroy(frame);
@@ -412,19 +449,19 @@ namespace wl {
       return;
     }
 
-    uint32_t stride = gbm_bo_get_stride(current_bo);
-    uint64_t modifier = gbm_bo_get_modifier(current_bo);
-
-    // Store in surface descriptor for later use
-    auto next_frame = get_next_frame();
-    next_frame->sd.fds[0] = fd;
-    next_frame->sd.pitches[0] = stride;
-    next_frame->sd.offsets[0] = 0;
-    next_frame->sd.modifier = modifier;
-
     // Create linux-dmabuf buffer
     auto params = zwp_linux_dmabuf_v1_create_params(dmabuf_interface);
-    zwp_linux_buffer_params_v1_add(params, fd, 0, 0, stride, modifier >> 32, modifier & 0xffffffff);
+    for (std::uint32_t plane = 0; plane < *plane_count; ++plane) {
+      zwp_linux_buffer_params_v1_add(
+        params,
+        next_frame->sd.fds[plane],
+        plane,
+        next_frame->sd.offsets[plane],
+        next_frame->sd.pitches[plane],
+        next_frame->sd.modifier >> 32,
+        next_frame->sd.modifier & 0xffffffff
+      );
+    }
 
     // Add listener for buffer creation
     zwp_linux_buffer_params_v1_add_listener(params, &params_listener, frame);
@@ -485,6 +522,7 @@ namespace wl {
 
     BOOST_LOG(error) << "[wayland] Failed to create buffer from params"sv;
     self->cleanup_gbm();
+    self->get_next_frame()->destroy();
 
     zwp_linux_buffer_params_v1_destroy(params);
     zwlr_screencopy_frame_v1_destroy(frame);

src/platform/linux/wayland.h

@@ -5,9 +5,11 @@
 #pragma once
 
 // standard includes
+#include <array>
 #include <bitset>
 #include <cstdint>
 #include <map>
+#include <optional>
 #include <vector>
 
 #ifdef SUNSHINE_BUILD_WAYLAND
@@ -25,6 +27,8 @@
  */
 #ifdef SUNSHINE_BUILD_WAYLAND
 
+struct gbm_bo;
+
 namespace wl {
   /**
    * @brief Determine whether wlroots capture should keep frames in VRAM for the requested memory type.
@@ -39,6 +43,17 @@ namespace wl {
    */
   using display_internal_t = util::safe_ptr<wl_display, wl_display_disconnect>;
 
+  /**
+   * @brief GBM buffer accessors used to export DMA-BUF plane metadata.
+   */
+  struct gbm_bo_accessors_t {
+    int (*get_plane_count)(gbm_bo *bo);  ///< Return the number of memory planes in the buffer.
+    int (*get_fd_for_plane)(gbm_bo *bo, int plane);  ///< Duplicate the DMA-BUF descriptor for one plane.
+    std::uint32_t (*get_stride_for_plane)(gbm_bo *bo, int plane);  ///< Return the row stride for one plane.
+    std::uint32_t (*get_offset)(gbm_bo *bo, int plane);  ///< Return the byte offset for one plane.
+    std::uint64_t (*get_modifier)(gbm_bo *bo);  ///< Return the DRM format modifier shared by the planes.
+  };
+
   /**
    * @brief Captured Wayland frame metadata and DMA-BUF surface state.
    */
@@ -54,6 +69,16 @@ namespace wl {
     std::optional<std::chrono::steady_clock::time_point> frame_timestamp;  ///< Capture timestamp associated with the frame.
   };
 
+  /**
+   * @brief Export every GBM buffer plane into a captured frame descriptor.
+   *
+   * @param bo GBM buffer whose DMA-BUF planes will be exported.
+   * @param frame Frame that takes ownership of the exported file descriptors.
+   * @param accessors GBM accessors used to query the buffer.
+   * @return Number of exported planes, or no value when the buffer cannot be exported.
+   */
+  std::optional<std::uint32_t> export_gbm_bo_planes(gbm_bo *bo, frame_t &frame, const gbm_bo_accessors_t &accessors);
+
   /**
    * @brief Listener state for Wayland screencopy frames backed by DMA-BUFs.
    */
@@ -358,12 +383,13 @@ namespace wl {
      */
     void dmabuf_format(zwp_linux_dmabuf_v1 *zwp_linux_dmabuf, uint32_t format);
     /**
-     * @brief Record a DMA-BUF format modifier advertised by the compositor.
+     * @brief Record an explicit DMA-BUF format modifier advertised by the compositor.
      *
      * @param zwp_linux_dmabuf DMA-BUF interface that emitted the modifier event.
      * @param format DRM format associated with the modifier.
      * @param modifier_hi High 32 bits of the DRM format modifier.
      * @param modifier_lo Low 32 bits of the DRM format modifier.
+     * @note The combined DRM_FORMAT_MOD_INVALID value is omitted because implicit allocation uses the fallback path.
      */
     void dmabuf_modifier(zwp_linux_dmabuf_v1 *zwp_linux_dmabuf, uint32_t format, uint32_t modifier_hi, uint32_t modifier_lo);
 

tests/unit/platform/linux/test_wayland.cpp

@@ -3,12 +3,70 @@
  * @brief Test Wayland output mode selection.
  */
 #ifdef SUNSHINE_BUILD_WAYLAND
+  // standard includes
+  #include <array>
+  #include <cerrno>
+
+  // system includes
+  #include <drm_fourcc.h>
+  #include <fcntl.h>
+  #include <unistd.h>
+
   // test includes
   #include "../../../tests_common.h"
 
   // local includes
   #include <src/platform/linux/wayland.h>
 
+namespace {
+  struct fake_gbm_bo_t {
+    int plane_count {};
+    int failed_plane {-1};
+    std::uint64_t modifier {};
+    std::array<std::uint32_t, 4> strides {};
+    std::array<std::uint32_t, 4> offsets {};
+    std::array<int, 4> exported_fds {-1, -1, -1, -1};
+  };
+
+  fake_gbm_bo_t &fake_bo(gbm_bo *bo) {
+    return *static_cast<fake_gbm_bo_t *>(static_cast<void *>(bo));
+  }
+
+  int get_plane_count(gbm_bo *bo) {
+    return fake_bo(bo).plane_count;
+  }
+
+  int get_fd_for_plane(gbm_bo *bo, int plane) {
+    auto &fake = fake_bo(bo);
+    if (plane == fake.failed_plane) {
+      return -1;
+    }
+
+    fake.exported_fds[plane] = open("/dev/null", O_RDONLY | O_CLOEXEC);
+    return fake.exported_fds[plane];
+  }
+
+  std::uint32_t get_stride_for_plane(gbm_bo *bo, int plane) {
+    return fake_bo(bo).strides[plane];
+  }
+
+  std::uint32_t get_offset(gbm_bo *bo, int plane) {
+    return fake_bo(bo).offsets[plane];
+  }
+
+  std::uint64_t get_modifier(gbm_bo *bo) {
+    return fake_bo(bo).modifier;
+  }
+
+  const wl::gbm_bo_accessors_t fake_accessors {
+    .get_plane_count = get_plane_count,
+    .get_fd_for_plane = get_fd_for_plane,
+    .get_stride_for_plane = get_stride_for_plane,
+    .get_offset = get_offset,
+    .get_modifier = get_modifier,
+  };
+}  // namespace
+
 TEST(WaylandMonitorTest, IgnoresNonCurrentModesAroundCurrentMode) {
   wl::monitor_t monitor {nullptr};
 
@@ -45,4 +103,69 @@ TEST(WaylandCaptureTest, UsesVramForCudaOnlyWhenCudaSupportIsBuilt) {
   EXPECT_FALSE(wl::use_vram_capture(platf::mem_type_e::cuda));
   #endif
 }
+
+TEST(WaylandInterfaceTest, RecordsOnlyExplicitDmabufModifiers) {
+  constexpr std::uint32_t format = DRM_FORMAT_XRGB8888;
+  constexpr std::uint64_t explicit_modifier = 0x100000000000004;
+  wl::interface_t interface;
+
+  interface.dmabuf_modifier(nullptr, format, DRM_FORMAT_MOD_INVALID >> 32, DRM_FORMAT_MOD_INVALID & 0xffffffff);
+  interface.dmabuf_modifier(nullptr, format, explicit_modifier >> 32, explicit_modifier & 0xffffffff);
+
+  const auto modifiers = interface.supported_modifiers.find(format);
+  ASSERT_NE(modifiers, interface.supported_modifiers.end());
+  ASSERT_EQ(modifiers->second.size(), 1);
+  EXPECT_EQ(modifiers->second.front(), explicit_modifier);
+}
+
+TEST(WaylandCaptureTest, ExportsEveryGbmBufferPlane) {
+  fake_gbm_bo_t bo {
+    .plane_count = 2,
+    .modifier = 0x100000000000004,
+    .strides = {10240, 256},
+    .offsets = {0, 0x800000},
+  };
+  wl::frame_t frame;
+
+  const auto plane_count = wl::export_gbm_bo_planes(static_cast<gbm_bo *>(static_cast<void *>(&bo)), frame, fake_accessors);
+
+  ASSERT_EQ(plane_count, 2);
+  EXPECT_EQ(frame.sd.modifier, bo.modifier);
+  for (std::size_t plane = 0; plane < *plane_count; ++plane) {
+    EXPECT_EQ(frame.sd.fds[plane], bo.exported_fds[plane]);
+    EXPECT_EQ(frame.sd.pitches[plane], bo.strides[plane]);
+    EXPECT_EQ(frame.sd.offsets[plane], bo.offsets[plane]);
+  }
+  EXPECT_EQ(frame.sd.fds[2], -1);
+  EXPECT_EQ(frame.sd.fds[3], -1);
+
+  frame.destroy();
+}
+
+TEST(WaylandCaptureTest, RejectsUnsupportedGbmPlaneCounts) {
+  wl::frame_t frame;
+  fake_gbm_bo_t empty_bo {};
+  fake_gbm_bo_t oversized_bo {
+    .plane_count = 5,
+  };
+
+  EXPECT_FALSE(wl::export_gbm_bo_planes(static_cast<gbm_bo *>(static_cast<void *>(&empty_bo)), frame, fake_accessors));
+  EXPECT_FALSE(wl::export_gbm_bo_planes(static_cast<gbm_bo *>(static_cast<void *>(&oversized_bo)), frame, fake_accessors));
+}
+
+TEST(WaylandCaptureTest, ClosesExportedPlanesAfterLaterPlaneFails) {
+  fake_gbm_bo_t bo {
+    .plane_count = 2,
+    .failed_plane = 1,
+  };
+  wl::frame_t frame;
+
+  EXPECT_FALSE(wl::export_gbm_bo_planes(static_cast<gbm_bo *>(static_cast<void *>(&bo)), frame, fake_accessors));
+  ASSERT_GE(bo.exported_fds[0], 0);
+  const auto fd_status = fcntl(bo.exported_fds[0], F_GETFD);
+  const auto fd_error = errno;
+  EXPECT_EQ(fd_status, -1);
+  EXPECT_EQ(fd_error, EBADF);
+  EXPECT_EQ(frame.sd.fds[0], -1);
+}
 #endif