feat(linux/wlr): enable Vulkan encoding for wlroots capture (#5734)

README.md

@@ -357,7 +357,7 @@ LizardByte has the full documentation hosted on [Read the Docs](https://docs.liz
     <tr>
         <td>Wayland (wlroots)</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>✅</td>
         <td>✅</td>
         <td>✅</td>
     </tr>

src/platform/linux/vulkan_encode.cpp

@@ -7,6 +7,7 @@
 #include <array>
 #include <cstdint>
 #include <drm_fourcc.h>
+#include <map>
 #include <sys/stat.h>
 #if defined(__FreeBSD__)
   #include <sys/types.h>
@@ -39,6 +40,132 @@ using namespace std::literals;
 
 namespace vk {
 
+  /**
+   * @brief Find Vulkan physical device matching a render node path.
+   *
+   * @param devs List of Vulkan physical devices.
+   * @param render_path Path to render node (e.g. /dev/dri/renderD128).
+   * @return Matching device, or devs[0] if no match found.
+   */
+  static VkPhysicalDevice find_device_by_render_node(const std::vector<VkPhysicalDevice> &devs, const std::string &render_path) {
+    if (render_path.empty() || render_path[0] != '/') {
+      return devs[0];
+    }
+
+    struct stat node_stat;
+    if (stat(render_path.c_str(), &node_stat) != 0) {
+      return devs[0];
+    }
+
+    auto target_major = major(node_stat.st_rdev);
+    auto target_minor = minor(node_stat.st_rdev);
+
+    for (const auto &dev : devs) {
+      VkPhysicalDeviceDrmPropertiesEXT drm = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRM_PROPERTIES_EXT};
+      VkPhysicalDeviceProperties2 props2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2};
+      props2.pNext = &drm;
+      vkGetPhysicalDeviceProperties2(dev, &props2);
+
+      if (drm.hasRender && drm.renderMajor == (int64_t) target_major && drm.renderMinor == (int64_t) target_minor) {
+        return dev;
+      }
+    }
+
+    return devs[0];
+  }
+
+  /**
+   * @brief Query supported modifiers for a single format.
+   *
+   * @param phys_dev Vulkan physical device.
+   * @param vk_fmt Vulkan format to query.
+   * @return Vector of supported modifiers, or empty if none.
+   */
+  static std::vector<std::uint64_t> query_format_modifiers(VkPhysicalDevice phys_dev, VkFormat vk_fmt) {
+    VkDrmFormatModifierPropertiesListEXT mod_list = {.sType = VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT};
+    VkFormatProperties2 fmt_props2 = {.sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2};
+    fmt_props2.pNext = &mod_list;
+
+    vkGetPhysicalDeviceFormatProperties2(phys_dev, vk_fmt, &fmt_props2);
+
+    if (mod_list.drmFormatModifierCount == 0) {
+      return {};
+    }
+
+    std::vector<VkDrmFormatModifierPropertiesEXT> mod_props(mod_list.drmFormatModifierCount);
+    mod_list.pDrmFormatModifierProperties = mod_props.data();
+    vkGetPhysicalDeviceFormatProperties2(phys_dev, vk_fmt, &fmt_props2);
+
+    std::vector<std::uint64_t> modifiers;
+    modifiers.reserve(mod_props.size());
+    for (const auto &mp : mod_props) {
+      // Only include modifiers that support sampled images (needed for compute shader input)
+      if (mp.drmFormatModifierTilingFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT) {
+        modifiers.push_back(mp.drmFormatModifier);
+      }
+    }
+
+    return modifiers;
+  }
+
+  /**
+   * @brief Query DRM format modifiers supported by the Vulkan driver for common capture formats.
+   *
+   * Creates a temporary Vulkan instance and physical device to query modifier support
+   * via VK_EXT_image_drm_format_modifier extension.
+   *
+   * @return Map of DRM format to supported modifiers, or empty map if query fails.
+   */
+  std::map<std::uint32_t, std::vector<std::uint64_t>> get_supported_capture_modifiers() {
+    std::map<std::uint32_t, std::vector<std::uint64_t>> result;
+
+    // Create temporary Vulkan instance
+    VkApplicationInfo app = {.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO};
+    app.apiVersion = VK_API_VERSION_1_1;
+
+    VkInstanceCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO};
+    ci.pApplicationInfo = &app;
+    VkInstance inst = VK_NULL_HANDLE;
+    if (vkCreateInstance(&ci, nullptr, &inst) != VK_SUCCESS) {
+      BOOST_LOG(warning) << "[vulkan] Failed to create instance for modifier query"sv;
+      return result;
+    }
+
+    // Get physical devices
+    uint32_t count = 0;
+    vkEnumeratePhysicalDevices(inst, &count, nullptr);
+    if (count == 0) {
+      vkDestroyInstance(inst, nullptr);
+      return result;
+    }
+
+    std::vector<VkPhysicalDevice> devs(count);
+    vkEnumeratePhysicalDevices(inst, &count, devs.data());
+
+    // Find the device matching the render node that will be used for encoding
+    VkPhysicalDevice phys_dev = find_device_by_render_node(devs, platf::resolve_render_device());
+
+    // Common capture formats (ARGB/XRGB variants)
+    static const std::array<std::pair<uint32_t, VkFormat>, 4> formats_to_query = {{
+      {DRM_FORMAT_ARGB8888, VK_FORMAT_B8G8R8A8_UNORM},
+      {DRM_FORMAT_XRGB8888, VK_FORMAT_B8G8R8A8_UNORM},
+      {DRM_FORMAT_ABGR8888, VK_FORMAT_R8G8B8A8_UNORM},
+      {DRM_FORMAT_XBGR8888, VK_FORMAT_R8G8B8A8_UNORM},
+    }};
+
+    for (const auto &[drm_fmt, vk_fmt] : formats_to_query) {
+      auto modifiers = query_format_modifiers(phys_dev, vk_fmt);
+      if (!modifiers.empty()) {
+        BOOST_LOG(debug) << "[vulkan] Format 0x"sv << std::hex << drm_fmt << std::dec
+                         << " has "sv << modifiers.size() << " supported modifiers"sv;
+        result[drm_fmt] = std::move(modifiers);
+      }
+    }
+
+    vkDestroyInstance(inst, nullptr);
+    return result;
+  }
+
   /**
    * @brief Query and validate encoder quality level for a given codec.
    *

src/platform/linux/vulkan_encode.h

@@ -6,10 +6,24 @@
 
 #include "src/platform/common.h"
 
+#include <cstdint>
+#include <map>
+#include <vector>
+
 extern "C" struct AVBufferRef;
 
 namespace vk {
 
+  /**
+   * @brief Query DRM format modifiers supported by the Vulkan driver for common capture formats.
+   *
+   * This queries the Vulkan driver for modifiers it can import via VK_EXT_image_drm_format_modifier.
+   * The returned map is keyed by DRM fourcc format code (e.g. DRM_FORMAT_ARGB8888).
+   *
+   * @return Map of DRM format to supported modifiers, or empty map if query fails.
+   */
+  std::map<std::uint32_t, std::vector<std::uint64_t>> get_supported_capture_modifiers();
+
   /**
    * @brief Initialize Vulkan hardware device for FFmpeg encoding.
    * @param encode_device The encode device (vk_t).

src/platform/linux/wayland.cpp

@@ -292,6 +292,37 @@ namespace wl {
     }
   }
 
+  std::map<std::uint32_t, std::vector<std::uint64_t>> intersect_modifiers(
+    const std::map<std::uint32_t, std::vector<std::uint64_t>> &compositor_modifiers,
+    const std::map<std::uint32_t, std::vector<std::uint64_t>> &encoder_modifiers
+  ) {
+    std::map<std::uint32_t, std::vector<std::uint64_t>> result;
+
+    for (const auto &[format, comp_mods] : compositor_modifiers) {
+      auto it = encoder_modifiers.find(format);
+      if (it == encoder_modifiers.end()) {
+        continue;
+      }
+
+      const auto &enc_mods = it->second;
+      std::vector<std::uint64_t> common;
+
+      for (auto mod : comp_mods) {
+        if (std::find(enc_mods.begin(), enc_mods.end(), mod) != enc_mods.end()) {
+          common.push_back(mod);
+        }
+      }
+
+      if (!common.empty()) {
+        BOOST_LOG(debug) << "[wayland] Format 0x"sv << std::hex << format << std::dec
+                         << " has "sv << common.size() << " common modifiers"sv;
+        result[format] = std::move(common);
+      }
+    }
+
+    return result;
+  }
+
   dmabuf_t::dmabuf_t():
       status {READY},
       frames {},
@@ -313,10 +344,12 @@ namespace wl {
     zwp_linux_dmabuf_v1 *dmabuf_interface,
     const std::map<std::uint32_t, std::vector<std::uint64_t>> *supported_modifiers,
     wl_output *output,
-    bool blend_cursor
+    bool blend_cursor,
+    const std::map<std::uint32_t, std::vector<std::uint64_t>> *encoder_modifiers
   ) {
     this->dmabuf_interface = dmabuf_interface;
     this->supported_modifiers = supported_modifiers;
+    this->encoder_modifiers = encoder_modifiers;
     // Reset state
     shm_info.supported = false;
     dmabuf_info.supported = false;
@@ -419,10 +452,11 @@ namespace wl {
       return;
     }
 
-    // Create GBM buffer
-    if (supported_modifiers) {
-      auto it = supported_modifiers->find(dmabuf_info.format);
-      if (it != supported_modifiers->end() && !it->second.empty()) {
+    // Create GBM buffer - prefer encoder-intersected modifiers when available
+    const std::map<std::uint32_t, std::vector<std::uint64_t>> *modifiers_to_use = encoder_modifiers ? encoder_modifiers : supported_modifiers;
+    if (modifiers_to_use) {
+      auto it = modifiers_to_use->find(dmabuf_info.format);
+      if (it != modifiers_to_use->end() && !it->second.empty()) {
         current_bo = gbm_bo_create_with_modifiers2(gbm_device, dmabuf_info.width, dmabuf_info.height, dmabuf_info.format, it->second.data(), it->second.size(), GBM_BO_USE_RENDERING);
       }
     }

src/platform/linux/wayland.h

@@ -38,6 +38,18 @@ namespace wl {
    */
   bool use_vram_capture(platf::mem_type_e hwdevice_type);
 
+  /**
+   * @brief Intersect compositor and encoder modifiers to find common formats.
+   *
+   * @param compositor_modifiers Modifiers supported by the compositor for each format.
+   * @param encoder_modifiers Modifiers supported by the encoder for each format.
+   * @return Intersection of formats and modifiers supported by both.
+   */
+  std::map<std::uint32_t, std::vector<std::uint64_t>> intersect_modifiers(
+    const std::map<std::uint32_t, std::vector<std::uint64_t>> &compositor_modifiers,
+    const std::map<std::uint32_t, std::vector<std::uint64_t>> &encoder_modifiers
+  );
+
   /**
    * @brief Owning pointer for a Wayland display connection.
    */
@@ -109,8 +121,9 @@ namespace wl {
      * @param supported_modifiers DMA-BUF format modifiers supported by the compositor.
      * @param output Wayland output to capture.
      * @param blend_cursor Whether the compositor should include the cursor in the frame.
+     * @param encoder_modifiers Optional modifiers supported by the encoder for format intersection.
      */
-    void listen(zwlr_screencopy_manager_v1 *screencopy_manager, zwp_linux_dmabuf_v1 *dmabuf_interface, const std::map<std::uint32_t, std::vector<std::uint64_t>> *supported_modifiers, wl_output *output, bool blend_cursor = false);
+    void listen(zwlr_screencopy_manager_v1 *screencopy_manager, zwp_linux_dmabuf_v1 *dmabuf_interface, const std::map<std::uint32_t, std::vector<std::uint64_t>> *supported_modifiers, wl_output *output, bool blend_cursor = false, const std::map<std::uint32_t, std::vector<std::uint64_t>> *encoder_modifiers = nullptr);
     /**
      * @brief Store the Wayland buffer created for a DMA-BUF parameter request.
      *
@@ -205,6 +218,7 @@ namespace wl {
 
     zwp_linux_dmabuf_v1 *dmabuf_interface {nullptr};
     const std::map<std::uint32_t, std::vector<std::uint64_t>> *supported_modifiers {nullptr};
+    const std::map<std::uint32_t, std::vector<std::uint64_t>> *encoder_modifiers {nullptr};  ///< Optional encoder modifiers for intersection.
 
     struct {
       bool supported {false};

src/platform/linux/wlgrab.cpp

@@ -11,6 +11,7 @@
 #include "src/platform/common.h"
 #include "src/video.h"
 #include "vaapi.h"
+#include "vulkan_encode.h"
 #include "wayland.h"
 
 using namespace std::literals;
@@ -24,6 +25,10 @@ namespace wl {
       return true;
     }
 
+    if (hwdevice_type == platf::mem_type_e::vulkan) {
+      return true;
+    }
+
 #ifdef SUNSHINE_BUILD_CUDA
     if (hwdevice_type == platf::mem_type_e::cuda) {
       return true;
@@ -169,13 +174,14 @@ namespace wl {
      * @param img_out Captured wlroots image returned to the streaming pipeline.
      * @param timeout Maximum time to wait for the operation.
      * @param cursor Cursor image or visibility state to composite.
+     * @param encoder_modifiers Optional encoder modifiers to intersect with compositor modifiers.
      * @return Capture status reported to the streaming pipeline.
      */
-    inline platf::capture_e snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool cursor) {
+    inline platf::capture_e snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool cursor, const std::map<std::uint32_t, std::vector<std::uint64_t>> *encoder_modifiers = nullptr) {
       auto to = std::chrono::steady_clock::now() + timeout;
 
       // Dispatch events until we get a new frame or the timeout expires
-      dmabuf.listen(interface.screencopy_manager, interface.dmabuf_interface, &interface.supported_modifiers, output, cursor);
+      dmabuf.listen(interface.screencopy_manager, interface.dmabuf_interface, &interface.supported_modifiers, output, cursor, encoder_modifiers);
       do {
         auto remaining_time_ms = std::chrono::duration_cast<std::chrono::milliseconds>(to - std::chrono::steady_clock::now());
         if (remaining_time_ms.count() < 0 || !display.dispatch(remaining_time_ms)) {
@@ -409,7 +415,9 @@ namespace wl {
      * @return Capture status reported to the streaming pipeline.
      */
     platf::capture_e snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool cursor) {
-      auto status = wlr_t::snapshot(pull_free_image_cb, img_out, timeout, cursor);
+      // For vulkan, use intersected modifiers; for others, pass nullptr (use compositor modifiers)
+      const std::map<std::uint32_t, std::vector<std::uint64_t>> *mods = intersected_modifiers.empty() ? nullptr : &intersected_modifiers;
+      auto status = wlr_t::snapshot(pull_free_image_cb, img_out, timeout, cursor, mods);
       if (status != platf::capture_e::ok) {
         return status;
       }
@@ -473,6 +481,10 @@ namespace wl {
       }
 #endif
 
+      if (mem_type == platf::mem_type_e::vulkan) {
+        return vk::make_avcodec_encode_device_vram(width, height, 0, 0);
+      }
+
       return std::make_unique<platf::avcodec_encode_device_t>();
     }
 
@@ -487,7 +499,26 @@ namespace wl {
       return 0;
     }
 
+    /**
+     * @brief Initialize encoder modifiers for vulkan capture.
+     *
+     * @param hwdevice_type Hardware device type requested for capture or encode.
+     */
+    void init_encoder_modifiers(platf::mem_type_e hwdevice_type) {
+      if (hwdevice_type == platf::mem_type_e::vulkan) {
+        encoder_modifiers = vk::get_supported_capture_modifiers();
+        if (!encoder_modifiers.empty()) {
+          intersected_modifiers = wl::intersect_modifiers(interface.supported_modifiers, encoder_modifiers);
+          if (intersected_modifiers.empty()) {
+            BOOST_LOG(warning) << "[wlgrab] No common modifiers between compositor and vulkan encoder"sv;
+          }
+        }
+      }
+    }
+
     std::uint64_t sequence {};  ///< Monotonic capture sequence assigned to Wayland frames.
+    std::map<std::uint32_t, std::vector<std::uint64_t>> encoder_modifiers;  ///< Modifiers supported by the vulkan encoder.
+    std::map<std::uint32_t, std::vector<std::uint64_t>> intersected_modifiers;  ///< Common modifiers between compositor and encoder.
   };
 
 }  // namespace wl
@@ -497,7 +528,7 @@ namespace platf {
    * @brief Create a Wayland capture backend for the requested memory type.
    */
   std::shared_ptr<display_t> wl_display(mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config) {
-    if (hwdevice_type != platf::mem_type_e::system && hwdevice_type != platf::mem_type_e::vaapi && hwdevice_type != platf::mem_type_e::cuda) {
+    if (hwdevice_type != platf::mem_type_e::system && hwdevice_type != platf::mem_type_e::vaapi && hwdevice_type != platf::mem_type_e::cuda && hwdevice_type != platf::mem_type_e::vulkan) {
       BOOST_LOG(error) << "[wlgrab] Could not initialize display with the given hw device type."sv;
       return nullptr;
     }
@@ -508,6 +539,9 @@ namespace platf {
         return nullptr;
       }
 
+      // Initialize encoder modifiers for vulkan capture after interface setup
+      wlr->init_encoder_modifiers(hwdevice_type);
+
       return wlr;
     }