obs-qsv11: Use device enumeration for choosing DRM device

Trying to use the display server as the QSV device was found to be
generally wrong in beta, so instead lets save defaults from the earlier
device enumeration similar to obs-ffmpeg-vaapi which is known to work
well.

cmake/Modules/FindLibva.cmake

@@ -15,8 +15,6 @@ find_package(PkgConfig QUIET)
 if(PKG_CONFIG_FOUND)
   pkg_check_modules(_LIBVA libva)
   pkg_check_modules(_LIBVA_DRM libva-drm)
-  pkg_check_modules(_LIBVA_WAYLAND libva-wayland)
-  pkg_check_modules(_LIBVA_X11 libva-x11)
 endif()
 
 find_path(
@@ -37,29 +35,14 @@ find_library(
   HINTS ${_LIBVA_DRM_LIBRARY_DIRS}
   PATHS /usr/lib /usr/local/lib /opt/local/lib)
 
-find_library(
-  LIBVA_WAYLAND_LIB
-  NAMES ${_LIBVA_WAYLAND_LIBRARIES} libva-wayland
-  HINTS ${_LIBVA_WAYLAND_LIBRARY_DIRS}
-  PATHS /usr/lib /usr/local/lib /opt/local/lib)
-
-find_library(
-  LIBVA_X11_LIB
-  NAMES ${_LIBVA_X11_LIBRARIES} libva-x11
-  HINTS ${_LIBVA_X11_LIBRARY_DIRS}
-  PATHS /usr/lib /usr/local/lib /opt/local/lib)
-
 include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(Libva REQUIRED_VARS LIBVA_INCLUDE_DIR LIBVA_LIB LIBVA_DRM_LIB LIBVA_WAYLAND_LIB
-                                                      LIBVA_X11_LIB)
-mark_as_advanced(LIBVA_INCLUDE_DIR LIBVA_LIB LIBVA_DRM_LIB LIBVA_WAYLAND_LIB LIBVA_X11_LIB)
+find_package_handle_standard_args(Libva REQUIRED_VARS LIBVA_INCLUDE_DIR LIBVA_LIB LIBVA_DRM_LIB)
+mark_as_advanced(LIBVA_INCLUDE_DIR LIBVA_LIB LIBVA_DRM_LIB)
 
 if(LIBVA_FOUND)
   set(LIBVA_INCLUDE_DIRS ${LIBVA_INCLUDE_DIR})
   set(LIBVA_LIBRARIES ${LIBVA_LIB})
   set(LIBVA_DRM_LIBRARIES ${LIBVA_DRM_LIB})
-  set(LIBVA_WAYLAND_LIBRARIES ${LIBVA_WAYLAND_LIB})
-  set(LIBVA_X11_LIBRARIES ${LIBVA_X11_LIB})
 
   if(NOT TARGET Libva::va)
     if(IS_ABSOLUTE "${LIBVA_LIBRARIES}")
@@ -85,27 +68,4 @@ if(LIBVA_FOUND)
     set_target_properties(Libva::drm PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LIBVA_INCLUDE_DIRS}")
   endif()
 
-  if(NOT TARGET Libva::wayland)
-    if(IS_ABSOLUTE "${LIBVA_WAYLAND_LIBRARIES}")
-      add_library(Libva::wayland UNKNOWN IMPORTED)
-      set_target_properties(Libva::wayland PROPERTIES IMPORTED_LOCATION "${LIBVA_WAYLAND_LIBRARIES}")
-    else()
-      add_library(Libva::wayland INTERFACE IMPORTED)
-      set_target_properties(Libva::wayland PROPERTIES IMPORTED_LIBNAME "${LIBVA_WAYLAND_LIBRARIES}")
-    endif()
-
-    set_target_properties(Libva::wayland PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LIBVA_INCLUDE_DIRS}")
-  endif()
-
-  if(NOT TARGET Libva::x11)
-    if(IS_ABSOLUTE "${LIBVA_X11_LIBRARIES}")
-      add_library(Libva::x11 UNKNOWN IMPORTED)
-      set_target_properties(Libva::x11 PROPERTIES IMPORTED_LOCATION "${LIBVA_X11_LIBRARIES}")
-    else()
-      add_library(Libva::x11 INTERFACE IMPORTED)
-      set_target_properties(Libva::x11 PROPERTIES IMPORTED_LIBNAME "${LIBVA_X11_LIBRARIES}")
-    endif()
-
-    set_target_properties(Libva::x11 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LIBVA_INCLUDE_DIRS}")
-  endif()
 endif()

plugins/obs-qsv11/CMakeLists.txt

@@ -58,5 +58,5 @@ elseif(OS_LINUX)
 
   target_sources(obs-qsv11 PRIVATE common_utils_linux.cpp)
 
-  target_link_libraries(obs-qsv11 PRIVATE Libva::va Libva::drm Libva::wayland Libva::x11)
+  target_link_libraries(obs-qsv11 PRIVATE Libva::va Libva::drm)
 endif()

plugins/obs-qsv11/QSV_Encoder.h

@@ -61,6 +61,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <stdint.h>
 #include <stdbool.h>
 #include <stddef.h>
+#include "common_utils.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -92,12 +93,6 @@ static const char *const qsv_latency_names[] = {"ultra-low", "low", "normal",
 						0};
 typedef struct qsv_t qsv_t;
 
-enum qsv_codec {
-	QSV_CODEC_AVC,
-	QSV_CODEC_AV1,
-	QSV_CODEC_HEVC,
-};
-
 typedef struct {
 	mfxU16 nTargetUsage; /* 1 through 7, 1 being best quality and 7
 				being the best speed */

plugins/obs-qsv11/QSV_Encoder_Internal.cpp

@@ -180,15 +180,16 @@ mfxStatus QSV_Encoder_Internal::Open(qsv_param_t *pParams, enum qsv_codec codec)
 	if (m_bUseD3D11)
 		// Use D3D11 surface
 		sts = Initialize(m_ver, &m_session, &m_mfxAllocator,
-				 &g_DX_Handle, false, false);
+				 &g_DX_Handle, false, false, codec);
 	else if (m_bD3D9HACK)
 		// Use hack
 		sts = Initialize(m_ver, &m_session, &m_mfxAllocator,
-				 &g_DX_Handle, false, true);
+				 &g_DX_Handle, false, true, codec);
 	else
-		sts = Initialize(m_ver, &m_session, NULL, NULL, NULL, NULL);
+		sts = Initialize(m_ver, &m_session, NULL, NULL, NULL, NULL,
+				 codec);
 #else
-	sts = Initialize(m_ver, &m_session, NULL, NULL, false, false);
+	sts = Initialize(m_ver, &m_session, NULL, NULL, false, false, codec);
 #endif
 
 	MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts);

plugins/obs-qsv11/cmake/legacy.cmake

@@ -55,7 +55,7 @@ elseif(OS_LINUX)
 
   target_sources(obs-qsv11 PRIVATE common_utils_linux.cpp)
 
-  target_link_libraries(obs-qsv11 PRIVATE Libva::va Libva::drm Libva::wayland Libva::x11)
+  target_link_libraries(obs-qsv11 PRIVATE Libva::va Libva::drm)
 endif()
 
 set_target_properties(obs-qsv11 PROPERTIES FOLDER "plugins/obs-qsv11")

plugins/obs-qsv11/common_utils.h

@@ -4,6 +4,31 @@
 
 // Most of this file shouldnt be accessed from C.
 #ifdef __cplusplus
+extern "C" {
+#endif // __cplusplus
+
+enum qsv_codec {
+	QSV_CODEC_AVC,
+	QSV_CODEC_AV1,
+	QSV_CODEC_HEVC,
+};
+
+struct adapter_info {
+	bool is_intel;
+	bool is_dgpu;
+	bool supports_av1;
+	bool supports_hevc;
+};
+
+#define MAX_ADAPTERS 10
+extern struct adapter_info adapters[MAX_ADAPTERS];
+extern size_t adapter_count;
+
+void util_cpuid(int cpuinfo[4], int flags);
+void check_adapters(struct adapter_info *adapters, size_t *adapter_count);
+
+#ifdef __cplusplus
+}
 
 #include <vpl/mfxvideo++.h>
 #include <vpl/mfxdispatcher.h>
@@ -136,7 +161,8 @@ int GetFreeTaskIndex(Task *pTaskPool, mfxU16 nPoolSize);
 // Initialize Intel VPL Session, device/display and memory manager
 mfxStatus Initialize(mfxVersion ver, mfxSession *pSession,
 		     mfxFrameAllocator *pmfxAllocator, mfxHDL *deviceHandle,
-		     bool bCreateSharedHandles, bool dx9hack); //vpl change
+		     bool bCreateSharedHandles, bool dx9hack,
+		     enum qsv_codec codec); //vpl change
 
 // Release resources (device/display)
 void Release();
@@ -149,23 +175,4 @@ void mfxGetTime(mfxTime *timestamp);
 //void mfxInitTime();  might need this for Windows
 double TimeDiffMsec(mfxTime tfinish, mfxTime tstart);
 
-extern "C" {
 #endif // __cplusplus
-
-struct adapter_info {
-	bool is_intel;
-	bool is_dgpu;
-	bool supports_av1;
-	bool supports_hevc;
-};
-
-#define MAX_ADAPTERS 10
-extern struct adapter_info adapters[MAX_ADAPTERS];
-extern size_t adapter_count;
-
-void util_cpuid(int cpuinfo[4], int flags);
-void check_adapters(struct adapter_info *adapters, size_t *adapter_count);
-
-#ifdef __cplusplus
-}
-#endif

plugins/obs-qsv11/common_utils_linux.cpp

@@ -5,8 +5,6 @@
 #include <util/c99defs.h>
 #include <util/dstr.h>
 #include <va/va_drm.h>
-#include <va/va_x11.h>
-#include <va/va_wayland.h>
 #include <va/va_str.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -18,6 +16,12 @@
 #include <obs.h>
 #include <obs-nix-platform.h>
 
+// Set during check_adapters to work-around VPL dispatcher not setting a VADisplay
+// for the MSDK runtime.
+static const char *default_h264_device = nullptr;
+static const char *default_hevc_device = nullptr;
+static const char *default_av1_device = nullptr;
+
 mfxStatus simple_alloc(mfxHDL pthis, mfxFrameAllocRequest *request,
 		       mfxFrameAllocResponse *response)
 {
@@ -77,7 +81,8 @@ void ClearRGBSurfaceVMem(mfxMemId memId);
 // Initialize Intel VPL Session, device/display and memory manager
 mfxStatus Initialize(mfxVersion ver, mfxSession *pSession,
 		     mfxFrameAllocator *pmfxAllocator, mfxHDL *deviceHandle,
-		     bool bCreateSharedHandles, bool dx9hack)
+		     bool bCreateSharedHandles, bool dx9hack,
+		     enum qsv_codec codec)
 {
 	UNUSED_PARAMETER(ver);
 	UNUSED_PARAMETER(pmfxAllocator);
@@ -107,22 +112,39 @@ mfxStatus Initialize(mfxVersion ver, mfxSession *pSession,
 		cfg, (const mfxU8 *)"mfxImplDescription.AccelerationMode",
 		impl);
 
-	mfxHDL vaDisplay = nullptr;
-	if (obs_get_nix_platform() == OBS_NIX_PLATFORM_X11_EGL) {
-		vaDisplay =
-			vaGetDisplay((Display *)obs_get_nix_platform_display());
-	} else if (obs_get_nix_platform() == OBS_NIX_PLATFORM_WAYLAND) {
-		vaDisplay = vaGetDisplayWl(
-			(wl_display *)obs_get_nix_platform_display());
+	// We cant cleanup FDs because Release() is only called for gpu texture sharing cases.
+	int fd = -1;
+	if (codec == QSV_CODEC_AVC)
+		fd = open(default_h264_device, O_RDWR);
+	if (codec == QSV_CODEC_HEVC)
+		fd = open(default_hevc_device, O_RDWR);
+	if (codec == QSV_CODEC_AV1)
+		fd = open(default_av1_device, O_RDWR);
+	if (fd < 0) {
+		blog(LOG_ERROR, "Failed to open device '%s'",
+		     default_h264_device);
+		return MFX_ERR_DEVICE_FAILED;
+	}
+
+	mfxHDL vaDisplay = vaGetDisplayDRM(fd);
+	if (!vaDisplay) {
+		return MFX_ERR_DEVICE_FAILED;
 	}
 
 	sts = MFXCreateSession(loader, 0, pSession);
-	MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts);
+	if (MFX_ERR_NONE > sts) {
+		blog(LOG_ERROR, "Failed to initialize MFX");
+		MSDK_PRINT_RET_MSG(sts);
+		close(fd);
+		return sts;
+	}
 
 	// VPL expects the VADisplay to be initialized.
 	int major;
 	int minor;
 	if (vaInitialize(vaDisplay, &major, &minor) != VA_STATUS_SUCCESS) {
+		blog(LOG_ERROR, "Failed to initialize VA-API");
+		close(fd);
 		vaTerminate(vaDisplay);
 		return MFX_ERR_DEVICE_FAILED;
 	}
@@ -315,6 +337,18 @@ void check_adapters(struct adapter_info *adapters, size_t *adapter_count)
 				vaapi_supports_av1(device.display);
 			adapter->supports_hevc =
 				vaapi_supports_hevc(device.display);
+
+			if (adapter->is_intel && default_h264_device == nullptr)
+				default_h264_device = strdup(full_path.array);
+
+			if (adapter->supports_av1 &&
+			    default_av1_device == nullptr)
+				default_av1_device = strdup(full_path.array);
+
+			if (adapter->supports_hevc &&
+			    default_hevc_device == nullptr)
+				default_hevc_device = strdup(full_path.array);
+
 			vaapi_close(&device);
 
 		next_entry:

plugins/obs-qsv11/common_utils_windows.cpp

@@ -24,10 +24,10 @@
 
 mfxStatus Initialize(mfxVersion ver, mfxSession *pSession,
 		     mfxFrameAllocator *pmfxAllocator, mfxHDL *deviceHandle,
-		     bool bCreateSharedHandles, bool dx9hack)
+		     bool bCreateSharedHandles, bool dx9hack,
+		     enum qsv_codec codec)
 {
-	bCreateSharedHandles; // (Lain) Currently unused
-	pmfxAllocator;        // (Lain) Currently unused
+	UNUSED_PARAMETER(codec);
 
 	mfxStatus sts = MFX_ERR_NONE;
 	mfxVariant impl;