graphics-hook: Handle VK_KHR_imageless_framebuffer

Applications that use this Vulkan extension would have image usage
mismatches because we add VK_IMAGE_USAGE_TRANSFER_SRC_BIT to
VkSwapchainCreateInfoKHR::imageUsage, so make the same modification to
VkFramebufferAttachmentImageInfo::usage.

plugins/win-capture/graphics-hook-ver.h

@@ -13,7 +13,7 @@
 
 #define HOOK_VER_MAJOR 1
 #define HOOK_VER_MINOR 8
-#define HOOK_VER_PATCH 0
+#define HOOK_VER_PATCH 1
 
 #ifndef STRINGIFY
 #define STRINGIFY(s) #s

plugins/win-capture/graphics-hook/vulkan-capture.c

@@ -66,6 +66,18 @@ struct vk_queue_data {
 	uint32_t frame_count;
 };
 
+struct vk_swap_view_data {
+	struct vk_obj_node node;
+};
+
+#define OBS_COLOR_ATTACHMENT_LIMIT 8
+struct vk_framebuffer_data {
+	struct vk_obj_node node;
+
+	VkFramebuffer alternates[1 << OBS_COLOR_ATTACHMENT_LIMIT];
+	uint32_t color_attachment_mask;
+};
+
 struct vk_frame_data {
 	VkCommandPool cmd_pool;
 	VkCommandBuffer cmd_buffer;
@@ -103,6 +115,8 @@ struct vk_data {
 	struct vk_swap_data *cur_swap;
 
 	struct vk_obj_list queues;
+	struct vk_obj_list swap_views;
+	struct vk_obj_list framebuffers;
 
 	VkExternalMemoryProperties external_mem_props;
 
@@ -247,6 +261,13 @@ static struct vk_data *get_device_data_by_queue(VkQueue queue)
 					      (uint64_t)GET_LDT(queue));
 }
 
+static struct vk_data *
+get_device_data_by_command_buffer(VkCommandBuffer commandBuffer)
+{
+	return (struct vk_data *)get_obj_data(&devices,
+					      (uint64_t)GET_LDT(commandBuffer));
+}
+
 static struct vk_data *remove_device_data(VkDevice device)
 {
 	return (struct vk_data *)remove_obj_data(&devices,
@@ -285,11 +306,6 @@ static struct vk_queue_data *get_queue_data(struct vk_data *data, VkQueue queue)
 						    (uint64_t)queue);
 }
 
-static VkQueue get_queue_key(const struct vk_queue_data *queue_data)
-{
-	return (VkQueue)(uintptr_t)queue_data->node.obj;
-}
-
 static void remove_free_queue_all(struct vk_data *data,
 				  const VkAllocationCallbacks *ac)
 {
@@ -321,6 +337,73 @@ static void queue_walk_end(struct vk_data *data)
 
 /* ------------------------------------------------------------------------- */
 
+static struct vk_swap_view_data *
+add_swap_view_data(struct vk_data *data, VkImageView imageView,
+		   const VkAllocationCallbacks *ac)
+{
+	struct vk_swap_view_data *const swap_view_data =
+		vk_alloc(ac, sizeof(struct vk_swap_view_data),
+			 _Alignof(struct vk_swap_view_data),
+			 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
+	add_obj_data(&data->swap_views, (uint64_t)imageView, swap_view_data);
+	return swap_view_data;
+}
+
+static struct vk_swap_view_data *get_swap_view_data(struct vk_data *data,
+						    VkImageView imageView)
+{
+	return (struct vk_swap_view_data *)get_obj_data(&data->swap_views,
+							(uint64_t)imageView);
+}
+
+static void remove_free_swap_view_data(struct vk_data *data,
+				       VkImageView imageView,
+				       const VkAllocationCallbacks *ac)
+{
+	struct vk_swap_data *const swap_view_data =
+		(struct vk_swap_data *)remove_obj_data(&data->swap_views,
+						       (uint64_t)imageView);
+	vk_free(ac, swap_view_data);
+}
+
+/* ------------------------------------------------------------------------- */
+
+static struct vk_framebuffer_data *
+add_framebuffer_data(struct vk_data *data, VkFramebuffer framebuffer,
+		     const VkAllocationCallbacks *ac)
+{
+	struct vk_framebuffer_data *const framebuffer_data =
+		vk_alloc(ac, sizeof(struct vk_framebuffer_data),
+			 _Alignof(struct vk_framebuffer_data),
+			 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
+	add_obj_data(&data->framebuffers, (uint64_t)framebuffer,
+		     framebuffer_data);
+	for (size_t i = 0; i < _countof(framebuffer_data->alternates); ++i) {
+		framebuffer_data->alternates[i] = VK_NULL_HANDLE;
+	}
+	framebuffer_data->color_attachment_mask = 0;
+	return framebuffer_data;
+}
+
+static struct vk_framebuffer_data *
+get_framebuffer_data(struct vk_data *data, VkFramebuffer framebuffer)
+{
+	return (struct vk_framebuffer_data *)get_obj_data(
+		&data->framebuffers, (uint64_t)framebuffer);
+}
+
+static void remove_free_framebuffer_data(struct vk_data *data,
+					 VkFramebuffer framebuffer,
+					 const VkAllocationCallbacks *ac)
+{
+	struct vk_swap_data *const framebuffer_data =
+		(struct vk_swap_data *)remove_obj_data(&data->swaps,
+						       (uint64_t)framebuffer);
+	vk_free(ac, framebuffer_data);
+}
+
+/* ------------------------------------------------------------------------- */
+
 static struct vk_swap_data *alloc_swap_data(const VkAllocationCallbacks *ac)
 {
 	struct vk_swap_data *const swap_data = vk_alloc(
@@ -1468,6 +1551,8 @@ static VkResult VKAPI_CALL OBS_CreateDevice(VkPhysicalDevice phy_device,
 		return VK_ERROR_OUT_OF_HOST_MEMORY;
 
 	init_obj_list(&data->queues);
+	init_obj_list(&data->swap_views);
+	init_obj_list(&data->framebuffers);
 
 	/* -------------------------------------------------------- */
 	/* create device and initialize hook data                   */
@@ -1502,6 +1587,10 @@ static VkResult VKAPI_CALL OBS_CreateDevice(VkPhysicalDevice phy_device,
 			funcs_found = false;               \
 		}                                          \
 	} while (false)
+#define GETADDR_OPTIONAL(x)                                \
+	do {                                               \
+		dfuncs->x = (void *)gdpa(device, "vk" #x); \
+	} while (false)
 
 	GETADDR(GetDeviceProcAddr);
 	GETADDR(DestroyDevice);
@@ -1531,6 +1620,14 @@ static VkResult VKAPI_CALL OBS_CreateDevice(VkPhysicalDevice phy_device,
 	GETADDR(DestroyFence);
 	GETADDR(WaitForFences);
 	GETADDR(ResetFences);
+	GETADDR(CreateImageView);
+	GETADDR(DestroyImageView);
+	GETADDR(CreateFramebuffer);
+	GETADDR(DestroyFramebuffer);
+	GETADDR(CmdBeginRenderPass);
+	GETADDR_OPTIONAL(CmdBeginRenderPass2KHR);
+	GETADDR_OPTIONAL(CmdBeginRenderPass2);
+#undef GETADDR_OPTIONAL
 #undef GETADDR
 
 	if (!funcs_found) {
@@ -1725,6 +1822,328 @@ OBS_CreateSwapchainKHR(VkDevice device, const VkSwapchainCreateInfoKHR *cinfo,
 	return VK_SUCCESS;
 }
 
+static VkResult VKAPI_CALL
+OBS_CreateImageView(VkDevice device, const VkImageViewCreateInfo *pCreateInfo,
+		    const VkAllocationCallbacks *pAllocator, VkImageView *pView)
+{
+	struct vk_data *const data = get_device_data(device);
+	const PFN_vkCreateImageView func = data->funcs.CreateImageView;
+	VkResult result = func(device, pCreateInfo, pAllocator, pView);
+	if (data->valid && (result == VK_SUCCESS)) {
+		struct vk_swap_data *swap = swap_walk_begin(data);
+
+		while (swap) {
+			bool match = false;
+			for (uint32_t i = 0, count = swap->image_count;
+			     i < count; ++i) {
+				match = swap->swap_images[i] ==
+					pCreateInfo->image;
+				if (match)
+					break;
+			}
+			if (match) {
+				add_swap_view_data(data, *pView, pAllocator);
+				break;
+			}
+
+			swap = swap_walk_next(swap);
+		}
+
+		swap_walk_end(data);
+	}
+
+	return result;
+}
+
+static void VKAPI_CALL
+OBS_DestroyImageView(VkDevice device, VkImageView imageView,
+		     const VkAllocationCallbacks *pAllocator)
+{
+	struct vk_data *const data = get_device_data(device);
+
+	if (data->valid && (imageView != VK_NULL_HANDLE)) {
+		struct vk_swap_view_data *swap_view_data =
+			get_swap_view_data(data, imageView);
+		if (swap_view_data)
+			remove_free_swap_view_data(data, imageView, pAllocator);
+	}
+
+	data->funcs.DestroyImageView(device, imageView, pAllocator);
+}
+
+static void generate_framebuffer_variants(
+	PFN_vkCreateFramebuffer func, VkDevice device,
+	const VkFramebufferCreateInfo *pCreateInfo,
+	const VkAllocationCallbacks *pAllocator,
+	struct vk_framebuffer_data *framebuffer_data,
+	const VkFramebufferAttachmentsCreateInfo *pAttachmentsCreateInfo,
+	uint32_t colorCount)
+{
+	const uint32_t variantCount = 1 << colorCount;
+	for (uint32_t colorMask = 0; colorMask < variantCount; ++colorMask) {
+		VkImageUsageFlags pPreviousUsage[OBS_COLOR_ATTACHMENT_LIMIT];
+		uint32_t colorIndex = 0;
+		for (uint32_t infoIndex = 0,
+			      count = pAttachmentsCreateInfo
+					      ->attachmentImageInfoCount;
+		     infoIndex < count; ++infoIndex) {
+			const VkFramebufferAttachmentImageInfo *const pInfo =
+				&pAttachmentsCreateInfo
+					 ->pAttachmentImageInfos[infoIndex];
+			const VkImageUsageFlags usage = pInfo->usage;
+			if (usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) {
+				pPreviousUsage[colorIndex] = usage;
+				if (colorMask & (1 << colorIndex)) {
+					((VkFramebufferAttachmentImageInfo *)
+						 pInfo)
+						->usage |=
+						VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
+				}
+
+				++colorIndex;
+				if (colorIndex == OBS_COLOR_ATTACHMENT_LIMIT)
+					break;
+			}
+		}
+
+		VkFramebuffer framebuffer;
+		const VkResult result =
+			func(device, pCreateInfo, pAllocator, &framebuffer);
+		if (result == VK_SUCCESS)
+			framebuffer_data->alternates[colorMask] = framebuffer;
+
+		colorIndex = 0;
+		for (uint32_t infoIndex = 0,
+			      count = pAttachmentsCreateInfo
+					      ->attachmentImageInfoCount;
+		     infoIndex < count; ++infoIndex) {
+			const VkFramebufferAttachmentImageInfo *const pInfo =
+				&pAttachmentsCreateInfo
+					 ->pAttachmentImageInfos[infoIndex];
+			if (pInfo->usage &
+			    VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) {
+				if (colorMask & (1 << colorIndex)) {
+					((VkFramebufferAttachmentImageInfo *)
+						 pInfo)
+						->usage =
+						pPreviousUsage[colorIndex];
+				}
+
+				++colorIndex;
+				if (colorIndex == OBS_COLOR_ATTACHMENT_LIMIT)
+					break;
+			}
+		}
+	}
+}
+
+static VkResult VKAPI_CALL OBS_CreateFramebuffer(
+	VkDevice device, const VkFramebufferCreateInfo *pCreateInfo,
+	const VkAllocationCallbacks *pAllocator, VkFramebuffer *pFramebuffer)
+{
+	struct vk_data *const data = get_device_data(device);
+	const PFN_vkCreateFramebuffer func = data->funcs.CreateFramebuffer;
+	VkResult result = func(device, pCreateInfo, pAllocator, pFramebuffer);
+	if (data->valid && (result == VK_SUCCESS) &&
+	    (pCreateInfo->flags & VK_FRAMEBUFFER_CREATE_IMAGELESS_BIT)) {
+		struct vk_framebuffer_data *const framebuffer_data =
+			add_framebuffer_data(data, *pFramebuffer, pAllocator);
+		const void *pCurrent = pCreateInfo->pNext;
+		while (pCurrent) {
+			VkBaseInStructure baseIn;
+			memcpy(&baseIn, pCurrent, sizeof(baseIn));
+			if (baseIn.sType ==
+			    VK_STRUCTURE_TYPE_FRAMEBUFFER_ATTACHMENTS_CREATE_INFO) {
+				uint32_t attachmentMask = 0;
+				uint32_t colorCount = 0;
+				const VkFramebufferAttachmentsCreateInfo *const
+					pAttachmentsCreateInfo = pCurrent;
+				for (uint32_t
+					     infoIndex = 0,
+					     count = min(
+						     sizeof(attachmentMask) * 8,
+						     pAttachmentsCreateInfo
+							     ->attachmentImageInfoCount);
+				     infoIndex < count; ++infoIndex) {
+					if (pAttachmentsCreateInfo
+						    ->pAttachmentImageInfos
+							    [infoIndex]
+						    .usage &
+					    VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) {
+						attachmentMask |= 1
+								  << infoIndex;
+						++colorCount;
+						if (colorCount ==
+						    OBS_COLOR_ATTACHMENT_LIMIT)
+							break;
+					}
+				}
+
+				generate_framebuffer_variants(
+					func, device, pCreateInfo, pAllocator,
+					framebuffer_data,
+					pAttachmentsCreateInfo, colorCount);
+				framebuffer_data->color_attachment_mask =
+					attachmentMask;
+				break;
+			}
+
+			pCurrent = baseIn.pNext;
+		}
+	}
+
+	return result;
+}
+
+static void VKAPI_CALL
+OBS_DestroyFramebuffer(VkDevice device, VkFramebuffer framebuffer,
+		       const VkAllocationCallbacks *pAllocator)
+{
+	struct vk_data *const data = get_device_data(device);
+	struct vk_device_funcs *const funcs = &data->funcs;
+
+	if (data->valid && (framebuffer != VK_NULL_HANDLE)) {
+		struct vk_framebuffer_data *framebuffer_data =
+			get_framebuffer_data(data, framebuffer);
+		if (framebuffer_data) {
+			for (size_t i = 0;
+			     i < _countof(framebuffer_data->alternates); ++i) {
+				if (framebuffer_data->alternates[i] !=
+				    VK_NULL_HANDLE) {
+					funcs->DestroyFramebuffer(
+						device,
+						framebuffer_data->alternates[i],
+						pAllocator);
+					framebuffer_data->alternates[i] =
+						VK_NULL_HANDLE;
+				}
+			}
+
+			remove_free_framebuffer_data(data, framebuffer,
+						     pAllocator);
+		}
+	}
+
+	funcs->DestroyFramebuffer(device, framebuffer, pAllocator);
+}
+
+static const VkRenderPassBeginInfo *
+process_render_pass_begin_info(const VkRenderPassBeginInfo *pRenderPassBegin,
+			       VkRenderPassBeginInfo *pAlternateBegin,
+			       struct vk_data *data)
+{
+	const void *pCurrent = pRenderPassBegin->pNext;
+	while (pCurrent) {
+		VkBaseInStructure baseIn;
+		memcpy(&baseIn, pCurrent, sizeof(baseIn));
+		if (baseIn.sType ==
+		    VK_STRUCTURE_TYPE_RENDER_PASS_ATTACHMENT_BEGIN_INFO) {
+			struct vk_framebuffer_data *const framebuffer_data =
+				get_framebuffer_data(
+					data, pRenderPassBegin->framebuffer);
+			if (framebuffer_data) {
+				const VkRenderPassAttachmentBeginInfo
+					*const pAttachmentInfo = pCurrent;
+				uint32_t swapMask = 0;
+				uint32_t colorIndex = 0;
+				for (uint32_t infoIndex = 0,
+					      count = pAttachmentInfo
+							      ->attachmentCount;
+				     infoIndex < count; ++infoIndex) {
+					if (framebuffer_data
+						    ->color_attachment_mask &
+					    (1 << infoIndex)) {
+						if (get_swap_view_data(
+							    data,
+							    pAttachmentInfo->pAttachments
+								    [infoIndex])) {
+							swapMask |=
+								1 << colorIndex;
+						}
+
+						++colorIndex;
+						if (colorIndex ==
+						    OBS_COLOR_ATTACHMENT_LIMIT)
+							break;
+					}
+				}
+
+				if (swapMask > 0) {
+					VkFramebuffer alternate =
+						framebuffer_data
+							->alternates[swapMask];
+					if (alternate != VK_NULL_HANDLE) {
+						*pAlternateBegin =
+							*pRenderPassBegin;
+						pAlternateBegin->framebuffer =
+							framebuffer_data->alternates
+								[swapMask];
+						pRenderPassBegin =
+							pAlternateBegin;
+					}
+				}
+			}
+
+			break;
+		}
+
+		pCurrent = baseIn.pNext;
+	}
+
+	return pRenderPassBegin;
+}
+
+static void VKAPI_CALL
+OBS_CmdBeginRenderPass(VkCommandBuffer commandBuffer,
+		       const VkRenderPassBeginInfo *pRenderPassBegin,
+		       VkSubpassContents contents)
+{
+	struct vk_data *const data =
+		get_device_data_by_command_buffer(commandBuffer);
+	if (data->valid) {
+		VkRenderPassBeginInfo alternateBegin;
+		pRenderPassBegin = process_render_pass_begin_info(
+			pRenderPassBegin, &alternateBegin, data);
+	}
+
+	data->funcs.CmdBeginRenderPass(commandBuffer, pRenderPassBegin,
+				       contents);
+}
+
+static void VKAPI_CALL
+OBS_CmdBeginRenderPass2KHR(VkCommandBuffer commandBuffer,
+			   const VkRenderPassBeginInfo *pRenderPassBegin,
+			   const VkSubpassBeginInfo *pSubpassBeginInfo)
+{
+	struct vk_data *const data =
+		get_device_data_by_command_buffer(commandBuffer);
+	if (data->valid) {
+		VkRenderPassBeginInfo alternateBegin;
+		pRenderPassBegin = process_render_pass_begin_info(
+			pRenderPassBegin, &alternateBegin, data);
+	}
+
+	data->funcs.CmdBeginRenderPass2KHR(commandBuffer, pRenderPassBegin,
+					   pSubpassBeginInfo);
+}
+
+static void VKAPI_CALL
+OBS_CmdBeginRenderPass2(VkCommandBuffer commandBuffer,
+			const VkRenderPassBeginInfo *pRenderPassBegin,
+			const VkSubpassBeginInfo *pSubpassBeginInfo)
+{
+	struct vk_data *const data =
+		get_device_data_by_command_buffer(commandBuffer);
+	if (data->valid) {
+		VkRenderPassBeginInfo alternateBegin;
+		pRenderPassBegin = process_render_pass_begin_info(
+			pRenderPassBegin, &alternateBegin, data);
+	}
+
+	data->funcs.CmdBeginRenderPass2(commandBuffer, pRenderPassBegin,
+					pSubpassBeginInfo);
+}
+
 static void VKAPI_CALL OBS_DestroySwapchainKHR(VkDevice device,
 					       VkSwapchainKHR sc,
 					       const VkAllocationCallbacks *ac)
@@ -1797,6 +2216,13 @@ static PFN_vkVoidFunction VKAPI_CALL OBS_GetDeviceProcAddr(VkDevice device,
 	GETPROCADDR_IF_SUPPORTED(CreateSwapchainKHR);
 	GETPROCADDR_IF_SUPPORTED(DestroySwapchainKHR);
 	GETPROCADDR_IF_SUPPORTED(QueuePresentKHR);
+	GETPROCADDR(CreateImageView);
+	GETPROCADDR(DestroyImageView);
+	GETPROCADDR(CreateFramebuffer);
+	GETPROCADDR(DestroyFramebuffer);
+	GETPROCADDR(CmdBeginRenderPass);
+	GETPROCADDR_IF_SUPPORTED(CmdBeginRenderPass2KHR);
+	GETPROCADDR_IF_SUPPORTED(CmdBeginRenderPass2);
 
 	if (funcs->GetDeviceProcAddr == NULL)
 		return NULL;

plugins/win-capture/graphics-hook/vulkan-capture.h

@@ -42,6 +42,13 @@ struct vk_device_funcs {
 	DEF_FUNC(DestroyFence);
 	DEF_FUNC(WaitForFences);
 	DEF_FUNC(ResetFences);
+	DEF_FUNC(CreateImageView);
+	DEF_FUNC(DestroyImageView);
+	DEF_FUNC(CreateFramebuffer);
+	DEF_FUNC(DestroyFramebuffer);
+	DEF_FUNC(CmdBeginRenderPass);
+	DEF_FUNC(CmdBeginRenderPass2KHR);
+	DEF_FUNC(CmdBeginRenderPass2);
 };
 
 #undef DEF_FUNC