From b376d179dbf7645d356db52e56e7851f382d2092 Mon Sep 17 00:00:00 2001 From: Mikhail Burakov Date: Sun, 18 Aug 2024 14:49:44 +0200 Subject: Add colorspace conversion facilities to video pipeline --- video_context.c | 293 ++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 252 insertions(+), 41 deletions(-) (limited to 'video_context.c') diff --git a/video_context.c b/video_context.c index 10dd7b9..5d2a2fd 100644 --- a/video_context.c +++ b/video_context.c @@ -17,7 +17,10 @@ #include "video_context.h" +#include +#include #include +#include #include #include #include @@ -26,11 +29,19 @@ #include #include +#include "encode_context.h" +#include "gpu_context.h" #include "util.h" #include "wlr-export-dmabuf-unstable-v1.h" +struct EGLAttribPair { + EGLAttrib key; + EGLAttrib value; +}; + struct VideoContext { struct IoContext* io_context; + struct EncodeContext* encode_context; // Wayland globals struct wl_display* display; @@ -38,11 +49,39 @@ struct VideoContext { struct wl_output* output; struct zwlr_export_dmabuf_manager_v1* export_dmabuf_manager; + // Colorspace conversion + struct GpuContext* gpu_context; + struct GpuContextImage** imported_images; + size_t imported_images_count; + // Threading struct pw_thread_loop* thread_loop; struct spa_source* source; + + // Volatile state + struct { + struct EGLAttribPair height; + struct EGLAttribPair width; + struct EGLAttribPair linux_drm_fourcc; + struct { + struct EGLAttribPair fd; + struct EGLAttribPair offset; + struct EGLAttribPair pitch; + struct EGLAttribPair modifier_lo; + struct EGLAttribPair modifier_hi; + } dma_buf_plane[4]; + EGLAttrib terminator; + } attrib_list; }; +static void SetEGLAttribPair(struct EGLAttribPair* pair, EGLAttrib key, + EGLAttrib value) { + *pair = (struct EGLAttribPair){ + .key = key, + .value = value, + }; +} + static void OnRegistryGlobal(void* data, struct wl_registry* wl_registry, uint32_t name, const char* interface, uint32_t version) { @@ -126,56 +165,206 @@ static void OnExportDmabufFrameFrame( uint32_t width, uint32_t height, uint32_t offset_x, uint32_t offset_y, uint32_t buffer_flags, uint32_t flags, uint32_t format, uint32_t mod_high, uint32_t mod_low, uint32_t num_objects) { - (void)data; (void)export_dmabuf_frame; - (void)width; - (void)height; - (void)offset_x; - (void)offset_y; - (void)buffer_flags; (void)flags; - (void)format; - (void)mod_high; - (void)mod_low; - (void)num_objects; - LOG("%s(data=%p, export_dmabuf_frame=%p, width=%u, height=%u, " - "offset_x=%u, offset_y=%u, buffer_flags=0x%x, flags=0x%x, " - "format=0x%08x, mod_high=%08x, mod_low=%08x, num_objects=%u)", - __FUNCTION__, data, (void*)export_dmabuf_frame, width, height, offset_x, - offset_y, buffer_flags, flags, format, mod_high, mod_low, num_objects); + + struct VideoContext* video_context = data; + if (!video_context->encode_context) { + video_context->encode_context = + EncodeContextCreate(video_context->io_context, width, height); + if (!video_context->encode_context) { + LOG("Failed to create encode context"); + // TODO(mburakov): Now what?.. + } + } + + // TODO(mburakov): Maybe handle those? + assert(!offset_x && !offset_y && !buffer_flags); + SetEGLAttribPair(&video_context->attrib_list.height, EGL_HEIGHT, height); + SetEGLAttribPair(&video_context->attrib_list.width, EGL_WIDTH, width); + SetEGLAttribPair(&video_context->attrib_list.linux_drm_fourcc, + EGL_LINUX_DRM_FOURCC_EXT, format); + + assert(num_objects <= LENGTH(video_context->attrib_list.dma_buf_plane)); + for (EGLAttrib index = 0; index < num_objects; index++) { + typeof(&video_context->attrib_list.dma_buf_plane[index]) plane = + &video_context->attrib_list.dma_buf_plane[index]; + SetEGLAttribPair(&plane->modifier_lo, + EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT + index * 2, mod_low); + SetEGLAttribPair(&plane->modifier_hi, + EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT + index * 2, mod_high); + } } static void OnExportDmabufFrameObject( void* data, struct zwlr_export_dmabuf_frame_v1* export_dmabuf_frame, uint32_t index, int32_t fd, uint32_t size, uint32_t offset, uint32_t stride, uint32_t plane_index) { - (void)data; (void)export_dmabuf_frame; - (void)index; - (void)fd; (void)size; - (void)offset; - (void)stride; - (void)plane_index; - LOG("%s(data=%p, export_dmabuf_frame=%p, index=%u, fd=%d, " - "size=%u, offset=%u, stride=%u, plane_index=%u)", - __FUNCTION__, data, (void*)export_dmabuf_frame, index, fd, size, offset, - stride, plane_index); + + assert(index == plane_index); + struct VideoContext* video_context = data; + + SetEGLAttribPair(&video_context->attrib_list.dma_buf_plane[index].fd, + EGL_DMA_BUF_PLANE0_FD_EXT + (EGLAttrib)index * 3, fd); + SetEGLAttribPair(&video_context->attrib_list.dma_buf_plane[index].offset, + EGL_DMA_BUF_PLANE0_OFFSET_EXT + (EGLAttrib)index * 3, + offset); + SetEGLAttribPair(&video_context->attrib_list.dma_buf_plane[index].pitch, + EGL_DMA_BUF_PLANE0_PITCH_EXT + (EGLAttrib)index * 3, stride); +} + +static void ResetAttribList(struct VideoContext* video_context) { + SetEGLAttribPair(&video_context->attrib_list.height, EGL_NONE, EGL_NONE); + SetEGLAttribPair(&video_context->attrib_list.width, EGL_NONE, EGL_NONE); + SetEGLAttribPair(&video_context->attrib_list.linux_drm_fourcc, EGL_NONE, + EGL_NONE); + + static const EGLAttrib kMaxPlanesCount = + LENGTH(video_context->attrib_list.dma_buf_plane); + for (EGLAttrib index = 0; index < kMaxPlanesCount; index++) { + typeof(&video_context->attrib_list.dma_buf_plane[index]) plane = + &video_context->attrib_list.dma_buf_plane[index]; + if (plane->fd.value != -1) { + assert(!close((int)plane->fd.value)); + } + SetEGLAttribPair(&plane->fd, EGL_NONE, -1); + SetEGLAttribPair(&plane->offset, EGL_NONE, EGL_NONE); + SetEGLAttribPair(&plane->pitch, EGL_NONE, EGL_NONE); + SetEGLAttribPair(&plane->modifier_hi, EGL_NONE, EGL_NONE); + SetEGLAttribPair(&plane->modifier_lo, EGL_NONE, EGL_NONE); + } + + video_context->attrib_list.terminator = EGL_NONE; +} + +static struct GpuContextImage* ImportEncodeContextFrame( + struct VideoContext* video_context, + struct EncodeContextFrame* encode_context_frame) { + static_assert(LENGTH(encode_context_frame->planes) == 2, + "Suspicious amount of imported frame planes"); + struct GpuContextImage* imported_frame_planes = + malloc(2 * sizeof(struct GpuContextImage)); + if (!imported_frame_planes) { + LOG("Failed to allocate imported frame planes (%s)", strerror(errno)); + return NULL; + } + + EGLAttrib attrib_list_luma[] = { +#define _(...) __VA_ARGS__ + _(EGL_HEIGHT, video_context->attrib_list.height.value), + _(EGL_WIDTH, video_context->attrib_list.width.value), + _(EGL_LINUX_DRM_FOURCC_EXT, DRM_FORMAT_R8), + _(EGL_DMA_BUF_PLANE0_FD_EXT, encode_context_frame->planes[0].fd), + _(EGL_DMA_BUF_PLANE0_OFFSET_EXT, encode_context_frame->planes[0].offset), + _(EGL_DMA_BUF_PLANE0_PITCH_EXT, encode_context_frame->planes[0].pitch), + EGL_NONE, +#undef _ + }; + if (GpuContextCreateImage(video_context->gpu_context, attrib_list_luma, + &imported_frame_planes[0])) { + LOG("Failed to import luma frame plane"); + goto rollback_imported_frame_planes; + } + + EGLAttrib attrib_list_chroma[] = { +#define _(...) __VA_ARGS__ + _(EGL_HEIGHT, video_context->attrib_list.height.value / 2), + _(EGL_WIDTH, video_context->attrib_list.width.value / 2), + _(EGL_LINUX_DRM_FOURCC_EXT, DRM_FORMAT_GR88), + _(EGL_DMA_BUF_PLANE0_FD_EXT, encode_context_frame->planes[1].fd), + _(EGL_DMA_BUF_PLANE0_OFFSET_EXT, encode_context_frame->planes[1].offset), + _(EGL_DMA_BUF_PLANE0_PITCH_EXT, encode_context_frame->planes[1].pitch), + EGL_NONE, +#undef _ + }; + if (GpuContextCreateImage(video_context->gpu_context, attrib_list_chroma, + &imported_frame_planes[1])) { + LOG("Failed to import chroma frame plane"); + goto rollback_luma_plane; + } + + size_t imported_images_count = video_context->imported_images_count + 1; + struct GpuContextImage** imported_images = + realloc(video_context->imported_images, + imported_images_count * sizeof(struct GpuContextCreateImage*)); + if (!imported_images) { + LOG("Failed to reallocate imported images list (%s)", strerror(errno)); + goto rollback_chroma_plane; + } + + imported_images[video_context->imported_images_count] = imported_frame_planes; + video_context->imported_images = imported_images; + video_context->imported_images_count = imported_images_count; + return imported_frame_planes; + +rollback_chroma_plane: + GpuContextDestroyImage(video_context->gpu_context, &imported_frame_planes[1]); +rollback_luma_plane: + GpuContextDestroyImage(video_context->gpu_context, &imported_frame_planes[0]); +rollback_imported_frame_planes: + free(imported_frame_planes); + return NULL; } static void OnExportDmabufFrameReady( void* data, struct zwlr_export_dmabuf_frame_v1* export_dmabuf_frame, uint32_t tv_sec_hi, uint32_t tv_sec_lo, uint32_t tv_nsec) { - (void)data; - (void)export_dmabuf_frame; - (void)tv_sec_hi; - (void)tv_sec_lo; - (void)tv_nsec; struct VideoContext* video_context = data; - LOG("%s(data=%p, export_dmabuf_frame=%p, " - "tv_sec_hi=%u, tv_sec_lo=%u, tv_nsec=%u)", - __FUNCTION__, data, (void*)export_dmabuf_frame, tv_sec_hi, tv_sec_lo, - tv_nsec); + struct GpuContextImage source_image; + if (!GpuContextCreateImage(video_context->gpu_context, + (EGLAttrib*)&video_context->attrib_list, + &source_image)) { + LOG("Failed to import Wayland frame"); + goto rollback_attrib_list; + } + + struct EncodeContextFrame* encode_context_frame = + EncodeContextDequeue(video_context->encode_context); + if (!encode_context_frame) { + LOG("Failed to dequeue encode context frame"); + // TODO(mburakov): Now what?.. + goto rollback_source_image; + } + + struct GpuContextImage* target_images = encode_context_frame->user_data; + if (!target_images) { + encode_context_frame->user_data = + ImportEncodeContextFrame(video_context, encode_context_frame); + target_images = encode_context_frame->user_data; + if (!target_images) { + LOG("Failed to import encode context frame"); + // TODO(mburakov): Now what?.. + goto rollback_encode_context_frame; + } + } + + if (!GpuContextConvertColorspace( + video_context->gpu_context, video_context->attrib_list.width.value, + video_context->attrib_list.height.value, source_image.gl_texture, + target_images[0].gl_texture, target_images[1].gl_texture)) { + LOG("Failed to convert Wayland frame colorspace"); + // TODO(mburakov): Now what?.. + goto rollback_encode_context_frame; + } + + if (!EncodeContextQueue(video_context->encode_context, encode_context_frame, + true)) { + LOG("Failed to encode video frame"); + // TODO(mburakov): Now what? + goto rollback_encode_context_frame; + } + + goto rollback_source_image; + +rollback_encode_context_frame: + assert(EncodeContextQueue(video_context->encode_context, encode_context_frame, + false)); +rollback_source_image: + GpuContextDestroyImage(video_context->gpu_context, &source_image); +rollback_attrib_list: + ResetAttribList(video_context); zwlr_export_dmabuf_frame_v1_destroy(export_dmabuf_frame); } @@ -183,16 +372,15 @@ static void OnExportDmabufFrameCancel( void* data, struct zwlr_export_dmabuf_frame_v1* export_dmabuf_frame, uint32_t reason) { (void)data; - (void)export_dmabuf_frame; - (void)reason; - struct VideoContext* video_context = data; static const char* const kCancelReasons[] = { "temporary", "permanent", "resizing", }; - LOG("%s(data=%p, export_dmabuf_frame=%p, reason=%s)", __FUNCTION__, data, - (void*)export_dmabuf_frame, kCancelReasons[reason]); + assert(reason < LENGTH(kCancelReasons)); + LOG("Capturing is cancelled (%s)", kCancelReasons[reason]); + struct VideoContext* video_context = data; + ResetAttribList(video_context); zwlr_export_dmabuf_frame_v1_destroy(export_dmabuf_frame); } @@ -248,16 +436,26 @@ struct VideoContext* VideoContextCreate(struct IoContext* io_context) { *video_context = (struct VideoContext){ .io_context = io_context, + .attrib_list.dma_buf_plane[0].fd.value = -1, + .attrib_list.dma_buf_plane[1].fd.value = -1, + .attrib_list.dma_buf_plane[2].fd.value = -1, + .attrib_list.dma_buf_plane[3].fd.value = -1, }; if (!InitWaylandGlobals(video_context)) { - LOG("Failed to init wayland globals"); + LOG("Failed to init Wayland globals"); goto rollback_video_context; } + video_context->gpu_context = GpuContextCreate(video_context->display); + if (!video_context->gpu_context) { + LOG("Failed to create gpu context"); + goto rollback_wayland_globals; + } + video_context->thread_loop = pw_thread_loop_new("video-capture", NULL); if (!video_context->thread_loop) { LOG("Failed to create thread loop"); - goto rollback_wayland_globals; + goto rollback_gpu_context; } pw_thread_loop_lock(video_context->thread_loop); @@ -291,12 +489,15 @@ struct VideoContext* VideoContextCreate(struct IoContext* io_context) { goto rollback_thread_loop; } + ResetAttribList(video_context); pw_thread_loop_unlock(video_context->thread_loop); return video_context; rollback_thread_loop: pw_thread_loop_unlock(video_context->thread_loop); pw_thread_loop_destroy(video_context->thread_loop); +rollback_gpu_context: + GpuContextDestroy(video_context->gpu_context); rollback_wayland_globals: zwlr_export_dmabuf_manager_v1_destroy(video_context->export_dmabuf_manager); wl_output_destroy(video_context->output); @@ -314,6 +515,16 @@ void VideoContextDestroy(struct VideoContext* video_context) { video_context->source)); pw_thread_loop_unlock(video_context->thread_loop); pw_thread_loop_destroy(video_context->thread_loop); + for (size_t index = 0; index < video_context->imported_images_count; + index++) { + GpuContextDestroyImage(video_context->gpu_context, + &video_context->imported_images[index][0]); + GpuContextDestroyImage(video_context->gpu_context, + &video_context->imported_images[index][1]); + free(video_context->imported_images[index]); + } + free(video_context->imported_images); + GpuContextDestroy(video_context->gpu_context); zwlr_export_dmabuf_manager_v1_destroy(video_context->export_dmabuf_manager); wl_output_release(video_context->output); wl_registry_destroy(video_context->registry); -- cgit v1.2.3