diff options
| -rw-r--r-- | chroma.glsl | 25 | ||||
| -rw-r--r-- | encode_context.c | 74 | ||||
| -rw-r--r-- | encode_context.h | 46 | ||||
| -rw-r--r-- | gpu_context.c | 456 | ||||
| -rw-r--r-- | gpu_context.h | 44 | ||||
| -rw-r--r-- | luma.glsl | 27 | ||||
| -rw-r--r-- | makefile | 3 | ||||
| -rw-r--r-- | video_context.c | 293 |
8 files changed, 916 insertions, 52 deletions
diff --git a/chroma.glsl b/chroma.glsl index dd103b1..44f1bc2 100644 --- a/chroma.glsl +++ b/chroma.glsl @@ -17,11 +17,26 @@ uniform sampler2D img_input; uniform mediump vec2 sample_offsets[4]; -uniform mediump mat3 colorspace; -uniform mediump vec3 ranges[2]; - varying mediump vec2 texcoord; +const mediump mat3 kColorSpace = mat3( + 0.299, 0.587, 0.114, + -0.168736, -0.331264, 0.5, + 0.5, -0.418688, -0.081312 +); + +const mediump vec3 kColorRangeBase = vec3( + 16.0 / 255.0, + 16.0 / 255.0, + 16.0 / 255.0 +); + +const mediump vec3 kColorRangeScale = vec3( + (235.0 - 16.0) / 255.0, + (240.0 - 16.0) / 255.0, + (240.0 - 16.0) / 255.0 +); + mediump vec4 supersample() { return texture2D(img_input, texcoord + sample_offsets[0]) + texture2D(img_input, texcoord + sample_offsets[1]) + @@ -30,8 +45,8 @@ mediump vec4 supersample() { } mediump vec3 rgb2yuv(in mediump vec3 rgb) { - mediump vec3 yuv = colorspace * rgb.rgb + vec3(0.0, 0.5, 0.5); - return ranges[0] + yuv * ranges[1]; + mediump vec3 yuv = kColorSpace * rgb.rgb + vec3(0.0, 0.5, 0.5); + return kColorRangeBase + yuv * kColorRangeScale; } void main() { diff --git a/encode_context.c b/encode_context.c new file mode 100644 index 0000000..353c3fa --- /dev/null +++ b/encode_context.c @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2024 Mikhail Burakov. This file is part of streamer. + * + * streamer is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * streamer is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with streamer. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "encode_context.h" + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <va/va.h> +#include <va/va_drm.h> +#include <va/va_drmcommon.h> + +#include "io_context.h" +#include "util.h" + +struct EncodeContext { + struct IoContext* io_context; + size_t width; + size_t height; +}; + +struct EncodeContext* EncodeContextCreate(struct IoContext* io_context, + uint32_t width, uint32_t height) { + LOG("Initializing encoder context for %ux%u resolution", width, height); + struct EncodeContext* encode_context = malloc(sizeof(struct EncodeContext)); + if (!encode_context) { + LOG("Failed to allocate encode context (%s)", strerror(errno)); + return NULL; + } + + *encode_context = (struct EncodeContext){ + .io_context = io_context, + .width = width, + .height = height, + }; + + return encode_context; +} + +struct EncodeContextFrame* EncodeContextDequeue( + struct EncodeContext* encode_context) { + (void)encode_context; + // TODO(mburakov): Implement this! + return NULL; +} + +bool EncodeContextQueue(struct EncodeContext* encode_context, + struct EncodeContextFrame* encode_context_frame, + bool encode) { + (void)encode_context; + (void)encode_context_frame; + (void)encode; + // TODO(mburakov): Implement this! + return true; +} + +void EncodeContextDestroy(struct EncodeContext* encode_context) { + free(encode_context); +} diff --git a/encode_context.h b/encode_context.h new file mode 100644 index 0000000..88d4d30 --- /dev/null +++ b/encode_context.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2024 Mikhail Burakov. This file is part of streamer. + * + * streamer is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * streamer is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with streamer. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef STREAMER_ENCODE_CONTEXT_H_ +#define STREAMER_ENCODE_CONTEXT_H_ + +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> + +struct EncodeContext; +struct IoContext; + +struct EncodeContextFrame { + void* user_data; + struct { + int fd; + uint32_t offset; + uint32_t pitch; + } const planes[2]; +}; + +struct EncodeContext* EncodeContextCreate(struct IoContext* io_context, + uint32_t width, uint32_t height); +struct EncodeContextFrame* EncodeContextDequeue( + struct EncodeContext* encode_context); +bool EncodeContextQueue(struct EncodeContext* encode_context, + struct EncodeContextFrame* encode_context_frame, + bool encode); +void EncodeContextDestroy(struct EncodeContext* encode_context); + +#endif // STREAMER_ENCODE_CONTEXT_H_ diff --git a/gpu_context.c b/gpu_context.c new file mode 100644 index 0000000..b44ce64 --- /dev/null +++ b/gpu_context.c @@ -0,0 +1,456 @@ +/* + * Copyright (C) 2024 Mikhail Burakov. This file is part of streamer. + * + * streamer is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * streamer is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with streamer. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "gpu_context.h" + +#include <EGL/eglext.h> +#include <GLES2/gl2ext.h> +#include <assert.h> +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "util.h" + +extern const char _binary_vertex_glsl_start[]; +extern const char _binary_vertex_glsl_end[]; +extern const char _binary_luma_glsl_start[]; +extern const char _binary_luma_glsl_end[]; +extern const char _binary_chroma_glsl_start[]; +extern const char _binary_chroma_glsl_end[]; + +struct GpuContext { + // EGL objects + EGLDisplay egl_display; + EGLContext egl_context; + + // OpenGL functions + PFNGLEGLIMAGETARGETTEXTURE2DOESPROC glEGLImageTargetTexture2DOES; + + // OpenGL objects + GLuint program_luma; + GLuint program_chroma; + GLint sample_offsets; + GLuint framebuffer; + GLuint vertices; +}; + +static const char* EglErrorString(EGLint error) { + static const char* const kEglErrorStrings[] = { + "EGL_SUCCESS", "EGL_NOT_INITIALIZED", "EGL_BAD_ACCESS", + "EGL_BAD_ALLOC", "EGL_BAD_ATTRIBUTE", "EGL_BAD_CONFIG", + "EGL_BAD_CONTEXT", "EGL_BAD_CURRENT_SURFACE", "EGL_BAD_DISPLAY", + "EGL_BAD_MATCH", "EGL_BAD_NATIVE_PIXMAP", "EGL_BAD_NATIVE_WINDOW", + "EGL_BAD_PARAMETER", "EGL_BAD_SURFACE", "EGL_CONTEXT_LOST", + }; + return EGL_SUCCESS <= error && + error < EGL_SUCCESS + (EGLint)LENGTH(kEglErrorStrings) + ? kEglErrorStrings[error - EGL_SUCCESS] + : "???"; +} + +static const char* GlErrorString(GLenum error) { + static const char* const kGlErrorStrings[] = { + "GL_INVALID_ENUM", + "GL_INVALID_VALUE", + "GL_INVALID_OPERATION", + "GL_STACK_OVERFLOW", + "GL_STACK_UNDERFLOW", + "GL_OUT_OF_MEMORY", + "GL_INVALID_FRAMEBUFFER_OPERATION", + "GL_CONTEXT_LOST", + }; + if (error == GL_NO_ERROR) return "GL_NO_ERROR"; + return GL_INVALID_ENUM <= error && + error < GL_INVALID_ENUM + LENGTH(kGlErrorStrings) + ? kGlErrorStrings[error - GL_INVALID_ENUM] + : "???"; +} + +#define DEFINE_CHECK_BUILDABLE_FUNCTION(what, err, op) \ + static bool CheckBuildable##what(GLuint buildable) { \ + GLenum error = glGetError(); \ + if (error != GL_NO_ERROR) { \ + LOG("Failed to " err " (%s)", GlErrorString(error)); \ + return false; \ + } \ + GLint status; \ + glGet##what##iv(buildable, op, &status); \ + if (status != GL_TRUE) { \ + GLint log_length; \ + glGet##what##iv(buildable, GL_INFO_LOG_LENGTH, &log_length); \ + char message[log_length]; \ + memset(message, 0, sizeof(message)); \ + glGet##what##InfoLog(buildable, log_length, NULL, message); \ + LOG("%s", message); \ + return false; \ + } \ + return true; \ + } + +DEFINE_CHECK_BUILDABLE_FUNCTION(Shader, "compile shader", GL_COMPILE_STATUS) +DEFINE_CHECK_BUILDABLE_FUNCTION(Program, "link program", GL_LINK_STATUS) + +static GLuint CreateGlProgram(const char* vs_begin, const char* vs_end, + const char* fs_begin, const char* fs_end) { + GLuint program = 0; + GLuint vertex = glCreateShader(GL_VERTEX_SHADER); + if (!vertex) { + LOG("Failed to create vertex shader (%s)", GlErrorString(glGetError())); + goto bail_out; + } + GLsizei size = (GLsizei)(vs_end - vs_begin); + glShaderSource(vertex, 1, &vs_begin, &size); + glCompileShader(vertex); + if (!CheckBuildableShader(vertex)) { + goto delete_vs; + } + + GLuint fragment = glCreateShader(GL_FRAGMENT_SHADER); + if (!fragment) { + LOG("Failed to create fragment shader (%s)", GlErrorString(glGetError())); + goto delete_vs; + } + size = (GLsizei)(fs_end - fs_begin); + glShaderSource(fragment, 1, &fs_begin, &size); + glCompileShader(fragment); + if (!CheckBuildableShader(fragment)) { + goto delete_fs; + } + + program = glCreateProgram(); + if (!program) { + LOG("Failed to create shader program (%s)", GlErrorString(glGetError())); + goto delete_fs; + } + glAttachShader(program, vertex); + glAttachShader(program, fragment); + glLinkProgram(program); + if (!CheckBuildableProgram(program)) { + glDeleteProgram(program); + program = 0; + goto delete_fs; + } + +delete_fs: + glDeleteShader(fragment); +delete_vs: + glDeleteShader(vertex); +bail_out: + return program; +} + +static bool GpuContextInitOpenGL(struct GpuContext* gpu_context) { + // TODO(mburakov): Check extensions? + gpu_context->glEGLImageTargetTexture2DOES = + (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglGetProcAddress( + "glEGLImageTargetTexture2DOES"); + if (!gpu_context->glEGLImageTargetTexture2DOES) { + LOG("Failed to get address of glEGLImageTargetTexture2DOES"); + return false; + } + + gpu_context->program_luma = + CreateGlProgram(_binary_vertex_glsl_start, _binary_vertex_glsl_end, + _binary_luma_glsl_start, _binary_luma_glsl_end); + if (!gpu_context->program_luma) { + LOG("Failed to create luma program"); + return false; + } + + gpu_context->program_chroma = + CreateGlProgram(_binary_vertex_glsl_start, _binary_vertex_glsl_end, + _binary_chroma_glsl_start, _binary_chroma_glsl_end); + if (!gpu_context->program_chroma) { + LOG("Failed to create chroma program"); + goto rollback_program_luma; + } + + gpu_context->sample_offsets = + glGetUniformLocation(gpu_context->program_chroma, "sample_offsets"); + if (gpu_context->sample_offsets == -1) { + LOG("Failed to get sample_offsets uniform location (%s)", + GlErrorString(glGetError())); + goto rollback_program_chroma; + } + + gpu_context->framebuffer = 0; + glGenFramebuffers(1, &gpu_context->framebuffer); + glBindFramebuffer(GL_FRAMEBUFFER, gpu_context->framebuffer); + if (!gpu_context->framebuffer) { + LOG("Failed to allocate framebuffer (%s)", GlErrorString(glGetError())); + goto rollback_program_chroma; + } + + gpu_context->vertices = 0; + glGenBuffers(1, &gpu_context->vertices); + glBindBuffer(GL_ARRAY_BUFFER, gpu_context->vertices); + if (!gpu_context->vertices) { + LOG("Failed to allocate buffer (%s)", GlErrorString(glGetError())); + goto rollback_framebuffer; + } + + static const GLfloat kVertices[] = {.0f, .0f, 1.f, 0.f, 1.f, 1.f, .0f, 1.f}; + glBufferData(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices, GL_STATIC_DRAW); + glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 0, NULL); + glEnableVertexAttribArray(0); + GLenum error = glGetError(); + if (error != GL_NO_ERROR) { + LOG("Failed to initialize array buffer (%s)", GlErrorString(error)); + goto rollback_vertices; + } + return true; + +rollback_vertices: + glDeleteBuffers(1, &gpu_context->vertices); +rollback_framebuffer: + glDeleteFramebuffers(1, &gpu_context->framebuffer); +rollback_program_chroma: + glDeleteProgram(gpu_context->program_chroma); +rollback_program_luma: + glDeleteProgram(gpu_context->program_luma); + return false; +} + +struct GpuContext* GpuContextCreate(EGLNativeDisplayType native_display) { + struct GpuContext* gpu_context = malloc(sizeof(struct GpuContext)); + if (!gpu_context) { + LOG("Failed to allocate gpu context (%s)", strerror(errno)); + return NULL; + } + + gpu_context->egl_display = + eglGetPlatformDisplay(EGL_PLATFORM_WAYLAND_KHR, native_display, NULL); + if (gpu_context->egl_display == EGL_NO_DISPLAY) { + LOG("Failed to get platform display (%s)", EglErrorString(eglGetError())); + goto rollback_gpu_context; + } + + EGLint major, minor; + if (!eglInitialize(gpu_context->egl_display, &major, &minor)) { + LOG("Failed to initialize display (%s)", EglErrorString(eglGetError())); + goto rollback_egl_display; + } + + LOG("Initialized EGL %d.%d", major, minor); + if (!eglBindAPI(EGL_OPENGL_ES_API)) { + LOG("Failed to bind EGL API (%s)", EglErrorString(eglGetError())); + goto rollback_egl_display; + } + + static const EGLint kEglContextAttribs[] = { +#define _(...) __VA_ARGS__ + _(EGL_CONTEXT_MAJOR_VERSION, 3), + _(EGL_CONTEXT_MINOR_VERSION, 1), + EGL_NONE, +#undef _ + }; + gpu_context->egl_context = + eglCreateContext(gpu_context->egl_display, EGL_NO_CONFIG_KHR, + EGL_NO_CONTEXT, kEglContextAttribs); + if (gpu_context->egl_context == EGL_NO_CONTEXT) { + LOG("Failed to create EGL context (%s)", EglErrorString(eglGetError())); + goto rollback_egl_display; + } + + if (!eglMakeCurrent(gpu_context->egl_display, EGL_NO_SURFACE, EGL_NO_SURFACE, + gpu_context->egl_context)) { + LOG("Failed to make EGL context current (%s)", + EglErrorString(eglGetError())); + goto rollback_egl_context; + } + + bool result = GpuContextInitOpenGL(gpu_context); + assert(eglMakeCurrent(gpu_context->egl_display, EGL_NO_SURFACE, + EGL_NO_SURFACE, EGL_NO_CONTEXT)); + if (!result) { + LOG("Failed to initialize OpenGL objects"); + goto rollback_egl_context; + } + return gpu_context; + +rollback_egl_context: + assert(eglDestroyContext(gpu_context->egl_display, gpu_context->egl_context)); +rollback_egl_display: + assert(eglTerminate(gpu_context->egl_display)); +rollback_gpu_context: + free(gpu_context); + return NULL; +} + +bool GpuContextCreateImage(const struct GpuContext* gpu_context, + const EGLAttrib* attrib_list, + struct GpuContextImage* gpu_context_image) { + EGLImage egl_image = eglCreateImage(gpu_context->egl_display, EGL_NO_CONTEXT, + EGL_LINUX_DMA_BUF_EXT, NULL, attrib_list); + if (egl_image == EGL_NO_IMAGE) { + LOG("Failed to create EGL image (%s)", EglErrorString(eglGetError())); + return false; + } + + if (!eglMakeCurrent(gpu_context->egl_display, EGL_NO_SURFACE, EGL_NO_SURFACE, + gpu_context->egl_context)) { + LOG("Failed to make EGL context current (%s)", + EglErrorString(eglGetError())); + goto rollback_egl_image; + } + + GLuint gl_texture = 0; + glGenTextures(1, &gl_texture); + if (!gl_texture) { + LOG("Failed to allocate texture (%s)", GlErrorString(glGetError())); + goto rollback_egl_make_current; + } + + glBindTexture(GL_TEXTURE_2D, gl_texture); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + gpu_context->glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, egl_image); + glBindTexture(GL_TEXTURE_2D, 0); + + GLenum error = glGetError(); + if (error != GL_NO_ERROR) { + LOG("Failed to initialize texture (%s)", GlErrorString(error)); + goto rollback_gl_texture; + } + + assert(eglMakeCurrent(gpu_context->egl_display, EGL_NO_SURFACE, + EGL_NO_SURFACE, EGL_NO_CONTEXT)); + *gpu_context_image = (struct GpuContextImage){ + .egl_image = egl_image, + .gl_texture = gl_texture, + }; + return true; + +rollback_gl_texture: + glDeleteTextures(1, &gl_texture); +rollback_egl_make_current: + assert(eglMakeCurrent(gpu_context->egl_context, EGL_NO_SURFACE, + EGL_NO_SURFACE, EGL_NO_CONTEXT)); +rollback_egl_image: + assert(eglDestroyImage(gpu_context->egl_display, egl_image)); + return false; +} + +static bool GpuContextRender(GLuint source, GLuint target) { + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, + target, 0); + GLenum framebuffer_status = glCheckFramebufferStatus(GL_FRAMEBUFFER); + if (framebuffer_status != GL_FRAMEBUFFER_COMPLETE) { + LOG("Framebuffer is incomplete (0x%x)", framebuffer_status); + return false; + } + + glBindTexture(GL_TEXTURE_2D, source); + glDrawArrays(GL_TRIANGLE_FAN, 0, 4); + GLenum error = glGetError(); + if (error != GL_NO_ERROR) { + LOG("Failed to render (%s)", GlErrorString(error)); + return false; + } + return true; +} + +bool GpuContextConvertColorspace(const struct GpuContext* gpu_context, + EGLAttrib width, EGLAttrib height, + GLuint source, GLuint luma, GLuint chroma) { + if (!eglMakeCurrent(gpu_context->egl_display, EGL_NO_SURFACE, EGL_NO_SURFACE, + gpu_context->egl_context)) { + LOG("Failed to make EGL context current (%s)", + EglErrorString(eglGetError())); + return false; + } + + glUseProgram(gpu_context->program_luma); + glViewport(0, 0, (GLsizei)width, (GLsizei)height); + if (!GpuContextRender(source, luma)) { + LOG("Failed to convert luma plane"); + goto rollback_egl_make_current; + } + + const GLfloat sample_offsets[] = { +#define _(...) __VA_ARGS__ + _(0.f, 0.f), + _(1.f / (GLfloat)width, 0.f), + _(0.f, 1.f / (GLfloat)height), + _(1.f / (GLfloat)width, 1.f / (GLfloat)height), +#undef _ + }; + + glUseProgram(gpu_context->program_chroma); + glUniform2fv(gpu_context->sample_offsets, 4, sample_offsets); + glViewport(0, 0, (GLsizei)width / 2, (GLsizei)height / 2); + if (!GpuContextRender(source, chroma)) { + LOG("Failed to convert chroma plane"); + goto rollback_egl_make_current; + } + + EGLSync sync = eglCreateSync(gpu_context->egl_display, EGL_SYNC_FENCE, NULL); + if (sync == EGL_NO_SYNC) { + LOG("Failed to create EGL fence sync (%s)", EglErrorString(eglGetError())); + goto rollback_egl_make_current; + } + + if (!eglClientWaitSync(gpu_context->egl_display, sync, 0, EGL_FOREVER)) { + LOG("Failed to wait EGLfence sync (%s)", EglErrorString(eglGetError())); + goto rollback_sync; + } + + assert(eglDestroySync(gpu_context->egl_display, sync)); + assert(eglMakeCurrent(gpu_context->egl_display, EGL_NO_SURFACE, + EGL_NO_SURFACE, EGL_NO_CONTEXT)); + return true; + +rollback_sync: + assert(eglMakeCurrent(gpu_context->egl_display, EGL_NO_SURFACE, + EGL_NO_SURFACE, EGL_NO_CONTEXT)); +rollback_egl_make_current: + assert(eglMakeCurrent(gpu_context->egl_display, EGL_NO_SURFACE, + EGL_NO_SURFACE, EGL_NO_CONTEXT)); + return false; +} + +void GpuContextDestroyImage(const struct GpuContext* gpu_context, + const struct GpuContextImage* gpu_context_image) { + assert(eglMakeCurrent(gpu_context->egl_display, EGL_NO_SURFACE, + EGL_NO_SURFACE, gpu_context->egl_context)); + glDeleteTextures(1, &gpu_context_image->gl_texture); + assert(eglMakeCurrent(gpu_context->egl_display, EGL_NO_SURFACE, + EGL_NO_SURFACE, EGL_NO_CONTEXT)); + assert( + eglDestroyImage(gpu_context->egl_display, gpu_context_image->egl_image)); +} + +void GpuContextDestroy(struct GpuContext* gpu_context) { + assert(eglMakeCurrent(gpu_context->egl_display, EGL_NO_SURFACE, + EGL_NO_SURFACE, gpu_context->egl_context)); + glBindBuffer(GL_ARRAY_BUFFER, 0); + glDeleteBuffers(1, &gpu_context->vertices); + glBindFramebuffer(GL_FRAMEBUFFER, 0); + glDeleteFramebuffers(1, &gpu_context->framebuffer); + glUseProgram(0); + glDeleteProgram(gpu_context->program_chroma); + glDeleteProgram(gpu_context->program_luma); + assert(eglMakeCurrent(gpu_context->egl_display, EGL_NO_SURFACE, + EGL_NO_SURFACE, EGL_NO_CONTEXT)); + assert(eglDestroyContext(gpu_context->egl_display, gpu_context->egl_context)); + assert(eglTerminate(gpu_context->egl_display)); + free(gpu_context); +} diff --git a/gpu_context.h b/gpu_context.h new file mode 100644 index 0000000..b9044e9 --- /dev/null +++ b/gpu_context.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2024 Mikhail Burakov. This file is part of streamer. + * + * streamer is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * streamer is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with streamer. If not, see <https://www.gnu.org/licenses/>. + */ + +#ifndef STREAMER_GPU_CONTEXT_H_ +#define STREAMER_GPU_CONTEXT_H_ + +#include <EGL/egl.h> +#include <GLES2/gl2.h> +#include <stdbool.h> +#include <stddef.h> + +struct GpuContextImage { + EGLImage egl_image; + GLuint gl_texture; +}; + +struct GpuContext; + +struct GpuContext* GpuContextCreate(EGLNativeDisplayType native_display); +bool GpuContextCreateImage(const struct GpuContext* gpu_context, + const EGLAttrib* attrib_list, + struct GpuContextImage* gpu_context_image); +bool GpuContextConvertColorspace(const struct GpuContext* gpu_context, + EGLAttrib width, EGLAttrib height, + GLuint source, GLuint luma, GLuint chroma); +void GpuContextDestroyImage(const struct GpuContext* gpu_context, + const struct GpuContextImage* gpu_context_image); +void GpuContextDestroy(struct GpuContext* gpu_context); + +#endif // STREAMER_GPU_CONTEXT_H_ @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Mikhail Burakov. This file is part of streamer. + * Copyright (C) 2024 Mikhail Burakov. This file is part of streamer. * * streamer is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,14 +16,29 @@ */ uniform sampler2D img_input; -uniform mediump mat3 colorspace; -uniform mediump vec3 ranges[2]; - varying mediump vec2 texcoord; +const mediump mat3 kColorSpace = mat3( + 0.299, 0.587, 0.114, + -0.168736, -0.331264, 0.5, + 0.5, -0.418688, -0.081312 +); + +const mediump vec3 kColorRangeBase = vec3( + 16.0 / 255.0, + 16.0 / 255.0, + 16.0 / 255.0 +); + +const mediump vec3 kColorRangeScale = vec3( + (235.0 - 16.0) / 255.0, + (240.0 - 16.0) / 255.0, + (240.0 - 16.0) / 255.0 +); + mediump vec3 rgb2yuv(in mediump vec3 rgb) { - mediump vec3 yuv = colorspace * rgb.rgb + vec3(0.0, 0.5, 0.5); - return ranges[0] + yuv * ranges[1]; + mediump vec3 yuv = kColorSpace * rgb.rgb + vec3(0.0, 0.5, 0.5); + return kColorRangeBase + yuv * kColorRangeScale; } void main() { @@ -3,6 +3,9 @@ src:=$(wildcard *.c) obj:=$(src:.c=.o) libs:=\ + egl \ + glesv2 \ + libdrm \ libpipewire-0.3 \ wayland-client diff --git a/video_context.c b/video_context.c index 10dd7b9..5d2a2fd 100644 --- a/video_context.c +++ b/video_context.c @@ -17,7 +17,10 @@ #include "video_context.h" +#include <EGL/egl.h> +#include <EGL/eglext.h> #include <assert.h> +#include <drm_fourcc.h> #include <errno.h> #include <pipewire/pipewire.h> #include <stdio.h> @@ -26,11 +29,19 @@ #include <threads.h> #include <wayland-client.h> +#include "encode_context.h" +#include "gpu_context.h" #include "util.h" #include "wlr-export-dmabuf-unstable-v1.h" +struct EGLAttribPair { + EGLAttrib key; + EGLAttrib value; +}; + struct VideoContext { struct IoContext* io_context; + struct EncodeContext* encode_context; // Wayland globals struct wl_display* display; @@ -38,11 +49,39 @@ struct VideoContext { struct wl_output* output; struct zwlr_export_dmabuf_manager_v1* export_dmabuf_manager; + // Colorspace conversion + struct GpuContext* gpu_context; + struct GpuContextImage** imported_images; + size_t imported_images_count; + // Threading struct pw_thread_loop* thread_loop; struct spa_source* source; + + // Volatile state + struct { + struct EGLAttribPair height; + struct EGLAttribPair width; + struct EGLAttribPair linux_drm_fourcc; + struct { + struct EGLAttribPair fd; + struct EGLAttribPair offset; + struct EGLAttribPair pitch; + struct EGLAttribPair modifier_lo; + struct EGLAttribPair modifier_hi; + } dma_buf_plane[4]; + EGLAttrib terminator; + } attrib_list; }; +static void SetEGLAttribPair(struct EGLAttribPair* pair, EGLAttrib key, + EGLAttrib value) { + *pair = (struct EGLAttribPair){ + .key = key, + .value = value, + }; +} + static void OnRegistryGlobal(void* data, struct wl_registry* wl_registry, uint32_t name, const char* interface, uint32_t version) { @@ -126,56 +165,206 @@ static void OnExportDmabufFrameFrame( uint32_t width, uint32_t height, uint32_t offset_x, uint32_t offset_y, uint32_t buffer_flags, uint32_t flags, uint32_t format, uint32_t mod_high, uint32_t mod_low, uint32_t num_objects) { - (void)data; (void)export_dmabuf_frame; - (void)width; - (void)height; - (void)offset_x; - (void)offset_y; - (void)buffer_flags; (void)flags; - (void)format; - (void)mod_high; - (void)mod_low; - (void)num_objects; - LOG("%s(data=%p, export_dmabuf_frame=%p, width=%u, height=%u, " - "offset_x=%u, offset_y=%u, buffer_flags=0x%x, flags=0x%x, " - "format=0x%08x, mod_high=%08x, mod_low=%08x, num_objects=%u)", - __FUNCTION__, data, (void*)export_dmabuf_frame, width, height, offset_x, - offset_y, buffer_flags, flags, format, mod_high, mod_low, num_objects); + + struct VideoContext* video_context = data; + if (!video_context->encode_context) { + video_context->encode_context = + EncodeContextCreate(video_context->io_context, width, height); + if (!video_context->encode_context) { + LOG("Failed to create encode context"); + // TODO(mburakov): Now what?.. + } + } + + // TODO(mburakov): Maybe handle those? + assert(!offset_x && !offset_y && !buffer_flags); + SetEGLAttribPair(&video_context->attrib_list.height, EGL_HEIGHT, height); + SetEGLAttribPair(&video_context->attrib_list.width, EGL_WIDTH, width); + SetEGLAttribPair(&video_context->attrib_list.linux_drm_fourcc, + EGL_LINUX_DRM_FOURCC_EXT, format); + + assert(num_objects <= LENGTH(video_context->attrib_list.dma_buf_plane)); + for (EGLAttrib index = 0; index < num_objects; index++) { + typeof(&video_context->attrib_list.dma_buf_plane[index]) plane = + &video_context->attrib_list.dma_buf_plane[index]; + SetEGLAttribPair(&plane->modifier_lo, + EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT + index * 2, mod_low); + SetEGLAttribPair(&plane->modifier_hi, + EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT + index * 2, mod_high); + } } static void OnExportDmabufFrameObject( void* data, struct zwlr_export_dmabuf_frame_v1* export_dmabuf_frame, uint32_t index, int32_t fd, uint32_t size, uint32_t offset, uint32_t stride, uint32_t plane_index) { - (void)data; (void)export_dmabuf_frame; - (void)index; - (void)fd; (void)size; - (void)offset; - (void)stride; - (void)plane_index; - LOG("%s(data=%p, export_dmabuf_frame=%p, index=%u, fd=%d, " - "size=%u, offset=%u, stride=%u, plane_index=%u)", - __FUNCTION__, data, (void*)export_dmabuf_frame, index, fd, size, offset, - stride, plane_index); + + assert(index == plane_index); + struct VideoContext* video_context = data; + + SetEGLAttribPair(&video_context->attrib_list.dma_buf_plane[index].fd, + EGL_DMA_BUF_PLANE0_FD_EXT + (EGLAttrib)index * 3, fd); + SetEGLAttribPair(&video_context->attrib_list.dma_buf_plane[index].offset, + EGL_DMA_BUF_PLANE0_OFFSET_EXT + (EGLAttrib)index * 3, + offset); + SetEGLAttribPair(&video_context->attrib_list.dma_buf_plane[index].pitch, + EGL_DMA_BUF_PLANE0_PITCH_EXT + (EGLAttrib)index * 3, stride); +} + +static void ResetAttribList(struct VideoContext* video_context) { + SetEGLAttribPair(&video_context->attrib_list.height, EGL_NONE, EGL_NONE); + SetEGLAttribPair(&video_context->attrib_list.width, EGL_NONE, EGL_NONE); + SetEGLAttribPair(&video_context->attrib_list.linux_drm_fourcc, EGL_NONE, + EGL_NONE); + + static const EGLAttrib kMaxPlanesCount = + LENGTH(video_context->attrib_list.dma_buf_plane); + for (EGLAttrib index = 0; index < kMaxPlanesCount; index++) { + typeof(&video_context->attrib_list.dma_buf_plane[index]) plane = + &video_context->attrib_list.dma_buf_plane[index]; + if (plane->fd.value != -1) { + assert(!close((int)plane->fd.value)); + } + SetEGLAttribPair(&plane->fd, EGL_NONE, -1); + SetEGLAttribPair(&plane->offset, EGL_NONE, EGL_NONE); + SetEGLAttribPair(&plane->pitch, EGL_NONE, EGL_NONE); + SetEGLAttribPair(&plane->modifier_hi, EGL_NONE, EGL_NONE); + SetEGLAttribPair(&plane->modifier_lo, EGL_NONE, EGL_NONE); + } + + video_context->attrib_list.terminator = EGL_NONE; +} + +static struct GpuContextImage* ImportEncodeContextFrame( + struct VideoContext* video_context, + struct EncodeContextFrame* encode_context_frame) { + static_assert(LENGTH(encode_context_frame->planes) == 2, + "Suspicious amount of imported frame planes"); + struct GpuContextImage* imported_frame_planes = + malloc(2 * sizeof(struct GpuContextImage)); + if (!imported_frame_planes) { + LOG("Failed to allocate imported frame planes (%s)", strerror(errno)); + return NULL; + } + + EGLAttrib attrib_list_luma[] = { +#define _(...) __VA_ARGS__ + _(EGL_HEIGHT, video_context->attrib_list.height.value), + _(EGL_WIDTH, video_context->attrib_list.width.value), + _(EGL_LINUX_DRM_FOURCC_EXT, DRM_FORMAT_R8), + _(EGL_DMA_BUF_PLANE0_FD_EXT, encode_context_frame->planes[0].fd), + _(EGL_DMA_BUF_PLANE0_OFFSET_EXT, encode_context_frame->planes[0].offset), + _(EGL_DMA_BUF_PLANE0_PITCH_EXT, encode_context_frame->planes[0].pitch), + EGL_NONE, +#undef _ + }; + if (GpuContextCreateImage(video_context->gpu_context, attrib_list_luma, + &imported_frame_planes[0])) { + LOG("Failed to import luma frame plane"); + goto rollback_imported_frame_planes; + } + + EGLAttrib attrib_list_chroma[] = { +#define _(...) __VA_ARGS__ + _(EGL_HEIGHT, video_context->attrib_list.height.value / 2), + _(EGL_WIDTH, video_context->attrib_list.width.value / 2), + _(EGL_LINUX_DRM_FOURCC_EXT, DRM_FORMAT_GR88), + _(EGL_DMA_BUF_PLANE0_FD_EXT, encode_context_frame->planes[1].fd), + _(EGL_DMA_BUF_PLANE0_OFFSET_EXT, encode_context_frame->planes[1].offset), + _(EGL_DMA_BUF_PLANE0_PITCH_EXT, encode_context_frame->planes[1].pitch), + EGL_NONE, +#undef _ + }; + if (GpuContextCreateImage(video_context->gpu_context, attrib_list_chroma, + &imported_frame_planes[1])) { + LOG("Failed to import chroma frame plane"); + goto rollback_luma_plane; + } + + size_t imported_images_count = video_context->imported_images_count + 1; + struct GpuContextImage** imported_images = + realloc(video_context->imported_images, + imported_images_count * sizeof(struct GpuContextCreateImage*)); + if (!imported_images) { + LOG("Failed to reallocate imported images list (%s)", strerror(errno)); + goto rollback_chroma_plane; + } + + imported_images[video_context->imported_images_count] = imported_frame_planes; + video_context->imported_images = imported_images; + video_context->imported_images_count = imported_images_count; + return imported_frame_planes; + +rollback_chroma_plane: + GpuContextDestroyImage(video_context->gpu_context, &imported_frame_planes[1]); +rollback_luma_plane: + GpuContextDestroyImage(video_context->gpu_context, &imported_frame_planes[0]); +rollback_imported_frame_planes: + free(imported_frame_planes); + return NULL; } static void OnExportDmabufFrameReady( void* data, struct zwlr_export_dmabuf_frame_v1* export_dmabuf_frame, uint32_t tv_sec_hi, uint32_t tv_sec_lo, uint32_t tv_nsec) { - (void)data; - (void)export_dmabuf_frame; - (void)tv_sec_hi; - (void)tv_sec_lo; - (void)tv_nsec; struct VideoContext* video_context = data; - LOG("%s(data=%p, export_dmabuf_frame=%p, " - "tv_sec_hi=%u, tv_sec_lo=%u, tv_nsec=%u)", - __FUNCTION__, data, (void*)export_dmabuf_frame, tv_sec_hi, tv_sec_lo, - tv_nsec); + struct GpuContextImage source_image; + if (!GpuContextCreateImage(video_context->gpu_context, + (EGLAttrib*)&video_context->attrib_list, + &source_image)) { + LOG("Failed to import Wayland frame"); + goto rollback_attrib_list; + } + + struct EncodeContextFrame* encode_context_frame = + EncodeContextDequeue(video_context->encode_context); + if (!encode_context_frame) { + LOG("Failed to dequeue encode context frame"); + // TODO(mburakov): Now what?.. + goto rollback_source_image; + } + + struct GpuContextImage* target_images = encode_context_frame->user_data; + if (!target_images) { + encode_context_frame->user_data = + ImportEncodeContextFrame(video_context, encode_context_frame); + target_images = encode_context_frame->user_data; + if (!target_images) { + LOG("Failed to import encode context frame"); + // TODO(mburakov): Now what?.. + goto rollback_encode_context_frame; + } + } + + if (!GpuContextConvertColorspace( + video_context->gpu_context, video_context->attrib_list.width.value, + video_context->attrib_list.height.value, source_image.gl_texture, + target_images[0].gl_texture, target_images[1].gl_texture)) { + LOG("Failed to convert Wayland frame colorspace"); + // TODO(mburakov): Now what?.. + goto rollback_encode_context_frame; + } + + if (!EncodeContextQueue(video_context->encode_context, encode_context_frame, + true)) { + LOG("Failed to encode video frame"); + // TODO(mburakov): Now what? + goto rollback_encode_context_frame; + } + + goto rollback_source_image; + +rollback_encode_context_frame: + assert(EncodeContextQueue(video_context->encode_context, encode_context_frame, + false)); +rollback_source_image: + GpuContextDestroyImage(video_context->gpu_context, &source_image); +rollback_attrib_list: + ResetAttribList(video_context); zwlr_export_dmabuf_frame_v1_destroy(export_dmabuf_frame); } @@ -183,16 +372,15 @@ static void OnExportDmabufFrameCancel( void* data, struct zwlr_export_dmabuf_frame_v1* export_dmabuf_frame, uint32_t reason) { (void)data; - (void)export_dmabuf_frame; - (void)reason; - struct VideoContext* video_context = data; static const char* const kCancelReasons[] = { "temporary", "permanent", "resizing", }; - LOG("%s(data=%p, export_dmabuf_frame=%p, reason=%s)", __FUNCTION__, data, - (void*)export_dmabuf_frame, kCancelReasons[reason]); + assert(reason < LENGTH(kCancelReasons)); + LOG("Capturing is cancelled (%s)", kCancelReasons[reason]); + struct VideoContext* video_context = data; + ResetAttribList(video_context); zwlr_export_dmabuf_frame_v1_destroy(export_dmabuf_frame); } @@ -248,16 +436,26 @@ struct VideoContext* VideoContextCreate(struct IoContext* io_context) { *video_context = (struct VideoContext){ .io_context = io_context, + .attrib_list.dma_buf_plane[0].fd.value = -1, + .attrib_list.dma_buf_plane[1].fd.value = -1, + .attrib_list.dma_buf_plane[2].fd.value = -1, + .attrib_list.dma_buf_plane[3].fd.value = -1, }; if (!InitWaylandGlobals(video_context)) { - LOG("Failed to init wayland globals"); + LOG("Failed to init Wayland globals"); goto rollback_video_context; } + video_context->gpu_context = GpuContextCreate(video_context->display); + if (!video_context->gpu_context) { + LOG("Failed to create gpu context"); + goto rollback_wayland_globals; + } + video_context->thread_loop = pw_thread_loop_new("video-capture", NULL); if (!video_context->thread_loop) { LOG("Failed to create thread loop"); - goto rollback_wayland_globals; + goto rollback_gpu_context; } pw_thread_loop_lock(video_context->thread_loop); @@ -291,12 +489,15 @@ struct VideoContext* VideoContextCreate(struct IoContext* io_context) { goto rollback_thread_loop; } + ResetAttribList(video_context); pw_thread_loop_unlock(video_context->thread_loop); return video_context; rollback_thread_loop: pw_thread_loop_unlock(video_context->thread_loop); pw_thread_loop_destroy(video_context->thread_loop); +rollback_gpu_context: + GpuContextDestroy(video_context->gpu_context); rollback_wayland_globals: zwlr_export_dmabuf_manager_v1_destroy(video_context->export_dmabuf_manager); wl_output_destroy(video_context->output); @@ -314,6 +515,16 @@ void VideoContextDestroy(struct VideoContext* video_context) { video_context->source)); pw_thread_loop_unlock(video_context->thread_loop); pw_thread_loop_destroy(video_context->thread_loop); + for (size_t index = 0; index < video_context->imported_images_count; + index++) { + GpuContextDestroyImage(video_context->gpu_context, + &video_context->imported_images[index][0]); + GpuContextDestroyImage(video_context->gpu_context, + &video_context->imported_images[index][1]); + free(video_context->imported_images[index]); + } + free(video_context->imported_images); + GpuContextDestroy(video_context->gpu_context); zwlr_export_dmabuf_manager_v1_destroy(video_context->export_dmabuf_manager); wl_output_release(video_context->output); wl_registry_destroy(video_context->registry); |