Recovering hevc encoding with va (WIP)

4 files changed, 181 insertions, 5 deletions

diff --git a/encode_context.c b/encode_context.c
index 353c3fa..5efe8ca 100644
--- a/encode_context.c
+++ b/encode_context.c
@@ -17,13 +17,14 @@
 
 #include "encode_context.h"
 
+#include <assert.h>
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <va/va.h>
-#include <va/va_drm.h>
 #include <va/va_drmcommon.h>
+#include <va/va_wayland.h>
 
 #include "io_context.h"
 #include "util.h"
@@ -32,10 +33,141 @@ struct EncodeContext {
   struct IoContext* io_context;
   size_t width;
   size_t height;
+  VADisplay display;
+  VAConfigID config_id;
+
+  uint32_t packed_headers;
+  VAConfigAttribValEncHEVCFeatures hevc_features;
+  VAConfigAttribValEncHEVCBlockSizes hevc_block_sizes;
 };
 
+static const char* VaErrorString(VAStatus error) {
+  static const char* kVaErrorStrings[] = {
+      "VA_STATUS_SUCCESS",
+      "VA_STATUS_ERROR_OPERATION_FAILED",
+      "VA_STATUS_ERROR_ALLOCATION_FAILED",
+      "VA_STATUS_ERROR_INVALID_DISPLAY",
+      "VA_STATUS_ERROR_INVALID_CONFIG",
+      "VA_STATUS_ERROR_INVALID_CONTEXT",
+      "VA_STATUS_ERROR_INVALID_SURFACE",
+      "VA_STATUS_ERROR_INVALID_BUFFER",
+      "VA_STATUS_ERROR_INVALID_IMAGE",
+      "VA_STATUS_ERROR_INVALID_SUBPICTURE",
+      "VA_STATUS_ERROR_ATTR_NOT_SUPPORTED",
+      "VA_STATUS_ERROR_MAX_NUM_EXCEEDED",
+      "VA_STATUS_ERROR_UNSUPPORTED_PROFILE",
+      "VA_STATUS_ERROR_UNSUPPORTED_ENTRYPOINT",
+      "VA_STATUS_ERROR_UNSUPPORTED_RT_FORMAT",
+      "VA_STATUS_ERROR_UNSUPPORTED_BUFFERTYPE",
+      "VA_STATUS_ERROR_SURFACE_BUSY",
+      "VA_STATUS_ERROR_FLAG_NOT_SUPPORTED",
+      "VA_STATUS_ERROR_INVALID_PARAMETER",
+      "VA_STATUS_ERROR_RESOLUTION_NOT_SUPPORTED",
+      "VA_STATUS_ERROR_UNIMPLEMENTED",
+      "VA_STATUS_ERROR_SURFACE_IN_DISPLAYING",
+      "VA_STATUS_ERROR_INVALID_IMAGE_FORMAT",
+      "VA_STATUS_ERROR_DECODING_ERROR",
+      "VA_STATUS_ERROR_ENCODING_ERROR",
+      "VA_STATUS_ERROR_INVALID_VALUE",
+      "???",
+      "???",
+      "???",
+      "???",
+      "???",
+      "???",
+      "VA_STATUS_ERROR_UNSUPPORTED_FILTER",
+      "VA_STATUS_ERROR_INVALID_FILTER_CHAIN",
+      "VA_STATUS_ERROR_HW_BUSY",
+      "???",
+      "VA_STATUS_ERROR_UNSUPPORTED_MEMORY_TYPE",
+      "VA_STATUS_ERROR_NOT_ENOUGH_BUFFER",
+      "VA_STATUS_ERROR_TIMEDOUT",
+  };
+  return VA_STATUS_SUCCESS <= error &&
+                 error < VA_STATUS_SUCCESS + (int)LENGTH(kVaErrorStrings)
+             ? kVaErrorStrings[error - VA_STATUS_SUCCESS]
+             : "???";
+}
+
+static bool InitializeCodecCaps(struct EncodeContext* encode_context) {
+  VAConfigAttrib attrib_list[] = {
+      {.type = VAConfigAttribEncPackedHeaders},
+      {.type = VAConfigAttribEncHEVCFeatures},
+      {.type = VAConfigAttribEncHEVCBlockSizes},
+  };
+  VAStatus status = vaGetConfigAttributes(
+      encode_context->display, VAProfileHEVCMain, VAEntrypointEncSlice,
+      attrib_list, LENGTH(attrib_list));
+  if (status != VA_STATUS_SUCCESS) {
+    LOG("Failed to get va config attributes (%s)", VaErrorString(status));
+    return false;
+  }
+
+  if (attrib_list[0].value == VA_ATTRIB_NOT_SUPPORTED) {
+    LOG("VAConfigAttribEncPackedHeaders is not supported");
+  } else {
+    LOG("VAConfigAttribEncPackedHeaders is 0x%08x", attrib_list[0].value);
+    encode_context->packed_headers = attrib_list[0].value;
+  }
+
+  if (attrib_list[1].value == VA_ATTRIB_NOT_SUPPORTED) {
+    LOG("VAConfigAttribEncHEVCFeatures is not supported");
+    encode_context->hevc_features = (VAConfigAttribValEncHEVCFeatures){
+        .bits =
+            {
+                // mburakov: ffmpeg hardcodes these for i965 Skylake driver.
+                .separate_colour_planes = 0,     // Table 6-1
+                .scaling_lists = 0,              // No scaling lists
+                .amp = 1,                        // hardcoded
+                .sao = 0,                        // hardcoded
+                .pcm = 0,                        // hardcoded
+                .temporal_mvp = 0,               // hardcoded
+                .strong_intra_smoothing = 0,     // TODO
+                .dependent_slices = 0,           // No slice segments
+                .sign_data_hiding = 0,           // TODO
+                .constrained_intra_pred = 0,     // TODO
+                .transform_skip = 0,             // defaulted
+                .cu_qp_delta = 0,                // Fixed quality
+                .weighted_prediction = 0,        // TODO
+                .transquant_bypass = 0,          // TODO
+                .deblocking_filter_disable = 0,  // TODO
+            },
+    };
+  } else {
+    LOG("VAConfigAttribEncHEVCFeatures is 0x%08x", attrib_list[1].value);
+    encode_context->hevc_features.value = attrib_list[1].value;
+  }
+
+  if (attrib_list[2].value == VA_ATTRIB_NOT_SUPPORTED) {
+    LOG("VAConfigAttribEncHEVCBlockSizes is not supported");
+    encode_context->hevc_block_sizes = (VAConfigAttribValEncHEVCBlockSizes){
+        .bits =
+            {
+                // mburakov: ffmpeg hardcodes these for i965 Skylake driver.
+                .log2_max_coding_tree_block_size_minus3 = 2,     // hardcoded
+                .log2_min_coding_tree_block_size_minus3 = 0,     // TODO
+                .log2_min_luma_coding_block_size_minus3 = 0,     // hardcoded
+                .log2_max_luma_transform_block_size_minus2 = 3,  // hardcoded
+                .log2_min_luma_transform_block_size_minus2 = 0,  // hardcoded
+                .max_max_transform_hierarchy_depth_inter = 3,    // hardcoded
+                .min_max_transform_hierarchy_depth_inter = 0,    // defaulted
+                .max_max_transform_hierarchy_depth_intra = 3,    // hardcoded
+                .min_max_transform_hierarchy_depth_intra = 0,    // defaulted
+                .log2_max_pcm_coding_block_size_minus3 = 0,      // TODO
+                .log2_min_pcm_coding_block_size_minus3 = 0,      // TODO
+            },
+    };
+  } else {
+    LOG("VAConfigAttribEncHEVCBlockSizes is 0x%08x", attrib_list[2].value);
+    encode_context->hevc_block_sizes.value = attrib_list[2].value;
+  }
+
+  return true;
+}
+
 struct EncodeContext* EncodeContextCreate(struct IoContext* io_context,
-                                          uint32_t width, uint32_t height) {
+                                          uint32_t width, uint32_t height,
+                                          struct wl_display* display) {
   LOG("Initializing encoder context for %ux%u resolution", width, height);
   struct EncodeContext* encode_context = malloc(sizeof(struct EncodeContext));
   if (!encode_context) {
@@ -47,9 +179,47 @@ struct EncodeContext* EncodeContextCreate(struct IoContext* io_context,
       .io_context = io_context,
       .width = width,
       .height = height,
+      .display = vaGetDisplayWl(display),
   };
+  if (!encode_context->display) {
+    LOG("Failed to get VA display");
+    goto rollback_encode_context;
+  }
+
+  int major, minor;
+  VAStatus status = vaInitialize(encode_context->display, &major, &minor);
+  if (status != VA_STATUS_SUCCESS) {
+    LOG("Failed to initialize VA (%s)", VaErrorString(status));
+    goto rollback_display;
+  }
+
+  LOG("Initialized VA %d.%d", major, minor);
+  VAConfigAttrib attrib_list[] = {
+      {.type = VAConfigAttribRTFormat, .value = VA_RT_FORMAT_YUV420},
+  };
+  status = vaCreateConfig(encode_context->display, VAProfileHEVCMain,
+                          VAEntrypointEncSlice, attrib_list,
+                          LENGTH(attrib_list), &encode_context->config_id);
+  if (status != VA_STATUS_SUCCESS) {
+    LOG("Failed to create VA config (%s)", VaErrorString(status));
+    goto rollback_display;
+  }
+
+  if (!InitializeCodecCaps(encode_context)) {
+    LOG("Failed to initialize codec caps");
+    goto rollback_config_id;
+  }
 
   return encode_context;
+
+rollback_config_id:
+  assert(vaDestroyConfig(encode_context->display, encode_context->config_id) ==
+         VA_STATUS_SUCCESS);
+rollback_display:
+  assert(vaTerminate(encode_context->display) == VA_STATUS_SUCCESS);
+rollback_encode_context:
+  free(encode_context);
+  return NULL;
 }
 
 struct EncodeContextFrame* EncodeContextDequeue(
@@ -70,5 +240,8 @@ bool EncodeContextQueue(struct EncodeContext* encode_context,
 }
 
 void EncodeContextDestroy(struct EncodeContext* encode_context) {
+  assert(vaDestroyConfig(encode_context->display, encode_context->config_id) ==
+         VA_STATUS_SUCCESS);
+  assert(vaTerminate(encode_context->display) == VA_STATUS_SUCCESS);
   free(encode_context);
 }
diff --git a/encode_context.h b/encode_context.h
index 88d4d30..b2724b3 100644
--- a/encode_context.h
+++ b/encode_context.h
@@ -24,6 +24,7 @@
 
 struct EncodeContext;
 struct IoContext;
+struct wl_display;
 
 struct EncodeContextFrame {
   void* user_data;
@@ -35,7 +36,8 @@ struct EncodeContextFrame {
 };
 
 struct EncodeContext* EncodeContextCreate(struct IoContext* io_context,
-                                          uint32_t width, uint32_t height);
+                                          uint32_t width, uint32_t height,
+                                          struct wl_display* display);
 struct EncodeContextFrame* EncodeContextDequeue(
     struct EncodeContext* encode_context);
 bool EncodeContextQueue(struct EncodeContext* encode_context,
diff --git a/makefile b/makefile
index 79b2b81..2b6cdbd 100644
--- a/makefile
+++ b/makefile
@@ -7,6 +7,7 @@ libs:=\
 	glesv2 \
 	libdrm \
 	libpipewire-0.3 \
+	libva-wayland \
 	wayland-client
 
 protocols_dir:=\
diff --git a/video_context.c b/video_context.c
index 5d2a2fd..5b157fe 100644
--- a/video_context.c
+++ b/video_context.c
@@ -170,8 +170,8 @@ static void OnExportDmabufFrameFrame(
 
   struct VideoContext* video_context = data;
   if (!video_context->encode_context) {
-    video_context->encode_context =
-        EncodeContextCreate(video_context->io_context, width, height);
+    video_context->encode_context = EncodeContextCreate(
+        video_context->io_context, width, height, video_context->display);
     if (!video_context->encode_context) {
       LOG("Failed to create encode context");
       // TODO(mburakov): Now what?..