decoder: h264: factor out allocation of reconstructed surfaces.

Add new avc_ensure_surface_bo() helper function to factor out the
allocatiion and initialization processes of the reconstructed VA
surface buffer stores.

Keep preferred native format (NV12) and initialize chroma values
to 0.0 (0x80) when needed for "fake" grayscale (Y800) surfaces
implemented on top of existing NV12.

Signed-off-by: Gwenole Beauchesne <gwenole.beauchesne@intel.com>

6 files changed, 65 insertions, 45 deletions

diff --git a/src/gen6_mfd.c b/src/gen6_mfd.c
index 4a22052..2092f69 100755
--- a/src/gen6_mfd.c
+++ b/src/gen6_mfd.c
@@ -840,18 +840,8 @@ gen6_mfd_avc_decode_init(VADriverContextP ctx,
     obj_surface = decode_state->render_object;
     obj_surface->flags &= ~SURFACE_REF_DIS_MASK;
     obj_surface->flags |= (pic_param->pic_fields.bits.reference_pic_flag ? SURFACE_REFERENCED : 0);
-    i965_check_alloc_surface_bo(ctx, obj_surface, 1, VA_FOURCC_NV12, SUBSAMPLE_YUV420);
-
-    /* initial uv component for YUV400 case */
-    if (pic_param->seq_fields.bits.chroma_format_idc == 0) {
-         unsigned int uv_offset = obj_surface->width * obj_surface->height; 
-         unsigned int uv_size   = obj_surface->width * obj_surface->height / 2; 
-
-         drm_intel_gem_bo_map_gtt(obj_surface->bo);
-         memset(obj_surface->bo->virtual + uv_offset, 0x80, uv_size);
-         drm_intel_gem_bo_unmap_gtt(obj_surface->bo);
-    }
 
+    avc_ensure_surface_bo(ctx, decode_state, obj_surface, pic_param);
     gen6_mfd_init_avc_surface(ctx, pic_param, obj_surface);
 
     dri_bo_unreference(gen6_mfd_context->post_deblocking_output.bo);
diff --git a/src/gen75_mfd.c b/src/gen75_mfd.c
index 2d4e236..5b023cf 100644
--- a/src/gen75_mfd.c
+++ b/src/gen75_mfd.c
@@ -1084,18 +1084,8 @@ gen75_mfd_avc_decode_init(VADriverContextP ctx,
     obj_surface = decode_state->render_object;
     obj_surface->flags &= ~SURFACE_REF_DIS_MASK;
     obj_surface->flags |= (pic_param->pic_fields.bits.reference_pic_flag ? SURFACE_REFERENCED : 0);
-    i965_check_alloc_surface_bo(ctx, obj_surface, 1, VA_FOURCC_NV12, SUBSAMPLE_YUV420);
-
-    /* initial uv component for YUV400 case */
-    if (pic_param->seq_fields.bits.chroma_format_idc == 0) {
-         unsigned int uv_offset = obj_surface->width * obj_surface->height; 
-         unsigned int uv_size   = obj_surface->width * obj_surface->height / 2; 
-
-         drm_intel_gem_bo_map_gtt(obj_surface->bo);
-         memset(obj_surface->bo->virtual + uv_offset, 0x80, uv_size);
-         drm_intel_gem_bo_unmap_gtt(obj_surface->bo);
-    }
 
+    avc_ensure_surface_bo(ctx, decode_state, obj_surface, pic_param);
     gen75_mfd_init_avc_surface(ctx, pic_param, obj_surface);
 
     dri_bo_unreference(gen7_mfd_context->post_deblocking_output.bo);
diff --git a/src/gen7_mfd.c b/src/gen7_mfd.c
index f4ccb12..06eb743 100755
--- a/src/gen7_mfd.c
+++ b/src/gen7_mfd.c
@@ -758,18 +758,8 @@ gen7_mfd_avc_decode_init(VADriverContextP ctx,
     obj_surface = decode_state->render_object;
     obj_surface->flags &= ~SURFACE_REF_DIS_MASK;
     obj_surface->flags |= (pic_param->pic_fields.bits.reference_pic_flag ? SURFACE_REFERENCED : 0);
-    i965_check_alloc_surface_bo(ctx, obj_surface, 1, VA_FOURCC_NV12, SUBSAMPLE_YUV420);
-
-    /* initial uv component for YUV400 case */
-    if (pic_param->seq_fields.bits.chroma_format_idc == 0) {
-         unsigned int uv_offset = obj_surface->width * obj_surface->height;
-         unsigned int uv_size   = obj_surface->width * obj_surface->height / 2;
-
-         drm_intel_gem_bo_map_gtt(obj_surface->bo);
-         memset(obj_surface->bo->virtual + uv_offset, 0x80, uv_size);
-         drm_intel_gem_bo_unmap_gtt(obj_surface->bo);
-    }
 
+    avc_ensure_surface_bo(ctx, decode_state, obj_surface, pic_param);
     gen7_mfd_init_avc_surface(ctx, pic_param, obj_surface);
 
     dri_bo_unreference(gen7_mfd_context->post_deblocking_output.bo);
diff --git a/src/gen8_mfd.c b/src/gen8_mfd.c
index 1742bea..e3e71fb 100644
--- a/src/gen8_mfd.c
+++ b/src/gen8_mfd.c
@@ -845,18 +845,8 @@ gen8_mfd_avc_decode_init(VADriverContextP ctx,
     obj_surface = decode_state->render_object;
     obj_surface->flags &= ~SURFACE_REF_DIS_MASK;
     obj_surface->flags |= (pic_param->pic_fields.bits.reference_pic_flag ? SURFACE_REFERENCED : 0);
-    i965_check_alloc_surface_bo(ctx, obj_surface, 1, VA_FOURCC_NV12, SUBSAMPLE_YUV420);
-
-    /* initial uv component for YUV400 case */
-    if (pic_param->seq_fields.bits.chroma_format_idc == 0) {
-         unsigned int uv_offset = obj_surface->width * obj_surface->height; 
-         unsigned int uv_size   = obj_surface->width * obj_surface->height / 2; 
-
-         drm_intel_gem_bo_map_gtt(obj_surface->bo);
-         memset(obj_surface->bo->virtual + uv_offset, 0x80, uv_size);
-         drm_intel_gem_bo_unmap_gtt(obj_surface->bo);
-    }
 
+    avc_ensure_surface_bo(ctx, decode_state, obj_surface, pic_param);
     gen8_mfd_init_avc_surface(ctx, pic_param, obj_surface);
 
     dri_bo_unreference(gen7_mfd_context->post_deblocking_output.bo);
diff --git a/src/i965_decoder_utils.c b/src/i965_decoder_utils.c
index 7ebc3fa..18704fe 100644
--- a/src/i965_decoder_utils.c
+++ b/src/i965_decoder_utils.c
@@ -174,6 +174,58 @@ mpeg2_set_reference_surfaces(
     }
 }
 
+/* Ensure the supplied VA surface has valid storage for decoding the
+   current picture */
+VAStatus
+avc_ensure_surface_bo(
+    VADriverContextP                    ctx,
+    struct decode_state                *decode_state,
+    struct object_surface              *obj_surface,
+    const VAPictureParameterBufferH264 *pic_param
+)
+{
+    VAStatus va_status;
+    uint32_t hw_fourcc, fourcc, subsample;
+
+    /* Validate chroma format */
+    switch (pic_param->seq_fields.bits.chroma_format_idc) {
+    case 0: // Grayscale
+        fourcc = VA_FOURCC_Y800;
+        subsample = SUBSAMPLE_YUV400;
+        break;
+    case 1: // YUV 4:2:0
+        fourcc = VA_FOURCC_NV12;
+        subsample = SUBSAMPLE_YUV420;
+        break;
+    default:
+        return VA_STATUS_ERROR_UNSUPPORTED_RT_FORMAT;
+    }
+
+    /* XXX: always allocate NV12 (YUV 4:2:0) surfaces for now */
+    hw_fourcc = VA_FOURCC_NV12;
+    subsample = SUBSAMPLE_YUV420;
+
+    /* (Re-)allocate the underlying surface buffer store, if necessary */
+    if (!obj_surface->bo || obj_surface->fourcc != hw_fourcc) {
+        i965_destroy_surface_storage(obj_surface);
+        va_status = i965_check_alloc_surface_bo(ctx, obj_surface, 1,
+            hw_fourcc, subsample);
+        if (va_status != VA_STATUS_SUCCESS)
+            return va_status;
+    }
+
+    /* Fake chroma components if grayscale is implemented on top of NV12 */
+    if (fourcc == VA_FOURCC_Y800 && hw_fourcc == VA_FOURCC_NV12) {
+        const uint32_t uv_offset = obj_surface->width * obj_surface->height;
+        const uint32_t uv_size   = obj_surface->width * obj_surface->height / 2;
+
+        drm_intel_gem_bo_map_gtt(obj_surface->bo);
+        memset(obj_surface->bo->virtual + uv_offset, 0x80, uv_size);
+        drm_intel_gem_bo_unmap_gtt(obj_surface->bo);
+    }
+    return VA_STATUS_SUCCESS;
+}
+
 /* Generate flat scaling matrices for H.264 decoding */
 void
 avc_gen_default_iq_matrix(VAIQMatrixBufferH264 *iq_matrix)
@@ -423,7 +475,7 @@ intel_update_avc_frame_store_index(VADriverContextP ctx,
              * Sometimes a dummy frame comes from the upper layer library, call i965_check_alloc_surface_bo()
              * to ake sure the store buffer is allocated for this reference frame
              */
-            i965_check_alloc_surface_bo(ctx, obj_surface, 1, VA_FOURCC_NV12, SUBSAMPLE_YUV420);
+            avc_ensure_surface_bo(ctx, decode_state, obj_surface, pic_param);
 
             slot_found = 0;
             frame_idx = -1;
diff --git a/src/i965_decoder_utils.h b/src/i965_decoder_utils.h
index b7b72b3..14a45fb 100644
--- a/src/i965_decoder_utils.h
+++ b/src/i965_decoder_utils.h
@@ -43,6 +43,14 @@ mpeg2_set_reference_surfaces(
     VAPictureParameterBufferMPEG2 *pic_param
 );
 
+VAStatus
+avc_ensure_surface_bo(
+    VADriverContextP                    ctx,
+    struct decode_state                *decode_state,
+    struct object_surface              *obj_surface,
+    const VAPictureParameterBufferH264 *pic_param
+);
+
 void
 avc_gen_default_iq_matrix(VAIQMatrixBufferH264 *iq_matrix);