diff options
author | Marco <marpan@google.com> | 2014-11-10 13:07:05 -0800 |
---|---|---|
committer | Marco <marpan@google.com> | 2014-12-03 16:01:28 -0800 |
commit | af898b56bb2cd01ff8af72d276032b8ad9988018 (patch) | |
tree | 2c6e8b4d9b1fd6f14dc775805053fbb9738ae6a2 | |
parent | d5937cd268330da4e80683444bc463d2016c07ee (diff) | |
download | libvpx-af898b56bb2cd01ff8af72d276032b8ad9988018.tar.gz libvpx-af898b56bb2cd01ff8af72d276032b8ad9988018.tar.bz2 libvpx-af898b56bb2cd01ff8af72d276032b8ad9988018.zip |
Various updates to vp8.
Change-Id: Icc7a816491897107764e4c936288e9000e6319b8
-rw-r--r-- | examples/vp8_multi_resolution_encoder.c | 954 | ||||
-rw-r--r-- | examples/vpx_temporal_svc_encoder.c | 6 | ||||
-rw-r--r-- | test/error_resilience_test.cc | 200 | ||||
-rw-r--r-- | vp8/common/blockd.h | 4 | ||||
-rw-r--r-- | vp8/common/onyx.h | 1 | ||||
-rw-r--r-- | vp8/encoder/block.h | 4 | ||||
-rw-r--r-- | vp8/encoder/denoising.c | 19 | ||||
-rw-r--r-- | vp8/encoder/denoising.h | 4 | ||||
-rw-r--r-- | vp8/encoder/encodeframe.c | 8 | ||||
-rw-r--r-- | vp8/encoder/ethreading.c | 9 | ||||
-rw-r--r-- | vp8/encoder/onyx_if.c | 308 | ||||
-rw-r--r-- | vp8/encoder/onyx_int.h | 9 | ||||
-rw-r--r-- | vp8/encoder/pickinter.c | 325 | ||||
-rw-r--r-- | vp8/encoder/quantize.c | 18 | ||||
-rw-r--r-- | vp8/encoder/ratectrl.c | 8 | ||||
-rw-r--r-- | vp8/vp8_cx_iface.c | 98 | ||||
-rw-r--r-- | vpx/vp8cx.h | 17 | ||||
-rw-r--r-- | vpxenc.c | 6 |
18 files changed, 1627 insertions, 371 deletions
diff --git a/examples/vp8_multi_resolution_encoder.c b/examples/vp8_multi_resolution_encoder.c index 9f50dc7cf..e623567b8 100644 --- a/examples/vp8_multi_resolution_encoder.c +++ b/examples/vp8_multi_resolution_encoder.c @@ -8,292 +8,730 @@ * be found in the AUTHORS file in the root of the source tree. */ +/* + * This is an example demonstrating multi-resolution encoding in VP8. + * High-resolution input video is down-sampled to lower-resolutions. The + * encoder then encodes the video and outputs multiple bitstreams with + * different resolutions. + * + * This test also allows for settings temporal layers for each spatial layer. + * Different number of temporal layers per spatial stream may be used. + * Currently up to 3 temporal layers per spatial stream (encoder) are supported + * in this test. + */ -// This is an example demonstrating multi-resolution encoding in VP8. -// High-resolution input video is down-sampled to lower-resolutions. The -// encoder then encodes the video and outputs multiple bitstreams with -// different resolutions. -// -// Configure with --enable-multi-res-encoding flag to enable this example. +#include "./vpx_config.h" #include <stdio.h> #include <stdlib.h> +#include <stdarg.h> #include <string.h> +#include <math.h> +#include <assert.h> +#include <sys/time.h> +#if USE_POSIX_MMAP +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/mman.h> +#include <fcntl.h> +#include <unistd.h> +#endif +#include "vpx_ports/vpx_timer.h" +#define VPX_CODEC_DISABLE_COMPAT 1 +#include "vpx/vpx_encoder.h" +#include "vpx/vp8cx.h" +#include "vpx_ports/mem_ops.h" +#include "./tools_common.h" +#define interface (vpx_codec_vp8_cx()) +#define fourcc 0x30385056 + +void usage_exit() { + exit(EXIT_FAILURE); +} + +/* + * The input video frame is downsampled several times to generate a multi-level + * hierarchical structure. NUM_ENCODERS is defined as the number of encoding + * levels required. For example, if the size of input video is 1280x720, + * NUM_ENCODERS is 3, and down-sampling factor is 2, the encoder outputs 3 + * bitstreams with resolution of 1280x720(level 0), 640x360(level 1), and + * 320x180(level 2) respectively. + */ + +/* Number of encoders (spatial resolutions) used in this test. */ +#define NUM_ENCODERS 3 + +/* Maximum number of temporal layers allowed for this test. */ +#define MAX_NUM_TEMPORAL_LAYERS 3 +/* This example uses the scaler function in libyuv. */ #include "third_party/libyuv/include/libyuv/basic_types.h" #include "third_party/libyuv/include/libyuv/scale.h" #include "third_party/libyuv/include/libyuv/cpu_id.h" -#include "vpx/vpx_encoder.h" -#include "vpx/vp8cx.h" +int (*read_frame_p)(FILE *f, vpx_image_t *img); -#include "./tools_common.h" -#include "./video_writer.h" +static int read_frame(FILE *f, vpx_image_t *img) { + size_t nbytes, to_read; + int res = 1; -// The input video frame is downsampled several times to generate a -// multi-level hierarchical structure. kNumEncoders is defined as the number -// of encoding levels required. For example, if the size of input video is -// 1280x720, kNumEncoders is 3, and down-sampling factor is 2, the encoder -// outputs 3 bitstreams with resolution of 1280x720(level 0), -// 640x360(level 1), and 320x180(level 2) respectively. -#define kNumEncoders 3 + to_read = img->w*img->h*3/2; + nbytes = fread(img->planes[0], 1, to_read, f); + if(nbytes != to_read) { + res = 0; + if(nbytes > 0) + printf("Warning: Read partial frame. Check your width & height!\n"); + } + return res; +} -static const char *exec_name; +static int read_frame_by_row(FILE *f, vpx_image_t *img) { + size_t nbytes, to_read; + int res = 1; + int plane; -void usage_exit() { - fprintf(stderr, - "Usage: %s <width> <height> <infile> <outfile(s)> <output psnr?>\n", - exec_name); - exit(EXIT_FAILURE); + for (plane = 0; plane < 3; plane++) + { + unsigned char *ptr; + int w = (plane ? (1 + img->d_w) / 2 : img->d_w); + int h = (plane ? (1 + img->d_h) / 2 : img->d_h); + int r; + + /* Determine the correct plane based on the image format. The for-loop + * always counts in Y,U,V order, but this may not match the order of + * the data on disk. + */ + switch (plane) + { + case 1: + ptr = img->planes[img->fmt==VPX_IMG_FMT_YV12? VPX_PLANE_V : VPX_PLANE_U]; + break; + case 2: + ptr = img->planes[img->fmt==VPX_IMG_FMT_YV12?VPX_PLANE_U : VPX_PLANE_V]; + break; + default: + ptr = img->planes[plane]; + } + + for (r = 0; r < h; r++) + { + to_read = w; + + nbytes = fread(ptr, 1, to_read, f); + if(nbytes != to_read) { + res = 0; + if(nbytes > 0) + printf("Warning: Read partial frame. Check your width & height!\n"); + break; + } + + ptr += img->stride[plane]; + } + if (!res) + break; + } + + return res; } -int main(int argc, char *argv[]) { - int frame_cnt = 0; - FILE *infile = NULL; - VpxVideoWriter *writers[kNumEncoders]; - vpx_codec_ctx_t codec[kNumEncoders]; - vpx_codec_enc_cfg_t cfg[kNumEncoders]; - vpx_image_t raw[kNumEncoders]; - const VpxInterface *const encoder = get_vpx_encoder_by_name("vp8"); - // Currently, only realtime mode is supported in multi-resolution encoding. - const int arg_deadline = VPX_DL_REALTIME; - int i; - int width = 0; - int height = 0; - int frame_avail = 0; - int got_data = 0; - - // Set show_psnr to 1/0 to show/not show PSNR. Choose show_psnr=0 if you - // don't need to know PSNR, which will skip PSNR calculation and save - // encoding time. - int show_psnr = 0; - uint64_t psnr_sse_total[kNumEncoders] = {0}; - uint64_t psnr_samples_total[kNumEncoders] = {0}; - double psnr_totals[kNumEncoders][4] = {{0, 0}}; - int psnr_count[kNumEncoders] = {0}; - - // Set the required target bitrates for each resolution level. - // If target bitrate for highest-resolution level is set to 0, - // (i.e. target_bitrate[0]=0), we skip encoding at that level. - unsigned int target_bitrate[kNumEncoders] = {1000, 500, 100}; - - // Enter the frame rate of the input video. - const int framerate = 30; - // Set down-sampling factor for each resolution level. - // dsf[0] controls down sampling from level 0 to level 1; - // dsf[1] controls down sampling from level 1 to level 2; - // dsf[2] is not used. - vpx_rational_t dsf[kNumEncoders] = {{2, 1}, {2, 1}, {1, 1}}; - - exec_name = argv[0]; - - if (!encoder) - die("Unsupported codec."); - - // exe_name, input width, input height, input file, - // output file 1, output file 2, output file 3, psnr on/off - if (argc != (5 + kNumEncoders)) - die("Invalid number of input options."); - - printf("Using %s\n", vpx_codec_iface_name(encoder->codec_interface())); - - width = strtol(argv[1], NULL, 0); - height = strtol(argv[2], NULL, 0); - - if (width < 16 || width % 2 || height < 16 || height % 2) - die("Invalid resolution: %ldx%ld", width, height); - - // Open input video file for encoding - if (!(infile = fopen(argv[3], "rb"))) - die("Failed to open %s for reading", argv[3]); - - show_psnr = strtol(argv[kNumEncoders + 4], NULL, 0); - - // Populate default encoder configuration - for (i = 0; i < kNumEncoders; ++i) { - vpx_codec_err_t res = - vpx_codec_enc_config_default(encoder->codec_interface(), &cfg[i], 0); - if (res != VPX_CODEC_OK) { - printf("Failed to get config: %s\n", vpx_codec_err_to_string(res)); - return EXIT_FAILURE; +static void write_ivf_file_header(FILE *outfile, + const vpx_codec_enc_cfg_t *cfg, + int frame_cnt) { + char header[32]; + + if(cfg->g_pass != VPX_RC_ONE_PASS && cfg->g_pass != VPX_RC_LAST_PASS) + return; + header[0] = 'D'; + header[1] = 'K'; + header[2] = 'I'; + header[3] = 'F'; + mem_put_le16(header+4, 0); /* version */ + mem_put_le16(header+6, 32); /* headersize */ + mem_put_le32(header+8, fourcc); /* headersize */ + mem_put_le16(header+12, cfg->g_w); /* width */ + mem_put_le16(header+14, cfg->g_h); /* height */ + mem_put_le32(header+16, cfg->g_timebase.den); /* rate */ + mem_put_le32(header+20, cfg->g_timebase.num); /* scale */ + mem_put_le32(header+24, frame_cnt); /* length */ + mem_put_le32(header+28, 0); /* unused */ + + (void) fwrite(header, 1, 32, outfile); +} + +static void write_ivf_frame_header(FILE *outfile, + const vpx_codec_cx_pkt_t *pkt) +{ + char header[12]; + vpx_codec_pts_t pts; + + if(pkt->kind != VPX_CODEC_CX_FRAME_PKT) + return; + + pts = pkt->data.frame.pts; + mem_put_le32(header, pkt->data.frame.sz); + mem_put_le32(header+4, pts&0xFFFFFFFF); + mem_put_le32(header+8, pts >> 32); + + (void) fwrite(header, 1, 12, outfile); +} + +/* Temporal scaling parameters */ +/* This sets all the temporal layer parameters given |num_temporal_layers|, + * including the target bit allocation across temporal layers. Bit allocation + * parameters will be passed in as user parameters in another version. + */ +static void set_temporal_layer_pattern(int num_temporal_layers, + vpx_codec_enc_cfg_t *cfg, + int bitrate, + int *layer_flags) +{ + assert(num_temporal_layers <= MAX_NUM_TEMPORAL_LAYERS); + switch (num_temporal_layers) + { + case 1: + { + /* 1-layer */ + cfg->ts_number_layers = 1; + cfg->ts_periodicity = 1; + cfg->ts_rate_decimator[0] = 1; + cfg->ts_layer_id[0] = 0; + cfg->ts_target_bitrate[0] = bitrate; + + // Update L only. + layer_flags[0] = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF; + break; + } + + case 2: + { + /* 2-layers, with sync point at first frame of layer 1. */ + cfg->ts_number_layers = 2; + cfg->ts_periodicity = 2; + cfg->ts_rate_decimator[0] = 2; + cfg->ts_rate_decimator[1] = 1; + cfg->ts_layer_id[0] = 0; + cfg->ts_layer_id[1] = 1; + // Use 60/40 bit allocation as example. + cfg->ts_target_bitrate[0] = 0.6f * bitrate; + cfg->ts_target_bitrate[1] = bitrate; + + /* 0=L, 1=GF */ + // ARF is used as predictor for all frames, and is only updated on + // key frame. Sync point every 8 frames. + + // Layer 0: predict from L and ARF, update L and G. + layer_flags[0] = VP8_EFLAG_NO_REF_GF | + VP8_EFLAG_NO_UPD_ARF; + + // Layer 1: sync point: predict from L and ARF, and update G. + layer_flags[1] = VP8_EFLAG_NO_REF_GF | + VP8_EFLAG_NO_UPD_LAST | + VP8_EFLAG_NO_UPD_ARF; + + // Layer 0, predict from L and ARF, update L. + layer_flags[2] = VP8_EFLAG_NO_REF_GF | + VP8_EFLAG_NO_UPD_GF | + VP8_EFLAG_NO_UPD_ARF; + + // Layer 1: predict from L, G and ARF, and update G. + layer_flags[3] = VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_UPD_LAST | + VP8_EFLAG_NO_UPD_ENTROPY; + + // Layer 0 + layer_flags[4] = layer_flags[2]; + + // Layer 1 + layer_flags[5] = layer_flags[3]; + + // Layer 0 + layer_flags[6] = layer_flags[4]; + + // Layer 1 + layer_flags[7] = layer_flags[5]; + break; } - } - - // Update the default configuration according to needs of the application. - // Highest-resolution encoder settings - cfg[0].g_w = width; - cfg[0].g_h = height; - cfg[0].g_threads = 1; - cfg[0].rc_dropframe_thresh = 30; - cfg[0].rc_end_usage = VPX_CBR; - cfg[0].rc_resize_allowed = 0; - cfg[0].rc_min_quantizer = 4; - cfg[0].rc_max_quantizer = 56; - cfg[0].rc_undershoot_pct = 98; - cfg[0].rc_overshoot_pct = 100; - cfg[0].rc_buf_initial_sz = 500; - cfg[0].rc_buf_optimal_sz = 600; - cfg[0].rc_buf_sz = 1000; - cfg[0].g_error_resilient = 1; - cfg[0].g_lag_in_frames = 0; - cfg[0].kf_mode = VPX_KF_AUTO; // VPX_KF_DISABLED - cfg[0].kf_min_dist = 3000; - cfg[0].kf_max_dist = 3000; - cfg[0].rc_target_bitrate = target_bitrate[0]; - cfg[0].g_timebase.num = 1; - cfg[0].g_timebase.den = framerate; - - // Other-resolution encoder settings - for (i = 1; i < kNumEncoders; ++i) { - cfg[i] = cfg[0]; - cfg[i].g_threads = 1; - cfg[i].rc_target_bitrate = target_bitrate[i]; - - // Note: Width & height of other-resolution encoders are calculated - // from the highest-resolution encoder's size and the corresponding - // down_sampling_factor. + + case 3: + default: { - unsigned int iw = cfg[i - 1].g_w * dsf[i - 1].den + dsf[i - 1].num - 1; - unsigned int ih = cfg[i - 1].g_h * dsf[i - 1].den + dsf[i - 1].num - 1; - cfg[i].g_w = iw / dsf[i - 1].num; - cfg[i].g_h = ih / dsf[i - 1].num; + // 3-layers structure where ARF is used as predictor for all frames, + // and is only updated on key frame. + // Sync points for layer 1 and 2 every 8 frames. + cfg->ts_number_layers = 3; + cfg->ts_periodicity = 4; + cfg->ts_rate_decimator[0] = 4; + cfg->ts_rate_decimator[1] = 2; + cfg->ts_rate_decimator[2] = 1; + cfg->ts_layer_id[0] = 0; + cfg->ts_layer_id[1] = 2; + cfg->ts_layer_id[2] = 1; + cfg->ts_layer_id[3] = 2; + // Use 40/20/40 bit allocation as example. + cfg->ts_target_bitrate[0] = 0.4f * bitrate; + cfg->ts_target_bitrate[1] = 0.6f * bitrate; + cfg->ts_target_bitrate[2] = bitrate; + + /* 0=L, 1=GF, 2=ARF */ + + // Layer 0: predict from L and ARF; update L and G. + layer_flags[0] = VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_REF_GF; + + // Layer 2: sync point: predict from L and ARF; update none. + layer_flags[1] = VP8_EFLAG_NO_REF_GF | + VP8_EFLAG_NO_UPD_GF | + VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_UPD_LAST | + VP8_EFLAG_NO_UPD_ENTROPY; + + // Layer 1: sync point: predict from L and ARF; update G. + layer_flags[2] = VP8_EFLAG_NO_REF_GF | + VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_UPD_LAST; + + // Layer 2: predict from L, G, ARF; update none. + layer_flags[3] = VP8_EFLAG_NO_UPD_GF | + VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_UPD_LAST | + VP8_EFLAG_NO_UPD_ENTROPY; + + // Layer 0: predict from L and ARF; update L. + layer_flags[4] = VP8_EFLAG_NO_UPD_GF | + VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_REF_GF; + + // Layer 2: predict from L, G, ARF; update none. + layer_flags[5] = layer_flags[3]; + + // Layer 1: predict from L, G, ARF; update G. + layer_flags[6] = VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_UPD_LAST; + + // Layer 2: predict from L, G, ARF; update none. + layer_flags[7] = layer_flags[3]; + break; + } } +} + +/* The periodicity of the pattern given the number of temporal layers. */ +static int periodicity_to_num_layers[MAX_NUM_TEMPORAL_LAYERS] = {1, 8, 8}; + +int main(int argc, char **argv) +{ + FILE *infile, *outfile[NUM_ENCODERS]; + FILE *downsampled_input[NUM_ENCODERS - 1]; + char filename[50]; + vpx_codec_ctx_t codec[NUM_ENCODERS]; + vpx_codec_enc_cfg_t cfg[NUM_ENCODERS]; + int frame_cnt = 0; + vpx_image_t raw[NUM_ENCODERS]; + vpx_codec_err_t res[NUM_ENCODERS]; + + int i; + long width; + long height; + int length_frame; + int frame_avail; + int got_data; + int flags = 0; + int layer_id = 0; + + int layer_flags[VPX_TS_MAX_PERIODICITY * NUM_ENCODERS] + = {0}; + int flag_periodicity; + + /*Currently, only realtime mode is supported in multi-resolution encoding.*/ + int arg_deadline = VPX_DL_REALTIME; + + /* Set show_psnr to 1/0 to show/not show PSNR. Choose show_psnr=0 if you + don't need to know PSNR, which will skip PSNR calculation and save + encoding time. */ + int show_psnr = 0; + int key_frame_insert = 0; + uint64_t psnr_sse_total[NUM_ENCODERS] = {0}; + uint64_t psnr_samples_total[NUM_ENCODERS] = {0}; + double psnr_totals[NUM_ENCODERS][4] = {{0,0}}; + int psnr_count[NUM_ENCODERS] = {0}; + + double cx_time = 0; + struct timeval tv1, tv2, difftv; + + /* Set the required target bitrates for each resolution level. + * If target bitrate for highest-resolution level is set to 0, + * (i.e. target_bitrate[0]=0), we skip encoding at that level. + */ + unsigned int target_bitrate[NUM_ENCODERS]={1000, 500, 100}; + + /* Enter the frame rate of the input video */ + int framerate = 30; + + /* Set down-sampling factor for each resolution level. + dsf[0] controls down sampling from level 0 to level 1; + dsf[1] controls down sampling from level 1 to level 2; + dsf[2] is not used. */ + vpx_rational_t dsf[NUM_ENCODERS] = {{2, 1}, {2, 1}, {1, 1}}; + + /* Set the number of temporal layers for each encoder/resolution level, + * starting from highest resoln down to lowest resoln. */ + unsigned int num_temporal_layers[NUM_ENCODERS] = {3, 3, 3}; + + if(argc!= (7 + 3 * NUM_ENCODERS)) + die("Usage: %s <width> <height> <frame_rate> <infile> <outfile(s)> " + "<rate_encoder(s)> <temporal_layer(s)> <key_frame_insert> <output psnr?> \n", + argv[0]); + + printf("Using %s\n",vpx_codec_iface_name(interface)); + + width = strtol(argv[1], NULL, 0); + height = strtol(argv[2], NULL, 0); + framerate = strtol(argv[3], NULL, 0); + + if(width < 16 || width%2 || height <16 || height%2) + die("Invalid resolution: %ldx%ld", width, height); + + /* Open input video file for encoding */ + if(!(infile = fopen(argv[4], "rb"))) + die("Failed to open %s for reading", argv[4]); + + /* Open output file for each encoder to output bitstreams */ + for (i=0; i< NUM_ENCODERS; i++) + { + if(!target_bitrate[i]) + { + outfile[i] = NULL; + continue; + } - // Make width & height to be multiplier of 2. - if ((cfg[i].g_w) % 2) - cfg[i].g_w++; - - if ((cfg[i].g_h) % 2) - cfg[i].g_h++; - } - - // Open output file for each encoder to output bitstreams - for (i = 0; i < kNumEncoders; ++i) { - VpxVideoInfo info = { - encoder->fourcc, - cfg[i].g_w, - cfg[i].g_h, - {cfg[i].g_timebase.num, cfg[i].g_timebase.den} - }; - - if (!(writers[i] = vpx_video_writer_open(argv[i+4], kContainerIVF, &info))) - die("Failed to open %s for writing", argv[i+4]); - } - - // Allocate image for each encoder - for (i = 0; i < kNumEncoders; ++i) - if (!vpx_img_alloc(&raw[i], VPX_IMG_FMT_I420, cfg[i].g_w, cfg[i].g_h, 32)) - die("Failed to allocate image", cfg[i].g_w, cfg[i].g_h); - - // Initialize multi-encoder - if (vpx_codec_enc_init_multi(&codec[0], encoder->codec_interface(), &cfg[0], - kNumEncoders, - show_psnr ? VPX_CODEC_USE_PSNR : 0, &dsf[0])) - die_codec(&codec[0], "Failed to initialize encoder"); - - // The extra encoding configuration parameters can be set as follows. - for (i = 0; i < kNumEncoders; i++) { - // Set encoding speed - if (vpx_codec_control(&codec[i], VP8E_SET_CPUUSED, -6)) - die_codec(&codec[i], "Failed to set cpu_used"); - - // Set static threshold. - if (vpx_codec_control(&codec[i], VP8E_SET_STATIC_THRESHOLD, 1)) - die_codec(&codec[i], "Failed to set static threshold"); - - // Set NOISE_SENSITIVITY to do TEMPORAL_DENOISING - // Enable denoising for the highest-resolution encoder. - if (vpx_codec_control(&codec[0], VP8E_SET_NOISE_SENSITIVITY, i == 0)) - die_codec(&codec[0], "Failed to set noise_sensitivity"); - } - - frame_avail = 1; - got_data = 0; - - while (frame_avail || got_data) { - vpx_codec_iter_t iter[kNumEncoders] = {NULL}; - const vpx_codec_cx_pkt_t *pkt[kNumEncoders]; - - frame_avail = vpx_img_read(&raw[0], infile); - - if (frame_avail) { - for (i = 1; i < kNumEncoders; ++i) { - vpx_image_t *const prev = &raw[i - 1]; - - // Scale the image down a number of times by downsampling factor - // FilterMode 1 or 2 give better psnr than FilterMode 0. - I420Scale(prev->planes[VPX_PLANE_Y], prev->stride[VPX_PLANE_Y], - prev->planes[VPX_PLANE_U], prev->stride[VPX_PLANE_U], - prev->planes[VPX_PLANE_V], prev->stride[VPX_PLANE_V], - prev->d_w, prev->d_h, - raw[i].planes[VPX_PLANE_Y], raw[i].stride[VPX_PLANE_Y], - raw[i].planes[VPX_PLANE_U], raw[i].stride[VPX_PLANE_U], - raw[i].planes[VPX_PLANE_V], raw[i].stride[VPX_PLANE_V], - raw[i].d_w, raw[i].d_h, 1); - } + if(!(outfile[i] = fopen(argv[i+5], "wb"))) + die("Failed to open %s for writing", argv[i+4]); } - // Encode frame. - if (vpx_codec_encode(&codec[0], frame_avail? &raw[0] : NULL, - frame_cnt, 1, 0, arg_deadline)) { - die_codec(&codec[0], "Failed to encode frame"); + // Bitrates per spatial layer: overwrite default rates above. + for (i=0; i< NUM_ENCODERS; i++) + { + target_bitrate[i] = strtol(argv[NUM_ENCODERS + 5 + i], NULL, 0); } - for (i = kNumEncoders - 1; i >= 0; i--) { - got_data = 0; - - while ((pkt[i] = vpx_codec_get_cx_data(&codec[i], &iter[i]))) { - got_data = 1; - switch (pkt[i]->kind) { - case VPX_CODEC_CX_FRAME_PKT: - vpx_video_writer_write_frame(writers[i], pkt[i]->data.frame.buf, - pkt[i]->data.frame.sz, frame_cnt - 1); - break; - case VPX_CODEC_PSNR_PKT: - if (show_psnr) { - int j; - psnr_sse_total[i] += pkt[i]->data.psnr.sse[0]; - psnr_samples_total[i] += pkt[i]->data.psnr.samples[0]; - for (j = 0; j < 4; j++) - psnr_totals[i][j] += pkt[i]->data.psnr.psnr[j]; - psnr_count[i]++; - } - break; - default: - break; + // Temporal layers per spatial layers: overwrite default settings above. + for (i=0; i< NUM_ENCODERS; i++) + { + num_temporal_layers[i] = strtol(argv[2 * NUM_ENCODERS + 5 + i], NULL, 0); + if (num_temporal_layers[i] < 1 || num_temporal_layers[i] > 3) + die("Invalid temporal layers: %d, Must be 1, 2, or 3. \n", + num_temporal_layers); + } + + /* Open file to write out each spatially downsampled input stream. */ + for (i=0; i< NUM_ENCODERS - 1; i++) + { + // Highest resoln is encoder 0. + if (sprintf(filename,"ds%d.yuv",NUM_ENCODERS - i) < 0) + { + return EXIT_FAILURE; + } + downsampled_input[i] = fopen(filename,"wb"); + } + + key_frame_insert = strtol(argv[3 * NUM_ENCODERS + 5], NULL, 0); + + show_psnr = strtol(argv[3 * NUM_ENCODERS + 6], NULL, 0); + + + /* Populate default encoder configuration */ + for (i=0; i< NUM_ENCODERS; i++) + { + res[i] = vpx_codec_enc_config_default(interface, &cfg[i], 0); + if(res[i]) { + printf("Failed to get config: %s\n", vpx_codec_err_to_string(res[i])); + return EXIT_FAILURE; } - printf(pkt[i]->kind == VPX_CODEC_CX_FRAME_PKT && - (pkt[i]->data.frame.flags & VPX_FRAME_IS_KEY)? "K":"."); - fflush(stdout); - } } - frame_cnt++; - } - printf("\n"); - - fclose(infile); - - printf("Processed %d frames.\n", frame_cnt - 1); - for (i = 0; i < kNumEncoders; ++i) { - // Calculate PSNR and print it out - if (show_psnr && psnr_count[i] > 0) { - int j; - double ovpsnr = sse_to_psnr(psnr_samples_total[i], 255.0, - psnr_sse_total[i]); - - fprintf(stderr, "\n ENC%d PSNR (Overall/Avg/Y/U/V)", i); - fprintf(stderr, " %.3lf", ovpsnr); - for (j = 0; j < 4; j++) - fprintf(stderr, " %.3lf", psnr_totals[i][j]/psnr_count[i]); + + /* + * Update the default configuration according to needs of the application. + */ + /* Highest-resolution encoder settings */ + cfg[0].g_w = width; + cfg[0].g_h = height; + cfg[0].rc_dropframe_thresh = 0; + cfg[0].rc_end_usage = VPX_CBR; + cfg[0].rc_resize_allowed = 0; + cfg[0].rc_min_quantizer = 2; + cfg[0].rc_max_quantizer = 56; + cfg[0].rc_undershoot_pct = 100; + cfg[0].rc_overshoot_pct = 15; + cfg[0].rc_buf_initial_sz = 500; + cfg[0].rc_buf_optimal_sz = 600; + cfg[0].rc_buf_sz = 1000; + cfg[0].g_error_resilient = 1; /* Enable error resilient mode */ + cfg[0].g_lag_in_frames = 0; + + /* Disable automatic keyframe placement */ + /* Note: These 3 settings are copied to all levels. But, except the lowest + * resolution level, all other levels are set to VPX_KF_DISABLED internally. + */ + cfg[0].kf_mode = VPX_KF_AUTO; + cfg[0].kf_min_dist = 3000; + cfg[0].kf_max_dist = 3000; + + cfg[0].rc_target_bitrate = target_bitrate[0]; /* Set target bitrate */ + cfg[0].g_timebase.num = 1; /* Set fps */ + cfg[0].g_timebase.den = framerate; + + /* Other-resolution encoder settings */ + for (i=1; i< NUM_ENCODERS; i++) + { + memcpy(&cfg[i], &cfg[0], sizeof(vpx_codec_enc_cfg_t)); + + cfg[i].rc_target_bitrate = target_bitrate[i]; + + /* Note: Width & height of other-resolution encoders are calculated + * from the highest-resolution encoder's size and the corresponding + * down_sampling_factor. + */ + { + unsigned int iw = cfg[i-1].g_w*dsf[i-1].den + dsf[i-1].num - 1; + unsigned int ih = cfg[i-1].g_h*dsf[i-1].den + dsf[i-1].num - 1; + cfg[i].g_w = iw/dsf[i-1].num; + cfg[i].g_h = ih/dsf[i-1].num; + } + + /* Make width & height to be multiplier of 2. */ + // Should support odd size ??? + if((cfg[i].g_w)%2)cfg[i].g_w++; + if((cfg[i].g_h)%2)cfg[i].g_h++; + } + + + // Set the number of threads per encode/spatial layer. + // (1, 1, 1) means no encoder threading. + cfg[0].g_threads = 2; + cfg[1].g_threads = 1; + cfg[2].g_threads = 1; + + /* Allocate image for each encoder */ + for (i=0; i< NUM_ENCODERS; i++) + if(!vpx_img_alloc(&raw[i], VPX_IMG_FMT_I420, cfg[i].g_w, cfg[i].g_h, 32)) + die("Failed to allocate image", cfg[i].g_w, cfg[i].g_h); + + if (raw[0].stride[VPX_PLANE_Y] == raw[0].d_w) + read_frame_p = read_frame; + else + read_frame_p = read_frame_by_row; + + for (i=0; i< NUM_ENCODERS; i++) + if(outfile[i]) + write_ivf_file_header(outfile[i], &cfg[i], 0); + + /* Temporal layers settings */ + for ( i=0; i<NUM_ENCODERS; i++) + { + set_temporal_layer_pattern(num_temporal_layers[i], + &cfg[i], + cfg[i].rc_target_bitrate, + &layer_flags[i * VPX_TS_MAX_PERIODICITY]); + } + + /* Initialize multi-encoder */ + if(vpx_codec_enc_init_multi(&codec[0], interface, &cfg[0], NUM_ENCODERS, + (show_psnr ? VPX_CODEC_USE_PSNR : 0), &dsf[0])) + die_codec(&codec[0], "Failed to initialize encoder"); + + /* The extra encoding configuration parameters can be set as follows. */ + /* Set encoding speed */ + for ( i=0; i<NUM_ENCODERS; i++) + { + int speed = -6; + /* Lower speed for the lowest resolution. */ + if (i == NUM_ENCODERS - 1) speed = -4; + if(vpx_codec_control(&codec[i], VP8E_SET_CPUUSED, speed)) + die_codec(&codec[i], "Failed to set cpu_used"); } - if (vpx_codec_destroy(&codec[i])) - die_codec(&codec[i], "Failed to destroy codec"); + /* Set static threshold = 1 for all encoders */ + for ( i=0; i<NUM_ENCODERS; i++) + { + if(vpx_codec_control(&codec[i], VP8E_SET_STATIC_THRESHOLD, 1)) + die_codec(&codec[i], "Failed to set static threshold"); + } - vpx_img_free(&raw[i]); - vpx_video_writer_close(writers[i]); - } - printf("\n"); + /* Set NOISE_SENSITIVITY to do TEMPORAL_DENOISING */ + /* Enable denoising for the highest-resolution encoder. */ + if(vpx_codec_control(&codec[0], VP8E_SET_NOISE_SENSITIVITY, 1)) + die_codec(&codec[0], "Failed to set noise_sensitivity"); + for ( i=1; i< NUM_ENCODERS; i++) + { + if(vpx_codec_control(&codec[i], VP8E_SET_NOISE_SENSITIVITY, 0)) + die_codec(&codec[i], "Failed to set noise_sensitivity"); + } + + /* Set the number of token partitions */ + for ( i=0; i<NUM_ENCODERS; i++) + { + if(vpx_codec_control(&codec[i], VP8E_SET_TOKEN_PARTITIONS, 1)) + die_codec(&codec[i], "Failed to set static threshold"); + } + + /* Set the max intra target bitrate */ + for ( i=0; i<NUM_ENCODERS; i++) + { + unsigned int max_intra_size_pct = + (int)(((double)cfg[0].rc_buf_optimal_sz * 0.5) * framerate / 10); + if(vpx_codec_control(&codec[i], VP8E_SET_MAX_INTRA_BITRATE_PCT, + max_intra_size_pct)) + die_codec(&codec[i], "Failed to set static threshold"); + //printf("%d %d \n",i,max_intra_size_pct); + } + + frame_avail = 1; + got_data = 0; + + while(frame_avail || got_data) + { + vpx_codec_iter_t iter[NUM_ENCODERS]={NULL}; + const vpx_codec_cx_pkt_t *pkt[NUM_ENCODERS]; + + flags = 0; + frame_avail = read_frame_p(infile, &raw[0]); + + if(frame_avail) + { + for ( i=1; i<NUM_ENCODERS; i++) + { + /*Scale the image down a number of times by downsampling factor*/ + /* FilterMode 1 or 2 give better psnr than FilterMode 0. */ + I420Scale(raw[i-1].planes[VPX_PLANE_Y], raw[i-1].stride[VPX_PLANE_Y], + raw[i-1].planes[VPX_PLANE_U], raw[i-1].stride[VPX_PLANE_U], + raw[i-1].planes[VPX_PLANE_V], raw[i-1].stride[VPX_PLANE_V], + raw[i-1].d_w, raw[i-1].d_h, + raw[i].planes[VPX_PLANE_Y], raw[i].stride[VPX_PLANE_Y], + raw[i].planes[VPX_PLANE_U], raw[i].stride[VPX_PLANE_U], + raw[i].planes[VPX_PLANE_V], raw[i].stride[VPX_PLANE_V], + raw[i].d_w, raw[i].d_h, 1); + /* Write out down-sampled input. */ + length_frame = cfg[i].g_w * cfg[i].g_h *3/2; + if (fwrite(raw[i].planes[0], 1, length_frame, + downsampled_input[NUM_ENCODERS - i - 1]) != + length_frame) + { + return EXIT_FAILURE; + } + } + } + + /* Set the flags (reference and update) for all the encoders.*/ + for ( i=0; i<NUM_ENCODERS; i++) + { + layer_id = cfg[i].ts_layer_id[frame_cnt % cfg[i].ts_periodicity]; + flags = 0; + flag_periodicity = periodicity_to_num_layers + [num_temporal_layers[i] - 1]; + flags = layer_flags[i * VPX_TS_MAX_PERIODICITY + + frame_cnt % flag_periodicity]; + // Key frame flag for first frame. + if (frame_cnt == 0) + { + flags |= VPX_EFLAG_FORCE_KF; + } + if (frame_cnt > 0 && frame_cnt == key_frame_insert) + { + flags = VPX_EFLAG_FORCE_KF; + } + + vpx_codec_control(&codec[i], VP8E_SET_FRAME_FLAGS, flags); + vpx_codec_control(&codec[i], VP8E_SET_TEMPORAL_LAYER_ID, layer_id); + } + + gettimeofday(&tv1, NULL); + /* Encode each frame at multi-levels */ + /* Note the flags must be set to 0 in the encode call if they are set + for each frame with the vpx_codec_control(), as done above. */ + if(vpx_codec_encode(&codec[0], frame_avail? &raw[0] : NULL, + frame_cnt, 1, 0, arg_deadline)) + { + die_codec(&codec[0], "Failed to encode frame"); + } + gettimeofday(&tv2, NULL); + timersub(&tv2, &tv1, &difftv); + cx_time += (double)(difftv.tv_sec * 1000000 + difftv.tv_usec); + for (i=NUM_ENCODERS-1; i>=0 ; i--) + { + got_data = 0; + while( (pkt[i] = vpx_codec_get_cx_data(&codec[i], &iter[i])) ) + { + got_data = 1; + switch(pkt[i]->kind) { + case VPX_CODEC_CX_FRAME_PKT: + write_ivf_frame_header(outfile[i], pkt[i]); + (void) fwrite(pkt[i]->data.frame.buf, 1, + pkt[i]->data.frame.sz, outfile[i]); + break; + case VPX_CODEC_PSNR_PKT: + if (show_psnr) + { + int j; + + psnr_sse_total[i] += pkt[i]->data.psnr.sse[0]; + psnr_samples_total[i] += pkt[i]->data.psnr.samples[0]; + for (j = 0; j < 4; j++) + { + psnr_totals[i][j] += pkt[i]->data.psnr.psnr[j]; + } + psnr_count[i]++; + } + + break; + default: + break; + } + printf(pkt[i]->kind == VPX_CODEC_CX_FRAME_PKT + && (pkt[i]->data.frame.flags & VPX_FRAME_IS_KEY)? "K":""); + fflush(stdout); + } + } + frame_cnt++; + } + printf("\n"); + printf("FPS for encoding %d %f %f \n", frame_cnt, (float)cx_time / 1000000, + 1000000 * (double)frame_cnt / (double)cx_time); + + fclose(infile); + + printf("Processed %ld frames.\n",(long int)frame_cnt-1); + for (i=0; i< NUM_ENCODERS; i++) + { + /* Calculate PSNR and print it out */ + if ( (show_psnr) && (psnr_count[i]>0) ) + { + int j; + double ovpsnr = sse_to_psnr(psnr_samples_total[i], 255.0, + psnr_sse_total[i]); + + fprintf(stderr, "\n ENC%d PSNR (Overall/Avg/Y/U/V)", i); + + fprintf(stderr, " %.3lf", ovpsnr); + for (j = 0; j < 4; j++) + { + fprintf(stderr, " %.3lf", psnr_totals[i][j]/psnr_count[i]); + } + } + + if(vpx_codec_destroy(&codec[i])) + die_codec(&codec[i], "Failed to destroy codec"); + + vpx_img_free(&raw[i]); + + if(!outfile[i]) + continue; + + /* Try to rewrite the file header with the actual frame count */ + if(!fseek(outfile[i], 0, SEEK_SET)) + write_ivf_file_header(outfile[i], &cfg[i], frame_cnt-1); + fclose(outfile[i]); + } + printf("\n"); - return EXIT_SUCCESS; + return EXIT_SUCCESS; } diff --git a/examples/vpx_temporal_svc_encoder.c b/examples/vpx_temporal_svc_encoder.c index 8cc7f4a95..cbe0157ff 100644 --- a/examples/vpx_temporal_svc_encoder.c +++ b/examples/vpx_temporal_svc_encoder.c @@ -675,6 +675,9 @@ int main(int argc, char **argv) { die_codec(&codec, "Failed to set SVC"); } } + if (strncmp(encoder->name, "vp8", 3) == 0) { + vpx_codec_control(&codec, VP8E_SET_SCREEN_CONTENT_MODE, 0); + } vpx_codec_control(&codec, VP8E_SET_STATIC_THRESHOLD, 1); vpx_codec_control(&codec, VP8E_SET_TOKEN_PARTITIONS, 1); // This controls the maximum target size of the key frame. @@ -697,6 +700,9 @@ int main(int argc, char **argv) { cfg.ts_layer_id[frame_cnt % cfg.ts_periodicity]; if (strncmp(encoder->name, "vp9", 3) == 0) { vpx_codec_control(&codec, VP9E_SET_SVC_LAYER_ID, &layer_id); + } else if (strncmp(encoder->name, "vp8", 3) == 0) { + vpx_codec_control(&codec, VP8E_SET_TEMPORAL_LAYER_ID, + layer_id.temporal_layer_id); } flags = layer_flags[frame_cnt % flag_periodicity]; frame_avail = vpx_img_read(&raw, infile); diff --git a/test/error_resilience_test.cc b/test/error_resilience_test.cc index 28cda2f32..182547bdf 100644 --- a/test/error_resilience_test.cc +++ b/test/error_resilience_test.cc @@ -316,7 +316,205 @@ TEST_P(ErrorResilienceTestLarge, 2LayersDropEnhancement) { Reset(); } +class ErrorResilienceTestLargeCodecControls : public ::libvpx_test::EncoderTest, + public ::libvpx_test::CodecTestWithParam<libvpx_test::TestMode> { + protected: + ErrorResilienceTestLargeCodecControls() + : EncoderTest(GET_PARAM(0)), + encoding_mode_(GET_PARAM(1)) { + Reset(); + } + + virtual ~ErrorResilienceTestLargeCodecControls() {} + + void Reset() { + last_pts_ = 0; + tot_frame_number_ = 0; + // For testing up to 3 layers. + for (int i = 0; i < 3; ++i) { + bits_total_[i] = 0; + } + duration_ = 0.0; + } + + virtual void SetUp() { + InitializeConfig(); + SetMode(encoding_mode_); + } + + // + // Frame flags and layer id for temporal layers. + // + + // For two layers, test pattern is: + // 1 3 + // 0 2 ..... + // For three layers, test pattern is: + // 1 3 5 7 + // 2 6 + // 0 4 .... + // LAST is always update on base/layer 0, GOLDEN is updated on layer 1, + // and ALTREF is updated on top layer for 3 layer pattern. + int SetFrameFlags(int frame_num, int num_temp_layers) { + int frame_flags = 0; + if (num_temp_layers == 2) { + if (frame_num % 2 == 0) { + // Layer 0: predict from L and ARF, update L. + frame_flags = VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | + VP8_EFLAG_NO_UPD_ARF; + } else { + // Layer 1: predict from L, G and ARF, and update G. + frame_flags = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST | + VP8_EFLAG_NO_UPD_ENTROPY; + } + } else if (num_temp_layers == 3) { + if (frame_num % 4 == 0) { + // Layer 0: predict from L, update L. + frame_flags = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF; + } else if ((frame_num - 2) % 4 == 0) { + // Layer 1: predict from L, G, update G. + frame_flags = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST | + VP8_EFLAG_NO_REF_ARF; + } else if ((frame_num - 1) % 2 == 0) { + // Layer 2: predict from L, G, ARF; update ARG. + frame_flags = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_LAST; + } + } + return frame_flags; + } + + int SetLayerId(int frame_num, int num_temp_layers) { + int layer_id = 0; + if (num_temp_layers == 2) { + if (frame_num % 2 == 0) { + layer_id = 0; + } else { + layer_id = 1; + } + } else if (num_temp_layers == 3) { + if (frame_num % 4 == 0) { + layer_id = 0; + } else if ((frame_num - 2) % 4 == 0) { + layer_id = 1; + } else if ((frame_num - 1) % 2 == 0) { + layer_id = 2; + } + } + return layer_id; + } + + virtual void PreEncodeFrameHook(libvpx_test::VideoSource *video, + libvpx_test::Encoder *encoder) { + if (cfg_.ts_number_layers > 1) { + int layer_id = SetLayerId(video->frame(), cfg_.ts_number_layers); + int frame_flags = SetFrameFlags(video->frame(), cfg_.ts_number_layers); + if (video->frame() > 0) { + encoder->Control(VP8E_SET_TEMPORAL_LAYER_ID, layer_id); + encoder->Control(VP8E_SET_FRAME_FLAGS, frame_flags); + } + const vpx_rational_t tb = video->timebase(); + timebase_ = static_cast<double>(tb.num) / tb.den; + duration_ = 0; + return; + } + } + + virtual void FramePktHook(const vpx_codec_cx_pkt_t *pkt) { + // Time since last timestamp = duration. + vpx_codec_pts_t duration = pkt->data.frame.pts - last_pts_; + if (duration > 1) { + // Update counter for total number of frames (#frames input to encoder). + // Needed for setting the proper layer_id below. + tot_frame_number_ += static_cast<int>(duration - 1); + } + int layer = SetLayerId(tot_frame_number_, cfg_.ts_number_layers); + const size_t frame_size_in_bits = pkt->data.frame.sz * 8; + // Update the total encoded bits. For temporal layers, update the cumulative + // encoded bits per layer. + for (int i = layer; i < static_cast<int>(cfg_.ts_number_layers); ++i) { + bits_total_[i] += frame_size_in_bits; + } + // Update the most recent pts. + last_pts_ = pkt->data.frame.pts; + ++tot_frame_number_; + } + + virtual void EndPassHook(void) { + duration_ = (last_pts_ + 1) * timebase_; + if (cfg_.ts_number_layers > 1) { + for (int layer = 0; layer < static_cast<int>(cfg_.ts_number_layers); + ++layer) { + if (bits_total_[layer]) { + // Effective file datarate: + effective_datarate_[layer] = (bits_total_[layer] / 1000.0) / duration_; + } + } + } + } + + double effective_datarate_[3]; + private: + libvpx_test::TestMode encoding_mode_; + vpx_codec_pts_t last_pts_; + double timebase_; + int64_t bits_total_[3]; + double duration_; + int tot_frame_number_; + }; + +// Check two codec controls used for: +// (1) for setting temporal layer id, and (2) for settings encoder flags. +// This test invokes those controls for each frame, and verifies encoder/decoder +// mismatch and basic rate control response. +// TODO(marpan): Maybe move this test to datarate_test.cc. +TEST_P(ErrorResilienceTestLargeCodecControls, CodecControl3TemporalLayers) { + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 500; + cfg_.rc_buf_sz = 1000; + cfg_.rc_dropframe_thresh = 1; + cfg_.rc_min_quantizer = 2; + cfg_.rc_max_quantizer = 56; + cfg_.rc_end_usage = VPX_CBR; + cfg_.rc_dropframe_thresh = 1; + cfg_.g_lag_in_frames = 0; + cfg_.kf_mode = VPX_KF_DISABLED; + cfg_.g_error_resilient = 1; + + // 3 Temporal layers. Framerate decimation (4, 2, 1). + cfg_.ts_number_layers = 3; + cfg_.ts_rate_decimator[0] = 4; + cfg_.ts_rate_decimator[1] = 2; + cfg_.ts_rate_decimator[2] = 1; + cfg_.ts_periodicity = 4; + cfg_.ts_layer_id[0] = 0; + cfg_.ts_layer_id[1] = 2; + cfg_.ts_layer_id[2] = 1; + cfg_.ts_layer_id[3] = 2; + + ::libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + 30, 1, 0, 200); + for (int i = 200; i <= 800; i += 200) { + cfg_.rc_target_bitrate = i; + Reset(); + // 40-20-40 bitrate allocation for 3 temporal layers. + cfg_.ts_target_bitrate[0] = 40 * cfg_.rc_target_bitrate / 100; + cfg_.ts_target_bitrate[1] = 60 * cfg_.rc_target_bitrate / 100; + cfg_.ts_target_bitrate[2] = cfg_.rc_target_bitrate; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + for (int j = 0; j < static_cast<int>(cfg_.ts_number_layers); ++j) { + ASSERT_GE(effective_datarate_[j], cfg_.ts_target_bitrate[j] * 0.75) + << " The datarate for the file is lower than target by too much, " + "for layer: " << j; + ASSERT_LE(effective_datarate_[j], cfg_.ts_target_bitrate[j] * 1.25) + << " The datarate for the file is greater than target by too much, " + "for layer: " << j; + } + } +} + VP8_INSTANTIATE_TEST_CASE(ErrorResilienceTestLarge, ONE_PASS_TEST_MODES); +VP8_INSTANTIATE_TEST_CASE(ErrorResilienceTestLargeCodecControls, + ONE_PASS_TEST_MODES); VP9_INSTANTIATE_TEST_CASE(ErrorResilienceTestLarge, ONE_PASS_TEST_MODES); - } // namespace diff --git a/vp8/common/blockd.h b/vp8/common/blockd.h index ea1a6a4ad..192108a06 100644 --- a/vp8/common/blockd.h +++ b/vp8/common/blockd.h @@ -187,8 +187,12 @@ typedef struct { FRAME_TYPE frame_type; int is_frame_dropped; + // The frame rate for the lowest resolution. + double low_res_framerate; /* The frame number of each reference frames */ unsigned int low_res_ref_frames[MAX_REF_FRAMES]; + // The video frame counter value for the key frame, for lowest resolution. + unsigned int key_frame_counter_value; LOWER_RES_MB_INFO *mb_info; } LOWER_RES_FRAME_INFO; #endif diff --git a/vp8/common/onyx.h b/vp8/common/onyx.h index d48c4fe5e..f39b675cd 100644 --- a/vp8/common/onyx.h +++ b/vp8/common/onyx.h @@ -122,6 +122,7 @@ extern "C" int Sharpness; int cpu_used; unsigned int rc_max_intra_bitrate_pct; + unsigned int screen_content_mode; /* mode -> *(0)=Realtime/Live Encoding. This mode is optimized for realtim diff --git a/vp8/encoder/block.h b/vp8/encoder/block.h index dbdcab957..248e79549 100644 --- a/vp8/encoder/block.h +++ b/vp8/encoder/block.h @@ -125,6 +125,8 @@ typedef struct macroblock int optimize; int q_index; + int is_skin; + int denoise_zeromv; #if CONFIG_TEMPORAL_DENOISING int increase_denoising; @@ -161,6 +163,8 @@ typedef struct macroblock void (*short_walsh4x4)(short *input, short *output, int pitch); void (*quantize_b)(BLOCK *b, BLOCKD *d); + unsigned int mbs_zero_last_dot_suppress; + int zero_last_dot_suppress; } MACROBLOCK; diff --git a/vp8/encoder/denoising.c b/vp8/encoder/denoising.c index c0eff4e55..b9fbf061e 100644 --- a/vp8/encoder/denoising.c +++ b/vp8/encoder/denoising.c @@ -391,7 +391,7 @@ void vp8_denoiser_set_parameters(VP8_DENOISER *denoiser, int mode) { denoiser->denoise_pars.scale_increase_filter = 1; denoiser->denoise_pars.denoise_mv_bias = 60; denoiser->denoise_pars.pickmode_mv_bias = 75; - denoiser->denoise_pars.qp_thresh = 85; + denoiser->denoise_pars.qp_thresh = 80; denoiser->denoise_pars.consec_zerolast = 15; denoiser->denoise_pars.spatial_blur = 0; } @@ -456,10 +456,10 @@ int vp8_denoiser_allocate(VP8_DENOISER *denoiser, int width, int height, denoiser->bitrate_threshold = 400000; // (bits/sec). denoiser->threshold_aggressive_mode = 80; if (width * height > 1280 * 720) { - denoiser->bitrate_threshold = 2500000; - denoiser->threshold_aggressive_mode = 180; + denoiser->bitrate_threshold = 3000000; + denoiser->threshold_aggressive_mode = 200; } else if (width * height > 960 * 540) { - denoiser->bitrate_threshold = 1000000; + denoiser->bitrate_threshold = 1200000; denoiser->threshold_aggressive_mode = 120; } else if (width * height > 640 * 480) { denoiser->bitrate_threshold = 600000; @@ -483,7 +483,6 @@ void vp8_denoiser_free(VP8_DENOISER *denoiser) vpx_free(denoiser->denoise_state); } - void vp8_denoiser_denoise_mb(VP8_DENOISER *denoiser, MACROBLOCK *x, unsigned int best_sse, @@ -554,6 +553,7 @@ void vp8_denoiser_denoise_mb(VP8_DENOISER *denoiser, * Note that any changes to the mode info only affects the * denoising. */ + x->denoise_zeromv = 1; mbmi->ref_frame = x->best_zeromv_reference_frame; @@ -603,6 +603,12 @@ void vp8_denoiser_denoise_mb(VP8_DENOISER *denoiser, motion_threshold = denoiser->denoise_pars.scale_motion_thresh * NOISE_MOTION_THRESHOLD; + // If block is considered to be skin area, lower the motion threshold. + // In current version set threshold = 1, so only denoise very low + // (i.e., zero) mv on skin. + if (x->is_skin) + motion_threshold = 1; + if (motion_magnitude2 < denoiser->denoise_pars.scale_increase_filter * NOISE_MOTION_THRESHOLD) x->increase_denoising = 1; @@ -662,6 +668,7 @@ void vp8_denoiser_denoise_mb(VP8_DENOISER *denoiser, /* No filtering of this block; it differs too much from the predictor, * or the motion vector magnitude is considered too big. */ + x->denoise_zeromv = 0; vp8_copy_mem16x16( x->thismb, 16, denoiser->yv12_running_avg[INTRA_FRAME].y_buffer + recon_yoffset, @@ -692,7 +699,7 @@ void vp8_denoiser_denoise_mb(VP8_DENOISER *denoiser, int uv_stride =denoiser->yv12_running_avg[INTRA_FRAME].uv_stride; // Fix filter level to some nominal value for now. - int filter_level = 32; + int filter_level = 48; int hev_index = lfi_n->hev_thr_lut[INTER_FRAME][filter_level]; lfi.mblim = lfi_n->mblim[filter_level]; diff --git a/vp8/encoder/denoising.h b/vp8/encoder/denoising.h index 6c1f9e22b..9a379a6a1 100644 --- a/vp8/encoder/denoising.h +++ b/vp8/encoder/denoising.h @@ -19,7 +19,7 @@ extern "C" { #endif #define SUM_DIFF_THRESHOLD (16 * 16 * 2) -#define SUM_DIFF_THRESHOLD_HIGH (600) +#define SUM_DIFF_THRESHOLD_HIGH (600) // ~(16 * 16 * 1.5) #define MOTION_MAGNITUDE_THRESHOLD (8*3) #define SUM_DIFF_THRESHOLD_UV (96) // (8 * 8 * 1.5) @@ -27,7 +27,7 @@ extern "C" { #define SUM_DIFF_FROM_AVG_THRESH_UV (8 * 8 * 8) #define MOTION_MAGNITUDE_THRESHOLD_UV (8*3) -#define MAX_GF_ARF_DENOISE_RANGE (16) +#define MAX_GF_ARF_DENOISE_RANGE (8) enum vp8_denoiser_decision { diff --git a/vp8/encoder/encodeframe.c b/vp8/encoder/encodeframe.c index 2a3f69cc8..70632c0c3 100644 --- a/vp8/encoder/encodeframe.c +++ b/vp8/encoder/encodeframe.c @@ -522,7 +522,8 @@ void encode_mb_row(VP8_COMP *cpi, } #endif - // Keep track of how many (consecutive) times a block is coded + + // Keep track of how many (consecutive) times a block is coded // as ZEROMV_LASTREF, for base layer frames. // Reset to 0 if its coded as anything else. if (cpi->current_layer == 0) { @@ -531,9 +532,14 @@ void encode_mb_row(VP8_COMP *cpi, // Increment, check for wrap-around. if (cpi->consec_zero_last[map_index+mb_col] < 255) cpi->consec_zero_last[map_index+mb_col] += 1; + if (cpi->consec_zero_last_mvbias[map_index+mb_col] < 255) + cpi->consec_zero_last_mvbias[map_index+mb_col] += 1; } else { cpi->consec_zero_last[map_index+mb_col] = 0; + cpi->consec_zero_last_mvbias[map_index+mb_col] = 0; } + if (x->zero_last_dot_suppress) + cpi->consec_zero_last_mvbias[map_index+mb_col] = 0; } /* Special case code for cyclic refresh diff --git a/vp8/encoder/ethreading.c b/vp8/encoder/ethreading.c index 3598a7a69..a6b30a6bc 100644 --- a/vp8/encoder/ethreading.c +++ b/vp8/encoder/ethreading.c @@ -215,11 +215,15 @@ THREAD_FUNCTION thread_encoding_proc(void *p_data) LAST_FRAME) { // Increment, check for wrap-around. if (cpi->consec_zero_last[map_index+mb_col] < 255) - cpi->consec_zero_last[map_index+mb_col] += - 1; + cpi->consec_zero_last[map_index+mb_col] += 1; + if (cpi->consec_zero_last_mvbias[map_index+mb_col] < 255) + cpi->consec_zero_last_mvbias[map_index+mb_col] += 1; } else { cpi->consec_zero_last[map_index+mb_col] = 0; + cpi->consec_zero_last_mvbias[map_index+mb_col] = 0; } + if (x->zero_last_dot_suppress) + cpi->consec_zero_last_mvbias[map_index+mb_col] = 0; } /* Special case code for cyclic refresh @@ -505,6 +509,7 @@ void vp8cx_init_mbrthread_data(VP8_COMP *cpi, mb->intra_error = 0; vp8_zero(mb->count_mb_ref_frame_usage); mb->mbs_tested_so_far = 0; + mb->mbs_zero_last_dot_suppress = 0; } } diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c index 3cceb5a80..b04645654 100644 --- a/vp8/encoder/onyx_if.c +++ b/vp8/encoder/onyx_if.c @@ -579,11 +579,31 @@ static void cyclic_background_refresh(VP8_COMP *cpi, int Q, int lf_adjustment) cpi->cyclic_refresh_q = Q / 2; + if (cpi->oxcf.screen_content_mode) { + // Modify quality ramp-up based on Q. Above some Q level, increase the + // number of blocks to be refreshed, and reduce it below the thredhold. + // Turn-off under certain conditions (i.e., away from key frame, and if + // we are at good quality (low Q) and most of the blocks were skipped-encoded + // in previous frame. + if (Q >= 100) { + cpi->cyclic_refresh_mode_max_mbs_perframe = + (cpi->common.mb_rows * cpi->common.mb_cols) / 10; + } else if (cpi->frames_since_key > 250 && + Q < 20 && + cpi->mb.skip_true_count > (int)(0.95 * mbs_in_frame)) { + cpi->cyclic_refresh_mode_max_mbs_perframe = 0; + } else { + cpi->cyclic_refresh_mode_max_mbs_perframe = + (cpi->common.mb_rows * cpi->common.mb_cols) / 20; + } + block_count = cpi->cyclic_refresh_mode_max_mbs_perframe; + } + // Set every macroblock to be eligible for update. // For key frame this will reset seg map to 0. vpx_memset(cpi->segmentation_map, 0, mbs_in_frame); - if (cpi->common.frame_type != KEY_FRAME) + if (cpi->common.frame_type != KEY_FRAME && block_count > 0) { /* Cycle through the macro_block rows */ /* MB loop to set local segmentation map */ @@ -617,15 +637,18 @@ static void cyclic_background_refresh(VP8_COMP *cpi, int Q, int lf_adjustment) #if CONFIG_TEMPORAL_DENOISING if (cpi->oxcf.noise_sensitivity > 0) { if (cpi->denoiser.denoiser_mode == kDenoiserOnYUVAggressive && - Q < (int)cpi->denoiser.denoise_pars.qp_thresh) { + Q < (int)cpi->denoiser.denoise_pars.qp_thresh && + (cpi->frames_since_key > + 2 * cpi->denoiser.denoise_pars.consec_zerolast)) { // Under aggressive denoising, use segmentation to turn off loop - // filter below some qp thresh. The filter is turned off for all + // filter below some qp thresh. The filter is reduced for all // blocks that have been encoded as ZEROMV LAST x frames in a row, // where x is set by cpi->denoiser.denoise_pars.consec_zerolast. // This is to avoid "dot" artifacts that can occur from repeated // loop filtering on noisy input source. cpi->cyclic_refresh_q = Q; - lf_adjustment = -MAX_LOOP_FILTER; + // lf_adjustment = -MAX_LOOP_FILTER; + lf_adjustment = -40; for (i = 0; i < mbs_in_frame; ++i) { seg_map[i] = (cpi->consec_zero_last[i] > cpi->denoiser.denoise_pars.consec_zerolast) ? 1 : 0; @@ -786,6 +809,7 @@ void vp8_set_speed_features(VP8_COMP *cpi) } cpi->mb.mbs_tested_so_far = 0; + cpi->mb.mbs_zero_last_dot_suppress = 0; /* best quality defaults */ sf->RD = 1; @@ -853,6 +877,25 @@ void vp8_set_speed_features(VP8_COMP *cpi) sf->thresh_mult[THR_SPLIT2] = sf->thresh_mult[THR_SPLIT3] = speed_map(Speed, thresh_mult_map_split2); + // Special case for temporal layers. + // Reduce the thresholds for zero/nearest/near for GOLDEN, if GOLDEN is + // used as second reference. We don't modify thresholds for ALTREF case + // since ALTREF is usually used as long-term reference in temporal layers. + if ((cpi->Speed <= 6) && + (cpi->oxcf.number_of_layers > 1) && + (cpi->ref_frame_flags & VP8_LAST_FRAME) && + (cpi->ref_frame_flags & VP8_GOLD_FRAME)) { + if (cpi->closest_reference_frame == GOLDEN_FRAME) { + sf->thresh_mult[THR_ZERO2] = sf->thresh_mult[THR_ZERO2] >> 3; + sf->thresh_mult[THR_NEAREST2] = sf->thresh_mult[THR_NEAREST2] >> 3; + sf->thresh_mult[THR_NEAR2] = sf->thresh_mult[THR_NEAR2] >> 3; + } else { + sf->thresh_mult[THR_ZERO2] = sf->thresh_mult[THR_ZERO2] >> 1; + sf->thresh_mult[THR_NEAREST2] = sf->thresh_mult[THR_NEAREST2] >> 1; + sf->thresh_mult[THR_NEAR2] = sf->thresh_mult[THR_NEAR2] >> 1; + } + } + cpi->mode_check_freq[THR_ZERO1] = cpi->mode_check_freq[THR_NEAREST1] = cpi->mode_check_freq[THR_NEAR1] = @@ -1380,6 +1423,12 @@ static void init_config(VP8_COMP *cpi, VP8_CONFIG *oxcf) cpi->ref_framerate = cpi->framerate; + cpi->ref_frame_flags = VP8_ALTR_FRAME | VP8_GOLD_FRAME | VP8_LAST_FRAME; + + cm->refresh_golden_frame = 0; + cm->refresh_last_frame = 1; + cm->refresh_entropy_probs = 1; + /* change includes all joint functionality */ vp8_change_config(cpi, oxcf); @@ -1600,12 +1649,6 @@ void vp8_change_config(VP8_COMP *cpi, VP8_CONFIG *oxcf) cpi->baseline_gf_interval = cpi->oxcf.alt_freq ? cpi->oxcf.alt_freq : DEFAULT_GF_INTERVAL; - cpi->ref_frame_flags = VP8_ALTR_FRAME | VP8_GOLD_FRAME | VP8_LAST_FRAME; - - cm->refresh_golden_frame = 0; - cm->refresh_last_frame = 1; - cm->refresh_entropy_probs = 1; - #if (CONFIG_REALTIME_ONLY & CONFIG_ONTHEFLY_BITPACKING) cpi->oxcf.token_partitions = 3; #endif @@ -1708,7 +1751,11 @@ void vp8_change_config(VP8_COMP *cpi, VP8_CONFIG *oxcf) if (cpi->oxcf.number_of_layers != prev_number_of_layers) { // If the number of temporal layers are changed we must start at the - // base of the pattern cycle, so reset temporal_pattern_counter. + // base of the pattern cycle, so set the layer id to 0 and reset + // the temporal pattern counter. + if (cpi->temporal_layer_id > 0) { + cpi->temporal_layer_id = 0; + } cpi->temporal_pattern_counter = 0; reset_temporal_layer_change(cpi, oxcf, prev_number_of_layers); } @@ -1855,6 +1902,7 @@ struct VP8_COMP* vp8_create_compressor(VP8_CONFIG *oxcf) memcpy(cpi->base_skip_false_prob, vp8cx_base_skip_false_prob, sizeof(vp8cx_base_skip_false_prob)); cpi->common.current_video_frame = 0; cpi->temporal_pattern_counter = 0; + cpi->temporal_layer_id = -1; cpi->kf_overspend_bits = 0; cpi->kf_bitrate_adjustment = 0; cpi->frames_till_gf_update_due = 0; @@ -1907,6 +1955,8 @@ struct VP8_COMP* vp8_create_compressor(VP8_CONFIG *oxcf) } #endif + cpi->mse_source_denoised = 0; + /* Should we use the cyclic refresh method. * Currently this is tied to error resilliant mode */ @@ -1930,7 +1980,9 @@ struct VP8_COMP* vp8_create_compressor(VP8_CONFIG *oxcf) cpi->cyclic_refresh_map = (signed char *) NULL; CHECK_MEM_ERROR(cpi->consec_zero_last, - vpx_calloc(cpi->common.mb_rows * cpi->common.mb_cols, 1)); + vpx_calloc(cm->mb_rows * cm->mb_cols, 1)); + CHECK_MEM_ERROR(cpi->consec_zero_last_mvbias, + vpx_calloc((cpi->common.mb_rows * cpi->common.mb_cols), 1)); #ifdef VP8_ENTROPY_STATS init_context_counters(); @@ -2453,6 +2505,7 @@ void vp8_remove_compressor(VP8_COMP **ptr) vpx_free(cpi->tok); vpx_free(cpi->cyclic_refresh_map); vpx_free(cpi->consec_zero_last); + vpx_free(cpi->consec_zero_last_mvbias); vp8_remove_common(&cpi->common); vpx_free(cpi); @@ -3296,6 +3349,49 @@ static void update_reference_frames(VP8_COMP *cpi) } +static int measure_square_diff_partial(YV12_BUFFER_CONFIG *source, + YV12_BUFFER_CONFIG *dest, + VP8_COMP *cpi) + { + int i, j; + int Total = 0; + int num_blocks = 0; + int skip = 2; + int min_consec_zero_last = 10; + int tot_num_blocks = (source->y_height * source->y_width) >> 8; + unsigned char *src = source->y_buffer; + unsigned char *dst = dest->y_buffer; + + /* Loop through the Y plane, every |skip| blocks along rows and colmumns, + * summing the square differences, and only for blocks that have been + * zero_last mode at least |x| frames in a row. + */ + for (i = 0; i < source->y_height; i += 16 * skip) + { + int block_index_row = (i >> 4) * cpi->common.mb_cols; + for (j = 0; j < source->y_width; j += 16 * skip) + { + int index = block_index_row + (j >> 4); + if (cpi->consec_zero_last[index] >= min_consec_zero_last) { + unsigned int sse; + Total += vp8_mse16x16(src + j, + source->y_stride, + dst + j, dest->y_stride, + &sse); + num_blocks++; + } + } + src += 16 * skip * source->y_stride; + dst += 16 * skip * dest->y_stride; + } + // Only return non-zero if we have at least ~1/16 samples for estimate. + if (num_blocks > (tot_num_blocks >> 4)) { + return (Total / num_blocks); + } else { + return 0; + } + } + #if CONFIG_TEMPORAL_DENOISING static void process_denoiser_mode_change(VP8_COMP *cpi) { const VP8_COMMON *const cm = &cpi->common; @@ -3350,7 +3446,7 @@ static void process_denoiser_mode_change(VP8_COMP *cpi) { // Only consider this block as valid for noise measurement // if the sum_diff average of the current and previous frame // is small (to avoid effects from lighting change). - if ((sse - var) < 256) { + if ((sse - var) < 128) { unsigned int sse2; const unsigned int act = vp8_variance16x16(src + j, ystride, @@ -3421,6 +3517,13 @@ void vp8_loopfilter_frame(VP8_COMP *cpi, VP8_COMMON *cm) { const FRAME_TYPE frame_type = cm->frame_type; + int update_any_ref_buffers = 1; + if (cpi->common.refresh_last_frame == 0 && + cpi->common.refresh_golden_frame == 0 && + cpi->common.refresh_alt_ref_frame == 0) { + update_any_ref_buffers = 0; + } + if (cm->no_lpf) { cm->filter_level = 0; @@ -3432,11 +3535,36 @@ void vp8_loopfilter_frame(VP8_COMP *cpi, VP8_COMMON *cm) vp8_clear_system_state(); vpx_usec_timer_start(&timer); - if (cpi->sf.auto_filter == 0) + if (cpi->sf.auto_filter == 0) { +#if CONFIG_TEMPORAL_DENOISING + if (cpi->oxcf.noise_sensitivity && cm->frame_type != KEY_FRAME) { + // Use the denoised buffer for selecting base loop filter level. + // Denoised signal for current frame is stored in INTRA_FRAME. + // No denoising on key frames. + vp8cx_pick_filter_level_fast( + &cpi->denoiser.yv12_running_avg[INTRA_FRAME], cpi); + } else { + vp8cx_pick_filter_level_fast(cpi->Source, cpi); + } +#else vp8cx_pick_filter_level_fast(cpi->Source, cpi); - - else +#endif + } else { +#if CONFIG_TEMPORAL_DENOISING + if (cpi->oxcf.noise_sensitivity && cm->frame_type != KEY_FRAME) { + // Use the denoised buffer for selecting base loop filter level. + // Denoised signal for current frame is stored in INTRA_FRAME. + // No denoising on key frames. + vp8cx_pick_filter_level( + &cpi->denoiser.yv12_running_avg[INTRA_FRAME], cpi); + } else { + vp8cx_pick_filter_level(cpi->Source, cpi); + } +#else vp8cx_pick_filter_level(cpi->Source, cpi); +#endif + } + if (cm->filter_level > 0) { @@ -3452,7 +3580,9 @@ void vp8_loopfilter_frame(VP8_COMP *cpi, VP8_COMMON *cm) sem_post(&cpi->h_event_end_lpf); /* signal that we have set filter_level */ #endif - if (cm->filter_level > 0) + // No need to apply loop-filter if the encoded frame does not update + // any reference buffers. + if (cm->filter_level > 0 && update_any_ref_buffers) { vp8_loop_filter_frame(cm, &cpi->mb.e_mbd, frame_type); } @@ -3582,39 +3712,78 @@ static void encode_frame_to_data_rate } #if CONFIG_MULTI_RES_ENCODING - /* In multi-resolution encoding, frame_type is decided by lowest-resolution - * encoder. Same frame_type is adopted while encoding at other resolution. - */ - if (cpi->oxcf.mr_encoder_id) - { - LOWER_RES_FRAME_INFO* low_res_frame_info - = (LOWER_RES_FRAME_INFO*)cpi->oxcf.mr_low_res_mode_info; + if (cpi->oxcf.mr_total_resolutions > 1) { + LOWER_RES_FRAME_INFO* low_res_frame_info + = (LOWER_RES_FRAME_INFO*)cpi->oxcf.mr_low_res_mode_info; + if (cpi->oxcf.mr_encoder_id) { + + // TODO(marpan): This constraint shouldn't be needed, as we would like + // to allow for key frame setting (forced or periodic) defined per + // spatial layer. For now, keep this in. cm->frame_type = low_res_frame_info->frame_type; + // Check if lower resolution is available for motion vector reuse. if(cm->frame_type != KEY_FRAME) { - cpi->mr_low_res_mv_avail = 1; - cpi->mr_low_res_mv_avail &= !(low_res_frame_info->is_frame_dropped); - - if (cpi->ref_frame_flags & VP8_LAST_FRAME) - cpi->mr_low_res_mv_avail &= (cpi->current_ref_frames[LAST_FRAME] - == low_res_frame_info->low_res_ref_frames[LAST_FRAME]); - - if (cpi->ref_frame_flags & VP8_GOLD_FRAME) - cpi->mr_low_res_mv_avail &= (cpi->current_ref_frames[GOLDEN_FRAME] - == low_res_frame_info->low_res_ref_frames[GOLDEN_FRAME]); + cpi->mr_low_res_mv_avail = 1; + cpi->mr_low_res_mv_avail &= !(low_res_frame_info->is_frame_dropped); + + if (cpi->ref_frame_flags & VP8_LAST_FRAME) + cpi->mr_low_res_mv_avail &= (cpi->current_ref_frames[LAST_FRAME] + == low_res_frame_info->low_res_ref_frames[LAST_FRAME]); + + if (cpi->ref_frame_flags & VP8_GOLD_FRAME) + cpi->mr_low_res_mv_avail &= (cpi->current_ref_frames[GOLDEN_FRAME] + == low_res_frame_info->low_res_ref_frames[GOLDEN_FRAME]); + + // Don't use altref to determine whether low res is available. + // TODO (marpan): Should we make this type of condition on a + // per-reference frame basis? + /* + if (cpi->ref_frame_flags & VP8_ALTR_FRAME) + cpi->mr_low_res_mv_avail &= (cpi->current_ref_frames[ALTREF_FRAME] + == low_res_frame_info->low_res_ref_frames[ALTREF_FRAME]); + */ + } + } - if (cpi->ref_frame_flags & VP8_ALTR_FRAME) - cpi->mr_low_res_mv_avail &= (cpi->current_ref_frames[ALTREF_FRAME] - == low_res_frame_info->low_res_ref_frames[ALTREF_FRAME]); + // On a key frame: For the lowest resolution, keep track of the key frame + // counter value. For the higher resolutions, reset the current video + // frame counter to that of the lowest resolution. + // This is done to the handle the case where we may stop/start encoding + // higher layer(s). The restart-encoding of higher layer is only signaled + // by a key frame for now. + // TODO (marpan): Add flag to indicate restart-encoding of higher layer. + if (cm->frame_type == KEY_FRAME) { + if (cpi->oxcf.mr_encoder_id) { + // If the initial starting value of the buffer level is zero (this can + // happen because we may have not started encoding this higher stream), + // then reset it to non-zero value based on |starting_buffer_level|. + if (cpi->common.current_video_frame == 0 && cpi->buffer_level == 0) { + unsigned int i; + cpi->bits_off_target = cpi->oxcf.starting_buffer_level; + cpi->buffer_level = cpi->oxcf.starting_buffer_level; + for (i = 0; i < cpi->oxcf.number_of_layers; i++) { + LAYER_CONTEXT *lc = &cpi->layer_context[i]; + lc->bits_off_target = lc->starting_buffer_level; + lc->buffer_level = lc->starting_buffer_level; + } + } + cpi->common.current_video_frame = + low_res_frame_info->key_frame_counter_value; + } else { + low_res_frame_info->key_frame_counter_value = + cpi->common.current_video_frame; } + } + } #endif // Find the reference frame closest to the current frame. cpi->closest_reference_frame = LAST_FRAME; - if (cm->frame_type != KEY_FRAME) { + if(cm->frame_type != KEY_FRAME) { int i; MV_REFERENCE_FRAME closest_ref = INTRA_FRAME; if (cpi->ref_frame_flags & VP8_LAST_FRAME) { @@ -3624,12 +3793,12 @@ static void encode_frame_to_data_rate } else if (cpi->ref_frame_flags & VP8_ALTR_FRAME) { closest_ref = ALTREF_FRAME; } - for (i = 1; i <= 3; i++) { + for(i = 1; i <= 3; i++) { vpx_ref_frame_type_t ref_frame_type = (vpx_ref_frame_type_t) ((i == 3) ? 4 : i); if (cpi->ref_frame_flags & ref_frame_type) { if ((cm->current_video_frame - cpi->current_ref_frames[i]) < - (cm->current_video_frame - cpi->current_ref_frames[closest_ref])) { + (cm->current_video_frame - cpi->current_ref_frames[closest_ref])) { closest_ref = i; } } @@ -3656,6 +3825,8 @@ static void encode_frame_to_data_rate // Reset the zero_last counter to 0 on key frame. vpx_memset(cpi->consec_zero_last, 0, cm->mb_rows * cm->mb_cols); + vpx_memset(cpi->consec_zero_last_mvbias, 0, + (cpi->common.mb_rows * cpi->common.mb_cols)); } #if 0 @@ -4184,8 +4355,10 @@ static void encode_frame_to_data_rate else disable_segmentation(cpi); } - // Reset the consec_zero_last counter on key frame. + // Reset the zero_last counter to 0 on key frame. vpx_memset(cpi->consec_zero_last, 0, cm->mb_rows * cm->mb_cols); + vpx_memset(cpi->consec_zero_last_mvbias, 0, + (cpi->common.mb_rows * cpi->common.mb_cols)); vp8_set_quantizer(cpi, Q); } @@ -4618,6 +4791,22 @@ static void encode_frame_to_data_rate cm->frame_to_show = &cm->yv12_fb[cm->new_fb_idx]; #if CONFIG_TEMPORAL_DENOISING + // Get some measure of the amount of noise, by measuring the (partial) mse + // between source and denoised buffer, for y channel. Partial refers to + // computing the sse for a sub-sample of the frame (i.e., skip x blocks along row/column), + // and only for blocks in that set that are consecutive ZEROMV_LAST mode. + // Do this every ~8 frames, to further reduce complexity. + // TODO(marpan): Keep this for now for the case cpi->oxcf.noise_sensitivity < 4, + // should be removed in favor of the process_denoiser_mode_change() function below. + if (cpi->oxcf.noise_sensitivity > 0 && + cpi->oxcf.noise_sensitivity < 4 && + !cpi->oxcf.screen_content_mode && + cpi->frames_since_key%8 == 0 && + cm->frame_type != KEY_FRAME) { + cpi->mse_source_denoised = measure_square_diff_partial( + &cpi->denoiser.yv12_running_avg[INTRA_FRAME], cpi->Source, cpi); + } + // For the adaptive denoising mode (noise_sensitivity == 4), sample the mse // of source diff (between current and previous frame), and determine if we // should switch the denoiser mode. Sampling refers to computing the mse for @@ -4626,6 +4815,7 @@ static void encode_frame_to_data_rate // constraint on the sum diff between blocks. This process is called every // ~8 frames, to further reduce complexity. if (cpi->oxcf.noise_sensitivity == 4 && + !cpi->oxcf.screen_content_mode && cpi->frames_since_key % 8 == 0 && cm->frame_type != KEY_FRAME) { process_denoiser_mode_change(cpi); @@ -4763,6 +4953,13 @@ static void encode_frame_to_data_rate if (cpi->bits_off_target > cpi->oxcf.maximum_buffer_size) cpi->bits_off_target = cpi->oxcf.maximum_buffer_size; + // If the frame dropper is not enabled, don't let the buffer level go below + // some threshold, given here by -|maximum_buffer_size|. For now we only do + // this for screen content input. + if (cpi->drop_frames_allowed == 0 && cpi->oxcf.screen_content_mode && + cpi->bits_off_target < -cpi->oxcf.maximum_buffer_size) + cpi->bits_off_target = -cpi->oxcf.maximum_buffer_size; + /* Rolling monitors of whether we are over or underspending used to * help regulate min and Max Q in two pass. */ @@ -5237,7 +5434,26 @@ int vp8_get_compressed_data(VP8_COMP *cpi, unsigned int *frame_flags, unsigned l cpi->ref_framerate = 10000000.0 / avg_duration; } - +#if CONFIG_MULTI_RES_ENCODING + if (cpi->oxcf.mr_total_resolutions > 1) { + LOWER_RES_FRAME_INFO* low_res_frame_info = (LOWER_RES_FRAME_INFO*) + cpi->oxcf.mr_low_res_mode_info; + // Frame rate should be the same for all spatial layers in + // multi-res-encoding (simulcast), so we constrain the frame for + // higher layers to be that of lowest resolution. This is needed + // as he application may decide to skip encoding a high layer and + // then start again, in which case a big jump in time-stamps will + // be received for that high layer, which will yield an incorrect + // frame rate (from time-stamp adjustment in above calculation). + if (cpi->oxcf.mr_encoder_id) { + cpi->ref_framerate = low_res_frame_info->low_res_framerate; + } + else { + // Keep track of frame rate for lowest resolution. + low_res_frame_info->low_res_framerate = cpi->ref_framerate; + } + } +#endif if (cpi->oxcf.number_of_layers > 1) { unsigned int i; @@ -5267,8 +5483,12 @@ int vp8_get_compressed_data(VP8_COMP *cpi, unsigned int *frame_flags, unsigned l update_layer_contexts (cpi); /* Restore layer specific context & set frame rate */ - layer = cpi->oxcf.layer_id[ - cpi->temporal_pattern_counter % cpi->oxcf.periodicity]; + if (cpi->temporal_layer_id >= 0) { + layer = cpi->temporal_layer_id; + } else { + layer = cpi->oxcf.layer_id[ + cpi->temporal_pattern_counter % cpi->oxcf.periodicity]; + } restore_layer_context (cpi, layer); vp8_new_framerate(cpi, cpi->layer_context[layer].framerate); } diff --git a/vp8/encoder/onyx_int.h b/vp8/encoder/onyx_int.h index f0424e69c..b1a749c1d 100644 --- a/vp8/encoder/onyx_int.h +++ b/vp8/encoder/onyx_int.h @@ -513,10 +513,18 @@ typedef struct VP8_COMP signed char *cyclic_refresh_map; // Count on how many (consecutive) times a macroblock uses ZER0MV_LAST. unsigned char *consec_zero_last; + // Counter that is reset when a block is checked for a mode-bias against + // ZEROMV_LASTREF. + unsigned char *consec_zero_last_mvbias; // Frame counter for the temporal pattern. Counter is rest when the temporal // layers are changed dynamically (run-time change). unsigned int temporal_pattern_counter; + // Temporal layer id. + int temporal_layer_id; + + // Measure of average squared difference between source and denoised signal. + int mse_source_denoised; #if CONFIG_MULTITHREAD /* multithread data */ @@ -687,6 +695,7 @@ typedef struct VP8_COMP #endif /* The frame number of each reference frames */ unsigned int current_ref_frames[MAX_REF_FRAMES]; + // Closest reference frame to current frame. MV_REFERENCE_FRAME closest_reference_frame; struct rd_costs_struct diff --git a/vp8/encoder/pickinter.c b/vp8/encoder/pickinter.c index 9d5556dcd..9eb69324b 100644 --- a/vp8/encoder/pickinter.c +++ b/vp8/encoder/pickinter.c @@ -40,6 +40,134 @@ extern const MB_PREDICTION_MODE vp8_mode_order[MAX_MODES]; extern int vp8_cost_mv_ref(MB_PREDICTION_MODE m, const int near_mv_ref_ct[4]); +// Fixed point implementation of a skin color classifier. Skin color +// is model by a Gaussian distribution in the CbCr color space. +// See ../../test/skin_color_detector_test.cc where the reference +// skin color classifier is defined. + +// Fixed-point skin color model parameters. +static const int skin_mean[2] = {7463, 9614}; // q6 +static const int skin_inv_cov[4] = {4107, 1663, 1663, 2157}; // q16 +static const int skin_threshold = 1570636; // q18 + +// Evaluates the Mahalanobis distance measure for the input CbCr values. +static int evaluate_skin_color_difference(int cb, int cr) +{ + const int cb_q6 = cb << 6; + const int cr_q6 = cr << 6; + const int cb_diff_q12 = (cb_q6 - skin_mean[0]) * (cb_q6 - skin_mean[0]); + const int cbcr_diff_q12 = (cb_q6 - skin_mean[0]) * (cr_q6 - skin_mean[1]); + const int cr_diff_q12 = (cr_q6 - skin_mean[1]) * (cr_q6 - skin_mean[1]); + const int cb_diff_q2 = (cb_diff_q12 + (1 << 9)) >> 10; + const int cbcr_diff_q2 = (cbcr_diff_q12 + (1 << 9)) >> 10; + const int cr_diff_q2 = (cr_diff_q12 + (1 << 9)) >> 10; + const int skin_diff = skin_inv_cov[0] * cb_diff_q2 + + skin_inv_cov[1] * cbcr_diff_q2 + + skin_inv_cov[2] * cbcr_diff_q2 + + skin_inv_cov[3] * cr_diff_q2; + return skin_diff; +} + +static int macroblock_corner_grad(unsigned char* signal, int stride, + int offsetx, int offsety, int sgnx, int sgny) +{ + int y1 = signal[offsetx * stride + offsety]; + int y2 = signal[offsetx * stride + offsety + sgny]; + int y3 = signal[(offsetx + sgnx) * stride + offsety]; + int y4 = signal[(offsetx + sgnx) * stride + offsety + sgny]; + return MAX(MAX(abs(y1 - y2), abs(y1 - y3)), abs(y1 - y4)); +} + +static int check_dot_artifact_candidate(VP8_COMP *cpi, + MACROBLOCK *x, + unsigned char *target_last, + int stride, + unsigned char* last_ref, + int mb_row, + int mb_col, + int channel) +{ + int threshold1 = 6; + int threshold2 = 3; + unsigned int max_num = (cpi->common.MBs) / 10; + int grad_last = 0; + int grad_source = 0; + int index = mb_row * cpi->common.mb_cols + mb_col; + // Threshold for #consecutive (base layer) frames using zero_last mode. + int num_frames = 30; + int shift = 15; + if (channel > 0) { + shift = 7; + } + if (cpi->oxcf.number_of_layers > 1) + { + num_frames = 20; + } + x->zero_last_dot_suppress = 0; + // Blocks on base layer frames that have been using ZEROMV_LAST repeatedly + // (i.e, at least |x| consecutive frames are candidates for increasing the + // rd adjustment for zero_last mode. + // Only allow this for at most |max_num| blocks per frame. + // Don't allow this for screen content input. + if (cpi->current_layer == 0 && + cpi->consec_zero_last_mvbias[index] > num_frames && + x->mbs_zero_last_dot_suppress < max_num && + !cpi->oxcf.screen_content_mode) + { + // If this block is checked here, label it so we don't check it again until + // ~|x| framaes later. + x->zero_last_dot_suppress = 1; + // Dot artifact is noticeable as strong gradient at corners of macroblock, + // for flat areas. As a simple detector for now, we look for a high + // corner gradient on last ref, and a smaller gradient on source. + // Check 4 corners, return if any satisfy condition. + // Top-left: + grad_last = macroblock_corner_grad(last_ref, stride, 0, 0, 1, 1); + grad_source = macroblock_corner_grad(target_last, stride, 0, 0, 1, 1); + if (grad_last >= threshold1 && grad_source <= threshold2) + { + x->mbs_zero_last_dot_suppress++; + return 1; + } + // Top-right: + grad_last = macroblock_corner_grad(last_ref, stride, 0, shift, 1, -1); + grad_source = macroblock_corner_grad(target_last, stride, 0, shift, 1, -1); + if (grad_last >= threshold1 && grad_source <= threshold2) + { + x->mbs_zero_last_dot_suppress++; + return 1; + } + // Bottom-left: + grad_last = macroblock_corner_grad(last_ref, stride, shift, 0, -1, 1); + grad_source = macroblock_corner_grad(target_last, stride, shift, 0, -1, 1); + if (grad_last >= threshold1 && grad_source <= threshold2) + { + x->mbs_zero_last_dot_suppress++; + return 1; + } + // Bottom-right: + grad_last = macroblock_corner_grad(last_ref, stride, shift, shift, -1, -1); + grad_source = macroblock_corner_grad(target_last, stride, shift, shift, -1, -1); + if (grad_last >= threshold1 && grad_source <= threshold2) + { + x->mbs_zero_last_dot_suppress++; + return 1; + } + return 0; + } + return 0; +} + +// Checks if the input yCbCr values corresponds to skin color. +static int is_skin_color(int y, int cb, int cr) +{ + if (y < 40 || y > 220) + { + return 0; + } + return (evaluate_skin_color_difference(cb, cr) < skin_threshold); +} + int vp8_skip_fractional_mv_step(MACROBLOCK *mb, BLOCK *b, BLOCKD *d, int_mv *bestmv, int_mv *ref_mv, int error_per_bit, @@ -514,10 +642,17 @@ static int evaluate_inter_mode(unsigned int* sse, int rate2, int* distortion2, #endif // Adjust rd for ZEROMV and LAST, if LAST is the closest reference frame. - if (this_mode == ZEROMV && - x->e_mbd.mode_info_context->mbmi.ref_frame == LAST_FRAME && - (denoise_aggressive || cpi->closest_reference_frame == LAST_FRAME)) { - this_rd = ((int64_t)this_rd) * rd_adj / 100; + // TODO: We should also add condition on distance of closest to current. + if(!cpi->oxcf.screen_content_mode && + this_mode == ZEROMV && + x->e_mbd.mode_info_context->mbmi.ref_frame == LAST_FRAME && + (denoise_aggressive || (cpi->closest_reference_frame == LAST_FRAME))) + { + // No adjustment if block is considered to be skin area. + if(x->is_skin) + rd_adj = 100; + + this_rd = ((int64_t)this_rd) * rd_adj / 100; } check_for_encode_breakout(*sse, x); @@ -597,6 +732,15 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, #endif int sf_improved_mv_pred = cpi->sf.improved_mv_pred; + +#if CONFIG_MULTI_RES_ENCODING + int dissim = INT_MAX; + int parent_ref_frame = 0; + int_mv parent_ref_mv; + MB_PREDICTION_MODE parent_mode = 0; + int parent_ref_valid = 0; +#endif + int_mv mvp; int near_sadidx[8] = {0, 1, 2, 3, 4, 5, 6, 7}; @@ -607,14 +751,55 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, unsigned char *plane[4][3]; int ref_frame_map[4]; int sign_bias = 0; + int dot_artifact_candidate = 0; + // For detecting dot artifact. + unsigned char* target = x->src.y_buffer; + unsigned char* target_u = x->block[16].src + *x->block[16].base_src; + unsigned char* target_v = x->block[20].src + *x->block[20].base_src; + int stride = x->src.y_stride; + int stride_uv = x->block[16].src_stride; +#if CONFIG_TEMPORAL_DENOISING + if (cpi->oxcf.noise_sensitivity) { + int uv_denoise = (cpi->oxcf.noise_sensitivity >= 2) ? 1 : 0; + target = + cpi->denoiser.yv12_running_avg[LAST_FRAME].y_buffer + recon_yoffset; + stride = cpi->denoiser.yv12_running_avg[LAST_FRAME].y_stride; + if (uv_denoise) { + target_u = + cpi->denoiser.yv12_running_avg[LAST_FRAME].u_buffer + recon_uvoffset; + target_v = + cpi->denoiser.yv12_running_avg[LAST_FRAME].v_buffer + recon_uvoffset; + stride_uv = cpi->denoiser.yv12_running_avg[LAST_FRAME].uv_stride; + } + } +#endif -#if CONFIG_MULTI_RES_ENCODING - int dissim = INT_MAX; - int parent_ref_frame = 0; - int parent_ref_valid = cpi->oxcf.mr_encoder_id && cpi->mr_low_res_mv_avail; - int_mv parent_ref_mv; - MB_PREDICTION_MODE parent_mode = 0; + get_predictor_pointers(cpi, plane, recon_yoffset, recon_uvoffset); + dot_artifact_candidate = + check_dot_artifact_candidate(cpi, x, + target, stride, + plane[LAST_FRAME][0], mb_row, mb_col, 0); + // If not found in Y channel, check UV channel. + if (!dot_artifact_candidate) { + dot_artifact_candidate = + check_dot_artifact_candidate(cpi, x, + target_u, stride_uv, + plane[LAST_FRAME][1], mb_row, mb_col, 1); + if (!dot_artifact_candidate) { + dot_artifact_candidate = + check_dot_artifact_candidate(cpi, x, + target_v, stride_uv, + plane[LAST_FRAME][2], mb_row, mb_col, 2); + } + } + +#if CONFIG_MULTI_RES_ENCODING + // |parent_ref_valid| will be set here if potentially we can do mv resue for + // this higher resol (|cpi->oxcf.mr_encoder_id| > 0) frame. + // |parent_ref_valid| may be reset depending on |parent_ref_frame| for + // the current macroblock below. + parent_ref_valid = cpi->oxcf.mr_encoder_id && cpi->mr_low_res_mv_avail; if (parent_ref_valid) { int parent_ref_flag; @@ -632,17 +817,44 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, * In this event, take the conservative approach of disabling the * lower res info for this MB. */ + parent_ref_flag = 0; + // Note availability for mv reuse is only based on last and golden. if (parent_ref_frame == LAST_FRAME) parent_ref_flag = (cpi->ref_frame_flags & VP8_LAST_FRAME); else if (parent_ref_frame == GOLDEN_FRAME) parent_ref_flag = (cpi->ref_frame_flags & VP8_GOLD_FRAME); - else if (parent_ref_frame == ALTREF_FRAME) - parent_ref_flag = (cpi->ref_frame_flags & VP8_ALTR_FRAME); //assert(!parent_ref_frame || parent_ref_flag); + + // If |parent_ref_frame| did not match either last or golden then + // shut off mv reuse. if (parent_ref_frame && !parent_ref_flag) parent_ref_valid = 0; + + // Don't do mv reuse since we want to allow for another mode besides + // ZEROMV_LAST to remove dot artifact. + if (dot_artifact_candidate) + parent_ref_valid = 0; + } +#endif + + // Check if current macroblock is in skin area. + { + const int y = x->src.y_buffer[7 * x->src.y_stride + 7]; + const int cb = x->src.u_buffer[3 * x->src.uv_stride + 3]; + const int cr = x->src.v_buffer[3 * x->src.uv_stride + 3]; + x->is_skin = 0; + if (!cpi->oxcf.screen_content_mode) + x->is_skin = is_skin_color(y, cb, cr); + } +#if CONFIG_TEMPORAL_DENOISING + if (cpi->oxcf.noise_sensitivity) { + // Under aggressive denoising mode, should we use skin map to reduce denoiser + // and ZEROMV bias? Will need to revisit the accuracy of this detection for + // very noisy input. For now keep this as is (i.e., don't turn it off). + // if (cpi->denoiser.denoiser_mode == kDenoiserOnYUVAggressive) + // x->is_skin = 0; } #endif @@ -680,8 +892,6 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, best_ref_mv.as_int = best_ref_mv_sb[sign_bias].as_int; } - get_predictor_pointers(cpi, plane, recon_yoffset, recon_uvoffset); - /* Count of the number of MBs tested so far this frame */ x->mbs_tested_so_far++; @@ -691,9 +901,13 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, x->e_mbd.mode_info_context->mbmi.ref_frame = INTRA_FRAME; /* If the frame has big static background and current MB is in low - * motion area, its mode decision is biased to ZEROMV mode. - */ - calculate_zeromv_rd_adjustment(cpi, x, &rd_adjustment); + * motion area, its mode decision is biased to ZEROMV mode. + * No adjustment if cpu_used is <= -12 (i.e., cpi->Speed >= 12). + * At such speed settings, ZEROMV is already heavily favored. + */ + if (cpi->Speed < 12) { + calculate_zeromv_rd_adjustment(cpi, x, &rd_adjustment); + } #if CONFIG_TEMPORAL_DENOISING if (cpi->oxcf.noise_sensitivity) { @@ -702,6 +916,13 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, } #endif + if (dot_artifact_candidate) + { + // Bias against ZEROMV_LAST mode. + rd_adjustment = 150; + } + + /* if we encode a new mv this is important * find the best new motion vector */ @@ -887,14 +1108,17 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, step_param = cpi->sf.first_step + speed_adjust; #if CONFIG_MULTI_RES_ENCODING - /* If lower-res drops this frame, then higher-res encoder does - motion search without any previous knowledge. Also, since - last frame motion info is not stored, then we can not + /* If lower-res frame is not available for mv reuse (because of + frame dropping or different temporal layer pattern), then higher + resol encoder does motion search without any previous knowledge. + Also, since last frame motion info is not stored, then we can not use improved_mv_pred. */ - if (cpi->oxcf.mr_encoder_id && !parent_ref_valid) + if (cpi->oxcf.mr_encoder_id) sf_improved_mv_pred = 0; - if (parent_ref_valid && parent_ref_frame) + // Only use parent MV as predictor if this candidate reference frame + // (|this_ref_frame|) is equal to |parent_ref_frame|. + if (parent_ref_valid && (parent_ref_frame == this_ref_frame)) { /* Use parent MV as predictor. Adjust search range * accordingly. @@ -938,7 +1162,8 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, } #if CONFIG_MULTI_RES_ENCODING - if (parent_ref_valid && parent_ref_frame && dissim <= 2 && + if (parent_ref_valid && (parent_ref_frame == this_ref_frame) && + dissim <= 2 && MAX(abs(best_ref_mv.as_mv.row - parent_ref_mv.as_mv.row), abs(best_ref_mv.as_mv.col - parent_ref_mv.as_mv.col)) <= 4) { @@ -975,10 +1200,12 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, * change the behavior in lowest-resolution encoder. * Will improve it later. */ - /* Set step_param to 0 to ensure large-range motion search - when encoder drops this frame at lower-resolution. - */ - if (!parent_ref_valid) + /* Set step_param to 0 to ensure large-range motion search + * when mv reuse if not valid (i.e. |parent_ref_valid| = 0), + * or if this candidate reference frame (|this_ref_frame|) is + * not equal to |parent_ref_frame|. + */ + if (!parent_ref_valid || (parent_ref_frame != this_ref_frame)) step_param = 0; #endif bestsme = vp8_hex_search(x, b, d, &mvp_full, &d->bmi.mv, @@ -1080,7 +1307,6 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, #if CONFIG_TEMPORAL_DENOISING if (cpi->oxcf.noise_sensitivity) { - /* Store for later use by denoiser. */ // Dont' denoise with GOLDEN OR ALTREF is they are old reference // frames (greater than MAX_GF_ARF_DENOISE_RANGE frames in past). @@ -1096,7 +1322,7 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, x->e_mbd.mode_info_context->mbmi.ref_frame; } - /* Store the best NEWMV in x for later use in the denoiser. */ + // Store the best NEWMV in x for later use in the denoiser. if (x->e_mbd.mode_info_context->mbmi.mode == NEWMV && sse < best_sse && !skip_old_reference) { @@ -1184,6 +1410,8 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, if (cpi->oxcf.noise_sensitivity) { int block_index = mb_row * cpi->common.mb_cols + mb_col; + int reevaluate = 0; + int is_noisy = 0; if (x->best_sse_inter_mode == DC_PRED) { /* No best MV found. */ @@ -1193,18 +1421,52 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, x->best_reference_frame = best_mbmode.ref_frame; best_sse = best_rd_sse; } + // For non-skin blocks that have selected ZEROMV for this current frame, + // and have been selecting ZEROMV_LAST (on the base layer frame) at + // least |x~20| consecutive past frames in a row, label the block for + // possible increase in denoising strength. We also condition this + // labeling on there being significant denoising in the scene + if (cpi->oxcf.noise_sensitivity == 4) { + if (cpi->denoiser.nmse_source_diff > + 70 * cpi->denoiser.threshold_aggressive_mode / 100) + is_noisy = 1; + } else { + if (cpi->mse_source_denoised > 1000) + is_noisy = 1; + } x->increase_denoising = 0; + if (!x->is_skin && + x->best_sse_inter_mode == ZEROMV && + (x->best_reference_frame == LAST_FRAME || + x->best_reference_frame == cpi->closest_reference_frame) && + cpi->consec_zero_last[block_index] >= 20 && + is_noisy) { + x->increase_denoising = 1; + } + x->denoise_zeromv = 0; vp8_denoiser_denoise_mb(&cpi->denoiser, x, best_sse, zero_mv_sse, recon_yoffset, recon_uvoffset, &cpi->common.lf_info, mb_row, mb_col, block_index); - /* Reevaluate ZEROMV after denoising. */ - if (best_mbmode.ref_frame == INTRA_FRAME && + // Reevaluate ZEROMV after denoising: for large noise content + // (i.e., cpi->mse_source_denoised is above threshold), do this for all + // blocks that did not pick ZEROMV as best mode but are using ZEROMV + // for denoising. Otherwise, always re-evaluate for blocks that picked + // INTRA mode as best mode. + // Avoid blocks that have been biased against ZERO_LAST + // (i.e., dot artifact candidate blocks). + reevaluate = (best_mbmode.ref_frame == INTRA_FRAME) || + (best_mbmode.mode != ZEROMV && + x->denoise_zeromv && + cpi->mse_source_denoised > 2000); + if (!dot_artifact_candidate && + reevaluate && x->best_zeromv_reference_frame != INTRA_FRAME) { int this_rd = 0; int this_ref_frame = x->best_zeromv_reference_frame; + rd_adjustment = 100; rate2 = x->ref_frame_cost[this_ref_frame] + vp8_cost_mv_ref(ZEROMV, mdcounts); distortion2 = 0; @@ -1264,7 +1526,6 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, update_mvcount(x, &best_ref_mv); } - void vp8_pick_intra_mode(MACROBLOCK *x, int *rate_) { int error4x4, error16x16 = INT_MAX; diff --git a/vp8/encoder/quantize.c b/vp8/encoder/quantize.c index 2feb316b3..bd47823dd 100644 --- a/vp8/encoder/quantize.c +++ b/vp8/encoder/quantize.c @@ -535,6 +535,7 @@ void vp8_set_quantizer(struct VP8_COMP *cpi, int Q) MACROBLOCKD *mbd = &cpi->mb.e_mbd; int update = 0; int new_delta_q; + int new_uv_delta_q; cm->base_qindex = Q; /* if any of the delta_q values are changing update flag has to be set */ @@ -542,8 +543,6 @@ void vp8_set_quantizer(struct VP8_COMP *cpi, int Q) cm->y1dc_delta_q = 0; cm->y2ac_delta_q = 0; - cm->uvdc_delta_q = 0; - cm->uvac_delta_q = 0; if (Q < 4) { @@ -555,6 +554,21 @@ void vp8_set_quantizer(struct VP8_COMP *cpi, int Q) update |= cm->y2dc_delta_q != new_delta_q; cm->y2dc_delta_q = new_delta_q; + new_uv_delta_q = 0; + // For screen content, lower the q value for UV channel. For now, select + // conservative delta; same delta for dc and ac, and decrease it with lower + // Q, and set to 0 below some threshold. May want to condition this in + // future on the variance/energy in UV channel. + if (cpi->oxcf.screen_content_mode && Q > 40) { + new_uv_delta_q = -(int)(0.15 * Q); + // Check range: magnitude of delta is 4 bits. + if (new_uv_delta_q < -15) { + new_uv_delta_q = -15; + } + } + update |= cm->uvdc_delta_q != new_uv_delta_q; + cm->uvdc_delta_q = new_uv_delta_q; + cm->uvac_delta_q = new_uv_delta_q; /* Set Segment specific quatizers */ mbd->segment_feature_data[MB_LVL_ALT_Q][0] = cpi->segment_feature_data[MB_LVL_ALT_Q][0]; diff --git a/vp8/encoder/ratectrl.c b/vp8/encoder/ratectrl.c index c51650c3c..e30ad9e28 100644 --- a/vp8/encoder/ratectrl.c +++ b/vp8/encoder/ratectrl.c @@ -708,7 +708,13 @@ static void calc_pframe_target_size(VP8_COMP *cpi) Adjustment = (cpi->this_frame_target - min_frame_target); if (cpi->frames_since_golden == (cpi->current_gf_interval >> 1)) - cpi->this_frame_target += ((cpi->current_gf_interval - 1) * Adjustment); + { + Adjustment = (cpi->current_gf_interval - 1) * Adjustment; + // Limit adjustment to 10% of current target. + if (Adjustment > (10 * cpi->this_frame_target) / 100) + Adjustment = (10 * cpi->this_frame_target) / 100; + cpi->this_frame_target += Adjustment; + } else cpi->this_frame_target -= Adjustment; } diff --git a/vp8/vp8_cx_iface.c b/vp8/vp8_cx_iface.c index b1b079cb2..3426f5986 100644 --- a/vp8/vp8_cx_iface.c +++ b/vp8/vp8_cx_iface.c @@ -37,6 +37,7 @@ struct vp8_extracfg vp8e_tuning tuning; unsigned int cq_level; /* constrained quality level */ unsigned int rc_max_intra_bitrate_pct; + unsigned int screen_content_mode; }; @@ -62,6 +63,7 @@ static struct vp8_extracfg default_extracfg = { 0, /* tuning*/ 10, /* cq_level */ 0, /* rc_max_intra_bitrate_pct */ + 0, /* screen_content_mode */ }; struct vpx_codec_alg_priv @@ -79,6 +81,7 @@ struct vpx_codec_alg_priv /* pkt_list size depends on the maximum number of lagged frames allowed. */ vpx_codec_pkt_list_decl(64) pkt_list; unsigned int fixed_kf_cntr; + vpx_enc_frame_flags_t control_frame_flags; }; @@ -194,6 +197,7 @@ static vpx_codec_err_t validate_config(vpx_codec_alg_priv_t *ctx, RANGE_CHECK_HI(vp8_cfg, arnr_strength, 6); RANGE_CHECK(vp8_cfg, arnr_type, 1, 3); RANGE_CHECK(vp8_cfg, cq_level, 0, 63); + RANGE_CHECK_BOOL(vp8_cfg, screen_content_mode); if (finalize && (cfg->rc_end_usage == VPX_CQ || cfg->rc_end_usage == VPX_Q)) RANGE_CHECK(vp8_cfg, cq_level, cfg->rc_min_quantizer, cfg->rc_max_quantizer); @@ -231,7 +235,8 @@ static vpx_codec_err_t validate_config(vpx_codec_alg_priv_t *ctx, RANGE_CHECK_HI(cfg, ts_periodicity, 16); for (i=1; i<cfg->ts_number_layers; i++) - if (cfg->ts_target_bitrate[i] <= cfg->ts_target_bitrate[i-1]) + if (cfg->ts_target_bitrate[i] <= cfg->ts_target_bitrate[i-1] && + cfg->rc_target_bitrate > 0) ERROR("ts_target_bitrate entries are not strictly increasing"); RANGE_CHECK(cfg, ts_rate_decimator[cfg->ts_number_layers-1], 1, 1); @@ -397,6 +402,8 @@ static vpx_codec_err_t set_vp8e_config(VP8_CONFIG *oxcf, oxcf->tuning = vp8_cfg.tuning; + oxcf->screen_content_mode = vp8_cfg.screen_content_mode; + /* printf("Current VP8 Settings: \n"); printf("target_bandwidth: %d\n", oxcf->target_bandwidth); @@ -586,6 +593,15 @@ static vpx_codec_err_t set_rc_max_intra_bitrate_pct(vpx_codec_alg_priv_t *ctx, return update_extracfg(ctx, &extra_cfg); } +static vpx_codec_err_t set_screen_content_mode(vpx_codec_alg_priv_t *ctx, + va_list args) +{ + struct vp8_extracfg extra_cfg = ctx->vp8_cfg; + extra_cfg.screen_content_mode = + CAST(VP8E_SET_SCREEN_CONTENT_MODE, args); + return update_extracfg(ctx, &extra_cfg); +} + static vpx_codec_err_t vp8e_mr_alloc_mem(const vpx_codec_enc_cfg_t *cfg, void **mem_loc) { @@ -768,27 +784,9 @@ static void pick_quickcompress_mode(vpx_codec_alg_priv_t *ctx, } } - -static vpx_codec_err_t vp8e_encode(vpx_codec_alg_priv_t *ctx, - const vpx_image_t *img, - vpx_codec_pts_t pts, - unsigned long duration, - vpx_enc_frame_flags_t flags, - unsigned long deadline) +static vpx_codec_err_t set_reference_and_update(vpx_codec_alg_priv_t *ctx, + int flags) { - vpx_codec_err_t res = VPX_CODEC_OK; - - if (!ctx->cfg.rc_target_bitrate) - return res; - - if (img) - res = validate_img(ctx, img); - - if (!res) - res = validate_config(ctx, &ctx->cfg, &ctx->vp8_cfg, 1); - - pick_quickcompress_mode(ctx, duration, deadline); - vpx_codec_pkt_list_init(&ctx->pkt_list); /* Handle Flags */ if (((flags & VP8_EFLAG_NO_UPD_GF) && (flags & VP8_EFLAG_FORCE_GF)) @@ -838,6 +836,42 @@ static vpx_codec_err_t vp8e_encode(vpx_codec_alg_priv_t *ctx, vp8_update_entropy(ctx->cpi, 0); } + return VPX_CODEC_OK; +} + +static vpx_codec_err_t vp8e_encode(vpx_codec_alg_priv_t *ctx, + const vpx_image_t *img, + vpx_codec_pts_t pts, + unsigned long duration, + vpx_enc_frame_flags_t flags, + unsigned long deadline) +{ + vpx_codec_err_t res = VPX_CODEC_OK; + + if (!ctx->cfg.rc_target_bitrate) + return res; + + if (!ctx->cfg.rc_target_bitrate) + return res; + + if (img) + res = validate_img(ctx, img); + + if (!res) + res = validate_config(ctx, &ctx->cfg, &ctx->vp8_cfg, 1); + + pick_quickcompress_mode(ctx, duration, deadline); + vpx_codec_pkt_list_init(&ctx->pkt_list); + + // If no flags are set in the encode call, then use the frame flags as + // defined via the control function: vp8e_set_frame_flags. + if (!flags) { + flags = ctx->control_frame_flags; + } + ctx->control_frame_flags = 0; + + res = set_reference_and_update(ctx, flags); + /* Handle fixed keyframe intervals */ if (ctx->cfg.kf_mode == VPX_KF_AUTO && ctx->cfg.kf_min_dist == ctx->cfg.kf_max_dist) @@ -1140,6 +1174,25 @@ static vpx_codec_err_t vp8e_use_reference(vpx_codec_alg_priv_t *ctx, return VPX_CODEC_OK; } +static vpx_codec_err_t vp8e_set_frame_flags(vpx_codec_alg_priv_t *ctx, + va_list args) +{ + int frame_flags = va_arg(args, int); + ctx->control_frame_flags = frame_flags; + return set_reference_and_update(ctx, frame_flags); +} + +static vpx_codec_err_t vp8e_set_temporal_layer_id(vpx_codec_alg_priv_t *ctx, + va_list args) +{ + int layer_id = va_arg(args, int); + if (layer_id < 0 || layer_id >= (int)ctx->cfg.ts_number_layers) { + return VPX_CODEC_INVALID_PARAM; + } + ctx->cpi->temporal_layer_id = layer_id; + return VPX_CODEC_OK; +} + static vpx_codec_err_t vp8e_set_roi_map(vpx_codec_alg_priv_t *ctx, va_list args) { @@ -1214,6 +1267,8 @@ static vpx_codec_ctrl_fn_map_t vp8e_ctf_maps[] = {VP8E_UPD_ENTROPY, vp8e_update_entropy}, {VP8E_UPD_REFERENCE, vp8e_update_reference}, {VP8E_USE_REFERENCE, vp8e_use_reference}, + {VP8E_SET_FRAME_FLAGS, vp8e_set_frame_flags}, + {VP8E_SET_TEMPORAL_LAYER_ID, vp8e_set_temporal_layer_id}, {VP8E_SET_ROI_MAP, vp8e_set_roi_map}, {VP8E_SET_ACTIVEMAP, vp8e_set_activemap}, {VP8E_SET_SCALEMODE, vp8e_set_scalemode}, @@ -1231,6 +1286,7 @@ static vpx_codec_ctrl_fn_map_t vp8e_ctf_maps[] = {VP8E_SET_TUNING, set_tuning}, {VP8E_SET_CQ_LEVEL, set_cq_level}, {VP8E_SET_MAX_INTRA_BITRATE_PCT, set_rc_max_intra_bitrate_pct}, + {VP8E_SET_SCREEN_CONTENT_MODE, set_screen_content_mode}, { -1, NULL}, }; diff --git a/vpx/vp8cx.h b/vpx/vp8cx.h index 0a45dbb16..4fc0fd62f 100644 --- a/vpx/vp8cx.h +++ b/vpx/vp8cx.h @@ -193,6 +193,7 @@ enum vp8e_enc_control_id { * */ VP8E_SET_MAX_INTRA_BITRATE_PCT, + VP8E_SET_FRAME_FLAGS, /**< control function to set reference and update frame flags */ /*!\brief Max data rate for Inter frames * @@ -222,6 +223,17 @@ enum vp8e_enc_control_id { */ VP8E_SET_GF_CBR_BOOST_PCT, + /*!\brief Codec control function to set the temporal layer id + * + * For temporal scalability: this control allows the application to set the + * layer id for each frame to be encoded. Note that this control must be set + * for every frame prior to encoding. The usage of this control function + * supersedes the internal temporal pattern counter, which is now deprecated. + */ + VP8E_SET_TEMPORAL_LAYER_ID, + + VP8E_SET_SCREEN_CONTENT_MODE, /**<control function to set encoder screen content mode */ + /* TODO(jkoleszar): Move to vp9cx.h */ VP9E_SET_LOSSLESS, VP9E_SET_TILE_COLUMNS, @@ -362,6 +374,8 @@ VPX_CTRL_USE_TYPE_DEPRECATED(VP8E_UPD_ENTROPY, int) VPX_CTRL_USE_TYPE_DEPRECATED(VP8E_UPD_REFERENCE, int) VPX_CTRL_USE_TYPE_DEPRECATED(VP8E_USE_REFERENCE, int) +VPX_CTRL_USE_TYPE(VP8E_SET_FRAME_FLAGS, int) +VPX_CTRL_USE_TYPE(VP8E_SET_TEMPORAL_LAYER_ID, int) VPX_CTRL_USE_TYPE(VP8E_SET_ROI_MAP, vpx_roi_map_t *) VPX_CTRL_USE_TYPE(VP8E_SET_ACTIVEMAP, vpx_active_map_t *) VPX_CTRL_USE_TYPE(VP8E_SET_SCALEMODE, vpx_scaling_mode_t *) @@ -395,6 +409,9 @@ VPX_CTRL_USE_TYPE(VP8E_SET_MAX_INTRA_BITRATE_PCT, unsigned int) VPX_CTRL_USE_TYPE(VP8E_SET_MAX_INTER_BITRATE_PCT, unsigned int) VPX_CTRL_USE_TYPE(VP8E_SET_GF_CBR_BOOST_PCT, unsigned int) + +VPX_CTRL_USE_TYPE(VP8E_SET_SCREEN_CONTENT_MODE, unsigned int) + VPX_CTRL_USE_TYPE(VP9E_SET_LOSSLESS, unsigned int) VPX_CTRL_USE_TYPE(VP9E_SET_FRAME_PARALLEL_DECODING, unsigned int) @@ -354,13 +354,16 @@ static const arg_def_t max_inter_rate_pct = ARG_DEF( static const arg_def_t gf_cbr_boost_pct = ARG_DEF( NULL, "gf-cbr-boost", 1, "Boost for Golden Frame in CBR mode (pct)"); +static const arg_def_t screen_content_mode = ARG_DEF(NULL, "screen-content-mode", 1, + "Screen content mode"); + #if CONFIG_VP8_ENCODER static const arg_def_t token_parts = ARG_DEF( NULL, "token-parts", 1, "Number of token partitions to use, log2"); static const arg_def_t *vp8_args[] = { &cpu_used, &auto_altref, &noise_sens, &sharpness, &static_thresh, &token_parts, &arnr_maxframes, &arnr_strength, &arnr_type, - &tune_ssim, &cq_level, &max_intra_rate_pct, + &tune_ssim, &cq_level, &max_intra_rate_pct, &screen_content_mode, NULL }; static const int vp8_arg_ctrl_map[] = { @@ -369,6 +372,7 @@ static const int vp8_arg_ctrl_map[] = { VP8E_SET_TOKEN_PARTITIONS, VP8E_SET_ARNR_MAXFRAMES, VP8E_SET_ARNR_STRENGTH, VP8E_SET_ARNR_TYPE, VP8E_SET_TUNING, VP8E_SET_CQ_LEVEL, VP8E_SET_MAX_INTRA_BITRATE_PCT, + VP8E_SET_SCREEN_CONTENT_MODE, 0 }; #endif |