AOMedia AV1 Codec
tpl_model.h
1 /*
2  * Copyright (c) 2019, Alliance for Open Media. All rights reserved
3  *
4  * This source code is subject to the terms of the BSD 2 Clause License and
5  * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6  * was not distributed with this source code in the LICENSE file, you can
7  * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8  * Media Patent License 1.0 was not distributed with this source code in the
9  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10  */
11 
12 #ifndef AOM_AV1_ENCODER_TPL_MODEL_H_
13 #define AOM_AV1_ENCODER_TPL_MODEL_H_
14 
15 #ifdef __cplusplus
16 extern "C" {
17 #endif
18 
21 struct AV1_PRIMARY;
22 struct AV1_COMP;
23 struct AV1_SEQ_CODING_TOOLS;
24 struct EncodeFrameParams;
25 struct EncodeFrameInput;
26 struct GF_GROUP;
27 struct TPL_INFO;
28 
29 #include "config/aom_config.h"
30 
31 #include "aom_scale/yv12config.h"
32 
33 #include "av1/common/mv.h"
34 #include "av1/common/scale.h"
35 #include "av1/encoder/block.h"
36 #include "av1/encoder/lookahead.h"
37 #include "av1/encoder/ratectrl.h"
38 
39 static INLINE BLOCK_SIZE convert_length_to_bsize(int length) {
40  switch (length) {
41  case 64: return BLOCK_64X64;
42  case 32: return BLOCK_32X32;
43  case 16: return BLOCK_16X16;
44  case 8: return BLOCK_8X8;
45  case 4: return BLOCK_4X4;
46  default:
47  assert(0 && "Invalid block size for tpl model");
48  return BLOCK_16X16;
49  }
50 }
51 
52 typedef struct AV1TplRowMultiThreadSync {
53 #if CONFIG_MULTITHREAD
54  // Synchronization objects for top-right dependency.
55  pthread_mutex_t *mutex_;
56  pthread_cond_t *cond_;
57 #endif
58  // Buffer to store the macroblock whose encoding is complete.
59  // num_finished_cols[i] stores the number of macroblocks which finished
60  // encoding in the ith macroblock row.
61  int *num_finished_cols;
62  // Number of extra macroblocks of the top row to be complete for encoding
63  // of the current macroblock to start. A value of 1 indicates top-right
64  // dependency.
65  int sync_range;
66  // Number of macroblock rows.
67  int rows;
68  // Number of threads processing the current tile.
69  int num_threads_working;
70 } AV1TplRowMultiThreadSync;
71 
72 typedef struct AV1TplRowMultiThreadInfo {
73  // Row synchronization related function pointers.
74  void (*sync_read_ptr)(AV1TplRowMultiThreadSync *tpl_mt_sync, int r, int c);
75  void (*sync_write_ptr)(AV1TplRowMultiThreadSync *tpl_mt_sync, int r, int c,
76  int cols);
77 } AV1TplRowMultiThreadInfo;
78 
79 // TODO(jingning): This needs to be cleaned up next.
80 
81 // TPL stats buffers are prepared for every frame in the GOP,
82 // including (internal) overlays and (internal) arfs.
83 // In addition, frames in the lookahead that are outside of the GOP
84 // are also used.
85 // Thus it should use
86 // (gop_length) + (# overlays) + (MAX_LAG_BUFFERS - gop_len) =
87 // MAX_LAG_BUFFERS + (# overlays)
88 // 2 * MAX_LAG_BUFFERS is therefore a safe estimate.
89 // TODO(bohanli): test setting it to 1.5 * MAX_LAG_BUFFER
90 #define MAX_TPL_FRAME_IDX (2 * MAX_LAG_BUFFERS)
91 // The first REF_FRAMES + 1 buffers are reserved.
92 // tpl_data->tpl_frame starts after REF_FRAMES + 1
93 #define MAX_LENGTH_TPL_FRAME_STATS (MAX_TPL_FRAME_IDX + REF_FRAMES + 1)
94 #define TPL_DEP_COST_SCALE_LOG2 4
95 
96 #define TPL_EPSILON 0.0000001
97 
98 typedef struct TplTxfmStats {
99  int ready; // Whether abs_coeff_mean is ready
100  double abs_coeff_sum[256]; // Assume we are using 16x16 transform block
101  double abs_coeff_mean[256];
102  int txfm_block_count;
103  int coeff_num;
104 } TplTxfmStats;
105 
106 typedef struct TplDepStats {
107  int64_t intra_cost;
108  int64_t inter_cost;
109  int64_t srcrf_dist;
110  int64_t recrf_dist;
111  int64_t cmp_recrf_dist[2];
112  int64_t srcrf_rate;
113  int64_t recrf_rate;
114  int64_t srcrf_sse;
115  int64_t cmp_recrf_rate[2];
116  int64_t mc_dep_rate;
117  int64_t mc_dep_dist;
118  int_mv mv[INTER_REFS_PER_FRAME];
119  int ref_frame_index[2];
120  int64_t pred_error[INTER_REFS_PER_FRAME];
121 } TplDepStats;
122 
123 typedef struct TplDepFrame {
124  uint8_t is_valid;
125  TplDepStats *tpl_stats_ptr;
126  const YV12_BUFFER_CONFIG *gf_picture;
127  YV12_BUFFER_CONFIG *rec_picture;
128  int ref_map_index[REF_FRAMES];
129  int stride;
130  int width;
131  int height;
132  int mi_rows;
133  int mi_cols;
134  int base_rdmult;
135  uint32_t frame_display_index;
136 } TplDepFrame;
137 
142 typedef struct TplParams {
146  int ready;
147 
152 
156  uint8_t tpl_bsize_1d;
157 
163  TplDepFrame tpl_stats_buffer[MAX_LENGTH_TPL_FRAME_STATS];
164 
170  TplDepStats *tpl_stats_pool[MAX_LAG_BUFFERS];
171 
178  TplTxfmStats *txfm_stats_list;
179 
185 
189  TplDepFrame *tpl_frame;
190 
194  struct scale_factors sf;
195 
200 
206  const YV12_BUFFER_CONFIG *src_ref_frame[INTER_REFS_PER_FRAME];
207 
213  const YV12_BUFFER_CONFIG *ref_frame[INTER_REFS_PER_FRAME];
214 
219  AV1TplRowMultiThreadSync tpl_mt_sync;
220 
225 
226 } TplParams;
227 
228 #if CONFIG_BITRATE_ACCURACY || CONFIG_RATECTRL_LOG
229 #define VBR_RC_INFO_MAX_FRAMES 500
230 #endif // CONFIG_BITRATE_ACCURACY || CONFIG_RATECTRL_LOG
231 
232 #if CONFIG_BITRATE_ACCURACY
233 
238 typedef struct {
239  int ready;
240  double total_bit_budget; // The total bit budget of the entire video
241  int show_frame_count; // Number of show frames in the entire video
242 
243  int gop_showframe_count; // The number of show frames in the current gop
244  double gop_bit_budget; // The bitbudget for the current gop
245  double scale_factors[FRAME_UPDATE_TYPES]; // Scale factors to improve the
246  // budget estimation
247  double mv_scale_factors[FRAME_UPDATE_TYPES]; // Scale factors to improve
248  // MV entropy estimation
249 
250  // === Below this line are GOP related data that will be updated per GOP ===
251  int base_q_index; // Stores the base q index.
252  int q_index_list_ready;
253  int q_index_list[VBR_RC_INFO_MAX_FRAMES]; // q indices for the current
254  // GOP
255 
256  // Array to store qstep_ratio for each frame in a GOP
257  double qstep_ratio_list[VBR_RC_INFO_MAX_FRAMES];
258 
259 #if CONFIG_THREE_PASS
260  TplTxfmStats txfm_stats_list[VBR_RC_INFO_MAX_FRAMES];
261  FRAME_UPDATE_TYPE update_type_list[VBR_RC_INFO_MAX_FRAMES];
262  int gop_start_idx_list[VBR_RC_INFO_MAX_FRAMES];
263  int gop_length_list[VBR_RC_INFO_MAX_FRAMES];
264  int cur_gop_idx;
265  int total_frame_count;
266  int gop_count;
267 #endif // CONFIG_THREE_PASS
268 } VBR_RATECTRL_INFO;
269 
270 static INLINE void vbr_rc_reset_gop_data(VBR_RATECTRL_INFO *vbr_rc_info) {
271  vbr_rc_info->q_index_list_ready = 0;
272  av1_zero(vbr_rc_info->q_index_list);
273 }
274 
275 void av1_vbr_rc_init(VBR_RATECTRL_INFO *vbr_rc_info, double total_bit_budget,
276  int show_frame_count);
277 
278 int av1_vbr_rc_frame_coding_idx(const VBR_RATECTRL_INFO *vbr_rc_info,
279  int gf_frame_index);
280 
281 void av1_vbr_rc_append_tpl_info(VBR_RATECTRL_INFO *vbr_rc_info,
282  const struct TPL_INFO *tpl_info);
283 
284 void av1_vbr_rc_set_gop_bit_budget(VBR_RATECTRL_INFO *vbr_rc_info,
285  int gop_showframe_count);
286 
287 void av1_vbr_rc_compute_q_indices(int base_q_index, int frame_count,
288  const double *qstep_ratio_list,
289  aom_bit_depth_t bit_depth, int *q_index_list);
290 
299 void av1_vbr_rc_update_q_index_list(VBR_RATECTRL_INFO *vbr_rc_info,
300  const TplParams *tpl_data,
301  const struct GF_GROUP *gf_group,
302  aom_bit_depth_t bit_depth);
303 /*
304  *!\brief Compute the number of bits needed to encode a GOP
305  *
306  * \param[in] base_q_index base layer q_index
307  * \param[in] bit_depth bit depth
308  * \param[in] update_type_scale_factors array of scale factors for each
309  * update_type
310  * \param[in] frame_count size of update_type_list,
311  * qstep_ratio_list stats_list,
312  * q_index_list and
313  * estimated_bitrate_byframe
314  * \param[in] update_type_list array of update_type, one per frame
315  * \param[in] qstep_ratio_list array of qstep_ratio, one per frame
316  * \param[in] stats_list array of transform stats, one per
317  * frame
318  * \param[out] q_index_list array of q_index, one per frame
319  * \param[out] estimated_bitrate_byframe array to keep track of frame
320  * bitrate
321  *
322  * \return The estimated GOP bitrate.
323  *
324  */
325 double av1_vbr_rc_info_estimate_gop_bitrate(
326  int base_q_index, aom_bit_depth_t bit_depth,
327  const double *update_type_scale_factors, int frame_count,
328  const FRAME_UPDATE_TYPE *update_type_list, const double *qstep_ratio_list,
329  const TplTxfmStats *stats_list, int *q_index_list,
330  double *estimated_bitrate_byframe);
331 
353 int av1_vbr_rc_info_estimate_base_q(
354  double bit_budget, aom_bit_depth_t bit_depth,
355  const double *update_type_scale_factors, int frame_count,
356  const FRAME_UPDATE_TYPE *update_type_list, const double *qstep_ratio_list,
357  const TplTxfmStats *stats_list, int *q_index_list,
358  double *estimated_bitrate_byframe);
359 
360 #endif // CONFIG_BITRATE_ACCURACY
361 
362 #if CONFIG_RD_COMMAND
363 typedef enum {
364  RD_OPTION_NONE,
365  RD_OPTION_SET_Q,
366  RD_OPTION_SET_Q_RDMULT
367 } RD_OPTION;
368 
369 typedef struct RD_COMMAND {
370  RD_OPTION option_ls[MAX_LENGTH_TPL_FRAME_STATS];
371  int q_index_ls[MAX_LENGTH_TPL_FRAME_STATS];
372  int rdmult_ls[MAX_LENGTH_TPL_FRAME_STATS];
373  int frame_count;
374  int frame_index;
375 } RD_COMMAND;
376 
377 void av1_read_rd_command(const char *filepath, RD_COMMAND *rd_command);
378 #endif // CONFIG_RD_COMMAND
379 
388 void av1_setup_tpl_buffers(struct AV1_PRIMARY *const ppi,
389  CommonModeInfoParams *const mi_params, int width,
390  int height, int byte_alignment, int lag_in_frames);
391 
403 int av1_tpl_setup_stats(struct AV1_COMP *cpi, int gop_eval,
404  const struct EncodeFrameParams *const frame_params);
405 
408 void av1_tpl_preload_rc_estimate(
409  struct AV1_COMP *cpi, const struct EncodeFrameParams *const frame_params);
410 
411 int av1_tpl_ptr_pos(int mi_row, int mi_col, int stride, uint8_t right_shift);
412 
413 void av1_init_tpl_stats(TplParams *const tpl_data);
414 
415 int av1_tpl_stats_ready(const TplParams *tpl_data, int gf_frame_index);
416 
417 void av1_tpl_rdmult_setup(struct AV1_COMP *cpi);
418 
419 void av1_tpl_rdmult_setup_sb(struct AV1_COMP *cpi, MACROBLOCK *const x,
420  BLOCK_SIZE sb_size, int mi_row, int mi_col);
421 
422 void av1_mc_flow_dispenser_row(struct AV1_COMP *cpi,
423  TplTxfmStats *tpl_txfm_stats, MACROBLOCK *x,
424  int mi_row, BLOCK_SIZE bsize, TX_SIZE tx_size);
425 
438 double av1_exponential_entropy(double q_step, double b);
439 
453 double av1_laplace_entropy(double q_step, double b, double zero_bin_ratio);
454 
472 double av1_laplace_estimate_frame_rate(int q_index, int block_count,
473  const double *abs_coeff_mean,
474  int coeff_num);
475 
476 /*
477  *!\brief Init TplTxfmStats
478  *
479  * \param[in] tpl_txfm_stats a structure for storing transform stats
480  *
481  */
482 void av1_init_tpl_txfm_stats(TplTxfmStats *tpl_txfm_stats);
483 
484 /*
485  *!\brief Accumulate TplTxfmStats
486  *
487  * \param[in] sub_stats a structure for storing sub transform stats
488  * \param[out] accumulated_stats a structure for storing accumulated
489  *transform stats
490  *
491  */
492 void av1_accumulate_tpl_txfm_stats(const TplTxfmStats *sub_stats,
493  TplTxfmStats *accumulated_stats);
494 
495 /*
496  *!\brief Record a transform block into TplTxfmStats
497  *
498  * \param[in] tpl_txfm_stats A structure for storing transform stats
499  * \param[out] coeff An array of transform coefficients. Its size
500  * should equal to tpl_txfm_stats.coeff_num.
501  *
502  */
503 void av1_record_tpl_txfm_block(TplTxfmStats *tpl_txfm_stats,
504  const tran_low_t *coeff);
505 
506 /*
507  *!\brief Update abs_coeff_mean and ready of txfm_stats
508  * If txfm_block_count > 0, this function will use abs_coeff_sum and
509  * txfm_block_count to compute abs_coeff_mean. Moreover, reday flag
510  * will be set to one.
511  *
512  * \param[in] txfm_stats A structure for storing transform stats
513  */
514 void av1_tpl_txfm_stats_update_abs_coeff_mean(TplTxfmStats *txfm_stats);
515 
531 double av1_estimate_coeff_entropy(double q_step, double b,
532  double zero_bin_ratio, int qcoeff);
533 
546 double av1_estimate_txfm_block_entropy(int q_index,
547  const double *abs_coeff_mean,
548  int *qcoeff_arr, int coeff_num);
549 
550 // TODO(angiebird): Add doxygen description here.
551 int64_t av1_delta_rate_cost(int64_t delta_rate, int64_t recrf_dist,
552  int64_t srcrf_dist, int pix_num);
553 
569 int av1_get_overlap_area(int row_a, int col_a, int row_b, int col_b, int width,
570  int height);
571 
581 int av1_tpl_get_q_index(const TplParams *tpl_data, int gf_frame_index,
582  int leaf_qindex, aom_bit_depth_t bit_depth);
583 
591 double av1_tpl_get_frame_importance(const TplParams *tpl_data,
592  int gf_frame_index);
593 
604 double av1_tpl_get_qstep_ratio(const TplParams *tpl_data, int gf_frame_index);
605 
614 int av1_get_q_index_from_qstep_ratio(int leaf_qindex, double qstep_ratio,
615  aom_bit_depth_t bit_depth);
616 
631 int_mv av1_compute_mv_difference(const TplDepFrame *tpl_frame, int row, int col,
632  int step, int tpl_stride, int right_shift);
633 
641 double av1_tpl_compute_frame_mv_entropy(const TplDepFrame *tpl_frame,
642  uint8_t right_shift);
643 
644 #if CONFIG_RATECTRL_LOG
645 typedef struct {
646  int coding_frame_count;
647  int base_q_index;
648 
649  // Encode decision
650  int q_index_list[VBR_RC_INFO_MAX_FRAMES];
651  double qstep_ratio_list[VBR_RC_INFO_MAX_FRAMES];
652  FRAME_UPDATE_TYPE update_type_list[VBR_RC_INFO_MAX_FRAMES];
653 
654  // Frame stats
655  TplTxfmStats txfm_stats_list[VBR_RC_INFO_MAX_FRAMES];
656 
657  // Estimated encode results
658  double est_coeff_rate_list[VBR_RC_INFO_MAX_FRAMES];
659 
660  // Actual encode results
661  double act_rate_list[VBR_RC_INFO_MAX_FRAMES];
662  double act_coeff_rate_list[VBR_RC_INFO_MAX_FRAMES];
663 } RATECTRL_LOG;
664 
665 static INLINE void rc_log_init(RATECTRL_LOG *rc_log) { av1_zero(*rc_log); }
666 
667 static INLINE void rc_log_frame_stats(RATECTRL_LOG *rc_log, int coding_index,
668  const TplTxfmStats *txfm_stats) {
669  rc_log->txfm_stats_list[coding_index] = *txfm_stats;
670 }
671 
672 static INLINE void rc_log_frame_encode_param(RATECTRL_LOG *rc_log,
673  int coding_index,
674  double qstep_ratio, int q_index,
675  FRAME_UPDATE_TYPE update_type) {
676  rc_log->qstep_ratio_list[coding_index] = qstep_ratio;
677  rc_log->q_index_list[coding_index] = q_index;
678  rc_log->update_type_list[coding_index] = update_type;
679  const TplTxfmStats *txfm_stats = &rc_log->txfm_stats_list[coding_index];
680  rc_log->est_coeff_rate_list[coding_index] = 0;
681  if (txfm_stats->ready) {
682  rc_log->est_coeff_rate_list[coding_index] = av1_laplace_estimate_frame_rate(
683  q_index, txfm_stats->txfm_block_count, txfm_stats->abs_coeff_mean,
684  txfm_stats->coeff_num);
685  }
686 }
687 
688 static INLINE void rc_log_frame_entropy(RATECTRL_LOG *rc_log, int coding_index,
689  double act_rate,
690  double act_coeff_rate) {
691  rc_log->act_rate_list[coding_index] = act_rate;
692  rc_log->act_coeff_rate_list[coding_index] = act_coeff_rate;
693 }
694 
695 static INLINE void rc_log_record_chunk_info(RATECTRL_LOG *rc_log,
696  int base_q_index,
697  int coding_frame_count) {
698  rc_log->base_q_index = base_q_index;
699  rc_log->coding_frame_count = coding_frame_count;
700 }
701 
702 static INLINE void rc_log_show(const RATECTRL_LOG *rc_log) {
703  printf("= chunk 1\n");
704  printf("coding_frame_count %d base_q_index %d\n", rc_log->coding_frame_count,
705  rc_log->base_q_index);
706  printf("= frame %d\n", rc_log->coding_frame_count);
707  for (int coding_idx = 0; coding_idx < rc_log->coding_frame_count;
708  coding_idx++) {
709  printf(
710  "coding_idx %d update_type %d q %d qstep_ratio %f est_coeff_rate %f "
711  "act_coeff_rate %f act_rate %f\n",
712  coding_idx, rc_log->update_type_list[coding_idx],
713  rc_log->q_index_list[coding_idx], rc_log->qstep_ratio_list[coding_idx],
714  rc_log->est_coeff_rate_list[coding_idx],
715  rc_log->act_coeff_rate_list[coding_idx],
716  rc_log->act_rate_list[coding_idx]);
717  }
718 }
719 #endif // CONFIG_RATECTRL_LOG
720 
722 #ifdef __cplusplus
723 } // extern "C"
724 #endif
725 
726 #endif // AOM_AV1_ENCODER_TPL_MODEL_H_
enum aom_bit_depth aom_bit_depth_t
Bit depth for codecThis enumeration determines the bit depth of the codec.
int av1_tpl_setup_stats(struct AV1_COMP *cpi, int gop_eval, const struct EncodeFrameParams *const frame_params)
Implements temporal dependency modelling for a GOP (GF/ARF group) and selects between 16 and 32 frame...
Describes look ahead buffer operations.
Top level encoder structure.
Definition: encoder.h:2664
Top level primary encoder structure.
Definition: encoder.h:2373
Params related to MB_MODE_INFO arrays and related info.
Definition: av1_common_int.h:501
Input frames and last input frame.
Definition: encoder.h:3344
contains per-frame encoding parameters decided upon by av1_encode_strategy() and passed down to av1_e...
Definition: encoder.h:3356
Data related to the current GF/ARF group and the individual frames within the group.
Definition: firstpass.h:344
Params related to temporal dependency model.
Definition: tpl_model.h:142
const YV12_BUFFER_CONFIG * src_ref_frame[INTER_REFS_PER_FRAME]
Definition: tpl_model.h:206
struct scale_factors sf
Definition: tpl_model.h:194
int ready
Definition: tpl_model.h:146
TplDepFrame tpl_stats_buffer[MAX_LENGTH_TPL_FRAME_STATS]
Definition: tpl_model.h:163
uint8_t tpl_bsize_1d
Definition: tpl_model.h:156
AV1TplRowMultiThreadSync tpl_mt_sync
Definition: tpl_model.h:219
TplDepFrame * tpl_frame
Definition: tpl_model.h:189
int border_in_pixels
Definition: tpl_model.h:224
TplDepStats * tpl_stats_pool[MAX_LAG_BUFFERS]
Definition: tpl_model.h:170
TplTxfmStats * txfm_stats_list
Definition: tpl_model.h:178
YV12_BUFFER_CONFIG tpl_rec_pool[MAX_LAG_BUFFERS]
Definition: tpl_model.h:184
uint8_t tpl_stats_block_mis_log2
Definition: tpl_model.h:151
int frame_idx
Definition: tpl_model.h:199
const YV12_BUFFER_CONFIG * ref_frame[INTER_REFS_PER_FRAME]
Definition: tpl_model.h:213
Encoder's parameters related to the current coding block.
Definition: block.h:813
YV12 frame buffer data structure.
Definition: yv12config.h:39