Imported Upstream version 1.4+222+hg5f9f7194267b
[deb_x265.git] / source / common / quant.h
1 /*****************************************************************************
2 * Copyright (C) 2014 x265 project
3 *
4 * Authors: Steve Borho <steve@borho.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.
19 *
20 * This program is also available under a commercial proprietary license.
21 * For more information, contact us at license @ x265.com.
22 *****************************************************************************/
23
24 #ifndef X265_QUANT_H
25 #define X265_QUANT_H
26
27 #include "common.h"
28 #include "scalinglist.h"
29 #include "contexts.h"
30
31 namespace x265 {
32 // private namespace
33
34 class CUData;
35 class Entropy;
36 struct TUEntropyCodingParameters;
37
38 struct QpParam
39 {
40 int rem;
41 int per;
42 int qp;
43 int64_t lambda2; /* FIX8 */
44 int64_t lambda; /* FIX8 */
45
46 QpParam() : qp(MAX_INT) {}
47
48 void setQpParam(int qpScaled)
49 {
50 if (qp != qpScaled)
51 {
52 rem = qpScaled % 6;
53 per = qpScaled / 6;
54 qp = qpScaled;
55 lambda2 = (int64_t)(x265_lambda2_tab[qp - QP_BD_OFFSET] * 256. + 0.5);
56 lambda = (int64_t)(x265_lambda_tab[qp - QP_BD_OFFSET] * 256. + 0.5);
57 }
58 }
59 };
60
61 #define MAX_NUM_TR_COEFFS MAX_TR_SIZE * MAX_TR_SIZE /* Maximum number of transform coefficients, for a 32x32 transform */
62 #define MAX_NUM_TR_CATEGORIES 16 /* 32, 16, 8, 4 transform categories each for luma and chroma */
63
64 // NOTE: MUST be 16-byte aligned for asm code
65 struct NoiseReduction
66 {
67 /* 0 = luma 4x4, 1 = luma 8x8, 2 = luma 16x16, 3 = luma 32x32
68 * 4 = chroma 4x4, 5 = chroma 8x8, 6 = chroma 16x16, 7 = chroma 32x32
69 * Intra 0..7 - Inter 8..15 */
70 uint16_t offsetDenoise[MAX_NUM_TR_CATEGORIES][MAX_NUM_TR_COEFFS];
71 uint32_t residualSum[MAX_NUM_TR_CATEGORIES][MAX_NUM_TR_COEFFS];
72 uint32_t count[MAX_NUM_TR_CATEGORIES];
73 };
74
75 class Quant
76 {
77 protected:
78
79 const ScalingList* m_scalingList;
80 Entropy* m_entropyCoder;
81
82 QpParam m_qpParam[3];
83
84 bool m_useRDOQ;
85 int64_t m_psyRdoqScale;
86 int16_t* m_resiDctCoeff;
87 int16_t* m_fencDctCoeff;
88 int16_t* m_fencShortBuf;
89
90 enum { IEP_RATE = 32768 }; /* FIX15 cost of an equal probable bit */
91
92 public:
93
94 NoiseReduction* m_nr;
95 NoiseReduction* m_frameNr; // Array of NR structures, one for each frameEncoder
96
97 Quant();
98 ~Quant();
99
100 /* one-time setup */
101 bool init(bool useRDOQ, double psyScale, const ScalingList& scalingList, Entropy& entropy);
102 bool allocNoiseReduction(const x265_param& param);
103
104 /* CU setup */
105 void setQPforQuant(const CUData& ctu);
106
107 uint32_t transformNxN(const CUData& cu, const pixel* fenc, uint32_t fencStride, const int16_t* residual, uint32_t resiStride, coeff_t* coeff,
108 uint32_t log2TrSize, TextType ttype, uint32_t absPartIdx, bool useTransformSkip);
109
110 void invtransformNxN(bool transQuantBypass, int16_t* residual, uint32_t resiStride, const coeff_t* coeff,
111 uint32_t log2TrSize, TextType ttype, bool bIntra, bool useTransformSkip, uint32_t numSig);
112
113 /* static methods shared with entropy.cpp */
114 static uint32_t calcPatternSigCtx(uint64_t sigCoeffGroupFlag64, uint32_t cgPosX, uint32_t cgPosY, uint32_t log2TrSizeCG);
115 static uint32_t getSigCtxInc(uint32_t patternSigCtx, uint32_t log2TrSize, uint32_t trSize, uint32_t blkPos, bool bIsLuma, uint32_t firstSignificanceMapContext);
116 static uint32_t getSigCoeffGroupCtxInc(uint64_t sigCoeffGroupFlag64, uint32_t cgPosX, uint32_t cgPosY, uint32_t log2TrSizeCG);
117
118 protected:
119
120 void setChromaQP(int qpin, TextType ttype, int chFmt);
121
122 uint32_t signBitHidingHDQ(int16_t* qcoeff, int32_t* deltaU, uint32_t numSig, const TUEntropyCodingParameters &codingParameters);
123
124 uint32_t rdoQuant(const CUData& cu, int16_t* dstCoeff, uint32_t log2TrSize, TextType ttype, uint32_t absPartIdx, bool usePsy);
125 inline uint32_t getRateLast(uint32_t posx, uint32_t posy) const;
126 };
127
128 static inline uint32_t getGroupIdx(const uint32_t idx)
129 {
130 // TODO: Why is this not a table lookup?
131
132 uint32_t group = (idx >> 3);
133
134 if (idx >= 24)
135 group = 2;
136 uint32_t groupIdx = ((idx >> (group + 1)) - 2) + 4 + (group << 1);
137 if (idx <= 3)
138 groupIdx = idx;
139
140 #ifdef _DEBUG
141 static const uint8_t g_groupIdx[32] = { 0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9 };
142 assert(groupIdx == g_groupIdx[idx]);
143 #endif
144
145 return groupIdx;
146 }
147
148 }
149
150 #endif // ifndef X265_QUANT_H