2 * Copyright (c) 2011 Smartjog S.A.S, Clément Bœsch <clement.boesch@smartjog.com>
4 * This file is part of FFmpeg.
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 * Potential thumbnail lookup filter to reduce the risk of an inappropriate
24 * selection (such as a black frame) we could get with an absolute seek.
26 * Simplified version of algorithm by Vadim Zaliva <lord@crocodile.org>.
27 * @see http://notbrainsurgery.livejournal.com/29773.html
30 #include "libavutil/opt.h"
34 #define HIST_SIZE (3*256)
37 AVFrame
*buf
; ///< cached frame
38 int histogram
[HIST_SIZE
]; ///< RGB color distribution histogram of the frame
43 int n
; ///< current frame
44 int n_frames
; ///< number of frames for analysis
45 struct thumb_frame
*frames
; ///< the n_frames frames
46 AVRational tb
; ///< copy of the input timebase to ease access
49 #define OFFSET(x) offsetof(ThumbContext, x)
50 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
52 static const AVOption thumbnail_options
[] = {
53 { "n", "set the frames batch size", OFFSET(n_frames
), AV_OPT_TYPE_INT
, {.i64
=100}, 2, INT_MAX
, FLAGS
},
57 AVFILTER_DEFINE_CLASS(thumbnail
);
59 static av_cold
int init(AVFilterContext
*ctx
)
61 ThumbContext
*thumb
= ctx
->priv
;
63 thumb
->frames
= av_calloc(thumb
->n_frames
, sizeof(*thumb
->frames
));
65 av_log(ctx
, AV_LOG_ERROR
,
66 "Allocation failure, try to lower the number of frames\n");
67 return AVERROR(ENOMEM
);
69 av_log(ctx
, AV_LOG_VERBOSE
, "batch size: %d frames\n", thumb
->n_frames
);
74 * @brief Compute Sum-square deviation to estimate "closeness".
75 * @param hist color distribution histogram
76 * @param median average color distribution histogram
77 * @return sum of squared errors
79 static double frame_sum_square_err(const int *hist
, const double *median
)
82 double err
, sum_sq_err
= 0;
84 for (i
= 0; i
< HIST_SIZE
; i
++) {
85 err
= median
[i
] - (double)hist
[i
];
86 sum_sq_err
+= err
*err
;
91 static AVFrame
*get_best_frame(AVFilterContext
*ctx
)
94 ThumbContext
*thumb
= ctx
->priv
;
95 int i
, j
, best_frame_idx
= 0;
96 int nb_frames
= thumb
->n
;
97 double avg_hist
[HIST_SIZE
] = {0}, sq_err
, min_sq_err
= -1;
99 // average histogram of the N frames
100 for (j
= 0; j
< FF_ARRAY_ELEMS(avg_hist
); j
++) {
101 for (i
= 0; i
< nb_frames
; i
++)
102 avg_hist
[j
] += (double)thumb
->frames
[i
].histogram
[j
];
103 avg_hist
[j
] /= nb_frames
;
106 // find the frame closer to the average using the sum of squared errors
107 for (i
= 0; i
< nb_frames
; i
++) {
108 sq_err
= frame_sum_square_err(thumb
->frames
[i
].histogram
, avg_hist
);
109 if (i
== 0 || sq_err
< min_sq_err
)
110 best_frame_idx
= i
, min_sq_err
= sq_err
;
113 // free and reset everything (except the best frame buffer)
114 for (i
= 0; i
< nb_frames
; i
++) {
115 memset(thumb
->frames
[i
].histogram
, 0, sizeof(thumb
->frames
[i
].histogram
));
116 if (i
!= best_frame_idx
)
117 av_frame_free(&thumb
->frames
[i
].buf
);
121 // raise the chosen one
122 picref
= thumb
->frames
[best_frame_idx
].buf
;
123 av_log(ctx
, AV_LOG_INFO
, "frame id #%d (pts_time=%f) selected "
124 "from a set of %d images\n", best_frame_idx
,
125 picref
->pts
* av_q2d(thumb
->tb
), nb_frames
);
126 thumb
->frames
[best_frame_idx
].buf
= NULL
;
131 static int filter_frame(AVFilterLink
*inlink
, AVFrame
*frame
)
134 AVFilterContext
*ctx
= inlink
->dst
;
135 ThumbContext
*thumb
= ctx
->priv
;
136 AVFilterLink
*outlink
= ctx
->outputs
[0];
137 int *hist
= thumb
->frames
[thumb
->n
].histogram
;
138 const uint8_t *p
= frame
->data
[0];
140 // keep a reference of each frame
141 thumb
->frames
[thumb
->n
].buf
= frame
;
143 // update current frame RGB histogram
144 for (j
= 0; j
< inlink
->h
; j
++) {
145 for (i
= 0; i
< inlink
->w
; i
++) {
146 hist
[0*256 + p
[i
*3 ]]++;
147 hist
[1*256 + p
[i
*3 + 1]]++;
148 hist
[2*256 + p
[i
*3 + 2]]++;
150 p
+= frame
->linesize
[0];
153 // no selection until the buffer of N frames is filled up
155 if (thumb
->n
< thumb
->n_frames
)
158 return ff_filter_frame(outlink
, get_best_frame(ctx
));
161 static av_cold
void uninit(AVFilterContext
*ctx
)
164 ThumbContext
*thumb
= ctx
->priv
;
165 for (i
= 0; i
< thumb
->n_frames
&& thumb
->frames
[i
].buf
; i
++)
166 av_frame_free(&thumb
->frames
[i
].buf
);
167 av_freep(&thumb
->frames
);
170 static int request_frame(AVFilterLink
*link
)
172 AVFilterContext
*ctx
= link
->src
;
173 ThumbContext
*thumb
= ctx
->priv
;
175 /* loop until a frame thumbnail is available (when a frame is queued,
176 * thumb->n is reset to zero) */
178 int ret
= ff_request_frame(ctx
->inputs
[0]);
179 if (ret
== AVERROR_EOF
&& thumb
->n
) {
180 ret
= ff_filter_frame(link
, get_best_frame(ctx
));
191 static int config_props(AVFilterLink
*inlink
)
193 AVFilterContext
*ctx
= inlink
->dst
;
194 ThumbContext
*thumb
= ctx
->priv
;
196 thumb
->tb
= inlink
->time_base
;
200 static int query_formats(AVFilterContext
*ctx
)
202 static const enum AVPixelFormat pix_fmts
[] = {
203 AV_PIX_FMT_RGB24
, AV_PIX_FMT_BGR24
,
206 ff_set_common_formats(ctx
, ff_make_format_list(pix_fmts
));
210 static const AVFilterPad thumbnail_inputs
[] = {
213 .type
= AVMEDIA_TYPE_VIDEO
,
214 .config_props
= config_props
,
215 .filter_frame
= filter_frame
,
220 static const AVFilterPad thumbnail_outputs
[] = {
223 .type
= AVMEDIA_TYPE_VIDEO
,
224 .request_frame
= request_frame
,
229 AVFilter ff_vf_thumbnail
= {
231 .description
= NULL_IF_CONFIG_SMALL("Select the most representative frame in a given sequence of consecutive frames."),
232 .priv_size
= sizeof(ThumbContext
),
235 .query_formats
= query_formats
,
236 .inputs
= thumbnail_inputs
,
237 .outputs
= thumbnail_outputs
,
238 .priv_class
= &thumbnail_class
,