| 1 | /* |
| 2 | * This file is part of FFmpeg. |
| 3 | * |
| 4 | * FFmpeg is free software; you can redistribute it and/or |
| 5 | * modify it under the terms of the GNU Lesser General Public |
| 6 | * License as published by the Free Software Foundation; either |
| 7 | * version 2.1 of the License, or (at your option) any later version. |
| 8 | * |
| 9 | * FFmpeg is distributed in the hope that it will be useful, |
| 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 12 | * Lesser General Public License for more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU Lesser General Public |
| 15 | * License along with FFmpeg; if not, write to the Free Software |
| 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| 17 | */ |
| 18 | |
| 19 | /** |
| 20 | * @file |
| 21 | * Audio join filter |
| 22 | * |
| 23 | * Join multiple audio inputs as different channels in |
| 24 | * a single output |
| 25 | */ |
| 26 | |
| 27 | #include "libavutil/avassert.h" |
| 28 | #include "libavutil/channel_layout.h" |
| 29 | #include "libavutil/common.h" |
| 30 | #include "libavutil/opt.h" |
| 31 | |
| 32 | #include "audio.h" |
| 33 | #include "avfilter.h" |
| 34 | #include "formats.h" |
| 35 | #include "internal.h" |
| 36 | |
| 37 | typedef struct ChannelMap { |
| 38 | int input; ///< input stream index |
| 39 | int in_channel_idx; ///< index of in_channel in the input stream data |
| 40 | uint64_t in_channel; ///< layout describing the input channel |
| 41 | uint64_t out_channel; ///< layout describing the output channel |
| 42 | } ChannelMap; |
| 43 | |
| 44 | typedef struct JoinContext { |
| 45 | const AVClass *class; |
| 46 | |
| 47 | int inputs; |
| 48 | char *map; |
| 49 | char *channel_layout_str; |
| 50 | uint64_t channel_layout; |
| 51 | |
| 52 | int nb_channels; |
| 53 | ChannelMap *channels; |
| 54 | |
| 55 | /** |
| 56 | * Temporary storage for input frames, until we get one on each input. |
| 57 | */ |
| 58 | AVFrame **input_frames; |
| 59 | |
| 60 | /** |
| 61 | * Temporary storage for buffer references, for assembling the output frame. |
| 62 | */ |
| 63 | AVBufferRef **buffers; |
| 64 | } JoinContext; |
| 65 | |
| 66 | #define OFFSET(x) offsetof(JoinContext, x) |
| 67 | #define A AV_OPT_FLAG_AUDIO_PARAM |
| 68 | #define F AV_OPT_FLAG_FILTERING_PARAM |
| 69 | static const AVOption join_options[] = { |
| 70 | { "inputs", "Number of input streams.", OFFSET(inputs), AV_OPT_TYPE_INT, { .i64 = 2 }, 1, INT_MAX, A|F }, |
| 71 | { "channel_layout", "Channel layout of the " |
| 72 | "output stream.", OFFSET(channel_layout_str), AV_OPT_TYPE_STRING, {.str = "stereo"}, 0, 0, A|F }, |
| 73 | { "map", "A comma-separated list of channels maps in the format " |
| 74 | "'input_stream.input_channel-output_channel.", |
| 75 | OFFSET(map), AV_OPT_TYPE_STRING, .flags = A|F }, |
| 76 | { NULL } |
| 77 | }; |
| 78 | |
| 79 | AVFILTER_DEFINE_CLASS(join); |
| 80 | |
| 81 | static int filter_frame(AVFilterLink *link, AVFrame *frame) |
| 82 | { |
| 83 | AVFilterContext *ctx = link->dst; |
| 84 | JoinContext *s = ctx->priv; |
| 85 | int i; |
| 86 | |
| 87 | for (i = 0; i < ctx->nb_inputs; i++) |
| 88 | if (link == ctx->inputs[i]) |
| 89 | break; |
| 90 | av_assert0(i < ctx->nb_inputs); |
| 91 | av_assert0(!s->input_frames[i]); |
| 92 | s->input_frames[i] = frame; |
| 93 | |
| 94 | return 0; |
| 95 | } |
| 96 | |
| 97 | static int parse_maps(AVFilterContext *ctx) |
| 98 | { |
| 99 | JoinContext *s = ctx->priv; |
| 100 | char separator = '|'; |
| 101 | char *cur = s->map; |
| 102 | |
| 103 | #if FF_API_OLD_FILTER_OPTS |
| 104 | if (cur && strchr(cur, ',')) { |
| 105 | av_log(ctx, AV_LOG_WARNING, "This syntax is deprecated, use '|' to " |
| 106 | "separate the mappings.\n"); |
| 107 | separator = ','; |
| 108 | } |
| 109 | #endif |
| 110 | |
| 111 | while (cur && *cur) { |
| 112 | char *sep, *next, *p; |
| 113 | uint64_t in_channel = 0, out_channel = 0; |
| 114 | int input_idx, out_ch_idx, in_ch_idx; |
| 115 | |
| 116 | next = strchr(cur, separator); |
| 117 | if (next) |
| 118 | *next++ = 0; |
| 119 | |
| 120 | /* split the map into input and output parts */ |
| 121 | if (!(sep = strchr(cur, '-'))) { |
| 122 | av_log(ctx, AV_LOG_ERROR, "Missing separator '-' in channel " |
| 123 | "map '%s'\n", cur); |
| 124 | return AVERROR(EINVAL); |
| 125 | } |
| 126 | *sep++ = 0; |
| 127 | |
| 128 | #define PARSE_CHANNEL(str, var, inout) \ |
| 129 | if (!(var = av_get_channel_layout(str))) { \ |
| 130 | av_log(ctx, AV_LOG_ERROR, "Invalid " inout " channel: %s.\n", str);\ |
| 131 | return AVERROR(EINVAL); \ |
| 132 | } \ |
| 133 | if (av_get_channel_layout_nb_channels(var) != 1) { \ |
| 134 | av_log(ctx, AV_LOG_ERROR, "Channel map describes more than one " \ |
| 135 | inout " channel.\n"); \ |
| 136 | return AVERROR(EINVAL); \ |
| 137 | } |
| 138 | |
| 139 | /* parse output channel */ |
| 140 | PARSE_CHANNEL(sep, out_channel, "output"); |
| 141 | if (!(out_channel & s->channel_layout)) { |
| 142 | av_log(ctx, AV_LOG_ERROR, "Output channel '%s' is not present in " |
| 143 | "requested channel layout.\n", sep); |
| 144 | return AVERROR(EINVAL); |
| 145 | } |
| 146 | |
| 147 | out_ch_idx = av_get_channel_layout_channel_index(s->channel_layout, |
| 148 | out_channel); |
| 149 | if (s->channels[out_ch_idx].input >= 0) { |
| 150 | av_log(ctx, AV_LOG_ERROR, "Multiple maps for output channel " |
| 151 | "'%s'.\n", sep); |
| 152 | return AVERROR(EINVAL); |
| 153 | } |
| 154 | |
| 155 | /* parse input channel */ |
| 156 | input_idx = strtol(cur, &cur, 0); |
| 157 | if (input_idx < 0 || input_idx >= s->inputs) { |
| 158 | av_log(ctx, AV_LOG_ERROR, "Invalid input stream index: %d.\n", |
| 159 | input_idx); |
| 160 | return AVERROR(EINVAL); |
| 161 | } |
| 162 | |
| 163 | if (*cur) |
| 164 | cur++; |
| 165 | |
| 166 | in_ch_idx = strtol(cur, &p, 0); |
| 167 | if (p == cur) { |
| 168 | /* channel specifier is not a number, |
| 169 | * try to parse as channel name */ |
| 170 | PARSE_CHANNEL(cur, in_channel, "input"); |
| 171 | } |
| 172 | |
| 173 | s->channels[out_ch_idx].input = input_idx; |
| 174 | if (in_channel) |
| 175 | s->channels[out_ch_idx].in_channel = in_channel; |
| 176 | else |
| 177 | s->channels[out_ch_idx].in_channel_idx = in_ch_idx; |
| 178 | |
| 179 | cur = next; |
| 180 | } |
| 181 | return 0; |
| 182 | } |
| 183 | |
| 184 | static av_cold int join_init(AVFilterContext *ctx) |
| 185 | { |
| 186 | JoinContext *s = ctx->priv; |
| 187 | int ret, i; |
| 188 | |
| 189 | if (!(s->channel_layout = av_get_channel_layout(s->channel_layout_str))) { |
| 190 | av_log(ctx, AV_LOG_ERROR, "Error parsing channel layout '%s'.\n", |
| 191 | s->channel_layout_str); |
| 192 | return AVERROR(EINVAL); |
| 193 | } |
| 194 | |
| 195 | s->nb_channels = av_get_channel_layout_nb_channels(s->channel_layout); |
| 196 | s->channels = av_mallocz(sizeof(*s->channels) * s->nb_channels); |
| 197 | s->buffers = av_mallocz(sizeof(*s->buffers) * s->nb_channels); |
| 198 | s->input_frames = av_mallocz(sizeof(*s->input_frames) * s->inputs); |
| 199 | if (!s->channels || !s->buffers|| !s->input_frames) |
| 200 | return AVERROR(ENOMEM); |
| 201 | |
| 202 | for (i = 0; i < s->nb_channels; i++) { |
| 203 | s->channels[i].out_channel = av_channel_layout_extract_channel(s->channel_layout, i); |
| 204 | s->channels[i].input = -1; |
| 205 | } |
| 206 | |
| 207 | if ((ret = parse_maps(ctx)) < 0) |
| 208 | return ret; |
| 209 | |
| 210 | for (i = 0; i < s->inputs; i++) { |
| 211 | char name[32]; |
| 212 | AVFilterPad pad = { 0 }; |
| 213 | |
| 214 | snprintf(name, sizeof(name), "input%d", i); |
| 215 | pad.type = AVMEDIA_TYPE_AUDIO; |
| 216 | pad.name = av_strdup(name); |
| 217 | pad.filter_frame = filter_frame; |
| 218 | |
| 219 | pad.needs_fifo = 1; |
| 220 | |
| 221 | ff_insert_inpad(ctx, i, &pad); |
| 222 | } |
| 223 | |
| 224 | return 0; |
| 225 | } |
| 226 | |
| 227 | static av_cold void join_uninit(AVFilterContext *ctx) |
| 228 | { |
| 229 | JoinContext *s = ctx->priv; |
| 230 | int i; |
| 231 | |
| 232 | for (i = 0; i < ctx->nb_inputs; i++) { |
| 233 | av_freep(&ctx->input_pads[i].name); |
| 234 | av_frame_free(&s->input_frames[i]); |
| 235 | } |
| 236 | |
| 237 | av_freep(&s->channels); |
| 238 | av_freep(&s->buffers); |
| 239 | av_freep(&s->input_frames); |
| 240 | } |
| 241 | |
| 242 | static int join_query_formats(AVFilterContext *ctx) |
| 243 | { |
| 244 | JoinContext *s = ctx->priv; |
| 245 | AVFilterChannelLayouts *layouts = NULL; |
| 246 | int i; |
| 247 | |
| 248 | ff_add_channel_layout(&layouts, s->channel_layout); |
| 249 | ff_channel_layouts_ref(layouts, &ctx->outputs[0]->in_channel_layouts); |
| 250 | |
| 251 | for (i = 0; i < ctx->nb_inputs; i++) { |
| 252 | layouts = ff_all_channel_layouts(); |
| 253 | if (!layouts) |
| 254 | return AVERROR(ENOMEM); |
| 255 | ff_channel_layouts_ref(layouts, &ctx->inputs[i]->out_channel_layouts); |
| 256 | } |
| 257 | |
| 258 | ff_set_common_formats (ctx, ff_planar_sample_fmts()); |
| 259 | ff_set_common_samplerates(ctx, ff_all_samplerates()); |
| 260 | |
| 261 | return 0; |
| 262 | } |
| 263 | |
| 264 | static void guess_map_matching(AVFilterContext *ctx, ChannelMap *ch, |
| 265 | uint64_t *inputs) |
| 266 | { |
| 267 | int i; |
| 268 | |
| 269 | for (i = 0; i < ctx->nb_inputs; i++) { |
| 270 | AVFilterLink *link = ctx->inputs[i]; |
| 271 | |
| 272 | if (ch->out_channel & link->channel_layout && |
| 273 | !(ch->out_channel & inputs[i])) { |
| 274 | ch->input = i; |
| 275 | ch->in_channel = ch->out_channel; |
| 276 | inputs[i] |= ch->out_channel; |
| 277 | return; |
| 278 | } |
| 279 | } |
| 280 | } |
| 281 | |
| 282 | static void guess_map_any(AVFilterContext *ctx, ChannelMap *ch, |
| 283 | uint64_t *inputs) |
| 284 | { |
| 285 | int i; |
| 286 | |
| 287 | for (i = 0; i < ctx->nb_inputs; i++) { |
| 288 | AVFilterLink *link = ctx->inputs[i]; |
| 289 | |
| 290 | if ((inputs[i] & link->channel_layout) != link->channel_layout) { |
| 291 | uint64_t unused = link->channel_layout & ~inputs[i]; |
| 292 | |
| 293 | ch->input = i; |
| 294 | ch->in_channel = av_channel_layout_extract_channel(unused, 0); |
| 295 | inputs[i] |= ch->in_channel; |
| 296 | return; |
| 297 | } |
| 298 | } |
| 299 | } |
| 300 | |
| 301 | static int join_config_output(AVFilterLink *outlink) |
| 302 | { |
| 303 | AVFilterContext *ctx = outlink->src; |
| 304 | JoinContext *s = ctx->priv; |
| 305 | uint64_t *inputs; // nth element tracks which channels are used from nth input |
| 306 | int i, ret = 0; |
| 307 | |
| 308 | /* initialize inputs to user-specified mappings */ |
| 309 | if (!(inputs = av_mallocz(sizeof(*inputs) * ctx->nb_inputs))) |
| 310 | return AVERROR(ENOMEM); |
| 311 | for (i = 0; i < s->nb_channels; i++) { |
| 312 | ChannelMap *ch = &s->channels[i]; |
| 313 | AVFilterLink *inlink; |
| 314 | |
| 315 | if (ch->input < 0) |
| 316 | continue; |
| 317 | |
| 318 | inlink = ctx->inputs[ch->input]; |
| 319 | |
| 320 | if (!ch->in_channel) |
| 321 | ch->in_channel = av_channel_layout_extract_channel(inlink->channel_layout, |
| 322 | ch->in_channel_idx); |
| 323 | |
| 324 | if (!(ch->in_channel & inlink->channel_layout)) { |
| 325 | av_log(ctx, AV_LOG_ERROR, "Requested channel %s is not present in " |
| 326 | "input stream #%d.\n", av_get_channel_name(ch->in_channel), |
| 327 | ch->input); |
| 328 | ret = AVERROR(EINVAL); |
| 329 | goto fail; |
| 330 | } |
| 331 | |
| 332 | inputs[ch->input] |= ch->in_channel; |
| 333 | } |
| 334 | |
| 335 | /* guess channel maps when not explicitly defined */ |
| 336 | /* first try unused matching channels */ |
| 337 | for (i = 0; i < s->nb_channels; i++) { |
| 338 | ChannelMap *ch = &s->channels[i]; |
| 339 | |
| 340 | if (ch->input < 0) |
| 341 | guess_map_matching(ctx, ch, inputs); |
| 342 | } |
| 343 | |
| 344 | /* if the above failed, try to find _any_ unused input channel */ |
| 345 | for (i = 0; i < s->nb_channels; i++) { |
| 346 | ChannelMap *ch = &s->channels[i]; |
| 347 | |
| 348 | if (ch->input < 0) |
| 349 | guess_map_any(ctx, ch, inputs); |
| 350 | |
| 351 | if (ch->input < 0) { |
| 352 | av_log(ctx, AV_LOG_ERROR, "Could not find input channel for " |
| 353 | "output channel '%s'.\n", |
| 354 | av_get_channel_name(ch->out_channel)); |
| 355 | goto fail; |
| 356 | } |
| 357 | |
| 358 | ch->in_channel_idx = av_get_channel_layout_channel_index(ctx->inputs[ch->input]->channel_layout, |
| 359 | ch->in_channel); |
| 360 | } |
| 361 | |
| 362 | /* print mappings */ |
| 363 | av_log(ctx, AV_LOG_VERBOSE, "mappings: "); |
| 364 | for (i = 0; i < s->nb_channels; i++) { |
| 365 | ChannelMap *ch = &s->channels[i]; |
| 366 | av_log(ctx, AV_LOG_VERBOSE, "%d.%s => %s ", ch->input, |
| 367 | av_get_channel_name(ch->in_channel), |
| 368 | av_get_channel_name(ch->out_channel)); |
| 369 | } |
| 370 | av_log(ctx, AV_LOG_VERBOSE, "\n"); |
| 371 | |
| 372 | for (i = 0; i < ctx->nb_inputs; i++) { |
| 373 | if (!inputs[i]) |
| 374 | av_log(ctx, AV_LOG_WARNING, "No channels are used from input " |
| 375 | "stream %d.\n", i); |
| 376 | } |
| 377 | |
| 378 | fail: |
| 379 | av_freep(&inputs); |
| 380 | return ret; |
| 381 | } |
| 382 | |
| 383 | static int join_request_frame(AVFilterLink *outlink) |
| 384 | { |
| 385 | AVFilterContext *ctx = outlink->src; |
| 386 | JoinContext *s = ctx->priv; |
| 387 | AVFrame *frame; |
| 388 | int linesize = INT_MAX; |
| 389 | int nb_samples = 0; |
| 390 | int nb_buffers = 0; |
| 391 | int i, j, ret; |
| 392 | |
| 393 | /* get a frame on each input */ |
| 394 | for (i = 0; i < ctx->nb_inputs; i++) { |
| 395 | AVFilterLink *inlink = ctx->inputs[i]; |
| 396 | |
| 397 | if (!s->input_frames[i] && |
| 398 | (ret = ff_request_frame(inlink)) < 0) |
| 399 | return ret; |
| 400 | |
| 401 | /* request the same number of samples on all inputs */ |
| 402 | if (i == 0) { |
| 403 | nb_samples = s->input_frames[0]->nb_samples; |
| 404 | |
| 405 | for (j = 1; !i && j < ctx->nb_inputs; j++) |
| 406 | ctx->inputs[j]->request_samples = nb_samples; |
| 407 | } |
| 408 | } |
| 409 | |
| 410 | /* setup the output frame */ |
| 411 | frame = av_frame_alloc(); |
| 412 | if (!frame) |
| 413 | return AVERROR(ENOMEM); |
| 414 | if (s->nb_channels > FF_ARRAY_ELEMS(frame->data)) { |
| 415 | frame->extended_data = av_mallocz(s->nb_channels * |
| 416 | sizeof(*frame->extended_data)); |
| 417 | if (!frame->extended_data) { |
| 418 | ret = AVERROR(ENOMEM); |
| 419 | goto fail; |
| 420 | } |
| 421 | } |
| 422 | |
| 423 | /* copy the data pointers */ |
| 424 | for (i = 0; i < s->nb_channels; i++) { |
| 425 | ChannelMap *ch = &s->channels[i]; |
| 426 | AVFrame *cur = s->input_frames[ch->input]; |
| 427 | AVBufferRef *buf; |
| 428 | |
| 429 | frame->extended_data[i] = cur->extended_data[ch->in_channel_idx]; |
| 430 | linesize = FFMIN(linesize, cur->linesize[0]); |
| 431 | |
| 432 | /* add the buffer where this plan is stored to the list if it's |
| 433 | * not already there */ |
| 434 | buf = av_frame_get_plane_buffer(cur, ch->in_channel_idx); |
| 435 | if (!buf) { |
| 436 | ret = AVERROR(EINVAL); |
| 437 | goto fail; |
| 438 | } |
| 439 | for (j = 0; j < nb_buffers; j++) |
| 440 | if (s->buffers[j]->buffer == buf->buffer) |
| 441 | break; |
| 442 | if (j == i) |
| 443 | s->buffers[nb_buffers++] = buf; |
| 444 | } |
| 445 | |
| 446 | /* create references to the buffers we copied to output */ |
| 447 | if (nb_buffers > FF_ARRAY_ELEMS(frame->buf)) { |
| 448 | frame->nb_extended_buf = nb_buffers - FF_ARRAY_ELEMS(frame->buf); |
| 449 | frame->extended_buf = av_mallocz(sizeof(*frame->extended_buf) * |
| 450 | frame->nb_extended_buf); |
| 451 | if (!frame->extended_buf) { |
| 452 | frame->nb_extended_buf = 0; |
| 453 | ret = AVERROR(ENOMEM); |
| 454 | goto fail; |
| 455 | } |
| 456 | } |
| 457 | for (i = 0; i < FFMIN(FF_ARRAY_ELEMS(frame->buf), nb_buffers); i++) { |
| 458 | frame->buf[i] = av_buffer_ref(s->buffers[i]); |
| 459 | if (!frame->buf[i]) { |
| 460 | ret = AVERROR(ENOMEM); |
| 461 | goto fail; |
| 462 | } |
| 463 | } |
| 464 | for (i = 0; i < frame->nb_extended_buf; i++) { |
| 465 | frame->extended_buf[i] = av_buffer_ref(s->buffers[i + |
| 466 | FF_ARRAY_ELEMS(frame->buf)]); |
| 467 | if (!frame->extended_buf[i]) { |
| 468 | ret = AVERROR(ENOMEM); |
| 469 | goto fail; |
| 470 | } |
| 471 | } |
| 472 | |
| 473 | frame->nb_samples = nb_samples; |
| 474 | frame->channel_layout = outlink->channel_layout; |
| 475 | av_frame_set_channels(frame, outlink->channels); |
| 476 | frame->sample_rate = outlink->sample_rate; |
| 477 | frame->format = outlink->format; |
| 478 | frame->pts = s->input_frames[0]->pts; |
| 479 | frame->linesize[0] = linesize; |
| 480 | if (frame->data != frame->extended_data) { |
| 481 | memcpy(frame->data, frame->extended_data, sizeof(*frame->data) * |
| 482 | FFMIN(FF_ARRAY_ELEMS(frame->data), s->nb_channels)); |
| 483 | } |
| 484 | |
| 485 | ret = ff_filter_frame(outlink, frame); |
| 486 | |
| 487 | for (i = 0; i < ctx->nb_inputs; i++) |
| 488 | av_frame_free(&s->input_frames[i]); |
| 489 | |
| 490 | return ret; |
| 491 | |
| 492 | fail: |
| 493 | av_frame_free(&frame); |
| 494 | return ret; |
| 495 | } |
| 496 | |
| 497 | static const AVFilterPad avfilter_af_join_outputs[] = { |
| 498 | { |
| 499 | .name = "default", |
| 500 | .type = AVMEDIA_TYPE_AUDIO, |
| 501 | .config_props = join_config_output, |
| 502 | .request_frame = join_request_frame, |
| 503 | }, |
| 504 | { NULL } |
| 505 | }; |
| 506 | |
| 507 | AVFilter ff_af_join = { |
| 508 | .name = "join", |
| 509 | .description = NULL_IF_CONFIG_SMALL("Join multiple audio streams into " |
| 510 | "multi-channel output."), |
| 511 | .priv_size = sizeof(JoinContext), |
| 512 | .priv_class = &join_class, |
| 513 | .init = join_init, |
| 514 | .uninit = join_uninit, |
| 515 | .query_formats = join_query_formats, |
| 516 | .inputs = NULL, |
| 517 | .outputs = avfilter_af_join_outputs, |
| 518 | .flags = AVFILTER_FLAG_DYNAMIC_INPUTS, |
| 519 | }; |