2 * VMware Screen Codec (VMnc) decoder
3 * Copyright (c) 2006 Konstantin Shishkov
5 * This file is part of FFmpeg.
7 * FFmpeg is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
12 * FFmpeg is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with FFmpeg; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 * VMware Screen Codec (VMnc) decoder
25 * As Alex Beregszaszi discovered, this is effectively RFB data dump
31 #include "libavutil/common.h"
32 #include "libavutil/intreadwrite.h"
35 #include "bytestream.h"
38 MAGIC_WMVd
= 0x574D5664,
48 HT_RAW
= 1, // tile is raw
49 HT_BKG
= 2, // background color is present
50 HT_FG
= 4, // foreground color is present
51 HT_SUB
= 8, // subrects are present
52 HT_CLR
= 16 // each subrect has own color
58 typedef struct VmncContext
{
59 AVCodecContext
*avctx
;
73 uint8_t *curbits
, *curmask
;
77 /* read pixel value from stream */
78 static av_always_inline
int vmnc_get_pixel(GetByteContext
*gb
, int bpp
, int be
)
80 switch (bpp
* 2 + be
) {
83 return bytestream2_get_byte(gb
);
85 return bytestream2_get_le16(gb
);
87 return bytestream2_get_be16(gb
);
89 return bytestream2_get_le32(gb
);
91 return bytestream2_get_be32(gb
);
96 static void load_cursor(VmncContext
*c
)
99 const int bpp
= c
->bpp2
;
100 uint8_t *dst8
= c
->curbits
;
101 uint16_t *dst16
= (uint16_t *)c
->curbits
;
102 uint32_t *dst32
= (uint32_t *)c
->curbits
;
104 for (j
= 0; j
< c
->cur_h
; j
++) {
105 for (i
= 0; i
< c
->cur_w
; i
++) {
106 p
= vmnc_get_pixel(&c
->gb
, bpp
, c
->bigendian
);
116 dst16
= (uint16_t*)c
->curmask
;
117 dst32
= (uint32_t*)c
->curmask
;
118 for (j
= 0; j
< c
->cur_h
; j
++) {
119 for (i
= 0; i
< c
->cur_w
; i
++) {
120 p
= vmnc_get_pixel(&c
->gb
, bpp
, c
->bigendian
);
131 static void put_cursor(uint8_t *dst
, int stride
, VmncContext
*c
, int dx
, int dy
)
136 if (c
->width
< c
->cur_x
+ c
->cur_w
)
137 w
= c
->width
- c
->cur_x
;
139 if (c
->height
< c
->cur_y
+ c
->cur_h
)
140 h
= c
->height
- c
->cur_y
;
152 if ((w
< 1) || (h
< 1))
154 dst
+= x
* c
->bpp2
+ y
* stride
;
157 uint8_t *cd
= c
->curbits
, *msk
= c
->curmask
;
158 for (j
= 0; j
< h
; j
++) {
159 for (i
= 0; i
< w
; i
++)
160 dst
[i
] = (dst
[i
] & cd
[i
]) ^ msk
[i
];
165 } else if (c
->bpp2
== 2) {
166 uint16_t *cd
= (uint16_t*)c
->curbits
, *msk
= (uint16_t*)c
->curmask
;
168 for (j
= 0; j
< h
; j
++) {
169 dst2
= (uint16_t*)dst
;
170 for (i
= 0; i
< w
; i
++)
171 dst2
[i
] = (dst2
[i
] & cd
[i
]) ^ msk
[i
];
176 } else if (c
->bpp2
== 4) {
177 uint32_t *cd
= (uint32_t*)c
->curbits
, *msk
= (uint32_t*)c
->curmask
;
179 for (j
= 0; j
< h
; j
++) {
180 dst2
= (uint32_t*)dst
;
181 for (i
= 0; i
< w
; i
++)
182 dst2
[i
] = (dst2
[i
] & cd
[i
]) ^ msk
[i
];
190 /* fill rectangle with given color */
191 static av_always_inline
void paint_rect(uint8_t *dst
, int dx
, int dy
,
192 int w
, int h
, int color
,
196 dst
+= dx
* bpp
+ dy
* stride
;
198 for (j
= 0; j
< h
; j
++) {
199 memset(dst
, color
, w
);
202 } else if (bpp
== 2) {
204 for (j
= 0; j
< h
; j
++) {
205 dst2
= (uint16_t*)dst
;
206 for (i
= 0; i
< w
; i
++)
210 } else if (bpp
== 4) {
212 for (j
= 0; j
< h
; j
++) {
213 dst2
= (uint32_t*)dst
;
214 for (i
= 0; i
< w
; i
++)
221 static av_always_inline
void paint_raw(uint8_t *dst
, int w
, int h
,
222 GetByteContext
*gb
, int bpp
,
226 for (j
= 0; j
< h
; j
++) {
227 for (i
= 0; i
< w
; i
++) {
228 p
= vmnc_get_pixel(gb
, bpp
, be
);
234 ((uint16_t*)dst
)[i
] = p
;
237 ((uint32_t*)dst
)[i
] = p
;
245 static int decode_hextile(VmncContext
*c
, uint8_t* dst
, GetByteContext
*gb
,
246 int w
, int h
, int stride
)
249 int bg
= 0, fg
= 0, rects
, color
, flags
, xy
, wh
;
250 const int bpp
= c
->bpp2
;
252 int bw
= 16, bh
= 16;
254 for (j
= 0; j
< h
; j
+= 16) {
259 for (i
= 0; i
< w
; i
+= 16, dst2
+= 16 * bpp
) {
260 if (bytestream2_get_bytes_left(gb
) <= 0) {
261 av_log(c
->avctx
, AV_LOG_ERROR
, "Premature end of data!\n");
262 return AVERROR_INVALIDDATA
;
266 flags
= bytestream2_get_byte(gb
);
267 if (flags
& HT_RAW
) {
268 if (bytestream2_get_bytes_left(gb
) < bw
* bh
* bpp
) {
269 av_log(c
->avctx
, AV_LOG_ERROR
, "Premature end of data!\n");
270 return AVERROR_INVALIDDATA
;
272 paint_raw(dst2
, bw
, bh
, gb
, bpp
, c
->bigendian
, stride
);
275 bg
= vmnc_get_pixel(gb
, bpp
, c
->bigendian
);
277 fg
= vmnc_get_pixel(gb
, bpp
, c
->bigendian
);
280 rects
= bytestream2_get_byte(gb
);
281 color
= !!(flags
& HT_CLR
);
283 paint_rect(dst2
, 0, 0, bw
, bh
, bg
, bpp
, stride
);
285 if (bytestream2_get_bytes_left(gb
) < rects
* (color
* bpp
+ 2)) {
286 av_log(c
->avctx
, AV_LOG_ERROR
, "Premature end of data!\n");
287 return AVERROR_INVALIDDATA
;
289 for (k
= 0; k
< rects
; k
++) {
291 fg
= vmnc_get_pixel(gb
, bpp
, c
->bigendian
);
292 xy
= bytestream2_get_byte(gb
);
293 wh
= bytestream2_get_byte(gb
);
294 if ( (xy
>> 4) + (wh
>> 4) + 1 > w
- i
295 || (xy
& 0xF) + (wh
& 0xF)+1 > h
- j
) {
296 av_log(c
->avctx
, AV_LOG_ERROR
, "Rectangle outside picture\n");
297 return AVERROR_INVALIDDATA
;
299 paint_rect(dst2
, xy
>> 4, xy
& 0xF,
300 (wh
>>4)+1, (wh
& 0xF)+1, fg
, bpp
, stride
);
309 static void reset_buffers(VmncContext
*c
)
311 av_freep(&c
->curbits
);
312 av_freep(&c
->curmask
);
313 av_freep(&c
->screendta
);
314 c
->cur_w
= c
->cur_h
= 0;
315 c
->cur_hx
= c
->cur_hy
= 0;
319 static int decode_frame(AVCodecContext
*avctx
, void *data
, int *got_frame
,
322 const uint8_t *buf
= avpkt
->data
;
323 int buf_size
= avpkt
->size
;
324 VmncContext
* const c
= avctx
->priv_data
;
325 GetByteContext
*gb
= &c
->gb
;
327 int dx
, dy
, w
, h
, depth
, enc
, chunks
, res
, size_left
, ret
;
329 if ((ret
= ff_reget_buffer(avctx
, c
->pic
)) < 0)
332 bytestream2_init(gb
, buf
, buf_size
);
334 c
->pic
->key_frame
= 0;
335 c
->pic
->pict_type
= AV_PICTURE_TYPE_P
;
337 // restore screen after cursor
341 if (c
->width
< c
->cur_x
+ w
)
342 w
= c
->width
- c
->cur_x
;
344 if (c
->height
< c
->cur_y
+ h
)
345 h
= c
->height
- c
->cur_y
;
356 if ((w
> 0) && (h
> 0)) {
357 outptr
= c
->pic
->data
[0] + dx
* c
->bpp2
+ dy
* c
->pic
->linesize
[0];
358 for (i
= 0; i
< h
; i
++) {
359 memcpy(outptr
, c
->screendta
+ i
* c
->cur_w
* c
->bpp2
,
361 outptr
+= c
->pic
->linesize
[0];
365 bytestream2_skip(gb
, 2);
366 chunks
= bytestream2_get_be16(gb
);
368 if (bytestream2_get_bytes_left(gb
) < 12) {
369 av_log(avctx
, AV_LOG_ERROR
, "Premature end of data!\n");
372 dx
= bytestream2_get_be16(gb
);
373 dy
= bytestream2_get_be16(gb
);
374 w
= bytestream2_get_be16(gb
);
375 h
= bytestream2_get_be16(gb
);
376 enc
= bytestream2_get_be32(gb
);
377 outptr
= c
->pic
->data
[0] + dx
* c
->bpp2
+ dy
* c
->pic
->linesize
[0];
378 size_left
= bytestream2_get_bytes_left(gb
);
380 case MAGIC_WMVd
: // cursor
381 if (w
*(int64_t)h
*c
->bpp2
> INT_MAX
/2 - 2) {
382 av_log(avctx
, AV_LOG_ERROR
, "dimensions too large\n");
383 return AVERROR_INVALIDDATA
;
385 if (size_left
< 2 + w
* h
* c
->bpp2
* 2) {
386 av_log(avctx
, AV_LOG_ERROR
,
387 "Premature end of data! (need %i got %i)\n",
388 2 + w
* h
* c
->bpp2
* 2, size_left
);
389 return AVERROR_INVALIDDATA
;
391 bytestream2_skip(gb
, 2);
396 if ((c
->cur_hx
> c
->cur_w
) || (c
->cur_hy
> c
->cur_h
)) {
397 av_log(avctx
, AV_LOG_ERROR
,
398 "Cursor hot spot is not in image: "
399 "%ix%i of %ix%i cursor size\n",
400 c
->cur_hx
, c
->cur_hy
, c
->cur_w
, c
->cur_h
);
401 c
->cur_hx
= c
->cur_hy
= 0;
403 if (c
->cur_w
* c
->cur_h
>= INT_MAX
/ c
->bpp2
) {
405 return AVERROR(EINVAL
);
407 int screen_size
= c
->cur_w
* c
->cur_h
* c
->bpp2
;
408 if ((ret
= av_reallocp(&c
->curbits
, screen_size
)) < 0 ||
409 (ret
= av_reallocp(&c
->curmask
, screen_size
)) < 0 ||
410 (ret
= av_reallocp(&c
->screendta
, screen_size
)) < 0) {
417 case MAGIC_WMVe
: // unknown
418 bytestream2_skip(gb
, 2);
420 case MAGIC_WMVf
: // update cursor position
421 c
->cur_x
= dx
- c
->cur_hx
;
422 c
->cur_y
= dy
- c
->cur_hy
;
424 case MAGIC_WMVg
: // unknown
425 bytestream2_skip(gb
, 10);
427 case MAGIC_WMVh
: // unknown
428 bytestream2_skip(gb
, 4);
430 case MAGIC_WMVi
: // ServerInitialization struct
431 c
->pic
->key_frame
= 1;
432 c
->pic
->pict_type
= AV_PICTURE_TYPE_I
;
433 depth
= bytestream2_get_byte(gb
);
434 if (depth
!= c
->bpp
) {
435 av_log(avctx
, AV_LOG_INFO
,
436 "Depth mismatch. Container %i bpp, "
437 "Frame data: %i bpp\n",
440 bytestream2_skip(gb
, 1);
441 c
->bigendian
= bytestream2_get_byte(gb
);
442 if (c
->bigendian
& (~1)) {
443 av_log(avctx
, AV_LOG_INFO
,
444 "Invalid header: bigendian flag = %i\n", c
->bigendian
);
445 return AVERROR_INVALIDDATA
;
447 //skip the rest of pixel format data
448 bytestream2_skip(gb
, 13);
450 case MAGIC_WMVj
: // unknown
451 bytestream2_skip(gb
, 2);
453 case 0x00000000: // raw rectangle data
454 if ((dx
+ w
> c
->width
) || (dy
+ h
> c
->height
)) {
455 av_log(avctx
, AV_LOG_ERROR
,
456 "Incorrect frame size: %ix%i+%ix%i of %ix%i\n",
457 w
, h
, dx
, dy
, c
->width
, c
->height
);
458 return AVERROR_INVALIDDATA
;
460 if (size_left
< w
* h
* c
->bpp2
) {
461 av_log(avctx
, AV_LOG_ERROR
,
462 "Premature end of data! (need %i got %i)\n",
463 w
* h
* c
->bpp2
, size_left
);
464 return AVERROR_INVALIDDATA
;
466 paint_raw(outptr
, w
, h
, gb
, c
->bpp2
, c
->bigendian
,
467 c
->pic
->linesize
[0]);
469 case 0x00000005: // HexTile encoded rectangle
470 if ((dx
+ w
> c
->width
) || (dy
+ h
> c
->height
)) {
471 av_log(avctx
, AV_LOG_ERROR
,
472 "Incorrect frame size: %ix%i+%ix%i of %ix%i\n",
473 w
, h
, dx
, dy
, c
->width
, c
->height
);
474 return AVERROR_INVALIDDATA
;
476 res
= decode_hextile(c
, outptr
, gb
, w
, h
, c
->pic
->linesize
[0]);
481 av_log(avctx
, AV_LOG_ERROR
, "Unsupported block type 0x%08X\n", enc
);
482 chunks
= 0; // leave chunks decoding loop
487 // save screen data before painting cursor
489 if (c
->width
< c
->cur_x
+ w
)
490 w
= c
->width
- c
->cur_x
;
492 if (c
->height
< c
->cur_y
+ h
)
493 h
= c
->height
- c
->cur_y
;
504 if ((w
> 0) && (h
> 0)) {
505 outptr
= c
->pic
->data
[0] + dx
* c
->bpp2
+ dy
* c
->pic
->linesize
[0];
506 for (i
= 0; i
< h
; i
++) {
507 memcpy(c
->screendta
+ i
* c
->cur_w
* c
->bpp2
, outptr
,
509 outptr
+= c
->pic
->linesize
[0];
511 outptr
= c
->pic
->data
[0];
512 put_cursor(outptr
, c
->pic
->linesize
[0], c
, c
->cur_x
, c
->cur_y
);
516 if ((ret
= av_frame_ref(data
, c
->pic
)) < 0)
519 /* always report that the buffer was completely consumed */
523 static av_cold
int decode_init(AVCodecContext
*avctx
)
525 VmncContext
* const c
= avctx
->priv_data
;
528 c
->width
= avctx
->width
;
529 c
->height
= avctx
->height
;
530 c
->bpp
= avctx
->bits_per_coded_sample
;
531 c
->bpp2
= c
->bpp
/ 8;
535 avctx
->pix_fmt
= AV_PIX_FMT_PAL8
;
538 avctx
->pix_fmt
= AV_PIX_FMT_RGB555
;
541 avctx
->pix_fmt
= AV_PIX_FMT_RGB32
;
544 av_log(avctx
, AV_LOG_ERROR
, "Unsupported bitdepth %i\n", c
->bpp
);
545 return AVERROR_INVALIDDATA
;
548 c
->pic
= av_frame_alloc();
550 return AVERROR(ENOMEM
);
555 static av_cold
int decode_end(AVCodecContext
*avctx
)
557 VmncContext
* const c
= avctx
->priv_data
;
559 av_frame_free(&c
->pic
);
561 av_freep(&c
->curbits
);
562 av_freep(&c
->curmask
);
563 av_freep(&c
->screendta
);
567 AVCodec ff_vmnc_decoder
= {
569 .long_name
= NULL_IF_CONFIG_SMALL("VMware Screen Codec / VMware Video"),
570 .type
= AVMEDIA_TYPE_VIDEO
,
571 .id
= AV_CODEC_ID_VMNC
,
572 .priv_data_size
= sizeof(VmncContext
),
575 .decode
= decode_frame
,
576 .capabilities
= CODEC_CAP_DR1
,