TiVo® HME Tech Note #001: Video Backgrounds
20041001

© 2004-2005 TiVo Inc.

What are Video Backgrounds?

The rendering hardware on the TiVo box allows for three or four graphics planes. Applications that include multiple transparent layers will suffer a performance penalty if they use more than the available planes. The underlying video decoder plane can be used as an additional graphics plane, but it only displays MPEG streams.

A video background is a single frame MPEG clip that can be placed in the video plane and used as an application background. This is an efficient way for performance intensive applications to squeeze a little more performance from the TiVo box.

This is an advanced feature that only applies to graphically intensive applications with many overlapping views. Most applications will not need this feature.

Using Video Backgrounds in your application

For a complete sample application that uses a video background, see the VideoBackground sample application included in the SDK. It is easy to use a video background - simply add it to a view as a resource.

The simulator does not support video backgrounds. The sample code shows how to work around that problem while developing with the simulator.

Here is a simple example that shows how to use a video background in your application: 

public class VideoBackgroundDemo extends Application
{
    /**
     * Create the appplication.
     */
    protected void init(Context context)
    {
        //
        // Set the mpeg file as the resource of the root view.  You can display
        // a video background in any view, but the coordinate space defined by
	// the view is ignored, the mpeg will be displayed at 0,0 in global screen
        // coordinates.
        //
        root.setResource("myloop.mpg");

        //
        // Display some text above the video
        //
        View fg = new View(root, 0, 0, root.width, root.height);
        fg.setResource(createText("default-20-bold.font",
                                  Color.black, "The picture is a movie!"));
    }
}

Creating a Video Background

This release of HME supports single frame mpegs as video backgrounds. We have provided the source code to a C program "mkloop.c" that will convert a JPEG into an MPEG. mkloop requires ffmpeg, an open source mpeg encoder.

Use mkloop to create a single frame MPEG (linux)

  1. Resize images to 720x480 or 640x480. Note: mkloop will attempt to calculate aspect ratios for inputs of any size, but resizing should ensure that the output mpg is always usable.
  2. Download ffmpeg-0.4.9-pre1 from http://ffmpeg.sourceforge.net/ and compile it.
  3. Create a directory inside the ffmpeg directory called mkloop, and put mkloop.c and the mkloop Makefile in it. (see below)
  4. Build and run mkloop on your jpg: 
    % make
% mkloop foo.jpg foo.mpg
  5. Use foo.mpg in your application.

mkloop.c
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>

#include "avformat.h"
#include "avcodec.h"

#define STREAM_NB_FRAMES 12
#define STREAM_GOP STREAM_NB_FRAMES

AVPicture input_picture;
enum PixelFormat img_fmt;
int img_width;
int img_height;

AVFrame *picture;
uint8_t *video_outbuf;
int frame_count, video_outbuf_size;

int img_alloc_cb(void *opaque, AVImageInfo *info)
{
    unsigned char *data = malloc(info->width * info->height * 4);
    img_width = info->width;
    img_height = info->height;
    img_fmt = info->pix_fmt;

    avpicture_fill(&info->pict, data, info->pix_fmt, (info->width+15)&(~15), (info->height+15)&(~15));
    memcpy(&input_picture, &info->pict, sizeof(AVPicture));
    return 0;
}

AVStream *add_video_stream(AVFormatContext *oc, int codec_id)
{
    AVCodecContext *c;
    AVStream *st;
    int aspect_ratio_numerator;

    st = av_new_stream(oc, 0);
    if (!st) {
        fprintf(stderr, "Could not alloc stream\n");
        exit(1);
    }
    
    c = &st->codec;
    c->codec_id = codec_id;
    c->codec_type = CODEC_TYPE_VIDEO;

    c->bit_rate = 400000;
    c->width = img_width;  
    c->height = img_height;
    c->frame_rate = 30000;
    c->frame_rate_base = 1001;
    c->max_b_frames = 0;
    c->gop_size = STREAM_GOP;
    c->flags |= CODEC_FLAG_CLOSED_GOP;
    c->scenechange_threshold = 1000000000;
    aspect_ratio_numerator = ((img_height / (float) img_width) * 1.33333) * 1000;
    c->sample_aspect_ratio.num = aspect_ratio_numerator;
    c->sample_aspect_ratio.den = 1000;
    
    return st;
}

AVFrame *alloc_picture(int pix_fmt, int width, int height)
{
    AVFrame *picture;
    uint8_t *picture_buf;
    int size;
    
    picture = avcodec_alloc_frame();
    if (!picture)
        return NULL;
    size = avpicture_get_size(pix_fmt, width, height);
    picture_buf = malloc(size);
    if (!picture_buf) {
        av_free(picture);
        return NULL;
    }
    avpicture_fill((AVPicture *)picture, picture_buf, 
                   pix_fmt, width, height);
    return picture;
}
    
void open_video(AVFormatContext *oc, AVStream *st)
{
    AVCodec *codec;
    AVCodecContext *c;

    c = &st->codec;

    // find the video encoder
    codec = avcodec_find_encoder(c->codec_id);
    if (!codec) {
        fprintf(stderr, "codec not found\n");
        exit(1);
    }

    // open the codec
    if (avcodec_open(c, codec) < 0) {
        fprintf(stderr, "could not open codec\n");
        exit(1);
    }

    video_outbuf = NULL;
    if (!(oc->oformat->flags & AVFMT_RAWPICTURE)) {
        video_outbuf_size = 200000;
        video_outbuf = malloc(video_outbuf_size);
    }

    // allocate the encoded raw picture
    picture = alloc_picture(c->pix_fmt, c->width, c->height);
    if (!picture) {
        fprintf(stderr, "Could not allocate picture\n");
        exit(1);
    }
}

void write_video_frame(AVFormatContext *oc, AVStream *st)
{
    int out_size, ret;
    AVCodecContext *c;
    AVFrame *picture_ptr;
    
    c = &st->codec;
    
    if (frame_count >= STREAM_NB_FRAMES) {
        /* no more frame to compress. The codec has a latency of a few
           frames if using B frames, so we get the last frames by
           passing a NULL picture */
        picture_ptr = NULL;
    } else {
        img_convert((AVPicture *)picture, c->pix_fmt, 
                    (AVPicture *)&input_picture, img_fmt,
                    c->width, c->height);
        picture_ptr = picture;

        if (frame_count == 0) {
            picture->pts = 1000000;
        }
    }

    
    /* encode the image */
    out_size = avcodec_encode_video(c, video_outbuf, video_outbuf_size, picture_ptr);
    /* if zero size, it means the image was buffered */
    if (out_size != 0) {
        AVPacket pkt;
        av_init_packet(&pkt);
            
        pkt.pts= c->coded_frame->pts;
        pkt.dts= c->coded_frame->pts;
        if(c->coded_frame->key_frame)
            pkt.flags |= PKT_FLAG_KEY;
        pkt.stream_index= st->index;
        pkt.data= video_outbuf;
        pkt.size= out_size;
            
        /* write the compressed frame in the media file */
        ret = av_write_frame(oc, &pkt);
    } else {
        ret = 0;
    }

    if (ret != 0) {
        fprintf(stderr, "Error while writing video frame\n");
        exit(1);
    }
    frame_count++;
}

void write_seq_end(AVFormatContext *oc, AVStream *st)
{
    AVPacket pkt;
    AVCodecContext *c;
    c = &st->codec;

    av_init_packet(&pkt);
    pkt.pts= c->coded_frame->pts;
    pkt.dts= c->coded_frame->pts;
    pkt.stream_index= st->index;
    pkt.data= video_outbuf;
    video_outbuf[0] = 0x00;
    video_outbuf[1] = 0x00;
    video_outbuf[2] = 0x01;
    video_outbuf[3] = 0xb7;
    pkt.size= 4;
            
    /* write the compressed frame in the media file */
    av_write_frame(oc, &pkt);
}

void close_video(AVFormatContext *oc, AVStream *st)
{
    avcodec_close(&st->codec);
    av_free(picture->data[0]);
    av_free(picture);
}

//
// convert image to video
//

int main(int argc, char **argv)
{
    const char *ifilename;
    const char *ofilename;
    AVOutputFormat *fmt;
    AVFormatContext *oc;
    AVStream *video_st;
    ByteIOContext bio;
    int i;

    // initialize libavcodec
    av_register_all();
    
    if (argc != 3) {
        printf("usage: %s <input>.jpg <output>.mpg\n", argv[0]);
        exit(1);
    }

    ifilename = argv[1];
    ofilename = argv[2];

    // read the image
    if (url_fopen(&bio, ifilename, URL_RDONLY) < 0) {
        fprintf(stderr, "failed to open image: %s\n", ifilename);
        exit(1);
    }
    if (av_read_image(&bio, ifilename, NULL, img_alloc_cb, NULL) < 0) {
        fprintf(stderr, "failed to read image: %s\n", ifilename);
        exit(1);
    }
    url_fclose(&bio);

    printf("input image %dx%d\n", img_width, img_height);

    // get MPEG format
    fmt = guess_format("vob", NULL, NULL);
    
    // allocate the output media context
    oc = av_alloc_format_context();
    oc->oformat = fmt;
    snprintf(oc->filename, sizeof(oc->filename), "%s", ofilename);

    video_st = NULL;
    if (fmt->video_codec != CODEC_ID_NONE) {
        video_st = add_video_stream(oc, fmt->video_codec);
    }
    //video_st->quality = 1.0;

    if (av_set_parameters(oc, NULL) < 0) {
        fprintf(stderr, "invalid output format parameters\n");
        exit(1);
    }

    dump_format(oc, 0, ofilename, 1);

    open_video(oc, video_st);

    // open the output file
    if (url_fopen(&oc->pb, ofilename, URL_WRONLY) < 0) {
        fprintf(stderr, "could not open '%s'\n", ofilename);
        exit(1);
    }
    
    // write the stream header
    av_write_header(oc);
    for (i = 0 ; i < STREAM_NB_FRAMES*2 ; i++) {
        write_video_frame(oc, video_st);
    }
    write_seq_end(oc, video_st);
    
    close_video(oc, video_st);
    av_write_trailer(oc);
    
    // cleanup
    for(i = 0; i < oc->nb_streams; i++) {
        av_freep(&oc->streams[i]);
    }

    // program end
    video_outbuf[0] = 0x00;
    video_outbuf[1] = 0x00;
    video_outbuf[2] = 0x01;
    video_outbuf[3] = 0xB9;
    put_buffer(&oc->pb, video_outbuf, 4);
    put_flush_packet(&oc->pb);
    url_fclose(&oc->pb);
    av_free(oc);
    av_free(video_outbuf);

    return 0;
}

Note: Line 13 and 16 of the Makefile below must begin with a TAB. The browser probably converted them to 8 spaces. If your make fails, then you may need to change 8 spaces to tabs.

Makefile 
include ../config.mak

VPATH=$(SRC_PATH)

CFLAGS=$(OPTFLAGS) -I. -I$(SRC_PATH) -I$(SRC_PATH)/libavcodec \
       -I$(SRC_PATH)/libavformat -D_FILE_OFFSET_BITS=64 \
       -D_LARGEFILE_SOURCE -D_GNU_SOURCE
FFLIBS = -L../libavformat -lavformat -L../libavcodec -lavcodec

default: mkloop

mkloop: mkloop.o
	$(CC) -o mkloop $< $(FFLIBS) $(EXTRALIBS)

mkloop.o: mkloop.c
	$(CC) $(CFLAGS) $(SDL_CFLAGS) -c -o $@ $<

Mpeg Guidelines

The MPEG should have the following characteristics, In the Video Sequence Header:

The chroma format must be 4:2:0.

Video should be encoded as MP@ML.

Caveat

Video background alignment may not be pixel perfect on different TiVo boxes or even between software releases. Therefore application developers should not rely upon the exact alignment of a video background. That said you can free up over 600kb of graphics memory by using a video background.