Welcome!

Welcome to the official BlackBerry Support Community Forums.

This is your resource to discuss support topics with your peers, and learn from each other.

inside custom component

Native Development

Reply
Developer
mreed
Posts: 1,041
Registered: ‎07-16-2008
My Device: ಠ_ಠ
Accepted Solution

Camera API NV12 frame to AVFrame (FFmpeg)

I'm trying to use the Camera API to stream video, but since it currently only writes to a file I am trying to use the callback with the video view finder.

 

void vf_callback(camera_handle_t handle, camera_buffer_t* buf, void* arg)

 

Since this only gets a video frame in NV12 (close enough to YUV420P? I think?) I'm trying to use FFmpeg to convert it. I already have FFmpeg ported and it runs fine, but I can't seem to get the frame to convert over into an MPEG frame.

 

My question is, does anyone know how to encode the video frame from the callback using FFmpeg?

 

This makes a dummy AVFrame that runs fine when the video file is created:

/* Y */
for(int y=0;y<c->height;y++) {
    for(int x=0;x<c->width;x++) {
        picture->data[0][y * picture->linesize[0] + x] = x + y + a * 3;
    }
}

/* Cb and Cr */
for(int y=0;y<c->height/2;y++) {
    for(int x=0;x<c->width/2;x++) {
        picture->data[1][y * picture->linesize[1] + x] = 128 + y + a * 2;
        picture->data[2][y * picture->linesize[2] + x] = 64 + x + a * 5;
    }
}

 

I found this in the FFmpeg source, but it doesn't quite work to convert the picture:

int8_t *y, *u, *v;
y = picture->data[0];
u = picture->data[1];
v = picture->data[2];
const uint8_t *src=buf->framebuf;

for (int i = 0; i < (c->height + 1) >> 1; i++)
{
	for (int j = 0; j < (c->width + 1) >> 1; j++)
	{
		u[j] = *src++ ^ 0x80;
		v[j] = *src++ ^ 0x80;
		y[2 * j] = *src++;
		y[2 * j + 1] = *src++;
		y[picture->linesize[0] + 2 * j] = *src++;
		y[picture->linesize[0] + 2 * j + 1] = *src++;
	}

	y += 2 * picture->linesize[0];
	u += picture->linesize[1];
	v += picture->linesize[2];
}

 

Here is the callback and other test code:

void vf_callback(camera_handle_t handle, camera_buffer_t* buf, void* arg)
{
	if (buf->frametype != CAMERA_FRAMETYPE_NV12)
	{
		return;
	}

	printf("got video buffer of size %d x %d, bytes: %d\n",
			buf->framedesc.nv12.width, buf->framedesc.nv12.height,
			(buf->framedesc.nv12.height + (buf->framedesc.nv12.height / 2))
					* buf->framedesc.nv12.stride);

	av_register_all();

	video_encode_example(buf, "/accounts/1000/shared/camera/VID_TEST.mpg",
			CODEC_ID_MPEG1VIDEO);
}

void video_encode_example(camera_buffer_t* buf, const char *filename,
		enum CodecID codec_id)
{
	AVCodec *codec;
	AVCodecContext *c = NULL;
	int out_size, outbuf_size;
	FILE *f;
	AVFrame *picture;
	uint8_t *outbuf;
	int had_output = 0;

	printf("Encode video file %s\n", filename);

	/* find the mpeg1 video encoder */
	codec = avcodec_find_encoder(codec_id);
	if (!codec)
	{
		fprintf(stderr, "codec not found\n");
		exit(1);
	}

	c = avcodec_alloc_context3(codec);
	picture = avcodec_alloc_frame();

	/* put sample parameters */
	c->bit_rate = 400000;
	/* resolution must be a multiple of two */
//    c->width = buf->framedesc.nv12.width;
//    c->height = buf->framedesc.nv12.height;
	c->width = 352;
	c->height = 288;
	/* frames per second */
	c->time_base = (AVRational)
	{	1,25};
	c->gop_size = 10; /* emit one intra frame every ten frames */
	c->max_b_frames = 1;
	c->pix_fmt = PIX_FMT_YUV420P;

//    if(codec_id == CODEC_ID_H264)
//        av_opt_set(c->priv_data, "preset", "slow", 0);

	/* open it */
	if (avcodec_open2(c, codec, NULL) < 0)
	{
		fprintf(stderr, "could not open codec\n");
		exit(1);
	}

	f = fopen(filename, "wb");
	if (!f)
	{
		fprintf(stderr, "could not open %s\n", filename);
		exit(1);
	}

		/* alloc image and output buffer */
	outbuf_size = 100000 + 12 * c->width * c->height;
	outbuf = (uint8_t *) malloc(outbuf_size);

	/* the image can be allocated by any means and av_image_alloc() is
	 * just the most convenient way if av_malloc() is to be used */
	av_image_alloc(picture->data, picture->linesize, c->width, c->height,
			c->pix_fmt, 1);

	/* encode 1 second of video */
	int a = 0;
	for (; a < 15; a++)
	{
//		fflush(stdout);

        /* Y */
        for(int y=0;y<c->height;y++) {
            for(int x=0;x<c->width;x++) {
                picture->data[0][y * picture->linesize[0] + x] = x + y + a * 3;
            }
        }

        /* Cb and Cr */
        for(int y=0;y<c->height/2;y++) {
            for(int x=0;x<c->width/2;x++) {
                picture->data[1][y * picture->linesize[1] + x] = 128 + y + a * 2;
                picture->data[2][y * picture->linesize[2] + x] = 64 + x + a * 5;
            }
        }

//		uint8_t *y, *u, *v;
//		y = picture->data[0];
//		u = picture->data[1];
//		v = picture->data[2];
//		const uint8_t *src=buf->framebuf;
//
//		for (int i = 0; i < (c->height + 1) >> 1; i++)
//		{
//			for (int j = 0; j < (c->width + 1) >> 1; j++)
//			{
//				u[j] = *src++ ^ 0x80;
//				v[j] = *src++ ^ 0x80;
//				y[2 * j] = *src++;
//				y[2 * j + 1] = *src++;
//				y[picture->linesize[0] + 2 * j] = *src++;
//				y[picture->linesize[0] + 2 * j + 1] = *src++;
//			}
//
//			y += 2 * picture->linesize[0];
//			u += picture->linesize[1];
//			v += picture->linesize[2];
//		}

		struct SwsContext* fooContext = sws_getContext(c->width, c->height,
				PIX_FMT_YUV420P, c->width, c->height, PIX_FMT_RGB8,
				SWS_FAST_BILINEAR, NULL, NULL, NULL);

		AVFrame* outpic = avcodec_alloc_frame();
		av_image_alloc(outpic->data, outpic->linesize, c->width, c->height,
				PIX_FMT_RGB8, 1);

		sws_scale(fooContext, picture->data, picture->linesize, 0, c->height,
				outpic->data, outpic->linesize);

		/* encode the image */
		out_size = avcodec_encode_video(c, outbuf, outbuf_size, outpic);
		had_output |= out_size;
		printf("encoding frame %3d (size=%5d)\n", a, out_size);
		fwrite(outbuf, 1, out_size, f);
	}

	/* get the delayed frames */
	for (; out_size || !had_output; a++)
	{
		fflush(stdout);

		out_size = avcodec_encode_video(c, outbuf, outbuf_size, NULL);
		had_output |= out_size;
		printf("write frame %3d (size=%5d)\n", a, out_size);
		fwrite(outbuf, 1, out_size, f);
	}

		/* add sequence end code to have a real mpeg file */
	outbuf[0] = 0x00;
	outbuf[1] = 0x00;
	outbuf[2] = 0x01;
	outbuf[3] = 0xb7;
	fwrite(outbuf, 1, 4, f);
	fclose(f);
	free(outbuf);

	avcodec_close(c);
	av_free(c);
	av_free(picture->data[0]);
	av_free(picture);
	printf("\n");
}

 

I am using this with the HelloVideoCamera sample, so if you want to run it you can plug the callback into that.

Please use plain text.
BlackBerry Development Advisor
smcveigh
Posts: 668
Registered: ‎11-29-2011
My Device: developer
My Carrier: other

Re: Camera API NV12 frame to AVFrame (FFmpeg)

YUV420P and NV12 should be reasonably similar.  Both are YCbCr 4:2:0 formats.

If you can direct me to some docs which indicate what formats ffmpeg can take as inputs, I can probably give you some help

 

Cheers,

Sean

Please use plain text.
BlackBerry Development Advisor
smcveigh
Posts: 668
Registered: ‎11-29-2011
My Device: developer
My Carrier: other

Re: Camera API NV12 frame to AVFrame (FFmpeg)

So I did a bit of preliminary investigation and it sounds like the following fields of the AVFrame struct are of interest in an NV12->YUV420P conversion:

 

 

uint8_t* data[];
uint8_t linesize[];
int width;
int height;

 

 

Sounds like in the YUV420P case, data[0] is a pointer to the Y pixel plane, data[1] is a pointer to the U pixel plane, and data[2] is a pointer to the V pixel plane.

You will note that in the NV12 format, there are only 2 planes: a Y plane and a combined UV plane.  The trick in this conversion process is going to be de-interleaving the U and V values from the combined UV plane into separate U and V planes.

The Y plane should be usable as-is.  You shouldn't even need to copy any of the pixel data.

 

picture->data[0] = buf->framebuf;
picture->linesize[0] = buf->framedesc.nv12.stride;

The above code should be sufficient to set up the Y plane.  If you really want, you could malloc the data[0] pixel plane and then memcpy() the Y data (line-by-line!) from buf->framebuf over, but this is likely a waste of time.  I notice you were using av_image_alloc(), which you will probably want to skip since you likely only want to alloc the data[1] and data[2] planes and will probably have to do that by hand.. you may want to consider implementing a pool rather than going to malloc() in realtime.

In any event, once you have the data[1] and data[2] planes malloc()'d, you should be able to do a de-interleave and copy of the U and V data from the NV12 buffer as follows:

uint8_t* srcuv = &buf->framebuf[buf->framedesc.nv12.uv_offset];
uint8_t* destu = picture->data[1];
uint8_t* destv = picture->data[2];

picture->linesize[1] = buf->framedesc.nv12.width / 2;
picture->linesize[2] = picture->linesize[1];

for (i=0; i<buf->framedesc.nv12.height/2; i++) { uint8_t* curuv = srcuv; for (j=0; i<buf->framedesc.nv12.width/2; j++) { *destu++ = *curuv++; *destv++ = *curuv++; } srcuv += buf->framedesc.nv12.stride; // uv_stride in later API versions }

Note that I'm assuming a de-strided U and V plane is desirable.  If your allocator for the data[1] and data[2] planes insists on striding, then stride the dest pointers as necessary at the end of the "j" loop.

 

Now you should have a YUV420P frame that will be compatible with your encoder.  Or at least this is how I interpret whatever headers I was looking at :smileyhappy:

 

Cheers,

Sean

 

 

 

Please use plain text.
Developer
mreed
Posts: 1,041
Registered: ‎07-16-2008
My Device: ಠ_ಠ

Re: Camera API NV12 frame to AVFrame (FFmpeg)

[ Edited ]

Thanks Sean!

 

So far it looks like its working. I'm only repeating one frame right now, but it appears to be encoding correctly, and the image is recognizable when I run the video file!

 

(btw there is a typo in your j loop where you compare i instead.)

 

=)

Please use plain text.
BlackBerry Development Advisor
smcveigh
Posts: 668
Registered: ‎11-29-2011
My Device: developer
My Carrier: other

Re: Camera API NV12 frame to AVFrame (FFmpeg)

good to hear. 

oh, I'm sure I had typos.  I should have used the "I'm entering code into a textbox" disclaimer.

Please use plain text.
Developer
mreed
Posts: 1,041
Registered: ‎07-16-2008
My Device: ಠ_ಠ

Re: Camera API NV12 frame to AVFrame (FFmpeg)

I think *curuv++ should be *curuv

 

*destu++ = *curuv;
*destv++ = *curuv++;

 

Please use plain text.
BlackBerry Development Advisor
smcveigh
Posts: 668
Registered: ‎11-29-2011
My Device: developer
My Carrier: other

Re: Camera API NV12 frame to AVFrame (FFmpeg)

No.

The UV plane is:

curuv="UVUVUVUV...."

and you want:

destu="UUUUU..."

destv="VVVVV..."

 

If you don't advance the pointer, you get:

destu="UVUVUV..."

destv="UVUVUV..."

which is not a U plane and a V plane.

 

Please use plain text.
Developer
mreed
Posts: 1,041
Registered: ‎07-16-2008
My Device: ಠ_ಠ

Re: Camera API NV12 frame to AVFrame (FFmpeg)

[ Edited ]

Yea, ok, makes sense. I thought I had with working last night before I posted, but it must've been a fluke because I can't get back to that point.

 

I'm trying to add frames to the video as they come in now, rather than repeating just one like before... but I'm having some issues. I've attached some screenshots. When I add just the U plane or V plane, or both, I get the correct video dimensions, but when I add in the Y plane it just collapses.

 

Codes:

void HelloVideoCameraApp::video_encode_example(camera_buffer_t* buf)
{
	if (!record || endFile)
		return;

	pthread_mutex_lock(&mutex);

	if (!record || endFile)
	{
		pthread_mutex_unlock(&mutex);
		return;
	}

	AVFrame *frame = avcodec_alloc_frame();

	frame->linesize[0] = buf->framedesc.nv12.stride;
	frame->linesize[1] = pVideoCodec->width / 2;
	frame->linesize[2] = frame->linesize[1];

	int uvBufferSize = frame->linesize[1] * (pVideoCodec->height / 2);

	frame->data[0] = buf->framebuf;
//	frame->data[0] = (uint8_t*) av_mallocz(frame->linesize[0] * pVideoCodec->height);
	frame->data[1] = (uint8_t*) av_mallocz(2 * uvBufferSize);
	frame->data[2] = frame->data[1] + uvBufferSize;

	uint8_t *srcuv = buf->framebuf + buf->framedesc.nv12.uv_offset;
	uint8_t *destu = frame->data[1];
	uint8_t *destv = frame->data[2];

	for (int i = 0; i < pVideoCodec->height / 2; i++)
	{
		uint8_t* curuv = srcuv;
		for (int j = 0; j < pVideoCodec->width / 2; j++)
		{
			*destu++ = *curuv++;
			*destv++ = *curuv++;
		}
		srcuv += buf->framedesc.nv12.stride; // uv_stride in later API versions
	}

	int nOutputSize = avcodec_encode_video(pVideoCodec, pVideoEncodeBuffer,
			nSizeVideoEncodeBuffer, frame);
	if (nOutputSize > 0)
	{
		printf("write frame %d (size=%5d)\n", a++, nOutputSize);
		fflush(stdout);

		fwrite(pVideoEncodeBuffer, 1, nOutputSize, file);
	}

	if (endFile)
	{
		for (int out_size = 1; out_size > 0; a++)
		{
			out_size = avcodec_encode_video(pVideoCodec, pVideoEncodeBuffer,
					nSizeVideoEncodeBuffer, NULL);
			fwrite(pVideoEncodeBuffer, 1, out_size, file);
			printf("write (delayed) frame %d (size=%5d)\n", a, out_size);
			fflush(stdout);
		}

		/* add sequence end code to have a real mpeg file */
		pVideoEncodeBuffer[0] = 0x00;
		pVideoEncodeBuffer[1] = 0x00;
		pVideoEncodeBuffer[2] = 0x01;
		pVideoEncodeBuffer[3] = 0xb7;
		fwrite(pVideoEncodeBuffer, 1, 4, file);

		fclose(file);
		file = NULL;

		avcodec_close(pVideoCodec);
		av_free(pVideoCodec);

		av_free(pVideoEncodeBuffer);
		pVideoEncodeBuffer = NULL;
		nSizeVideoEncodeBuffer = 0;
	}

//	av_free(frame->data[0]);
	av_free(frame->data[1]);
	av_free(frame);
	pthread_mutex_unlock(&mutex);
}

 

The screenshots are of recording the Chrome logo:

http://www.blogcdn.com/downloadsquad.switched.com/media/2011/03/chrome-logo-1301044215.jpg

Please use plain text.
BlackBerry Development Advisor
smcveigh
Posts: 668
Registered: ‎11-29-2011
My Device: developer
My Carrier: other

Re: Camera API NV12 frame to AVFrame (FFmpeg)

Can you dump the complete NV12 frame to disk and the complete Y, U, and V planes as well?  Just plain .bin files would be fine.

Y plane size should be linesize[0] * height, U and V planes should be (width*height)/4 bytes.

NV12 buffer will be uv_offset+stride*height/2 bytes starting at buf->framebuf.   (note: don't use this convention in practise, as there's no guarantee you won't bus error in there).

I will take a look at your files and see if there is anything wrong with the conversion.  If not, then the problem is something ffmpeg-related.

 

Cheers,

Sean

Please use plain text.
Developer
mreed
Posts: 1,041
Registered: ‎07-16-2008
My Device: ಠ_ಠ

Re: Camera API NV12 frame to AVFrame (FFmpeg)

[ Edited ]

K, attached as frame.zip. Thanks.

 

EDIT: I think I did the wrong y plane, one sec...

 

FILE *binFile = fopen("/accounts/1000/shared/camera/nv12.bin", "wb");
if (binFile)
{
	int size = buf->framedesc.nv12.uv_offset
			+ buf->framedesc.nv12.stride * pVideoCodec->height / 2;
	fwrite(buf->framebuf, 1, size, binFile);
	fclose(binFile);
}

binFile = fopen("/accounts/1000/shared/camera/y.bin", "wb");
if (binFile)
{
	int size = frame->linesize[0] * pVideoCodec->height;
	fwrite(frame->data[0], 1, size, binFile);
	fclose(binFile);
}

binFile = fopen("/accounts/1000/shared/camera/u.bin", "wb");
if (binFile)
{
	int size = (pVideoCodec->width * pVideoCodec->height) / 4;
	fwrite(frame->data[1], 1, size, binFile);
	fclose(binFile);
}

binFile = fopen("/accounts/1000/shared/camera/v.bin", "wb");
if (binFile)
{
	int size = (pVideoCodec->width * pVideoCodec->height) / 4;
	fwrite(frame->data[2], 1, size, binFile);
	fclose(binFile);
}

 

Please use plain text.