ppm vhook follow up patch by (Charles Yates <charles dot yates at pandora dot be>)

mostly tab -> space cosmetics

Originally committed as revision 2313 to svn://svn.ffmpeg.org/ffmpeg/trunk

vhook/Makefile

@@ -7,7 +7,7 @@ CFLAGS=-fPIC $(OPTFLAGS) -Wall -I.. -I$(SRC_PATH) -I$(SRC_PATH)/libavformat -I$(
 ifeq ($(CONFIG_DARWIN),yes)
 SHFLAGS+=-bundle -flat_namespace -undefined suppress
 endif
-HOOKS=null.so fish.so
+HOOKS=null.so fish.so ppm.so
 
 ifeq ($(HAVE_IMLIB2),yes)
     HOOKS += imlib2.so

vhook/ppm.c

@@ -111,37 +111,9 @@ FILE *rwpipe_writer( rwpipe *this )
         return NULL;
 }
 
-/** Close the pipe and process.
+/* Read a number from the pipe - assumes PNM style headers.
 */
 
-void rwpipe_close( rwpipe *this )
-{
-	if ( this != NULL )
-	{
-		fclose( this->reader );
-		fclose( this->writer );
-		waitpid( this->pid, NULL, 0 );
-		av_free( this );
-	}
-}
-
-typedef struct 
-{
-	rwpipe *rw;
-} 
-ContextInfo;
-
-int Configure(void **ctxp, int argc, char *argv[])
-{
-    *ctxp = av_mallocz(sizeof(ContextInfo));
-	if ( ctxp != NULL && argc > 1 )
-	{
-		ContextInfo *info = (ContextInfo *)*ctxp;
-		info->rw = rwpipe_open( argc - 1, &argv[ 1 ] );
-	}
-    return 0;
-}
-
 int rwpipe_read_number( rwpipe *rw )
 {
     int value = 0;
@@ -170,6 +142,9 @@ int rwpipe_read_number( rwpipe *rw )
     return value;
 }
 
+/** Read a PPM P6 header.
+*/
+
 int rwpipe_read_ppm_header( rwpipe *rw, int *width, int *height )
 {
     char line[ 3 ];
@@ -187,12 +162,58 @@ int rwpipe_read_ppm_header( rwpipe *rw, int *width, int *height )
     return 1;
 }
 
+/** Close the pipe and process.
+*/
+
+void rwpipe_close( rwpipe *this )
+{
+    if ( this != NULL )
+    {
+        fclose( this->reader );
+        fclose( this->writer );
+        waitpid( this->pid, NULL, 0 );
+        av_free( this );
+    }
+}
+
+/** Context info for this vhook - stores the pipe and image buffers.
+*/
+
+typedef struct 
+{
+    rwpipe *rw;
+    int size1;
+    char *buf1;
+    int size2;
+    char *buf2;
+} 
+ContextInfo;
+
+/** Initialise the context info for this vhook.
+*/
+
+int Configure(void **ctxp, int argc, char *argv[])
+{
+    if ( argc > 1 )
+    {
+        *ctxp = av_mallocz(sizeof(ContextInfo));
+        if ( ctxp != NULL && argc > 1 )
+        {
+            ContextInfo *info = (ContextInfo *)*ctxp;
+            info->rw = rwpipe_open( argc - 1, &argv[ 1 ] );
+            return 0;
+        }
+    }
+    return 1;
+}
+
+/** Process a frame.
+*/
+
 void Process(void *ctx, AVPicture *picture, enum PixelFormat pix_fmt, int width, int height, int64_t pts)
 {
     int err = 0;
     ContextInfo *ci = (ContextInfo *) ctx;
-    char *buf1 = 0;
-    char *buf2 = 0;
     AVPicture picture1;
     AVPicture picture2;
     AVPicture *pict = picture;
@@ -203,20 +224,31 @@ void Process(void *ctx, AVPicture *picture, enum PixelFormat pix_fmt, int width,
     FILE *in = rwpipe_reader( ci->rw );
     FILE *out = rwpipe_writer( ci->rw );
 
+    /* Check that we have a pipe to talk to. */
+    if ( in == NULL || out == NULL )
+        err = 1;
+
     /* Convert to RGB24 if necessary */
-    if (pix_fmt != PIX_FMT_RGB24) {
+    if ( !err && pix_fmt != PIX_FMT_RGB24 ) 
-        int size;
+    {
+        int size = avpicture_get_size(PIX_FMT_RGB24, width, height);
 
-        size = avpicture_get_size(PIX_FMT_RGB24, width, height);
+        if ( size != ci->size1 )
-        buf1 = av_malloc(size);
+        {
+            av_free( ci->buf1 );
+            ci->buf1 = av_malloc(size);
+            ci->size1 = size;
+            err = ci->buf1 == NULL;
+        }
 
-        avpicture_fill(&picture1, buf1, PIX_FMT_RGB24, width, height);
+        if ( !err )
-        if (img_convert(&picture1, PIX_FMT_RGB24, 
+        {
-                        picture, pix_fmt, width, height) < 0) {
+            avpicture_fill(&picture1, ci->buf1, PIX_FMT_RGB24, width, height);
+            if (img_convert(&picture1, PIX_FMT_RGB24, picture, pix_fmt, width, height) < 0)
                 err = 1;
-        }
             pict = &picture1;
         }
+    }
 
     /* Write out the PPM */
     if ( !err )
@@ -236,8 +268,18 @@ void Process(void *ctx, AVPicture *picture, enum PixelFormat pix_fmt, int width,
     if ( !err && !rwpipe_read_ppm_header( ci->rw, &out_width, &out_height ) )
     {
         int size = avpicture_get_size(PIX_FMT_RGB24, out_width, out_height);
-        buf2 = av_malloc(size);
+
-        avpicture_fill(&picture2, buf2, PIX_FMT_RGB24, out_width, out_height);
+        if ( size != ci->size2 )
+        {
+            av_free( ci->buf2 );
+            ci->buf2 = av_malloc(size);
+            ci->size2 = size;
+            err = ci->buf2 == NULL;
+        }
+
+        if ( !err )
+        {
+            avpicture_fill(&picture2, ci->buf2, PIX_FMT_RGB24, out_width, out_height);
             ptr = picture2.data[ 0 ];
             for ( i = 0; !err && i < out_height; i ++ )
             {
@@ -245,18 +287,33 @@ void Process(void *ctx, AVPicture *picture, enum PixelFormat pix_fmt, int width,
                 ptr += picture2.linesize[ 0 ];
             }
         }
+    }
 
     /* Convert the returned PPM back to the input format */
     if ( !err )
     {
-        if (img_convert(picture, pix_fmt, &picture2, PIX_FMT_RGB24, width, height) < 0) {
+        /* Actually, this is wrong, since the out_width/out_height returned from the
+         * filter won't necessarily be the same as width and height - img_resample 
+         * won't scale rgb24, so the only way out of this is to convert to something 
+         * that img_resample does like [which may or may not be pix_fmt], rescale 
+         * and finally convert to pix_fmt... slow, but would provide the most flexibility.
+         *
+         * Currently, we take the upper left width/height pixels from the filtered image,
+         * smaller images are going to be corrupted or cause a crash.
+         *
+         * Finally, what should we do in case of this call failing? Up to now, failures
+         * are gracefully ignored and the original image is returned - in this case, a
+         * failure may corrupt the input.
+         */
+        if (img_convert(picture, pix_fmt, &picture2, PIX_FMT_RGB24, width, height) < 0) 
+        {
         }
     }
-
-    av_free(buf1);
-    av_free(buf2);
 }
 
+/** Clean up the effect.
+*/
+
 void Release(void *ctx)
 {
     ContextInfo *ci;
@@ -265,6 +322,8 @@ void Release(void *ctx)
     if (ctx)
     {
         rwpipe_close( ci->rw );
+        av_free( ci->buf1 );
+        av_free( ci->buf2 );
         av_free(ctx);
     }
 }