avcodec/libx264: Check for av_malloc() failure

Fixes CID1257813
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/libx264.c

@@ -700,6 +700,8 @@ static av_cold int X264_init(AVCodecContext *avctx)
 
         s = x264_encoder_headers(x4->enc, &nal, &nnal);
         avctx->extradata = p = av_malloc(s);
+        if (!p)
+            goto nomem;
 
         for (i = 0; i < nnal; i++) {
             /* Don't put the SEI in extradata. */
@@ -707,6 +709,8 @@ static av_cold int X264_init(AVCodecContext *avctx)
                 av_log(avctx, AV_LOG_INFO, "%s\n", nal[i].p_payload+25);
                 x4->sei_size = nal[i].i_payload;
                 x4->sei      = av_malloc(x4->sei_size);
+                if (!x4->sei)
+                    goto nomem;
                 memcpy(x4->sei, nal[i].p_payload, nal[i].i_payload);
                 continue;
             }
@@ -717,6 +721,9 @@ static av_cold int X264_init(AVCodecContext *avctx)
     }
 
     return 0;
+nomem:
+    X264_close(avctx);
+    return AVERROR(ENOMEM);
 }
 
 static const enum AVPixelFormat pix_fmts_8bit[] = {