diff --git a/libavcodec/motion_est.c b/libavcodec/motion_est.c
index 775544b26a..f9c0d0a601 100644
--- a/libavcodec/motion_est.c
+++ b/libavcodec/motion_est.c
@@ -1978,6 +1978,8 @@ void ff_estimate_b_frame_motion(MpegEncContext * s,
         }
          //FIXME something smarter
         if(dmin>256*256*16) type&= ~CANDIDATE_MB_TYPE_DIRECT; //dont try direct mode if its invalid for this MB
+        if(s->codec_id == CODEC_ID_MPEG4 && type&CANDIDATE_MB_TYPE_DIRECT && s->flags&CODEC_FLAG_MV0 && *(uint32_t*)s->b_direct_mv_table[xy])
+            type |= CANDIDATE_MB_TYPE_DIRECT0;
 #if 0
         if(s->out_format == FMT_MPEG1)
             type |= CANDIDATE_MB_TYPE_INTRA;
diff --git a/libavcodec/motion_est_template.c b/libavcodec/motion_est_template.c
index 1bfda10472..897c08e3df 100644
--- a/libavcodec/motion_est_template.c
+++ b/libavcodec/motion_est_template.c
@@ -1025,7 +1025,7 @@ static av_always_inline int epzs_motion_search_internal(MpegEncContext * s, int
     score_map[0]= dmin;
 
     //FIXME precalc first term below?
-    if((s->pict_type == B_TYPE  || s->flags&CODEC_FLAG_MV0) && !(c->flags & FLAG_DIRECT))
+    if((s->pict_type == B_TYPE && !(c->flags & FLAG_DIRECT)) || s->flags&CODEC_FLAG_MV0)
         dmin += (mv_penalty[pred_x] + mv_penalty[pred_y])*penalty_factor;
 
     /* first line */
diff --git a/tests/ffmpeg.regression.ref b/tests/ffmpeg.regression.ref
index 06d2ca81fb..8ff4c5e172 100644
--- a/tests/ffmpeg.regression.ref
+++ b/tests/ffmpeg.regression.ref
@@ -75,14 +75,14 @@ stddev: 10.40 PSNR:27.78 bytes:7602176
 600188 ./data/a-mpeg4-adv.avi
 60edc5a67271e425d0a2a52981895b81 *./data/out.yuv
 stddev: 10.25 PSNR:27.91 bytes:7602176
-8cbcdc72a33e5a8d624f38529efd93c8 *./data/a-mpeg4-qprd.avi
-657582 ./data/a-mpeg4-qprd.avi
-5b620f592a795b1caad323ab3fab0859 *./data/out.yuv
-stddev: 12.12 PSNR:26.45 bytes:7602176
-7573ec318b965cfb83da45daa23a9fc2 *./data/a-mpeg4-adap.avi
-400752 ./data/a-mpeg4-adap.avi
-820c4231d4fd192a93102f17f163d621 *./data/out.yuv
-stddev: 14.65 PSNR:24.80 bytes:7602176
+d099307d14c1a4daa145618ca0522888 *./data/a-mpeg4-qprd.avi
+657996 ./data/a-mpeg4-qprd.avi
+8a52c562082bad78cabb40ffa292ceec *./data/out.yuv
+stddev: 12.12 PSNR:26.44 bytes:7602176
+9192b10ae298ba325d53abf7b5b91ba3 *./data/a-mpeg4-adap.avi
+400650 ./data/a-mpeg4-adap.avi
+0ce1d1fbebc9e9178e1a1d4a32a5804c *./data/out.yuv
+stddev: 14.66 PSNR:24.80 bytes:7602176
 5fff534f0b958547dfdb811d4f289931 *./data/a-mpeg4-thread.avi
 761170 ./data/a-mpeg4-thread.avi
 fe1d119938f8a26174b38eeaa18dff85 *./data/out.yuv
diff --git a/tests/rotozoom.regression.ref b/tests/rotozoom.regression.ref
index d1aac7073c..643ec3c4f6 100644
--- a/tests/rotozoom.regression.ref
+++ b/tests/rotozoom.regression.ref
@@ -75,14 +75,14 @@ stddev:  4.24 PSNR:35.56 bytes:7602176
 173590 ./data/a-mpeg4-adv.avi
 699edf05648fdc42196b7bebef9be269 *./data/out.yuv
 stddev:  4.84 PSNR:34.41 bytes:7602176
-edd969be2816c13ae078b7a0416a5715 *./data/a-mpeg4-qprd.avi
-234852 ./data/a-mpeg4-qprd.avi
-51fa46add28e145c1a5ce6b27a4c57b8 *./data/out.yuv
-stddev:  4.24 PSNR:35.57 bytes:7602176
-0784800b9914cf69a6a3dc950747d376 *./data/a-mpeg4-adap.avi
-204726 ./data/a-mpeg4-adap.avi
-3172924d9449b83586e30ae73c86be2a *./data/out.yuv
-stddev:  4.04 PSNR:35.98 bytes:7602176
+1e12bb209dae0ab4b64265b0a4262257 *./data/a-mpeg4-qprd.avi
+234048 ./data/a-mpeg4-qprd.avi
+1ec355ffb30d2adf302a550cf5812636 *./data/out.yuv
+stddev:  4.23 PSNR:35.58 bytes:7602176
+d581d6f4a331325905b8ffb05cd3bfd3 *./data/a-mpeg4-adap.avi
+204284 ./data/a-mpeg4-adap.avi
+c2c7f1c7844ab92d34247ccb70198c61 *./data/out.yuv
+stddev:  4.04 PSNR:35.99 bytes:7602176
 8cd8940d7451925784536fe9b2f2a5e3 *./data/a-mpeg4-thread.avi
 254260 ./data/a-mpeg4-thread.avi
 d160a4224ea1af66c85178912f8d3a7c *./data/out.yuv