diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 690efb1a45..64f10f95fe 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -1008,7 +1008,7 @@ static inline void pred_direct_motion(H264Context * const h, int *mb_type){
             int col_parity = FFABS(col_poc[0] - cur_poc) >= FFABS(col_poc[1] - cur_poc);
             mb_xy= s->mb_x + ((s->mb_y&~1) + col_parity)*s->mb_stride;
             b8_stride = 0;
-        }else if(!(s->picture_structure & h->ref_list[1][0].reference)){// FL -> FL & differ parity
+        }else if(!(s->picture_structure & h->ref_list[1][0].reference) && !h->ref_list[1][0].mbaff){// FL -> FL & differ parity
             int fieldoff= 2*(h->ref_list[1][0].reference)-3;
             mb_xy += s->mb_stride*fieldoff;
         }