diff --git a/libswscale/ops_optimizer.c b/libswscale/ops_optimizer.c
index abba9df2ff..520069f859 100644
--- a/libswscale/ops_optimizer.c
+++ b/libswscale/ops_optimizer.c
@@ -518,9 +518,9 @@ retry:
 
         case SWS_OP_DITHER:
             for (int i = 0; i < 4; i++) {
-                if (next->comps.unused[i] || op->dither.y_offset[i] < 0)
+                if (op->dither.y_offset[i] < 0)
                     continue;
-                if (prev->comps.flags[i] & SWS_COMP_EXACT) {
+                if (next->comps.unused[i] || (prev->comps.flags[i] & SWS_COMP_EXACT)) {
                     op->dither.y_offset[i] = -1; /* unnecessary dither */
                     goto retry;
                 } else {
diff --git a/tests/ref/fate/sws-ops-list b/tests/ref/fate/sws-ops-list
index 13049a0c14..c83105406f 100644
--- a/tests/ref/fate/sws-ops-list
+++ b/tests/ref/fate/sws-ops-list
@@ -1 +1 @@
-a312bd79cadff3e2e02fd14ae7e54e26
+121d60ee0261ca5d9650a8d1e9e8a060