Enforce constraints at the start of nonlinear fitting

Bug: oss-fuzz:8130
Change-Id: I4dd5a2d85a9efa4b4bcaf83c520ef5d2168f997e
Reviewed-on: https://skia-review.googlesource.com/125732
Commit-Queue: Brian Osman <brianosman@google.com>
Commit-Queue: Mike Klein <mtklein@google.com>
Reviewed-by: Mike Klein <mtklein@google.com>
diff --git a/profiles/fuzz/inverse_tf_adb_negative.icc b/profiles/fuzz/inverse_tf_adb_negative.icc
new file mode 100644
index 0000000..667e5af
--- /dev/null
+++ b/profiles/fuzz/inverse_tf_adb_negative.icc
Binary files differ
diff --git a/profiles/fuzz/inverse_tf_adb_negative.icc.txt b/profiles/fuzz/inverse_tf_adb_negative.icc.txt
new file mode 100644
index 0000000..19fb7c9
--- /dev/null
+++ b/profiles/fuzz/inverse_tf_adb_negative.icc.txt
@@ -0,0 +1,38 @@
+                Size : 0x00000A34 : 2612
+    Data color space : 0x20202020 : '    '
+                 PCS : 0x58595A20 : 'XYZ '
+           Tag count : 0x0000000B : 11
+
+ Tag    : Type   : Size   : Offset
+ ------ : ------ : ------ : --------
+ '    ' : '    ' :     32 : 288
+ '    ' : '    ' :     32 : 288
+ '    ' : '    ' :     32 : 288
+ '    ' : '    ' :     32 : 288
+ 'rXYZ' : 'XYZ ' :     32 : 448
+ 'gXYZ' : 'XYZ ' :     32 : 468
+ 'bXYZ' : 'XYZ ' :     32 : 488
+ 'rTRC' : 'curv' :   2080 : 508
+ '    ' : '' :     32 : 2568
+ 'bTRC' : 'curv' :   2080 : 508
+ 'gTRC' : 'curv' :   2080 : 508
+
+rTRC : 16-bit table with 1024 entries
+gTRC : 16-bit table with 1024 entries
+bTRC : 16-bit table with 1024 entries
+ XYZ : | 8224.125000000 8224.125000000 8224.125000000 |
+       | 8224.125000000 8224.125000000 8224.125000000 |
+       | -0.003417969 8224.125000000 8224.125000000 |
+252 random bytes transformed to linear XYZD50 bytes:
+	ffffff ffffff ffffff ffffff ffffff ffffff ffffff
+	ffffff ffffff ffffff ffffff ffffff ffffff ffffff
+	ffffff ffffff ffffff ffffff ffffff ffffff ffffff
+	ffffff ffffff ffffff ffffff ffffff ffffff ffffff
+	ffffff ffffff ffffff ffffff ffffff ffffff ffffff
+	ffffff ffffff ffffff ffffff ffffff ffffff ffffff
+	ffffff ffffff ffffff ffffff ffffff ffffff ffffff
+	ffffff ffffff ffffff ffffff ffffff ffffff ffffff
+	ffffff ffffff ffffff ffffff ffffff ffffff ffffff
+	ffffff ffffff ffffff ffffff ffffff ffffff ffffff
+	ffffff ffffff ffffff ffffff ffffff ffffff ffffff
+	ffffff ffffff ffffff ffffff ffffff ffffff ffffff
diff --git a/src/TransferFunction.c b/src/TransferFunction.c
index df728e1..c1fce37 100644
--- a/src/TransferFunction.c
+++ b/src/TransferFunction.c
@@ -238,6 +238,14 @@
 
     for (int j = 0; j < 3/*TODO: tune*/; j++) {
         // These extra constraints a >= 0 and ad+b >= 0 are not modeled in the optimization.
+        // We don't really know how to fix up a if it goes negative.
+        if (P[1] < 0) {
+            return false;
+        }
+        // If ad+b goes negative, we feel just barely not uneasy enough to tweak b so ad+b is zero.
+        if (P[1] * tf->d + P[2] < 0) {
+            P[2] = -P[1] * tf->d;
+        }
         assert (P[1] >= 0 &&
                 P[1] * tf->d + P[2] >= 0);
 
@@ -247,15 +255,14 @@
                                      start*x_scale, 1, N-start)) {
             return false;
         }
+    }
 
-        // We don't really know how to fix up a if it goes negative.
-        if (P[1] < 0) {
-            return false;
-        }
-        // If ad+b goes negative, we feel just barely not uneasy enough to tweak b so ad+b is zero.
-        if (P[1] * tf->d + P[2] < 0) {
-            P[2] = -P[1] * tf->d;
-        }
+    // We need to apply our fixups one last time
+    if (P[1] < 0) {
+        return false;
+    }
+    if (P[1] * tf->d + P[2] < 0) {
+        P[2] = -P[1] * tf->d;
     }
 
     tf->g = P[0];
diff --git a/tests.c b/tests.c
index a6154f9..6015db1 100644
--- a/tests.c
+++ b/tests.c
@@ -513,6 +513,9 @@
 
     // Once caused skcms_PolyTF fit to round trip an index to infinity.
     "profiles/fuzz/infinite_roundtrip.icc",           // oss-fuzz:8101
+
+    // Caused skcms_ApproximateCurve to violate the a*d+b >= 0 constraint.
+    "profiles/fuzz/inverse_tf_adb_negative.icc",      // oss-fuzz:8130
 };
 
 static void test_Parse(bool regen) {