skcms: Increase transform op capacity and add bounds checks

Complex color transforms (e.g. A2B to B2A with 4-channel curve tables,
matrices, format and space conversion) can require ~40 ops, which
exceeded the previous hardcoded limit of 32 ops and resulted in a
stack buffer overflow.

Increase SKCMS_MAX_PROGRAM_OPS to 64, providing ample padding while
keeping stack overhead small. Harden add_op and add_op_ctx to avoid
writing past the program buffer during compilation, and add runtime
checks in both skcms_Transform and run_program to gracefully reject
programs that exceed the maximum capacity.

This limit is only hit when doing an A2B to B2A conversion, which
Chromium and Android do not do.

Fixed: b/530238257
TAG=agy
CONV=56f173c3-5496-4fd5-ad6b-34aebcff742d
Change-Id: I1868e43405dcd109c01c007ee0ade4d0aebe3198
Reviewed-on: https://skia-review.googlesource.com/c/skcms/+/1351256
Reviewed-by: Florin Malita <fmalita@google.com>
Commit-Queue: Christopher Cameron <ccameron@google.com>

skcms.cc

diff --git a/skcms.cc b/skcms.cc
index 692e6af..c23dbc1 100644
--- a/skcms.cc
+++ b/skcms.cc
@@ -2767,20 +2767,28 @@
     }
     // TODO: more careful alias rejection (like, dst == src + 1)?
 
-    Op          program[32];
-    const void* context[32];
+    Op          program[SKCMS_MAX_PROGRAM_OPS];
+    const void* context[SKCMS_MAX_PROGRAM_OPS];
 
     Op*          ops      = program;
     const void** contexts = context;
 
     auto add_op = [&](Op o) {
-        *ops++ = o;
-        *contexts++ = nullptr;
+        if (ops < program + ARRAY_COUNT(program)) {
+            *ops = o;
+            *contexts = nullptr;
+        }
+        ops++;
+        contexts++;
     };
 
     auto add_op_ctx = [&](Op o, const void* c) {
-        *ops++ = o;
-        *contexts++ = c;
+        if (ops < program + ARRAY_COUNT(program)) {
+            *ops = o;
+            *contexts = c;
+        }
+        ops++;
+        contexts++;
     };
 
     auto add_curve_ops = [&](const skcms_Curve* curves, int numChannels) -> bool {
@@ -3099,6 +3107,9 @@
 
     assert(ops      <= program + ARRAY_COUNT(program));
     assert(contexts <= context + ARRAY_COUNT(context));
+    if (ops > program + ARRAY_COUNT(program)) {
+        return false;
+    }
 
     auto run = baseline::run_program;
     switch (cpu_type()) {

src/Transform_inl.h

diff --git a/src/Transform_inl.h b/src/Transform_inl.h
index 53a35ef..51f369d 100644
--- a/src/Transform_inl.h
+++ b/src/Transform_inl.h
@@ -1565,8 +1565,11 @@
                 size_t src_bpp, size_t dst_bpp) {
 #if SKCMS_HAS_MUSTTAIL
     // Convert the program into an array of tailcall stages.
-    StageFn stages[32];
+    StageFn stages[SKCMS_MAX_PROGRAM_OPS];
     assert(programSize <= ARRAY_COUNT(stages));
+    if (programSize > ARRAY_COUNT(stages)) {
+        return;
+    }
 
     static constexpr StageFn kStageFns[] = {
 #define M(name) &Exec_##name,

src/skcms_Transform.h

diff --git a/src/skcms_Transform.h b/src/skcms_Transform.h
index 868d71a..c29b524 100644
--- a/src/skcms_Transform.h
+++ b/src/skcms_Transform.h
@@ -118,6 +118,12 @@
 
 /** Constants */
 
+// The maximum number of ops in a transform program.
+// A complex transform (e.g. A2B to B2A with 4 channels, multi-stage curves, matrices,
+// format conversion and colorspace conversion) can require up to ~40 ops.
+// 64 gives plenty of padding while keeping stack frames small.
+static constexpr int SKCMS_MAX_PROGRAM_OPS = 64;
+
 #if defined(__clang__) || defined(__GNUC__)
     static constexpr float INFINITY_ = __builtin_inff();
 #else