Fixing undefined behavior in basisu_etc.cpp
The value inside the static cast can be negative. Casting a negative floating point value to an unsigned integer is an undefined behavior and we were running into issues when executing the code with an address sanitizer turned on.
Not sure if this is the proper way to fix this but it seems to work ok. Either way, the undefined behavior should be fixed.
diff --git a/basisu_etc.cpp b/basisu_etc.cpp
index 244f1d2..2a23afb 100644
--- a/basisu_etc.cpp
+++ b/basisu_etc.cpp
@@ -730,9 +730,9 @@
const float avg_delta_g_f = static_cast<float>(delta_sum_g) / 8;
const float avg_delta_b_f = static_cast<float>(delta_sum_b) / 8;
- const int br1 = clamp<int>(static_cast<uint32_t>((m_avg_color[0] - avg_delta_r_f) * m_limit / 255.0f + .5f), 0, m_limit);
- const int bg1 = clamp<int>(static_cast<uint32_t>((m_avg_color[1] - avg_delta_g_f) * m_limit / 255.0f + .5f), 0, m_limit);
- const int bb1 = clamp<int>(static_cast<uint32_t>((m_avg_color[2] - avg_delta_b_f) * m_limit / 255.0f + .5f), 0, m_limit);
+ const int br1 = clamp<int>(static_cast<int32_t>((m_avg_color[0] - avg_delta_r_f) * m_limit / 255.0f + .5f), 0, m_limit);
+ const int bg1 = clamp<int>(static_cast<int32_t>((m_avg_color[1] - avg_delta_g_f) * m_limit / 255.0f + .5f), 0, m_limit);
+ const int bb1 = clamp<int>(static_cast<int32_t>((m_avg_color[2] - avg_delta_b_f) * m_limit / 255.0f + .5f), 0, m_limit);
#if BASISU_DEBUG_ETC_ENCODER_DEEPER
printf("Second refinement trial %u, avg_delta %f %f %f\n", i, avg_delta_r_f, avg_delta_g_f, avg_delta_b_f);
