TurboJPEG: Properly handle gigapixel images
Prevent several integer overflow issues and subsequent segfaults that
occurred when attempting to compress or decompress gigapixel images with
the TurboJPEG API:
- Modify tjBufSize(), tjBufSizeYUV2(), and tjPlaneSizeYUV() to avoid
integer overflow when computing the return values and to return an
error if such an overflow is unavoidable.
- Modify tjunittest to validate the above.
- Modify tjCompress2(), tjEncodeYUVPlanes(), tjDecompress2(), and
tjDecodeYUVPlanes() to avoid integer overflow when computing the row
pointers in the 64-bit TurboJPEG C API.
- Modify TJBench (both C and Java versions) to avoid overflowing the
size argument to malloc()/new and to fail gracefully if such an
overflow is unavoidable.
In general, this allows gigapixel images to be accommodated by the
64-bit TurboJPEG C API when using automatic JPEG buffer (re)allocation.
Such images cannot currently be accommodated without automatic JPEG
buffer (re)allocation, due to the fact that tjAlloc() accepts a 32-bit
integer argument (oops.) Such images cannot be accommodated in the
TurboJPEG Java API due to the fact that Java always uses a signed 32-bit
integer as an array index.
Fixes #361
diff --git a/ChangeLog.md b/ChangeLog.md
index 0b015c1..b646c93 100644
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -21,6 +21,10 @@
transposed prior to decompression (in the frequency domain) or after
decompression (in the spatial domain.)
+4. Fixed an integer overflow and subsequent segfault that occurred when
+attempting to compress or decompress images with more than 1 billion pixels
+using the TurboJPEG API.
+
2.0.2
=====
diff --git a/java/TJBench.java b/java/TJBench.java
index 7829e53..cc3178e 100644
--- a/java/TJBench.java
+++ b/java/TJBench.java
@@ -121,6 +121,8 @@
int rindex = TJ.getRedOffset(pixelFormat);
int gindex = TJ.getGreenOffset(pixelFormat);
int bindex = TJ.getBlueOffset(pixelFormat);
+ if ((long)w[0] * (long)h[0] * (long)ps > (long)Integer.MAX_VALUE)
+ throw new Exception("Image is too large");
byte[] dstBuf = new byte[w[0] * h[0] * ps];
int pixels = w[0] * h[0], dstPtr = 0, rgbPtr = 0;
@@ -175,8 +177,11 @@
tjd = new TJDecompressor();
- if (dstBuf == null)
+ if (dstBuf == null) {
+ if ((long)pitch * (long)scaledh > (long)Integer.MAX_VALUE)
+ throw new Exception("Image is too large");
dstBuf = new byte[pitch * scaledh];
+ }
/* Set the destination buffer to gray so we know whether the decompressor
attempted to write to it */
@@ -331,6 +336,8 @@
String pfStr = PIXFORMATSTR[pf];
YUVImage yuvImage = null;
+ if ((long)pitch * (long)h > (long)Integer.MAX_VALUE)
+ throw new Exception("Image is too large");
tmpBuf = new byte[pitch * h];
if (quiet == 0)
@@ -491,6 +498,8 @@
int tw, th, ttilew, ttileh, tntilesw, tntilesh, tsubsamp;
FileInputStream fis = new FileInputStream(fileName);
+ if (fis.getChannel().size() > (long)Integer.MAX_VALUE)
+ throw new Exception("Image is too large");
int srcSize = (int)fis.getChannel().size();
srcBuf = new byte[srcSize];
fis.read(srcBuf, 0, srcSize);
diff --git a/tjbench.c b/tjbench.c
index b94637b..120fccb 100644
--- a/tjbench.c
+++ b/tjbench.c
@@ -32,6 +32,7 @@
#include <ctype.h>
#include <math.h>
#include <errno.h>
+#include <limits.h>
#include <cdjpeg.h>
#include "./tjutil.h"
#include "./turbojpeg.h"
@@ -161,7 +162,10 @@
THROW_TJ("executing tjInitDecompress()");
if (dstBuf == NULL) {
- if ((dstBuf = (unsigned char *)malloc(pitch * scaledh)) == NULL)
+ if ((unsigned long long)pitch * (unsigned long long)scaledh >
+ (unsigned long long)((size_t)-1))
+ THROW("allocating destination buffer", "Image is too large");
+ if ((dstBuf = (unsigned char *)malloc((size_t)pitch * scaledh)) == NULL)
THROW_UNIX("allocating destination buffer");
dstBufAlloc = 1;
}
@@ -172,8 +176,10 @@
if (doYUV) {
int width = doTile ? tilew : scaledw;
int height = doTile ? tileh : scaledh;
- int yuvSize = tjBufSizeYUV2(width, yuvPad, height, subsamp);
+ unsigned long yuvSize = tjBufSizeYUV2(width, yuvPad, height, subsamp);
+ if (yuvSize == (unsigned long)-1)
+ THROW_TJ("allocating YUV buffer");
if ((yuvBuf = (unsigned char *)malloc(yuvSize)) == NULL)
THROW_UNIX("allocating YUV buffer");
memset(yuvBuf, 127, yuvSize);
@@ -267,13 +273,13 @@
if (srcBuf && sf.num == 1 && sf.denom == 1) {
if (!quiet) printf("Compression error written to %s.\n", tempStr);
if (subsamp == TJ_GRAYSCALE) {
- int index, index2;
+ unsigned long index, index2;
for (row = 0, index = 0; row < h; row++, index += pitch) {
for (col = 0, index2 = index; col < w; col++, index2 += ps) {
- int rindex = index2 + tjRedOffset[pf];
- int gindex = index2 + tjGreenOffset[pf];
- int bindex = index2 + tjBlueOffset[pf];
+ unsigned long rindex = index2 + tjRedOffset[pf];
+ unsigned long gindex = index2 + tjGreenOffset[pf];
+ unsigned long bindex = index2 + tjBlueOffset[pf];
int y = (int)((double)srcBuf[rindex] * 0.299 +
(double)srcBuf[gindex] * 0.587 +
(double)srcBuf[bindex] * 0.114 + 0.5);
@@ -314,13 +320,16 @@
*srcPtr2;
double start, elapsed, elapsedEncode;
int totalJpegSize = 0, row, col, i, tilew = w, tileh = h, retval = 0;
- int iter, yuvSize = 0;
- unsigned long *jpegSize = NULL;
+ int iter;
+ unsigned long *jpegSize = NULL, yuvSize = 0;
int ps = tjPixelSize[pf];
int ntilesw = 1, ntilesh = 1, pitch = w * ps;
const char *pfStr = pixFormatStr[pf];
- if ((tmpBuf = (unsigned char *)malloc(pitch * h)) == NULL)
+ if ((unsigned long long)pitch * (unsigned long long)h >
+ (unsigned long long)((size_t)-1))
+ THROW("allocating temporary image buffer", "Image is too large");
+ if ((tmpBuf = (unsigned char *)malloc((size_t)pitch * h)) == NULL)
THROW_UNIX("allocating temporary image buffer");
if (!quiet)
@@ -346,6 +355,8 @@
if ((flags & TJFLAG_NOREALLOC) != 0)
for (i = 0; i < ntilesw * ntilesh; i++) {
+ if (tjBufSize(tilew, tileh, subsamp) > (unsigned long)INT_MAX)
+ THROW("getting buffer size", "Image is too large");
if ((jpegBuf[i] = (unsigned char *)
tjAlloc(tjBufSize(tilew, tileh, subsamp))) == NULL)
THROW_UNIX("allocating JPEG tiles");
@@ -363,6 +374,8 @@
if (doYUV) {
yuvSize = tjBufSizeYUV2(tilew, yuvPad, tileh, subsamp);
+ if (yuvSize == (unsigned long)-1)
+ THROW_TJ("allocating YUV buffer");
if ((yuvBuf = (unsigned char *)malloc(yuvSize)) == NULL)
THROW_UNIX("allocating YUV buffer");
memset(yuvBuf, 127, yuvSize);
@@ -437,7 +450,7 @@
if (doYUV) {
printf("Encode YUV --> Frame rate: %f fps\n",
(double)iter / elapsedEncode);
- printf(" Output image size: %d bytes\n", yuvSize);
+ printf(" Output image size: %lu bytes\n", yuvSize);
printf(" Compression ratio: %f:1\n",
(double)(w * h * ps) / (double)yuvSize);
printf(" Throughput: %f Megapixels/sec\n",
@@ -578,8 +591,11 @@
THROW_UNIX("allocating JPEG size array");
memset(jpegSize, 0, sizeof(unsigned long) * ntilesw * ntilesh);
- if ((flags & TJFLAG_NOREALLOC) != 0 || !doTile)
+ if ((flags & TJFLAG_NOREALLOC) != 0 &&
+ (doTile || xformOp != TJXOP_NONE || xformOpt != 0 || customFilter))
for (i = 0; i < ntilesw * ntilesh; i++) {
+ if (tjBufSize(tilew, tileh, subsamp) > (unsigned long)INT_MAX)
+ THROW("getting buffer size", "Image is too large");
if ((jpegBuf[i] = (unsigned char *)
tjAlloc(tjBufSize(tilew, tileh, subsamp))) == NULL)
THROW_UNIX("allocating JPEG tiles");
@@ -685,7 +701,7 @@
}
} else {
if (quiet == 1) printf("N/A N/A ");
- tjFree(jpegBuf[0]);
+ if(jpegBuf[0]) tjFree(jpegBuf[0]);
jpegBuf[0] = NULL;
decompsrc = 1;
}
@@ -700,7 +716,8 @@
} else if (quiet == 1) printf("N/A\n");
for (i = 0; i < ntilesw * ntilesh; i++) {
- tjFree(jpegBuf[i]); jpegBuf[i] = NULL;
+ if(jpegBuf[i]) tjFree(jpegBuf[i]);
+ jpegBuf[i] = NULL;
}
free(jpegBuf); jpegBuf = NULL;
if (jpegSize) { free(jpegSize); jpegSize = NULL; }
diff --git a/tjunittest.c b/tjunittest.c
index a5baf5b..da7c6ff 100644
--- a/tjunittest.c
+++ b/tjunittest.c
@@ -554,6 +554,42 @@
}
+#if SIZEOF_SIZE_T == 8
+#define CHECKSIZE(function) { \
+ if ((unsigned long long)size < (unsigned long long)0xFFFFFFFF) \
+ THROW(#function " overflow"); \
+}
+#else
+#define CHECKSIZE(function) { \
+ if (size != (unsigned long)(-1) || \
+ !strcmp(tjGetErrorStr2(NULL), "No error")) \
+ THROW(#function " overflow"); \
+}
+#endif
+
+static void overflowTest(void)
+{
+ /* Ensure that the various buffer size functions don't overflow */
+ unsigned long size;
+
+ size = tjBufSize(26755, 26755, TJSAMP_444);
+ CHECKSIZE(tjBufSize());
+ size = TJBUFSIZE(26755, 26755);
+ CHECKSIZE(TJBUFSIZE());
+ size = tjBufSizeYUV2(37838, 1, 37838, TJSAMP_444);
+ CHECKSIZE(tjBufSizeYUV2());
+ size = TJBUFSIZEYUV(37838, 37838, TJSAMP_444);
+ CHECKSIZE(TJBUFSIZEYUV());
+ size = tjBufSizeYUV(37838, 37838, TJSAMP_444);
+ CHECKSIZE(tjBufSizeYUV());
+ size = tjPlaneSizeYUV(0, 65536, 0, 65536, TJSAMP_444);
+ CHECKSIZE(tjPlaneSizeYUV());
+
+bailout:
+ return;
+}
+
+
static void bufSizeTest(void)
{
int w, h, i, subsamp;
@@ -865,6 +901,7 @@
}
if (alloc) printf("Testing automatic buffer allocation\n");
if (doYUV) num4bf = 4;
+ overflowTest();
doTest(35, 39, _3byteFormats, 2, TJSAMP_444, "test");
doTest(39, 41, _4byteFormats, num4bf, TJSAMP_444, "test");
doTest(41, 35, _3byteFormats, 2, TJSAMP_422, "test");
diff --git a/turbojpeg.c b/turbojpeg.c
index d491972..b6a1f20 100644
--- a/turbojpeg.c
+++ b/turbojpeg.c
@@ -491,7 +491,7 @@
DLLEXPORT unsigned long tjBufSize(int width, int height, int jpegSubsamp)
{
- unsigned long retval = 0;
+ unsigned long long retval = 0;
int mcuw, mcuh, chromasf;
if (width < 1 || height < 1 || jpegSubsamp < 0 || jpegSubsamp >= NUMSUBOPT)
@@ -503,15 +503,17 @@
mcuw = tjMCUWidth[jpegSubsamp];
mcuh = tjMCUHeight[jpegSubsamp];
chromasf = jpegSubsamp == TJSAMP_GRAY ? 0 : 4 * 64 / (mcuw * mcuh);
- retval = PAD(width, mcuw) * PAD(height, mcuh) * (2 + chromasf) + 2048;
+ retval = PAD(width, mcuw) * PAD(height, mcuh) * (2ULL + chromasf) + 2048ULL;
+ if (retval > (unsigned long long)((unsigned long)-1))
+ THROWG("tjBufSize(): Image is too large");
bailout:
- return retval;
+ return (unsigned long)retval;
}
DLLEXPORT unsigned long TJBUFSIZE(int width, int height)
{
- unsigned long retval = 0;
+ unsigned long long retval = 0;
if (width < 1 || height < 1)
THROWG("TJBUFSIZE(): Invalid argument");
@@ -519,17 +521,20 @@
/* This allows for rare corner cases in which a JPEG image can actually be
larger than the uncompressed input (we wouldn't mention it if it hadn't
happened before.) */
- retval = PAD(width, 16) * PAD(height, 16) * 6 + 2048;
+ retval = PAD(width, 16) * PAD(height, 16) * 6ULL + 2048ULL;
+ if (retval > (unsigned long long)((unsigned long)-1))
+ THROWG("TJBUFSIZE(): Image is too large");
bailout:
- return retval;
+ return (unsigned long)retval;
}
DLLEXPORT unsigned long tjBufSizeYUV2(int width, int pad, int height,
int subsamp)
{
- int retval = 0, nc, i;
+ unsigned long long retval = 0;
+ int nc, i;
if (subsamp < 0 || subsamp >= NUMSUBOPT)
THROWG("tjBufSizeYUV2(): Invalid argument");
@@ -541,11 +546,13 @@
int ph = tjPlaneHeight(i, height, subsamp);
if (pw < 0 || ph < 0) return -1;
- else retval += stride * ph;
+ else retval += (unsigned long long)stride * ph;
}
+ if (retval > (unsigned long long)((unsigned long)-1))
+ THROWG("tjBufSizeYUV2(): Image is too large");
bailout:
- return retval;
+ return (unsigned long)retval;
}
DLLEXPORT unsigned long tjBufSizeYUV(int width, int height, int subsamp)
@@ -604,7 +611,7 @@
DLLEXPORT unsigned long tjPlaneSizeYUV(int componentID, int width, int stride,
int height, int subsamp)
{
- unsigned long retval = 0;
+ unsigned long long retval = 0;
int pw, ph;
if (width < 1 || height < 1 || subsamp < 0 || subsamp >= NUMSUBOPT)
@@ -617,10 +624,12 @@
if (stride == 0) stride = pw;
else stride = abs(stride);
- retval = stride * (ph - 1) + pw;
+ retval = (unsigned long long)stride * (ph - 1) + pw;
+ if (retval > (unsigned long long)((unsigned long)-1))
+ THROWG("tjPlaneSizeYUV(): Image is too large");
bailout:
- return retval;
+ return (unsigned long)retval;
}
@@ -672,9 +681,9 @@
jpeg_start_compress(cinfo, TRUE);
for (i = 0; i < height; i++) {
if (flags & TJFLAG_BOTTOMUP)
- row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * pitch];
+ row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * (size_t)pitch];
else
- row_pointer[i] = (JSAMPROW)&srcBuf[i * pitch];
+ row_pointer[i] = (JSAMPROW)&srcBuf[i * (size_t)pitch];
}
while (cinfo->next_scanline < cinfo->image_height)
jpeg_write_scanlines(cinfo, &row_pointer[cinfo->next_scanline],
@@ -783,9 +792,9 @@
THROW("tjEncodeYUVPlanes(): Memory allocation failure");
for (i = 0; i < height; i++) {
if (flags & TJFLAG_BOTTOMUP)
- row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * pitch];
+ row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * (size_t)pitch];
else
- row_pointer[i] = (JSAMPROW)&srcBuf[i * pitch];
+ row_pointer[i] = (JSAMPROW)&srcBuf[i * (size_t)pitch];
}
if (height < ph0)
for (i = height; i < ph0; i++) row_pointer[i] = row_pointer[height - 1];
@@ -1293,9 +1302,9 @@
}
for (i = 0; i < (int)dinfo->output_height; i++) {
if (flags & TJFLAG_BOTTOMUP)
- row_pointer[i] = &dstBuf[(dinfo->output_height - i - 1) * pitch];
+ row_pointer[i] = &dstBuf[(dinfo->output_height - i - 1) * (size_t)pitch];
else
- row_pointer[i] = &dstBuf[i * pitch];
+ row_pointer[i] = &dstBuf[i * (size_t)pitch];
}
while (dinfo->output_scanline < dinfo->output_height)
jpeg_read_scanlines(dinfo, &row_pointer[dinfo->output_scanline],
@@ -1450,9 +1459,9 @@
THROW("tjDecodeYUVPlanes(): Memory allocation failure");
for (i = 0; i < height; i++) {
if (flags & TJFLAG_BOTTOMUP)
- row_pointer[i] = &dstBuf[(height - i - 1) * pitch];
+ row_pointer[i] = &dstBuf[(height - i - 1) * (size_t)pitch];
else
- row_pointer[i] = &dstBuf[i * pitch];
+ row_pointer[i] = &dstBuf[i * (size_t)pitch];
}
if (height < ph0)
for (i = height; i < ph0; i++) row_pointer[i] = row_pointer[height - 1];
