Add BROTLI_PARAM_MIN_BASE64_REGION_LEN to gate Base64 literal mode by region size. PiperOrigin-RevId: 974739694
diff --git a/c/enc/backward_references_inc.h b/c/enc/backward_references_inc.h index 02e1f4f..007f382 100644 --- a/c/enc/backward_references_inc.h +++ b/c/enc/backward_references_inc.h
@@ -43,6 +43,15 @@ } while (position + FN(HashTypeLength)() < pos_end) { if (position >= next_base64_pos) { + if (position > next_base64_pos) { + /* A previous backward match jumped over the trigger position. + Find the next trigger from the current position safely checking pos_end. */ + next_base64_pos = FindNextBase64Trigger( + ringbuffer, ringbuffer_mask, position, pos_end); + if (position < next_base64_pos) { + goto skip_base64_trigger; + } + } /* Find where it ends */ size_t scan_pos = position + kBase64TriggerLen; size_t first_equal_pos = 0; @@ -64,7 +73,6 @@ } } /* Jump directly to the end of base64 block */ - /* Skip the ';base64,' trigger */ size_t start_pos = position + kBase64TriggerLen; size_t length = scan_pos - start_pos; /* Exclude '=' characters from the flat 6-bit entropy block */ @@ -72,23 +80,28 @@ ringbuffer[(start_pos + length - 1) & ringbuffer_mask] == '=') { length--; } - if (length > 0) { + if (length >= params->min_base64_region_len && length > 0) { hasher->common.base64_regions[hasher->common.num_base64_regions] .start_literal_pos = start_pos; hasher->common.base64_regions[hasher->common.num_base64_regions] .length = length; hasher->common.num_base64_regions++; - } - insert_length += (scan_pos - position); - position = scan_pos; - if (hasher->common.num_base64_regions < params->max_base64_regions) { - next_base64_pos = FindNextBase64Trigger(ringbuffer, ringbuffer_mask, - position, pos_end); + insert_length += (scan_pos - position); + position = scan_pos; + if (hasher->common.num_base64_regions < params->max_base64_regions) { + next_base64_pos = FindNextBase64Trigger( + ringbuffer, ringbuffer_mask, position, pos_end); + } else { + next_base64_pos = pos_end; + } + continue; } else { - next_base64_pos = pos_end; + next_base64_pos = FindNextBase64Trigger( + ringbuffer, ringbuffer_mask, position + 1, pos_end); + goto skip_base64_trigger; } - continue; } +skip_base64_trigger: ; size_t max_length = pos_end - position; size_t max_distance = BROTLI_MIN(size_t, position, max_backward_limit); size_t dictionary_start = BROTLI_MIN(size_t,
diff --git a/c/enc/block_encoder_inc.h b/c/enc/block_encoder_inc.h index 85e2a0d..5a90658 100644 --- a/c/enc/block_encoder_inc.h +++ b/c/enc/block_encoder_inc.h
@@ -21,16 +21,29 @@ if (self->histogram_length_ == 256 && is_base64_histogram && i < 256 && is_base64_histogram[i]) { size_t k; - memset(&self->depths_[ix], 0, 256); + BROTLI_BOOL all_base64 = BROTLI_TRUE; for (k = 0; k < 256; ++k) { - if (kIsBase64[k]) { - self->depths_[ix + k] = 6; + if (histograms[i].data_[k] > 0 && !kIsBase64[k]) { + all_base64 = BROTLI_FALSE; + break; } } - BrotliConvertBitDepthsToSymbols(&self->depths_[ix], 256, - &self->bits_[ix]); - BrotliStoreHuffmanTree(&self->depths_[ix], 256, tree, storage_ix, - storage); + if (all_base64) { + memset(&self->depths_[ix], 0, 256); + for (k = 0; k < 256; ++k) { + if (kIsBase64[k]) { + self->depths_[ix + k] = 6; + } + } + BrotliConvertBitDepthsToSymbols(&self->depths_[ix], 256, + &self->bits_[ix]); + BrotliStoreHuffmanTree(&self->depths_[ix], 256, tree, storage_ix, + storage); + } else { + BuildAndStoreHuffmanTree( + &histograms[i].data_[0], self->histogram_length_, alphabet_size, + tree, &self->depths_[ix], &self->bits_[ix], storage_ix, storage); + } } else { BuildAndStoreHuffmanTree( &histograms[i].data_[0], self->histogram_length_, alphabet_size,
diff --git a/c/enc/encode.c b/c/enc/encode.c index 0d424c3..0a7132f 100644 --- a/c/enc/encode.c +++ b/c/enc/encode.c
@@ -113,6 +113,10 @@ state->params.max_base64_regions = value; return BROTLI_TRUE; + case BROTLI_PARAM_MIN_BASE64_REGION_LEN: + state->params.min_base64_region_len = value; + return BROTLI_TRUE; + case BROTLI_PARAM_SIMD_HASHER: if (value > 2) return BROTLI_FALSE; state->params.simd_hasher = (BrotliEncoderSimdHasher)value; @@ -707,6 +711,7 @@ BrotliInitSharedEncoderDictionary(¶ms->dictionary); params->base64_mode = (int)BROTLI_DEFAULT_BASE64_MODE; params->max_base64_regions = BROTLI_DEFAULT_MAX_BASE64_REGIONS; + params->min_base64_region_len = BROTLI_DEFAULT_MIN_BASE64_REGION_LEN; params->simd_hasher = BROTLI_DEFAULT_SIMD_HASHER; params->dist.distance_postfix_bits = 0; params->dist.num_direct_distance_codes = 0;
diff --git a/c/enc/params.h b/c/enc/params.h index b34eb4b..08c663f 100644 --- a/c/enc/params.h +++ b/c/enc/params.h
@@ -43,6 +43,7 @@ SharedEncoderDictionary dictionary; int base64_mode; size_t max_base64_regions; + size_t min_base64_region_len; BrotliEncoderSimdHasher simd_hasher; } BrotliEncoderParams;
diff --git a/c/include/brotli/encode.h b/c/include/brotli/encode.h index 076025e..5e16dab 100644 --- a/c/include/brotli/encode.h +++ b/c/include/brotli/encode.h
@@ -70,6 +70,8 @@ #define BROTLI_DEFAULT_MAX_BASE64_REGIONS 16 +#define BROTLI_DEFAULT_MIN_BASE64_REGION_LEN 2048 + /** Options for ::BROTLI_PARAM_SIMD_HASHER parameter. */ typedef enum BrotliEncoderSimdHasher { /** Use SIMD hasher when recommended for the quality level. */ @@ -261,7 +263,14 @@ * Controls whether the encoder uses SIMD hashers. * See ::BrotliEncoderSimdHasher for options. */ - BROTLI_PARAM_SIMD_HASHER = 12 + BROTLI_PARAM_SIMD_HASHER = 12, + /** + * Minimum length of a Base64 region to trigger detection and literal block + * splitting. Below this threshold, Base64 regions are encoded using standard + * LZ77 and Huffman coding. + * Default is 2048. + */ + BROTLI_PARAM_MIN_BASE64_REGION_LEN = 14 } BrotliEncoderParameter; /**
diff --git a/docs/encode.h.3 b/docs/encode.h.3 index c24fb97..e379ff0 100644 --- a/docs/encode.h.3 +++ b/docs/encode.h.3
@@ -362,6 +362,9 @@ .TP \fB\fIBROTLI_PARAM_SIMD_HASHER \fP\fP SIMD hasher usage mode\&. Controls whether the encoder uses SIMD hashers\&. See \fBBrotliEncoderSimdHasher\fP for options\&. +.TP +\fB\fIBROTLI_PARAM_MIN_BASE64_REGION_LEN \fP\fP +Minimum length of a Base64 region to trigger detection and literal block splitting\&. Below this threshold, Base64 regions are encoded using standard LZ77 and Huffman coding\&. Default is 2048\&. .SS "enum \fBBrotliEncoderSimdHasher\fP" .PP