Prefetch the dictionary contents. PiperOrigin-RevId: 982113761
diff --git a/c/enc/backward_references_inc_opt.h b/c/enc/backward_references_inc_opt.h index 9e1ef64..d26c282 100644 --- a/c/enc/backward_references_inc_opt.h +++ b/c/enc/backward_references_inc_opt.h
@@ -109,6 +109,10 @@ sr.len_code_delta = 0; sr.distance = 0; sr.score = kMinScore; + if (ENABLE_COMPOUND_DICTIONARY) { + PrefetchCompoundDictionaryMatchOpt(¶ms->dictionary.compound, + ringbuffer, ringbuffer_mask, position); + } FN(FindLongestMatch)(privat, params->dictionary.contextual.dict[dict_id], ringbuffer, ringbuffer_mask, dist_cache, position, max_length, max_distance, dictionary_start + gap, params->dist.max_distance, &sr); @@ -138,6 +142,10 @@ dict_id = params->dictionary.contextual.context_map[ BROTLI_CONTEXT(p1, p2, literal_context_lut)]; } + if (ENABLE_COMPOUND_DICTIONARY) { + PrefetchCompoundDictionaryMatchOpt(¶ms->dictionary.compound, + ringbuffer, ringbuffer_mask, position + 1); + } FN(FindLongestMatch)(privat, params->dictionary.contextual.dict[dict_id], ringbuffer, ringbuffer_mask, dist_cache, position + 1, max_length,
diff --git a/c/enc/hash.h b/c/enc/hash.h index 2e0b047..8df275a 100644 --- a/c/enc/hash.h +++ b/c/enc/hash.h
@@ -747,6 +747,25 @@ return total_found; } +static BROTLI_INLINE void PrefetchCompoundDictionaryMatchOpt( + const CompoundDictionary* addon, const uint8_t* BROTLI_RESTRICT data, + const size_t ring_buffer_mask, const size_t cur_ix) { + const size_t cur_ix_masked = cur_ix & ring_buffer_mask; + const uint64_t bytes = BROTLI_UNALIGNED_LOAD64LE(&data[cur_ix_masked]); + size_t d; + for (d = 0; d < addon->num_chunks; ++d) { + const PreparedDictionaryView* view = &addon->chunk_views[d]; + const uint64_t h = + (bytes & view->hash_mask) * kPreparedDictionaryHashMul64Long; + const uint32_t key = (uint32_t)(h >> view->hash_shift); + const uint32_t slot = key & view->slot_mask; + const uint32_t head = view->heads[key]; + /* Deliberately branchless - if head == 0xFFFF (no items), we'll prefetch + * some garbage address. Prefetch can't fault, so this is safe.*/ + PREFETCH_L1(&view->items[view->slot_offsets[slot] + head]); + } +} + static BROTLI_INLINE void FindCompoundDictionaryMatchOpt( const PreparedDictionaryView* self, const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache,
diff --git a/c/enc/hash_longest_match_simd_opt_inc.h b/c/enc/hash_longest_match_simd_opt_inc.h index 6b92380..df9aa66 100644 --- a/c/enc/hash_longest_match_simd_opt_inc.h +++ b/c/enc/hash_longest_match_simd_opt_inc.h
@@ -105,9 +105,9 @@ buckets[offset] = (uint32_t)ix; tags[offset] = tag; } -static BROTLI_INLINE void FN(StoreRange)(HashLongestMatch* BROTLI_RESTRICT self, - const uint8_t* BROTLI_RESTRICT data, const size_t mask, - const size_t ix_start, const size_t ix_end) { +static BROTLI_NOINLINE void FN(StoreRange)( + HashLongestMatch* BROTLI_RESTRICT self, const uint8_t* BROTLI_RESTRICT data, + const size_t mask, const size_t ix_start, const size_t ix_end) { size_t i; for (i = ix_start; i < ix_end; ++i) { FN(Store)(self, data, mask, i); @@ -273,4 +273,11 @@ max_distance, out, BROTLI_FALSE); } } +/* `StoreRange` is deliberately not inlined, so it is not eligible for the + `BROTLI_INLINE` treatment that keeps unused hasher methods quiet; reference + it explicitly for the translation units that do not call it. */ +BROTLI_UNUSED_FUNCTION void FN(SuppressUnusedFunctions)(void) { + BROTLI_UNUSED(&FN(SuppressUnusedFunctions)); + BROTLI_UNUSED(&FN(StoreRange)); +} #undef HashLongestMatch