Prefetch compound dictionary heads in the Brotli encoder. PiperOrigin-RevId: 990719175
diff --git a/c/enc/backward_references_opt_inc.h b/c/enc/backward_references_opt_inc.h index b9a14eb..1b8fb14 100644 --- a/c/enc/backward_references_opt_inc.h +++ b/c/enc/backward_references_opt_inc.h
@@ -206,6 +206,13 @@ range_start = BROTLI_MIN(size_t, range_end, BROTLI_MAX(size_t, range_start, position + sr.len - (sr.distance << 2))); } + /* The next search is at position + sr.len (the one-ahead in the + prefetch helper covers only the cur_ix + 1 successor): prefetch its + dictionary heads[] line before the StoreRange loop. */ + if (ENABLE_COMPOUND_DICTIONARY) { + PrefetchCompoundDictionaryHeadsOpt(¶ms->dictionary.compound, + ringbuffer, ringbuffer_mask, position + sr.len); + } FN(StoreRange)(privat, ringbuffer, ringbuffer_mask, range_start, range_end); }
diff --git a/c/enc/hash.h b/c/enc/hash.h index dc16d47..c875e6f 100644 --- a/c/enc/hash.h +++ b/c/enc/hash.h
@@ -774,6 +774,36 @@ PREFETCH_L1(chain); probes[d].chain = chain; probes[d].item = (head == 0xFFFF) ? 1 : 0; + + /* One-ahead for the heads[] line of the next probe: the next search is + at cur_ix + 1 (the lazy search, or the next position after a miss), + whose hash covers bytes cur_ix+1.. -- the same 8-byte load shifted by + one byte (hash_mask spans at most 56 bits for this to be exact; a + mismatch would only waste the prefetch). The heads[key] load above + misses L1 ~94% of the time (mostly L3 fills). */ + { + const uint64_t h1 = + ((bytes >> 8) & view->hash_mask) * kPreparedDictionaryHashMul64Long; + PREFETCH_L1(&view->heads[(uint32_t)(h1 >> view->hash_shift)]); + } + } +} + +/* The heads[] half of PrefetchCompoundDictionaryMatchOpt: hash the position + and prefetch its heads[] line, with no dependent items[] load. Used at the + commit site, where the successor position is known but the probe state for + it would not survive the intervening StoreRange. */ +static BROTLI_INLINE void PrefetchCompoundDictionaryHeadsOpt( + const CompoundDictionary* addon, const uint8_t* BROTLI_RESTRICT data, + const size_t ring_buffer_mask, const size_t cur_ix) { + const uint64_t bytes = + BROTLI_UNALIGNED_LOAD64LE(&data[cur_ix & ring_buffer_mask]); + size_t d; + for (d = 0; d < addon->num_chunks; ++d) { + const PreparedDictionaryView* view = &addon->chunk_views[d]; + const uint64_t h = + (bytes & view->hash_mask) * kPreparedDictionaryHashMul64Long; + PREFETCH_L1(&view->heads[(uint32_t)(h >> view->hash_shift)]); } }