Move serialized dictionary feature behind the flag.
BROTLI_SHARED_DICTIONARY_SERIALIZED enum value is a part of API,
but it should not be used (will cause failures).
Changing how serialized dictionaries work won't be considered as an API change, until this feature is enabled.
Enabling this feature in the future will be considered as a "compatible" change.
PiperOrigin-RevId: 557489434
diff --git a/c/common/shared_dictionary.c b/c/common/shared_dictionary.c
index 3ca40c0..49f1c9b 100644
--- a/c/common/shared_dictionary.c
+++ b/c/common/shared_dictionary.c
@@ -20,6 +20,8 @@
extern "C" {
#endif
+#if defined(BROTLI_EXPERIMENTAL)
+
#define BROTLI_NUM_ENCODED_LENGTHS (SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH \
- SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH + 1)
@@ -442,6 +444,8 @@
return ParseDictionary(encoded, size, dict);
}
+#endif /* BROTLI_EXPERIMENTAL */
+
void BrotliSharedDictionaryDestroyInstance(
BrotliSharedDictionary* dict) {
if (!dict) {
@@ -464,9 +468,12 @@
if (!dict) {
return BROTLI_FALSE;
}
+#if defined(BROTLI_EXPERIMENTAL)
if (type == BROTLI_SHARED_DICTIONARY_SERIALIZED) {
return DecodeSharedDictionary(data, data_size, dict);
- } else if (type == BROTLI_SHARED_DICTIONARY_RAW) {
+ }
+#endif /* BROTLI_EXPERIMENTAL */
+ if (type == BROTLI_SHARED_DICTIONARY_RAW) {
if (dict->num_prefix >= SHARED_BROTLI_MAX_COMPOUND_DICTS) {
return BROTLI_FALSE;
}
@@ -474,9 +481,8 @@
dict->prefix[dict->num_prefix] = data;
dict->num_prefix++;
return BROTLI_TRUE;
- } else {
- return BROTLI_FALSE;
}
+ return BROTLI_FALSE;
}
BrotliSharedDictionary* BrotliSharedDictionaryCreateInstance(
diff --git a/c/enc/encode.c b/c/enc/encode.c
index 0303094..81ea40e 100644
--- a/c/enc/encode.c
+++ b/c/enc/encode.c
@@ -7,6 +7,8 @@
/* Implementation of Brotli compressor. */
#include <brotli/encode.h>
+#include <brotli/shared_dictionary.h>
+#include <brotli/types.h>
#include <stdlib.h> /* free, malloc */
#include <string.h> /* memcpy, memset */
@@ -1707,8 +1709,12 @@
const uint8_t data[BROTLI_ARRAY_PARAM(size)], int quality,
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
ManagedDictionary* managed_dictionary = NULL;
- if (type != BROTLI_SHARED_DICTIONARY_RAW &&
- type != BROTLI_SHARED_DICTIONARY_SERIALIZED) {
+ BROTLI_BOOL type_is_known = BROTLI_FALSE;
+ type_is_known |= (type == BROTLI_SHARED_DICTIONARY_RAW);
+#if defined(BROTLI_EXPERIMENTAL)
+ type_is_known |= (type == BROTLI_SHARED_DICTIONARY_SERIALIZED);
+#endif /* BROTLI_EXPERIMENTAL */
+ if (!type_is_known) {
return NULL;
}
managed_dictionary =
@@ -1719,7 +1725,9 @@
if (type == BROTLI_SHARED_DICTIONARY_RAW) {
managed_dictionary->dictionary = (uint32_t*)CreatePreparedDictionary(
&managed_dictionary->memory_manager_, data, size);
- } else {
+ }
+#if defined(BROTLI_EXPERIMENTAL)
+ if (type == BROTLI_SHARED_DICTIONARY_SERIALIZED) {
SharedEncoderDictionary* dict = (SharedEncoderDictionary*)BrotliAllocate(
&managed_dictionary->memory_manager_, sizeof(SharedEncoderDictionary));
managed_dictionary->dictionary = (uint32_t*)dict;
@@ -1732,6 +1740,9 @@
}
}
}
+#else /* BROTLI_EXPERIMENTAL */
+ (void)quality;
+#endif /* BROTLI_EXPERIMENTAL */
if (managed_dictionary->dictionary == NULL) {
BrotliDestroyManagedDictionary(managed_dictionary);
return NULL;
diff --git a/c/enc/encoder_dict.c b/c/enc/encoder_dict.c
index 0c93e00..6602c55 100644
--- a/c/enc/encoder_dict.c
+++ b/c/enc/encoder_dict.c
@@ -77,6 +77,7 @@
BrotliTrieFree(m, &dict->trie);
}
+#if defined(BROTLI_EXPERIMENTAL)
/* Word length must be at least 4 bytes */
static uint32_t Hash(const uint8_t* data, int bits) {
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
@@ -481,6 +482,7 @@
return BROTLI_TRUE;
}
+#endif /* BROTLI_EXPERIMENTAL */
void BrotliInitSharedEncoderDictionary(SharedEncoderDictionary* dict) {
dict->magic = kSharedDictionaryMagic;
@@ -501,6 +503,7 @@
dict->max_quality = BROTLI_MAX_QUALITY;
}
+#if defined(BROTLI_EXPERIMENTAL)
/* TODO(eustas): make sure that tooling will warn user if not all the cutoff
transforms are available (for low-quality encoder). */
static BROTLI_BOOL InitCustomSharedEncoderDictionary(
@@ -586,6 +589,7 @@
BrotliSharedDictionaryDestroyInstance(decoded_dict);
return success;
}
+#endif /* BROTLI_EXPERIMENTAL */
void BrotliCleanupSharedEncoderDictionary(MemoryManager* m,
SharedEncoderDictionary* dict) {
diff --git a/c/enc/encoder_dict.h b/c/enc/encoder_dict.h
index b291f98..27dcbcd 100644
--- a/c/enc/encoder_dict.h
+++ b/c/enc/encoder_dict.h
@@ -52,8 +52,11 @@
BrotliTrieNode root;
} BrotliTrie;
+#if defined(BROTLI_EXPERIMENTAL)
BROTLI_INTERNAL const BrotliTrieNode* BrotliTrieSub(const BrotliTrie* trie,
const BrotliTrieNode* node, uint8_t c);
+#endif /* BROTLI_EXPERIMENTAL */
+
/* Dictionary data (words and transforms) for 1 possible context */
typedef struct BrotliEncoderDictionary {
const BrotliDictionary* words;
@@ -129,12 +132,14 @@
BROTLI_INTERNAL void BrotliInitSharedEncoderDictionary(
SharedEncoderDictionary* dict);
+#if defined(BROTLI_EXPERIMENTAL)
/* Initializes to shared dictionary that will be parsed from
encoded_dict. Requires that you keep the encoded_dict buffer
around, parts of data will point to it. */
BROTLI_INTERNAL BROTLI_BOOL BrotliInitCustomSharedEncoderDictionary(
MemoryManager* m, const uint8_t* encoded_dict, size_t size,
int quality, SharedEncoderDictionary* dict);
+#endif /* BROTLI_EXPERIMENTAL */
BROTLI_INTERNAL void BrotliCleanupSharedEncoderDictionary(
MemoryManager* m, SharedEncoderDictionary* dict);
diff --git a/c/enc/memory.c b/c/enc/memory.c
index 51e1b7f..8811821 100644
--- a/c/enc/memory.c
+++ b/c/enc/memory.c
@@ -20,9 +20,11 @@
extern "C" {
#endif
-#define MAX_PERM_ALLOCATED 128
+/* TODO(eustas): fine-tune for "many slots" case */
#define MAX_NEW_ALLOCATED 64
#define MAX_NEW_FREED 64
+#define MAX_PERM_ALLOCATED \
+ (BROTLI_ENCODER_MEMORY_MANAGER_SLOTS - MAX_NEW_ALLOCATED - MAX_NEW_FREED)
#define PERM_ALLOCATED_OFFSET 0
#define NEW_ALLOCATED_OFFSET MAX_PERM_ALLOCATED
@@ -68,6 +70,7 @@
static void SortPointers(void** items, const size_t n) {
/* Shell sort. */
+ /* TODO(eustas): fine-tune for "many slots" case */
static const size_t gaps[] = {23, 10, 4, 1};
int g = 0;
for (; g < 4; ++g) {
diff --git a/c/enc/memory.h b/c/enc/memory.h
index cbe4e30..9dbed55 100644
--- a/c/enc/memory.h
+++ b/c/enc/memory.h
@@ -24,6 +24,14 @@
#define BROTLI_ENCODER_EXIT_ON_OOM
#endif
+#if !defined(BROTLI_ENCODER_EXIT_ON_OOM)
+#if defined(BROTLI_EXPERIMENTAL)
+#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS 6144
+#else /* BROTLI_EXPERIMENTAL */
+#define BROTLI_ENCODER_MEMORY_MANAGER_SLOTS 256
+#endif /* BROTLI_EXPERIMENTAL */
+#endif /* BROTLI_ENCODER_EXIT_ON_OOM */
+
typedef struct MemoryManager {
brotli_alloc_func alloc_func;
brotli_free_func free_func;
@@ -33,7 +41,7 @@
size_t perm_allocated;
size_t new_allocated;
size_t new_freed;
- void* pointers[256];
+ void* pointers[BROTLI_ENCODER_MEMORY_MANAGER_SLOTS];
#endif /* BROTLI_ENCODER_EXIT_ON_OOM */
} MemoryManager;
diff --git a/c/enc/static_dict.c b/c/enc/static_dict.c
index 9e6f270..291d283 100644
--- a/c/enc/static_dict.c
+++ b/c/enc/static_dict.c
@@ -79,6 +79,7 @@
const BrotliEncoderDictionary* dictionary, const uint8_t* data,
size_t min_length, size_t max_length, uint32_t* matches) {
BROTLI_BOOL has_found_match = BROTLI_FALSE;
+#if defined(BROTLI_EXPERIMENTAL)
if (dictionary->has_words_heavy) {
const BrotliTrieNode* node = &dictionary->trie.root;
size_t l = 0;
@@ -93,6 +94,7 @@
}
return has_found_match;
}
+#endif /* BROTLI_EXPERIMENTAL */
{
size_t offset = dictionary->buckets[Hash(data)];
BROTLI_BOOL end = !offset;
diff --git a/c/include/brotli/shared_dictionary.h b/c/include/brotli/shared_dictionary.h
index ceb6cf1..2970c2d 100644
--- a/c/include/brotli/shared_dictionary.h
+++ b/c/include/brotli/shared_dictionary.h
@@ -35,7 +35,10 @@
typedef enum BrotliSharedDictionaryType {
/** Raw LZ77 prefix dictionary. */
BROTLI_SHARED_DICTIONARY_RAW = 0,
- /** Serialized shared dictionary. */
+ /** Serialized shared dictionary.
+ *
+ * DO NOT USE: methods accepting this value will fail.
+ */
BROTLI_SHARED_DICTIONARY_SERIALIZED = 1
} BrotliSharedDictionaryType;
