icu4c/source/i18n/collationdatabuilder.h - external/github.com/unicode-org/icu - Git at Google

 // © 2016 and later: Unicode, Inc. and others.
 // License & terms of use: http://www.unicode.org/copyright.html
 /*
 *******************************************************************************
 * Copyright (C) 2012-2014, International Business Machines
 * Corporation and others.  All Rights Reserved.
 *******************************************************************************
 * collationdatabuilder.h
 *
 * created on: 2012apr01
 * created by: Markus W. Scherer
 */

 #ifndef __COLLATIONDATABUILDER_H__
 #define __COLLATIONDATABUILDER_H__

 #include "unicode/utypes.h"

 #if !UCONFIG_NO_COLLATION

 #include "unicode/uniset.h"
 #include "unicode/unistr.h"
 #include "unicode/uversion.h"
 #include "collation.h"
 #include "collationdata.h"
 #include "collationsettings.h"
 #include "normalizer2impl.h"
 #include "utrie2.h"
 #include "uvectr32.h"
 #include "uvectr64.h"
 #include "uvector.h"

 U_NAMESPACE_BEGIN

 struct ConditionalCE32;

 class CollationFastLatinBuilder;
 class CopyHelper;
 class DataBuilderCollationIterator;
 class UCharsTrieBuilder;

 /**
  * Low-level CollationData builder.
  * Takes (character, CE) pairs and builds them into runtime data structures.
  * Supports characters with context prefixes and contraction suffixes.
  */
 class U_I18N_API CollationDataBuilder : public UObject {
 public:
     /**
      * Collation element modifier. Interface class for a modifier
      * that changes a tailoring builder's temporary CEs to final CEs.
      * Called for every non-special CE32 and every expansion CE.
      */
     class CEModifier : public UObject {
     public:
         virtual ~CEModifier();
         /** Returns a new CE to replace the non-special input CE32, or else Collation::NO_CE. */
         virtual int64_t modifyCE32(uint32_t ce32) const = 0;
         /** Returns a new CE to replace the input CE, or else Collation::NO_CE. */
         virtual int64_t modifyCE(int64_t ce) const = 0;
     };

     CollationDataBuilder(UErrorCode &errorCode);

     virtual ~CollationDataBuilder();

     void initForTailoring(const CollationData *b, UErrorCode &errorCode);

     virtual UBool isCompressibleLeadByte(uint32_t b) const;

     inline UBool isCompressiblePrimary(uint32_t p) const {
         return isCompressibleLeadByte(p >> 24);
     }

     /**
      * @return true if this builder has mappings (e.g., add() has been called)
      */
     UBool hasMappings() const { return modified; }

     /**
      * @return true if c has CEs in this builder
      */
     UBool isAssigned(UChar32 c) const;

     /**
      * @return the three-byte primary if c maps to a single such CE and has no context data,
      * otherwise returns 0.
      */
     uint32_t getLongPrimaryIfSingleCE(UChar32 c) const;

     /**
      * @return the single CE for c.
      * Sets an error code if c does not have a single CE.
      */
     int64_t getSingleCE(UChar32 c, UErrorCode &errorCode) const;

     void add(const UnicodeString &prefix, const UnicodeString &s,
              const int64_t ces[], int32_t cesLength,
              UErrorCode &errorCode);

     /**
      * Encodes the ces as either the returned ce32 by itself,
      * or by storing an expansion, with the returned ce32 referring to that.
      *
      * add(p, s, ces, cesLength) = addCE32(p, s, encodeCEs(ces, cesLength))
      */
     virtual uint32_t encodeCEs(const int64_t ces[], int32_t cesLength, UErrorCode &errorCode);
     void addCE32(const UnicodeString &prefix, const UnicodeString &s,
                  uint32_t ce32, UErrorCode &errorCode);

     /**
      * Sets three-byte-primary CEs for a range of code points in code point order,
      * if it is worth doing; otherwise no change is made.
      * None of the code points in the range should have complex mappings so far
      * (expansions/contractions/prefixes).
      * @param start first code point
      * @param end last code point (inclusive)
      * @param primary primary weight for 'start'
      * @param step per-code point primary-weight increment
      * @param errorCode ICU in/out error code
      * @return true if an OFFSET_TAG range was used for start..end
      */
     UBool maybeSetPrimaryRange(UChar32 start, UChar32 end,
                                uint32_t primary, int32_t step,
                                UErrorCode &errorCode);

     /**
      * Sets three-byte-primary CEs for a range of code points in code point order.
      * Sets range values if that is worth doing, or else individual values.
      * None of the code points in the range should have complex mappings so far
      * (expansions/contractions/prefixes).
      * @param start first code point
      * @param end last code point (inclusive)
      * @param primary primary weight for 'start'
      * @param step per-code point primary-weight increment
      * @param errorCode ICU in/out error code
      * @return the next primary after 'end': start primary incremented by ((end-start)+1)*step
      */
     uint32_t setPrimaryRangeAndReturnNext(UChar32 start, UChar32 end,
                                           uint32_t primary, int32_t step,
                                           UErrorCode &errorCode);

     /**
      * Copies all mappings from the src builder, with modifications.
      * This builder here must not be built yet, and should be empty.
      */
     void copyFrom(const CollationDataBuilder &src, const CEModifier &modifier,
                   UErrorCode &errorCode);

     void optimize(const UnicodeSet &set, UErrorCode &errorCode);
     void suppressContractions(const UnicodeSet &set, UErrorCode &errorCode);

     void enableFastLatin() { fastLatinEnabled = true; }
     virtual void build(CollationData &data, UErrorCode &errorCode);

     /**
      * Looks up CEs for s and appends them to the ces array.
      * Does not handle normalization: s should be in FCD form.
      *
      * Does not write completely ignorable CEs.
      * Does not write beyond Collation::MAX_EXPANSION_LENGTH.
      *
      * @return incremented cesLength
      */
     int32_t getCEs(const UnicodeString &s, int64_t ces[], int32_t cesLength);
     int32_t getCEs(const UnicodeString &prefix, const UnicodeString &s,
                    int64_t ces[], int32_t cesLength);

 protected:
     friend class CopyHelper;
     friend class DataBuilderCollationIterator;

     uint32_t getCE32FromOffsetCE32(UBool fromBase, UChar32 c, uint32_t ce32) const;

     int32_t addCE(int64_t ce, UErrorCode &errorCode);
     int32_t addCE32(uint32_t ce32, UErrorCode &errorCode);
     int32_t addConditionalCE32(const UnicodeString &context, uint32_t ce32, UErrorCode &errorCode);

     inline ConditionalCE32 *getConditionalCE32(int32_t index) const {
         return static_cast<ConditionalCE32 *>(conditionalCE32s[index]);
     }
     inline ConditionalCE32 *getConditionalCE32ForCE32(uint32_t ce32) const {
         return getConditionalCE32(Collation::indexFromCE32(ce32));
     }

     static uint32_t makeBuilderContextCE32(int32_t index) {
         return Collation::makeCE32FromTagAndIndex(Collation::BUILDER_DATA_TAG, index);
     }
     static inline UBool isBuilderContextCE32(uint32_t ce32) {
         return Collation::hasCE32Tag(ce32, Collation::BUILDER_DATA_TAG);
     }

     static uint32_t encodeOneCEAsCE32(int64_t ce);
     uint32_t encodeOneCE(int64_t ce, UErrorCode &errorCode);
     uint32_t encodeExpansion(const int64_t ces[], int32_t length, UErrorCode &errorCode);
     uint32_t encodeExpansion32(const int32_t newCE32s[], int32_t length, UErrorCode &errorCode);

     uint32_t copyFromBaseCE32(UChar32 c, uint32_t ce32, UBool withContext, UErrorCode &errorCode);
     /**
      * Copies base contractions to a list of ConditionalCE32.
      * Sets cond->next to the index of the first new item
      * and returns the index of the last new item.
      */
     int32_t copyContractionsFromBaseCE32(UnicodeString &context, UChar32 c, uint32_t ce32,
                                          ConditionalCE32 *cond, UErrorCode &errorCode);

     UBool getJamoCE32s(uint32_t jamoCE32s[], UErrorCode &errorCode);
     void setDigitTags(UErrorCode &errorCode);
     void setLeadSurrogates(UErrorCode &errorCode);

     void buildMappings(CollationData &data, UErrorCode &errorCode);

     void clearContexts();
     void buildContexts(UErrorCode &errorCode);
     uint32_t buildContext(ConditionalCE32 *head, UErrorCode &errorCode);
     int32_t addContextTrie(uint32_t defaultCE32, UCharsTrieBuilder &trieBuilder,
                            UErrorCode &errorCode);

     void buildFastLatinTable(CollationData &data, UErrorCode &errorCode);

     int32_t getCEs(const UnicodeString &s, int32_t start, int64_t ces[], int32_t cesLength);

     static UChar32 jamoCpFromIndex(int32_t i) {
         // 0 <= i < CollationData::JAMO_CE32S_LENGTH = 19 + 21 + 27
         if(i < Hangul::JAMO_L_COUNT) { return Hangul::JAMO_L_BASE + i; }
         i -= Hangul::JAMO_L_COUNT;
         if(i < Hangul::JAMO_V_COUNT) { return Hangul::JAMO_V_BASE + i; }
         i -= Hangul::JAMO_V_COUNT;
         // i < 27
         return Hangul::JAMO_T_BASE + 1 + i;
     }

     /** @see Collation::BUILDER_DATA_TAG */
     static const uint32_t IS_BUILDER_JAMO_CE32 = 0x100;

     const Normalizer2Impl &nfcImpl;
     const CollationData *base;
     const CollationSettings *baseSettings;
     UTrie2 *trie;
     UVector32 ce32s;
     UVector64 ce64s;
     UVector conditionalCE32s;  // vector of ConditionalCE32
     // Characters that have context (prefixes or contraction suffixes).
     UnicodeSet contextChars;
     // Serialized UCharsTrie structures for finalized contexts.
     UnicodeString contexts;
     UnicodeSet unsafeBackwardSet;
     UBool modified;

     UBool fastLatinEnabled;
     CollationFastLatinBuilder *fastLatinBuilder;

     DataBuilderCollationIterator *collIter;
 };

 U_NAMESPACE_END

 #endif  // !UCONFIG_NO_COLLATION
 #endif  // __COLLATIONDATABUILDER_H__
	// © 2016 and later: Unicode, Inc. and others.
	// License & terms of use: http://www.unicode.org/copyright.html
	/*
	*******************************************************************************
	* Copyright (C) 2012-2014, International Business Machines
	* Corporation and others. All Rights Reserved.
	*******************************************************************************
	* collationdatabuilder.h
	*
	* created on: 2012apr01
	* created by: Markus W. Scherer
	*/

	#ifndef __COLLATIONDATABUILDER_H__
	#define __COLLATIONDATABUILDER_H__

	#include "unicode/utypes.h"

	#if !UCONFIG_NO_COLLATION

	#include "unicode/uniset.h"
	#include "unicode/unistr.h"
	#include "unicode/uversion.h"
	#include "collation.h"
	#include "collationdata.h"
	#include "collationsettings.h"
	#include "normalizer2impl.h"
	#include "utrie2.h"
	#include "uvectr32.h"
	#include "uvectr64.h"
	#include "uvector.h"

	U_NAMESPACE_BEGIN

	struct ConditionalCE32;

	class CollationFastLatinBuilder;
	class CopyHelper;
	class DataBuilderCollationIterator;
	class UCharsTrieBuilder;

	/**
	* Low-level CollationData builder.
	* Takes (character, CE) pairs and builds them into runtime data structures.
	* Supports characters with context prefixes and contraction suffixes.
	*/
	class U_I18N_API CollationDataBuilder : public UObject {
	public:
	/**
	* Collation element modifier. Interface class for a modifier
	* that changes a tailoring builder's temporary CEs to final CEs.
	* Called for every non-special CE32 and every expansion CE.
	*/
	class CEModifier : public UObject {
	public:
	virtual ~CEModifier();
	/** Returns a new CE to replace the non-special input CE32, or else Collation::NO_CE. */
	virtual int64_t modifyCE32(uint32_t ce32) const = 0;
	/** Returns a new CE to replace the input CE, or else Collation::NO_CE. */
	virtual int64_t modifyCE(int64_t ce) const = 0;
	};

	CollationDataBuilder(UErrorCode &errorCode);

	virtual ~CollationDataBuilder();

	void initForTailoring(const CollationData *b, UErrorCode &errorCode);

	virtual UBool isCompressibleLeadByte(uint32_t b) const;

	inline UBool isCompressiblePrimary(uint32_t p) const {
	return isCompressibleLeadByte(p >> 24);
	}

	/**
	* @return true if this builder has mappings (e.g., add() has been called)
	*/
	UBool hasMappings() const { return modified; }

	/**
	* @return true if c has CEs in this builder
	*/
	UBool isAssigned(UChar32 c) const;

	/**
	* @return the three-byte primary if c maps to a single such CE and has no context data,
	* otherwise returns 0.
	*/
	uint32_t getLongPrimaryIfSingleCE(UChar32 c) const;

	/**
	* @return the single CE for c.
	* Sets an error code if c does not have a single CE.
	*/
	int64_t getSingleCE(UChar32 c, UErrorCode &errorCode) const;

	void add(const UnicodeString &prefix, const UnicodeString &s,
	const int64_t ces[], int32_t cesLength,
	UErrorCode &errorCode);

	/**
	* Encodes the ces as either the returned ce32 by itself,
	* or by storing an expansion, with the returned ce32 referring to that.
	*
	* add(p, s, ces, cesLength) = addCE32(p, s, encodeCEs(ces, cesLength))
	*/
	virtual uint32_t encodeCEs(const int64_t ces[], int32_t cesLength, UErrorCode &errorCode);
	void addCE32(const UnicodeString &prefix, const UnicodeString &s,
	uint32_t ce32, UErrorCode &errorCode);

	/**
	* Sets three-byte-primary CEs for a range of code points in code point order,
	* if it is worth doing; otherwise no change is made.
	* None of the code points in the range should have complex mappings so far
	* (expansions/contractions/prefixes).
	* @param start first code point
	* @param end last code point (inclusive)
	* @param primary primary weight for 'start'
	* @param step per-code point primary-weight increment
	* @param errorCode ICU in/out error code
	* @return true if an OFFSET_TAG range was used for start..end
	*/
	UBool maybeSetPrimaryRange(UChar32 start, UChar32 end,
	uint32_t primary, int32_t step,
	UErrorCode &errorCode);

	/**
	* Sets three-byte-primary CEs for a range of code points in code point order.
	* Sets range values if that is worth doing, or else individual values.
	* None of the code points in the range should have complex mappings so far
	* (expansions/contractions/prefixes).
	* @param start first code point
	* @param end last code point (inclusive)
	* @param primary primary weight for 'start'
	* @param step per-code point primary-weight increment
	* @param errorCode ICU in/out error code
	* @return the next primary after 'end': start primary incremented by ((end-start)+1)*step
	*/
	uint32_t setPrimaryRangeAndReturnNext(UChar32 start, UChar32 end,
	uint32_t primary, int32_t step,
	UErrorCode &errorCode);

	/**
	* Copies all mappings from the src builder, with modifications.
	* This builder here must not be built yet, and should be empty.
	*/
	void copyFrom(const CollationDataBuilder &src, const CEModifier &modifier,
	UErrorCode &errorCode);

	void optimize(const UnicodeSet &set, UErrorCode &errorCode);
	void suppressContractions(const UnicodeSet &set, UErrorCode &errorCode);

	void enableFastLatin() { fastLatinEnabled = true; }
	virtual void build(CollationData &data, UErrorCode &errorCode);

	/**
	* Looks up CEs for s and appends them to the ces array.
	* Does not handle normalization: s should be in FCD form.
	*
	* Does not write completely ignorable CEs.
	* Does not write beyond Collation::MAX_EXPANSION_LENGTH.
	*
	* @return incremented cesLength
	*/
	int32_t getCEs(const UnicodeString &s, int64_t ces[], int32_t cesLength);
	int32_t getCEs(const UnicodeString &prefix, const UnicodeString &s,
	int64_t ces[], int32_t cesLength);

	protected:
	friend class CopyHelper;
	friend class DataBuilderCollationIterator;

	uint32_t getCE32FromOffsetCE32(UBool fromBase, UChar32 c, uint32_t ce32) const;

	int32_t addCE(int64_t ce, UErrorCode &errorCode);
	int32_t addCE32(uint32_t ce32, UErrorCode &errorCode);
	int32_t addConditionalCE32(const UnicodeString &context, uint32_t ce32, UErrorCode &errorCode);

	inline ConditionalCE32 *getConditionalCE32(int32_t index) const {
	return static_cast<ConditionalCE32 *>(conditionalCE32s[index]);
	}
	inline ConditionalCE32 *getConditionalCE32ForCE32(uint32_t ce32) const {
	return getConditionalCE32(Collation::indexFromCE32(ce32));
	}

	static uint32_t makeBuilderContextCE32(int32_t index) {
	return Collation::makeCE32FromTagAndIndex(Collation::BUILDER_DATA_TAG, index);
	}
	static inline UBool isBuilderContextCE32(uint32_t ce32) {
	return Collation::hasCE32Tag(ce32, Collation::BUILDER_DATA_TAG);
	}

	static uint32_t encodeOneCEAsCE32(int64_t ce);
	uint32_t encodeOneCE(int64_t ce, UErrorCode &errorCode);
	uint32_t encodeExpansion(const int64_t ces[], int32_t length, UErrorCode &errorCode);
	uint32_t encodeExpansion32(const int32_t newCE32s[], int32_t length, UErrorCode &errorCode);

	uint32_t copyFromBaseCE32(UChar32 c, uint32_t ce32, UBool withContext, UErrorCode &errorCode);
	/**
	* Copies base contractions to a list of ConditionalCE32.
	* Sets cond->next to the index of the first new item
	* and returns the index of the last new item.
	*/
	int32_t copyContractionsFromBaseCE32(UnicodeString &context, UChar32 c, uint32_t ce32,
	ConditionalCE32 *cond, UErrorCode &errorCode);

	UBool getJamoCE32s(uint32_t jamoCE32s[], UErrorCode &errorCode);
	void setDigitTags(UErrorCode &errorCode);
	void setLeadSurrogates(UErrorCode &errorCode);

	void buildMappings(CollationData &data, UErrorCode &errorCode);

	void clearContexts();
	void buildContexts(UErrorCode &errorCode);
	uint32_t buildContext(ConditionalCE32 *head, UErrorCode &errorCode);
	int32_t addContextTrie(uint32_t defaultCE32, UCharsTrieBuilder &trieBuilder,
	UErrorCode &errorCode);

	void buildFastLatinTable(CollationData &data, UErrorCode &errorCode);

	int32_t getCEs(const UnicodeString &s, int32_t start, int64_t ces[], int32_t cesLength);

	static UChar32 jamoCpFromIndex(int32_t i) {
	// 0 <= i < CollationData::JAMO_CE32S_LENGTH = 19 + 21 + 27
	if(i < Hangul::JAMO_L_COUNT) { return Hangul::JAMO_L_BASE + i; }
	i -= Hangul::JAMO_L_COUNT;
	if(i < Hangul::JAMO_V_COUNT) { return Hangul::JAMO_V_BASE + i; }
	i -= Hangul::JAMO_V_COUNT;
	// i < 27
	return Hangul::JAMO_T_BASE + 1 + i;
	}

	/** @see Collation::BUILDER_DATA_TAG */
	static const uint32_t IS_BUILDER_JAMO_CE32 = 0x100;

	const Normalizer2Impl &nfcImpl;
	const CollationData *base;
	const CollationSettings *baseSettings;
	UTrie2 *trie;
	UVector32 ce32s;
	UVector64 ce64s;
	UVector conditionalCE32s; // vector of ConditionalCE32
	// Characters that have context (prefixes or contraction suffixes).
	UnicodeSet contextChars;
	// Serialized UCharsTrie structures for finalized contexts.
	UnicodeString contexts;
	UnicodeSet unsafeBackwardSet;
	UBool modified;

	UBool fastLatinEnabled;
	CollationFastLatinBuilder *fastLatinBuilder;

	DataBuilderCollationIterator *collIter;
	};

	U_NAMESPACE_END

	#endif // !UCONFIG_NO_COLLATION
	#endif // __COLLATIONDATABUILDER_H__