| /* |
| ******************************************************************************* |
| * Copyright (C) 2013-2015, International Business Machines |
| * Corporation and others. All Rights Reserved. |
| ******************************************************************************* |
| * CollationDataReader.java, ported from collationdatareader.h/.cpp |
| * |
| * C++ version created on: 2013feb07 |
| * created by: Markus W. Scherer |
| */ |
| |
| package com.ibm.icu.impl.coll; |
| |
| import java.io.IOException; |
| import java.nio.ByteBuffer; |
| import java.nio.CharBuffer; |
| import java.util.Arrays; |
| |
| import com.ibm.icu.impl.ICUBinary; |
| import com.ibm.icu.impl.Trie2_32; |
| import com.ibm.icu.impl.USerializedSet; |
| import com.ibm.icu.text.Collator; |
| import com.ibm.icu.text.UnicodeSet; |
| import com.ibm.icu.util.ICUException; |
| |
| /** |
| * Collation binary data reader. |
| */ |
| final class CollationDataReader /* all static */ { |
| // The following constants are also copied into source/common/ucol_swp.cpp. |
| // Keep them in sync! |
| /** |
| * Number of int indexes. |
| * |
| * Can be 2 if there are only options. |
| * Can be 7 or 8 if there are only options and a script reordering. |
| * The loader treats any index>=indexes[IX_INDEXES_LENGTH] as 0. |
| */ |
| static final int IX_INDEXES_LENGTH = 0; |
| /** |
| * Bits 31..24: numericPrimary, for numeric collation |
| * 23..16: fast Latin format version (0 = no fast Latin table) |
| * 15.. 0: options bit set |
| */ |
| static final int IX_OPTIONS = 1; |
| static final int IX_RESERVED2 = 2; |
| static final int IX_RESERVED3 = 3; |
| |
| /** Array offset to Jamo CE32s in ce32s[], or <0 if none. */ |
| static final int IX_JAMO_CE32S_START = 4; |
| |
| // Byte offsets from the start of the data, after the generic header. |
| // The indexes[] are at byte offset 0, other data follows. |
| // Each data item is aligned properly. |
| // The data items should be in descending order of unit size, |
| // to minimize the need for padding. |
| // Each item's byte length is given by the difference between its offset and |
| // the next index/offset value. |
| /** Byte offset to int reorderCodes[]. */ |
| static final int IX_REORDER_CODES_OFFSET = 5; |
| /** |
| * Byte offset to uint8_t reorderTable[]. |
| * Empty table if <256 bytes (padding only). |
| * Otherwise 256 bytes or more (with padding). |
| */ |
| static final int IX_REORDER_TABLE_OFFSET = 6; |
| /** Byte offset to the collation trie. Its length is a multiple of 8 bytes. */ |
| static final int IX_TRIE_OFFSET = 7; |
| |
| static final int IX_RESERVED8_OFFSET = 8; |
| /** Byte offset to long ces[]. */ |
| static final int IX_CES_OFFSET = 9; |
| static final int IX_RESERVED10_OFFSET = 10; |
| /** Byte offset to int ce32s[]. */ |
| static final int IX_CE32S_OFFSET = 11; |
| |
| /** Byte offset to uint32_t rootElements[]. */ |
| static final int IX_ROOT_ELEMENTS_OFFSET = 12; |
| /** Byte offset to UChar *contexts[]. */ |
| static final int IX_CONTEXTS_OFFSET = 13; |
| /** Byte offset to char [] with serialized unsafeBackwardSet. */ |
| static final int IX_UNSAFE_BWD_OFFSET = 14; |
| /** Byte offset to char fastLatinTable[]. */ |
| static final int IX_FAST_LATIN_TABLE_OFFSET = 15; |
| |
| /** Byte offset to char scripts[]. */ |
| static final int IX_SCRIPTS_OFFSET = 16; |
| /** |
| * Byte offset to boolean compressibleBytes[]. |
| * Empty table if <256 bytes (padding only). |
| * Otherwise 256 bytes or more (with padding). |
| */ |
| static final int IX_COMPRESSIBLE_BYTES_OFFSET = 17; |
| static final int IX_RESERVED18_OFFSET = 18; |
| static final int IX_TOTAL_SIZE = 19; |
| |
| static void read(CollationTailoring base, ByteBuffer inBytes, |
| CollationTailoring tailoring) throws IOException { |
| tailoring.version = ICUBinary.readHeader(inBytes, DATA_FORMAT, IS_ACCEPTABLE); |
| if(base != null && base.getUCAVersion() != tailoring.getUCAVersion()) { |
| throw new ICUException("Tailoring UCA version differs from base data UCA version"); |
| } |
| |
| int inLength = inBytes.remaining(); |
| if(inLength < 8) { |
| throw new ICUException("not enough bytes"); |
| } |
| int indexesLength = inBytes.getInt(); // inIndexes[IX_INDEXES_LENGTH] |
| if(indexesLength < 2 || inLength < indexesLength * 4) { |
| throw new ICUException("not enough indexes"); |
| } |
| int[] inIndexes = new int[IX_TOTAL_SIZE + 1]; |
| inIndexes[0] = indexesLength; |
| for(int i = 1; i < indexesLength && i < inIndexes.length; ++i) { |
| inIndexes[i] = inBytes.getInt(); |
| } |
| for(int i = indexesLength; i < inIndexes.length; ++i) { |
| inIndexes[i] = -1; |
| } |
| if(indexesLength > inIndexes.length) { |
| ICUBinary.skipBytes(inBytes, (indexesLength - inIndexes.length) * 4); |
| } |
| |
| // Assume that the tailoring data is in initial state, |
| // with null pointers and 0 lengths. |
| |
| // Set pointers to non-empty data parts. |
| // Do this in order of their byte offsets. (Should help porting to Java.) |
| |
| int index; // one of the indexes[] slots |
| int offset; // byte offset for the index part |
| int length; // number of bytes in the index part |
| |
| if(indexesLength > IX_TOTAL_SIZE) { |
| length = inIndexes[IX_TOTAL_SIZE]; |
| } else if(indexesLength > IX_REORDER_CODES_OFFSET) { |
| length = inIndexes[indexesLength - 1]; |
| } else { |
| length = 0; // only indexes, and inLength was already checked for them |
| } |
| if(inLength < length) { |
| throw new ICUException("not enough bytes"); |
| } |
| |
| CollationData baseData = base == null ? null : base.data; |
| int[] reorderCodes; |
| int reorderCodesLength; |
| index = IX_REORDER_CODES_OFFSET; |
| offset = inIndexes[index]; |
| length = inIndexes[index + 1] - offset; |
| if(length >= 4) { |
| if(baseData == null) { |
| // We assume for collation settings that |
| // the base data does not have a reordering. |
| throw new ICUException("Collation base data must not reorder scripts"); |
| } |
| reorderCodesLength = length / 4; |
| reorderCodes = new int[reorderCodesLength]; |
| for(int i = 0; i < reorderCodesLength; ++i) { |
| reorderCodes[i] = inBytes.getInt(); |
| } |
| length &= 3; |
| |
| // The reorderRanges (if any) are the trailing reorderCodes entries. |
| // Split the array at the boundary. |
| // Script or reorder codes do not exceed 16-bit values. |
| // Range limits are stored in the upper 16 bits, and are never 0. |
| int reorderRangesLength = 0; |
| while(reorderRangesLength < reorderCodesLength && |
| (reorderCodes[reorderCodesLength - reorderRangesLength - 1] & 0xffff0000) != 0) { |
| ++reorderRangesLength; |
| } |
| assert(reorderRangesLength < reorderCodesLength); |
| reorderCodesLength -= reorderRangesLength; |
| } else { |
| reorderCodes = new int[0]; |
| reorderCodesLength = 0; |
| } |
| ICUBinary.skipBytes(inBytes, length); |
| |
| // There should be a reorder table only if there are reorder codes. |
| // However, when there are reorder codes the reorder table may be omitted to reduce |
| // the data size. |
| byte[] reorderTable = null; |
| index = IX_REORDER_TABLE_OFFSET; |
| offset = inIndexes[index]; |
| length = inIndexes[index + 1] - offset; |
| if(length >= 256) { |
| if(reorderCodesLength == 0) { |
| throw new ICUException("Reordering table without reordering codes"); |
| } |
| reorderTable = new byte[256]; |
| inBytes.get(reorderTable); |
| length -= 256; |
| } else { |
| // If we have reorder codes, then build the reorderTable at the end, |
| // when the CollationData is otherwise complete. |
| } |
| ICUBinary.skipBytes(inBytes, length); |
| |
| if(baseData != null && baseData.numericPrimary != (inIndexes[IX_OPTIONS] & 0xff000000L)) { |
| throw new ICUException("Tailoring numeric primary weight differs from base data"); |
| } |
| CollationData data = null; // Remains null if there are no mappings. |
| |
| index = IX_TRIE_OFFSET; |
| offset = inIndexes[index]; |
| length = inIndexes[index + 1] - offset; |
| if(length >= 8) { |
| tailoring.ensureOwnedData(); |
| data = tailoring.ownedData; |
| data.base = baseData; |
| data.numericPrimary = inIndexes[IX_OPTIONS] & 0xff000000L; |
| data.trie = tailoring.trie = Trie2_32.createFromSerialized(inBytes); |
| int trieLength = data.trie.getSerializedLength(); |
| if(trieLength > length) { |
| throw new ICUException("Not enough bytes for the mappings trie"); // No mappings. |
| } |
| length -= trieLength; |
| } else if(baseData != null) { |
| // Use the base data. Only the settings are tailored. |
| tailoring.data = baseData; |
| } else { |
| throw new ICUException("Missing collation data mappings"); // No mappings. |
| } |
| ICUBinary.skipBytes(inBytes, length); |
| |
| index = IX_RESERVED8_OFFSET; |
| offset = inIndexes[index]; |
| length = inIndexes[index + 1] - offset; |
| ICUBinary.skipBytes(inBytes, length); |
| |
| index = IX_CES_OFFSET; |
| offset = inIndexes[index]; |
| length = inIndexes[index + 1] - offset; |
| if(length >= 8) { |
| if(data == null) { |
| throw new ICUException("Tailored ces without tailored trie"); |
| } |
| data.ces = new long[length / 8]; |
| for(int i = 0; i < length / 8; ++i) { |
| data.ces[i] = inBytes.getLong(); |
| } |
| length &= 7; |
| } |
| ICUBinary.skipBytes(inBytes, length); |
| |
| index = IX_RESERVED10_OFFSET; |
| offset = inIndexes[index]; |
| length = inIndexes[index + 1] - offset; |
| ICUBinary.skipBytes(inBytes, length); |
| |
| index = IX_CE32S_OFFSET; |
| offset = inIndexes[index]; |
| length = inIndexes[index + 1] - offset; |
| if(length >= 4) { |
| if(data == null) { |
| throw new ICUException("Tailored ce32s without tailored trie"); |
| } |
| data.ce32s = new int[length / 4]; |
| for(int i = 0; i < length / 4; ++i) { |
| data.ce32s[i] = inBytes.getInt(); |
| } |
| length &= 3; |
| } |
| ICUBinary.skipBytes(inBytes, length); |
| |
| int jamoCE32sStart = inIndexes[IX_JAMO_CE32S_START]; |
| if(jamoCE32sStart >= 0) { |
| if(data == null || data.ce32s == null) { |
| throw new ICUException("JamoCE32sStart index into non-existent ce32s[]"); |
| } |
| data.jamoCE32s = new int[CollationData.JAMO_CE32S_LENGTH]; |
| System.arraycopy(data.ce32s, jamoCE32sStart, data.jamoCE32s, 0, CollationData.JAMO_CE32S_LENGTH); |
| } else if(data == null) { |
| // Nothing to do. |
| } else if(baseData != null) { |
| data.jamoCE32s = baseData.jamoCE32s; |
| } else { |
| throw new ICUException("Missing Jamo CE32s for Hangul processing"); |
| } |
| |
| index = IX_ROOT_ELEMENTS_OFFSET; |
| offset = inIndexes[index]; |
| length = inIndexes[index + 1] - offset; |
| if(length >= 4) { |
| int rootElementsLength = length / 4; |
| if(data == null) { |
| throw new ICUException("Root elements but no mappings"); |
| } |
| if(rootElementsLength <= CollationRootElements.IX_SEC_TER_BOUNDARIES) { |
| throw new ICUException("Root elements array too short"); |
| } |
| data.rootElements = new long[rootElementsLength]; |
| for(int i = 0; i < rootElementsLength; ++i) { |
| data.rootElements[i] = inBytes.getInt() & 0xffffffffL; // unsigned int -> long |
| } |
| long commonSecTer = data.rootElements[CollationRootElements.IX_COMMON_SEC_AND_TER_CE]; |
| if(commonSecTer != Collation.COMMON_SEC_AND_TER_CE) { |
| throw new ICUException("Common sec/ter weights in base data differ from the hardcoded value"); |
| } |
| long secTerBoundaries = data.rootElements[CollationRootElements.IX_SEC_TER_BOUNDARIES]; |
| if((secTerBoundaries >>> 24) < CollationKeys.SEC_COMMON_HIGH) { |
| // [fixed last secondary common byte] is too low, |
| // and secondary weights would collide with compressed common secondaries. |
| throw new ICUException("[fixed last secondary common byte] is too low"); |
| } |
| length &= 3; |
| } |
| ICUBinary.skipBytes(inBytes, length); |
| |
| index = IX_CONTEXTS_OFFSET; |
| offset = inIndexes[index]; |
| length = inIndexes[index + 1] - offset; |
| if(length >= 2) { |
| if(data == null) { |
| throw new ICUException("Tailored contexts without tailored trie"); |
| } |
| StringBuilder sb = new StringBuilder(length / 2); |
| for(int i = 0; i < length / 2; ++i) { |
| sb.append(inBytes.getChar()); |
| } |
| data.contexts = sb.toString(); |
| length &= 1; |
| } |
| ICUBinary.skipBytes(inBytes, length); |
| |
| index = IX_UNSAFE_BWD_OFFSET; |
| offset = inIndexes[index]; |
| length = inIndexes[index + 1] - offset; |
| if(length >= 2) { |
| if(data == null) { |
| throw new ICUException("Unsafe-backward-set but no mappings"); |
| } |
| if(baseData == null) { |
| // Create the unsafe-backward set for the root collator. |
| // Include all non-zero combining marks and trail surrogates. |
| // We do this at load time, rather than at build time, |
| // to simplify Unicode version bootstrapping: |
| // The root data builder only needs the new FractionalUCA.txt data, |
| // but it need not be built with a version of ICU already updated to |
| // the corresponding new Unicode Character Database. |
| // |
| // The following is an optimized version of |
| // new UnicodeSet("[[:^lccc=0:][\\udc00-\\udfff]]"). |
| // It is faster and requires fewer code dependencies. |
| tailoring.unsafeBackwardSet = new UnicodeSet(0xdc00, 0xdfff); // trail surrogates |
| data.nfcImpl.addLcccChars(tailoring.unsafeBackwardSet); |
| } else { |
| // Clone the root collator's set contents. |
| tailoring.unsafeBackwardSet = baseData.unsafeBackwardSet.cloneAsThawed(); |
| } |
| // Add the ranges from the data file to the unsafe-backward set. |
| USerializedSet sset = new USerializedSet(); |
| char[] unsafeData = new char[length / 2]; |
| for(int i = 0; i < length / 2; ++i) { |
| unsafeData[i] = inBytes.getChar(); |
| } |
| length &= 1; |
| sset.getSet(unsafeData, 0); |
| int count = sset.countRanges(); |
| int[] range = new int[2]; |
| for(int i = 0; i < count; ++i) { |
| sset.getRange(i, range); |
| tailoring.unsafeBackwardSet.add(range[0], range[1]); |
| } |
| // Mark each lead surrogate as "unsafe" |
| // if any of its 1024 associated supplementary code points is "unsafe". |
| int c = 0x10000; |
| for(int lead = 0xd800; lead < 0xdc00; ++lead, c += 0x400) { |
| if(!tailoring.unsafeBackwardSet.containsNone(c, c + 0x3ff)) { |
| tailoring.unsafeBackwardSet.add(lead); |
| } |
| } |
| tailoring.unsafeBackwardSet.freeze(); |
| data.unsafeBackwardSet = tailoring.unsafeBackwardSet; |
| } else if(data == null) { |
| // Nothing to do. |
| } else if(baseData != null) { |
| // No tailoring-specific data: Alias the root collator's set. |
| data.unsafeBackwardSet = baseData.unsafeBackwardSet; |
| } else { |
| throw new ICUException("Missing unsafe-backward-set"); |
| } |
| ICUBinary.skipBytes(inBytes, length); |
| |
| // If the fast Latin format version is different, |
| // or the version is set to 0 for "no fast Latin table", |
| // then just always use the normal string comparison path. |
| index = IX_FAST_LATIN_TABLE_OFFSET; |
| offset = inIndexes[index]; |
| length = inIndexes[index + 1] - offset; |
| if(data != null) { |
| data.fastLatinTable = null; |
| data.fastLatinTableHeader = null; |
| if(((inIndexes[IX_OPTIONS] >> 16) & 0xff) == CollationFastLatin.VERSION) { |
| if(length >= 2) { |
| char header0 = inBytes.getChar(); |
| int headerLength = header0 & 0xff; |
| data.fastLatinTableHeader = new char[headerLength]; |
| data.fastLatinTableHeader[0] = header0; |
| for(int i = 1; i < headerLength; ++i) { |
| data.fastLatinTableHeader[i] = inBytes.getChar(); |
| } |
| int tableLength = length / 2 - headerLength; |
| data.fastLatinTable = new char[tableLength]; |
| for(int i = 0; i < tableLength; ++i) { |
| data.fastLatinTable[i] = inBytes.getChar(); |
| } |
| length &= 1; |
| if((header0 >> 8) != CollationFastLatin.VERSION) { |
| throw new ICUException("Fast-Latin table version differs from version in data header"); |
| } |
| } else if(baseData != null) { |
| data.fastLatinTable = baseData.fastLatinTable; |
| data.fastLatinTableHeader = baseData.fastLatinTableHeader; |
| } |
| } |
| } |
| ICUBinary.skipBytes(inBytes, length); |
| |
| index = IX_SCRIPTS_OFFSET; |
| offset = inIndexes[index]; |
| length = inIndexes[index + 1] - offset; |
| if(length >= 2) { |
| if(data == null) { |
| throw new ICUException("Script order data but no mappings"); |
| } |
| int scriptsLength = length / 2; |
| CharBuffer inChars = inBytes.asCharBuffer(); |
| data.numScripts = inChars.get(); |
| // There must be enough entries for both arrays, including more than two range starts. |
| int scriptStartsLength = scriptsLength - (1 + data.numScripts + 16); |
| if(scriptStartsLength <= 2) { |
| throw new ICUException("Script order data too short"); |
| } |
| inChars.get(data.scriptsIndex = new char[data.numScripts + 16]); |
| inChars.get(data.scriptStarts = new char[scriptStartsLength]); |
| if(!(data.scriptStarts[0] == 0 && |
| data.scriptStarts[1] == ((Collation.MERGE_SEPARATOR_BYTE + 1) << 8) && |
| data.scriptStarts[scriptStartsLength - 1] == |
| (Collation.TRAIL_WEIGHT_BYTE << 8))) { |
| throw new ICUException("Script order data not valid"); |
| } |
| } else if(data == null) { |
| // Nothing to do. |
| } else if(baseData != null) { |
| data.numScripts = baseData.numScripts; |
| data.scriptsIndex = baseData.scriptsIndex; |
| data.scriptStarts = baseData.scriptStarts; |
| } |
| ICUBinary.skipBytes(inBytes, length); |
| |
| index = IX_COMPRESSIBLE_BYTES_OFFSET; |
| offset = inIndexes[index]; |
| length = inIndexes[index + 1] - offset; |
| if(length >= 256) { |
| if(data == null) { |
| throw new ICUException("Data for compressible primary lead bytes but no mappings"); |
| } |
| data.compressibleBytes = new boolean[256]; |
| for(int i = 0; i < 256; ++i) { |
| data.compressibleBytes[i] = inBytes.get() != 0; |
| } |
| length -= 256; |
| } else if(data == null) { |
| // Nothing to do. |
| } else if(baseData != null) { |
| data.compressibleBytes = baseData.compressibleBytes; |
| } else { |
| throw new ICUException("Missing data for compressible primary lead bytes"); |
| } |
| ICUBinary.skipBytes(inBytes, length); |
| |
| index = IX_RESERVED18_OFFSET; |
| offset = inIndexes[index]; |
| length = inIndexes[index + 1] - offset; |
| ICUBinary.skipBytes(inBytes, length); |
| |
| CollationSettings ts = tailoring.settings.readOnly(); |
| int options = inIndexes[IX_OPTIONS] & 0xffff; |
| char[] fastLatinPrimaries = new char[CollationFastLatin.LATIN_LIMIT]; |
| int fastLatinOptions = CollationFastLatin.getOptions( |
| tailoring.data, ts, fastLatinPrimaries); |
| if(options == ts.options && ts.variableTop != 0 && |
| Arrays.equals(reorderCodes, ts.reorderCodes) && |
| fastLatinOptions == ts.fastLatinOptions && |
| (fastLatinOptions < 0 || |
| Arrays.equals(fastLatinPrimaries, ts.fastLatinPrimaries))) { |
| return; |
| } |
| |
| CollationSettings settings = tailoring.settings.copyOnWrite(); |
| settings.options = options; |
| // Set variableTop from options and scripts data. |
| settings.variableTop = tailoring.data.getLastPrimaryForGroup( |
| Collator.ReorderCodes.FIRST + settings.getMaxVariable()); |
| if(settings.variableTop == 0) { |
| throw new ICUException("The maxVariable could not be mapped to a variableTop"); |
| } |
| |
| if(reorderCodesLength != 0) { |
| settings.aliasReordering(baseData, reorderCodes, reorderCodesLength, reorderTable); |
| } |
| |
| settings.fastLatinOptions = CollationFastLatin.getOptions( |
| tailoring.data, settings, |
| settings.fastLatinPrimaries); |
| } |
| |
| private static final class IsAcceptable implements ICUBinary.Authenticate { |
| // @Override when we switch to Java 6 |
| public boolean isDataVersionAcceptable(byte version[]) { |
| return version[0] == 5; |
| } |
| } |
| private static final IsAcceptable IS_ACCEPTABLE = new IsAcceptable(); |
| private static final int DATA_FORMAT = 0x55436f6c; // "UCol" |
| |
| private CollationDataReader() {} // no constructor |
| } |
| |
| /* |
| * Format of collation data (ucadata.icu, binary data in coll/ *.res files): |
| * See ICU4C source/common/collationdatareader.h. |
| */ |
