diff options
| -rw-r--r-- | java/com/google/flatbuffers/FlatBufferBuilder.java | 66 | ||||
| -rw-r--r-- | java/com/google/flatbuffers/Table.java | 38 | ||||
| -rw-r--r-- | java/com/google/flatbuffers/Utf8.java | 191 | ||||
| -rw-r--r-- | java/com/google/flatbuffers/Utf8Old.java | 99 | ||||
| -rw-r--r-- | java/com/google/flatbuffers/Utf8Safe.java | 451 | ||||
| -rw-r--r-- | pom.xml | 6 |
6 files changed, 781 insertions, 70 deletions
diff --git a/java/com/google/flatbuffers/FlatBufferBuilder.java b/java/com/google/flatbuffers/FlatBufferBuilder.java index c34ae3f8..4a62f33c 100644 --- a/java/com/google/flatbuffers/FlatBufferBuilder.java +++ b/java/com/google/flatbuffers/FlatBufferBuilder.java @@ -21,11 +21,7 @@ import static com.google.flatbuffers.Constants.*; import java.io.IOException; import java.io.InputStream; import java.nio.*; -import java.nio.charset.CharacterCodingException; -import java.nio.charset.CharsetEncoder; -import java.nio.charset.CoderResult; import java.util.Arrays; -import java.nio.charset.Charset; /// @file /// @addtogroup flatbuffers_java_api @@ -39,7 +35,6 @@ public class FlatBufferBuilder { /// @cond FLATBUFFERS_INTERNAL ByteBuffer bb; // Where we construct the FlatBuffer. int space; // Remaining space in the ByteBuffer. - static final Charset utf8charset = Charset.forName("UTF-8"); // The UTF-8 character set used by FlatBuffers. int minalign = 1; // Minimum alignment encountered so far. int[] vtable = null; // The vtable for the current table. int vtable_in_use = 0; // The amount of fields we're actually using. @@ -50,9 +45,8 @@ public class FlatBufferBuilder { int num_vtables = 0; // Number of entries in `vtables` in use. int vector_num_elems = 0; // For the current vector being built. boolean force_defaults = false; // False omits default values from the serialized data. - CharsetEncoder encoder = utf8charset.newEncoder(); - ByteBuffer dst; ByteBufferFactory bb_factory; // Factory for allocating the internal buffer + final Utf8 utf8; // UTF-8 encoder to use /// @endcond /** @@ -62,10 +56,31 @@ public class FlatBufferBuilder { * @param bb_factory The factory to be used for allocating the internal buffer */ public FlatBufferBuilder(int initial_size, ByteBufferFactory bb_factory) { - if (initial_size <= 0) initial_size = 1; + this(initial_size, bb_factory, null, Utf8.getDefault()); + } + + /** + * Start with a buffer of size `initial_size`, then grow as required. + * + * @param initial_size The initial size of the internal buffer to use. + * @param bb_factory The factory to be used for allocating the internal buffer + * @param existing_bb The byte buffer to reuse. + */ + public FlatBufferBuilder(int initial_size, ByteBufferFactory bb_factory, + ByteBuffer existing_bb, Utf8 utf8) { + if (initial_size <= 0) { + initial_size = 1; + } space = initial_size; this.bb_factory = bb_factory; - bb = bb_factory.newByteBuffer(initial_size); + if (existing_bb != null) { + bb = existing_bb; + bb.clear(); + } else { + bb = bb_factory.newByteBuffer(initial_size); + } + bb.order(ByteOrder.LITTLE_ENDIAN); + this.utf8 = utf8; } /** @@ -74,7 +89,7 @@ public class FlatBufferBuilder { * @param initial_size The initial size of the internal buffer to use. */ public FlatBufferBuilder(int initial_size) { - this(initial_size, new HeapByteBufferFactory()); + this(initial_size, new HeapByteBufferFactory(), null, Utf8.getDefault()); } /** @@ -94,7 +109,7 @@ public class FlatBufferBuilder { * the existing buffer needs to grow */ public FlatBufferBuilder(ByteBuffer existing_bb, ByteBufferFactory bb_factory) { - init(existing_bb, bb_factory); + this(existing_bb.capacity(), bb_factory, existing_bb, Utf8.getDefault()); } /** @@ -105,7 +120,7 @@ public class FlatBufferBuilder { * @param existing_bb The byte buffer to reuse. */ public FlatBufferBuilder(ByteBuffer existing_bb) { - init(existing_bb, new HeapByteBufferFactory()); + this(existing_bb, new HeapByteBufferFactory()); } /** @@ -503,27 +518,12 @@ public class FlatBufferBuilder { * @return The offset in the buffer where the encoded string starts. */ public int createString(CharSequence s) { - int length = s.length(); - int estimatedDstCapacity = (int) (length * encoder.maxBytesPerChar()); - if (dst == null || dst.capacity() < estimatedDstCapacity) { - dst = ByteBuffer.allocate(Math.max(128, estimatedDstCapacity)); - } - - dst.clear(); - - CharBuffer src = s instanceof CharBuffer ? (CharBuffer) s : - CharBuffer.wrap(s); - CoderResult result = encoder.encode(src, dst, true); - if (result.isError()) { - try { - result.throwException(); - } catch (CharacterCodingException x) { - throw new Error(x); - } - } - - dst.flip(); - return createString(dst); + int length = utf8.encodedLength(s); + addByte((byte)0); + startVector(1, length, 1); + bb.position(space -= length); + utf8.encodeUtf8(s, bb); + return endVector(); } /** diff --git a/java/com/google/flatbuffers/Table.java b/java/com/google/flatbuffers/Table.java index 489893b6..703ce28a 100644 --- a/java/com/google/flatbuffers/Table.java +++ b/java/com/google/flatbuffers/Table.java @@ -19,11 +19,7 @@ package com.google.flatbuffers; import static com.google.flatbuffers.Constants.*; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.CharBuffer; -import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CoderResult; /// @cond FLATBUFFERS_INTERNAL @@ -31,23 +27,17 @@ import java.nio.charset.CoderResult; * All tables in the generated code derive from this class, and add their own accessors. */ public class Table { - private final static ThreadLocal<CharsetDecoder> UTF8_DECODER = new ThreadLocal<CharsetDecoder>() { - @Override - protected CharsetDecoder initialValue() { - return Charset.forName("UTF-8").newDecoder(); - } - }; public final static ThreadLocal<Charset> UTF8_CHARSET = new ThreadLocal<Charset>() { @Override protected Charset initialValue() { return Charset.forName("UTF-8"); } }; - private final static ThreadLocal<CharBuffer> CHAR_BUFFER = new ThreadLocal<CharBuffer>(); /** Used to hold the position of the `bb` buffer. */ protected int bb_pos; /** The underlying ByteBuffer to hold the data of the Table. */ protected ByteBuffer bb; + Utf8 utf8 = Utf8.getDefault(); /** * Get the underlying ByteBuffer. @@ -98,34 +88,10 @@ public class Table { * @return Returns a `String` from the data stored inside the FlatBuffer at `offset`. */ protected String __string(int offset) { - CharsetDecoder decoder = UTF8_DECODER.get(); - decoder.reset(); - offset += bb.getInt(offset); ByteBuffer src = bb.duplicate().order(ByteOrder.LITTLE_ENDIAN); int length = src.getInt(offset); - src.position(offset + SIZEOF_INT); - src.limit(offset + SIZEOF_INT + length); - - int required = (int)((float)length * decoder.maxCharsPerByte()); - CharBuffer dst = CHAR_BUFFER.get(); - if (dst == null || dst.capacity() < required) { - dst = CharBuffer.allocate(required); - CHAR_BUFFER.set(dst); - } - - dst.clear(); - - try { - CoderResult cr = decoder.decode(src, dst, true); - if (!cr.isUnderflow()) { - cr.throwException(); - } - } catch (CharacterCodingException x) { - throw new RuntimeException(x); - } - - return dst.flip().toString(); + return utf8.decodeUtf8(bb, offset + SIZEOF_INT, length); } /** diff --git a/java/com/google/flatbuffers/Utf8.java b/java/com/google/flatbuffers/Utf8.java new file mode 100644 index 00000000..a2640637 --- /dev/null +++ b/java/com/google/flatbuffers/Utf8.java @@ -0,0 +1,191 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.flatbuffers; + +import java.nio.ByteBuffer; + +import static java.lang.Character.MIN_HIGH_SURROGATE; +import static java.lang.Character.MIN_LOW_SURROGATE; +import static java.lang.Character.MIN_SUPPLEMENTARY_CODE_POINT; + +public abstract class Utf8 { + + /** + * Returns the number of bytes in the UTF-8-encoded form of {@code sequence}. For a string, + * this method is equivalent to {@code string.getBytes(UTF_8).length}, but is more efficient in + * both time and space. + * + * @throws IllegalArgumentException if {@code sequence} contains ill-formed UTF-16 (unpaired + * surrogates) + */ + public abstract int encodedLength(CharSequence sequence); + + /** + * Encodes the given characters to the target {@link ByteBuffer} using UTF-8 encoding. + * + * <p>Selects an optimal algorithm based on the type of {@link ByteBuffer} (i.e. heap or direct) + * and the capabilities of the platform. + * + * @param in the source string to be encoded + * @param out the target buffer to receive the encoded string. + */ + public abstract void encodeUtf8(CharSequence in, ByteBuffer out); + + /** + * Decodes the given UTF-8 portion of the {@link ByteBuffer} into a {@link String}. + * + * @throws IllegalArgumentException if the input is not valid UTF-8. + */ + public abstract String decodeUtf8(ByteBuffer buffer, int offset, int length); + + private static Utf8 DEFAULT; + + /** + * Get the default UTF-8 processor. + * @return the default processor + */ + public static Utf8 getDefault() { + if (DEFAULT == null) { + DEFAULT = new Utf8Safe(); + } + return DEFAULT; + } + + /** + * Set the default instance of the UTF-8 processor. + * @param instance the new instance to use + */ + public static void setDefault(Utf8 instance) { + DEFAULT = instance; + } + + /** + * Utility methods for decoding bytes into {@link String}. Callers are responsible for extracting + * bytes (possibly using Unsafe methods), and checking remaining bytes. All other UTF-8 validity + * checks and codepoint conversion happen in this class. + */ + static class DecodeUtil { + + /** + * Returns whether this is a single-byte codepoint (i.e., ASCII) with the form '0XXXXXXX'. + */ + static boolean isOneByte(byte b) { + return b >= 0; + } + + /** + * Returns whether this is a two-byte codepoint with the form '10XXXXXX'. + */ + static boolean isTwoBytes(byte b) { + return b < (byte) 0xE0; + } + + /** + * Returns whether this is a three-byte codepoint with the form '110XXXXX'. + */ + static boolean isThreeBytes(byte b) { + return b < (byte) 0xF0; + } + + static void handleOneByte(byte byte1, char[] resultArr, int resultPos) { + resultArr[resultPos] = (char) byte1; + } + + static void handleTwoBytes( + byte byte1, byte byte2, char[] resultArr, int resultPos) + throws IllegalArgumentException { + // Simultaneously checks for illegal trailing-byte in leading position (<= '11000000') and + // overlong 2-byte, '11000001'. + if (byte1 < (byte) 0xC2 + || isNotTrailingByte(byte2)) { + throw new IllegalArgumentException("Invalid UTF-8"); + } + resultArr[resultPos] = (char) (((byte1 & 0x1F) << 6) | trailingByteValue(byte2)); + } + + static void handleThreeBytes( + byte byte1, byte byte2, byte byte3, char[] resultArr, int resultPos) + throws IllegalArgumentException { + if (isNotTrailingByte(byte2) + // overlong? 5 most significant bits must not all be zero + || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) + // check for illegal surrogate codepoints + || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) + || isNotTrailingByte(byte3)) { + throw new IllegalArgumentException("Invalid UTF-8"); + } + resultArr[resultPos] = (char) + (((byte1 & 0x0F) << 12) | (trailingByteValue(byte2) << 6) | trailingByteValue(byte3)); + } + + static void handleFourBytes( + byte byte1, byte byte2, byte byte3, byte byte4, char[] resultArr, int resultPos) + throws IllegalArgumentException{ + if (isNotTrailingByte(byte2) + // Check that 1 <= plane <= 16. Tricky optimized form of: + // valid 4-byte leading byte? + // if (byte1 > (byte) 0xF4 || + // overlong? 4 most significant bits must not all be zero + // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || + // codepoint larger than the highest code point (U+10FFFF)? + // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) + || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 + || isNotTrailingByte(byte3) + || isNotTrailingByte(byte4)) { + throw new IllegalArgumentException("Invalid UTF-8"); + } + int codepoint = ((byte1 & 0x07) << 18) + | (trailingByteValue(byte2) << 12) + | (trailingByteValue(byte3) << 6) + | trailingByteValue(byte4); + resultArr[resultPos] = DecodeUtil.highSurrogate(codepoint); + resultArr[resultPos + 1] = DecodeUtil.lowSurrogate(codepoint); + } + + /** + * Returns whether the byte is not a valid continuation of the form '10XXXXXX'. + */ + private static boolean isNotTrailingByte(byte b) { + return b > (byte) 0xBF; + } + + /** + * Returns the actual value of the trailing byte (removes the prefix '10') for composition. + */ + private static int trailingByteValue(byte b) { + return b & 0x3F; + } + + private static char highSurrogate(int codePoint) { + return (char) ((MIN_HIGH_SURROGATE - (MIN_SUPPLEMENTARY_CODE_POINT >>> 10)) + + (codePoint >>> 10)); + } + + private static char lowSurrogate(int codePoint) { + return (char) (MIN_LOW_SURROGATE + (codePoint & 0x3ff)); + } + } + + // These UTF-8 handling methods are copied from Guava's Utf8Unsafe class with a modification to throw + // a protocol buffer local exception. This exception is then caught in CodedOutputStream so it can + // fallback to more lenient behavior. + static class UnpairedSurrogateException extends IllegalArgumentException { + UnpairedSurrogateException(int index, int length) { + super("Unpaired surrogate at index " + index + " of " + length); + } + } +} diff --git a/java/com/google/flatbuffers/Utf8Old.java b/java/com/google/flatbuffers/Utf8Old.java new file mode 100644 index 00000000..97933332 --- /dev/null +++ b/java/com/google/flatbuffers/Utf8Old.java @@ -0,0 +1,99 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.flatbuffers; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.nio.charset.StandardCharsets; + +/** + * This class implements the Utf8 API using the Java Utf8 encoder. Use + * Utf8.setDefault(new Utf8Old()); to use it. + */ +public class Utf8Old extends Utf8 { + + private static class Cache { + final CharsetEncoder encoder; + final CharsetDecoder decoder; + CharSequence lastInput = null; + ByteBuffer lastOutput = null; + + Cache() { + encoder = StandardCharsets.UTF_8.newEncoder(); + decoder = StandardCharsets.UTF_8.newDecoder(); + } + } + + private static final ThreadLocal<Cache> CACHE = + ThreadLocal.withInitial(() -> new Cache()); + + // Play some games so that the old encoder doesn't pay twice for computing + // the length of the encoded string. + + @Override + public int encodedLength(CharSequence in) { + final Cache cache = CACHE.get(); + int estimated = (int) (in.length() * cache.encoder.maxBytesPerChar()); + if (cache.lastOutput == null || cache.lastOutput.capacity() < estimated) { + cache.lastOutput = ByteBuffer.allocate(Math.max(128, estimated)); + } + cache.lastOutput.clear(); + cache.lastInput = in; + CharBuffer wrap = (in instanceof CharBuffer) ? + (CharBuffer) in : CharBuffer.wrap(in); + CoderResult result = cache.encoder.encode(wrap, cache.lastOutput, true); + if (result.isError()) { + try { + result.throwException(); + } catch (CharacterCodingException e) { + throw new IllegalArgumentException("bad character encoding", e); + } + } + return cache.lastOutput.remaining(); + } + + @Override + public void encodeUtf8(CharSequence in, ByteBuffer out) { + final Cache cache = CACHE.get(); + if (cache.lastInput != in) { + // Update the lastOutput to match our input, although flatbuffer should + // never take this branch. + encodedLength(in); + } + out.put(cache.lastOutput); + } + + @Override + public String decodeUtf8(ByteBuffer buffer, int offset, int length) { + CharsetDecoder decoder = CACHE.get().decoder; + decoder.reset(); + buffer = buffer.duplicate(); + buffer.position(offset); + buffer.limit(offset + length); + try { + CharBuffer result = decoder.decode(buffer); + result.flip(); + return result.toString(); + } catch (CharacterCodingException e) { + throw new IllegalArgumentException("Bad encoding", e); + } + } +} diff --git a/java/com/google/flatbuffers/Utf8Safe.java b/java/com/google/flatbuffers/Utf8Safe.java new file mode 100644 index 00000000..06ea420b --- /dev/null +++ b/java/com/google/flatbuffers/Utf8Safe.java @@ -0,0 +1,451 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.flatbuffers; + +import java.nio.ByteBuffer; +import static java.lang.Character.MAX_SURROGATE; +import static java.lang.Character.MIN_SUPPLEMENTARY_CODE_POINT; +import static java.lang.Character.MIN_SURROGATE; +import static java.lang.Character.isSurrogatePair; +import static java.lang.Character.toCodePoint; + +/** + * A set of low-level, high-performance static utility methods related + * to the UTF-8 character encoding. This class has no dependencies + * outside of the core JDK libraries. + * + * <p>There are several variants of UTF-8. The one implemented by + * this class is the restricted definition of UTF-8 introduced in + * Unicode 3.1, which mandates the rejection of "overlong" byte + * sequences as well as rejection of 3-byte surrogate codepoint byte + * sequences. Note that the UTF-8 decoder included in Oracle's JDK + * has been modified to also reject "overlong" byte sequences, but (as + * of 2011) still accepts 3-byte surrogate codepoint byte sequences. + * + * <p>The byte sequences considered valid by this class are exactly + * those that can be roundtrip converted to Strings and back to bytes + * using the UTF-8 charset, without loss: <pre> {@code + * Arrays.equals(bytes, new String(bytes, Internal.UTF_8).getBytes(Internal.UTF_8)) + * }</pre> + * + * <p>See the Unicode Standard,</br> + * Table 3-6. <em>UTF-8 Bit Distribution</em>,</br> + * Table 3-7. <em>Well Formed UTF-8 Byte Sequences</em>. + */ +final public class Utf8Safe extends Utf8 { + + /** + * Returns the number of bytes in the UTF-8-encoded form of {@code sequence}. For a string, + * this method is equivalent to {@code string.getBytes(UTF_8).length}, but is more efficient in + * both time and space. + * + * @throws IllegalArgumentException if {@code sequence} contains ill-formed UTF-16 (unpaired + * surrogates) + */ + private static int computeEncodedLength(CharSequence sequence) { + // Warning to maintainers: this implementation is highly optimized. + int utf16Length = sequence.length(); + int utf8Length = utf16Length; + int i = 0; + + // This loop optimizes for pure ASCII. + while (i < utf16Length && sequence.charAt(i) < 0x80) { + i++; + } + + // This loop optimizes for chars less than 0x800. + for (; i < utf16Length; i++) { + char c = sequence.charAt(i); + if (c < 0x800) { + utf8Length += ((0x7f - c) >>> 31); // branch free! + } else { + utf8Length += encodedLengthGeneral(sequence, i); + break; + } + } + + if (utf8Length < utf16Length) { + // Necessary and sufficient condition for overflow because of maximum 3x expansion + throw new IllegalArgumentException("UTF-8 length does not fit in int: " + + (utf8Length + (1L << 32))); + } + return utf8Length; + } + + private static int encodedLengthGeneral(CharSequence sequence, int start) { + int utf16Length = sequence.length(); + int utf8Length = 0; + for (int i = start; i < utf16Length; i++) { + char c = sequence.charAt(i); + if (c < 0x800) { + utf8Length += (0x7f - c) >>> 31; // branch free! + } else { + utf8Length += 2; + // jdk7+: if (Character.isSurrogate(c)) { + if (Character.MIN_SURROGATE <= c && c <= Character.MAX_SURROGATE) { + // Check that we have a well-formed surrogate pair. + int cp = Character.codePointAt(sequence, i); + if (cp < MIN_SUPPLEMENTARY_CODE_POINT) { + throw new Utf8Safe.UnpairedSurrogateException(i, utf16Length); + } + i++; + } + } + } + return utf8Length; + } + + private static String decodeUtf8Array(byte[] bytes, int index, int size) { + // Bitwise OR combines the sign bits so any negative value fails the check. + if ((index | size | bytes.length - index - size) < 0) { + throw new ArrayIndexOutOfBoundsException( + String.format("buffer length=%d, index=%d, size=%d", bytes.length, index, size)); + } + + int offset = index; + final int limit = offset + size; + + // The longest possible resulting String is the same as the number of input bytes, when it is + // all ASCII. For other cases, this over-allocates and we will truncate in the end. + char[] resultArr = new char[size]; + int resultPos = 0; + + // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this). + // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). + while (offset < limit) { + byte b = bytes[offset]; + if (!DecodeUtil.isOneByte(b)) { + break; + } + offset++; + DecodeUtil.handleOneByte(b, resultArr, resultPos++); + } + + while (offset < limit) { + byte byte1 = bytes[offset++]; + if (DecodeUtil.isOneByte(byte1)) { + DecodeUtil.handleOneByte(byte1, resultArr, resultPos++); + // It's common for there to be multiple ASCII characters in a run mixed in, so add an + // extra optimized loop to take care of these runs. + while (offset < limit) { + byte b = bytes[offset]; + if (!DecodeUtil.isOneByte(b)) { + break; + } + offset++; + DecodeUtil.handleOneByte(b, resultArr, resultPos++); + } + } else if (DecodeUtil.isTwoBytes(byte1)) { + if (offset >= limit) { + throw new IllegalArgumentException("Invalid UTF-8"); + } + DecodeUtil.handleTwoBytes(byte1, /* byte2 */ bytes[offset++], resultArr, resultPos++); + } else if (DecodeUtil.isThreeBytes(byte1)) { + if (offset >= limit - 1) { + throw new IllegalArgumentException("Invalid UTF-8"); + } + DecodeUtil.handleThreeBytes( + byte1, + /* byte2 */ bytes[offset++], + /* byte3 */ bytes[offset++], + resultArr, + resultPos++); + } else { + if (offset >= limit - 2) { + throw new IllegalArgumentException("Invalid UTF-8"); + } + DecodeUtil.handleFourBytes( + byte1, + /* byte2 */ bytes[offset++], + /* byte3 */ bytes[offset++], + /* byte4 */ bytes[offset++], + resultArr, + resultPos++); + // 4-byte case requires two chars. + resultPos++; + } + } + + return new String(resultArr, 0, resultPos); + } + + private static String decodeUtf8Buffer(ByteBuffer buffer, int offset, + int length) { + // Bitwise OR combines the sign bits so any negative value fails the check. + if ((offset | length | buffer.limit() - offset - length) < 0) { + throw new ArrayIndexOutOfBoundsException( + String.format("buffer limit=%d, index=%d, limit=%d", buffer.limit(), + offset, length)); + } + + final int limit = offset + length; + + // The longest possible resulting String is the same as the number of input bytes, when it is + // all ASCII. For other cases, this over-allocates and we will truncate in the end. + char[] resultArr = new char[length]; + int resultPos = 0; + + // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this). + // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). + while (offset < limit) { + byte b = buffer.get(offset); + if (!DecodeUtil.isOneByte(b)) { + break; + } + offset++; + DecodeUtil.handleOneByte(b, resultArr, resultPos++); + } + + while (offset < limit) { + byte byte1 = buffer.get(offset++); + if (DecodeUtil.isOneByte(byte1)) { + DecodeUtil.handleOneByte(byte1, resultArr, resultPos++); + // It's common for there to be multiple ASCII characters in a run mixed in, so add an + // extra optimized loop to take care of these runs. + while (offset < limit) { + byte b = buffer.get(offset); + if (!DecodeUtil.isOneByte(b)) { + break; + } + offset++; + DecodeUtil.handleOneByte(b, resultArr, resultPos++); + } + } else if (DecodeUtil.isTwoBytes(byte1)) { + if (offset >= limit) { + throw new IllegalArgumentException("Invalid UTF-8"); + } + DecodeUtil.handleTwoBytes( + byte1, /* byte2 */ buffer.get(offset++), resultArr, resultPos++); + } else if (DecodeUtil.isThreeBytes(byte1)) { + if (offset >= limit - 1) { + throw new IllegalArgumentException("Invalid UTF-8"); + } + DecodeUtil.handleThreeBytes( + byte1, + /* byte2 */ buffer.get(offset++), + /* byte3 */ buffer.get(offset++), + resultArr, + resultPos++); + } else { + if (offset >= limit - 2) { + throw new IllegalArgumentException("Invalid UTF-8"); + } + DecodeUtil.handleFourBytes( + byte1, + /* byte2 */ buffer.get(offset++), + /* byte3 */ buffer.get(offset++), + /* byte4 */ buffer.get(offset++), + resultArr, + resultPos++); + // 4-byte case requires two chars. + resultPos++; + } + } + + return new String(resultArr, 0, resultPos); + } + + @Override + public int encodedLength(CharSequence in) { + return computeEncodedLength(in); + } + + /** + * Decodes the given UTF-8 portion of the {@link ByteBuffer} into a {@link String}. + * + * @throws IllegalArgumentException if the input is not valid UTF-8. + */ + @Override + public String decodeUtf8(ByteBuffer buffer, int offset, int length) + throws IllegalArgumentException { + if (buffer.hasArray()) { + return decodeUtf8Array(buffer.array(), buffer.arrayOffset() + offset, length); + } else { + return decodeUtf8Buffer(buffer, offset, length); + } + } + + + private static void encodeUtf8Buffer(CharSequence in, ByteBuffer out) { + final int inLength = in.length(); + int outIx = out.position(); + int inIx = 0; + + // Since ByteBuffer.putXXX() already checks boundaries for us, no need to explicitly check + // access. Assume the buffer is big enough and let it handle the out of bounds exception + // if it occurs. + try { + // Designed to take advantage of + // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination + for (char c; inIx < inLength && (c = in.charAt(inIx)) < 0x80; ++inIx) { + out.put(outIx + inIx, (byte) c); + } + if (inIx == inLength) { + // Successfully encoded the entire string. + out.position(outIx + inIx); + return; + } + + outIx += inIx; + for (char c; inIx < inLength; ++inIx, ++outIx) { + c = in.charAt(inIx); + if (c < 0x80) { + // One byte (0xxx xxxx) + out.put(outIx, (byte) c); + } else if (c < 0x800) { + // Two bytes (110x xxxx 10xx xxxx) + + // Benchmarks show put performs better than putShort here (for HotSpot). + out.put(outIx++, (byte) (0xC0 | (c >>> 6))); + out.put(outIx, (byte) (0x80 | (0x3F & c))); + } else if (c < MIN_SURROGATE || MAX_SURROGATE < c) { + // Three bytes (1110 xxxx 10xx xxxx 10xx xxxx) + // Maximum single-char code point is 0xFFFF, 16 bits. + + // Benchmarks show put performs better than putShort here (for HotSpot). + out.put(outIx++, (byte) (0xE0 | (c >>> 12))); + out.put(outIx++, (byte) (0x80 | (0x3F & (c >>> 6)))); + out.put(outIx, (byte) (0x80 | (0x3F & c))); + } else { + // Four bytes (1111 xxxx 10xx xxxx 10xx xxxx 10xx xxxx) + + // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8 + // bytes + final char low; + if (inIx + 1 == inLength || !isSurrogatePair(c, (low = in.charAt(++inIx)))) { + throw new UnpairedSurrogateException(inIx, inLength); + } + // TODO(nathanmittler): Consider using putInt() to improve performance. + int codePoint = toCodePoint(c, low); + out.put(outIx++, (byte) ((0xF << 4) | (codePoint >>> 18))); + out.put(outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 12)))); + out.put(outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 6)))); + out.put(outIx, (byte) (0x80 | (0x3F & codePoint))); + } + } + + // Successfully encoded the entire string. + out.position(outIx); + } catch (IndexOutOfBoundsException e) { + // TODO(nathanmittler): Consider making the API throw IndexOutOfBoundsException instead. + + // If we failed in the outer ASCII loop, outIx will not have been updated. In this case, + // use inIx to determine the bad write index. + int badWriteIndex = out.position() + Math.max(inIx, outIx - out.position() + 1); + throw new ArrayIndexOutOfBoundsException( + "Failed writing " + in.charAt(inIx) + " at index " + badWriteIndex); + } + } + + private static int encodeUtf8Array(CharSequence in, byte[] out, + int offset, int length) { + int utf16Length = in.length(); + int j = offset; + int i = 0; + int limit = offset + length; + // Designed to take advantage of + // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination + for (char c; i < utf16Length && i + j < limit && (c = in.charAt(i)) < 0x80; i++) { + out[j + i] = (byte) c; + } + if (i == utf16Length) { + return j + utf16Length; + } + j += i; + for (char c; i < utf16Length; i++) { + c = in.charAt(i); + if (c < 0x80 && j < limit) { + out[j++] = (byte) c; + } else if (c < 0x800 && j <= limit - 2) { // 11 bits, two UTF-8 bytes + out[j++] = (byte) ((0xF << 6) | (c >>> 6)); + out[j++] = (byte) (0x80 | (0x3F & c)); + } else if ((c < Character.MIN_SURROGATE || Character.MAX_SURROGATE < c) && j <= limit - 3) { + // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes + out[j++] = (byte) ((0xF << 5) | (c >>> 12)); + out[j++] = (byte) (0x80 | (0x3F & (c >>> 6))); + out[j++] = (byte) (0x80 | (0x3F & c)); + } else if (j <= limit - 4) { + // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, + // four UTF-8 bytes + final char low; + if (i + 1 == in.length() + || !Character.isSurrogatePair(c, (low = in.charAt(++i)))) { + throw new UnpairedSurrogateException((i - 1), utf16Length); + } + int codePoint = Character.toCodePoint(c, low); + out[j++] = (byte) ((0xF << 4) | (codePoint >>> 18)); + out[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 12))); + out[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 6))); + out[j++] = (byte) (0x80 | (0x3F & codePoint)); + } else { + // If we are surrogates and we're not a surrogate pair, always throw an + // UnpairedSurrogateException instead of an ArrayOutOfBoundsException. + if ((Character.MIN_SURROGATE <= c && c <= Character.MAX_SURROGATE) + && (i + 1 == in.length() + || !Character.isSurrogatePair(c, in.charAt(i + 1)))) { + throw new UnpairedSurrogateException(i, utf16Length); + } + throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + j); + } + } + return j; + } + + /** + * Encodes the given characters to the target {@link ByteBuffer} using UTF-8 encoding. + * + * <p>Selects an optimal algorithm based on the type of {@link ByteBuffer} (i.e. heap or direct) + * and the capabilities of the platform. + * + * @param in the source string to be encoded + * @param out the target buffer to receive the encoded string. + */ + @Override + public void encodeUtf8(CharSequence in, ByteBuffer out) { + if (out.hasArray()) { + int start = out.arrayOffset(); + int end = encodeUtf8Array(in, out.array(), start + out.position(), + out.remaining()); + out.position(end - start); + } else { + encodeUtf8Buffer(in, out); + } + } + + // These UTF-8 handling methods are copied from Guava's Utf8Unsafe class with + // a modification to throw a local exception. This exception can be caught + // to fallback to more lenient behavior. + static class UnpairedSurrogateException extends IllegalArgumentException { + UnpairedSurrogateException(int index, int length) { + super("Unpaired surrogate at index " + index + " of " + length); + } + } +} @@ -5,7 +5,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>com.google.flatbuffers</groupId> <artifactId>flatbuffers-java</artifactId> - <version>1.10.0</version> + <version>1.10.1-SNAPSHOT</version> <packaging>bundle</packaging> <name>FlatBuffers Java API</name> <description> @@ -78,6 +78,10 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-javadoc-plugin</artifactId> <version>2.9.1</version> + <configuration> + <additionalparam>-Xdoclint:none</additionalparam> + <additionalOptions>-Xdoclint:none</additionalOptions> + </configuration> <executions> <execution> <id>attach-javadocs</id> |