Skip to content

F
flatbuffers
Commit cb99116a authored by Owen O'Malley's avatar Owen O'Malley Committed by Wouter van Oortmerssen
Browse files
Options

Java: Pulling in protobuf's faster UTF-8 encoder. (#5035) 

* Pulling in protobuf's faster UTF-8 encoder.

* Remove Utf8 unsafe code.
parent 9ad73bf5
Show whitespace changes
Inline Side-by-side
Showing with 780 additions and 69 deletions
java/com/google/flatbuffers/FlatBufferBuilder.java
View file @ cb99116a
......@@ -21,11 +21,7 @@ import static com.google.flatbuffers.Constants.*;
import java.io.IOException;
import java.io.InputStream;
import java.nio.*;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CoderResult;
import java.util.Arrays;
import java.nio.charset.Charset;
/// @file
/// @addtogroup flatbuffers_java_api
......@@ -39,7 +35,6 @@ public class FlatBufferBuilder {
/// @cond FLATBUFFERS_INTERNAL
ByteBuffer bb; // Where we construct the FlatBuffer.
int space; // Remaining space in the ByteBuffer.
static final Charset utf8charset = Charset.forName("UTF-8"); // The UTF-8 character set used by FlatBuffers.
int minalign = 1; // Minimum alignment encountered so far.
int[] vtable = null; // The vtable for the current table.
int vtable_in_use = 0; // The amount of fields we're actually using.
......@@ -50,9 +45,8 @@ public class FlatBufferBuilder {
int num_vtables = 0; // Number of entries in `vtables` in use.
int vector_num_elems = 0; // For the current vector being built.
boolean force_defaults = false; // False omits default values from the serialized data.
CharsetEncoder encoder = utf8charset.newEncoder();
ByteBuffer dst;
ByteBufferFactory bb_factory; // Factory for allocating the internal buffer
final Utf8 utf8; // UTF-8 encoder to use
/// @endcond
/**
......@@ -62,11 +56,32 @@ public class FlatBufferBuilder {
* @param bb_factory The factory to be used for allocating the internal buffer
*/
public FlatBufferBuilder(int initial_size, ByteBufferFactory bb_factory) {
if (initial_size <= 0) initial_size = 1;
this(initial_size, bb_factory, null, Utf8.getDefault());
}
/**
* Start with a buffer of size `initial_size`, then grow as required.
*
* @param initial_size The initial size of the internal buffer to use.
* @param bb_factory The factory to be used for allocating the internal buffer
* @param existing_bb The byte buffer to reuse.
*/
public FlatBufferBuilder(int initial_size, ByteBufferFactory bb_factory,
ByteBuffer existing_bb, Utf8 utf8) {
if (initial_size <= 0) {
initial_size = 1;
}
space = initial_size;
this.bb_factory = bb_factory;
if (existing_bb != null) {
bb = existing_bb;
bb.clear();
} else {
bb = bb_factory.newByteBuffer(initial_size);
}
bb.order(ByteOrder.LITTLE_ENDIAN);
this.utf8 = utf8;
}
/**
* Start with a buffer of size `initial_size`, then grow as required.
......@@ -74,7 +89,7 @@ public class FlatBufferBuilder {
* @param initial_size The initial size of the internal buffer to use.
*/
public FlatBufferBuilder(int initial_size) {
this(initial_size, new HeapByteBufferFactory());
this(initial_size, new HeapByteBufferFactory(), null, Utf8.getDefault());
}
/**
......@@ -94,7 +109,7 @@ public class FlatBufferBuilder {
* the existing buffer needs to grow
*/
public FlatBufferBuilder(ByteBuffer existing_bb, ByteBufferFactory bb_factory) {
init(existing_bb, bb_factory);
this(existing_bb.capacity(), bb_factory, existing_bb, Utf8.getDefault());
}
/**
......@@ -105,7 +120,7 @@ public class FlatBufferBuilder {
* @param existing_bb The byte buffer to reuse.
*/
public FlatBufferBuilder(ByteBuffer existing_bb) {
init(existing_bb, new HeapByteBufferFactory());
this(existing_bb, new HeapByteBufferFactory());
}
/**
......@@ -503,27 +518,12 @@ public class FlatBufferBuilder {
* @return The offset in the buffer where the encoded string starts.
*/
public int createString(CharSequence s) {
int length = s.length();
int estimatedDstCapacity = (int) (length * encoder.maxBytesPerChar());
if (dst == null || dst.capacity() < estimatedDstCapacity) {
dst = ByteBuffer.allocate(Math.max(128, estimatedDstCapacity));
}
dst.clear();
CharBuffer src = s instanceof CharBuffer ? (CharBuffer) s :
CharBuffer.wrap(s);
CoderResult result = encoder.encode(src, dst, true);
if (result.isError()) {
try {
result.throwException();
} catch (CharacterCodingException x) {
throw new Error(x);
}
}
dst.flip();
return createString(dst);
int length = utf8.encodedLength(s);
addByte((byte)0);
startVector(1, length, 1);
bb.position(space -= length);
utf8.encodeUtf8(s, bb);
return endVector();
}
/**
......
java/com/google/flatbuffers/Table.java
View file @ cb99116a
......@@ -19,11 +19,7 @@ package com.google.flatbuffers;
import static com.google.flatbuffers.Constants.*;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.CharBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CoderResult;
/// @cond FLATBUFFERS_INTERNAL
......@@ -31,23 +27,17 @@ import java.nio.charset.CoderResult;
* All tables in the generated code derive from this class, and add their own accessors.
*/
public class Table {
private final static ThreadLocal<CharsetDecoder> UTF8_DECODER = new ThreadLocal<CharsetDecoder>() {
@Override
protected CharsetDecoder initialValue() {
return Charset.forName("UTF-8").newDecoder();
}
};
public final static ThreadLocal<Charset> UTF8_CHARSET = new ThreadLocal<Charset>() {
@Override
protected Charset initialValue() {
return Charset.forName("UTF-8");
}
};
private final static ThreadLocal<CharBuffer> CHAR_BUFFER = new ThreadLocal<CharBuffer>();
/** Used to hold the position of the `bb` buffer. */
protected int bb_pos;
/** The underlying ByteBuffer to hold the data of the Table. */
protected ByteBuffer bb;
Utf8 utf8 = Utf8.getDefault();
/**
* Get the underlying ByteBuffer.
......@@ -98,34 +88,10 @@ public class Table {
* @return Returns a `String` from the data stored inside the FlatBuffer at `offset`.
*/
protected String __string(int offset) {
CharsetDecoder decoder = UTF8_DECODER.get();
decoder.reset();
offset += bb.getInt(offset);
ByteBuffer src = bb.duplicate().order(ByteOrder.LITTLE_ENDIAN);
int length = src.getInt(offset);
src.position(offset + SIZEOF_INT);
src.limit(offset + SIZEOF_INT + length);
int required = (int)((float)length * decoder.maxCharsPerByte());
CharBuffer dst = CHAR_BUFFER.get();
if (dst == null || dst.capacity() < required) {
dst = CharBuffer.allocate(required);
CHAR_BUFFER.set(dst);
}
dst.clear();
try {
CoderResult cr = decoder.decode(src, dst, true);
if (!cr.isUnderflow()) {
cr.throwException();
}
} catch (CharacterCodingException x) {
throw new RuntimeException(x);
}
return dst.flip().toString();
return utf8.decodeUtf8(bb, offset + SIZEOF_INT, length);
}
/**
......
java/com/google/flatbuffers/Utf8.java 0 → 100644
View file @ cb99116a
/*
* Copyright 2014 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.flatbuffers;
import java.nio.ByteBuffer;
import static java.lang.Character.MIN_HIGH_SURROGATE;
import static java.lang.Character.MIN_LOW_SURROGATE;
import static java.lang.Character.MIN_SUPPLEMENTARY_CODE_POINT;
public abstract class Utf8 {
/**
* Returns the number of bytes in the UTF-8-encoded form of {@code sequence}. For a string,
* this method is equivalent to {@code string.getBytes(UTF_8).length}, but is more efficient in
* both time and space.
*
* @throws IllegalArgumentException if {@code sequence} contains ill-formed UTF-16 (unpaired
* surrogates)
*/
public abstract int encodedLength(CharSequence sequence);
/**
* Encodes the given characters to the target {@link ByteBuffer} using UTF-8 encoding.
*
* <p>Selects an optimal algorithm based on the type of {@link ByteBuffer} (i.e. heap or direct)
* and the capabilities of the platform.
*
* @param in the source string to be encoded
* @param out the target buffer to receive the encoded string.
*/
public abstract void encodeUtf8(CharSequence in, ByteBuffer out);
/**
* Decodes the given UTF-8 portion of the {@link ByteBuffer} into a {@link String}.
*
* @throws IllegalArgumentException if the input is not valid UTF-8.
*/
public abstract String decodeUtf8(ByteBuffer buffer, int offset, int length);
private static Utf8 DEFAULT;
/**
* Get the default UTF-8 processor.
* @return the default processor
*/
public static Utf8 getDefault() {
if (DEFAULT == null) {
DEFAULT = new Utf8Safe();
}
return DEFAULT;
}
/**
* Set the default instance of the UTF-8 processor.
* @param instance the new instance to use
*/
public static void setDefault(Utf8 instance) {
DEFAULT = instance;
}
/**
* Utility methods for decoding bytes into {@link String}. Callers are responsible for extracting
* bytes (possibly using Unsafe methods), and checking remaining bytes. All other UTF-8 validity
* checks and codepoint conversion happen in this class.
*/
static class DecodeUtil {
/**
* Returns whether this is a single-byte codepoint (i.e., ASCII) with the form '0XXXXXXX'.
*/
static boolean isOneByte(byte b) {
return b >= 0;
}
/**
* Returns whether this is a two-byte codepoint with the form '10XXXXXX'.
*/
static boolean isTwoBytes(byte b) {
return b < (byte) 0xE0;
}
/**
* Returns whether this is a three-byte codepoint with the form '110XXXXX'.
*/
static boolean isThreeBytes(byte b) {
return b < (byte) 0xF0;
}
static void handleOneByte(byte byte1, char[] resultArr, int resultPos) {
resultArr[resultPos] = (char) byte1;
}
static void handleTwoBytes(
byte byte1, byte byte2, char[] resultArr, int resultPos)
throws IllegalArgumentException {
// Simultaneously checks for illegal trailing-byte in leading position (<= '11000000') and
// overlong 2-byte, '11000001'.
if (byte1 < (byte) 0xC2
|| isNotTrailingByte(byte2)) {
throw new IllegalArgumentException("Invalid UTF-8");
}
resultArr[resultPos] = (char) (((byte1 & 0x1F) << 6) | trailingByteValue(byte2));
}
static void handleThreeBytes(
byte byte1, byte byte2, byte byte3, char[] resultArr, int resultPos)
throws IllegalArgumentException {
if (isNotTrailingByte(byte2)
// overlong? 5 most significant bits must not all be zero
|| (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0)
// check for illegal surrogate codepoints
|| (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0)
|| isNotTrailingByte(byte3)) {
throw new IllegalArgumentException("Invalid UTF-8");
}
resultArr[resultPos] = (char)
(((byte1 & 0x0F) << 12) | (trailingByteValue(byte2) << 6) | trailingByteValue(byte3));
}
static void handleFourBytes(
byte byte1, byte byte2, byte byte3, byte byte4, char[] resultArr, int resultPos)
throws IllegalArgumentException{
if (isNotTrailingByte(byte2)
// Check that 1 <= plane <= 16. Tricky optimized form of:
// valid 4-byte leading byte?
// if (byte1 > (byte) 0xF4 ||
// overlong? 4 most significant bits must not all be zero
// byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 ||
// codepoint larger than the highest code point (U+10FFFF)?
// byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F)
|| (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0
|| isNotTrailingByte(byte3)
|| isNotTrailingByte(byte4)) {
throw new IllegalArgumentException("Invalid UTF-8");
}
int codepoint = ((byte1 & 0x07) << 18)
| (trailingByteValue(byte2) << 12)
| (trailingByteValue(byte3) << 6)
| trailingByteValue(byte4);
resultArr[resultPos] = DecodeUtil.highSurrogate(codepoint);
resultArr[resultPos + 1] = DecodeUtil.lowSurrogate(codepoint);
}
/**
* Returns whether the byte is not a valid continuation of the form '10XXXXXX'.
*/
private static boolean isNotTrailingByte(byte b) {
return b > (byte) 0xBF;
}
/**
* Returns the actual value of the trailing byte (removes the prefix '10') for composition.
*/
private static int trailingByteValue(byte b) {
return b & 0x3F;
}
private static char highSurrogate(int codePoint) {
return (char) ((MIN_HIGH_SURROGATE - (MIN_SUPPLEMENTARY_CODE_POINT >>> 10))
+ (codePoint >>> 10));
}
private static char lowSurrogate(int codePoint) {
return (char) (MIN_LOW_SURROGATE + (codePoint & 0x3ff));
}
}
// These UTF-8 handling methods are copied from Guava's Utf8Unsafe class with a modification to throw
// a protocol buffer local exception. This exception is then caught in CodedOutputStream so it can
// fallback to more lenient behavior.
static class UnpairedSurrogateException extends IllegalArgumentException {
UnpairedSurrogateException(int index, int length) {
super("Unpaired surrogate at index " + index + " of " + length);
}
}
}
java/com/google/flatbuffers/Utf8Old.java 0 → 100644
View file @ cb99116a
/*
* Copyright 2014 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.flatbuffers;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CoderResult;
import java.nio.charset.StandardCharsets;
/**
* This class implements the Utf8 API using the Java Utf8 encoder. Use
* Utf8.setDefault(new Utf8Old()); to use it.
*/
public class Utf8Old extends Utf8 {
private static class Cache {
final CharsetEncoder encoder;
final CharsetDecoder decoder;
CharSequence lastInput = null;
ByteBuffer lastOutput = null;
Cache() {
encoder = StandardCharsets.UTF_8.newEncoder();
decoder = StandardCharsets.UTF_8.newDecoder();
}
}
private static final ThreadLocal<Cache> CACHE =
ThreadLocal.withInitial(() -> new Cache());
// Play some games so that the old encoder doesn't pay twice for computing
// the length of the encoded string.
@Override
public int encodedLength(CharSequence in) {
final Cache cache = CACHE.get();
int estimated = (int) (in.length() * cache.encoder.maxBytesPerChar());
if (cache.lastOutput == null || cache.lastOutput.capacity() < estimated) {
cache.lastOutput = ByteBuffer.allocate(Math.max(128, estimated));
}
cache.lastOutput.clear();
cache.lastInput = in;
CharBuffer wrap = (in instanceof CharBuffer) ?
(CharBuffer) in : CharBuffer.wrap(in);
CoderResult result = cache.encoder.encode(wrap, cache.lastOutput, true);
if (result.isError()) {
try {
result.throwException();
} catch (CharacterCodingException e) {
throw new IllegalArgumentException("bad character encoding", e);
}
}
return cache.lastOutput.remaining();
}
@Override
public void encodeUtf8(CharSequence in, ByteBuffer out) {
final Cache cache = CACHE.get();
if (cache.lastInput != in) {
// Update the lastOutput to match our input, although flatbuffer should
// never take this branch.
encodedLength(in);
}
out.put(cache.lastOutput);
}
@Override
public String decodeUtf8(ByteBuffer buffer, int offset, int length) {
CharsetDecoder decoder = CACHE.get().decoder;
decoder.reset();
buffer = buffer.duplicate();
buffer.position(offset);
buffer.limit(offset + length);
try {
CharBuffer result = decoder.decode(buffer);
result.flip();
return result.toString();
} catch (CharacterCodingException e) {
throw new IllegalArgumentException("Bad encoding", e);
}
}
}
java/com/google/flatbuffers/Utf8Safe.java 0 → 100644
View file @ cb99116a
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.flatbuffers;
import java.nio.ByteBuffer;
import static java.lang.Character.MAX_SURROGATE;
import static java.lang.Character.MIN_SUPPLEMENTARY_CODE_POINT;
import static java.lang.Character.MIN_SURROGATE;
import static java.lang.Character.isSurrogatePair;
import static java.lang.Character.toCodePoint;
/**
* A set of low-level, high-performance static utility methods related
* to the UTF-8 character encoding. This class has no dependencies
* outside of the core JDK libraries.
*
* <p>There are several variants of UTF-8. The one implemented by
* this class is the restricted definition of UTF-8 introduced in
* Unicode 3.1, which mandates the rejection of "overlong" byte
* sequences as well as rejection of 3-byte surrogate codepoint byte
* sequences. Note that the UTF-8 decoder included in Oracle's JDK
* has been modified to also reject "overlong" byte sequences, but (as
* of 2011) still accepts 3-byte surrogate codepoint byte sequences.
*
* <p>The byte sequences considered valid by this class are exactly
* those that can be roundtrip converted to Strings and back to bytes
* using the UTF-8 charset, without loss: <pre> {@code
* Arrays.equals(bytes, new String(bytes, Internal.UTF_8).getBytes(Internal.UTF_8))
* }</pre>
*
* <p>See the Unicode Standard,</br>
* Table 3-6. <em>UTF-8 Bit Distribution</em>,</br>
* Table 3-7. <em>Well Formed UTF-8 Byte Sequences</em>.
*/
final public class Utf8Safe extends Utf8 {
/**
* Returns the number of bytes in the UTF-8-encoded form of {@code sequence}. For a string,
* this method is equivalent to {@code string.getBytes(UTF_8).length}, but is more efficient in
* both time and space.
*
* @throws IllegalArgumentException if {@code sequence} contains ill-formed UTF-16 (unpaired
* surrogates)
*/
private static int computeEncodedLength(CharSequence sequence) {
// Warning to maintainers: this implementation is highly optimized.
int utf16Length = sequence.length();
int utf8Length = utf16Length;
int i = 0;
// This loop optimizes for pure ASCII.
while (i < utf16Length && sequence.charAt(i) < 0x80) {
i++;
}
// This loop optimizes for chars less than 0x800.
for (; i < utf16Length; i++) {
char c = sequence.charAt(i);
if (c < 0x800) {
utf8Length += ((0x7f - c) >>> 31); // branch free!
} else {
utf8Length += encodedLengthGeneral(sequence, i);
break;
}
}
if (utf8Length < utf16Length) {
// Necessary and sufficient condition for overflow because of maximum 3x expansion
throw new IllegalArgumentException("UTF-8 length does not fit in int: "
+ (utf8Length + (1L << 32)));
}
return utf8Length;
}
private static int encodedLengthGeneral(CharSequence sequence, int start) {
int utf16Length = sequence.length();
int utf8Length = 0;
for (int i = start; i < utf16Length; i++) {
char c = sequence.charAt(i);
if (c < 0x800) {
utf8Length += (0x7f - c) >>> 31; // branch free!
} else {
utf8Length += 2;
// jdk7+: if (Character.isSurrogate(c)) {
if (Character.MIN_SURROGATE <= c && c <= Character.MAX_SURROGATE) {
// Check that we have a well-formed surrogate pair.
int cp = Character.codePointAt(sequence, i);
if (cp < MIN_SUPPLEMENTARY_CODE_POINT) {
throw new Utf8Safe.UnpairedSurrogateException(i, utf16Length);
}
i++;
}
}
}
return utf8Length;
}
private static String decodeUtf8Array(byte[] bytes, int index, int size) {
// Bitwise OR combines the sign bits so any negative value fails the check.
if ((index | size | bytes.length - index - size) < 0) {
throw new ArrayIndexOutOfBoundsException(
String.format("buffer length=%d, index=%d, size=%d", bytes.length, index, size));
}
int offset = index;
final int limit = offset + size;
// The longest possible resulting String is the same as the number of input bytes, when it is
// all ASCII. For other cases, this over-allocates and we will truncate in the end.
char[] resultArr = new char[size];
int resultPos = 0;
// Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this).
// This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII).
while (offset < limit) {
byte b = bytes[offset];
if (!DecodeUtil.isOneByte(b)) {
break;
}
offset++;
DecodeUtil.handleOneByte(b, resultArr, resultPos++);
}
while (offset < limit) {
byte byte1 = bytes[offset++];
if (DecodeUtil.isOneByte(byte1)) {
DecodeUtil.handleOneByte(byte1, resultArr, resultPos++);
// It's common for there to be multiple ASCII characters in a run mixed in, so add an
// extra optimized loop to take care of these runs.
while (offset < limit) {
byte b = bytes[offset];
if (!DecodeUtil.isOneByte(b)) {
break;
}
offset++;
DecodeUtil.handleOneByte(b, resultArr, resultPos++);
}
} else if (DecodeUtil.isTwoBytes(byte1)) {
if (offset >= limit) {
throw new IllegalArgumentException("Invalid UTF-8");
}
DecodeUtil.handleTwoBytes(byte1, /* byte2 */ bytes[offset++], resultArr, resultPos++);
} else if (DecodeUtil.isThreeBytes(byte1)) {
if (offset >= limit - 1) {
throw new IllegalArgumentException("Invalid UTF-8");
}
DecodeUtil.handleThreeBytes(
byte1,
/* byte2 */ bytes[offset++],
/* byte3 */ bytes[offset++],
resultArr,
resultPos++);
} else {
if (offset >= limit - 2) {
throw new IllegalArgumentException("Invalid UTF-8");
}
DecodeUtil.handleFourBytes(
byte1,
/* byte2 */ bytes[offset++],
/* byte3 */ bytes[offset++],
/* byte4 */ bytes[offset++],
resultArr,
resultPos++);
// 4-byte case requires two chars.
resultPos++;
}
}
return new String(resultArr, 0, resultPos);
}
private static String decodeUtf8Buffer(ByteBuffer buffer, int offset,
int length) {
// Bitwise OR combines the sign bits so any negative value fails the check.
if ((offset | length | buffer.limit() - offset - length) < 0) {
throw new ArrayIndexOutOfBoundsException(
String.format("buffer limit=%d, index=%d, limit=%d", buffer.limit(),
offset, length));
}
final int limit = offset + length;
// The longest possible resulting String is the same as the number of input bytes, when it is
// all ASCII. For other cases, this over-allocates and we will truncate in the end.
char[] resultArr = new char[length];
int resultPos = 0;
// Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this).
// This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII).
while (offset < limit) {
byte b = buffer.get(offset);
if (!DecodeUtil.isOneByte(b)) {
break;
}
offset++;
DecodeUtil.handleOneByte(b, resultArr, resultPos++);
}
while (offset < limit) {
byte byte1 = buffer.get(offset++);
if (DecodeUtil.isOneByte(byte1)) {
DecodeUtil.handleOneByte(byte1, resultArr, resultPos++);
// It's common for there to be multiple ASCII characters in a run mixed in, so add an
// extra optimized loop to take care of these runs.
while (offset < limit) {
byte b = buffer.get(offset);
if (!DecodeUtil.isOneByte(b)) {
break;
}
offset++;
DecodeUtil.handleOneByte(b, resultArr, resultPos++);
}
} else if (DecodeUtil.isTwoBytes(byte1)) {
if (offset >= limit) {
throw new IllegalArgumentException("Invalid UTF-8");
}
DecodeUtil.handleTwoBytes(
byte1, /* byte2 */ buffer.get(offset++), resultArr, resultPos++);
} else if (DecodeUtil.isThreeBytes(byte1)) {
if (offset >= limit - 1) {
throw new IllegalArgumentException("Invalid UTF-8");
}
DecodeUtil.handleThreeBytes(
byte1,
/* byte2 */ buffer.get(offset++),
/* byte3 */ buffer.get(offset++),
resultArr,
resultPos++);
} else {
if (offset >= limit - 2) {
throw new IllegalArgumentException("Invalid UTF-8");
}
DecodeUtil.handleFourBytes(
byte1,
/* byte2 */ buffer.get(offset++),
/* byte3 */ buffer.get(offset++),
/* byte4 */ buffer.get(offset++),
resultArr,
resultPos++);
// 4-byte case requires two chars.
resultPos++;
}
}
return new String(resultArr, 0, resultPos);
}
@Override
public int encodedLength(CharSequence in) {
return computeEncodedLength(in);
}
/**
* Decodes the given UTF-8 portion of the {@link ByteBuffer} into a {@link String}.
*
* @throws IllegalArgumentException if the input is not valid UTF-8.
*/
@Override
public String decodeUtf8(ByteBuffer buffer, int offset, int length)
throws IllegalArgumentException {
if (buffer.hasArray()) {
return decodeUtf8Array(buffer.array(), buffer.arrayOffset() + offset, length);
} else {
return decodeUtf8Buffer(buffer, offset, length);
}
}
private static void encodeUtf8Buffer(CharSequence in, ByteBuffer out) {
final int inLength = in.length();
int outIx = out.position();
int inIx = 0;
// Since ByteBuffer.putXXX() already checks boundaries for us, no need to explicitly check
// access. Assume the buffer is big enough and let it handle the out of bounds exception
// if it occurs.
try {
// Designed to take advantage of
// https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination
for (char c; inIx < inLength && (c = in.charAt(inIx)) < 0x80; ++inIx) {
out.put(outIx + inIx, (byte) c);
}
if (inIx == inLength) {
// Successfully encoded the entire string.
out.position(outIx + inIx);
return;
}
outIx += inIx;
for (char c; inIx < inLength; ++inIx, ++outIx) {
c = in.charAt(inIx);
if (c < 0x80) {
// One byte (0xxx xxxx)
out.put(outIx, (byte) c);
} else if (c < 0x800) {
// Two bytes (110x xxxx 10xx xxxx)
// Benchmarks show put performs better than putShort here (for HotSpot).
out.put(outIx++, (byte) (0xC0 | (c >>> 6)));
out.put(outIx, (byte) (0x80 | (0x3F & c)));
} else if (c < MIN_SURROGATE || MAX_SURROGATE < c) {
// Three bytes (1110 xxxx 10xx xxxx 10xx xxxx)
// Maximum single-char code point is 0xFFFF, 16 bits.
// Benchmarks show put performs better than putShort here (for HotSpot).
out.put(outIx++, (byte) (0xE0 | (c >>> 12)));
out.put(outIx++, (byte) (0x80 | (0x3F & (c >>> 6))));
out.put(outIx, (byte) (0x80 | (0x3F & c)));
} else {
// Four bytes (1111 xxxx 10xx xxxx 10xx xxxx 10xx xxxx)
// Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8
// bytes
final char low;
if (inIx + 1 == inLength || !isSurrogatePair(c, (low = in.charAt(++inIx)))) {
throw new UnpairedSurrogateException(inIx, inLength);
}
// TODO(nathanmittler): Consider using putInt() to improve performance.
int codePoint = toCodePoint(c, low);
out.put(outIx++, (byte) ((0xF << 4) | (codePoint >>> 18)));
out.put(outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 12))));
out.put(outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 6))));
out.put(outIx, (byte) (0x80 | (0x3F & codePoint)));
}
}
// Successfully encoded the entire string.
out.position(outIx);
} catch (IndexOutOfBoundsException e) {
// TODO(nathanmittler): Consider making the API throw IndexOutOfBoundsException instead.
// If we failed in the outer ASCII loop, outIx will not have been updated. In this case,
// use inIx to determine the bad write index.
int badWriteIndex = out.position() + Math.max(inIx, outIx - out.position() + 1);
throw new ArrayIndexOutOfBoundsException(
"Failed writing " + in.charAt(inIx) + " at index " + badWriteIndex);
}
}
private static int encodeUtf8Array(CharSequence in, byte[] out,
int offset, int length) {
int utf16Length = in.length();
int j = offset;
int i = 0;
int limit = offset + length;
// Designed to take advantage of
// https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination
for (char c; i < utf16Length && i + j < limit && (c = in.charAt(i)) < 0x80; i++) {
out[j + i] = (byte) c;
}
if (i == utf16Length) {
return j + utf16Length;
}
j += i;
for (char c; i < utf16Length; i++) {
c = in.charAt(i);
if (c < 0x80 && j < limit) {
out[j++] = (byte) c;
} else if (c < 0x800 && j <= limit - 2) { // 11 bits, two UTF-8 bytes
out[j++] = (byte) ((0xF << 6) | (c >>> 6));
out[j++] = (byte) (0x80 | (0x3F & c));
} else if ((c < Character.MIN_SURROGATE || Character.MAX_SURROGATE < c) && j <= limit - 3) {
// Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes
out[j++] = (byte) ((0xF << 5) | (c >>> 12));
out[j++] = (byte) (0x80 | (0x3F & (c >>> 6)));
out[j++] = (byte) (0x80 | (0x3F & c));
} else if (j <= limit - 4) {
// Minimum code point represented by a surrogate pair is 0x10000, 17 bits,
// four UTF-8 bytes
final char low;
if (i + 1 == in.length()
|| !Character.isSurrogatePair(c, (low = in.charAt(++i)))) {
throw new UnpairedSurrogateException((i - 1), utf16Length);
}
int codePoint = Character.toCodePoint(c, low);
out[j++] = (byte) ((0xF << 4) | (codePoint >>> 18));
out[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 12)));
out[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 6)));
out[j++] = (byte) (0x80 | (0x3F & codePoint));
} else {
// If we are surrogates and we're not a surrogate pair, always throw an
// UnpairedSurrogateException instead of an ArrayOutOfBoundsException.
if ((Character.MIN_SURROGATE <= c && c <= Character.MAX_SURROGATE)
&& (i + 1 == in.length()
|| !Character.isSurrogatePair(c, in.charAt(i + 1)))) {
throw new UnpairedSurrogateException(i, utf16Length);
}
throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + j);
}
}
return j;
}
/**
* Encodes the given characters to the target {@link ByteBuffer} using UTF-8 encoding.
*
* <p>Selects an optimal algorithm based on the type of {@link ByteBuffer} (i.e. heap or direct)
* and the capabilities of the platform.
*
* @param in the source string to be encoded
* @param out the target buffer to receive the encoded string.
*/
@Override
public void encodeUtf8(CharSequence in, ByteBuffer out) {
if (out.hasArray()) {
int start = out.arrayOffset();
int end = encodeUtf8Array(in, out.array(), start + out.position(),
out.remaining());
out.position(end - start);
} else {
encodeUtf8Buffer(in, out);
}
}
// These UTF-8 handling methods are copied from Guava's Utf8Unsafe class with
// a modification to throw a local exception. This exception can be caught
// to fallback to more lenient behavior.
static class UnpairedSurrogateException extends IllegalArgumentException {
UnpairedSurrogateException(int index, int length) {
super("Unpaired surrogate at index " + index + " of " + length);
}
}
}
pom.xml
View file @ cb99116a
......@@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.google.flatbuffers</groupId>
<artifactId>flatbuffers-java</artifactId>
<version>1.10.0</version>
<version>1.10.1-SNAPSHOT</version>
<packaging>bundle</packaging>
<name>FlatBuffers Java API</name>
<description>
......@@ -78,6 +78,10 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9.1</version>
<configuration>
<additionalparam>-Xdoclint:none</additionalparam>
<additionalOptions>-Xdoclint:none</additionalOptions>
</configuration>
<executions>
<execution>
<id>attach-javadocs</id>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment