Chiebot-Mirror
/
protobuf
mirror of https://github.com/protocolbuffers/protobuf.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Protobuf Lite ArrayLists: Defer allocating backing array until we have some idea how much to allocate.

Browse Source 
This avoids allocating a backing array of size 10 when we are adding >10 elements.
- If we are adding objects one at a time, inflate a Object[10] and continue.
- If we are adding objects from a Collection, assume this is the only collection we are adding, and inflate a `Object[collection.size()]`
- If the existing array is non-empty, resize the array by 1.5x exponential growth as usual.

There's another small change where if we're addAll(<10 elements), we allocate an exact-sized backing array (e.g. size=3), rather than rounding up to size 10. See android/LiteAllocationTest. I think this is good: this will save memory in the common case of just calling .addAll() once, and if we call addAll twice, we grow still exponentially. But we could decide to avoid this or split it out into its own change.

This change involves moving some logic out of GeneratedMessageLite. I think the default size of the backing array is better handled inside ProtobufArrayList than inside GeneratedMessageLite's wrapper function.

PiperOrigin-RevId: 673612155
pull/18094/head
Mark Hansen 2 months ago committed by Copybara-Service
parent 4e8469cb45
commit 05a8a40123
8 changed files with 89 additions and 47 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 18
      java/core/src/main/java/com/google/protobuf/BooleanArrayList.java
  2. 18
      java/core/src/main/java/com/google/protobuf/DoubleArrayList.java
  3. 18
      java/core/src/main/java/com/google/protobuf/FloatArrayList.java
  4. 18
      java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java
  5. 18
      java/core/src/main/java/com/google/protobuf/IntArrayList.java
  6. 18
      java/core/src/main/java/com/google/protobuf/LongArrayList.java
  7. 4
      java/core/src/main/java/com/google/protobuf/MessageSchema.java
  8. 24
      java/core/src/main/java/com/google/protobuf/ProtobufArrayList.java

18
java/core/src/main/java/com/google/protobuf/BooleanArrayList.java
Unescape View File

@ -8,6 +8,7 @@
package com.google.protobuf;
import static com.google.protobuf.Internal.checkNotNull;
import static java.lang.Math.max;
import com.google.protobuf.Internal.BooleanList;
import java.util.Arrays;
@ -22,7 +23,9 @@ import java.util.RandomAccess;
final class BooleanArrayList extends AbstractProtobufList<Boolean>
implements BooleanList, RandomAccess, PrimitiveNonBoxingCollection {
private static final BooleanArrayList EMPTY_LIST = new BooleanArrayList(new boolean[0], 0, false);
private static final boolean[] EMPTY_ARRAY = new boolean[0];
private static final BooleanArrayList EMPTY_LIST = new BooleanArrayList(EMPTY_ARRAY, 0, false);
public static BooleanArrayList emptyList() {
return EMPTY_LIST;
@ -39,7 +42,7 @@ final class BooleanArrayList extends AbstractProtobufList<Boolean>
/** Constructs a new mutable {@code BooleanArrayList} with default capacity. */
BooleanArrayList() {
this(new boolean[DEFAULT_CAPACITY], 0, true);
this(EMPTY_ARRAY, 0, true);
}
/**
@ -101,7 +104,8 @@ final class BooleanArrayList extends AbstractProtobufList<Boolean>
if (capacity < size) {
throw new IllegalArgumentException();
}
return new BooleanArrayList(Arrays.copyOf(array, capacity), size, true);
boolean[] newArray = capacity == 0 ? EMPTY_ARRAY : Arrays.copyOf(array, capacity);
return new BooleanArrayList(newArray, size, true);
}
@Override
@ -258,6 +262,10 @@ final class BooleanArrayList extends AbstractProtobufList<Boolean>
if (minCapacity <= array.length) {
return;
}
if (array.length == 0) {
array = new boolean[max(minCapacity, DEFAULT_CAPACITY)];
return;
}
// To avoid quadratic copying when calling .addAllFoo(List) in a loop, we must not size to
// exactly the requested capacity, but must exponentially grow instead. This is similar
// behaviour to ArrayList.
@ -269,8 +277,8 @@ final class BooleanArrayList extends AbstractProtobufList<Boolean>
}
private static int growSize(int previousSize) {
// Resize to 1.5x the size
return ((previousSize * 3) / 2) + 1;
// Resize to 1.5x the size, rounding up to DEFAULT_CAPACITY.
return max(((previousSize * 3) / 2) + 1, DEFAULT_CAPACITY);
}
/**

18
java/core/src/main/java/com/google/protobuf/DoubleArrayList.java
Unescape View File

@ -8,6 +8,7 @@
package com.google.protobuf;
import static com.google.protobuf.Internal.checkNotNull;
import static java.lang.Math.max;
import com.google.protobuf.Internal.DoubleList;
import java.util.Arrays;
@ -22,7 +23,9 @@ import java.util.RandomAccess;
final class DoubleArrayList extends AbstractProtobufList<Double>
implements DoubleList, RandomAccess, PrimitiveNonBoxingCollection {
private static final DoubleArrayList EMPTY_LIST = new DoubleArrayList(new double[0], 0, false);
private static final double[] EMPTY_ARRAY = new double[0];
private static final DoubleArrayList EMPTY_LIST = new DoubleArrayList(EMPTY_ARRAY, 0, false);
public static DoubleArrayList emptyList() {
return EMPTY_LIST;
@ -39,7 +42,7 @@ final class DoubleArrayList extends AbstractProtobufList<Double>
/** Constructs a new mutable {@code DoubleArrayList} with default capacity. */
DoubleArrayList() {
this(new double[DEFAULT_CAPACITY], 0, true);
this(EMPTY_ARRAY, 0, true);
}
/**
@ -101,7 +104,8 @@ final class DoubleArrayList extends AbstractProtobufList<Double>
if (capacity < size) {
throw new IllegalArgumentException();
}
return new DoubleArrayList(Arrays.copyOf(array, capacity), size, true);
double[] newArray = capacity == 0 ? EMPTY_ARRAY : Arrays.copyOf(array, capacity);
return new DoubleArrayList(newArray, size, true);
}
@Override
@ -258,6 +262,10 @@ final class DoubleArrayList extends AbstractProtobufList<Double>
if (minCapacity <= array.length) {
return;
}
if (array.length == 0) {
array = new double[max(minCapacity, DEFAULT_CAPACITY)];
return;
}
// To avoid quadratic copying when calling .addAllFoo(List) in a loop, we must not size to
// exactly the requested capacity, but must exponentially grow instead. This is similar
// behaviour to ArrayList.
@ -269,8 +277,8 @@ final class DoubleArrayList extends AbstractProtobufList<Double>
}
private static int growSize(int previousSize) {
// Resize to 1.5x the size
return ((previousSize * 3) / 2) + 1;
// Resize to 1.5x the size, rounding up to DEFAULT_CAPACITY.
return max(((previousSize * 3) / 2) + 1, DEFAULT_CAPACITY);
}
/**

18
java/core/src/main/java/com/google/protobuf/FloatArrayList.java
Unescape View File

@ -8,6 +8,7 @@
package com.google.protobuf;
import static com.google.protobuf.Internal.checkNotNull;
import static java.lang.Math.max;
import com.google.protobuf.Internal.FloatList;
import java.util.Arrays;
@ -22,7 +23,9 @@ import java.util.RandomAccess;
final class FloatArrayList extends AbstractProtobufList<Float>
implements FloatList, RandomAccess, PrimitiveNonBoxingCollection {
private static final FloatArrayList EMPTY_LIST = new FloatArrayList(new float[0], 0, false);
private static final float[] EMPTY_ARRAY = new float[0];
private static final FloatArrayList EMPTY_LIST = new FloatArrayList(EMPTY_ARRAY, 0, false);
public static FloatArrayList emptyList() {
return EMPTY_LIST;
@ -39,7 +42,7 @@ final class FloatArrayList extends AbstractProtobufList<Float>
/** Constructs a new mutable {@code FloatArrayList} with default capacity. */
FloatArrayList() {
this(new float[DEFAULT_CAPACITY], 0, true);
this(EMPTY_ARRAY, 0, true);
}
/**
@ -100,7 +103,8 @@ final class FloatArrayList extends AbstractProtobufList<Float>
if (capacity < size) {
throw new IllegalArgumentException();
}
return new FloatArrayList(Arrays.copyOf(array, capacity), size, true);
float[] newArray = capacity == 0 ? EMPTY_ARRAY : Arrays.copyOf(array, capacity);
return new FloatArrayList(newArray, size, true);
}
@Override
@ -257,6 +261,10 @@ final class FloatArrayList extends AbstractProtobufList<Float>
if (minCapacity <= array.length) {
return;
}
if (array.length == 0) {
array = new float[max(minCapacity, DEFAULT_CAPACITY)];
return;
}
// To avoid quadratic copying when calling .addAllFoo(List) in a loop, we must not size to
// exactly the requested capacity, but must exponentially grow instead. This is similar
// behaviour to ArrayList.
@ -268,8 +276,8 @@ final class FloatArrayList extends AbstractProtobufList<Float>
}
private static int growSize(int previousSize) {
// Resize to 1.5x the size
return ((previousSize * 3) / 2) + 1;
// Resize to 1.5x the size, rounding up to DEFAULT_CAPACITY.
return max(((previousSize * 3) / 2) + 1, DEFAULT_CAPACITY);
}
/**

18
java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java
Unescape View File

@ -1496,8 +1496,7 @@ public abstract class GeneratedMessageLite<
protected static IntList mutableCopy(IntList list) {
int size = list.size();
return list.mutableCopyWithCapacity(
size == 0 ? AbstractProtobufList.DEFAULT_CAPACITY : size * 2);
return list.mutableCopyWithCapacity(size * 2);
}
protected static LongList emptyLongList() {
@ -1506,8 +1505,7 @@ public abstract class GeneratedMessageLite<
protected static LongList mutableCopy(LongList list) {
int size = list.size();
return list.mutableCopyWithCapacity(
size == 0 ? AbstractProtobufList.DEFAULT_CAPACITY : size * 2);
return list.mutableCopyWithCapacity(size * 2);
}
protected static FloatList emptyFloatList() {
@ -1516,8 +1514,7 @@ public abstract class GeneratedMessageLite<
protected static FloatList mutableCopy(FloatList list) {
int size = list.size();
return list.mutableCopyWithCapacity(
size == 0 ? AbstractProtobufList.DEFAULT_CAPACITY : size * 2);
return list.mutableCopyWithCapacity(size * 2);
}
protected static DoubleList emptyDoubleList() {
@ -1526,8 +1523,7 @@ public abstract class GeneratedMessageLite<
protected static DoubleList mutableCopy(DoubleList list) {
int size = list.size();
return list.mutableCopyWithCapacity(
size == 0 ? AbstractProtobufList.DEFAULT_CAPACITY : size * 2);
return list.mutableCopyWithCapacity(size * 2);
}
protected static BooleanList emptyBooleanList() {
@ -1536,8 +1532,7 @@ public abstract class GeneratedMessageLite<
protected static BooleanList mutableCopy(BooleanList list) {
int size = list.size();
return list.mutableCopyWithCapacity(
size == 0 ? AbstractProtobufList.DEFAULT_CAPACITY : size * 2);
return list.mutableCopyWithCapacity(size * 2);
}
protected static <E> ProtobufList<E> emptyProtobufList() {
@ -1546,8 +1541,7 @@ public abstract class GeneratedMessageLite<
protected static <E> ProtobufList<E> mutableCopy(ProtobufList<E> list) {
int size = list.size();
return list.mutableCopyWithCapacity(
size == 0 ? AbstractProtobufList.DEFAULT_CAPACITY : size * 2);
return list.mutableCopyWithCapacity(size * 2);
}
/**

18
java/core/src/main/java/com/google/protobuf/IntArrayList.java
Unescape View File

@ -8,6 +8,7 @@
package com.google.protobuf;
import static com.google.protobuf.Internal.checkNotNull;
import static java.lang.Math.max;
import com.google.protobuf.Internal.IntList;
import java.util.Arrays;
@ -22,7 +23,9 @@ import java.util.RandomAccess;
final class IntArrayList extends AbstractProtobufList<Integer>
implements IntList, RandomAccess, PrimitiveNonBoxingCollection {
private static final IntArrayList EMPTY_LIST = new IntArrayList(new int[0], 0, false);
private static final int[] EMPTY_ARRAY = new int[0];
private static final IntArrayList EMPTY_LIST = new IntArrayList(EMPTY_ARRAY, 0, false);
public static IntArrayList emptyList() {
return EMPTY_LIST;
@ -39,7 +42,7 @@ final class IntArrayList extends AbstractProtobufList<Integer>
/** Constructs a new mutable {@code IntArrayList} with default capacity. */
IntArrayList() {
this(new int[DEFAULT_CAPACITY], 0, true);
this(EMPTY_ARRAY, 0, true);
}
/**
@ -100,7 +103,8 @@ final class IntArrayList extends AbstractProtobufList<Integer>
if (capacity < size) {
throw new IllegalArgumentException();
}
return new IntArrayList(Arrays.copyOf(array, capacity), size, true);
int[] newArray = capacity == 0 ? EMPTY_ARRAY : Arrays.copyOf(array, capacity);
return new IntArrayList(newArray, size, true);
}
@Override
@ -257,6 +261,10 @@ final class IntArrayList extends AbstractProtobufList<Integer>
if (minCapacity <= array.length) {
return;
}
if (array.length == 0) {
array = new int[max(minCapacity, DEFAULT_CAPACITY)];
return;
}
// To avoid quadratic copying when calling .addAllFoo(List) in a loop, we must not size to
// exactly the requested capacity, but must exponentially grow instead. This is similar
// behaviour to ArrayList.
@ -268,8 +276,8 @@ final class IntArrayList extends AbstractProtobufList<Integer>
}
private static int growSize(int previousSize) {
// Resize to 1.5x the size
return ((previousSize * 3) / 2) + 1;
// Resize to 1.5x the size, rounding up to DEFAULT_CAPACITY.
return max(((previousSize * 3) / 2) + 1, DEFAULT_CAPACITY);
}
/**

18
java/core/src/main/java/com/google/protobuf/LongArrayList.java
Unescape View File

@ -8,6 +8,7 @@
package com.google.protobuf;
import static com.google.protobuf.Internal.checkNotNull;
import static java.lang.Math.max;
import com.google.protobuf.Internal.LongList;
import java.util.Arrays;
@ -22,7 +23,9 @@ import java.util.RandomAccess;
final class LongArrayList extends AbstractProtobufList<Long>
implements LongList, RandomAccess, PrimitiveNonBoxingCollection {
private static final LongArrayList EMPTY_LIST = new LongArrayList(new long[0], 0, false);
private static final long[] EMPTY_ARRAY = new long[0];
private static final LongArrayList EMPTY_LIST = new LongArrayList(EMPTY_ARRAY, 0, false);
public static LongArrayList emptyList() {
return EMPTY_LIST;
@ -39,7 +42,7 @@ final class LongArrayList extends AbstractProtobufList<Long>
/** Constructs a new mutable {@code LongArrayList} with default capacity. */
LongArrayList() {
this(new long[DEFAULT_CAPACITY], 0, true);
this(EMPTY_ARRAY, 0, true);
}
/**
@ -100,7 +103,8 @@ final class LongArrayList extends AbstractProtobufList<Long>
if (capacity < size) {
throw new IllegalArgumentException();
}
return new LongArrayList(Arrays.copyOf(array, capacity), size, true);
long[] newArray = capacity == 0 ? EMPTY_ARRAY : Arrays.copyOf(array, capacity);
return new LongArrayList(newArray, size, true);
}
@Override
@ -257,6 +261,10 @@ final class LongArrayList extends AbstractProtobufList<Long>
if (minCapacity <= array.length) {
return;
}
if (array.length == 0) {
array = new long[max(minCapacity, DEFAULT_CAPACITY)];
return;
}
// To avoid quadratic copying when calling .addAllFoo(List) in a loop, we must not size to
// exactly the requested capacity, but must exponentially grow instead. This is similar
// behaviour to ArrayList.
@ -268,8 +276,8 @@ final class LongArrayList extends AbstractProtobufList<Long>
}
private static int growSize(int previousSize) {
// Resize to 1.5x the size
return ((previousSize * 3) / 2) + 1;
// Resize to 1.5x the size, rounding up to DEFAULT_CAPACITY.
return max(((previousSize * 3) / 2) + 1, DEFAULT_CAPACITY);
}
/**

4
java/core/src/main/java/com/google/protobuf/MessageSchema.java
Unescape View File

@ -3577,9 +3577,7 @@ final class MessageSchema<T> implements Schema<T> {
ProtobufList<?> list = (ProtobufList<?>) UNSAFE.getObject(message, fieldOffset);
if (!list.isModifiable()) {
final int size = list.size();
list =
list.mutableCopyWithCapacity(
size == 0 ? AbstractProtobufList.DEFAULT_CAPACITY : size * 2);
list = list.mutableCopyWithCapacity(size * 2);
UNSAFE.putObject(message, fieldOffset, list);
}
switch (fieldType) {

24
java/core/src/main/java/com/google/protobuf/ProtobufArrayList.java
Unescape View File

@ -7,6 +7,8 @@
package com.google.protobuf;
import static java.lang.Math.max;
import com.google.protobuf.Internal.ProtobufList;
import java.util.Arrays;
import java.util.RandomAccess;
@ -14,8 +16,10 @@ import java.util.RandomAccess;
/** Implements {@link ProtobufList} for non-primitive and {@link String} types. */
final class ProtobufArrayList<E> extends AbstractProtobufList<E> implements RandomAccess {
private static final Object[] EMPTY_ARRAY = new Object[0];
private static final ProtobufArrayList<Object> EMPTY_LIST =
new ProtobufArrayList<Object>(new Object[0], 0, false);
new ProtobufArrayList<>(EMPTY_ARRAY, 0, false);
@SuppressWarnings("unchecked") // Guaranteed safe by runtime.
public static <E> ProtobufArrayList<E> emptyList() {
@ -27,7 +31,7 @@ final class ProtobufArrayList<E> extends AbstractProtobufList<E> implements Rand
@SuppressWarnings("unchecked")
ProtobufArrayList() {
this((E[]) new Object[DEFAULT_CAPACITY], 0, true);
this((E[]) EMPTY_ARRAY, 0, true);
}
private ProtobufArrayList(E[] array, int size, boolean isMutable) {
@ -36,13 +40,14 @@ final class ProtobufArrayList<E> extends AbstractProtobufList<E> implements Rand
this.size = size;
}
@SuppressWarnings("unchecked") // Casting an empty Object[] to a generic array is safe.
@Override
public ProtobufArrayList<E> mutableCopyWithCapacity(int capacity) {
if (capacity < size) {
throw new IllegalArgumentException();
}
E[] newArray = Arrays.copyOf(array, capacity);
E[] newArray = capacity == 0 ? (E[]) EMPTY_ARRAY : Arrays.copyOf(array, capacity);
return new ProtobufArrayList<E>(newArray, size, true);
}
@ -52,7 +57,7 @@ final class ProtobufArrayList<E> extends AbstractProtobufList<E> implements Rand
ensureIsMutable();
if (size == array.length) {
int length = growSize(size);
int length = growSize(array.length);
E[] newArray = Arrays.copyOf(array, length);
array = newArray;
@ -65,8 +70,8 @@ final class ProtobufArrayList<E> extends AbstractProtobufList<E> implements Rand
}
private static int growSize(int previousSize) {
// Resize to 1.5x the size
return ((previousSize * 3) / 2) + 1;
// Resize to 1.5x the size, rounding up to DEFAULT_CAPACITY.
return max(((previousSize * 3) / 2) + 1, DEFAULT_CAPACITY);
}
@Override
@ -81,7 +86,7 @@ final class ProtobufArrayList<E> extends AbstractProtobufList<E> implements Rand
// Shift everything over to make room
System.arraycopy(array, index, array, index + 1, size - index);
} else {
int length = growSize(size);
int length = growSize(array.length);
E[] newArray = createArray(length);
// Copy the first part directly
@ -136,10 +141,15 @@ final class ProtobufArrayList<E> extends AbstractProtobufList<E> implements Rand
}
/** Ensures the backing array can fit at least minCapacity elements. */
@SuppressWarnings("unchecked") // Casting an Object[] with no values inside to E[] is safe.
void ensureCapacity(int minCapacity) {
if (minCapacity <= array.length) {
return;
}
if (array.length == 0) {
array = (E[]) new Object[max(minCapacity, DEFAULT_CAPACITY)];
return;
}
// To avoid quadratic copying when calling .addAllFoo(List) in a loop, we must not size to
// exactly the requested capacity, but must exponentially grow instead. This is similar
// behaviour to ArrayList.

Write Preview
Loading…
Cancel
Save