Merge pull request #2431 from saintstack/2228v2

Change CodedInputStream#DEFAULT_SIZE_LIMIT from 64MB to

java/core/src/main/java/com/google/protobuf/CodedInputStream.java

@@ -60,7 +60,8 @@ import java.util.List;
 public abstract class CodedInputStream {
   private static final int DEFAULT_BUFFER_SIZE = 4096;
   private static final int DEFAULT_RECURSION_LIMIT = 100;
-  private static final int DEFAULT_SIZE_LIMIT = 64 << 20; // 64MB
+  // Integer.MAX_VALUE == 0x7FFFFFF == INT_MAX from limits.h
+  private static final int DEFAULT_SIZE_LIMIT = Integer.MAX_VALUE;
 
   /** Visible for subclasses. See setRecursionLimit() */
   int recursionDepth;
@@ -2762,9 +2763,9 @@ public abstract class CodedInputStream {
         throw InvalidProtocolBufferException.negativeSize();
       }
 
-      // Verify that the message size so far has not exceeded sizeLimit.
+      // Integer-overflow-conscious check that the message size so far has not exceeded sizeLimit.
       int currentMessageSize = totalBytesRetired + pos + size;
-      if (currentMessageSize > sizeLimit) {
+      if (currentMessageSize - sizeLimit > 0) {
         throw InvalidProtocolBufferException.sizeLimitExceeded();
       }
 

java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java

@@ -30,8 +30,6 @@
 
 package com.google.protobuf;
 
-import static org.junit.Assert.assertArrayEquals;
-
 import protobuf_unittest.UnittestProto.BoolMessage;
 import protobuf_unittest.UnittestProto.Int32Message;
 import protobuf_unittest.UnittestProto.Int64Message;
@@ -445,6 +443,82 @@ public class CodedInputStreamTest extends TestCase {
     }
   }
 
+  /**
+   * Test we can do messages that are up to CodedInputStream#DEFAULT_SIZE_LIMIT
+   * in size (2G or Integer#MAX_SIZE).
+   * @throws IOException
+   */
+  public void testParseMessagesCloseTo2G() throws IOException {
+    byte[] serializedMessage = getBigSerializedMessage();
+    // How many of these big messages do we need to take us near our 2G limit?
+    int count = Integer.MAX_VALUE / serializedMessage.length;
+    // Now make an inputstream that will fake a near 2G message of messages
+    // returning our big serialized message 'count' times.
+    InputStream is = new RepeatingInputStream(serializedMessage, count);
+    // Parse should succeed!
+    TestAllTypes.parseFrom(is);
+  }
+
+  /**
+   * Test there is an exception if a message exceeds
+   * CodedInputStream#DEFAULT_SIZE_LIMIT in size (2G or Integer#MAX_SIZE).
+   * @throws IOException
+   */
+  public void testParseMessagesOver2G() throws IOException {
+    byte[] serializedMessage = getBigSerializedMessage();
+    // How many of these big messages do we need to take us near our 2G limit?
+    int count = Integer.MAX_VALUE / serializedMessage.length;
+    // Now add one to take us over the limit
+    count++;
+    // Now make an inputstream that will fake a near 2G message of messages
+    // returning our big serialized message 'count' times.
+    InputStream is = new RepeatingInputStream(serializedMessage, count);
+    try {
+      TestAllTypes.parseFrom(is);
+      fail("Should have thrown an exception!");
+    } catch (InvalidProtocolBufferException e) {
+      assertTrue(e.getMessage().contains("too large"));
+    }
+  }
+
+  /*
+   * @return A serialized big message.
+   */
+  private static byte[] getBigSerializedMessage() {
+    byte[] value = new byte[16 * 1024 * 1024]; 
+    ByteString bsValue = ByteString.wrap(value);
+    return TestAllTypes.newBuilder().setOptionalBytes(bsValue).build().toByteArray();
+  }
+
+  /*
+   * An input stream that repeats a byte arrays' content a number of times.
+   * Simulates really large input without consuming loads of memory. Used above
+   * to test the parsing behavior when the input size exceeds 2G or close to it.
+   */
+  private static class RepeatingInputStream extends InputStream {
+    private final byte[] serializedMessage;
+    private final int count;
+    private int index = 0;
+    private int offset = 0;
+
+    RepeatingInputStream(byte[] serializedMessage, int count) {
+      this.serializedMessage = serializedMessage;
+      this.count = count;
+    }
+
+    @Override
+    public int read() throws IOException {
+      if (this.offset == this.serializedMessage.length) {
+        this.index++;
+        this.offset = 0;
+      }
+      if (this.index == this.count) {
+        return -1;
+      }
+      return this.serializedMessage[offset++];
+    }
+  }
+
   private TestRecursiveMessage makeRecursiveMessage(int depth) {
     if (depth == 0) {
       return TestRecursiveMessage.newBuilder().setI(5).build();