// Protocol Buffers - Google's data interchange format // Copyright 2008 Google Inc. // http://code.google.com/p/protobuf/ // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. using System; using System.IO; using System.Text; using Google.ProtocolBuffers.Descriptors; namespace Google.ProtocolBuffers { /// /// Encodes and writes protocol message fields. /// /// /// This class contains two kinds of methods: methods that write specific /// protocol message constructs and field types (e.g. WriteTag and /// WriteInt32) and methods that write low-level values (e.g. /// WriteRawVarint32 and WriteRawBytes). If you are writing encoded protocol /// messages, you should use the former methods, but if you are writing some /// other format of your own design, use the latter. The names of the former /// methods are taken from the protocol buffer type names, not .NET types. /// (Hence WriteFloat instead of WriteSingle, and WriteBool instead of WriteBoolean.) /// public sealed class CodedOutputStream { /// /// The buffer size used by CreateInstance(Stream). /// public static readonly int DefaultBufferSize = 4096; private readonly byte[] buffer; private readonly int limit; private int position; private readonly Stream output; #region Construction private CodedOutputStream(byte[] buffer, int offset, int length) { this.output = null; this.buffer = buffer; this.position = offset; this.limit = offset + length; } private CodedOutputStream(Stream output, byte[] buffer) { this.output = output; this.buffer = buffer; this.position = 0; this.limit = buffer.Length; } /// /// Creates a new CodedOutputStream which write to the given stream. /// public static CodedOutputStream CreateInstance(Stream output) { return CreateInstance(output, DefaultBufferSize); } /// /// Creates a new CodedOutputStream which write to the given stream and uses /// the specified buffer size. /// public static CodedOutputStream CreateInstance(Stream output, int bufferSize) { return new CodedOutputStream(output, new byte[bufferSize]); } /// /// Creates a new CodedOutputStream that writes directly to the given /// byte array. If more bytes are written than fit in the array, /// OutOfSpaceException will be thrown. /// public static CodedOutputStream CreateInstance(byte[] flatArray) { return CreateInstance(flatArray, 0, flatArray.Length); } /// /// Creates a new CodedOutputStream that writes directly to the given /// byte array slice. If more bytes are written than fit in the array, /// OutOfSpaceException will be thrown. /// public static CodedOutputStream CreateInstance(byte[] flatArray, int offset, int length) { return new CodedOutputStream(flatArray, offset, length); } #endregion #region Writing of tags etc /// /// Writes a double field value, including tag, to the stream. /// public void WriteDouble(int fieldNumber, double value) { WriteTag(fieldNumber, WireFormat.WireType.Fixed64); WriteRawLittleEndian64(BitConverter.DoubleToInt64Bits(value)); } /// /// Writes a float field value, including tag, to the stream. /// public void WriteFloat(int fieldNumber, float value) { WriteTag(fieldNumber, WireFormat.WireType.Fixed32); // FIXME: How do we convert a single to 32 bits? (Without unsafe code) //WriteRawLittleEndian32(BitConverter.SingleT(value)); } /// /// Writes a uint64 field value, including tag, to the stream. /// public void WriteUInt64(int fieldNumber, ulong value) { WriteTag(fieldNumber, WireFormat.WireType.Varint); WriteRawVarint64(value); } /// /// Writes an int64 field value, including tag, to the stream. /// public void WriteInt64(int fieldNumber, long value) { WriteTag(fieldNumber, WireFormat.WireType.Varint); WriteRawVarint64((ulong)value); } /// /// Writes an int32 field value, including tag, to the stream. /// public void WriteInt32(int fieldNumber, int value) { WriteTag(fieldNumber, WireFormat.WireType.Varint); if (value >= 0) { WriteRawVarint32((uint)value); } else { // Must sign-extend. WriteRawVarint64((ulong)value); } } /// /// Writes a fixed64 field value, including tag, to the stream. /// public void WriteFixed64(int fieldNumber, long value) { WriteTag(fieldNumber, WireFormat.WireType.Fixed64); WriteRawLittleEndian64(value); } /// /// Writes a fixed32 field value, including tag, to the stream. /// public void WriteFixed32(int fieldNumber, int value) { WriteTag(fieldNumber, WireFormat.WireType.Fixed32); WriteRawLittleEndian32(value); } /// /// Writes a bool field value, including tag, to the stream. /// public void WriteBool(int fieldNumber, bool value) { WriteTag(fieldNumber, WireFormat.WireType.Varint); WriteRawByte(value ? (byte)1 : (byte)0); } /// /// Writes a string field value, including tag, to the stream. /// public void WriteString(int fieldNumber, string value) { WriteTag(fieldNumber, WireFormat.WireType.LengthDelimited); // TODO(jonskeet): Optimise this if possible // Unfortunately there does not appear to be any way to tell Java to encode // UTF-8 directly into our buffer, so we have to let it create its own byte // array and then copy. In .NET we can do the same thing very easily, // so we don't need to worry about only writing one buffer at a time. // We can optimise later. byte[] bytes = Encoding.UTF8.GetBytes(value); WriteRawVarint32((uint)bytes.Length); WriteRawBytes(bytes); } /// /// Writes a group field value, including tag, to the stream. /// public void WriteGroup(int fieldNumber, IMessage value) { WriteTag(fieldNumber, WireFormat.WireType.StartGroup); value.WriteTo(this); WriteTag(fieldNumber, WireFormat.WireType.EndGroup); } public void WriteUnknownGroup(int fieldNumber, UnknownFieldSet value) { WriteTag(fieldNumber, WireFormat.WireType.StartGroup); value.WriteTo(this); WriteTag(fieldNumber, WireFormat.WireType.EndGroup); } public void WriteMessage(int fieldNumber, IMessage value) { WriteTag(fieldNumber, WireFormat.WireType.LengthDelimited); WriteRawVarint32((uint)value.SerializedSize); value.WriteTo(this); } public void WriteBytes(int fieldNumber, ByteString value) { // TODO(jonskeet): Optimise this! (No need to copy the bytes twice.) byte[] bytes = value.ToByteArray(); WriteRawVarint32((uint)bytes.Length); WriteRawBytes(bytes); } public void WriteUInt32(int fieldNumber, uint value) { WriteTag(fieldNumber, WireFormat.WireType.Varint); WriteRawVarint32(value); } public void WriteEnum(int fieldNumber, int value) { WriteTag(fieldNumber, WireFormat.WireType.Varint); WriteRawVarint32((uint)value); } public void WriteSFixed32(int fieldNumber, int value) { WriteTag(fieldNumber, WireFormat.WireType.Fixed32); WriteRawVarint32((uint)value); } public void WriteSFixed64(int fieldNumber, long value) { WriteTag(fieldNumber, WireFormat.WireType.Fixed64); WriteRawVarint64((ulong)value); } public void WriteSInt32(int fieldNumber, int value) { WriteTag(fieldNumber, WireFormat.WireType.Varint); WriteRawVarint32(EncodeZigZag32(value)); } public void WriteSInt64(int fieldNumber, long value) { WriteTag(fieldNumber, WireFormat.WireType.Varint); WriteRawVarint64(EncodeZigZag64(value)); } public void WriteMessageSetExtension(int fieldNumber, IMessage value) { WriteTag(WireFormat.MessageSetField.Item, WireFormat.WireType.StartGroup); WriteUInt32(WireFormat.MessageSetField.TypeID, (uint)fieldNumber); WriteMessage(WireFormat.MessageSetField.Message, value); WriteTag(WireFormat.MessageSetField.Item, WireFormat.WireType.EndGroup); } public void WriteRawMessageSetExtension(int fieldNumber, ByteString value) { WriteTag(WireFormat.MessageSetField.Item, WireFormat.WireType.StartGroup); WriteUInt32(WireFormat.MessageSetField.TypeID, (uint)fieldNumber); WriteBytes(WireFormat.MessageSetField.Message, value); WriteTag(WireFormat.MessageSetField.Item, WireFormat.WireType.EndGroup); } public void WriteField(FieldType fieldType, int fieldNumber, object value) { switch (fieldType) { case FieldType.Double: WriteDouble(fieldNumber, (double)value); break; case FieldType.Float: WriteFloat(fieldNumber, (float)value); break; case FieldType.Int64: WriteInt64(fieldNumber, (long)value); break; case FieldType.UInt64: WriteUInt64(fieldNumber, (ulong)value); break; case FieldType.Int32: WriteInt32(fieldNumber, (int)value); break; case FieldType.Fixed64: WriteFixed64(fieldNumber, (long)value); break; case FieldType.Fixed32: WriteFixed32(fieldNumber, (int)value); break; case FieldType.Bool: WriteBool(fieldNumber, (bool)value); break; case FieldType.String: WriteString(fieldNumber, (string)value); break; case FieldType.Group: WriteGroup(fieldNumber, (IMessage)value); break; case FieldType.Message: WriteMessage(fieldNumber, (IMessage)value); break; case FieldType.Bytes: WriteBytes(fieldNumber, (ByteString)value); break; case FieldType.UInt32: WriteUInt32(fieldNumber, (uint)value); break; case FieldType.SFixed32: WriteSFixed32(fieldNumber, (int)value); break; case FieldType.SFixed64: WriteSFixed64(fieldNumber, (long)value); break; case FieldType.SInt32: WriteSInt32(fieldNumber, (int)value); break; case FieldType.SInt64: WriteSInt64(fieldNumber, (long)value); break; case FieldType.Enum: WriteEnum(fieldNumber, ((EnumValueDescriptor)value).Number); break; } } #endregion #region Underlying writing primitives /// /// Encodes and writes a tag. /// public void WriteTag(int fieldNumber, WireFormat.WireType type) { WriteRawVarint32(WireFormat.MakeTag(fieldNumber, type)); } public void WriteRawVarint32(uint value) { while (true) { if ((value & ~0x7F) == 0) { WriteRawByte(value); return; } else { WriteRawByte((value & 0x7F) | 0x80); value >>= 7; } } } public void WriteRawVarint64(ulong value) { while (true) { if ((value & ~0x7FUL) == 0) { WriteRawByte((uint)value); return; } else { WriteRawByte(((uint)value & 0x7F) | 0x80); value >>= 7; } } } public void WriteRawLittleEndian32(int value) { WriteRawByte((byte)value); WriteRawByte((byte)(value >> 8)); WriteRawByte((byte)(value >> 16)); WriteRawByte((byte)(value >> 24)); } public void WriteRawLittleEndian64(long value) { WriteRawByte((byte)value); WriteRawByte((byte)(value >> 8)); WriteRawByte((byte)(value >> 16)); WriteRawByte((byte)(value >> 24)); WriteRawByte((byte)(value >> 32)); WriteRawByte((byte)(value >> 40)); WriteRawByte((byte)(value >> 48)); WriteRawByte((byte)(value >> 56)); } public void WriteRawByte(byte value) { if (position == limit) { RefreshBuffer(); } buffer[position++] = value; } public void WriteRawByte(uint value) { WriteRawByte((byte)value); } /// /// Writes out an array of bytes. /// public void WriteRawBytes(byte[] value) { WriteRawBytes(value, 0, value.Length); } /// /// Writes out part of an array of bytes. /// public void WriteRawBytes(byte[] value, int offset, int length) { if (limit - position >= length) { Array.Copy(value, offset, buffer, position, length); // We have room in the current buffer. position += length; } else { // Write extends past current buffer. Fill the rest of this buffer and // flush. int bytesWritten = limit - position; Array.Copy(value, offset, buffer, position, bytesWritten); offset += bytesWritten; length -= bytesWritten; position = limit; RefreshBuffer(); // Now deal with the rest. // Since we have an output stream, this is our buffer // and buffer offset == 0 if (length <= limit) { // Fits in new buffer. Array.Copy(value, offset, buffer, 0, length); position = length; } else { // Write is very big. Let's do it all at once. output.Write(value, offset, length); } } } #endregion #region Size computations const int LittleEndian64Size = 8; const int LittleEndian32Size = 4; /// /// Compute the number of bytes that would be needed to encode a /// double field, including the tag. /// public static int ComputeDoubleSize(int fieldNumber, double value) { return ComputeTagSize(fieldNumber) + LittleEndian64Size; } /// /// Compute the number of bytes that would be needed to encode a /// float field, including the tag. /// public static int ComputeFloatSize(int fieldNumber, float value) { return ComputeTagSize(fieldNumber) + LittleEndian32Size; } /// /// Compute the number of bytes that would be needed to encode a /// uint64 field, including the tag. /// public static int ComputeUInt64Size(int fieldNumber, ulong value) { return ComputeTagSize(fieldNumber) + ComputeRawVarint64Size(value); } /// /// Compute the number of bytes that would be needed to encode an /// int64 field, including the tag. /// public static int ComputeInt64Size(int fieldNumber, long value) { return ComputeTagSize(fieldNumber) + ComputeRawVarint64Size((ulong)value); } /// /// Compute the number of bytes that would be needed to encode an /// int32 field, including the tag. /// public static int ComputeInt32Size(int fieldNumber, int value) { if (value >= 0) { return ComputeTagSize(fieldNumber) + ComputeRawVarint32Size((uint)value); } else { // Must sign-extend. return ComputeTagSize(fieldNumber) + 10; } } /// /// Compute the number of bytes that would be needed to encode a /// fixed64 field, including the tag. /// public static int ComputeFixed64Size(int fieldNumber, long value) { return ComputeTagSize(fieldNumber) + LittleEndian64Size; } /// /// Compute the number of bytes that would be needed to encode a /// fixed32 field, including the tag. /// public static int ComputeFixed32Size(int fieldNumber, int value) { return ComputeTagSize(fieldNumber) + LittleEndian32Size; } /// /// Compute the number of bytes that would be needed to encode a /// bool field, including the tag. /// public static int ComputeBoolSize(int fieldNumber, bool value) { return ComputeTagSize(fieldNumber) + 1; } /// /// Compute the number of bytes that would be needed to encode a /// string field, including the tag. /// public static int ComputeStringSize(int fieldNumber, String value) { int byteArraySize = Encoding.UTF8.GetByteCount(value); return ComputeTagSize(fieldNumber) + ComputeRawVarint32Size((uint)byteArraySize) + byteArraySize; } /// /// Compute the number of bytes that would be needed to encode a /// group field, including the tag. /// public static int ComputeGroupSize(int fieldNumber, IMessage value) { return ComputeTagSize(fieldNumber) * 2 + value.SerializedSize; } /// /// Compute the number of bytes that would be needed to encode a /// group field represented by an UnknownFieldSet, including the tag. /// public static int ComputeUnknownGroupSize(int fieldNumber, UnknownFieldSet value) { return ComputeTagSize(fieldNumber) * 2 + value.SerializedSize; } /// /// Compute the number of bytes that would be needed to encode an /// embedded message field, including the tag. /// public static int ComputeMessageSize(int fieldNumber, IMessage value) { int size = value.SerializedSize; return ComputeTagSize(fieldNumber) + ComputeRawVarint32Size((uint)size) + size; } /// /// Compute the number of bytes that would be needed to encode a /// bytes field, including the tag. /// public static int ComputeBytesSize(int fieldNumber, ByteString value) { return ComputeTagSize(fieldNumber) + ComputeRawVarint32Size((uint)value.Length) + value.Length; } /// /// Compute the number of bytes that would be needed to encode a /// uint32 field, including the tag. /// public static int ComputeUInt32Size(int fieldNumber, uint value) { return ComputeTagSize(fieldNumber) + ComputeRawVarint32Size(value); } /// /// Compute the number of bytes that would be needed to encode a /// enum field, including the tag. The caller is responsible for /// converting the enum value to its numeric value. /// public static int ComputeEnumSize(int fieldNumber, int value) { return ComputeTagSize(fieldNumber) + ComputeRawVarint32Size((uint)value); } /// /// Compute the number of bytes that would be needed to encode an /// sfixed32 field, including the tag. /// public static int ComputeSFixed32Size(int fieldNumber, int value) { return ComputeTagSize(fieldNumber) + LittleEndian32Size; } /// /// Compute the number of bytes that would be needed to encode an /// sfixed64 field, including the tag. /// public static int ComputeSFixed64Size(int fieldNumber, long value) { return ComputeTagSize(fieldNumber) + LittleEndian64Size; } /// /// Compute the number of bytes that would be needed to encode an /// sint32 field, including the tag. /// public static int ComputeSInt32Size(int fieldNumber, int value) { return ComputeTagSize(fieldNumber) + ComputeRawVarint32Size(EncodeZigZag32(value)); } /// /// Compute the number of bytes that would be needed to encode an /// sint64 field, including the tag. /// public static int ComputeSInt64Size(int fieldNumber, long value) { return ComputeTagSize(fieldNumber) + ComputeRawVarint64Size(EncodeZigZag64(value)); } /* * Compute the number of bytes that would be needed to encode a * MessageSet extension to the stream. For historical reasons, * the wire format differs from normal fields. */ /// /// Compute the number of bytes that would be needed to encode a /// MessageSet extension to the stream. For historical reasons, /// the wire format differs from normal fields. /// public static int ComputeMessageSetExtensionSize(int fieldNumber, IMessage value) { return ComputeTagSize(WireFormat.MessageSetField.Item) * 2 + ComputeUInt32Size(WireFormat.MessageSetField.TypeID, (uint) fieldNumber) + ComputeMessageSize(WireFormat.MessageSetField.Message, value); } /// /// Compute the number of bytes that would be needed to encode an /// unparsed MessageSet extension field to the stream. For /// historical reasons, the wire format differs from normal fields. /// public static int ComputeRawMessageSetExtensionSize(int fieldNumber, ByteString value) { return ComputeTagSize(WireFormat.MessageSetField.Item) * 2 + ComputeUInt32Size(WireFormat.MessageSetField.TypeID, (uint) fieldNumber) + ComputeBytesSize(WireFormat.MessageSetField.Message, value); } /// /// Compute the number of bytes that would be needed to encode a varint. /// public static int ComputeRawVarint32Size(uint value) { if ((value & (0xffffffff << 7)) == 0) return 1; if ((value & (0xffffffff << 14)) == 0) return 2; if ((value & (0xffffffff << 21)) == 0) return 3; if ((value & (0xffffffff << 28)) == 0) return 4; return 5; } /// /// Compute the number of bytes that would be needed to encode a varint. /// public static int ComputeRawVarint64Size(ulong value) { if ((value & (0xffffffffffffffffL << 7)) == 0) return 1; if ((value & (0xffffffffffffffffL << 14)) == 0) return 2; if ((value & (0xffffffffffffffffL << 21)) == 0) return 3; if ((value & (0xffffffffffffffffL << 28)) == 0) return 4; if ((value & (0xffffffffffffffffL << 35)) == 0) return 5; if ((value & (0xffffffffffffffffL << 42)) == 0) return 6; if ((value & (0xffffffffffffffffL << 49)) == 0) return 7; if ((value & (0xffffffffffffffffL << 56)) == 0) return 8; if ((value & (0xffffffffffffffffL << 63)) == 0) return 9; return 10; } /* * Compute the number of bytes that would be needed to encode a * field of arbitrary type, including tag, to the stream. * * @param type The field's type. * @param number The field's number. * @param value Object representing the field's value. Must be of the exact * type which would be returned by * {@link Message#getField(FieldDescriptor)} for * this field. */ public static int ComputeFieldSize(FieldType fieldType, int fieldNumber, Object value) { switch (fieldType) { case FieldType.Double: return ComputeDoubleSize(fieldNumber, (double)value); case FieldType.Float: return ComputeFloatSize(fieldNumber, (float)value); case FieldType.Int64: return ComputeInt64Size(fieldNumber, (long)value); case FieldType.UInt64: return ComputeUInt64Size(fieldNumber, (ulong)value); case FieldType.Int32: return ComputeInt32Size(fieldNumber, (int)value); case FieldType.Fixed64: return ComputeFixed64Size(fieldNumber, (long)value); case FieldType.Fixed32: return ComputeFixed32Size(fieldNumber, (int)value); case FieldType.Bool: return ComputeBoolSize(fieldNumber, (bool)value); case FieldType.String: return ComputeStringSize(fieldNumber, (string)value); case FieldType.Group: return ComputeGroupSize(fieldNumber, (IMessage)value); case FieldType.Message: return ComputeMessageSize(fieldNumber, (IMessage)value); case FieldType.Bytes: return ComputeBytesSize(fieldNumber, (ByteString)value); case FieldType.UInt32: return ComputeUInt32Size(fieldNumber, (uint)value); case FieldType.SFixed32: return ComputeSFixed32Size(fieldNumber, (int)value); case FieldType.SFixed64: return ComputeSFixed64Size(fieldNumber, (long)value); case FieldType.SInt32: return ComputeSInt32Size(fieldNumber, (int)value); case FieldType.SInt64: return ComputeSInt64Size(fieldNumber, (long)value); case FieldType.Enum: return ComputeEnumSize(fieldNumber, ((EnumValueDescriptor)value).Number); default: throw new ArgumentOutOfRangeException("Invalid field type " + fieldType); } } /// /// Compute the number of bytes that would be needed to encode a tag. /// public static int ComputeTagSize(int fieldNumber) { return ComputeRawVarint32Size(WireFormat.MakeTag(fieldNumber, 0)); } #endregion /// /// Encode a 32-bit value with ZigZag encoding. /// /// /// ZigZag encodes signed integers into values that can be efficiently /// encoded with varint. (Otherwise, negative values must be /// sign-extended to 64 bits to be varint encoded, thus always taking /// 10 bytes on the wire.) /// public static uint EncodeZigZag32(int n) { // Note: the right-shift must be arithmetic return (uint)((n << 1) ^ (n >> 31)); } /// /// Encode a 64-bit value with ZigZag encoding. /// /// /// ZigZag encodes signed integers into values that can be efficiently /// encoded with varint. (Otherwise, negative values must be /// sign-extended to 64 bits to be varint encoded, thus always taking /// 10 bytes on the wire.) /// public static ulong EncodeZigZag64(long n) { return (ulong)((n << 1) ^ (n >> 63)); } private void RefreshBuffer() { if (output == null) { // We're writing to a single buffer. throw new OutOfSpaceException(); } // Since we have an output stream, this is our buffer // and buffer offset == 0 output.Write(buffer, 0, position); position = 0; } /// /// Indicates that a CodedOutputStream wrapping a flat byte array /// ran out of space. /// public class OutOfSpaceException : IOException { internal OutOfSpaceException() : base("CodedOutputStream was writing to a flat byte array and ran out of space.") { } } public void Flush() { if (output != null) { RefreshBuffer(); } } /// /// Verifies that SpaceLeft returns zero. It's common to create a byte array /// that is exactly big enough to hold a message, then write to it with /// a CodedOutputStream. Calling CheckNoSpaceLeft after writing verifies that /// the message was actually as big as expected, which can help bugs. /// public void CheckNoSpaceLeft() { if (SpaceLeft != 0) { throw new InvalidOperationException("Did not write as much data as expected."); } } /// /// If writing to a flat array, returns the space left in the array. Otherwise, /// throws an InvalidOperationException. /// public int SpaceLeft { get { if (output == null) { return limit - position; } else { throw new InvalidOperationException( "SpaceLeft can only be called on CodedOutputStreams that are " + "writing to a flat array."); } } } } }