propper buffering in place

MQTTnet.Core/Channel/BufferConstants.cs

@@ -0,0 +1,7 @@
+namespace MQTTnet.Core.Channel
+{
+    public static class BufferConstants
+    {
+        public const int Size = 4096 * 20;
+    }
+}

MQTTnet.Core/Channel/BufferedCommunicationChannel.cs

@@ -0,0 +1,77 @@
+using System.Threading.Tasks;
+using MQTTnet.Core.Client;
+using System;
+
+namespace MQTTnet.Core.Channel
+{
+    public class BufferedCommunicationChannel : IMqttCommunicationChannel
+    {
+        private IMqttCommunicationChannel _inner { get; }
+        private int _bufferSize = 0;
+        private int _bufferOffset = 0;
+
+        public BufferedCommunicationChannel(IMqttCommunicationChannel inner)
+        {
+            _inner = inner;
+        }
+
+        public Task ConnectAsync(MqttClientOptions options)
+        {
+            return _inner.ConnectAsync(options);
+        }
+
+        public Task DisconnectAsync()
+        {
+            return _inner.DisconnectAsync();
+        }
+
+        public int Peek()
+        {
+            return _inner.Peek();
+        }
+
+        public async Task<ArraySegment<byte>> ReadAsync(int length, byte[] buffer)
+        {
+            //read from buffer
+            if (_bufferSize > 0)
+            {
+                return ReadFomBuffer(length, buffer);
+            }
+
+            var available = _inner.Peek();
+            // if there are less or equal bytes available then requested then just read em
+            if (available <= length)
+            {
+                return await _inner.ReadAsync(length, buffer);
+            }
+
+            //if more bytes are available than requested do buffer them to reduce calls to network buffers
+            await WriteToBuffer(available, buffer).ConfigureAwait(false);
+            return ReadFomBuffer(length, buffer);
+        }
+
+        private async Task WriteToBuffer(int available, byte[] buffer)
+        {
+            await _inner.ReadAsync(available, buffer).ConfigureAwait(false);
+            _bufferSize = available;
+            _bufferOffset = 0;
+        }
+
+        private ArraySegment<byte> ReadFomBuffer(int length, byte[] buffer)
+        {
+            var result = new ArraySegment<byte>(buffer, _bufferOffset, length);
+            _bufferSize -= length;
+            _bufferOffset += length;
+
+            if (_bufferSize < 0)
+            {
+            }
+            return result;
+        }
+
+        public Task WriteAsync(byte[] buffer)
+        {
+            return _inner.WriteAsync(buffer);
+        }
+    }
+}

MQTTnet.Core/Channel/IMqttCommunicationChannel.cs

@@ -1,5 +1,6 @@
 using System.Threading.Tasks;
 using MQTTnet.Core.Client;
+using System;
 
 namespace MQTTnet.Core.Channel
 {
@@ -11,6 +12,11 @@ namespace MQTTnet.Core.Channel
 
         Task WriteAsync(byte[] buffer);
 
-        Task ReadAsync(byte[] buffer);
+        /// <summary>
+        /// get the currently available number of bytes without reading them
+        /// </summary>
+        int Peek();
+
+        Task<ArraySegment<byte>> ReadAsync(int length, byte[] buffer);
     }
 }

MQTTnet.Core/Serializer/MqttPacketReader.cs

@@ -49,13 +49,13 @@ namespace MQTTnet.Core.Serializer
             return ReadBytes(_header.BodyLength - (int)BaseStream.Position);
         }
 
-        public static async Task<MqttPacketHeader> ReadHeaderFromSourceAsync(IMqttCommunicationChannel source)
+        public static async Task<MqttPacketHeader> ReadHeaderFromSourceAsync(IMqttCommunicationChannel source, byte[] buffer)
         {
-            var fixedHeader = await ReadStreamByteAsync(source).ConfigureAwait(false);
+            var fixedHeader = await ReadStreamByteAsync(source, buffer).ConfigureAwait(false);
             var byteReader = new ByteReader(fixedHeader);
             byteReader.Read(4);
             var controlPacketType = (MqttControlPacketType)byteReader.Read(4);
-            var bodyLength = await ReadBodyLengthFromSourceAsync(source).ConfigureAwait(false);
+            var bodyLength = await ReadBodyLengthFromSourceAsync(source, buffer).ConfigureAwait(false);
 
             return new MqttPacketHeader()
             {
@@ -65,18 +65,17 @@ namespace MQTTnet.Core.Serializer
             };
         }
 
-        private static async Task<byte> ReadStreamByteAsync(IMqttCommunicationChannel source)
+        private static async Task<byte> ReadStreamByteAsync(IMqttCommunicationChannel source, byte[] readBuffer)
         {
-            var buffer = new byte[1];
-            await ReadFromSourceAsync(source, buffer).ConfigureAwait(false);
-            return buffer[0];
+            var result = await ReadFromSourceAsync(source, 1, readBuffer).ConfigureAwait(false);
+            return result.Array[result.Offset];
         }
 
-        public static async Task ReadFromSourceAsync(IMqttCommunicationChannel source, byte[] buffer)
+        public static async Task<ArraySegment<byte>> ReadFromSourceAsync(IMqttCommunicationChannel source, int length, byte[] buffer)
         {
             try
             {
-                await source.ReadAsync(buffer);
+                return await source.ReadAsync(length, buffer);
             }
             catch (Exception exception)
             {
@@ -84,7 +83,7 @@ namespace MQTTnet.Core.Serializer
             }
         }
         
-        private static async Task<int> ReadBodyLengthFromSourceAsync(IMqttCommunicationChannel source)
+        private static async Task<int> ReadBodyLengthFromSourceAsync(IMqttCommunicationChannel source, byte[] buffer)
         {
             // Alorithm taken from http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html.
             var multiplier = 1;
@@ -92,7 +91,7 @@ namespace MQTTnet.Core.Serializer
             byte encodedByte;
             do
             {
-                encodedByte = await ReadStreamByteAsync(source).ConfigureAwait(false);
+                encodedByte = await ReadStreamByteAsync(source, buffer).ConfigureAwait(false);
                 value += (encodedByte & 127) * multiplier;
                 multiplier *= 128;
                 if (multiplier > 128 * 128 * 128)

MQTTnet.Core/Serializer/MqttPacketSerializer.cs

@@ -17,6 +17,7 @@ namespace MQTTnet.Core.Serializer
         private static byte[] ProtocolVersionV310Name { get; } = Encoding.UTF8.GetBytes("MQIs");
 
         public MqttProtocolVersion ProtocolVersion { get; set; } = MqttProtocolVersion.V311;
+        private byte[] _readBuffer = new byte[BufferConstants.Size]; 
 
         public async Task SerializeAsync(MqttBasePacket packet, IMqttCommunicationChannel destination)
         {
@@ -115,17 +116,13 @@ namespace MQTTnet.Core.Serializer
         public async Task<MqttBasePacket> DeserializeAsync(IMqttCommunicationChannel source)
         {
             if (source == null) throw new ArgumentNullException(nameof(source));
-            
-            var header = await MqttPacketReader.ReadHeaderFromSourceAsync(source).ConfigureAwait(false);
 
-            var body = new byte[header.BodyLength];
-            if (header.BodyLength > 0)
+            var header = await MqttPacketReader.ReadHeaderFromSourceAsync(source, _readBuffer).ConfigureAwait(false);
+
+            var body = await GetBody(source, header).ConfigureAwait(false);
+
+            using (var mqttPacketReader = new MqttPacketReader(body, header))
             {
-                await MqttPacketReader.ReadFromSourceAsync(source, body).ConfigureAwait(false);
-            }
-
-            using (var mqttPacketReader = new MqttPacketReader(new MemoryStream(body), header))
-            {                
                 switch (header.ControlPacketType)
                 {
                     case MqttControlPacketType.Connect:
@@ -221,6 +218,17 @@ namespace MQTTnet.Core.Serializer
             }
         }
 
+        private async Task<MemoryStream> GetBody(IMqttCommunicationChannel source, MqttPacketHeader header)
+        {
+            if (header.BodyLength > 0)
+            {
+                var segment = await MqttPacketReader.ReadFromSourceAsync(source, header.BodyLength, _readBuffer).ConfigureAwait(false);
+                return new MemoryStream(segment.Array, segment.Offset, segment.Count);
+            }
+
+            return new MemoryStream();
+        }
+
         private static MqttBasePacket DeserializeUnsubscribe(MqttPacketReader reader)
         {
             var packet = new MqttUnsubscribePacket