LZ4Stream

Apr 25, 2013 at 5:57 PM
I've taken the liberty to create a small stream wrapper for your port of LZ4. While it's not perfect, it might help a lot of people to port existing code to an LZ4 variant.

Note that reading should use a BufferedReader; I haven't optimized read block-length! Also I simply use LZ4n - but you can change it easily into a Func if you like.

The code is pretty straight forward:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;

namespace LZ4n
{
    public class LZ4CompressStream : Stream
    {
        public LZ4CompressStream(Stream baseStream)
        {
            this.baseStream = baseStream;
        }

        private Stream baseStream;
        private byte[] buffer = new byte[BufferSize];
        private int ptr = 0;

        internal const int BufferSize = 1024 * 1024;

        public override bool CanRead
        {
            get { return false; }
        }

        public override bool CanSeek
        {
            get { return false; }
        }

        public override bool CanWrite
        {
            get { return true; }
        }

        public override void Flush()
        {
        }

        public override long Length
        {
            get { throw new NotSupportedException(); }
        }

        public override long Position
        {
            get
            {
                throw new NotSupportedException();
            }
            set
            {
                throw new NotSupportedException();
            }
        }

        public override int Read(byte[] buffer, int offset, int count)
        {
            throw new NotSupportedException();
        }

        private void WritePage()
        {
            if (this.ptr > 0)
            {
                byte[] buffer = new byte[LZ4Codec.MaximumOutputLength(this.ptr) + 8];
                int n = LZ4Codec.Encode64(this.buffer, 0, this.ptr, buffer, 8, buffer.Length - 8);

                Buffer.BlockCopy(BitConverter.GetBytes(this.ptr), 0, buffer, 0, 4);
                Buffer.BlockCopy(BitConverter.GetBytes(n), 0, buffer, 4, 4);

                baseStream.Write(buffer, 0, n + 8);

                this.ptr = 0;
            }
        }

        public override long Seek(long offset, SeekOrigin origin)
        {
            throw new NotSupportedException();
        }

        public override void SetLength(long value)
        {
            throw new NotSupportedException();
        }

        public override void Write(byte[] buffer, int offset, int count)
        {
            while (count + this.ptr >= BufferSize)
            {
                int towrite = BufferSize - this.ptr;
                Buffer.BlockCopy(buffer, offset, this.buffer, this.ptr, towrite);
                this.ptr += towrite;

                WritePage();
                this.ptr = 0;

                count -= towrite;
                offset += towrite;
            }

            Buffer.BlockCopy(buffer, offset, this.buffer, this.ptr, count);
            this.ptr += count;
        }

        public override void Close()
        {
            if (baseStream != null)
            {
                WritePage();
                base.Close();
                baseStream.Close();
                baseStream = null;
            }
        }
    }

    public class LZ4DecompressStream : Stream
    {
        public LZ4DecompressStream(Stream baseStream)
        {
            compressedBuffer = new byte[LZ4Codec.MaximumOutputLength(LZ4CompressStream.BufferSize) + 4];

            this.baseStream = baseStream;
            this.eof = ReadPage();
        }

        private bool eof;
        private Stream baseStream;

        private byte[] compressedBuffer;
        private byte[] uncompressedBuffer = new byte[LZ4CompressStream.BufferSize];
        private int uncompressedPtr = 0;
        private int uncompressedLength;

        public override bool CanRead
        {
            get { return true; }
        }

        public override bool CanSeek
        {
            get { return false; }
        }

        public override bool CanWrite
        {
            get { return false; }
        }

        public override void Flush()
        {
        }

        public override long Length
        {
            get { throw new NotSupportedException(); }
        }

        public override long Position
        {
            get
            {
                throw new NotSupportedException();
            }
            set
            {
                throw new NotSupportedException();
            }
        }

        public override int Read(byte[] buffer, int offset, int count)
        {
            int read = 0;
            while (count > 0 && uncompressedPtr < uncompressedBuffer.Length)
            {
                int tocopy = Math.Min(uncompressedLength - uncompressedPtr, count);
                Buffer.BlockCopy(uncompressedBuffer, uncompressedPtr, buffer, offset, tocopy);

                uncompressedPtr += tocopy;
                count -= tocopy;
                offset += tocopy;
                read += tocopy;

                if ((uncompressedPtr == uncompressedLength && !ReadPage()) || count == 0)
                {
                    return read;
                }
            }

            return read;
        }

        private bool ReadPage()
        {
            var read = baseStream.Read(this.compressedBuffer, 0, 8);
            if (read < 8)
            {
                return false; // EOF
            }

            uncompressedLength = BitConverter.ToInt32(compressedBuffer, 0);
            int toread = BitConverter.ToInt32(compressedBuffer, 4);

            int off = 8;
            do
            {
                read = baseStream.Read(compressedBuffer, off, toread);
                toread -= read;
                off += read;
            }
            while (read > 0 && toread > 0);

            if (toread > 0)
            {
                throw new Exception("Error reading compressed file; expected data.");
            }
            else
            {
                LZ4Codec.Decode64(compressedBuffer, 8, off - 8, uncompressedBuffer, 0, uncompressedLength, true);
                uncompressedPtr = 0;
                return true;
            }
        }

        public override long Seek(long offset, SeekOrigin origin)
        {
            throw new NotSupportedException();
        }

        public override void SetLength(long value)
        {
            throw new NotSupportedException();
        }

        public override void Write(byte[] buffer, int offset, int count)
        {
            throw new NotSupportedException();
        }

        public override void Close()
        {
            base.Close();
            baseStream.Close();
        }
    }
}