BinarySerializer.h

Go to the documentation of this file.

/*
 * Copyright (C) 2012 by
 *   MetraLabs GmbH (MLAB), GERMANY
 * and
 *   Neuroinformatics and Cognitive Robotics Labs (NICR) at TU Ilmenau, GERMANY
 * All rights reserved.
 *
 * Contact: info@mira-project.org
 *
 * Commercial Usage:
 *   Licensees holding valid commercial licenses may use this file in
 *   accordance with the commercial license agreement provided with the
 *   software or, alternatively, in accordance with the terms contained in
 *   a written agreement between you and MLAB or NICR.
 *
 * GNU General Public License Usage:
 *   Alternatively, this file may be used under the terms of the GNU
 *   General Public License version 3.0 as published by the Free Software
 *   Foundation and appearing in the file LICENSE.GPL3 included in the
 *   packaging of this file. Please review the following information to
 *   ensure the GNU General Public License version 3.0 requirements will be
 *   met: http://www.gnu.org/copyleft/gpl.html.
 *   Alternatively you may (at your option) use any later version of the GNU
 *   General Public License if such license has been publicly approved by
 *   MLAB and NICR (or its successors, if any).
 *
 * IN NO EVENT SHALL "MLAB" OR "NICR" BE LIABLE TO ANY PARTY FOR DIRECT,
 * INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF
 * THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF "MLAB" OR
 * "NICR" HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * "MLAB" AND "NICR" SPECIFICALLY DISCLAIM ANY WARRANTIES, INCLUDING,
 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
 * FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
 * ON AN "AS IS" BASIS, AND "MLAB" AND "NICR" HAVE NO OBLIGATION TO
 * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS OR MODIFICATIONS.
 */

#ifndef _MIRA_BINARYSERIALIZER_H_
#define _MIRA_BINARYSERIALIZER_H_

#include <tuple>

#include <stream/BinaryStream.h>

#include <serialization/Serializer.h>
#include <serialization/Deserializer.h>

#include <serialization/BinarySerializerCodec.h>

#include <serialization/IsBitwiseSerializable.h>
#include <platform/Types.h>
#include <platform/Typename.h>

namespace mira {


// a marker to mark occurrence of format version number in serialized data.
// not 100% unique when looking at serialized data created before binary format
// version was included, but it will have to do
#define BINARY_VERSION_MARKER 65432u

#ifndef NDEBUG
#define CHECK_FORCE_SERIALIZE_BINARY_VERSION
#endif


class BinarySerializerMixin
{
public:
    enum PointerType
    {
        NULL_POINTER        = 0,
        POINTER_REFERENCE   = 1,
        NORMAL_POINTER      = 2,
        POLYMORPHIC_POINTER = 3
    };
};


template <typename BinaryStream>
class StreamAccessMixinBase
{
protected:
    StreamAccessMixinBase(typename BinaryStream::streambuffer_pointer buffer)
        : mStream(buffer) {}

    StreamAccessMixinBase(BinaryStream& stream) : mStream(stream.rdbuf()) {}

public:
    void reassign(typename BinaryStream::streambuffer_pointer buffer) { mStream = buffer; }

protected:
    BinaryStream& stream() { return mStream; }

private:
    BinaryStream mStream;
};

template <typename BinaryStream, bool Buffered>
class StreamAccessMixin : public StreamAccessMixinBase<BinaryStream>
{
    typedef StreamAccessMixinBase<BinaryStream> Base;

public:
    typedef BinaryStream                        StreamType;

protected:
    StreamAccessMixin(typename BinaryStream::streambuffer_pointer buffer)
        : Base(buffer) {}

    StreamAccessMixin(BinaryStream& stream) : Base(stream) {}

    void flushBuffer() {}
};

template <typename BinaryStream>
class StreamAccessMixin<BinaryStream, true> : public StreamAccessMixinBase<BinaryStream>
{
    typedef StreamAccessMixinBase<BinaryStream> Base;

public:
    typedef BinaryBufferOstream                 StreamType;

protected:
    StreamAccessMixin(typename BinaryStream::streambuffer_pointer buffer)
        : Base(buffer), mBufferStream(&mBuffer) {}

    StreamAccessMixin(BinaryStream& stream) : Base(stream), mBufferStream(&mBuffer) {}

    ~StreamAccessMixin()
    {
        flushBuffer(); // just to make sure everything is written out
    }

    BinaryBufferOstream& stream() { return mBufferStream; }

    void flushBuffer()
    {
        Base::stream().write(mBuffer.data(), mBuffer.size());
        mBuffer.clear();
        mBufferStream.seekp(0);
    }

private:
    BinaryBufferOstream::buffer_type mBuffer;
    BinaryBufferOstream mBufferStream;
};


struct BinarySerializerTag {};

template <typename Derived>
class BinarySerializer : public Serializer<Derived>
{
public:
    typedef BinarySerializerTag Tag;

public:
    typedef serialization::VersionType VersionType;
    typedef serialization::AcceptDesiredVersion AcceptDesiredVersion;

    template <typename T>
    VersionType version(VersionType version, const T* caller = NULL) {
        return this->This()->template version<T>(version);
    }

    MIRA_DEPRECATED("Please call as version<MyType>(v) or version(v, this)",
    VersionType version(VersionType version)) {
        return this->template version<void>(version);
    }

    template <typename T>
    VersionType version(VersionType version, AcceptDesiredVersion, const T* caller = NULL)
    {
        return this->This()->template version<T>(version, AcceptDesiredVersion());
    }

public:
    template <typename T>
    void write(const T* data, std::size_t count) {
        this->This()->write(data,count);
    }

public:
    template <typename T>
    bool hasCodec() const {
        return this->This()->template hasCodec<T>();
    }

    template <typename T>
    bool codec(const T& obj)
    {
        return this->This()->template codec(obj);
    }
};


template <typename BinaryStream, uint8 BinaryFormatVersion, bool Buffered>
class ConcreteBinarySerializer;

template <typename BinaryStream, uint8 BinaryFormatVersion, bool Buffered, typename StreamType>
class SerializerFormatMixin
{
};

template <typename Serializer, typename StreamType>
class SerializerFormatMixin01Base
{
public:
    static size_t formatVersionOverhead() { return 0; }

protected:
    typedef boost::mpl::bool_<false> requireReflectBarriers;
    typedef void*                    ReflectState;

    typedef int VersionType; // format version 0 had int32 serialization versions

    ReflectState insertVersionPlaceholder(const char* context,
                                          Serializer& serializer,
                                          StreamType& stream) { return NULL; }

    template <typename T>
    void writeVersion(VersionType version, Serializer& serializer, StreamType& stream) {
        serializer.atomic(version);
    }

    void restoreVersionPtr(const ReflectState& prev) {}
};

template <typename BinaryStream, bool Buffered, typename StreamType>
class SerializerFormatMixin<BinaryStream, 0, Buffered, StreamType> :
    public SerializerFormatMixin01Base<ConcreteBinarySerializer<BinaryStream, 0, Buffered>, StreamType>
{
public:
    static uint8 getSerializerFormatVersion() { return 0; }

protected:
    template <typename T>
    bool writeFormatVersion(T& value, bool enableTypeCheck, StreamType& stream) { return true; }
};

// This is just for sake of completeness, actually. There is no use case for serializing to v1.
// Where needed, v1 data should be created from v0 serialized data by just prepending the version
// marker bytes.
template <typename BinaryStream, bool Buffered, typename StreamType>
class SerializerFormatMixin<BinaryStream, 1, Buffered, StreamType> :
    public SerializerFormatMixin01Base<ConcreteBinarySerializer<BinaryStream, 1, Buffered>, StreamType>
{
public:
    static uint8 getSerializerFormatVersion() { return 1; }

protected:
    template <typename T>
    bool writeFormatVersion(T& value, bool enableTypeCheck, StreamType& stream)
    {
        stream << (uint16)BINARY_VERSION_MARKER;
        stream << getSerializerFormatVersion();
        return true;
    }
};

template <typename BinaryStream, bool Buffered, typename StreamType>
class SerializerFormatMixin<BinaryStream, 2, Buffered, StreamType>
{
public:
    static uint8 getSerializerFormatVersion() { return 2; }

    static size_t formatVersionOverhead() { return sizeof(uint16) + sizeof(uint8); }
protected:
    typedef ConcreteBinarySerializer<BinaryStream, 2, Buffered> Serializer;

    // Needs to know about independent reflection blocks to reserve space
    // for a version number in the output for each of them.
    typedef boost::mpl::bool_<true> requireReflectBarriers;

    // State consists of position for a version (placeholder) in the buffer
    // and a flag to know if it was set by the reflected object.
    // The third element is a context string, which should indicate the reflect() method
    // we are in (typically the typename of the reflected class).
    typedef std::tuple<typename StreamType::pos_type, bool, const char*> ReflectState;

    typedef serialization::VersionType VersionType;

    SerializerFormatMixin() : mVersionPos(0), mContext(NULL) {}

    template <typename T>
    bool writeFormatVersion(T& value, bool enableTypeCheck, StreamType& stream)
    {
#ifdef CHECK_FORCE_SERIALIZE_BINARY_VERSION
        int v = Serializer::forcedSerializeVersion();
        if (v >= 0) {
            MIRA_LOG(NOTICE) << "BinarySerializer: Forced serializing as binary "
                             << "format version " << v << ".";

            if (v == 0) {
                // have to make this artificially depend on the template parameter,
                // otherwise compiler complains about 'incomplete type'
                typename Serializer::SimpleStreamType::buffer_type buffer;
                ConcreteBinarySerializer<typename Serializer::SimpleStreamType, 0, false> v0(&buffer);
                v0.serialize(value, enableTypeCheck);
                stream.write(buffer.data(), buffer.size());
                return false;
            }

            if (v == 1) {
                typename Serializer::SimpleStreamType::buffer_type buffer;
                ConcreteBinarySerializer<typename Serializer::SimpleStreamType, 1, false> v1(&buffer);
                v1.serialize(value, enableTypeCheck);
                stream.write(buffer.data(), buffer.size());
                return false;
            }

            if (v != getSerializerFormatVersion()) {
                MIRA_THROW(XIO, "BinarySerializer: Unhandled binary format, expected version "
                                 << (int)this->getSerializerFormatVersion() << ", requested " << (int)v << ".");
            }
        }
#endif

        // TODO: omit this when serializing an AtomicSerializable type? (optimization)
        // again, the tricky part is how to deal with it in the BinaryJSONConverter stuff??
        stream << (uint16)BINARY_VERSION_MARKER;
        stream << getSerializerFormatVersion();
        return true;
    }

    ReflectState insertVersionPlaceholder(const char* context, Serializer& serializer, StreamType& stream) {
        ReflectState prevState = std::make_tuple(mVersionPos, mVersionSet, mContext);

        mVersionPos = stream.tellp();
        if (mVersionPos == typename StreamType::pos_type(-1))
            MIRA_THROW(XIO, "Failed querying stream position. The stream type does not seem to support repositioning. "
                            "Please use a buffered serializer (BufferedBinaryStreamSerializer).");

        mVersionSet = false;
        mContext = context;
        VersionType tmp = 0;
        serializer.atomic(tmp);

        return prevState;
    }

    template <typename T>
    void writeVersion(VersionType version, Serializer& serializer, StreamType& stream) {

        if (mVersionSet) {
            MIRA_THROW(XIO, "BinarySerializer: version() called repeatedly by serialization of type '"
                            << typeName<T>() << "'. Context = '" << (mContext ? mContext : "") << "'.");
        }
        std::size_t pos = stream.tellp();
        stream.seekp(mVersionPos);
        serializer.atomic(version);
        stream.seekp(pos);
        mVersionSet = true;
    }

    void restoreVersionPtr(const ReflectState& prev) {
        mVersionPos = std::get<0>(prev);
        mVersionSet = std::get<1>(prev);
        mContext = std::get<2>(prev);
    }

private:
    typename StreamType::pos_type mVersionPos;
    bool mVersionSet;
    const char* mContext;
};


template <typename BinaryStream, uint8 BinaryFormatVersion, bool Buffered = false>
class ConcreteBinarySerializer : public BinarySerializer<ConcreteBinarySerializer<BinaryStream,
                                                                                  BinaryFormatVersion,
                                                                                  Buffered> >,
                                 public BinarySerializerMixin,
                                 public StreamAccessMixin<BinaryStream, Buffered>,
                                 public SerializerFormatMixin<BinaryStream, BinaryFormatVersion, Buffered,
                                                              typename StreamAccessMixin<BinaryStream, Buffered>::StreamType>
{
    typedef BinarySerializer<ConcreteBinarySerializer<BinaryStream, BinaryFormatVersion, Buffered> > Base;
    typedef StreamAccessMixin<BinaryStream, Buffered>                                                StreamAccess;
    typedef SerializerFormatMixin<BinaryStream, BinaryFormatVersion, Buffered,
                                  typename StreamAccessMixin<BinaryStream, Buffered>::StreamType>    Format;

public:

    typedef BinaryBufferOstream SimpleStreamType;

public:

    // We are not using human readable ids here. This will turn off the
    // whole id-handling system of the serialization/reflection framework
    // and dramatically improve the performance
    typedef boost::mpl::bool_<false> useHumanReadableIDs;

    typedef typename Format::requireReflectBarriers requireReflectBarriers;
    typedef typename Format::ReflectState           ReflectState;

public:
    ConcreteBinarySerializer(typename BinaryStream::streambuffer_pointer buffer)
        : StreamAccess(buffer), mLevel(0) {}

    ConcreteBinarySerializer(BinaryStream& stream) : StreamAccess(stream), mLevel(0) {}

public:

    template <typename T>
    void serialize(const T& value, bool enableTypeCheck = true)
    {
#ifdef CHECK_FORCE_SERIALIZE_BINARY_VERSION
        if (!Format::writeFormatVersion(value, enableTypeCheck, this->stream()))
            return;
#else
        Format::writeFormatVersion(value, enableTypeCheck, this->stream());
#endif

        if(enableTypeCheck) {
            std::string fulltypename = typeName<T>();
            Base::serialize("", fulltypename);
        }
        Base::serialize("", value);
    }

    typedef typename Format::VersionType VersionType;

    template <typename T>
    VersionType version(VersionType version, const T* caller = NULL) {
        version = this->template queryDesiredClassVersion<T>(version, false);
        this->template writeVersion<T>(version, *this, this->stream());
        return version;
    }

    MIRA_DEPRECATED("Please call as version<MyType>(v) or version(v, this)",
    VersionType version(VersionType version)) {
        return this->template version<void>(version);
    }

    typedef typename Base::AcceptDesiredVersion AcceptDesiredVersion;

    template <typename T>
    VersionType version(VersionType version, AcceptDesiredVersion, const T* caller = NULL) {
        version = this->template queryDesiredClassVersion<T>(version, true);
        this->template writeVersion<T>(version, *this, this->stream());
        return version;
    }


    template<typename T>
    void atomic(T& member)
    {
        Base::atomic(member);
        // write out the value in binary format
        this->stream() << member;
    }

    typename Format::ReflectState preReflect(const char* context = "")
    {
        ++mLevel;
        return Format::insertVersionPlaceholder(context, *this, this->stream());
    }

    void postReflect(const typename Format::ReflectState& prev)
    {
        Format::restoreVersionPtr(prev);
        if (--mLevel == 0)
            this->flushBuffer();
    }


    /*
     * Pointers are stored with a prepended 1-byte marker that
     * indicates the type of the pointer:
     *    1 = PointerReference
     * It is followed immediately by the object id (int) to the
     * referenced object.
     */
    void pointerReference(int referencedObjectID) {
        this->stream() << (uint8)POINTER_REFERENCE;
        this->stream() << referencedObjectID;
    }

    /*
     * Pointers are stored with a prepended 1-byte marker that
     * indicates the type of the pointer:
     *    2 = NormalPointer
     * It is followed by the serialized object.
     */
    void pointerWithoutClassType() {
        this->stream() << (uint8)NORMAL_POINTER;
    }

    /*
     * Pointers are stored with a prepended 1-byte marker that
     * indicates the type of the pointer:
     *    3 = PolymorphicPointer
     * It is followed immediately by the string of the class
     * type of the object and afterwards the serialized object
     * follows.
     */
    void pointerWithClassType(const std::string& type) {
        this->stream() << (uint8)POLYMORPHIC_POINTER;
        this->stream() << type;
    }

    /*
     * Pointers are stored with a prepended 1-byte marker that
     * indicates the type of the pointer:
     *    0 = NullPointer
     */
    void pointerNull() {
        this->stream() << (uint8)NULL_POINTER;
    }

    template<typename T>
    void invokeOverwrite(T& object)
    {
        static const std::string context = "invokeOverwrite " + typeName<T>();
        typename Format::ReflectState prevState = preReflect(context.c_str());

        Base::invokeOverwrite(object);

        postReflect(prevState);
    }

    template<typename T>
    void invokeOverwrite(serialization::PlainArray<T>& array)
    {
        static const std::string context = "invokeOverwrite PlainArray<" + typeName<T>() + ">";
        typename Format::ReflectState prevState = preReflect(context.c_str());

        if(this->template isTrackingEnabled<T>() || !IsBitwiseSerializable<T>::value) {
            // if tracking is enabled or the array's elements cannot be
            // serialized bitwise we cannot even think about using optimized
            // array serialization
            Base::invokeOverwrite(array);
        } else {
            // otherwise, USE the optimized variant and dump the whole array
            // into the stream as one block.
            const char* buffer = reinterpret_cast<const char*>(array.data);
            this->stream().write(buffer, array.getSizeInBytes());
        }

        postReflect(prevState);
    }

public:

    template <typename T>
    void write(const T* data, std::size_t count) {
        this->stream().write(reinterpret_cast<const char*>(data),count*sizeof(T));
        if (mLevel == 0)
            this->flushBuffer();
    }

public:
    // Codec support


    void registerCodec(BinarySerializerCodecPtr codec) {
        if(!codec)
            return;

        TypeId type = codec->getSupportedTypeId();
        auto p = mCodecs.insert(std::make_pair(type, codec));
        if(!p.second)
            MIRA_THROW(XLogical, "A codec for the same type exists already");
    }

    void unregisterCodec(BinarySerializerCodecPtr codec) {
        if(!codec)
            return;

        TypeId type = codec->getSupportedTypeId();
        auto it = mCodecs.find(type);
        if(it==mCodecs.end())
            return; // not registered

        if(it->second!=codec)
            return; // not the same codec

        mCodecs.erase(it);
    }

    template <typename T>
    bool hasCodec() const {
        return mCodecs.count(typeId<T>())!=0;
    }

    template <typename T>
    bool codec(const T& obj)
    {
        BinarySerializerCodecPtr codec;
        BinarySerializerCodec::Fourcc s = BinarySerializerCodec::Fourcc::null();

        auto it = mCodecs.find(typeId<T>());
        if(it!=mCodecs.end()) {
            // found codec
            codec = it->second;
            s = codec->getFourcc();
        }

        if(!codec) {
            write(s.fourcc, sizeof(s.fourcc));
            return false;
        }

        try {
            // encode the data using the codec
            const Buffer<uint8>& encodedData = codec->encode(obj);

            // write the signature of the codec not before successful encoding
            write(s.fourcc, sizeof(s.fourcc));

            // write the encoded data immediately
            this->stream() << encodedData;

        } catch(...) {

            // was not able to encode data, so serialize it normally
            s = BinarySerializerCodec::Fourcc::null();
            write(s.fourcc, sizeof(s.fourcc));
            return false;
        }

        // done
        return true;
    }

private:
    std::map<TypeId, BinarySerializerCodecPtr> mCodecs;

    uint32 mLevel;
};

extern template class ConcreteBinarySerializer<BinaryBufferOstream, 2>;
using BinaryBufferSerializer = ConcreteBinarySerializer<BinaryBufferOstream, 2>;
extern template class ConcreteBinarySerializer<BinaryBufferOstream, 0>;
using BinaryBufferSerializerLegacy = ConcreteBinarySerializer<BinaryBufferOstream, 0>;

extern template class ConcreteBinarySerializer<BinaryStlOstream, 2>;
using BinaryStreamSerializer = ConcreteBinarySerializer<BinaryStlOstream, 2>;
extern template class ConcreteBinarySerializer<BinaryStlOstream, 2, true>;
using BufferedBinaryStreamSerializer = ConcreteBinarySerializer<BinaryStlOstream, 2, true>;
extern template class ConcreteBinarySerializer<BinaryStlOstream, 0>;
using BinaryStreamSerializerLegacy = ConcreteBinarySerializer<BinaryStlOstream, 0>;


template <typename Derived>
class BinaryDeserializer : public Deserializer<Derived>
{
public:
    typedef serialization::VersionType VersionType;
    typedef serialization::AcceptDesiredVersion AcceptDesiredVersion;

    template <typename T>
    VersionType version(VersionType version, const T* caller = NULL) {
        return this->This()->template version<T>(version);
    }

    MIRA_DEPRECATED("Please call as version<MyType>(v) or version(v, this)",
    VersionType version(VersionType version)) {
        return this->template version<void>(version);
    }

    template <typename T>
    VersionType version(VersionType version, AcceptDesiredVersion, const T* caller = NULL)
    {
        return this->This()->template version<T>(version, AcceptDesiredVersion());
    }

public:
    template <typename T>
    void read(T* data, std::size_t count) {
        this->This()->read(data,count);
    }

public:
    template <typename T>
    bool hasCodec() const {
        return this->This()->template hasCodec<T>();
    }

    template <typename T>
    bool codec(T& obj)
    {
        return this->This()->template codec(obj);
    }
};


template <typename BinaryStream, uint8 BinaryFormatVersion>
class ConcreteBinaryDeserializer;

template <typename BinaryStream, uint8 BinaryVersionFormat>
class DeserializerFormatMixin
{
};

template <typename Deserializer, typename BinaryStream>
class DeserializerFormatMixin01Base
{
protected:
    typedef boost::mpl::bool_<false>              requireReflectBarriers;
    typedef void*                                 ReflectState;

    typedef int VersionType; // format version 0 had int32 serialization versions

    ReflectState readVersion(const char* context, Deserializer& deserializer) {
        return NULL;
    }

    template <typename T>
    VersionType getVersion(Deserializer& deserializer) {
        VersionType v;
        deserializer.atomic(v);
        return v;
    }

    void restoreVersion(const ReflectState& prev) {}

    void reportVersionChecked(std::ostream& os) {}
};

template <typename BinaryStream>
class DeserializerFormatMixin<BinaryStream, 0> :
    public DeserializerFormatMixin01Base<ConcreteBinaryDeserializer<BinaryStream, 0>, BinaryStream>
{
public:
    static uint8 getSerializerFormatVersion() { return 0; }

protected:
    template <typename T>
    bool checkFormatVersion(T& value, bool enableTypeCheck, BinaryStream& stream) { return true; }
};

template <typename BinaryStream>
class DeserializerFormatMixin<BinaryStream, 1> :
    public DeserializerFormatMixin01Base<ConcreteBinaryDeserializer<BinaryStream, 1>, BinaryStream>
{
public:
    static uint8 getSerializerFormatVersion() { return 1; }

protected:
    template <typename T>
    bool checkFormatVersion(T& value, bool enableTypeCheck, BinaryStream& stream)
    {
        uint16 m;
        stream >> m;

        if (stream.fail())
            MIRA_THROW(XIO, "Failed to read binary format version number from binary stream: "
                            "Reading from stream failed, e.g. empty or non-existent file.");

        if (m != BINARY_VERSION_MARKER)
            MIRA_THROW(XIO, "Failed to read binary format version number from binary stream: "
                            "Expected marker not found, data invalid for binary format version 1.");

        uint8 v;
        stream >> v;

        if (v != getSerializerFormatVersion())
            MIRA_THROW(XIO, "BinaryDeserializer: Unhandled binary format, expected version "
                             << (int)getSerializerFormatVersion() << ", found " << (int)v << ".");

        return true;
    }
};

template <typename BinaryStream>
class DeserializerFormatMixin<BinaryStream, 2>
{
public:
    static uint8 getSerializerFormatVersion() { return 2; }

    static uint8 getDataFormatVersion(BinaryStream& stream)
    {
        uint16 m;

        typename BinaryStream::pos_type pos = stream.tellg();

        stream >> m;

        if (m == BINARY_VERSION_MARKER) {
            uint8 v;
            stream >> v;
            return v;
        } else {
            if (stream.fail())
                MIRA_THROW(XIO, "Failed to read binary format version number from binary stream: "
                                "Reading from stream failed, e.g. empty or non-existent file.");
            // v0 did not store a version in binary data, so if we don't find it, it is v0
            if (pos == typename BinaryStream::pos_type(-1))
                MIRA_THROW(XIO, "Requiring fallback to legacy deserializer, but failing to query (and set) stream position. "
                                "The stream type does not seem to support repositioning. "
                                "You may try using a legacy deserializer directly, avoiding the need for setting the stream read position.");
            stream.seekg(pos);
            return 0;
        }
    }

protected:
    typedef ConcreteBinaryDeserializer<BinaryStream, 2> Deserializer;

    // Needs to know about independent reflection blocks to read
    // a version number from the input for each of them.
    typedef boost::mpl::bool_<true> requireReflectBarriers;

    // State consists of a version number and a flag to know if it
    // was queried by the reflected object, plus context
    // provided to preReflect() (indicating the reflect() method
    // we are in, typically the typename of the reflected class).
    typedef std::tuple<serialization::VersionType, bool, const char*, bool> ReflectState;

    typedef serialization::VersionType VersionType;

    DeserializerFormatMixin() : mVersion(0), mContext(NULL),
                                mUncheckedVersion(0), mUncheckedContext(NULL) {}

    template <typename T>
    bool checkFormatVersion(T& value, bool enableTypeCheck, BinaryStream& stream)
    {
        uint8 v;
#ifdef CHECK_FORCE_SERIALIZE_BINARY_VERSION
        int vf = Deserializer::forcedDeserializeVersion();
        if (vf >= 0) {
            MIRA_LOG(NOTICE) << "BinaryDeserializer: Forced deserializing as binary "
                             << "format version " << vf << ".";
            v = vf;
            if (v >= 1)                        // consume version data bytes from the stream
                getDataFormatVersion(stream);  // (and ignore them)
        }
        else
#endif
            v = getDataFormatVersion(stream);

#ifndef NDEBUG
        if (v < getSerializerFormatVersion()) {
            // give a NOTICE in debug builds, else nothing (as it quickly turns into major spam,
            // and there is no efficient way to make sure it is displayed only once)
            MIRA_LOG(NOTICE) << "BinaryDeserializer: Found outdated binary format, version "
                             << (int)v << ". Using legacy deserializer.";
        }
#endif

        if (v == 0) {
            ConcreteBinaryDeserializer<BinaryStream, 0> v0(stream, stream.tellg());
            v0.deserialize(value, enableTypeCheck, true);
            stream.seekg(v0.streamPosition());
            return false;
        }

        if (v == 1) {
            ConcreteBinaryDeserializer<BinaryStream, 1> v1(stream, stream.tellg());
            v1.deserialize(value, enableTypeCheck, true);
            stream.seekg(v1.streamPosition());
            return false;
        }

        if (v != getSerializerFormatVersion()) {
            MIRA_THROW(XIO, "BinaryDeserializer: Unhandled binary format, expected version "
                             << (int)getSerializerFormatVersion() << ", found " << (int)v << ".");
        }

        return true;
    }

    ReflectState readVersion(const char* context, Deserializer& deserializer) {
        ReflectState prevState;
        if ((mVersion > 0) && !mVersionChecked && (mUncheckedVersion == 0)) {
            mUncheckedVersion = mVersion;
            mUncheckedContext = mContext;
            prevState = std::make_tuple(mVersion, mVersionChecked, mContext, true);
        } else
            prevState = std::make_tuple(mVersion, mVersionChecked, mContext, false);
        deserializer.atomic(mVersion);
        mVersionChecked = false;
        mContext = context;
        return prevState;
    }

    template <typename T>
    VersionType getVersion(Deserializer& deserializer) {
        // we could check for double query here, but rather not worth the extra work
        // if (mVersionChecked) {
        //  MIRA_LOG(WARNING) << "BinaryDeserializer: version() called twice for type '"
        //                    << typeName<T>() << "'. Context = '" << (mContext ? mContext : "") << "'.";
        // }
        mVersionChecked = true;
        return mVersion;
    }

    void restoreVersion(const ReflectState& prev) {
        if ((mVersion > 0) && !mVersionChecked)
            MIRA_THROW(XIO, "Tried to deserialize versioned binary data (version " << (int)mVersion <<
                            ") into unversioned type. Context = '" << (mContext ? mContext : "") << "'.");
        mVersion = std::get<0>(prev);
        mVersionChecked = std::get<1>(prev);
        mContext = std::get<2>(prev);
        if (std::get<3>(prev)) {
            mUncheckedVersion = 0;
        }
    }

    void reportVersionChecked(std::ostream& os) {
        if (mUncheckedVersion > 0) {
            os << " While trying to deserialize versioned binary data (version " << (int)mUncheckedVersion <<
                  ") into type that is unversioned or not checking version before critical read. " <<
                  "Context = '" << (mUncheckedContext ? mUncheckedContext : "") << "'.";
        } else if ((mVersion > 0) && !mVersionChecked) {
            os << " While trying to deserialize versioned binary data (version " << (int)mVersion <<
                  ") into type that is unversioned or not checking version before critical read. " <<
                  "Context = '" << (mContext ? mContext : "") << "'.";
        }
    }

private:
    VersionType mVersion;
    bool mVersionChecked; // make sure the receiver of versioned data actually checks the version
    const char* mContext;

    VersionType mUncheckedVersion; // if there is some unchecked version, keep it in mind
    const char* mUncheckedContext; // to report when an error occurs (possibly somewhere "further down")
};


template <typename BinaryStream, uint8 BinaryFormatVersion>
class ConcreteBinaryDeserializer : public BinaryDeserializer<ConcreteBinaryDeserializer<BinaryStream,
                                                                                        BinaryFormatVersion> >,
                                   public BinarySerializerMixin,
                                   public DeserializerFormatMixin<BinaryStream, BinaryFormatVersion>
{
    typedef BinaryDeserializer<ConcreteBinaryDeserializer<BinaryStream, BinaryFormatVersion> >     Base;
    typedef DeserializerFormatMixin<BinaryStream, BinaryFormatVersion>                             Format;
public:

    // We are not using human readable ids here. This will turn off the
    // whole id-handling system of the serialization/reflection framework
    // and dramatically improve the performance
    typedef boost::mpl::bool_<false> useHumanReadableIDs;

    typedef std::map<TypeId, BinarySerializerCodecPtr> CodecsMap;

    typedef typename Format::requireReflectBarriers requireReflectBarriers;
    typedef typename Format::ReflectState           ReflectState;

public:

    ConcreteBinaryDeserializer(typename BinaryStream::streambuffer_pointer buffer)
        : mStream(buffer) {}

    ConcreteBinaryDeserializer(BinaryStream& stream) : mStream(stream.rdbuf()) {}

    ConcreteBinaryDeserializer(BinaryStream& stream, typename BinaryStream::pos_type pos) :
        mStream(stream, pos) {}

public:

    void reassign(typename BinaryStream::streambuffer_pointer buffer) {
        mStream = buffer;
    }

    typename BinaryStream::pos_type streamPosition() { return mStream.tellg(); }

public:

    template <typename T>
    void deserialize(T& value, bool enableTypeCheck = true, bool recursive = false)
    {
        if (!recursive && !Format::checkFormatVersion(value, enableTypeCheck, mStream))
            return;

        if(enableTypeCheck) {
            std::string fulltypename;
            try {
                Base::deserialize("", fulltypename);
            }
            catch (...) {
                MIRA_THROW(XBadCast, "Failed reading type from binary data while " <<
                                     "deserializing into a variable of type '" << typeName<T>() << "'");
            }

            if(fulltypename != typeName<T>())
                MIRA_THROW(XBadCast, "Cannot deserialize the type '" << fulltypename <<
                                     "' into a variable of type '" << typeName<T>() << "'");
        }
        Base::deserialize("", value);
    }

public:
    typedef typename Format::VersionType VersionType;

    template <typename T>
    VersionType version(VersionType expectedVersion, const T* object = NULL) {
        VersionType version = this->template getVersion<T>(*this);
        if (version > expectedVersion)
            MIRA_THROW(XIO, "Trying to deserialize binary data of a newer version (" << (int)version <<
                ") of type " << typeName<T>() << " into an older version (" << (int)expectedVersion << ").");
        return version;
    }

    MIRA_DEPRECATED("Please call as version<MyType>(v) or version(v, this)",
    VersionType version(VersionType expectedVersion)) {
        return version<void>(expectedVersion);
    }

    typedef typename Base::AcceptDesiredVersion AcceptDesiredVersion;

    template <typename T>
    VersionType version(VersionType expectedVersion, AcceptDesiredVersion, const T* object = NULL) {
        return this->template getVersion<T>(*this);
    }

    template<typename T>
    void atomic(T& member)
    {
        Base::atomic(member);
        // read in the value from binary stream
        mStream >> member;
        if(mStream.fail()) {
            std::stringstream ss;
            ss << "Failed to read member of type '" << typeName<T>()
               << "' from binary stream at stream position " << streamPosition() << ".";

            Format::reportVersionChecked(ss);

            MIRA_THROW(XIO, ss.str());
        }
    }

    typename Format::ReflectState preReflect(const char* context = "")
    {
        return Format::readVersion(context, *this);
    }

    void postReflect(const typename Format::ReflectState& prev)
    {
        Format::restoreVersion(prev);
    }

    template<typename T>
    void pointer(T* &pointer)
    {
        // we always store a marker before each pointer
        // in Serializer::pointerReference / pointerNoReference
        uint8 pointerType8U;
        mStream >> pointerType8U;

        PointerType pointerType = (PointerType) pointerType8U;

        switch(pointerType)
        {
            case NULL_POINTER:
                pointer=NULL;
                break;

            case POINTER_REFERENCE:
            {
                int ref;
                mStream >> ref; // read the ref id
                // ... and resolve it
                pointer = this->template resolveReference<T>(ref);
                break;
            }

            case NORMAL_POINTER:
                // we have a "normal" pointer, so deserialize it
                mClassType.clear(); // we have no class type information here
                Base::pointer(pointer);
                break;

            case POLYMORPHIC_POINTER:
                // we have a "polymorphic" pointer, so get class type
                mStream >> mClassType;
                // and deserialize it
                Base::pointer(pointer);
                break;
        }
    }

    std::string pointerClassType() {
        return mClassType;
    }


    template<typename T>
    void invokeOverwrite(T& object)
    {
        // an object starts here - we are about to call a reflect() method
        static const std::string context = "invokeOverwrite " + typeName<T>();
        typename Format::ReflectState prevState = preReflect(context.c_str());

        Base::invokeOverwrite(object);

        // we are done with this object
        postReflect(prevState);
    }

    template<typename T>
    void invokeOverwrite(serialization::PlainArray<T>& array)
    {
        // an object starts here - we are about to call a reflect() method
        static const std::string context = "invokeOverwrite PlainArray<" + typeName<T>() +">";
        typename Format::ReflectState prevState = preReflect(context.c_str());

        if(this->template isTrackingEnabled<T>() || !IsBitwiseSerializable<T>::value) {
            // if tracking is enabled or the array's elements cannot be
            // serialized bitwise we cannot even think about using optimized
            // array deserialization
            Base::invokeOverwrite(array);
        } else {

            // otherwise, USE the optimized variant and read the whole array
            // from the stream as one block.
            char* buffer = reinterpret_cast<char*>(array.data);
            mStream.read(buffer, array.getSizeInBytes());
        }

        // we are done with this object
        postReflect(prevState);
    }

public:

    template <typename T>
    void read(T* data, std::size_t count) {
        mStream.read(reinterpret_cast<char*>(data),count*sizeof(T));
    }

public:
    // codec support

    template <typename T>
    bool hasCodec() const {
        //return mCodecs.count(typeId<T>())!=0;
        return false;
    }

    template <typename T>
    bool codec(T& obj)
    {
        BinarySerializerCodecPtr codec;
        BinarySerializerCodec::Fourcc s;

        // read the codec fourcc
        read(s.fourcc, sizeof(s.fourcc));

        if(s==BinarySerializerCodec::Fourcc::null())
            return false; // no codec was used by serializer

        // try to find appropriate codec within our created codecs
        auto it = mCodecs.find(typeId<T>());
        if(it==mCodecs.end() || it->second->getFourcc()!=s) {
            // codec not found, so we must create one
            codec = BinarySerializerCodec::createCodec<T>(s);
            assert(codec);
            mCodecs[typeId<T>()] = codec;
        } else
            codec = it->second; // use existing codec

        assert(codec->getFourcc()==s);
        assert(codec->getSupportedTypeId()==typeId<T>());

        // read the encoded data
        Buffer<uint8> encodedData;
        mStream >> encodedData;

        // and decode
        codec->decode(encodedData, obj);

        // done
        return true;
    }

    const CodecsMap& getCodecs() const {
        return mCodecs;
    }

    void setCodecs(const CodecsMap& codecs) {
        mCodecs = codecs;
    }

private:

    BinaryStream mStream;
    std::string  mClassType;

    CodecsMap mCodecs;
};

extern template class ConcreteBinaryDeserializer<BinaryBufferIstream, 2>;
using BinaryBufferDeserializer = ConcreteBinaryDeserializer<BinaryBufferIstream, 2>;
extern template class ConcreteBinaryDeserializer<BinaryBufferIstream, 1>;
using BinaryBufferDeserializerLegacyMarked = ConcreteBinaryDeserializer<BinaryBufferIstream, 1>;
extern template class ConcreteBinaryDeserializer<BinaryBufferIstream, 0>;
using BinaryBufferDeserializerLegacy = ConcreteBinaryDeserializer<BinaryBufferIstream, 0>;

extern template class ConcreteBinaryDeserializer<BinaryStlIstream, 2>;
using BinaryStreamDeserializer = ConcreteBinaryDeserializer<BinaryStlIstream, 2>;
extern template class ConcreteBinaryDeserializer<BinaryStlIstream, 1>;
using BinaryStreamDeserializerLegacyMarked = ConcreteBinaryDeserializer<BinaryStlIstream, 1>;
extern template class ConcreteBinaryDeserializer<BinaryStlIstream, 0>;
using BinaryStreamDeserializerLegacy = ConcreteBinaryDeserializer<BinaryStlIstream, 0>;


} // namespace

#endif