XMLSerializer.h

Go to the documentation of this file.

/*
 * Copyright (C) 2012 by
 *   MetraLabs GmbH (MLAB), GERMANY
 * and
 *   Neuroinformatics and Cognitive Robotics Labs (NICR) at TU Ilmenau, GERMANY
 * All rights reserved.
 *
 * Contact: info@mira-project.org
 *
 * Commercial Usage:
 *   Licensees holding valid commercial licenses may use this file in
 *   accordance with the commercial license agreement provided with the
 *   software or, alternatively, in accordance with the terms contained in
 *   a written agreement between you and MLAB or NICR.
 *
 * GNU General Public License Usage:
 *   Alternatively, this file may be used under the terms of the GNU
 *   General Public License version 3.0 as published by the Free Software
 *   Foundation and appearing in the file LICENSE.GPL3 included in the
 *   packaging of this file. Please review the following information to
 *   ensure the GNU General Public License version 3.0 requirements will be
 *   met: http://www.gnu.org/copyleft/gpl.html.
 *   Alternatively you may (at your option) use any later version of the GNU
 *   General Public License if such license has been publicly approved by
 *   MLAB and NICR (or its successors, if any).
 *
 * IN NO EVENT SHALL "MLAB" OR "NICR" BE LIABLE TO ANY PARTY FOR DIRECT,
 * INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF
 * THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF "MLAB" OR
 * "NICR" HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * "MLAB" AND "NICR" SPECIFICALLY DISCLAIM ANY WARRANTIES, INCLUDING,
 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
 * FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
 * ON AN "AS IS" BASIS, AND "MLAB" AND "NICR" HAVE NO OBLIGATION TO
 * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS OR MODIFICATIONS.
 */

#ifndef _MIRA_XMLSERIALIZER_H_
#define _MIRA_XMLSERIALIZER_H_

#include <iostream>
#include <list>
#include <algorithm>
#include <vector>

#ifndef Q_MOC_RUN 
#include <boost/algorithm/string.hpp>
#include <boost/range/algorithm.hpp>
#include <boost/range/algorithm_ext/erase.hpp>
#endif

#include <serialization/Serializer.h>
#include <serialization/Deserializer.h>
#include <serialization/IsPointerOrSharedPointer.h>

#include <xml/XMLDom.h>

namespace mira {


namespace Private {

inline std::string replaceSpaces(const std::string& s)
{
    return boost::replace_all_copy(s, " ", "_");
}

inline std::string replaceSpecialChars(const std::string& s)
{
    std::string result = boost::replace_all_copy(s, " ", "_");
    boost::remove_erase_if(result, boost::is_any_of("[]"));
    return result;
}

} // namespace


struct XMLSerializerTag {};

class XMLSerializer : public Serializer<XMLSerializer>
{
    typedef Serializer<XMLSerializer> Base;

public:
    typedef XMLSerializerTag Tag;

    enum OutputFormat
    {
        STANDARD      = 0x00, 
        NO_COMMENTS   = 0x01, 
        COMPRESSED    = 0x02, 
    };

    MIRA_ENUM_TO_FLAGS_INCLASS(OutputFormat) // generate logical operators for enum values

    
    XMLSerializer(XMLDom& iXmlDom, OutputFormat format = STANDARD) :
        mRoot(iXmlDom.root()), mNode(iXmlDom.root()), mOutputFormat(format)
    {
    }

    XMLSerializer(XMLDom::iterator& iRoot, OutputFormat format = STANDARD) :
        mRoot(iRoot), mNode(iRoot), mOutputFormat(format)
    {
    }

    typedef Base::VersionType VersionType;

    template <typename T>
    VersionType version(VersionType version, const T* caller = NULL)
    {
        return this->version<T>(version, false);
    }

    MIRA_DEPRECATED("Please call as version<MyType>(v) or version(v, this)",
    VersionType version(VersionType version))
    {
        // add version - backward compatible variant if no caller type provided
        return versionLegacy(version);
    }

    template <typename T>
    VersionType version(VersionType version, AcceptDesiredVersion, const T* caller = NULL)
    {
        return this->version<T>(version, true);
    }

private:

    template <typename T>
    VersionType version(VersionType version, bool acceptDesiredVersion)
    {
        version = this->template queryDesiredClassVersion<T>(version, acceptDesiredVersion);

        int vf = this->forcedSerializeVersion();
        if ((vf == 0) || (vf == 1))
            return versionLegacy(version);

        // add version
        mNode.add_child("version").add_attribute("type", typeName<T>())
                                  .add_content(toString((int)version));
        return version;
    }

    VersionType versionLegacy(VersionType version)
    {
        mNode.add_attribute("version", toString((int)version) );
        return version;
    }

public:

    template<typename T>
    void atomic(T& member)
    {
        const ReflectMemberMeta& meta = getCurrentMemberMeta();
        if (mOutputFormat & COMPRESSED)
            mNode.add_attribute(Private::replaceSpecialChars(meta.id), toString(member,12));
        else
        {
            if(!(mOutputFormat & NO_COMMENTS) && meta.comment!=NULL && strlen(meta.comment)>0)
                mNode.add_comment(meta.comment);
            mNode.add_child(Private::replaceSpaces(meta.name)).add_content(toString(member,12));
        }
    }

    template<typename T>
    void object(T& member)
    {
        const ReflectMemberMeta& meta = getCurrentMemberMeta();

        // create a new node
        if(!(mOutputFormat & NO_COMMENTS) && meta.comment!=NULL && strlen(meta.comment)>0)
            mNode.add_comment(meta.comment);
        XMLDom::iterator i = mNode.add_child(Private::replaceSpaces(meta.name));

        // enter the new node
        mNode = i;

        if(!mPointerClass.empty()) {
            mNode.add_attribute("class", mPointerClass);
            mPointerClass.clear();
        }

        Base::object(member);

        // leave the node
        mNode = mNode.parent();
        assert(mNode!=mRoot.end());

    }

    // Stores a reference to a previously serialized object using the "ref" attribute.
    void pointerReference(int referencedObjectID) {
        const char* name = getCurrentMemberMeta().name;
        mNode.add_child(Private::replaceSpaces(name)).add_attribute("ref", this->getHumanReadableFullID(referencedObjectID));
    }

    // Stores the class type the "class" attribute.
    void pointerWithClassType(const std::string& type) {
        mPointerClass = type;
        //mNode.add_attribute("class", type);
    }

    // Stores a NULL pointer using the "nullptr" attribute.
    void pointerNull() {
        const char* name = getCurrentMemberMeta().name;
        mNode.add_child(Private::replaceSpaces(name)).add_attribute("nullptr", "true");
    }

public:

    XMLDom::iterator getNode() { return mNode; }

private:

    XMLDom::iterator mRoot;

    // iterator to the current node
    XMLDom::iterator mNode;

    std::string mPointerClass;

    OutputFormat mOutputFormat;

};


class XMLDeserializer : public Deserializer<XMLDeserializer>
{
    typedef Deserializer<XMLDeserializer> Base;
public:

    XMLDeserializer(const XMLDom& iXmlDom) :
        mRoot(iXmlDom.root()), mNode(iXmlDom.root())
    {
    }

    XMLDeserializer(const XMLDom::const_iterator& iRoot) :
        mRoot(iRoot), mNode(iRoot)
    {
    }


    template <typename T>
    void deserialize(const std::string& name, T& value)
    {
        mDeserializeFromThisNode=false;
        mNextFindIteratorStack.clear();
        pushFindStack();
        // since we deserialize just an element of root we must not add all children from root
        mUnprocessedChildren.emplace();
        try {
            Base::deserialize(name,value);
        } catch (Exception& ex) {
            MIRA_RETHROW(ex, "in tag starting at: " <<  mNode.uri() << "(" << mNode.line() << ")");
        }
        popFindStack();
        popUnprocessedChildren();
    }

    template <typename T>
    void deserializeFromNode(const char* name, T& value)
    {
        mDeserializeFromThisNode=true;
        mNextFindIteratorStack.clear();
        pushFindStack();
        pushUnprocessedChildren(mNode);
        try {
            Base::deserialize(name,value);
        } catch (Exception& ex) {
            MIRA_RETHROW(ex, "in tag starting at: " <<  mNode.uri()
                        << "(" << mNode.line() << ")");
        }
        popFindStack();
        popUnprocessedChildren();
    }

    typedef serialization::VersionType VersionType;

    template <typename T>
    VersionType version(VersionType expectedVersion, const T* caller = NULL)
    {
        // obtain version

        int vf = this->forcedDeserializeVersion();
        if ((vf == 0) || (vf == 1))
            return versionLegacy(expectedVersion);

        std::string type = typeName<T>();
        for (int n = 0; true; ++n) {
            XMLDom::const_sibling_iterator versionNode = mNode.find("version", n);
            if (versionNode == mNode.cend()) {
                if (mNode.has_attribute("version")) {
                    MIRA_LOG(WARNING) << "Failed finding version for type '" << type << "', "
                                         "but read an anonymous version attribute instead. "
                                         "Please update your config xml ("
                                         << mNode.uri() << " , line " << mNode.line() << ").";

                    VersionType version = mNode.get_attribute<int>("version");
                    checkVersion<T>(version, expectedVersion);
                    return version;
                }
                return 0;
            }

            markChildAsProcessed(versionNode);

            // 'version' elements without 'type' attribute are ignored here, allowing
            // to still use 'version' as part of the actual data
            if (versionNode.get_attribute<std::string>("type", "") == type) {
                VersionType version = fromString<int>(*versionNode.content_begin());
                checkVersion<T>(version, expectedVersion);
                return version;
            }
        }
    }

    MIRA_DEPRECATED("Please call as version<MyType>(v) or version(v, this)",
    VersionType version(VersionType expectedVersion))
    {
        // obtain version - backward compatible variant if no caller type provided
        return versionLegacy(expectedVersion);
    }

    template <typename T>
    VersionType version(VersionType expectedVersion, AcceptDesiredVersion, const T* caller = NULL)
    {
        return this->version<T>(expectedVersion);
    }

private:

    VersionType versionLegacy(VersionType expectedVersion)
    {
        VersionType version = mNode.get_attribute<int>("version", 0);
        checkVersion<void>(version, expectedVersion);
        return version;
    }

    template <typename T>
    void checkVersion(VersionType version, VersionType expectedVersion)
    {
        if (version > expectedVersion) {
            MIRA_LOG(WARNING) << "Trying to deserialize XML data of a newer version (" << (int)version <<
                ") of type " << typeName<T>() << " into an older version (" << (int)expectedVersion << ").";
        }
    }

public:

    template<typename T>
    void atomic(T& member)
    {
        Base::atomic(member);

        if(mAtomicFromAttribute.empty())
            member = fromString<T>(*mNode.content_begin());
        else // try to get the value from the attribute
             // (which should exists, since this is checked in invokeMemberOverwrite)
            member = mNode.get_attribute<T>(mAtomicFromAttribute); //uses fromString<T> internally
    }

    void beginTag(const ReflectMemberMeta& meta)
    {
        if(meta.name!=NULL) {
            if(!mAtomicFromAttribute.empty()) {
                MIRA_THROW(XIO, "Cannot deserialize '" << mAtomicFromAttribute <<
                                "' from an attribute as it is a complex type, that "
                                "must be stored as an XML node.");
            }

            XMLDom::const_iterator i = findNext(Private::replaceSpaces(meta.name));
            if(i!=mNode.end()) {
                // found it, so set it as current node
                mNode = i;
            } else {
                // node not found
                MIRA_THROW(XMemberNotFound, "Node '"
                          << Private::replaceSpaces(meta.name)
                          << "' is missing"); // hence we have to abort with an exception
            }
            pushFindStack();
            pushUnprocessedChildren(mNode);
        }
    }

    void endTag(const ReflectMemberMeta& meta)
    {
        if(meta.name!=NULL) {
            assert(mNode!=mRoot); // we will never go above the root node.
            popFindStack();
            popUnprocessedChildren();
            markChildAsProcessed(mNode);
            mNode = mNode.parent();
            assert(mNode!=mRoot.end());
        }
    }

    /*
     * Overwrites Deserializer::pointer.
     * This method checks, if there is a pointer reference specified
     * (the ref attribute is present). If so, it reads the referenced full id
     * of the previously stored object and uses resolveReference() to get the
     * pointer to that previously deserialized object.
     * Otherwise it calls pointer() of the Deserializer base class to
     * deserialize the full object.
     */
    template<typename T>
    void pointer(T* &pointer)
    {
        // do we have a null pointer?
        if(fromString<bool>(mNode.get_attribute<std::string>("nullptr", "false"))) {
            pointer = NULL;
            return;
        }

        // do we have a reference?
        std::string ref = mNode.get_attribute<std::string>("ref","");
        if(ref.size()>0) {
            // we have a reference, so resolve it
            pointer = resolveReference<T>(ref);
            return;
        }

        // we have a "normal" pointer, so deserialize it
        Base::pointer(pointer);
    }

    std::string pointerClassType()
    {
        return mNode.get_attribute<std::string>("class","");
    }

    // hijack the invokeOverwrite method for adding a starting and closing tag
    // around the serialized data
    template<typename T>
    void invokeMemberOverwrite(T& member, const ReflectMemberMeta& meta)
    {
        bool fromThisNode = mDeserializeFromThisNode;
        bool fromAttribute = false;

        if(!fromThisNode) {
            try {
                beginTag(meta);
            } catch(XMemberNotFound& ex) {
                // member of value not found, if it is an atomic type
                // we can still try to get the value from an attribute with
                // the same name
                // (however, the following reserved attributes are
                //  not allowed: version, nullptr, ref, class)
                std::string id = Private::replaceSpecialChars(meta.id);
                try {
                    if(id!="version" &&
                       id!="nullptr" &&
                       id!="ref" &&
                       id!="class")
                    {
                        // check if the attribute exists
                        if(!mNode.has_attribute(id)) {
                            MIRA_THROW(XMemberNotFound, "Node (or attribute) '" << id <<
                                "' is missing");
                        }
                        mAtomicFromAttribute = id;
                        fromAttribute = true;
                    } else {
                        // otherwise rethrow the exception since we cannot do anything
                        throw;
                    }
                }
                catch(Exception& ex) {
                    MIRA_RETHROW(ex, "in tag starting at: " <<  mNode.uri() << "(" << mNode.line() << ")");
                }

            }
        }

        mDeserializeFromThisNode=false;

        try {
            Base::invokeMemberOverwrite(member, meta);
        } catch(...) {
            // make sure our find stack is correctly unwound and the correct
            // node in the XML file is maintained if an exception
            // is thrown somewhere in our child nodes (fixes #536)
            mAtomicFromAttribute.clear();
            if(!fromThisNode && !fromAttribute) {
                endTag(meta); // finish this tag and ...
                rewindFindIterator(meta.getName()); // ... 'undo' read so we can re-try in a handler and ...
            }
            throw; // ... rethrow
        }

        // hm, this cleanup code is a duplicate of the above cleanup
        // code in the event of an exception
        mAtomicFromAttribute.clear();
        if(!fromThisNode && !fromAttribute)
            endTag(meta);
    }

public: // accessors for special serialize/deserialize methods below

    XMLDom::const_iterator getNode() { return mNode; }

private:

    XMLDom::const_iterator mRoot;

    XMLDom::const_iterator mNode;

    bool mDeserializeFromThisNode;

    std::string mAtomicFromAttribute;

private:
    // Necessary to allow multiple nodes with the same name.
    // Here we store the iterators of previous searches for
    // a certain nodes, the next search will then continue
    // after the previous found iterator.
    // Those iterators must be stored in a stack for each
    // level of the XML tree.
    typedef std::map<std::string, XMLDom::const_iterator> NextFindIteratorMap;
    std::list<NextFindIteratorMap> mNextFindIteratorStack;

    XMLDom::const_iterator findNext(const std::string& name)
    {
        assert(mNextFindIteratorStack.size()>=1);
        NextFindIteratorMap& m = mNextFindIteratorStack.back();

        XMLDom::const_iterator i = mNode.end();
        XMLDom::const_iterator j = mNode.begin();

        // check if we have searched for the name already
        NextFindIteratorMap::iterator p = m.find(name);
        if(p!=m.end()) {
            // we already searched for name
            // so continue search from successor of previous find
            j = p->second;
            ++j;
        }
        i = find(j, mNode.end(), name);

        // if we found something, save the position for next search
        if(i!=mNode.end())
            m[name] = i;

        return i;
    }

    // is called when we enter a new node
    void pushFindStack()
    {
        mNextFindIteratorStack.push_back(NextFindIteratorMap());
    }

    // is called when we leave a node
    void popFindStack()
    {
        mNextFindIteratorStack.pop_back();
    }

    void rewindFindIterator(const std::string& name)
    {
        assert(mNextFindIteratorStack.size()>=1);
        NextFindIteratorMap& m = mNextFindIteratorStack.back();
        
        NextFindIteratorMap::iterator p = m.find(name);
        if(p==m.end())
            return;

        if (m[name] == mNode.begin())
            m.erase(name);
        else
            --(m[name]);
    }

    std::stack<std::vector<XMLDom::const_iterator>> mUnprocessedChildren;

    void pushUnprocessedChildren(XMLDom::const_iterator parent)
    {
        std::vector<XMLDom::const_iterator> children;
        for (auto it = parent.begin(); it != parent.end(); ++it) {
            children.push_back(it);
        }
        mUnprocessedChildren.push(std::move(children));
    }

    void popUnprocessedChildren()
    {
        auto& unprocessedChildren = mUnprocessedChildren.top();
        for (auto& child : unprocessedChildren) {
            constexpr static auto mayBeIgnoredAttr = "__may_be_ignored__";
            if (child.get_attribute(mayBeIgnoredAttr, false)) {
                continue;
            }
            MIRA_LOG(ERROR) << "XML tag " << *child << " (originating from " << child.uri() << ":" << child.line()
                            << ") is not used during deserialization";
        }
        mUnprocessedChildren.pop();
    }

    void markChildAsProcessed(XMLDom::const_iterator child)
    {
        auto& unprocessedChildren = mUnprocessedChildren.top();
        auto it = std::remove(unprocessedChildren.begin(), unprocessedChildren.end(), child);
        unprocessedChildren.erase(it, unprocessedChildren.end());
    }

public:
    void markAllChildrenAsProcessed()
    {
        auto& unprocessedChildren = mUnprocessedChildren.top();
        unprocessedChildren.clear();
    }
};


// specializations for STL containers

namespace serialization { // our private namespace


template<typename Collection>
struct ReflectCollectionCount<XMLSerializer, Collection>
{
    static void reflect(XMLSerializer& r, uint32& ioCount)
    {
        // do nothing
    }
};

template<typename Collection, bool HasKeyType = true, bool HasMappedType = true>
struct CheckKeyCount
{
    static void check(XMLDom::const_iterator node, uint32 itemCount)
    {
        // nothing to do here if Collection is not a map (default)
    }
};

// if Collection has a definition for key_type and mapped_type (other than void), assume it is a map
// -> expect a <key> tag for each <item> tag
// (key_type alone is not sufficient as it also exists e.g. in std::set)
template<typename Collection>
struct CheckKeyCount<Collection,
                     (bool)!std::is_void_v<typename Collection::key_type>,
                     (bool)!std::is_void_v<typename Collection::mapped_type>>
{
    struct IsKey {
        bool operator()(const XMLDom::Attribute& attr) {
            return boost::algorithm::starts_with(attr.first, "key");
        }
    };

    static void check(XMLDom::const_iterator node, uint32 itemCount)
    {
        uint32 keyCount = (uint32)std::count(node.begin(), node.end(), "key");

        if (keyCount == 0) {
            // there might be attributes instead
            keyCount = (uint32)std::count_if(node.attribute_cbegin(), node.attribute_cend(),
                                             IsKey());
        }
        if (keyCount != itemCount) {
            MIRA_THROW(XIO, "Node for map (or map-like collection) has " << keyCount
                         << " <key> tags and " << itemCount << " <item> tags.");
        }
    }
};

template<typename Collection>
struct ReflectCollectionCount<XMLDeserializer, Collection>
{
    struct IsItem {
        bool operator()(const XMLDom::Attribute& attr) {
            return boost::algorithm::starts_with(attr.first, "item");
        }
    };

    static void reflect(XMLDeserializer& r, uint32& ioCount)
    {
        XMLDom::const_iterator node = r.getNode();

        // count the "item" nodes
        ioCount = (uint32)std::count(node.begin(), node.end(), "item");

        if (ioCount == 0) {
            // there might be attributes instead
            ioCount = (uint32)std::count_if(node.attribute_cbegin(), node.attribute_cend(),
                                            IsItem());
        }

        CheckKeyCount<Collection>::check(node, ioCount);
    }
};

template<typename Reflector, typename Container> struct ReflectReadMapItems;

template<typename Container>
struct ReflectReadMapItems<XMLSerializer, Container>
{
    typedef typename Container::value_type  value_type;
    typedef typename Container::key_type    key_type;

    static void reflect(XMLSerializer& r, Container& c)
    {
        // store each item
        int id=0;
        foreach(value_type& p, c)
        {
            key_type& nonconstkey = const_cast<key_type&>(p.first);
            // no tracking on this one, see  ReflectReadMapPair
            MIRA_MEMBER_WITH_ID(r, "key", "key["+toString(id)+"]", nonconstkey, "",
                                REFLECT_CTRLFLAG_TEMP_TRACKING);

            // the values can be reflected/deserialized with tracking
            MIRA_PROPERTY_WITH_ID(r, "item", "item["+toString(id)+"]", p.second, "");

            ++id;
        }
    }
};

template<typename Reflector, typename Container> struct ReflectWriteMapItems;

template<typename Container>
struct ReflectWriteMapItems<XMLDeserializer, Container>
{
    typedef typename Container::iterator iterator;
    typedef typename Container::value_type   value_type;
    typedef typename Container::key_type     key_type;
    typedef typename Container::mapped_type  mapped_type;

    static void reflect(XMLDeserializer& r, Container& c, uint32 count)
    {
        iterator hint = c.begin();

        // reflect key
        key_type key;

        for(uint32 id=0; id<count; ++id) {

            if(IsPointerOrSharedPointer<key_type>::value) {
                MIRA_MEMBER_WITH_ID(r, "key", "key["+toString(id)+"]", key, "");
            } else {
                // we have to deserialize the key to a temp object, cannot restore references to it !!!
                MIRA_MEMBER_WITH_ID(r, "key", "key["+toString(id)+"]", key, "", REFLECT_CTRLFLAG_TEMP_TRACKING);
            }

            // insert a dummy value with the key
            iterator it = c.insert(hint, value_type(key, mapped_type()));
            hint = it; // the iterator will be insertion hint for next element

            // reflect the value directly into the map item
            // the values can be reflected/deserialized with tracking
            MIRA_PROPERTY_WITH_ID(r, "item", "item["+toString(id)+"]", it->second, "");
        }
    }
};


} // namespace



} // namespace

#endif