serialization: add direct backend

Change-Id: I5c9345310f58fa62c7fec3d8a1ccc56978e4c53a

7 files changed, 1144 insertions, 1 deletions

diff --git a/Makefile.am b/Makefile.am
index f17af0b..da4f145 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -64,8 +64,10 @@ COMMON_SRC =\
 	src/internal/tslog.cpp \
 	src/internal/serializer.cpp \
 	src/internal/serializer_flatbuffers.cpp \
+	src/internal/serializer_direct.cpp \
 	src/internal/policy_containers.cpp \
 	src/internal/print_content.cpp \
+	src/internal/storage_backend_direct.cpp \
 	src/internal/storage_backend_flatbuffers.cpp \
 	src/internal/storage_backend_serialized.cpp \
 	src/internal/storage_backend_xml.cpp
diff --git a/src/internal/serialization_backend.hpp b/src/internal/serialization_backend.hpp
index 2b3395a..159ae0d 100644
--- a/src/internal/serialization_backend.hpp
+++ b/src/internal/serialization_backend.hpp
@@ -22,6 +22,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE. */
 
+#include "serializer_direct.hpp"
+#include "storage_backend_direct.hpp"
+
 #include "serializer_flatbuffers.hpp"
 #include "storage_backend_flatbuffers.hpp"
 
@@ -32,6 +35,11 @@ struct SerializationBackendFlatbuffers {
 	typedef ldp_serializer::SerializerFlatbuffers Serializer;
 };
 
-typedef SerializationBackendFlatbuffers SerializationBackend;
+struct SerializationBackendDirect {
+	typedef ldp_serialized::StorageBackendDirect Storage;
+	typedef ldp_serializer::SerializerDirect Serializer;
+};
+
+typedef SerializationBackendDirect SerializationBackend;
 
 }
diff --git a/src/internal/serialized.hpp b/src/internal/serialized.hpp
new file mode 100644
index 0000000..70589af
--- /dev/null
+++ b/src/internal/serialized.hpp
@@ -0,0 +1,259 @@
+#pragma once
+
+/*  MIT License
+ *
+ * Copyright (c) 2020 Samsung Electronics Co., Ltd
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is furnished
+ * to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE. */
+
+#include <cassert>
+#include <boost/utility/string_ref.hpp>
+#include <string>
+#include <vector>
+
+namespace ldp_serializer {
+
+/* The Serialized class represents serialized data created from some structures.
+ * It serializes values of the following types:
+ * - uint32_t;
+ * - enum types (serialized as uint32_t);
+ * - bool type (serialized as uint32_t)
+ * - container types (maps, sets, vectors, serialized as a kind of array, needs special preparation, see below);
+ * - string types, as a special case of container types.
+ *
+ * Usage: prepare your data, and pass them to the constructor.
+ * Example: if your data is:
+ * - single integer: 123;
+ * - single enum value: MyEnumValue;
+ * - single string: "MyString";
+ * - a single, already serialized object Already;
+ * - a map v: int->string;
+ * you may serialize the data by writing:
+ * auto serialized = Serialized{123, MyEnumValue, "MyString", Already,
+ *                              serialize_container(v, [](const auto &elem) {
+ *                                  return Serialized{elem.first, elem.second};
+ *                              })};
+ * const uint8_t *data = serialized.releaseData(); // or better const auto *data = ...
+ *
+ * when you're done serializing, get your data with releaseData() method. You're now responsible for
+ * freeing the data. The Serialized object can be now disposed.
+ *
+ * NOTE: Never try to serialize just a single already serialized object, as this probably won't work.
+ *       It may destroy your object.
+ *       ... and it makes little sense. It would just wrap it in another, unnecessary layer.
+ *       Unfortunately, I didn't come up with a way to prevent it yet.
+ *
+ * The map above is serialized with helper function serialize_container, which takes 2 arguments:
+ * - a container;
+ * - a function object used to prepare a serialized element of the container.
+ * This is all for the special preparation. It creates a Serialized vector of Serialized elements,
+ * which you can pass as an argument for further serialization.
+ *
+ * Internal layout:
+ * Each Serialized object data contains a data buffer.
+ * A data buffer contains a number of fields and additional data.
+ * The number of fields is equal to the number of the parameters of the constructor.
+ * Each field is the same size as uint32_t.
+ * Any integer, enum, or bool is serialized as a field.
+ * Every other type is serialized like this:
+ * - the data is appended directly after the fields.
+ * - the corresponding field contains offset (from the start of the buffer) of the start of the data.
+ * The appended data may be a string. String data is serialized as:
+ * - a single uint32_t value, the string's length;
+ * - the characters;
+ * - nul byte;
+ * - padding.
+ * The appended data may be a vector. Vector data is serialized as fields:
+ * - field at index 0 contains number of elements in the vector;
+ * - first input element is serialized as a field at index 1, taking into account the rules
+ *   above, that is integer = in-place field, other type = offset + data.
+ * - the indexes are relative to the first field at which vector is appended. The idea is
+ *   that a vector should serialized to its own Serialized object, and then to another Serialized object as a field,
+ *   but it should also work with serializing it "inline", as a series of fields.
+ */
+
+class Serialized {
+	typedef uint32_t HeaderFieldType;
+	constexpr size_t alignment() { return sizeof(HeaderFieldType); }
+
+	std::unique_ptr<uint8_t[]> _data;
+	size_t _size{0};		// keeps the total allocated _data size
+	size_t _done{0};		// indicates how many fields were already written
+	size_t _done_bytes{0};	// indicates an offset where another field's data can be written
+
+	HeaderFieldType *fields() { return reinterpret_cast<HeaderFieldType*>(_data.get()); }
+	uint8_t *buf() { return &_data[_done_bytes]; }
+
+	void startNextDataField() {
+		fields()[_done++] = _done_bytes;
+	}
+
+	void written(size_t len) {
+		_done_bytes += len;
+	}
+
+	template <typename T>
+	void writeContainerData(T iterator, T end_iterator) {
+		std::copy(iterator, end_iterator, buf());
+		written((end_iterator - iterator) * sizeof(*iterator));
+	}
+
+	template <typename T>
+	void writeString(const T &field) {
+		startNextDataField();
+
+		// write string length (without nul-byte)
+		*reinterpret_cast<uint32_t *>(buf()) = field.size();
+		written(sizeof(uint32_t));
+
+		writeContainerData(field.begin(), field.end());
+
+		// add nul byte and padding
+		do {
+			*buf() = 0;
+			written(1);
+		} while (_done_bytes % alignment() != 0);
+	}
+
+	template <typename ...Args>
+	size_t computeSize(const Serialized &s, const Args &...args)
+	{ return s.size() + computeSize(args...); }
+
+	inline size_t strSize(size_t length) {
+		uint32_t size = length + 1 + sizeof(uint32_t);   // 1: nul byte; placeholder for string length
+		return size + alignment() - (size%alignment());  // padding to alignment boundary
+	}
+
+	template <typename ...Args>
+	size_t computeSize(const std::string &s, const Args &...args)
+	{ return strSize(s.size()) + computeSize(args...); }
+
+	template <typename ...Args>
+	size_t computeSize(boost::string_ref s, const Args &...args)
+	{ return strSize(s.size()) + computeSize(args...); }
+
+	template <typename ...Args>
+	size_t computeSize(uint32_t, const Args &...args)
+	{ return computeSize(args...); }
+
+	template <typename T, typename ...Args>
+	size_t computeSize(const std::vector<T> &v, const Args &...args) {
+		size_t size = sizeof(uint32_t);
+		for (const auto &elem: v)
+			size += computeSize(elem);
+		return size + computeSize(args...);
+	}
+
+	template <typename T,
+			 typename = std::enable_if_t<std::is_enum<T>::value>,
+			 typename ...Args>
+	size_t computeSize(T, const Args &...args)
+	{ return computeSize(args...); }
+
+	size_t computeSize() { return 0; }
+
+	template <typename T, typename ...Args>
+	size_t computeNumOfFields(const T &, const Args &...args)
+	{ return 1 + computeNumOfFields(args...); }
+
+	template <typename ...Args>
+	size_t computeNumOfFields(const Serialized &s, const Args &...args)
+	{ return (s.size() ? 1 : 0) + computeNumOfFields(args...); }
+
+	template <typename T, typename ...Args>
+	size_t computeNumOfFields(const std::vector<T> &v, const Args &...args)
+	{ return 1 + v.size() + computeNumOfFields(args...); } // 1 for list size
+
+	size_t computeNumOfFields() { return 0; }
+
+	template <typename T, typename ...Args>
+	void writeFields(const T &t, const Args &...args) {
+		writeField(t);
+		writeFields(args...);
+	}
+
+	void writeFields() {}
+
+	void writeField(const Serialized &field) {
+		// we ignore empty fields
+		if (field.size() == 0)
+			return;
+		assert(field.data());
+
+		startNextDataField();
+
+		writeContainerData(field.data(), field.data() + field.size());
+	}
+	void writeField(const std::string &field) {
+		writeString(field);
+	}
+	void writeField(boost::string_ref field) {
+		writeString(field);
+	}
+	void writeField(uint32_t field) {
+		fields()[_done++] = field;
+	}
+	template <typename T>
+	void writeField(const std::vector<T> &v) {
+		writeField(v.size());
+		for (const auto &elem: v)
+			writeField(elem);
+	}
+	template <typename T,
+			 typename = std::enable_if_t<std::is_enum<T>::value>,
+			 typename ...Args>
+	void writeField(T field) {
+		writeField(static_cast<uint32_t>(field));
+	}
+	const uint8_t *data() const { return _data.get(); }
+public:
+	// NOTE: never serialize single Serialized object into Serialized object as it may just swap data
+	//       ... and it makes little sense anyway
+	Serialized(Serialized &&s) : _data{s._data.release()}, _size{s._size}, _done{s._done}, _done_bytes{s._done_bytes} {}
+
+	template <typename ...Args>
+	explicit Serialized(const Args &...args)
+	{
+		_done_bytes = computeNumOfFields(args...) * sizeof(uint32_t);
+		_size = _done_bytes + computeSize(args...);
+		if (_size)
+			_data = std::make_unique<uint8_t[]>(_size);
+
+		writeFields(args...);
+	}
+
+	uint32_t size() const { return _size; }
+
+	const uint8_t *releaseData() {
+		return _data.release();
+	}
+};
+
+template <typename T, typename F>
+Serialized serialize_container(const T &container, const F &serialize_elem_fun) {
+	std::vector<Serialized> sx;
+	sx.reserve(container.size());
+
+	for (const auto &e: container)
+		sx.push_back(serialize_elem_fun(e));
+
+	return Serialized{sx};
+}
+
+}
diff --git a/src/internal/serializer_direct.cpp b/src/internal/serializer_direct.cpp
new file mode 100644
index 0000000..d74ce9b
--- /dev/null
+++ b/src/internal/serializer_direct.cpp
@@ -0,0 +1,170 @@
+/*  MIT License
+ *
+ * Copyright (c) 2020 Samsung Electronics Co., Ltd
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is furnished
+ * to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE. */
+
+#include "policy_containers.hpp"
+#include "serialization_traits.hpp"
+#include "serialized.hpp"
+#include "serializer_direct.hpp"
+
+using namespace ldp_serializer;
+using ldp_serializer::Serialized;
+using ldp_serializer::serialize_container;
+
+class SerializerDirectImpl {
+	const ldp_xml::StorageBackendXML &_db;
+
+	template <typename ItemType>
+	Serialized serializeItem(const ItemType &v);
+
+	template <typename ItemType>
+	Serialized serializeItems(const std::vector<ItemType> &v) {
+		return serialize_container(v,
+				[&](const auto &item) {
+					return serializeItem(item);
+				});
+	}
+
+	template <typename PolicyType>
+	Serialized serializePolicy(const PolicyType &policy);
+
+	template <typename PolicyType>
+	Serialized serializeUserGroup(const std::map<uid_t, PolicyType> &m) {
+		return serialize_container(m,
+				[&](const auto &u) {
+					return Serialized{
+						u.first,
+						this->serializePolicy(u.second)};
+				});
+	}
+
+	Serialized serializeDecisionItem(const ldp_xml_parser::DecisionItem &item) {
+		return Serialized{
+				item.getDecision(),
+				item.getPrivilege()};
+	}
+
+	Serialized serializeTree(const std::shared_ptr<ldp_xml_parser::TreeNode> &node) {
+		return Serialized{
+				node->getToken(),
+				serializeDecisionItem(node->getOwnPrefixDecisionItem()),
+				serializeDecisionItem(node->getOwnDecisionItem()),
+				serialize_container(node->getChildren(),
+						[&](const auto &child) {
+							return serializeTree(child.second);
+						})
+				};
+	}
+
+	template <typename PolicyType>
+	Serialized serializeSet();
+
+public:
+	SerializerDirectImpl(const ldp_xml::StorageBackendXML &db)
+		: _db{db}
+	{}
+
+	const uint8_t *serialize(size_t &size);
+};
+
+template <typename ItemType>
+Serialized SerializerDirectImpl::serializeItem(const ItemType &item) {
+	return Serialized{
+		serializeDecisionItem(item.getDecision()),
+		item.getName(),
+		item.getInterface(),
+		item.getMember(),
+		item.getPath(),
+		item.getType(),
+		item.isNamePrefix()};
+}
+
+template <>
+Serialized SerializerDirectImpl::serializeItem(const ldp_xml_parser::ItemAccess &item) {
+	return Serialized{
+		serializeDecisionItem(item.getDecision()),
+		item.getUid(),
+		item.getGid(),
+		item.getType()};
+}
+
+template <typename PolicyType>
+Serialized SerializerDirectImpl::serializePolicy(const PolicyType &policy) {
+	return serializeItems(policy.getItems());
+}
+
+template <>
+Serialized SerializerDirectImpl::serializePolicy(const ldp_xml_parser::PolicyOwn &policy) {
+	return serializeTree(policy.getTree().getRoot());
+}
+
+template <>
+Serialized SerializerDirectImpl::serializePolicy(const ldp_xml_parser::PolicySend &policy) {
+	return Serialized{
+			serializeItems(policy.getItems()),
+			serialize_container(policy.getIndex(),
+				[&](const auto &elem) {
+					return Serialized{
+						elem.first,
+						elem.second.m_bestScore,
+						Serialized{elem.second.m_itemRefs}
+					};
+				}),
+			Serialized{policy.getPrefixIndex()}};
+}
+
+template <typename PolicyType>
+Serialized SerializerDirectImpl::serializeSet() {
+	return Serialized{
+			serializePolicy(_db.getPolicyContextMandatory<PolicyType>()),
+			serializePolicy(_db.getPolicyContextDefault<PolicyType>()),
+			serializeUserGroup(_db.getPoliciesUser<PolicyType>()),
+			serializeUserGroup(_db.getPoliciesGroup<PolicyType>())
+		};
+}
+
+template <>
+Serialized SerializerDirectImpl::serializeSet<ldp_xml_parser::PolicyAccess>() {
+	return Serialized{
+			serializePolicy(_db.getPolicyContextMandatory<ldp_xml_parser::PolicyAccess>()),
+			serializePolicy(_db.getPolicyContextDefault<ldp_xml_parser::PolicyAccess>())
+		};
+}
+
+const uint8_t *SerializerDirectImpl::serialize(size_t &size) {
+	Serialized r{
+		serializeSet<ldp_xml_parser::PolicyOwn>(),
+		serializeSet<ldp_xml_parser::PolicySend>(),
+		serializeSet<ldp_xml_parser::PolicyReceive>(),
+		serializeSet<ldp_xml_parser::PolicyAccess>()};
+
+	size = r.size();
+	return r.releaseData();
+}
+
+SerializerDirect::SerializerDirect()
+{}
+
+const uint8_t *SerializerDirect::serialize(const ldp_xml::StorageBackendXML &db, size_t &size) {
+	SerializerDirectImpl impl{db};
+	_data.reset(impl.serialize(size));
+	return _data.get();
+}
diff --git a/src/internal/serializer_direct.hpp b/src/internal/serializer_direct.hpp
new file mode 100644
index 0000000..8b75d19
--- /dev/null
+++ b/src/internal/serializer_direct.hpp
@@ -0,0 +1,37 @@
+#pragma once
+
+/*  MIT License
+ *
+ * Copyright (c) 2020 Samsung Electronics Co., Ltd
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is furnished
+ * to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE. */
+
+#include "storage_backend_xml.hpp"
+#include <memory>
+
+namespace ldp_serializer {
+
+class SerializerDirect {
+	std::unique_ptr<const uint8_t> _data;
+public:
+	SerializerDirect();
+	const uint8_t *serialize(const ldp_xml::StorageBackendXML &db, size_t &size);
+};
+
+}
diff --git a/src/internal/storage_backend_direct.cpp b/src/internal/storage_backend_direct.cpp
new file mode 100644
index 0000000..455f443
--- /dev/null
+++ b/src/internal/storage_backend_direct.cpp
@@ -0,0 +1,31 @@
+/*  MIT License
+ *
+ * Copyright (c) 2020 Samsung Electronics Co., Ltd
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is furnished
+ * to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE. */
+
+#include "storage_backend_direct.hpp"
+
+using namespace ldp_serialized;
+
+bool StorageBackendDirect::initFromData(const uint8_t *mem, size_t , bool ) {
+
+	file = reinterpret_cast<const File*>(mem);
+	return file != nullptr;
+}
diff --git a/src/internal/storage_backend_direct.hpp b/src/internal/storage_backend_direct.hpp
new file mode 100644
index 0000000..4b3b47b
--- /dev/null
+++ b/src/internal/storage_backend_direct.hpp
@@ -0,0 +1,636 @@
+#pragma once
+
+/*  MIT License
+ *
+ * Copyright (c) 2020 Samsung Electronics Co., Ltd
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is furnished
+ * to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE. */
+
+#include "serialized_convert.hpp"
+#include "serialization_traits.hpp"
+#include <cassert>
+
+namespace ldp_serialized {
+
+/* This file contains definition of classes that help with deserializing data
+ * serialized with Serialized objects.
+ *
+ * It works like this:
+ * - the objects are not created in the traditional way (with constructors);
+ * - instead they are "mapped" into their serialized data - so they should not have
+ *   any data members;
+ * - an existing object operates on its "this" pointer to compute offsets proper
+ *   to the object class to find pointers to contained types, or copy integer
+ *   values.
+ *
+ * It all starts with File object, which is mapped directly into the first byte of
+ * the serialized data. Then the File object knows how to find further pointers.
+ * This is roughly the schema for the data:
+ *
+ * File {
+ * 	OwnSet {
+ * 		context_default: PolicyOwn;
+ * 		context_mandatory: PolicyOwn;
+ * 		user: array of {id, PolicyOwn};
+ * 		group: array of {id, PolicyOwn};
+ *
+ * 		// id = integer, mapped to respectively uid_t and gid_t
+ * 		define PolicyOwn = {
+ * 			tree: PolicyOwnNode;
+ *
+ * 			define PolicyOwnNode = {
+ * 				token: string;
+ * 				prefix_decision_item: DecisionItem;
+ * 				decision_item: DecisionItem;
+ * 				children: array of PolicyOwnNode;
+ * 			}
+ * 		}
+ * 	}
+ * 	SendSet {
+ * 		context_default: PolicySend;
+ * 		context_mandatory: PolicySend;
+ * 		user: array of {id, PolicySend};
+ * 		group: array of {id, PolicySend};
+ *
+ * 		define PolicySend = {
+ * 			items: array of {
+ * 				define ItemSend = {
+ * 					decision: DecisionItem;
+ * 					name: string;
+ * 					interface: string;
+ * 					member: string;
+ * 					path: string;
+ * 					type: MessageType;
+ * 					is_name_prefix: bool;
+ * 				}
+ * 			}
+ * 			index: array of {
+ * 				name: string;
+ * 				best_score: uint32_t;
+ * 				item_refs: array of {uint32_t};
+ * 			}
+ * 			prefix_index: array of {uint32_t};
+ * 		}
+ * 	}
+ * 	ReceiveSet {
+ * 		context_default: PolicyReceive;
+ * 		context_mandatory: PolicyReceive;
+ * 		user: array of {id, PolicyReceive};
+ * 		group: array of {id, PolicyReceive};
+ *
+ * 		define PolicyReceive = {
+ * 			items: array of {
+ * 				define ItemReceive = {
+ * 					decision: DecisionItem;
+ * 					name: string;
+ * 					interface: string;
+ * 					member: string;
+ * 					path: string;
+ * 					type: MessageType;
+ * 					is_name_prefix: bool;
+ * 				}
+ * 			}
+ * 	}
+ * 	AccessSet {
+ * 		context_default: PolicyAccess;
+ * 		context_mandatory: PolicyAccess;
+ *
+ * 		define PolicyAccess = {
+ * 			items: array of {
+ * 				define ItemAccess = {
+ * 					uid: uint;
+ * 					gid: uint;
+ * 					decision: DecisionItem;
+ * 					type: BusAccessType;
+ * 				}
+ * 			}
+ * 		}
+ * 	}
+ *
+ * 	define DecisionItem = {
+ * 		decision: Decision;
+ * 		privilege: string;
+ * 	}
+ * }
+ *
+ */
+
+/* This class is a base for all the data extractors just to provide them some handy helpers.
+ */
+class BufferBase {
+	BufferBase() = delete;
+protected:
+	// a short-typing caster
+	template <typename Type, typename AnyType>
+	const Type *T(const AnyType *t) const
+	{ return reinterpret_cast<const Type *>(t); }
+
+	// this gives pointer to the buffer with a given offset
+	const uint8_t *buf(size_t offset=0) const { return T<uint8_t>(this) + offset; }
+	// this gives value of specified field 'num' - either the field itself
+	// or offset to additional data
+	uint32_t offset(size_t num) const { return T<uint32_t>(this)[num]; }
+
+	// this gives pointer to additional data of the field 'num'
+	template <typename Type>
+	const Type *ptr(size_t num) const
+	{ return T<Type>(buf(offset(num))); }
+};
+
+// String extracting class
+class Str : private BufferBase {
+public:
+	typedef uint32_t SizeType;
+
+	const char *c_str() const
+	{ return T<char>(buf(sizeof(SizeType))); }
+	SizeType size() const
+	{ return offset(0); }
+	const boost::string_ref toStringRef() const
+	{ return boost::string_ref(c_str(), size()); }
+};
+
+// Simple array extracting class - works for arrays of integers
+template <typename Elem = uint32_t>
+class SimpleList : protected BufferBase {
+public:
+	// first uint32_t is size (number of elements in the list)
+	size_t size() const { return offset(0); }
+	// iterators are plain pointers
+	const Elem *begin() const
+	{ return T<Elem>(buf(sizeof(uint32_t))); }
+	const Elem *end() const
+	{ return begin() + size(); }
+	// at() returns by value, because SimpleList is used only for uint32_t anyway
+	Elem at(size_t idx) const
+	{ return begin()[idx]; }
+};
+
+// Iterator for non-simple arrays
+template <typename Elem>
+class Iterator {
+	const uint8_t *_root;
+	size_t _pos{0};
+	uint32_t offset(size_t num) const { return reinterpret_cast<const uint32_t *>(_root)[num]; }
+public:
+	Iterator(const uint8_t *buf, size_t pos) : _root{buf}, _pos{pos} {}
+
+	const Elem *operator*() const
+	{ return reinterpret_cast<const Elem *>(&_root[offset(_pos)]); }
+
+	Iterator &operator++() {
+		_pos++;
+		return *this;
+	}
+	Iterator operator++(int)
+	{ return Iterator(_root, _pos++); }
+	Iterator &operator--() {
+		_pos--;
+		return *this;
+	}
+	Iterator operator--(int)
+	{ return Iterator(_root, _pos--); }
+	bool operator!=(const Iterator &rhs)
+	{ return _pos != rhs._pos; }
+
+	int operator-(Iterator rhs)
+	{ return static_cast<int>(_pos) - rhs._pos; }
+	void operator+=(int val)
+	{ _pos += val; }
+
+	// This Iterator class lacks some operators (counterparts "==" , "+", "-=" to the above)
+	// only because we don't need them for our goals.
+	// This is very internal code, so we don't push it to be nice to everyone.
+};
+
+// Non-simple array extracting class
+template <typename Elem>
+class List : protected BufferBase {
+public:
+	size_t size() const { return offset(0); }
+	auto begin() const
+	{ return Iterator<Elem>(BufferBase::buf(), 1); }
+	auto end() const
+	{ return Iterator<Elem>(BufferBase::buf(), size()+1); }
+	const auto *at(size_t idx) const
+	{ return *Iterator<Elem>(buf(), idx+1); }
+};
+
+class DecisionItem : private BufferBase {
+public:
+	auto getDecision() const
+	{ return makeDecision(offset(0)); }
+	const auto *getPrivilege() const
+	{ return ptr<Str>(1); }
+};
+
+class PolicyOwnNode : private BufferBase {
+public:
+	typedef DecisionItem DecisionItemType;
+	typedef List<PolicyOwnNode> ChildrenType;
+
+	const auto *getToken() const
+	{ return ptr<Str>(0); }
+	const auto *getPrefixDecisionItem() const
+	{ return ptr<DecisionItemType>(1); }
+	const auto *getDecisionItem() const
+	{ return ptr<DecisionItemType>(2); }
+	const auto *getChildren() const
+	{ return ptr<ChildrenType>(3); }
+};
+
+class NameScoresPair : private BufferBase {
+public:
+	const auto *getName() const
+	{ return ptr<Str>(0); }
+	uint32_t getBestScore() const
+	{ return offset(1); }
+	const auto *getItemRefs() const
+	{ return ptr<SimpleList<>>(2); }
+};
+
+class ItemAccess : private BufferBase {
+public:
+	const auto *getDecisionItem() const
+	{ return ptr<DecisionItem>(0); }
+	auto getUid() const
+	{ return static_cast<uid_t>(offset(1)); }
+	auto getGid() const
+	{ return static_cast<gid_t>(offset(2)); }
+	auto getType() const
+	{ return makeBusAccessType(offset(3)); }
+};
+
+class ItemSR : private BufferBase {
+public:
+	const auto *getDecisionItem() const
+	{ return ptr<DecisionItem>(0); }
+	auto getName() const
+	{ return ptr<Str>(1); }
+	auto getInterface() const
+	{ return ptr<Str>(2); }
+	auto getMember() const
+	{ return ptr<Str>(3); }
+	auto getPath() const
+	{ return ptr<Str>(4); }
+	auto getMessageType() const
+	{ return makeMessageType(offset(5)); }
+	auto getIsNamePrefix() const
+	{ return !!offset(6); }
+};
+
+class ItemSend : public ItemSR {};
+class ItemReceive : public ItemSR {};
+
+class PolicyOwn : private BufferBase {
+public:
+	const auto *getTree() const
+	{ return T<PolicyOwnNode>(this); }
+};
+
+template <typename ItemType>
+class Policy : protected BufferBase {
+public:
+	typedef List<ItemType> ListType;
+
+	const auto *getItems() const
+	{ return T<ListType>(this); }
+};
+
+using PolicyAccess = Policy<ItemAccess>;
+using PolicyReceive = Policy<ItemReceive>;
+
+class PolicySend : protected BufferBase {
+public:
+	typedef List<ItemSend> ListType;
+	typedef NameScoresPair IndexElemType;
+	typedef List<IndexElemType> IndexType;
+	typedef SimpleList<> PrefixIndexType;
+
+	const auto *getItems() const
+	{ return ptr<ListType>(0); }
+	const auto *getIndex() const
+	{ return ptr<IndexType>(1); }
+	const auto *getPrefixIndex() const
+	{ return ptr<PrefixIndexType>(2); }
+};
+
+template <typename PolicyType>
+class PolicyPair : private BufferBase {
+public:
+	id_t getId() const
+	{ return static_cast<id_t>(offset(0)); }
+
+	const auto *getPolicy() const
+	{ return ptr<PolicyType>(1); }
+};
+
+template <typename PolicyType>
+class Set : private BufferBase {
+public:
+	typedef List<PolicyPair<PolicyType>> ListType;
+
+	const auto *getContextMandatory() const
+	{ return ptr<PolicyType>(0); }
+	const auto *getContextDefault() const
+	{ return ptr<PolicyType>(1); }
+	const auto *getUser() const
+	{ return ptr<ListType>(2); }
+	const auto *getGroup() const
+	{ return ptr<ListType>(3); }
+};
+
+class File : private BufferBase {
+public:
+	typedef Set<PolicyOwn> OwnSetType;
+	typedef Set<PolicySend> SendSetType;
+	typedef Set<PolicyReceive> ReceiveSetType;
+	typedef Set<PolicyAccess> AccessSetType;
+
+	const auto *getOwnSet() const
+	{ return ptr<OwnSetType>(0); }
+	const auto *getSendSet() const
+	{ return ptr<SendSetType>(1); }
+	const auto *getReceiveSet() const
+	{ return ptr<ReceiveSetType>(2); }
+	const auto *getAccessSet() const
+	{ return ptr<AccessSetType>(3); }
+};
+
+class StorageBackendDirect {
+	// this private subclass is needed for looking up elements in containers by their key with std::lower_bound
+	template <typename ExtractKey>
+	class Comparator {
+		ExtractKey _f;
+	public:
+		Comparator(const ExtractKey &fun) : _f{fun} {}
+		template <typename R, typename K>
+		bool operator()(const R &r, const K &k) {
+			return _f(r) < k;
+		}
+	};
+
+	template <typename T>
+	auto makeComparator(const T &fun) const { return Comparator<T>{fun}; }
+
+	template <typename T, typename K, typename E>
+	auto containerLookupByKey(const T *container, const K &key, const E &extractKey) const {
+		assert(container->size());
+		auto found = std::lower_bound(container->begin(), container->end(), key, makeComparator(extractKey));
+		if (found != container->end() && extractKey(*found) == key)
+			return std::make_pair(true, *found);
+		return std::make_pair(false, *container->begin());
+	}
+public:
+	bool initFromData(const uint8_t *serialized_data, size_t size, bool verify = false);
+	void release() {}
+
+	auto getFile() const
+	{ return file; }
+
+	auto fileGetOwnSet(const File *file) const
+	{ return file->getOwnSet(); }
+	auto fileGetSendSet(const File *file) const
+	{ return file->getSendSet(); }
+	auto fileGetReceiveSet(const File *file) const
+	{ return file->getReceiveSet(); }
+	auto fileGetAccessSet(const File *file) const
+	{ return file->getAccessSet(); }
+
+	template <typename PolicyType>
+	auto setGetContextDefault(const Set<PolicyType> *set) const
+	{ return set->getContextDefault(); }
+
+	template <typename PolicyType>
+	auto setGetContextMandatory(const Set<PolicyType> *set) const
+	{ return set->getContextMandatory(); }
+
+	template <typename PolicyType>
+	auto setGetUser(const Set<PolicyType> *set) const
+	{ return set->getUser(); }
+
+	template <typename PolicyType>
+	auto setGetGroup(const Set<PolicyType> *set) const
+	{ return set->getGroup(); }
+
+	template <typename PolicyType>
+	int setUserGroupGetId(const PolicyPair<PolicyType> *set_user_group) const
+	{ return set_user_group->getId(); }
+
+	template <typename PolicyType>
+	auto setUserGroupGetPolicy(const PolicyPair<PolicyType> *set_user_group) const
+	{ return set_user_group->getPolicy(); }
+
+	auto policyGetTree(const PolicyOwn *policy) const
+	{ return policy->getTree(); }
+
+	template <typename PolicyType>
+	auto policyGetItems(const PolicyType *policy) const
+	{ return policy->getItems(); }
+
+	auto policyHasIndex(const PolicySend *policy) const
+	{ return policy->getIndex()->size() > 0; }
+
+	auto policyGetIndex(const PolicySend *policy) const
+	{ return policy->getIndex(); }
+
+	auto policyGetPrefixIndex(const PolicySend *policy) const
+	{ return policy->getPrefixIndex(); }
+
+	auto policyIndexGetName(const NameScoresPair *p) const
+	{ return p->getName(); }
+
+	auto policyIndexGetBestScore(const NameScoresPair *p) const
+	{ return p->getBestScore(); }
+
+	auto policyIndexGetItemRefs(const NameScoresPair *p) const
+	{ return p->getItemRefs(); }
+
+	template <typename Container>
+	auto containerGetReverseIterator(const Container *container) const
+	{ return makeReverseIterator(container->end()); }
+
+	template <typename Container>
+	auto containerGetReverseIteratorEnd(const Container *container) const
+	{ return makeReverseIterator(container->begin()); }
+
+	template <typename Container>
+	auto containerGetIterator(const Container *container) const
+	{ return container->begin(); }
+
+	template <typename Container>
+	auto containerGetIteratorEnd(const Container *container) const
+	{ return container->end(); }
+
+	template <typename Container>
+	auto containerGetSize(const Container *container) const
+	{ return container->size(); }
+
+	template <typename Container>
+	auto containerLookupByIndex(const Container *container, size_t index) const
+	{ return container->at(index); }
+
+	template <typename Container>
+	auto containerEmpty(const Container *container) const
+	{ return container->size() == 0; }
+
+	auto containerLookupByKey(const List<PolicyOwnNode> *container, const char *key) const {
+		return containerLookupByKey(container, key,
+			                        [](const auto *node) { return node->getToken()->toStringRef(); });
+	}
+
+	template <typename ContainerType>
+	auto containerLookupByKey(const ContainerType *container, id_t id) const {
+		return containerLookupByKey(container, id,
+			                        [](const auto *elem) { return static_cast<id_t>(elem->getId()); });
+	}
+
+	auto containerLookupByKey(const List<NameScoresPair> *container, const char *key) const {
+		return containerLookupByKey(container, key,
+			                        [](const auto *elem) { return elem->getName()->toStringRef(); });
+	}
+
+	auto decisionItemGetDecision(const DecisionItem *item) const
+	{ return item->getDecision(); }
+
+	auto decisionItemGetPrivilege(const DecisionItem *item) const
+	{ return item->getPrivilege(); }
+
+	auto ownNodeGetToken(const PolicyOwnNode *node) const
+	{ return node->getToken(); }
+
+	auto ownNodeGetDecisionItem(const PolicyOwnNode *node) const
+	{ return node->getDecisionItem(); }
+
+	auto ownNodeGetPrefixDecisionItem(const PolicyOwnNode *node) const
+	{ return node->getPrefixDecisionItem(); }
+
+	auto ownNodeGetChildren(const PolicyOwnNode *node) const
+	{ return node->getChildren(); }
+
+	auto itemAccessGetType(const ItemAccess *item) const
+	{ return item->getType(); }
+
+	auto itemAccessGetUid(const ItemAccess *item) const
+	{ return item->getUid(); }
+
+	auto itemAccessGetGid(const ItemAccess *item) const
+	{ return item->getGid(); }
+
+	auto itemSrGetName(const ItemSR *item) const
+	{ return item->getName(); }
+
+	auto itemSrGetIsNamePrefix(const ItemSR *item) const
+	{ return item->getIsNamePrefix(); }
+
+	auto itemSrGetInterface(const ItemSR *item) const
+	{ return item->getInterface(); }
+
+	auto itemSrGetMember(const ItemSR *item) const
+	{ return item->getMember(); }
+
+	auto itemSrGetPath(const ItemSR *item) const
+	{ return item->getPath(); }
+
+	template <typename ItemSR>
+	auto itemSrGetMessageType(const ItemSR *item) const
+	{ return item->getMessageType(); }
+
+	template <typename Item>
+	auto itemGetDecisionItem(const Item *item) const
+	{ return item->getDecisionItem(); }
+
+	const char *stringGetCStr(const Str *str) const
+	{ return str->c_str(); }
+
+	size_t stringGetSize(const Str *str) const
+	{ return str->size(); }
+
+private:
+	const File *file{nullptr};
+	template <typename T>
+	class ReverseIterator {
+		T __iterator;
+		public:
+			ReverseIterator(const T &iterator) : __iterator{iterator} {}
+
+			// note: we copy the element here, but it is ok (FOR US!)
+			// because it is pointer anyway
+			auto operator*() {
+				auto tmp = __iterator;
+				return *--tmp;
+			}
+
+			auto operator++() { return __iterator--; }
+			auto operator++(int) {
+				auto tmp = __iterator;
+				__iterator--;
+				return tmp;
+			}
+			auto operator!=(const ReverseIterator<T> &rhs) { return __iterator != rhs.__iterator; }
+	};
+
+	template <typename T>
+	auto makeReverseIterator(const T &iterator) const
+	{ return ReverseIterator<T>(iterator); }
+};
+
+}
+
+namespace std {
+
+template <typename Elem>
+struct iterator_traits<typename ldp_serialized::Iterator<Elem>>
+	: public std::iterator<std::random_access_iterator_tag, const Elem *, int> {};
+
+}  // namespace std
+
+namespace ldp_serialization {
+template <>
+struct HasUserGroup<const ldp_serialized::File::OwnSetType *>
+{ typedef std::true_type result; };
+template <>
+struct HasUserGroup<const ldp_serialized::File::SendSetType *>
+{ typedef std::true_type result; };
+template <>
+struct HasUserGroup<const ldp_serialized::File::ReceiveSetType *>
+{ typedef std::true_type result; };
+template <>
+struct HasUserGroup<const ldp_serialized::File::AccessSetType *>
+{ typedef std::false_type result; };
+
+template <> struct PolicyContentType<const ldp_serialized::PolicyOwn *>
+{ typedef TreeType result; };
+template <> struct PolicyContentType<const ldp_serialized::PolicySend *>
+{ typedef ItemsType result; };
+template <> struct PolicyContentType<const ldp_serialized::PolicyReceive *>
+{ typedef ItemsType result; };
+template <> struct PolicyContentType<const ldp_serialized::PolicyAccess *>
+{ typedef ItemsType result; };
+
+template <> struct PolicyHasIndex<const ldp_serialized::PolicySend *>
+{ typedef std::true_type result; };
+
+template <> struct ItemType<const ldp_serialized::ItemSend *>
+{ typedef SendType result; };
+template <> struct ItemType<const ldp_serialized::ItemReceive *>
+{ typedef ReceiveType result; };
+template <> struct ItemType<const ldp_serialized::ItemAccess *>
+{ typedef AccessType result; };
+}