From d6442068ede93b4cf377aa6a5ad72dd6c56f9687 Mon Sep 17 00:00:00 2001 From: Joshua Scott Date: Fri, 16 Nov 2018 15:04:00 +0000 Subject: [PATCH] serialization: Start implementing SerializerBinary This current implementation can fully save objects, but cannot currently load them. --- CMakeLists.txt | 10 +- include/ki/serialization/SerializerBinary.h | 71 ++++++ include/ki/serialization/SerializerFile.h | 0 include/ki/serialization/SerializerXML.h | 0 src/serialization/CMakeLists.txt | 4 + src/serialization/SerializerBinary.cpp | 258 ++++++++++++++++++++ 6 files changed, 339 insertions(+), 4 deletions(-) create mode 100644 include/ki/serialization/SerializerBinary.h create mode 100644 include/ki/serialization/SerializerFile.h create mode 100644 include/ki/serialization/SerializerXML.h create mode 100644 src/serialization/CMakeLists.txt create mode 100644 src/serialization/SerializerBinary.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 10ba4dc..969b538 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,9 @@ set(RAPIDXML_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/third_party) add_library(RapidXML INTERFACE) target_include_directories(RapidXML INTERFACE ${RAPIDXML_INCLUDE_DIR}) +find_package(ZLIB REQUIRED) +include_directories(${ZLIB_INCLUDE_DIRS}) + add_library(${PROJECT_NAME}) set_target_properties(${PROJECT_NAME} PROPERTIES @@ -13,16 +16,15 @@ set_target_properties(${PROJECT_NAME} CXX_STANDARD 11 ) target_include_directories(${PROJECT_NAME} - PRIVATE - ${PROJECT_SOURCE_DIR}/include - INTERFACE + PUBLIC ${PROJECT_SOURCE_DIR}/include ) -target_link_libraries(${PROJECT_NAME} RapidXML) +target_link_libraries(${PROJECT_NAME} RapidXML ${ZLIB_LIBRARIES}) add_subdirectory("src/dml") add_subdirectory("src/pclass") add_subdirectory("src/protocol") +add_subdirectory("src/serialization") add_subdirectory("src/util") option(COVERALLS "Generate coveralls data" OFF) diff --git a/include/ki/serialization/SerializerBinary.h b/include/ki/serialization/SerializerBinary.h new file mode 100644 index 0000000..0f8a5a5 --- /dev/null +++ b/include/ki/serialization/SerializerBinary.h @@ -0,0 +1,71 @@ +#pragma once +#include +#include "ki/pclass/TypeSystem.h" +#include "ki/pclass/Property.h" +#include "ki/pclass/PropertyClass.h" +#include "ki/util/BitStream.h" +#include "ki/util/FlagsEnum.h" + +namespace ki +{ +namespace serialization +{ + /** + * TODO: Documentation + */ + class SerializerBinary + { + public: + /** + * These flags control how the serializer reads/writes data. + */ + enum class flags : uint32_t + { + NONE = 0, + + /** + * When enabled, the flags the serializer was constructed with are written + * into an unsigned 32-bit integer before writing any data. + */ + WRITE_SERIALIZER_FLAGS = 0x01, + + /** + * When enabled, the serialized data (after the flags, if present) is compressed. + */ + COMPRESSED = 0x08 + }; + + /** + * Construct a new binary serializer. + * @param type_system The TypeSystem instance to acquire Type information from. + * @param is_file Determines whether or not to write type sizes, and property headers. + * @param flags Determines how serialized data is formatted. + */ + explicit SerializerBinary(const pclass::TypeSystem *type_system, + bool is_file, flags flags); + virtual ~SerializerBinary() {} + + void save(const pclass::PropertyClass *object, BitStream &stream); + void load(pclass::PropertyClass *&dest, BitStream &stream); + + protected: + virtual void presave_object(const pclass::PropertyClass *object, BitStream &stream) const; + void save_object(const pclass::PropertyClass *object, BitStream &stream) const; + void save_property(const pclass::PropertyBase *prop, BitStream &stream) const; + + virtual void preload_object(pclass::PropertyClass *&dest, BitStream &stream) const; + void load_object(pclass::PropertyClass *&dest, BitStream &stream) const; + void load_property(pclass::PropertyBase *prop, BitStream &stream) const; + + private: + const pclass::TypeSystem *m_type_system; + bool m_is_file; + flags m_flags; + + const pclass::PropertyClass *m_root_object; + }; +} + + // Make sure the flags enum can be used like a bitflag + MAKE_FLAGS_ENUM(ki::serialization::SerializerBinary::flags); +} diff --git a/include/ki/serialization/SerializerFile.h b/include/ki/serialization/SerializerFile.h new file mode 100644 index 0000000..e69de29 diff --git a/include/ki/serialization/SerializerXML.h b/include/ki/serialization/SerializerXML.h new file mode 100644 index 0000000..e69de29 diff --git a/src/serialization/CMakeLists.txt b/src/serialization/CMakeLists.txt new file mode 100644 index 0000000..e19681a --- /dev/null +++ b/src/serialization/CMakeLists.txt @@ -0,0 +1,4 @@ +target_sources(${PROJECT_NAME} + PRIVATE + ${PROJECT_SOURCE_DIR}/src/serialization/SerializerBinary.cpp +) \ No newline at end of file diff --git a/src/serialization/SerializerBinary.cpp b/src/serialization/SerializerBinary.cpp new file mode 100644 index 0000000..a4fef18 --- /dev/null +++ b/src/serialization/SerializerBinary.cpp @@ -0,0 +1,258 @@ +#include "ki/serialization/SerializerBinary.h" +#include +#include + +namespace ki +{ +namespace serialization +{ + SerializerBinary::SerializerBinary(const pclass::TypeSystem* type_system, + const bool is_file, const flags flags) + { + m_type_system = type_system; + m_is_file = is_file; + m_flags = flags; + m_root_object = nullptr; + } + + void SerializerBinary::save(const pclass::PropertyClass* object, BitStream& stream) + { + // Write the serializer flags + if (FLAG_IS_SET(m_flags, flags::WRITE_SERIALIZER_FLAGS)) + stream.write(static_cast(m_flags)); + + // Remember where we started writing data, so we can compress later + // if necessary. + const auto start_pos = stream.tell(); + + // If the contents of the stream are going to be compressed, + // reserve space to put the length + if (FLAG_IS_SET(m_flags, flags::COMPRESSED)) + stream.write(0); + + // Write the object to the stream + m_root_object = object; + save_object(object, stream); + + // Compress the contents of the stream + if (FLAG_IS_SET(m_flags, flags::COMPRESSED)) + { + // Remember where the ending of the data is + const auto end_pos = stream.tell(); + const auto size_bits = (end_pos - (start_pos + bitsizeof::value)).as_bits(); + const auto size_bytes = (size_bits / 8) + (size_bits % 8 > 0 ? 1 : 0); + + // Make a copy of the uncompressed data + auto *uncompressed = new uint8_t[size_bytes]; + stream.read_copy(uncompressed, size_bits); + + // Setup compression + static const std::size_t bufsize = 1024; + auto *temp_buffer = new uint8_t[bufsize]; + std::vector compressed(size_bytes); + z_stream z; + z.zalloc = nullptr; + z.zfree = nullptr; + z.next_in = uncompressed; + z.avail_in = size_bytes; + z.next_out = temp_buffer; + z.avail_out = bufsize; + + // Compress the uncompressed data + deflateInit(&z, Z_DEFAULT_COMPRESSION); + while (z.avail_in != 0) + { + const auto result = deflate(&z, Z_NO_FLUSH); + assert(result == Z_OK); + + // Have we filled up the temporary buffer? + if (z.avail_out == 0) + { + // Copy all data from the temporary buffer into the + // compressed vector + compressed.insert(compressed.end(), + temp_buffer, temp_buffer + bufsize); + z.next_out = temp_buffer; + z.avail_out = bufsize; + } + } + auto deflate_result = Z_OK; + while (deflate_result == Z_OK) + { + if (z.avail_out == 0) + { + compressed.insert(compressed.end(), + temp_buffer, temp_buffer + bufsize); + z.next_out = temp_buffer; + z.avail_out = bufsize; + } + deflate_result = deflate(&z, Z_FINISH); + } + assert(deflate_result == Z_STREAM_END); + compressed.insert(compressed.end(), + temp_buffer, temp_buffer + bufsize - z.avail_out); + deflateEnd(&z); + + // Move the contents of the compressed buffer if the compressed + // buffer is smaller + stream.seek(start_pos); + stream.write(size_bytes); + if (compressed.size() < size_bytes) + stream.write_copy(compressed.data(), compressed.size() * 8); + else + // Go back to the end of the data + stream.seek(end_pos); + + // Cleanup temporary buffers + delete[] uncompressed; + delete[] temp_buffer; + } + } + + void SerializerBinary::presave_object(const pclass::PropertyClass *object, BitStream& stream) const + { + // If we have an object, write the type hash, otherwise, write NULL (0). + if (object) + stream.write(object->get_type().get_hash()); + else + stream.write(NULL); + } + + void SerializerBinary::save_object(const pclass::PropertyClass *object, BitStream& stream) const + { + // Write any object headers + presave_object(object, stream); + + // Make sure we have an object to write + if (!object) + return; + + // Remember where we started writing the object data + const auto start_pos = stream.tell(); + + // If we're using the file format, reserve space to put the size of this object + if (m_is_file) + stream.write(0); + + // Write each of the object's properties + auto &properties = object->get_properties(); + for (auto it = properties.begin(); + it != properties.end(); ++it) + { + save_property(*it, stream); + } + + // If we're using the file format, we need to write how big the object is + if (m_is_file) + { + // Remember where the ending of the data is + const auto end_pos = stream.tell(); + const auto size_bits = (end_pos - start_pos).as_bits(); + + // Write the size, and then move back to the end + stream.seek(start_pos); + stream.write(size_bits); + stream.seek(end_pos); + } + } + + void SerializerBinary::save_property(const pclass::PropertyBase *prop, BitStream& stream) const + { + // Remember where we started writing the property data + const auto start_pos = stream.tell(); + + // Do we need to write the property header? + if (m_is_file) + { + // Reserve space to put the size of the property + stream.write(0); + stream.write(prop->get_full_hash()); + } + + // Is the property dynamic? (holding more than one value) + auto &property_type = prop->get_type(); + if (prop->is_dynamic()) + { + // Cast the property to a DynamicPropertyBase + const auto *dynamic_property = + dynamic_cast(prop); + + // Write the number of elements + stream.write(dynamic_property->get_element_count()); + + // Iterate through the elements + for (auto i = 0; i < dynamic_property->get_element_count(); i++) + { + // Is this a collection of pointers? + if (prop->is_pointer()) + { + // Is the property a collection of pointers to other objects? + if (property_type.get_kind() == pclass::Type::kind::CLASS) + // Write the value as a nested object + save_object(dynamic_property->get_object(i), stream); + else + // Write the value as normal (let the property deal with dereferencing) + dynamic_property->write_value_to(stream, i); + } + else + // If the value isn't a pointer, and it's not dynamic, just write it as a value + dynamic_property->write_value_to(stream, i); + } + } + else if (prop->is_pointer()) + { + // Does this property hold a pointer to another object? + if (property_type.get_kind() == pclass::Type::kind::CLASS) + // Write the value as a nested object + save_object(prop->get_object(), stream); + else + // Write the value as normal (let the property deal with dereferencing) + prop->write_value_to(stream); + } + else + // If the value isn't a pointer, and it's not dynamic, just write it as a value + prop->write_value_to(stream); + + // Finish writing the property header by writing the length + if (m_is_file) + { + // Remember where the ending of the data is + const auto end_pos = stream.tell(); + const auto size_bits = (end_pos - start_pos).as_bits(); + + // Write the size, and then move back to the end + stream.seek(start_pos); + stream.write(size_bits); + stream.seek(end_pos); + } + } + + void SerializerBinary::load(pclass::PropertyClass*& dest, BitStream& stream) + { + // Read the serializer flags + if (FLAG_IS_SET(m_flags, flags::WRITE_SERIALIZER_FLAGS)) + m_flags = static_cast(stream.read()); + + // Decompress the contents of the stream + if (FLAG_IS_SET(m_flags, flags::COMPRESSED)) + { + + } + } + + void SerializerBinary::preload_object(pclass::PropertyClass*& dest, BitStream& stream) const + { + + } + + void SerializerBinary::load_object(pclass::PropertyClass*& dest, BitStream& stream) const + { + + } + + void SerializerBinary::load_property(pclass::PropertyBase* prop, BitStream& stream) const + { + + } +} +}