SeanOMik
/
libki
mirror of https://github.com/SeanOMik/libki.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

serialization: Implement saving via JsonSerializer

This commit is contained in:
Joshua Scott 2018-12-15 16:52:36 +00:00
parent def6549bcb
commit 1a3dfbea48
12 changed files with 633 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
include/ki/serialization/BinarySerializer.h
View File

@ -64,19 +64,7 @@ namespace serialization
* @param object
* @param stream
*/
virtual void presave_object(const pclass::PropertyClass *object, BitStream &stream) const;
/**
* @param object
* @param stream
*/
void save_object(const pclass::PropertyClass *object, BitStream &stream) const;
/**
* @param prop
* @param stream
*/
void save_property(const pclass::IProperty &prop, BitStream &stream) const;
virtual bool presave_object(const pclass::PropertyClass *object, BitStream &stream) const;
/**
* Read an object header, and instantiate the necessary PropertyClass.
@ -85,19 +73,6 @@ namespace serialization
*/
virtual void preload_object(
std::unique_ptr<pclass::PropertyClass> &dest, BitStream &stream) const;
/**
* @param dest
* @param stream
*/
void load_object(
std::unique_ptr<pclass::PropertyClass> &dest, BitStream &stream) const;
/**
* @param prop
* @param stream
*/
void load_property(pclass::IProperty &prop, BitStream &stream) const;
private:
const pclass::TypeSystem *m_type_system;
@ -105,6 +80,16 @@ namespace serialization
flags m_flags;
const pclass::PropertyClass *m_root_object;
void save_object(const pclass::PropertyClass *object, BitStream &stream) const;
void save_property(const pclass::IProperty &prop, BitStream &stream) const;
void save_dynamic_property(
const pclass::IDynamicProperty &prop, BitStream &stream) const;
void load_object(
std::unique_ptr<pclass::PropertyClass> &dest, BitStream &stream) const;
void load_property(pclass::IProperty &prop, BitStream &stream) const;
void load_dynamic_property(pclass::IDynamicProperty &prop, BitStream &stream) const;
};
}
}

35
include/ki/serialization/JsonSerializer.h Normal file
View File

@ -0,0 +1,35 @@
#pragma once
#include <memory>
#include <json.hpp>
#include "ki/pclass/Property.h"
#include "ki/pclass/TypeSystem.h"
namespace ki
{
namespace serialization
{
/**
* TODO: Documentation
*/
class JsonSerializer
{
static const int FILE_INDENT_VALUE = 2;
public:
explicit JsonSerializer(pclass::TypeSystem &type_system, bool is_file);
virtual ~JsonSerializer() {}
std::string save(const pclass::PropertyClass *object) const;
void load(std::unique_ptr<pclass::PropertyClass> &dest, const std::string &json_string) const;
private:
pclass::TypeSystem *m_type_system;
bool m_is_file;
virtual bool presave_object(nlohmann::json &j, const pclass::PropertyClass *object) const;
nlohmann::json save_object(const pclass::PropertyClass *object) const;
void save_property(nlohmann::json &j, const pclass::IProperty &prop) const;
void save_dynamic_property(nlohmann::json &j, const pclass::IDynamicProperty &prop) const;
};
}
}

123
src/serialization/BinarySerializer.cpp
View File

@ -114,22 +114,20 @@ namespace serialization
}
}
void BinarySerializer::presave_object(const pclass::PropertyClass *object, BitStream &stream) const
bool BinarySerializer::presave_object(const pclass::PropertyClass *object, BitStream &stream) const
{
// If we have an object, write the type hash, otherwise, write NULL (0).
if (object)
stream.write<uint32_t>(object->get_type().get_hash());
else
stream.write<uint32_t>(NULL);
return object != nullptr;
}
void BinarySerializer::save_object(const pclass::PropertyClass *object, BitStream &stream) const
{
// Write any object headers
presave_object(object, stream);
// Make sure we have an object to write
if (!object)
if (!presave_object(object, stream))
return;
// Remember where we started writing the object data
@ -179,39 +177,17 @@ namespace serialization
}
// Is the property dynamic? (holding more than one value)
auto &property_type = prop.get_type();
if (prop.is_dynamic())
{
// Cast the property to a IDynamicProperty
const auto &dynamic_property =
dynamic_cast<const pclass::IDynamicProperty &>(prop);
// Write the number of elements
stream.write<uint32_t>(dynamic_property.get_element_count());
// Iterate through the elements
for (auto i = 0; i < dynamic_property.get_element_count(); i++)
{
// Is this a collection of pointers?
if (prop.is_pointer())
{
// Is the property a collection of pointers to other objects?
if (property_type.get_kind() == pclass::Type::kind::CLASS)
// Write the value as a nested object
save_object(dynamic_property.get_object(i), stream);
else
// Write the value as normal (let the property deal with dereferencing)
dynamic_property.write_value_to(stream, i);
}
else
// If the value isn't a pointer, and it's not dynamic, just write it as a value
dynamic_property.write_value_to(stream, i);
}
save_dynamic_property(dynamic_property, stream);
}
else if (prop.is_pointer())
{
// Does this property hold a pointer to another object?
if (property_type.get_kind() == pclass::Type::kind::CLASS)
if (prop.get_type().get_kind() == pclass::Type::kind::CLASS)
// Write the value as a nested object
save_object(prop.get_object(), stream);
else
@ -236,6 +212,32 @@ namespace serialization
}
}
void BinarySerializer::save_dynamic_property(
const pclass::IDynamicProperty& prop, BitStream& stream) const
{
// Write the number of elements
stream.write<uint32_t>(prop.get_element_count());
// Iterate through the elements
for (auto i = 0; i < prop.get_element_count(); i++)
{
// Is this a collection of pointers?
if (prop.is_pointer())
{
// Is the property a collection of pointers to other objects?
if (prop.get_type().get_kind() == pclass::Type::kind::CLASS)
// Write the value as a nested object
save_object(prop.get_object(i), stream);
else
// Write the value as normal (let the property deal with dereferencing)
prop.write_value_to(stream, i);
}
else
// If the value isn't a pointer, and it's not dynamic, just write it as a value
prop.write_value_to(stream, i);
}
}
void BinarySerializer::load(
std::unique_ptr<pclass::PropertyClass> &dest,
BitStream &stream, const std::size_t size)
@ -378,43 +380,16 @@ namespace serialization
void BinarySerializer::load_property(pclass::IProperty &prop, BitStream &stream) const
{
auto &property_type = prop.get_type();
if (prop.is_dynamic())
{
auto &dynamic_property =
dynamic_cast<pclass::IDynamicProperty &>(prop);
// How many elements are there in this dynamic property?
const auto element_count = stream.read<uint32_t>();
dynamic_property.set_element_count(element_count);
// Load each of the elements
for (uint16_t i = 0; i < element_count; i++)
{
// Is this a collection of pointers?
if (dynamic_property.is_pointer())
{
// Is the property a collection of pointers to other objects?
if (property_type.get_kind() == pclass::Type::kind::CLASS)
{
// Read the object as a nested object
std::unique_ptr<pclass::PropertyClass> object = nullptr;
load_object(object, stream);
dynamic_property.set_object(object, i);
}
else
// Read the value as normal (let the property deal with dereferencing)
dynamic_property.read_value_from(stream, i);
}
else
// If the value isn't a pointer, and it's not dynamic, just read it as a value
dynamic_property.read_value_from(stream, i);
}
load_dynamic_property(dynamic_property, stream);
}
else if (prop.is_pointer())
{
// Does this property hold a pointer to another object?
if (property_type.get_kind() == pclass::Type::kind::CLASS)
if (prop.get_type().get_kind() == pclass::Type::kind::CLASS)
{
// Read the object as a nested object
std::unique_ptr<pclass::PropertyClass> object = nullptr;
@ -429,5 +404,37 @@ namespace serialization
// If the value isn't a pointer, and it's not dynamic, just read it as a value
prop.read_value_from(stream);
}
void BinarySerializer::load_dynamic_property(
pclass::IDynamicProperty& prop, BitStream& stream) const
{
// How many elements are there in this dynamic property?
const auto element_count = stream.read<uint32_t>();
prop.set_element_count(element_count);
// Load each of the elements
for (uint16_t i = 0; i < element_count; i++)
{
// Is this a collection of pointers?
if (prop.is_pointer())
{
// Is the property a collection of pointers to other objects?
if (prop.get_type().get_kind() == pclass::Type::kind::CLASS)
{
// Read the object as a nested object
std::unique_ptr<pclass::PropertyClass> object = nullptr;
load_object(object, stream);
prop.set_object(object, i);
}
else
// Read the value as normal (let the property deal with dereferencing)
prop.read_value_from(stream, i);
}
else
// If the value isn't a pointer, and it's not dynamic, just read it as a value
prop.read_value_from(stream, i);
}
}
}
}

1
src/serialization/CMakeLists.txt
View File

@ -1,4 +1,5 @@
target_sources(${PROJECT_NAME}
PRIVATE
${PROJECT_SOURCE_DIR}/src/serialization/BinarySerializer.cpp
${PROJECT_SOURCE_DIR}/src/serialization/JsonSerializer.cpp
)

105
src/serialization/JsonSerializer.cpp Normal file
View File

@ -0,0 +1,105 @@
#include "ki/serialization/JsonSerializer.h"
using namespace nlohmann;
namespace ki
{
namespace serialization
{
JsonSerializer::JsonSerializer(pclass::TypeSystem& type_system,
const bool is_file)
{
m_type_system = &type_system;
m_is_file = is_file;
}
std::string JsonSerializer::save(const pclass::PropertyClass* object) const
{
return save_object(object).dump(
m_is_file ? FILE_INDENT_VALUE : -1,
' ',
true
);
}
bool JsonSerializer::presave_object(json& j, const pclass::PropertyClass* object) const
{
// Add the object's meta information
j["_pclass_meta"] = {
{ "type_hash", object ? object->get_type().get_hash() : NULL }
};
return object != nullptr;
}
json JsonSerializer::save_object(const pclass::PropertyClass* object) const
{
json j;
if (!presave_object(j, object))
return j;
// Add the object's properties
auto &property_list = object->get_properties();
for (auto it = property_list.begin();
it != property_list.end(); ++it)
{
auto &prop = *it;
save_property(j, prop);
}
return j;
}
void JsonSerializer::save_property(json& j,
const pclass::IProperty &prop) const
{
if (prop.is_dynamic())
{
// Cast the property to a IDynamicProperty
const auto &dynamic_property =
dynamic_cast<const pclass::IDynamicProperty &>(prop);
return save_dynamic_property(j, dynamic_property);
}
if (prop.get_type().get_kind() == pclass::Type::kind::CLASS)
{
auto *other_object = prop.get_object();
j[prop.get_name()] = save_object(other_object);
}
else
{
auto value = prop.get_value().dereference<json>();
j[prop.get_name()] = value.get<json>();
}
}
void JsonSerializer::save_dynamic_property(json& j,
const pclass::IDynamicProperty &prop) const
{
json property_value;
for (auto i = 0; i < prop.get_element_count(); ++i)
{
if (prop.get_type().get_kind() == pclass::Type::kind::CLASS)
{
auto *other_object = prop.get_object(i);
property_value.push_back(save_object(other_object));
}
else
{
auto value = prop.get_value(i).dereference<json>();
property_value.push_back(value.get<json>());
}
}
j[prop.get_name()] = property_value;
}
void JsonSerializer::load(std::unique_ptr<pclass::PropertyClass>& dest,
const std::string& json_string) const
{
// TODO: JSON Deserialization
auto j = json::parse(json_string);
dest = nullptr;
}
}
}

0
test/samples/serialization_binary_file.bin → test/samples/serialization/file.bin
View File

285
test/samples/serialization/file.json Normal file
View File

@ -0,0 +1,285 @@
{
"_pclass_meta": {
"type_hash": 2069042008
},
"collection": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35,
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53,
54,
55,
56,
57,
58,
59,
60,
61,
62,
63,
64,
65,
66,
67,
68,
69,
70,
71,
72,
73,
74,
75,
76,
77,
78,
79,
80,
81,
82,
83,
84,
85,
86,
87,
88,
89,
90,
91,
92,
93,
94,
95,
96,
97,
98,
99
],
"float32": 3.1415927410125732,
"float64": 3.141592653589793,
"int16": 515,
"int24": 263430,
"int32": 117967114,
"int4": -6,
"int64": 796025588171149586,
"int8": 1,
"int_ptr": 52,
"not_null_object": {
"_pclass_meta": {
"type_hash": 1181435066
},
"m_kind": 1
},
"null_object": {
"_pclass_meta": {
"type_hash": 0
}
},
"objects": [
{
"_pclass_meta": {
"type_hash": 1180894394
},
"extra_value": 10,
"m_kind": 2
},
{
"_pclass_meta": {
"type_hash": 1180943546
},
"m_kind": 3
}
],
"ptr_collection": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35,
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53,
54,
55,
56,
57,
58,
59,
60,
61,
62,
63,
64,
65,
66,
67,
68,
69,
70,
71,
72,
73,
74,
75,
76,
77,
78,
79,
80,
81,
82,
83,
84,
85,
86,
87,
88,
89,
90,
91,
92,
93,
94,
95,
96,
97,
98,
99
],
"string": "This is a test value",
"uint16": 515,
"uint24": 263430,
"uint32": 117967114,
"uint4": 5,
"uint64": 796025588171149586,
"uint8": 1,
"value_object": {
"_pclass_meta": {
"type_hash": 1180894394
},
"extra_value": 20,
"m_kind": 2
},
"vector3d": {
"x": 24.0,
"y": 61.0,
"z": 3.619999885559082
},
"wstring": [
84,
104,
105,
115,
32,
105,
115,
32,
97,
32,
116,
101,
115,
116,
32,
118,
97,
108,
117,
101
]
}

0
test/samples/serialization_binary_file_compressed.bin → test/samples/serialization/file_compressed.bin
View File

0
test/samples/serialization_binary_regular.bin → test/samples/serialization/regular.bin
View File

1
test/samples/serialization/regular.json Normal file
View File

@ -0,0 +1 @@
{"_pclass_meta":{"type_hash":2069042008},"collection":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],"float32":3.1415927410125732,"float64":3.141592653589793,"int16":515,"int24":263430,"int32":117967114,"int4":-6,"int64":796025588171149586,"int8":1,"int_ptr":52,"not_null_object":{"_pclass_meta":{"type_hash":1181435066},"m_kind":1},"null_object":{"_pclass_meta":{"type_hash":0}},"objects":[{"_pclass_meta":{"type_hash":1180894394},"extra_value":10,"m_kind":2},{"_pclass_meta":{"type_hash":1180943546},"m_kind":3}],"ptr_collection":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],"string":"This is a test value","uint16":515,"uint24":263430,"uint32":117967114,"uint4":5,"uint64":796025588171149586,"uint8":1,"value_object":{"_pclass_meta":{"type_hash":1180894394},"extra_value":20,"m_kind":2},"vector3d":{"x":24.0,"y":61.0,"z":3.619999885559082},"wstring":[84,104,105,115,32,105,115,32,97,32,116,101,115,116,32,118,97,108,117,101]}

0
test/samples/serialization_binary_regular_compressed.bin → test/samples/serialization/regular_compressed.bin
View File

136
test/src/unit-serialization.cpp
View File

@ -9,6 +9,7 @@
#include <ki/pclass/VectorProperty.h>
#include <ki/serialization/BinarySerializer.h>
#include "ki/util/unique.h"
#include "ki/serialization/JsonSerializer.h"
using namespace ki;
@ -17,7 +18,11 @@ using namespace ki;
*/
struct Vector3D
{
Vector3D(
// Allow json caster to access private members
friend pclass::detail::value_caster<Vector3D, nlohmann::json>;
friend pclass::detail::value_caster<nlohmann::json, Vector3D>;
explicit Vector3D(
const float x = 0.0f,
const float y = 0.0f,
const float z = 0.0f)
@ -95,6 +100,45 @@ namespace detail
return Value::make_value<Vector3D>(value);
}
};
/**
* value_caster specialization for casting Vector3D to json object.
*/
template <>
struct value_caster<Vector3D, nlohmann::json>
: value_caster_impl<Vector3D, nlohmann::json>
{
Value cast(const Value &value) const override
{
auto &vector = value.get<Vector3D>();
const nlohmann::json j = {
{ "x", vector.m_x },
{ "y", vector.m_y },
{ "z", vector.m_z }
};
return Value::make_value<nlohmann::json>(j);
}
};
/**
* value_caster specialization for casting json object to Vector3D.
*/
template <>
struct value_caster<nlohmann::json, Vector3D>
: value_caster_impl<nlohmann::json, Vector3D>
{
Value cast(const Value &value) const override
{
auto &j = value.get<nlohmann::json>();
return Value::make_value<Vector3D>(
Vector3D(
j["x"].get<float>(),
j["y"].get<float>(),
j["z"].get<float>()
)
);
}
};
}
}
}
@ -276,6 +320,10 @@ void define_types()
g_type_system->define_class<NestedTestObjectA>("class NestedTestObjectA", nested_test_object);
g_type_system->define_class<NestedTestObjectB>("class NestedTestObjectB", nested_test_object);
g_type_system->define_class<TestObject>("class TestObject");
pclass::ValueCaster::declare<Vector3D, nlohmann::json>();
pclass::ValueCaster::declare<nlohmann::json, Vector3D>();
g_types_defined = true;
}
@ -407,7 +455,7 @@ void test_serializer(
// Open the sample data
std::ifstream sample(
"samples/serialization_binary" + file_suffix + ".bin",
"samples/serialization/" + file_suffix + ".bin",
std::ios::binary
);
REQUIRE(sample.is_open());
@ -468,6 +516,67 @@ void test_serializer(
delete[] sample_data;
}
/**
* Conduct save/load tests with a BinarySerializer instance.
*/
void test_serializer(
std::unique_ptr<TestObject> &test_object,
serialization::JsonSerializer &serializer,
const std::string &file_suffix)
{
// Open the sample data
std::ifstream sample_file(
"samples/serialization/" + file_suffix + ".json",
std::ios::binary
);
REQUIRE(sample_file.is_open());
// Load the sample data into a buffer
const auto begin = sample_file.tellg();
sample_file.seekg(0, std::ios::end);
const auto end = sample_file.tellg();
const size_t sample_size = end - begin;
sample_file.seekg(std::ios::beg);
auto *sample_data = new char[sample_size];
sample_file.read(sample_data, sample_size);
sample_file.close();
// Load the sample data into a string
const auto sample = std::string(sample_data, sample_size);
delete[] sample_data;
SECTION("Saving objects")
{
// Create a test object, configure it, and write it to our stream
test_object = g_type_system->instantiate<TestObject>("class TestObject");
configure_test_object(*test_object);
const auto json_string = serializer.save(test_object.get());
// Delete the test object here so that it is not
// unnecessarily validated by the caller
test_object = nullptr;
// Validate the JSON string
REQUIRE(json_string == sample);
}
SECTION("Loading objects")
{
// Load an object from the sample
std::unique_ptr<pclass::PropertyClass> object = nullptr;
serializer.load(object, sample);
// Set test_object so that it is validated by the caller
/*
REQUIRE(object != nullptr);
test_object = std::unique_ptr<TestObject>(
dynamic_cast<TestObject *>(object.release())
);
REQUIRE(test_object != nullptr);
*/
}
}
TEST_CASE("Serialization tests", "[serialization]")
{
std::unique_ptr<TestObject> test_object = nullptr;
@ -481,7 +590,7 @@ TEST_CASE("Serialization tests", "[serialization]")
*g_type_system.get(), false,
serialization::BinarySerializer::flags::NONE
);
test_serializer(test_object, serializer, "_regular");
test_serializer(test_object, serializer, "regular");
}
SECTION("File format without compression")
{
@ -489,7 +598,7 @@ TEST_CASE("Serialization tests", "[serialization]")
*g_type_system.get(), true,
serialization::BinarySerializer::flags::WRITE_SERIALIZER_FLAGS
);
test_serializer(test_object, serializer, "_file");
test_serializer(test_object, serializer, "file");
}
SECTION("Regular format with compression")
{
@ -497,7 +606,7 @@ TEST_CASE("Serialization tests", "[serialization]")
*g_type_system.get(), false,
serialization::BinarySerializer::flags::COMPRESSED
);
test_serializer(test_object, serializer, "_regular_compressed");
test_serializer(test_object, serializer, "regular_compressed");
}
SECTION("File format with compression")
{
@ -506,7 +615,22 @@ TEST_CASE("Serialization tests", "[serialization]")
serialization::BinarySerializer::flags::WRITE_SERIALIZER_FLAGS |
serialization::BinarySerializer::flags::COMPRESSED
);
test_serializer(test_object, serializer, "_file_compressed");
test_serializer(test_object, serializer, "file_compressed");
}
}
SECTION("JsonSerializer")
{
SECTION("Regular format")
{
serialization::JsonSerializer serializer(*g_type_system, false);
test_serializer(test_object, serializer, "regular");
}
SECTION("File format")
{
serialization::JsonSerializer serializer(*g_type_system, true);
test_serializer(test_object, serializer, "file");
}
}