Fix #116 Verify Database Serialization Version
This commit is contained in:
Daniel Larimer
parent
ce3fbbf93e
commit
be9c771254
4 changed files with 350 additions and 6 deletions
|
|
@ -1,4 +1,4 @@
|
||||||
file(GLOB HEADERS "include/graphene/db/*.hpp")
|
file(GLOB HEADERS "include/graphene/db/*.hpp")
|
||||||
add_library( graphene_db undo_database.cpp index.cpp object_database.cpp ${HEADERS} )
|
add_library( graphene_db undo_database.cpp index.cpp object_database.cpp type_serializer.cpp ${HEADERS} )
|
||||||
target_link_libraries( graphene_db fc )
|
target_link_libraries( graphene_db fc )
|
||||||
target_include_directories( graphene_db PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" )
|
target_include_directories( graphene_db PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" )
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,7 @@
|
||||||
*/
|
*/
|
||||||
#pragma once
|
#pragma once
|
||||||
#include <graphene/db/object.hpp>
|
#include <graphene/db/object.hpp>
|
||||||
|
#include <graphene/db/type_serializer.hpp>
|
||||||
#include <fc/interprocess/file_mapping.hpp>
|
#include <fc/interprocess/file_mapping.hpp>
|
||||||
#include <fc/io/raw.hpp>
|
#include <fc/io/raw.hpp>
|
||||||
#include <fc/io/json.hpp>
|
#include <fc/io/json.hpp>
|
||||||
|
|
@ -209,11 +210,8 @@ namespace graphene { namespace db {
|
||||||
|
|
||||||
fc::sha256 get_object_version()const
|
fc::sha256 get_object_version()const
|
||||||
{
|
{
|
||||||
// TODO: use something other than json to describe the type and detect changes in serialization
|
auto desc = get_type_description<object_type>();
|
||||||
// json will only detect adding, removing, or renaming fields but will not detect changing types
|
return fc::sha256::hash(desc);
|
||||||
// or changes in the contents of optional values.
|
|
||||||
auto json_obj = fc::json::to_string( object_type() );
|
|
||||||
return fc::sha256::hash(json_obj);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
virtual void open( const path& db )override
|
virtual void open( const path& db )override
|
||||||
|
|
|
||||||
283
libraries/db/include/graphene/db/type_serializer.hpp
Normal file
283
libraries/db/include/graphene/db/type_serializer.hpp
Normal file
|
|
@ -0,0 +1,283 @@
|
||||||
|
#include <fc/reflect/variant.hpp>
|
||||||
|
#include <fc/crypto/ripemd160.hpp>
|
||||||
|
#include <fc/crypto/sha224.hpp>
|
||||||
|
#include <sstream>
|
||||||
|
#include <set>
|
||||||
|
|
||||||
|
namespace graphene { namespace db {
|
||||||
|
using std::set;
|
||||||
|
using std::map;
|
||||||
|
using fc::time_point_sec;
|
||||||
|
using fc::time_point;
|
||||||
|
using std::string;
|
||||||
|
|
||||||
|
std::set<std::string>& processed_types();
|
||||||
|
std::stringstream& current_stream( std::unique_ptr<std::stringstream>&& s = std::unique_ptr<std::stringstream>() );
|
||||||
|
string remove_tail_if( const string& str, char c, const string& match );
|
||||||
|
string remove_namespace_if( const string& str, const string& match );
|
||||||
|
string remove_namespace( string str );
|
||||||
|
|
||||||
|
template<typename T>
|
||||||
|
void generate_serializer();
|
||||||
|
template<typename T>
|
||||||
|
void register_serializer();
|
||||||
|
|
||||||
|
extern map<string, size_t > st;
|
||||||
|
/*
|
||||||
|
extern vector<std::function<void()>> serializers;
|
||||||
|
*/
|
||||||
|
|
||||||
|
bool register_serializer( const string& name, std::function<void()> sr );
|
||||||
|
|
||||||
|
template<typename T> struct js_name { static std::string name(){ return remove_namespace(fc::get_typename<T>::name()); }; };
|
||||||
|
|
||||||
|
template<typename T, size_t N>
|
||||||
|
struct js_name<fc::array<T,N>>
|
||||||
|
{
|
||||||
|
static std::string name(){ return "fixed_array "+ fc::to_string(N) + ", " + remove_namespace(fc::get_typename<T>::name()); };
|
||||||
|
};
|
||||||
|
template<size_t N> struct js_name<fc::array<char,N>> { static std::string name(){ return "bytes "+ fc::to_string(N); }; };
|
||||||
|
template<size_t N> struct js_name<fc::array<uint8_t,N>> { static std::string name(){ return "bytes "+ fc::to_string(N); }; };
|
||||||
|
template<typename T> struct js_name< fc::optional<T> > { static std::string name(){ return "optional " + js_name<T>::name(); } };
|
||||||
|
template<> struct js_name< object_id_type > { static std::string name(){ return "object_id_type"; } };
|
||||||
|
template<typename T> struct js_name< fc::flat_set<T> > { static std::string name(){ return "set " + js_name<T>::name(); } };
|
||||||
|
template<typename T> struct js_name< std::vector<T> > { static std::string name(){ return "array " + js_name<T>::name(); } };
|
||||||
|
template<typename T> struct js_name< fc::safe<T> > { static std::string name(){ return js_name<T>::name(); } };
|
||||||
|
|
||||||
|
|
||||||
|
template<> struct js_name< std::vector<char> > { static std::string name(){ return "bytes()"; } };
|
||||||
|
//template<> struct js_name< op_wrapper > { static std::string name(){ return "operation "; } };
|
||||||
|
template<> struct js_name<fc::uint160> { static std::string name(){ return "bytes 20"; } };
|
||||||
|
template<> struct js_name<fc::sha224> { static std::string name(){ return "bytes 28"; } };
|
||||||
|
template<> struct js_name<fc::unsigned_int> { static std::string name(){ return "varuint32"; } };
|
||||||
|
template<> struct js_name<fc::signed_int> { static std::string name(){ return "varint32"; } };
|
||||||
|
template<> struct js_name< time_point_sec > { static std::string name(){ return "time_point_sec"; } };
|
||||||
|
|
||||||
|
template<uint8_t S, uint8_t T, typename O>
|
||||||
|
struct js_name<graphene::db::object_id<S,T,O> >
|
||||||
|
{
|
||||||
|
static std::string name(){
|
||||||
|
return "protocol_id_type \"" + remove_namespace(fc::get_typename<O>::name()) + "\"";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
template<typename T> struct js_name< std::set<T> > { static std::string name(){ return "set " + js_name<T>::name(); } };
|
||||||
|
|
||||||
|
template<typename K, typename V>
|
||||||
|
struct js_name< std::map<K,V> > { static std::string name(){ return "map (" + js_name<K>::name() + "), (" + js_name<V>::name() +")"; } };
|
||||||
|
|
||||||
|
template<typename K, typename V>
|
||||||
|
struct js_name< fc::flat_map<K,V> > { static std::string name(){ return "map (" + js_name<K>::name() + "), (" + js_name<V>::name() +")"; } };
|
||||||
|
|
||||||
|
|
||||||
|
template<typename... T> struct js_sv_name;
|
||||||
|
|
||||||
|
template<typename A> struct js_sv_name<A>
|
||||||
|
{ static std::string name(){ return "\n " + js_name<A>::name(); } };
|
||||||
|
|
||||||
|
template<typename A, typename... T>
|
||||||
|
struct js_sv_name<A,T...> { static std::string name(){ return "\n " + js_name<A>::name() +" " + js_sv_name<T...>::name(); } };
|
||||||
|
|
||||||
|
|
||||||
|
template<typename... T>
|
||||||
|
struct js_name< fc::static_variant<T...> >
|
||||||
|
{
|
||||||
|
static std::string name( std::string n = ""){
|
||||||
|
static const std::string name = n;
|
||||||
|
if( name == "" )
|
||||||
|
return "static_variant [" + js_sv_name<T...>::name() + "\n]";
|
||||||
|
else return name;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
template<typename T, bool reflected = fc::reflector<T>::is_defined::value>
|
||||||
|
struct serializer;
|
||||||
|
|
||||||
|
|
||||||
|
struct register_type_visitor
|
||||||
|
{
|
||||||
|
typedef void result_type;
|
||||||
|
|
||||||
|
template<typename Type>
|
||||||
|
result_type operator()( const Type& op )const { serializer<Type>::init(); }
|
||||||
|
};
|
||||||
|
|
||||||
|
class register_member_visitor;
|
||||||
|
|
||||||
|
struct serialize_type_visitor
|
||||||
|
{
|
||||||
|
typedef void result_type;
|
||||||
|
|
||||||
|
fc::mutable_variant_object& obj;
|
||||||
|
int t = 0;
|
||||||
|
serialize_type_visitor( fc::mutable_variant_object& o, int _t ):obj(o),t(_t){}
|
||||||
|
|
||||||
|
template<typename Type>
|
||||||
|
result_type operator()( const Type& op )const
|
||||||
|
{
|
||||||
|
obj(remove_namespace( fc::get_typename<Type>::name()),t);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
class serialize_member_visitor
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
template<typename Member, class Class, Member (Class::*member)>
|
||||||
|
void operator()( const char* name )const
|
||||||
|
{
|
||||||
|
current_stream() << name << " : " << "";//js_name<Member>::name();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
template<typename T>
|
||||||
|
struct serializer<T,false>
|
||||||
|
{
|
||||||
|
static_assert( fc::reflector<T>::is_defined::value == false, "invalid template arguments" );
|
||||||
|
static void init()
|
||||||
|
{}
|
||||||
|
|
||||||
|
static void generate()
|
||||||
|
{}
|
||||||
|
};
|
||||||
|
|
||||||
|
template<typename T, size_t N>
|
||||||
|
struct serializer<fc::array<T,N>,false>
|
||||||
|
{
|
||||||
|
static void init() { serializer<T>::init(); }
|
||||||
|
static void generate() {}
|
||||||
|
};
|
||||||
|
template<typename T>
|
||||||
|
struct serializer<std::vector<T>,false>
|
||||||
|
{
|
||||||
|
static void init() { serializer<T>::init(); }
|
||||||
|
static void generate() {}
|
||||||
|
};
|
||||||
|
/*
|
||||||
|
template<>
|
||||||
|
struct serializer<std::vector<operation>,false>
|
||||||
|
{
|
||||||
|
static void init() { }
|
||||||
|
static void generate() {}
|
||||||
|
};
|
||||||
|
*/
|
||||||
|
|
||||||
|
template<>
|
||||||
|
struct serializer<object_id_type,true>
|
||||||
|
{
|
||||||
|
static void init() {}
|
||||||
|
|
||||||
|
static void generate() {}
|
||||||
|
};
|
||||||
|
template<>
|
||||||
|
struct serializer<uint64_t,false>
|
||||||
|
{
|
||||||
|
static void init() {}
|
||||||
|
static void generate() {}
|
||||||
|
};
|
||||||
|
#ifdef __APPLE__
|
||||||
|
// on mac, size_t is a distinct type from uint64_t or uint32_t and needs a separate specialization
|
||||||
|
template<> struct serializer<size_t,false> { static void init() {} static void generate() {} };
|
||||||
|
#endif
|
||||||
|
template<> struct serializer<int64_t,false> { static void init() {} static void generate() {} };
|
||||||
|
template<> struct serializer<int64_t,true> { static void init() {} static void generate() {} };
|
||||||
|
|
||||||
|
template<typename T>
|
||||||
|
struct serializer<fc::optional<T>,false>
|
||||||
|
{
|
||||||
|
static void init() { serializer<T>::init(); }
|
||||||
|
static void generate(){}
|
||||||
|
};
|
||||||
|
|
||||||
|
template<uint8_t SpaceID, uint8_t TypeID, typename T>
|
||||||
|
struct serializer< graphene::db::object_id<SpaceID,TypeID,T> ,true>
|
||||||
|
{
|
||||||
|
static void init() {}
|
||||||
|
static void generate() {}
|
||||||
|
};
|
||||||
|
|
||||||
|
template<typename... T>
|
||||||
|
struct serializer< fc::static_variant<T...>, false >
|
||||||
|
{
|
||||||
|
static void init()
|
||||||
|
{
|
||||||
|
auto name = js_name<fc::static_variant<T...>>::name();
|
||||||
|
if( processed_types().find( name ) == processed_types().end() )
|
||||||
|
{
|
||||||
|
processed_types().insert( name );
|
||||||
|
fc::static_variant<T...> var;
|
||||||
|
for( int i = 0; i < var.count(); ++i )
|
||||||
|
{
|
||||||
|
var.set_which(i);
|
||||||
|
var.visit( register_type_visitor() );
|
||||||
|
}
|
||||||
|
register_serializer( js_name<fc::static_variant<T...>>::name(), [=](){ generate(); } );
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void generate()
|
||||||
|
{
|
||||||
|
current_stream() << js_name<fc::static_variant<T...>>::name() << " = static_variant [" + js_sv_name<T...>::name() + "\n]\n\n";
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
class register_member_visitor
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
template<typename Member, class Class, Member (Class::*member)>
|
||||||
|
void operator()( const char* name )const
|
||||||
|
{
|
||||||
|
serializer<Member>::init();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
template<typename T, bool reflected>
|
||||||
|
struct serializer
|
||||||
|
{
|
||||||
|
static_assert( fc::reflector<T>::is_defined::value == reflected, "invalid template arguments" );
|
||||||
|
static void init()
|
||||||
|
{
|
||||||
|
auto name = js_name<T>::name();
|
||||||
|
if( st.find(name) == st.end() )
|
||||||
|
{
|
||||||
|
fc::reflector<T>::visit( register_member_visitor() );
|
||||||
|
register_serializer( name, [=](){ generate(); } );
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void generate()
|
||||||
|
{
|
||||||
|
auto name = remove_namespace( js_name<T>::name() );
|
||||||
|
if( name == "int64" ) return;
|
||||||
|
current_stream() << "" << name
|
||||||
|
<< " = new Serializer( \n"
|
||||||
|
<< " \"" + name + "\"\n";
|
||||||
|
|
||||||
|
fc::reflector<T>::visit( serialize_member_visitor() );
|
||||||
|
|
||||||
|
current_stream() <<")\n\n";
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
template<typename T>
|
||||||
|
std::string get_type_description()
|
||||||
|
{
|
||||||
|
current_stream( std::unique_ptr<std::stringstream>(new std::stringstream()) );
|
||||||
|
processed_types().clear();
|
||||||
|
serializer<T>::init();
|
||||||
|
return current_stream().str();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
} } // graphene::db
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
63
libraries/db/type_serializer.cpp
Normal file
63
libraries/db/type_serializer.cpp
Normal file
|
|
@ -0,0 +1,63 @@
|
||||||
|
#include <graphene/db/index.hpp>
|
||||||
|
|
||||||
|
namespace graphene { namespace db {
|
||||||
|
map<string, size_t > st;
|
||||||
|
vector<std::function<void()>> serializers;
|
||||||
|
|
||||||
|
std::set<std::string>& processed_types()
|
||||||
|
{
|
||||||
|
static std::set<std::string> p;
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::stringstream& current_stream( std::unique_ptr<std::stringstream>&& s )
|
||||||
|
{
|
||||||
|
static std::unique_ptr<std::stringstream> stream;
|
||||||
|
if( s ) stream = std::move(s);
|
||||||
|
FC_ASSERT( stream );
|
||||||
|
return *stream;
|
||||||
|
}
|
||||||
|
|
||||||
|
string remove_tail_if( const string& str, char c, const string& match )
|
||||||
|
{
|
||||||
|
auto last = str.find_last_of( c );
|
||||||
|
if( last != std::string::npos )
|
||||||
|
if( str.substr( last + 1 ) == match )
|
||||||
|
return str.substr( 0, last );
|
||||||
|
return str;
|
||||||
|
}
|
||||||
|
string remove_namespace_if( const string& str, const string& match )
|
||||||
|
{
|
||||||
|
auto last = str.find( match );
|
||||||
|
if( last != std::string::npos )
|
||||||
|
return str.substr( match.size()+2 );
|
||||||
|
return str;
|
||||||
|
}
|
||||||
|
|
||||||
|
string remove_namespace( string str )
|
||||||
|
{
|
||||||
|
str = remove_tail_if( str, '_', "operation" );
|
||||||
|
str = remove_tail_if( str, '_', "t" );
|
||||||
|
str = remove_tail_if( str, '_', "object" );
|
||||||
|
str = remove_tail_if( str, '_', "type" );
|
||||||
|
str = remove_namespace_if( str, "graphene::chain" );
|
||||||
|
str = remove_namespace_if( str, "graphene::db" );
|
||||||
|
str = remove_namespace_if( str, "std" );
|
||||||
|
str = remove_namespace_if( str, "fc" );
|
||||||
|
auto pos = str.find( ":" );
|
||||||
|
if( pos != str.npos )
|
||||||
|
str.replace( pos, 2, "_" );
|
||||||
|
return str;
|
||||||
|
}
|
||||||
|
bool register_serializer( const string& name, std::function<void()> sr )
|
||||||
|
{
|
||||||
|
if( st.find(name) == st.end() )
|
||||||
|
{
|
||||||
|
serializers.push_back( sr );
|
||||||
|
st[name] = serializers.size() - 1;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
} }// graphene
|
||||||
Loading…
Reference in a new issue