From abb4b9b0ab94cf904f784eb3eb92dc70ea016228 Mon Sep 17 00:00:00 2001 From: John Jones Date: Tue, 20 Feb 2018 15:18:47 -0500 Subject: [PATCH 01/46] Added editline as submodule --- .gitmodules | 3 + CMakeLists.txt | 25 ++++---- CMakeModules/FindEditline.cmake | 50 +++++++++++++++ CMakeModules/FindReadline.cmake | 49 --------------- src/rpc/cli.cpp | 106 +++++++++++++++++--------------- vendor/editline | 1 + 6 files changed, 124 insertions(+), 110 deletions(-) create mode 100644 CMakeModules/FindEditline.cmake delete mode 100644 CMakeModules/FindReadline.cmake create mode 160000 vendor/editline diff --git a/.gitmodules b/.gitmodules index 0c2a833..7abf7dd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "vendor/websocketpp"] path = vendor/websocketpp url = https://github.com/zaphoyd/websocketpp.git +[submodule "vendor/editline"] + path = vendor/editline + url = https://github.com/troglobit/editline.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f7787d..3766b1a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -251,24 +251,25 @@ add_subdirectory( vendor/websocketpp EXCLUDE_FROM_ALL ) setup_library( fc SOURCES ${sources} LIBRARY_TYPE STATIC ) install( DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/include/" DESTINATION include ) -# begin readline stuff -find_package(Curses) -find_package(Readline) +# begin editline stuff +#find_package(Curses) +set(EDITLINE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline" ) +find_package(Editline) file(GLOB HEADERS "include/bts/cli/*.hpp") -if (READLINE_FOUND) - target_compile_definitions (fc PRIVATE HAVE_READLINE) - set(readline_libraries ${Readline_LIBRARY}) - if (CURSES_FOUND) - list(APPEND readline_libraries ${CURSES_LIBRARY}) - endif() - set(readline_includes ${Readline_INCLUDE_DIR}) +if (EDITLINE_FOUND) + target_compile_definitions (fc PRIVATE HAVE_EDITLINE) + set(editline_libraries ${Editline_LIBRARY}) + #if (CURSES_FOUND) + # list(APPEND editline_libraries ${CURSES_LIBRARY}) + #endif() + set(editline_includes ${Editline_INCLUDE_DIR}) endif() if(WIN32) target_compile_definitions( fc PRIVATE _CRT_NONSTDC_NO_DEPRECATE ) endif(WIN32) -# end readline stuff +# end editline stuff if( NOT CPP_STANDARD ) set( CPP_STANDARD, "-std=c++11" ) @@ -362,7 +363,7 @@ target_include_directories(fc ${OPENSSL_INCLUDE_DIR} "vendor/diff-match-patch-cpp-stl" ${CMAKE_CURRENT_SOURCE_DIR}/vendor/websocketpp - "${readline_includes}" + "${editline_includes}" PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/vendor/boost_1.51/include diff --git a/CMakeModules/FindEditline.cmake b/CMakeModules/FindEditline.cmake new file mode 100644 index 0000000..3508fe1 --- /dev/null +++ b/CMakeModules/FindEditline.cmake @@ -0,0 +1,50 @@ +# - Try to find editline include dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Editline) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Editline_ROOT_DIR Set this variable to the root installation of +# editline if the module has problems finding the +# proper installation path. +# +# Variables defined by this module: +# +# EDITLINE_FOUND System has editline, include and lib dirs found +# Editline_INCLUDE_DIR The editline include directories. +# Editline_LIBRARY The editline library. + +find_path(Editline_ROOT_DIR + NAMES include/editline.h + HINTS ${EDITLINE_DIR} +) + +find_path(Editline_INCLUDE_DIR + NAMES editline.h + HINTS ${Editline_ROOT_DIR}/include +) + +find_library(Editline_LIBRARY + NAMES libeditline.a + HINTS ${Editline_ROOT_DIR}/src/.libs +) + +if(Editline_INCLUDE_DIR AND Editline_LIBRARY) + set(EDITLINE_FOUND TRUE) +else(Editline_INCLUDE_DIR AND Editline_LIBRARY) + FIND_LIBRARY(Editline_LIBRARY NAMES editline) + include(FindPackageHandleStandardArgs) + FIND_PACKAGE_HANDLE_STANDARD_ARGS(Editline DEFAULT_MSG Editline_INCLUDE_DIR Editline_LIBRARY ) + MARK_AS_ADVANCED(Editline_INCLUDE_DIR Editline_LIBRARY) +endif(Editline_INCLUDE_DIR AND Editline_LIBRARY) + +mark_as_advanced( + Editline_ROOT_DIR + Editline_INCLUDE_DIR + Editline_LIBRARY +) + +MESSAGE( STATUS "Found Editline: ${Editline_LIBRARY}" ) diff --git a/CMakeModules/FindReadline.cmake b/CMakeModules/FindReadline.cmake deleted file mode 100644 index f1d0d74..0000000 --- a/CMakeModules/FindReadline.cmake +++ /dev/null @@ -1,49 +0,0 @@ -# - Try to find readline include dirs and libraries -# -# Usage of this module as follows: -# -# find_package(Readline) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# Readline_ROOT_DIR Set this variable to the root installation of -# readline if the module has problems finding the -# proper installation path. -# -# Variables defined by this module: -# -# READLINE_FOUND System has readline, include and lib dirs found -# Readline_INCLUDE_DIR The readline include directories. -# Readline_LIBRARY The readline library. - -find_path(Readline_ROOT_DIR - NAMES include/readline/readline.h -) - -find_path(Readline_INCLUDE_DIR - NAMES readline/readline.h - HINTS ${Readline_ROOT_DIR}/include -) - -find_library(Readline_LIBRARY - NAMES readline - HINTS ${Readline_ROOT_DIR}/lib -) - -if(Readline_INCLUDE_DIR AND Readline_LIBRARY AND Ncurses_LIBRARY) - set(READLINE_FOUND TRUE) -else(Readline_INCLUDE_DIR AND Readline_LIBRARY AND Ncurses_LIBRARY) - FIND_LIBRARY(Readline_LIBRARY NAMES readline) - include(FindPackageHandleStandardArgs) - FIND_PACKAGE_HANDLE_STANDARD_ARGS(Readline DEFAULT_MSG Readline_INCLUDE_DIR Readline_LIBRARY ) - MARK_AS_ADVANCED(Readline_INCLUDE_DIR Readline_LIBRARY) -endif(Readline_INCLUDE_DIR AND Readline_LIBRARY AND Ncurses_LIBRARY) - -mark_as_advanced( - Readline_ROOT_DIR - Readline_INCLUDE_DIR - Readline_LIBRARY -) - -MESSAGE( STATUS "Found Readline: ${Readline_LIBRARY}" ) diff --git a/src/rpc/cli.cpp b/src/rpc/cli.cpp index d3070fb..487e444 100644 --- a/src/rpc/cli.cpp +++ b/src/rpc/cli.cpp @@ -7,22 +7,8 @@ #include #endif -#ifdef HAVE_READLINE -# include -# include -// I don't know exactly what version of readline we need. I know the 4.2 version that ships on some macs is -// missing some functions we require. We're developing against 6.3, but probably anything in the 6.x -// series is fine -# if RL_VERSION_MAJOR < 6 -# ifdef _MSC_VER -# pragma message("You have an old version of readline installed that might not support some of the features we need") -# pragma message("Readline support will not be compiled in") -# else -# warning "You have an old version of readline installed that might not support some of the features we need" -# warning "Readline support will not be compiled in" -# endif -# undef HAVE_READLINE -# endif +#ifdef HAVE_EDITLINE +# include "editline.h" # ifdef WIN32 # include # endif @@ -134,53 +120,73 @@ char * dupstr (const char* s) { return (r); } -char* my_generator(const char* text, int state) +/**** + * @brief loop through list of commands, attempting to find a match + * @param token what the user typed + * @param match + * @returns the full name of the command or NULL if no matches found + */ +static char *my_rl_complete(char *token, int *match) { - static int list_index, len; - const char *name; + int matchlen = 0; + int count = 0; + const char* method_name; - if (!state) { - list_index = 0; - len = strlen (text); - } + auto& cmd = cli_commands(); + for (auto it = cmd.begin(); it != cmd.end(); it++) { + int partlen = strlen (token); /* Part of token */ - auto& cmd = cli_commands(); + if (!strncmp ((*it).c_str(), token, partlen)) { + method_name = (*it).c_str(); + matchlen = partlen; + count ++; + } + } - while( list_index < cmd.size() ) - { - name = cmd[list_index].c_str(); - list_index++; + if (count == 1) { + *match = 1; + return strdup (method_name + matchlen); + } - if (strncmp (name, text, len) == 0) - return (dupstr(name)); - } - - /* If no names matched, then return NULL. */ - return ((char *)NULL); + return NULL; } - -static char** cli_completion( const char * text , int start, int end) +/*** + * @brief return an array of matching commands + * @param token the incoming text + * @param av the resultant array of possible matches + * @returns the number of matches + */ +static int cli_completion(char *token, char ***av) { - char **matches; - matches = (char **)NULL; + int num, total = 0; + char **copy; -#ifdef HAVE_READLINE - if (start == 0) - matches = rl_completion_matches ((char*)text, &my_generator); - else - rl_bind_key('\t',rl_abort); -#endif + auto& cmd = cli_commands(); + num = cmd.size(); - return (matches); + copy = (char **) malloc (num * sizeof(char *)); + for (auto it = cmd.begin(); it != cmd.end(); ++it) { + if (!strncmp ((*it).c_str(), token, strlen (token))) { + copy[total] = strdup ( (*it).c_str() ); + total ++; + } + } + *av = copy; + + return total; } - +/*** + * @brief Read input from the user + * @param prompt the prompt to display + * @param line what the user typed + */ void cli::getline( const fc::string& prompt, fc::string& line) { // getting file descriptor for C++ streams is near impossible // so we just assume it's the same as the C stream... -#ifdef HAVE_READLINE +#ifdef HAVE_EDITLINE #ifndef WIN32 if( isatty( fileno( stdin ) ) ) #else @@ -192,7 +198,9 @@ void cli::getline( const fc::string& prompt, fc::string& line) if( _isatty( _fileno( stdin ) ) ) #endif { - rl_attempted_completion_function = cli_completion; + //rl_attempted_completion_function = cli_completion; + rl_set_complete_func(my_rl_complete); + rl_set_list_possib_func(cli_completion); static fc::thread getline_thread("getline"); getline_thread.async( [&](){ @@ -201,7 +209,7 @@ void cli::getline( const fc::string& prompt, fc::string& line) line_read = readline(prompt.c_str()); if( line_read == nullptr ) FC_THROW_EXCEPTION( fc::eof_exception, "" ); - rl_bind_key( '\t', rl_complete ); + //el_bind_key( '\t', rl_complete ); if( *line_read ) add_history(line_read); line = line_read; diff --git a/vendor/editline b/vendor/editline new file mode 160000 index 0000000..5be965d --- /dev/null +++ b/vendor/editline @@ -0,0 +1 @@ +Subproject commit 5be965deec0527b48b393b7aca9cb2a76aec2a99 From 79cf3d2b3a5dfa994d2fa9e416061eeeadba3f2d Mon Sep 17 00:00:00 2001 From: John Jones Date: Mon, 26 Feb 2018 16:43:54 -0500 Subject: [PATCH 02/46] code cleanup --- .gitmodules | 4 ++-- src/rpc/cli.cpp | 22 ++++++---------------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/.gitmodules b/.gitmodules index 7abf7dd..69f6e7c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,5 +8,5 @@ path = vendor/websocketpp url = https://github.com/zaphoyd/websocketpp.git [submodule "vendor/editline"] - path = vendor/editline - url = https://github.com/troglobit/editline.git + path = vendor/editline + url = https://github.com/troglobit/editline.git diff --git a/src/rpc/cli.cpp b/src/rpc/cli.cpp index 487e444..5d7c903 100644 --- a/src/rpc/cli.cpp +++ b/src/rpc/cli.cpp @@ -111,15 +111,6 @@ void cli::run() } } - -char * dupstr (const char* s) { - char *r; - - r = (char*) malloc ((strlen (s) + 1)); - strcpy (r, s); - return (r); -} - /**** * @brief loop through list of commands, attempting to find a match * @param token what the user typed @@ -133,11 +124,11 @@ static char *my_rl_complete(char *token, int *match) const char* method_name; auto& cmd = cli_commands(); - for (auto it = cmd.begin(); it != cmd.end(); it++) { + for (auto it : cmd) { int partlen = strlen (token); /* Part of token */ - if (!strncmp ((*it).c_str(), token, partlen)) { - method_name = (*it).c_str(); + if (!strncmp (it.c_str(), token, partlen)) { + method_name = it.c_str(); matchlen = partlen; count ++; } @@ -166,9 +157,9 @@ static int cli_completion(char *token, char ***av) num = cmd.size(); copy = (char **) malloc (num * sizeof(char *)); - for (auto it = cmd.begin(); it != cmd.end(); ++it) { - if (!strncmp ((*it).c_str(), token, strlen (token))) { - copy[total] = strdup ( (*it).c_str() ); + for (auto it : cmd) { + if (!strncmp (it.c_str(), token, strlen (token))) { + copy[total] = strdup ( it.c_str() ); total ++; } } @@ -209,7 +200,6 @@ void cli::getline( const fc::string& prompt, fc::string& line) line_read = readline(prompt.c_str()); if( line_read == nullptr ) FC_THROW_EXCEPTION( fc::eof_exception, "" ); - //el_bind_key( '\t', rl_complete ); if( *line_read ) add_history(line_read); line = line_read; From 9d5a32c536d9adc0e73b95c14199f9266ceed6fe Mon Sep 17 00:00:00 2001 From: John Jones Date: Mon, 26 Feb 2018 16:47:38 -0500 Subject: [PATCH 03/46] Moved assignment from within a loop --- src/rpc/cli.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/rpc/cli.cpp b/src/rpc/cli.cpp index 5d7c903..dc63491 100644 --- a/src/rpc/cli.cpp +++ b/src/rpc/cli.cpp @@ -124,9 +124,8 @@ static char *my_rl_complete(char *token, int *match) const char* method_name; auto& cmd = cli_commands(); + int partlen = strlen (token); /* Part of token */ for (auto it : cmd) { - int partlen = strlen (token); /* Part of token */ - if (!strncmp (it.c_str(), token, partlen)) { method_name = it.c_str(); matchlen = partlen; From 046c65016b6f3b8f471e12895b69e37c22f945da Mon Sep 17 00:00:00 2001 From: John Jones Date: Mon, 26 Feb 2018 16:54:32 -0500 Subject: [PATCH 04/46] Replaced tab with spaces --- src/rpc/cli.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/rpc/cli.cpp b/src/rpc/cli.cpp index dc63491..caf85b7 100644 --- a/src/rpc/cli.cpp +++ b/src/rpc/cli.cpp @@ -188,9 +188,8 @@ void cli::getline( const fc::string& prompt, fc::string& line) if( _isatty( _fileno( stdin ) ) ) #endif { - //rl_attempted_completion_function = cli_completion; - rl_set_complete_func(my_rl_complete); - rl_set_list_possib_func(cli_completion); + rl_set_complete_func(my_rl_complete); + rl_set_list_possib_func(cli_completion); static fc::thread getline_thread("getline"); getline_thread.async( [&](){ From bbe2a8b289d1c8a1dce2a3702a9c4b81670fb96d Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Thu, 1 Mar 2018 18:58:23 +0100 Subject: [PATCH 05/46] JSON parsing fix from steem PR 2178 --- src/io/json.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/io/json.cpp b/src/io/json.cpp index a53b3a5..288f40d 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -390,8 +390,9 @@ namespace fc { skip_white_space(in); variant var; - while( signed char c = in.peek() ) + while( true ) { + signed char c = in.peek(); switch( c ) { case ' ': From b12205caf6d01a5062e6d73e4e8eac966c8ce454 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Fri, 2 Mar 2018 21:13:03 +0100 Subject: [PATCH 06/46] Minor optimization --- include/fc/smart_ref_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/fc/smart_ref_impl.hpp b/include/fc/smart_ref_impl.hpp index 7c31cea..fd69694 100644 --- a/include/fc/smart_ref_impl.hpp +++ b/include/fc/smart_ref_impl.hpp @@ -89,7 +89,7 @@ namespace fc { template T& smart_ref::operator = ( smart_ref&& u ) { if( &u == this ) return *impl; - if( impl ) delete impl; + delete impl; impl = u.impl; u.impl = nullptr; return *impl; From ade96c1213d2af39679183c51402121a2285518b Mon Sep 17 00:00:00 2001 From: abitmore Date: Mon, 5 Mar 2018 00:46:04 +0000 Subject: [PATCH 07/46] Build editline TODO: the cmake script is copied from the secp256k1 part in the same file, may need to review/fix the MSVC and MinGW part. --- CMakeLists.txt | 72 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3766b1a..575e5da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,6 +102,60 @@ else ( MSVC ) endif ( MSVC ) # End configure secp256k1-zkp +# Configure editline +if ( MSVC ) + # autoconf won't work here, hard code the defines + set( EDITLINE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline" ) + + file( GLOB EDITLINE_SOURCES "${EDITLINE_DIR}/src/editline.c" ) + add_library( editline ${EDITLINE_SOURCES} ) + + target_include_directories( editline PRIVATE "${EDITLINE_DIR}" PUBLIC "${EDITLINE_DIR}/include" ) + + set( EDITLINE_BUILD_DEFINES + USE_FIELD_10X26 + USE_FIELD_INV_BUILTIN + USE_NUM_NONE + USE_SCALAR_8X32 + USE_SCALAR_INV_BUILTIN ) + set_target_properties( editline PROPERTIES COMPILE_DEFINITIONS "${EDITLINE_BUILD_DEFINES}" LINKER_LANGUAGE C ) +else ( MSVC ) + include(ExternalProject) + if ( MINGW ) + ExternalProject_Add( project_editline + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/vendor/editline + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline + CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline/configure --prefix=${CMAKE_CURRENT_BINARY_DIR}/vendor/editline --with-bignum=no --host=x86_64-w64-mingw32 + BUILD_COMMAND make + INSTALL_COMMAND true + BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/vendor/editline/src/project_editline-build/.libs/libeditline.a + ) + else ( MINGW ) + ExternalProject_Add( project_editline + PREFIX ${CMAKE_CURRENT_BINARY_DIR}/vendor/editline + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline + CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline/configure --prefix=${CMAKE_CURRENT_BINARY_DIR}/vendor/editline --with-bignum=no + BUILD_COMMAND make + INSTALL_COMMAND true + BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/vendor/editline/src/project_editline-build/.libs/libeditline.a + ) + endif ( MINGW ) + ExternalProject_Add_Step(project_editline autogen + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline/autogen.sh + DEPENDERS configure + ) + + ExternalProject_Get_Property(project_editline binary_dir) + + add_library(editline STATIC IMPORTED) + set_property(TARGET editline PROPERTY IMPORTED_LOCATION ${binary_dir}/src/.libs/libeditline${CMAKE_STATIC_LIBRARY_SUFFIX}) + set_property(TARGET editline PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline/include) + add_dependencies(editline project_editline) + install( FILES ${binary_dir}/src/.libs/libeditline${CMAKE_STATIC_LIBRARY_SUFFIX} DESTINATION lib/cryptonomex ) +endif ( MSVC ) +# End configure editline + IF( WIN32 ) MESSAGE(STATUS "Configuring fc to build on Win32") @@ -252,20 +306,10 @@ setup_library( fc SOURCES ${sources} LIBRARY_TYPE STATIC ) install( DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/include/" DESTINATION include ) # begin editline stuff -#find_package(Curses) set(EDITLINE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline" ) -find_package(Editline) - file(GLOB HEADERS "include/bts/cli/*.hpp") - -if (EDITLINE_FOUND) - target_compile_definitions (fc PRIVATE HAVE_EDITLINE) - set(editline_libraries ${Editline_LIBRARY}) - #if (CURSES_FOUND) - # list(APPEND editline_libraries ${CURSES_LIBRARY}) - #endif() - set(editline_includes ${Editline_INCLUDE_DIR}) -endif() +target_compile_definitions (fc PRIVATE HAVE_EDITLINE) +set(editline_libraries editline) if(WIN32) target_compile_definitions( fc PRIVATE _CRT_NONSTDC_NO_DEPRECATE ) endif(WIN32) @@ -363,7 +407,7 @@ target_include_directories(fc ${OPENSSL_INCLUDE_DIR} "vendor/diff-match-patch-cpp-stl" ${CMAKE_CURRENT_SOURCE_DIR}/vendor/websocketpp - "${editline_includes}" + ${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/vendor/boost_1.51/include @@ -374,7 +418,7 @@ target_include_directories(fc IF(NOT WIN32) set(LINK_USR_LOCAL_LIB -L/usr/local/lib) ENDIF() -target_link_libraries( fc PUBLIC ${LINK_USR_LOCAL_LIB} ${Boost_LIBRARIES} ${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${PLATFORM_SPECIFIC_LIBS} ${RPCRT4} ${CMAKE_DL_LIBS} ${rt_library} ${readline_libraries} ${ECC_LIB} ) +target_link_libraries( fc PUBLIC ${LINK_USR_LOCAL_LIB} ${Boost_LIBRARIES} ${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${PLATFORM_SPECIFIC_LIBS} ${RPCRT4} ${CMAKE_DL_LIBS} ${rt_library} ${editline_libraries} ${ECC_LIB} ) if(MSVC) set_source_files_properties( src/network/http/websocket.cpp PROPERTIES COMPILE_FLAGS "/bigobj" ) From e766ea94290ec1ffa09ee711a3e54389a5b2deb7 Mon Sep 17 00:00:00 2001 From: John Jones Date: Mon, 5 Mar 2018 14:37:14 -0500 Subject: [PATCH 08/46] correctly handle list of methods --- src/rpc/cli.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/rpc/cli.cpp b/src/rpc/cli.cpp index caf85b7..3f0b065 100644 --- a/src/rpc/cli.cpp +++ b/src/rpc/cli.cpp @@ -114,20 +114,20 @@ void cli::run() /**** * @brief loop through list of commands, attempting to find a match * @param token what the user typed - * @param match - * @returns the full name of the command or NULL if no matches found + * @param match sets to 1 if only 1 match was found + * @returns the remaining letters of the name of the command or NULL if 1 match not found */ static char *my_rl_complete(char *token, int *match) { int matchlen = 0; int count = 0; - const char* method_name; + std::string method_name; auto& cmd = cli_commands(); int partlen = strlen (token); /* Part of token */ - for (auto it : cmd) { - if (!strncmp (it.c_str(), token, partlen)) { - method_name = it.c_str(); + for (std::string it : cmd) { + if (!strncmp ( it.c_str(), token, partlen)) { + method_name = it; matchlen = partlen; count ++; } @@ -135,7 +135,7 @@ static char *my_rl_complete(char *token, int *match) if (count == 1) { *match = 1; - return strdup (method_name + matchlen); + return strdup (method_name.c_str() + matchlen); } return NULL; From 5f3ace5ca31088f26e56c1cbe7353eabcb607a43 Mon Sep 17 00:00:00 2001 From: John Jones Date: Tue, 6 Mar 2018 08:06:18 -0500 Subject: [PATCH 09/46] Added space at end of completed command --- src/rpc/cli.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/rpc/cli.cpp b/src/rpc/cli.cpp index 3f0b065..b585bc2 100644 --- a/src/rpc/cli.cpp +++ b/src/rpc/cli.cpp @@ -126,16 +126,17 @@ static char *my_rl_complete(char *token, int *match) auto& cmd = cli_commands(); int partlen = strlen (token); /* Part of token */ for (std::string it : cmd) { - if (!strncmp ( it.c_str(), token, partlen)) { - method_name = it; - matchlen = partlen; - count ++; - } + if (!strncmp ( it.c_str(), token, partlen)) { + method_name = it; + matchlen = partlen; + count ++; + } } if (count == 1) { - *match = 1; - return strdup (method_name.c_str() + matchlen); + *match = 1; + method_name += " "; + return strdup (method_name.c_str() + matchlen); } return NULL; From 4258f4f98bbf7b1d3112aa8768782a6c936256bb Mon Sep 17 00:00:00 2001 From: abitmore Date: Tue, 6 Mar 2018 13:41:40 +0000 Subject: [PATCH 10/46] Update cmake file for editline --- CMakeLists.txt | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 575e5da..ac9715b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,20 +112,14 @@ if ( MSVC ) target_include_directories( editline PRIVATE "${EDITLINE_DIR}" PUBLIC "${EDITLINE_DIR}/include" ) - set( EDITLINE_BUILD_DEFINES - USE_FIELD_10X26 - USE_FIELD_INV_BUILTIN - USE_NUM_NONE - USE_SCALAR_8X32 - USE_SCALAR_INV_BUILTIN ) - set_target_properties( editline PROPERTIES COMPILE_DEFINITIONS "${EDITLINE_BUILD_DEFINES}" LINKER_LANGUAGE C ) + set_target_properties( editline PROPERTIES COMPILE_DEFINITIONS LINKER_LANGUAGE C ) else ( MSVC ) include(ExternalProject) if ( MINGW ) ExternalProject_Add( project_editline PREFIX ${CMAKE_CURRENT_BINARY_DIR}/vendor/editline SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline - CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline/configure --prefix=${CMAKE_CURRENT_BINARY_DIR}/vendor/editline --with-bignum=no --host=x86_64-w64-mingw32 + CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline/configure --prefix=${CMAKE_CURRENT_BINARY_DIR}/vendor/editline --host=x86_64-w64-mingw32 BUILD_COMMAND make INSTALL_COMMAND true BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/vendor/editline/src/project_editline-build/.libs/libeditline.a @@ -134,7 +128,7 @@ else ( MSVC ) ExternalProject_Add( project_editline PREFIX ${CMAKE_CURRENT_BINARY_DIR}/vendor/editline SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline - CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline/configure --prefix=${CMAKE_CURRENT_BINARY_DIR}/vendor/editline --with-bignum=no + CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline/configure --prefix=${CMAKE_CURRENT_BINARY_DIR}/vendor/editline BUILD_COMMAND make INSTALL_COMMAND true BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/vendor/editline/src/project_editline-build/.libs/libeditline.a @@ -307,7 +301,6 @@ install( DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/include/" DESTINATION include ) # begin editline stuff set(EDITLINE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline" ) -file(GLOB HEADERS "include/bts/cli/*.hpp") target_compile_definitions (fc PRIVATE HAVE_EDITLINE) set(editline_libraries editline) if(WIN32) From 0c757094884c0835e09c1e89cae950c644222925 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sat, 3 Mar 2018 11:00:21 +0100 Subject: [PATCH 11/46] Added unit tests for stringstream, [io]fstream and buffered_[io]stream --- tests/CMakeLists.txt | 1 + tests/io/stream_tests.cpp | 178 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 tests/io/stream_tests.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index bf7a01d..48f7a05 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -49,6 +49,7 @@ add_executable( all_tests all_tests.cpp crypto/dh_test.cpp crypto/rand_test.cpp crypto/sha_tests.cpp + io/stream_tests.cpp network/http/websocket_test.cpp thread/task_cancel.cpp thread/thread_tests.cpp diff --git a/tests/io/stream_tests.cpp b/tests/io/stream_tests.cpp new file mode 100644 index 0000000..038c419 --- /dev/null +++ b/tests/io/stream_tests.cpp @@ -0,0 +1,178 @@ +#include + +#include +#include +#include +#include +#include + +#include + +BOOST_AUTO_TEST_SUITE(stream_tests) + +BOOST_AUTO_TEST_CASE(stringstream_test) +{ + const fc::string constant( "Hello", 6 ); // includes trailing \0 + fc::string writable( "World" ); + fc::stringstream in1( constant ); + fc::stringstream in2( writable ); + fc::stringstream out; + + std::shared_ptr buf( new char[15], [](char* p){ delete[] p; } ); + *buf = 'w'; + in2.writesome( buf, 1, 0 ); + + BOOST_CHECK_EQUAL( 3, in1.readsome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 3, out.writesome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 'l', in1.peek() ); + BOOST_CHECK_EQUAL( 3, in1.readsome( buf, 4, 0 ) ); + BOOST_CHECK_EQUAL( '\0', (&(*buf))[2] ); + BOOST_CHECK_EQUAL( 2, out.writesome( buf, 2, 0 ) ); + *buf = ' '; + out.writesome( buf, 1, 0 ); + BOOST_CHECK_THROW( in1.readsome( buf, 3, 0 ), fc::eof_exception ); + BOOST_CHECK_EQUAL( 5, in2.readsome( buf, 6, 0 ) ); + BOOST_CHECK_EQUAL( 5, out.writesome( buf, 5, 0 ) ); + BOOST_CHECK_THROW( in2.readsome( buf, 3, 0 ), fc::eof_exception ); + + BOOST_CHECK_EQUAL( "Hello world", out.str() ); + BOOST_CHECK_THROW( in1.peek(), fc::eof_exception ); + BOOST_CHECK( in1.eof() ); + BOOST_CHECK_THROW( in2.readsome( buf, 3, 0 ), fc::eof_exception ); + // BOOST_CHECK( in2.eof() ); // fails, apparently readsome doesn't set eof +} + +BOOST_AUTO_TEST_CASE(buffered_stringstream_test) +{ + const fc::string constant( "Hello", 6 ); // includes trailing \0 + fc::string writable( "World" ); + fc::istream_ptr in1( new fc::stringstream( constant ) ); + std::shared_ptr in2( new fc::stringstream( writable ) ); + std::shared_ptr out1( new fc::stringstream() ); + fc::buffered_istream bin1( in1 ); + fc::buffered_istream bin2( in2 ); + fc::buffered_ostream bout( out1 ); + + std::shared_ptr buf( new char[15], [](char* p){ delete[] p; } ); + *buf = 'w'; + in2->writesome( buf, 1, 0 ); + + BOOST_CHECK_EQUAL( 3, bin1.readsome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 3, bout.writesome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 'l', bin1.peek() ); + BOOST_CHECK_EQUAL( 3, bin1.readsome( buf, 4, 0 ) ); + BOOST_CHECK_EQUAL( '\0', (&(*buf))[2] ); + BOOST_CHECK_EQUAL( 2, bout.writesome( buf, 2, 0 ) ); + *buf = ' '; + bout.writesome( buf, 1, 0 ); + BOOST_CHECK_THROW( bin1.readsome( buf, 3, 0 ), fc::eof_exception ); + BOOST_CHECK_EQUAL( 5, bin2.readsome( buf, 6, 0 ) ); + BOOST_CHECK_EQUAL( 5, bout.writesome( buf, 5, 0 ) ); + BOOST_CHECK_THROW( bin2.readsome( buf, 3, 0 ), fc::eof_exception ); + + bout.flush(); + + BOOST_CHECK_EQUAL( "Hello world", out1->str() ); +} + +BOOST_AUTO_TEST_CASE(fstream_test) +{ + fc::temp_file inf1( fc::temp_directory_path(), true ); + fc::temp_file inf2( fc::temp_directory_path(), true ); + fc::temp_file outf( fc::temp_directory_path(), true ); + + { + std::fstream init( inf1.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.write( "Hello", 6 ); // includes trailing \0 + init.close(); + + init.open( inf2.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.write( "world", 5 ); + init.close(); + + init.open( outf.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.close(); + } + + fc::ifstream in1( inf1.path() ); + fc::ifstream in2( inf2.path() ); + fc::ofstream out( outf.path() ); + + std::shared_ptr buf( new char[15], [](char* p){ delete[] p; } ); + BOOST_CHECK_EQUAL( 3, in1.readsome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 3, out.writesome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 3, in1.readsome( buf, 4, 0 ) ); + BOOST_CHECK_EQUAL( '\0', (&(*buf))[2] ); + BOOST_CHECK_EQUAL( 2, out.writesome( buf, 2, 0 ) ); + *buf = ' '; + out.writesome( buf, 1, 0 ); + BOOST_CHECK_THROW( in1.readsome( buf, 3, 0 ), fc::eof_exception ); + BOOST_CHECK_EQUAL( 5, in2.readsome( buf, 6, 0 ) ); + BOOST_CHECK_EQUAL( 5, out.writesome( buf, 5, 0 ) ); + BOOST_CHECK_THROW( in2.readsome( buf, 3, 0 ), fc::eof_exception ); + + { + out.flush(); + std::fstream test( outf.path().to_native_ansi_path(), std::fstream::in ); + BOOST_CHECK_EQUAL( 11, test.readsome( (&(*buf)), 11 ) ); + BOOST_CHECK_EQUAL( "Hello world", std::string( (&(*buf)), 11 ) ); + BOOST_CHECK_EQUAL( 0, test.readsome( (&(*buf)), 11 ) ); + test.close(); + } + + BOOST_CHECK( in1.eof() ); + BOOST_CHECK( in2.eof() ); +} + +BOOST_AUTO_TEST_CASE(buffered_fstream_test) +{ + fc::temp_file inf1( fc::temp_directory_path(), true ); + fc::temp_file inf2( fc::temp_directory_path(), true ); + fc::temp_file outf( fc::temp_directory_path(), true ); + + { + std::fstream init( inf1.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.write( "Hello", 6 ); // includes trailing \0 + init.close(); + + init.open( inf2.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.write( "world", 5 ); + init.close(); + + init.open( outf.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.close(); + } + + fc::istream_ptr in1( new fc::ifstream( inf1.path() ) ); + fc::istream_ptr in2( new fc::ifstream( inf2.path() ) ); + fc::ostream_ptr out( new fc::ofstream( outf.path() ) ); + fc::buffered_istream bin1( in1 ); + fc::buffered_istream bin2( in2 ); + fc::buffered_ostream bout( out ); + + std::shared_ptr buf( new char[15], [](char* p){ delete[] p; } ); + + BOOST_CHECK_EQUAL( 3, bin1.readsome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 3, bout.writesome( buf, 3, 0 ) ); + BOOST_CHECK_EQUAL( 'l', bin1.peek() ); + BOOST_CHECK_EQUAL( 3, bin1.readsome( buf, 4, 0 ) ); + BOOST_CHECK_EQUAL( '\0', (&(*buf))[2] ); + BOOST_CHECK_EQUAL( 2, bout.writesome( buf, 2, 0 ) ); + *buf = ' '; + bout.writesome( buf, 1, 0 ); + BOOST_CHECK_THROW( bin1.readsome( buf, 3, 0 ), fc::eof_exception ); + BOOST_CHECK_EQUAL( 5, bin2.readsome( buf, 6, 0 ) ); + BOOST_CHECK_EQUAL( 5, bout.writesome( buf, 5, 0 ) ); + BOOST_CHECK_THROW( bin2.readsome( buf, 3, 0 ), fc::eof_exception ); + + { + bout.flush(); + std::fstream test( outf.path().to_native_ansi_path(), std::fstream::in ); + BOOST_CHECK_EQUAL( 11, test.readsome( (&(*buf)), 11 ) ); + BOOST_CHECK_EQUAL( "Hello world", std::string( (&(*buf)), 11 ) ); + BOOST_CHECK_EQUAL( 0, test.readsome( (&(*buf)), 11 ) ); + test.close(); + } +} + +BOOST_AUTO_TEST_SUITE_END() From 72e96e3c1d53b4e8d54b1aa0d7b2146d18fa622d Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sat, 3 Mar 2018 18:40:02 +0100 Subject: [PATCH 12/46] Added first json test case --- tests/CMakeLists.txt | 1 + tests/io/json_tests.cpp | 117 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 tests/io/json_tests.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 48f7a05..fe1b412 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -49,6 +49,7 @@ add_executable( all_tests all_tests.cpp crypto/dh_test.cpp crypto/rand_test.cpp crypto/sha_tests.cpp + io/json_tests.cpp io/stream_tests.cpp network/http/websocket_test.cpp thread/task_cancel.cpp diff --git a/tests/io/json_tests.cpp b/tests/io/json_tests.cpp new file mode 100644 index 0000000..8db3e08 --- /dev/null +++ b/tests/io/json_tests.cpp @@ -0,0 +1,117 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include + +BOOST_AUTO_TEST_SUITE(json_tests) + +static void replace_some( std::string& str ) +{ + for( size_t i = 0; i < str.length(); i++ ) + if( str[i] == '\1' ) str[i] = '\0'; + else if( str[i] == '\'' ) str[i] = '"'; +} + +static void test_fail_string( const std::string& str ) +{ + try { + fc::json::from_string( str ); + BOOST_FAIL( "json::from_string('" + str + "') failed" ); + } catch( const fc::parse_error_exception& ) { // ignore, ok + } catch( const fc::eof_exception& ) { // ignore, ok + } FC_CAPTURE_LOG_AND_RETHROW( ("json::from_string failed")(str) ) +} + +static void test_fail_stream( const std::string& str ) +{ + fc::temp_file file( fc::temp_directory_path(), true ); + { + std::fstream init( file.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.write( str.c_str(), str.length() ); + init.close(); + } + try { + fc::istream_ptr in( new fc::ifstream( file.path() ) ); + fc::buffered_istream bin( in ); + fc::json::from_stream( bin ); + BOOST_FAIL( "json::from_stream('" + str + "') failed using ifstream" ); + } catch( const fc::parse_error_exception& ) { // ignore, ok + } catch( const fc::eof_exception& ) { // ignore, ok + } FC_CAPTURE_LOG_AND_RETHROW( ("json::from_stream failed using ifstream")(str) ) + try { + fc::istream_ptr in( new fc::stringstream( str ) ); + fc::buffered_istream bin( in ); + fc::json::from_stream( bin ); + BOOST_FAIL( "json::from_stream('" + str + "') failed using stringstream" ); + } catch( const fc::parse_error_exception& ) { // ignore, ok + } catch( const fc::eof_exception& ) { // ignore, ok + } FC_CAPTURE_LOG_AND_RETHROW( ("json::from_stream failed using stringstream")(str) ) +} + +static void test_fail_file( const std::string& str ) +{ + fc::temp_file file( fc::temp_directory_path(), true ); + { + std::fstream init( file.path().to_native_ansi_path(), std::fstream::out | std::fstream::trunc ); + init.write( str.c_str(), str.length() ); + init.close(); + } + try { + fc::json::from_file( file.path() ); + BOOST_FAIL( "json::from_file('" + str + "') failed using" ); + } catch( const fc::parse_error_exception& ) { // ignore, ok + } catch( const fc::eof_exception& ) { // ignore, ok + } FC_CAPTURE_LOG_AND_RETHROW( ("json::from_file failed")(str) ) +} + +BOOST_AUTO_TEST_CASE(imbalanced_test) +{ + std::vector tests + { // for easier handling and better readability, in the following test + // strings ' is used instead of " and \1 instead of \0 + "{", + "{'", + "{'}", + "{'a'", + "{'a':", + "{'a':5", + "[", + "['", + "[']", + "[ 13", + "' end", + "{ 13: }", + "{\1", + "{\1}", + "{'\1", + "{'\1}", + "{'a'\1", + "{'a'\1}", + "{'a': \1", + "{'a': \1}", + "[\1", + "[\1]", + "['\1", + "['\1]", + "[ 13\1", + "[ 13\1]", + "' end\1" + }; + + for( std::string test : tests ) + { + replace_some( test ); + test_fail_string( test ); + test_fail_stream( test ); + test_fail_file( test ); + } +} + + +BOOST_AUTO_TEST_SUITE_END() From 52f68107350e047ee040d6abf77d319e25d7743f Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sat, 3 Mar 2018 21:59:28 +0100 Subject: [PATCH 13/46] Fixed from_file --- src/io/json.cpp | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/src/io/json.cpp b/src/io/json.cpp index 288f40d..2c453cf 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -798,23 +798,9 @@ namespace fc } variant json::from_file( const fc::path& p, parse_type ptype ) { - //auto tmp = std::make_shared( p, ifstream::binary ); - //auto tmp = std::make_shared( p.generic_string().c_str(), std::ios::binary ); - //buffered_istream bi( tmp ); - boost::filesystem::ifstream bi( p, std::ios::binary ); - switch( ptype ) - { - case legacy_parser: - return variant_from_stream( bi ); - case legacy_parser_with_string_doubles: - return variant_from_stream( bi ); - case strict_parser: - return json_relaxed::variant_from_stream( bi ); - case relaxed_parser: - return json_relaxed::variant_from_stream( bi ); - default: - FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); - } + fc::istream_ptr in( new fc::ifstream( p ) ); + fc::buffered_istream bin( in ); + return from_stream( bin, ptype ); } variant json::from_stream( buffered_istream& in, parse_type ptype ) { From 4d782e894f9866e87ca82448526881666ebd3082 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sat, 3 Mar 2018 22:03:23 +0100 Subject: [PATCH 14/46] Deduplicate some code --- src/io/json.cpp | 41 +++++++---------------------------------- 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/src/io/json.cpp b/src/io/json.cpp index 2c453cf..ca1c7fb 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -461,21 +461,9 @@ namespace fc { try { check_string_depth( utf8_str ); - fc::stringstream in( utf8_str ); - //in.exceptions( std::ifstream::eofbit ); - switch( ptype ) - { - case legacy_parser: - return variant_from_stream( in ); - case legacy_parser_with_string_doubles: - return variant_from_stream( in ); - case strict_parser: - return json_relaxed::variant_from_stream( in ); - case relaxed_parser: - return json_relaxed::variant_from_stream( in ); - default: - FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); - } + fc::istream_ptr in( new fc::stringstream( utf8_str ) ); + fc::buffered_istream bin( in ); + return from_stream( bin, ptype ); } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) } variants json::variants_from_string( const std::string& utf8_str, parse_type ptype ) @@ -838,25 +826,10 @@ namespace fc bool json::is_valid( const std::string& utf8_str, parse_type ptype ) { if( utf8_str.size() == 0 ) return false; - fc::stringstream in( utf8_str ); - switch( ptype ) - { - case legacy_parser: - variant_from_stream( in ); - break; - case legacy_parser_with_string_doubles: - variant_from_stream( in ); - break; - case strict_parser: - json_relaxed::variant_from_stream( in ); - break; - case relaxed_parser: - json_relaxed::variant_from_stream( in ); - break; - default: - FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); - } - try { in.peek(); } catch ( const eof_exception& e ) { return true; } + fc::istream_ptr in( new fc::stringstream( utf8_str ) ); + fc::buffered_istream bin( in ); + from_stream( bin, ptype ); + try { bin.peek(); } catch ( const eof_exception& e ) { return true; } return false; } From d33092f86c916b146c50382a6a040f3dbbe5d451 Mon Sep 17 00:00:00 2001 From: abitmore Date: Tue, 6 Mar 2018 13:47:45 +0000 Subject: [PATCH 15/46] More changes on cmake file for editline --- CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ac9715b..7b9c7e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -300,7 +300,6 @@ setup_library( fc SOURCES ${sources} LIBRARY_TYPE STATIC ) install( DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/include/" DESTINATION include ) # begin editline stuff -set(EDITLINE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline" ) target_compile_definitions (fc PRIVATE HAVE_EDITLINE) set(editline_libraries editline) if(WIN32) @@ -400,7 +399,6 @@ target_include_directories(fc ${OPENSSL_INCLUDE_DIR} "vendor/diff-match-patch-cpp-stl" ${CMAKE_CURRENT_SOURCE_DIR}/vendor/websocketpp - ${CMAKE_CURRENT_SOURCE_DIR}/vendor/editline PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/vendor/boost_1.51/include From f5d68e933652aa5a3395fe6fb1762e41cb664ae7 Mon Sep 17 00:00:00 2001 From: John Jones Date: Tue, 6 Mar 2018 09:25:38 -0500 Subject: [PATCH 16/46] Adjusting for formatting guidelines --- src/rpc/cli.cpp | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/rpc/cli.cpp b/src/rpc/cli.cpp index b585bc2..f85b7fd 100644 --- a/src/rpc/cli.cpp +++ b/src/rpc/cli.cpp @@ -119,27 +119,30 @@ void cli::run() */ static char *my_rl_complete(char *token, int *match) { - int matchlen = 0; - int count = 0; - std::string method_name; + int matchlen = 0; + int count = 0; + std::string method_name; - auto& cmd = cli_commands(); - int partlen = strlen (token); /* Part of token */ - for (std::string it : cmd) { - if (!strncmp ( it.c_str(), token, partlen)) { - method_name = it; - matchlen = partlen; - count ++; - } - } + auto& cmd = cli_commands(); + int partlen = strlen (token); /* Part of token */ + for (const std::string it : cmd) + { + if (it.compare(0, partlen, token) == 0) + { + method_name = it; + matchlen = partlen; + count ++; + } + } - if (count == 1) { - *match = 1; - method_name += " "; - return strdup (method_name.c_str() + matchlen); - } + if (count == 1) + { + *match = 1; + method_name += " "; + return strdup (method_name.c_str() + matchlen); + } - return NULL; + return NULL; } /*** From 0242a051115972449ca9544ed7020e45ed64269a Mon Sep 17 00:00:00 2001 From: abitmore Date: Tue, 6 Mar 2018 16:16:52 +0000 Subject: [PATCH 17/46] Remove FindEditline.cmake --- CMakeModules/FindEditline.cmake | 50 --------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 CMakeModules/FindEditline.cmake diff --git a/CMakeModules/FindEditline.cmake b/CMakeModules/FindEditline.cmake deleted file mode 100644 index 3508fe1..0000000 --- a/CMakeModules/FindEditline.cmake +++ /dev/null @@ -1,50 +0,0 @@ -# - Try to find editline include dirs and libraries -# -# Usage of this module as follows: -# -# find_package(Editline) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# Editline_ROOT_DIR Set this variable to the root installation of -# editline if the module has problems finding the -# proper installation path. -# -# Variables defined by this module: -# -# EDITLINE_FOUND System has editline, include and lib dirs found -# Editline_INCLUDE_DIR The editline include directories. -# Editline_LIBRARY The editline library. - -find_path(Editline_ROOT_DIR - NAMES include/editline.h - HINTS ${EDITLINE_DIR} -) - -find_path(Editline_INCLUDE_DIR - NAMES editline.h - HINTS ${Editline_ROOT_DIR}/include -) - -find_library(Editline_LIBRARY - NAMES libeditline.a - HINTS ${Editline_ROOT_DIR}/src/.libs -) - -if(Editline_INCLUDE_DIR AND Editline_LIBRARY) - set(EDITLINE_FOUND TRUE) -else(Editline_INCLUDE_DIR AND Editline_LIBRARY) - FIND_LIBRARY(Editline_LIBRARY NAMES editline) - include(FindPackageHandleStandardArgs) - FIND_PACKAGE_HANDLE_STANDARD_ARGS(Editline DEFAULT_MSG Editline_INCLUDE_DIR Editline_LIBRARY ) - MARK_AS_ADVANCED(Editline_INCLUDE_DIR Editline_LIBRARY) -endif(Editline_INCLUDE_DIR AND Editline_LIBRARY) - -mark_as_advanced( - Editline_ROOT_DIR - Editline_INCLUDE_DIR - Editline_LIBRARY -) - -MESSAGE( STATUS "Found Editline: ${Editline_LIBRARY}" ) From b5fec061529d342e825132bb85167857a2da3e4f Mon Sep 17 00:00:00 2001 From: abitmore Date: Tue, 6 Mar 2018 16:28:20 +0000 Subject: [PATCH 18/46] Update websocketpp to a forked and patched version --- .gitmodules | 2 +- vendor/websocketpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 69f6e7c..1ab151e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,7 +6,7 @@ url = https://github.com/bitshares/secp256k1-zkp.git [submodule "vendor/websocketpp"] path = vendor/websocketpp - url = https://github.com/zaphoyd/websocketpp.git + url = https://github.com/bitshares/websocketpp.git [submodule "vendor/editline"] path = vendor/editline url = https://github.com/troglobit/editline.git diff --git a/vendor/websocketpp b/vendor/websocketpp index 378437a..e6c4e3c 160000 --- a/vendor/websocketpp +++ b/vendor/websocketpp @@ -1 +1 @@ -Subproject commit 378437aecdcb1dfe62096ffd5d944bf1f640ccc3 +Subproject commit e6c4e3c54bf9cf1892c85a6ed6289486bba36fa1 From 1fe7d4be46cf11c346fc35d24571e6829a9966ca Mon Sep 17 00:00:00 2001 From: John Jones Date: Tue, 6 Mar 2018 11:44:00 -0500 Subject: [PATCH 19/46] Trying to get the #$%@$ tabs to disappear --- src/rpc/cli.cpp | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/rpc/cli.cpp b/src/rpc/cli.cpp index f85b7fd..8687e0b 100644 --- a/src/rpc/cli.cpp +++ b/src/rpc/cli.cpp @@ -119,30 +119,30 @@ void cli::run() */ static char *my_rl_complete(char *token, int *match) { - int matchlen = 0; - int count = 0; - std::string method_name; + int matchlen = 0; + int count = 0; + std::string method_name; - auto& cmd = cli_commands(); - int partlen = strlen (token); /* Part of token */ - for (const std::string it : cmd) - { - if (it.compare(0, partlen, token) == 0) - { - method_name = it; - matchlen = partlen; - count ++; - } - } + auto& cmd = cli_commands(); + int partlen = strlen (token); /* Part of token */ + for (const std::string it : cmd) + { + if (it.compare(0, partlen, token) == 0) + { + method_name = it; + matchlen = partlen; + count ++; + } + } - if (count == 1) - { - *match = 1; - method_name += " "; - return strdup (method_name.c_str() + matchlen); - } + if (count == 1) + { + *match = 1; + method_name += " "; + return strdup (method_name.c_str() + matchlen); + } - return NULL; + return NULL; } /*** From ebdb5eaceaf901a83aeb6871a474b8b36e7638fb Mon Sep 17 00:00:00 2001 From: Abit Date: Tue, 6 Mar 2018 20:37:23 +0100 Subject: [PATCH 20/46] Correctly set default C++ standard to c++11 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3766b1a..557609a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -272,7 +272,7 @@ endif(WIN32) # end editline stuff if( NOT CPP_STANDARD ) - set( CPP_STANDARD, "-std=c++11" ) + set( CPP_STANDARD "-std=c++11" ) endif() IF(WIN32) From c225488cd4070a9e3df8cbf06914b327dab91608 Mon Sep 17 00:00:00 2001 From: John Jones Date: Tue, 6 Mar 2018 17:17:09 -0500 Subject: [PATCH 21/46] replace tabs with spaces --- src/rpc/cli.cpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/rpc/cli.cpp b/src/rpc/cli.cpp index 8687e0b..22ef34b 100644 --- a/src/rpc/cli.cpp +++ b/src/rpc/cli.cpp @@ -153,22 +153,22 @@ static char *my_rl_complete(char *token, int *match) */ static int cli_completion(char *token, char ***av) { - int num, total = 0; - char **copy; + int num, total = 0; + char **copy; - auto& cmd = cli_commands(); - num = cmd.size(); + auto& cmd = cli_commands(); + num = cmd.size(); - copy = (char **) malloc (num * sizeof(char *)); - for (auto it : cmd) { - if (!strncmp (it.c_str(), token, strlen (token))) { - copy[total] = strdup ( it.c_str() ); - total ++; - } - } - *av = copy; + copy = (char **) malloc (num * sizeof(char *)); + for (auto it : cmd) { + if (!strncmp (it.c_str(), token, strlen (token))) { + copy[total] = strdup ( it.c_str() ); + total ++; + } + } + *av = copy; - return total; + return total; } /*** From b3053d04217c6b47b7511b0791a52e46183b3290 Mon Sep 17 00:00:00 2001 From: John Jones Date: Wed, 7 Mar 2018 02:55:59 -0500 Subject: [PATCH 22/46] adjust spacing, add const, remove unnecessary variable, bad malloc check --- src/rpc/cli.cpp | 54 +++++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/src/rpc/cli.cpp b/src/rpc/cli.cpp index 22ef34b..fdb3a25 100644 --- a/src/rpc/cli.cpp +++ b/src/rpc/cli.cpp @@ -119,27 +119,33 @@ void cli::run() */ static char *my_rl_complete(char *token, int *match) { - int matchlen = 0; - int count = 0; + bool have_one = false; std::string method_name; auto& cmd = cli_commands(); int partlen = strlen (token); /* Part of token */ + for (const std::string it : cmd) { if (it.compare(0, partlen, token) == 0) { - method_name = it; - matchlen = partlen; - count ++; + if (have_one) { + // we can only have 1, but we found a second + return NULL; + } + else + { + method_name = it; + have_one = true; + } } } - if (count == 1) + if (have_one) { *match = 1; method_name += " "; - return strdup (method_name.c_str() + matchlen); + return strdup (method_name.c_str() + partlen); } return NULL; @@ -148,27 +154,35 @@ static char *my_rl_complete(char *token, int *match) /*** * @brief return an array of matching commands * @param token the incoming text - * @param av the resultant array of possible matches + * @param array the resultant array of possible matches * @returns the number of matches */ -static int cli_completion(char *token, char ***av) +static int cli_completion(char *token, char ***array) { - int num, total = 0; - char **copy; - auto& cmd = cli_commands(); - num = cmd.size(); + int num_commands = cmd.size(); - copy = (char **) malloc (num * sizeof(char *)); - for (auto it : cmd) { - if (!strncmp (it.c_str(), token, strlen (token))) { - copy[total] = strdup ( it.c_str() ); - total ++; + char **copy = (char **) malloc (num_commands * sizeof(char *)); + if (copy == NULL) + { + // possible out of memory + return 0; + } + int total_matches = 0; + + int partlen = strlen(token); + + for (const std::string it : cmd) + { + if ( it.compare(0, partlen, token) == 0) + { + copy[total_matches] = strdup ( it.c_str() ); + ++total_matches; } } - *av = copy; + *array = copy; - return total; + return total_matches; } /*** From 8c09ff09fc4a4ee47ee9f4e530064d1d460f2094 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Mon, 5 Mar 2018 21:09:39 +0100 Subject: [PATCH 23/46] Added json write/validate/read test --- tests/io/json_tests.cpp | 173 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/tests/io/json_tests.cpp b/tests/io/json_tests.cpp index 8db3e08..35ff110 100644 --- a/tests/io/json_tests.cpp +++ b/tests/io/json_tests.cpp @@ -1,5 +1,8 @@ #include +#include +#include + #include #include #include @@ -75,6 +78,7 @@ BOOST_AUTO_TEST_CASE(imbalanced_test) std::vector tests { // for easier handling and better readability, in the following test // strings ' is used instead of " and \1 instead of \0 + "", "{", "{'", "{'}", @@ -87,6 +91,7 @@ BOOST_AUTO_TEST_CASE(imbalanced_test) "[ 13", "' end", "{ 13: }", + "\1", "{\1", "{\1}", "{'\1", @@ -113,5 +118,173 @@ BOOST_AUTO_TEST_CASE(imbalanced_test) } } +static bool equal( const fc::variant& a, const fc::variant& b ) +{ + auto a_type = a.get_type(); + auto b_type = b.get_type(); + if( a_type == fc::variant::type_id::int64_type && a.as() > 0 ) + a_type = fc::variant::type_id::uint64_type; + if( b_type == fc::variant::type_id::int64_type && b.as() > 0 ) + b_type = fc::variant::type_id::uint64_type; + if( a_type != b_type ) return false; + switch( a_type ) + { + case fc::variant::type_id::null_type: return true; + case fc::variant::type_id::int64_type: return a.as() == b.as(); + case fc::variant::type_id::uint64_type: return a.as() == b.as(); + case fc::variant::type_id::double_type: return a.as() == b.as(); + case fc::variant::type_id::bool_type: return a.as() == b.as(); + case fc::variant::type_id::string_type: return a.as() == b.as(); + case fc::variant::type_id::array_type: + if( a.get_array().size() != b.get_array().size() ) return false; + else + { + std::vector::const_iterator b_it = b.get_array().begin(); + for( const auto& a_it : a.get_array() ) + { + if( !equal( a_it, *b_it ) ) return false; + b_it++; + } + } + return true; + case fc::variant::type_id::object_type: + if( a.get_object().size() != b.get_object().size() ) return false; + for( const auto& a_it : a.get_object() ) + { + const auto& b_obj = b.get_object().find( a_it.key() ); + if( b_obj == b.get_object().end() || !equal( a_it.value(), b_obj->value() ) ) return false; + } + return true; + case fc::variant::type_id::blob_type: + default: + FC_THROW_EXCEPTION( fc::invalid_arg_exception, "Unsupported variant type: " + a.get_type() ); + } +} + +static void test_recursive( const fc::variant& v ) +{ try { + const std::string json = fc::json::to_string( v, fc::json::output_formatting::legacy_generator ); + BOOST_CHECK( fc::json::is_valid( json ) ); + BOOST_CHECK( !fc::json::is_valid( json + " " ) ); + + const std::string pretty = fc::json::to_pretty_string( v, fc::json::output_formatting::legacy_generator ); + BOOST_CHECK( fc::json::is_valid( pretty ) ); + BOOST_CHECK( !fc::json::is_valid( pretty + " " ) ); + + fc::temp_file file( fc::temp_directory_path(), true ); + fc::json::save_to_file( v, file.path(), false, fc::json::output_formatting::legacy_generator ); + + BOOST_CHECK( equal( v, fc::json::from_string( json + " " ) ) ); + BOOST_CHECK( equal( v, fc::json::from_string( pretty + " " ) ) ); + BOOST_CHECK( equal( v, fc::json::from_file( file.path() ) ) ); + + if( v.get_type() == fc::variant::type_id::array_type ) + for( const auto& item : v.get_array() ) + test_recursive( item ); + else if( v.get_type() == fc::variant::type_id::object_type ) + for( const auto& item : v.get_object() ) + test_recursive( item.value() ); +} FC_CAPTURE_LOG_AND_RETHROW( (v) ) } + +BOOST_AUTO_TEST_CASE(structured_test) +{ + fc::variant_object v_empty_obj; + fc::variants v_empty_array; + fc::variant v_null; + fc::variant v_true( true ); + fc::variant v_false( false ); + fc::variant v_empty_str( "" ); + fc::variant v_str( "false" ); + fc::variant v_int8_1( (int8_t) 1 ); + fc::variant v_int8_2( (int8_t) -2 ); + fc::variant v_uint8_1( (int8_t) 1 ); + fc::variant v_int16_1( (int16_t) 1 ); + fc::variant v_int16_2( (int16_t) -2 ); + fc::variant v_uint16_1( (int16_t) 1 ); + fc::variant v_int32_1( (int32_t) 1 ); + fc::variant v_int32_2( (int32_t) -2 ); + fc::variant v_uint32_1( (int32_t) 1 ); + fc::variant v_int64_1( (int8_t) 1 ); + fc::variant v_int64_2( (int8_t) -2 ); + fc::variant v_uint64_1( (int8_t) 1 ); + fc::variant v_float_1( 0.0f ); + fc::variant v_float_2( -2.0f ); + fc::variant v_double_1( 0.0d ); + fc::variant v_double_2( -2.0d ); + fc::variants v_small_array + { + v_empty_obj, + v_empty_array, + v_null, + v_true, + v_false, + v_empty_str + }; + fc::mutable_variant_object v_small_obj; + v_small_obj( "", v_empty_str ) + ( "1", v_empty_array ) + ( "2", v_null ) + ( "a", v_true ) + ( "b", v_false ) + ( "x", v_small_array ) + ( "y", v_empty_obj ); + fc::variants v_big_array + { + v_empty_obj, + v_empty_array, + v_null, + v_true, + v_false, + v_empty_str, + v_str, + v_int8_1, + v_int8_2, + v_uint8_1, + v_int16_1, + v_int16_2, + v_uint16_1, + v_int32_1, + v_int32_2, + v_uint32_1, + v_int64_1, + v_int64_2, + v_uint64_1, + v_float_1, + v_float_2, + v_double_1, + v_double_2, + v_small_array, + v_small_obj + }; + fc::mutable_variant_object v_big_obj; + v_big_obj( "v_empty_obj", v_empty_obj ) + ( "v_empty_array", v_empty_array ) + ( "v_null", v_null ) + ( "v_true", v_true ) + ( "v_false", v_false ) + ( "v_empty_str", v_empty_str ) + ( "v_str", v_str ) + ( "v_int8_1", v_int8_1 ) + ( "v_int8_2", v_int8_2 ) + ( "v_uint8_1", v_uint8_1 ) + ( "v_int16_1", v_int16_1 ) + ( "v_int16_2", v_int16_2 ) + ( "v_uint16_1", v_uint16_1 ) + ( "v_int32_1", v_int32_1 ) + ( "v_int32_2", v_int32_2 ) + ( "v_uint32_1", v_uint32_1 ) + ( "v_int64_1", v_int64_1 ) + ( "v_int64_2", v_int64_2 ) + ( "v_uint64_1", v_uint64_1 ) + ( "v_float_1", v_float_1 ) + ( "v_float_2", v_float_2 ) + ( "v_double_1", v_double_1 ) + ( "v_double_2", v_double_2 ) + ( "v_small_array", v_small_array ) + ( "v_small_obj", v_small_obj ); + v_big_array.push_back( v_big_obj ); + + test_recursive( v_big_array ); +} BOOST_AUTO_TEST_SUITE_END() From e37d9a50514122853791c43befd50286fbfd1a4a Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Mon, 5 Mar 2018 21:57:11 +0100 Subject: [PATCH 24/46] Code cleanups and simplifications --- src/io/json.cpp | 151 +++++++++++++++++------------------------------- 1 file changed, 54 insertions(+), 97 deletions(-) diff --git a/src/io/json.cpp b/src/io/json.cpp index ca1c7fb..99feab9 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -5,7 +5,6 @@ #include #include #include -//#include #include #include #include @@ -179,7 +178,6 @@ namespace fc "Expected '{', but read '${char}'", ("char",string(&c, &c + 1)) ); in.get(); - skip_white_space(in); while( in.peek() != '}' ) { if( in.peek() == ',' ) @@ -199,7 +197,6 @@ namespace fc auto val = variant_from_stream( in ); obj(std::move(key),std::move(val)); - skip_white_space(in); } if( in.peek() == '}' ) { @@ -227,7 +224,6 @@ namespace fc if( in.peek() != '[' ) FC_THROW_EXCEPTION( parse_error_exception, "Expected '['" ); in.get(); - skip_white_space(in); while( in.peek() != ']' ) { @@ -238,7 +234,6 @@ namespace fc } if( skip_white_space(in) ) continue; ar.push_back( variant_from_stream(in) ); - skip_white_space(in); } if( in.peek() != ']' ) FC_THROW_EXCEPTION( parse_error_exception, "Expected ']' after parsing ${variant}", @@ -276,6 +271,7 @@ namespace fc if (dot) FC_THROW_EXCEPTION(parse_error_exception, "Can't parse a number with two decimal places"); dot = true; + [[fallthrough]]; case '0': case '1': case '2': @@ -299,10 +295,10 @@ namespace fc } } catch (fc::eof_exception&) - { + { // EOF ends the loop } catch (const std::ios_base::failure&) - { + { // read error ends the loop } fc::string str = ss.str(); if (str == "-." || str == ".") // check the obviously wrong things we could have encountered @@ -379,7 +375,7 @@ namespace fc // make out ("falfe") // A strict JSON parser would signal this as an error, but we // will just treat the malformed token as an un-quoted string. - return str + stringFromToken(in);; + return str + stringFromToken(in); } } } @@ -389,53 +385,42 @@ namespace fc variant variant_from_stream( T& in ) { skip_white_space(in); - variant var; - while( true ) + signed char c = in.peek(); + switch( c ) { - signed char c = in.peek(); - switch( c ) - { - case ' ': - case '\t': - case '\n': - case '\r': - in.get(); - continue; - case '"': - return stringFromStream( in ); - case '{': - return objectFromStream( in ); - case '[': - return arrayFromStream( in ); - case '-': - case '.': - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - return number_from_stream( in ); - // null, true, false, or 'warning' / string - case 'n': - case 't': - case 'f': - return token_from_stream( in ); - case 0x04: // ^D end of transmission - case EOF: - case 0: - FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" ); - default: - FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"", - ("c", c)("s", stringFromToken(in)) ); - } + case '"': + return stringFromStream( in ); + case '{': + return objectFromStream( in ); + case '[': + return arrayFromStream( in ); + case '-': + case '.': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return number_from_stream( in ); + // null, true, false, or 'warning' / string + case 'n': + case 't': + case 'f': + return token_from_stream( in ); + case 0x04: // ^D end of transmission + case EOF: + FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" ); + case 0: + default: + FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"", + ("c", c)("s", stringFromToken(in)) ); } - return variant(); - } + } /** the purpose of this check is to verify that we will not get a stack overflow in the recursive descent parser */ @@ -467,33 +452,18 @@ namespace fc } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) } variants json::variants_from_string( const std::string& utf8_str, parse_type ptype ) - { try { - check_string_depth( utf8_str ); + { variants result; - fc::stringstream in( utf8_str ); - //in.exceptions( std::ifstream::eofbit ); try { + check_string_depth( utf8_str ); + fc::stringstream in( utf8_str ); while( true ) - { - // result.push_back( variant_from_stream( in )); - result.push_back(json_relaxed::variant_from_stream( in )); - } - } catch ( const fc::eof_exception& ){} - return result; - } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) } - /* - void toUTF8( const char str, ostream& os ) - { - // validate str == valid utf8 - utf8::replace_invalid( &str, &str + 1, ostream_iterator(os) ); + result.push_back(json_relaxed::variant_from_stream( in )); + } catch ( const fc::eof_exception& ) { + return result; + } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) } - void toUTF8( const wchar_t c, ostream& os ) - { - utf8::utf16to8( &c, (&c)+1, ostream_iterator(os) ); - } - */ - /** * Convert '\t', '\a', '\n', '\\' and '"' to "\t\a\n\\\"" * @@ -563,7 +533,6 @@ namespace fc default: os << *itr; - //toUTF8( *itr, os ); } } os << '"'; @@ -616,27 +585,19 @@ namespace fc os << "null"; return; case variant::int64_type: - { - int64_t i = v.as_int64(); if( format == json::stringify_large_ints_and_doubles && - i > 0xffffffff ) + v.as_int64() > 0xffffffff ) os << '"'< 0xffffffff ) + v.as_uint64() > 0xffffffff ) os << '"'< Date: Mon, 5 Mar 2018 22:36:01 +0100 Subject: [PATCH 25/46] Code deduplication --- include/fc/io/json_relaxed.hpp | 76 ++-------------------------------- src/io/json.cpp | 27 +++++++++--- 2 files changed, 24 insertions(+), 79 deletions(-) diff --git a/include/fc/io/json_relaxed.hpp b/include/fc/io/json_relaxed.hpp index e4876f2..26a89a5 100644 --- a/include/fc/io/json_relaxed.hpp +++ b/include/fc/io/json_relaxed.hpp @@ -564,84 +564,14 @@ namespace fc { namespace json_relaxed template variant_object objectFromStream( T& in ) { - mutable_variant_object obj; - try - { - char c = in.peek(); - if( c != '{' ) - FC_THROW_EXCEPTION( parse_error_exception, - "Expected '{', but read '${char}'", - ("char",string(&c, &c + 1)) ); - in.get(); - skip_white_space(in); - while( in.peek() != '}' ) - { - if( in.peek() == ',' ) - { - in.get(); - continue; - } - if( skip_white_space(in) ) continue; - string key = json_relaxed::stringFromStream( in ); - skip_white_space(in); - if( in.peek() != ':' ) - { - FC_THROW_EXCEPTION( parse_error_exception, "Expected ':' after key \"${key}\"", - ("key", key) ); - } - in.get(); - auto val = json_relaxed::variant_from_stream( in ); - - obj(std::move(key),std::move(val)); - skip_white_space(in); - } - if( in.peek() == '}' ) - { - in.get(); - return obj; - } - FC_THROW_EXCEPTION( parse_error_exception, "Expected '}' after ${variant}", ("variant", obj ) ); - } - catch( const fc::eof_exception& e ) - { - FC_THROW_EXCEPTION( parse_error_exception, "Unexpected EOF: ${e}", ("e", e.to_detail_string() ) ); - } - catch( const std::ios_base::failure& e ) - { - FC_THROW_EXCEPTION( parse_error_exception, "Unexpected EOF: ${e}", ("e", e.what() ) ); - } FC_RETHROW_EXCEPTIONS( warn, "Error parsing object" ); + return objectFromStreamBase( in, []( T& in ){ return json_relaxed::stringFromStream( in ); }, + []( T& in ){ return json_relaxed::variant_from_stream( in ); } ); } template variants arrayFromStream( T& in ) { - variants ar; - try - { - if( in.peek() != '[' ) - FC_THROW_EXCEPTION( parse_error_exception, "Expected '['" ); - in.get(); - skip_white_space(in); - - while( in.peek() != ']' ) - { - if( in.peek() == ',' ) - { - in.get(); - continue; - } - if( skip_white_space(in) ) continue; - ar.push_back( json_relaxed::variant_from_stream(in) ); - skip_white_space(in); - } - if( in.peek() != ']' ) - FC_THROW_EXCEPTION( parse_error_exception, "Expected ']' after parsing ${variant}", - ("variant", ar) ); - - in.get(); - } FC_RETHROW_EXCEPTIONS( warn, "Attempting to parse array ${array}", - ("array", ar ) ); - return ar; + return arrayFromStreamBase( in, []( T& in ){ return json_relaxed::variant_from_stream( in ); } ); } template diff --git a/src/io/json.cpp b/src/io/json.cpp index 99feab9..8f4f2c1 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -19,7 +19,9 @@ namespace fc template fc::string stringFromStream( T& in ); template bool skip_white_space( T& in ); template fc::string stringFromToken( T& in ); + template variant_object objectFromStreamBase( T& in, std::function& get_key, std::function& get_value ); template variant_object objectFromStream( T& in ); + template variants arrayFromStreamBase( T& in, std::function& get_value ); template variants arrayFromStream( T& in ); template variant number_from_stream( T& in ); template variant token_from_stream( T& in ); @@ -166,8 +168,8 @@ namespace fc ("token", token.str() ) ); } - template - variant_object objectFromStream( T& in ) + template + variant_object objectFromStreamBase( T& in, std::string (*get_key)(T&), variant (*get_value)(T&) ) { mutable_variant_object obj; try @@ -186,7 +188,7 @@ namespace fc continue; } if( skip_white_space(in) ) continue; - string key = stringFromStream( in ); + string key = get_key( in ); skip_white_space(in); if( in.peek() != ':' ) { @@ -194,7 +196,7 @@ namespace fc ("key", key) ); } in.get(); - auto val = variant_from_stream( in ); + auto val = get_value( in ); obj(std::move(key),std::move(val)); } @@ -216,7 +218,14 @@ namespace fc } template - variants arrayFromStream( T& in ) + variant_object objectFromStream( T& in ) + { + return objectFromStreamBase( in, []( T& in ){ return stringFromStream( in ); }, + []( T& in ){ return variant_from_stream( in ); } ); + } + + template + variants arrayFromStreamBase( T& in, variant (*get_value)(T&) ) { variants ar; try @@ -233,7 +242,7 @@ namespace fc continue; } if( skip_white_space(in) ) continue; - ar.push_back( variant_from_stream(in) ); + ar.push_back( get_value(in) ); } if( in.peek() != ']' ) FC_THROW_EXCEPTION( parse_error_exception, "Expected ']' after parsing ${variant}", @@ -245,6 +254,12 @@ namespace fc return ar; } + template + variants arrayFromStream( T& in ) + { + return arrayFromStreamBase( in, []( T& in ){ return variant_from_stream( in ); } ); + } + template variant number_from_stream( T& in ) { From cb9c61fa2d20e85c1002a2cfd755262c288ba6fa Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Tue, 6 Mar 2018 22:13:17 +0100 Subject: [PATCH 26/46] Avoid legacy_generator --- tests/io/json_tests.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/io/json_tests.cpp b/tests/io/json_tests.cpp index 35ff110..25bd975 100644 --- a/tests/io/json_tests.cpp +++ b/tests/io/json_tests.cpp @@ -126,7 +126,15 @@ static bool equal( const fc::variant& a, const fc::variant& b ) a_type = fc::variant::type_id::uint64_type; if( b_type == fc::variant::type_id::int64_type && b.as() > 0 ) b_type = fc::variant::type_id::uint64_type; - if( a_type != b_type ) return false; + if( a_type != b_type ) + { + if( ( a_type == fc::variant::type_id::double_type + && b_type == fc::variant::type_id::string_type ) + || ( a_type == fc::variant::type_id::string_type + && b_type == fc::variant::type_id::double_type ) ) + return a.as() == b.as(); + return false; + } switch( a_type ) { case fc::variant::type_id::null_type: return true; @@ -163,16 +171,16 @@ static bool equal( const fc::variant& a, const fc::variant& b ) static void test_recursive( const fc::variant& v ) { try { - const std::string json = fc::json::to_string( v, fc::json::output_formatting::legacy_generator ); + const std::string json = fc::json::to_string( v ); BOOST_CHECK( fc::json::is_valid( json ) ); BOOST_CHECK( !fc::json::is_valid( json + " " ) ); - const std::string pretty = fc::json::to_pretty_string( v, fc::json::output_formatting::legacy_generator ); + const std::string pretty = fc::json::to_pretty_string( v ); BOOST_CHECK( fc::json::is_valid( pretty ) ); BOOST_CHECK( !fc::json::is_valid( pretty + " " ) ); fc::temp_file file( fc::temp_directory_path(), true ); - fc::json::save_to_file( v, file.path(), false, fc::json::output_formatting::legacy_generator ); + fc::json::save_to_file( v, file.path(), false ); BOOST_CHECK( equal( v, fc::json::from_string( json + " " ) ) ); BOOST_CHECK( equal( v, fc::json::from_string( pretty + " " ) ) ); From 1ae3cc2fad06aa17bdaf1f5fc7713df0a1f24c92 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Tue, 6 Mar 2018 22:14:30 +0100 Subject: [PATCH 27/46] Make unused parsers compile-time optional --- include/fc/io/json.hpp | 4 ++++ include/fc/io/json_relaxed.hpp | 1 - src/io/json.cpp | 8 +++++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/include/fc/io/json.hpp b/include/fc/io/json.hpp index 8a46d13..10b5314 100644 --- a/include/fc/io/json.hpp +++ b/include/fc/io/json.hpp @@ -18,14 +18,18 @@ namespace fc enum parse_type { legacy_parser = 0, +#ifdef WITH_EXOTIC_JSON_PARSERS strict_parser = 1, relaxed_parser = 2, legacy_parser_with_string_doubles = 3 +#endif }; enum output_formatting { stringify_large_ints_and_doubles = 0, +#ifdef WITH_EXOTIC_JSON_PARSERS legacy_generator = 1 +#endif }; static ostream& to_stream( ostream& out, const fc::string&); diff --git a/include/fc/io/json_relaxed.hpp b/include/fc/io/json_relaxed.hpp index 26a89a5..dd6db73 100644 --- a/include/fc/io/json_relaxed.hpp +++ b/include/fc/io/json_relaxed.hpp @@ -671,5 +671,4 @@ namespace fc { namespace json_relaxed } return variant(); } - } } // fc::json_relaxed diff --git a/src/io/json.cpp b/src/io/json.cpp index 8f4f2c1..d3296c9 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -319,7 +319,11 @@ namespace fc if (str == "-." || str == ".") // check the obviously wrong things we could have encountered FC_THROW_EXCEPTION(parse_error_exception, "Can't parse token \"${token}\" as a JSON numeric constant", ("token", str)); if( dot ) - return parser_type == json::legacy_parser_with_string_doubles ? variant(str) : variant(to_double(str)); + return +#ifdef WITH_EXOTIC_JSON_PARSERS + parser_type == json::legacy_parser_with_string_doubles ? variant(str) : +#endif + variant(to_double(str)); if( neg ) return to_int64(str); return to_uint64(str); @@ -768,12 +772,14 @@ namespace fc { case legacy_parser: return variant_from_stream( in ); +#ifdef WITH_EXOTIC_JSON_PARSERS case legacy_parser_with_string_doubles: return variant_from_stream( in ); case strict_parser: return json_relaxed::variant_from_stream( in ); case relaxed_parser: return json_relaxed::variant_from_stream( in ); +#endif default: FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); } From b4f3947cb1bd9b5194c9988ecf6ab2977e24d547 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Tue, 6 Mar 2018 22:47:02 +0100 Subject: [PATCH 28/46] Include variants_from_string in parser test --- tests/io/json_tests.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/io/json_tests.cpp b/tests/io/json_tests.cpp index 25bd975..d86b743 100644 --- a/tests/io/json_tests.cpp +++ b/tests/io/json_tests.cpp @@ -182,6 +182,14 @@ static void test_recursive( const fc::variant& v ) fc::temp_file file( fc::temp_directory_path(), true ); fc::json::save_to_file( v, file.path(), false ); + fc::variants list = fc::json::variants_from_string( json ); + BOOST_CHECK_EQUAL( 1, list.size() ); + BOOST_CHECK( equal( v, list[0] ) ); + + list = fc::json::variants_from_string( pretty ); + BOOST_CHECK_EQUAL( 1, list.size() ); + BOOST_CHECK( equal( v, list[0] ) ); + BOOST_CHECK( equal( v, fc::json::from_string( json + " " ) ) ); BOOST_CHECK( equal( v, fc::json::from_string( pretty + " " ) ) ); BOOST_CHECK( equal( v, fc::json::from_file( file.path() ) ) ); From 4bb8bf7832b29a3be6f3c1ef1002e11ffbd11270 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Tue, 6 Mar 2018 22:47:59 +0100 Subject: [PATCH 29/46] Fixed relaxed parser wrt "" input --- include/fc/io/json_relaxed.hpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/include/fc/io/json_relaxed.hpp b/include/fc/io/json_relaxed.hpp index dd6db73..36f11c8 100644 --- a/include/fc/io/json_relaxed.hpp +++ b/include/fc/io/json_relaxed.hpp @@ -104,8 +104,15 @@ namespace fc { namespace json_relaxed if( in.peek() == q ) { in.get(); - if( in.peek() != q ) - return fc::string(); + try + { + if( in.peek() != q ) + return fc::string(); + } + catch( const fc::eof_exception& e ) + { + return fc::string(); + } // triple quote processing if( strict ) From d336af82a68ed03d9b1ebd22b3df9be846f9fd26 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Tue, 6 Mar 2018 23:01:40 +0100 Subject: [PATCH 30/46] Applied variant_from_stream fix from regular to relaxed --- include/fc/io/json_relaxed.hpp | 90 ++++++++++++++++------------------ 1 file changed, 41 insertions(+), 49 deletions(-) diff --git a/include/fc/io/json_relaxed.hpp b/include/fc/io/json_relaxed.hpp index 36f11c8..1ed424b 100644 --- a/include/fc/io/json_relaxed.hpp +++ b/include/fc/io/json_relaxed.hpp @@ -626,56 +626,48 @@ namespace fc { namespace json_relaxed variant variant_from_stream( T& in ) { skip_white_space(in); - variant var; - while( signed char c = in.peek() ) + signed char c = in.peek(); + switch( c ) { - switch( c ) - { - case ' ': - case '\t': - case '\n': - case '\r': - in.get(); - continue; - case '"': - return json_relaxed::stringFromStream( in ); - case '{': - return json_relaxed::objectFromStream( in ); - case '[': - return json_relaxed::arrayFromStream( in ); - case '-': - case '+': - case '.': - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - return json_relaxed::numberFromStream( in ); - // null, true, false, or 'warning' / string - case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': - case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': - case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': - case 'y': case 'z': - case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': - case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': - case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': - case 'Y': case 'Z': - case '_': case '/': - return json_relaxed::wordFromStream( in ); - case 0x04: // ^D end of transmission - case EOF: - FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" ); - default: - FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"", - ("c", c)("s", stringFromToken(in)) ); - } + case '"': + return json_relaxed::stringFromStream( in ); + case '{': + return json_relaxed::objectFromStream( in ); + case '[': + return json_relaxed::arrayFromStream( in ); + case '-': + case '+': + case '.': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return json_relaxed::numberFromStream( in ); + // null, true, false, or 'warning' / string + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': + case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': + case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': + case 'y': case 'z': + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': + case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': + case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': + case 'Y': case 'Z': + case '_': case '/': + return json_relaxed::wordFromStream( in ); + case 0x04: // ^D end of transmission + case EOF: + FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" ); + case 0: + default: + FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"", + ("c", c)("s", stringFromToken(in)) ); } - return variant(); } + } } // fc::json_relaxed From a7e0c887dbc7a33ba8eec470d05711022cbc4530 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Wed, 7 Mar 2018 15:19:42 +0100 Subject: [PATCH 31/46] Added test case for recursion depth 240 --- tests/io/json_tests.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/io/json_tests.cpp b/tests/io/json_tests.cpp index d86b743..4e4cb5c 100644 --- a/tests/io/json_tests.cpp +++ b/tests/io/json_tests.cpp @@ -75,6 +75,10 @@ static void test_fail_file( const std::string& str ) BOOST_AUTO_TEST_CASE(imbalanced_test) { + std::string open40("[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[["); + std::string close40("]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]"); + std::string open80 = open40 + open40; + std::string close80 = close40 + close40; std::vector tests { // for easier handling and better readability, in the following test // strings ' is used instead of " and \1 instead of \0 @@ -106,7 +110,9 @@ BOOST_AUTO_TEST_CASE(imbalanced_test) "['\1]", "[ 13\1", "[ 13\1]", - "' end\1" + "' end\1", + open80 + "'" + close80 + close80 + "'," + open80 + open80 + + close80 + close80 + close80 }; for( std::string test : tests ) From 90137d400df93c9bd41f1ebb187c01102e4f9c78 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Wed, 7 Mar 2018 15:20:31 +0100 Subject: [PATCH 32/46] Fix for recursion depth limitation --- include/fc/io/json_relaxed.hpp | 21 +++++++------ src/io/json.cpp | 57 ++++++++++++---------------------- 2 files changed, 32 insertions(+), 46 deletions(-) diff --git a/include/fc/io/json_relaxed.hpp b/include/fc/io/json_relaxed.hpp index 1ed424b..a9196c2 100644 --- a/include/fc/io/json_relaxed.hpp +++ b/include/fc/io/json_relaxed.hpp @@ -21,7 +21,7 @@ namespace fc { namespace json_relaxed { template - variant variant_from_stream( T& in ); + variant variant_from_stream( T& in, uint32_t depth ); template fc::string tokenFromStream( T& in ) @@ -569,16 +569,18 @@ namespace fc { namespace json_relaxed } FC_CAPTURE_AND_RETHROW( (token) ) } template - variant_object objectFromStream( T& in ) + variant_object objectFromStream( T& in, uint32_t depth ) { - return objectFromStreamBase( in, []( T& in ){ return json_relaxed::stringFromStream( in ); }, - []( T& in ){ return json_relaxed::variant_from_stream( in ); } ); + std::function get_key = []( T& in ){ return json_relaxed::stringFromStream( in ); }; + std::function get_value = [depth]( T& in ){ return json_relaxed::variant_from_stream( in, depth ); }; + return objectFromStreamBase( in, get_key, get_value ); } template - variants arrayFromStream( T& in ) + variants arrayFromStream( T& in, uint32_t depth ) { - return arrayFromStreamBase( in, []( T& in ){ return json_relaxed::variant_from_stream( in ); } ); + std::function get_value = [depth]( T& in ){ return json_relaxed::variant_from_stream( in, depth ); }; + return arrayFromStreamBase( in, get_value ); } template @@ -623,8 +625,9 @@ namespace fc { namespace json_relaxed } template - variant variant_from_stream( T& in ) + variant variant_from_stream( T& in, uint32_t depth ) { + FC_ASSERT( depth < MAX_RECURSION_DEPTH, "Too many nested items in JSON string!" ); skip_white_space(in); signed char c = in.peek(); switch( c ) @@ -632,9 +635,9 @@ namespace fc { namespace json_relaxed case '"': return json_relaxed::stringFromStream( in ); case '{': - return json_relaxed::objectFromStream( in ); + return json_relaxed::objectFromStream( in, depth + 1 ); case '[': - return json_relaxed::arrayFromStream( in ); + return json_relaxed::arrayFromStream( in, depth + 1 ); case '-': case '+': case '.': diff --git a/src/io/json.cpp b/src/io/json.cpp index d3296c9..5d23e70 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -14,15 +14,15 @@ namespace fc { // forward declarations of provided functions - template variant variant_from_stream( T& in ); + template variant variant_from_stream( T& in, uint32_t depth = 0 ); template char parseEscape( T& in ); template fc::string stringFromStream( T& in ); template bool skip_white_space( T& in ); template fc::string stringFromToken( T& in ); template variant_object objectFromStreamBase( T& in, std::function& get_key, std::function& get_value ); - template variant_object objectFromStream( T& in ); + template variant_object objectFromStream( T& in, uint32_t depth ); template variants arrayFromStreamBase( T& in, std::function& get_value ); - template variants arrayFromStream( T& in ); + template variants arrayFromStream( T& in, uint32_t depth ); template variant number_from_stream( T& in ); template variant token_from_stream( T& in ); void escape_string( const string& str, ostream& os ); @@ -32,6 +32,8 @@ namespace fc fc::string pretty_print( const fc::string& v, uint8_t indent ); } +#define MAX_RECURSION_DEPTH 200 + #include namespace fc @@ -169,7 +171,7 @@ namespace fc } template - variant_object objectFromStreamBase( T& in, std::string (*get_key)(T&), variant (*get_value)(T&) ) + variant_object objectFromStreamBase( T& in, std::function& get_key, std::function& get_value ) { mutable_variant_object obj; try @@ -218,14 +220,15 @@ namespace fc } template - variant_object objectFromStream( T& in ) + variant_object objectFromStream( T& in, uint32_t depth ) { - return objectFromStreamBase( in, []( T& in ){ return stringFromStream( in ); }, - []( T& in ){ return variant_from_stream( in ); } ); + std::function get_key = []( T& in ){ return stringFromStream( in ); }; + std::function get_value = [depth]( T& in ){ return variant_from_stream( in, depth ); }; + return objectFromStreamBase( in, get_key, get_value ); } template - variants arrayFromStreamBase( T& in, variant (*get_value)(T&) ) + variants arrayFromStreamBase( T& in, std::function& get_value ) { variants ar; try @@ -255,9 +258,10 @@ namespace fc } template - variants arrayFromStream( T& in ) + variants arrayFromStream( T& in, uint32_t depth ) { - return arrayFromStreamBase( in, []( T& in ){ return variant_from_stream( in ); } ); + std::function get_value = [depth]( T& in ){ return variant_from_stream( in, depth ); }; + return arrayFromStreamBase( in, get_value ); } template @@ -401,8 +405,10 @@ namespace fc template - variant variant_from_stream( T& in ) + variant variant_from_stream( T& in, uint32_t depth ) { + if( depth > MAX_RECURSION_DEPTH ) + FC_THROW_EXCEPTION( parse_error_exception, "Too many nested items in JSON input!" ); skip_white_space(in); signed char c = in.peek(); switch( c ) @@ -410,9 +416,9 @@ namespace fc case '"': return stringFromStream( in ); case '{': - return objectFromStream( in ); + return objectFromStream( in, depth + 1 ); case '[': - return arrayFromStream( in ); + return arrayFromStream( in, depth + 1 ); case '-': case '.': case '0': @@ -441,30 +447,8 @@ namespace fc } } - - /** the purpose of this check is to verify that we will not get a stack overflow in the recursive descent parser */ - void check_string_depth( const string& utf8_str ) - { - int32_t open_object = 0; - int32_t open_array = 0; - for( auto c : utf8_str ) - { - switch( c ) - { - case '{': open_object++; break; - case '}': open_object--; break; - case '[': open_array++; break; - case ']': open_array--; break; - default: break; - } - FC_ASSERT( open_object < 100 && open_array < 100, "object graph too deep", ("object depth",open_object)("array depth", open_array) ); - } - } - variant json::from_string( const std::string& utf8_str, parse_type ptype ) { try { - check_string_depth( utf8_str ); - fc::istream_ptr in( new fc::stringstream( utf8_str ) ); fc::buffered_istream bin( in ); return from_stream( bin, ptype ); @@ -474,10 +458,9 @@ namespace fc { variants result; try { - check_string_depth( utf8_str ); fc::stringstream in( utf8_str ); while( true ) - result.push_back(json_relaxed::variant_from_stream( in )); + result.push_back(json_relaxed::variant_from_stream( in, 0 )); } catch ( const fc::eof_exception& ) { return result; } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) From 2bacd5fda8232a302fcd65e31912bf8a23e997b9 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Wed, 7 Mar 2018 15:26:30 +0100 Subject: [PATCH 33/46] Added broken_nul_parser to preserve previous behaviour --- include/fc/io/json.hpp | 3 ++- src/io/json.cpp | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/fc/io/json.hpp b/include/fc/io/json.hpp index 10b5314..d7c1b0f 100644 --- a/include/fc/io/json.hpp +++ b/include/fc/io/json.hpp @@ -21,8 +21,9 @@ namespace fc #ifdef WITH_EXOTIC_JSON_PARSERS strict_parser = 1, relaxed_parser = 2, - legacy_parser_with_string_doubles = 3 + legacy_parser_with_string_doubles = 3, #endif + broken_nul_parser = 4 }; enum output_formatting { diff --git a/src/io/json.cpp b/src/io/json.cpp index 5d23e70..6230fd1 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -441,6 +441,9 @@ namespace fc case EOF: FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" ); case 0: + if( parser_type == fc::json::broken_nul_parser ) + return variant(); + [[fallthrough]]; default: FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"", ("c", c)("s", stringFromToken(in)) ); From 5b5190a55c16ae852020aa53f390383ab5e8b69f Mon Sep 17 00:00:00 2001 From: John Jones Date: Wed, 7 Mar 2018 09:44:35 -0500 Subject: [PATCH 34/46] changed const to const ref --- src/rpc/cli.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpc/cli.cpp b/src/rpc/cli.cpp index fdb3a25..993e19f 100644 --- a/src/rpc/cli.cpp +++ b/src/rpc/cli.cpp @@ -125,7 +125,7 @@ static char *my_rl_complete(char *token, int *match) auto& cmd = cli_commands(); int partlen = strlen (token); /* Part of token */ - for (const std::string it : cmd) + for (const std::string& it : cmd) { if (it.compare(0, partlen, token) == 0) { @@ -172,7 +172,7 @@ static int cli_completion(char *token, char ***array) int partlen = strlen(token); - for (const std::string it : cmd) + for (const std::string& it : cmd) { if ( it.compare(0, partlen, token) == 0) { From 66ed9fc3dcf5cef4ed49ecb18a8ec3a8aba60773 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Wed, 7 Mar 2018 22:41:45 +0100 Subject: [PATCH 35/46] Minor fixes --- src/io/json.cpp | 16 +++++++++++----- tests/io/json_tests.cpp | 12 ++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/io/json.cpp b/src/io/json.cpp index 6230fd1..606fa85 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -32,6 +32,12 @@ namespace fc fc::string pretty_print( const fc::string& v, uint8_t indent ); } +#if __cplusplus > 201402L +#define FALLTHROUGH [[fallthrough]]; +#else +#define FALLTHROUGH +#endif + #define MAX_RECURSION_DEPTH 200 #include @@ -290,7 +296,7 @@ namespace fc if (dot) FC_THROW_EXCEPTION(parse_error_exception, "Can't parse a number with two decimal places"); dot = true; - [[fallthrough]]; + FALLTHROUGH case '0': case '1': case '2': @@ -320,7 +326,7 @@ namespace fc { // read error ends the loop } fc::string str = ss.str(); - if (str == "-." || str == ".") // check the obviously wrong things we could have encountered + if (str == "-." || str == "." || str == "-") // check the obviously wrong things we could have encountered FC_THROW_EXCEPTION(parse_error_exception, "Can't parse token \"${token}\" as a JSON numeric constant", ("token", str)); if( dot ) return @@ -443,7 +449,7 @@ namespace fc case 0: if( parser_type == fc::json::broken_nul_parser ) return variant(); - [[fallthrough]]; + FALLTHROUGH default: FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"", ("c", c)("s", stringFromToken(in)) ); @@ -591,7 +597,7 @@ namespace fc return; case variant::int64_type: if( format == json::stringify_large_ints_and_doubles && - v.as_int64() > 0xffffffff ) + ( v.as_int64() > 0xffffffff || v.as_int64() < -int64_t(0xffffffff) ) ) os << '"'< Date: Thu, 8 Mar 2018 07:40:20 -0500 Subject: [PATCH 36/46] assuring that malloc gets freed --- src/rpc/cli.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/rpc/cli.cpp b/src/rpc/cli.cpp index 993e19f..2570c9f 100644 --- a/src/rpc/cli.cpp +++ b/src/rpc/cli.cpp @@ -216,10 +216,20 @@ void cli::getline( const fc::string& prompt, fc::string& line) line_read = readline(prompt.c_str()); if( line_read == nullptr ) FC_THROW_EXCEPTION( fc::eof_exception, "" ); - if( *line_read ) - add_history(line_read); line = line_read; - free(line_read); + if (*line_read) + { + try + { + add_history(line_read); + free(line_read); + } + catch(...) + { + free(line_read); + throw; + } + } }).wait(); } else From 2017ed911a9a1e62fd0b824478ea063ac9ce60a4 Mon Sep 17 00:00:00 2001 From: John Jones Date: Thu, 8 Mar 2018 10:46:23 -0500 Subject: [PATCH 37/46] fix memory leak --- src/rpc/cli.cpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/rpc/cli.cpp b/src/rpc/cli.cpp index 2570c9f..29d5051 100644 --- a/src/rpc/cli.cpp +++ b/src/rpc/cli.cpp @@ -217,18 +217,16 @@ void cli::getline( const fc::string& prompt, fc::string& line) if( line_read == nullptr ) FC_THROW_EXCEPTION( fc::eof_exception, "" ); line = line_read; - if (*line_read) + try { - try - { + if (*line_read) add_history(line_read); - free(line_read); - } - catch(...) - { - free(line_read); - throw; - } + free(line_read); + } + catch(...) + { + free(line_read); + throw; } }).wait(); } From 9ee2bcf0a5f44b8f414f893bb3d001ec8abb97c8 Mon Sep 17 00:00:00 2001 From: John Jones Date: Thu, 8 Mar 2018 11:05:34 -0500 Subject: [PATCH 38/46] avoid double free --- src/rpc/cli.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/cli.cpp b/src/rpc/cli.cpp index 29d5051..c4030f8 100644 --- a/src/rpc/cli.cpp +++ b/src/rpc/cli.cpp @@ -221,13 +221,13 @@ void cli::getline( const fc::string& prompt, fc::string& line) { if (*line_read) add_history(line_read); - free(line_read); } catch(...) { free(line_read); throw; } + free(line_read); }).wait(); } else From 1412df18d15cd67af4b4ce9d99abd5fb4c7f08fa Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Thu, 8 Mar 2018 18:54:49 +0100 Subject: [PATCH 39/46] Make broken_nul_parser usable --- src/io/json.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/io/json.cpp b/src/io/json.cpp index 606fa85..9553ff3 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -772,6 +772,8 @@ namespace fc case relaxed_parser: return json_relaxed::variant_from_stream( in ); #endif + case broken_nul_parser: + return variant_from_stream( in ); default: FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); } From f9802f686007e7efeaa88cba31f647ca96f1ee0c Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Thu, 8 Mar 2018 22:12:28 +0100 Subject: [PATCH 40/46] Added max_depth parameter to all to_/from_ methods --- include/fc/io/json.hpp | 48 +++++++++-------- include/fc/io/json_relaxed.hpp | 19 +++---- src/io/json.cpp | 99 +++++++++++++++++----------------- tests/io/json_tests.cpp | 11 ++++ 4 files changed, 96 insertions(+), 81 deletions(-) diff --git a/include/fc/io/json.hpp b/include/fc/io/json.hpp index d7c1b0f..21d07c4 100644 --- a/include/fc/io/json.hpp +++ b/include/fc/io/json.hpp @@ -2,6 +2,8 @@ #include #include +#define DEFAULT_MAX_RECURSION_DEPTH 200 + namespace fc { class ostream; @@ -33,52 +35,54 @@ namespace fc #endif }; - static ostream& to_stream( ostream& out, const fc::string&); - static ostream& to_stream( ostream& out, const variant& v, output_formatting format = stringify_large_ints_and_doubles ); - static ostream& to_stream( ostream& out, const variants& v, output_formatting format = stringify_large_ints_and_doubles ); - static ostream& to_stream( ostream& out, const variant_object& v, output_formatting format = stringify_large_ints_and_doubles ); + static ostream& to_stream( ostream& out, const fc::string& ); + static ostream& to_stream( ostream& out, const variant& v, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); + static ostream& to_stream( ostream& out, const variants& v, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); + static ostream& to_stream( ostream& out, const variant_object& v, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); - static variant from_stream( buffered_istream& in, parse_type ptype = legacy_parser ); + static variant from_stream( buffered_istream& in, parse_type ptype = legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); - static variant from_string( const string& utf8_str, parse_type ptype = legacy_parser ); - static variants variants_from_string( const string& utf8_str, parse_type ptype = legacy_parser ); - static string to_string( const variant& v, output_formatting format = stringify_large_ints_and_doubles ); - static string to_pretty_string( const variant& v, output_formatting format = stringify_large_ints_and_doubles ); + static variant from_string( const string& utf8_str, parse_type ptype = legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); + static variants variants_from_string( const string& utf8_str, parse_type ptype = legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); + static string to_string( const variant& v, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); + static string to_pretty_string( const variant& v, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); - static bool is_valid( const std::string& json_str, parse_type ptype = legacy_parser ); + static bool is_valid( const std::string& json_str, parse_type ptype = legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); template - static void save_to_file( const T& v, const fc::path& fi, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles ) + static void save_to_file( const T& v, const fc::path& fi, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ) { - save_to_file( variant(v), fi, pretty, format ); + save_to_file( variant(v), fi, pretty, format, max_depth ); } - static void save_to_file( const variant& v, const fc::path& fi, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles ); - static variant from_file( const fc::path& p, parse_type ptype = legacy_parser ); + static void save_to_file( const variant& v, const fc::path& fi, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); + static variant from_file( const fc::path& p, parse_type ptype = legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ); template - static T from_file( const fc::path& p, parse_type ptype = legacy_parser ) + static T from_file( const fc::path& p, parse_type ptype = legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ) { - return json::from_file(p, ptype).as(); + return json::from_file(p, ptype, max_depth).as(); } template - static string to_string( const T& v, output_formatting format = stringify_large_ints_and_doubles ) + static string to_string( const T& v, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ) { - return to_string( variant(v), format ); + return to_string( variant(v), format, max_depth ); } template - static string to_pretty_string( const T& v, output_formatting format = stringify_large_ints_and_doubles ) + static string to_pretty_string( const T& v, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ) { - return to_pretty_string( variant(v), format ); + return to_pretty_string( variant(v), format, max_depth ); } template - static void save_to_file( const T& v, const std::string& p, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles ) + static void save_to_file( const T& v, const std::string& p, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH ) { - save_to_file( variant(v), fc::path(p), pretty ); + save_to_file( variant(v), fc::path(p), pretty, format, max_depth ); } }; } // fc + +#undef DEFAULT_MAX_RECURSION_DEPTH diff --git a/include/fc/io/json_relaxed.hpp b/include/fc/io/json_relaxed.hpp index a9196c2..555b21a 100644 --- a/include/fc/io/json_relaxed.hpp +++ b/include/fc/io/json_relaxed.hpp @@ -21,7 +21,7 @@ namespace fc { namespace json_relaxed { template - variant variant_from_stream( T& in, uint32_t depth ); + variant variant_from_stream( T& in, uint32_t max_depth ); template fc::string tokenFromStream( T& in ) @@ -569,17 +569,17 @@ namespace fc { namespace json_relaxed } FC_CAPTURE_AND_RETHROW( (token) ) } template - variant_object objectFromStream( T& in, uint32_t depth ) + variant_object objectFromStream( T& in, uint32_t max_depth ) { std::function get_key = []( T& in ){ return json_relaxed::stringFromStream( in ); }; - std::function get_value = [depth]( T& in ){ return json_relaxed::variant_from_stream( in, depth ); }; + std::function get_value = [max_depth]( T& in ){ return json_relaxed::variant_from_stream( in, max_depth ); }; return objectFromStreamBase( in, get_key, get_value ); } template - variants arrayFromStream( T& in, uint32_t depth ) + variants arrayFromStream( T& in, uint32_t max_depth ) { - std::function get_value = [depth]( T& in ){ return json_relaxed::variant_from_stream( in, depth ); }; + std::function get_value = [max_depth]( T& in ){ return json_relaxed::variant_from_stream( in, max_depth ); }; return arrayFromStreamBase( in, get_value ); } @@ -625,9 +625,10 @@ namespace fc { namespace json_relaxed } template - variant variant_from_stream( T& in, uint32_t depth ) + variant variant_from_stream( T& in, uint32_t max_depth ) { - FC_ASSERT( depth < MAX_RECURSION_DEPTH, "Too many nested items in JSON string!" ); + if( max_depth == 0 ) + FC_THROW_EXCEPTION( parse_error_exception, "Too many nested items in JSON input!" ); skip_white_space(in); signed char c = in.peek(); switch( c ) @@ -635,9 +636,9 @@ namespace fc { namespace json_relaxed case '"': return json_relaxed::stringFromStream( in ); case '{': - return json_relaxed::objectFromStream( in, depth + 1 ); + return json_relaxed::objectFromStream( in, max_depth - 1 ); case '[': - return json_relaxed::arrayFromStream( in, depth + 1 ); + return json_relaxed::arrayFromStream( in, max_depth - 1 ); case '-': case '+': case '.': diff --git a/src/io/json.cpp b/src/io/json.cpp index 9553ff3..a0f484e 100644 --- a/src/io/json.cpp +++ b/src/io/json.cpp @@ -14,21 +14,21 @@ namespace fc { // forward declarations of provided functions - template variant variant_from_stream( T& in, uint32_t depth = 0 ); + template variant variant_from_stream( T& in, uint32_t max_depth ); template char parseEscape( T& in ); template fc::string stringFromStream( T& in ); template bool skip_white_space( T& in ); template fc::string stringFromToken( T& in ); template variant_object objectFromStreamBase( T& in, std::function& get_key, std::function& get_value ); - template variant_object objectFromStream( T& in, uint32_t depth ); + template variant_object objectFromStream( T& in, uint32_t max_depth ); template variants arrayFromStreamBase( T& in, std::function& get_value ); - template variants arrayFromStream( T& in, uint32_t depth ); + template variants arrayFromStream( T& in, uint32_t max_depth ); template variant number_from_stream( T& in ); template variant token_from_stream( T& in ); void escape_string( const string& str, ostream& os ); - template void to_stream( T& os, const variants& a, json::output_formatting format ); - template void to_stream( T& os, const variant_object& o, json::output_formatting format ); - template void to_stream( T& os, const variant& v, json::output_formatting format ); + template void to_stream( T& os, const variants& a, json::output_formatting format, uint32_t max_depth ); + template void to_stream( T& os, const variant_object& o, json::output_formatting format, uint32_t max_depth ); + template void to_stream( T& os, const variant& v, json::output_formatting format, uint32_t max_depth ); fc::string pretty_print( const fc::string& v, uint8_t indent ); } @@ -38,8 +38,6 @@ namespace fc #define FALLTHROUGH #endif -#define MAX_RECURSION_DEPTH 200 - #include namespace fc @@ -226,10 +224,10 @@ namespace fc } template - variant_object objectFromStream( T& in, uint32_t depth ) + variant_object objectFromStream( T& in, uint32_t max_depth ) { std::function get_key = []( T& in ){ return stringFromStream( in ); }; - std::function get_value = [depth]( T& in ){ return variant_from_stream( in, depth ); }; + std::function get_value = [max_depth]( T& in ){ return variant_from_stream( in, max_depth ); }; return objectFromStreamBase( in, get_key, get_value ); } @@ -264,9 +262,9 @@ namespace fc } template - variants arrayFromStream( T& in, uint32_t depth ) + variants arrayFromStream( T& in, uint32_t max_depth ) { - std::function get_value = [depth]( T& in ){ return variant_from_stream( in, depth ); }; + std::function get_value = [max_depth]( T& in ){ return variant_from_stream( in, max_depth ); }; return arrayFromStreamBase( in, get_value ); } @@ -411,9 +409,9 @@ namespace fc template - variant variant_from_stream( T& in, uint32_t depth ) + variant variant_from_stream( T& in, uint32_t max_depth ) { - if( depth > MAX_RECURSION_DEPTH ) + if( max_depth == 0 ) FC_THROW_EXCEPTION( parse_error_exception, "Too many nested items in JSON input!" ); skip_white_space(in); signed char c = in.peek(); @@ -422,9 +420,9 @@ namespace fc case '"': return stringFromStream( in ); case '{': - return objectFromStream( in, depth + 1 ); + return objectFromStream( in, max_depth - 1 ); case '[': - return arrayFromStream( in, depth + 1 ); + return arrayFromStream( in, max_depth - 1 ); case '-': case '.': case '0': @@ -456,20 +454,20 @@ namespace fc } } - variant json::from_string( const std::string& utf8_str, parse_type ptype ) + variant json::from_string( const std::string& utf8_str, parse_type ptype, uint32_t max_depth ) { try { fc::istream_ptr in( new fc::stringstream( utf8_str ) ); fc::buffered_istream bin( in ); - return from_stream( bin, ptype ); + return from_stream( bin, ptype, max_depth ); } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) } - variants json::variants_from_string( const std::string& utf8_str, parse_type ptype ) + variants json::variants_from_string( const std::string& utf8_str, parse_type ptype, uint32_t max_depth ) { variants result; try { fc::stringstream in( utf8_str ); while( true ) - result.push_back(json_relaxed::variant_from_stream( in, 0 )); + result.push_back(json_relaxed::variant_from_stream( in, max_depth )); } catch ( const fc::eof_exception& ) { return result; } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) @@ -555,14 +553,14 @@ namespace fc } template - void to_stream( T& os, const variants& a, json::output_formatting format ) + void to_stream( T& os, const variants& a, json::output_formatting format, uint32_t max_depth ) { os << '['; auto itr = a.begin(); while( itr != a.end() ) { - to_stream( os, *itr, format ); + to_stream( os, *itr, format, max_depth ); ++itr; if( itr != a.end() ) os << ','; @@ -570,7 +568,7 @@ namespace fc os << ']'; } template - void to_stream( T& os, const variant_object& o, json::output_formatting format ) + void to_stream( T& os, const variant_object& o, json::output_formatting format, uint32_t max_depth ) { os << '{'; auto itr = o.begin(); @@ -579,7 +577,7 @@ namespace fc { escape_string( itr->key(), os ); os << ':'; - to_stream( os, itr->value(), format ); + to_stream( os, itr->value(), format, max_depth ); ++itr; if( itr != o.end() ) os << ','; @@ -588,8 +586,9 @@ namespace fc } template - void to_stream( T& os, const variant& v, json::output_formatting format ) + void to_stream( T& os, const variant& v, json::output_formatting format, uint32_t max_depth ) { + FC_ASSERT( max_depth > 0, "Too many nested objects!" ); switch( v.get_type() ) { case variant::null_type: @@ -625,20 +624,20 @@ namespace fc escape_string( v.as_string(), os ); return; case variant::array_type: - to_stream( os, v.get_array(), format ); + to_stream( os, v.get_array(), format, max_depth - 1 ); return; case variant::object_type: - to_stream(os, v.get_object(), format ); + to_stream(os, v.get_object(), format, max_depth - 1 ); return; default: FC_THROW_EXCEPTION( fc::invalid_arg_exception, "Unsupported variant type: " + v.get_type() ); } } - fc::string json::to_string( const variant& v, output_formatting format /* = stringify_large_ints_and_doubles */ ) + fc::string json::to_string( const variant& v, output_formatting format, uint32_t max_depth ) { fc::stringstream ss; - fc::to_stream( ss, v, format ); + fc::to_stream( ss, v, format, max_depth ); return ss.str(); } @@ -733,74 +732,74 @@ namespace fc - fc::string json::to_pretty_string( const variant& v, output_formatting format /* = stringify_large_ints_and_doubles */ ) + fc::string json::to_pretty_string( const variant& v, output_formatting format, uint32_t max_depth ) { - return pretty_print(to_string(v, format), 2); + return pretty_print(to_string(v, format, max_depth), 2); } - void json::save_to_file( const variant& v, const fc::path& fi, bool pretty, output_formatting format /* = stringify_large_ints_and_doubles */ ) + void json::save_to_file( const variant& v, const fc::path& fi, bool pretty, output_formatting format, uint32_t max_depth ) { if( pretty ) { - auto str = json::to_pretty_string( v, format ); + auto str = json::to_pretty_string( v, format, max_depth ); fc::ofstream o(fi); o.write( str.c_str(), str.size() ); } else { fc::ofstream o(fi); - fc::to_stream( o, v, format ); + fc::to_stream( o, v, format, max_depth ); } } - variant json::from_file( const fc::path& p, parse_type ptype ) + variant json::from_file( const fc::path& p, parse_type ptype, uint32_t max_depth ) { fc::istream_ptr in( new fc::ifstream( p ) ); fc::buffered_istream bin( in ); - return from_stream( bin, ptype ); + return from_stream( bin, ptype, max_depth ); } - variant json::from_stream( buffered_istream& in, parse_type ptype ) + variant json::from_stream( buffered_istream& in, parse_type ptype, uint32_t max_depth ) { switch( ptype ) { case legacy_parser: - return variant_from_stream( in ); + return variant_from_stream( in, max_depth ); #ifdef WITH_EXOTIC_JSON_PARSERS case legacy_parser_with_string_doubles: - return variant_from_stream( in ); + return variant_from_stream( in, max_depth ); case strict_parser: - return json_relaxed::variant_from_stream( in ); + return json_relaxed::variant_from_stream( in, max_depth ); case relaxed_parser: - return json_relaxed::variant_from_stream( in ); + return json_relaxed::variant_from_stream( in, max_depth ); #endif case broken_nul_parser: - return variant_from_stream( in ); + return variant_from_stream( in, max_depth ); default: FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); } } - ostream& json::to_stream( ostream& out, const variant& v, output_formatting format /* = stringify_large_ints_and_doubles */ ) + ostream& json::to_stream( ostream& out, const variant& v, output_formatting format, uint32_t max_depth ) { - fc::to_stream( out, v, format ); + fc::to_stream( out, v, format, max_depth ); return out; } - ostream& json::to_stream( ostream& out, const variants& v, output_formatting format /* = stringify_large_ints_and_doubles */ ) + ostream& json::to_stream( ostream& out, const variants& v, output_formatting format, uint32_t max_depth ) { - fc::to_stream( out, v, format ); + fc::to_stream( out, v, format, max_depth ); return out; } - ostream& json::to_stream( ostream& out, const variant_object& v, output_formatting format /* = stringify_large_ints_and_doubles */ ) + ostream& json::to_stream( ostream& out, const variant_object& v, output_formatting format, uint32_t max_depth ) { - fc::to_stream( out, v, format ); + fc::to_stream( out, v, format, max_depth ); return out; } - bool json::is_valid( const std::string& utf8_str, parse_type ptype ) + bool json::is_valid( const std::string& utf8_str, parse_type ptype, uint32_t max_depth ) { if( utf8_str.size() == 0 ) return false; fc::istream_ptr in( new fc::stringstream( utf8_str ) ); fc::buffered_istream bin( in ); - from_stream( bin, ptype ); + from_stream( bin, ptype, max_depth ); try { bin.peek(); } catch ( const eof_exception& e ) { return true; } return false; } diff --git a/tests/io/json_tests.cpp b/tests/io/json_tests.cpp index 9be97fe..e5421f2 100644 --- a/tests/io/json_tests.cpp +++ b/tests/io/json_tests.cpp @@ -321,4 +321,15 @@ BOOST_AUTO_TEST_CASE(precision_test) BOOST_CHECK_EQUAL( "0.5", half ); } +BOOST_AUTO_TEST_CASE(recursion_test) +{ + std::string ten_levels = "[[[[[[[[[[]]]]]]]]]]"; + fc::variant nested = fc::json::from_string( ten_levels ); + BOOST_CHECK_THROW( fc::json::from_string( ten_levels, fc::json::legacy_parser, 9 ), fc::parse_error_exception ); + + std::string back = fc::json::to_string( nested ); + BOOST_CHECK_EQUAL( ten_levels, back ); + BOOST_CHECK_THROW( fc::json::to_string( nested, fc::json::stringify_large_ints_and_doubles, 9 ), fc::assert_exception ); +} + BOOST_AUTO_TEST_SUITE_END() From 1331485c08d5e28110bd1f136590cca1fe91c223 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Thu, 8 Mar 2018 22:25:52 +0100 Subject: [PATCH 41/46] Stringify numbers >MAXINT or #include #include +#include #include #include #include @@ -596,7 +597,7 @@ namespace fc return; case variant::int64_type: if( format == json::stringify_large_ints_and_doubles && - ( v.as_int64() > 0xffffffff || v.as_int64() < -int64_t(0xffffffff) ) ) + ( v.as_int64() > INT32_MAX || v.as_int64() < INT32_MIN ) ) os << '"'< Date: Fri, 9 Mar 2018 18:47:24 +0100 Subject: [PATCH 42/46] Added test case for exception in exception handling --- tests/io/json_tests.cpp | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/io/json_tests.cpp b/tests/io/json_tests.cpp index e5421f2..42af936 100644 --- a/tests/io/json_tests.cpp +++ b/tests/io/json_tests.cpp @@ -332,4 +332,45 @@ BOOST_AUTO_TEST_CASE(recursion_test) BOOST_CHECK_THROW( fc::json::to_string( nested, fc::json::stringify_large_ints_and_doubles, 9 ), fc::assert_exception ); } +BOOST_AUTO_TEST_CASE(rethrow_test) +{ + fc::variants biggie; + for( int i = 0; i < 250; i++ ) + { + fc::variant tmp( std::move(biggie) ); + biggie.reserve(1); + biggie.push_back( std::move(tmp) ); + } + + auto test_r = [&biggie](){ + try { + FC_THROW_EXCEPTION( fc::unknown_host_exception, "WTF?" ); + } FC_RETHROW_EXCEPTIONS( warn, "Argh! ${biggie}", ("biggie",biggie) ) }; + BOOST_CHECK_THROW( test_r(), fc::unknown_host_exception ); + + auto test_lr = [&biggie](){ + try { + FC_THROW_EXCEPTION( fc::unknown_host_exception, "WTF?" ); + } FC_LOG_AND_RETHROW() }; + BOOST_CHECK_THROW( test_lr(), fc::unknown_host_exception ); + + auto test_clr = [&biggie](){ + try { + FC_THROW_EXCEPTION( fc::unknown_host_exception, "WTF?" ); + } FC_CAPTURE_LOG_AND_RETHROW( (biggie) ) }; + BOOST_CHECK_THROW( test_clr(), fc::unknown_host_exception ); + + auto test_cl = [&biggie](){ + try { + FC_THROW_EXCEPTION( fc::unknown_host_exception, "WTF?" ); + } FC_CAPTURE_AND_LOG( (biggie) ) }; + test_cl(); + + auto test_cr = [&biggie](){ + try { + FC_THROW_EXCEPTION( fc::unknown_host_exception, "WTF?" ); + } FC_CAPTURE_AND_RETHROW( (biggie) ) }; + BOOST_CHECK_THROW( test_cr(), fc::unknown_host_exception ); +} + BOOST_AUTO_TEST_SUITE_END() From 527daab6b73410a5bc88a28d3e0ede46a28c8219 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Fri, 9 Mar 2018 18:49:21 +0100 Subject: [PATCH 43/46] Moved format_string from variant.cpp to string.cpp (it is declared in string.hpp), added handling of recursion errors --- src/exception.cpp | 23 +++++++++++---- src/log/gelf_appender.cpp | 11 +++++-- src/string.cpp | 56 ++++++++++++++++++++++++++++++++++++ src/variant.cpp | 60 --------------------------------------- 4 files changed, 83 insertions(+), 67 deletions(-) diff --git a/src/exception.cpp b/src/exception.cpp index 0e0b956..e173be4 100644 --- a/src/exception.cpp +++ b/src/exception.cpp @@ -162,7 +162,14 @@ namespace fc for( auto itr = my->_elog.begin(); itr != my->_elog.end(); ) { ss << itr->get_message() <<"\n"; //fc::format_string( itr->get_format(), itr->get_data() ) <<"\n"; - ss << " " << json::to_string( itr->get_data() )<<"\n"; + try + { + ss << " " << json::to_string( itr->get_data() )<<"\n"; + } + catch( const fc::assert_exception& e ) + { + ss << "ERROR: Failed to convert log data to string!\n"; + } ss << " " << itr->get_context().to_string(); ++itr; if( itr != my->_elog.end() ) ss<<"\n"; @@ -256,10 +263,16 @@ namespace fc ("source_lineno", lineno) ("expr", expr) ; - std::cout - << "FC_ASSERT triggered: " - << fc::json::to_string( assert_trip_info ) << "\n"; - return; + try + { + std::cout + << "FC_ASSERT triggered: " + << fc::json::to_string( assert_trip_info ) << "\n"; + } + catch( const fc::assert_exception& e ) + { // this should never happen. assert_trip_info is flat. + std::cout << "ERROR: Failed to convert info to string?!\n"; + } } bool enable_record_assert_trip = false; diff --git a/src/log/gelf_appender.cpp b/src/log/gelf_appender.cpp index b88387c..2f14d78 100644 --- a/src/log/gelf_appender.cpp +++ b/src/log/gelf_appender.cpp @@ -128,8 +128,15 @@ namespace fc if (!context.get_task_name().empty()) gelf_message["_task_name"] = context.get_task_name(); - string gelf_message_as_string = json::to_string(gelf_message); - //unsigned uncompressed_size = gelf_message_as_string.size(); + string gelf_message_as_string; + try + { + gelf_message_as_string = json::to_string(gelf_message); + } + catch( const fc::assert_exception& e ) + { + gelf_message_as_string = "{\"level\":3,\"short_message\":\"ERROR while generating log message\"}"; + } gelf_message_as_string = zlib_compress(gelf_message_as_string); // graylog2 expects the zlib header to be 0x78 0x9c diff --git a/src/string.cpp b/src/string.cpp index 84edb88..e04f439 100644 --- a/src/string.cpp +++ b/src/string.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include #include @@ -256,6 +258,60 @@ namespace fc { } } + string format_string( const string& format, const variant_object& args ) + { + stringstream ss; + size_t prev = 0; + auto next = format.find( '$' ); + while( prev < format.size() ) + { + ss << format.substr( prev, next == string::npos ? string::npos : next - prev ); + + // if we got to the end, return it. + if( next == size_t(string::npos) || next == format.size() ) + return ss.str(); + + // if we are not at the end, then update the start + prev = next + 1; + + if( format[prev] == '{' ) + { + // if the next char is a open, then find close + next = format.find( '}', prev ); + // if we found close... + if( next != string::npos ) + { + // the key is between prev and next + string key = format.substr( prev+1, (next-prev-1) ); + + auto val = args.find( key ); + if( val != args.end() ) + { + if( val->value().is_object() || val->value().is_array() ) + { + try + { + ss << json::to_string( val->value() ); + } + catch( const fc::assert_exception& e ) + { + ss << "[\"ERROR_WHILE_CONVERTING_VALUE_TO_STRING\"]"; + } + } + else + ss << val->value().as_string(); + } + else + ss << "${"<& vo ) // vo = std::vector( b64.c_str(), b64.c_str() + b64.size() ); } -string format_string( const string& format, const variant_object& args ) -{ - stringstream ss; - size_t prev = 0; - auto next = format.find( '$' ); - while( prev != size_t(string::npos) && prev < size_t(format.size()) ) - { - ss << format.substr( prev, size_t(next-prev) ); - - // if we got to the end, return it. - if( next == size_t(string::npos) ) - return ss.str(); - - // if we are not at the end, then update the start - prev = next + 1; - - if( format[prev] == '{' ) - { - // if the next char is a open, then find close - next = format.find( '}', prev ); - // if we found close... - if( next != size_t(string::npos) ) - { - // the key is between prev and next - string key = format.substr( prev+1, (next-prev-1) ); - - auto val = args.find( key ); - if( val != args.end() ) - { - if( val->value().is_object() || val->value().is_array() ) - { - ss << json::to_string( val->value() ); - } - else - { - ss << val->value().as_string(); - } - } - else - { - ss << "${"< Date: Sun, 11 Mar 2018 15:56:45 -0400 Subject: [PATCH 44/46] Add test case for nested-object serialization --- tests/CMakeLists.txt | 1 + tests/serialization_test.cpp | 108 +++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 tests/serialization_test.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index bf7a01d..6151fad 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -54,6 +54,7 @@ add_executable( all_tests all_tests.cpp thread/thread_tests.cpp bloom_test.cpp real128_test.cpp + serialization_test.cpp time_test.cpp utf8_test.cpp ) diff --git a/tests/serialization_test.cpp b/tests/serialization_test.cpp new file mode 100644 index 0000000..1e35f86 --- /dev/null +++ b/tests/serialization_test.cpp @@ -0,0 +1,108 @@ +#include +#include + +#include +#include + +namespace fc { namespace test { + + struct item; + inline bool operator < ( const item& a, const item& b ); + inline bool operator == ( const item& a, const item& b ); + + struct item_wrapper + { + item_wrapper() {} + item_wrapper(item&& it) { v.reserve(1); v.insert( it ); } + boost::container::flat_set v; + }; + inline bool operator < ( const item_wrapper& a, const item_wrapper& b ); + inline bool operator == ( const item_wrapper& a, const item_wrapper& b ); + + struct item + { + item(int32_t lvl = 0) : level(lvl) {} + item(item_wrapper&& wp, int32_t lvl = 0) : level(lvl), w(wp) {} + int32_t level; + item_wrapper w; + }; + + inline bool operator == ( const item& a, const item& b ) + { return ( std::tie( a.level, a.w ) == std::tie( b.level, b.w ) ); } + + inline bool operator < ( const item& a, const item& b ) + { return ( std::tie( a.level, a.w ) < std::tie( b.level, b.w ) ); } + + inline bool operator == ( const item_wrapper& a, const item_wrapper& b ) + { return ( std::tie( a.v ) == std::tie( b.v ) ); } + + inline bool operator < ( const item_wrapper& a, const item_wrapper& b ) + { return ( std::tie( a.v ) < std::tie( b.v ) ); } + +} } + +FC_REFLECT( fc::test::item_wrapper, (v) ); +FC_REFLECT( fc::test::item, (level)(w) ); + +BOOST_AUTO_TEST_SUITE(fc_serialization) + +BOOST_AUTO_TEST_CASE( nested_objects_test ) +{ try { + + auto create_nested_object = []( uint32_t level ) + { + ilog( "Creating nested object with ${lv} level(s)", ("lv",level) ); + fc::test::item nested; + for( uint32_t i = 1; i <= level; i++ ) + { + if( i % 100 == 0 ) + ilog( "Creating level ${lv}", ("lv",i) ); + fc::test::item_wrapper wp( std::move(nested) ); + nested = fc::test::item( std::move(wp), i ); + } + return nested; + }; + + // 100 levels, should be allowed + { + auto nested = create_nested_object( 100 ); + + std::stringstream ss; + + BOOST_TEST_MESSAGE( "About to pack." ); + fc::raw::pack( ss, nested ); + + BOOST_TEST_MESSAGE( "About to unpack." ); + fc::test::item unpacked; + fc::raw::unpack( ss, unpacked ); + + BOOST_CHECK( unpacked == nested ); + } + + // 150 levels, by default packing will fail + { + auto nested = create_nested_object( 150 ); + + std::stringstream ss; + + BOOST_TEST_MESSAGE( "About to pack." ); + BOOST_CHECK_THROW( fc::raw::pack( ss, nested ), fc::assert_exception ); + } + + // 150 levels and allow packing, unpacking will fail + { + auto nested = create_nested_object( 150 ); + + std::stringstream ss; + + BOOST_TEST_MESSAGE( "About to pack." ); + fc::raw::pack( ss, nested, 1500 ); + + BOOST_TEST_MESSAGE( "About to unpack." ); + fc::test::item unpacked; + BOOST_CHECK_THROW( fc::raw::unpack( ss, unpacked ), fc::assert_exception ); + } + +} FC_CAPTURE_LOG_AND_RETHROW ( (0) ) } + +BOOST_AUTO_TEST_SUITE_END() From 86e18663a0048868a933b18e76be9a875f6d938d Mon Sep 17 00:00:00 2001 From: abitmore Date: Sun, 11 Mar 2018 16:14:22 -0400 Subject: [PATCH 45/46] Add max_depth parameter to pack/unpack functions --- include/fc/config.hpp | 4 + include/fc/container/deque_fwd.hpp | 5 +- include/fc/container/flat.hpp | 42 +- include/fc/container/flat_fwd.hpp | 15 +- include/fc/crypto/elliptic.hpp | 20 +- include/fc/crypto/pke.hpp | 20 +- include/fc/fixed_string.hpp | 14 +- include/fc/interprocess/container.hpp | 30 +- include/fc/io/enum_type.hpp | 22 +- include/fc/io/raw.hpp | 635 +++++++++++++++----------- include/fc/io/raw_fwd.hpp | 133 +++--- include/fc/io/raw_variant.hpp | 80 ++-- include/fc/network/ip.hpp | 48 +- include/fc/real128.hpp | 13 +- include/fc/uint128.hpp | 5 +- 15 files changed, 610 insertions(+), 476 deletions(-) create mode 100644 include/fc/config.hpp diff --git a/include/fc/config.hpp b/include/fc/config.hpp new file mode 100644 index 0000000..2f646bd --- /dev/null +++ b/include/fc/config.hpp @@ -0,0 +1,4 @@ +#ifndef FC_PACK_MAX_DEPTH + // The maximum level of object nesting is around 20% of this value + #define FC_PACK_MAX_DEPTH 1000 +#endif diff --git a/include/fc/container/deque_fwd.hpp b/include/fc/container/deque_fwd.hpp index 80359ea..71df18d 100644 --- a/include/fc/container/deque_fwd.hpp +++ b/include/fc/container/deque_fwd.hpp @@ -1,13 +1,14 @@ #pragma once +#include #include namespace fc { namespace raw { template - void pack( Stream& s, const std::deque& value ); + void pack( Stream& s, const std::deque& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); template - void unpack( Stream& s, std::deque& value ); + void unpack( Stream& s, std::deque& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); } } // namespace fc diff --git a/include/fc/container/flat.hpp b/include/fc/container/flat.hpp index 48fdf75..0b494f2 100644 --- a/include/fc/container/flat.hpp +++ b/include/fc/container/flat.hpp @@ -8,61 +8,66 @@ namespace fc { namespace raw { template - inline void pack( Stream& s, const flat_set& value ) { - pack( s, unsigned_int((uint32_t)value.size()) ); + inline void pack( Stream& s, const flat_set& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { - fc::raw::pack( s, *itr ); + fc::raw::pack( s, *itr, _max_depth - 1 ); ++itr; } } template - inline void unpack( Stream& s, flat_set& value ) { - unsigned_int size; unpack( s, size ); + inline void unpack( Stream& s, flat_set& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + unsigned_int size; unpack( s, size, _max_depth - 1 ); value.clear(); FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); value.reserve(size.value); for( uint32_t i = 0; i < size.value; ++i ) { T tmp; - fc::raw::unpack( s, tmp ); + fc::raw::unpack( s, tmp, _max_depth - 1 ); value.insert( std::move(tmp) ); } } template - inline void pack( Stream& s, const flat_map& value ) { - pack( s, unsigned_int((uint32_t)value.size()) ); + inline void pack( Stream& s, const flat_map& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { - fc::raw::pack( s, *itr ); + fc::raw::pack( s, *itr, _max_depth - 1 ); ++itr; } } template - inline void unpack( Stream& s, flat_map& value ) + inline void unpack( Stream& s, flat_map& value, uint32_t _max_depth ) { - unsigned_int size; unpack( s, size ); + FC_ASSERT( _max_depth > 0 ); + unsigned_int size; unpack( s, size, _max_depth - 1 ); value.clear(); FC_ASSERT( size.value*(sizeof(K)+sizeof(V)) < MAX_ARRAY_ALLOC_SIZE ); value.reserve(size.value); for( uint32_t i = 0; i < size.value; ++i ) { std::pair tmp; - fc::raw::unpack( s, tmp ); + fc::raw::unpack( s, tmp, _max_depth - 1 ); value.insert( std::move(tmp) ); } } template - void pack( Stream& s, const bip::vector& value ) { - pack( s, unsigned_int((uint32_t)value.size()) ); + void pack( Stream& s, const bip::vector& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); if( !std::is_fundamental::value ) { auto itr = value.begin(); auto end = value.end(); while( itr != end ) { - fc::raw::pack( s, *itr ); + fc::raw::pack( s, *itr, _max_depth - 1 ); ++itr; } } else { @@ -71,13 +76,14 @@ namespace fc { } template - void unpack( Stream& s, bip::vector& value ) { + void unpack( Stream& s, bip::vector& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); unsigned_int size; - unpack( s, size ); + unpack( s, size, _max_depth - 1 ); value.resize( size ); if( !std::is_fundamental::value ) { for( auto& item : value ) - unpack( s, item ); + unpack( s, item, _max_depth - 1 ); } else { s.read( (char*)value.data(), value.size() ); } diff --git a/include/fc/container/flat_fwd.hpp b/include/fc/container/flat_fwd.hpp index d56323a..8db36b6 100644 --- a/include/fc/container/flat_fwd.hpp +++ b/include/fc/container/flat_fwd.hpp @@ -1,7 +1,8 @@ -#pragma once +#pragma once #include #include #include +#include namespace fc { @@ -11,19 +12,19 @@ namespace fc { namespace raw { template - void pack( Stream& s, const flat_set& value ); + void pack( Stream& s, const flat_set& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); template - void unpack( Stream& s, flat_set& value ); + void unpack( Stream& s, flat_set& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); template - void pack( Stream& s, const flat_map& value ); + void pack( Stream& s, const flat_map& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); template - void unpack(Stream& s, flat_map& value); + void unpack(Stream& s, flat_map& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); template - void pack( Stream& s, const bip::vector& value ); + void pack( Stream& s, const bip::vector& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); template - void unpack( Stream& s, bip::vector& value ); + void unpack( Stream& s, bip::vector& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); } // namespace raw } // fc diff --git a/include/fc/crypto/elliptic.hpp b/include/fc/crypto/elliptic.hpp index 7d3046f..971ae3d 100644 --- a/include/fc/crypto/elliptic.hpp +++ b/include/fc/crypto/elliptic.hpp @@ -260,31 +260,35 @@ namespace fc { namespace raw { template - void unpack( Stream& s, fc::ecc::public_key& pk) + void unpack( Stream& s, fc::ecc::public_key& pk, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); ecc::public_key_data ser; - fc::raw::unpack(s,ser); + fc::raw::unpack( s, ser, _max_depth - 1 ); pk = fc::ecc::public_key( ser ); } template - void pack( Stream& s, const fc::ecc::public_key& pk) + void pack( Stream& s, const fc::ecc::public_key& pk, uint32_t _max_depth ) { - fc::raw::pack( s, pk.serialize() ); + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, pk.serialize(), _max_depth - 1 ); } template - void unpack( Stream& s, fc::ecc::private_key& pk) + void unpack( Stream& s, fc::ecc::private_key& pk, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); fc::sha256 sec; - unpack( s, sec ); + unpack( s, sec, _max_depth - 1 ); pk = ecc::private_key::regenerate(sec); } template - void pack( Stream& s, const fc::ecc::private_key& pk) + void pack( Stream& s, const fc::ecc::private_key& pk, uint32_t _max_depth ) { - fc::raw::pack( s, pk.get_secret() ); + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, pk.get_secret(), _max_depth - 1 ); } } // namespace raw diff --git a/include/fc/crypto/pke.hpp b/include/fc/crypto/pke.hpp index 3f1c607..551ef99 100644 --- a/include/fc/crypto/pke.hpp +++ b/include/fc/crypto/pke.hpp @@ -76,31 +76,35 @@ namespace fc { namespace raw { template - void unpack( Stream& s, fc::public_key& pk) + void unpack( Stream& s, fc::public_key& pk, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { + FC_ASSERT( _max_depth > 0 ); bytes ser; - fc::raw::unpack(s,ser); + fc::raw::unpack( s, ser, _max_depth - 1 ); pk = fc::public_key( ser ); } template - void pack( Stream& s, const fc::public_key& pk) + void pack( Stream& s, const fc::public_key& pk, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { - fc::raw::pack( s, pk.serialize() ); + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, pk.serialize(), _max_depth - 1 ); } template - void unpack( Stream& s, fc::private_key& pk) + void unpack( Stream& s, fc::private_key& pk, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { + FC_ASSERT( _max_depth > 0 ); bytes ser; - fc::raw::unpack(s,ser); + fc::raw::unpack( s, ser, _max_depth - 1 ); pk = fc::private_key( ser ); } template - void pack( Stream& s, const fc::private_key& pk) + void pack( Stream& s, const fc::private_key& pk, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { - fc::raw::pack( s, pk.serialize() ); + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, pk.serialize(), _max_depth - 1 ); } } class variant; diff --git a/include/fc/fixed_string.hpp b/include/fc/fixed_string.hpp index ca6ea1f..ce00390 100644 --- a/include/fc/fixed_string.hpp +++ b/include/fc/fixed_string.hpp @@ -98,16 +98,18 @@ namespace fc { namespace raw { template - inline void pack( Stream& s, const fc::fixed_string& u ) { + inline void pack( Stream& s, const fc::fixed_string& u, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { + FC_ASSERT( _max_depth > 0 ); unsigned_int size = u.size(); - pack( s, size ); + pack( s, size, _max_depth - 1 ); s.write( (const char*)&u.data, size ); } template - inline void unpack( Stream& s, fc::fixed_string& u ) { + inline void unpack( Stream& s, fc::fixed_string& u, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { + FC_ASSERT( _max_depth > 0 ); unsigned_int size; - fc::raw::unpack( s, size ); + fc::raw::unpack( s, size, _max_depth - 1 ); if( size.value > 0 ) { if( size.value > sizeof(Storage) ) { s.read( (char*)&u.data, sizeof(Storage) ); @@ -135,12 +137,12 @@ namespace fc { /* template - inline void pack( Stream& s, const boost::multiprecision::number& d ) { + inline void pack( Stream& s, const boost::multiprecision::number& d, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { s.write( (const char*)&d, sizeof(d) ); } template - inline void unpack( Stream& s, boost::multiprecision::number& u ) { + inline void unpack( Stream& s, boost::multiprecision::number& u, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { s.read( (const char*)&u, sizeof(u) ); } */ diff --git a/include/fc/interprocess/container.hpp b/include/fc/interprocess/container.hpp index ed761b8..6ed73f2 100644 --- a/include/fc/interprocess/container.hpp +++ b/include/fc/interprocess/container.hpp @@ -122,22 +122,24 @@ namespace fc { namespace bip = boost::interprocess; template - inline void pack( Stream& s, const bip::vector& value ) { - pack( s, unsigned_int((uint32_t)value.size()) ); - auto itr = value.begin(); - auto end = value.end(); - while( itr != end ) { - fc::raw::pack( s, *itr ); - ++itr; - } + inline void pack( Stream& s, const bip::vector& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { + FC_ASSERT( _max_depth > 0 ); + pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + auto itr = value.begin(); + auto end = value.end(); + while( itr != end ) { + fc::raw::pack( s, *itr, _max_depth - 1 ); + ++itr; + } } template - inline void unpack( Stream& s, bip::vector& value ) { - unsigned_int size; - unpack( s, size ); - value.clear(); value.resize(size); - for( auto& item : value ) - fc::raw::unpack( s, item ); + inline void unpack( Stream& s, bip::vector& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { + FC_ASSERT( _max_depth > 0 ); + unsigned_int size; + unpack( s, size, _max_depth - 1 ); + value.clear(); value.resize(size); + for( auto& item : value ) + fc::raw::unpack( s, item, _max_depth - 1 ); } } } diff --git a/include/fc/io/enum_type.hpp b/include/fc/io/enum_type.hpp index e41770c..46da877 100644 --- a/include/fc/io/enum_type.hpp +++ b/include/fc/io/enum_type.hpp @@ -11,16 +11,16 @@ namespace fc public: enum_type( EnumType t ) :value(t){} - + enum_type( IntType t ) :value( (EnumType)t ){} - + enum_type(){} - + explicit operator IntType()const { return static_cast(value); } operator EnumType()const { return value; } operator std::string()const { return fc::reflector::to_string(value); } - + enum_type& operator=( IntType i ) { value = (EnumType)i; return *this;} enum_type& operator=( EnumType i ) { value = i; return *this;} bool operator<( EnumType i ) const { return value < i; } @@ -60,19 +60,21 @@ namespace fc /** serializes like an IntType */ - namespace raw - { + namespace raw + { template - inline void pack( Stream& s, const fc::enum_type& tp ) + inline void pack( Stream& s, const fc::enum_type& tp, uint32_t _max_depth ) { - fc::raw::pack( s, static_cast(tp) ); + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, static_cast(tp), _max_depth - 1 ); } template - inline void unpack( Stream& s, fc::enum_type& tp ) + inline void unpack( Stream& s, fc::enum_type& tp, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); IntType t; - fc::raw::unpack( s, t ); + fc::raw::unpack( s, t, _max_depth - 1 ); tp = t; } } diff --git a/include/fc/io/raw.hpp b/include/fc/io/raw.hpp index 288c3c9..02873fc 100644 --- a/include/fc/io/raw.hpp +++ b/include/fc/io/raw.hpp @@ -19,70 +19,77 @@ namespace fc { namespace raw { template - inline void pack( Stream& s, const Arg0& a0, Args... args ) { - pack( s, a0 ); - pack( s, args... ); + inline void pack( Stream& s, const Arg0& a0, Args... args, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + pack( s, a0, _max_depth - 1 ); + pack( s, args..., _max_depth - 1 ); } template - inline void pack( Stream& s, const fc::exception& e ) + inline void pack( Stream& s, const fc::exception& e, uint32_t _max_depth ) { - fc::raw::pack( s, e.code() ); - fc::raw::pack( s, std::string(e.name()) ); - fc::raw::pack( s, std::string(e.what()) ); - fc::raw::pack( s, e.get_log() ); + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, e.code(), _max_depth - 1 ); + fc::raw::pack( s, std::string(e.name()), _max_depth - 1 ); + fc::raw::pack( s, std::string(e.what()), _max_depth - 1 ); + fc::raw::pack( s, e.get_log(), _max_depth - 1 ); } template - inline void unpack( Stream& s, fc::exception& e ) + inline void unpack( Stream& s, fc::exception& e, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); int64_t code; std::string name, what; log_messages msgs; - fc::raw::unpack( s, code ); - fc::raw::unpack( s, name ); - fc::raw::unpack( s, what ); - fc::raw::unpack( s, msgs ); + fc::raw::unpack( s, code, _max_depth - 1 ); + fc::raw::unpack( s, name, _max_depth - 1 ); + fc::raw::unpack( s, what, _max_depth - 1 ); + fc::raw::unpack( s, msgs, _max_depth - 1 ); e = fc::exception( fc::move(msgs), code, name, what ); } template - inline void pack( Stream& s, const fc::log_message& msg ) + inline void pack( Stream& s, const fc::log_message& msg, uint32_t _max_depth ) { - fc::raw::pack( s, variant(msg) ); + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, variant(msg), _max_depth - 1 ); } template - inline void unpack( Stream& s, fc::log_message& msg ) + inline void unpack( Stream& s, fc::log_message& msg, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); fc::variant vmsg; - fc::raw::unpack( s, vmsg ); + fc::raw::unpack( s, vmsg, _max_depth - 1 ); msg = vmsg.as(); } template - inline void pack( Stream& s, const fc::path& tp ) + inline void pack( Stream& s, const fc::path& tp, uint32_t _max_depth ) { - fc::raw::pack( s, tp.generic_string() ); + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, tp.generic_string(), _max_depth - 1 ); } template - inline void unpack( Stream& s, fc::path& tp ) + inline void unpack( Stream& s, fc::path& tp, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); std::string p; - fc::raw::unpack( s, p ); + fc::raw::unpack( s, p, _max_depth - 1 ); tp = p; } template - inline void pack( Stream& s, const fc::time_point_sec& tp ) + inline void pack( Stream& s, const fc::time_point_sec& tp, uint32_t _max_depth ) { uint32_t usec = tp.sec_since_epoch(); s.write( (const char*)&usec, sizeof(usec) ); } template - inline void unpack( Stream& s, fc::time_point_sec& tp ) + inline void unpack( Stream& s, fc::time_point_sec& tp, uint32_t _max_depth ) { try { uint32_t sec; s.read( (char*)&sec, sizeof(sec) ); @@ -90,14 +97,14 @@ namespace fc { } FC_RETHROW_EXCEPTIONS( warn, "" ) } template - inline void pack( Stream& s, const fc::time_point& tp ) + inline void pack( Stream& s, const fc::time_point& tp, uint32_t _max_depth ) { uint64_t usec = tp.time_since_epoch().count(); s.write( (const char*)&usec, sizeof(usec) ); } template - inline void unpack( Stream& s, fc::time_point& tp ) + inline void unpack( Stream& s, fc::time_point& tp, uint32_t _max_depth ) { try { uint64_t usec; s.read( (char*)&usec, sizeof(usec) ); @@ -105,14 +112,14 @@ namespace fc { } FC_RETHROW_EXCEPTIONS( warn, "" ) } template - inline void pack( Stream& s, const fc::microseconds& usec ) + inline void pack( Stream& s, const fc::microseconds& usec, uint32_t _max_depth ) { uint64_t usec_as_int64 = usec.count(); s.write( (const char*)&usec_as_int64, sizeof(usec_as_int64) ); } template - inline void unpack( Stream& s, fc::microseconds& usec ) + inline void unpack( Stream& s, fc::microseconds& usec, uint32_t _max_depth ) { try { uint64_t usec_as_int64; s.read( (char*)&usec_as_int64, sizeof(usec_as_int64) ); @@ -120,30 +127,32 @@ namespace fc { } FC_RETHROW_EXCEPTIONS( warn, "" ) } template - inline void pack( Stream& s, const fc::array& v) { + inline void pack( Stream& s, const fc::array& v, uint32_t _max_depth ) { s.write((const char*)&v.data[0],N*sizeof(T)); } template - inline void pack( Stream& s, const std::shared_ptr& v) + inline void pack( Stream& s, const std::shared_ptr& v, uint32_t _max_depth ) { - fc::raw::pack( s, *v ); + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, *v, _max_depth - 1 ); } template - inline void unpack( Stream& s, fc::array& v) + inline void unpack( Stream& s, fc::array& v, uint32_t _max_depth ) { try { s.read((char*)&v.data[0],N*sizeof(T)); } FC_RETHROW_EXCEPTIONS( warn, "fc::array", ("type",fc::get_typename::name())("length",N) ) } template - inline void unpack( Stream& s, std::shared_ptr& v) + inline void unpack( Stream& s, std::shared_ptr& v, uint32_t _max_depth ) { try { - v = std::make_shared(); - fc::raw::unpack( s, *v ); + FC_ASSERT( _max_depth > 0 ); + v = std::make_shared(); + fc::raw::unpack( s, *v, _max_depth - 1 ); } FC_RETHROW_EXCEPTIONS( warn, "std::shared_ptr", ("type",fc::get_typename::name()) ) } - template inline void pack( Stream& s, const signed_int& v ) { + template inline void pack( Stream& s, const signed_int& v, uint32_t _max_depth ) { uint32_t val = (v.value<<1) ^ (v.value>>31); do { uint8_t b = uint8_t(val) & 0x7f; @@ -153,7 +162,7 @@ namespace fc { } while( val ); } - template inline void pack( Stream& s, const unsigned_int& v ) { + template inline void pack( Stream& s, const unsigned_int& v, uint32_t _max_depth ) { uint64_t val = v.value; do { uint8_t b = uint8_t(val) & 0x7f; @@ -163,7 +172,7 @@ namespace fc { }while( val ); } - template inline void unpack( Stream& s, signed_int& vi ) { + template inline void unpack( Stream& s, signed_int& vi, uint32_t _max_depth ) { uint32_t v = 0; char b = 0; int by = 0; do { s.get(b); @@ -174,7 +183,7 @@ namespace fc { vi.value = v&0x01 ? vi.value : -vi.value; vi.value = -vi.value; } - template inline void unpack( Stream& s, unsigned_int& vi ) { + template inline void unpack( Stream& s, unsigned_int& vi, uint32_t _max_depth ) { uint64_t v = 0; char b = 0; uint8_t by = 0; do { s.get(b); @@ -184,83 +193,117 @@ namespace fc { vi.value = static_cast(v); } - template inline void unpack( Stream& s, const T& vi ) + template inline void unpack( Stream& s, const T& vi, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); T tmp; - fc::raw::unpack( s, tmp ); + fc::raw::unpack( s, tmp, _max_depth - 1 ); FC_ASSERT( vi == tmp ); } - template inline void pack( Stream& s, const char* v ) { fc::raw::pack( s, fc::string(v) ); } + template inline void pack( Stream& s, const char* v, uint32_t _max_depth ) + { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, fc::string(v), _max_depth - 1 ); + } template - void pack( Stream& s, const safe& v ) { fc::raw::pack( s, v.value ); } + void pack( Stream& s, const safe& v, uint32_t _max_depth ) + { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, v.value, _max_depth - 1 ); + } template - void unpack( Stream& s, fc::safe& v ) { fc::raw::unpack( s, v.value ); } - - template - void pack( Stream& s, const fc::fwd& v ) { - fc::raw::pack( *v ); + void unpack( Stream& s, fc::safe& v, uint32_t _max_depth ) + { + FC_ASSERT( _max_depth > 0 ); + fc::raw::unpack( s, v.value, _max_depth - 1 ); } template - void unpack( Stream& s, fc::fwd& v ) { - fc::raw::unpack( *v ); + void pack( Stream& s, const fc::fwd& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( *v, _max_depth - 1 ); // TODO not sure about this + } + + template + void unpack( Stream& s, fc::fwd& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::raw::unpack( *v, _max_depth - 1 ); // TODO not sure about this } template - void pack( Stream& s, const fc::smart_ref& v ) { fc::raw::pack( s, *v ); } + void pack( Stream& s, const fc::smart_ref& v, uint32_t _max_depth ) + { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, *v, _max_depth - 1 ); + } template - void unpack( Stream& s, fc::smart_ref& v ) { fc::raw::unpack( s, *v ); } + void unpack( Stream& s, fc::smart_ref& v, uint32_t _max_depth ) + { + FC_ASSERT( _max_depth > 0 ); + fc::raw::unpack( s, *v, _max_depth - 1 ); + } // optional template - void pack( Stream& s, const fc::optional& v ) { - fc::raw::pack( s, bool(!!v) ); - if( !!v ) fc::raw::pack( s, *v ); + void pack( Stream& s, const fc::optional& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, bool(!!v), _max_depth - 1 ); + if( !!v ) fc::raw::pack( s, *v, _max_depth - 1 ); } template - void unpack( Stream& s, fc::optional& v ) + void unpack( Stream& s, fc::optional& v, uint32_t _max_depth ) { try { - bool b; fc::raw::unpack( s, b ); - if( b ) { v = T(); fc::raw::unpack( s, *v ); } + FC_ASSERT( _max_depth > 0 ); + bool b; fc::raw::unpack( s, b, _max_depth - 1 ); + if( b ) { v = T(); fc::raw::unpack( s, *v, _max_depth - 1 ); } } FC_RETHROW_EXCEPTIONS( warn, "optional<${type}>", ("type",fc::get_typename::name() ) ) } // std::vector - template inline void pack( Stream& s, const std::vector& value ) { - fc::raw::pack( s, unsigned_int((uint32_t)value.size()) ); - if( value.size() ) - s.write( &value.front(), (uint32_t)value.size() ); + template inline void pack( Stream& s, const std::vector& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + if( value.size() ) + s.write( &value.front(), (uint32_t)value.size() ); } - template inline void unpack( Stream& s, std::vector& value ) { - unsigned_int size; fc::raw::unpack( s, size ); - FC_ASSERT( size.value < MAX_ARRAY_ALLOC_SIZE ); - value.resize(size.value); - if( value.size() ) - s.read( value.data(), value.size() ); + template inline void unpack( Stream& s, std::vector& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + unsigned_int size; fc::raw::unpack( s, size, _max_depth - 1 ); + FC_ASSERT( size.value < MAX_ARRAY_ALLOC_SIZE ); + value.resize(size.value); + if( value.size() ) + s.read( value.data(), value.size() ); } // fc::string - template inline void pack( Stream& s, const fc::string& v ) { - fc::raw::pack( s, unsigned_int((uint32_t)v.size())); - if( v.size() ) s.write( v.c_str(), v.size() ); + template inline void pack( Stream& s, const fc::string& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, unsigned_int((uint32_t)v.size()), _max_depth - 1 ); + if( v.size() ) s.write( v.c_str(), v.size() ); } - template inline void unpack( Stream& s, fc::string& v ) { - std::vector tmp; fc::raw::unpack(s,tmp); - if( tmp.size() ) - v = fc::string(tmp.data(),tmp.data()+tmp.size()); - else v = fc::string(); + template inline void unpack( Stream& s, fc::string& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + std::vector tmp; fc::raw::unpack( s, tmp, _max_depth - 1 ); + if( tmp.size() ) + v = fc::string( tmp.data(), tmp.data()+tmp.size() ); + else v = fc::string(); } // bool - template inline void pack( Stream& s, const bool& v ) { fc::raw::pack( s, uint8_t(v) ); } - template inline void unpack( Stream& s, bool& v ) + template inline void pack( Stream& s, const bool& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, uint8_t(v), _max_depth - 1 ); + } + template inline void unpack( Stream& s, bool& v, uint32_t _max_depth ) + { + FC_ASSERT( _max_depth > 0 ); uint8_t b; - fc::raw::unpack( s, b ); + fc::raw::unpack( s, b, _max_depth - 1 ); FC_ASSERT( (b & ~1) == 0 ); v=(b!=0); } @@ -269,49 +312,56 @@ namespace fc { template struct pack_object_visitor { - pack_object_visitor(const Class& _c, Stream& _s) - :c(_c),s(_s){} + pack_object_visitor( const Class& _c, Stream& _s, uint32_t _max_depth ) + :c(_c),s(_s),max_depth(_max_depth) + { + FC_ASSERT( _max_depth > 0 ); + } template void operator()( const char* name )const { - fc::raw::pack( s, c.*p ); + fc::raw::pack( s, c.*p, max_depth - 1 ); } private: const Class& c; Stream& s; + uint32_t max_depth; }; template struct unpack_object_visitor { - unpack_object_visitor(Class& _c, Stream& _s) - :c(_c),s(_s){} + unpack_object_visitor( Class& _c, Stream& _s, uint32_t _max_depth ) : c(_c),s(_s),max_depth(_max_depth) + { + FC_ASSERT( _max_depth > 0 ); + } template inline void operator()( const char* name )const { try { - fc::raw::unpack( s, c.*p ); + fc::raw::unpack( s, c.*p, max_depth - 1 ); } FC_RETHROW_EXCEPTIONS( warn, "Error unpacking field ${field}", ("field",name) ) } private: Class& c; Stream& s; + uint32_t max_depth; }; template struct if_class{ template - static inline void pack( Stream& s, const T& v ) { s << v; } + static inline void pack( Stream& s, const T& v, uint32_t _max_depth ) { s << v; } template - static inline void unpack( Stream& s, T& v ) { s >> v; } + static inline void unpack( Stream& s, T& v, uint32_t _max_depth ) { s >> v; } }; template<> struct if_class { template - static inline void pack( Stream& s, const T& v ) { + static inline void pack( Stream& s, const T& v, uint32_t _max_depth ) { s.write( (char*)&v, sizeof(v) ); } template - static inline void unpack( Stream& s, T& v ) { + static inline void unpack( Stream& s, T& v, uint32_t _max_depth ) { s.read( (char*)&v, sizeof(v) ); } }; @@ -319,24 +369,28 @@ namespace fc { template struct if_enum { template - static inline void pack( Stream& s, const T& v ) { - fc::reflector::visit( pack_object_visitor( v, s ) ); + static inline void pack( Stream& s, const T& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::reflector::visit( pack_object_visitor( v, s, _max_depth - 1 ) ); } template - static inline void unpack( Stream& s, T& v ) { - fc::reflector::visit( unpack_object_visitor( v, s ) ); + static inline void unpack( Stream& s, T& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::reflector::visit( unpack_object_visitor( v, s, _max_depth - 1 ) ); } }; template<> struct if_enum { template - static inline void pack( Stream& s, const T& v ) { - fc::raw::pack(s, (int64_t)v); + static inline void pack( Stream& s, const T& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, (int64_t)v, _max_depth - 1 ); } template - static inline void unpack( Stream& s, T& v ) { + static inline void unpack( Stream& s, T& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); int64_t temp; - fc::raw::unpack(s, temp); + fc::raw::unpack( s, temp, _max_depth - 1 ); v = (T)temp; } }; @@ -344,283 +398,314 @@ namespace fc { template struct if_reflected { template - static inline void pack( Stream& s, const T& v ) { - if_class::type>::pack(s,v); + static inline void pack( Stream& s, const T& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + if_class::type>::pack( s, v, _max_depth - 1 ); } template - static inline void unpack( Stream& s, T& v ) { - if_class::type>::unpack(s,v); + static inline void unpack( Stream& s, T& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + if_class::type>::unpack( s, v, _max_depth - 1 ); } }; template<> struct if_reflected { template - static inline void pack( Stream& s, const T& v ) { - if_enum< typename fc::reflector::is_enum >::pack(s,v); + static inline void pack( Stream& s, const T& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + if_enum< typename fc::reflector::is_enum >::pack( s, v, _max_depth - 1 ); } template - static inline void unpack( Stream& s, T& v ) { - if_enum< typename fc::reflector::is_enum >::unpack(s,v); + static inline void unpack( Stream& s, T& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + if_enum< typename fc::reflector::is_enum >::unpack( s, v, _max_depth - 1 ); } }; } // namesapce detail template - inline void pack( Stream& s, const std::unordered_set& value ) { - fc::raw::pack( s, unsigned_int((uint32_t)value.size()) ); - auto itr = value.begin(); - auto end = value.end(); - while( itr != end ) { - fc::raw::pack( s, *itr ); - ++itr; - } + inline void pack( Stream& s, const std::unordered_set& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + auto itr = value.begin(); + auto end = value.end(); + while( itr != end ) { + fc::raw::pack( s, *itr, _max_depth - 1 ); + ++itr; + } } template - inline void unpack( Stream& s, std::unordered_set& value ) { - unsigned_int size; fc::raw::unpack( s, size ); - value.clear(); - FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); - value.reserve(size.value); - for( uint32_t i = 0; i < size.value; ++i ) - { + inline void unpack( Stream& s, std::unordered_set& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + unsigned_int size; fc::raw::unpack( s, size, _max_depth - 1 ); + value.clear(); + FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); + value.reserve(size.value); + for( uint32_t i = 0; i < size.value; ++i ) + { T tmp; - fc::raw::unpack( s, tmp ); + fc::raw::unpack( s, tmp, _max_depth - 1 ); value.insert( std::move(tmp) ); - } + } } template - inline void pack( Stream& s, const std::pair& value ) { - fc::raw::pack( s, value.first ); - fc::raw::pack( s, value.second ); + inline void pack( Stream& s, const std::pair& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, value.first, _max_depth - 1 ); + fc::raw::pack( s, value.second, _max_depth - 1 ); } template - inline void unpack( Stream& s, std::pair& value ) + inline void unpack( Stream& s, std::pair& value, uint32_t _max_depth ) { - fc::raw::unpack( s, value.first ); - fc::raw::unpack( s, value.second ); + FC_ASSERT( _max_depth > 0 ); + fc::raw::unpack( s, value.first, _max_depth - 1 ); + fc::raw::unpack( s, value.second, _max_depth - 1 ); } template - inline void pack( Stream& s, const std::unordered_map& value ) { - fc::raw::pack( s, unsigned_int((uint32_t)value.size()) ); - auto itr = value.begin(); - auto end = value.end(); - while( itr != end ) { - fc::raw::pack( s, *itr ); - ++itr; - } + inline void pack( Stream& s, const std::unordered_map& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + auto itr = value.begin(); + auto end = value.end(); + while( itr != end ) { + fc::raw::pack( s, *itr, _max_depth - 1 ); + ++itr; + } } template - inline void unpack( Stream& s, std::unordered_map& value ) + inline void unpack( Stream& s, std::unordered_map& value, uint32_t _max_depth ) { - unsigned_int size; fc::raw::unpack( s, size ); - value.clear(); - FC_ASSERT( size.value*(sizeof(K)+sizeof(V)) < MAX_ARRAY_ALLOC_SIZE ); - value.reserve(size.value); - for( uint32_t i = 0; i < size.value; ++i ) - { + FC_ASSERT( _max_depth > 0 ); + unsigned_int size; fc::raw::unpack( s, size, _max_depth - 1 ); + value.clear(); + FC_ASSERT( size.value*(sizeof(K)+sizeof(V)) < MAX_ARRAY_ALLOC_SIZE ); + value.reserve(size.value); + for( uint32_t i = 0; i < size.value; ++i ) + { std::pair tmp; - fc::raw::unpack( s, tmp ); + fc::raw::unpack( s, tmp, _max_depth - 1 ); value.insert( std::move(tmp) ); - } + } } template - inline void pack( Stream& s, const std::map& value ) { - fc::raw::pack( s, unsigned_int((uint32_t)value.size()) ); - auto itr = value.begin(); - auto end = value.end(); - while( itr != end ) { - fc::raw::pack( s, *itr ); - ++itr; - } + inline void pack( Stream& s, const std::map& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + auto itr = value.begin(); + auto end = value.end(); + while( itr != end ) { + fc::raw::pack( s, *itr, _max_depth - 1 ); + ++itr; + } } template - inline void unpack( Stream& s, std::map& value ) + inline void unpack( Stream& s, std::map& value, uint32_t _max_depth ) { - unsigned_int size; fc::raw::unpack( s, size ); - value.clear(); - FC_ASSERT( size.value*(sizeof(K)+sizeof(V)) < MAX_ARRAY_ALLOC_SIZE ); - for( uint32_t i = 0; i < size.value; ++i ) - { + FC_ASSERT( _max_depth > 0 ); + unsigned_int size; fc::raw::unpack( s, size, _max_depth - 1 ); + value.clear(); + FC_ASSERT( size.value*(sizeof(K)+sizeof(V)) < MAX_ARRAY_ALLOC_SIZE ); + for( uint32_t i = 0; i < size.value; ++i ) + { std::pair tmp; - fc::raw::unpack( s, tmp ); + fc::raw::unpack( s, tmp, _max_depth - 1 ); value.insert( std::move(tmp) ); - } + } } template - inline void pack( Stream& s, const std::deque& value ) { - fc::raw::pack( s, unsigned_int((uint32_t)value.size()) ); - auto itr = value.begin(); - auto end = value.end(); - while( itr != end ) { - fc::raw::pack( s, *itr ); - ++itr; - } + inline void pack( Stream& s, const std::deque& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + auto itr = value.begin(); + auto end = value.end(); + while( itr != end ) { + fc::raw::pack( s, *itr, _max_depth - 1 ); + ++itr; + } } template - inline void unpack( Stream& s, std::deque& value ) { - unsigned_int size; fc::raw::unpack( s, size ); - FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); - value.resize(size.value); - auto itr = value.begin(); - auto end = value.end(); - while( itr != end ) { - fc::raw::unpack( s, *itr ); - ++itr; - } + inline void unpack( Stream& s, std::deque& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + unsigned_int size; fc::raw::unpack( s, size, _max_depth - 1 ); + FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); + value.resize(size.value); + auto itr = value.begin(); + auto end = value.end(); + while( itr != end ) { + fc::raw::unpack( s, *itr, _max_depth - 1 ); + ++itr; + } } template - inline void pack( Stream& s, const std::vector& value ) { - fc::raw::pack( s, unsigned_int((uint32_t)value.size()) ); - auto itr = value.begin(); - auto end = value.end(); - while( itr != end ) { - fc::raw::pack( s, *itr ); - ++itr; - } + inline void pack( Stream& s, const std::vector& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + auto itr = value.begin(); + auto end = value.end(); + while( itr != end ) { + fc::raw::pack( s, *itr, _max_depth - 1 ); + ++itr; + } } template - inline void unpack( Stream& s, std::vector& value ) { - unsigned_int size; fc::raw::unpack( s, size ); - FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); - value.resize(size.value); - auto itr = value.begin(); - auto end = value.end(); - while( itr != end ) { - fc::raw::unpack( s, *itr ); - ++itr; - } + inline void unpack( Stream& s, std::vector& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + unsigned_int size; fc::raw::unpack( s, size, _max_depth - 1 ); + FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); + value.resize(size.value); + auto itr = value.begin(); + auto end = value.end(); + while( itr != end ) { + fc::raw::unpack( s, *itr, _max_depth - 1 ); + ++itr; + } } template - inline void pack( Stream& s, const std::set& value ) { - fc::raw::pack( s, unsigned_int((uint32_t)value.size()) ); - auto itr = value.begin(); - auto end = value.end(); - while( itr != end ) { - fc::raw::pack( s, *itr ); - ++itr; - } + inline void pack( Stream& s, const std::set& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + auto itr = value.begin(); + auto end = value.end(); + while( itr != end ) { + fc::raw::pack( s, *itr, _max_depth - 1 ); + ++itr; + } } template - inline void unpack( Stream& s, std::set& value ) { - unsigned_int size; fc::raw::unpack( s, size ); - for( uint64_t i = 0; i < size.value; ++i ) - { - T tmp; - fc::raw::unpack( s, tmp ); - value.insert( std::move(tmp) ); - } + inline void unpack( Stream& s, std::set& value, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + unsigned_int size; fc::raw::unpack( s, size, _max_depth - 1 ); + for( uint64_t i = 0; i < size.value; ++i ) + { + T tmp; + fc::raw::unpack( s, tmp, _max_depth - 1 ); + value.insert( std::move(tmp) ); + } } template - inline void pack( Stream& s, const T& v ) { - fc::raw::detail::if_reflected< typename fc::reflector::is_defined >::pack(s,v); + inline void pack( Stream& s, const T& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + fc::raw::detail::if_reflected< typename fc::reflector::is_defined >::pack( s, v, _max_depth - 1 ); } template - inline void unpack( Stream& s, T& v ) + inline void unpack( Stream& s, T& v, uint32_t _max_depth ) { try { - fc::raw::detail::if_reflected< typename fc::reflector::is_defined >::unpack(s,v); + FC_ASSERT( _max_depth > 0 ); + fc::raw::detail::if_reflected< typename fc::reflector::is_defined >::unpack( s, v, _max_depth - 1 ); } FC_RETHROW_EXCEPTIONS( warn, "error unpacking ${type}", ("type",fc::get_typename::name() ) ) } template - inline size_t pack_size( const T& v ) + inline size_t pack_size( const T& v ) { - datastream ps; - fc::raw::pack(ps,v ); - return ps.tellp(); + datastream ps; + fc::raw::pack( ps, v ); + return ps.tellp(); } template - inline std::vector pack( const T& v ) { - datastream ps; - fc::raw::pack(ps,v ); - std::vector vec(ps.tellp()); + inline std::vector pack( const T& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + datastream ps; + fc::raw::pack( ps, v, _max_depth - 1 ); + std::vector vec(ps.tellp()); - if( vec.size() ) { - datastream ds( vec.data(), size_t(vec.size()) ); - fc::raw::pack(ds,v); - } - return vec; + if( vec.size() ) { + datastream ds( vec.data(), size_t(vec.size()) ); + fc::raw::pack( ds, v, _max_depth - 1 ); + } + return vec; } template - inline std::vector pack( const T& v, Next... next ) { - datastream ps; - fc::raw::pack(ps,v,next...); - std::vector vec(ps.tellp()); + inline std::vector pack( const T& v, Next... next, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + datastream ps; + fc::raw::pack( ps, v, next..., _max_depth - 1 ); + std::vector vec(ps.tellp()); - if( vec.size() ) { - datastream ds( vec.data(), size_t(vec.size()) ); - fc::raw::pack(ds,v,next...); - } - return vec; + if( vec.size() ) { + datastream ds( vec.data(), size_t(vec.size()) ); + fc::raw::pack( ds, v, next..., _max_depth - 1 ); + } + return vec; } template - inline T unpack( const std::vector& s ) + inline T unpack( const std::vector& s, uint32_t _max_depth ) { try { - T tmp; - if( s.size() ) { - datastream ds( s.data(), size_t(s.size()) ); - fc::raw::unpack(ds,tmp); - } - return tmp; + FC_ASSERT( _max_depth > 0 ); + T tmp; + if( s.size() ) { + datastream ds( s.data(), size_t(s.size()) ); + fc::raw::unpack( ds, tmp, _max_depth - 1 ); + } + return tmp; } FC_RETHROW_EXCEPTIONS( warn, "error unpacking ${type}", ("type",fc::get_typename::name() ) ) } template - inline void unpack( const std::vector& s, T& tmp ) + inline void unpack( const std::vector& s, T& tmp, uint32_t _max_depth ) { try { - if( s.size() ) { - datastream ds( s.data(), size_t(s.size()) ); - fc::raw::unpack(ds,tmp); - } + FC_ASSERT( _max_depth > 0 ); + if( s.size() ) { + datastream ds( s.data(), size_t(s.size()) ); + fc::raw::unpack( ds, tmp, _max_depth - 1 ); + } } FC_RETHROW_EXCEPTIONS( warn, "error unpacking ${type}", ("type",fc::get_typename::name() ) ) } template - inline void pack( char* d, uint32_t s, const T& v ) { - datastream ds(d,s); - fc::raw::pack(ds,v ); + inline void pack( char* d, uint32_t s, const T& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); + datastream ds(d,s); + fc::raw::pack( ds, v, _max_depth - 1 ); } template - inline T unpack( const char* d, uint32_t s ) + inline T unpack( const char* d, uint32_t s, uint32_t _max_depth ) { try { - T v; - datastream ds( d, s ); - fc::raw::unpack(ds,v); - return v; + FC_ASSERT( _max_depth > 0 ); + T v; + datastream ds( d, s ); + fc::raw::unpack( ds, v, _max_depth - 1 ); + return v; } FC_RETHROW_EXCEPTIONS( warn, "error unpacking ${type}", ("type",fc::get_typename::name() ) ) } template - inline void unpack( const char* d, uint32_t s, T& v ) + inline void unpack( const char* d, uint32_t s, T& v, uint32_t _max_depth ) { try { - datastream ds( d, s ); - fc::raw::unpack(ds,v); - return v; + FC_ASSERT( _max_depth > 0 ); + datastream ds( d, s ); + fc::raw::unpack( ds, v, _max_depth - 1 ); + return v; } FC_RETHROW_EXCEPTIONS( warn, "error unpacking ${type}", ("type",fc::get_typename::name() ) ) } template struct pack_static_variant { Stream& stream; - pack_static_variant( Stream& s ):stream(s){} + uint32_t max_depth; + pack_static_variant( Stream& s, uint32_t _max_depth ):stream(s),max_depth(_max_depth) + { + FC_ASSERT( _max_depth > 0 ); + } typedef void result_type; template void operator()( const T& v )const { - fc::raw::pack( stream, v ); + fc::raw::pack( stream, v, max_depth - 1 ); } }; @@ -628,29 +713,35 @@ namespace fc { struct unpack_static_variant { Stream& stream; - unpack_static_variant( Stream& s ):stream(s){} + uint32_t max_depth; + unpack_static_variant( Stream& s, uint32_t _max_depth ) : stream(s),max_depth(_max_depth) + { + FC_ASSERT( _max_depth > 0 ); + } typedef void result_type; template void operator()( T& v )const { - fc::raw::unpack( stream, v ); + fc::raw::unpack( stream, v, max_depth - 1 ); } }; template - void pack( Stream& s, const static_variant& sv ) + void pack( Stream& s, const static_variant& sv, uint32_t _max_depth ) { - fc::raw::pack( s, unsigned_int(sv.which()) ); - sv.visit( pack_static_variant(s) ); + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, unsigned_int(sv.which()), _max_depth - 1 ); + sv.visit( pack_static_variant( s, _max_depth - 1 ) ); } - template void unpack( Stream& s, static_variant& sv ) + template void unpack( Stream& s, static_variant& sv, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); unsigned_int w; - fc::raw::unpack( s, w ); + fc::raw::unpack( s, w, _max_depth - 1 ); sv.set_which(w.value); - sv.visit( unpack_static_variant(s) ); + sv.visit( unpack_static_variant( s, _max_depth - 1 ) ); } } } // namespace fc::raw diff --git a/include/fc/io/raw_fwd.hpp b/include/fc/io/raw_fwd.hpp index f397202..42e955a 100644 --- a/include/fc/io/raw_fwd.hpp +++ b/include/fc/io/raw_fwd.hpp @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -11,9 +12,9 @@ #include #include -#define MAX_ARRAY_ALLOC_SIZE (1024*1024*10) +#define MAX_ARRAY_ALLOC_SIZE (1024*1024*10) -namespace fc { +namespace fc { class time_point; class time_point_sec; class variant; @@ -31,94 +32,94 @@ namespace fc { template inline size_t pack_size( const T& v ); - template inline void pack( Stream& s, const fc::fixed_string& u ); - template inline void unpack( Stream& s, fc::fixed_string& u ); + template inline void pack( Stream& s, const fc::fixed_string& u, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, fc::fixed_string& u, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); template - inline void pack( Stream& s, const fc::enum_type& tp ); + inline void pack( Stream& s, const fc::enum_type& tp, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); template - inline void unpack( Stream& s, fc::enum_type& tp ); + inline void unpack( Stream& s, fc::enum_type& tp, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline void pack( Stream& s, const std::set& value ); - template inline void unpack( Stream& s, std::set& value ); - template inline void pack( Stream& s, const std::unordered_set& value ); - template inline void unpack( Stream& s, std::unordered_set& value ); + template inline void pack( Stream& s, const std::set& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, std::set& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void pack( Stream& s, const std::unordered_set& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, std::unordered_set& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template void pack( Stream& s, const static_variant& sv ); - template void unpack( Stream& s, static_variant& sv ); + template void pack( Stream& s, const static_variant& sv, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template void unpack( Stream& s, static_variant& sv, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline void pack( Stream& s, const flat_set& value ); - template inline void unpack( Stream& s, flat_set& value ); + //template inline void pack( Stream& s, const flat_set& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + //template inline void unpack( Stream& s, flat_set& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline void pack( Stream& s, const std::deque& value ); - template inline void unpack( Stream& s, std::deque& value ); + //template inline void pack( Stream& s, const std::deque& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + //template inline void unpack( Stream& s, std::deque& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline void pack( Stream& s, const std::unordered_map& value ); - template inline void unpack( Stream& s, std::unordered_map& value ); + template inline void pack( Stream& s, const std::unordered_map& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, std::unordered_map& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline void pack( Stream& s, const std::map& value ); - template inline void unpack( Stream& s, std::map& value ); + template inline void pack( Stream& s, const std::map& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, std::map& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline void pack( Stream& s, const flat_map& value ); - template inline void unpack( Stream& s, flat_map& value ); + //template inline void pack( Stream& s, const flat_map& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + //template inline void unpack( Stream& s, flat_map& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline void pack( Stream& s, const std::pair& value ); - template inline void unpack( Stream& s, std::pair& value ); + template inline void pack( Stream& s, const std::pair& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, std::pair& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline void pack( Stream& s, const variant_object& v ); - template inline void unpack( Stream& s, variant_object& v ); - template inline void pack( Stream& s, const variant& v ); - template inline void unpack( Stream& s, variant& v ); + template inline void pack( Stream& s, const variant_object& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, variant_object& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void pack( Stream& s, const variant& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, variant& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline void pack( Stream& s, const path& v ); - template inline void unpack( Stream& s, path& v ); - template inline void pack( Stream& s, const ip::endpoint& v ); - template inline void unpack( Stream& s, ip::endpoint& v ); + template inline void pack( Stream& s, const path& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, path& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void pack( Stream& s, const ip::endpoint& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, ip::endpoint& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template void unpack( Stream& s, fc::optional& v ); - template void unpack( Stream& s, const T& v ); - template void pack( Stream& s, const fc::optional& v ); - template void pack( Stream& s, const safe& v ); - template void unpack( Stream& s, fc::safe& v ); + template void unpack( Stream& s, fc::optional& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template void unpack( Stream& s, const T& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template void pack( Stream& s, const fc::optional& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template void pack( Stream& s, const safe& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template void unpack( Stream& s, fc::safe& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template void unpack( Stream& s, time_point& ); - template void pack( Stream& s, const time_point& ); - template void unpack( Stream& s, time_point_sec& ); - template void pack( Stream& s, const time_point_sec& ); - template void unpack( Stream& s, std::string& ); - template void pack( Stream& s, const std::string& ); - template void unpack( Stream& s, fc::ecc::public_key& ); - template void pack( Stream& s, const fc::ecc::public_key& ); - template void unpack( Stream& s, fc::ecc::private_key& ); - template void pack( Stream& s, const fc::ecc::private_key& ); + template void unpack( Stream& s, time_point&, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template void pack( Stream& s, const time_point&, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template void unpack( Stream& s, time_point_sec&, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template void pack( Stream& s, const time_point_sec&, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template void unpack( Stream& s, std::string&, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template void pack( Stream& s, const std::string&, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template void unpack( Stream& s, fc::ecc::public_key&, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template void pack( Stream& s, const fc::ecc::public_key&, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template void unpack( Stream& s, fc::ecc::private_key&, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template void pack( Stream& s, const fc::ecc::private_key&, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline void pack( Stream& s, const T& v ); - template inline void unpack( Stream& s, T& v ); + template inline void pack( Stream& s, const T& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, T& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline void pack( Stream& s, const std::vector& v ); - template inline void unpack( Stream& s, std::vector& v ); + template inline void pack( Stream& s, const std::vector& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, std::vector& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline void pack( Stream& s, const signed_int& v ); - template inline void unpack( Stream& s, signed_int& vi ); + template inline void pack( Stream& s, const signed_int& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, signed_int& vi, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline void pack( Stream& s, const unsigned_int& v ); - template inline void unpack( Stream& s, unsigned_int& vi ); + template inline void pack( Stream& s, const unsigned_int& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, unsigned_int& vi, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline void pack( Stream& s, const char* v ); - template inline void pack( Stream& s, const std::vector& value ); - template inline void unpack( Stream& s, std::vector& value ); + template inline void pack( Stream& s, const char* v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void pack( Stream& s, const std::vector& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, std::vector& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline void pack( Stream& s, const fc::array& v); - template inline void unpack( Stream& s, fc::array& v); + template inline void pack( Stream& s, const fc::array& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, fc::array& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH); - template inline void pack( Stream& s, const bool& v ); - template inline void unpack( Stream& s, bool& v ); + template inline void pack( Stream& s, const bool& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( Stream& s, bool& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); - template inline std::vector pack( const T& v ); - template inline T unpack( const std::vector& s ); - template inline T unpack( const char* d, uint32_t s ); - template inline void unpack( const char* d, uint32_t s, T& v ); + template inline std::vector pack( const T& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline T unpack( const std::vector& s, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline T unpack( const char* d, uint32_t s, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); + template inline void unpack( const char* d, uint32_t s, T& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ); } } diff --git a/include/fc/io/raw_variant.hpp b/include/fc/io/raw_variant.hpp index 24fe3f6..9df0118 100644 --- a/include/fc/io/raw_variant.hpp +++ b/include/fc/io/raw_variant.hpp @@ -10,53 +10,59 @@ namespace fc { namespace raw { class variant_packer : public variant::visitor { public: - variant_packer( Stream& _s ):s(_s){} + variant_packer( Stream& _s, uint32_t _max_depth ):s(_s),max_depth(_max_depth) + { + FC_ASSERT( _max_depth > 0 ); + } virtual void handle()const { } virtual void handle( const int64_t& v )const { - fc::raw::pack( s, v ); + fc::raw::pack( s, v, max_depth - 1 ); } virtual void handle( const uint64_t& v )const { - fc::raw::pack( s, v ); + fc::raw::pack( s, v, max_depth - 1 ); } - virtual void handle( const double& v )const + virtual void handle( const double& v )const { - fc::raw::pack( s, v ); + fc::raw::pack( s, v, max_depth - 1 ); } virtual void handle( const bool& v )const { - fc::raw::pack( s, v ); + fc::raw::pack( s, v, max_depth - 1 ); } virtual void handle( const string& v )const { - fc::raw::pack( s, v ); + fc::raw::pack( s, v, max_depth - 1 ); } virtual void handle( const variant_object& v)const { - fc::raw::pack( s, v ); + fc::raw::pack( s, v, max_depth - 1 ); } virtual void handle( const variants& v)const { - fc::raw::pack( s, v ); + fc::raw::pack( s, v, max_depth - 1 ); } - + Stream& s; - + uint32_t max_depth; + }; - template - inline void pack( Stream& s, const variant& v ) + template + inline void pack( Stream& s, const variant& v, uint32_t _max_depth ) { - pack( s, uint8_t(v.get_type()) ); - v.visit( variant_packer(s) ); + FC_ASSERT( _max_depth > 0 ); + pack( s, uint8_t(v.get_type()), _max_depth - 1 ); + v.visit( variant_packer( s, _max_depth - 1 ) ); } - template - inline void unpack( Stream& s, variant& v ) + template + inline void unpack( Stream& s, variant& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); uint8_t t; - unpack( s, t ); + unpack( s, t, _max_depth - 1 ); switch( t ) { case variant::null_type: @@ -64,49 +70,49 @@ namespace fc { namespace raw { case variant::int64_type: { int64_t val; - raw::unpack(s,val); + raw::unpack( s, val, _max_depth - 1 ); v = val; return; } case variant::uint64_type: { uint64_t val; - raw::unpack(s,val); + raw::unpack( s, val, _max_depth - 1 ); v = val; return; } case variant::double_type: { double val; - raw::unpack(s,val); + raw::unpack( s, val, _max_depth - 1 ); v = val; return; } case variant::bool_type: { bool val; - raw::unpack(s,val); + raw::unpack( s, val, _max_depth - 1 ); v = val; return; } case variant::string_type: { fc::string val; - raw::unpack(s,val); + raw::unpack( s, val, _max_depth - 1 ); v = fc::move(val); return; } case variant::array_type: { variants val; - raw::unpack(s,val); + raw::unpack( s, val, _max_depth - 1 ); v = fc::move(val); return; } case variant::object_type: { - variant_object val; - raw::unpack(s,val); + variant_object val; + raw::unpack( s, val, _max_depth - 1 ); v = fc::move(val); return; } @@ -115,22 +121,24 @@ namespace fc { namespace raw { } } - template - inline void pack( Stream& s, const variant_object& v ) + template + inline void pack( Stream& s, const variant_object& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); unsigned_int vs = (uint32_t)v.size(); - pack( s, vs ); + pack( s, vs, _max_depth - 1 ); for( auto itr = v.begin(); itr != v.end(); ++itr ) { - pack( s, itr->key() ); - pack( s, itr->value() ); + pack( s, itr->key(), _max_depth - 1 ); + pack( s, itr->value(), _max_depth - 1 ); } } - template - inline void unpack( Stream& s, variant_object& v ) + template + inline void unpack( Stream& s, variant_object& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); unsigned_int vs; - unpack( s, vs ); + unpack( s, vs, _max_depth - 1 ); mutable_variant_object mvo; mvo.reserve(vs.value); @@ -138,8 +146,8 @@ namespace fc { namespace raw { { fc::string key; fc::variant value; - fc::raw::unpack(s,key); - fc::raw::unpack(s,value); + fc::raw::unpack( s, key, _max_depth - 1 ); + fc::raw::unpack( s, value, _max_depth - 1 ); mvo.set( fc::move(key), fc::move(value) ); } v = fc::move(mvo); diff --git a/include/fc/network/ip.hpp b/include/fc/network/ip.hpp index 5a1bd0c..5292dbf 100644 --- a/include/fc/network/ip.hpp +++ b/include/fc/network/ip.hpp @@ -40,7 +40,7 @@ namespace fc { private: uint32_t _ip; }; - + class endpoint { public: endpoint(); @@ -58,16 +58,16 @@ namespace fc { friend bool operator==( const endpoint& a, const endpoint& b ); friend bool operator!=( const endpoint& a, const endpoint& b ); friend bool operator< ( const endpoint& a, const endpoint& b ); - + private: /** * The compiler pads endpoint to a full 8 bytes, so while * a port number is limited in range to 16 bits, we specify - * a full 32 bits so that memcmp can be used with sizeof(), - * otherwise 2 bytes will be 'random' and you do not know + * a full 32 bits so that memcmp can be used with sizeof(), + * otherwise 2 bytes will be 'random' and you do not know * where they are stored. */ - uint32_t _port; + uint32_t _port; address _ip; }; @@ -80,41 +80,45 @@ namespace fc { void from_variant( const variant& var, ip::address& vo ); - namespace raw + namespace raw { - template - inline void pack( Stream& s, const ip::address& v ) + template + inline void pack( Stream& s, const ip::address& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { - fc::raw::pack( s, uint32_t(v) ); + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, uint32_t(v), _max_depth - 1 ); } - template - inline void unpack( Stream& s, ip::address& v ) + template + inline void unpack( Stream& s, ip::address& v, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { + FC_ASSERT( _max_depth > 0 ); uint32_t _ip; - fc::raw::unpack( s, _ip ); + fc::raw::unpack( s, _ip, _max_depth - 1 ); v = ip::address(_ip); } - template - inline void pack( Stream& s, const ip::endpoint& v ) + template + inline void pack( Stream& s, const ip::endpoint& v, uint32_t _max_depth ) { - fc::raw::pack( s, v.get_address() ); - fc::raw::pack( s, v.port() ); + FC_ASSERT( _max_depth > 0 ); + fc::raw::pack( s, v.get_address(), _max_depth - 1 ); + fc::raw::pack( s, v.port(), _max_depth - 1 ); } - template - inline void unpack( Stream& s, ip::endpoint& v ) + template + inline void unpack( Stream& s, ip::endpoint& v, uint32_t _max_depth ) { + FC_ASSERT( _max_depth > 0 ); ip::address a; uint16_t p; - fc::raw::unpack( s, a ); - fc::raw::unpack( s, p ); + fc::raw::unpack( s, a, _max_depth - 1 ); + fc::raw::unpack( s, p, _max_depth - 1 ); v = ip::endpoint(a,p); } } } // namespace fc -FC_REFLECT_TYPENAME( fc::ip::address ) -FC_REFLECT_TYPENAME( fc::ip::endpoint ) +FC_REFLECT_TYPENAME( fc::ip::address ) +FC_REFLECT_TYPENAME( fc::ip::endpoint ) namespace std { template<> diff --git a/include/fc/real128.hpp b/include/fc/real128.hpp index 3a7d26d..142f5ec 100644 --- a/include/fc/real128.hpp +++ b/include/fc/real128.hpp @@ -1,5 +1,5 @@ #pragma once -#include +#include #define FC_REAL128_PRECISION (uint64_t(1000000) * uint64_t(1000000) * uint64_t(1000000)) @@ -27,7 +27,7 @@ namespace fc { real128& operator -= ( const real128& o ); real128& operator /= ( const real128& o ); real128& operator *= ( const real128& o ); - + static real128 from_fixed( const uint128& fixed ); uint64_t to_uint64()const; @@ -39,12 +39,15 @@ namespace fc { void to_variant( const real128& var, variant& vo ); void from_variant( const variant& var, real128& vo ); - namespace raw + namespace raw { template - inline void pack( Stream& s, const real128& value_to_pack ) { s.write( (char*)&value_to_pack, sizeof(value_to_pack) ); } + inline void pack( Stream& s, const real128& value_to_pack, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) + { s.write( (char*)&value_to_pack, sizeof(value_to_pack) ); } + template - inline void unpack( Stream& s, real128& value_to_unpack ) { s.read( (char*)&value_to_unpack, sizeof(value_to_unpack) ); } + inline void unpack( Stream& s, real128& value_to_unpack, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) + { s.read( (char*)&value_to_unpack, sizeof(value_to_unpack) ); } } diff --git a/include/fc/uint128.hpp b/include/fc/uint128.hpp index 5e599dc..e2a72cc 100644 --- a/include/fc/uint128.hpp +++ b/include/fc/uint128.hpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -126,9 +127,9 @@ namespace fc namespace raw { template - inline void pack( Stream& s, const uint128& u ) { s.write( (char*)&u, sizeof(u) ); } + inline void pack( Stream& s, const uint128& u, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { s.write( (char*)&u, sizeof(u) ); } template - inline void unpack( Stream& s, uint128& u ) { s.read( (char*)&u, sizeof(u) ); } + inline void unpack( Stream& s, uint128& u, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { s.read( (char*)&u, sizeof(u) ); } } size_t city_hash_size_t(const char *buf, size_t len); From 43ac0b0520613fc3256e7b2dae0b91d0c27c1d2a Mon Sep 17 00:00:00 2001 From: abitmore Date: Tue, 13 Mar 2018 13:03:37 -0400 Subject: [PATCH 46/46] Slightly improve performance for pack/unpack --- include/fc/container/flat.hpp | 30 +++-- include/fc/interprocess/container.hpp | 10 +- include/fc/io/raw.hpp | 155 +++++++++++++++----------- include/fc/io/raw_variant.hpp | 54 ++++----- include/fc/network/ip.hpp | 10 +- 5 files changed, 148 insertions(+), 111 deletions(-) diff --git a/include/fc/container/flat.hpp b/include/fc/container/flat.hpp index 0b494f2..b323c25 100644 --- a/include/fc/container/flat.hpp +++ b/include/fc/container/flat.hpp @@ -10,36 +10,39 @@ namespace fc { template inline void pack( Stream& s, const flat_set& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + --_max_depth; + pack( s, unsigned_int((uint32_t)value.size()), _max_depth ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { - fc::raw::pack( s, *itr, _max_depth - 1 ); + fc::raw::pack( s, *itr, _max_depth ); ++itr; } } template inline void unpack( Stream& s, flat_set& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - unsigned_int size; unpack( s, size, _max_depth - 1 ); + --_max_depth; + unsigned_int size; unpack( s, size, _max_depth ); value.clear(); FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); value.reserve(size.value); for( uint32_t i = 0; i < size.value; ++i ) { T tmp; - fc::raw::unpack( s, tmp, _max_depth - 1 ); + fc::raw::unpack( s, tmp, _max_depth ); value.insert( std::move(tmp) ); } } template inline void pack( Stream& s, const flat_map& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + --_max_depth; + pack( s, unsigned_int((uint32_t)value.size()), _max_depth ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { - fc::raw::pack( s, *itr, _max_depth - 1 ); + fc::raw::pack( s, *itr, _max_depth ); ++itr; } } @@ -47,14 +50,15 @@ namespace fc { inline void unpack( Stream& s, flat_map& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - unsigned_int size; unpack( s, size, _max_depth - 1 ); + --_max_depth; + unsigned_int size; unpack( s, size, _max_depth ); value.clear(); FC_ASSERT( size.value*(sizeof(K)+sizeof(V)) < MAX_ARRAY_ALLOC_SIZE ); value.reserve(size.value); for( uint32_t i = 0; i < size.value; ++i ) { std::pair tmp; - fc::raw::unpack( s, tmp, _max_depth - 1 ); + fc::raw::unpack( s, tmp, _max_depth ); value.insert( std::move(tmp) ); } } @@ -62,12 +66,13 @@ namespace fc { template void pack( Stream& s, const bip::vector& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + --_max_depth; + pack( s, unsigned_int((uint32_t)value.size()), _max_depth ); if( !std::is_fundamental::value ) { auto itr = value.begin(); auto end = value.end(); while( itr != end ) { - fc::raw::pack( s, *itr, _max_depth - 1 ); + fc::raw::pack( s, *itr, _max_depth ); ++itr; } } else { @@ -78,12 +83,13 @@ namespace fc { template void unpack( Stream& s, bip::vector& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); + --_max_depth; unsigned_int size; - unpack( s, size, _max_depth - 1 ); + unpack( s, size, _max_depth ); value.resize( size ); if( !std::is_fundamental::value ) { for( auto& item : value ) - unpack( s, item, _max_depth - 1 ); + unpack( s, item, _max_depth ); } else { s.read( (char*)value.data(), value.size() ); } diff --git a/include/fc/interprocess/container.hpp b/include/fc/interprocess/container.hpp index 6ed73f2..f61a68c 100644 --- a/include/fc/interprocess/container.hpp +++ b/include/fc/interprocess/container.hpp @@ -124,22 +124,24 @@ namespace fc { template inline void pack( Stream& s, const bip::vector& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { FC_ASSERT( _max_depth > 0 ); - pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + --_max_depth; + pack( s, unsigned_int((uint32_t)value.size()), _max_depth ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { - fc::raw::pack( s, *itr, _max_depth - 1 ); + fc::raw::pack( s, *itr, _max_depth ); ++itr; } } template inline void unpack( Stream& s, bip::vector& value, uint32_t _max_depth=FC_PACK_MAX_DEPTH ) { FC_ASSERT( _max_depth > 0 ); + --_max_depth; unsigned_int size; - unpack( s, size, _max_depth - 1 ); + unpack( s, size, _max_depth ); value.clear(); value.resize(size); for( auto& item : value ) - fc::raw::unpack( s, item, _max_depth - 1 ); + fc::raw::unpack( s, item, _max_depth ); } } } diff --git a/include/fc/io/raw.hpp b/include/fc/io/raw.hpp index 02873fc..d7e8bcd 100644 --- a/include/fc/io/raw.hpp +++ b/include/fc/io/raw.hpp @@ -21,31 +21,34 @@ namespace fc { template inline void pack( Stream& s, const Arg0& a0, Args... args, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - pack( s, a0, _max_depth - 1 ); - pack( s, args..., _max_depth - 1 ); + --_max_depth; + pack( s, a0, _max_depth ); + pack( s, args..., _max_depth ); } template inline void pack( Stream& s, const fc::exception& e, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - fc::raw::pack( s, e.code(), _max_depth - 1 ); - fc::raw::pack( s, std::string(e.name()), _max_depth - 1 ); - fc::raw::pack( s, std::string(e.what()), _max_depth - 1 ); - fc::raw::pack( s, e.get_log(), _max_depth - 1 ); + --_max_depth; + fc::raw::pack( s, e.code(), _max_depth ); + fc::raw::pack( s, std::string(e.name()), _max_depth ); + fc::raw::pack( s, std::string(e.what()), _max_depth ); + fc::raw::pack( s, e.get_log(), _max_depth ); } template inline void unpack( Stream& s, fc::exception& e, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); + --_max_depth; int64_t code; std::string name, what; log_messages msgs; - fc::raw::unpack( s, code, _max_depth - 1 ); - fc::raw::unpack( s, name, _max_depth - 1 ); - fc::raw::unpack( s, what, _max_depth - 1 ); - fc::raw::unpack( s, msgs, _max_depth - 1 ); + fc::raw::unpack( s, code, _max_depth ); + fc::raw::unpack( s, name, _max_depth ); + fc::raw::unpack( s, what, _max_depth ); + fc::raw::unpack( s, msgs, _max_depth ); e = fc::exception( fc::move(msgs), code, name, what ); } @@ -54,7 +57,7 @@ namespace fc { inline void pack( Stream& s, const fc::log_message& msg, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - fc::raw::pack( s, variant(msg), _max_depth - 1 ); + fc::raw::pack( s, variant(msg), _max_depth - 1 ); // TODO check variant depth? } template inline void unpack( Stream& s, fc::log_message& msg, uint32_t _max_depth ) @@ -62,7 +65,7 @@ namespace fc { FC_ASSERT( _max_depth > 0 ); fc::variant vmsg; fc::raw::unpack( s, vmsg, _max_depth - 1 ); - msg = vmsg.as(); + msg = vmsg.as(); // TODO check depth? } template @@ -250,16 +253,18 @@ namespace fc { template void pack( Stream& s, const fc::optional& v, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - fc::raw::pack( s, bool(!!v), _max_depth - 1 ); - if( !!v ) fc::raw::pack( s, *v, _max_depth - 1 ); + --_max_depth; + fc::raw::pack( s, bool(!!v), _max_depth ); + if( !!v ) fc::raw::pack( s, *v, _max_depth ); } template void unpack( Stream& s, fc::optional& v, uint32_t _max_depth ) { try { FC_ASSERT( _max_depth > 0 ); - bool b; fc::raw::unpack( s, b, _max_depth - 1 ); - if( b ) { v = T(); fc::raw::unpack( s, *v, _max_depth - 1 ); } + --_max_depth; + bool b; fc::raw::unpack( s, b, _max_depth ); + if( b ) { v = T(); fc::raw::unpack( s, *v, _max_depth ); } } FC_RETHROW_EXCEPTIONS( warn, "optional<${type}>", ("type",fc::get_typename::name() ) ) } // std::vector @@ -313,24 +318,24 @@ namespace fc { template struct pack_object_visitor { pack_object_visitor( const Class& _c, Stream& _s, uint32_t _max_depth ) - :c(_c),s(_s),max_depth(_max_depth) + :c(_c),s(_s),max_depth(_max_depth - 1) { FC_ASSERT( _max_depth > 0 ); } template void operator()( const char* name )const { - fc::raw::pack( s, c.*p, max_depth - 1 ); + fc::raw::pack( s, c.*p, max_depth ); } private: - const Class& c; - Stream& s; - uint32_t max_depth; + const Class& c; + Stream& s; + const uint32_t max_depth; }; template struct unpack_object_visitor { - unpack_object_visitor( Class& _c, Stream& _s, uint32_t _max_depth ) : c(_c),s(_s),max_depth(_max_depth) + unpack_object_visitor( Class& _c, Stream& _s, uint32_t _max_depth ) : c(_c),s(_s),max_depth(_max_depth - 1) { FC_ASSERT( _max_depth > 0 ); } @@ -338,12 +343,12 @@ namespace fc { template inline void operator()( const char* name )const { try { - fc::raw::unpack( s, c.*p, max_depth - 1 ); + fc::raw::unpack( s, c.*p, max_depth ); } FC_RETHROW_EXCEPTIONS( warn, "Error unpacking field ${field}", ("field",name) ) } private: Class& c; Stream& s; - uint32_t max_depth; + const uint32_t max_depth; }; template @@ -427,25 +432,27 @@ namespace fc { template inline void pack( Stream& s, const std::unordered_set& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + --_max_depth; + fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { - fc::raw::pack( s, *itr, _max_depth - 1 ); + fc::raw::pack( s, *itr, _max_depth ); ++itr; } } template inline void unpack( Stream& s, std::unordered_set& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - unsigned_int size; fc::raw::unpack( s, size, _max_depth - 1 ); + --_max_depth; + unsigned_int size; fc::raw::unpack( s, size, _max_depth ); value.clear(); FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); value.reserve(size.value); for( uint32_t i = 0; i < size.value; ++i ) { T tmp; - fc::raw::unpack( s, tmp, _max_depth - 1 ); + fc::raw::unpack( s, tmp, _max_depth ); value.insert( std::move(tmp) ); } } @@ -454,25 +461,28 @@ namespace fc { template inline void pack( Stream& s, const std::pair& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - fc::raw::pack( s, value.first, _max_depth - 1 ); - fc::raw::pack( s, value.second, _max_depth - 1 ); + --_max_depth; + fc::raw::pack( s, value.first, _max_depth ); + fc::raw::pack( s, value.second, _max_depth ); } template inline void unpack( Stream& s, std::pair& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - fc::raw::unpack( s, value.first, _max_depth - 1 ); - fc::raw::unpack( s, value.second, _max_depth - 1 ); + --_max_depth; + fc::raw::unpack( s, value.first, _max_depth ); + fc::raw::unpack( s, value.second, _max_depth ); } template inline void pack( Stream& s, const std::unordered_map& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + --_max_depth; + fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { - fc::raw::pack( s, *itr, _max_depth - 1 ); + fc::raw::pack( s, *itr, _max_depth ); ++itr; } } @@ -480,25 +490,27 @@ namespace fc { inline void unpack( Stream& s, std::unordered_map& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - unsigned_int size; fc::raw::unpack( s, size, _max_depth - 1 ); + --_max_depth; + unsigned_int size; fc::raw::unpack( s, size, _max_depth ); value.clear(); FC_ASSERT( size.value*(sizeof(K)+sizeof(V)) < MAX_ARRAY_ALLOC_SIZE ); value.reserve(size.value); for( uint32_t i = 0; i < size.value; ++i ) { std::pair tmp; - fc::raw::unpack( s, tmp, _max_depth - 1 ); + fc::raw::unpack( s, tmp, _max_depth ); value.insert( std::move(tmp) ); } } template inline void pack( Stream& s, const std::map& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + --_max_depth; + fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { - fc::raw::pack( s, *itr, _max_depth - 1 ); + fc::raw::pack( s, *itr, _max_depth ); ++itr; } } @@ -506,13 +518,14 @@ namespace fc { inline void unpack( Stream& s, std::map& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - unsigned_int size; fc::raw::unpack( s, size, _max_depth - 1 ); + --_max_depth; + unsigned_int size; fc::raw::unpack( s, size, _max_depth ); value.clear(); FC_ASSERT( size.value*(sizeof(K)+sizeof(V)) < MAX_ARRAY_ALLOC_SIZE ); for( uint32_t i = 0; i < size.value; ++i ) { std::pair tmp; - fc::raw::unpack( s, tmp, _max_depth - 1 ); + fc::raw::unpack( s, tmp, _max_depth ); value.insert( std::move(tmp) ); } } @@ -520,11 +533,12 @@ namespace fc { template inline void pack( Stream& s, const std::deque& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + --_max_depth; + fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { - fc::raw::pack( s, *itr, _max_depth - 1 ); + fc::raw::pack( s, *itr, _max_depth ); ++itr; } } @@ -532,13 +546,14 @@ namespace fc { template inline void unpack( Stream& s, std::deque& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - unsigned_int size; fc::raw::unpack( s, size, _max_depth - 1 ); + --_max_depth; + unsigned_int size; fc::raw::unpack( s, size, _max_depth ); FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); value.resize(size.value); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { - fc::raw::unpack( s, *itr, _max_depth - 1 ); + fc::raw::unpack( s, *itr, _max_depth ); ++itr; } } @@ -546,11 +561,12 @@ namespace fc { template inline void pack( Stream& s, const std::vector& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + --_max_depth; + fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { - fc::raw::pack( s, *itr, _max_depth - 1 ); + fc::raw::pack( s, *itr, _max_depth ); ++itr; } } @@ -558,13 +574,14 @@ namespace fc { template inline void unpack( Stream& s, std::vector& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - unsigned_int size; fc::raw::unpack( s, size, _max_depth - 1 ); + --_max_depth; + unsigned_int size; fc::raw::unpack( s, size, _max_depth ); FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); value.resize(size.value); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { - fc::raw::unpack( s, *itr, _max_depth - 1 ); + fc::raw::unpack( s, *itr, _max_depth ); ++itr; } } @@ -572,11 +589,12 @@ namespace fc { template inline void pack( Stream& s, const std::set& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth - 1 ); + --_max_depth; + fc::raw::pack( s, unsigned_int((uint32_t)value.size()), _max_depth ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { - fc::raw::pack( s, *itr, _max_depth - 1 ); + fc::raw::pack( s, *itr, _max_depth ); ++itr; } } @@ -584,11 +602,12 @@ namespace fc { template inline void unpack( Stream& s, std::set& value, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - unsigned_int size; fc::raw::unpack( s, size, _max_depth - 1 ); + --_max_depth; + unsigned_int size; fc::raw::unpack( s, size, _max_depth ); for( uint64_t i = 0; i < size.value; ++i ) { T tmp; - fc::raw::unpack( s, tmp, _max_depth - 1 ); + fc::raw::unpack( s, tmp, _max_depth ); value.insert( std::move(tmp) ); } } @@ -618,13 +637,14 @@ namespace fc { template inline std::vector pack( const T& v, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); + --_max_depth; datastream ps; - fc::raw::pack( ps, v, _max_depth - 1 ); + fc::raw::pack( ps, v, _max_depth ); std::vector vec(ps.tellp()); if( vec.size() ) { datastream ds( vec.data(), size_t(vec.size()) ); - fc::raw::pack( ds, v, _max_depth - 1 ); + fc::raw::pack( ds, v, _max_depth ); } return vec; } @@ -632,13 +652,14 @@ namespace fc { template inline std::vector pack( const T& v, Next... next, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); + --_max_depth; datastream ps; - fc::raw::pack( ps, v, next..., _max_depth - 1 ); + fc::raw::pack( ps, v, next..., _max_depth ); std::vector vec(ps.tellp()); if( vec.size() ) { datastream ds( vec.data(), size_t(vec.size()) ); - fc::raw::pack( ds, v, next..., _max_depth - 1 ); + fc::raw::pack( ds, v, next..., _max_depth ); } return vec; } @@ -696,8 +717,8 @@ namespace fc { struct pack_static_variant { Stream& stream; - uint32_t max_depth; - pack_static_variant( Stream& s, uint32_t _max_depth ):stream(s),max_depth(_max_depth) + const uint32_t max_depth; + pack_static_variant( Stream& s, uint32_t _max_depth ):stream(s),max_depth(_max_depth - 1) { FC_ASSERT( _max_depth > 0 ); } @@ -705,7 +726,7 @@ namespace fc { typedef void result_type; template void operator()( const T& v )const { - fc::raw::pack( stream, v, max_depth - 1 ); + fc::raw::pack( stream, v, max_depth ); } }; @@ -713,8 +734,8 @@ namespace fc { struct unpack_static_variant { Stream& stream; - uint32_t max_depth; - unpack_static_variant( Stream& s, uint32_t _max_depth ) : stream(s),max_depth(_max_depth) + const uint32_t max_depth; + unpack_static_variant( Stream& s, uint32_t _max_depth ) : stream(s),max_depth(_max_depth - 1) { FC_ASSERT( _max_depth > 0 ); } @@ -722,7 +743,7 @@ namespace fc { typedef void result_type; template void operator()( T& v )const { - fc::raw::unpack( stream, v, max_depth - 1 ); + fc::raw::unpack( stream, v, max_depth ); } }; @@ -731,17 +752,19 @@ namespace fc { void pack( Stream& s, const static_variant& sv, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - fc::raw::pack( s, unsigned_int(sv.which()), _max_depth - 1 ); - sv.visit( pack_static_variant( s, _max_depth - 1 ) ); + --_max_depth; + fc::raw::pack( s, unsigned_int(sv.which()), _max_depth ); + sv.visit( pack_static_variant( s, _max_depth ) ); } template void unpack( Stream& s, static_variant& sv, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); + --_max_depth; unsigned_int w; - fc::raw::unpack( s, w, _max_depth - 1 ); + fc::raw::unpack( s, w, _max_depth ); sv.set_which(w.value); - sv.visit( unpack_static_variant( s, _max_depth - 1 ) ); + sv.visit( unpack_static_variant( s, _max_depth ) ); } } } // namespace fc::raw diff --git a/include/fc/io/raw_variant.hpp b/include/fc/io/raw_variant.hpp index 9df0118..b07fbaa 100644 --- a/include/fc/io/raw_variant.hpp +++ b/include/fc/io/raw_variant.hpp @@ -10,42 +10,42 @@ namespace fc { namespace raw { class variant_packer : public variant::visitor { public: - variant_packer( Stream& _s, uint32_t _max_depth ):s(_s),max_depth(_max_depth) + variant_packer( Stream& _s, uint32_t _max_depth ):s(_s),max_depth(_max_depth - 1) { FC_ASSERT( _max_depth > 0 ); } virtual void handle()const { } virtual void handle( const int64_t& v )const { - fc::raw::pack( s, v, max_depth - 1 ); + fc::raw::pack( s, v, max_depth ); } virtual void handle( const uint64_t& v )const { - fc::raw::pack( s, v, max_depth - 1 ); + fc::raw::pack( s, v, max_depth ); } virtual void handle( const double& v )const { - fc::raw::pack( s, v, max_depth - 1 ); + fc::raw::pack( s, v, max_depth ); } virtual void handle( const bool& v )const { - fc::raw::pack( s, v, max_depth - 1 ); + fc::raw::pack( s, v, max_depth ); } virtual void handle( const string& v )const { - fc::raw::pack( s, v, max_depth - 1 ); + fc::raw::pack( s, v, max_depth ); } virtual void handle( const variant_object& v)const { - fc::raw::pack( s, v, max_depth - 1 ); + fc::raw::pack( s, v, max_depth ); } virtual void handle( const variants& v)const { - fc::raw::pack( s, v, max_depth - 1 ); + fc::raw::pack( s, v, max_depth ); } Stream& s; - uint32_t max_depth; + const uint32_t max_depth; }; @@ -54,15 +54,17 @@ namespace fc { namespace raw { inline void pack( Stream& s, const variant& v, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - pack( s, uint8_t(v.get_type()), _max_depth - 1 ); - v.visit( variant_packer( s, _max_depth - 1 ) ); + --_max_depth; + pack( s, uint8_t(v.get_type()), _max_depth ); + v.visit( variant_packer( s, _max_depth ) ); } template inline void unpack( Stream& s, variant& v, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); + --_max_depth; uint8_t t; - unpack( s, t, _max_depth - 1 ); + unpack( s, t, _max_depth ); switch( t ) { case variant::null_type: @@ -70,49 +72,49 @@ namespace fc { namespace raw { case variant::int64_type: { int64_t val; - raw::unpack( s, val, _max_depth - 1 ); + raw::unpack( s, val, _max_depth ); v = val; return; } case variant::uint64_type: { uint64_t val; - raw::unpack( s, val, _max_depth - 1 ); + raw::unpack( s, val, _max_depth ); v = val; return; } case variant::double_type: { double val; - raw::unpack( s, val, _max_depth - 1 ); + raw::unpack( s, val, _max_depth ); v = val; return; } case variant::bool_type: { bool val; - raw::unpack( s, val, _max_depth - 1 ); + raw::unpack( s, val, _max_depth ); v = val; return; } case variant::string_type: { fc::string val; - raw::unpack( s, val, _max_depth - 1 ); + raw::unpack( s, val, _max_depth ); v = fc::move(val); return; } case variant::array_type: { variants val; - raw::unpack( s, val, _max_depth - 1 ); + raw::unpack( s, val, _max_depth ); v = fc::move(val); return; } case variant::object_type: { variant_object val; - raw::unpack( s, val, _max_depth - 1 ); + raw::unpack( s, val, _max_depth ); v = fc::move(val); return; } @@ -125,20 +127,22 @@ namespace fc { namespace raw { inline void pack( Stream& s, const variant_object& v, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); + --_max_depth; unsigned_int vs = (uint32_t)v.size(); - pack( s, vs, _max_depth - 1 ); + pack( s, vs, _max_depth ); for( auto itr = v.begin(); itr != v.end(); ++itr ) { - pack( s, itr->key(), _max_depth - 1 ); - pack( s, itr->value(), _max_depth - 1 ); + pack( s, itr->key(), _max_depth ); + pack( s, itr->value(), _max_depth ); } } template inline void unpack( Stream& s, variant_object& v, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); + --_max_depth; unsigned_int vs; - unpack( s, vs, _max_depth - 1 ); + unpack( s, vs, _max_depth ); mutable_variant_object mvo; mvo.reserve(vs.value); @@ -146,8 +150,8 @@ namespace fc { namespace raw { { fc::string key; fc::variant value; - fc::raw::unpack( s, key, _max_depth - 1 ); - fc::raw::unpack( s, value, _max_depth - 1 ); + fc::raw::unpack( s, key, _max_depth ); + fc::raw::unpack( s, value, _max_depth ); mvo.set( fc::move(key), fc::move(value) ); } v = fc::move(mvo); diff --git a/include/fc/network/ip.hpp b/include/fc/network/ip.hpp index 5292dbf..04c5030 100644 --- a/include/fc/network/ip.hpp +++ b/include/fc/network/ip.hpp @@ -101,17 +101,19 @@ namespace fc { inline void pack( Stream& s, const ip::endpoint& v, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); - fc::raw::pack( s, v.get_address(), _max_depth - 1 ); - fc::raw::pack( s, v.port(), _max_depth - 1 ); + --_max_depth; + fc::raw::pack( s, v.get_address(), _max_depth ); + fc::raw::pack( s, v.port(), _max_depth ); } template inline void unpack( Stream& s, ip::endpoint& v, uint32_t _max_depth ) { FC_ASSERT( _max_depth > 0 ); + --_max_depth; ip::address a; uint16_t p; - fc::raw::unpack( s, a, _max_depth - 1 ); - fc::raw::unpack( s, p, _max_depth - 1 ); + fc::raw::unpack( s, a, _max_depth ); + fc::raw::unpack( s, p, _max_depth ); v = ip::endpoint(a,p); }