From 232d59525af8d2e3b56e8b18b2fdb7f58435a7f5 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Fri, 19 Feb 2016 16:36:29 -0500 Subject: [PATCH 1/5] raw.hpp: Use fc::raw:: namespace explicitly everywhere, cleanup whitespace --- include/fc/io/raw.hpp | 212 +++++++++++++++++++++--------------------- 1 file changed, 106 insertions(+), 106 deletions(-) diff --git a/include/fc/io/raw.hpp b/include/fc/io/raw.hpp index 1b62fa9..bc5cd22 100644 --- a/include/fc/io/raw.hpp +++ b/include/fc/io/raw.hpp @@ -15,7 +15,7 @@ #include #include -namespace fc { +namespace fc { namespace raw { template inline void pack( Stream& s, const fc::exception& e ) @@ -76,7 +76,7 @@ namespace fc { template inline void unpack( Stream& s, fc::time_point_sec& tp ) - { try { + { try { uint32_t sec; s.read( (char*)&sec, sizeof(sec) ); tp = fc::time_point() + fc::seconds(sec); @@ -112,7 +112,7 @@ namespace fc { usec = fc::microseconds(usec_as_int64); } FC_RETHROW_EXCEPTIONS( warn, "" ) } - template + template inline void pack( Stream& s, const fc::array& v) { s.write((const char*)&v.data[0],N*sizeof(T)); } @@ -120,20 +120,20 @@ namespace fc { template inline void pack( Stream& s, const std::shared_ptr& v) { - pack( s, *v ); + fc::raw::pack( s, *v ); } - template - inline void unpack( Stream& s, fc::array& v) + template + inline void unpack( Stream& s, fc::array& v) { try { s.read((char*)&v.data[0],N*sizeof(T)); } FC_RETHROW_EXCEPTIONS( warn, "fc::array", ("type",fc::get_typename::name())("length",N) ) } - template - inline void unpack( Stream& s, std::shared_ptr& v) + template + inline void unpack( Stream& s, std::shared_ptr& v) { try { v = std::make_shared(); - unpack( s, *v ); + fc::raw::unpack( s, *v ); } FC_RETHROW_EXCEPTIONS( warn, "std::shared_ptr", ("type",fc::get_typename::name()) ) } template inline void pack( Stream& s, const signed_int& v ) { @@ -177,58 +177,58 @@ namespace fc { vi.value = static_cast(v); } - template inline void unpack( Stream& s, const T& vi ) + template inline void unpack( Stream& s, const T& vi ) { T tmp; - unpack( s, tmp ); + fc::raw::unpack( s, tmp ); FC_ASSERT( vi == tmp ); } - template inline void pack( Stream& s, const char* v ) { pack( s, fc::string(v) ); } - - template - void pack( Stream& s, const safe& v ) { pack( s, v.value ); } + template inline void pack( Stream& s, const char* v ) { fc::raw::pack( s, fc::string(v) ); } template - void unpack( Stream& s, fc::safe& v ) { unpack( s, v.value ); } + void pack( Stream& s, const safe& v ) { fc::raw::pack( s, v.value ); } - template + template + void unpack( Stream& s, fc::safe& v ) { fc::raw::unpack( s, v.value ); } + + template void pack( Stream& s, const fc::fwd& v ) { - pack( *v ); + fc::raw::pack( *v ); } - template + template void unpack( Stream& s, fc::fwd& v ) { - unpack( *v ); + fc::raw::unpack( *v ); } - template - void pack( Stream& s, const fc::smart_ref& v ) { pack( s, *v ); } + template + void pack( Stream& s, const fc::smart_ref& v ) { fc::raw::pack( s, *v ); } - template - void unpack( Stream& s, fc::smart_ref& v ) { unpack( s, *v ); } + template + void unpack( Stream& s, fc::smart_ref& v ) { fc::raw::unpack( s, *v ); } // optional - template + template void pack( Stream& s, const fc::optional& v ) { - pack( s, bool(!!v) ); - if( !!v ) pack( s, *v ); + fc::raw::pack( s, bool(!!v) ); + if( !!v ) fc::raw::pack( s, *v ); } template - void unpack( Stream& s, fc::optional& v ) + void unpack( Stream& s, fc::optional& v ) { try { - bool b; unpack( s, b ); - if( b ) { v = T(); unpack( s, *v ); } + bool b; fc::raw::unpack( s, b ); + if( b ) { v = T(); fc::raw::unpack( s, *v ); } } FC_RETHROW_EXCEPTIONS( warn, "optional<${type}>", ("type",fc::get_typename::name() ) ) } // std::vector - template inline void pack( Stream& s, const std::vector& value ) { - pack( s, unsigned_int((uint32_t)value.size()) ); + template inline void pack( Stream& s, const std::vector& value ) { + fc::raw::pack( s, unsigned_int((uint32_t)value.size()) ); if( value.size() ) s.write( &value.front(), (uint32_t)value.size() ); } - template inline void unpack( Stream& s, std::vector& value ) { - unsigned_int size; unpack( s, size ); + template inline void unpack( Stream& s, std::vector& value ) { + unsigned_int size; fc::raw::unpack( s, size ); FC_ASSERT( size.value < MAX_ARRAY_ALLOC_SIZE ); value.resize(size.value); if( value.size() ) @@ -237,29 +237,29 @@ namespace fc { // fc::string template inline void pack( Stream& s, const fc::string& v ) { - pack( s, unsigned_int((uint32_t)v.size())); + fc::raw::pack( s, unsigned_int((uint32_t)v.size())); if( v.size() ) s.write( v.c_str(), v.size() ); } template inline void unpack( Stream& s, fc::string& v ) { - std::vector tmp; unpack(s,tmp); + std::vector tmp; fc::raw::unpack(s,tmp); if( tmp.size() ) v = fc::string(tmp.data(),tmp.data()+tmp.size()); else v = fc::string(); } // bool - template inline void pack( Stream& s, const bool& v ) { pack( s, uint8_t(v) ); } + template inline void pack( Stream& s, const bool& v ) { fc::raw::pack( s, uint8_t(v) ); } template inline void unpack( Stream& s, bool& v ) { uint8_t b; - unpack( s, b ); + fc::raw::unpack( s, b ); FC_ASSERT( (b & ~1) == 0 ); v=(b!=0); } namespace detail { - + template struct pack_object_visitor { pack_object_visitor(const Class& _c, Stream& _s) @@ -267,9 +267,9 @@ namespace fc { template void operator()( const char* name )const { - raw::pack( s, c.*p ); + fc::raw::pack( s, c.*p ); } - private: + private: const Class& c; Stream& s; }; @@ -280,11 +280,11 @@ namespace fc { :c(_c),s(_s){} template - inline void operator()( const char* name )const + inline void operator()( const char* name )const { try { - raw::unpack( s, c.*p ); + fc::raw::unpack( s, c.*p ); } FC_RETHROW_EXCEPTIONS( warn, "Error unpacking field ${field}", ("field",name) ) } - private: + private: Class& c; Stream& s; }; @@ -300,23 +300,23 @@ namespace fc { template<> struct if_class { template - static inline void pack( Stream& s, const T& v ) { - s.write( (char*)&v, sizeof(v) ); + static inline void pack( Stream& s, const T& v ) { + s.write( (char*)&v, sizeof(v) ); } template - static inline void unpack( Stream& s, T& v ) { - s.read( (char*)&v, sizeof(v) ); + static inline void unpack( Stream& s, T& v ) { + s.read( (char*)&v, sizeof(v) ); } }; template struct if_enum { template - static inline void pack( Stream& s, const T& v ) { + static inline void pack( Stream& s, const T& v ) { fc::reflector::visit( pack_object_visitor( v, s ) ); } template - static inline void unpack( Stream& s, T& v ) { + static inline void unpack( Stream& s, T& v ) { fc::reflector::visit( unpack_object_visitor( v, s ) ); } }; @@ -327,32 +327,32 @@ namespace fc { fc::raw::pack(s, (int64_t)v); } template - static inline void unpack( Stream& s, T& v ) { + static inline void unpack( Stream& s, T& v ) { int64_t temp; fc::raw::unpack(s, temp); v = (T)temp; } - }; + }; template struct if_reflected { template - static inline void pack( Stream& s, const T& v ) { + static inline void pack( Stream& s, const T& v ) { if_class::type>::pack(s,v); } template - static inline void unpack( Stream& s, T& v ) { + static inline void unpack( Stream& s, T& v ) { if_class::type>::unpack(s,v); } }; template<> struct if_reflected { template - static inline void pack( Stream& s, const T& v ) { + static inline void pack( Stream& s, const T& v ) { if_enum< typename fc::reflector::is_enum >::pack(s,v); } template - static inline void unpack( Stream& s, T& v ) { + static inline void unpack( Stream& s, T& v ) { if_enum< typename fc::reflector::is_enum >::unpack(s,v); } }; @@ -361,7 +361,7 @@ namespace fc { template inline void pack( Stream& s, const std::unordered_set& value ) { - pack( s, unsigned_int((uint32_t)value.size()) ); + fc::raw::pack( s, unsigned_int((uint32_t)value.size()) ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { @@ -371,7 +371,7 @@ namespace fc { } template inline void unpack( Stream& s, std::unordered_set& value ) { - unsigned_int size; unpack( s, size ); + unsigned_int size; fc::raw::unpack( s, size ); value.clear(); FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); value.reserve(size.value); @@ -386,19 +386,19 @@ namespace fc { template inline void pack( Stream& s, const std::pair& value ) { - pack( s, value.first ); - pack( s, value.second ); + fc::raw::pack( s, value.first ); + fc::raw::pack( s, value.second ); } template - inline void unpack( Stream& s, std::pair& value ) + inline void unpack( Stream& s, std::pair& value ) { - unpack( s, value.first ); - unpack( s, value.second ); + fc::raw::unpack( s, value.first ); + fc::raw::unpack( s, value.second ); } - template + template inline void pack( Stream& s, const std::unordered_map& value ) { - pack( s, unsigned_int((uint32_t)value.size()) ); + fc::raw::pack( s, unsigned_int((uint32_t)value.size()) ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { @@ -407,9 +407,9 @@ namespace fc { } } template - inline void unpack( Stream& s, std::unordered_map& value ) + inline void unpack( Stream& s, std::unordered_map& value ) { - unsigned_int size; unpack( s, size ); + unsigned_int size; fc::raw::unpack( s, size ); value.clear(); FC_ASSERT( size.value*(sizeof(K)+sizeof(V)) < MAX_ARRAY_ALLOC_SIZE ); value.reserve(size.value); @@ -422,7 +422,7 @@ namespace fc { } template inline void pack( Stream& s, const std::map& value ) { - pack( s, unsigned_int((uint32_t)value.size()) ); + fc::raw::pack( s, unsigned_int((uint32_t)value.size()) ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { @@ -431,9 +431,9 @@ namespace fc { } } template - inline void unpack( Stream& s, std::map& value ) + inline void unpack( Stream& s, std::map& value ) { - unsigned_int size; unpack( s, size ); + unsigned_int size; fc::raw::unpack( s, size ); value.clear(); FC_ASSERT( size.value*(sizeof(K)+sizeof(V)) < MAX_ARRAY_ALLOC_SIZE ); for( uint32_t i = 0; i < size.value; ++i ) @@ -446,7 +446,7 @@ namespace fc { template inline void pack( Stream& s, const std::deque& value ) { - pack( s, unsigned_int((uint32_t)value.size()) ); + fc::raw::pack( s, unsigned_int((uint32_t)value.size()) ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { @@ -457,7 +457,7 @@ namespace fc { template inline void unpack( Stream& s, std::deque& value ) { - unsigned_int size; unpack( s, size ); + unsigned_int size; fc::raw::unpack( s, size ); FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); value.resize(size.value); auto itr = value.begin(); @@ -470,7 +470,7 @@ namespace fc { template inline void pack( Stream& s, const std::vector& value ) { - pack( s, unsigned_int((uint32_t)value.size()) ); + fc::raw::pack( s, unsigned_int((uint32_t)value.size()) ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { @@ -481,7 +481,7 @@ namespace fc { template inline void unpack( Stream& s, std::vector& value ) { - unsigned_int size; unpack( s, size ); + unsigned_int size; fc::raw::unpack( s, size ); FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); value.resize(size.value); auto itr = value.begin(); @@ -494,7 +494,7 @@ namespace fc { template inline void pack( Stream& s, const std::set& value ) { - pack( s, unsigned_int((uint32_t)value.size()) ); + fc::raw::pack( s, unsigned_int((uint32_t)value.size()) ); auto itr = value.begin(); auto end = value.end(); while( itr != end ) { @@ -505,93 +505,93 @@ namespace fc { template inline void unpack( Stream& s, std::set& value ) { - unsigned_int size; unpack( s, size ); + unsigned_int size; fc::raw::unpack( s, size ); for( uint64_t i = 0; i < size.value; ++i ) { T tmp; - unpack( s, tmp ); + fc::raw::unpack( s, tmp ); value.insert( std::move(tmp) ); } } - template + template inline void pack( Stream& s, const T& v ) { fc::raw::detail::if_reflected< typename fc::reflector::is_defined >::pack(s,v); } - template - inline void unpack( Stream& s, T& v ) + template + inline void unpack( Stream& s, T& v ) { try { fc::raw::detail::if_reflected< typename fc::reflector::is_defined >::unpack(s,v); } FC_RETHROW_EXCEPTIONS( warn, "error unpacking ${type}", ("type",fc::get_typename::name() ) ) } template - inline size_t pack_size( const T& v ) + inline size_t pack_size( const T& v ) { - datastream ps; - raw::pack(ps,v ); + datastream ps; + fc::raw::pack(ps,v ); return ps.tellp(); } template inline std::vector pack( const T& v ) { - datastream ps; - raw::pack(ps,v ); + datastream ps; + fc::raw::pack(ps,v ); std::vector vec(ps.tellp()); if( vec.size() ) { - datastream ds( vec.data(), size_t(vec.size()) ); - raw::pack(ds,v); + datastream ds( vec.data(), size_t(vec.size()) ); + fc::raw::pack(ds,v); } return vec; } template - inline T unpack( const std::vector& s ) + inline T unpack( const std::vector& s ) { try { T tmp; if( s.size() ) { - datastream ds( s.data(), size_t(s.size()) ); - raw::unpack(ds,tmp); + datastream ds( s.data(), size_t(s.size()) ); + fc::raw::unpack(ds,tmp); } return tmp; } FC_RETHROW_EXCEPTIONS( warn, "error unpacking ${type}", ("type",fc::get_typename::name() ) ) } template - inline void unpack( const std::vector& s, T& tmp ) + inline void unpack( const std::vector& s, T& tmp ) { try { if( s.size() ) { - datastream ds( s.data(), size_t(s.size()) ); - raw::unpack(ds,tmp); + datastream ds( s.data(), size_t(s.size()) ); + fc::raw::unpack(ds,tmp); } } FC_RETHROW_EXCEPTIONS( warn, "error unpacking ${type}", ("type",fc::get_typename::name() ) ) } template inline void pack( char* d, uint32_t s, const T& v ) { - datastream ds(d,s); - raw::pack(ds,v ); + datastream ds(d,s); + fc::raw::pack(ds,v ); } template - inline T unpack( const char* d, uint32_t s ) + inline T unpack( const char* d, uint32_t s ) { try { T v; datastream ds( d, s ); - raw::unpack(ds,v); + fc::raw::unpack(ds,v); return v; } FC_RETHROW_EXCEPTIONS( warn, "error unpacking ${type}", ("type",fc::get_typename::name() ) ) } template - inline void unpack( const char* d, uint32_t s, T& v ) + inline void unpack( const char* d, uint32_t s, T& v ) { try { datastream ds( d, s ); - raw::unpack(ds,v); + fc::raw::unpack(ds,v); return v; } FC_RETHROW_EXCEPTIONS( warn, "error unpacking ${type}", ("type",fc::get_typename::name() ) ) } template - struct pack_static_variant + struct pack_static_variant { Stream& stream; pack_static_variant( Stream& s ):stream(s){} @@ -599,7 +599,7 @@ namespace fc { typedef void result_type; template void operator()( const T& v )const { - pack( stream, v ); + fc::raw::pack( stream, v ); } }; @@ -612,25 +612,25 @@ namespace fc { typedef void result_type; template void operator()( T& v )const { - unpack( stream, v ); + fc::raw::unpack( stream, v ); } }; - template + template void pack( Stream& s, const static_variant& sv ) { - pack( s, unsigned_int(sv.which()) ); + fc::raw::pack( s, unsigned_int(sv.which()) ); sv.visit( pack_static_variant(s) ); } template void unpack( Stream& s, static_variant& sv ) { unsigned_int w; - unpack( s, w ); + fc::raw::unpack( s, w ); sv.set_which(w.value); sv.visit( unpack_static_variant(s) ); } - + } } // namespace fc::raw From 83b4de067a6c99704a6d21d6dfc8b1e838ddcaf7 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Thu, 25 Feb 2016 01:53:32 -0500 Subject: [PATCH 2/5] future.cpp: Fix use-after-free bug cryptonomex/graphene#597 --- src/thread/future.cpp | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/thread/future.cpp b/src/thread/future.cpp index 1950086..2111584 100644 --- a/src/thread/future.cpp +++ b/src/thread/future.cpp @@ -63,18 +63,42 @@ namespace fc { _enqueue_thread(); } std::exception_ptr e; - try { thread::current().wait_until( ptr(this,true), timeout_us ); } + + // + // Create shared_ptr to take ownership of this; i.e. this will + // be deleted when p_this goes out of scope. Consequently, + // it would be Very Bad to let p_this go out of scope + // before we're done reading/writing instance variables! + // See https://github.com/cryptonomex/graphene/issues/597 + // + + ptr p_this = ptr( this, true ); + + try + { + // + // We clone p_this here because the wait_until() API requires us + // to use std::move(). I.e. wait_until() takes ownership of any + // pointer passed to it. Since we want to keep ownership ourselves, + // we need to have two shared_ptr's to this: + // + // - p_this to keep this alive until the end of the current function + // - p_this2 to be owned by wait_until() as the wait_until() API requires + // + ptr p_this2 = p_this; + thread::current().wait_until( std::move( p_this2 ), timeout_us ); + } catch (...) { e = std::current_exception(); } _dequeue_thread(); if( e ) std::rethrow_exception(e); - if( _ready ) + if( _ready ) { - if( _exceptp ) + if( _exceptp ) _exceptp->dynamic_rethrow_exception(); - return; + return; } FC_THROW_EXCEPTION( timeout_exception, "" ); } From 38419164b6f1ade468bf4b1d27929b3ac2503b6e Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Fri, 26 Feb 2016 14:22:11 -0500 Subject: [PATCH 3/5] optional.hpp: Allow easy access to contained type --- include/fc/optional.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/fc/optional.hpp b/include/fc/optional.hpp index dfb725d..bb760a5 100644 --- a/include/fc/optional.hpp +++ b/include/fc/optional.hpp @@ -18,9 +18,11 @@ namespace fc { * fc::optional adds less than 400. */ template - class optional + class optional { public: + typedef T value_type; + optional():_valid(false){} ~optional(){ reset(); } From d5370fc2ea436fa4340cda01af98c7993fbc67d5 Mon Sep 17 00:00:00 2001 From: Eric Frias Date: Thu, 3 Mar 2016 16:42:36 -0500 Subject: [PATCH 4/5] Update the websocketpp library and change the configuration options we pass when creating the websocket servers to enable deflate compression on websocket frames. This is relevant to cryptonomex/graphene#540 because the spammed data is higly compressible. In my tests, it reduces bandwidth for a single idle node by a factor of ~16, from 577kbps down to 36kbps. This doesn't require any changes to the wallets, simply upgrading the public servers will begin sending compressed data to all clients that support it. Note: this commit adds a dependency on zlib for non-apple platforms (it was already required on apple) --- CMakeLists.txt | 19 ++++++----- src/network/http/websocket.cpp | 62 ++++++++++------------------------ vendor/websocketpp | 2 +- 3 files changed, 29 insertions(+), 54 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9931b9a..eeadd2b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,6 +142,17 @@ find_package(OpenSSL REQUIRED) set( CMAKE_FIND_LIBRARY_SUFFIXES ${ORIGINAL_LIB_SUFFIXES} ) +# We are now building in support for deflate compression into our websockets layer by default, +# which requires zlib. Aside from that, all of fc compiles without zlib, so this could be +# made optional without much effort +# (important exception, apple: as of 10.10 yosemite, the OpenSSL static libraries shipped with +# os x have a dependency on zlib) +# On a side note, fc's fc::zlib_compress() function uses a separate implementation of zlib +# from the miniz library. If we're comfortable requiring an external zlib, we can +# reimplement fc::zlib_compress() to call the real zlib, and remove miniz.c from our +# repository. +find_package( ZLIB REQUIRED ) + option( UNITY_BUILD OFF ) set( fc_sources @@ -473,14 +484,6 @@ if(WIN32) endif(WIN32) -IF(APPLE) - # As of 10.10 yosemite, the OpenSSL static libraries shipped with os x have a dependency - # on zlib, so any time you link in openssl you also need to link zlib. . We really want to detect whether openssl was configured with the --no-zlib - # option or not when it was built, but that's difficult to do in practice, so we - # just always try to link it in on mac. - find_package( ZLIB REQUIRED ) -ENDIF(APPLE) - SET(OPENSSL_CONF_TARGET ) IF(DEFINED CMAKE_RUNTIME_OUTPUT_DIRECTORY) SET (OPENSSL_CONF_TARGET ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/src/network/http/websocket.cpp b/src/network/http/websocket.cpp index 67745d9..c75f0e8 100644 --- a/src/network/http/websocket.cpp +++ b/src/network/http/websocket.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -61,47 +62,12 @@ namespace fc { namespace http { transport_type; static const long timeout_open_handshake = 0; - }; - struct asio_tls_with_stub_log : public websocketpp::config::asio_tls { - typedef asio_with_stub_log type; - typedef asio_tls base; + /// permessage_compress extension + struct permessage_deflate_config {}; - typedef base::concurrency_type concurrency_type; - - typedef base::request_type request_type; - typedef base::response_type response_type; - - typedef base::message_type message_type; - typedef base::con_msg_manager_type con_msg_manager_type; - typedef base::endpoint_msg_manager_type endpoint_msg_manager_type; - - /// Custom Logging policies - /*typedef websocketpp::log::syslog elog_type; - typedef websocketpp::log::syslog alog_type; - */ - //typedef base::alog_type alog_type; - //typedef base::elog_type elog_type; - typedef websocketpp::log::stub elog_type; - typedef websocketpp::log::stub alog_type; - - typedef base::rng_type rng_type; - - struct transport_config : public base::transport_config { - typedef type::concurrency_type concurrency_type; - typedef type::alog_type alog_type; - typedef type::elog_type elog_type; - typedef type::request_type request_type; - typedef type::response_type response_type; - typedef websocketpp::transport::asio::tls_socket::endpoint socket_type; - }; - - typedef websocketpp::transport::asio::endpoint - transport_type; - - static const long timeout_open_handshake = 0; + typedef websocketpp::extensions::permessage_deflate::enabled + permessage_deflate_type; }; struct asio_tls_stub_log : public websocketpp::config::asio_tls { typedef asio_tls_stub_log type; @@ -124,16 +90,22 @@ namespace fc { namespace http { typedef base::rng_type rng_type; struct transport_config : public base::transport_config { - typedef type::concurrency_type concurrency_type; - typedef type::alog_type alog_type; - typedef type::elog_type elog_type; - typedef type::request_type request_type; - typedef type::response_type response_type; - typedef websocketpp::transport::asio::tls_socket::endpoint socket_type; + typedef type::concurrency_type concurrency_type; + typedef type::alog_type alog_type; + typedef type::elog_type elog_type; + typedef type::request_type request_type; + typedef type::response_type response_type; + typedef websocketpp::transport::asio::tls_socket::endpoint socket_type; }; typedef websocketpp::transport::asio::endpoint transport_type; + + /// permessage_compress extension + struct permessage_deflate_config {}; + + typedef websocketpp::extensions::permessage_deflate::enabled + permessage_deflate_type; }; diff --git a/vendor/websocketpp b/vendor/websocketpp index c5510d6..378437a 160000 --- a/vendor/websocketpp +++ b/vendor/websocketpp @@ -1 +1 @@ -Subproject commit c5510d6de04917812b910a8dd44735c1f17061d9 +Subproject commit 378437aecdcb1dfe62096ffd5d944bf1f640ccc3 From 21045dde5faa8fcf5f43b97c85f9df210317633b Mon Sep 17 00:00:00 2001 From: Eric Frias Date: Thu, 10 Mar 2016 17:22:28 -0500 Subject: [PATCH 5/5] Add an option to the websocket server constructor to allow disabling deflate compression. Refactored the code to remove duplication between tls/non-tls versions of the server, which appear to have been cut & paste copies of one another that had diverged slightly. This makes some of the fixes to the non-tls server available in the tls server. cryptonomex/graphene#619 --- include/fc/network/http/websocket.hpp | 16 +- src/network/http/websocket.cpp | 374 +++++++++++++------------- 2 files changed, 196 insertions(+), 194 deletions(-) diff --git a/include/fc/network/http/websocket.hpp b/include/fc/network/http/websocket.hpp index c0c7266..1cb6417 100644 --- a/include/fc/network/http/websocket.hpp +++ b/include/fc/network/http/websocket.hpp @@ -8,8 +8,7 @@ namespace fc { namespace http { namespace detail { - class websocket_server_impl; - class websocket_tls_server_impl; + class abstract_websocket_server; class websocket_client_impl; class websocket_tls_client_impl; } // namespace detail; @@ -42,7 +41,7 @@ namespace fc { namespace http { class websocket_server { public: - websocket_server(); + websocket_server(bool enable_permessage_deflate = true); ~websocket_server(); void on_connection( const on_connection_handler& handler); @@ -51,16 +50,16 @@ namespace fc { namespace http { void start_accept(); private: - friend class detail::websocket_server_impl; - std::unique_ptr my; + std::unique_ptr my; }; class websocket_tls_server { public: - websocket_tls_server( const std::string& server_pem = std::string(), - const std::string& ssl_password = std::string()); + websocket_tls_server(const std::string& server_pem = std::string(), + const std::string& ssl_password = std::string(), + bool enable_permessage_deflate = true); ~websocket_tls_server(); void on_connection( const on_connection_handler& handler); @@ -69,8 +68,7 @@ namespace fc { namespace http { void start_accept(); private: - friend class detail::websocket_tls_server_impl; - std::unique_ptr my; + std::unique_ptr my; }; class websocket_client diff --git a/src/network/http/websocket.cpp b/src/network/http/websocket.cpp index c75f0e8..81a7cad 100644 --- a/src/network/http/websocket.cpp +++ b/src/network/http/websocket.cpp @@ -20,12 +20,11 @@ namespace fc { namespace http { namespace detail { - struct asio_with_stub_log : public websocketpp::config::asio { - typedef asio_with_stub_log type; typedef asio base; + //// All boilerplate copying the base class's config, except as noted typedef base::concurrency_type concurrency_type; typedef base::request_type request_type; @@ -34,15 +33,8 @@ namespace fc { namespace http { typedef base::message_type message_type; typedef base::con_msg_manager_type con_msg_manager_type; typedef base::endpoint_msg_manager_type endpoint_msg_manager_type; - - /// Custom Logging policies - /*typedef websocketpp::log::syslog elog_type; - typedef websocketpp::log::syslog alog_type; - */ - //typedef base::alog_type alog_type; - //typedef base::elog_type elog_type; + + /// Custom Logging policies, use do-nothing log::stub instead of log::basic typedef websocketpp::log::stub elog_type; typedef websocketpp::log::stub alog_type; @@ -61,60 +53,124 @@ namespace fc { namespace http { typedef websocketpp::transport::asio::endpoint transport_type; + // override default value of 5 sec timeout static const long timeout_open_handshake = 0; + }; + + struct asio_with_stub_log_and_deflate : public websocketpp::config::asio { + typedef asio_with_stub_log_and_deflate type; + typedef asio base; - /// permessage_compress extension + //// All boilerplate copying the base class's config, except as noted + typedef base::concurrency_type concurrency_type; + + typedef base::request_type request_type; + typedef base::response_type response_type; + + typedef base::message_type message_type; + typedef base::con_msg_manager_type con_msg_manager_type; + typedef base::endpoint_msg_manager_type endpoint_msg_manager_type; + + /// Custom Logging policies, use do-nothing log::stub instead of log::basic + typedef websocketpp::log::stub elog_type; + typedef websocketpp::log::stub alog_type; + + typedef base::rng_type rng_type; + + struct transport_config : public base::transport_config { + typedef type::concurrency_type concurrency_type; + typedef type::alog_type alog_type; + typedef type::elog_type elog_type; + typedef type::request_type request_type; + typedef type::response_type response_type; + typedef websocketpp::transport::asio::basic_socket::endpoint + socket_type; + }; + + typedef websocketpp::transport::asio::endpoint + transport_type; + + /// enable the permessage_compress extension struct permessage_deflate_config {}; + typedef websocketpp::extensions::permessage_deflate::enabled + permessage_deflate_type; + // override default value of 5 sec timeout + static const long timeout_open_handshake = 0; + }; + + struct asio_tls_stub_log : public websocketpp::config::asio_tls { + typedef asio_tls_stub_log type; + typedef asio_tls base; + + //// All boilerplate copying the base class's config, except as noted + typedef base::concurrency_type concurrency_type; + + typedef base::request_type request_type; + typedef base::response_type response_type; + + typedef base::message_type message_type; + typedef base::con_msg_manager_type con_msg_manager_type; + typedef base::endpoint_msg_manager_type endpoint_msg_manager_type; + + /// Custom Logging policies, use do-nothing log::stub instead of log::basic + typedef websocketpp::log::stub elog_type; + typedef websocketpp::log::stub alog_type; + + typedef base::rng_type rng_type; + + struct transport_config : public base::transport_config { + typedef type::concurrency_type concurrency_type; + typedef type::alog_type alog_type; + typedef type::elog_type elog_type; + typedef type::request_type request_type; + typedef type::response_type response_type; + typedef websocketpp::transport::asio::tls_socket::endpoint socket_type; + }; + + typedef websocketpp::transport::asio::endpoint + transport_type; + }; + + struct asio_tls_stub_log_and_deflate : public websocketpp::config::asio_tls { + typedef asio_tls_stub_log_and_deflate type; + typedef asio_tls base; + + //// All boilerplate copying the base class's config, except as noted + typedef base::concurrency_type concurrency_type; + + typedef base::request_type request_type; + typedef base::response_type response_type; + + typedef base::message_type message_type; + typedef base::con_msg_manager_type con_msg_manager_type; + typedef base::endpoint_msg_manager_type endpoint_msg_manager_type; + + /// Custom Logging policies, use do-nothing log::stub instead of log::basic + typedef websocketpp::log::stub elog_type; + typedef websocketpp::log::stub alog_type; + + typedef base::rng_type rng_type; + + struct transport_config : public base::transport_config { + typedef type::concurrency_type concurrency_type; + typedef type::alog_type alog_type; + typedef type::elog_type elog_type; + typedef type::request_type request_type; + typedef type::response_type response_type; + typedef websocketpp::transport::asio::tls_socket::endpoint socket_type; + }; + + typedef websocketpp::transport::asio::endpoint + transport_type; + + /// enable the permessage_compress extension + struct permessage_deflate_config {}; typedef websocketpp::extensions::permessage_deflate::enabled permessage_deflate_type; }; - struct asio_tls_stub_log : public websocketpp::config::asio_tls { - typedef asio_tls_stub_log type; - typedef asio_tls base; - - typedef base::concurrency_type concurrency_type; - - typedef base::request_type request_type; - typedef base::response_type response_type; - - typedef base::message_type message_type; - typedef base::con_msg_manager_type con_msg_manager_type; - typedef base::endpoint_msg_manager_type endpoint_msg_manager_type; - - //typedef base::alog_type alog_type; - //typedef base::elog_type elog_type; - typedef websocketpp::log::stub elog_type; - typedef websocketpp::log::stub alog_type; - - typedef base::rng_type rng_type; - - struct transport_config : public base::transport_config { - typedef type::concurrency_type concurrency_type; - typedef type::alog_type alog_type; - typedef type::elog_type elog_type; - typedef type::request_type request_type; - typedef type::response_type response_type; - typedef websocketpp::transport::asio::tls_socket::endpoint socket_type; - }; - - typedef websocketpp::transport::asio::endpoint - transport_type; - - /// permessage_compress extension - struct permessage_deflate_config {}; - - typedef websocketpp::extensions::permessage_deflate::enabled - permessage_deflate_type; - }; - - - - using websocketpp::connection_hdl; - typedef websocketpp::server websocket_server_type; - typedef websocketpp::server websocket_tls_server_type; template class websocket_connection_impl : public websocket_connection @@ -145,7 +201,19 @@ namespace fc { namespace http { typedef websocketpp::lib::shared_ptr context_ptr; - class websocket_server_impl + class abstract_websocket_server + { + public: + virtual ~abstract_websocket_server() {} + + virtual void on_connection( const on_connection_handler& handler) = 0; + virtual void listen( uint16_t port ) = 0; + virtual void listen( const fc::ip::endpoint& ep ) = 0; + virtual void start_accept() = 0; + }; + + template + class websocket_server_impl : public abstract_websocket_server { public: websocket_server_impl() @@ -157,15 +225,15 @@ namespace fc { namespace http { _server.set_reuse_addr(true); _server.set_open_handler( [&]( connection_hdl hdl ){ _server_thread.async( [&](){ - auto new_con = std::make_shared>( _server.get_con_from_hdl(hdl) ); + websocket_connection_ptr new_con = std::make_shared::connection_ptr>>( _server.get_con_from_hdl(hdl) ); _on_connection( _connections[hdl] = new_con ); }).wait(); }); - _server.set_message_handler( [&]( connection_hdl hdl, websocket_server_type::message_ptr msg ){ + _server.set_message_handler( [&]( connection_hdl hdl, typename websocketpp::server::message_ptr msg ){ _server_thread.async( [&](){ auto current_con = _connections.find(hdl); assert( current_con != _connections.end() ); - wdump(("server")(msg->get_payload())); + //wdump(("server")(msg->get_payload())); //std::cerr<<"recv: "<get_payload()<<"\n"; auto payload = msg->get_payload(); std::shared_ptr con = current_con->second; @@ -178,13 +246,13 @@ namespace fc { namespace http { _server.set_http_handler( [&]( connection_hdl hdl ){ _server_thread.async( [&](){ - auto current_con = std::make_shared>( _server.get_con_from_hdl(hdl) ); + auto current_con = std::make_shared::connection_ptr>>( _server.get_con_from_hdl(hdl) ); _on_connection( current_con ); auto con = _server.get_con_from_hdl(hdl); con->defer_http_response(); std::string request_body = con->get_request_body(); - wdump(("server")(request_body)); + //wdump(("server")(request_body)); fc::async([current_con, request_body, con] { std::string response = current_con->on_http(request_body); @@ -246,132 +314,62 @@ namespace fc { namespace http { if( _closed ) _closed->wait(); } + void on_connection( const on_connection_handler& handler ) override + { + _on_connection = handler; + } + + void listen( uint16_t port ) override + { + _server.listen(port); + } + + void listen( const fc::ip::endpoint& ep ) override + { + _server.listen( boost::asio::ip::tcp::endpoint( boost::asio::ip::address_v4(uint32_t(ep.get_address())),ep.port()) ); + } + + void start_accept() override + { + _server.start_accept(); + } + typedef std::map > con_map; con_map _connections; fc::thread& _server_thread; - websocket_server_type _server; + websocketpp::server _server; on_connection_handler _on_connection; fc::promise::ptr _closed; uint32_t _pending_messages = 0; }; - class websocket_tls_server_impl + template + class websocket_tls_server_impl : public websocket_server_impl { public: websocket_tls_server_impl( const string& server_pem, const string& ssl_password ) - :_server_thread( fc::thread::current() ) { - //if( server_pem.size() ) - { - _server.set_tls_init_handler( [=]( websocketpp::connection_hdl hdl ) -> context_ptr { - context_ptr ctx = websocketpp::lib::make_shared(boost::asio::ssl::context::tlsv1); - try { - ctx->set_options(boost::asio::ssl::context::default_workarounds | - boost::asio::ssl::context::no_sslv2 | - boost::asio::ssl::context::no_sslv3 | - boost::asio::ssl::context::single_dh_use); - ctx->set_password_callback([=](std::size_t max_length, boost::asio::ssl::context::password_purpose){ return ssl_password;}); - ctx->use_certificate_chain_file(server_pem); - ctx->use_private_key_file(server_pem, boost::asio::ssl::context::pem); - } catch (std::exception& e) { - std::cout << e.what() << std::endl; - } - return ctx; - }); - } - - _server.clear_access_channels( websocketpp::log::alevel::all ); - _server.init_asio(&fc::asio::default_io_service()); - _server.set_reuse_addr(true); - _server.set_open_handler( [&]( connection_hdl hdl ){ - _server_thread.async( [&](){ - auto new_con = std::make_shared>( _server.get_con_from_hdl(hdl) ); - _on_connection( _connections[hdl] = new_con ); - }).wait(); - }); - _server.set_message_handler( [&]( connection_hdl hdl, websocket_server_type::message_ptr msg ){ - _server_thread.async( [&](){ - auto current_con = _connections.find(hdl); - assert( current_con != _connections.end() ); - auto received = msg->get_payload(); - std::shared_ptr con = current_con->second; - fc::async([con,received](){ con->on_message( received ); }); - }).wait(); - }); - - _server.set_http_handler( [&]( connection_hdl hdl ){ - _server_thread.async( [&](){ - - auto current_con = std::make_shared>( _server.get_con_from_hdl(hdl) ); - try{ - _on_connection( current_con ); - - auto con = _server.get_con_from_hdl(hdl); - wdump(("server")(con->get_request_body())); - auto response = current_con->on_http( con->get_request_body() ); - - con->set_body( response ); - con->set_status( websocketpp::http::status_code::ok ); - } catch ( const fc::exception& e ) - { - edump((e.to_detail_string())); - } - current_con->closed(); - - }).wait(); - }); - - _server.set_close_handler( [&]( connection_hdl hdl ){ - _server_thread.async( [&](){ - _connections[hdl]->closed(); - _connections.erase( hdl ); - }).wait(); - }); - - _server.set_fail_handler( [&]( connection_hdl hdl ){ - if( _server.is_listening() ) - { - _server_thread.async( [&](){ - if( _connections.find(hdl) != _connections.end() ) - { - _connections[hdl]->closed(); - _connections.erase( hdl ); - } - }).wait(); - } + this->_server.set_tls_init_handler( [=]( websocketpp::connection_hdl hdl ) -> context_ptr { + context_ptr ctx = websocketpp::lib::make_shared(boost::asio::ssl::context::tlsv1); + try { + ctx->set_options(boost::asio::ssl::context::default_workarounds | + boost::asio::ssl::context::no_sslv2 | + boost::asio::ssl::context::no_sslv3 | + boost::asio::ssl::context::single_dh_use); + ctx->set_password_callback([=](std::size_t max_length, boost::asio::ssl::context::password_purpose){ return ssl_password;}); + ctx->use_certificate_chain_file(server_pem); + ctx->use_private_key_file(server_pem, boost::asio::ssl::context::pem); + } catch (std::exception& e) { + std::cout << e.what() << std::endl; + } + return ctx; }); } - ~websocket_tls_server_impl() - { - if( _server.is_listening() ) - _server.stop_listening(); - auto cpy_con = _connections; - for( auto item : cpy_con ) - _server.close( item.first, 0, "server exit" ); - } - - typedef std::map > con_map; - - con_map _connections; - fc::thread& _server_thread; - websocket_tls_server_type _server; - on_connection_handler _on_connection; - fc::promise::ptr _closed; }; - - - - - - - - - - typedef websocketpp::client websocket_client_type; typedef websocketpp::client websocket_tls_client_type; @@ -514,57 +512,63 @@ namespace fc { namespace http { } // namespace detail - websocket_server::websocket_server():my( new detail::websocket_server_impl() ) {} + websocket_server::websocket_server(bool enable_permessage_deflate /* = true */) : + my( enable_permessage_deflate ? + (detail::abstract_websocket_server*)new detail::websocket_server_impl : + (detail::abstract_websocket_server*)new detail::websocket_server_impl ) + {} websocket_server::~websocket_server(){} void websocket_server::on_connection( const on_connection_handler& handler ) { - my->_on_connection = handler; + my->on_connection(handler); } void websocket_server::listen( uint16_t port ) { - my->_server.listen(port); + my->listen(port); } void websocket_server::listen( const fc::ip::endpoint& ep ) { - my->_server.listen( boost::asio::ip::tcp::endpoint( boost::asio::ip::address_v4(uint32_t(ep.get_address())),ep.port()) ); + my->listen(ep); } void websocket_server::start_accept() { - my->_server.start_accept(); + my->start_accept(); } - websocket_tls_server::websocket_tls_server( const string& server_pem, const string& ssl_password ):my( new detail::websocket_tls_server_impl(server_pem, ssl_password) ) {} + websocket_tls_server::websocket_tls_server(const string& server_pem, + const string& ssl_password, + bool enable_permessage_deflate /* = true */) : + my( enable_permessage_deflate ? + (detail::abstract_websocket_server*)new detail::websocket_tls_server_impl(server_pem, ssl_password) : + (detail::abstract_websocket_server*)new detail::websocket_tls_server_impl(server_pem, ssl_password) ) + {} websocket_tls_server::~websocket_tls_server(){} void websocket_tls_server::on_connection( const on_connection_handler& handler ) { - my->_on_connection = handler; + my->on_connection(handler); } void websocket_tls_server::listen( uint16_t port ) { - my->_server.listen(port); + my->listen(port); } void websocket_tls_server::listen( const fc::ip::endpoint& ep ) { - my->_server.listen( boost::asio::ip::tcp::endpoint( boost::asio::ip::address_v4(uint32_t(ep.get_address())),ep.port()) ); + my->listen(ep); } - void websocket_tls_server::start_accept() { - my->_server.start_accept(); + void websocket_tls_server::start_accept() + { + my->start_accept(); } - websocket_tls_client::websocket_tls_client():my( new detail::websocket_tls_client_impl() ) {} - websocket_tls_client::~websocket_tls_client(){ } - - - websocket_client::websocket_client():my( new detail::websocket_client_impl() ),smy(new detail::websocket_tls_client_impl()) {} websocket_client::~websocket_client(){ }