From ce7139c0733339c2dc457028e248ba4beceb3e6a Mon Sep 17 00:00:00 2001 From: Eric Frias Date: Thu, 17 Apr 2014 12:00:52 -0400 Subject: [PATCH 01/19] Start of work towards throttling TCP connections --- CMakeLists.txt | 1 + include/fc/asio.hpp | 24 ++- include/fc/network/rate_limiting.hpp | 26 +++ src/network/rate_limiting.cpp | 228 +++++++++++++++++++++++++++ src/network/tcp_socket.cpp | 4 +- 5 files changed, 277 insertions(+), 6 deletions(-) create mode 100644 include/fc/network/rate_limiting.hpp create mode 100644 src/network/rate_limiting.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 87ceef0..a987ff2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -147,6 +147,7 @@ set( fc_sources src/network/http/http_connection.cpp src/network/http/http_server.cpp src/network/ip.cpp + src/network/rate_limiting.cpp src/network/resolve.cpp src/network/url.cpp src/compress/smaz.cpp diff --git a/include/fc/asio.hpp b/include/fc/asio.hpp index 36a2727..8723290 100644 --- a/include/fc/asio.hpp +++ b/include/fc/asio.hpp @@ -79,11 +79,17 @@ namespace asio { * @return the number of bytes read. */ template - size_t read_some( AsyncReadStream& s, const MutableBufferSequence& buf ) + size_t read_some(AsyncReadStream& s, const MutableBufferSequence& buf) { - promise::ptr p(new promise("fc::asio::async_read_some")); - s.async_read_some( buf, boost::bind( detail::read_write_handler, p, _1, _2 ) ); - return p->wait(); + promise::ptr p(new promise("fc::asio::async_read_some")); + s.async_read_some(buf, boost::bind(detail::read_write_handler, p, _1, _2)); + return p->wait(); + } + + template + void async_read_some(AsyncReadStream& s, const MutableBufferSequence& buf, promise::ptr completion_promise) + { + s.async_read_some(buf, boost::bind(detail::read_write_handler, completion_promise, _1, _2)); } template @@ -117,6 +123,16 @@ namespace asio { return p->wait(); } + /** + * @pre s.non_blocking() == true + * @brief wraps boost::asio::async_write_some + * @return the number of bytes written + */ + template + void async_write_some(AsyncWriteStream& s, const ConstBufferSequence& buf, promise::ptr completion_promise) { + s.async_write_some(buf, boost::bind(detail::read_write_handler, completion_promise, _1, _2)); + } + namespace tcp { typedef boost::asio::ip::tcp::endpoint endpoint; typedef boost::asio::ip::tcp::resolver::iterator resolver_iterator; diff --git a/include/fc/network/rate_limiting.hpp b/include/fc/network/rate_limiting.hpp new file mode 100644 index 0000000..10ac9bb --- /dev/null +++ b/include/fc/network/rate_limiting.hpp @@ -0,0 +1,26 @@ +#pragma once +#include +#include +#include +#include + +namespace fc +{ + namespace detail + { + class rate_limiting_group_impl; + } + + class rate_limiting_group + { + public: + rate_limiting_group(uint32_t upload_bytes_per_second, uint32_t download_bytes_per_second); + ~rate_limiting_group(); + + private: + std::unique_ptr my; + }; + typedef std::shared_ptr rate_limiting_group_ptr; + +} // namesapce fc + diff --git a/src/network/rate_limiting.cpp b/src/network/rate_limiting.cpp new file mode 100644 index 0000000..6182169 --- /dev/null +++ b/src/network/rate_limiting.cpp @@ -0,0 +1,228 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fc +{ + + namespace detail + { + // data about a read or write we're managing + class rate_limited_operation + { + public: + size_t length; + size_t permitted_length; + promise::ptr completion_promise; + + rate_limited_operation(size_t length, + promise::ptr&& completion_promise) : + length(length), + permitted_length(0), + completion_promise(completion_promise) + {} + + virtual void perform_operation() = 0; + }; + + class rate_limited_tcp_write_operation : public rate_limited_operation + { + public: + boost::asio::ip::tcp::socket& socket; + const char* buffer; + + rate_limited_tcp_write_operation(boost::asio::ip::tcp::socket& socket, + const char* buffer, + size_t length, + promise::ptr completion_promise) : + rate_limited_operation(length, std::move(completion_promise)), + socket(socket), + buffer(buffer) + {} + virtual void perform_operation() override + { + asio::async_write_some(socket, + boost::asio::buffer(buffer, permitted_length), + completion_promise); + } + }; + class rate_limited_tcp_read_operation : public rate_limited_operation + { + public: + boost::asio::ip::tcp::socket& socket; + char* buffer; + + rate_limited_tcp_read_operation(boost::asio::ip::tcp::socket& socket, + char* buffer, + size_t length, + promise::ptr completion_promise) : + rate_limited_operation(length, std::move(completion_promise)), + socket(socket), + buffer(buffer) + {} + virtual void perform_operation() override + { + asio::async_read_some(socket, + boost::asio::buffer(buffer, permitted_length), + completion_promise); + } + }; + + struct is_operation_shorter + { + // less than operator designed to bring the shortest operations to the end + bool operator()(const rate_limited_operation* lhs, const rate_limited_operation* rhs) + { + return lhs->length > rhs->length; + } + }; + + class rate_limiting_group_impl + { + public: + uint32_t _upload_bytes_per_second; + uint32_t _download_bytes_per_second; + + microseconds _granularity; + + typedef std::list > rate_limited_operation_list; + rate_limited_operation_list _read_operations_in_progress; + rate_limited_operation_list _read_operations_for_next_iteration; + rate_limited_operation_list _write_operations_in_progress; + rate_limited_operation_list _write_operations_for_next_iteration; + + time_point _last_write_iteration_time; + + rate_limiting_group_impl(uint32_t upload_bytes_per_second, uint32_t download_bytes_per_second); + + size_t readsome(boost::asio::ip::tcp::socket& socket, char* buffer, size_t length); + size_t writesome(boost::asio::ip::tcp::socket& socket, const char* buf, size_t len); + + void process_pending_reads(); + void process_pending_writes(); + void process_pending_operations(rate_limited_operation_list& operations_in_progress, + rate_limited_operation_list& operations_for_next_iteration); + }; + + rate_limiting_group_impl::rate_limiting_group_impl(uint32_t upload_bytes_per_second, uint32_t download_bytes_per_second) : + _upload_bytes_per_second(upload_bytes_per_second), + _download_bytes_per_second(download_bytes_per_second), + _granularity(fc::milliseconds(50)) + { + } + + size_t rate_limiting_group_impl::readsome(boost::asio::ip::tcp::socket& socket, char* buffer, size_t length) + { + promise::ptr completion_promise(new promise()); + _read_operations_for_next_iteration.emplace_back(std::make_unique(socket, buffer, length, completion_promise)); + return completion_promise->wait(); + } + size_t rate_limiting_group_impl::writesome(boost::asio::ip::tcp::socket& socket, const char* buffer, size_t length) + { + if (_upload_bytes_per_second) + { + promise::ptr completion_promise(new promise()); + _write_operations_for_next_iteration.emplace_back(std::make_unique(socket, buffer, length, completion_promise)); + return completion_promise->wait(); + } + else + return asio::write_some(socket, boost::asio::buffer(buffer, length)); + } + void rate_limiting_group_impl::process_pending_reads() + { + process_pending_operations(_read_operations_in_progress, _read_operations_for_next_iteration); + } + void rate_limiting_group_impl::process_pending_writes() + { + process_pending_operations(_write_operations_in_progress, _write_operations_for_next_iteration); + } + void rate_limiting_group_impl::process_pending_operations(rate_limited_operation_list& operations_in_progress, + rate_limited_operation_list& operations_for_next_iteration) + { + // lock here for multithreaded + std::copy(std::make_move_iterator(operations_for_next_iteration.begin()), + std::make_move_iterator(operations_for_next_iteration.end()), + std::back_inserter(operations_in_progress)); + operations_for_next_iteration.clear(); + + // find out how much time since our last write + time_point this_write_iteration_start_time = time_point::now(); + if (_upload_bytes_per_second) // the we are limiting upload speed + { + microseconds time_since_last_iteration = this_write_iteration_start_time - _last_write_iteration_time; + if (time_since_last_iteration > seconds(1)) + time_since_last_iteration = seconds(1); + else if (time_since_last_iteration < microseconds(0)) + time_since_last_iteration = microseconds(0); + + uint32_t total_bytes_for_this_iteration = + (uint32_t)(time_since_last_iteration.count() / _upload_bytes_per_second / seconds(1).count()); + if (total_bytes_for_this_iteration) + { + // sort the pending writes in order of the number of bytes they need to write, smallest first + std::vector operations_sorted_by_length; + operations_sorted_by_length.reserve(operations_in_progress.size()); + for (std::unique_ptr& operation_data : operations_in_progress) + operations_sorted_by_length.push_back(operation_data.get()); + std::sort(operations_sorted_by_length.begin(), operations_sorted_by_length.end(), is_operation_shorter()); + + // figure out how many bytes each writer is allowed to write + uint32_t bytes_remaining_to_allocate = total_bytes_for_this_iteration; + while (!operations_sorted_by_length.empty()) + { + uint32_t bytes_permitted_for_this_writer = bytes_remaining_to_allocate / operations_sorted_by_length.size(); + uint32_t bytes_allocated_for_this_writer = std::min(operations_sorted_by_length.back()->length, bytes_permitted_for_this_writer); + operations_sorted_by_length.back()->permitted_length = bytes_allocated_for_this_writer; + bytes_remaining_to_allocate -= bytes_allocated_for_this_writer; + operations_sorted_by_length.pop_back(); + } + + // kick off the writes in first-come order + for (auto iter = operations_in_progress.begin(); iter != operations_in_progress.end();) + { + if ((*iter)->permitted_length > 0) + { + (*iter)->perform_operation(); + iter = operations_in_progress.erase(iter); + } + else + ++iter; + } + } + } + else // upload speed is unlimited + { + // we shouldn't end up here often. If the rate is unlimited, we should just execute + // the operation immediately without being queued up. This should only be hit if + // we change from a limited rate to unlimited + for (auto iter = operations_in_progress.begin(); + iter != operations_in_progress.end(); + ++iter) + { + (*iter)->permitted_length = (*iter)->length; + (*iter)->perform_operation(); + } + operations_in_progress.clear(); + } + _last_write_iteration_time = this_write_iteration_start_time; + } + + } + + rate_limiting_group::rate_limiting_group(uint32_t upload_bytes_per_second, uint32_t download_bytes_per_second) : + my(new detail::rate_limiting_group_impl(upload_bytes_per_second, download_bytes_per_second)) + { + } + + rate_limiting_group::~rate_limiting_group() + { + } + + + +} // namespace fc diff --git a/src/network/tcp_socket.cpp b/src/network/tcp_socket.cpp index 864199a..a9bdab1 100644 --- a/src/network/tcp_socket.cpp +++ b/src/network/tcp_socket.cpp @@ -77,8 +77,8 @@ namespace fc { #if defined _WIN32 || defined WIN32 || defined OS_WIN64 || defined _WIN64 || defined WIN64 || defined WINNT struct tcp_keepalive keepalive_settings; keepalive_settings.onoff = 1; - keepalive_settings.keepalivetime = interval.count() / fc::milliseconds(1).count(); - keepalive_settings.keepaliveinterval = interval.count() / fc::milliseconds(1).count(); + keepalive_settings.keepalivetime = (ULONG)(interval.count() / fc::milliseconds(1).count()); + keepalive_settings.keepaliveinterval = (ULONG)(interval.count() / fc::milliseconds(1).count()); DWORD dwBytesRet = 0; if (WSAIoctl(my->_sock.native(), SIO_KEEPALIVE_VALS, &keepalive_settings, sizeof(keepalive_settings), From 06df18c6903700b074043942b2ec30e02690bbd5 Mon Sep 17 00:00:00 2001 From: Eric Frias Date: Thu, 17 Apr 2014 19:39:15 -0400 Subject: [PATCH 02/19] More work towards rate-limited TCP sockets --- CMakeLists.txt | 2 + include/fc/network/rate_limiting.hpp | 17 +++- include/fc/network/tcp_socket.hpp | 8 +- include/fc/network/tcp_socket_io_hooks.hpp | 12 +++ src/network/rate_limiting.cpp | 107 ++++++++++++++++----- src/network/tcp_socket.cpp | 39 ++++++-- tests/rate_limiting.cpp | 24 +++++ 7 files changed, 171 insertions(+), 38 deletions(-) create mode 100644 include/fc/network/tcp_socket_io_hooks.hpp create mode 100644 tests/rate_limiting.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a987ff2..0c6de49 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -218,6 +218,8 @@ target_link_libraries( fc PUBLIC easylzma_static ${Boost_LIBRARIES} ${OPENSSL_LI #target_link_libraries( test_aes fc ${rt_library} ${pthread_library} ) #add_executable( test_sleep tests/sleep.cpp ) #target_link_libraries( test_sleep fc ) +add_executable( test_rate_limiting tests/rate_limiting.cpp ) +target_link_libraries( test_rate_limiting fc ) if(WIN32) # add addtional import library on windows platform diff --git a/include/fc/network/rate_limiting.hpp b/include/fc/network/rate_limiting.hpp index 10ac9bb..7694789 100644 --- a/include/fc/network/rate_limiting.hpp +++ b/include/fc/network/rate_limiting.hpp @@ -1,8 +1,7 @@ #pragma once -#include -#include -#include -#include +#include + +#include namespace fc { @@ -11,12 +10,22 @@ namespace fc class rate_limiting_group_impl; } + class tcp_socket; + class rate_limiting_group { public: rate_limiting_group(uint32_t upload_bytes_per_second, uint32_t download_bytes_per_second); ~rate_limiting_group(); + void set_upload_limit(uint32_t upload_bytes_per_second); + uint32_t get_upload_limit() const; + + void set_download_limit(uint32_t download_bytes_per_second); + uint32_t get_download_limit() const; + + void add_tcp_socket(tcp_socket* tcp_socket_to_limit); + void remove_tcp_socket(tcp_socket* tcp_socket_to_stop_limiting); private: std::unique_ptr my; }; diff --git a/include/fc/network/tcp_socket.hpp b/include/fc/network/tcp_socket.hpp index 3229f87..70fb9d8 100644 --- a/include/fc/network/tcp_socket.hpp +++ b/include/fc/network/tcp_socket.hpp @@ -6,6 +6,9 @@ namespace fc { namespace ip { class endpoint; } + + class tcp_socket_io_hooks; + class tcp_socket : public virtual iostream { public: @@ -15,6 +18,7 @@ namespace fc { void connect_to( const fc::ip::endpoint& remote_endpoint ); void connect_to( const fc::ip::endpoint& remote_endpoint, const fc::ip::endpoint& local_endpoint ); void enable_keep_alives(const fc::microseconds& interval); + void set_io_hooks(tcp_socket_io_hooks* new_hooks); fc::ip::endpoint remote_endpoint()const; void get( char& c ) @@ -41,9 +45,9 @@ namespace fc { friend class tcp_server; class impl; #ifdef _WIN64 - fc::fwd my; + fc::fwd my; #else - fc::fwd my; + fc::fwd my; #endif }; typedef std::shared_ptr tcp_socket_ptr; diff --git a/include/fc/network/tcp_socket_io_hooks.hpp b/include/fc/network/tcp_socket_io_hooks.hpp new file mode 100644 index 0000000..a317ed1 --- /dev/null +++ b/include/fc/network/tcp_socket_io_hooks.hpp @@ -0,0 +1,12 @@ +#include + +namespace fc +{ + class tcp_socket_io_hooks + { + public: + virtual ~tcp_socket_io_hooks() {} + virtual size_t readsome(boost::asio::ip::tcp::socket& socket, char* buffer, size_t length) = 0; + virtual size_t writesome(boost::asio::ip::tcp::socket& socket, const char* buffer, size_t length) = 0; + }; +} // namesapce fc diff --git a/src/network/rate_limiting.cpp b/src/network/rate_limiting.cpp index 6182169..f41eaa0 100644 --- a/src/network/rate_limiting.cpp +++ b/src/network/rate_limiting.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6,6 +7,7 @@ #include #include #include +#include namespace fc { @@ -82,7 +84,7 @@ namespace fc } }; - class rate_limiting_group_impl + class rate_limiting_group_impl : public tcp_socket_io_hooks { public: uint32_t _upload_bytes_per_second; @@ -96,16 +98,22 @@ namespace fc rate_limited_operation_list _write_operations_in_progress; rate_limited_operation_list _write_operations_for_next_iteration; + time_point _last_read_iteration_time; time_point _last_write_iteration_time; + fc::future _process_pending_reads_loop_complete; + fc::future _process_pending_writes_loop_complete; + rate_limiting_group_impl(uint32_t upload_bytes_per_second, uint32_t download_bytes_per_second); - size_t readsome(boost::asio::ip::tcp::socket& socket, char* buffer, size_t length); - size_t writesome(boost::asio::ip::tcp::socket& socket, const char* buf, size_t len); + virtual size_t readsome(boost::asio::ip::tcp::socket& socket, char* buffer, size_t length) override; + virtual size_t writesome(boost::asio::ip::tcp::socket& socket, const char* buffer, size_t length) override; void process_pending_reads(); void process_pending_writes(); - void process_pending_operations(rate_limited_operation_list& operations_in_progress, + void process_pending_operations(time_point& last_iteration_start_time, + uint32_t& limit_bytes_per_second, + rate_limited_operation_list& operations_in_progress, rate_limited_operation_list& operations_for_next_iteration); }; @@ -118,9 +126,16 @@ namespace fc size_t rate_limiting_group_impl::readsome(boost::asio::ip::tcp::socket& socket, char* buffer, size_t length) { - promise::ptr completion_promise(new promise()); - _read_operations_for_next_iteration.emplace_back(std::make_unique(socket, buffer, length, completion_promise)); - return completion_promise->wait(); + if (_download_bytes_per_second) + { + promise::ptr completion_promise(new promise()); + _read_operations_for_next_iteration.emplace_back(std::make_unique(socket, buffer, length, completion_promise)); + if (!_process_pending_reads_loop_complete.valid()) + _process_pending_reads_loop_complete = async([=](){ process_pending_reads(); }); + return completion_promise->wait(); + } + else + return asio::read_some(socket, boost::asio::buffer(buffer, length)); } size_t rate_limiting_group_impl::writesome(boost::asio::ip::tcp::socket& socket, const char* buffer, size_t length) { @@ -128,6 +143,8 @@ namespace fc { promise::ptr completion_promise(new promise()); _write_operations_for_next_iteration.emplace_back(std::make_unique(socket, buffer, length, completion_promise)); + if (!_process_pending_writes_loop_complete.valid()) + _process_pending_writes_loop_complete = async([=](){ process_pending_writes(); }); return completion_promise->wait(); } else @@ -135,13 +152,25 @@ namespace fc } void rate_limiting_group_impl::process_pending_reads() { - process_pending_operations(_read_operations_in_progress, _read_operations_for_next_iteration); + for (;;) + { + process_pending_operations(_last_read_iteration_time, _download_bytes_per_second, + _read_operations_in_progress, _read_operations_for_next_iteration); + fc::usleep(_granularity); + } } void rate_limiting_group_impl::process_pending_writes() { - process_pending_operations(_write_operations_in_progress, _write_operations_for_next_iteration); + for (;;) + { + process_pending_operations(_last_write_iteration_time, _upload_bytes_per_second, + _write_operations_in_progress, _write_operations_for_next_iteration); + fc::usleep(_granularity); + } } - void rate_limiting_group_impl::process_pending_operations(rate_limited_operation_list& operations_in_progress, + void rate_limiting_group_impl::process_pending_operations(time_point& last_iteration_start_time, + uint32_t& limit_bytes_per_second, + rate_limited_operation_list& operations_in_progress, rate_limited_operation_list& operations_for_next_iteration) { // lock here for multithreaded @@ -150,39 +179,40 @@ namespace fc std::back_inserter(operations_in_progress)); operations_for_next_iteration.clear(); - // find out how much time since our last write - time_point this_write_iteration_start_time = time_point::now(); - if (_upload_bytes_per_second) // the we are limiting upload speed + // find out how much time since our last read/write + time_point this_iteration_start_time = time_point::now(); + if (limit_bytes_per_second) // the we are limiting up/download speed { - microseconds time_since_last_iteration = this_write_iteration_start_time - _last_write_iteration_time; + microseconds time_since_last_iteration = this_iteration_start_time - last_iteration_start_time; if (time_since_last_iteration > seconds(1)) time_since_last_iteration = seconds(1); else if (time_since_last_iteration < microseconds(0)) time_since_last_iteration = microseconds(0); - uint32_t total_bytes_for_this_iteration = - (uint32_t)(time_since_last_iteration.count() / _upload_bytes_per_second / seconds(1).count()); + uint32_t total_bytes_for_this_iteration = time_since_last_iteration.count() ? + (uint32_t)((1000000 * limit_bytes_per_second) / time_since_last_iteration.count()) : + 0; if (total_bytes_for_this_iteration) { - // sort the pending writes in order of the number of bytes they need to write, smallest first + // sort the pending reads/writes in order of the number of bytes they need to write, smallest first std::vector operations_sorted_by_length; operations_sorted_by_length.reserve(operations_in_progress.size()); for (std::unique_ptr& operation_data : operations_in_progress) operations_sorted_by_length.push_back(operation_data.get()); std::sort(operations_sorted_by_length.begin(), operations_sorted_by_length.end(), is_operation_shorter()); - // figure out how many bytes each writer is allowed to write + // figure out how many bytes each reader/writer is allowed to read/write uint32_t bytes_remaining_to_allocate = total_bytes_for_this_iteration; while (!operations_sorted_by_length.empty()) { - uint32_t bytes_permitted_for_this_writer = bytes_remaining_to_allocate / operations_sorted_by_length.size(); - uint32_t bytes_allocated_for_this_writer = std::min(operations_sorted_by_length.back()->length, bytes_permitted_for_this_writer); - operations_sorted_by_length.back()->permitted_length = bytes_allocated_for_this_writer; - bytes_remaining_to_allocate -= bytes_allocated_for_this_writer; + uint32_t bytes_permitted_for_this_operation = bytes_remaining_to_allocate / operations_sorted_by_length.size(); + uint32_t bytes_allocated_for_this_operation = std::min(operations_sorted_by_length.back()->length, bytes_permitted_for_this_operation); + operations_sorted_by_length.back()->permitted_length = bytes_allocated_for_this_operation; + bytes_remaining_to_allocate -= bytes_allocated_for_this_operation; operations_sorted_by_length.pop_back(); } - // kick off the writes in first-come order + // kick off the reads/writes in first-come order for (auto iter = operations_in_progress.begin(); iter != operations_in_progress.end();) { if ((*iter)->permitted_length > 0) @@ -209,7 +239,7 @@ namespace fc } operations_in_progress.clear(); } - _last_write_iteration_time = this_write_iteration_start_time; + last_iteration_start_time = this_iteration_start_time; } } @@ -223,6 +253,35 @@ namespace fc { } + void rate_limiting_group::set_upload_limit(uint32_t upload_bytes_per_second) + { + my->_upload_bytes_per_second = upload_bytes_per_second; + } + + uint32_t rate_limiting_group::get_upload_limit() const + { + return my->_upload_bytes_per_second; + } + + void rate_limiting_group::set_download_limit(uint32_t download_bytes_per_second) + { + my->_download_bytes_per_second = download_bytes_per_second; + } + + uint32_t rate_limiting_group::get_download_limit() const + { + return my->_download_bytes_per_second; + } + + void rate_limiting_group::add_tcp_socket(tcp_socket* tcp_socket_to_limit) + { + tcp_socket_to_limit->set_io_hooks(my.get()); + } + + void rate_limiting_group::remove_tcp_socket(tcp_socket* tcp_socket_to_stop_limiting) + { + tcp_socket_to_stop_limiting->set_io_hooks(NULL); + } } // namespace fc diff --git a/src/network/tcp_socket.cpp b/src/network/tcp_socket.cpp index 9212419..e588be1 100644 --- a/src/network/tcp_socket.cpp +++ b/src/network/tcp_socket.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -12,14 +13,32 @@ namespace fc { - class tcp_socket::impl { + class tcp_socket::impl : public tcp_socket_io_hooks{ public: - impl():_sock( fc::asio::default_io_service() ){} - ~impl(){ + impl() : + _sock(fc::asio::default_io_service()), + _io_hooks(this) + {} + ~impl() + { if( _sock.is_open() ) _sock.close(); } + virtual size_t readsome(boost::asio::ip::tcp::socket& socket, char* buffer, size_t length) override; + virtual size_t writesome(boost::asio::ip::tcp::socket& socket, const char* buffer, size_t length) override; + boost::asio::ip::tcp::socket _sock; + tcp_socket_io_hooks* _io_hooks; }; + + size_t tcp_socket::impl::readsome(boost::asio::ip::tcp::socket& socket, char* buffer, size_t length) + { + return fc::asio::read_some(socket, boost::asio::buffer(buffer, length)); + } + size_t tcp_socket::impl::writesome(boost::asio::ip::tcp::socket& socket, const char* buffer, size_t length) + { + return fc::asio::write_some(socket, boost::asio::buffer(buffer, length)); + } + bool tcp_socket::is_open()const { return my->_sock.is_open(); } @@ -42,8 +61,8 @@ namespace fc { return !my->_sock.is_open(); } - size_t tcp_socket::writesome( const char* buf, size_t len ) { - return fc::asio::write_some( my->_sock, boost::asio::buffer( buf, len ) ); + size_t tcp_socket::writesome(const char* buf, size_t len) { + return my->_io_hooks->writesome(my->_sock, buf, len); } fc::ip::endpoint tcp_socket::remote_endpoint()const @@ -52,9 +71,8 @@ namespace fc { return fc::ip::endpoint(rep.address().to_v4().to_ulong(), rep.port() ); } - size_t tcp_socket::readsome( char* buf, size_t len ) { - auto r = fc::asio::read_some( my->_sock, boost::asio::buffer( buf, len ) ); - return r; + size_t tcp_socket::readsome(char* buf, size_t len) { + return my->_io_hooks->readsome(my->_sock, buf, len); } void tcp_socket::connect_to( const fc::ip::endpoint& remote_endpoint ) { @@ -107,6 +125,11 @@ namespace fc { } } + void tcp_socket::set_io_hooks(tcp_socket_io_hooks* new_hooks) + { + my->_io_hooks = new_hooks ? new_hooks : &*my; + } + class tcp_server::impl { public: impl(uint16_t port) diff --git a/tests/rate_limiting.cpp b/tests/rate_limiting.cpp new file mode 100644 index 0000000..1ca31f3 --- /dev/null +++ b/tests/rate_limiting.cpp @@ -0,0 +1,24 @@ +#include +#include +#include +#include +#include + +#include + +int main( int argc, char** argv ) +{ + fc::rate_limiting_group rate_limiter(1000000,1000000); + fc::http::connection http_connection; + rate_limiter.add_tcp_socket(&http_connection.get_socket()); + http_connection.connect_to(fc::ip::endpoint(fc::ip::address("162.243.115.24"),80)); + std::cout << "Starting download...\n"; + fc::time_point start_time(fc::time_point::now()); + fc::http::reply reply = http_connection.request("GET", "http://invictus.io/bin/Keyhotee-0.7.0.dmg"); + fc::time_point end_time(fc::time_point::now()); + + std::cout << "HTTP return code: " << reply.status << "\n"; + std::cout << "Retreived " << reply.body.size() << " bytes in " << ((end_time - start_time).count() / fc::milliseconds(1).count()) << "ms\n"; + std::cout << "Average speed " << ((1000 * (uint64_t)reply.body.size()) / ((end_time - start_time).count() / fc::milliseconds(1).count())) << " bytes per second"; + return 0; +} From 62b479568e2dcad49cfa4e7149c230b49a1829c0 Mon Sep 17 00:00:00 2001 From: Eric Frias Date: Thu, 8 May 2014 14:55:51 -0400 Subject: [PATCH 03/19] Get rate limiting mostly working --- src/network/rate_limiting.cpp | 111 +++++++++++++++++++++++++--------- tests/rate_limiting.cpp | 18 ++++-- 2 files changed, 98 insertions(+), 31 deletions(-) diff --git a/src/network/rate_limiting.cpp b/src/network/rate_limiting.cpp index f41eaa0..0055abb 100644 --- a/src/network/rate_limiting.cpp +++ b/src/network/rate_limiting.cpp @@ -90,9 +90,13 @@ namespace fc uint32_t _upload_bytes_per_second; uint32_t _download_bytes_per_second; - microseconds _granularity; + microseconds _granularity; // how often to add tokens to the bucket + uint32_t _read_tokens; + uint32_t _unused_read_tokens; // gets filled with tokens for unused bytes (if I'm allowed to read 200 bytes and I try to read 200 bytes, but can only read 50, tokens for the other 150 get returned here) + uint32_t _write_tokens; + uint32_t _unused_write_tokens; - typedef std::list > rate_limited_operation_list; + typedef std::list rate_limited_operation_list; rate_limited_operation_list _read_operations_in_progress; rate_limited_operation_list _read_operations_for_next_iteration; rate_limited_operation_list _write_operations_in_progress; @@ -101,8 +105,10 @@ namespace fc time_point _last_read_iteration_time; time_point _last_write_iteration_time; - fc::future _process_pending_reads_loop_complete; - fc::future _process_pending_writes_loop_complete; + future _process_pending_reads_loop_complete; + promise::ptr _new_read_operation_available_promise; + future _process_pending_writes_loop_complete; + promise::ptr _new_write_operation_available_promise; rate_limiting_group_impl(uint32_t upload_bytes_per_second, uint32_t download_bytes_per_second); @@ -114,13 +120,19 @@ namespace fc void process_pending_operations(time_point& last_iteration_start_time, uint32_t& limit_bytes_per_second, rate_limited_operation_list& operations_in_progress, - rate_limited_operation_list& operations_for_next_iteration); + rate_limited_operation_list& operations_for_next_iteration, + uint32_t& tokens, + uint32_t& unused_tokens); }; rate_limiting_group_impl::rate_limiting_group_impl(uint32_t upload_bytes_per_second, uint32_t download_bytes_per_second) : _upload_bytes_per_second(upload_bytes_per_second), _download_bytes_per_second(download_bytes_per_second), - _granularity(fc::milliseconds(50)) + _granularity(milliseconds(50)), + _read_tokens(_download_bytes_per_second), + _unused_read_tokens(0), + _write_tokens(_upload_bytes_per_second), + _unused_write_tokens(0) { } @@ -129,10 +141,18 @@ namespace fc if (_download_bytes_per_second) { promise::ptr completion_promise(new promise()); - _read_operations_for_next_iteration.emplace_back(std::make_unique(socket, buffer, length, completion_promise)); - if (!_process_pending_reads_loop_complete.valid()) + rate_limited_tcp_read_operation read_operation(socket, buffer, length, completion_promise); + _read_operations_for_next_iteration.push_back(&read_operation); + + // launch the read processing loop it if isn't running, or signal it to resume if it's paused. + if (!_process_pending_reads_loop_complete.valid() || _process_pending_reads_loop_complete.ready()) _process_pending_reads_loop_complete = async([=](){ process_pending_reads(); }); - return completion_promise->wait(); + else if (_new_read_operation_available_promise) + _new_read_operation_available_promise->set_value(); + + size_t bytes_read = completion_promise->wait(); + _unused_read_tokens += read_operation.permitted_length - bytes_read; + return bytes_read; } else return asio::read_some(socket, boost::asio::buffer(buffer, length)); @@ -142,10 +162,18 @@ namespace fc if (_upload_bytes_per_second) { promise::ptr completion_promise(new promise()); - _write_operations_for_next_iteration.emplace_back(std::make_unique(socket, buffer, length, completion_promise)); - if (!_process_pending_writes_loop_complete.valid()) + rate_limited_tcp_write_operation write_operation(socket, buffer, length, completion_promise); + _write_operations_for_next_iteration.push_back(&write_operation); + + // launch the write processing loop it if isn't running, or signal it to resume if it's paused. + if (!_process_pending_writes_loop_complete.valid() || _process_pending_writes_loop_complete.ready()) _process_pending_writes_loop_complete = async([=](){ process_pending_writes(); }); - return completion_promise->wait(); + else if (_new_write_operation_available_promise) + _new_write_operation_available_promise->set_value(); + + size_t bytes_written = completion_promise->wait(); + _unused_write_tokens += write_operation.permitted_length - bytes_written; + return bytes_written; } else return asio::write_some(socket, boost::asio::buffer(buffer, length)); @@ -155,8 +183,20 @@ namespace fc for (;;) { process_pending_operations(_last_read_iteration_time, _download_bytes_per_second, - _read_operations_in_progress, _read_operations_for_next_iteration); - fc::usleep(_granularity); + _read_operations_in_progress, _read_operations_for_next_iteration, _read_tokens, _unused_read_tokens); + + _new_read_operation_available_promise = new promise(); + try + { + if (_read_operations_in_progress.empty()) + _new_read_operation_available_promise->wait(); + else + _new_read_operation_available_promise->wait(_granularity); + } + catch (const timeout_exception&) + { + } + _new_read_operation_available_promise.reset(); } } void rate_limiting_group_impl::process_pending_writes() @@ -164,18 +204,32 @@ namespace fc for (;;) { process_pending_operations(_last_write_iteration_time, _upload_bytes_per_second, - _write_operations_in_progress, _write_operations_for_next_iteration); - fc::usleep(_granularity); + _write_operations_in_progress, _write_operations_for_next_iteration, _write_tokens, _unused_write_tokens); + + _new_write_operation_available_promise = new promise(); + try + { + if (_write_operations_in_progress.empty()) + _new_write_operation_available_promise->wait(); + else + _new_write_operation_available_promise->wait(_granularity); + } + catch (const timeout_exception&) + { + } + _new_write_operation_available_promise.reset(); } } void rate_limiting_group_impl::process_pending_operations(time_point& last_iteration_start_time, uint32_t& limit_bytes_per_second, rate_limited_operation_list& operations_in_progress, - rate_limited_operation_list& operations_for_next_iteration) + rate_limited_operation_list& operations_for_next_iteration, + uint32_t& tokens, + uint32_t& unused_tokens) { // lock here for multithreaded - std::copy(std::make_move_iterator(operations_for_next_iteration.begin()), - std::make_move_iterator(operations_for_next_iteration.end()), + std::copy(operations_for_next_iteration.begin(), + operations_for_next_iteration.end(), std::back_inserter(operations_in_progress)); operations_for_next_iteration.clear(); @@ -188,21 +242,23 @@ namespace fc time_since_last_iteration = seconds(1); else if (time_since_last_iteration < microseconds(0)) time_since_last_iteration = microseconds(0); + + tokens += (uint32_t)((limit_bytes_per_second * time_since_last_iteration.count()) / 1000000); + tokens += unused_tokens; + unused_tokens = 0; + tokens = std::min(tokens, limit_bytes_per_second); - uint32_t total_bytes_for_this_iteration = time_since_last_iteration.count() ? - (uint32_t)((1000000 * limit_bytes_per_second) / time_since_last_iteration.count()) : - 0; - if (total_bytes_for_this_iteration) + if (tokens) { // sort the pending reads/writes in order of the number of bytes they need to write, smallest first std::vector operations_sorted_by_length; operations_sorted_by_length.reserve(operations_in_progress.size()); - for (std::unique_ptr& operation_data : operations_in_progress) - operations_sorted_by_length.push_back(operation_data.get()); + for (rate_limited_operation* operation_data : operations_in_progress) + operations_sorted_by_length.push_back(operation_data); std::sort(operations_sorted_by_length.begin(), operations_sorted_by_length.end(), is_operation_shorter()); // figure out how many bytes each reader/writer is allowed to read/write - uint32_t bytes_remaining_to_allocate = total_bytes_for_this_iteration; + uint32_t bytes_remaining_to_allocate = tokens; while (!operations_sorted_by_length.empty()) { uint32_t bytes_permitted_for_this_operation = bytes_remaining_to_allocate / operations_sorted_by_length.size(); @@ -211,6 +267,7 @@ namespace fc bytes_remaining_to_allocate -= bytes_allocated_for_this_operation; operations_sorted_by_length.pop_back(); } + tokens = bytes_remaining_to_allocate; // kick off the reads/writes in first-come order for (auto iter = operations_in_progress.begin(); iter != operations_in_progress.end();) @@ -225,7 +282,7 @@ namespace fc } } } - else // upload speed is unlimited + else // down/upload speed is unlimited { // we shouldn't end up here often. If the rate is unlimited, we should just execute // the operation immediately without being queued up. This should only be hit if diff --git a/tests/rate_limiting.cpp b/tests/rate_limiting.cpp index 1ca31f3..dd454a1 100644 --- a/tests/rate_limiting.cpp +++ b/tests/rate_limiting.cpp @@ -5,20 +5,30 @@ #include #include +fc::rate_limiting_group rate_limiter(1000000, 1000000); -int main( int argc, char** argv ) +void download_url(const std::string& ip_address, const std::string& url) { - fc::rate_limiting_group rate_limiter(1000000,1000000); fc::http::connection http_connection; rate_limiter.add_tcp_socket(&http_connection.get_socket()); - http_connection.connect_to(fc::ip::endpoint(fc::ip::address("162.243.115.24"),80)); + http_connection.connect_to(fc::ip::endpoint(fc::ip::address(ip_address.c_str()), 80)); std::cout << "Starting download...\n"; fc::time_point start_time(fc::time_point::now()); - fc::http::reply reply = http_connection.request("GET", "http://invictus.io/bin/Keyhotee-0.7.0.dmg"); + fc::http::reply reply = http_connection.request("GET", "http://mirror.cs.vt.edu/pub/cygwin/glibc/releases/glibc-2.9.tar.gz"); fc::time_point end_time(fc::time_point::now()); std::cout << "HTTP return code: " << reply.status << "\n"; std::cout << "Retreived " << reply.body.size() << " bytes in " << ((end_time - start_time).count() / fc::milliseconds(1).count()) << "ms\n"; std::cout << "Average speed " << ((1000 * (uint64_t)reply.body.size()) / ((end_time - start_time).count() / fc::milliseconds(1).count())) << " bytes per second"; +} + +int main( int argc, char** argv ) +{ + std::vector > download_futures; + download_futures.push_back(fc::async([](){ download_url("198.82.184.145", "http://mirror.cs.vt.edu/pub/cygwin/glibc/releases/glibc-2.9.tar.gz"); })); + download_futures.push_back(fc::async([](){ download_url("198.82.184.145", "http://mirror.cs.vt.edu/pub/cygwin/glibc/releases/glibc-2.7.tar.gz"); })); + + for (int i = 0; i < download_futures.size(); ++i) + download_futures[i].wait(); return 0; } From 6b455ab8508f7f33b432e268c0d99551fd5c9200 Mon Sep 17 00:00:00 2001 From: Vikram Rajkumar Date: Mon, 19 May 2014 02:52:21 -0400 Subject: [PATCH 04/19] Fix typos --- src/crypto/aes.cpp | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/crypto/aes.cpp b/src/crypto/aes.cpp index 3f0933f..954f0b4 100644 --- a/src/crypto/aes.cpp +++ b/src/crypto/aes.cpp @@ -38,7 +38,7 @@ void aes_encoder::init( const fc::sha256& key, const fc::uint128& init_value ) * is 128 bits */ if(1 != EVP_EncryptInit_ex(my->ctx, EVP_aes_256_cbc(), NULL, (unsigned char*)&key, (unsigned char*)&init_value)) { - FC_THROW_EXCEPTION( exception, "error durring aes 256 cbc encryption init", + FC_THROW_EXCEPTION( exception, "error during aes 256 cbc encryption init", ("s", ERR_error_string( ERR_get_error(), nullptr) ) ); } EVP_CIPHER_CTX_set_padding( my->ctx, 0 ); @@ -52,7 +52,7 @@ uint32_t aes_encoder::encode( const char* plaintxt, uint32_t plaintext_len, char * */ if(1 != EVP_EncryptUpdate(my->ctx, (unsigned char*)ciphertxt, &ciphertext_len, (const unsigned char*)plaintxt, plaintext_len)) { - FC_THROW_EXCEPTION( exception, "error durring aes 256 cbc encryption update", + FC_THROW_EXCEPTION( exception, "error during aes 256 cbc encryption update", ("s", ERR_error_string( ERR_get_error(), nullptr) ) ); } FC_ASSERT( ciphertext_len == plaintext_len, "", ("ciphertext_len",ciphertext_len)("plaintext_len",plaintext_len) ); @@ -67,7 +67,7 @@ uint32_t aes_encoder::final_encode( char* ciphertxt ) * */ if(1 != EVP_EncryptFinal_ex(my->ctx, (unsigned char*)ciphertxt, &ciphertext_len)) { - FC_THROW_EXCEPTION( exception, "error durring aes 256 cbc encryption final", + FC_THROW_EXCEPTION( exception, "error during aes 256 cbc encryption final", ("s", ERR_error_string( ERR_get_error(), nullptr) ) ); } return ciphertext_len; @@ -98,7 +98,7 @@ void aes_decoder::init( const fc::sha256& key, const fc::uint128& init_value ) * is 128 bits */ if(1 != EVP_DecryptInit_ex(my->ctx, EVP_aes_256_cbc(), NULL, (unsigned char*)&key, (unsigned char*)&init_value)) { - FC_THROW_EXCEPTION( exception, "error durring aes 256 cbc encryption init", + FC_THROW_EXCEPTION( exception, "error during aes 256 cbc encryption init", ("s", ERR_error_string( ERR_get_error(), nullptr) ) ); } EVP_CIPHER_CTX_set_padding( my->ctx, 0 ); @@ -115,7 +115,7 @@ uint32_t aes_decoder::decode( const char* ciphertxt, uint32_t plaintext_len, cha * */ if(1 != EVP_DecryptUpdate(my->ctx, (unsigned char*)plaintext, &ciphertext_len, (const unsigned char*)ciphertxt, plaintext_len)) { - FC_THROW_EXCEPTION( exception, "error durring aes 256 cbc encryption update", + FC_THROW_EXCEPTION( exception, "error during aes 256 cbc encryption update", ("s", ERR_error_string( ERR_get_error(), nullptr) ) ); } FC_ASSERT( ciphertext_len == plaintext_len, "", ("ciphertext_len",ciphertext_len)("plaintext_len",plaintext_len) ); @@ -131,7 +131,7 @@ uint32_t aes_decoder::final_decode( char* plaintext ) * */ if(1 != EVP_DecryptFinal_ex(my->ctx, (unsigned char*)plaintext, &ciphertext_len)) { - FC_THROW_EXCEPTION( exception, "error durring aes 256 cbc encryption final", + FC_THROW_EXCEPTION( exception, "error during aes 256 cbc encryption final", ("s", ERR_error_string( ERR_get_error(), nullptr) ) ); } return ciphertext_len; @@ -172,7 +172,7 @@ int aes_encrypt(unsigned char *plaintext, int plaintext_len, unsigned char *key, * is 128 bits */ if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, key, iv)) { - FC_THROW_EXCEPTION( exception, "error durring aes 256 cbc encryption init", + FC_THROW_EXCEPTION( exception, "error during aes 256 cbc encryption init", ("s", ERR_error_string( ERR_get_error(), nullptr) ) ); } @@ -181,7 +181,7 @@ int aes_encrypt(unsigned char *plaintext, int plaintext_len, unsigned char *key, * */ if(1 != EVP_EncryptUpdate(ctx, ciphertext, &len, plaintext, plaintext_len)) { - FC_THROW_EXCEPTION( exception, "error durring aes 256 cbc encryption update", + FC_THROW_EXCEPTION( exception, "error during aes 256 cbc encryption update", ("s", ERR_error_string( ERR_get_error(), nullptr) ) ); } ciphertext_len = len; @@ -191,7 +191,7 @@ int aes_encrypt(unsigned char *plaintext, int plaintext_len, unsigned char *key, * */ if(1 != EVP_EncryptFinal_ex(ctx, ciphertext + len, &len)) { - FC_THROW_EXCEPTION( exception, "error durring aes 256 cbc encryption final", + FC_THROW_EXCEPTION( exception, "error during aes 256 cbc encryption final", ("s", ERR_error_string( ERR_get_error(), nullptr) ) ); } ciphertext_len += len; @@ -220,7 +220,7 @@ int aes_decrypt(unsigned char *ciphertext, int ciphertext_len, unsigned char *ke * * is 128 bits */ if(1 != EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, key, iv)) { - FC_THROW_EXCEPTION( exception, "error durring aes 256 cbc decrypt init", + FC_THROW_EXCEPTION( exception, "error during aes 256 cbc decrypt init", ("s", ERR_error_string( ERR_get_error(), nullptr) ) ); } @@ -229,7 +229,7 @@ int aes_decrypt(unsigned char *ciphertext, int ciphertext_len, unsigned char *ke * */ if(1 != EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len)) { - FC_THROW_EXCEPTION( exception, "error durring aes 256 cbc decrypt update", + FC_THROW_EXCEPTION( exception, "error during aes 256 cbc decrypt update", ("s", ERR_error_string( ERR_get_error(), nullptr) ) ); } @@ -240,7 +240,7 @@ int aes_decrypt(unsigned char *ciphertext, int ciphertext_len, unsigned char *ke * */ if(1 != EVP_DecryptFinal_ex(ctx, plaintext + len, &len)) { - FC_THROW_EXCEPTION( exception, "error durring aes 256 cbc decrypt final", + FC_THROW_EXCEPTION( exception, "error during aes 256 cbc decrypt final", ("s", ERR_error_string( ERR_get_error(), nullptr) ) ); } plaintext_len += len; @@ -269,7 +269,7 @@ int aes_cfb_decrypt(unsigned char *ciphertext, int ciphertext_len, unsigned char * * is 128 bits */ if(1 != EVP_DecryptInit_ex(ctx, EVP_aes_256_cfb128(), NULL, key, iv)) { - FC_THROW_EXCEPTION( exception, "error durring aes 256 cbc decrypt init", + FC_THROW_EXCEPTION( exception, "error during aes 256 cbc decrypt init", ("s", ERR_error_string( ERR_get_error(), nullptr) ) ); } @@ -278,7 +278,7 @@ int aes_cfb_decrypt(unsigned char *ciphertext, int ciphertext_len, unsigned char * */ if(1 != EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len)) { - FC_THROW_EXCEPTION( exception, "error durring aes 256 cbc decrypt update", + FC_THROW_EXCEPTION( exception, "error during aes 256 cbc decrypt update", ("s", ERR_error_string( ERR_get_error(), nullptr) ) ); } @@ -289,7 +289,7 @@ int aes_cfb_decrypt(unsigned char *ciphertext, int ciphertext_len, unsigned char * */ if(1 != EVP_DecryptFinal_ex(ctx, plaintext + len, &len)) { - FC_THROW_EXCEPTION( exception, "error durring aes 256 cbc decrypt final", + FC_THROW_EXCEPTION( exception, "error during aes 256 cbc decrypt final", ("s", ERR_error_string( ERR_get_error(), nullptr) ) ); } plaintext_len += len; From 174096c3ab2e06156df8c47c935b36cddf7ac29e Mon Sep 17 00:00:00 2001 From: HackFisher Date: Tue, 20 May 2014 19:17:19 +0800 Subject: [PATCH 05/19] Fix issue #25 eofbit std exception is replaced by eof_exception, checked in peek(), so no need to throw std exception when encounter EOF. --- src/io/sstream.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/io/sstream.cpp b/src/io/sstream.cpp index 28a5aac..aed1575 100644 --- a/src/io/sstream.cpp +++ b/src/io/sstream.cpp @@ -9,13 +9,13 @@ namespace fc { public: impl( fc::string&s ) :ss( s ) - { ss.exceptions( std::stringstream::badbit | std::stringstream::eofbit ); } + { ss.exceptions( std::stringstream::badbit ); } impl( const fc::string&s ) :ss( s ) - { ss.exceptions( std::stringstream::badbit | std::stringstream::eofbit ); } + { ss.exceptions( std::stringstream::badbit ); } - impl(){ss.exceptions( std::stringstream::badbit | std::stringstream::eofbit ); } + impl(){ss.exceptions( std::stringstream::badbit ); } std::stringstream ss; }; From 5719d28a190358427836f2d492443384b2a902cc Mon Sep 17 00:00:00 2001 From: Eric Frias Date: Tue, 20 May 2014 18:02:20 -0400 Subject: [PATCH 06/19] Move/rename git revision info compiled into FC to be more self-explanatory --- CMakeLists.txt | 8 ++++---- GitSHA3.cpp.in | 7 ------- GitSHA3.h | 9 --------- include/fc/git_revision.hpp | 9 +++++++++ src/git_revision.cpp.in | 11 +++++++++++ 5 files changed, 24 insertions(+), 20 deletions(-) delete mode 100644 GitSHA3.cpp.in delete mode 100644 GitSHA3.h create mode 100644 include/fc/git_revision.hpp create mode 100644 src/git_revision.cpp.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 87ceef0..4f306a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,8 +16,8 @@ INCLUDE( VersionMacros ) INCLUDE( SetupTargetMacros ) INCLUDE(GetGitRevisionDescription) -get_git_head_revision(GIT_REFSPEC GIT_SHA3) -get_git_unix_timestamp(GIT_UNIX_TIMESTAMP3) +get_git_head_revision(GIT_REFSPEC FC_GIT_REVISION_SHA) +get_git_unix_timestamp(FC_GIT_REVISION_UNIX_TIMESTAMP) SET( DEFAULT_HEADER_INSTALL_DIR include/\${target} ) SET( DEFAULT_LIBRARY_INSTALL_DIR lib/ ) @@ -168,8 +168,8 @@ set( sources ${fc_sources} ) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/GitSHA3.cpp.in" "${CMAKE_CURRENT_BINARY_DIR}/GitSHA3.cpp" @ONLY) -list(APPEND sources "${CMAKE_CURRENT_BINARY_DIR}/GitSHA3.cpp" GitSHA3.h) +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/src/git_revision.cpp.in" "${CMAKE_CURRENT_BINARY_DIR}/git_revision.cpp" @ONLY) +list(APPEND sources "${CMAKE_CURRENT_BINARY_DIR}/git_revision.cpp") list(APPEND sources ${fc_headers}) add_subdirectory( vendor/easylzma ) diff --git a/GitSHA3.cpp.in b/GitSHA3.cpp.in deleted file mode 100644 index 045015e..0000000 --- a/GitSHA3.cpp.in +++ /dev/null @@ -1,7 +0,0 @@ -#include -#include "GitSHA3.h" - -#define GIT_SHA3 "@GIT_SHA3@" -const char* const g_GIT_SHA3 = GIT_SHA3; -#define GIT_UNIX_TIMESTAMP3 @GIT_UNIX_TIMESTAMP3@ -const uint32_t g_GIT_UNIX_TIMESTAMP3 = GIT_UNIX_TIMESTAMP3; diff --git a/GitSHA3.h b/GitSHA3.h deleted file mode 100644 index cb8830e..0000000 --- a/GitSHA3.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef __GITSHA3_H -#define __GITSHA3_H - -extern const char* const g_GIT_SHA3; -extern const uint32_t g_GIT_UNIX_TIMESTAMP3; - -#define APPLICATION_VERSION "1.0 Beta1" - -#endif ///__GITSHA3_H diff --git a/include/fc/git_revision.hpp b/include/fc/git_revision.hpp new file mode 100644 index 0000000..6232f3c --- /dev/null +++ b/include/fc/git_revision.hpp @@ -0,0 +1,9 @@ +#pragma once +#include + +namespace fc { + +extern const char* const git_revision_sha; +extern const uint32_t git_revision_unix_timestamp; + +} // end namespace fc diff --git a/src/git_revision.cpp.in b/src/git_revision.cpp.in new file mode 100644 index 0000000..40977b7 --- /dev/null +++ b/src/git_revision.cpp.in @@ -0,0 +1,11 @@ +#include + +#define FC_GIT_REVISION_SHA "@FC_GIT_REVISION_SHA@" +#define FC_GIT_REVISION_UNIX_TIMESTAMP @FC_GIT_REVISION_UNIX_TIMESTAMP@ + +namespace fc { + +const char* const git_revision_sha = FC_GIT_REVISION_SHA; +const uint32_t git_revision_unix_timestamp = FC_GIT_REVISION_UNIX_TIMESTAMP; + +} // end namespace fc From 1eac85e9e6888e8ee5a66fb1cc55bda2c3d58372 Mon Sep 17 00:00:00 2001 From: Vikram Rajkumar Date: Tue, 20 May 2014 19:02:13 -0400 Subject: [PATCH 07/19] Update .gitignore --- .gitignore | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index bbccfeb..f1cf69a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,6 @@ *.a *.lib - #CMake->MSVC artifacts *.sln *.vcxproj @@ -41,5 +40,7 @@ libfc.a libfc_debug.a fc_automoc.cpp -*.swp +git_revision.cpp GitSHA3.cpp + +*.swp From ba4839198a083c159c31d14ce62dbe08c4d794fc Mon Sep 17 00:00:00 2001 From: Vikram Rajkumar Date: Wed, 21 May 2014 03:11:07 -0400 Subject: [PATCH 08/19] Update .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f1cf69a..5fc5eea 100644 --- a/.gitignore +++ b/.gitignore @@ -43,4 +43,4 @@ fc_automoc.cpp git_revision.cpp GitSHA3.cpp -*.swp +*.sw* From c9529c8f8781486f4ad691cc3163e4a268b66dce Mon Sep 17 00:00:00 2001 From: Vikram Rajkumar Date: Thu, 22 May 2014 03:52:59 -0400 Subject: [PATCH 09/19] Return synonym "uint160_t" for uint160 typename to not break leveldb upgrade routine --- include/fc/crypto/ripemd160.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/fc/crypto/ripemd160.hpp b/include/fc/crypto/ripemd160.hpp index e2d13a9..998b01d 100644 --- a/include/fc/crypto/ripemd160.hpp +++ b/include/fc/crypto/ripemd160.hpp @@ -77,7 +77,7 @@ class ripemd160 typedef ripemd160 uint160_t; typedef ripemd160 uint160; - template<> struct get_typename { static const char* name() { return "uint160"; } }; + template<> struct get_typename { static const char* name() { return "uint160_t"; } }; } // namespace fc From 9bafe4d64fc5bd0bb0aba6fc24e747d614689630 Mon Sep 17 00:00:00 2001 From: Vikram Rajkumar Date: Thu, 22 May 2014 04:52:30 -0400 Subject: [PATCH 10/19] Remove unnecessary compiler flags to get rid of Clang warning messages --- CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4f306a1..b7a68cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -186,13 +186,13 @@ IF(WIN32) # Add /U options to be sure settings specific to dynamic boost link are ineffective target_compile_options(fc PUBLIC /EHsc /UBOOST_ALL_DYN_LINK /UBOOST_LINKING_PYTHON /UBOOST_DEBUG_PYTHON) ELSE() - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -Wall" ) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -Wall") IF(APPLE) - SET(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -std=c++11 -stdlib=libc++ -Wall" ) + SET(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -std=c++11 -stdlib=libc++ -Wall") ELSE() - target_compile_options(fc PUBLIC -std=c++11 -Wall -fnon-call-exceptions -Wno-unused-local-typedefs -fmax-errors=3) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-local-typedefs -fmax-errors=3 ") + target_compile_options(fc PUBLIC -std=c++11 -Wall -fnon-call-exceptions) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") ENDIF() ENDIF() From 9874fc907603b742245043614d37ad8169c771ab Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Thu, 22 May 2014 14:24:31 -0400 Subject: [PATCH 11/19] adding != operator to fc::microseconds --- include/fc/time.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/fc/time.hpp b/include/fc/time.hpp index f14c4f0..888eefd 100644 --- a/include/fc/time.hpp +++ b/include/fc/time.hpp @@ -18,6 +18,7 @@ namespace fc { bool operator==(const microseconds& c)const { return _count == c._count; } + bool operator!=(const microseconds& c)const { return _count != c._count; } friend bool operator>(const microseconds& a, const microseconds& b){ return a._count > b._count; } friend bool operator>=(const microseconds& a, const microseconds& b){ return a._count >= b._count; } friend bool operator<(const microseconds& a, const microseconds& b){ return a._count < b._count; } From ba45a9b09034b4ad477b3aa904ad47c4d8d5a205 Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Thu, 22 May 2014 14:56:52 -0400 Subject: [PATCH 12/19] improve error reporting on std::exceptions --- include/fc/exception/exception.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/fc/exception/exception.hpp b/include/fc/exception/exception.hpp index 5b02e24..9a54db8 100644 --- a/include/fc/exception/exception.hpp +++ b/include/fc/exception/exception.hpp @@ -239,10 +239,10 @@ do { if( !(TEST) ) { FC_THROW_EXCEPTION( assert_exception, #TEST ": " __VA_ARGS catch( fc::exception& er ) { \ FC_RETHROW_EXCEPTION( er, LOG_LEVEL, FORMAT, __VA_ARGS__ ); \ } catch( const std::exception& e ) { \ - throw fc::std_exception( \ - FC_LOG_MESSAGE( LOG_LEVEL, FORMAT,__VA_ARGS__), \ + fc::std_exception fce( \ + FC_LOG_MESSAGE( LOG_LEVEL, "what: ${what} - " FORMAT,__VA_ARGS__("what",e.what())), \ std::current_exception(), \ - e.what() ) ; \ + e.what() ) ; throw fce;\ } catch( ... ) { \ throw fc::unhandled_exception( \ FC_LOG_MESSAGE( LOG_LEVEL, FORMAT,__VA_ARGS__), \ From 7c3767c2082d417cfe439da843d7ff39b7f7e859 Mon Sep 17 00:00:00 2001 From: vogel76 Date: Fri, 23 May 2014 17:37:20 +0200 Subject: [PATCH 13/19] [BW]: [NIP] Changes to support shared libraries build on linux and further installationpackage configuration --- CMakeLists.txt | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4f306a1..d754e76 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,10 +38,6 @@ LIST(APPEND BOOST_COMPONENTS thread date_time system filesystem program_options IF( WIN32 ) MESSAGE(STATUS "Configuring fc to build on Win32") - #You need to set OPENSSL_ROOT environment variable for your system on WIN32 - message(STATUS "Setting up OpenSSL root and include vars on Win32 platform") - set( OPENSSL_ROOT_DIR $ENV{OPENSSL_ROOT} ) - set( RPCRT4 Rpcrt4 ) #boost @@ -77,7 +73,13 @@ ELSE(WIN32) ENDIF(NOT APPLE) ENDIF(WIN32) -find_package( OpenSSL ) +#IF($ENV{OPENSSL_ROOT_DIR}) + set(OPENSSL_ROOT_DIR $ENV{OPENSSL_ROOT_DIR} ) + set(OPENSSL_INCLUDE_DIR ${OPENSSL_ROOT_DIR}/include) + message(STATUS "Setting up OpenSSL root and include vars to ${OPENSSL_ROOT_DIR}, ${OPENSSL_INCLUDE_DIR}") +#ENDIF() + +find_package(OpenSSL) set( CMAKE_FIND_LIBRARY_SUFFIXES ${ORIGINAL_LIB_SUFFIXES} ) From c1ff37889bd0022c7884b836ee1fbc22ac97008b Mon Sep 17 00:00:00 2001 From: Eric Frias Date: Fri, 23 May 2014 18:03:27 -0400 Subject: [PATCH 14/19] Add + operator to time_point_sec --- include/fc/time.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/fc/time.hpp b/include/fc/time.hpp index 888eefd..22e577f 100644 --- a/include/fc/time.hpp +++ b/include/fc/time.hpp @@ -95,6 +95,7 @@ namespace fc { friend bool operator != ( const time_point_sec& a, const time_point_sec& b ) { return a.utc_seconds != b.utc_seconds; } time_point_sec& operator += ( uint32_t m ) { utc_seconds+=m; return *this; } time_point_sec& operator -= ( uint32_t m ) { utc_seconds-=m; return *this; } + time_point_sec operator+( uint32_t offset ) { return time_point_sec(utc_seconds + offset); } friend time_point operator - ( const time_point_sec& t, const microseconds& m ) { return time_point(t) - m; } friend microseconds operator - ( const time_point_sec& t, const time_point_sec& m ) { return time_point(t) - time_point(m); } From 072eb63b17f58d548efe23eaf0af2f663cb66dd1 Mon Sep 17 00:00:00 2001 From: Eric Frias Date: Fri, 23 May 2014 18:09:44 -0400 Subject: [PATCH 15/19] Disable test from compiling by default --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 17d4d57..9f3e8f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -220,8 +220,8 @@ target_link_libraries( fc PUBLIC easylzma_static ${Boost_LIBRARIES} ${OPENSSL_LI #target_link_libraries( test_aes fc ${rt_library} ${pthread_library} ) #add_executable( test_sleep tests/sleep.cpp ) #target_link_libraries( test_sleep fc ) -add_executable( test_rate_limiting tests/rate_limiting.cpp ) -target_link_libraries( test_rate_limiting fc ) +#add_executable( test_rate_limiting tests/rate_limiting.cpp ) +#target_link_libraries( test_rate_limiting fc ) if(WIN32) # add addtional import library on windows platform From 3eaa6a3f653e1d4b6d6fbd616cf8a7ba74f9e37c Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Fri, 23 May 2014 21:38:05 -0400 Subject: [PATCH 16/19] adding mod to uint128 --- include/fc/uint128.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/fc/uint128.hpp b/include/fc/uint128.hpp index 35c5083..dd129bc 100644 --- a/include/fc/uint128.hpp +++ b/include/fc/uint128.hpp @@ -61,6 +61,7 @@ namespace fc friend uint128 operator - ( const uint128& l, const uint128& r ) { return uint128(l)-=r; } friend uint128 operator * ( const uint128& l, const uint128& r ) { return uint128(l)*=r; } friend uint128 operator / ( const uint128& l, const uint128& r ) { return uint128(l)/=r; } + friend uint128 operator % ( const uint128& l, const uint128& r ) { return uint128(l)%=r; } friend uint128 operator | ( const uint128& l, const uint128& r ) { return uint128(l)=(r); } friend uint128 operator & ( const uint128& l, const uint128& r ) { return uint128(l)&=r; } friend uint128 operator ^ ( const uint128& l, const uint128& r ) { return uint128(l)^=r; } From 9f6b52eac2221f398896b318bd46824ee54623e0 Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Fri, 23 May 2014 21:54:59 -0400 Subject: [PATCH 17/19] fix build on OS X --- src/network/rate_limiting.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/network/rate_limiting.cpp b/src/network/rate_limiting.cpp index 0055abb..59db1a5 100644 --- a/src/network/rate_limiting.cpp +++ b/src/network/rate_limiting.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include #include @@ -262,7 +264,7 @@ namespace fc while (!operations_sorted_by_length.empty()) { uint32_t bytes_permitted_for_this_operation = bytes_remaining_to_allocate / operations_sorted_by_length.size(); - uint32_t bytes_allocated_for_this_operation = std::min(operations_sorted_by_length.back()->length, bytes_permitted_for_this_operation); + uint32_t bytes_allocated_for_this_operation = std::min(operations_sorted_by_length.back()->length, bytes_permitted_for_this_operation); operations_sorted_by_length.back()->permitted_length = bytes_allocated_for_this_operation; bytes_remaining_to_allocate -= bytes_allocated_for_this_operation; operations_sorted_by_length.pop_back(); From ac03a71f3da1ea2cd78e5923cc930e2238c4b28c Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Sun, 25 May 2014 14:18:26 -0400 Subject: [PATCH 18/19] improve variant cast error reporting --- include/fc/variant.hpp | 1 + src/variant.cpp | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/include/fc/variant.hpp b/include/fc/variant.hpp index 4bed054..fea92b2 100644 --- a/include/fc/variant.hpp +++ b/include/fc/variant.hpp @@ -430,3 +430,4 @@ namespace fc #include FC_REFLECT_TYPENAME( fc::variant ) +FC_REFLECT_ENUM( fc::variant::type_id, (null_type)(int64_type)(uint64_type)(double_type)(bool_type)(string_type)(array_type)(object_type) ) diff --git a/src/variant.cpp b/src/variant.cpp index c6077f1..fe69efe 100644 --- a/src/variant.cpp +++ b/src/variant.cpp @@ -8,6 +8,7 @@ //#include #include #include +#include namespace fc { @@ -351,7 +352,7 @@ int64_t variant::as_int64()const case null_type: return 0; default: - FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to int64", ("type", "") ); + FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to int64", ("type", get_type()) ); } } @@ -372,7 +373,7 @@ uint64_t variant::as_uint64()const case null_type: return 0; default: - FC_THROW_EXCEPTION( bad_cast_exception,"Invalid cast from ${type} to uint64", ("type","")); + FC_THROW_EXCEPTION( bad_cast_exception,"Invalid cast from ${type} to uint64", ("type",get_type())); } } @@ -394,7 +395,7 @@ double variant::as_double()const case null_type: return 0; default: - FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to double" ); + FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to double", ("type",get_type()) ); } } @@ -415,7 +416,7 @@ bool variant::as_bool()const case null_type: return false; default: - FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to bool" ); + FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to bool" , ("type",get_type())); } } @@ -436,7 +437,7 @@ string variant::as_string()const case null_type: return string(); default: - FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to string", ("type", int64_t(get_type()) ) ); + FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to string", ("type", get_type() ) ); } } @@ -447,7 +448,7 @@ variants& variant::get_array() if( get_type() == array_type ) return **reinterpret_cast(this); - FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to Array" ); + FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to Array", ("type",get_type()) ); } @@ -456,7 +457,7 @@ const variants& variant::get_array()const { if( get_type() == array_type ) return **reinterpret_cast(this); - FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to Array" ); + FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to Array", ("type",get_type()) ); } @@ -465,7 +466,7 @@ variant_object& variant::get_object() { if( get_type() == object_type ) return **reinterpret_cast(this); - FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to Object" ); + FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to Object", ("type",get_type()) ); } const variant& variant::operator[]( const char* key )const @@ -486,7 +487,7 @@ const string& variant::get_string()const { if( get_type() == string_type ) return **reinterpret_cast(this); - FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from type '${type}' to Object", ("type",int(get_type())) ); + FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from type '${type}' to Object", ("type",get_type()) ); } @@ -495,7 +496,7 @@ const variant_object& variant::get_object()const { if( get_type() == object_type ) return **reinterpret_cast(this); - FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from type '${type}' to Object", ("type",int(get_type())) ); + FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from type '${type}' to Object", ("type",get_type()) ); } void to_variant( const std::string& s, variant& v ) From 65361f1fce1fb2565ef76bb046b69286e05cfabb Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Sun, 25 May 2014 23:12:33 -0400 Subject: [PATCH 19/19] updated typedefs for clarity --- include/fc/crypto/elliptic.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/fc/crypto/elliptic.hpp b/include/fc/crypto/elliptic.hpp index 03502be..e7145a4 100644 --- a/include/fc/crypto/elliptic.hpp +++ b/include/fc/crypto/elliptic.hpp @@ -16,6 +16,7 @@ namespace fc { } typedef fc::array public_key_data; + typedef fc::sha256 private_key_secret; typedef fc::array public_key_point_data; ///< the full non-compressed version of the ECC point typedef fc::array signature; typedef fc::array compact_signature; @@ -92,7 +93,9 @@ namespace fc { */ static private_key generate_from_seed( const fc::sha256& seed, const fc::sha256& offset = fc::sha256() ); - fc::sha256 get_secret()const; // get the private key secret + private_key_secret get_secret()const; // get the private key secret + + operator private_key_secret ()const { return get_secret(); } /** * Given a public key, calculatse a 512 bit shared secret between that