Merge branch 'phoenix' into tcp_rate_limiting

Conflicts:
	include/fc/network/tcp_socket.hpp
	src/network/tcp_socket.cpp

include/fc/io/datastream.hpp

@@ -7,7 +7,7 @@ namespace fc {
 
 namespace detail 
 {
-  NO_RETURN void throw_datastream_range_error( const char* file, size_t len, size_t over );
+  NO_RETURN void throw_datastream_range_error( const char* file, size_t len, int64_t over );
 }
 
 /**

include/fc/io/json.hpp

@@ -1,9 +1,9 @@
 #pragma once
 #include <fc/variant.hpp>
+#include <fc/filesystem.hpp>
 
 namespace fc
 {
-   class path;
    class ostream;
    class buffered_istream;
 
@@ -54,9 +54,9 @@ namespace fc
          }
 
          template<typename T>
-         static void save_to_file( const T& v, const string& p, bool pretty = true ) 
+         static void save_to_file( const T& v, const std::string& p, bool pretty = true ) 
          {
-            save_to_file( variant(v), p, pretty );
+            save_to_file( variant(v), fc::path(p), pretty );
          } 
    };
 

include/fc/io/varint.hpp

@@ -27,6 +27,11 @@ struct unsigned_int {
     friend bool operator>=( const unsigned_int& i, const unsigned_int& v ) { return v >= i.value; }
 };
 
+/**
+ *  @brief serializes a 32 bit signed interger in as few bytes as possible
+ *
+ *  Uses the google protobuf algorithm for seralizing signed numbers
+ */
 struct signed_int {
     signed_int( int32_t v = 0 ):value(v){}
     operator int32_t()const { return value; }

include/fc/network/http/connection.hpp

@@ -25,6 +25,8 @@ namespace fc {
         enum status_code {
             OK                  = 200,
             RecordCreated       = 201,
+            BadRequest          = 400,
+            NotAuthorized       = 401,
             NotFound            = 404,
             Found               = 302,
             InternalServerError = 500
@@ -74,3 +76,6 @@ namespace fc {
 
 } } // fc::http
 
+#include <fc/reflect/reflect.hpp>
+FC_REFLECT( fc::http::header, (key)(val) )
+

include/fc/network/tcp_socket.hpp

@@ -19,13 +19,16 @@ namespace fc {
       void     connect_to( const fc::ip::endpoint& remote_endpoint, const fc::ip::endpoint& local_endpoint );
       void     enable_keep_alives(const fc::microseconds& interval);
       void set_io_hooks(tcp_socket_io_hooks* new_hooks);
+      void set_reuse_address(bool enable = true); // set SO_REUSEADDR
       fc::ip::endpoint remote_endpoint() const;
+      fc::ip::endpoint local_endpoint() const;
 
       void get( char& c )
       {
           read( &c, 1 );
       }
 
+
       /// istream interface
       /// @{
       virtual size_t   readsome( char* buffer, size_t max );
@@ -39,6 +42,7 @@ namespace fc {
       virtual void     close();
       /// @}
 
+      void open();
       bool   is_open()const;
 
     private:
@@ -61,6 +65,7 @@ namespace fc {
 
       void     close();
       void     accept( tcp_socket& s );
+      void     set_reuse_address(bool enable = true); // set SO_REUSEADDR, call before listen
       void     listen( uint16_t port );
       void     listen( const fc::ip::endpoint& ep );
       uint16_t get_port()const;

include/fc/reflect/reflect.hpp

@@ -132,6 +132,14 @@ template<> struct reflector<ENUM> { \
       }\
       return nullptr; \
     } \
+    static const char* to_string(ENUM elem) { \
+      switch( elem ) { \
+        BOOST_PP_SEQ_FOR_EACH( FC_REFLECT_ENUM_TO_STRING, ENUM, FIELDS ) \
+        default: \
+           fc::throw_bad_enum_cast( BOOST_PP_STRINGIZE(elem), BOOST_PP_STRINGIZE(ENUM) ); \
+      }\
+      return nullptr; \
+    } \
     static ENUM from_string( const char* s ) { \
         BOOST_PP_SEQ_FOR_EACH( FC_REFLECT_ENUM_FROM_STRING, ENUM, FIELDS ) \
         fc::throw_bad_enum_cast( s, BOOST_PP_STRINGIZE(ENUM) ); \

include/fc/rpc/json_connection.hpp

@@ -76,6 +76,12 @@ namespace fc { namespace rpc  {
                                      const variant& a2, 
                                      const variant& a3 );
 
+         future<variant> async_call( const fc::string& method, 
+                                     const variant& a1, 
+                                     const variant& a2, 
+                                     const variant& a3, 
+                                     const variant& a4 );
+
          template<typename Result>
          Result call( const fc::string& method, 
                                const variant& a1, 
@@ -86,6 +92,17 @@ namespace fc { namespace rpc  {
             return async_call( method, a1, a2, a3 ).wait(timeout).as<Result>();
          }
 
+         template<typename Result>
+         Result call( const fc::string& method, 
+                               const variant& a1, 
+                               const variant& a2, 
+                               const variant& a3,
+                               const variant& a4,
+                               microseconds timeout = microseconds::maximum())
+         {
+            return async_call( method, a1, a2, a3, a4).wait(timeout).as<Result>();
+         }
+
          template<typename Result>
          Result call( const fc::string& method, 
                                const variant& a1, 

include/fc/time.hpp

@@ -45,6 +45,7 @@ namespace fc {
         static time_point from_iso_string( const fc::string& s );
 
         const microseconds& time_since_epoch()const { return elapsed; }
+        uint32_t            sec_since_epoch()const  { return elapsed.count() / 1000000; }
         bool   operator > ( const time_point& t )const                              { return elapsed._count > t.elapsed._count; }
         bool   operator >=( const time_point& t )const                              { return elapsed._count >=t.elapsed._count; }
         bool   operator < ( const time_point& t )const                              { return elapsed._count < t.elapsed._count; }
@@ -87,6 +88,7 @@ namespace fc {
         friend bool      operator <= ( const time_point_sec& a, const time_point_sec& b )  { return a.utc_seconds <= b.utc_seconds; }
         friend bool      operator >= ( const time_point_sec& a, const time_point_sec& b )  { return a.utc_seconds >= b.utc_seconds; }
         friend bool      operator == ( const time_point_sec& a, const time_point_sec& b ) { return a.utc_seconds == b.utc_seconds; }
+        friend bool      operator != ( const time_point_sec& a, const time_point_sec& b ) { return a.utc_seconds != b.utc_seconds; }
         time_point_sec&  operator += ( uint32_t m ) { utc_seconds+=m; return *this; }
 
         friend time_point   operator - ( const time_point_sec& t, const microseconds& m )   { return time_point(t) - m;             }

include/fc/variant_object.hpp

@@ -59,7 +59,7 @@ namespace fc
       const variant& operator[]( const string& key )const;
       const variant& operator[]( const char* key )const;
       size_t size()const;
-      bool   contains( const char* key ) { return find(key) != end(); }
+      bool   contains( const char* key ) const { return find(key) != end(); }
       ///@}
 
       variant_object();

src/io/datastream.cpp

@@ -1,7 +1,7 @@
 #include <fc/io/datastream.hpp>
 #include <fc/exception/exception.hpp>
 
-NO_RETURN void fc::detail::throw_datastream_range_error(char const* method, size_t len, size_t over)
+NO_RETURN void fc::detail::throw_datastream_range_error(char const* method, size_t len, int64_t over)
 {
   FC_THROW_EXCEPTION( out_of_range_exception, "${method} datastream of length ${len} over by ${over}", ("method",fc::string(method))("len",len)("over",over) );
 }

src/io/iostream.cpp

@@ -264,12 +264,14 @@ namespace fc {
 
   istream& operator>>( istream& o, std::string& v )
   {
+     assert(false && "not implemented");
      return o;
   }
 
 #ifdef USE_FC_STRING
   istream& operator>>( istream& o, fc::string& v )
   {
+     assert(false && "not implemented");
      return o;
   }
 #endif
@@ -282,51 +284,61 @@ namespace fc {
 
   istream& operator>>( istream& o, double& v )
   {
+     assert(false && "not implemented");
      return o;
   }
 
   istream& operator>>( istream& o, float& v )
   {
+     assert(false && "not implemented");
      return o;
   }
 
   istream& operator>>( istream& o, int64_t& v )
   {
+     assert(false && "not implemented");
      return o;
   }
 
   istream& operator>>( istream& o, uint64_t& v )
   {
+     assert(false && "not implemented");
      return o;
   }
 
   istream& operator>>( istream& o, int32_t& v )
   {
+     assert(false && "not implemented");
      return o;
   }
 
   istream& operator>>( istream& o, uint32_t& v )
   {
+     assert(false && "not implemented");
      return o;
   }
 
   istream& operator>>( istream& o, int16_t& v )
   {
+     assert(false && "not implemented");
      return o;
   }
 
   istream& operator>>( istream& o, uint16_t& v )
   {
+     assert(false && "not implemented");
      return o;
   }
 
   istream& operator>>( istream& o, int8_t& v )
   {
+     assert(false && "not implemented");
      return o;
   }
 
   istream& operator>>( istream& o, uint8_t& v )
   {
+     assert(false && "not implemented");
      return o;
   }
 

src/io/json.cpp

@@ -211,32 +211,29 @@ namespace fc
    template<typename T>
    variant number_from_stream( T& in )
    {
-      char buf[30];
+      fc::stringstream ss;
-      memset( buf, 0, sizeof(buf) );
+
-      char* pos = buf;
       bool  dot = false;
       bool  neg = false;
       if( in.peek() == '-')
       {
         neg = true;
-        *pos = in.get();
+        ss.put( in.get() );
-        ++pos;
       }
       bool done = false;
-      while(  !done)
+
+      try
       {
-        char c = in.peek();
+        char c;
+        while((c = in.peek()) && !done)
+        {
+          
           switch( c )
           {
               case '.':
-            {
                  if (dot) 
-               {
+                    FC_THROW_EXCEPTION(parse_error_exception, "Can't parse a number with two decimal places");
-                  done = true;
-                  break;
-               }
                  dot = true;
-            }
               case '0':
               case '1':
               case '2':
@@ -247,24 +244,36 @@ namespace fc
               case '7':
               case '8':
               case '9':
-               *pos = c;
+                 ss.put( in.get() );
-               ++pos;
-               in.get();
                  break;
               default:
                 done = true;
                 break;
           }
         }
-      if( dot ) return to_double(buf);
+      }
-      if( neg ) return to_int64(buf);
+      catch (fc::eof_exception&)
-      return to_uint64(buf);
+      {
+      }
+      fc::string str = ss.str();
+      if (str == "-." || str == ".") // check the obviously wrong things we could have encountered
+        FC_THROW_EXCEPTION(parse_error_exception, "Can't parse token \"${token}\" as a JSON numeric constant", ("token", str));
+      if( dot )
+        return to_double(str);
+      if( neg ) 
+        return to_int64(str);
+      return to_uint64(str);
    }
    template<typename T>
    variant token_from_stream( T& in )
    {
       fc::stringstream ss;
-      while( char c = in.peek() )
+      bool parsed_unexpected_character = false;
+      bool received_eof = false;
+      try
+      {
+        char c;
+        while((c = in.peek()) && !parsed_unexpected_character)
         {
            switch( c )
            {
@@ -280,22 +289,38 @@ namespace fc
                  ss.put( in.get() );
                  break;
               default:
+                 parsed_unexpected_character = true;
+                 break;
+           }
+        }
+      }
+      catch (fc::eof_exception&)
       {
+        received_eof = true;
+      }
+
+      // we can get here either by processing a delimiter as in "null,"
+      // an EOF like "null<EOF>", or an invalid token like "nullZ"
       fc::string str = ss.str();
       if( str == "null" )  return variant();
       if( str == "true" )  return true;
       if( str == "false" ) return false;
       else 
       {
+        if (received_eof)
+          FC_THROW_EXCEPTION( parse_error_exception, "Unexpected EOF" );
+        else
+        {
+          // I'm not sure why we do this, a comment would be helpful.
+          // if we've reached this point, we've either seen a partial
+          // token ("tru<EOF>") or something our simple parser couldn't
+          // make out ("falfe")
           return str;
           // FC_THROW_EXCEPTION( parse_error_exception, "Invalid token '${token}'",
           //                          ("token",str) );
         }
       }
    }
-      }
-	  FC_THROW_EXCEPTION( parse_error_exception, "Unexpected EOF" );
-   }
 
 
    template<typename T>

src/network/tcp_socket.cpp

@@ -21,7 +21,8 @@ namespace fc {
       {}
       ~impl()
       {
-        if( _sock.is_open() ) _sock.close();
+        if( _sock.is_open() ) 
+          _sock.close();
       }
       virtual size_t readsome(boost::asio::ip::tcp::socket& socket, char* buffer, size_t length) override;
       virtual size_t writesome(boost::asio::ip::tcp::socket& socket, const char* buffer, size_t length) override;
@@ -39,6 +40,12 @@ namespace fc {
     return fc::asio::write_some(socket, boost::asio::buffer(buffer, length));
   }
 
+
+  void tcp_socket::open()
+  {
+    my->_sock.open(boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), 0).protocol());
+  }
+
   bool tcp_socket::is_open()const {
     return my->_sock.is_open();
   }
@@ -71,6 +78,13 @@ namespace fc {
     return  fc::ip::endpoint(rep.address().to_v4().to_ulong(), rep.port() );
   }
 
+
+  fc::ip::endpoint tcp_socket::local_endpoint() const
+  {
+    auto boost_local_endpoint = my->_sock.local_endpoint();
+    return fc::ip::endpoint(boost_local_endpoint.address().to_v4().to_ulong(), boost_local_endpoint.port() );
+  }
+
   size_t tcp_socket::readsome( char* buf, size_t len ) {
     return my->_io_hooks->readsome(my->_sock, buf, len);
   }
@@ -80,9 +94,16 @@ namespace fc {
   }
 
   void tcp_socket::connect_to( const fc::ip::endpoint& remote_endpoint, const fc::ip::endpoint& local_endpoint ) {
-    my->_sock = boost::asio::ip::tcp::socket(fc::asio::default_io_service(), 
+    try
-                                             boost::asio::ip::tcp::endpoint(boost::asio::ip::address_v4(local_endpoint.get_address()), 
+    {
+    my->_sock.bind(boost::asio::ip::tcp::endpoint(boost::asio::ip::address_v4(local_endpoint.get_address()), 
                                                                               local_endpoint.port()));
+    }
+    catch (std::exception& except)
+    {
+      elog("Exception binding outgoing connection to desired local endpoint: ${what}", ("what", except.what()));
+      throw;
+    }
     fc::asio::tcp::connect(my->_sock, fc::asio::tcp::endpoint(boost::asio::ip::address_v4(remote_endpoint.get_address()), remote_endpoint.port())); 
   }
 
@@ -102,7 +123,7 @@ namespace fc {
       if (WSAIoctl(my->_sock.native(), SIO_KEEPALIVE_VALS, &keepalive_settings, sizeof(keepalive_settings),
                    NULL, 0, &dwBytesRet, NULL, NULL) == SOCKET_ERROR)
         wlog("Error setting TCP keepalive values");
-#else
+#elif !defined(__clang__) || (__clang_major__ >= 6)
       // This should work for modern Linuxes and for OSX >= Mountain Lion
       int timeout_sec = interval.count() / fc::seconds(1).count();
       if (setsockopt(my->_sock.native(), IPPROTO_TCP, 
@@ -113,10 +134,12 @@ namespace fc {
        #endif
                      (char*)&timeout_sec, sizeof(timeout_sec)) < 0)
         wlog("Error setting TCP keepalive idle time");
+# if !defined(__APPLE__) || defined(TCP_KEEPINTVL) // TCP_KEEPINTVL not defined before 10.9
       if (setsockopt(my->_sock.native(), IPPROTO_TCP, TCP_KEEPINTVL, 
                      (char*)&timeout_sec, sizeof(timeout_sec)) < 0)
         wlog("Error setting TCP keepalive interval");
-#endif
+# endif // !__APPLE__ || TCP_KEEPINTVL
+#endif // !WIN32
     }
     else
     {
@@ -130,13 +153,21 @@ namespace fc {
     my->_io_hooks = new_hooks ? new_hooks : &*my;
   }
 
+  void tcp_socket::set_reuse_address(bool enable /* = true */)
+  {
+    FC_ASSERT(my->_sock.is_open());
+    boost::asio::socket_base::reuse_address option(enable);
+    my->_sock.set_option(option);
+  }
+
+
   class tcp_server::impl {
     public:
-      impl(uint16_t port)
+      impl()
-      :_accept( fc::asio::default_io_service(), boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), port) ){}
+      :_accept( fc::asio::default_io_service() )
-
+      {
-      impl(const fc::ip::endpoint& ep  )
+        _accept.open(boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), 0).protocol());
-      :_accept( fc::asio::default_io_service(), boost::asio::ip::tcp::endpoint(boost::asio::ip::address_v4( ep.get_address()), ep.port()) ){}
+      }
 
       ~impl(){
         try {
@@ -151,8 +182,10 @@ namespace fc {
       boost::asio::ip::tcp::acceptor _accept;
   };
   void tcp_server::close() {
-    if( my && my->_accept.is_open() ) my->_accept.close();
+    if( my && my->_accept.is_open() ) 
-    delete my; my = nullptr;
+      my->_accept.close();
+    delete my;
+    my = nullptr;
   }
   tcp_server::tcp_server()
   :my(nullptr) {
@@ -170,20 +203,31 @@ namespace fc {
       fc::asio::tcp::accept( my->_accept, s.my->_sock  ); 
     } FC_RETHROW_EXCEPTIONS( warn, "Unable to accept connection on socket." );
   }
-
+  void tcp_server::set_reuse_address(bool enable /* = true */)
+  {
+    if( !my ) 
+      my = new impl;
+    boost::asio::ip::tcp::acceptor::reuse_address option(enable);
+    my->_accept.set_option(option);
+  }
   void tcp_server::listen( uint16_t port ) 
   {
-    if( my ) delete my;
+    if( !my ) 
-    my = new impl(port);
+      my = new impl;
+    my->_accept.bind(boost::asio::ip::tcp::endpoint(boost::asio::ip::address_v4(), port));
+    my->_accept.listen();
   }
   void tcp_server::listen( const fc::ip::endpoint& ep ) 
   {
-    if( my ) delete my;
+    if( !my ) 
-    my = new impl(ep);
+      my = new impl;
+    my->_accept.bind(boost::asio::ip::tcp::endpoint(boost::asio::ip::address_v4::from_string((string)ep.get_address()), ep.port()));
+    my->_accept.listen();
   }
 
   uint16_t tcp_server::get_port()const
   {
+     FC_ASSERT( my != nullptr );
      return my->_accept.local_endpoint().port();
   }
 

src/rpc/json_connection.cpp

@@ -35,7 +35,7 @@ namespace fc { namespace rpc {
 
             void send_result( variant id, variant result )
             {
-               ilog( "send ${i} ${r}", ("i",id)("r",result) );
+               ilog( "send: {\"id\": ${i}, \"result\": ${r}}", ("i",id)("r",result) );
                {
                  fc::scoped_lock<fc::mutex> lock(_write_mutex);
                  *_out << "{\"id\":";
@@ -48,6 +48,8 @@ namespace fc { namespace rpc {
             }
             void send_error( variant id, fc::exception& e )
             {
+               ilog( "send: {\"id\": ${i}, \"error\":{\"message\": ${what},\"code\":0,\"data\":${data}}}",
+                     ("i",id)("what",e.what())("data", e) );
                {
                  fc::scoped_lock<fc::mutex> lock(_write_mutex);
                  *_out << "{\"id\":";
@@ -141,7 +143,7 @@ namespace fc { namespace rpc {
                         }
                      }
                   }
-                  else if( i != obj.end() )
+                  else if( i != obj.end() ) //handle any received JSON response
                   {
                      uint64_t id = i->value().as_int64();
                      auto await = _awaiting.find(id);
@@ -149,11 +151,11 @@ namespace fc { namespace rpc {
                      {
                         auto r = obj.find("result");
                         auto e = obj.find("error");
-                        if( r != obj.end() )
+                        if( r != obj.end() ) //if regular result response
                         {
                            await->second->set_value( r->value() ); 
                         }
-                        else if( e != obj.end() )
+                        else if( e != obj.end() ) //if error response
                         {
                           try
                           {
@@ -406,7 +408,30 @@ namespace fc { namespace rpc {
       return my->_awaiting[id];
    }
 
+   future<variant> json_connection::async_call( const fc::string& method, const variant& a1, const variant& a2, const variant& a3, const variant& a4 )
+   {
+      auto id = my->_next_id++;
+      my->_awaiting[id] = fc::promise<variant>::ptr( new fc::promise<variant>() );
 
+      {
+         fc::scoped_lock<fc::mutex> lock(my->_write_mutex);
+         *my->_out << "{\"id\":";
+         *my->_out << id;
+         *my->_out << ",\"method\":";
+         json::to_stream( *my->_out, method );
+         *my->_out << ",\"params\":[";
+         fc::json::to_stream( *my->_out, a1 );
+         *my->_out << ",";
+         fc::json::to_stream( *my->_out, a2 );
+         *my->_out << ",";
+         fc::json::to_stream( *my->_out, a3 );
+         *my->_out << ",";
+         fc::json::to_stream( *my->_out, a4 );
+         *my->_out << "]}\n";
+      }
+      my->_out->flush();
+      return my->_awaiting[id];
+   }
 
    future<variant> json_connection::async_call( const fc::string& method, mutable_variant_object named_args )
    {

src/variant.cpp

@@ -486,7 +486,7 @@ const string&        variant::get_string()const
 {
   if( get_type() == string_type )
      return **reinterpret_cast<const const_string_ptr*>(this);
-  FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to Object" );
+  FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from type '${type}' to Object", ("type",int(get_type())) );
 }
 
 
@@ -495,7 +495,7 @@ const variant_object&  variant::get_object()const
 {
   if( get_type() == object_type )
      return **reinterpret_cast<const const_variant_object_ptr*>(this);
-  FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from ${type} to Object" );
+  FC_THROW_EXCEPTION( bad_cast_exception, "Invalid cast from type '${type}' to Object", ("type",int(get_type())) );
 }
 
 void to_variant( const std::string& s, variant& v )