Merge Elasticplugin, snapshot plugin and graphene updates to beatrice (#304)

* check witness signature before adding block to fork db

* Replace verify_no_send_in_progress with no_parallel_execution_guard

* fixed cli_wallet log issue

* Port plugin sanitization code

* avoid directly overwriting wallet file

* Implemented "plugins" config variable

* allow plugin to have descriptions

* Merge pull request #444 from oxarbitrage/elasticsearch

Elasticsearch plugin

* Merge pull request #500 from oxarbitrage/elasticsearch-extras

es_objects plugin

* Merge pull request #873 from pmconrad/585_fix_history_ids

Fix history ids

* Merge pull request #1201 from oxarbitrage/elasticsearch_tests2

Elasticsearch refactor

* Merge pull request #1271 from oxarbitrage/es_objects

refine es_objects plugin

* Merge pull request #1429 from oxarbitrage/es_objects_templates

Add an adaptor to es_objects and template function to reduce code

* Merge pull request #1458 from oxarbitrage/issue1455

add option elasticsearch-start-es-after-block to es plugin

* Merge pull request #1541 from oxarbitrage/es_objects_start_after_block

add es-objects-start-es-after-block option

* explicitly cleanup external library facilities

* Merge pull request #1717 from oxarbitrage/issue1652

add genesis data to es_objects

* Merge pull request #1073 from xiangxn/merge-impacted

merge impacted into db_notify

* Merge pull request #1725 from oxarbitrage/issue1682

elasticsearch history api #1682

* change ES index prefixes to Peerplays-specific

* sync develop with beatrice

* fix the data writing to ES during sync issues

* fix CLI tests

* brought updates from mainnet branch (#285)

* Fix unit test failures (#289)

* fixed unit test failures from the recent merges

* fixed unit test failures from the recent merges

* enable snapshot plugin (#288)

* sync fc branch(build optimization changes)

* update to es plugin

* fix verify witness signature method (#295)

* enable mandatory plugins to have smooth transition for next release

* updated tests to keep in-line with plugin changes

Co-authored-by: Sandip Patel <sandip@knackroot.com>
Co-authored-by: Peter Conrad <conrad@quisquis.de>
Co-authored-by: Alfredo <oxarbitrage@gmail.com>
Co-authored-by: Abit <abitmore@users.noreply.github.com>
Co-authored-by: crypto-ape <43807588+crypto-ape@users.noreply.github.com>
Co-authored-by: gladcow <s.gladkov@pbsa.info>

.gitmodules

@@ -5,5 +5,5 @@
 [submodule "libraries/fc"]
     path = libraries/fc
     url = https://github.com/peerplays-network/peerplays-fc.git
-    branch = beatrice
+    branch = latest-fc
     ignore = dirty

libraries/app/CMakeLists.txt

@@ -5,7 +5,6 @@ add_library( graphene_app
              api.cpp
              application.cpp
              database_api.cpp
-             impacted.cpp
              plugin.cpp
              config_util.cpp
              ${HEADERS}
@@ -14,7 +13,7 @@ add_library( graphene_app
 
 # need to link graphene_debug_witness because plugins aren't sufficiently isolated #246
 #target_link_libraries( graphene_app graphene_market_history graphene_account_history graphene_chain fc graphene_db graphene_net graphene_utilities graphene_debug_witness )
-target_link_libraries( graphene_app graphene_market_history graphene_account_history graphene_accounts_list graphene_affiliate_stats graphene_chain fc graphene_db graphene_net graphene_time graphene_utilities graphene_debug_witness graphene_bookie )
+target_link_libraries( graphene_app graphene_market_history graphene_account_history graphene_accounts_list graphene_affiliate_stats graphene_chain fc graphene_db graphene_net graphene_time graphene_utilities graphene_debug_witness graphene_bookie graphene_elasticsearch )
 target_include_directories( graphene_app
                             PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include"
                             "${CMAKE_CURRENT_SOURCE_DIR}/../egenesis/include" )

libraries/app/api.cpp

@@ -26,7 +26,6 @@
 #include <graphene/app/api.hpp>
 #include <graphene/app/api_access.hpp>
 #include <graphene/app/application.hpp>
-#include <graphene/app/impacted.hpp>
 #include <graphene/chain/database.hpp>
 #include <graphene/chain/get_config.hpp>
 #include <graphene/utilities/key_conversion.hpp>
@@ -581,6 +580,18 @@ namespace graphene { namespace app {
               start = node.operation_id;
         } catch(...) { return result; }
 
+        if(_app.is_plugin_enabled("elasticsearch")) {
+           auto es = _app.get_plugin<elasticsearch::elasticsearch_plugin>("elasticsearch");
+           if(es.get()->get_running_mode() != elasticsearch::mode::only_save) {
+              if(!_app.elasticsearch_thread)
+                 _app.elasticsearch_thread= std::make_shared<fc::thread>("elasticsearch");
+
+              return _app.elasticsearch_thread->async([&es, &account, &stop, &limit, &start]() {
+                 return es->get_account_history(account, stop, limit, start);
+              }, "thread invoke for method " BOOST_PP_STRINGIZE(method_name)).wait();
+           }
+        }
+
         const auto& hist_idx = db.get_index_type<account_transaction_history_index>();
         const auto& by_op_idx = hist_idx.indices().get<by_op>();
         auto index_start = by_op_idx.begin();

libraries/app/application.cpp

@@ -891,7 +891,8 @@ namespace detail {
       std::shared_ptr<fc::http::websocket_server>      _websocket_server;
       std::shared_ptr<fc::http::websocket_tls_server>  _websocket_tls_server;
 
-      std::map<string, std::shared_ptr<abstract_plugin>> _plugins;
+      std::map<string, std::shared_ptr<abstract_plugin>> _active_plugins;
+      std::map<string, std::shared_ptr<abstract_plugin>> _available_plugins;
 
       bool _is_finished_syncing = false;
    };
@@ -933,6 +934,7 @@ void application::set_program_options(boost::program_options::options_descriptio
          ("enable-standby-votes-tracking", bpo::value<bool>()->implicit_value(true),
           "Whether to enable tracking of votes of standby witnesses and committee members. "
           "Set it to true to provide accurate data to API clients, set to false for slightly better performance.")
+         ("plugins", bpo::value<string>(), "Space-separated list of plugins to activate")
          ;
    command_line_options.add(configuration_file_options);
    command_line_options.add_options()
@@ -978,6 +980,36 @@ void application::initialize(const fc::path& data_dir, const boost::program_opti
 
       std::exit(EXIT_SUCCESS);
    }
+
+   std::set<string> wanted;
+   if( options.count("plugins") )
+   {
+      boost::split(wanted, options.at("plugins").as<std::string>(), [](char c){return c == ' ';});
+   }
+   else
+   {      
+      wanted.insert("account_history");
+      wanted.insert("market_history");
+      wanted.insert("accounts_list");
+      wanted.insert("affiliate_stats");
+   }
+   wanted.insert("witness");
+   wanted.insert("bookie");
+
+   int es_ah_conflict_counter = 0;
+   for (auto& it : wanted)
+   {
+      if(it == "account_history")
+         ++es_ah_conflict_counter;
+      if(it == "elasticsearch")
+         ++es_ah_conflict_counter;
+
+      if(es_ah_conflict_counter > 1) {
+         elog("Can't start program with elasticsearch and account_history plugin at the same time");
+         std::exit(EXIT_FAILURE);
+      }
+      if (!it.empty()) enable_plugin(it);
+   }
 }
 
 void application::startup()
@@ -995,7 +1027,12 @@ void application::startup()
 
 std::shared_ptr<abstract_plugin> application::get_plugin(const string& name) const
 {
-   return my->_plugins[name];
+   return my->_active_plugins[name];
+}
+
+bool application::is_plugin_enabled(const string& name) const
+{
+   return !(my->_active_plugins.find(name) == my->_active_plugins.end());
 }
 
 net::node_ptr application::p2p_node()
@@ -1028,14 +1065,21 @@ bool application::is_finished_syncing() const
    return my->_is_finished_syncing;
 }
 
-void graphene::app::application::add_plugin(const string& name, std::shared_ptr<graphene::app::abstract_plugin> p)
+void graphene::app::application::enable_plugin(const string& name)
 {
-   my->_plugins[name] = p;
+   FC_ASSERT(my->_available_plugins[name], "Unknown plugin '" + name + "'");
+   my->_active_plugins[name] = my->_available_plugins[name];
+   my->_active_plugins[name]->plugin_set_app(this);
+}
+
+void graphene::app::application::add_available_plugin(std::shared_ptr<graphene::app::abstract_plugin> p)
+{
+   my->_available_plugins[p->plugin_name()] = p;
 }
 
 void application::shutdown_plugins()
 {
-   for( auto& entry : my->_plugins )
+   for( auto& entry : my->_active_plugins )
       entry.second->plugin_shutdown();
    return;
 }
@@ -1049,14 +1093,14 @@ void application::shutdown()
 
 void application::initialize_plugins( const boost::program_options::variables_map& options )
 {
-   for( auto& entry : my->_plugins )
+   for( auto& entry : my->_active_plugins )
       entry.second->plugin_initialize( options );
    return;
 }
 
 void application::startup_plugins()
 {
-   for( auto& entry : my->_plugins )
+   for( auto& entry : my->_active_plugins )
       entry.second->plugin_startup();
    return;
 }

libraries/app/impacted.cpp

@@ -1,315 +0,0 @@
-/*
- * Copyright (c) 2015 Cryptonomex, Inc., and contributors.
- *
- * The MIT License
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include <graphene/chain/protocol/authority.hpp>
-#include <graphene/app/impacted.hpp>
-
-namespace graphene { namespace app {
-
-using namespace fc;
-using namespace graphene::chain;
-
-// TODO:  Review all of these, especially no-ops
-struct get_impacted_account_visitor
-{
-   flat_set<account_id_type>& _impacted;
-   get_impacted_account_visitor( flat_set<account_id_type>& impact ):_impacted(impact) {}
-   typedef void result_type;
-
-   void operator()( const transfer_operation& op )
-   {
-      _impacted.insert( op.to );
-   }
-
-   void operator()( const asset_claim_fees_operation& op ){}
-   void operator()( const limit_order_create_operation& op ) {}
-   void operator()( const limit_order_cancel_operation& op )
-   {
-      _impacted.insert( op.fee_paying_account );
-   }
-   void operator()( const call_order_update_operation& op ) {}
-   void operator()( const fill_order_operation& op )
-   {
-      _impacted.insert( op.account_id );
-   }
-
-   void operator()( const account_create_operation& op )
-   {
-      _impacted.insert( op.registrar );
-      _impacted.insert( op.referrer );
-      add_authority_accounts( _impacted, op.owner );
-      add_authority_accounts( _impacted, op.active );
-   }
-
-   void operator()( const account_update_operation& op )
-   {
-      _impacted.insert( op.account );
-      if( op.owner )
-         add_authority_accounts( _impacted, *(op.owner) );
-      if( op.active )
-         add_authority_accounts( _impacted, *(op.active) );
-   }
-
-   void operator()( const account_whitelist_operation& op )
-   {
-      _impacted.insert( op.account_to_list );
-   }
-
-   void operator()( const account_upgrade_operation& op ) {}
-   void operator()( const account_transfer_operation& op )
-   {
-      _impacted.insert( op.new_owner );
-   }
-
-   void operator()( const asset_create_operation& op ) {}
-   void operator()( const asset_update_operation& op )
-   {
-      if( op.new_issuer )
-         _impacted.insert( *(op.new_issuer) );
-   }
-
-   void operator()( const asset_update_bitasset_operation& op ) {}
-   void operator()( const asset_update_dividend_operation& op ) {}
-   void operator()( const asset_dividend_distribution_operation& op ) 
-   {
-      _impacted.insert( op.account_id );
-   }
-
-   void operator()( const asset_update_feed_producers_operation& op ) {}
-
-   void operator()( const asset_issue_operation& op )
-   {
-      _impacted.insert( op.issue_to_account );
-   }
-
-   void operator()( const asset_reserve_operation& op ) {}
-   void operator()( const asset_fund_fee_pool_operation& op ) {}
-   void operator()( const asset_settle_operation& op ) {}
-   void operator()( const asset_global_settle_operation& op ) {}
-   void operator()( const asset_publish_feed_operation& op ) {}
-   void operator()( const witness_create_operation& op )
-   {
-      _impacted.insert( op.witness_account );
-   }
-   void operator()( const witness_update_operation& op )
-   {
-      _impacted.insert( op.witness_account );
-   }
-
-   void operator()( const proposal_create_operation& op )
-   {
-      vector<authority> other;
-      for( const auto& proposed_op : op.proposed_ops )
-         operation_get_required_authorities( proposed_op.op, _impacted, _impacted, other );
-      for( auto& o : other )
-         add_authority_accounts( _impacted, o );
-   }
-
-   void operator()( const proposal_update_operation& op ) {}
-   void operator()( const proposal_delete_operation& op ) {}
-
-   void operator()( const withdraw_permission_create_operation& op )
-   {
-      _impacted.insert( op.authorized_account );
-   }
-
-   void operator()( const withdraw_permission_update_operation& op )
-   {
-      _impacted.insert( op.authorized_account );
-   }
-
-   void operator()( const withdraw_permission_claim_operation& op )
-   {
-      _impacted.insert( op.withdraw_from_account );
-   }
-
-   void operator()( const withdraw_permission_delete_operation& op )
-   {
-      _impacted.insert( op.authorized_account );
-   }
-
-   void operator()( const committee_member_create_operation& op )
-   {
-      _impacted.insert( op.committee_member_account );
-   }
-   void operator()( const committee_member_update_operation& op )
-   {
-      _impacted.insert( op.committee_member_account );
-   }
-   void operator()( const committee_member_update_global_parameters_operation& op ) {}
-
-   void operator()( const vesting_balance_create_operation& op )
-   {
-      _impacted.insert( op.owner );
-   }
-
-   void operator()( const vesting_balance_withdraw_operation& op ) {}
-   void operator()( const worker_create_operation& op ) {}
-   void operator()( const custom_operation& op ) {}
-   void operator()( const assert_operation& op ) {}
-   void operator()( const balance_claim_operation& op ) {}
-
-   void operator()( const override_transfer_operation& op )
-   {
-      _impacted.insert( op.to );
-      _impacted.insert( op.from );
-      _impacted.insert( op.issuer );
-   }
-
-   void operator()( const transfer_to_blind_operation& op )
-   {
-      _impacted.insert( op.from );
-      for( const auto& out : op.outputs )
-         add_authority_accounts( _impacted, out.owner );
-   }
-
-   void operator()( const blind_transfer_operation& op )
-   {
-      for( const auto& in : op.inputs )
-         add_authority_accounts( _impacted, in.owner );
-      for( const auto& out : op.outputs )
-         add_authority_accounts( _impacted, out.owner );
-   }
-
-   void operator()( const transfer_from_blind_operation& op )
-   {
-      _impacted.insert( op.to );
-      for( const auto& in : op.inputs )
-         add_authority_accounts( _impacted, in.owner );
-   }
-
-   void operator()( const asset_settle_cancel_operation& op )
-   {
-      _impacted.insert( op.account );
-   }
-
-   void operator()( const fba_distribute_operation& op )
-   {
-      _impacted.insert( op.account_id );
-   }
-
-   void operator()( const sport_create_operation& op ) {}
-   void operator()( const sport_update_operation& op ) {}
-   void operator()( const sport_delete_operation& op ) {}
-   void operator()( const event_group_create_operation& op ) {}
-   void operator()( const event_group_update_operation& op ) {}
-   void operator()( const event_group_delete_operation& op ) {}
-   void operator()( const event_create_operation& op ) {}
-   void operator()( const event_update_operation& op ) {}
-   void operator()( const event_update_status_operation& op ) {}
-   void operator()( const betting_market_rules_create_operation& op ) {}
-   void operator()( const betting_market_rules_update_operation& op ) {}
-   void operator()( const betting_market_group_create_operation& op ) {}
-   void operator()( const betting_market_group_update_operation& op ) {}
-   void operator()( const betting_market_create_operation& op ) {}
-   void operator()( const betting_market_update_operation& op ) {}
-   void operator()( const betting_market_group_resolve_operation& op ) {}
-   void operator()( const betting_market_group_cancel_unmatched_bets_operation& op ) {}
-
-   void operator()( const bet_place_operation& op )
-   {
-      _impacted.insert( op.bettor_id );
-   }
-   void operator()( const bet_cancel_operation& op )
-   {
-      _impacted.insert( op.bettor_id );
-   }
-   void operator()( const bet_canceled_operation& op )
-   {
-      _impacted.insert( op.bettor_id );
-   }
-   void operator()( const bet_adjusted_operation& op )
-   {
-      _impacted.insert( op.bettor_id );
-   }
-   void operator()( const bet_matched_operation& op )
-   {
-      _impacted.insert( op.bettor_id );
-   }
-   void operator()( const betting_market_group_resolved_operation& op )
-   {
-      _impacted.insert( op.bettor_id );
-   }
-
-   void operator()( const tournament_create_operation& op )
-   {
-      _impacted.insert( op.creator );
-      _impacted.insert( op.options.whitelist.begin(), op.options.whitelist.end() );
-   }
-   void operator()( const tournament_join_operation& op )
-   {
-      _impacted.insert( op.payer_account_id );
-      _impacted.insert( op.player_account_id );
-   }
-   void operator()( const tournament_leave_operation& op )
-   {
-      //if account canceling registration is not the player, it must be the payer
-      if (op.canceling_account_id != op.player_account_id)
-        _impacted.erase( op.canceling_account_id );
-      _impacted.erase( op.player_account_id );
-   }
-   void operator()( const game_move_operation& op )
-   {
-      _impacted.insert( op.player_account_id );
-   }
-   void operator()( const tournament_payout_operation& op )
-   {
-      _impacted.insert( op.payout_account_id );
-   }
-   void operator()( const affiliate_payout_operation& op )
-   {
-      _impacted.insert( op.affiliate );
-   }
-   void operator()( const affiliate_referral_payout_operation& op ) { }
-   void operator()( const lottery_asset_create_operation& op) { }
-   void operator()( const ticket_purchase_operation& op )
-   {
-      _impacted.insert( op.buyer );
-   }
-   void operator()( const lottery_reward_operation& op ) {
-      _impacted.insert( op.winner );
-   }
-   void operator()( const lottery_end_operation& op ) {
-      for( auto participant : op.participants ) {
-         _impacted.insert(participant.first); 
-      }
-   }
-   void operator()( const sweeps_vesting_claim_operation& op ) { 
-      _impacted.insert( op.account );
-   }
-};
-
-void operation_get_impacted_accounts( const operation& op, flat_set<account_id_type>& result )
-{
-   get_impacted_account_visitor vtor = get_impacted_account_visitor( result );
-   op.visit( vtor );
-}
-
-void transaction_get_impacted_accounts( const transaction& tx, flat_set<account_id_type>& result )
-{
-   for( const auto& op : tx.operations )
-      operation_get_impacted_accounts( op, result );
-}
-
-} }

libraries/app/include/graphene/app/api.hpp

@@ -31,6 +31,8 @@
 #include <graphene/market_history/market_history_plugin.hpp>
 #include <graphene/accounts_list/accounts_list_plugin.hpp>
 
+#include <graphene/elasticsearch/elasticsearch_plugin.hpp>
+
 #include <graphene/debug_witness/debug_api.hpp>
 #include <graphene/affiliate_stats/affiliate_stats_api.hpp>
 #include <graphene/bookie/bookie_api.hpp>

libraries/app/include/graphene/app/application.hpp

@@ -56,14 +56,15 @@ namespace graphene { namespace app {
             auto plug = std::make_shared<PluginType>();
             plug->plugin_set_app(this);
 
-            boost::program_options::options_description plugin_cli_options("Options for plugin " + plug->plugin_name()), plugin_cfg_options;
+            boost::program_options::options_description plugin_cli_options(plug->plugin_name() + " plugin. " + plug->plugin_description() + "\nOptions"), plugin_cfg_options;
+            //boost::program_options::options_description plugin_cli_options("Options for plugin " + plug->plugin_name()), plugin_cfg_options;
             plug->plugin_set_program_options(plugin_cli_options, plugin_cfg_options);
             if( !plugin_cli_options.options().empty() )
                _cli_options.add(plugin_cli_options);
             if( !plugin_cfg_options.options().empty() )
                _cfg_options.add(plugin_cfg_options);
 
-            add_plugin( plug->plugin_name(), plug );
+            add_available_plugin( plug );
             return plug;
          }
          std::shared_ptr<abstract_plugin> get_plugin( const string& name )const;
@@ -88,8 +89,14 @@ namespace graphene { namespace app {
          /// Emitted when syncing finishes (is_finished_syncing will return true)
          boost::signals2::signal<void()> syncing_finished;
 
-      private:
+         void enable_plugin( const string& name );
-         void add_plugin( const string& name, std::shared_ptr<abstract_plugin> p );
+
+         bool is_plugin_enabled(const string& name) const;
+
+         std::shared_ptr<fc::thread> elasticsearch_thread;
+
+   private:
+         void add_available_plugin( std::shared_ptr<abstract_plugin> p );
          std::shared_ptr<detail::application_impl> my;
 
          boost::program_options::options_description _cli_options;

libraries/app/include/graphene/app/plugin.hpp

@@ -35,6 +35,7 @@ class abstract_plugin
    public:
       virtual ~abstract_plugin(){}
       virtual std::string plugin_name()const = 0;
+      virtual std::string plugin_description()const = 0;
 
       /**
        * @brief Perform early startup routines and register plugin indexes, callbacks, etc.
@@ -100,6 +101,7 @@ class plugin : public abstract_plugin
       virtual ~plugin() override;
 
       virtual std::string plugin_name()const override;
+      virtual std::string plugin_description()const override;
       virtual void plugin_initialize( const boost::program_options::variables_map& options ) override;
       virtual void plugin_startup() override;
       virtual void plugin_shutdown() override;

libraries/app/plugin.cpp

@@ -43,6 +43,11 @@ std::string plugin::plugin_name()const
    return "<unknown plugin>";
 }
 
+std::string plugin::plugin_description()const
+{
+   return "<no description>";
+}
+
 void plugin::plugin_initialize( const boost::program_options::variables_map& options )
 {
    return;

libraries/chain/db_block.cpp

@@ -39,6 +39,7 @@
 #include <graphene/chain/protocol/fee_schedule.hpp>
 #include <graphene/chain/exceptions.hpp>
 #include <graphene/chain/evaluator.hpp>
+#include <graphene/chain/witness_schedule_object.hpp>
 #include <fc/crypto/digest.hpp>
 
 #include <fc/smart_ref_impl.hpp>
@@ -197,82 +198,90 @@ bool database::push_block(const signed_block& new_block, uint32_t skip)
 bool database::_push_block(const signed_block& new_block)
 { try {
    uint32_t skip = get_node_properties().skip_flags;
-   if( !(skip&skip_fork_db) )
+   const auto now = fc::time_point::now().sec_since_epoch();
+
+   if( _fork_db.head() && new_block.timestamp.sec_since_epoch() > now - 86400 )
    {
-      /// TODO: if the block is greater than the head block and before the next maitenance interval
       // verify that the block signer is in the current set of active witnesses.
+      shared_ptr<fork_item> prev_block = _fork_db.fetch_block( new_block.previous );
+      GRAPHENE_ASSERT( prev_block, unlinkable_block_exception, "block does not link to known chain" );
+      if( prev_block->scheduled_witnesses && !(skip&(skip_witness_schedule_check|skip_witness_signature)) )
+         verify_signing_witness( new_block, *prev_block );
+   }
+   shared_ptr<fork_item> new_head = _fork_db.push_block(new_block);
 
-      shared_ptr<fork_item> new_head = _fork_db.push_block(new_block);
+   //If the head block from the longest chain does not build off of the current head, we need to switch forks.
-      //If the head block from the longest chain does not build off of the current head, we need to switch forks.
+   if( new_head->data.previous != head_block_id() )
-      if( new_head->data.previous != head_block_id() )
+   {
+      //If the newly pushed block is the same height as head, we get head back in new_head
+      //Only switch forks if new_head is actually higher than head
+      if( new_head->data.block_num() > head_block_num() )
       {
-         //If the newly pushed block is the same height as head, we get head back in new_head
+         wlog( "Switching to fork: ${id}", ("id",new_head->data.id()) );
-         //Only switch forks if new_head is actually higher than head
+         auto branches = _fork_db.fetch_branch_from(new_head->data.id(), head_block_id());
-         if( new_head->data.block_num() > head_block_num() )
+
+         // pop blocks until we hit the forked block
+         while( head_block_id() != branches.second.back()->data.previous )
          {
-            wlog( "Switching to fork: ${id}", ("id",new_head->data.id()) );
+            ilog( "popping block #${n} ${id}", ("n",head_block_num())("id",head_block_id()) );
-            auto branches = _fork_db.fetch_branch_from(new_head->data.id(), head_block_id());
+            pop_block();
-
-            // pop blocks until we hit the forked block
-            while( head_block_id() != branches.second.back()->data.previous )
-            {
-               ilog( "popping block #${n} ${id}", ("n",head_block_num())("id",head_block_id()) );
-               pop_block();
-            }
-
-            // push all blocks on the new fork
-            for( auto ritr = branches.first.rbegin(); ritr != branches.first.rend(); ++ritr )
-            {
-                ilog( "pushing block from fork #${n} ${id}", ("n",(*ritr)->data.block_num())("id",(*ritr)->id) );
-                optional<fc::exception> except;
-                try {
-                   undo_database::session session = _undo_db.start_undo_session();
-                   apply_block( (*ritr)->data, skip );
-                   _block_id_to_block.store( (*ritr)->id, (*ritr)->data );
-                   session.commit();
-                }
-                catch ( const fc::exception& e ) { except = e; }
-                if( except )
-                {
-                   wlog( "exception thrown while switching forks ${e}", ("e",except->to_detail_string() ) );
-                   // remove the rest of branches.first from the fork_db, those blocks are invalid
-                   while( ritr != branches.first.rend() )
-                   {
-                      ilog( "removing block from fork_db #${n} ${id}", ("n",(*ritr)->data.block_num())("id",(*ritr)->id) );
-                      _fork_db.remove( (*ritr)->id );
-                      ++ritr;
-                   }
-                   _fork_db.set_head( branches.second.front() );
-
-                   // pop all blocks from the bad fork
-                   while( head_block_id() != branches.second.back()->data.previous )
-                   {
-                      ilog( "popping block #${n} ${id}", ("n",head_block_num())("id",head_block_id()) );
-                      pop_block();
-                   }
-
-                   ilog( "Switching back to fork: ${id}", ("id",branches.second.front()->data.id()) );
-                   // restore all blocks from the good fork
-                   for( auto ritr2 = branches.second.rbegin(); ritr2 != branches.second.rend(); ++ritr2 )
-                   {
-                      ilog( "pushing block #${n} ${id}", ("n",(*ritr2)->data.block_num())("id",(*ritr2)->id) );
-                      auto session = _undo_db.start_undo_session();
-                      apply_block( (*ritr2)->data, skip );
-                      _block_id_to_block.store( (*ritr2)->id, (*ritr2)->data );
-                      session.commit();
-                   }
-                   throw *except;
-                }
-            }
-            return true;
          }
-         else return false;
+
+         // push all blocks on the new fork
+         for( auto ritr = branches.first.rbegin(); ritr != branches.first.rend(); ++ritr )
+         {
+               ilog( "pushing block from fork #${n} ${id}", ("n",(*ritr)->data.block_num())("id",(*ritr)->id) );
+               optional<fc::exception> except;
+               try {
+                  undo_database::session session = _undo_db.start_undo_session();
+                  apply_block( (*ritr)->data, skip );
+                  update_witnesses( **ritr );
+                  _block_id_to_block.store( (*ritr)->id, (*ritr)->data );
+                  session.commit();
+               }
+               catch ( const fc::exception& e ) { except = e; }
+               if( except )
+               {
+                  wlog( "exception thrown while switching forks ${e}", ("e",except->to_detail_string() ) );
+                  // remove the rest of branches.first from the fork_db, those blocks are invalid
+                  while( ritr != branches.first.rend() )
+                  {
+                     ilog( "removing block from fork_db #${n} ${id}", ("n",(*ritr)->data.block_num())("id",(*ritr)->id) );
+                     _fork_db.remove( (*ritr)->id );
+                     ++ritr;
+                  }
+                  _fork_db.set_head( branches.second.front() );
+
+                  // pop all blocks from the bad fork
+                  while( head_block_id() != branches.second.back()->data.previous )
+                  {
+                     ilog( "popping block #${n} ${id}", ("n",head_block_num())("id",head_block_id()) );
+                     pop_block();
+                  }
+
+                  ilog( "Switching back to fork: ${id}", ("id",branches.second.front()->data.id()) );
+                  // restore all blocks from the good fork
+                  for( auto ritr2 = branches.second.rbegin(); ritr2 != branches.second.rend(); ++ritr2 )
+                  {
+                     ilog( "pushing block #${n} ${id}", ("n",(*ritr2)->data.block_num())("id",(*ritr2)->id) );
+                     auto session = _undo_db.start_undo_session();
+                     apply_block( (*ritr2)->data, skip );
+                     _block_id_to_block.store( (*ritr2)->id, (*ritr2)->data );
+                     session.commit();
+                  }
+                  throw *except;
+               }
+         }
+         return true;
       }
+      else return false;
    }
 
    try {
       auto session = _undo_db.start_undo_session();
       apply_block(new_block, skip);
+      if( new_block.timestamp.sec_since_epoch() > now - 86400 )
+         update_witnesses( *new_head );
       _block_id_to_block.store(new_block.id(), new_block);
       session.commit();
    } catch ( const fc::exception& e ) {
@@ -284,6 +293,73 @@ bool database::_push_block(const signed_block& new_block)
    return false;
 } FC_CAPTURE_AND_RETHROW( (new_block) ) }
 
+void database::verify_signing_witness( const signed_block& new_block, const fork_item& fork_entry )const
+{
+   FC_ASSERT( new_block.timestamp >= fork_entry.next_block_time );
+   uint32_t slot_num = ( new_block.timestamp - fork_entry.next_block_time ).to_seconds() / block_interval();
+   const global_property_object& gpo = get_global_properties();
+
+   if (gpo.parameters.witness_schedule_algorithm == GRAPHENE_WITNESS_SHUFFLED_ALGORITHM)
+   {
+      uint64_t index = ( fork_entry.next_block_aslot + slot_num ) % fork_entry.scheduled_witnesses->size();
+      const auto& scheduled_witness = (*fork_entry.scheduled_witnesses)[index];
+      FC_ASSERT( new_block.witness == scheduled_witness.first, "Witness produced block at wrong time",
+               ("block witness",new_block.witness)("scheduled",scheduled_witness)("slot_num",slot_num) );
+      FC_ASSERT( new_block.validate_signee( scheduled_witness.second ) );
+   }
+   if (gpo.parameters.witness_schedule_algorithm == GRAPHENE_WITNESS_SCHEDULED_ALGORITHM &&
+       slot_num != 0 )
+   {
+      witness_id_type wid;
+      const witness_schedule_object& wso = get_witness_schedule_object();
+      // ask the near scheduler who goes in the given slot
+      bool slot_is_near = wso.scheduler.get_slot(slot_num, wid);
+      if(! slot_is_near)
+      {
+         // if the near scheduler doesn't know, we have to extend it to
+         //   a far scheduler.
+         // n.b. instantiating it is slow, but block gaps long enough to
+         //   need it are likely pretty rare.
+
+         witness_scheduler_rng far_rng(wso.rng_seed.begin(), GRAPHENE_FAR_SCHEDULE_CTR_IV);
+
+         far_future_witness_scheduler far_scheduler =
+            far_future_witness_scheduler(wso.scheduler, far_rng);
+         if(!far_scheduler.get_slot(slot_num, wid))
+         {
+            // no scheduled witness -- somebody set up us the bomb
+            // n.b. this code path is impossible, the present
+            // implementation of far_future_witness_scheduler
+            // returns true unconditionally
+            assert( false );
+         }
+      }
+
+      FC_ASSERT( new_block.witness == wid, "Witness produced block at wrong time",
+               ("block witness",new_block.witness)("scheduled",wid)("slot_num",slot_num) );
+      FC_ASSERT( new_block.validate_signee( wid(*this).signing_key ) );
+   }   
+}
+
+void database::update_witnesses( fork_item& fork_entry )const
+{
+   if( fork_entry.scheduled_witnesses ) return;
+
+   const dynamic_global_property_object& dpo = get_dynamic_global_properties();
+   fork_entry.next_block_aslot = dpo.current_aslot + 1;
+   fork_entry.next_block_time = get_slot_time( 1 );
+
+   const witness_schedule_object& wso = get_witness_schedule_object();
+   fork_entry.scheduled_witnesses = std::make_shared< vector< pair< witness_id_type, public_key_type > > >();
+   fork_entry.scheduled_witnesses->reserve( wso.current_shuffled_witnesses.size() );
+   
+   for( size_t i = 0; i < wso.current_shuffled_witnesses.size(); ++i )
+   {
+      const auto& witness = wso.current_shuffled_witnesses[i](*this);
+      fork_entry.scheduled_witnesses->emplace_back( wso.current_shuffled_witnesses[i], witness.signing_key );
+   }
+}
+
 /**
  * Attempts to push the transaction into the pending queue
  *
@@ -324,7 +400,7 @@ processed_transaction database::_push_transaction( const signed_transaction& trx
    temp_session.merge();
 
    // notify anyone listening to pending transactions
-   on_pending_transaction( trx );
+   notify_on_pending_transaction( trx );
    return processed_trx;
 }
 
@@ -658,7 +734,7 @@ void database::_apply_block( const signed_block& next_block )
       apply_debug_updates();
 
    // notify observers that the block has been applied
-   applied_block( next_block ); //emit
+   notify_applied_block( next_block ); //emit
    _applied_ops.clear();
 
    notify_changed_objects();

libraries/chain/db_notify.cpp

@@ -33,6 +33,14 @@
 #include <graphene/chain/confidential_object.hpp>
 #include <graphene/chain/market_object.hpp>
 #include <graphene/chain/committee_member_object.hpp>
+#include <graphene/chain/exceptions.hpp>
+#include <graphene/chain/witness_object.hpp>
+#include <graphene/chain/proposal_object.hpp>
+#include <graphene/chain/operation_history_object.hpp>
+#include <graphene/chain/vesting_balance_object.hpp>
+#include <graphene/chain/transaction_object.hpp>
+#include <graphene/chain/impacted.hpp>
+
 
 using namespace fc;
 using namespace graphene::chain;
@@ -287,13 +295,13 @@ struct get_impacted_account_visitor
    }
 };
 
-void operation_get_impacted_accounts( const operation& op, flat_set<account_id_type>& result )
+void graphene::chain::operation_get_impacted_accounts( const operation& op, flat_set<account_id_type>& result )
 {
   get_impacted_account_visitor vtor = get_impacted_account_visitor( result );
   op.visit( vtor );
 }
 
-void transaction_get_impacted_accounts( const transaction& tx, flat_set<account_id_type>& result )
+void graphene::chain::transaction_get_impacted_accounts( const transaction& tx, flat_set<account_id_type>& result )
 {
   for( const auto& op : tx.operations )
     operation_get_impacted_accounts( op, result );
@@ -433,6 +441,16 @@ void get_relevant_accounts( const object* obj, flat_set<account_id_type>& accoun
 
 namespace graphene { namespace chain {
 
+void database::notify_applied_block( const signed_block& block )
+{
+   GRAPHENE_TRY_NOTIFY( applied_block, block )
+}
+
+void database::notify_on_pending_transaction( const signed_transaction& tx )
+{
+   GRAPHENE_TRY_NOTIFY( on_pending_transaction, tx )
+}
+
 void database::notify_changed_objects()
 { try {
    if( _undo_db.enabled() ) 
@@ -452,7 +470,7 @@ void database::notify_changed_objects()
             get_relevant_accounts(obj, new_accounts_impacted);
         }
 
-        new_objects(new_ids, new_accounts_impacted);
+        GRAPHENE_TRY_NOTIFY( new_objects, new_ids, new_accounts_impacted)
       }
 
       // Changed
@@ -466,7 +484,7 @@ void database::notify_changed_objects()
           get_relevant_accounts(item.second.get(), changed_accounts_impacted);
         }
 
-        changed_objects(changed_ids, changed_accounts_impacted);
+        GRAPHENE_TRY_NOTIFY( changed_objects, changed_ids, changed_accounts_impacted)
       }
 
       // Removed
@@ -483,7 +501,7 @@ void database::notify_changed_objects()
           get_relevant_accounts(obj, removed_accounts_impacted);
         }
 
-        removed_objects(removed_ids, removed, removed_accounts_impacted);
+        GRAPHENE_TRY_NOTIFY( removed_objects, removed_ids, removed, removed_accounts_impacted)
       }
    }
 } FC_CAPTURE_AND_LOG( (0) ) }

libraries/chain/db_witness_schedule.cpp

@@ -45,7 +45,7 @@ witness_id_type database::get_scheduled_witness( uint32_t slot_num )const
    if (gpo.parameters.witness_schedule_algorithm == GRAPHENE_WITNESS_SCHEDULED_ALGORITHM &&
        slot_num != 0 )
    {
-       const witness_schedule_object& wso = get_witness_schedule_object();;
+       const witness_schedule_object& wso = get_witness_schedule_object();
        // ask the near scheduler who goes in the given slot
        bool slot_is_near = wso.scheduler.get_slot(slot_num-1, wid);
        if(! slot_is_near)

libraries/chain/include/graphene/chain/database.hpp

@@ -469,6 +469,8 @@ namespace graphene { namespace chain {
    protected:
          //Mark pop_undo() as protected -- we do not want outside calling pop_undo(); it should call pop_block() instead
          void pop_undo() { object_database::pop_undo(); }
+         void notify_applied_block( const signed_block& block );
+         void notify_on_pending_transaction( const signed_transaction& tx );
          void notify_changed_objects();
 
       private:
@@ -494,6 +496,8 @@ namespace graphene { namespace chain {
 
          const witness_object& validate_block_header( uint32_t skip, const signed_block& next_block )const;
          const witness_object& _validate_block_header( const signed_block& next_block )const;
+         void verify_signing_witness( const signed_block& new_block, const fork_item& fork_entry )const;
+         void update_witnesses( fork_item& fork_entry )const;
          void create_block_summary(const signed_block& next_block);
 
          //////////////////// db_witness_schedule.cpp ////////////////////

libraries/chain/include/graphene/chain/exceptions.hpp

@@ -65,6 +65,21 @@
       msg                                                             \
       )
 
+#define GRAPHENE_TRY_NOTIFY( signal, ... )                                    \
+   try                                                                        \
+   {                                                                          \
+      signal( __VA_ARGS__ );                                                  \
+   }                                                                          \
+   catch( const graphene::chain::plugin_exception& e )                        \
+   {                                                                          \
+      elog( "Caught plugin exception: ${e}", ("e", e.to_detail_string() ) );  \
+      throw;                                                                  \
+   }                                                                          \
+   catch( ... )                                                               \
+   {                                                                          \
+      wlog( "Caught unexpected exception in plugin" );                        \
+   }
+
 namespace graphene { namespace chain {
 
    FC_DECLARE_EXCEPTION( chain_exception, 3000000, "blockchain exception" )
@@ -77,6 +92,7 @@ namespace graphene { namespace chain {
    FC_DECLARE_DERIVED_EXCEPTION( undo_database_exception,           graphene::chain::chain_exception, 3070000, "undo database exception" )
    FC_DECLARE_DERIVED_EXCEPTION( unlinkable_block_exception,        graphene::chain::chain_exception, 3080000, "unlinkable block" )
    FC_DECLARE_DERIVED_EXCEPTION( black_swan_exception,              graphene::chain::chain_exception, 3090000, "black swan" )
+   FC_DECLARE_DERIVED_EXCEPTION( plugin_exception,                  graphene::chain::chain_exception, 3100000, "plugin exception" )
 
    FC_DECLARE_DERIVED_EXCEPTION( tx_missing_active_auth,            graphene::chain::transaction_exception, 3030001, "missing required active authority" )
    FC_DECLARE_DERIVED_EXCEPTION( tx_missing_owner_auth,             graphene::chain::transaction_exception, 3030002, "missing required owner authority" )

libraries/chain/include/graphene/chain/fork_database.hpp

@@ -51,6 +51,11 @@ namespace graphene { namespace chain {
       bool                  invalid = false;
       block_id_type         id;
       signed_block          data;
+
+      // contains witness block signing keys scheduled *after* the block has been applied
+      shared_ptr< vector< pair< witness_id_type, public_key_type > > > scheduled_witnesses;
+      uint64_t                                                         next_block_aslot = 0;
+      fc::time_point_sec                                               next_block_time;
    };
    typedef shared_ptr<fork_item> item_ptr;
 

libraries/chain/include/graphene/chain/impacted.hpp

@@ -28,7 +28,7 @@
 #include <graphene/chain/protocol/transaction.hpp>
 #include <graphene/chain/protocol/types.hpp>
 
-namespace graphene { namespace app {
+namespace graphene { namespace chain {
 
 void operation_get_impacted_accounts(
    const graphene::chain::operation& op,
@@ -39,4 +39,4 @@ void transaction_get_impacted_accounts(
    fc::flat_set<graphene::chain::account_id_type>& result
    );
 
-} } // graphene::app
+} } // graphene::app

libraries/chain/include/graphene/chain/operation_history_object.hpp

@@ -102,6 +102,16 @@ namespace graphene { namespace chain {
 struct by_seq;
 struct by_op;
 struct by_opid;
+
+typedef multi_index_container<
+   operation_history_object,
+   indexed_by<
+      ordered_unique< tag<by_id>, member< object, object_id_type, &object::id > >
+   >
+> operation_history_multi_index_type;
+
+typedef generic_index<operation_history_object, operation_history_multi_index_type> operation_history_index;
+
 typedef multi_index_container<
    account_transaction_history_object,
    indexed_by<

libraries/chain/include/graphene/chain/protocol/chain_parameters.hpp

@@ -35,12 +35,11 @@ namespace graphene { namespace chain { struct fee_schedule; } }
 namespace graphene { namespace chain {
    struct parameter_extension
    {
-      optional< bet_multiplier_type > min_bet_multiplier                = GRAPHENE_DEFAULT_MIN_BET_MULTIPLIER;
+      optional< bet_multiplier_type > min_bet_multiplier;
-      optional< bet_multiplier_type > max_bet_multiplier                = GRAPHENE_DEFAULT_MAX_BET_MULTIPLIER;
+      optional< bet_multiplier_type > max_bet_multiplier;
-      optional< uint16_t >            betting_rake_fee_percentage       = GRAPHENE_DEFAULT_RAKE_FEE_PERCENTAGE;
+      optional< uint16_t >            betting_rake_fee_percentage;
-      optional< flat_map<bet_multiplier_type, bet_multiplier_type> > 
+      optional< flat_map<bet_multiplier_type, bet_multiplier_type> > permitted_betting_odds_increments;
-      permitted_betting_odds_increments                                 = flat_map<bet_multiplier_type, bet_multiplier_type>(GRAPHENE_DEFAULT_PERMITTED_BETTING_ODDS_INCREMENTS);
+      optional< uint16_t >            live_betting_delay_time;
-      optional< uint16_t >            live_betting_delay_time           = GRAPHENE_DEFAULT_LIVE_BETTING_DELAY_TIME;
       optional< uint16_t >            sweeps_distribution_percentage    = SWEEPS_DEFAULT_DISTRIBUTION_PERCENTAGE;
       optional< asset_id_type >       sweeps_distribution_asset         = SWEEPS_DEFAULT_DISTRIBUTION_ASSET;
       optional< account_id_type >     sweeps_vesting_accumulator_account= SWEEPS_ACCUMULATOR_ACCOUNT;
@@ -148,6 +147,7 @@ FC_REFLECT( graphene::chain::parameter_extension,
    (min_bet_multiplier)
    (max_bet_multiplier)
    (betting_rake_fee_percentage)
+   (permitted_betting_odds_increments)
    (live_betting_delay_time)
    (sweeps_distribution_percentage)
    (sweeps_distribution_asset)

libraries/fc

@@ -1 +1 @@
-Subproject commit 31e289c53d3625afea87c54edb6d97c3bca4c626
+Subproject commit a76b9ff81c6887ebe1dc9fa03ef15e1433029c65

libraries/net/include/graphene/net/peer_connection.hpp

@@ -261,13 +261,13 @@ namespace graphene { namespace net
       fc::future<void> accept_or_connect_task_done;
 
       firewall_check_state_data *firewall_check_state = nullptr;
-#ifndef NDEBUG
+
     private:
+#ifndef NDEBUG
       fc::thread* _thread = nullptr;
       unsigned _send_message_queue_tasks_running = 0; // temporary debugging
 #endif
       bool _currently_handling_message = false; // true while we're in the middle of handling a message from the remote system
-    private:
       peer_connection(peer_connection_delegate* delegate);
       void destroy();
     public:

libraries/net/message_oriented_connection.cpp

@@ -62,7 +62,8 @@ namespace graphene { namespace net {
       fc::time_point _last_message_received_time;
       fc::time_point _last_message_sent_time;
 
-      bool _send_message_in_progress;
+      std::atomic_bool _send_message_in_progress;
+      std::atomic_bool _read_loop_in_progress;
 #ifndef NDEBUG
       fc::thread* _thread;
 #endif
@@ -98,7 +99,8 @@ namespace graphene { namespace net {
       _delegate(delegate),
       _bytes_received(0),
       _bytes_sent(0),
-      _send_message_in_progress(false)
+      _send_message_in_progress(false),
+      _read_loop_in_progress(false)
 #ifndef NDEBUG
       ,_thread(&fc::thread::current())
 #endif
@@ -138,6 +140,21 @@ namespace graphene { namespace net {
       _sock.bind(local_endpoint);
     }
 
+    class no_parallel_execution_guard final
+    {
+      std::atomic_bool* _flag;
+    public:
+      explicit no_parallel_execution_guard(std::atomic_bool* flag) : _flag(flag)
+      {
+          bool expected = false;
+          FC_ASSERT( flag->compare_exchange_strong( expected, true ), "Only one thread at time can visit it");
+      }
+      ~no_parallel_execution_guard()
+      {
+          *_flag = false;
+      }
+    };
+
     void message_oriented_connection_impl::read_loop()
     {
       VERIFY_CORRECT_THREAD();
@@ -145,6 +162,7 @@ namespace graphene { namespace net {
       const int LEFTOVER = BUFFER_SIZE - sizeof(message_header);
       static_assert(BUFFER_SIZE >= sizeof(message_header), "insufficient buffer");
 
+      no_parallel_execution_guard guard( &_read_loop_in_progress );
       _connected_time = fc::time_point::now();
 
       fc::oexception exception_to_rethrow;
@@ -241,17 +259,7 @@ namespace graphene { namespace net {
       } send_message_scope_logger(remote_endpoint);
 #endif
 #endif
-      struct verify_no_send_in_progress {
+      no_parallel_execution_guard guard( &_send_message_in_progress );
-        bool& var;
-        verify_no_send_in_progress(bool& var) : var(var)
-        {
-          if (var)
-            elog("Error: two tasks are calling message_oriented_connection::send_message() at the same time");
-          assert(!var);
-          var = true;
-        }
-        ~verify_no_send_in_progress() { var = false; }
-      } _verify_no_send_in_progress(_send_message_in_progress);
 
       try
       {

libraries/plugins/CMakeLists.txt

@@ -2,6 +2,7 @@ add_subdirectory( witness )
 add_subdirectory( account_history )
 add_subdirectory( accounts_list )
 add_subdirectory( affiliate_stats )
+add_subdirectory( elasticsearch )
 add_subdirectory( market_history )
 add_subdirectory( delayed_node )
 add_subdirectory( bookie )
@@ -9,3 +10,4 @@ add_subdirectory( generate_genesis )
 add_subdirectory( generate_uia_sharedrop_genesis )
 add_subdirectory( debug_witness )
 add_subdirectory( snapshot )
+add_subdirectory( es_objects )

libraries/plugins/account_history/account_history_plugin.cpp

@@ -24,7 +24,7 @@
 
 #include <graphene/account_history/account_history_plugin.hpp>
 
-#include <graphene/app/impacted.hpp>
+#include <graphene/chain/impacted.hpp>
 
 #include <graphene/chain/account_evaluator.hpp>
 #include <graphene/chain/account_object.hpp>
@@ -128,8 +128,8 @@ void account_history_plugin_impl::update_account_histories( const signed_block&
       if( op.op.which() == operation::tag< account_create_operation >::value )
          impacted.insert( op.result.get<object_id_type>() );
       else
-         graphene::app::operation_get_impacted_accounts( op.op, impacted );
+         graphene::chain::operation_get_impacted_accounts( op.op, impacted );
-      if( op.op.which() == operation::tag< lottery_end_operation >::value ) 
+      if( op.op.which() == operation::tag< lottery_end_operation >::value )
       {
          auto lop = op.op.get< lottery_end_operation >();
          auto asset_object = lop.lottery( db );
@@ -137,6 +137,7 @@ void account_history_plugin_impl::update_account_histories( const signed_block&
          for( auto benefactor : asset_object.lottery_options->benefactors )
             impacted.insert( benefactor.id );
       }
+
       for( auto& a : other )
          for( auto& item : a.account_auths )
             impacted.insert( item.first );

libraries/plugins/accounts_list/accounts_list_plugin.cpp

@@ -24,7 +24,7 @@
 
 #include <graphene/accounts_list/accounts_list_plugin.hpp>
 
-#include <graphene/app/impacted.hpp>
+#include <graphene/chain/impacted.hpp>
 
 #include <graphene/chain/account_evaluator.hpp>
 #include <graphene/chain/account_object.hpp>

libraries/plugins/affiliate_stats/affiliate_stats_plugin.cpp

@@ -25,7 +25,7 @@
 #include <graphene/affiliate_stats/affiliate_stats_plugin.hpp>
 #include <graphene/affiliate_stats/affiliate_stats_objects.hpp>
 
-#include <graphene/app/impacted.hpp>
+#include <graphene/chain/impacted.hpp>
 
 #include <graphene/chain/account_evaluator.hpp>
 #include <graphene/chain/account_object.hpp>

libraries/plugins/bookie/bookie_plugin.cpp

@@ -24,7 +24,7 @@
 #include <graphene/bookie/bookie_plugin.hpp>
 #include <graphene/bookie/bookie_objects.hpp>
 
-#include <graphene/app/impacted.hpp>
+#include <graphene/chain/impacted.hpp>
 
 #include <graphene/chain/account_evaluator.hpp>
 #include <graphene/chain/account_object.hpp>
@@ -370,8 +370,8 @@ void bookie_plugin_impl::on_block_applied( const signed_block& )
          assert(bet_iter != persistent_bets_by_bet_id.end());
          if (bet_iter != persistent_bets_by_bet_id.end())
          {
-            ilog("Adding bet_canceled_operation ${canceled_id} to bet ${bet_id}'s associated operations", 
+            // ilog("Adding bet_canceled_operation ${canceled_id} to bet ${bet_id}'s associated operations", 
-                 ("canceled_id", op.id)("bet_id", bet_canceled_op.bet_id));
+            //     ("canceled_id", op.id)("bet_id", bet_canceled_op.bet_id));
             if (is_operation_history_object_stored(op.id))
                db.modify(*bet_iter, [&]( persistent_bet_object& obj ) {
                   obj.associated_operations.emplace_back(op.id);
@@ -386,8 +386,8 @@ void bookie_plugin_impl::on_block_applied( const signed_block& )
          assert(bet_iter != persistent_bets_by_bet_id.end());
          if (bet_iter != persistent_bets_by_bet_id.end())
          {
-            ilog("Adding bet_adjusted_operation ${adjusted_id} to bet ${bet_id}'s associated operations", 
+            // ilog("Adding bet_adjusted_operation ${adjusted_id} to bet ${bet_id}'s associated operations", 
-                 ("adjusted_id", op.id)("bet_id", bet_adjusted_op.bet_id));
+            //     ("adjusted_id", op.id)("bet_id", bet_adjusted_op.bet_id));
             if (is_operation_history_object_stored(op.id))
                db.modify(*bet_iter, [&]( persistent_bet_object& obj ) {
                   obj.associated_operations.emplace_back(op.id);

libraries/plugins/delayed_node/delayed_node_plugin.cpp

@@ -58,7 +58,7 @@ delayed_node_plugin::~delayed_node_plugin()
 void delayed_node_plugin::plugin_set_program_options(bpo::options_description& cli, bpo::options_description& cfg)
 {
    cli.add_options()
-         ("trusted-node", boost::program_options::value<std::string>()->required(), "RPC endpoint of a trusted validating node (required)")
+         ("trusted-node", boost::program_options::value<std::string>(), "RPC endpoint of a trusted validating node (required)")
          ;
    cfg.add(cli);
 }
@@ -74,6 +74,7 @@ void delayed_node_plugin::connect()
 
 void delayed_node_plugin::plugin_initialize(const boost::program_options::variables_map& options)
 {
+   FC_ASSERT(options.count("trusted-node") > 0);
    my->remote_endpoint = "ws://" + options.at("trusted-node").as<std::string>();
 }
 

libraries/plugins/elasticsearch/CMakeLists.txt

@@ -0,0 +1,23 @@
+file(GLOB HEADERS "include/graphene/elasticsearch/*.hpp")
+
+add_library( graphene_elasticsearch
+        elasticsearch_plugin.cpp
+           )
+
+target_link_libraries( graphene_elasticsearch graphene_chain graphene_app curl )
+target_include_directories( graphene_elasticsearch
+                            PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" )
+
+if(MSVC)
+  set_source_files_properties(elasticsearch_plugin.cpp PROPERTIES COMPILE_FLAGS "/bigobj" )
+endif(MSVC)
+
+install( TARGETS
+   graphene_elasticsearch
+
+   RUNTIME DESTINATION bin
+   LIBRARY DESTINATION lib
+   ARCHIVE DESTINATION lib
+)
+INSTALL( FILES ${HEADERS} DESTINATION "include/graphene/elasticsearch" )
+

libraries/plugins/elasticsearch/elasticsearch_plugin.cpp

@@ -0,0 +1,622 @@
+/*
+ * Copyright (c) 2017 Cryptonomex, Inc., and contributors.
+ *
+ * The MIT License
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <graphene/elasticsearch/elasticsearch_plugin.hpp>
+#include <graphene/chain/impacted.hpp>
+#include <graphene/chain/account_evaluator.hpp>
+#include <fc/smart_ref_impl.hpp>
+#include <curl/curl.h>
+
+namespace graphene { namespace elasticsearch {
+
+namespace detail
+{
+
+class elasticsearch_plugin_impl
+{
+   public:
+      elasticsearch_plugin_impl(elasticsearch_plugin& _plugin)
+         : _self( _plugin )
+      {  curl = curl_easy_init(); }
+      virtual ~elasticsearch_plugin_impl();
+
+      bool update_account_histories( const signed_block& b );
+
+      graphene::chain::database& database()
+      {
+         return _self.database();
+      }
+
+      elasticsearch_plugin& _self;
+      primary_index< operation_history_index >* _oho_index;
+
+      std::string _elasticsearch_node_url = "http://localhost:9200/";
+      uint32_t _elasticsearch_bulk_replay = 10000;
+      uint32_t _elasticsearch_bulk_sync = 100;
+      bool _elasticsearch_visitor = false;
+      std::string _elasticsearch_basic_auth = "";
+      std::string _elasticsearch_index_prefix = "peerplays-";
+      bool _elasticsearch_operation_object = false;
+      uint32_t _elasticsearch_start_es_after_block = 0;
+      bool _elasticsearch_operation_string = true;
+      mode _elasticsearch_mode = mode::only_save;
+      CURL *curl; // curl handler
+      vector <string> bulk_lines; //  vector of op lines
+      vector<std::string> prepare;
+
+      graphene::utilities::ES es;
+      uint32_t limit_documents;
+      int16_t op_type;
+      operation_history_struct os;
+      block_struct bs;
+      visitor_struct vs;
+      bulk_struct bulk_line_struct;
+      std::string bulk_line;
+      std::string index_name;
+      bool is_sync = false;
+      fc::time_point last_sync;
+   private:
+      bool add_elasticsearch( const account_id_type account_id, const optional<operation_history_object>& oho, const uint32_t block_number );
+      const account_transaction_history_object& addNewEntry(const account_statistics_object& stats_obj,
+                                                            const account_id_type account_id,
+                                                            const optional <operation_history_object>& oho);
+      const account_statistics_object& getStatsObject(const account_id_type account_id);
+      void growStats(const account_statistics_object& stats_obj, const account_transaction_history_object& ath);
+      void getOperationType(const optional <operation_history_object>& oho);
+      void doOperationHistory(const optional <operation_history_object>& oho);
+      void doBlock(const optional <operation_history_object>& oho, const signed_block& b);
+      void doVisitor(const optional <operation_history_object>& oho);
+      void checkState(const fc::time_point_sec& block_time);
+      void cleanObjects(const account_transaction_history_object& ath, account_id_type account_id);
+      void createBulkLine(const account_transaction_history_object& ath);
+      void prepareBulk(const account_transaction_history_id_type& ath_id);
+      void populateESstruct();
+};
+
+elasticsearch_plugin_impl::~elasticsearch_plugin_impl()
+{
+   if (curl) {
+      curl_easy_cleanup(curl);
+      curl = nullptr;
+   }
+   return;
+}
+
+bool elasticsearch_plugin_impl::update_account_histories( const signed_block& b )
+{
+   checkState(b.timestamp);
+   index_name = graphene::utilities::generateIndexName(b.timestamp, _elasticsearch_index_prefix);
+
+   graphene::chain::database& db = database();
+   const vector<optional< operation_history_object > >& hist = db.get_applied_operations();
+   bool is_first = true;
+   auto skip_oho_id = [&is_first,&db,this]() {
+      if( is_first && db._undo_db.enabled() ) // this ensures that the current id is rolled back on undo
+      {
+         db.remove( db.create<operation_history_object>( []( operation_history_object& obj) {} ) );
+         is_first = false;
+      }
+      else
+         _oho_index->use_next_id();
+   };
+   for( const optional< operation_history_object >& o_op : hist ) {
+      optional <operation_history_object> oho;
+
+      auto create_oho = [&]() {
+         is_first = false;
+         return optional<operation_history_object>(
+               db.create<operation_history_object>([&](operation_history_object &h) {
+                  if (o_op.valid())
+                  {
+                     h.op           = o_op->op;
+                     h.result       = o_op->result;
+                     h.block_num    = o_op->block_num;
+                     h.trx_in_block = o_op->trx_in_block;
+                     h.op_in_trx    = o_op->op_in_trx;
+                     h.virtual_op   = o_op->virtual_op;
+                  }
+               }));
+      };
+
+      if( !o_op.valid() ) {
+         skip_oho_id();
+         continue;
+      }
+      oho = create_oho();
+
+      // populate what we can before impacted loop
+      getOperationType(oho);
+      doOperationHistory(oho);
+      doBlock(oho, b);
+      if(_elasticsearch_visitor)
+         doVisitor(oho);
+
+      const operation_history_object& op = *o_op;
+
+      // get the set of accounts this operation applies to
+      flat_set<account_id_type> impacted;
+      vector<authority> other;
+      operation_get_required_authorities( op.op, impacted, impacted, other ); // fee_payer is added here
+
+      if( op.op.which() == operation::tag< account_create_operation >::value )
+         impacted.insert( op.result.get<object_id_type>() );
+      else
+         graphene::chain::operation_get_impacted_accounts( op.op, impacted );
+
+      for( auto& a : other )
+         for( auto& item : a.account_auths )
+            impacted.insert( item.first );
+
+      for( auto& account_id : impacted )
+      {
+         if(!add_elasticsearch( account_id, oho, b.block_num() ))
+            return false;
+      }
+   }
+   // we send bulk at end of block when we are in sync for better real time client experience
+   if(is_sync)
+   {
+      populateESstruct();
+      if(es.bulk_lines.size() > 0)
+      {
+         prepare.clear();
+         if(!graphene::utilities::SendBulk(es))
+            return false;
+         else
+            bulk_lines.clear();
+      }
+   }
+
+   return true;
+}
+
+void elasticsearch_plugin_impl::checkState(const fc::time_point_sec& block_time)
+{
+   fc::time_point current_time(fc::time_point::now());
+   if(((current_time - block_time) < fc::seconds(30)) || (current_time - last_sync > fc::seconds(60)))
+   {
+      limit_documents = _elasticsearch_bulk_sync;
+      is_sync = true;
+      last_sync = current_time;
+   }
+   else
+   {
+      limit_documents = _elasticsearch_bulk_replay;
+      is_sync = false;
+   }
+}
+
+void elasticsearch_plugin_impl::getOperationType(const optional <operation_history_object>& oho)
+{
+   if (!oho->id.is_null())
+      op_type = oho->op.which();
+}
+
+void elasticsearch_plugin_impl::doOperationHistory(const optional <operation_history_object>& oho)
+{
+   os.trx_in_block = oho->trx_in_block;
+   os.op_in_trx = oho->op_in_trx;
+   os.operation_result = fc::json::to_string(oho->result);
+   os.virtual_op = oho->virtual_op;
+
+   if(_elasticsearch_operation_object) {
+      oho->op.visit(fc::from_static_variant(os.op_object, FC_PACK_MAX_DEPTH));
+      adaptor_struct adaptor;
+      os.op_object = adaptor.adapt(os.op_object.get_object());
+   }
+   if(_elasticsearch_operation_string)
+      os.op = fc::json::to_string(oho->op);
+}
+
+void elasticsearch_plugin_impl::doBlock(const optional <operation_history_object>& oho, const signed_block& b)
+{
+   std::string trx_id = "";
+   if(oho->trx_in_block < b.transactions.size())
+      trx_id = b.transactions[oho->trx_in_block].id().str();
+   bs.block_num = b.block_num();
+   bs.block_time = b.timestamp;
+   bs.trx_id = trx_id;
+}
+
+void elasticsearch_plugin_impl::doVisitor(const optional <operation_history_object>& oho)
+{
+   operation_visitor o_v;
+   oho->op.visit(o_v);
+
+   vs.fee_data.asset = o_v.fee_asset;
+   vs.fee_data.amount = o_v.fee_amount;
+
+   vs.transfer_data.asset = o_v.transfer_asset_id;
+   vs.transfer_data.amount = o_v.transfer_amount;
+   vs.transfer_data.from = o_v.transfer_from;
+   vs.transfer_data.to = o_v.transfer_to;
+
+   vs.fill_data.order_id = o_v.fill_order_id;
+   vs.fill_data.account_id = o_v.fill_account_id;
+   vs.fill_data.pays_asset_id = o_v.fill_pays_asset_id;
+   vs.fill_data.pays_amount = o_v.fill_pays_amount;
+   vs.fill_data.receives_asset_id = o_v.fill_receives_asset_id;
+   vs.fill_data.receives_amount = o_v.fill_receives_amount;
+   //vs.fill_data.fill_price = o_v.fill_fill_price;
+   //vs.fill_data.is_maker = o_v.fill_is_maker;
+}
+
+bool elasticsearch_plugin_impl::add_elasticsearch( const account_id_type account_id,
+                                                   const optional <operation_history_object>& oho,
+                                                   const uint32_t block_number)
+{
+   const auto &stats_obj = getStatsObject(account_id);
+   const auto &ath = addNewEntry(stats_obj, account_id, oho);
+   growStats(stats_obj, ath);
+   if(block_number > _elasticsearch_start_es_after_block)  {
+      createBulkLine(ath);
+      prepareBulk(ath.id);
+   }
+   cleanObjects(ath, account_id);
+
+   if (curl && bulk_lines.size() >= limit_documents) { // we are in bulk time, ready to add data to elasticsearech
+      prepare.clear();
+      populateESstruct();
+      if(!graphene::utilities::SendBulk(es))
+         return false;
+      else
+         bulk_lines.clear();
+   }
+
+   return true;
+}
+
+const account_statistics_object& elasticsearch_plugin_impl::getStatsObject(const account_id_type account_id)
+{
+   graphene::chain::database& db = database();
+   const auto &acct = db.get<account_object>(account_id);
+   return acct.statistics(db);
+}
+
+const account_transaction_history_object& elasticsearch_plugin_impl::addNewEntry(const account_statistics_object& stats_obj,
+                                                                                 const account_id_type account_id,
+                                                                                 const optional <operation_history_object>& oho)
+{
+   graphene::chain::database& db = database();
+   const auto &ath = db.create<account_transaction_history_object>([&](account_transaction_history_object &obj) {
+      obj.operation_id = oho->id;
+      obj.account = account_id;
+      obj.sequence = stats_obj.total_ops + 1;
+      obj.next = stats_obj.most_recent_op;
+   });
+
+   return ath;
+}
+
+void elasticsearch_plugin_impl::growStats(const account_statistics_object& stats_obj,
+                                          const account_transaction_history_object& ath)
+{
+   graphene::chain::database& db = database();
+   db.modify(stats_obj, [&](account_statistics_object &obj) {
+      obj.most_recent_op = ath.id;
+      obj.total_ops = ath.sequence;
+   });
+}
+
+void elasticsearch_plugin_impl::createBulkLine(const account_transaction_history_object& ath)
+{
+   bulk_line_struct.account_history = ath;
+   bulk_line_struct.operation_history = os;
+   bulk_line_struct.operation_type = op_type;
+   bulk_line_struct.operation_id_num = ath.operation_id.instance.value;
+   bulk_line_struct.block_data = bs;
+   if(_elasticsearch_visitor)
+      bulk_line_struct.additional_data = vs;
+   bulk_line = fc::json::to_string(bulk_line_struct);
+}
+
+void elasticsearch_plugin_impl::prepareBulk(const account_transaction_history_id_type& ath_id)
+{
+   const std::string _id = fc::json::to_string(ath_id);
+   fc::mutable_variant_object bulk_header;
+   bulk_header["_index"] = index_name;
+   bulk_header["_type"] = "data";
+   bulk_header["_id"] = fc::to_string(ath_id.space_id) + "." + fc::to_string(ath_id.type_id) + "." + ath_id.instance;
+   prepare = graphene::utilities::createBulk(bulk_header, bulk_line);
+   bulk_lines.insert(bulk_lines.end(), prepare.begin(), prepare.end());
+}
+
+void elasticsearch_plugin_impl::cleanObjects(const account_transaction_history_object& ath, account_id_type account_id)
+{
+   graphene::chain::database& db = database();
+   // remove everything except current object from ath
+   const auto &his_idx = db.get_index_type<account_transaction_history_index>();
+   const auto &by_seq_idx = his_idx.indices().get<by_seq>();
+   auto itr = by_seq_idx.lower_bound(boost::make_tuple(account_id, 0));
+   if (itr != by_seq_idx.end() && itr->account == account_id && itr->id != ath.id) {
+      // if found, remove the entry
+      const auto remove_op_id = itr->operation_id;
+      const auto itr_remove = itr;
+      ++itr;
+      db.remove( *itr_remove );
+      // modify previous node's next pointer
+      // this should be always true, but just have a check here
+      if( itr != by_seq_idx.end() && itr->account == account_id )
+      {
+         db.modify( *itr, [&]( account_transaction_history_object& obj ){
+            obj.next = account_transaction_history_id_type();
+         });
+      }
+      // do the same on oho
+      const auto &by_opid_idx = his_idx.indices().get<by_opid>();
+      if (by_opid_idx.find(remove_op_id) == by_opid_idx.end()) {
+         db.remove(remove_op_id(db));
+      }
+   }
+}
+
+void elasticsearch_plugin_impl::populateESstruct()
+{
+   es.curl = curl;
+   es.bulk_lines = bulk_lines;
+   es.elasticsearch_url = _elasticsearch_node_url;
+   es.auth = _elasticsearch_basic_auth;
+}
+
+} // end namespace detail
+
+elasticsearch_plugin::elasticsearch_plugin() :
+   my( new detail::elasticsearch_plugin_impl(*this) )
+{
+}
+
+elasticsearch_plugin::~elasticsearch_plugin()
+{
+}
+
+std::string elasticsearch_plugin::plugin_name()const
+{
+   return "elasticsearch";
+}
+std::string elasticsearch_plugin::plugin_description()const
+{
+   return "Stores account history data in elasticsearch database(EXPERIMENTAL).";
+}
+
+void elasticsearch_plugin::plugin_set_program_options(
+   boost::program_options::options_description& cli,
+   boost::program_options::options_description& cfg
+   )
+{
+   cli.add_options()
+         ("elasticsearch-node-url", boost::program_options::value<std::string>(),
+               "Elastic Search database node url(http://localhost:9200/)")
+         ("elasticsearch-bulk-replay", boost::program_options::value<uint32_t>(),
+               "Number of bulk documents to index on replay(10000)")
+         ("elasticsearch-bulk-sync", boost::program_options::value<uint32_t>(),
+               "Number of bulk documents to index on a syncronied chain(100)")
+         ("elasticsearch-visitor", boost::program_options::value<bool>(),
+               "Use visitor to index additional data(slows down the replay(false))")
+         ("elasticsearch-basic-auth", boost::program_options::value<std::string>(),
+               "Pass basic auth to elasticsearch database('')")
+         ("elasticsearch-index-prefix", boost::program_options::value<std::string>(),
+               "Add a prefix to the index(peerplays-)")
+         ("elasticsearch-operation-object", boost::program_options::value<bool>(),
+               "Save operation as object(false)")
+         ("elasticsearch-start-es-after-block", boost::program_options::value<uint32_t>(),
+               "Start doing ES job after block(0)")
+         ("elasticsearch-operation-string", boost::program_options::value<bool>(),
+               "Save operation as string. Needed to serve history api calls(true)")
+         ("elasticsearch-mode", boost::program_options::value<uint16_t>(),
+               "Mode of operation: only_save(0), only_query(1), all(2) - Default: 0")
+         ;
+   cfg.add(cli);
+}
+
+void elasticsearch_plugin::plugin_initialize(const boost::program_options::variables_map& options)
+{
+   my->_oho_index = database().add_index< primary_index< operation_history_index > >();
+   database().add_index< primary_index< account_transaction_history_index > >();
+
+   if (options.count("elasticsearch-node-url")) {
+      my->_elasticsearch_node_url = options["elasticsearch-node-url"].as<std::string>();
+   }
+   if (options.count("elasticsearch-bulk-replay")) {
+      my->_elasticsearch_bulk_replay = options["elasticsearch-bulk-replay"].as<uint32_t>();
+   }
+   if (options.count("elasticsearch-bulk-sync")) {
+      my->_elasticsearch_bulk_sync = options["elasticsearch-bulk-sync"].as<uint32_t>();
+   }
+   if (options.count("elasticsearch-visitor")) {
+      my->_elasticsearch_visitor = options["elasticsearch-visitor"].as<bool>();
+   }
+   if (options.count("elasticsearch-basic-auth")) {
+      my->_elasticsearch_basic_auth = options["elasticsearch-basic-auth"].as<std::string>();
+   }
+   if (options.count("elasticsearch-index-prefix")) {
+      my->_elasticsearch_index_prefix = options["elasticsearch-index-prefix"].as<std::string>();
+   }
+   if (options.count("elasticsearch-operation-object")) {
+      my->_elasticsearch_operation_object = options["elasticsearch-operation-object"].as<bool>();
+   }
+   if (options.count("elasticsearch-start-es-after-block")) {
+      my->_elasticsearch_start_es_after_block = options["elasticsearch-start-es-after-block"].as<uint32_t>();
+   }
+   if (options.count("elasticsearch-operation-string")) {
+      my->_elasticsearch_operation_string = options["elasticsearch-operation-string"].as<bool>();
+   }
+   if (options.count("elasticsearch-mode")) {
+      const auto option_number = options["elasticsearch-mode"].as<uint16_t>();
+      if(option_number > mode::all)
+         FC_THROW_EXCEPTION(fc::exception, "Elasticsearch mode not valid");
+      my->_elasticsearch_mode = static_cast<mode>(options["elasticsearch-mode"].as<uint16_t>());
+   }
+
+   if(my->_elasticsearch_mode != mode::only_query) {
+      if (my->_elasticsearch_mode == mode::all && !my->_elasticsearch_operation_string)
+         FC_THROW_EXCEPTION(fc::exception,
+               "If elasticsearch-mode is set to all then elasticsearch-operation-string need to be true");
+
+      database().applied_block.connect([this](const signed_block &b) {
+         if (!my->update_account_histories(b))
+            FC_THROW_EXCEPTION(fc::exception,
+                  "Error populating ES database, we are going to keep trying.");
+      });
+   }
+}
+
+void elasticsearch_plugin::plugin_startup()
+{
+   graphene::utilities::ES es;
+   es.curl = my->curl;
+   es.elasticsearch_url = my->_elasticsearch_node_url;
+   es.auth = my->_elasticsearch_basic_auth;
+
+   if(!graphene::utilities::checkES(es))
+      FC_THROW_EXCEPTION(fc::exception, "ES database is not up in url ${url}", ("url", my->_elasticsearch_node_url));
+   ilog("elasticsearch ACCOUNT HISTORY: plugin_startup() begin");
+}
+
+operation_history_object elasticsearch_plugin::get_operation_by_id(operation_history_id_type id)
+{
+   const string operation_id_string = std::string(object_id_type(id));
+
+   const string query = R"(
+   {
+      "query": {
+         "match":
+         {
+            "account_history.operation_id": )" + operation_id_string + R"("
+         }
+      }
+   }
+   )";
+
+   auto es = prepareHistoryQuery(query);
+   const auto response = graphene::utilities::simpleQuery(es);
+   variant variant_response = fc::json::from_string(response);
+   const auto source = variant_response["hits"]["hits"][size_t(0)]["_source"];
+   return fromEStoOperation(source);
+}
+
+vector<operation_history_object> elasticsearch_plugin::get_account_history(
+      const account_id_type account_id,
+      operation_history_id_type stop = operation_history_id_type(),
+      unsigned limit = 100,
+      operation_history_id_type start = operation_history_id_type())
+{
+   const string account_id_string = std::string(object_id_type(account_id));
+
+   const auto stop_number = stop.instance.value;
+   const auto start_number = start.instance.value;
+
+   string range = "";
+   if(stop_number == 0)
+      range = " AND operation_id_num: ["+fc::to_string(stop_number)+" TO "+fc::to_string(start_number)+"]";
+   else if(stop_number > 0)
+      range = " AND operation_id_num: {"+fc::to_string(stop_number)+" TO "+fc::to_string(start_number)+"]";
+
+   const string query = R"(
+   {
+      "size": )" + fc::to_string(limit) + R"(,
+      "sort" : [{ "operation_id_num" : {"order" : "desc"}}],
+      "query": {
+         "bool": {
+            "must": [
+            {
+               "query_string": {
+                  "query": "account_history.account: )" + account_id_string +  range + R"("
+               }
+            }
+            ]
+         }
+      }
+   }
+   )";
+
+   auto es = prepareHistoryQuery(query);
+
+   vector<operation_history_object> result;
+
+   if(!graphene::utilities::checkES(es))
+      return result;
+
+   const auto response = graphene::utilities::simpleQuery(es);
+   variant variant_response = fc::json::from_string(response);
+   
+   const auto hits = variant_response["hits"]["total"]["value"];
+   uint32_t size;
+   if( hits.is_object() ) // ES-7 ?
+      size = static_cast<uint32_t>(hits["value"].as_uint64());
+   else // probably ES-6
+      size = static_cast<uint32_t>(hits.as_uint64());
+      
+   size = std::min( size, limit );
+
+   for(unsigned i=0; i<size; i++)
+   {
+      const auto source = variant_response["hits"]["hits"][size_t(i)]["_source"];
+      result.push_back(fromEStoOperation(source));
+   }
+   return result;
+}
+
+operation_history_object elasticsearch_plugin::fromEStoOperation(variant source)
+{
+   operation_history_object result;
+
+   const auto operation_id = source["account_history"]["operation_id"];
+   fc::from_variant( operation_id, result.id, GRAPHENE_MAX_NESTED_OBJECTS );
+
+   const auto op = fc::json::from_string(source["operation_history"]["op"].as_string());
+   fc::from_variant( op, result.op, GRAPHENE_MAX_NESTED_OBJECTS );
+
+   const auto operation_result = fc::json::from_string(source["operation_history"]["operation_result"].as_string());
+   fc::from_variant( operation_result, result.result, GRAPHENE_MAX_NESTED_OBJECTS );
+
+   result.block_num = source["block_data"]["block_num"].as_uint64();
+   result.trx_in_block = source["operation_history"]["trx_in_block"].as_uint64();
+   result.op_in_trx = source["operation_history"]["op_in_trx"].as_uint64();
+   result.trx_in_block = source["operation_history"]["virtual_op"].as_uint64();
+
+   return result;
+}
+
+graphene::utilities::ES elasticsearch_plugin::prepareHistoryQuery(string query)
+{
+   CURL *curl;
+   curl = curl_easy_init();
+
+   graphene::utilities::ES es;
+   es.curl = curl;
+   es.elasticsearch_url = my->_elasticsearch_node_url;
+   es.index_prefix = my->_elasticsearch_index_prefix;
+   es.endpoint = es.index_prefix + "*/data/_search";
+   es.query = query;
+
+   return es;
+}
+
+mode elasticsearch_plugin::get_running_mode()
+{
+   return my->_elasticsearch_mode;
+}
+
+
+} }

libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp

@@ -0,0 +1,289 @@
+/*
+ * Copyright (c) 2017 Cryptonomex, Inc., and contributors.
+ *
+ * The MIT License
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#pragma once
+
+#include <graphene/app/plugin.hpp>
+#include <graphene/chain/database.hpp>
+#include <graphene/chain/operation_history_object.hpp>
+#include <graphene/utilities/elasticsearch.hpp>
+
+namespace graphene { namespace elasticsearch {
+   using namespace chain;
+
+//
+// Plugins should #define their SPACE_ID's so plugins with
+// conflicting SPACE_ID assignments can be compiled into the
+// same binary (by simply re-assigning some of the conflicting #defined
+// SPACE_ID's in a build script).
+//
+// Assignment of SPACE_ID's cannot be done at run-time because
+// various template automagic depends on them being known at compile
+// time.
+//
+#ifndef ELASTICSEARCH_SPACE_ID
+#define ELASTICSEARCH_SPACE_ID 6
+#endif
+
+namespace detail
+{
+    class elasticsearch_plugin_impl;
+}
+
+enum mode { only_save = 0 , only_query = 1, all = 2 };
+
+class elasticsearch_plugin : public graphene::app::plugin
+{
+   public:
+      elasticsearch_plugin();
+      virtual ~elasticsearch_plugin();
+
+      std::string plugin_name()const override;
+      std::string plugin_description()const override;
+      virtual void plugin_set_program_options(
+         boost::program_options::options_description& cli,
+         boost::program_options::options_description& cfg) override;
+      virtual void plugin_initialize(const boost::program_options::variables_map& options) override;
+      virtual void plugin_startup() override;
+
+      operation_history_object get_operation_by_id(operation_history_id_type id);
+      vector<operation_history_object> get_account_history(const account_id_type account_id,
+            operation_history_id_type stop, unsigned limit, operation_history_id_type start);
+      mode get_running_mode();
+
+      friend class detail::elasticsearch_plugin_impl;
+      std::unique_ptr<detail::elasticsearch_plugin_impl> my;
+
+   private:
+      operation_history_object fromEStoOperation(variant source);
+      graphene::utilities::ES prepareHistoryQuery(string query);
+};
+
+
+struct operation_visitor
+{
+   typedef void result_type;
+
+   share_type fee_amount;
+   asset_id_type fee_asset;
+
+   asset_id_type transfer_asset_id;
+   share_type transfer_amount;
+   account_id_type transfer_from;
+   account_id_type transfer_to;
+
+   void operator()( const graphene::chain::transfer_operation& o )
+   {
+      fee_asset = o.fee.asset_id;
+      fee_amount = o.fee.amount;
+
+      transfer_asset_id = o.amount.asset_id;
+      transfer_amount = o.amount.amount;
+      transfer_from = o.from;
+      transfer_to = o.to;
+   }
+
+   object_id_type      fill_order_id;
+   account_id_type     fill_account_id;
+   asset_id_type       fill_pays_asset_id;
+   share_type          fill_pays_amount;
+   asset_id_type       fill_receives_asset_id;
+   share_type          fill_receives_amount;
+   //double              fill_fill_price;
+   //bool                fill_is_maker;
+
+   void operator()( const graphene::chain::fill_order_operation& o )
+   {
+      fee_asset = o.fee.asset_id;
+      fee_amount = o.fee.amount;
+
+      fill_order_id = o.order_id;
+      fill_account_id = o.account_id;
+      fill_pays_asset_id = o.pays.asset_id;
+      fill_pays_amount = o.pays.amount;
+      fill_receives_asset_id = o.receives.asset_id;
+      fill_receives_amount = o.receives.amount;
+      //fill_fill_price = o.fill_price.to_real();
+      //fill_is_maker = o.is_maker;
+   }
+
+   template<typename T>
+   void operator()( const T& o )
+   {
+      fee_asset = o.fee.asset_id;
+      fee_amount = o.fee.amount;
+   }
+};
+
+struct operation_history_struct {
+   int trx_in_block;
+   int op_in_trx;
+   std::string operation_result;
+   int virtual_op;
+   std::string op;
+   variant op_object;
+};
+
+struct block_struct {
+   int block_num;
+   fc::time_point_sec block_time;
+   std::string trx_id;
+};
+
+struct fee_struct {
+   asset_id_type asset;
+   share_type amount;
+};
+
+struct transfer_struct {
+   asset_id_type asset;
+   share_type amount;
+   account_id_type from;
+   account_id_type to;
+};
+
+struct fill_struct {
+   object_id_type order_id;
+   account_id_type account_id;
+   asset_id_type pays_asset_id;
+   share_type pays_amount;
+   asset_id_type receives_asset_id;
+   share_type receives_amount;
+   double fill_price;
+   bool is_maker;
+};
+
+struct visitor_struct {
+   fee_struct fee_data;
+   transfer_struct transfer_data;
+   fill_struct fill_data;
+};
+
+struct bulk_struct {
+   account_transaction_history_object account_history;
+   operation_history_struct operation_history;
+   int operation_type;
+   int operation_id_num;
+   block_struct block_data;
+   optional<visitor_struct> additional_data;
+};
+
+struct adaptor_struct {
+   variant adapt(const variant_object& op)
+   {
+      fc::mutable_variant_object o(op);
+      vector<string> keys_to_rename;
+      for (auto i = o.begin(); i != o.end(); ++i)
+      {
+         auto& element = (*i).value();
+         if (element.is_object())
+         {
+            const string& name = (*i).key();
+            auto& vo = element.get_object();
+            if (vo.contains(name.c_str()))
+               keys_to_rename.emplace_back(name);
+            element = adapt(vo);
+         }
+         else if (element.is_array())
+            adapt(element.get_array());
+      }
+      for (const auto& i : keys_to_rename)
+      {
+         string new_name = i + "_";
+         o[new_name] = variant(o[i]);
+         o.erase(i);
+      }
+
+      if (o.find("memo") != o.end())
+      {
+         auto& memo = o["memo"];
+         if (memo.is_string())
+         {
+            o["memo_"] = o["memo"];
+            o.erase("memo");
+         }
+         else if (memo.is_object())
+         {
+            fc::mutable_variant_object tmp(memo.get_object());
+            if (tmp.find("nonce") != tmp.end())
+            {
+               tmp["nonce"] = tmp["nonce"].as_string();
+               o["memo"] = tmp;
+            }
+         }
+      }
+      if (o.find("new_parameters") != o.end())
+      {
+         auto& tmp = o["new_parameters"];
+         if (tmp.is_object())
+         {
+            fc::mutable_variant_object tmp2(tmp.get_object());
+            if (tmp2.find("current_fees") != tmp2.end())
+            {
+               tmp2.erase("current_fees");
+               o["new_parameters"] = tmp2;
+            }
+         }
+      }
+      if (o.find("owner") != o.end() && o["owner"].is_string())
+      {
+         o["owner_"] = o["owner"].as_string();
+         o.erase("owner");
+      }
+      if (o.find("proposed_ops") != o.end())
+      {
+         o["proposed_ops"] = fc::json::to_string(o["proposed_ops"]);
+      }
+      if (o.find("initializer") != o.end())
+      {
+         o["initializer"] = fc::json::to_string(o["initializer"]);
+      }
+
+      variant v;
+      fc::to_variant(o, v, FC_PACK_MAX_DEPTH);
+      return v;
+   }
+
+   void adapt(fc::variants& v)
+   {
+      for (auto& array_element : v)
+      {
+         if (array_element.is_object())
+            array_element = adapt(array_element.get_object());
+         else if (array_element.is_array())
+            adapt(array_element.get_array());
+         else
+            array_element = array_element.as_string();
+      }
+   }
+};
+} } //graphene::elasticsearch
+
+FC_REFLECT_ENUM( graphene::elasticsearch::mode, (only_save)(only_query)(all) )
+FC_REFLECT( graphene::elasticsearch::operation_history_struct, (trx_in_block)(op_in_trx)(operation_result)(virtual_op)(op)(op_object) )
+FC_REFLECT( graphene::elasticsearch::block_struct, (block_num)(block_time)(trx_id) )
+FC_REFLECT( graphene::elasticsearch::fee_struct, (asset)(amount) )
+FC_REFLECT( graphene::elasticsearch::transfer_struct, (asset)(amount)(from)(to) )
+FC_REFLECT( graphene::elasticsearch::fill_struct, (order_id)(account_id)(pays_asset_id)(pays_amount)(receives_asset_id)(receives_amount)(fill_price)(is_maker))
+FC_REFLECT( graphene::elasticsearch::visitor_struct, (fee_data)(transfer_data)(fill_data) )
+FC_REFLECT( graphene::elasticsearch::bulk_struct, (account_history)(operation_history)(operation_type)(operation_id_num)(block_data)(additional_data) )

libraries/plugins/es_objects/CMakeLists.txt

@@ -0,0 +1,23 @@
+file(GLOB HEADERS "include/graphene/es_objects/*.hpp")
+
+add_library( graphene_es_objects
+        es_objects.cpp
+           )
+
+target_link_libraries( graphene_es_objects graphene_chain graphene_app curl )
+target_include_directories( graphene_es_objects
+                            PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" )
+
+if(MSVC)
+  set_source_files_properties(es_objects.cpp PROPERTIES COMPILE_FLAGS "/bigobj" )
+endif(MSVC)
+
+install( TARGETS
+   graphene_es_objects
+
+   RUNTIME DESTINATION bin
+   LIBRARY DESTINATION lib
+   ARCHIVE DESTINATION lib
+)
+INSTALL( FILES ${HEADERS} DESTINATION "include/graphene/es_objects" )
+

libraries/plugins/es_objects/es_objects.cpp

@@ -0,0 +1,401 @@
+/*
+ * Copyright (c) 2018 oxarbitrage, and contributors.
+ *
+ * The MIT License
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <graphene/es_objects/es_objects.hpp>
+
+#include <fc/smart_ref_impl.hpp>
+
+#include <curl/curl.h>
+#include <graphene/chain/proposal_object.hpp>
+#include <graphene/chain/balance_object.hpp>
+#include <graphene/chain/market_object.hpp>
+#include <graphene/chain/asset_object.hpp>
+#include <graphene/chain/account_object.hpp>
+
+#include <graphene/utilities/elasticsearch.hpp>
+
+namespace graphene { namespace es_objects {
+
+namespace detail
+{
+
+class es_objects_plugin_impl
+{
+   public:
+      es_objects_plugin_impl(es_objects_plugin& _plugin)
+         : _self( _plugin )
+      {  curl = curl_easy_init(); }
+      virtual ~es_objects_plugin_impl();
+
+      bool index_database(const vector<object_id_type>& ids, std::string action);
+      bool genesis();
+      void remove_from_database(object_id_type id, std::string index);
+
+      es_objects_plugin& _self;
+      std::string _es_objects_elasticsearch_url = "http://localhost:9200/";
+      std::string _es_objects_auth = "";
+      uint32_t _es_objects_bulk_replay = 10000;
+      uint32_t _es_objects_bulk_sync = 100;
+      bool _es_objects_proposals = true;
+      bool _es_objects_accounts = true;
+      bool _es_objects_assets = true;
+      bool _es_objects_balances = true;
+      bool _es_objects_limit_orders = true;
+      bool _es_objects_asset_bitasset = true;
+      std::string _es_objects_index_prefix = "ppobjects-";
+      uint32_t _es_objects_start_es_after_block = 0;
+      CURL *curl; // curl handler
+      vector <std::string> bulk;
+      vector<std::string> prepare;
+
+      bool _es_objects_keep_only_current = true;
+
+      uint32_t block_number;
+      fc::time_point_sec block_time;
+
+   private:
+      template<typename T>
+      void prepareTemplate(T blockchain_object, string index_name);
+};
+
+bool es_objects_plugin_impl::genesis()
+{
+
+   ilog("elasticsearch OBJECTS: inserting data from genesis");
+
+   graphene::chain::database &db = _self.database();
+
+   block_number = db.head_block_num();
+   block_time = db.head_block_time();
+
+   if (_es_objects_accounts) {
+      auto &index_accounts = db.get_index(1, 2);
+      index_accounts.inspect_all_objects([this, &db](const graphene::db::object &o) {
+         auto obj = db.find_object(o.id);
+         auto a = static_cast<const account_object *>(obj);
+         prepareTemplate<account_object>(*a, "account");
+      });
+   }
+   if (_es_objects_assets) {
+      auto &index_assets = db.get_index(1, 3);
+      index_assets.inspect_all_objects([this, &db](const graphene::db::object &o) {
+         auto obj = db.find_object(o.id);
+         auto a = static_cast<const asset_object *>(obj);
+         prepareTemplate<asset_object>(*a, "asset");
+      });
+   }
+   if (_es_objects_balances) {
+      auto &index_balances = db.get_index(2, 5);
+      index_balances.inspect_all_objects([this, &db](const graphene::db::object &o) {
+         auto obj = db.find_object(o.id);
+         auto b = static_cast<const account_balance_object *>(obj);
+         prepareTemplate<account_balance_object>(*b, "balance");
+      });
+   }
+
+   graphene::utilities::ES es;
+   es.curl = curl;
+   es.bulk_lines = bulk;
+   es.elasticsearch_url = _es_objects_elasticsearch_url;
+   es.auth = _es_objects_auth;
+   if (!graphene::utilities::SendBulk(es))
+      FC_THROW_EXCEPTION(fc::exception, "Error inserting genesis data.");
+   else
+      bulk.clear();
+
+   return true;
+}
+
+bool es_objects_plugin_impl::index_database(const vector<object_id_type>& ids, std::string action)
+{
+   graphene::chain::database &db = _self.database();
+
+   block_time = db.head_block_time();
+   block_number = db.head_block_num();
+
+   if(block_number > _es_objects_start_es_after_block) {
+
+      // check if we are in replay or in sync and change number of bulk documents accordingly
+      uint32_t limit_documents = 0;
+      if ((fc::time_point::now() - block_time) < fc::seconds(30))
+         limit_documents = _es_objects_bulk_sync;
+      else
+         limit_documents = _es_objects_bulk_replay;
+
+
+      for (auto const &value: ids) {
+         if (value.is<proposal_object>() && _es_objects_proposals) {
+            auto obj = db.find_object(value);
+            auto p = static_cast<const proposal_object *>(obj);
+            if (p != nullptr) {
+               if (action == "delete")
+                  remove_from_database(p->id, "proposal");
+               else
+                  prepareTemplate<proposal_object>(*p, "proposal");
+            }
+         } else if (value.is<account_object>() && _es_objects_accounts) {
+            auto obj = db.find_object(value);
+            auto a = static_cast<const account_object *>(obj);
+            if (a != nullptr) {
+               if (action == "delete")
+                  remove_from_database(a->id, "account");
+               else
+                  prepareTemplate<account_object>(*a, "account");
+            }
+         } else if (value.is<asset_object>() && _es_objects_assets) {
+            auto obj = db.find_object(value);
+            auto a = static_cast<const asset_object *>(obj);
+            if (a != nullptr) {
+               if (action == "delete")
+                  remove_from_database(a->id, "asset");
+               else
+                  prepareTemplate<asset_object>(*a, "asset");
+            }
+         } else if (value.is<account_balance_object>() && _es_objects_balances) {
+            auto obj = db.find_object(value);
+            auto b = static_cast<const account_balance_object *>(obj);
+            if (b != nullptr) {
+               if (action == "delete")
+                  remove_from_database(b->id, "balance");
+               else
+                  prepareTemplate<account_balance_object>(*b, "balance");
+            }
+         } else if (value.is<limit_order_object>() && _es_objects_limit_orders) {
+            auto obj = db.find_object(value);
+            auto l = static_cast<const limit_order_object *>(obj);
+            if (l != nullptr) {
+               if (action == "delete")
+                  remove_from_database(l->id, "limitorder");
+               else
+                  prepareTemplate<limit_order_object>(*l, "limitorder");
+            }
+         } else if (value.is<asset_bitasset_data_object>() && _es_objects_asset_bitasset) {
+            auto obj = db.find_object(value);
+            auto ba = static_cast<const asset_bitasset_data_object *>(obj);
+            if (ba != nullptr) {
+               if (action == "delete")
+                  remove_from_database(ba->id, "bitasset");
+               else
+                  prepareTemplate<asset_bitasset_data_object>(*ba, "bitasset");
+            }
+         }
+      }
+
+      if (curl && bulk.size() >= limit_documents) { // we are in bulk time, ready to add data to elasticsearech
+
+         graphene::utilities::ES es;
+         es.curl = curl;
+         es.bulk_lines = bulk;
+         es.elasticsearch_url = _es_objects_elasticsearch_url;
+         es.auth = _es_objects_auth;
+
+         if (!graphene::utilities::SendBulk(es))
+            return false;
+         else
+            bulk.clear();
+      }
+   }
+
+   return true;
+}
+
+void es_objects_plugin_impl::remove_from_database( object_id_type id, std::string index)
+{
+   if(_es_objects_keep_only_current)
+   {
+      fc::mutable_variant_object delete_line;
+      delete_line["_id"] = string(id);
+      delete_line["_index"] = _es_objects_index_prefix + index;
+      delete_line["_type"] = "data";
+      fc::mutable_variant_object final_delete_line;
+      final_delete_line["delete"] = delete_line;
+      prepare.push_back(fc::json::to_string(final_delete_line));
+      std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk));
+      prepare.clear();
+   }
+}
+
+template<typename T>
+void es_objects_plugin_impl::prepareTemplate(T blockchain_object, string index_name)
+{
+   fc::mutable_variant_object bulk_header;
+   bulk_header["_index"] = _es_objects_index_prefix + index_name;
+   bulk_header["_type"] = "data";
+   if(_es_objects_keep_only_current)
+   {
+      bulk_header["_id"] = string(blockchain_object.id);
+   }
+
+   adaptor_struct adaptor;
+   fc::variant blockchain_object_variant;
+   fc::to_variant( blockchain_object, blockchain_object_variant, GRAPHENE_NET_MAX_NESTED_OBJECTS );
+   fc::mutable_variant_object o = adaptor.adapt(blockchain_object_variant.get_object());
+
+   o["object_id"] = string(blockchain_object.id);
+   o["block_time"] = block_time;
+   o["block_number"] = block_number;
+
+   string data = fc::json::to_string(o);
+
+   prepare = graphene::utilities::createBulk(bulk_header, std::move(data));
+   std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk));
+   prepare.clear();
+}
+
+es_objects_plugin_impl::~es_objects_plugin_impl()
+{
+   if (curl) {
+      curl_easy_cleanup(curl);
+      curl = nullptr;
+   }
+   return;
+}
+
+} // end namespace detail
+
+es_objects_plugin::es_objects_plugin() :
+   my( new detail::es_objects_plugin_impl(*this) )
+{
+}
+
+es_objects_plugin::~es_objects_plugin()
+{
+}
+
+std::string es_objects_plugin::plugin_name()const
+{
+   return "es_objects";
+}
+std::string es_objects_plugin::plugin_description()const
+{
+   return "Stores blockchain objects in ES database. Experimental.";
+}
+
+void es_objects_plugin::plugin_set_program_options(
+   boost::program_options::options_description& cli,
+   boost::program_options::options_description& cfg
+   )
+{
+   cli.add_options()
+         ("es-objects-elasticsearch-url", boost::program_options::value<std::string>(), "Elasticsearch node url(http://localhost:9200/)")
+         ("es-objects-auth", boost::program_options::value<std::string>(), "Basic auth username:password('')")
+         ("es-objects-bulk-replay", boost::program_options::value<uint32_t>(), "Number of bulk documents to index on replay(10000)")
+         ("es-objects-bulk-sync", boost::program_options::value<uint32_t>(), "Number of bulk documents to index on a synchronized chain(100)")
+         ("es-objects-proposals", boost::program_options::value<bool>(), "Store proposal objects(true)")
+         ("es-objects-accounts", boost::program_options::value<bool>(), "Store account objects(true)")
+         ("es-objects-assets", boost::program_options::value<bool>(), "Store asset objects(true)")
+         ("es-objects-balances", boost::program_options::value<bool>(), "Store balances objects(true)")
+         ("es-objects-limit-orders", boost::program_options::value<bool>(), "Store limit order objects(true)")
+         ("es-objects-asset-bitasset", boost::program_options::value<bool>(), "Store feed data(true)")
+         ("es-objects-index-prefix", boost::program_options::value<std::string>(), "Add a prefix to the index(ppobjects-)")
+         ("es-objects-keep-only-current", boost::program_options::value<bool>(), "Keep only current state of the objects(true)")
+         ("es-objects-start-es-after-block", boost::program_options::value<uint32_t>(), "Start doing ES job after block(0)")
+         ;
+   cfg.add(cli);
+}
+
+void es_objects_plugin::plugin_initialize(const boost::program_options::variables_map& options)
+{
+   database().applied_block.connect([this](const signed_block &b) {
+      if(b.block_num() == 1) {
+         if (!my->genesis())
+            FC_THROW_EXCEPTION(fc::exception, "Error populating genesis data.");
+      }
+   });
+
+   database().new_objects.connect([this]( const vector<object_id_type>& ids, const flat_set<account_id_type>& impacted_accounts ) {
+      if(!my->index_database(ids, "create"))
+      {
+         FC_THROW_EXCEPTION(fc::exception, "Error creating object from ES database, we are going to keep trying.");
+      }
+   });
+   database().changed_objects.connect([this]( const vector<object_id_type>& ids, const flat_set<account_id_type>& impacted_accounts ) {
+      if(!my->index_database(ids, "update"))
+      {
+         FC_THROW_EXCEPTION(fc::exception, "Error updating object from ES database, we are going to keep trying.");
+      }
+   });
+   database().removed_objects.connect([this](const vector<object_id_type>& ids, const vector<const object*>& objs, const flat_set<account_id_type>& impacted_accounts) {
+       if(!my->index_database(ids, "delete"))
+       {
+          FC_THROW_EXCEPTION(fc::exception, "Error deleting object from ES database, we are going to keep trying.");
+       }
+   });
+
+
+   if (options.count("es-objects-elasticsearch-url")) {
+      my->_es_objects_elasticsearch_url = options["es-objects-elasticsearch-url"].as<std::string>();
+   }
+   if (options.count("es-objects-auth")) {
+      my->_es_objects_auth = options["es-objects-auth"].as<std::string>();
+   }
+   if (options.count("es-objects-bulk-replay")) {
+      my->_es_objects_bulk_replay = options["es-objects-bulk-replay"].as<uint32_t>();
+   }
+   if (options.count("es-objects-bulk-sync")) {
+      my->_es_objects_bulk_sync = options["es-objects-bulk-sync"].as<uint32_t>();
+   }
+   if (options.count("es-objects-proposals")) {
+      my->_es_objects_proposals = options["es-objects-proposals"].as<bool>();
+   }
+   if (options.count("es-objects-accounts")) {
+      my->_es_objects_accounts = options["es-objects-accounts"].as<bool>();
+   }
+   if (options.count("es-objects-assets")) {
+      my->_es_objects_assets = options["es-objects-assets"].as<bool>();
+   }
+   if (options.count("es-objects-balances")) {
+      my->_es_objects_balances = options["es-objects-balances"].as<bool>();
+   }
+   if (options.count("es-objects-limit-orders")) {
+      my->_es_objects_limit_orders = options["es-objects-limit-orders"].as<bool>();
+   }
+   if (options.count("es-objects-asset-bitasset")) {
+      my->_es_objects_asset_bitasset = options["es-objects-asset-bitasset"].as<bool>();
+   }
+   if (options.count("es-objects-index-prefix")) {
+      my->_es_objects_index_prefix = options["es-objects-index-prefix"].as<std::string>();
+   }
+   if (options.count("es-objects-keep-only-current")) {
+      my->_es_objects_keep_only_current = options["es-objects-keep-only-current"].as<bool>();
+   }
+   if (options.count("es-objects-start-es-after-block")) {
+      my->_es_objects_start_es_after_block = options["es-objects-start-es-after-block"].as<uint32_t>();
+   }
+}
+
+void es_objects_plugin::plugin_startup()
+{
+   graphene::utilities::ES es;
+   es.curl = my->curl;
+   es.elasticsearch_url = my->_es_objects_elasticsearch_url;
+   es.auth = my->_es_objects_auth;
+   es.auth = my->_es_objects_index_prefix;
+
+   if(!graphene::utilities::checkES(es))
+      FC_THROW_EXCEPTION(fc::exception, "ES database is not up in url ${url}", ("url", my->_es_objects_elasticsearch_url));
+   ilog("elasticsearch OBJECTS: plugin_startup() begin");
+}
+
+} }

libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp

@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2018 oxarbitrage, and contributors.
+ *
+ * The MIT License
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#pragma once
+
+#include <graphene/app/plugin.hpp>
+#include <graphene/chain/database.hpp>
+
+namespace graphene { namespace es_objects {
+
+using namespace chain;
+
+namespace detail
+{
+    class es_objects_plugin_impl;
+}
+
+class es_objects_plugin : public graphene::app::plugin
+{
+   public:
+      es_objects_plugin();
+      virtual ~es_objects_plugin();
+
+      std::string plugin_name()const override;
+      std::string plugin_description()const override;
+      virtual void plugin_set_program_options(
+         boost::program_options::options_description& cli,
+         boost::program_options::options_description& cfg) override;
+      virtual void plugin_initialize(const boost::program_options::variables_map& options) override;
+      virtual void plugin_startup() override;
+
+      friend class detail::es_objects_plugin_impl;
+      std::unique_ptr<detail::es_objects_plugin_impl> my;
+};
+
+struct adaptor_struct {
+    fc::mutable_variant_object adapt(const variant_object &obj) {
+      fc::mutable_variant_object o(obj);
+      vector<string> keys_to_rename;
+      for (auto i = o.begin(); i != o.end(); ++i) {
+         auto &element = (*i).value();
+         if (element.is_object()) {
+            const string &name = (*i).key();
+            auto &vo = element.get_object();
+            if (vo.contains(name.c_str()))
+               keys_to_rename.emplace_back(name);
+            element = adapt(vo);
+         } else if (element.is_array())
+            adapt(element.get_array());
+      }
+      for (const auto &i : keys_to_rename) {
+         string new_name = i + "_";
+         o[new_name] = variant(o[i]);
+         o.erase(i);
+      }
+      if (o.find("owner") != o.end() && o["owner"].is_string())
+      {
+         o["owner_"] = o["owner"].as_string();
+         o.erase("owner");
+      }
+      if (o.find("active_special_authority") != o.end())
+      {
+         o["active_special_authority"] = fc::json::to_string(o["active_special_authority"]);
+      }
+      if (o.find("owner_special_authority") != o.end())
+      {
+         o["owner_special_authority"] = fc::json::to_string(o["owner_special_authority"]);
+      }
+      if (o.find("feeds") != o.end())
+      {
+         o["feeds"] = fc::json::to_string(o["feeds"]);
+      }
+      if (o.find("operations") != o.end())
+      {
+         o["operations"] = fc::json::to_string(o["operations"]);
+      }
+      
+      return o;
+   }
+
+   void adapt(fc::variants &v) {
+      for (auto &array_element : v) {
+         if (array_element.is_object())
+            array_element = adapt(array_element.get_object());
+         else if (array_element.is_array())
+            adapt(array_element.get_array());
+         else
+            array_element = array_element.as_string();
+      }
+   }
+};
+
+} } //graphene::es_objects

libraries/plugins/snapshot/CMakeLists.txt

@@ -15,3 +15,5 @@ install( TARGETS
    LIBRARY DESTINATION lib
    ARCHIVE DESTINATION lib
 )
+
+INSTALL( FILES ${HEADERS} DESTINATION "include/graphene/snapshot" )

libraries/plugins/snapshot/include/graphene/snapshot/snapshot.hpp

@@ -35,6 +35,7 @@ class snapshot_plugin : public graphene::app::plugin {
       ~snapshot_plugin() {}
 
       std::string plugin_name()const override;
+      std::string plugin_description()const override;
 
       virtual void plugin_set_program_options(
          boost::program_options::options_description &command_line_options,

libraries/plugins/snapshot/snapshot.cpp

@@ -54,6 +54,11 @@ std::string snapshot_plugin::plugin_name()const
    return "snapshot";
 }
 
+std::string snapshot_plugin::plugin_description()const
+{
+   return "Create snapshots at a specified time or block number.";
+}
+
 void snapshot_plugin::plugin_initialize(const boost::program_options::variables_map& options)
 { try {
    ilog("snapshot plugin: plugin_initialize() begin");

libraries/utilities/CMakeLists.txt

@@ -14,6 +14,7 @@ set(sources
    string_escape.cpp
    tempdir.cpp
    words.cpp
+   elasticsearch.cpp
    ${HEADERS})
 
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/git_revision.cpp.in" "${CMAKE_CURRENT_BINARY_DIR}/git_revision.cpp" @ONLY)

libraries/utilities/elasticsearch.cpp

@@ -0,0 +1,190 @@
+/*
+ * Copyright (c) 2018 oxarbitrage, and contributors.
+ *
+ * The MIT License
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include <graphene/utilities/elasticsearch.hpp>
+
+#include <boost/algorithm/string/join.hpp>
+#include <boost/algorithm/string.hpp>
+#include <fc/log/logger.hpp>
+#include <fc/io/json.hpp>
+
+size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
+{
+   ((std::string*)userp)->append((char*)contents, size * nmemb);
+   return size * nmemb;
+}
+
+namespace graphene { namespace utilities {
+
+bool checkES(ES& es)
+{
+   graphene::utilities::CurlRequest curl_request;
+   curl_request.handler = es.curl;
+   curl_request.url = es.elasticsearch_url + "_nodes";
+   curl_request.auth = es.auth;
+   curl_request.type = "GET";
+
+   if(doCurl(curl_request).empty())
+      return false;
+   return true;
+
+}
+const std::string simpleQuery(ES& es)
+{
+   graphene::utilities::CurlRequest curl_request;
+   curl_request.handler = es.curl;
+   curl_request.url = es.elasticsearch_url + es.endpoint;
+   curl_request.auth = es.auth;
+   curl_request.type = "POST";
+   curl_request.query = es.query;
+
+   return doCurl(curl_request);
+}
+
+bool SendBulk(ES& es)
+{
+   std::string bulking = joinBulkLines(es.bulk_lines);
+
+   graphene::utilities::CurlRequest curl_request;
+   curl_request.handler = es.curl;
+   curl_request.url = es.elasticsearch_url + "_bulk";
+   curl_request.auth = es.auth;
+   curl_request.type = "POST";
+   curl_request.query = bulking;
+
+   auto curlResponse = doCurl(curl_request);
+
+   if(handleBulkResponse(getResponseCode(curl_request.handler), curlResponse))
+      return true;
+   return false;
+}
+
+const std::string joinBulkLines(const std::vector<std::string>& bulk)
+{
+   auto bulking = boost::algorithm::join(bulk, "\n");
+   bulking = bulking + "\n";
+
+   return bulking;
+}
+long getResponseCode(CURL *handler)
+{
+   long http_code = 0;
+   curl_easy_getinfo (handler, CURLINFO_RESPONSE_CODE, &http_code);
+   return http_code;
+}
+
+bool handleBulkResponse(long http_code, const std::string& CurlReadBuffer)
+{
+   if(http_code == 200) {
+      // all good, but check errors in response
+      fc::variant j = fc::json::from_string(CurlReadBuffer);
+      bool errors = j["errors"].as_bool();
+      if(errors == true) {
+         return false;
+      }
+   }
+   else {
+      if(http_code == 413) {
+         elog( "413 error: Can be low disk space" );
+      }
+      else if(http_code == 401) {
+         elog( "401 error: Unauthorized" );
+      }
+      else {
+         elog( std::to_string(http_code) + " error: Unknown error" );
+      }
+      return false;
+   }
+   return true;
+}
+
+const std::vector<std::string> createBulk(const fc::mutable_variant_object& bulk_header, const std::string& data)
+{
+   std::vector<std::string> bulk;
+   fc::mutable_variant_object final_bulk_header;
+   final_bulk_header["index"] = bulk_header;
+   bulk.push_back(fc::json::to_string(final_bulk_header));
+   bulk.push_back(data);
+
+   return bulk;
+}
+
+bool deleteAll(ES& es)
+{
+   graphene::utilities::CurlRequest curl_request;
+   curl_request.handler = es.curl;
+   curl_request.url = es.elasticsearch_url + es.index_prefix + "*";
+   curl_request.auth = es.auth;
+   curl_request.type = "DELETE";
+
+   auto curl_response =  doCurl(curl_request);
+   if(curl_response.empty())
+      return false;
+   else
+      return true;
+}
+const std::string getEndPoint(ES& es)
+{
+   graphene::utilities::CurlRequest curl_request;
+   curl_request.handler = es.curl;
+   curl_request.url = es.elasticsearch_url + es.endpoint;
+   curl_request.auth = es.auth;
+   curl_request.type = "GET";
+
+   return doCurl(curl_request);
+}
+
+const std::string generateIndexName(const fc::time_point_sec& block_date, const std::string& _elasticsearch_index_prefix)
+{
+   auto block_date_string = block_date.to_iso_string();
+   std::vector<std::string> parts;
+   boost::split(parts, block_date_string, boost::is_any_of("-"));
+   std::string index_name = _elasticsearch_index_prefix + parts[0] + "-" + parts[1];
+   return index_name;
+}
+
+const std::string doCurl(CurlRequest& curl)
+{
+   std::string CurlReadBuffer;
+   struct curl_slist *headers = NULL;
+   headers = curl_slist_append(headers, "Content-Type: application/json");
+
+   curl_easy_setopt(curl.handler, CURLOPT_HTTPHEADER, headers);
+   curl_easy_setopt(curl.handler, CURLOPT_URL, curl.url.c_str());
+   curl_easy_setopt(curl.handler, CURLOPT_CUSTOMREQUEST, curl.type.c_str());
+   if(curl.type == "POST")
+   {
+      curl_easy_setopt(curl.handler, CURLOPT_POST, true);
+      curl_easy_setopt(curl.handler, CURLOPT_POSTFIELDS, curl.query.c_str());
+   }
+   curl_easy_setopt(curl.handler, CURLOPT_WRITEFUNCTION, WriteCallback);
+   curl_easy_setopt(curl.handler, CURLOPT_WRITEDATA, (void *)&CurlReadBuffer);
+   curl_easy_setopt(curl.handler, CURLOPT_USERAGENT, "libcrp/0.1");
+   if(!curl.auth.empty())
+      curl_easy_setopt(curl.handler, CURLOPT_USERPWD, curl.auth.c_str());
+   curl_easy_perform(curl.handler);
+
+   return CurlReadBuffer;
+}
+
+} } // end namespace graphene::utilities

libraries/utilities/include/graphene/utilities/elasticsearch.hpp

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2018 oxarbitrage, and contributors.
+ *
+ * The MIT License
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#pragma once
+#include <cstddef>
+#include <string>
+#include <vector>
+
+#include <curl/curl.h>
+#include <fc/time.hpp>
+#include <fc/variant_object.hpp>
+
+size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp);
+
+namespace graphene { namespace utilities {
+
+   class ES {
+      public:
+         CURL *curl;
+         std::vector <std::string> bulk_lines;
+         std::string elasticsearch_url;
+         std::string index_prefix;
+         std::string auth;
+         std::string endpoint;
+         std::string query;
+   };
+   class CurlRequest {
+      public:
+         CURL *handler;
+         std::string url;
+         std::string type;
+         std::string auth;
+         std::string query;
+   };
+
+   bool SendBulk(ES& es);
+   const std::vector<std::string> createBulk(const fc::mutable_variant_object& bulk_header, const std::string& data);
+   bool checkES(ES& es);
+   const std::string simpleQuery(ES& es);
+   bool deleteAll(ES& es);
+   bool handleBulkResponse(long http_code, const std::string& CurlReadBuffer);
+   const std::string getEndPoint(ES& es);
+   const std::string generateIndexName(const fc::time_point_sec& block_date, const std::string& _elasticsearch_index_prefix);
+   const std::string doCurl(CurlRequest& curl);
+   const std::string joinBulkLines(const std::vector<std::string>& bulk);
+   long getResponseCode(CURL *handler);
+
+} } // end namespace graphene::utilities

libraries/wallet/wallet.cpp

@@ -1100,7 +1100,7 @@ public:
       if( wallet_filename == "" )
          wallet_filename = _wallet_filename;
 
-      wlog( "saving wallet to file ${fn}", ("fn", wallet_filename) );
+      ilog( "saving wallet to file ${fn}", ("fn", wallet_filename) );
 
       string data = fc::json::to_pretty_string( _wallet );
       try
@@ -1112,14 +1112,38 @@ public:
          //
          // http://en.wikipedia.org/wiki/Most_vexing_parse
          //
-         fc::ofstream outfile{ fc::path( wallet_filename ) };
+         std::string tmp_wallet_filename = wallet_filename + ".tmp";
+         fc::ofstream outfile{ fc::path( tmp_wallet_filename ) };
          outfile.write( data.c_str(), data.length() );
          outfile.flush();
          outfile.close();
+
+         ilog( "saved successfully wallet to tmp file ${fn}", ("fn", tmp_wallet_filename) );
+
+         std::string wallet_file_content;
+         fc::read_file_contents(tmp_wallet_filename, wallet_file_content);
+
+         if (wallet_file_content == data) {
+            dlog( "validated successfully tmp wallet file ${fn}", ("fn", tmp_wallet_filename) );
+            fc::rename( tmp_wallet_filename, wallet_filename );
+            dlog( "renamed successfully tmp wallet file ${fn}", ("fn", tmp_wallet_filename) );
+         } 
+         else 
+         {
+            FC_THROW("tmp wallet file cannot be validated ${fn}", ("fn", tmp_wallet_filename) );
+         }
+
+         ilog( "successfully saved wallet to file ${fn}", ("fn", wallet_filename) );
+
          disable_umask_protection();
       }
       catch(...)
       {
+         string ws_password = _wallet.ws_password;
+         _wallet.ws_password = "";
+         elog("wallet file content is: ${data}", ("data", fc::json::to_pretty_string( _wallet ) ) );
+         _wallet.ws_password = ws_password;
+
          disable_umask_protection();
          throw;
       }
@@ -2122,6 +2146,7 @@ public:
 
       asset_object asset_obj = get_asset( asset_symbol );
       vector< vesting_balance_object > vbos;
+      vesting_balance_object vbo;
       fc::optional<vesting_balance_id_type> vbid = maybe_id<vesting_balance_id_type>(account_name);
       if( !vbid )
       {
@@ -2134,7 +2159,13 @@ public:
       if(!vbos.size())
          vbos.emplace_back( get_object<vesting_balance_object>(*vbid) );
  
-      const vesting_balance_object& vbo = vbos.front();
+      for (const vesting_balance_object& vesting_balance_obj: vbos) {
+         if(vesting_balance_obj.balance_type == vesting_balance_type::gpos)
+         {
+            vbo = vesting_balance_obj;
+            break;
+         }
+      }
 
       vesting_balance_withdraw_operation vesting_balance_withdraw_op;
 

programs/cli_wallet/main.cpp

@@ -113,11 +113,13 @@ int main( int argc, char** argv )
       cfg.appenders.push_back(fc::appender_config( "rpc", "file", fc::variant(ac, 5)));
 
       cfg.loggers = { fc::logger_config("default"), fc::logger_config( "rpc") };
-      cfg.loggers.front().level = fc::log_level::info;
+      cfg.loggers.front().level = fc::log_level::warn;
       cfg.loggers.front().appenders = {"default"};
-      cfg.loggers.back().level = fc::log_level::debug;
+      cfg.loggers.back().level = fc::log_level::info;
       cfg.loggers.back().appenders = {"rpc"};
 
+      fc::configure_logging( cfg );
+
       fc::ecc::private_key committee_private_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")));
 
       idump( (key_to_wif( committee_private_key ) ) );

programs/witness_node/CMakeLists.txt

@@ -11,7 +11,7 @@ endif()
 
 # We have to link against graphene_debug_witness because deficiency in our API infrastructure doesn't allow plugins to be fully abstracted #246
 target_link_libraries( witness_node
-                       PRIVATE graphene_app graphene_account_history graphene_affiliate_stats graphene_market_history graphene_witness graphene_chain graphene_debug_witness graphene_bookie graphene_egenesis_full fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} )
+                       PRIVATE graphene_app graphene_account_history graphene_affiliate_stats graphene_elasticsearch graphene_market_history graphene_witness graphene_chain graphene_debug_witness graphene_bookie graphene_egenesis_full graphene_snapshot graphene_es_objects fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} )
 # also add dependencies to graphene_generate_genesis graphene_generate_uia_sharedrop_genesis if you want those plugins
 
 install( TARGETS

programs/witness_node/genesis.json

@@ -1,5 +1,5 @@
 {
-  "initial_timestamp": "2019-05-14T18:47:51",
+  "initial_timestamp": "2020-01-13T06:03:15",
   "max_core_supply": "1000000000000000",
   "initial_parameters": {
     "current_fees": {
@@ -307,6 +307,27 @@
           75,{}
         ],[
           76,{}
+        ],[
+          77,{
+            "lottery_asset": 2000000,
+            "price_per_kbyte": 10
+          }
+        ],[
+          78,{
+            "fee": 0
+          }
+        ],[
+          79,{
+            "fee": 0
+          }
+        ],[
+          80,{
+            "fee": 0
+          }
+        ],[
+          81,{
+            "fee": 2000000
+          }
         ]
       ],
       "scale": 10000
@@ -352,7 +373,19 @@
     "maximum_tournament_start_time_in_future": 2419200,
     "maximum_tournament_start_delay": 604800,
     "maximum_tournament_number_of_wins": 100,
-    "extensions": {}
+    "extensions": {
+      "min_bet_multiplier": 10100,
+      "max_bet_multiplier": 10000000,
+      "betting_rake_fee_percentage": 300,
+      "live_betting_delay_time": 5,
+      "sweeps_distribution_percentage": 200,
+      "sweeps_distribution_asset": "1.3.0",
+      "sweeps_vesting_accumulator_account": "1.2.0",
+      "gpos_period": 15552000,
+      "gpos_subperiod": 2592000,
+      "gpos_period_start": 1601528400,
+      "gpos_vesting_lockin_period": 2592000
+    }
   },
   "initial_bts_accounts": [],
   "initial_accounts": [{

programs/witness_node/main.cpp

@@ -25,15 +25,18 @@
 #include <graphene/app/config_util.hpp>
 
 #include <graphene/witness/witness.hpp>
+#include <graphene/debug_witness/debug_witness.hpp>
 #include <graphene/account_history/account_history_plugin.hpp>
 #include <graphene/accounts_list/accounts_list_plugin.hpp>
+#include <graphene/elasticsearch/elasticsearch_plugin.hpp>
+#include <graphene/es_objects/es_objects.hpp>
 #include <graphene/market_history/market_history_plugin.hpp>
 //#include <graphene/generate_genesis/generate_genesis_plugin.hpp>
 //#include <graphene/generate_uia_sharedrop_genesis/generate_uia_sharedrop_genesis.hpp>
 #include <graphene/affiliate_stats/affiliate_stats_plugin.hpp>
 #include <graphene/bookie/bookie_plugin.hpp>
 #include <graphene/utilities/git_revision.hpp>
-//#include <graphene/snapshot/snapshot.hpp>
+#include <graphene/snapshot/snapshot.hpp>
 
 #include <fc/thread/thread.hpp>
 #include <fc/interprocess/signals.hpp>
@@ -71,14 +74,17 @@ int main(int argc, char** argv) {
       bpo::variables_map options;
 
       auto witness_plug = node->register_plugin<witness_plugin::witness_plugin>();
+      auto debug_witness_plug = node->register_plugin<debug_witness_plugin::debug_witness_plugin>();
       auto history_plug = node->register_plugin<account_history::account_history_plugin>();
+      auto elasticsearch_plug = node->register_plugin<elasticsearch::elasticsearch_plugin>();
+      auto es_objects_plug = node->register_plugin<es_objects::es_objects_plugin>();
       auto market_history_plug = node->register_plugin<market_history::market_history_plugin>();
       //auto generate_genesis_plug = node->register_plugin<generate_genesis_plugin::generate_genesis_plugin>();
       //auto generate_uia_sharedrop_genesis_plug = node->register_plugin<generate_uia_sharedrop_genesis::generate_uia_sharedrop_genesis_plugin>();
       auto list_plug = node->register_plugin<accounts_list::accounts_list_plugin>();
       auto affiliate_stats_plug = node->register_plugin<affiliate_stats::affiliate_stats_plugin>();
       auto bookie_plug = node->register_plugin<bookie::bookie_plugin>();
-//      auto snapshot_plug = node->register_plugin<snapshot_plugin::snapshot_plugin>();
+      auto snapshot_plug = node->register_plugin<snapshot_plugin::snapshot_plugin>();
 
       try
       {
@@ -142,7 +148,7 @@ int main(int argc, char** argv) {
          exit_promise->set_value(signal);
       }, SIGTERM);
 
-      ilog("Started witness node on a chain with ${h} blocks.", ("h", node->chain_database()->head_block_num()));
+      ilog("Started Peerplays node on a chain with ${h} blocks.", ("h", node->chain_database()->head_block_num()));
       ilog("Chain ID is ${id}", ("id", node->chain_database()->get_chain_id()) );
 
       int signal = exit_promise->wait();
@@ -163,4 +169,4 @@ int main(int argc, char** argv) {
       delete node;
       return 1;
    }
-}
+}

tests/CMakeLists.txt

@@ -8,38 +8,38 @@ endif()
 
 file(GLOB UNIT_TESTS "tests/*.cpp")
 add_executable( chain_test ${UNIT_TESTS} ${COMMON_SOURCES} )
-target_link_libraries( chain_test graphene_chain graphene_app graphene_account_history graphene_bookie graphene_egenesis_none fc graphene_wallet ${PLATFORM_SPECIFIC_LIBS} )
+target_link_libraries( chain_test graphene_chain graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_bookie graphene_egenesis_none fc graphene_wallet ${PLATFORM_SPECIFIC_LIBS} )
 if(MSVC)
   set_source_files_properties( tests/serialization_tests.cpp PROPERTIES COMPILE_FLAGS "/bigobj" )
 endif(MSVC)
 
 file(GLOB PERFORMANCE_TESTS "performance/*.cpp")
 add_executable( performance_test ${PERFORMANCE_TESTS} ${COMMON_SOURCES} )
-target_link_libraries( performance_test graphene_chain graphene_app graphene_account_history graphene_bookie graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} )
+target_link_libraries( performance_test graphene_chain graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_bookie graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} )
 
 file(GLOB BENCH_MARKS "benchmarks/*.cpp")
 add_executable( chain_bench ${BENCH_MARKS} ${COMMON_SOURCES} )
-target_link_libraries( chain_bench graphene_chain graphene_app graphene_account_history graphene_bookie graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} )
+target_link_libraries( chain_bench graphene_chain graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_bookie graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} )
 
 file(GLOB APP_SOURCES "app/*.cpp")
 add_executable( app_test ${APP_SOURCES} )
-target_link_libraries( app_test graphene_app graphene_account_history graphene_bookie graphene_net graphene_chain graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} )
+target_link_libraries( app_test graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_witness graphene_bookie graphene_net graphene_chain graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} )
 
 file(GLOB INTENSE_SOURCES "intense/*.cpp")
 add_executable( intense_test ${INTENSE_SOURCES} ${COMMON_SOURCES} )
-target_link_libraries( intense_test graphene_chain graphene_app graphene_account_history graphene_bookie graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} )
+target_link_libraries( intense_test graphene_chain graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_bookie graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} )
 
 file(GLOB BETTING_TESTS "betting/*.cpp")
 add_executable( betting_test ${BETTING_TESTS} ${COMMON_SOURCES} )
-target_link_libraries( betting_test graphene_chain graphene_app graphene_account_history graphene_bookie graphene_egenesis_none fc graphene_wallet ${PLATFORM_SPECIFIC_LIBS} )
+target_link_libraries( betting_test graphene_chain graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_bookie graphene_egenesis_none fc graphene_wallet ${PLATFORM_SPECIFIC_LIBS} )
 
 file(GLOB TOURNAMENT_TESTS "tournament/*.cpp")
 add_executable( tournament_test ${TOURNAMENT_TESTS} ${COMMON_SOURCES} )
-target_link_libraries( tournament_test graphene_chain graphene_app graphene_account_history graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} )
+target_link_libraries( tournament_test graphene_chain graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} )
 
 file(GLOB RANDOM_SOURCES "random/*.cpp")
 add_executable( random_test ${RANDOM_SOURCES} ${COMMON_SOURCES} )
-target_link_libraries( random_test graphene_chain graphene_app graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} )
+target_link_libraries( random_test graphene_chain graphene_app graphene_elasticsearch graphene_es_objects graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} )
 
 file(GLOB CLI_SOURCES "cli/*.cpp")
 add_executable( cli_test ${CLI_SOURCES} )
@@ -51,4 +51,8 @@ if(MSVC)
     set_source_files_properties( cli/main.cpp PROPERTIES COMPILE_FLAGS "/bigobj" )
 endif(MSVC)
 
+file(GLOB ES_SOURCES "elasticsearch/*.cpp")
+add_executable( es_test ${ES_SOURCES} ${COMMON_SOURCES} )
+target_link_libraries( es_test graphene_chain graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} )
+
 add_subdirectory( generate_empty_blocks )

tests/app/main.cpp

@@ -28,8 +28,12 @@
 
 #include <graphene/utilities/tempdir.hpp>
 
+#include <graphene/witness/witness.hpp>
 #include <graphene/account_history/account_history_plugin.hpp>
-
+#include <graphene/bookie/bookie_plugin.hpp>
+#include <graphene/accounts_list/accounts_list_plugin.hpp>
+#include <graphene/affiliate_stats/affiliate_stats_plugin.hpp>
+#include <graphene/market_history/market_history_plugin.hpp>
 #include <fc/thread/thread.hpp>
 #include <fc/smart_ref_impl.hpp>
 
@@ -56,7 +60,13 @@ BOOST_AUTO_TEST_CASE( two_node_network )
       BOOST_TEST_MESSAGE( "Creating and initializing app1" );
 
       graphene::app::application app1;
+      app1.register_plugin<graphene::witness_plugin::witness_plugin>();
       app1.register_plugin<graphene::account_history::account_history_plugin>();
+      app1.register_plugin<graphene::bookie::bookie_plugin>();
+      app1.register_plugin<graphene::accounts_list::accounts_list_plugin>();
+      app1.register_plugin<graphene::affiliate_stats::affiliate_stats_plugin>();
+      app1.register_plugin<graphene::market_history::market_history_plugin>();
+
       boost::program_options::variables_map cfg;
       cfg.emplace("p2p-endpoint", boost::program_options::variable_value(string("127.0.0.1:0"), false));
       app1.initialize(app_dir.path(), cfg);
@@ -71,7 +81,12 @@ BOOST_AUTO_TEST_CASE( two_node_network )
       auto cfg2 = cfg;
 
       graphene::app::application app2;
-      app2.register_plugin<account_history::account_history_plugin>();
+      app2.register_plugin<graphene::witness_plugin::witness_plugin>();
+      app2.register_plugin<graphene::account_history::account_history_plugin>();
+      app2.register_plugin<graphene::bookie::bookie_plugin>();
+      app2.register_plugin<graphene::accounts_list::accounts_list_plugin>();
+      app2.register_plugin<graphene::affiliate_stats::affiliate_stats_plugin>();
+      app2.register_plugin<graphene::market_history::market_history_plugin>();
       cfg2.erase("p2p-endpoint");
       cfg2.emplace("p2p-endpoint", boost::program_options::variable_value(string("127.0.0.1:0"), false));
       cfg2.emplace("seed-node", boost::program_options::variable_value(vector<string>{endpoint1}, false));

tests/cli/main.cpp

@@ -30,6 +30,8 @@
 #include <graphene/account_history/account_history_plugin.hpp>
 #include <graphene/witness/witness.hpp>
 #include <graphene/market_history/market_history_plugin.hpp>
+#include <graphene/accounts_list/accounts_list_plugin.hpp>
+#include <graphene/affiliate_stats/affiliate_stats_plugin.hpp>
 
 #include <graphene/egenesis/egenesis.hpp>
 #include <graphene/wallet/wallet.hpp>
@@ -125,9 +127,11 @@ std::shared_ptr<graphene::app::application> start_application(fc::temp_directory
    std::shared_ptr<graphene::app::application> app1(new graphene::app::application{});
 
    app1->register_plugin< graphene::bookie::bookie_plugin>();
-   app1->register_plugin<graphene::account_history::account_history_plugin>();
+   app1->register_plugin< graphene::account_history::account_history_plugin>();
    app1->register_plugin< graphene::market_history::market_history_plugin >();
    app1->register_plugin< graphene::witness_plugin::witness_plugin >();
+   app1->register_plugin< graphene::accounts_list::accounts_list_plugin >();
+   app1->register_plugin< graphene::affiliate_stats::affiliate_stats_plugin >();
    app1->startup_plugins();
    boost::program_options::variables_map cfg;
 #ifdef _WIN32

tests/common/database_fixture.cpp

@@ -29,11 +29,9 @@
 #include <graphene/bookie/bookie_plugin.hpp>
 #include <graphene/bookie/bookie_api.hpp>
 #include <graphene/affiliate_stats/affiliate_stats_plugin.hpp>
+#include <graphene/elasticsearch/elasticsearch_plugin.hpp>
+#include <graphene/es_objects/es_objects.hpp>
 
-#include <graphene/db/simple_index.hpp>
-
-#include <graphene/chain/account_object.hpp>
-#include <graphene/chain/asset_object.hpp>
 #include <graphene/chain/committee_member_object.hpp>
 #include <graphene/chain/fba_object.hpp>
 #include <graphene/chain/market_object.hpp>
@@ -51,9 +49,7 @@
 #include <fc/crypto/digest.hpp>
 #include <fc/smart_ref_impl.hpp>
 
-#include <iostream>
 #include <iomanip>
-#include <sstream>
 
 #include "database_fixture.hpp"
 
@@ -81,7 +77,7 @@ database_fixture::database_fixture()
          std::cout << "running test " << boost::unit_test::framework::current_test_case().p_name << std::endl;
    }
 
-   auto ahplugin = app.register_plugin<graphene::account_history::account_history_plugin>();
+   //auto ahplugin = app.register_plugin<graphene::account_history::account_history_plugin>();
    auto mhplugin = app.register_plugin<graphene::market_history::market_history_plugin>();
    auto bookieplugin = app.register_plugin<graphene::bookie::bookie_plugin>();
    auto affiliateplugin = app.register_plugin<graphene::affiliate_stats::affiliate_stats_plugin>();
@@ -134,8 +130,51 @@ database_fixture::database_fixture()
    }
    
    // app.initialize();
-   ahplugin->plugin_set_app(&app);
+
-   ahplugin->plugin_initialize(options);
+   auto test_name = boost::unit_test::framework::current_test_case().p_name.value;
+   if(test_name == "elasticsearch_account_history" || test_name == "elasticsearch_suite" ||
+      test_name == "elasticsearch_history_api") {
+      auto esplugin = app.register_plugin<graphene::elasticsearch::elasticsearch_plugin>();
+      esplugin->plugin_set_app(&app);
+
+      options.insert(std::make_pair("elasticsearch-node-url", boost::program_options::variable_value(string("http://localhost:9200/"), false)));
+      options.insert(std::make_pair("elasticsearch-bulk-replay", boost::program_options::variable_value(uint32_t(2), false)));
+      options.insert(std::make_pair("elasticsearch-bulk-sync", boost::program_options::variable_value(uint32_t(2), false)));
+      options.insert(std::make_pair("elasticsearch-start-es-after-block", boost::program_options::variable_value(uint32_t(0), false)));
+      options.insert(std::make_pair("elasticsearch-visitor", boost::program_options::variable_value(false, false)));
+      options.insert(std::make_pair("elasticsearch-operation-object", boost::program_options::variable_value(true, false)));
+      options.insert(std::make_pair("elasticsearch-operation-string", boost::program_options::variable_value(true, false)));
+      options.insert(std::make_pair("elasticsearch-mode", boost::program_options::variable_value(uint16_t(2), false)));
+
+      esplugin->plugin_initialize(options);
+      esplugin->plugin_startup();
+   }
+   else {
+      auto ahplugin = app.register_plugin<graphene::account_history::account_history_plugin>();
+      app.enable_plugin("affiliate_stats");
+      ahplugin->plugin_set_app(&app);
+      ahplugin->plugin_initialize(options);
+      ahplugin->plugin_startup();
+   }
+
+   if(test_name == "elasticsearch_objects" || test_name == "elasticsearch_suite") {
+      auto esobjects_plugin = app.register_plugin<graphene::es_objects::es_objects_plugin>();
+      esobjects_plugin->plugin_set_app(&app);
+
+      options.insert(std::make_pair("es-objects-elasticsearch-url", boost::program_options::variable_value(string("http://localhost:9200/"), false)));
+      options.insert(std::make_pair("es-objects-bulk-replay", boost::program_options::variable_value(uint32_t(2), false)));
+      options.insert(std::make_pair("es-objects-bulk-sync", boost::program_options::variable_value(uint32_t(2), false)));
+      options.insert(std::make_pair("es-objects-proposals", boost::program_options::variable_value(true, false)));
+      options.insert(std::make_pair("es-objects-accounts", boost::program_options::variable_value(true, false)));
+      options.insert(std::make_pair("es-objects-assets", boost::program_options::variable_value(true, false)));
+      options.insert(std::make_pair("es-objects-balances", boost::program_options::variable_value(true, false)));
+      options.insert(std::make_pair("es-objects-limit-orders", boost::program_options::variable_value(true, false)));
+      options.insert(std::make_pair("es-objects-asset-bitasset", boost::program_options::variable_value(true, false)));
+
+      esobjects_plugin->plugin_initialize(options);
+      esobjects_plugin->plugin_startup();
+   }
+
    mhplugin->plugin_set_app(&app);
    mhplugin->plugin_initialize(options);
    bookieplugin->plugin_set_app(&app);
@@ -143,7 +182,6 @@ database_fixture::database_fixture()
    affiliateplugin->plugin_set_app(&app);
    affiliateplugin->plugin_initialize(options);
 
-   ahplugin->plugin_startup();
    mhplugin->plugin_startup();
    bookieplugin->plugin_startup();
    affiliateplugin->plugin_startup();

tests/elasticsearch/main.cpp

@@ -0,0 +1,535 @@
+/*
+ * Copyright (c) 2018 oxarbitrage and contributors.
+ *
+ * The MIT License
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <graphene/app/api.hpp>
+#include <graphene/utilities/tempdir.hpp>
+#include <fc/crypto/digest.hpp>
+
+#include <graphene/utilities/elasticsearch.hpp>
+#include <graphene/elasticsearch/elasticsearch_plugin.hpp>
+
+#include "../common/database_fixture.hpp"
+
+#define BOOST_TEST_MODULE Elastic Search Database Tests
+#include <boost/test/included/unit_test.hpp>
+
+using namespace graphene::chain;
+using namespace graphene::chain::test;
+using namespace graphene::app;
+
+BOOST_FIXTURE_TEST_SUITE( elasticsearch_tests, database_fixture )
+
+BOOST_AUTO_TEST_CASE(elasticsearch_account_history) {
+   try {
+
+      CURL *curl; // curl handler
+      curl = curl_easy_init();
+
+      graphene::utilities::ES es;
+      es.curl = curl;
+      es.elasticsearch_url = "http://localhost:9200/";
+      es.index_prefix = "peerplays-";
+      //es.auth = "elastic:changeme";
+
+      // delete all first
+      auto delete_account_history = graphene::utilities::deleteAll(es);
+      fc::usleep(fc::milliseconds(1000)); // this is because index.refresh_interval, nothing to worry
+
+      if(delete_account_history) { // all records deleted
+
+         //account_id_type() do 3 ops
+         create_bitasset("USD", account_id_type());
+         auto dan = create_account("dan");
+         auto bob = create_account("bob");
+
+         generate_block();
+         fc::usleep(fc::milliseconds(1000));
+
+         // for later use
+         //int asset_crobjeate_op_id = operation::tag<asset_create_operation>::value;
+         //int account_create_op_id = operation::tag<account_create_operation>::value;
+
+         string query = "{ \"query\" : { \"bool\" : { \"must\" : [{\"match_all\": {}}] } } }";
+         es.endpoint = es.index_prefix + "*/data/_count";
+         es.query = query;
+
+         auto res = graphene::utilities::simpleQuery(es);
+         variant j = fc::json::from_string(res);
+         auto total = j["count"].as_string();
+         BOOST_CHECK_EQUAL(total, "5");
+
+         es.endpoint = es.index_prefix + "*/data/_search";
+         res = graphene::utilities::simpleQuery(es);
+         j = fc::json::from_string(res);
+         auto first_id = j["hits"]["hits"][size_t(0)]["_id"].as_string();
+         BOOST_CHECK_EQUAL(first_id, "2.9.0");
+
+         generate_block();
+         auto willie = create_account("willie");
+         generate_block();
+
+         fc::usleep(fc::milliseconds(1000)); // index.refresh_interval
+
+         es.endpoint = es.index_prefix + "*/data/_count";
+         res = graphene::utilities::simpleQuery(es);
+         j = fc::json::from_string(res);
+
+         total = j["count"].as_string();
+         BOOST_CHECK_EQUAL(total, "7");
+
+         // do some transfers in 1 block
+         transfer(account_id_type()(db), bob, asset(100));
+         transfer(account_id_type()(db), bob, asset(200));
+         transfer(account_id_type()(db), bob, asset(300));
+
+         generate_block();
+         fc::usleep(fc::milliseconds(1000)); // index.refresh_interval
+
+         res = graphene::utilities::simpleQuery(es);
+         j = fc::json::from_string(res);
+
+         total = j["count"].as_string();
+         BOOST_CHECK_EQUAL(total, "13");
+
+         // check the visitor data
+         auto block_date = db.head_block_time();
+         std::string index_name = graphene::utilities::generateIndexName(block_date, "peerplays-");
+
+         es.endpoint = index_name + "/data/2.9.12"; // we know last op is a transfer of amount 300
+         res = graphene::utilities::getEndPoint(es);
+         j = fc::json::from_string(res);
+         auto last_transfer_amount = j["_source"]["operation_history"]["op_object"]["amount_"]["amount"].as_string();
+         BOOST_CHECK_EQUAL(last_transfer_amount, "300");
+      }
+   }
+   catch (fc::exception &e) {
+      edump((e.to_detail_string()));
+      throw;
+   }
+}
+
+BOOST_AUTO_TEST_CASE(elasticsearch_objects) {
+   try {
+
+      CURL *curl; // curl handler
+      curl = curl_easy_init();
+
+      graphene::utilities::ES es;
+      es.curl = curl;
+      es.elasticsearch_url = "http://localhost:9200/";
+      es.index_prefix = "ppobjects-";
+      //es.auth = "elastic:changeme";
+
+      // delete all first
+      auto delete_objects = graphene::utilities::deleteAll(es);
+
+      generate_block();
+      fc::usleep(fc::milliseconds(1000));
+
+      if(delete_objects) { // all records deleted
+
+         // asset and bitasset
+         create_bitasset("USD", account_id_type());
+         generate_block();
+         fc::usleep(fc::milliseconds(1000));
+
+         string query = "{ \"query\" : { \"bool\" : { \"must\" : [{\"match_all\": {}}] } } }";
+         es.endpoint = es.index_prefix + "*/data/_count";
+         es.query = query;
+
+         auto res = graphene::utilities::simpleQuery(es);
+         variant j = fc::json::from_string(res);
+         auto total = j["count"].as_string();
+         BOOST_CHECK_EQUAL(total, "2");
+
+         es.endpoint = es.index_prefix + "asset/data/_search";
+         res = graphene::utilities::simpleQuery(es);
+         j = fc::json::from_string(res);
+         auto first_id = j["hits"]["hits"][size_t(0)]["_source"]["symbol"].as_string();
+         BOOST_CHECK_EQUAL(first_id, "USD");
+
+         auto bitasset_data_id = j["hits"]["hits"][size_t(0)]["_source"]["bitasset_data_id"].as_string();
+         es.endpoint = es.index_prefix + "bitasset/data/_search";
+         es.query = "{ \"query\" : { \"bool\": { \"must\" : [{ \"term\": { \"object_id\": \""+bitasset_data_id+"\"}}] } } }";
+         res = graphene::utilities::simpleQuery(es);
+         j = fc::json::from_string(res);
+         auto bitasset_object_id = j["hits"]["hits"][size_t(0)]["_source"]["object_id"].as_string();
+         BOOST_CHECK_EQUAL(bitasset_object_id, bitasset_data_id);
+      }
+   }
+   catch (fc::exception &e) {
+      edump((e.to_detail_string()));
+      throw;
+   }
+}
+
+BOOST_AUTO_TEST_CASE(elasticsearch_suite) {
+   try {
+
+      CURL *curl; // curl handler
+      curl = curl_easy_init();
+
+      graphene::utilities::ES es;
+      es.curl = curl;
+      es.elasticsearch_url = "http://localhost:9200/";
+      es.index_prefix = "peerplays-";
+      auto delete_account_history = graphene::utilities::deleteAll(es);
+      fc::usleep(fc::milliseconds(1000));
+      es.index_prefix = "ppobjects-";
+      auto delete_objects = graphene::utilities::deleteAll(es);
+      fc::usleep(fc::milliseconds(1000));
+
+      if(delete_account_history && delete_objects) { // all records deleted
+
+
+      }
+   }
+   catch (fc::exception &e) {
+      edump((e.to_detail_string()));
+      throw;
+   }
+}
+
+BOOST_AUTO_TEST_CASE(elasticsearch_history_api) {
+   try {
+      CURL *curl; // curl handler
+      curl = curl_easy_init();
+
+      graphene::utilities::ES es;
+      es.curl = curl;
+      es.elasticsearch_url = "http://localhost:9200/";
+      es.index_prefix = "peerplays-";
+
+      auto delete_account_history = graphene::utilities::deleteAll(es);
+
+      generate_block();
+      fc::usleep(fc::milliseconds(1000));
+
+      if(delete_account_history) {
+
+         create_bitasset("USD", account_id_type()); // create op 0
+         const account_object& dan = create_account("dan"); // create op 1
+         create_bitasset("CNY", dan.id); // create op 2
+         create_bitasset("BTC", account_id_type()); // create op 3
+         create_bitasset("XMR", dan.id); // create op 4
+         create_bitasset("EUR", account_id_type()); // create op 5
+         create_bitasset("OIL", dan.id); // create op 6
+
+         generate_block();
+         fc::usleep(fc::milliseconds(1000));
+
+         graphene::app::history_api hist_api(app);
+         app.enable_plugin("elasticsearch");
+
+         // f(A, 0, 4, 9) = { 5, 3, 1, 0 }
+         auto histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(9));
+
+         BOOST_CHECK_EQUAL(histories.size(), 4u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
+         BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u);
+
+         // f(A, 0, 4, 6) = { 5, 3, 1, 0 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(6));
+         BOOST_CHECK_EQUAL(histories.size(), 4u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
+         BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u);
+
+         // f(A, 0, 4, 5) = { 5, 3, 1, 0 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(5));
+         BOOST_CHECK_EQUAL(histories.size(), 4u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
+         BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u);
+
+         // f(A, 0, 4, 4) = { 3, 1, 0 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(4));
+         BOOST_CHECK_EQUAL(histories.size(), 3u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u);
+
+         // f(A, 0, 4, 3) = { 3, 1, 0 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(3));
+         BOOST_CHECK_EQUAL(histories.size(), 3u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u);
+
+         // f(A, 0, 4, 2) = { 1, 0 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(2));
+         BOOST_CHECK_EQUAL(histories.size(), 2u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u);
+
+         // f(A, 0, 4, 1) = { 1, 0 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(1));
+         BOOST_CHECK_EQUAL(histories.size(), 2u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u);
+
+         // f(A, 0, 4, 0) = { 5, 3, 1, 0 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type());
+         BOOST_CHECK_EQUAL(histories.size(), 4u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
+         BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u);
+
+         // f(A, 1, 5, 9) = { 5, 3 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(9));
+         BOOST_CHECK_EQUAL(histories.size(), 2u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
+
+         // f(A, 1, 5, 6) = { 5, 3 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(6));
+         BOOST_CHECK_EQUAL(histories.size(), 2u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
+
+         // f(A, 1, 5, 5) = { 5, 3 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(5));
+         BOOST_CHECK_EQUAL(histories.size(), 2u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
+
+         // f(A, 1, 5, 4) = { 3 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(4));
+         BOOST_CHECK_EQUAL(histories.size(), 1u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u);
+
+         // f(A, 1, 5, 3) = { 3 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(3));
+         BOOST_CHECK_EQUAL(histories.size(), 1u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u);
+
+         // f(A, 1, 5, 2) = { }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(2));
+         BOOST_CHECK_EQUAL(histories.size(), 0u);
+
+         // f(A, 1, 5, 1) = { }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(1));
+         BOOST_CHECK_EQUAL(histories.size(), 0u);
+
+         // f(A, 1, 5, 0) = { 5, 3 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(0));
+         BOOST_CHECK_EQUAL(histories.size(), 2u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
+
+         // f(A, 0, 3, 9) = { 5, 3, 1 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(9));
+         BOOST_CHECK_EQUAL(histories.size(), 3u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
+
+         // f(A, 0, 3, 6) = { 5, 3, 1 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(6));
+         BOOST_CHECK_EQUAL(histories.size(), 3u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
+
+         // f(A, 0, 3, 5) = { 5, 3, 1 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(5));
+         BOOST_CHECK_EQUAL(histories.size(), 3u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
+
+         // f(A, 0, 3, 4) = { 3, 1, 0 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(4));
+         BOOST_CHECK_EQUAL(histories.size(), 3u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u);
+
+         // f(A, 0, 3, 3) = { 3, 1, 0 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(3));
+         BOOST_CHECK_EQUAL(histories.size(), 3u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u);
+
+         // f(A, 0, 3, 2) = { 1, 0 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(2));
+         BOOST_CHECK_EQUAL(histories.size(), 2u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u);
+
+         // f(A, 0, 3, 1) = { 1, 0 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(1));
+         BOOST_CHECK_EQUAL(histories.size(), 2u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u);
+
+         // f(A, 0, 3, 0) = { 5, 3, 1 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type());
+         BOOST_CHECK_EQUAL(histories.size(), 3u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
+
+         // f(B, 0, 4, 9) = { 6, 4, 2, 1 }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(9));
+         BOOST_CHECK_EQUAL(histories.size(), 4u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 2u);
+         BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u);
+
+         // f(B, 0, 4, 6) = { 6, 4, 2, 1 }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(6));
+         BOOST_CHECK_EQUAL(histories.size(), 4u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 2u);
+         BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u);
+
+         // f(B, 0, 4, 5) = { 4, 2, 1 }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(5));
+         BOOST_CHECK_EQUAL(histories.size(), 3u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 2u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
+
+         // f(B, 0, 4, 4) = { 4, 2, 1 }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(4));
+         BOOST_CHECK_EQUAL(histories.size(), 3u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 2u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
+
+         // f(B, 0, 4, 3) = { 2, 1 }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(3));
+         BOOST_CHECK_EQUAL(histories.size(), 2u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 2u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u);
+
+         // f(B, 0, 4, 2) = { 2, 1 }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(2));
+         BOOST_CHECK_EQUAL(histories.size(), 2u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 2u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u);
+
+         // f(B, 0, 4, 1) = { 1 }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(1));
+         BOOST_CHECK_EQUAL(histories.size(), 1u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u);
+
+         // f(B, 0, 4, 0) = { 6, 4, 2, 1 }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type());
+         BOOST_CHECK_EQUAL(histories.size(), 4u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 2u);
+         BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u);
+
+         // f(B, 2, 4, 9) = { 6, 4 }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(9));
+         BOOST_CHECK_EQUAL(histories.size(), 2u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u);
+
+         // f(B, 2, 4, 6) = { 6, 4 }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(6));
+         BOOST_CHECK_EQUAL(histories.size(), 2u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u);
+
+         // f(B, 2, 4, 5) = { 4 }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(5));
+         BOOST_CHECK_EQUAL(histories.size(), 1u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u);
+
+         // f(B, 2, 4, 4) = { 4 }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(4));
+         BOOST_CHECK_EQUAL(histories.size(), 1u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u);
+
+         // f(B, 2, 4, 3) = { }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(3));
+         BOOST_CHECK_EQUAL(histories.size(), 0u);
+
+         // f(B, 2, 4, 2) = { }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(2));
+         BOOST_CHECK_EQUAL(histories.size(), 0u);
+
+         // f(B, 2, 4, 1) = { }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(1));
+         BOOST_CHECK_EQUAL(histories.size(), 0u);
+
+         // f(B, 2, 4, 0) = { 6, 4 }
+         histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(0));
+         BOOST_CHECK_EQUAL(histories.size(), 2u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u);
+
+         // 0 limits
+         histories = hist_api.get_account_history("dan", operation_history_id_type(0), 0, operation_history_id_type(0));
+         BOOST_CHECK_EQUAL(histories.size(), 0u);
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(3), 0, operation_history_id_type(9));
+         BOOST_CHECK_EQUAL(histories.size(), 0u);
+
+         // non existent account
+         histories = hist_api.get_account_history("1.2.18", operation_history_id_type(0), 4, operation_history_id_type(0));
+         BOOST_CHECK_EQUAL(histories.size(), 0u);
+
+         // create a new account C = alice { 7 }
+         auto alice = create_account("alice");
+
+         generate_block();
+         fc::usleep(fc::milliseconds(1000));
+
+         // f(C, 0, 4, 10) = { 7 }
+         histories = hist_api.get_account_history("alice", operation_history_id_type(0), 4, operation_history_id_type(10));
+         BOOST_CHECK_EQUAL(histories.size(), 1u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 7u);
+
+         // f(C, 8, 4, 10) = { }
+         histories = hist_api.get_account_history("alice", operation_history_id_type(8), 4, operation_history_id_type(10));
+         BOOST_CHECK_EQUAL(histories.size(), 0u);
+
+         // f(A, 0, 10, 0) = { 7, 5, 3, 1, 0 }
+         histories = hist_api.get_account_history("1.2.0", operation_history_id_type(0), 10, operation_history_id_type(0));
+         BOOST_CHECK_EQUAL(histories.size(), 5u);
+         BOOST_CHECK_EQUAL(histories[0].id.instance(), 7u);
+         BOOST_CHECK_EQUAL(histories[1].id.instance(), 5u);
+         BOOST_CHECK_EQUAL(histories[2].id.instance(), 3u);
+         BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u);
+         BOOST_CHECK_EQUAL(histories[4].id.instance(), 0u);
+      }
+   }
+   catch (fc::exception &e) {
+      edump((e.to_detail_string()));
+      throw;
+   }
+}
+BOOST_AUTO_TEST_SUITE_END()

tests/tests/block_tests.cpp

@@ -745,6 +745,8 @@ BOOST_FIXTURE_TEST_CASE( maintenance_interval, database_fixture )
          PUSH_TX( db, trx, ~0 );
          trx.operations.clear();
       }
+
+      generate_block();
       transfer(account_id_type()(db), nathan, asset(5000));
 
       generate_blocks(maintenence_time - initial_properties.parameters.block_interval);
@@ -959,18 +961,23 @@ BOOST_FIXTURE_TEST_CASE( pop_block_twice, database_fixture )
       processed_transaction ptx;
 
       account_object committee_account_object = committee_account(db);
+      generate_block(skip_flags);
       // transfer from committee account to Sam account
       transfer(committee_account_object, sam_account_object, core.amount(100000));
 
       generate_block(skip_flags);
 
-      create_account("alice");
+      private_key_type charlie_key = generate_private_key("charlie");
+      create_account("charlie", charlie_key);
       generate_block(skip_flags);
-      create_account("bob");
       generate_block(skip_flags);
-
+      private_key_type bob_key = generate_private_key("bob");
+      create_account("bob", bob_key);
+      generate_block(skip_flags);
+                 
       db.pop_block();
       db.pop_block();
+      
    } catch(const fc::exception& e) {
       edump( (e.to_detail_string()) );
       throw;

tests/tests/history_api_tests.cpp

@@ -595,4 +595,4 @@ BOOST_AUTO_TEST_CASE(get_account_history_operations) {
    }
 }
 
-BOOST_AUTO_TEST_SUITE_END()
+BOOST_AUTO_TEST_SUITE_END()