Peerplays_Blockchain/peerplays_migrated
1
0
Fork 0
Code Issues 35 Pull requests 4 Projects Releases 20 Packages Wiki Activity Actions

Merge pull request #1541 from oxarbitrage/es_objects_start_after_block

Browse source 
add es-objects-start-es-after-block option
This commit is contained in:
Alfredo Garcia 2019-01-30 11:46:17 -03:00 committed by gladcow
parent 837b665ef9
commit f2da1f4a5b
2 changed files with 86 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
libraries/plugins/elasticsearch/elasticsearch_plugin.cpp
View file

@ -59,7 +59,7 @@ class elasticsearch_plugin_impl
std::string _elasticsearch_basic_auth = ""; std::string _elasticsearch_basic_auth = "";
std::string _elasticsearch_index_prefix = "bitshares-"; std::string _elasticsearch_index_prefix = "bitshares-";
bool _elasticsearch_operation_object = false; bool _elasticsearch_operation_object = false;
uint32_t _elasticsearch_start_es_after_block = 0; // disabled uint32_t _elasticsearch_start_es_after_block = 0;
CURL *curl; // curl handler CURL *curl; // curl handler
vector <string> bulk_lines; // vector of op lines vector <string> bulk_lines; // vector of op lines
vector<std::string> prepare; vector<std::string> prepare;
@ -254,7 +254,7 @@ bool elasticsearch_plugin_impl::add_elasticsearch( const account_id_type account
const auto &stats_obj = getStatsObject(account_id); const auto &stats_obj = getStatsObject(account_id);
const auto &ath = addNewEntry(stats_obj, account_id, oho); const auto &ath = addNewEntry(stats_obj, account_id, oho);
growStats(stats_obj, ath); growStats(stats_obj, ath);
if(_elasticsearch_start_es_after_block == 0 || block_number > _elasticsearch_start_es_after_block) { if(block_number > _elasticsearch_start_es_after_block) {
createBulkLine(ath); createBulkLine(ath);
prepareBulk(ath.id); prepareBulk(ath.id);
} }

24
libraries/plugins/es_objects/es_objects.cpp
View file

@ -63,6 +63,7 @@ class es_objects_plugin_impl
bool _es_objects_limit_orders = true; bool _es_objects_limit_orders = true;
bool _es_objects_asset_bitasset = true; bool _es_objects_asset_bitasset = true;
std::string _es_objects_index_prefix = "objects-"; std::string _es_objects_index_prefix = "objects-";
uint32_t _es_objects_start_es_after_block = 0;
CURL *curl; // curl handler CURL *curl; // curl handler
vector <std::string> bulk; vector <std::string> bulk;
vector<std::string> prepare; vector<std::string> prepare;
@ -84,6 +85,8 @@ bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids,
block_time = db.head_block_time(); block_time = db.head_block_time();
block_number = db.head_block_num(); block_number = db.head_block_num();
if(block_number > _es_objects_start_es_after_block) {
// check if we are in replay or in sync and change number of bulk documents accordingly // check if we are in replay or in sync and change number of bulk documents accordingly
uint32_t limit_documents = 0; uint32_t limit_documents = 0;
if ((fc::time_point::now() - block_time) < fc::seconds(30)) if ((fc::time_point::now() - block_time) < fc::seconds(30))
@ -91,6 +94,7 @@ bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids,
else else
limit_documents = _es_objects_bulk_replay; limit_documents = _es_objects_bulk_replay;
for (auto const &value: ids) { for (auto const &value: ids) {
if (value.is<proposal_object>() && _es_objects_proposals) { if (value.is<proposal_object>() && _es_objects_proposals) {
auto obj = db.find_object(value); auto obj = db.find_object(value);
@ -101,8 +105,7 @@ bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids,
else else
prepareTemplate<proposal_object>(*p, "proposal"); prepareTemplate<proposal_object>(*p, "proposal");
} }
} } else if (value.is<account_object>() && _es_objects_accounts) {
else if(value.is<account_object>() && _es_objects_accounts) {
auto obj = db.find_object(value); auto obj = db.find_object(value);
auto a = static_cast<const account_object *>(obj); auto a = static_cast<const account_object *>(obj);
if (a != nullptr) { if (a != nullptr) {
@ -111,8 +114,7 @@ bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids,
else else
prepareTemplate<account_object>(*a, "account"); prepareTemplate<account_object>(*a, "account");
} }
} } else if (value.is<asset_object>() && _es_objects_assets) {
else if(value.is<asset_object>() && _es_objects_assets) {
auto obj = db.find_object(value); auto obj = db.find_object(value);
auto a = static_cast<const asset_object *>(obj); auto a = static_cast<const asset_object *>(obj);
if (a != nullptr) { if (a != nullptr) {
@ -121,8 +123,7 @@ bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids,
else else
prepareTemplate<asset_object>(*a, "asset"); prepareTemplate<asset_object>(*a, "asset");
} }
} } else if (value.is<account_balance_object>() && _es_objects_balances) {
else if(value.is<account_balance_object>() && _es_objects_balances) {
auto obj = db.find_object(value); auto obj = db.find_object(value);
auto b = static_cast<const account_balance_object *>(obj); auto b = static_cast<const account_balance_object *>(obj);
if (b != nullptr) { if (b != nullptr) {
@ -131,8 +132,7 @@ bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids,
else else
prepareTemplate<account_balance_object>(*b, "balance"); prepareTemplate<account_balance_object>(*b, "balance");
} }
} } else if (value.is<limit_order_object>() && _es_objects_limit_orders) {
else if(value.is<limit_order_object>() && _es_objects_limit_orders) {
auto obj = db.find_object(value); auto obj = db.find_object(value);
auto l = static_cast<const limit_order_object *>(obj); auto l = static_cast<const limit_order_object *>(obj);
if (l != nullptr) { if (l != nullptr) {
@ -141,8 +141,7 @@ bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids,
else else
prepareTemplate<limit_order_object>(*l, "limitorder"); prepareTemplate<limit_order_object>(*l, "limitorder");
} }
} } else if (value.is<asset_bitasset_data_object>() && _es_objects_asset_bitasset) {
else if(value.is<asset_bitasset_data_object>() && _es_objects_asset_bitasset) {
auto obj = db.find_object(value); auto obj = db.find_object(value);
auto ba = static_cast<const asset_bitasset_data_object *>(obj); auto ba = static_cast<const asset_bitasset_data_object *>(obj);
if (ba != nullptr) { if (ba != nullptr) {
@ -167,6 +166,7 @@ bool es_objects_plugin_impl::index_database( const vector<object_id_type>& ids,
else else
bulk.clear(); bulk.clear();
} }
}
return true; return true;
} }
@ -257,6 +257,7 @@ void es_objects_plugin::plugin_set_program_options(
("es-objects-asset-bitasset", boost::program_options::value<bool>(), "Store feed data(true)") ("es-objects-asset-bitasset", boost::program_options::value<bool>(), "Store feed data(true)")
("es-objects-index-prefix", boost::program_options::value<std::string>(), "Add a prefix to the index(objects-)") ("es-objects-index-prefix", boost::program_options::value<std::string>(), "Add a prefix to the index(objects-)")
("es-objects-keep-only-current", boost::program_options::value<bool>(), "Keep only current state of the objects(true)") ("es-objects-keep-only-current", boost::program_options::value<bool>(), "Keep only current state of the objects(true)")
("es-objects-start-es-after-block", boost::program_options::value<uint32_t>(), "Start doing ES job after block(0)")
; ;
cfg.add(cli); cfg.add(cli);
} }
@ -319,6 +320,9 @@ void es_objects_plugin::plugin_initialize(const boost::program_options::variable
if (options.count("es-objects-keep-only-current")) { if (options.count("es-objects-keep-only-current")) {
my->_es_objects_keep_only_current = options["es-objects-keep-only-current"].as<bool>(); my->_es_objects_keep_only_current = options["es-objects-keep-only-current"].as<bool>();
} }
if (options.count("es-objects-start-es-after-block")) {
my->_es_objects_start_es_after_block = options["es-objects-start-es-after-block"].as<uint32_t>();
}
} }
void es_objects_plugin::plugin_startup() void es_objects_plugin::plugin_startup()