Peerplays_Blockchain/peerplays_migrated
1
0
Fork 0
Code Issues 35 Pull requests 4 Projects Releases 20 Packages Wiki Activity Actions

237-es7-fix - Switch type from 'data' to default '_doc' for ES

Browse source
This commit is contained in:
Vladislav Dobromyslov 2022-01-05 19:30:28 +03:00
parent df2696e49a
commit d04e0c1c99
3 changed files with 36 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
libraries/plugins/elasticsearch/elasticsearch_plugin.cpp
View file

@ -75,6 +75,8 @@ class elasticsearch_plugin_impl
std::string bulk_line;
std::string index_name;
bool is_sync = false;
bool is_es_version_7_or_above = true;
private:
bool add_elasticsearch( const account_id_type account_id, const optional<operation_history_object>& oho, const uint32_t block_number );
const account_transaction_history_object& addNewEntry(const account_statistics_object& stats_obj,
@ -542,7 +544,8 @@ void elasticsearch_plugin_impl::prepareBulk(const account_transaction_history_id
const std::string _id = fc::json::to_string(ath_id);
fc::mutable_variant_object bulk_header;
bulk_header["_index"] = index_name;
bulk_header["_type"] = "data";
if( !is_es_version_7_or_above )
bulk_header["_type"] = "_doc";
bulk_header["_id"] = fc::to_string(ath_id.space_id) + "." + fc::to_string(ath_id.type_id) + "."
+ fc::to_string(ath_id.instance.value);
prepare = graphene::utilities::createBulk(bulk_header, std::move(bulk_line));
@ -690,10 +693,7 @@ void elasticsearch_plugin::plugin_initialize(const boost::program_options::varia
"Error populating ES database, we are going to keep trying.");
});
}
}
void elasticsearch_plugin::plugin_startup()
{
graphene::utilities::ES es;
es.curl = my->curl;
es.elasticsearch_url = my->_elasticsearch_node_url;
@ -701,6 +701,16 @@ void elasticsearch_plugin::plugin_startup()
if(!graphene::utilities::checkES(es))
FC_THROW_EXCEPTION(fc::exception, "ES database is not up in url ${url}", ("url", my->_elasticsearch_node_url));
graphene::utilities::checkESVersion7OrAbove(es, my->is_es_version_7_or_above);
ilog("elasticsearch ACCOUNT HISTORY: plugin_initialize() begin");
}
void elasticsearch_plugin::plugin_startup()
{
// Nothing to do
ilog("elasticsearch ACCOUNT HISTORY: plugin_startup() begin");
}
@ -817,7 +827,7 @@ graphene::utilities::ES elasticsearch_plugin::prepareHistoryQuery(string query)
es.curl = curl;
es.elasticsearch_url = my->_elasticsearch_node_url;
es.index_prefix = my->_elasticsearch_index_prefix;
es.endpoint = es.index_prefix + "*/data/_search";
es.endpoint = es.index_prefix + "*/_doc/_search";
es.query = query;
return es;

19
libraries/plugins/es_objects/es_objects.cpp
View file

@ -97,6 +97,7 @@ class es_objects_plugin_impl
uint32_t block_number;
fc::time_point_sec block_time;
bool is_es_version_7_or_above = true;
private:
template<typename T>
@ -523,7 +524,8 @@ void es_objects_plugin_impl::remove_from_database( object_id_type id, std::strin
fc::mutable_variant_object delete_line;
delete_line["_id"] = string(id);
delete_line["_index"] = _es_objects_index_prefix + index;
delete_line["_type"] = "data";
if( !is_es_version_7_or_above )
delete_line["_type"] = "_doc";
fc::mutable_variant_object final_delete_line;
final_delete_line["delete"] = delete_line;
prepare.push_back(fc::json::to_string(final_delete_line));
@ -537,7 +539,8 @@ void es_objects_plugin_impl::prepareTemplate(T blockchain_object, string index_n
{
fc::mutable_variant_object bulk_header;
bulk_header["_index"] = _es_objects_index_prefix + index_name;
bulk_header["_type"] = "data";
if( !is_es_version_7_or_above )
bulk_header["_type"] = "_doc";
if(_es_objects_keep_only_current)
{
bulk_header["_id"] = string(blockchain_object.id);
@ -721,10 +724,7 @@ void es_objects_plugin::plugin_initialize(const boost::program_options::variable
"Error deleting object from ES database, we are going to keep trying.");
}
});
}
void es_objects_plugin::plugin_startup()
{
graphene::utilities::ES es;
es.curl = my->curl;
es.elasticsearch_url = my->_es_objects_elasticsearch_url;
@ -733,6 +733,15 @@ void es_objects_plugin::plugin_startup()
if(!graphene::utilities::checkES(es))
FC_THROW_EXCEPTION(fc::exception, "ES database is not up in url ${url}", ("url", my->_es_objects_elasticsearch_url));
graphene::utilities::checkESVersion7OrAbove(es, my->is_es_version_7_or_above);
ilog("elasticsearch OBJECTS: plugin_initialize() begin");
}
void es_objects_plugin::plugin_startup()
{
// Nothing to do
ilog("elasticsearch OBJECTS: plugin_startup() begin");
}

14
tests/elasticsearch/main.cpp
View file

@ -71,7 +71,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_account_history) {
//int account_create_op_id = operation::tag<account_create_operation>::value;
string query = "{ \"query\" : { \"bool\" : { \"must\" : [{\"match_all\": {}}] } } }";
es.endpoint = es.index_prefix + "*/data/_count";
es.endpoint = es.index_prefix + "*/_doc/_count";
es.query = query;
auto res = graphene::utilities::simpleQuery(es);
@ -79,7 +79,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_account_history) {
auto total = j["count"].as_string();
BOOST_CHECK_EQUAL(total, "5");
es.endpoint = es.index_prefix + "*/data/_search";
es.endpoint = es.index_prefix + "*/_doc/_search";
res = graphene::utilities::simpleQuery(es);
j = fc::json::from_string(res);
auto first_id = j["hits"]["hits"][size_t(0)]["_id"].as_string();
@ -91,7 +91,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_account_history) {
fc::usleep(fc::milliseconds(1000)); // index.refresh_interval
es.endpoint = es.index_prefix + "*/data/_count";
es.endpoint = es.index_prefix + "*/_doc/_count";
res = graphene::utilities::simpleQuery(es);
j = fc::json::from_string(res);
@ -116,7 +116,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_account_history) {
auto block_date = db.head_block_time();
std::string index_name = graphene::utilities::generateIndexName(block_date, "peerplays-");
es.endpoint = index_name + "/data/2.9.12"; // we know last op is a transfer of amount 300
es.endpoint = index_name + "/_doc/2.9.12"; // we know last op is a transfer of amount 300
res = graphene::utilities::getEndPoint(es);
j = fc::json::from_string(res);
auto last_transfer_amount = j["_source"]["operation_history"]["op_object"]["amount_"]["amount"].as_string();
@ -155,7 +155,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) {
fc::usleep(fc::milliseconds(1000));
string query = "{ \"query\" : { \"bool\" : { \"must\" : [{\"match_all\": {}}] } } }";
es.endpoint = es.index_prefix + "*/data/_count";
es.endpoint = es.index_prefix + "*/_doc/_count";
es.query = query;
auto res = graphene::utilities::simpleQuery(es);
@ -163,14 +163,14 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) {
auto total = j["count"].as_string();
BOOST_CHECK_EQUAL(total, "2");
es.endpoint = es.index_prefix + "asset/data/_search";
es.endpoint = es.index_prefix + "asset/_doc/_search";
res = graphene::utilities::simpleQuery(es);
j = fc::json::from_string(res);
auto first_id = j["hits"]["hits"][size_t(0)]["_source"]["symbol"].as_string();
BOOST_CHECK_EQUAL(first_id, "USD");
auto bitasset_data_id = j["hits"]["hits"][size_t(0)]["_source"]["bitasset_data_id"].as_string();
es.endpoint = es.index_prefix + "bitasset/data/_search";
es.endpoint = es.index_prefix + "bitasset/_doc/_search";
es.query = "{ \"query\" : { \"bool\": { \"must\" : [{ \"term\": { \"object_id\": \""+bitasset_data_id+"\"}}] } } }";
res = graphene::utilities::simpleQuery(es);
j = fc::json::from_string(res);