Merge pull request #1725 from oxarbitrage/issue1682
elasticsearch history api #1682
This commit is contained in:
Alfredo Garcia
gladcow
parent
de4faee7f0
commit
62247c543d
9 changed files with 634 additions and 23 deletions
|
|
@ -13,7 +13,7 @@ add_library( graphene_app
|
|||
|
||||
# need to link graphene_debug_witness because plugins aren't sufficiently isolated #246
|
||||
#target_link_libraries( graphene_app graphene_market_history graphene_account_history graphene_chain fc graphene_db graphene_net graphene_utilities graphene_debug_witness )
|
||||
target_link_libraries( graphene_app graphene_market_history graphene_account_history graphene_accounts_list graphene_affiliate_stats graphene_chain fc graphene_db graphene_net graphene_time graphene_utilities graphene_debug_witness graphene_bookie )
|
||||
target_link_libraries( graphene_app graphene_market_history graphene_account_history graphene_accounts_list graphene_affiliate_stats graphene_chain fc graphene_db graphene_net graphene_time graphene_utilities graphene_debug_witness graphene_bookie graphene_elasticsearch )
|
||||
target_include_directories( graphene_app
|
||||
PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/../egenesis/include" )
|
||||
|
|
|
|||
|
|
@ -580,6 +580,18 @@ namespace graphene { namespace app {
|
|||
start = node.operation_id;
|
||||
} catch(...) { return result; }
|
||||
|
||||
if(_app.is_plugin_enabled("elasticsearch")) {
|
||||
auto es = _app.get_plugin<elasticsearch::elasticsearch_plugin>("elasticsearch");
|
||||
if(es.get()->get_running_mode() != elasticsearch::mode::only_save) {
|
||||
if(!_app.elasticsearch_thread)
|
||||
_app.elasticsearch_thread= std::make_shared<fc::thread>("elasticsearch");
|
||||
|
||||
return _app.elasticsearch_thread->async([&es, &account, &stop, &limit, &start]() {
|
||||
return es->get_account_history(account, stop, limit, start);
|
||||
}, "thread invoke for method " BOOST_PP_STRINGIZE(method_name)).wait();
|
||||
}
|
||||
}
|
||||
|
||||
const auto& hist_idx = db.get_index_type<account_transaction_history_index>();
|
||||
const auto& by_op_idx = hist_idx.indices().get<by_op>();
|
||||
auto index_start = by_op_idx.begin();
|
||||
|
|
|
|||
|
|
@ -1026,6 +1026,11 @@ std::shared_ptr<abstract_plugin> application::get_plugin(const string& name) con
|
|||
return my->_active_plugins[name];
|
||||
}
|
||||
|
||||
bool application::is_plugin_enabled(const string& name) const
|
||||
{
|
||||
return !(my->_active_plugins.find(name) == my->_active_plugins.end());
|
||||
}
|
||||
|
||||
net::node_ptr application::p2p_node()
|
||||
{
|
||||
return my->_p2p_network;
|
||||
|
|
|
|||
|
|
@ -31,6 +31,8 @@
|
|||
#include <graphene/market_history/market_history_plugin.hpp>
|
||||
#include <graphene/accounts_list/accounts_list_plugin.hpp>
|
||||
|
||||
#include <graphene/elasticsearch/elasticsearch_plugin.hpp>
|
||||
|
||||
#include <graphene/debug_witness/debug_api.hpp>
|
||||
#include <graphene/affiliate_stats/affiliate_stats_api.hpp>
|
||||
#include <graphene/bookie/bookie_api.hpp>
|
||||
|
|
|
|||
|
|
@ -89,8 +89,13 @@ namespace graphene { namespace app {
|
|||
/// Emitted when syncing finishes (is_finished_syncing will return true)
|
||||
boost::signals2::signal<void()> syncing_finished;
|
||||
|
||||
private:
|
||||
void enable_plugin( const string& name );
|
||||
|
||||
bool is_plugin_enabled(const string& name) const;
|
||||
|
||||
std::shared_ptr<fc::thread> elasticsearch_thread;
|
||||
|
||||
private:
|
||||
void add_available_plugin( std::shared_ptr<abstract_plugin> p );
|
||||
std::shared_ptr<detail::application_impl> my;
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@
|
|||
#include <graphene/chain/account_evaluator.hpp>
|
||||
#include <fc/smart_ref_impl.hpp>
|
||||
#include <curl/curl.h>
|
||||
#include <graphene/utilities/elasticsearch.hpp>
|
||||
|
||||
namespace graphene { namespace elasticsearch {
|
||||
|
||||
|
|
@ -60,6 +59,8 @@ class elasticsearch_plugin_impl
|
|||
std::string _elasticsearch_index_prefix = "bitshares-";
|
||||
bool _elasticsearch_operation_object = false;
|
||||
uint32_t _elasticsearch_start_es_after_block = 0;
|
||||
bool _elasticsearch_operation_string = true;
|
||||
mode _elasticsearch_mode = mode::only_save;
|
||||
CURL *curl; // curl handler
|
||||
vector <string> bulk_lines; // vector of op lines
|
||||
vector<std::string> prepare;
|
||||
|
|
@ -215,7 +216,14 @@ void elasticsearch_plugin_impl::doOperationHistory(const optional <operation_his
|
|||
os.op_in_trx = oho->op_in_trx;
|
||||
os.operation_result = fc::json::to_string(oho->result);
|
||||
os.virtual_op = oho->virtual_op;
|
||||
os.op = fc::json::to_string(oho->op);
|
||||
|
||||
if(_elasticsearch_operation_object) {
|
||||
oho->op.visit(fc::from_static_variant(os.op_object, FC_PACK_MAX_DEPTH));
|
||||
adaptor_struct adaptor;
|
||||
os.op_object = adaptor.adapt(os.op_object.get_object());
|
||||
}
|
||||
if(_elasticsearch_operation_string)
|
||||
os.op = fc::json::to_string(oho->op);
|
||||
}
|
||||
|
||||
void elasticsearch_plugin_impl::doBlock(const optional <operation_history_object>& oho, const signed_block& b)
|
||||
|
|
@ -394,25 +402,32 @@ void elasticsearch_plugin::plugin_set_program_options(
|
|||
)
|
||||
{
|
||||
cli.add_options()
|
||||
("elasticsearch-node-url", boost::program_options::value<std::string>(), "Elastic Search database node url")
|
||||
("elasticsearch-bulk-replay", boost::program_options::value<uint32_t>(), "Number of bulk documents to index on replay(5000)")
|
||||
("elasticsearch-bulk-sync", boost::program_options::value<uint32_t>(), "Number of bulk documents to index on a syncronied chain(10)")
|
||||
("elasticsearch-visitor", boost::program_options::value<bool>(), "Use visitor to index additional data(slows down the replay)")
|
||||
("elasticsearch-basic-auth", boost::program_options::value<std::string>(), "Pass basic auth to elasticsearch database ")
|
||||
("elasticsearch-index-prefix", boost::program_options::value<std::string>(), "Add a prefix to the index(bitshares-)")
|
||||
("elasticsearch-operation-object", boost::program_options::value<bool>(), "Save operation as object(false)")
|
||||
("elasticsearch-start-es-after-block", boost::program_options::value<uint32_t>(), "Start doing ES job after block(0)")
|
||||
("elasticsearch-node-url", boost::program_options::value<std::string>(),
|
||||
"Elastic Search database node url(http://localhost:9200/)")
|
||||
("elasticsearch-bulk-replay", boost::program_options::value<uint32_t>(),
|
||||
"Number of bulk documents to index on replay(10000)")
|
||||
("elasticsearch-bulk-sync", boost::program_options::value<uint32_t>(),
|
||||
"Number of bulk documents to index on a syncronied chain(100)")
|
||||
("elasticsearch-visitor", boost::program_options::value<bool>(),
|
||||
"Use visitor to index additional data(slows down the replay(false))")
|
||||
("elasticsearch-basic-auth", boost::program_options::value<std::string>(),
|
||||
"Pass basic auth to elasticsearch database('')")
|
||||
("elasticsearch-index-prefix", boost::program_options::value<std::string>(),
|
||||
"Add a prefix to the index(bitshares-)")
|
||||
("elasticsearch-operation-object", boost::program_options::value<bool>(),
|
||||
"Save operation as object(false)")
|
||||
("elasticsearch-start-es-after-block", boost::program_options::value<uint32_t>(),
|
||||
"Start doing ES job after block(0)")
|
||||
("elasticsearch-operation-string", boost::program_options::value<bool>(),
|
||||
"Save operation as string. Needed to serve history api calls(true)")
|
||||
("elasticsearch-mode", boost::program_options::value<uint16_t>(),
|
||||
"Mode of operation: only_save(0), only_query(1), all(2) - Default: 0")
|
||||
;
|
||||
cfg.add(cli);
|
||||
}
|
||||
|
||||
void elasticsearch_plugin::plugin_initialize(const boost::program_options::variables_map& options)
|
||||
{
|
||||
database().applied_block.connect( [&]( const signed_block& b) {
|
||||
if (!my->update_account_histories(b))
|
||||
FC_THROW_EXCEPTION(fc::exception, "Error populating ES database, we are going to keep trying.");
|
||||
} );
|
||||
|
||||
my->_oho_index = database().add_index< primary_index< operation_history_index > >();
|
||||
database().add_index< primary_index< account_transaction_history_index > >();
|
||||
|
||||
|
|
@ -439,7 +454,28 @@ void elasticsearch_plugin::plugin_initialize(const boost::program_options::varia
|
|||
}
|
||||
if (options.count("elasticsearch-start-es-after-block")) {
|
||||
my->_elasticsearch_start_es_after_block = options["elasticsearch-start-es-after-block"].as<uint32_t>();
|
||||
}
|
||||
}
|
||||
if (options.count("elasticsearch-operation-string")) {
|
||||
my->_elasticsearch_operation_string = options["elasticsearch-operation-string"].as<bool>();
|
||||
}
|
||||
if (options.count("elasticsearch-mode")) {
|
||||
const auto option_number = options["elasticsearch-mode"].as<uint16_t>();
|
||||
if(option_number > mode::all)
|
||||
FC_THROW_EXCEPTION(fc::exception, "Elasticsearch mode not valid");
|
||||
my->_elasticsearch_mode = static_cast<mode>(options["elasticsearch-mode"].as<uint16_t>());
|
||||
}
|
||||
|
||||
if(my->_elasticsearch_mode != mode::only_query) {
|
||||
if (my->_elasticsearch_mode == mode::all && !my->_elasticsearch_operation_string)
|
||||
FC_THROW_EXCEPTION(fc::exception,
|
||||
"If elasticsearch-mode is set to all then elasticsearch-operation-string need to be true");
|
||||
|
||||
database().applied_block.connect([this](const signed_block &b) {
|
||||
if (!my->update_account_histories(b))
|
||||
FC_THROW_EXCEPTION(fc::exception,
|
||||
"Error populating ES database, we are going to keep trying.");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void elasticsearch_plugin::plugin_startup()
|
||||
|
|
@ -454,4 +490,124 @@ void elasticsearch_plugin::plugin_startup()
|
|||
ilog("elasticsearch ACCOUNT HISTORY: plugin_startup() begin");
|
||||
}
|
||||
|
||||
operation_history_object elasticsearch_plugin::get_operation_by_id(operation_history_id_type id)
|
||||
{
|
||||
const string operation_id_string = std::string(object_id_type(id));
|
||||
|
||||
const string query = R"(
|
||||
{
|
||||
"query": {
|
||||
"match":
|
||||
{
|
||||
"account_history.operation_id": )" + operation_id_string + R"("
|
||||
}
|
||||
}
|
||||
}
|
||||
)";
|
||||
|
||||
auto es = prepareHistoryQuery(query);
|
||||
const auto response = graphene::utilities::simpleQuery(es);
|
||||
variant variant_response = fc::json::from_string(response);
|
||||
const auto source = variant_response["hits"]["hits"][size_t(0)]["_source"];
|
||||
return fromEStoOperation(source);
|
||||
}
|
||||
|
||||
vector<operation_history_object> elasticsearch_plugin::get_account_history(
|
||||
const account_id_type account_id,
|
||||
operation_history_id_type stop = operation_history_id_type(),
|
||||
unsigned limit = 100,
|
||||
operation_history_id_type start = operation_history_id_type())
|
||||
{
|
||||
const string account_id_string = std::string(object_id_type(account_id));
|
||||
|
||||
const auto stop_number = stop.instance.value;
|
||||
const auto start_number = start.instance.value;
|
||||
|
||||
string range = "";
|
||||
if(stop_number == 0)
|
||||
range = " AND operation_id_num: ["+fc::to_string(stop_number)+" TO "+fc::to_string(start_number)+"]";
|
||||
else if(stop_number > 0)
|
||||
range = " AND operation_id_num: {"+fc::to_string(stop_number)+" TO "+fc::to_string(start_number)+"]";
|
||||
|
||||
const string query = R"(
|
||||
{
|
||||
"size": )" + fc::to_string(limit) + R"(,
|
||||
"sort" : [{ "operation_id_num" : {"order" : "desc"}}],
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"query_string": {
|
||||
"query": "account_history.account: )" + account_id_string + range + R"("
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
)";
|
||||
|
||||
auto es = prepareHistoryQuery(query);
|
||||
|
||||
vector<operation_history_object> result;
|
||||
|
||||
if(!graphene::utilities::checkES(es))
|
||||
return result;
|
||||
|
||||
const auto response = graphene::utilities::simpleQuery(es);
|
||||
variant variant_response = fc::json::from_string(response);
|
||||
|
||||
const auto hits = variant_response["hits"]["total"]["value"];
|
||||
const auto size = std::min(static_cast<uint32_t>(hits.as_uint64()), limit);
|
||||
|
||||
for(unsigned i=0; i<size; i++)
|
||||
{
|
||||
const auto source = variant_response["hits"]["hits"][size_t(i)]["_source"];
|
||||
result.push_back(fromEStoOperation(source));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
operation_history_object elasticsearch_plugin::fromEStoOperation(variant source)
|
||||
{
|
||||
operation_history_object result;
|
||||
|
||||
const auto operation_id = source["account_history"]["operation_id"];
|
||||
fc::from_variant( operation_id, result.id, GRAPHENE_MAX_NESTED_OBJECTS );
|
||||
|
||||
const auto op = fc::json::from_string(source["operation_history"]["op"].as_string());
|
||||
fc::from_variant( op, result.op, GRAPHENE_MAX_NESTED_OBJECTS );
|
||||
|
||||
const auto operation_result = fc::json::from_string(source["operation_history"]["operation_result"].as_string());
|
||||
fc::from_variant( operation_result, result.result, GRAPHENE_MAX_NESTED_OBJECTS );
|
||||
|
||||
result.block_num = source["block_data"]["block_num"].as_uint64();
|
||||
result.trx_in_block = source["operation_history"]["trx_in_block"].as_uint64();
|
||||
result.op_in_trx = source["operation_history"]["op_in_trx"].as_uint64();
|
||||
result.trx_in_block = source["operation_history"]["virtual_op"].as_uint64();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
graphene::utilities::ES elasticsearch_plugin::prepareHistoryQuery(string query)
|
||||
{
|
||||
CURL *curl;
|
||||
curl = curl_easy_init();
|
||||
|
||||
graphene::utilities::ES es;
|
||||
es.curl = curl;
|
||||
es.elasticsearch_url = my->_elasticsearch_node_url;
|
||||
es.index_prefix = my->_elasticsearch_index_prefix;
|
||||
es.endpoint = es.index_prefix + "*/data/_search";
|
||||
es.query = query;
|
||||
|
||||
return es;
|
||||
}
|
||||
|
||||
mode elasticsearch_plugin::get_running_mode()
|
||||
{
|
||||
return my->_elasticsearch_mode;
|
||||
}
|
||||
|
||||
|
||||
} }
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@
|
|||
#include <graphene/app/plugin.hpp>
|
||||
#include <graphene/chain/database.hpp>
|
||||
#include <graphene/chain/operation_history_object.hpp>
|
||||
#include <graphene/utilities/elasticsearch.hpp>
|
||||
|
||||
namespace graphene { namespace elasticsearch {
|
||||
using namespace chain;
|
||||
|
|
@ -49,6 +50,8 @@ namespace detail
|
|||
class elasticsearch_plugin_impl;
|
||||
}
|
||||
|
||||
enum mode { only_save = 0 , only_query = 1, all = 2 };
|
||||
|
||||
class elasticsearch_plugin : public graphene::app::plugin
|
||||
{
|
||||
public:
|
||||
|
|
@ -63,10 +66,20 @@ class elasticsearch_plugin : public graphene::app::plugin
|
|||
virtual void plugin_initialize(const boost::program_options::variables_map& options) override;
|
||||
virtual void plugin_startup() override;
|
||||
|
||||
operation_history_object get_operation_by_id(operation_history_id_type id);
|
||||
vector<operation_history_object> get_account_history(const account_id_type account_id,
|
||||
operation_history_id_type stop, unsigned limit, operation_history_id_type start);
|
||||
mode get_running_mode();
|
||||
|
||||
friend class detail::elasticsearch_plugin_impl;
|
||||
std::unique_ptr<detail::elasticsearch_plugin_impl> my;
|
||||
|
||||
private:
|
||||
operation_history_object fromEStoOperation(variant source);
|
||||
graphene::utilities::ES prepareHistoryQuery(string query);
|
||||
};
|
||||
|
||||
|
||||
struct operation_visitor
|
||||
{
|
||||
typedef void result_type;
|
||||
|
|
@ -128,6 +141,7 @@ struct operation_history_struct {
|
|||
std::string operation_result;
|
||||
int virtual_op;
|
||||
std::string op;
|
||||
variant op_object;
|
||||
};
|
||||
|
||||
struct block_struct {
|
||||
|
|
@ -174,9 +188,99 @@ struct bulk_struct {
|
|||
optional<visitor_struct> additional_data;
|
||||
};
|
||||
|
||||
struct adaptor_struct {
|
||||
variant adapt(const variant_object& op)
|
||||
{
|
||||
fc::mutable_variant_object o(op);
|
||||
vector<string> keys_to_rename;
|
||||
for (auto i = o.begin(); i != o.end(); ++i)
|
||||
{
|
||||
auto& element = (*i).value();
|
||||
if (element.is_object())
|
||||
{
|
||||
const string& name = (*i).key();
|
||||
auto& vo = element.get_object();
|
||||
if (vo.contains(name.c_str()))
|
||||
keys_to_rename.emplace_back(name);
|
||||
element = adapt(vo);
|
||||
}
|
||||
else if (element.is_array())
|
||||
adapt(element.get_array());
|
||||
}
|
||||
for (const auto& i : keys_to_rename)
|
||||
{
|
||||
string new_name = i + "_";
|
||||
o[new_name] = variant(o[i]);
|
||||
o.erase(i);
|
||||
}
|
||||
|
||||
if (o.find("memo") != o.end())
|
||||
{
|
||||
auto& memo = o["memo"];
|
||||
if (memo.is_string())
|
||||
{
|
||||
o["memo_"] = o["memo"];
|
||||
o.erase("memo");
|
||||
}
|
||||
else if (memo.is_object())
|
||||
{
|
||||
fc::mutable_variant_object tmp(memo.get_object());
|
||||
if (tmp.find("nonce") != tmp.end())
|
||||
{
|
||||
tmp["nonce"] = tmp["nonce"].as_string();
|
||||
o["memo"] = tmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (o.find("new_parameters") != o.end())
|
||||
{
|
||||
auto& tmp = o["new_parameters"];
|
||||
if (tmp.is_object())
|
||||
{
|
||||
fc::mutable_variant_object tmp2(tmp.get_object());
|
||||
if (tmp2.find("current_fees") != tmp2.end())
|
||||
{
|
||||
tmp2.erase("current_fees");
|
||||
o["new_parameters"] = tmp2;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (o.find("owner") != o.end() && o["owner"].is_string())
|
||||
{
|
||||
o["owner_"] = o["owner"].as_string();
|
||||
o.erase("owner");
|
||||
}
|
||||
if (o.find("proposed_ops") != o.end())
|
||||
{
|
||||
o["proposed_ops"] = fc::json::to_string(o["proposed_ops"]);
|
||||
}
|
||||
if (o.find("initializer") != o.end())
|
||||
{
|
||||
o["initializer"] = fc::json::to_string(o["initializer"]);
|
||||
}
|
||||
|
||||
variant v;
|
||||
fc::to_variant(o, v, FC_PACK_MAX_DEPTH);
|
||||
return v;
|
||||
}
|
||||
|
||||
void adapt(fc::variants& v)
|
||||
{
|
||||
for (auto& array_element : v)
|
||||
{
|
||||
if (array_element.is_object())
|
||||
array_element = adapt(array_element.get_object());
|
||||
else if (array_element.is_array())
|
||||
adapt(array_element.get_array());
|
||||
else
|
||||
array_element = array_element.as_string();
|
||||
}
|
||||
}
|
||||
};
|
||||
} } //graphene::elasticsearch
|
||||
|
||||
FC_REFLECT( graphene::elasticsearch::operation_history_struct, (trx_in_block)(op_in_trx)(operation_result)(virtual_op)(op) )
|
||||
FC_REFLECT_ENUM( graphene::elasticsearch::mode, (only_save)(only_query)(all) )
|
||||
FC_REFLECT( graphene::elasticsearch::operation_history_struct, (trx_in_block)(op_in_trx)(operation_result)(virtual_op)(op)(op_object) )
|
||||
FC_REFLECT( graphene::elasticsearch::block_struct, (block_num)(block_time)(trx_id) )
|
||||
FC_REFLECT( graphene::elasticsearch::fee_struct, (asset)(amount) )
|
||||
FC_REFLECT( graphene::elasticsearch::transfer_struct, (asset)(amount)(from)(to) )
|
||||
|
|
|
|||
|
|
@ -132,21 +132,26 @@ database_fixture::database_fixture()
|
|||
// app.initialize();
|
||||
|
||||
auto test_name = boost::unit_test::framework::current_test_case().p_name.value;
|
||||
if(test_name == "elasticsearch_account_history" || test_name == "elasticsearch_suite") {
|
||||
if(test_name == "elasticsearch_account_history" || test_name == "elasticsearch_suite" ||
|
||||
test_name == "elasticsearch_history_api") {
|
||||
auto esplugin = app.register_plugin<graphene::elasticsearch::elasticsearch_plugin>();
|
||||
esplugin->plugin_set_app(&app);
|
||||
|
||||
options.insert(std::make_pair("elasticsearch-node-url", boost::program_options::variable_value(string("http://localhost:9200/"), false)));
|
||||
options.insert(std::make_pair("elasticsearch-bulk-replay", boost::program_options::variable_value(uint32_t(2), false)));
|
||||
options.insert(std::make_pair("elasticsearch-bulk-sync", boost::program_options::variable_value(uint32_t(2), false)));
|
||||
options.insert(std::make_pair("elasticsearch-visitor", boost::program_options::variable_value(true, false)));
|
||||
//options.insert(std::make_pair("elasticsearch-basic-auth", boost::program_options::variable_value(string("elastic:changeme"), false)));
|
||||
options.insert(std::make_pair("elasticsearch-start-es-after-block", boost::program_options::variable_value(uint32_t(0), false)));
|
||||
options.insert(std::make_pair("elasticsearch-visitor", boost::program_options::variable_value(false, false)));
|
||||
options.insert(std::make_pair("elasticsearch-operation-object", boost::program_options::variable_value(true, false)));
|
||||
options.insert(std::make_pair("elasticsearch-operation-string", boost::program_options::variable_value(true, false)));
|
||||
options.insert(std::make_pair("elasticsearch-mode", boost::program_options::variable_value(uint16_t(2), false)));
|
||||
|
||||
esplugin->plugin_initialize(options);
|
||||
esplugin->plugin_startup();
|
||||
}
|
||||
else {
|
||||
auto ahplugin = app.register_plugin<graphene::account_history::account_history_plugin>();
|
||||
app.enable_plugin("affiliate_stats");
|
||||
ahplugin->plugin_set_app(&app);
|
||||
ahplugin->plugin_initialize(options);
|
||||
ahplugin->plugin_startup();
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@
|
|||
#include <fc/crypto/digest.hpp>
|
||||
|
||||
#include <graphene/utilities/elasticsearch.hpp>
|
||||
#include <graphene/elasticsearch/elasticsearch_plugin.hpp>
|
||||
|
||||
#include "../common/database_fixture.hpp"
|
||||
|
||||
|
|
@ -118,7 +119,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_account_history) {
|
|||
es.endpoint = index_name + "/data/2.9.12"; // we know last op is a transfer of amount 300
|
||||
res = graphene::utilities::getEndPoint(es);
|
||||
j = fc::json::from_string(res);
|
||||
auto last_transfer_amount = j["_source"]["additional_data"]["transfer_data"]["amount"].as_string();
|
||||
auto last_transfer_amount = j["_source"]["operation_history"]["op_object"]["amount_"]["amount"].as_string();
|
||||
BOOST_CHECK_EQUAL(last_transfer_amount, "300");
|
||||
}
|
||||
}
|
||||
|
|
@ -210,4 +211,325 @@ BOOST_AUTO_TEST_CASE(elasticsearch_suite) {
|
|||
}
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_CASE(elasticsearch_history_api) {
|
||||
try {
|
||||
CURL *curl; // curl handler
|
||||
curl = curl_easy_init();
|
||||
|
||||
graphene::utilities::ES es;
|
||||
es.curl = curl;
|
||||
es.elasticsearch_url = "http://localhost:9200/";
|
||||
es.index_prefix = "bitshares-";
|
||||
|
||||
auto delete_account_history = graphene::utilities::deleteAll(es);
|
||||
|
||||
generate_block();
|
||||
fc::usleep(fc::milliseconds(1000));
|
||||
|
||||
if(delete_account_history) {
|
||||
|
||||
create_bitasset("USD", account_id_type()); // create op 0
|
||||
const account_object& dan = create_account("dan"); // create op 1
|
||||
create_bitasset("CNY", dan.id); // create op 2
|
||||
create_bitasset("BTC", account_id_type()); // create op 3
|
||||
create_bitasset("XMR", dan.id); // create op 4
|
||||
create_bitasset("EUR", account_id_type()); // create op 5
|
||||
create_bitasset("OIL", dan.id); // create op 6
|
||||
|
||||
generate_block();
|
||||
fc::usleep(fc::milliseconds(1000));
|
||||
|
||||
graphene::app::history_api hist_api(app);
|
||||
app.enable_plugin("elasticsearch");
|
||||
|
||||
// f(A, 0, 4, 9) = { 5, 3, 1, 0 }
|
||||
auto histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(9));
|
||||
|
||||
BOOST_CHECK_EQUAL(histories.size(), 4u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u);
|
||||
|
||||
// f(A, 0, 4, 6) = { 5, 3, 1, 0 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(6));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 4u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u);
|
||||
|
||||
// f(A, 0, 4, 5) = { 5, 3, 1, 0 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(5));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 4u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u);
|
||||
|
||||
// f(A, 0, 4, 4) = { 3, 1, 0 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(4));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u);
|
||||
|
||||
// f(A, 0, 4, 3) = { 3, 1, 0 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(3));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u);
|
||||
|
||||
// f(A, 0, 4, 2) = { 1, 0 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(2));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u);
|
||||
|
||||
// f(A, 0, 4, 1) = { 1, 0 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(1));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u);
|
||||
|
||||
// f(A, 0, 4, 0) = { 5, 3, 1, 0 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type());
|
||||
BOOST_CHECK_EQUAL(histories.size(), 4u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u);
|
||||
|
||||
// f(A, 1, 5, 9) = { 5, 3 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(9));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
|
||||
|
||||
// f(A, 1, 5, 6) = { 5, 3 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(6));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
|
||||
|
||||
// f(A, 1, 5, 5) = { 5, 3 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(5));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
|
||||
|
||||
// f(A, 1, 5, 4) = { 3 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(4));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u);
|
||||
|
||||
// f(A, 1, 5, 3) = { 3 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(3));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u);
|
||||
|
||||
// f(A, 1, 5, 2) = { }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(2));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 0u);
|
||||
|
||||
// f(A, 1, 5, 1) = { }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(1));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 0u);
|
||||
|
||||
// f(A, 1, 5, 0) = { 5, 3 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(0));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
|
||||
|
||||
// f(A, 0, 3, 9) = { 5, 3, 1 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(9));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
|
||||
|
||||
// f(A, 0, 3, 6) = { 5, 3, 1 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(6));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
|
||||
|
||||
// f(A, 0, 3, 5) = { 5, 3, 1 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(5));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
|
||||
|
||||
// f(A, 0, 3, 4) = { 3, 1, 0 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(4));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u);
|
||||
|
||||
// f(A, 0, 3, 3) = { 3, 1, 0 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(3));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u);
|
||||
|
||||
// f(A, 0, 3, 2) = { 1, 0 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(2));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u);
|
||||
|
||||
// f(A, 0, 3, 1) = { 1, 0 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(1));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u);
|
||||
|
||||
// f(A, 0, 3, 0) = { 5, 3, 1 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type());
|
||||
BOOST_CHECK_EQUAL(histories.size(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
|
||||
|
||||
// f(B, 0, 4, 9) = { 6, 4, 2, 1 }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(9));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 4u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u);
|
||||
|
||||
// f(B, 0, 4, 6) = { 6, 4, 2, 1 }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(6));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 4u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u);
|
||||
|
||||
// f(B, 0, 4, 5) = { 4, 2, 1 }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(5));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
|
||||
|
||||
// f(B, 0, 4, 4) = { 4, 2, 1 }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(4));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u);
|
||||
|
||||
// f(B, 0, 4, 3) = { 2, 1 }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(3));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u);
|
||||
|
||||
// f(B, 0, 4, 2) = { 2, 1 }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(2));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u);
|
||||
|
||||
// f(B, 0, 4, 1) = { 1 }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(1));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u);
|
||||
|
||||
// f(B, 0, 4, 0) = { 6, 4, 2, 1 }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type());
|
||||
BOOST_CHECK_EQUAL(histories.size(), 4u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u);
|
||||
|
||||
// f(B, 2, 4, 9) = { 6, 4 }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(9));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u);
|
||||
|
||||
// f(B, 2, 4, 6) = { 6, 4 }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(6));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u);
|
||||
|
||||
// f(B, 2, 4, 5) = { 4 }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(5));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u);
|
||||
|
||||
// f(B, 2, 4, 4) = { 4 }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(4));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u);
|
||||
|
||||
// f(B, 2, 4, 3) = { }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(3));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 0u);
|
||||
|
||||
// f(B, 2, 4, 2) = { }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(2));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 0u);
|
||||
|
||||
// f(B, 2, 4, 1) = { }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(1));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 0u);
|
||||
|
||||
// f(B, 2, 4, 0) = { 6, 4 }
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(0));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 2u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u);
|
||||
|
||||
// 0 limits
|
||||
histories = hist_api.get_account_history("dan", operation_history_id_type(0), 0, operation_history_id_type(0));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 0u);
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(3), 0, operation_history_id_type(9));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 0u);
|
||||
|
||||
// non existent account
|
||||
histories = hist_api.get_account_history("1.2.18", operation_history_id_type(0), 4, operation_history_id_type(0));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 0u);
|
||||
|
||||
// create a new account C = alice { 7 }
|
||||
auto alice = create_account("alice");
|
||||
|
||||
generate_block();
|
||||
fc::usleep(fc::milliseconds(1000));
|
||||
|
||||
// f(C, 0, 4, 10) = { 7 }
|
||||
histories = hist_api.get_account_history("alice", operation_history_id_type(0), 4, operation_history_id_type(10));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 7u);
|
||||
|
||||
// f(C, 8, 4, 10) = { }
|
||||
histories = hist_api.get_account_history("alice", operation_history_id_type(8), 4, operation_history_id_type(10));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 0u);
|
||||
|
||||
// f(A, 0, 10, 0) = { 7, 5, 3, 1, 0 }
|
||||
histories = hist_api.get_account_history("1.2.0", operation_history_id_type(0), 10, operation_history_id_type(0));
|
||||
BOOST_CHECK_EQUAL(histories.size(), 5u);
|
||||
BOOST_CHECK_EQUAL(histories[0].id.instance(), 7u);
|
||||
BOOST_CHECK_EQUAL(histories[1].id.instance(), 5u);
|
||||
BOOST_CHECK_EQUAL(histories[2].id.instance(), 3u);
|
||||
BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u);
|
||||
BOOST_CHECK_EQUAL(histories[4].id.instance(), 0u);
|
||||
}
|
||||
}
|
||||
catch (fc::exception &e) {
|
||||
edump((e.to_detail_string()));
|
||||
throw;
|
||||
}
|
||||
}
|
||||
BOOST_AUTO_TEST_SUITE_END()
|
||||
|
|
|
|||
Loading…
Reference in a new issue