From ceac07ed42ca026fb2f2805cc31390b591ab52cd Mon Sep 17 00:00:00 2001 From: Eric Frias Date: Thu, 1 Jun 2017 10:02:40 -0400 Subject: [PATCH] Improve the generate_genesis plugin to: - exclude more exchanges - exclude scam accounts - credit users for any vesting balances they could have withdrawn at the time of the snapshot - other minor cleanup --- .../generate_genesis/generate_genesis.cpp | 316 ++++++++++++------ .../generate_genesis_plugin.hpp | 23 +- 2 files changed, 226 insertions(+), 113 deletions(-) diff --git a/libraries/plugins/generate_genesis/generate_genesis.cpp b/libraries/plugins/generate_genesis/generate_genesis.cpp index 8f1b20d9..48b1cfeb 100644 --- a/libraries/plugins/generate_genesis/generate_genesis.cpp +++ b/libraries/plugins/generate_genesis/generate_genesis.cpp @@ -33,6 +33,7 @@ #include #include +#include #include #include @@ -50,7 +51,7 @@ void generate_genesis_plugin::plugin_set_program_options( command_line_options.add_options() ("output-genesis-file,o", bpo::value()->default_value("genesis.json"), "Genesis file to create") ("output-csvlog-file,o", bpo::value()->default_value("log.csv"), "CSV log file to create") - ("snapshot-block-number", bpo::value(), "Block number at which to snapshot balances") + ("snapshot-block-number", bpo::value()->default_value(0), "Block number at which to snapshot balances") ; config_file_options.add(command_line_options); } @@ -67,33 +68,32 @@ void generate_genesis_plugin::plugin_initialize(const boost::program_options::va _genesis_filename = options["output-genesis-file"].as(); _csvlog_filename = options["output-csvlog-file"].as(); - if (options.count("snapshot-block-number")) - _block_to_snapshot = options["snapshot-block-number"].as(); + _block_to_snapshot = options["snapshot-block-number"].as(); database().applied_block.connect([this](const graphene::chain::signed_block& b){ block_applied(b); }); ilog("generate genesis plugin: plugin_initialize() end"); } FC_LOG_AND_RETHROW() } void generate_genesis_plugin::plugin_startup() { try { + ilog("generate genesis plugin: plugin_startup() begin"); chain::database& d = database(); - if (_block_to_snapshot) + if (d.head_block_num() == _block_to_snapshot) { - if (d.head_block_num() == *_block_to_snapshot) - { - ilog("generate genesis plugin: already at snapshot block"); - generate_snapshot(); - } - else if (d.head_block_num() > *_block_to_snapshot) - elog("generate genesis plugin: already passed snapshot block, you must reindex to return to the snapshot state"); - else - elog("generate genesis plugin: waiting for block ${snapshot_block} to generate snapshot, current head is ${head}", - ("snapshot_block", _block_to_snapshot)("head", d.head_block_num())); + ilog("generate genesis plugin: already at snapshot block"); + generate_snapshot(); } + else if (d.head_block_num() > _block_to_snapshot) + elog("generate genesis plugin: already passed snapshot block, you must reindex to return to the snapshot state"); + else + elog("generate genesis plugin: waiting for block ${snapshot_block} to generate snapshot, current head is ${head}", + ("snapshot_block", _block_to_snapshot)("head", d.head_block_num())); + + ilog("generate genesis plugin: plugin_startup() end"); } FC_CAPTURE_AND_RETHROW() } void generate_genesis_plugin::block_applied(const graphene::chain::signed_block& b) { - if (_block_to_snapshot && b.block_num() == *_block_to_snapshot) + if (b.block_num() == _block_to_snapshot) { ilog("generate genesis plugin: snapshot block has arrived"); generate_snapshot(); @@ -110,145 +110,245 @@ bool is_special_account(const graphene::chain::account_id_type& account_id) return account_id.instance < 100; } -bool is_exchange(const std::string& account_name) -{ - return account_name == "poloniexcoldstorage" || - account_name == "btc38-public-for-bts-cold" || - account_name == "poloniexwallet" || - account_name == "btercom" || - account_name == "yunbi-cold-wallet" || - account_name == "btc38-btsx-octo-72722" || - account_name == "bittrex-deposit" || - account_name == "btc38btsxwithdrawal"; -} + bool is_scam(const std::string& account_name) + { + static std::set scam_accounts{ + "polonie-wallet", + "polonie-xwallet", + "poloniewallet", + "poloniex-deposit", + "poloniex-wallet", + "poloniexwall-et", + "poloniexwallett", + "poloniexwall-t", + "poloniexwalle", + "poloniex", + "poloneix", + "poloniex1", + "bittrex-deopsit", + "bittrex-deposi", + "bittrex-depositt", + "bittrex-dposit", + "bittrex", + "bittrex-deposits", + "coinbase", + "blocktrade", + "locktrades", + "yun.bts", + "transwiser-walle", + "transwiser-wallets", + "ranswiser-wallet", + "yun.btc", + "pay.coinbase.com", + "pay.bts.com", + "btc38.com", + "yunbi.com", + "coinbase.com", + "ripple.com" + }; + return scam_accounts.find(account_name) != scam_accounts.end(); + } + + bool is_exchange(const std::string& account_name) + { + static std::set exchange_accounts{ + "poloniexcoldstorage", + "btc38-public-for-bts-cold", + "poloniexwallet", + "btercom", + "yunbi-cold-wallet", + "btc38-btsx-octo-72722", + "bittrex-deposit", + "btc38btsxwithdrawal" + }; + return exchange_accounts.find(account_name) != exchange_accounts.end(); + } + + bool exclude_account_from_sharedrop(graphene::chain::database& d, const graphene::chain::account_id_type& account_id) + { + if (is_special_account(account_id)) + return true; + const std::string& account_name = account_id(d).name; + return is_exchange(account_name) || is_scam(account_name); + } + + void generate_genesis_plugin::generate_snapshot() -{ +{ try { ilog("generate genesis plugin: generating snapshot now"); graphene::chain::genesis_state_type new_genesis_state; chain::database& d = database(); - // we'll distribute 5% of 1,000,000 tokens, so: - graphene::chain::share_type total_amount_to_distribute = 50000 * GRAPHENE_BLOCKCHAIN_PRECISION; + // we'll distribute 5% of (some amount of tokens), so: + graphene::chain::share_type total_amount_to_distribute(27302662972); + + my_account_balance_object_index_type db_balances; + graphene::chain::share_type total_bts_balance; - // we need collection of mutable objects - std::vector db_balances; - // copy const objects to our collection auto& balance_index = d.get_index_type().indices().get(); for (auto balance_iter = balance_index.begin(); balance_iter != balance_index.end() && balance_iter->asset_type == graphene::chain::asset_id_type(); ++balance_iter) - if (!is_special_account(balance_iter->owner) && !is_exchange(balance_iter->owner(d).name)) + { + if (balance_iter->balance > 0 && !exclude_account_from_sharedrop(d, balance_iter->owner)) { - // it is possible due to constructor - db_balances.emplace_back(*balance_iter); + total_bts_balance += balance_iter->balance; + + my_account_balance_object new_balance_object; + new_balance_object.account_id = balance_iter->owner; + new_balance_object.balance = balance_iter->balance; + db_balances.insert(new_balance_object); + } + } + + + // account for BTS tied up in market orders + auto limit_order_index = d.get_index_type().indices(); + for (const graphene::chain::limit_order_object& limit_order : limit_order_index) + if (limit_order.amount_for_sale().asset_id == graphene::chain::asset_id_type()) + { + graphene::chain::share_type limit_order_amount = limit_order.amount_for_sale().amount; + if (limit_order_amount > 0 && !exclude_account_from_sharedrop(d, limit_order.seller)) + { + total_bts_balance += limit_order_amount; + + auto my_balance_iter = db_balances.find(limit_order.seller); + if (my_balance_iter == db_balances.end()) + { + my_account_balance_object balance_object; + balance_object.account_id = limit_order.seller; + balance_object.orders = limit_order_amount; + db_balances.insert(balance_object); + } + else + { + db_balances.modify(my_balance_iter, [&](my_account_balance_object& balance_object) { + balance_object.orders += limit_order_amount; + }); + } + } } - // walk through the balances; this index has the largest BTS balances first - // first, calculate orders and collaterals - // second, update balance - graphene::chain::share_type orders; - graphene::chain::share_type collaterals; - graphene::chain::share_type total_bts_balance; - std::ofstream logfile; - - bool sort = false; - for (auto balance_iter = db_balances.begin(); balance_iter != db_balances.end(); ++balance_iter) - { - orders = 0; - collaterals = 0; - // BTS tied up in market orders - auto order_range = d.get_index_type().indices().get().equal_range(balance_iter->owner); - std::for_each(order_range.first, order_range.second, - [&orders] (const graphene::chain::limit_order_object& order) { - if (order.amount_for_sale().asset_id == graphene::chain::asset_id_type()) - orders += order.amount_for_sale().amount; - }); - // BTS tied up in collateral for SmartCoins - auto collateral_range = d.get_index_type().indices().get().equal_range(balance_iter->owner); - - std::for_each(collateral_range.first, collateral_range.second, - [&collaterals] (const graphene::chain::call_order_object& order) { - collaterals += order.collateral; - }); - - balance_iter->initial_balance = balance_iter->balance; - balance_iter->orders = orders; - balance_iter->collaterals = collaterals; - balance_iter->balance += orders + collaterals; - sort = sort || orders.value > 0 || collaterals.value > 0; - total_bts_balance += balance_iter->balance; - } - - if (sort) - { - ilog("generate genesis plugin: sorting"); - std::sort(db_balances.begin(), db_balances.end(), - [](const my_account_balance_object & a, const my_account_balance_object & b) -> bool + // account for BTS tied up in collateral for SmartCoins + auto call_order_index = d.get_index_type().indices(); + for (const graphene::chain::call_order_object& call_order : call_order_index) + if (call_order.get_collateral().asset_id == graphene::chain::asset_id_type()) { - return a.balance.value > b.balance.value; - }); - } + graphene::chain::share_type call_order_amount = call_order.get_collateral().amount; + if (call_order_amount > 0 && !exclude_account_from_sharedrop(d, call_order.borrower)) + { + total_bts_balance += call_order_amount; + + auto my_balance_iter = db_balances.find(call_order.borrower); + if (my_balance_iter == db_balances.end()) + { + my_account_balance_object balance_object; + balance_object.account_id = call_order.borrower; + balance_object.collateral = call_order_amount; + db_balances.insert(balance_object); + } + else + { + db_balances.modify(my_balance_iter, [&](my_account_balance_object& balance_object) { + balance_object.collateral += call_order_amount; + }); + } + } + } + + // account available-but-unclaimed BTS in vesting balances + auto vesting_balance_index = d.get_index_type().indices(); + for (const graphene::chain::vesting_balance_object& vesting_balance : vesting_balance_index) + if (vesting_balance.balance.asset_id == graphene::chain::asset_id_type()) + { + graphene::chain::share_type vesting_balance_amount = vesting_balance.get_allowed_withdraw(d.head_block_time()).amount; + if (vesting_balance_amount > 0 && !exclude_account_from_sharedrop(d, vesting_balance.owner)) + { + total_bts_balance += vesting_balance_amount; + + auto my_balance_iter = db_balances.find(vesting_balance.owner); + if (my_balance_iter == db_balances.end()) + { + my_account_balance_object balance_object; + balance_object.account_id = vesting_balance.owner; + balance_object.vesting = vesting_balance_amount; + db_balances.insert(balance_object); + } + else + { + db_balances.modify(my_balance_iter, [&](my_account_balance_object& balance_object) { + balance_object.vesting += vesting_balance_amount; + }); + } + } + } graphene::chain::share_type total_shares_dropped; + // Now, we assume we're distributing balances to all BTS holders proportionally, figure // the smallest balance we can distribute and still assign the user a satoshi of the share drop graphene::chain::share_type effective_total_bts_balance; - auto balance_iter = db_balances.begin(); - for (balance_iter = db_balances.begin(); balance_iter != db_balances.end(); ++balance_iter) + auto& by_effective_balance_index = db_balances.get(); + auto balance_iter = by_effective_balance_index.begin(); + for (; balance_iter != by_effective_balance_index.end(); ++balance_iter) { fc::uint128 share_drop_amount = total_amount_to_distribute.value; - share_drop_amount *= balance_iter->balance.value; + share_drop_amount *= balance_iter->get_effective_balance().value; share_drop_amount /= total_bts_balance.value; if (!share_drop_amount.to_uint64()) break; // balances are decreasing, so every balance after will also round to zero total_shares_dropped += share_drop_amount.to_uint64(); - effective_total_bts_balance += balance_iter->balance; + effective_total_bts_balance += balance_iter->get_effective_balance(); } // our iterator is just after the smallest balance we will process, // walk it backwards towards the larger balances, distributing the sharedrop as we go graphene::chain::share_type remaining_amount_to_distribute = total_amount_to_distribute; graphene::chain::share_type bts_balance_remaining = effective_total_bts_balance; - std::map sharedrop_balances; do { --balance_iter; fc::uint128 share_drop_amount = remaining_amount_to_distribute.value; - share_drop_amount *= balance_iter->balance.value; + share_drop_amount *= balance_iter->get_effective_balance().value; share_drop_amount /= bts_balance_remaining.value; graphene::chain::share_type amount_distributed = share_drop_amount.to_uint64(); - sharedrop_balances[balance_iter->owner] = amount_distributed; - balance_iter->sharedrop = amount_distributed; + + by_effective_balance_index.modify(balance_iter, [&](my_account_balance_object& balance_object) { + balance_object.sharedrop += amount_distributed; + }); remaining_amount_to_distribute -= amount_distributed; - bts_balance_remaining -= balance_iter->balance.value; - } while (balance_iter != db_balances.begin()); + bts_balance_remaining -= balance_iter->get_effective_balance(); + } while (balance_iter != by_effective_balance_index.begin()); assert(remaining_amount_to_distribute == 0); + std::ofstream logfile; logfile.open(_csvlog_filename); assert(logfile.is_open()); - logfile << "name,balance+orders+collaterals,balance,orders,collaterals,sharedrop\n"; - char del = ','; - char nl = '\n'; - for(const auto& o : db_balances) - { - logfile << o.owner(d).name << del << o.balance.value << del << o.initial_balance.value << del << o.orders.value << del << o.collaterals.value << del << o.sharedrop.value << nl; - } + logfile << "name,balance+orders+collateral+vesting,balance,orders,collateral,vesting,sharedrop\n"; + for (const my_account_balance_object& balance : by_effective_balance_index) + logfile << balance.account_id(d).name << "," << + balance.get_effective_balance().value << "," << + balance.balance.value << "," << + balance.orders.value << "," << + balance.collateral.value << "," << + balance.vesting.value << "," << + balance.sharedrop.value << "\n"; ilog("CSV log written to file ${filename}", ("filename", _csvlog_filename)); logfile.close(); - //auto& account_index = d.get_index_type(); - //auto& account_by_id_index = account_index.indices().get(); + // remove all balance objects with zero sharedrops + by_effective_balance_index.erase(by_effective_balance_index.lower_bound(0), + by_effective_balance_index.end()); + // inefficient way of crawling the graph, but we only do it once std::set already_generated; for (;;) { unsigned accounts_generated_this_round = 0; - for (const auto& sharedrop_value : sharedrop_balances) + for (const my_account_balance_object& balance : by_effective_balance_index) { - const graphene::chain::account_id_type& account_id = sharedrop_value.first; - const graphene::chain::share_type& sharedrop_amount = sharedrop_value.second; - const graphene::chain::account_object& account_obj = account_id(d); - if (already_generated.find(account_id) == already_generated.end()) + const graphene::chain::account_object& account_obj = balance.account_id(d); + if (already_generated.find(balance.account_id) == already_generated.end()) { graphene::chain::genesis_state_type::initial_bts_account_type::initial_authority owner; owner.weight_threshold = account_obj.owner.weight_threshold; @@ -256,7 +356,7 @@ void generate_genesis_plugin::generate_snapshot() for (const auto& value : account_obj.owner.account_auths) { owner.account_auths.insert(std::make_pair(modify_account_name(value.first(d).name), value.second)); - sharedrop_balances[value.first] += 0; // make sure the account is generated, even if it has a zero balance + db_balances.insert(my_account_balance_object{value.first}); // make sure the account is generated, even if it has a zero balance } owner.key_auths = account_obj.owner.key_auths; owner.address_auths = account_obj.owner.address_auths; @@ -267,7 +367,7 @@ void generate_genesis_plugin::generate_snapshot() for (const auto& value : account_obj.active.account_auths) { active.account_auths.insert(std::make_pair(modify_account_name(value.first(d).name), value.second)); - sharedrop_balances[value.first] += 0; // make sure the account is generated, even if it has a zero balance + db_balances.insert(my_account_balance_object{value.first}); // make sure the account is generated, even if it has a zero balance } active.key_auths = account_obj.active.key_auths; active.address_auths = account_obj.active.address_auths; @@ -275,8 +375,8 @@ void generate_genesis_plugin::generate_snapshot() new_genesis_state.initial_bts_accounts.emplace_back( graphene::chain::genesis_state_type::initial_bts_account_type(modify_account_name(account_obj.name), owner, active, - sharedrop_amount)); - already_generated.insert(account_id); + balance.sharedrop)); + already_generated.insert(balance.account_id); ++accounts_generated_this_round; } } @@ -285,7 +385,7 @@ void generate_genesis_plugin::generate_snapshot() } fc::json::save_to_file(new_genesis_state, _genesis_filename); ilog("New genesis state written to file ${filename}", ("filename", _genesis_filename)); -} +} FC_LOG_AND_RETHROW() } void generate_genesis_plugin::plugin_shutdown() { diff --git a/libraries/plugins/generate_genesis/include/graphene/generate_genesis/generate_genesis_plugin.hpp b/libraries/plugins/generate_genesis/include/graphene/generate_genesis/generate_genesis_plugin.hpp index c1ffbaff..5557625a 100644 --- a/libraries/plugins/generate_genesis/include/graphene/generate_genesis/generate_genesis_plugin.hpp +++ b/libraries/plugins/generate_genesis/include/graphene/generate_genesis/generate_genesis_plugin.hpp @@ -52,21 +52,34 @@ private: boost::program_options::variables_map _options; - fc::optional _block_to_snapshot; + uint32_t _block_to_snapshot; std::string _genesis_filename; std::string _csvlog_filename; }; -class my_account_balance_object : public graphene::chain::account_balance_object +class my_account_balance_object { public: // constructor copying from base class - my_account_balance_object(const graphene::chain::account_balance_object& abo) : graphene::chain::account_balance_object(abo) {} + //my_account_balance_object(const graphene::chain::account_balance_object& abo) : graphene::chain::account_balance_object(abo) {} + graphene::chain::account_id_type account_id; - graphene::chain::share_type initial_balance; + graphene::chain::share_type balance; graphene::chain::share_type orders; - graphene::chain::share_type collaterals; + graphene::chain::share_type collateral; + graphene::chain::share_type vesting; graphene::chain::share_type sharedrop; + graphene::chain::share_type get_effective_balance() const { return balance + orders + collateral + vesting; } }; +using namespace boost::multi_index; +struct by_account{}; +struct by_effective_balance{}; +typedef multi_index_container, + member >, + ordered_non_unique, + const_mem_fun, + std::greater > > > my_account_balance_object_index_type; + } } //graphene::generate_genesis_plugin