Further cleanup after LevelDB removal

This commit is contained in:
Vikram Rajkumar 2015-06-17 12:33:38 -04:00
parent 8f739ac767
commit 34388fabee
10 changed files with 37 additions and 524 deletions

View file

@ -66,6 +66,7 @@ IF(NOT "${Boost_VERSION}" MATCHES "1.53(.*)")
ENDIF()
if( WIN32 )
message( STATUS "Configuring Graphene on WIN32")
set( DB_VERSION 60 )
set( BDB_STATIC_LIBS 1 )
@ -98,13 +99,6 @@ if( WIN32 )
SET(TCL_LIBRARY ${TCL_LIBS})
else( WIN32 ) # Apple AND Linux
list(APPEND LEVELDB_BUILD_DEFINES LEVELDB_PLATFORM_POSIX LEVELDB_ATOMIC_PRESENT)
if( APPLE )
list(APPEND LEVELDB_BUILD_DEFINES OS_MACOSX)
else() # Linux
list(APPEND LEVELDB_BUILD_DEFINES OS_LINUX)
list(APPEND LEVELDB_BUILD_LIBRARIES pthread)
endif()
find_library(READLINE_LIBRARIES NAMES readline)
find_path(READLINE_INCLUDE_DIR readline/readline.h)

View file

@ -53,7 +53,6 @@ namespace graphene { namespace chain {
checksum_type signed_block::calculate_merkle_root()const
{
if( transactions.size() == 0 ) return checksum_type();
wdump((transactions.size()));
vector<digest_type> ids;
ids.resize( ((transactions.size() + 1)/2)*2 );

View file

@ -1,3 +1,20 @@
/*
* Copyright (c) 2015, Cryptonomex, Inc.
* All rights reserved.
*
* This source code is provided for evaluation in private test networks only, until September 8, 2015. After this date, this license expires and
* the code may not be used, modified or distributed for any purpose. Redistribution and use in source and binary forms, with or without modification,
* are permitted until September 8, 2015, provided that the following conditions are met:
*
* 1. The code and/or derivative works are used only for private test networks consisting of no more than 10 P2P nodes.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <graphene/chain/block_database.hpp>
namespace graphene { namespace chain {
@ -8,7 +25,7 @@ struct index_entry
uint32_t block_size = 0;
block_id_type block_id;
};
}}
}}
FC_REFLECT( graphene::chain::index_entry, (block_pos)(block_size)(block_id) );
namespace graphene { namespace chain {
@ -146,9 +163,7 @@ optional<signed_block> block_database::fetch_by_number( uint32_t block_num )cons
FC_ASSERT( _block_num_to_pos.tellg() > index_pos );
_block_num_to_pos.seekg( index_pos, _block_num_to_pos.beg );
wdump((int64_t(_block_num_to_pos.tellg())) );
_block_num_to_pos.read( (char*)&e, sizeof(e) );
wdump((block_num)(e));
vector<char> data( e.block_size );
_blocks.seekg( e.block_pos );
@ -175,14 +190,13 @@ optional<signed_block> block_database::last()const
_block_num_to_pos.read( (char*)&e, sizeof(e) );
}
if( e.block_size == 0 )
if( e.block_size == 0 )
return optional<signed_block>();
vector<char> data( e.block_size );
_blocks.seekg( e.block_pos );
_blocks.read( data.data(), e.block_size );
auto result = fc::raw::unpack<signed_block>(data);
wdump((result));
return result;
}

View file

@ -1,3 +1,20 @@
/*
* Copyright (c) 2015, Cryptonomex, Inc.
* All rights reserved.
*
* This source code is provided for evaluation in private test networks only, until September 8, 2015. After this date, this license expires and
* the code may not be used, modified or distributed for any purpose. Redistribution and use in source and binary forms, with or without modification,
* are permitted until September 8, 2015, provided that the following conditions are met:
*
* 1. The code and/or derivative works are used only for private test networks consisting of no more than 10 P2P nodes.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#pragma once
#include <fstream>
#include <graphene/chain/block.hpp>

View file

@ -1,224 +0,0 @@
/*
* Copyright (c) 2015, Cryptonomex, Inc.
* All rights reserved.
*
* This source code is provided for evaluation in private test networks only, until September 8, 2015. After this date, this license expires and
* the code may not be used, modified or distributed for any purpose. Redistribution and use in source and binary forms, with or without modification,
* are permitted until September 8, 2015, provided that the following conditions are met:
*
* 1. The code and/or derivative works are used only for private test networks consisting of no more than 10 P2P nodes.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#pragma once
#include <graphene/db/level_map.hpp>
#include <fc/thread/thread.hpp>
#include <map>
namespace graphene { namespace db {
template<typename Key, typename Value, class CacheType = std::map<Key,Value>>
class cached_level_map
{
public:
void open( const fc::path& dir, bool create = true, size_t leveldb_cache_size = 0, bool write_through = true, bool sync_on_write = false )
{ try {
_db.open( dir, create, leveldb_cache_size );
for( auto itr = _db.begin(); itr.valid(); ++itr )
_cache.emplace_hint( _cache.end(), itr.key(), itr.value() );
_write_through = write_through;
_sync_on_write = sync_on_write;
} FC_CAPTURE_AND_RETHROW( (dir)(create)(leveldb_cache_size)(write_through)(sync_on_write) ) }
void close()
{ try {
if( _db.is_open() ) flush();
_db.close();
_cache.clear();
_dirty_store.clear();
_dirty_remove.clear();
} FC_CAPTURE_AND_RETHROW() }
void set_write_through( bool write_through )
{ try {
if( write_through == _write_through )
return;
if( write_through )
flush();
_write_through = write_through;
} FC_CAPTURE_AND_RETHROW( (write_through) ) }
void flush()
{ try {
typename level_map<Key, Value>::write_batch batch = _db.create_batch( _sync_on_write );
for( const auto& key : _dirty_store )
batch.store( key, _cache.at( key ) );
for( const auto& key : _dirty_remove )
batch.remove( key );
batch.commit();
_dirty_store.clear();
_dirty_remove.clear();
} FC_CAPTURE_AND_RETHROW() }
fc::optional<Value> fetch_optional( const Key& key )const
{ try {
const auto itr = _cache.find( key );
if( itr != _cache.end() )
return itr->second;
return fc::optional<Value>();
} FC_CAPTURE_AND_RETHROW( (key) ) }
Value fetch( const Key& key )const
{ try {
const auto itr = _cache.find( key );
if( itr != _cache.end() )
return itr->second;
FC_CAPTURE_AND_THROW( fc::key_not_found_exception, (key) );
} FC_CAPTURE_AND_RETHROW( (key) ) }
void store( const Key& key, const Value& value )
{ try {
_cache[ key ] = value;
if( _write_through )
{
_db.store( key, value, _sync_on_write );
}
else
{
_dirty_store.insert( key );
_dirty_remove.erase( key );
}
} FC_CAPTURE_AND_RETHROW( (key)(value) ) }
void remove( const Key& key )
{ try {
_cache.erase( key );
if( _write_through )
{
_db.remove( key, _sync_on_write );
}
else
{
_dirty_store.erase( key );
_dirty_remove.insert( key );
}
} FC_CAPTURE_AND_RETHROW( (key) ) }
size_t size()const
{ try {
return _cache.size();
} FC_CAPTURE_AND_RETHROW() }
bool last( Key& key )const
{ try {
const auto ritr = _cache.crbegin();
if( ritr != _cache.crend() )
{
key = ritr->first;
return true;
}
return false;
} FC_CAPTURE_AND_RETHROW( (key) ) }
bool last( Key& key, Value& value )
{ try {
const auto ritr = _cache.crbegin();
if( ritr != _cache.crend() )
{
key = ritr->first;
value = ritr->second;
return true;
}
return false;
} FC_CAPTURE_AND_RETHROW( (key)(value) ) }
class iterator
{
public:
iterator(){}
bool valid()const { return _it != _end; }
Key key()const { return _it->first; }
Value value()const { return _it->second; }
iterator& operator++() { ++_it; return *this; }
iterator operator++(int) {
auto backup = *this;
++_it;
return backup;
}
iterator& operator--()
{
if( _it == _begin )
_it = _end;
else
--_it;
return *this;
}
iterator operator--(int) {
auto backup = *this;
operator--();
return backup;
}
void reset() { _it = _end; }
protected:
friend class cached_level_map;
iterator( typename CacheType::const_iterator it, typename CacheType::const_iterator begin, typename CacheType::const_iterator end )
:_it(it),_begin(begin),_end(end)
{ }
typename CacheType::const_iterator _it;
typename CacheType::const_iterator _begin;
typename CacheType::const_iterator _end;
};
iterator begin()const
{
return iterator( _cache.begin(), _cache.begin(), _cache.end() );
}
iterator last()const
{
if( _cache.empty() )
return iterator( _cache.end(), _cache.begin(), _cache.end() );
return iterator( --_cache.end(), _cache.begin(), _cache.end() );
}
iterator find( const Key& key )const
{
return iterator( _cache.find(key), _cache.begin(), _cache.end() );
}
iterator lower_bound( const Key& key )const
{
return iterator( _cache.lower_bound(key), _cache.begin(), _cache.end() );
}
// TODO: Iterate over cache instead
void export_to_json( const fc::path& path )const
{ try {
_db.export_to_json( path );
} FC_CAPTURE_AND_RETHROW( (path) ) }
private:
level_map<Key, Value> _db;
CacheType _cache;
std::set<Key> _dirty_store;
std::set<Key> _dirty_remove;
bool _write_through = true;
bool _sync_on_write = false;
};
} }

View file

@ -1,143 +0,0 @@
/*
* Copyright (c) 2015, Cryptonomex, Inc.
* All rights reserved.
*
* This source code is provided for evaluation in private test networks only, until September 8, 2015. After this date, this license expires and
* the code may not be used, modified or distributed for any purpose. Redistribution and use in source and binary forms, with or without modification,
* are permitted until September 8, 2015, provided that the following conditions are met:
*
* 1. The code and/or derivative works are used only for private test networks consisting of no more than 10 P2P nodes.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#pragma once
#include <graphene/db/level_map.hpp>
namespace graphene { namespace db {
template<typename K, typename V>
class fast_level_map
{
level_map<K, V> _ldb;
fc::optional<fc::path> _ldb_path;
bool _ldb_enabled = true;
std::unordered_map<K, V> _cache;
public:
~fast_level_map()
{
close();
}
void open( const fc::path& path )
{ try {
FC_ASSERT( !_ldb_path.valid() );
_ldb_path = path;
_ldb.open( *_ldb_path );
_cache.reserve( _ldb.size() );
for( auto iter = _ldb.begin(); iter.valid(); ++iter )
_cache.emplace( iter.key(), iter.value() );
} FC_CAPTURE_AND_RETHROW( (path) ) }
void close()
{ try {
if( _ldb_path.valid() )
{
if( !_ldb_enabled ) toggle_leveldb( true );
_ldb.close();
_ldb_path = fc::optional<fc::path>();
}
_cache.clear();
} FC_CAPTURE_AND_RETHROW() }
void toggle_leveldb( const bool enabled )
{ try {
FC_ASSERT( _ldb_path.valid() );
if( enabled == _ldb_enabled )
return;
if( enabled )
{
_ldb.open( *_ldb_path );
auto batch = _ldb.create_batch();
for( const auto& item : _cache )
batch.store( item.first, item.second );
batch.commit();
}
else
{
_ldb.close();
fc::remove_all( *_ldb_path );
}
_ldb_enabled = enabled;
} FC_CAPTURE_AND_RETHROW( (enabled) ) }
void store( const K& key, const V& value )
{ try {
_cache[ key ] = value;
if( _ldb_enabled )
_ldb.store( key, value );
} FC_CAPTURE_AND_RETHROW( (key)(value) ) }
void remove( const K& key )
{ try {
_cache.erase( key );
if( _ldb_enabled )
_ldb.remove( key );
} FC_CAPTURE_AND_RETHROW( (key) ) }
auto empty()const -> decltype( _cache.empty() )
{
return _cache.empty();
}
auto size()const -> decltype( _cache.size() )
{
return _cache.size();
}
auto count( const K& key )const -> decltype( _cache.count( key ) )
{
return _cache.count( key );
}
auto unordered_begin()const -> decltype( _cache.cbegin() )
{
return _cache.cbegin();
}
auto unordered_end()const -> decltype( _cache.cend() )
{
return _cache.cend();
}
auto unordered_find( const K& key )const -> decltype( _cache.find( key ) )
{
return _cache.find( key );
}
auto ordered_first()const -> decltype( _ldb.begin() )
{ try {
return _ldb.begin();
} FC_CAPTURE_AND_RETHROW() }
auto ordered_last()const -> decltype( _ldb.last() )
{ try {
return _ldb.last();
} FC_CAPTURE_AND_RETHROW() }
auto ordered_lower_bound( const K& key )const -> decltype( _ldb.lower_bound( key ) )
{ try {
return _ldb.lower_bound( key );
} FC_CAPTURE_AND_RETHROW( (key) ) }
};
} } // graphene::db

View file

@ -1,132 +0,0 @@
/*
* Copyright (c) 2015, Cryptonomex, Inc.
* All rights reserved.
*
* This source code is provided for evaluation in private test networks only, until September 8, 2015. After this date, this license expires and
* the code may not be used, modified or distributed for any purpose. Redistribution and use in source and binary forms, with or without modification,
* are permitted until September 8, 2015, provided that the following conditions are met:
*
* 1. The code and/or derivative works are used only for private test networks consisting of no more than 10 P2P nodes.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#pragma once
#include <leveldb/db.h>
#include <leveldb/comparator.h>
#include <fc/reflect/reflect.hpp>
#include <fc/io/raw.hpp>
#include <fc/exception/exception.hpp>
#include <functional>
#include <map>
namespace fc { class path; }
/**
* This code has no graphene dependencies, and it
* could be moved to fc, if fc ever adds a leveldb dependency
*
* This code enables legacy databases files created by older programs to
* be upgraded to the current database formats. Whenever a database is first opened,
* this code checks if the database is stored in an old format and looks for an
* upgrade function to upgrade it to the current format. If found, the objects
* in the database will be immediately upgraded to the current format.
*
* Upgrades are performed by executing a series of chained copy constructors
* from the legacy object format to the current object format. This means
* that only one new copy constructor typically needs to be written to support
* upgrading any previous version of the object when an object type is modified.
*
* - Database versioning is only supported for changes to database value types
* (databases with modified key types cannot currently be upgraded).
* - The database versioning code requires that fc::get_typename is defined for
* all value types which are to be versioned.
*/
/*
Below is a simple example of how client code needs to be written to support
database versioning. Originally, a database stored values of record0, and
record was typedef'd to be record0. A new type record1 was created to add
"new_field" to record type, and record was typedef'd to record1. The typedef
is used to minimize required changes to the client code that references
record objects.
@code
struct record0
{
record0() : points(0) {}
double points;
};
FC_REFLECT( record0, (points) )
REGISTER_DB_OBJECT(record,0) //This creates an upgrade function for record0 databases
struct record1
{
record1() : points(0), new_field("EMPTY") {}
record1(const record0& r0) //convert from record0 to record1 for legacy files
{
key = r0.key;
new_field = "EMPTY";
}
std::string new_field;
double points;
};
FC_REFLECT( record1, (points)(new_field) )
typedef record1 record; //current databases store record1 objects
@endcode
*/
namespace graphene { namespace db {
typedef std::function<void(leveldb::DB*)> upgrade_db_function;
class upgrade_db_mapper
{
public:
static upgrade_db_mapper& instance();
int32_t add_type( const std::string& type_name, const upgrade_db_function& function);
std::map<std::string,upgrade_db_function> _upgrade_db_function_registry;
};
#define REGISTER_DB_OBJECT(TYPE,VERSIONNUM) \
void UpgradeDb ## TYPE ## VERSIONNUM(leveldb::DB* dbase) \
{ \
std::unique_ptr<leveldb::Iterator> dbase_itr( dbase->NewIterator(leveldb::ReadOptions()) ); \
dbase_itr->SeekToFirst(); \
if( dbase_itr->status().IsNotFound() ) /*if empty database, do nothing*/ \
return; \
if( !dbase_itr->status().ok() ) \
FC_THROW_EXCEPTION( exception, "database error: ${msg}", ("msg", dbase_itr->status().ToString() ) ); \
while( dbase_itr->Valid() ) /* convert dbase objects from legacy TypeVersionNum to current Type */ \
{ \
TYPE ## VERSIONNUM old_value; /*load old record type*/ \
fc::datastream<const char*> dstream( dbase_itr->value().data(), dbase_itr->value().size() ); \
fc::raw::unpack( dstream, old_value ); \
TYPE new_value(old_value); /*convert to new record type*/ \
leveldb::Slice key_slice = dbase_itr->key(); \
auto vec = fc::raw::pack(new_value); \
leveldb::Slice value_slice( vec.data(), vec.size() ); \
auto status = dbase->Put( leveldb::WriteOptions(), key_slice, value_slice ); \
if( !status.ok() ) \
{ \
FC_THROW_EXCEPTION( exception, "database error: ${msg}", ("msg", status.ToString() ) ); \
} \
dbase_itr->Next(); \
} /*while*/ \
} \
static int dummyResult ## TYPE ## VERSIONNUM = \
upgrade_db_mapper::instance()->add_type(fc::get_typename<TYPE ## VERSIONNUM>::name(), UpgradeDb ## TYPE ## VERSIONNUM);
void try_upgrade_db( const fc::path& dir, leveldb::DB* dbase, const char* record_type, size_t record_type_size );
} } // namespace db

View file

@ -1,6 +0,0 @@
#pragma once
#include <BaseTsd.h>
typedef SSIZE_T ssize_t;
#include "../../../leveldb/include/leveldb/db.h"

View file

@ -1 +0,0 @@
/* empty unistd to allow leveldb to compile with msvc which is missing this file */

View file

@ -1,5 +0,0 @@
This directory contains the files needed to make the stock LevelDB distribution
from https://github.com/bitcoin/leveldb.git compile on Windows with Visual C++.
Add this 'include' directory to yur include path before the regular includes
only when building leveldb (it isn't needed when compiling code that uses
leveldb).