From f56dd6d20840cae63f2e167681a5c68e388ec0e3 Mon Sep 17 00:00:00 2001 From: Vikram Rajkumar Date: Wed, 2 Jul 2014 22:36:10 -0400 Subject: [PATCH 1/8] Implement LZMA file compression --- CMakeLists.txt | 5 +- include/fc/compress/lzma.hpp | 11 ++- src/compress/lzma.cpp | 145 +++++++++++++++++++++++++++++++++++ tests/lzma_compress_file.cpp | 23 ++++++ 4 files changed, 181 insertions(+), 3 deletions(-) create mode 100644 tests/lzma_compress_file.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1494c29..6775d43 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,7 +33,7 @@ endif() SET (ORIGINAL_LIB_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) SET(BOOST_COMPONENTS) -LIST(APPEND BOOST_COMPONENTS thread date_time system filesystem program_options signals serialization chrono unit_test_framework context locale) +LIST(APPEND BOOST_COMPONENTS thread date_time system filesystem program_options signals serialization chrono unit_test_framework context locale iostreams) IF( WIN32 ) MESSAGE(STATUS "Configuring fc to build on Win32") @@ -237,6 +237,9 @@ target_link_libraries( udt_server fc udt ) add_executable( udt_client tests/udtc.cpp ) target_link_libraries( udt_client fc udt ) +add_executable( lzma_compress_file tests/lzma_compress_file.cpp ) +target_link_libraries( lzma_compress_file fc ) + #add_executable( test_compress tests/compress.cpp ) #target_link_libraries( test_compress fc ) #add_executable( test_aes tests/aes_test.cpp ) diff --git a/include/fc/compress/lzma.hpp b/include/fc/compress/lzma.hpp index a6d4f81..e1b411c 100644 --- a/include/fc/compress/lzma.hpp +++ b/include/fc/compress/lzma.hpp @@ -1,9 +1,16 @@ #pragma once + +#include #include namespace fc { - std::vector lzma_compress( const std::vector& in ); - std::vector lzma_decompress( const std::vector& compressed ); +std::vector lzma_compress( const std::vector& in ); +std::vector lzma_decompress( const std::vector& compressed ); + +void lzma_compress_file( path src_path, + path dst_path, + unsigned char level = 5, + unsigned int dict_size = (1 << 20) ); } // namespace fc diff --git a/src/compress/lzma.cpp b/src/compress/lzma.cpp index 60db130..91d5475 100644 --- a/src/compress/lzma.cpp +++ b/src/compress/lzma.cpp @@ -3,6 +3,14 @@ #include +#include + +#include + + + +#include + namespace fc { std::vector lzma_compress(const std::vector& in) @@ -50,4 +58,141 @@ std::vector lzma_decompress( const std::vector& compressed ) return out; } +struct lzma_file_ctx +{ + const unsigned char* src_buf; + size_t src_len; + + path dst_path; +}; + +static int input_callback( void* input_ctx, void* input_buf, size_t* input_len ) +{ + FC_ASSERT( input_ctx != NULL ); + FC_ASSERT( input_buf != NULL ); + + const auto ctx = ( struct lzma_file_ctx* )input_ctx; + const auto size = ( ctx->src_len < *input_len ) ? ctx->src_len : *input_len; + + if( size > 0 ) + { + memcpy( input_buf, ( void * )ctx->src_buf, size ); + ctx->src_buf += size; + ctx->src_len -= size; + } + + *input_len = size; + + return 0; +} + +static size_t output_callback( void* output_ctx, const void* output_buf, size_t output_len ) +{ + FC_ASSERT( output_ctx != NULL ); + FC_ASSERT( output_buf != NULL ); + + const auto ctx = ( struct lzma_file_ctx* )output_ctx; + + if( output_len > 0 ) + { + size_t dst_len = 0; + if( !exists( ctx->dst_path ) ) + { + auto fs = std::ofstream( ctx->dst_path.string() ); + fs.close(); + } + else + { + dst_len = file_size( ctx->dst_path ); + } + + resize_file( ctx->dst_path, dst_len + output_len ); + + boost::iostreams::mapped_file_sink dst_file; + dst_file.open( ctx->dst_path.string() ); + FC_ASSERT( dst_file.is_open() ); + + memcpy( ( void* )(dst_file.data() + dst_len), output_buf, output_len); + + dst_file.close(); + } + + return output_len; +} + +void lzma_compress_file( path src_path, + path dst_path, + unsigned char level, + unsigned int dict_size ) +{ + FC_ASSERT( exists( src_path ) ); + FC_ASSERT( !exists( dst_path ) ); + + boost::iostreams::mapped_file_source src_file; + src_file.open( src_path.string() ); + FC_ASSERT( src_file.is_open() ); + + elzma_compress_handle handle = NULL; + handle = elzma_compress_alloc(); + FC_ASSERT( handle != NULL ); + + struct lzma_file_ctx ctx; + ctx.src_buf = ( const unsigned char* )src_file.data(); + ctx.src_len = src_file.size(); + ctx.dst_path = dst_path; + + auto rc = elzma_compress_config( handle, + ELZMA_LC_DEFAULT, + ELZMA_LP_DEFAULT, + ELZMA_PB_DEFAULT, + level, + dict_size, + elzma_file_format::ELZMA_lzma, + ctx.src_len ); + + try + { + FC_ASSERT( rc == ELZMA_E_OK ); + } + catch( ... ) + { + elzma_compress_free( &handle ); + throw; + } + + rc = elzma_compress_run( handle, + input_callback, + ( void * )&ctx, + output_callback, + ( void * )&ctx, + NULL, + NULL ); + + elzma_compress_free( &handle ); + FC_ASSERT( rc == ELZMA_E_OK ); + + + /* TEST */ + FC_ASSERT( exists( dst_path ) ); + + boost::iostreams::mapped_file_source dst_file; + dst_file.open( dst_path.string() ); + FC_ASSERT( dst_file.is_open() ); + + std::vector result( dst_file.data(), dst_file.data() + dst_file.size() ); + dst_file.close(); + + for( const auto& c : result ) + { + std::cout << c; + } + std::cout << "\n"; + + result = lzma_decompress( result ); + for( const auto& c : result ) + { + std::cout << c; + } +} + } // namespace fc diff --git a/tests/lzma_compress_file.cpp b/tests/lzma_compress_file.cpp new file mode 100644 index 0000000..3f76733 --- /dev/null +++ b/tests/lzma_compress_file.cpp @@ -0,0 +1,23 @@ +#include +#include + +#include +#include + +using namespace fc; + +int main( int argc, char** argv ) +{ + if( argc != 2 && argc != 3 ) + { + std::cout << "usage: " << argv[0] << " [dst_path = src_path.lzma]\n"; + exit( -1 ); + } + + auto src = std::string( argv[1] ); + auto dst = (argc == 3) ? std::string( argv[2] ) : src + ".lzma"; + + lzma_compress_file( src, dst ); + + return 0; +} From 39c7f05ea96bea83f97d3b80056046314b7f27c8 Mon Sep 17 00:00:00 2001 From: Vikram Rajkumar Date: Wed, 2 Jul 2014 23:28:08 -0400 Subject: [PATCH 2/8] Implement LZMA file decompression --- .gitignore | 5 +- CMakeLists.txt | 4 +- include/fc/compress/lzma.hpp | 7 ++- src/compress/lzma.cpp | 56 +++++++++---------- .../{lzma_compress_file.cpp => lzma_test.cpp} | 9 +-- 5 files changed, 44 insertions(+), 37 deletions(-) rename tests/{lzma_compress_file.cpp => lzma_test.cpp} (59%) diff --git a/.gitignore b/.gitignore index 0a8ee06..e956fca 100644 --- a/.gitignore +++ b/.gitignore @@ -33,9 +33,11 @@ ZERO_CHECK Debug/ Release/ +CMakeCache.txt CMakeFiles Makefile -cmake_install.cmake +*.cmake + libfc.a libfc_debug.a @@ -45,6 +47,7 @@ fc_automoc.cpp git_revision.cpp GitSHA3.cpp +lzma_test ntp_test task_cancel_test udt_client diff --git a/CMakeLists.txt b/CMakeLists.txt index 6775d43..78812b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -237,8 +237,8 @@ target_link_libraries( udt_server fc udt ) add_executable( udt_client tests/udtc.cpp ) target_link_libraries( udt_client fc udt ) -add_executable( lzma_compress_file tests/lzma_compress_file.cpp ) -target_link_libraries( lzma_compress_file fc ) +add_executable( lzma_test tests/lzma_test.cpp ) +target_link_libraries( lzma_test fc ) #add_executable( test_compress tests/compress.cpp ) #target_link_libraries( test_compress fc ) diff --git a/include/fc/compress/lzma.hpp b/include/fc/compress/lzma.hpp index e1b411c..44d17b7 100644 --- a/include/fc/compress/lzma.hpp +++ b/include/fc/compress/lzma.hpp @@ -8,9 +8,12 @@ namespace fc { std::vector lzma_compress( const std::vector& in ); std::vector lzma_decompress( const std::vector& compressed ); -void lzma_compress_file( path src_path, - path dst_path, +void lzma_compress_file( const path& src_path, + const path& dst_path, unsigned char level = 5, unsigned int dict_size = (1 << 20) ); +void lzma_decompress_file( const path& src_path, + const path& dst_path ); + } // namespace fc diff --git a/src/compress/lzma.cpp b/src/compress/lzma.cpp index 91d5475..21ad80c 100644 --- a/src/compress/lzma.cpp +++ b/src/compress/lzma.cpp @@ -1,15 +1,8 @@ +#include #include #include - -#include - -#include - #include - - - -#include +#include namespace fc { @@ -120,8 +113,8 @@ static size_t output_callback( void* output_ctx, const void* output_buf, size_t return output_len; } -void lzma_compress_file( path src_path, - path dst_path, +void lzma_compress_file( const path& src_path, + const path& dst_path, unsigned char level, unsigned int dict_size ) { @@ -170,29 +163,36 @@ void lzma_compress_file( path src_path, elzma_compress_free( &handle ); FC_ASSERT( rc == ELZMA_E_OK ); +} +void lzma_decompress_file( const path& src_path, + const path& dst_path ) +{ + FC_ASSERT( exists( src_path ) ); + FC_ASSERT( !exists( dst_path ) ); - /* TEST */ - FC_ASSERT( exists( dst_path ) ); + boost::iostreams::mapped_file_source src_file; + src_file.open( src_path.string() ); + FC_ASSERT( src_file.is_open() ); - boost::iostreams::mapped_file_source dst_file; - dst_file.open( dst_path.string() ); - FC_ASSERT( dst_file.is_open() ); + elzma_decompress_handle handle = NULL; + handle = elzma_decompress_alloc(); + FC_ASSERT( handle != NULL ); - std::vector result( dst_file.data(), dst_file.data() + dst_file.size() ); - dst_file.close(); + struct lzma_file_ctx ctx; + ctx.src_buf = ( const unsigned char* )src_file.data(); + ctx.src_len = src_file.size(); + ctx.dst_path = dst_path; - for( const auto& c : result ) - { - std::cout << c; - } - std::cout << "\n"; + auto rc = elzma_decompress_run( handle, + input_callback, + ( void * )&ctx, + output_callback, + ( void * )&ctx, + elzma_file_format::ELZMA_lzma ); - result = lzma_decompress( result ); - for( const auto& c : result ) - { - std::cout << c; - } + elzma_decompress_free( &handle ); + FC_ASSERT( rc == ELZMA_E_OK ); } } // namespace fc diff --git a/tests/lzma_compress_file.cpp b/tests/lzma_test.cpp similarity index 59% rename from tests/lzma_compress_file.cpp rename to tests/lzma_test.cpp index 3f76733..3457597 100644 --- a/tests/lzma_compress_file.cpp +++ b/tests/lzma_test.cpp @@ -8,16 +8,17 @@ using namespace fc; int main( int argc, char** argv ) { - if( argc != 2 && argc != 3 ) + if( argc != 2 ) { - std::cout << "usage: " << argv[0] << " [dst_path = src_path.lzma]\n"; + std::cout << "usage: " << argv[0] << " \n"; exit( -1 ); } auto src = std::string( argv[1] ); - auto dst = (argc == 3) ? std::string( argv[2] ) : src + ".lzma"; - + auto dst = src + ".compressed"; lzma_compress_file( src, dst ); + lzma_decompress_file( dst, src + ".decompressed" ); + return 0; } From cc1ef61cd67b1bb3f8798700ca6d62809455b1fd Mon Sep 17 00:00:00 2001 From: Vikram Rajkumar Date: Thu, 3 Jul 2014 01:16:46 -0400 Subject: [PATCH 3/8] Logs are now rotated on startup if rotation is enabled --- include/fc/log/file_appender.hpp | 7 ++- src/log/file_appender.cpp | 105 ++++++++++++++----------------- 2 files changed, 53 insertions(+), 59 deletions(-) diff --git a/include/fc/log/file_appender.hpp b/include/fc/log/file_appender.hpp index a6f0e76..5963692 100644 --- a/include/fc/log/file_appender.hpp +++ b/include/fc/log/file_appender.hpp @@ -1,7 +1,8 @@ #pragma once + +#include #include #include -#include #include namespace fc { @@ -20,6 +21,7 @@ class file_appender : public appender { bool rotate; microseconds rotation_interval; microseconds rotation_limit; + bool rotation_compression; }; file_appender( const variant& args ); ~file_appender(); @@ -32,4 +34,5 @@ class file_appender : public appender { } // namespace fc #include -FC_REFLECT( fc::file_appender::config, (format)(filename)(flush)(truncate)(rotate)(rotation_interval)(rotation_limit) ) +FC_REFLECT( fc::file_appender::config, + (format)(filename)(flush)(truncate)(rotate)(rotation_interval)(rotation_limit)(rotation_compression) ) diff --git a/src/log/file_appender.cpp b/src/log/file_appender.cpp index 55a4693..c37c6f3 100644 --- a/src/log/file_appender.cpp +++ b/src/log/file_appender.cpp @@ -1,14 +1,11 @@ -#include -#include -#include -#include #include -#include +#include #include +#include +#include +#include #include #include -#include - namespace fc { class file_appender::impl : public fc::retainable { @@ -24,10 +21,49 @@ namespace fc { const auto file_number = timestamp.sec_since_epoch() / interval_seconds; return time_point_sec( file_number * interval_seconds ); } + + void rotate_files( bool initializing = false ) + { + if( !cfg.rotate ) return; + const auto now = time_point::now(); + const auto start_time = get_file_start_time( now, cfg.rotation_interval ); + if( !initializing ) + { + if( start_time <= current_file_start_time ) return; + out.close(); + } + + auto log_filename = cfg.filename.string(); + + /* Delete old log files */ + const auto limit_time = now - cfg.rotation_limit; + auto itr = directory_iterator( fc::path( log_filename ).parent_path() ); + for( ; itr != directory_iterator(); itr++ ) + { + const auto current_filename = itr->string(); + const auto current_pos = current_filename.find( log_filename ); + if( current_pos != 0 ) continue; + const auto current_timestamp = current_filename.substr( log_filename.size() + 1 ); // TODO SKI{P EXTENSION + try + { + if( std::stoi( current_timestamp ) < limit_time.sec_since_epoch() ) + remove_all( current_filename ); + //else compress if not already compressed + } + catch( ... ) + { + } + } + + // TODO: Convert to proper timestamp string + log_filename += "." + std::to_string( start_time.sec_since_epoch() ); + out.open( log_filename.c_str() ); + current_file_start_time = start_time; + } }; file_appender::config::config( const fc::path& p ) :format( "${timestamp} ${thread_name} ${context} ${file}:${line} ${method} ${level}] ${message}" ), - filename(p),flush(true),truncate(true),rotate(false){} + filename(p),flush(true),truncate(true),rotate(false),rotation_compression(false){} file_appender::file_appender( const variant& args ) :my( new impl() ) @@ -40,19 +76,12 @@ namespace fc { fc::create_directories( fc::path( log_filename ).parent_path() ); - if( my->cfg.rotate ) - { - const auto start_time = my->get_file_start_time( time_point::now(), my->cfg.rotation_interval ); - // TODO: Convert to proper timestamp string - log_filename += "." + std::to_string( start_time.sec_since_epoch() ); - my->current_file_start_time = start_time; - } - - my->out.open( log_filename.c_str() ); + if( !my->cfg.rotate ) my->out.open( log_filename.c_str() ); + else my->rotate_files( true ); } catch( ... ) { - std::cerr << "error opening log file: " << my->cfg.filename.string() << "\n"; + std::cerr << "error opening log file: " << log_filename << "\n"; //elog( "%s", fc::except_str().c_str() ); } } @@ -61,7 +90,6 @@ namespace fc { // MS THREAD METHOD MESSAGE \t\t\t File:Line void file_appender::log( const log_message& m ) { - const auto now = time_point::now(); std::stringstream line; //line << (m.get_context().get_timestamp().time_since_epoch().count() % (1000ll*1000ll*60ll*60))/1000 <<"ms "; line << std::string(m.get_context().get_timestamp()) << " "; @@ -91,44 +119,7 @@ namespace fc { /* Write to log file (rotating file beforehand if necessary) */ { fc::scoped_lock lock( my->slock ); - - if( my->cfg.rotate ) - { - const auto start_time = my->get_file_start_time( now, my->cfg.rotation_interval ); - if( start_time > my->current_file_start_time ) - { - my->out.close(); - auto log_filename = my->cfg.filename.string(); - - /* Delete old log files */ - // TODO: Delete on startup as well - const auto limit_time = now - my->cfg.rotation_limit; - auto itr = directory_iterator( fc::path( log_filename ).parent_path() ); - for( ; itr != directory_iterator(); itr++ ) - { - const auto current_filename = itr->string(); - const auto current_pos = current_filename.find( log_filename ); - if( current_pos != 0 ) continue; - const auto current_timestamp = current_filename.substr( log_filename.size() + 1 ); - try - { - if( std::stoi( current_timestamp ) < limit_time.sec_since_epoch() ) - remove_all( current_filename ); - //else compress if not already compressed - } - catch( ... ) - { - } - } - - // TODO: Convert to proper timestamp string - log_filename += "." + std::to_string( start_time.sec_since_epoch() ); - my->out.open( log_filename.c_str() ); - - my->current_file_start_time = start_time; - } - } - + if( my->cfg.rotate ) my->rotate_files(); my->out << line.str() << "\t\t\t" << m.get_context().get_file() <<":"<cfg.flush ) my->out.flush(); } From b1e677bceeb68a8d41d53a7458c8c89410281473 Mon Sep 17 00:00:00 2001 From: Vikram Rajkumar Date: Thu, 3 Jul 2014 01:27:58 -0400 Subject: [PATCH 4/8] Implement optional LZMA compression for rotated logs --- src/log/file_appender.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/log/file_appender.cpp b/src/log/file_appender.cpp index c37c6f3..82e8f0b 100644 --- a/src/log/file_appender.cpp +++ b/src/log/file_appender.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -41,14 +42,22 @@ namespace fc { for( ; itr != directory_iterator(); itr++ ) { const auto current_filename = itr->string(); - const auto current_pos = current_filename.find( log_filename ); + auto current_pos = current_filename.find( log_filename ); if( current_pos != 0 ) continue; - const auto current_timestamp = current_filename.substr( log_filename.size() + 1 ); // TODO SKI{P EXTENSION + current_pos = log_filename.size() + 1; + const auto current_timestamp = current_filename.substr( current_pos, current_pos + 10 ); try { if( std::stoi( current_timestamp ) < limit_time.sec_since_epoch() ) + { remove_all( current_filename ); - //else compress if not already compressed + } + else if( cfg.rotation_compression ) + { + if( current_filename.substr( current_filename.size() - 3 ) == ".7z" ) continue; + lzma_compress_file( current_filename, current_filename + ".7z" ); + remove_all( current_filename ); + } } catch( ... ) { From b522f12a0e71e57eefcbe5601844e3afde8ad94d Mon Sep 17 00:00:00 2001 From: Vikram Rajkumar Date: Thu, 3 Jul 2014 02:17:03 -0400 Subject: [PATCH 5/8] Use ISO format timestamps for rotated log files --- src/log/file_appender.cpp | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/log/file_appender.cpp b/src/log/file_appender.cpp index 82e8f0b..246521a 100644 --- a/src/log/file_appender.cpp +++ b/src/log/file_appender.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,17 @@ namespace fc { return time_point_sec( file_number * interval_seconds ); } + string timestamp_to_string( const time_point_sec& timestamp ) + { + auto ptime = boost::posix_time::from_time_t( time_t ( timestamp.sec_since_epoch() ) ); + return boost::posix_time::to_iso_string( ptime ); + } + + time_point_sec string_to_timestamp( const string& str ) + { + return time_point::from_iso_string( str ); + } + void rotate_files( bool initializing = false ) { if( !cfg.rotate ) return; @@ -35,6 +47,7 @@ namespace fc { } auto log_filename = cfg.filename.string(); + const auto timestamp_string = timestamp_to_string( start_time ); /* Delete old log files */ const auto limit_time = now - cfg.rotation_limit; @@ -45,10 +58,11 @@ namespace fc { auto current_pos = current_filename.find( log_filename ); if( current_pos != 0 ) continue; current_pos = log_filename.size() + 1; - const auto current_timestamp = current_filename.substr( current_pos, current_pos + 10 ); + const auto current_timestamp = string( current_filename.begin() + current_pos, + current_filename.begin() + current_pos + timestamp_string.size() ); try { - if( std::stoi( current_timestamp ) < limit_time.sec_since_epoch() ) + if( string_to_timestamp( current_timestamp ) < limit_time ) { remove_all( current_filename ); } @@ -64,10 +78,9 @@ namespace fc { } } - // TODO: Convert to proper timestamp string - log_filename += "." + std::to_string( start_time.sec_since_epoch() ); - out.open( log_filename.c_str() ); current_file_start_time = start_time; + log_filename += "." + timestamp_string; + out.open( log_filename.c_str() ); } }; file_appender::config::config( const fc::path& p ) From 223f0005f54ff48cea300fc5ee4a2ecdacd5a7b1 Mon Sep 17 00:00:00 2001 From: Vikram Rajkumar Date: Thu, 3 Jul 2014 02:35:18 -0400 Subject: [PATCH 6/8] Hardlink from most recent log to original filename during rotation --- src/log/file_appender.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/log/file_appender.cpp b/src/log/file_appender.cpp index 246521a..203ecb1 100644 --- a/src/log/file_appender.cpp +++ b/src/log/file_appender.cpp @@ -47,6 +47,8 @@ namespace fc { } auto log_filename = cfg.filename.string(); + const auto link_filename = log_filename; + remove_all( link_filename ); const auto timestamp_string = timestamp_to_string( start_time ); /* Delete old log files */ @@ -69,6 +71,7 @@ namespace fc { else if( cfg.rotation_compression ) { if( current_filename.substr( current_filename.size() - 3 ) == ".7z" ) continue; + /* TODO: Possibly move compression to another thread */ lzma_compress_file( current_filename, current_filename + ".7z" ); remove_all( current_filename ); } @@ -81,6 +84,7 @@ namespace fc { current_file_start_time = start_time; log_filename += "." + timestamp_string; out.open( log_filename.c_str() ); + create_hard_link( log_filename, link_filename ); } }; file_appender::config::config( const fc::path& p ) From adf8c10ee54a5a2bedc694d58cc80c8b1d3d56ff Mon Sep 17 00:00:00 2001 From: Vikram Rajkumar Date: Thu, 3 Jul 2014 02:49:35 -0400 Subject: [PATCH 7/8] Fix Linux compilation --- src/compress/lzma.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/compress/lzma.cpp b/src/compress/lzma.cpp index 21ad80c..d69cc86 100644 --- a/src/compress/lzma.cpp +++ b/src/compress/lzma.cpp @@ -1,7 +1,7 @@ +#include #include #include #include -#include #include namespace fc { @@ -59,7 +59,7 @@ struct lzma_file_ctx path dst_path; }; -static int input_callback( void* input_ctx, void* input_buf, size_t* input_len ) +static int lzma_file_input_callback( void* input_ctx, void* input_buf, size_t* input_len ) { FC_ASSERT( input_ctx != NULL ); FC_ASSERT( input_buf != NULL ); @@ -79,7 +79,7 @@ static int input_callback( void* input_ctx, void* input_buf, size_t* input_len ) return 0; } -static size_t output_callback( void* output_ctx, const void* output_buf, size_t output_len ) +static size_t lzma_file_output_callback( void* output_ctx, const void* output_buf, size_t output_len ) { FC_ASSERT( output_ctx != NULL ); FC_ASSERT( output_buf != NULL ); @@ -91,8 +91,8 @@ static size_t output_callback( void* output_ctx, const void* output_buf, size_t size_t dst_len = 0; if( !exists( ctx->dst_path ) ) { - auto fs = std::ofstream( ctx->dst_path.string() ); - fs.close(); + boost::filesystem::ofstream ofs( ctx->dst_path.string() ); + ofs.close(); } else { @@ -154,9 +154,9 @@ void lzma_compress_file( const path& src_path, } rc = elzma_compress_run( handle, - input_callback, + lzma_file_input_callback, ( void * )&ctx, - output_callback, + lzma_file_output_callback, ( void * )&ctx, NULL, NULL ); @@ -185,9 +185,9 @@ void lzma_decompress_file( const path& src_path, ctx.dst_path = dst_path; auto rc = elzma_decompress_run( handle, - input_callback, + lzma_file_input_callback, ( void * )&ctx, - output_callback, + lzma_file_output_callback, ( void * )&ctx, elzma_file_format::ELZMA_lzma ); From e9b1b8ec2eb55aa920466d4774bf9c2619b3c556 Mon Sep 17 00:00:00 2001 From: Vikram Rajkumar Date: Thu, 3 Jul 2014 10:02:40 -0400 Subject: [PATCH 8/8] Use a separate thread for log compression --- include/fc/log/file_appender.hpp | 2 - include/fc/thread/thread.hpp | 2 +- src/compress/lzma.cpp | 8 +- src/log/file_appender.cpp | 129 ++++++++++++++++++++++--------- src/thread/thread.cpp | 8 +- 5 files changed, 101 insertions(+), 48 deletions(-) diff --git a/include/fc/log/file_appender.hpp b/include/fc/log/file_appender.hpp index 5963692..af7aa62 100644 --- a/include/fc/log/file_appender.hpp +++ b/include/fc/log/file_appender.hpp @@ -7,8 +7,6 @@ namespace fc { -class varaint; - class file_appender : public appender { public: struct config { diff --git a/include/fc/thread/thread.hpp b/include/fc/thread/thread.hpp index e0863ad..a133ba4 100644 --- a/include/fc/thread/thread.hpp +++ b/include/fc/thread/thread.hpp @@ -9,7 +9,7 @@ namespace fc { class thread { public: - thread( const char* name = "" ); + thread( const std::string& name = "" ); thread( thread&& m ); thread& operator=(thread&& t ); diff --git a/src/compress/lzma.cpp b/src/compress/lzma.cpp index d69cc86..fb669d0 100644 --- a/src/compress/lzma.cpp +++ b/src/compress/lzma.cpp @@ -1,9 +1,11 @@ -#include #include #include #include +#include #include +#include + namespace fc { std::vector lzma_compress(const std::vector& in) @@ -91,8 +93,8 @@ static size_t lzma_file_output_callback( void* output_ctx, const void* output_bu size_t dst_len = 0; if( !exists( ctx->dst_path ) ) { - boost::filesystem::ofstream ofs( ctx->dst_path.string() ); - ofs.close(); + ofstream fs( ctx->dst_path ); + fs.close(); } else { diff --git a/src/log/file_appender.cpp b/src/log/file_appender.cpp index 203ecb1..e012b5a 100644 --- a/src/log/file_appender.cpp +++ b/src/log/file_appender.cpp @@ -1,21 +1,31 @@ #include +#include #include #include #include #include +#include #include #include #include #include +#include #include namespace fc { + + static const string compression_extension( ".lzma" ); + class file_appender::impl : public fc::retainable { public: - config cfg; - ofstream out; - boost::mutex slock; - time_point_sec current_file_start_time; + config cfg; + ofstream out; + boost::mutex slock; + + private: + future _rotation_task; + time_point_sec _current_file_start_time; + std::unique_ptr _compression_thread; time_point_sec get_file_start_time( const time_point_sec& timestamp, const microseconds& interval ) { @@ -35,45 +45,96 @@ namespace fc { return time_point::from_iso_string( str ); } + void compress_file( const string& filename ) + { + FC_ASSERT( cfg.rotate && cfg.rotation_compression ); + FC_ASSERT( _compression_thread ); + if( !_compression_thread->is_current() ) + { + _compression_thread->async( [this, filename]() { compress_file( filename ); } ).wait(); + return; + } + + try + { + lzma_compress_file( filename, filename + compression_extension ); + remove_all( filename ); + } + catch( ... ) + { + } + } + + public: + impl( const config& c) : cfg( c ) + { + if( cfg.rotate ) + { + if( cfg.rotation_compression ) + _compression_thread.reset( new thread( "compression") ); + + _rotation_task = async( [this]() { rotate_files( true ); } ); + } + } + + ~impl() + { + try + { + _rotation_task.cancel_and_wait(); + if( _compression_thread ) _compression_thread->quit(); + } + catch( ... ) + { + } + } + void rotate_files( bool initializing = false ) { - if( !cfg.rotate ) return; + FC_ASSERT( cfg.rotate ); const auto now = time_point::now(); const auto start_time = get_file_start_time( now, cfg.rotation_interval ); + const auto timestamp_string = timestamp_to_string( start_time ); + const auto link_filename = cfg.filename.string(); + const auto log_filename = link_filename + "." + timestamp_string; + if( !initializing ) { - if( start_time <= current_file_start_time ) return; + if( start_time <= _current_file_start_time ) return; + fc::scoped_lock lock( slock ); + out.flush(); out.close(); } - auto log_filename = cfg.filename.string(); - const auto link_filename = log_filename; + out.open( log_filename.c_str() ); remove_all( link_filename ); - const auto timestamp_string = timestamp_to_string( start_time ); + create_hard_link( log_filename, link_filename ); /* Delete old log files */ const auto limit_time = now - cfg.rotation_limit; - auto itr = directory_iterator( fc::path( log_filename ).parent_path() ); + auto itr = directory_iterator( fc::path( link_filename ).parent_path() ); for( ; itr != directory_iterator(); itr++ ) { - const auto current_filename = itr->string(); - auto current_pos = current_filename.find( log_filename ); - if( current_pos != 0 ) continue; - current_pos = log_filename.size() + 1; - const auto current_timestamp = string( current_filename.begin() + current_pos, - current_filename.begin() + current_pos + timestamp_string.size() ); try { - if( string_to_timestamp( current_timestamp ) < limit_time ) + const auto current_filename = itr->string(); + auto current_pos = current_filename.find( link_filename ); + if( current_pos != 0 ) continue; + current_pos = link_filename.size() + 1; + const auto current_timestamp_str = string( current_filename.begin() + current_pos, /* substr not working */ + current_filename.begin() + current_pos + timestamp_string.size() ); + const auto current_timestamp = string_to_timestamp( current_timestamp_str ); + if( current_timestamp < start_time ) { - remove_all( current_filename ); - } - else if( cfg.rotation_compression ) - { - if( current_filename.substr( current_filename.size() - 3 ) == ".7z" ) continue; - /* TODO: Possibly move compression to another thread */ - lzma_compress_file( current_filename, current_filename + ".7z" ); - remove_all( current_filename ); + if( current_timestamp < limit_time || file_size( current_filename ) <= 0 ) + { + remove_all( current_filename ); + continue; + } + + if( !cfg.rotation_compression ) continue; + if( current_filename.find( compression_extension ) != string::npos ) continue; + compress_file( current_filename ); } } catch( ... ) @@ -81,34 +142,29 @@ namespace fc { } } - current_file_start_time = start_time; - log_filename += "." + timestamp_string; - out.open( log_filename.c_str() ); - create_hard_link( log_filename, link_filename ); + _current_file_start_time = start_time; + _rotation_task = schedule( [this]() { rotate_files(); }, now + cfg.rotation_interval ); } }; file_appender::config::config( const fc::path& p ) :format( "${timestamp} ${thread_name} ${context} ${file}:${line} ${method} ${level}] ${message}" ), - filename(p),flush(true),truncate(true),rotate(false),rotation_compression(false){} + filename(p),flush(true),truncate(true),rotate(false),rotation_compression(true){} file_appender::file_appender( const variant& args ) - :my( new impl() ) + :my( new impl( args.as() ) ) { std::string log_filename; try { - my->cfg = args.as(); log_filename = my->cfg.filename.string(); fc::create_directories( fc::path( log_filename ).parent_path() ); if( !my->cfg.rotate ) my->out.open( log_filename.c_str() ); - else my->rotate_files( true ); } catch( ... ) { std::cerr << "error opening log file: " << log_filename << "\n"; - //elog( "%s", fc::except_str().c_str() ); } } file_appender::~file_appender(){} @@ -142,12 +198,11 @@ namespace fc { // fc::string fmt_str = fc::format_string( my->cfg.format, mutable_variant_object(m.get_context())( "message", message) ); - /* Write to log file (rotating file beforehand if necessary) */ { fc::scoped_lock lock( my->slock ); - if( my->cfg.rotate ) my->rotate_files(); my->out << line.str() << "\t\t\t" << m.get_context().get_file() <<":"<cfg.flush ) my->out.flush(); } } -} + +} // fc diff --git a/src/thread/thread.cpp b/src/thread/thread.cpp index 4d856fa..17016fa 100644 --- a/src/thread/thread.cpp +++ b/src/thread/thread.cpp @@ -70,11 +70,11 @@ namespace fc { return t; } - thread::thread( const char* name ) { + thread::thread( const std::string& name ) { promise::ptr p(new promise()); boost::thread* t = new boost::thread( [this,p,name]() { try { - set_thread_name(name); // set thread's name for the debugger to display + set_thread_name(name.c_str()); // set thread's name for the debugger to display this->my = new thread_d(*this); current_thread() = this; p->set_value(); @@ -126,17 +126,15 @@ namespace fc { void thread::debug( const fc::string& d ) { /*my->debug(d);*/ } void thread::quit() { - //if quiting from a different thread, start quit task on thread. + //if quitting from a different thread, start quit task on thread. //If we have and know our attached boost thread, wait for it to finish, then return. if( ¤t() != this ) { async( [=](){quit();} );//.wait(); if( my->boost_thread ) { auto n = name(); - ilog( "joining... ${n}", ("n",n) );//n.c_str() ); my->boost_thread->join(); delete my; my = nullptr; - ilog( "done joining...${n}", ("n",n) ); //n.c_str() ); } return; }