Peerplays_Blockchain/peerplays-fc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement gzip-compression of rotated log files (this effectively

Browse source 
restores the functionality removed in 701b9ac00c,
which had lzma-compression of rotated logs)
This commit is contained in:
Eric Frias 2016-08-12 18:39:57 -04:00
parent 605e9ed7ef
commit 57d14c7de8
6 changed files with 127 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
CMakeLists.txt
View file

@ -25,6 +25,12 @@ SET( DEFAULT_EXECUTABLE_INSTALL_DIR bin/ )
SET( CMAKE_DEBUG_POSTFIX _debug )
SET( BUILD_SHARED_LIBS NO )
SET( ECC_IMPL secp256k1 CACHE STRING "secp256k1 or openssl or mixed" )
SET( FC_USE_FULL_ZLIB FALSE CACHE BOOL "TRUE to try to use full zlib for compression, FALSE to use miniz.c")
if( FC_USE_FULL_ZLIB )
find_package( ZLIB REQUIRED )
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DFC_USE_FULL_ZLIB")
endif()
set(platformBitness 32)
if(CMAKE_SIZEOF_VOID_P EQUAL 8)
@ -306,6 +312,7 @@ target_include_directories(fc
${CMAKE_CURRENT_SOURCE_DIR}/vendor/udt4/src
${CMAKE_CURRENT_SOURCE_DIR}/vendor/websocketpp
${CMAKE_CURRENT_SOURCE_DIR}/vendor/secp256k1-zkp
${ZLIB_INCLUDE_DIRS}
)
#target_link_libraries( fc PUBLIC udt ${Boost_LIBRARIES} ${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${PLATFORM_SPECIFIC_LIBS} ${RPCRT4} ${CMAKE_DL_LIBS} ${rt_library} ${ECC_LIB} )

6
include/fc/compress/zlib.hpp
View file

@ -1,10 +1,16 @@
#pragma once
#include <fc/string.hpp>
#ifdef FC_USE_FULL_ZLIB
# include <fc/filesystem.hpp>
#endif
namespace fc
{
string zlib_compress(const string& in);
#ifdef FC_USE_FULL_ZLIB
void gzip_compress_file(const path& input_filename, const path& output_filename);
#endif
} // namespace fc

3
include/fc/log/file_appender.hpp
View file

@ -18,6 +18,7 @@ class file_appender : public appender {
bool rotate = false;
microseconds rotation_interval;
microseconds rotation_limit;
bool rotation_compression = false;
};
file_appender( const variant& args );
~file_appender();
@ -31,4 +32,4 @@ class file_appender : public appender {
#include <fc/reflect/reflect.hpp>
FC_REFLECT( fc::file_appender::config,
(format)(filename)(flush)(rotate)(rotation_interval)(rotation_limit) )
(format)(filename)(flush)(rotate)(rotation_interval)(rotation_limit)(rotation_compression) )

71
src/compress/zlib.cpp
View file

@ -1,9 +1,77 @@
#include <fc/compress/zlib.hpp>
#include "miniz.c"
#ifdef FC_USE_FULL_ZLIB
# include <zlib.h>
# include <memory>
# include <fstream>
#else
# include "miniz.c"
#endif
namespace fc
{
#ifdef FC_USE_FULL_ZLIB
string zlib_compress(const string& in)
{
unsigned long bufferLen = compressBound(in.size());
std::unique_ptr<char[]> buffer(new char[bufferLen]);
compress((unsigned char*)buffer.get(), &bufferLen, (const unsigned char*)in.c_str(), in.size());
string result(buffer.get(), bufferLen);
return result;
}
void gzip_compress_file(const path& input_filename, const path& output_filename)
{
std::ifstream infile(input_filename.generic_string().c_str(), std::ios::binary);
std::ofstream outfile(output_filename.generic_string().c_str(), std::ios::out | std::ios::binary);
unsigned bufferLen = 1024 * 1024;
std::unique_ptr<char[]> inputBuffer(new char[bufferLen]);
std::unique_ptr<char[]> outputBuffer(new char[bufferLen]);
z_stream outputStream;
outputStream.zalloc = 0;
outputStream.zfree = 0;
outputStream.opaque = 0;
int windowBits = 15;
int GZIP_ENCODING = 16;
deflateInit2(&outputStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, windowBits | GZIP_ENCODING,
8, Z_DEFAULT_STRATEGY);
if (infile)
{
do
{
infile.read(inputBuffer.get(), bufferLen);
int bytesRead = infile.gcount();
if (bytesRead == 0)
break;
outputStream.avail_in = bytesRead;
outputStream.next_in = (unsigned char*)inputBuffer.get();
do
{
outputStream.avail_out = bufferLen;
outputStream.next_out = (unsigned char*)outputBuffer.get();
deflate(&outputStream, Z_NO_FLUSH);
int compressedBytesGenerated = bufferLen - outputStream.avail_out;
outfile.write(outputBuffer.get(), compressedBytesGenerated);
}
while (outputStream.avail_out == 0);
}
while (infile);
}
do
{
outputStream.avail_out = bufferLen;
outputStream.next_out = (unsigned char*)outputBuffer.get();
deflate(&outputStream, Z_FINISH);
int compressedBytesGenerated = bufferLen - outputStream.avail_out;
outfile.write(outputBuffer.get(), compressedBytesGenerated);
}
while (outputStream.avail_out == 0);
deflateEnd(&outputStream);
}
#else
string zlib_compress(const string& in)
{
size_t compressed_message_length;
@ -12,4 +80,5 @@ namespace fc
free(compressed_message);
return result;
}
#endif
}

44
src/log/file_appender.cpp
View file

@ -5,6 +5,9 @@
#include <fc/thread/scoped_lock.hpp>
#include <fc/thread/thread.hpp>
#include <fc/variant.hpp>
#ifdef FC_USE_FULL_ZLIB
# include <fc/compress/zlib.hpp>
#endif
#include <boost/thread/mutex.hpp>
#include <iomanip>
#include <queue>
@ -12,6 +15,8 @@
namespace fc {
static const string compression_extension( ".gz" );
class file_appender::impl : public fc::retainable
{
public:
@ -22,6 +27,7 @@ namespace fc {
private:
future<void> _rotation_task;
time_point_sec _current_file_start_time;
std::unique_ptr<thread> _compression_thread;
time_point_sec get_file_start_time( const time_point_sec& timestamp, const microseconds& interval )
{
@ -30,6 +36,28 @@ namespace fc {
return time_point_sec( (uint32_t)(file_number * interval_seconds) );
}
void compress_file( const fc::path& filename )
{
#ifdef FC_USE_FULL_ZLIB
FC_ASSERT( cfg.rotate && cfg.rotation_compression );
FC_ASSERT( _compression_thread );
if( !_compression_thread->is_current() )
{
_compression_thread->async( [this, filename]() { compress_file( filename ); }, "compress_file" ).wait();
return;
}
try
{
gzip_compress_file( filename, filename.parent_path() / (filename.filename().string() + compression_extension) );
remove_all( filename );
}
catch( ... )
{
}
#endif
}
public:
impl( const config& c) : cfg( c )
{
@ -38,8 +66,10 @@ namespace fc {
FC_ASSERT( cfg.rotation_interval >= seconds( 1 ) );
FC_ASSERT( cfg.rotation_limit >= cfg.rotation_interval );
#ifdef FC_USE_FULL_ZLIB
if( cfg.rotation_compression )
_compression_thread.reset( new thread( "compression") );
#endif
_rotation_task = async( [this]() { rotate_files( true ); }, "rotate_files(1)" );
}
@ -107,11 +137,16 @@ namespace fc {
fc::time_point_sec current_timestamp = fc::time_point_sec::from_iso_string( current_timestamp_str );
if( current_timestamp < start_time )
{
if( current_timestamp < limit_time || file_size( current_filename ) <= 0 )
if( current_timestamp < limit_time || file_size( link_filename.parent_path() / itr->filename() ) <= 0 )
{
remove_all( *itr );
continue;
}
if( !cfg.rotation_compression )
continue;
if( current_filename.find( compression_extension ) != string::npos )
continue;
compress_file( *itr );
}
}
catch (const fc::canceled_exception&)
@ -134,7 +169,8 @@ namespace fc {
format( "${timestamp} ${thread_name} ${context} ${file}:${line} ${method} ${level}] ${message}" ),
filename(p),
flush(true),
rotate(false)
rotate(false),
rotation_compression(false)
{}
file_appender::file_appender( const variant& args ) :

3
tests/compress/compress.cpp
View file

@ -38,9 +38,9 @@ BOOST_AUTO_TEST_CASE(smaz_test)
BOOST_CHECK_EQUAL( decomp, line );
}
#ifndef FC_USE_FULL_ZLIB
extern "C" {
enum
{
TINFL_FLAG_PARSE_ZLIB_HEADER = 1,
@ -85,5 +85,6 @@ BOOST_AUTO_TEST_CASE(zlib_test)
std::string decomp = zlib_decompress( compressed );
BOOST_CHECK_EQUAL( decomp, line );
}
#endif
BOOST_AUTO_TEST_SUITE_END()