Implement gzip-compression of rotated log files (this effectively

restores the functionality removed in 701b9ac00c,
which had lzma-compression of rotated logs)

CMakeLists.txt

@@ -25,6 +25,12 @@ SET( DEFAULT_EXECUTABLE_INSTALL_DIR bin/ )
 SET( CMAKE_DEBUG_POSTFIX _debug )
 SET( BUILD_SHARED_LIBS NO )
 SET( ECC_IMPL secp256k1 CACHE STRING "secp256k1 or openssl or mixed" )
+SET( FC_USE_FULL_ZLIB FALSE CACHE BOOL "TRUE to try to use full zlib for compression, FALSE to use miniz.c")
+
+if( FC_USE_FULL_ZLIB )
+  find_package( ZLIB REQUIRED )
+  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DFC_USE_FULL_ZLIB")
+endif()
 
 set(platformBitness 32)
 if(CMAKE_SIZEOF_VOID_P EQUAL 8)
@@ -306,6 +312,7 @@ target_include_directories(fc
     ${CMAKE_CURRENT_SOURCE_DIR}/vendor/udt4/src
     ${CMAKE_CURRENT_SOURCE_DIR}/vendor/websocketpp
     ${CMAKE_CURRENT_SOURCE_DIR}/vendor/secp256k1-zkp
+    ${ZLIB_INCLUDE_DIRS}
   )
 
 #target_link_libraries( fc PUBLIC udt ${Boost_LIBRARIES} ${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${PLATFORM_SPECIFIC_LIBS} ${RPCRT4} ${CMAKE_DL_LIBS} ${rt_library} ${ECC_LIB} )

include/fc/compress/zlib.hpp

@@ -1,10 +1,16 @@
 #pragma once
 
 #include <fc/string.hpp>
+#ifdef FC_USE_FULL_ZLIB
+# include <fc/filesystem.hpp>
+#endif
 
 namespace fc 
 {
 
   string zlib_compress(const string& in);
+#ifdef FC_USE_FULL_ZLIB
+  void gzip_compress_file(const path& input_filename, const path& output_filename);
+#endif
 
 } // namespace fc

include/fc/log/file_appender.hpp

@@ -18,6 +18,7 @@ class file_appender : public appender {
             bool                               rotate = false;
             microseconds                       rotation_interval;
             microseconds                       rotation_limit;
+            bool                               rotation_compression = false;
          };
          file_appender( const variant& args );
          ~file_appender();
@@ -31,4 +32,4 @@ class file_appender : public appender {
 
 #include <fc/reflect/reflect.hpp>
 FC_REFLECT( fc::file_appender::config,
-            (format)(filename)(flush)(rotate)(rotation_interval)(rotation_limit) )
+            (format)(filename)(flush)(rotate)(rotation_interval)(rotation_limit)(rotation_compression) )

src/compress/zlib.cpp

@@ -1,9 +1,77 @@
 #include <fc/compress/zlib.hpp>
 
-#include "miniz.c"
+#ifdef FC_USE_FULL_ZLIB
+# include <zlib.h>
+# include <memory>
+# include <fstream>
+#else
+# include "miniz.c"
+#endif
 
 namespace fc
 {
+#ifdef FC_USE_FULL_ZLIB
+  string zlib_compress(const string& in)
+  {
+    unsigned long bufferLen = compressBound(in.size());
+    std::unique_ptr<char[]> buffer(new char[bufferLen]);
+    compress((unsigned char*)buffer.get(), &bufferLen, (const unsigned char*)in.c_str(), in.size());
+    string result(buffer.get(), bufferLen);
+    return result;
+  }
+
+  void gzip_compress_file(const path& input_filename, const path& output_filename)
+  {
+    std::ifstream infile(input_filename.generic_string().c_str(), std::ios::binary);
+    std::ofstream outfile(output_filename.generic_string().c_str(), std::ios::out | std::ios::binary);
+    unsigned bufferLen = 1024 * 1024;
+    std::unique_ptr<char[]> inputBuffer(new char[bufferLen]);
+    std::unique_ptr<char[]> outputBuffer(new char[bufferLen]);
+
+    z_stream outputStream;
+    outputStream.zalloc = 0;
+    outputStream.zfree = 0;
+    outputStream.opaque = 0;
+    int windowBits = 15;
+    int GZIP_ENCODING = 16;
+
+    deflateInit2(&outputStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, windowBits | GZIP_ENCODING,
+                 8, Z_DEFAULT_STRATEGY);
+
+    if (infile)
+    {
+      do
+      {
+        infile.read(inputBuffer.get(), bufferLen);
+        int bytesRead = infile.gcount();
+        if (bytesRead == 0)
+          break;
+        outputStream.avail_in = bytesRead;
+        outputStream.next_in = (unsigned char*)inputBuffer.get();
+        do
+        {
+          outputStream.avail_out = bufferLen;
+          outputStream.next_out = (unsigned char*)outputBuffer.get();
+          deflate(&outputStream, Z_NO_FLUSH);
+          int compressedBytesGenerated = bufferLen - outputStream.avail_out;
+          outfile.write(outputBuffer.get(), compressedBytesGenerated);
+        }
+        while (outputStream.avail_out == 0);
+      }
+      while (infile);
+    }
+    do
+    {
+      outputStream.avail_out = bufferLen;
+      outputStream.next_out = (unsigned char*)outputBuffer.get();
+      deflate(&outputStream, Z_FINISH);
+      int compressedBytesGenerated = bufferLen - outputStream.avail_out;
+      outfile.write(outputBuffer.get(), compressedBytesGenerated);
+    }
+    while (outputStream.avail_out == 0);
+    deflateEnd(&outputStream);
+  }
+#else
   string zlib_compress(const string& in)
   {
     size_t compressed_message_length;
@@ -12,4 +80,5 @@ namespace fc
     free(compressed_message);
     return result;
   }
+#endif
 }

src/log/file_appender.cpp

@@ -5,6 +5,9 @@
 #include <fc/thread/scoped_lock.hpp>
 #include <fc/thread/thread.hpp>
 #include <fc/variant.hpp>
+#ifdef FC_USE_FULL_ZLIB
+# include <fc/compress/zlib.hpp>
+#endif
 #include <boost/thread/mutex.hpp>
 #include <iomanip>
 #include <queue>
@@ -12,6 +15,8 @@
 
 namespace fc {
 
+   static const string compression_extension( ".gz" );
+
    class file_appender::impl : public fc::retainable
    {
       public:
@@ -22,6 +27,7 @@ namespace fc {
       private:
          future<void>               _rotation_task;
          time_point_sec             _current_file_start_time;
+         std::unique_ptr<thread>    _compression_thread;
 
          time_point_sec get_file_start_time( const time_point_sec& timestamp, const microseconds& interval )
          {
@@ -30,6 +36,28 @@ namespace fc {
              return time_point_sec( (uint32_t)(file_number * interval_seconds) );
          }
 
+         void compress_file( const fc::path& filename )
+         {
+#ifdef FC_USE_FULL_ZLIB
+             FC_ASSERT( cfg.rotate && cfg.rotation_compression );
+             FC_ASSERT( _compression_thread );
+             if( !_compression_thread->is_current() )
+             {
+                 _compression_thread->async( [this, filename]() { compress_file( filename ); }, "compress_file" ).wait();
+                 return;
+             }
+
+             try
+             {
+                 gzip_compress_file( filename, filename.parent_path() / (filename.filename().string() + compression_extension) );
+                 remove_all( filename );
+             }
+             catch( ... )
+             {
+             }
+#endif
+         }
+
       public:
          impl( const config& c) : cfg( c )
          {
@@ -38,8 +66,10 @@ namespace fc {
                  FC_ASSERT( cfg.rotation_interval >= seconds( 1 ) );
                  FC_ASSERT( cfg.rotation_limit >= cfg.rotation_interval );
 
-
+#ifdef FC_USE_FULL_ZLIB
-
+                 if( cfg.rotation_compression )
+                     _compression_thread.reset( new thread( "compression") );
+#endif
 
                  _rotation_task = async( [this]() { rotate_files( true ); }, "rotate_files(1)" );
              }
@@ -107,11 +137,16 @@ namespace fc {
                      fc::time_point_sec current_timestamp = fc::time_point_sec::from_iso_string( current_timestamp_str );
                      if( current_timestamp < start_time )
                      {
-                         if( current_timestamp < limit_time || file_size( current_filename ) <= 0 )
+                         if( current_timestamp < limit_time || file_size( link_filename.parent_path() / itr->filename() ) <= 0 )
                          {
                              remove_all( *itr );
                              continue;
                          }
+                         if( !cfg.rotation_compression )
+                           continue;
+                         if( current_filename.find( compression_extension ) != string::npos )
+                           continue;
+                         compress_file( *itr );
                      }
                  }
                  catch (const fc::canceled_exception&)
@@ -134,7 +169,8 @@ namespace fc {
      format( "${timestamp} ${thread_name} ${context} ${file}:${line} ${method} ${level}]  ${message}" ),
      filename(p),
      flush(true),
-     rotate(false)
+     rotate(false),
+     rotation_compression(false)
    {}
 
    file_appender::file_appender( const variant& args ) :

tests/compress/compress.cpp

@@ -38,9 +38,9 @@ BOOST_AUTO_TEST_CASE(smaz_test)
     BOOST_CHECK_EQUAL( decomp, line );
 }
 
+#ifndef FC_USE_FULL_ZLIB
 
 extern "C" {
-
 enum
 {
   TINFL_FLAG_PARSE_ZLIB_HEADER = 1,
@@ -85,5 +85,6 @@ BOOST_AUTO_TEST_CASE(zlib_test)
     std::string decomp = zlib_decompress( compressed );
     BOOST_CHECK_EQUAL( decomp, line );
 }
+#endif
 
 BOOST_AUTO_TEST_SUITE_END()