Improvements for handling unicode filenames on Windows. When converting fc::path to/from fc::variant, use utf8 encoding. Replace several places where we use a std::ifstream and open it with a char* filename with a boost::filesystem::ifstream and open it with a boost::filesystem::path, which does unicode correctly.

src/compress/lzma.cpp

@@ -1,3 +1,4 @@
+#include <boost/filesystem/path.hpp>
 #include <boost/iostreams/device/mapped_file.hpp>
 #include <fc/compress/lzma.hpp>
 #include <fc/exception/exception.hpp>
@@ -104,7 +105,7 @@ static size_t lzma_file_output_callback( void* output_ctx, const void* output_bu
         resize_file( ctx->dst_path, dst_len + output_len );
 
         boost::iostreams::mapped_file_sink dst_file;
-        dst_file.open( ctx->dst_path.string() );
+        dst_file.open( (boost::filesystem::path)ctx->dst_path );
         FC_ASSERT( dst_file.is_open() );
 
         memcpy( ( void* )(dst_file.data() + dst_len), output_buf, output_len);
@@ -124,7 +125,7 @@ void lzma_compress_file( const path& src_path,
     FC_ASSERT( !exists( dst_path ) );
 
     boost::iostreams::mapped_file_source src_file;
-    src_file.open( src_path.to_native_ansi_path() );
+    src_file.open( (boost::filesystem::path)src_path );
     FC_ASSERT( src_file.is_open() );
 
     elzma_compress_handle handle = NULL;
@@ -174,7 +175,7 @@ void lzma_decompress_file( const path& src_path,
     FC_ASSERT( !exists( dst_path ) );
 
     boost::iostreams::mapped_file_source src_file;
-    src_file.open( src_path.to_native_ansi_path() );
+    src_file.open( (boost::filesystem::path)src_path );
     FC_ASSERT( src_file.is_open() );
 
     elzma_decompress_handle handle = NULL;

src/filesystem.cpp

@@ -21,19 +21,24 @@
 #endif
 
 namespace fc {
-  void to_variant( const fc::path& t, variant& v ) {
+  // when converting to and from a variant, store utf-8 in the variant
-    std::string path = t.to_native_ansi_path();
+  void to_variant( const fc::path& path_to_convert, variant& variant_output ) 
-    for(auto& c : path)
+  {
-    {
+    std::wstring wide_string = path_to_convert.generic_wstring();
-      if(c == '\\')
+    std::string utf8_string;
-        c = '/';
+    fc::encodeUtf8(wide_string, &utf8_string);
-    }
+    variant_output = utf8_string;
 
-    v = path;
+    //std::string path = t.to_native_ansi_path();
+    //std::replace(path.begin(), path.end(), '\\', '/');
+    //v = path;
   }
 
-  void from_variant( const fc::variant& v, fc::path& t ) {
+  void from_variant( const fc::variant& variant_to_convert, fc::path& path_output ) 
-    t = fc::path(v.as_string());
+  {
+    std::wstring wide_string;
+    fc::decodeUtf8(variant_to_convert.as_string(), &wide_string);
+    path_output = path(wide_string);
   }
 
    // Note: we can do this cast because the separator should be an ASCII character
@@ -135,11 +140,9 @@ namespace fc {
     *  faster performance
     */
    fc::string path::windows_string()const {
-     auto gs = _p->generic_string();
+     std::string result = _p->generic_string();
-     for( size_t i =0 ; i < gs.size(); ++i ) {
+     std::replace(result.begin(), result.end(), '/', '\\');
-       if( gs[i] == '/' ) gs[i] = '\\';
+     return result;
-     }
-     return gs;
    }
 
    fc::string path::string()const {

src/interprocess/file_mapping.cpp

@@ -6,18 +6,36 @@
 namespace fc {
 
 
-  file_mapping::file_mapping( const char* file, mode_t m )
+  file_mapping::file_mapping( const char* file, mode_t m ) :
-  :my(file, m == read_only ? boost::interprocess::read_only : boost::interprocess::read_write ){}
+    my(file, m == read_only ? boost::interprocess::read_only : boost::interprocess::read_write )
-  file_mapping::~file_mapping(){}
+  {}
+  
+  file_mapping::~file_mapping() {}
 
 
 
-  mapped_region::mapped_region( const file_mapping& fm, mode_t m, uint64_t start, size_t size )
+  mapped_region::mapped_region( const file_mapping& fm, mode_t m, uint64_t start, size_t size ) :
-  :my( *fm.my, m == read_only ? boost::interprocess::read_only : boost::interprocess::read_write  ,start, size) { }
+    my( *fm.my, m == read_only ? boost::interprocess::read_only : boost::interprocess::read_write  ,start, size) 
-  mapped_region::mapped_region( const file_mapping& fm, mode_t m )
+  {}
-  :my( *fm.my, m == read_only ? boost::interprocess::read_only : boost::interprocess::read_write) { }
+
+  mapped_region::mapped_region( const file_mapping& fm, mode_t m ) :
+    my( *fm.my, m == read_only ? boost::interprocess::read_only : boost::interprocess::read_write) 
+  {}
+
   mapped_region::~mapped_region(){}
-  void* mapped_region::get_address()const { return my->get_address(); }
+
-  void mapped_region::flush(){ my->flush(); }
+  void* mapped_region::get_address() const 
-  size_t mapped_region::get_size()const { return my->get_size();    }
+  {
+    return my->get_address(); 
+  }
+
+  void mapped_region::flush()
+  {
+    my->flush(); 
+  }
+
+  size_t mapped_region::get_size() const 
+  {
+    return my->get_size();
+  }
 }

src/io/fstream.cpp

@@ -5,15 +5,16 @@
 #include <fc/log/logger.hpp>
 
 #include <boost/filesystem/path.hpp>
+#include <boost/filesystem/fstream.hpp>
 
 namespace fc {
    class ofstream::impl : public fc::retainable {
       public:
-         std::ofstream ofs;
+         boost::filesystem::ofstream ofs;
    };
    class ifstream::impl : public fc::retainable {
       public:
-         std::ifstream ifs;
+         boost::filesystem::ifstream ifs;
    };
 
    ofstream::ofstream()
@@ -25,7 +26,7 @@ namespace fc {
 
    void ofstream::open( const fc::path& file, int m ) {
      const boost::filesystem::path& bfp = file; 
-     my->ofs.open( bfp.native(), std::ios::binary );
+     my->ofs.open( bfp, std::ios::binary );
    }
    size_t ofstream::writesome( const char* buf, size_t len ) {
         my->ofs.write(buf,len);
@@ -57,7 +58,7 @@ namespace fc {
 
    void ifstream::open( const fc::path& file, int m ) {
      const boost::filesystem::path& bfp = file; 
-      my->ifs.open( bfp.native(), std::ios::binary );
+      my->ifs.open( bfp, std::ios::binary );
    }
    size_t ifstream::readsome( char* buf, size_t len ) {
       auto s = size_t(my->ifs.readsome( buf, len ));

src/io/json.cpp

@@ -10,6 +10,8 @@
 #include <fstream>
 #include <sstream>
 
+#include <boost/filesystem/fstream.hpp>
+
 namespace fc
 {
    template<typename T>
@@ -645,12 +647,12 @@ namespace fc
 
 
 
-   fc::string   json::to_pretty_string( const variant& v )
+   fc::string json::to_pretty_string( const variant& v )
    {
 	   return pretty_print(to_string(v), 2);
    }
 
-   void          json::save_to_file( const variant& v, const fc::path& fi, bool pretty )
+   void json::save_to_file( const variant& v, const fc::path& fi, bool pretty )
    {
       if( pretty )
       {
@@ -669,8 +671,8 @@ namespace fc
       //auto tmp = std::make_shared<fc::ifstream>( p, ifstream::binary );
       //auto tmp = std::make_shared<std::ifstream>( p.generic_string().c_str(), std::ios::binary );
       //buffered_istream bi( tmp );
-      std::ifstream bi( p.generic_string().c_str(), std::ios::binary );
+      boost::filesystem::ifstream bi( p, std::ios::binary );
-      return variant_from_stream( bi  );
+      return variant_from_stream( bi );
    }
    variant json::from_stream( buffered_istream& in )
    {
@@ -693,7 +695,7 @@ namespace fc
       return out;
    }
 
-   bool     json::is_valid( const std::string& utf8_str )
+   bool json::is_valid( const std::string& utf8_str )
    {
       if( utf8_str.size() == 0 ) return false;
       fc::stringstream in( utf8_str );

src/log/file_appender.cpp

@@ -125,14 +125,14 @@ namespace fc {
                    out.close();
                }
 
-               out.open( log_filename.to_native_ansi_path() );
+               out.open( log_filename );
              }
              remove_all( link_filename );
              create_hard_link( log_filename, link_filename );
 
              /* Delete old log files */
              fc::time_point limit_time = now - cfg.rotation_limit;
-             string link_filename_string = link_filename.filename().to_native_ansi_path();
+             string link_filename_string = link_filename.filename().string();
              directory_iterator itr(link_filename.parent_path());
              for( ; itr != directory_iterator(); itr++ )
              {
@@ -189,11 +189,11 @@ namespace fc {
          fc::create_directories(my->cfg.filename.parent_path());
 
          if(!my->cfg.rotate) 
-           my->out.open(my->cfg.filename.to_native_ansi_path().c_str());
+           my->out.open(my->cfg.filename);
       }
       catch( ... )
       {
-         std::cerr << "error opening log file: " << my->cfg.filename.string() << "\n";
+         std::cerr << "error opening log file: " << my->cfg.filename.preferred_string() << "\n";
       }
    }