include/fc/utf8.hpp

@@ -8,19 +8,21 @@
 namespace fc
 {
 
-bool is_utf8( const std::string& str );
+   std::string prune_invalid_utf8( const std::string& str );
 
-/** Decodes utf 8 std::string into unicode string.
-    @param input   - input string to be decoded and stored in 'storage'
-    @param storage - buffer for converted text. Cannot be nullptr.
-*/
-void decodeUtf8(const std::string& input, std::wstring* storage);
-  
-/** Encodes given wide (unicode) string into UTF-8 representation.
-    @param input - input string to be encoded and stored in 'storage'
-    @param storage - buffer for converted text. Cannot be nullptr.
-*/
-void encodeUtf8(const std::wstring& input, std::string* storage);
+   bool is_utf8( const std::string& str );
+
+   /** Decodes utf 8 std::string into unicode string.
+       @param input   - input string to be decoded and stored in 'storage'
+       @param storage - buffer for converted text. Cannot be nullptr.
+   */
+   void decodeUtf8(const std::string& input, std::wstring* storage);
+     
+   /** Encodes given wide (unicode) string into UTF-8 representation.
+       @param input - input string to be encoded and stored in 'storage'
+       @param storage - buffer for converted text. Cannot be nullptr.
+   */
+   void encodeUtf8(const std::wstring& input, std::string* storage);
 
 } /// namespace fc
 

src/utf8.cpp

@@ -3,16 +3,36 @@
 #include "utf8/checked.h"
 #include "utf8/core.h"
 #include "utf8/unchecked.h"
+#include <websocketpp/utf8_validator.hpp>
 
 #include <assert.h>
+#include <fc/log/logger.hpp>
+#include <iostream>
 
 namespace fc {
 
    bool is_utf8( const std::string& str )
    {
+      auto itr = utf8::find_invalid(str.begin(), str.end()); 
       return utf8::is_valid( str.begin(), str.end() );
    }
 
+   string prune_invalid_utf8( const string& str ) {
+      string result;
+
+      auto itr = utf8::find_invalid(str.begin(), str.end()); 
+      if( itr == str.end() ) return str;
+
+      result = string( str.begin(), itr );
+      while( itr != str.end() ) {
+         ++itr;
+         auto start = itr;
+         itr = utf8::find_invalid( start, str.end()); 
+         result += string( start, itr );
+      }
+      return result;
+   }
+
    void decodeUtf8(const std::string& input, std::wstring* storage)
    {
      assert(storage != nullptr);

vendor/websocketpp

@@ -1 +1 @@
-Subproject commit c5510d6de04917812b910a8dd44735c1f17061d9
+Subproject commit 378437aecdcb1dfe62096ffd5d944bf1f640ccc3