Merge pull request #7 from steemit/equihash_security

Implement test_canonical_order and test_intermediate_zeros for equihash

include/fc/crypto/equihash.hpp

@@ -11,7 +11,8 @@ namespace fc { namespace equihash {
       sha256   seed;
       std::vector< uint32_t > inputs;
 
-      bool is_valid() const;
+      bool is_valid( bool test_canonical_order = false, bool test_intermediate_zeros = false ) const;
+      void canonize_indexes();
 
       static proof hash( uint32_t n, uint32_t k, sha256 seed );
    };

src/crypto/equihash.cpp

@@ -23,11 +23,21 @@ namespace fc { namespace equihash {
       return new_seed;
    }
 
-   bool proof::is_valid() const
+   bool proof::is_valid( bool test_canonical_order, bool test_intermediate_zeros ) const
    {
       _POW::Proof test( n, k, sha_to_seed( seed ), EQUIHASH_NONCE, inputs );
+      if( test_canonical_order && !test.CheckIndexesCanon() )
+         return false;
+      if( test_intermediate_zeros )
+         return test.FullTest();
       return test.Test();
+   }
 
+   void proof::canonize_indexes()
+   {
+      _POW::Proof p( n, k, sha_to_seed( seed ), EQUIHASH_NONCE, inputs );
+      _POW::Proof p_canon = p.CanonizeIndexes();
+      inputs = p_canon.inputs;
    }
 
    proof proof::hash( uint32_t n, uint32_t k, sha256 seed )

vendor/equihash/include/equihash/pow.hpp

@@ -64,6 +64,9 @@ namespace _POW{
         Proof():n(0),k(1),seed(0),nonce(0),inputs(std::vector<Input>()) {};
 
         bool Test();
+        bool FullTest()const;
+        bool CheckIndexesCanon()const;
+        Proof CanonizeIndexes()const;
 	};
 
     class Tuple {

vendor/equihash/src/pow.cpp

@@ -8,6 +8,15 @@ Modifications by Steemit, Inc. 2016
 #include <equihash/blake2.h>
 #include <algorithm>
 
+#ifdef EQUIHASH_POW_VERBOSE
+#include <iomanip>
+#include <iostream>
+
+#define EQUIHASH_LOG(s) \
+   std::cerr << s << std::endl;
+#else
+#define EQUIHASH_LOG(s)
+#endif
 
 static uint64_t rdtsc(void) {
 #ifdef _MSC_VER
@@ -158,7 +167,9 @@ Proof Equihash::FindProof(){
                     dup = true;
             }
             if (!dup)
-                return solutions[i];
+            {
+                return solutions[i].CanonizeIndexes();
+            }
         }
     }
     return Proof(n, k, seed, nonce, std::vector<uint32_t>());
@@ -188,12 +199,85 @@ Proof Equihash::FindProof( Nonce _nonce )
                 dup = true;
        }
        if (!dup)
-          return solutions[i];
+       {
+          return solutions[i].CanonizeIndexes();
+       }
     }
 
     return Proof(n, k, seed, nonce, std::vector<uint32_t>());
 }
 
+Proof Proof::CanonizeIndexes()const
+{
+   // We consider the index values in the inputs array to be the leaf nodes of a binary
+   // tree, and the inner nodes to be labelled with the XOR of the corresponding vector
+   // elements.
+   //
+   // Define a binary tree to be canonically sorted if, for each inner node, the least
+   // leaf descendant of the left child is less than the least leaf descendant of the
+   // right child.
+   //
+   // This method puts the inputs into canonical order without altering the inner node
+   // labels.  Thus canonization preserves the validity of the proof and the
+   // footprint of Wagner's algorithm.
+   //
+   // We use a bottom-up traversal, dividing the input into successively larger power-of-2
+   // blocks and swapping the two half-blocks if non-canonical.
+   //
+   // Say a block is least-first if the least element is the first element.
+   //
+   // If each half-block is least-first, the conditional swap ensures the full block will also
+   // be least-first. The half-blocks in the initial iteration are obviously least-first
+   // (they only have a single element!).  So by induction, at each later iteration the half-blocks
+   // of that iteration are least-first (since they were the full blocks of the previous iteration,
+   // which were made least-first by the previous iteration's conditional swap).
+   //
+   // As a consequence, no search is necessary to find the least element in each half-block,
+   // it is always the first element in the half-block.
+
+   std::vector< uint32_t > new_inputs = inputs;
+
+   size_t input_size = inputs.size();
+   size_t half_size = 1;
+   size_t block_size = 2;
+   while( block_size <= input_size )
+   {
+      for( size_t i=0; i+block_size<=input_size; i+=block_size )
+      {
+         auto ita = new_inputs.begin()+i, itb = ita+half_size;
+         if( (*ita) >= (*itb) )
+         {
+            std::swap_ranges( ita, itb, itb );
+         }
+      }
+      half_size = block_size;
+      block_size += block_size;
+   }
+   return Proof(n, k, seed, nonce, new_inputs);
+}
+
+bool Proof::CheckIndexesCanon()const
+{
+   // This method is logically identical to CanonizeIndexes() but will return false
+   // instead of swapping elements.
+
+   size_t input_size = inputs.size();
+   size_t half_size = 1;
+   size_t block_size = 2;
+   while( block_size <= input_size )
+   {
+      for( size_t i=0; i+block_size<=input_size; i+=block_size )
+      {
+         auto ita = inputs.begin()+i, itb = ita+half_size;
+         if( (*ita) >= (*itb) )
+            return false;
+      }
+      half_size = block_size;
+      block_size += block_size;
+   }
+   return true;
+}
+
 bool Proof::Test()
 {
     uint32_t input[SEED_LENGTH + 2];
@@ -218,3 +302,114 @@ bool Proof::Test()
 
     return b;
 }
+
+bool Proof::FullTest()const
+{
+   // Length must be 2**k
+   if( inputs.size() != size_t(1 << k) )
+   {
+      EQUIHASH_LOG( "PoW failed length test" );
+      return false;
+   }
+
+   // Ensure all values are distinct
+   std::vector<Input> sorted_inputs = inputs;
+   std::sort( sorted_inputs.begin(), sorted_inputs.end() );
+   for( size_t i=1; i<inputs.size(); i++ )
+   {
+      if( sorted_inputs[i-1] >= sorted_inputs[i] )
+      {
+         EQUIHASH_LOG( "PoW failed distinct test" );
+         return false;
+      }
+   }
+
+   // Ensure all values are canonically indexed
+   /*
+   if( !CheckIndexesCanon() )
+      return false;
+   */
+
+   // Initialize blocks array
+   uint32_t input[SEED_LENGTH + 2];
+   for( size_t i=0; i<SEED_LENGTH; i++ )
+      input[i] = seed[i];
+   input[SEED_LENGTH] = nonce;
+   input[SEED_LENGTH + 1] = 0;
+   uint32_t buf[MAX_N / 4];
+
+   std::vector< std::vector< uint32_t > > blocks;
+
+   const uint32_t max_input = uint32_t(1) << (n / (k + 1) + 1);
+
+   for( size_t i=0; i<inputs.size(); i++ )
+   {
+      input[SEED_LENGTH + 1] = inputs[i];
+      if( inputs[i] >= max_input )
+      {
+         EQUIHASH_LOG( "PoW failed max_input test" );
+         return false;
+      }
+
+      blake2b((uint8_t*)buf, &input, NULL, sizeof(buf), sizeof(input), 0);
+      blocks.emplace_back();
+      std::vector<uint32_t>& x = blocks.back();
+      x.resize(k+1);
+      for( size_t j=0; j<(k+1); j++ )
+      {
+         //select j-th block of n/(k+1) bits
+         x[j] = buf[j] >> (32 - n / (k + 1));
+      }
+   }
+
+   while( true )
+   {
+#ifdef EQUIHASH_POW_VERBOSE
+      std::cerr << "\n\nBegin loop iteration\n";
+      for( const std::vector< uint32_t >& x : blocks )
+      {
+         for( const uint32_t& e : x )
+            std::cerr << std::hex << std::setw(5) << e << " ";
+         std::cerr << std::endl;
+      }
+#endif
+
+      size_t count = blocks.size();
+      if( count == 0 )
+      {
+         EQUIHASH_LOG( "PoW failed with count == 0" );
+         return false;
+      }
+      if( count == 1 )
+      {
+         if( blocks[0].size() != 1 )
+         {
+            EQUIHASH_LOG( "PoW failed due to vector size" );
+            return false;
+         }
+         if( blocks[0][0] != 0 )
+         {
+            EQUIHASH_LOG( "PoW failed because final bits are not zero" );
+            return false;
+         }
+         return true;
+      }
+      if( (count&1) != 0 )
+      {
+         EQUIHASH_LOG( "PoW failed with odd count" );
+         return false;
+      }
+      for( size_t i=0,new_i=0; i<count; i+=2,new_i++ )
+      {
+         if( blocks[i][0] != blocks[i+1][0] )
+         {
+            EQUIHASH_LOG( "PoW failed because leading element of vector pair does not match" );
+            return false;
+         }
+         for( size_t j=1; j<blocks[i].size(); j++ )
+            blocks[new_i][j-1] = blocks[i][j] ^ blocks[i+1][j];
+         blocks[new_i].resize(blocks[new_i].size()-1);
+      }
+      blocks.resize(blocks.size() >> 1);
+   }
+}