From fc13ce44781b86511ef5b90ac1ad8e836df7a401 Mon Sep 17 00:00:00 2001
From: theoreticalbts <theoreticalbts@users.noreply.github.com>
Date: Wed, 21 Oct 2015 13:27:40 -0400
Subject: [PATCH] genesis_util:  Implement create_bloom_filter

---
 programs/genesis_util/change_key_prefix.py   | 63 +++++++++++++++++
 programs/genesis_util/create_bloom_filter.py | 72 ++++++++++++++++++++
 2 files changed, 135 insertions(+)
 create mode 100755 programs/genesis_util/change_key_prefix.py
 create mode 100755 programs/genesis_util/create_bloom_filter.py

diff --git a/programs/genesis_util/change_key_prefix.py b/programs/genesis_util/change_key_prefix.py
new file mode 100755
index 00000000..7ca4b876
--- /dev/null
+++ b/programs/genesis_util/change_key_prefix.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import sys
+
+def dump_json(obj, out, pretty):
+    if pretty:
+        json.dump(obj, out, indent=2, sort_keys=True)
+    else:
+        json.dump(obj, out, separators=(",", ":"), sort_keys=True)
+    return
+
+def main():
+    parser = argparse.ArgumentParser(description="Set is_prefixed=false for all asset owners")
+    parser.add_argument("-o", "--output", metavar="OUT", default="-", help="output filename (default: stdout)")
+    parser.add_argument("-i", "--input", metavar="IN", default="-", help="input filename (default: stdin)")
+    parser.add_argument("-f", "--from", metavar="PREFIX", default="", help="initial prefix")
+    parser.add_argument("-t", "--to", metavar="PREFIX", default="", help="new prefix")
+    parser.add_argument("-p", "--pretty", action="store_true", default=False, help="pretty print output")
+    opts = parser.parse_args()
+
+    if opts.input == "-":
+        genesis = json.load(sys.stdin)
+    else:
+        with open(opts.input, "r") as f:
+            genesis = json.load(f)
+
+    frum = opts.__dict__["from"]    # from is a language keyword and cannot be an attribute name
+
+    def convert(k):
+        if not k.startswith(frum):
+            print("key {k} does not start with prefix {p}".format(k=repr(k), p=repr(frum)))
+            raise RuntimeError()
+        return opts.to + k[len(frum):]
+
+    for account in genesis["initial_accounts"]:
+        account["owner_key"] = convert(account["owner_key"])
+        account["active_key"] = convert(account["active_key"])
+
+    for asset in genesis["initial_assets"]:
+        for cr in asset.get("collateral_records", []):
+            cr["owner"] = convert(cr["owner"])
+
+    for balance in genesis["initial_balances"]:
+        balance["owner"] = convert(balance["owner"])
+
+    for vb in genesis["initial_vesting_balances"]:
+        vb["owner"] = convert(vb["owner"])
+
+    for witness in genesis["initial_witness_candidates"]:
+        witness["block_signing_key"] = convert(witness["block_signing_key"])
+
+    if opts.output == "-":
+        dump_json( genesis, sys.stdout, opts.pretty )
+        sys.stdout.flush()
+    else:
+        with open(opts.output, "w") as f:
+            dump_json( genesis, f, opts.pretty )
+    return
+
+if __name__ == "__main__":
+    main()
diff --git a/programs/genesis_util/create_bloom_filter.py b/programs/genesis_util/create_bloom_filter.py
new file mode 100755
index 00000000..17576d35
--- /dev/null
+++ b/programs/genesis_util/create_bloom_filter.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+import argparse
+import hashlib
+import json
+import sys
+
+def main():
+    parser = argparse.ArgumentParser(description="Set is_prefixed=false for all asset owners")
+    parser.add_argument("-o", "--output", metavar="OUT", default="-", help="output filename (default: stdout)")
+    parser.add_argument("-i", "--input", metavar="IN", default="-", help="input filename (default: stdin)")
+    parser.add_argument("-n", "--num", metavar="N", default=3, type=int, help="number of hashes per key (default: 3)")
+    parser.add_argument("-s", "--size", metavar="BITS", default=8*1048576, type=int, help="number of bits in filter")
+    parser.add_argument("-a", "--algorithm", metavar="NAME", default="sha256", type=str, help="hash algorithm (must exist in hashlib)")
+    opts = parser.parse_args()
+
+    if opts.input == "-":
+        genesis = json.load(sys.stdin)
+    else:
+        with open(opts.input, "r") as f:
+            genesis = json.load(f)
+
+    keys = set()
+
+    for account in genesis["initial_accounts"]:
+        keys.add(account["owner_key"])
+        keys.add(account["active_key"])
+
+    for asset in genesis["initial_assets"]:
+        for cr in asset.get("collateral_records", []):
+            keys.add(cr["owner"])
+
+    for balance in genesis["initial_balances"]:
+        keys.add(balance["owner"])
+
+    for vb in genesis["initial_vesting_balances"]:
+        keys.add(vb["owner"])
+
+    for witness in genesis["initial_witness_candidates"]:
+        keys.add(witness["block_signing_key"])
+
+    sys.stderr.write("got {n} distinct keys\n".format(n=len(keys)))
+
+    keys = [(str(i) + ":" + k).encode("UTF-8") for k in sorted(keys) for i in range(opts.num)]
+
+    data = bytearray((opts.size + 7) >> 3)
+
+    h = getattr(hashlib, opts.algorithm)
+    for k in keys:
+        address = int(h(k).hexdigest(), 16) % opts.size
+        data[address >> 3] |= (1 << (address & 7))
+
+    popcount = [bin(i).count("1") for i in range(256)]
+    w = sum(popcount[x] for x in data)
+    sys.stderr.write("""w={w}   o={o:.3%}   p={p:.3%}
+w: Hamming weight    o: Occupancy    p: False positive probability
+""".format(
+w=w,
+o=float(w) / float(opts.size),
+p=(float(w) / float(opts.size))**opts.num,
+))
+
+    if opts.output == "-":
+        sys.stdout.write(data)
+        sys.stdout.flush()
+    else:
+        with open(opts.output, "wb") as f:
+            f.write(data)
+    return
+
+if __name__ == "__main__":
+    main()