Resolve all crashes issues #512

New issue

Open

opened 2023-02-13 08:29:42 +00:00 by vampik · 4 comments

vampik commented

2023-02-13 08:29:42 +00:00

(Migrated from gitlab.com)

https://gitlab.com/PBSA/peerplays/-/issues/344

https://gitlab.com/PBSA/peerplays/-/issues/354

https://gitlab.com/PBSA/peerplays/-/issues/415

https://gitlab.com/PBSA/peerplays/-/issues/419

https://gitlab.com/PBSA/peerplays/-/issues/454

https://gitlab.com/PBSA/peerplays/-/issues/490

https://gitlab.com/PBSA/peerplays/-/issues/491

https://gitlab.com/PBSA/peerplays/-/issues/501

Currently, we have several issues that report crashes of witness_node. Some of them are old (and we can already fix them), some created more recently. We need to resolve it and close all related issues: https://gitlab.com/PBSA/peerplays/-/issues/344 https://gitlab.com/PBSA/peerplays/-/issues/354 https://gitlab.com/PBSA/peerplays/-/issues/415 https://gitlab.com/PBSA/peerplays/-/issues/419 https://gitlab.com/PBSA/peerplays/-/issues/454 https://gitlab.com/PBSA/peerplays/-/issues/490 https://gitlab.com/PBSA/peerplays/-/issues/491 https://gitlab.com/PBSA/peerplays/-/issues/501

vampik commented

2023-02-13 08:29:42 +00:00

(Migrated from gitlab.com)

assigned to @vampik

vampik commented

2023-02-13 08:29:55 +00:00

(Migrated from gitlab.com)

changed the description

prandnum commented

2023-02-17 18:50:48 +00:00

(Migrated from gitlab.com)

Attempted 11 bitcoin deposits to the same address and after 1 min, mined blocks to a different address. It was seen that the witness nodes started crashing after 2 mins

commands

root@f5d4bae77756:~/bitcoin-network# 
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 10 "" "" false
a9023785237bb1811873c684f17b2aae146012383c250bbe66e44de50f3936b2
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 20 "" "" false
190d96d105ea6e3c40cc44c6aaeefad427bff918ca9f10feb3f23b1973a3e438
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 30 "" "" false
b92416aa15ef942a4088c301a0973c0074d9d525eeeeab1cf6e418890ffc6365
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 40 "" "" false
809d4851c861afffabecda467c9fb36ed3c0db186b657d06f35bd4d997892118
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 50 "" "" false
d9dacc3b79b20ac980b5d21b524499f46f1af3ea74b4591b45b1c840fab4709a
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 60 "" "" false
54109af6e6f7c9802c220be6f2c9c3f7b9ccfdf61ab00ce6dec7345e8833da92
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 70 "" "" false
c8fae260a8411d9659326efccf2f7687bea82068b9b24b65c27881b883ae7a67
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 80 "" "" false
5108620e5c89b255494ab61fdfad9054e3c53c7c0a084ea82eeb25bc0836d1db
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 90 "" "" false
3ab2650a625c4d59dfbcd4c325ad3fd9f9f024bdcc948882aa95f3ec5fd51798
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 16 "" "" false
18a4730a821e3b9cfbef29d5cf9e70e8e4eac1463763af5dce7549dd62e98837
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 34 "" "" false
54de49099dc44400f492c26d4720a83b94dec35b0aba219f56ba442bc3990430
root@f5d4bae77756:~/bitcoin-network# 
root@f5d4bae77756:~/bitcoin-network#

root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz
[
  "56f0426b2aa0d854be20d8820750e1ea72e5c4bfece27c11fbf49b78097a5da8"
]
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz
[
  "3619c3da85bdb0f774bc9f90a6935a0eed6eeaa106f5a30a0ec48cc4b26a5d73"
]
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz
[
  "4d9b14e2588306ca6fb32d758f80cdff797e79b4793a95923301c9a552c0d9e4"
]
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz
[
  "2d5b7930a2c2b3ab2b523251daa18b1b826b5c612933236102b354c350d1c95e"
]
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz
[
  "1e88f5dadb19a96c9d7e75a85f687c52d8fbbfe92d4eb987cf5794a119b9dd47"
]
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz
[
  "41a8a5fcbd01ab9f4cbde2e3fe0e070f024a35d8716e0d96978854573ebbd64f"
]
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz
[
  "3ef85813b8df9599078f2f2ec23f4a19b83eeac218ec6f2eb4ad9dc30e52ed20"
]
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz
[
  "7f1ede7f1baafa29051b25bb970188d8a26e4e495af9a9ca1b21ea57d7982826"
]
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz
[
  "7b7fbde29e116122effd02e8e4a26a658276d7f0d6407d3a07808e89c935d937"
]
root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz
[
  "2c680428a231ffca4b9266a144d22f4011f3d4a46407c8d28a6d2a3475eed085"
]
root@f5d4bae77756:

kiran@warpspeed:/home$ sudo coredumpctl debug 463596
           PID: 463596 (witness_node)
           UID: 0 (root)
           GID: 0 (root)
        Signal: 11 (SEGV)
     Timestamp: Sat 2023-02-18 00:07:05 +0530 (5min ago)
  Command Line: ./witness_node -d ./witness_node_data_dir --replay-blockchain
    Executable: /home/peerplays/src/peerplays/build/programs/witness_node/witness_node
 Control Group: /docker/1f62fc6098cc06590fedb44524d97e332b89e37e2d0720c6c20b0e7ea0f74a4c
         Slice: -.slice
       Boot ID: 0244c133ef924f71b5283ff8ebd6fad0
    Machine ID: a388625228614d18b5373ad1be39a93f
      Hostname: warpspeed
       Storage: /var/lib/systemd/coredump/core.witness_node.0.0244c133ef924f71b5283ff8ebd6fad0.463596.1676659025000000000000.lz4
       Message: Process 463596 (witness_node) of user 0 dumped core.
                
                Stack trace of thread 15:
                #0  0x00007f23ac889941 __GI_abort (libc.so.6 + 0x22941)
                #1  0x00007f23ac889729 __assert_fail_base (libc.so.6 + 0x22729)
                #2  0x00007f23ac89afd6 __GI___assert_fail (libc.so.6 + 0x33fd6)
                #3  0x000055a7def27e30 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x58c5e30)
                #4  0x000055a7def453bb n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x58e33bb)
                #5  0x000055a7df092476 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x5a30476)
                #6  0x000055a7df098f6b n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x5a36f6b)
                #7  0x000055a7df098881 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x5a36881)
                #8  0x000055a7df09e25e n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x5a3c25e)
                #9  0x000055a7e03f7ae2 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6d95ae2)
                #10 0x000055a7e03f7a5e n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6d95a5e)
                #11 0x000055a7e03e960c n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6d8760c)
                #12 0x000055a7e03e99bb n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6d879bb)
                #13 0x000055a7e03e8f8b n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6d86f8b)
                #14 0x000055a7e065e00f n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6ffc00f)

GNU gdb (Ubuntu 9.2-0ubuntu1~20.04.1) 9.2
Copyright (C) 2020 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
    <http://www.gnu.org/software/gdb/documentation/>.

For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /home/peerplays/src/peerplays/build/programs/witness_node/witness_node...
[New LWP 15]
[New LWP 7]
[New LWP 1]
[New LWP 13]
[New LWP 8]
[New LWP 11]
[New LWP 16]
[New LWP 9]
[New LWP 19]
[New LWP 23]
[New LWP 12]
[New LWP 18]
[New LWP 20]
[New LWP 21]
[New LWP 17]
[New LWP 14]
[New LWP 10]
[New LWP 24]
[New LWP 22]

warning: .dynamic section for "/usr/local/lib/libbitcoin-protocol.so.0" is not at the expected address (wrong library or version mismatch?)

warning: .dynamic section for "/usr/local/lib/libbitcoin-client.so.0" is not at the expected address (wrong library or version mismatch?)

warning: .dynamic section for "/usr/local/lib/libbitcoin-explorer.so.0" is not at the expected address (wrong library or version mismatch?)

warning: .dynamic section for "/usr/local//lib/libsecp256k1.so.0" is not at the expected address (wrong library or version mismatch?)

warning: .dynamic section for "/usr/local//lib/libbitcoin-network.so.0" is not at the expected address (wrong library or version mismatch?)
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `./witness_node -d ./witness_node_data_dir --replay-blockchain'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  __GI_abort () at abort.c:107
107	abort.c: No such file or directory.
[Current thread is 1 (Thread 0x7f2388ffe700 (LWP 15))]
(gdb) 
(gdb) 
(gdb) 
(gdb) bt
#0  __GI_abort () at abort.c:107
#1  0x00007f23ac889729 in __assert_fail_base (fmt=0x7f23aca1f588 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=0x55a7e0b267f8 "_closing_connections.find(peer_to_delete) == _closing_connections.end()", 
    file=0x55a7e0b24d88 "/home/peerplays/src/peerplays/libraries/net/node.cpp", line=1654, function=<optimized out>) at assert.c:92
#2  0x00007f23ac89afd6 in __GI___assert_fail (assertion=0x55a7e0b267f8 "_closing_connections.find(peer_to_delete) == _closing_connections.end()", file=0x55a7e0b24d88 "/home/peerplays/src/peerplays/libraries/net/node.cpp", 
    line=1654, function=0x55a7e0b26700 "void graphene::net::detail::node_impl::schedule_peer_for_deletion(const peer_connection_ptr&)") at assert.c:101
#3  0x000055a7def27e30 in graphene::net::detail::node_impl::schedule_peer_for_deletion (this=0x55a7e20d8a10, peer_to_delete=std::shared_ptr<class graphene::net::peer_connection> (use count 4, weak count 2) = {...})
    at /home/peerplays/src/peerplays/libraries/net/node.cpp:1654
#4  0x000055a7def453bb in graphene::net::detail::node_impl::on_connection_closed (this=0x55a7e20d8a10, originating_peer=0x7f23800d0330) at /home/peerplays/src/peerplays/libraries/net/node.cpp:3106
#5  0x000055a7df092476 in graphene::net::peer_connection::on_connection_closed (this=0x7f23800d0330, originating_connection=0x7f23800d0360) at /home/peerplays/src/peerplays/libraries/net/peer_connection.cpp:283
#6  0x000055a7df098f6b in graphene::net::detail::message_oriented_connection_impl::read_loop (this=0x7f2380098190) at /home/peerplays/src/peerplays/libraries/net/message_oriented_connection.cpp:241
#7  0x000055a7df098881 in graphene::net::detail::message_oriented_connection_impl::<lambda()>::operator()(void) const (__closure=0x7f23800ab798) at /home/peerplays/src/peerplays/libraries/net/message_oriented_connection.cpp:134
#8  0x000055a7df09e25e in fc::detail::void_functor_run<graphene::net::detail::message_oriented_connection_impl::connect_to(const fc::ip::endpoint&)::<lambda()> >::run(void *, void *) (functor=0x7f23800ab798, prom=0x7f23800ab790)
    at /home/peerplays/src/peerplays/libraries/fc/include/fc/thread/task.hpp:83
#9  0x000055a7e03f7ae2 in fc::task_base::run_impl (this=0x7f23800ab7a0) at /home/peerplays/src/peerplays/libraries/fc/src/thread/task.cpp:43
#10 0x000055a7e03f7a5e in fc::task_base::run (this=0x7f23800ab7a0) at /home/peerplays/src/peerplays/libraries/fc/src/thread/task.cpp:32
#11 0x000055a7e03e960c in fc::thread_d::run_next_task (this=0x7f2380000b60) at /home/peerplays/src/peerplays/libraries/fc/src/thread/thread_d.hpp:520
#12 0x000055a7e03e99bb in fc::thread_d::process_tasks (this=0x7f2380000b60) at /home/peerplays/src/peerplays/libraries/fc/src/thread/thread_d.hpp:568
#13 0x000055a7e03e8f8b in fc::thread_d::start_process_tasks (my=...) at /home/peerplays/src/peerplays/libraries/fc/src/thread/thread_d.hpp:499
#14 0x000055a7e065e00f in make_fcontext ()
#15 0x0000000000000000 in ?? ()
(gdb)

@bobinson @serkixenos @vampik

CC: @tai.sama

Attempted 11 bitcoin deposits to the same address and after 1 min, mined blocks to a different address. It was seen that the witness nodes started crashing after 2 mins commands ``` root@f5d4bae77756:~/bitcoin-network# root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 10 "" "" false a9023785237bb1811873c684f17b2aae146012383c250bbe66e44de50f3936b2 root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 20 "" "" false 190d96d105ea6e3c40cc44c6aaeefad427bff918ca9f10feb3f23b1973a3e438 root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 30 "" "" false b92416aa15ef942a4088c301a0973c0074d9d525eeeeab1cf6e418890ffc6365 root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 40 "" "" false 809d4851c861afffabecda467c9fb36ed3c0db186b657d06f35bd4d997892118 root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 50 "" "" false d9dacc3b79b20ac980b5d21b524499f46f1af3ea74b4591b45b1c840fab4709a root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 60 "" "" false 54109af6e6f7c9802c220be6f2c9c3f7b9ccfdf61ab00ce6dec7345e8833da92 root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 70 "" "" false c8fae260a8411d9659326efccf2f7687bea82068b9b24b65c27881b883ae7a67 root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 80 "" "" false 5108620e5c89b255494ab61fdfad9054e3c53c7c0a084ea82eeb25bc0836d1db root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 90 "" "" false 3ab2650a625c4d59dfbcd4c325ad3fd9f9f024bdcc948882aa95f3ec5fd51798 root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 16 "" "" false 18a4730a821e3b9cfbef29d5cf9e70e8e4eac1463763af5dce7549dd62e98837 root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default sendtoaddress 2N7xWqbLSj5kgnz9D542J3RgMn9UTAW8s8H 34 "" "" false 54de49099dc44400f492c26d4720a83b94dec35b0aba219f56ba442bc3990430 root@f5d4bae77756:~/bitcoin-network# root@f5d4bae77756:~/bitcoin-network# ``` ``` root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz [ "56f0426b2aa0d854be20d8820750e1ea72e5c4bfece27c11fbf49b78097a5da8" ] root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz [ "3619c3da85bdb0f774bc9f90a6935a0eed6eeaa106f5a30a0ec48cc4b26a5d73" ] root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz [ "4d9b14e2588306ca6fb32d758f80cdff797e79b4793a95923301c9a552c0d9e4" ] root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz [ "2d5b7930a2c2b3ab2b523251daa18b1b826b5c612933236102b354c350d1c95e" ] root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz [ "1e88f5dadb19a96c9d7e75a85f687c52d8fbbfe92d4eb987cf5794a119b9dd47" ] root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz [ "41a8a5fcbd01ab9f4cbde2e3fe0e070f024a35d8716e0d96978854573ebbd64f" ] root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz [ "3ef85813b8df9599078f2f2ec23f4a19b83eeac218ec6f2eb4ad9dc30e52ed20" ] root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz [ "7f1ede7f1baafa29051b25bb970188d8a26e4e495af9a9ca1b21ea57d7982826" ] root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz [ "7b7fbde29e116122effd02e8e4a26a658276d7f0d6407d3a07808e89c935d937" ] root@f5d4bae77756:~/bitcoin-network# ./bitcoin-cli -rpcuser=1 -rpcpassword=1 -rpcwallet=default generatetoaddress 1 bcrt1qsa7h0v6taa5t6gjh4fgdm0annv4knarf5up2yz [ "2c680428a231ffca4b9266a144d22f4011f3d4a46407c8d28a6d2a3475eed085" ] root@f5d4bae77756: ``` ``` kiran@warpspeed:/home$ sudo coredumpctl debug 463596 PID: 463596 (witness_node) UID: 0 (root) GID: 0 (root) Signal: 11 (SEGV) Timestamp: Sat 2023-02-18 00:07:05 +0530 (5min ago) Command Line: ./witness_node -d ./witness_node_data_dir --replay-blockchain Executable: /home/peerplays/src/peerplays/build/programs/witness_node/witness_node Control Group: /docker/1f62fc6098cc06590fedb44524d97e332b89e37e2d0720c6c20b0e7ea0f74a4c Slice: -.slice Boot ID: 0244c133ef924f71b5283ff8ebd6fad0 Machine ID: a388625228614d18b5373ad1be39a93f Hostname: warpspeed Storage: /var/lib/systemd/coredump/core.witness_node.0.0244c133ef924f71b5283ff8ebd6fad0.463596.1676659025000000000000.lz4 Message: Process 463596 (witness_node) of user 0 dumped core. Stack trace of thread 15: #0 0x00007f23ac889941 __GI_abort (libc.so.6 + 0x22941) #1 0x00007f23ac889729 __assert_fail_base (libc.so.6 + 0x22729) #2 0x00007f23ac89afd6 __GI___assert_fail (libc.so.6 + 0x33fd6) #3 0x000055a7def27e30 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x58c5e30) #4 0x000055a7def453bb n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x58e33bb) #5 0x000055a7df092476 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x5a30476) #6 0x000055a7df098f6b n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x5a36f6b) #7 0x000055a7df098881 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x5a36881) #8 0x000055a7df09e25e n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x5a3c25e) #9 0x000055a7e03f7ae2 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6d95ae2) #10 0x000055a7e03f7a5e n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6d95a5e) #11 0x000055a7e03e960c n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6d8760c) #12 0x000055a7e03e99bb n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6d879bb) #13 0x000055a7e03e8f8b n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6d86f8b) #14 0x000055a7e065e00f n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6ffc00f) GNU gdb (Ubuntu 9.2-0ubuntu1~20.04.1) 9.2 Copyright (C) 2020 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>. Find the GDB manual and other documentation resources online at: <http://www.gnu.org/software/gdb/documentation/>. For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /home/peerplays/src/peerplays/build/programs/witness_node/witness_node... [New LWP 15] [New LWP 7] [New LWP 1] [New LWP 13] [New LWP 8] [New LWP 11] [New LWP 16] [New LWP 9] [New LWP 19] [New LWP 23] [New LWP 12] [New LWP 18] [New LWP 20] [New LWP 21] [New LWP 17] [New LWP 14] [New LWP 10] [New LWP 24] [New LWP 22] warning: .dynamic section for "/usr/local/lib/libbitcoin-protocol.so.0" is not at the expected address (wrong library or version mismatch?) warning: .dynamic section for "/usr/local/lib/libbitcoin-client.so.0" is not at the expected address (wrong library or version mismatch?) warning: .dynamic section for "/usr/local/lib/libbitcoin-explorer.so.0" is not at the expected address (wrong library or version mismatch?) warning: .dynamic section for "/usr/local//lib/libsecp256k1.so.0" is not at the expected address (wrong library or version mismatch?) warning: .dynamic section for "/usr/local//lib/libbitcoin-network.so.0" is not at the expected address (wrong library or version mismatch?) [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `./witness_node -d ./witness_node_data_dir --replay-blockchain'. Program terminated with signal SIGSEGV, Segmentation fault. #0 __GI_abort () at abort.c:107 107 abort.c: No such file or directory. [Current thread is 1 (Thread 0x7f2388ffe700 (LWP 15))] (gdb) (gdb) (gdb) (gdb) bt #0 __GI_abort () at abort.c:107 #1 0x00007f23ac889729 in __assert_fail_base (fmt=0x7f23aca1f588 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=0x55a7e0b267f8 "_closing_connections.find(peer_to_delete) == _closing_connections.end()", file=0x55a7e0b24d88 "/home/peerplays/src/peerplays/libraries/net/node.cpp", line=1654, function=<optimized out>) at assert.c:92 #2 0x00007f23ac89afd6 in __GI___assert_fail (assertion=0x55a7e0b267f8 "_closing_connections.find(peer_to_delete) == _closing_connections.end()", file=0x55a7e0b24d88 "/home/peerplays/src/peerplays/libraries/net/node.cpp", line=1654, function=0x55a7e0b26700 "void graphene::net::detail::node_impl::schedule_peer_for_deletion(const peer_connection_ptr&)") at assert.c:101 #3 0x000055a7def27e30 in graphene::net::detail::node_impl::schedule_peer_for_deletion (this=0x55a7e20d8a10, peer_to_delete=std::shared_ptr<class graphene::net::peer_connection> (use count 4, weak count 2) = {...}) at /home/peerplays/src/peerplays/libraries/net/node.cpp:1654 #4 0x000055a7def453bb in graphene::net::detail::node_impl::on_connection_closed (this=0x55a7e20d8a10, originating_peer=0x7f23800d0330) at /home/peerplays/src/peerplays/libraries/net/node.cpp:3106 #5 0x000055a7df092476 in graphene::net::peer_connection::on_connection_closed (this=0x7f23800d0330, originating_connection=0x7f23800d0360) at /home/peerplays/src/peerplays/libraries/net/peer_connection.cpp:283 #6 0x000055a7df098f6b in graphene::net::detail::message_oriented_connection_impl::read_loop (this=0x7f2380098190) at /home/peerplays/src/peerplays/libraries/net/message_oriented_connection.cpp:241 #7 0x000055a7df098881 in graphene::net::detail::message_oriented_connection_impl::<lambda()>::operator()(void) const (__closure=0x7f23800ab798) at /home/peerplays/src/peerplays/libraries/net/message_oriented_connection.cpp:134 #8 0x000055a7df09e25e in fc::detail::void_functor_run<graphene::net::detail::message_oriented_connection_impl::connect_to(const fc::ip::endpoint&)::<lambda()> >::run(void *, void *) (functor=0x7f23800ab798, prom=0x7f23800ab790) at /home/peerplays/src/peerplays/libraries/fc/include/fc/thread/task.hpp:83 #9 0x000055a7e03f7ae2 in fc::task_base::run_impl (this=0x7f23800ab7a0) at /home/peerplays/src/peerplays/libraries/fc/src/thread/task.cpp:43 #10 0x000055a7e03f7a5e in fc::task_base::run (this=0x7f23800ab7a0) at /home/peerplays/src/peerplays/libraries/fc/src/thread/task.cpp:32 #11 0x000055a7e03e960c in fc::thread_d::run_next_task (this=0x7f2380000b60) at /home/peerplays/src/peerplays/libraries/fc/src/thread/thread_d.hpp:520 #12 0x000055a7e03e99bb in fc::thread_d::process_tasks (this=0x7f2380000b60) at /home/peerplays/src/peerplays/libraries/fc/src/thread/thread_d.hpp:568 #13 0x000055a7e03e8f8b in fc::thread_d::start_process_tasks (my=...) at /home/peerplays/src/peerplays/libraries/fc/src/thread/thread_d.hpp:499 #14 0x000055a7e065e00f in make_fcontext () #15 0x0000000000000000 in ?? () (gdb) ``` @bobinson @serkixenos @vampik CC: @tai.sama

prandnum commented

2023-02-22 19:25:41 +00:00

(Migrated from gitlab.com)

bitcoin deposit crash is reproducible, seems like similar trace as above:

kiran@warpspeed:/home$ sudo coredumpctl debug 1297952
           PID: 1297952 (witness_node)
           UID: 0 (root)
           GID: 0 (root)
        Signal: 11 (SEGV)
     Timestamp: Thu 2023-02-23 00:48:25 +0530 (3min 55s ago)
  Command Line: ./witness_node -d ./witness_node_data_dir --replay-blockchain
    Executable: /home/peerplays/src/peerplays/build/programs/witness_node/witness_node
 Control Group: /docker/c1df8708ba774395860d359e695a45df55f08ee6b48998d7c96eef28844f338c
         Slice: -.slice
       Boot ID: 2467b5983e234846a5eb4d9cd930077a
    Machine ID: a388625228614d18b5373ad1be39a93f
      Hostname: warpspeed
       Storage: /var/lib/systemd/coredump/core.witness_node.0.2467b5983e234846a5eb4d9cd930077a.1297952.1677093505000000000000.lz4
       Message: Process 1297952 (witness_node) of user 0 dumped core.
                
                Stack trace of thread 81784:
                #0  0x00007fe5d382a941 __GI_abort (libc.so.6 + 0x22941)
                #1  0x00007fe5d3c02911 n/a (libstdc++.so.6 + 0x9e911)
                #2  0x00007fe5d3c0e38c n/a (libstdc++.so.6 + 0xaa38c)
                #3  0x00007fe5d3c0d369 n/a (libstdc++.so.6 + 0xa9369)
                #4  0x00007fe5d3c0dd21 __gxx_personality_v0 (libstdc++.so.6 + 0xa9d21)
                #5  0x00007fe5d3a0abef n/a (libgcc_s.so.1 + 0x10bef)
                #6  0x00007fe5d3a0b281 _Unwind_RaiseException (libgcc_s.so.1 + 0x11281)
                #7  0x00007fe5d3c0e69c __cxa_throw (libstdc++.so.6 + 0xaa69c)
                #8  0x0000561e0d62ddfe n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6d7cdfe)
                #9  0x0000561e0b2222ec n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x49712ec)
                #10 0x0000561e0b218f24 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4967f24)
                #11 0x0000561e0b7dd19a n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4f2c19a)
                #12 0x0000561e0b79d61f n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4eec61f)
                #13 0x0000561e0b798505 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ee7505)
                #14 0x0000561e0b79769c n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ee669c)
                #15 0x0000561e0b7a3755 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2755)
                #16 0x0000561e0b7a36f6 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef26f6)
                #17 0x0000561e0b7a3686 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2686)
                #18 0x0000561e0b7a3613 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2613)
                #19 0x0000561e0b7a3346 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2346)
                #20 0x0000561e0b7a3005 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2005)
                #21 0x0000561e0b7a5ece n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef4ece)
                #22 0x0000561e0b7a4f53 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef3f53)
                #23 0x0000561e0b7ae5d5 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4efd5d5)
                #24 0x0000561e0b7a9680 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef8680)
                #25 0x0000561e0b7a5c82 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef4c82)
                #26 0x0000561e0b7a5cb1 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef4cb1)
                #27 0x0000561e0b7a5cc6 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef4cc6)
                #28 0x00007fe5d3d5d4df __pthread_once_slow (libpthread.so.0 + 0x114df)
                #29 0x0000561e0b78d8a2 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4edc8a2)
                #30 0x0000561e0b7a5d70 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef4d70)
                #31 0x0000561e0b7a4caf n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef3caf)
                #32 0x0000561e0b7a2a7e n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef1a7e)
                #33 0x0000561e0b7a404d n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef304d)
                #34 0x0000561e0b7a3fee n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2fee)
                #35 0x0000561e0b7a3f30 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2f30)
                #36 0x0000561e0b7a3e71 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2e71)
                #37 0x0000561e0b7a39aa n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef29aa)
                #38 0x00007fe5d3c3ade4 n/a (libstdc++.so.6 + 0xd6de4)
                #39 0x00007fe5d3d54609 start_thread (libpthread.so.0 + 0x8609)
                #40 0x00007fe5d3927133 __clone (libc.so.6 + 0x11f133)
                
                Stack trace of thread 16:
                #0  0x00007fe5d3d5b376 futex_wait_cancelable (libpthread.so.0 + 0xf376)
                #1  0x0000561e0bfa92e7 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56f82e7)
                #2  0x0000561e0bf99a28 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56e8a28)
                #3  0x0000561e0bf9cc87 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56ebc87)
                #4  0x0000561e0bf9c650 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56eb650)
                #5  0x0000561e0bfa0103 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56ef103)
                #6  0x0000561e0bff1780 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x5740780)
                #7  0x0000561e0bf99beb n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56e8beb)
                #8  0x00007fe5d3d54609 start_thread (libpthread.so.0 + 0x8609)
                #9  0x00007fe5d3927133 __clone (libc.so.6 + 0x11f133)
                
                Stack trace of thread 7:
                #0  0x00007fe5d3d5b376 futex_wait_cancelable (libpthread.so.0 + 0xf376)
                #1  0x0000561e0bfa92e7 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56f82e7)
                #2  0x0000561e0bf99a28 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56e8a28)
                #3  0x0000561e0bf9cc87 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56ebc87)
                #4  0x0000561e0bf9c650 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56eb650)
                #5  0x0000561e0d816afc n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6f65afc)
                #6  0x0000561e0d816d60 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6f65d60)
                #7  0x0000561e0d823ed0 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6f72ed0)
                #8  0x0000561e0d8a94bb n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6ff84bb)

GNU gdb (Ubuntu 9.2-0ubuntu1~20.04.1) 9.2
Copyright (C) 2020 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
    <http://www.gnu.org/software/gdb/documentation/>.

For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /home/peerplays/src/peerplays/build/programs/witness_node/witness_node...
[New LWP 81784]
[New LWP 16]
[New LWP 7]
[New LWP 8]
[New LWP 18]
[New LWP 23]
[New LWP 1]
[New LWP 19]
[New LWP 81782]
[New LWP 21]
[New LWP 24]
[New LWP 20]
[New LWP 11]
[New LWP 14]
[New LWP 9]
[New LWP 13]
[New LWP 15]
[New LWP 22]
[New LWP 10]
[New LWP 12]
[New LWP 17]

warning: .dynamic section for "/usr/local/lib/libbitcoin-protocol.so.0" is not at the expected address (wrong library or version mismatch?)

warning: .dynamic section for "/usr/local/lib/libbitcoin-client.so.0" is not at the expected address (wrong library or version mismatch?)

warning: .dynamic section for "/usr/local/lib/libbitcoin-explorer.so.0" is not at the expected address (wrong library or version mismatch?)

warning: .dynamic section for "/usr/local//lib/libsecp256k1.so.0" is not at the expected address (wrong library or version mismatch?)

warning: .dynamic section for "/usr/local//lib/libbitcoin-network.so.0" is not at the expected address (wrong library or version mismatch?)
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `./witness_node -d ./witness_node_data_dir --replay-blockchain'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  __GI_abort () at abort.c:107
107	abort.c: No such file or directory.
[Current thread is 1 (Thread 0x7fe5873fc700 (LWP 81784))]
(gdb) bt
#0  __GI_abort () at abort.c:107
#1  0x00007fe5d3c02911 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6
#2  0x00007fe5d3c0e38c in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6
#3  0x00007fe5d3c0d369 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6
#4  0x00007fe5d3c0dd21 in __gxx_personality_v0 () from /lib/x86_64-linux-gnu/libstdc++.so.6
#5  0x00007fe5d3a0abef in ?? () from /lib/x86_64-linux-gnu/libgcc_s.so.1
#6  0x00007fe5d3a0b281 in _Unwind_RaiseException () from /lib/x86_64-linux-gnu/libgcc_s.so.1
#7  0x00007fe5d3c0e69c in __cxa_throw () from /lib/x86_64-linux-gnu/libstdc++.so.6
#8  0x0000561e0d62ddfe in fc::impl::dynamic_storage::data (this=0x7fe5873fb0d8) at /home/peerplays/src/peerplays/libraries/fc/src/static_variant.cpp:15
#9  0x0000561e0b2222ec in _ZN2fc14static_variantIJN8graphene5chain18transfer_operationENS2_28limit_order_create_operationENS2_28limit_order_cancel_operationENS2_27call_order_update_operationENS2_20fill_order_operationENS2_24account_create_operationENS2_24account_update_operationENS2_27account_whitelist_operationENS2_25account_upgrade_operationENS2_26account_transfer_operationENS2_22asset_create_operationENS2_22asset_update_operationENS2_31asset_update_bitasset_operationENS2_37asset_update_feed_producers_operationENS2_21asset_issue_operationENS2_23asset_reserve_operationENS2_29asset_fund_fee_pool_operationENS2_22asset_settle_operationENS2_29asset_global_settle_operationENS2_28asset_publish_feed_operationENS2_24witness_create_operationENS2_24witness_update_operationENS2_25proposal_create_operationENS2_25proposal_update_operationENS2_25proposal_delete_operationENS2_36withdraw_permission_create_operationENS2_36withdraw_permission_update_operationENS2_35withdraw_permission_claim_operationENS2_36withdraw_permission_delete_operationENS2_33committee_member_create_operationENS2_33committee_member_update_operationENS2_51committee_member_update_global_parameters_operationENS2_32vesting_balance_create_operationENS2_34vesting_balance_withdraw_operationENS2_23worker_create_operationENS2_16custom_operationENS2_16assert_operationENS2_23balance_claim_operationENS2_27override_transfer_operationENS2_27transfer_to_blind_operationENS2_24blind_transfer_operationENS2_29transfer_from_blind_operationENS2_29asset_settle_cancel_operationENS2_26asset_claim_fees_operationENS2_24fba_distribute_operationENS2_27tournament_create_operationENS2_25tournament_join_operationENS2_19game_move_operationENS2_31asset_update_dividend_operationENS2_37asset_dividend_distribution_operationENS2_27tournament_payout_operationENS2_26tournament_leave_operationENS2_22sport_create_operationENS2_22sport_update_operationENS2_28event_group_create_operationENS2_28event_group_update_operationENS2_22event_create_operationENS2_22event_update_operationENS2_37betting_market_rules_create_operationENS2_37betting_market_rules_update_operationENS2_37betting_market_group_create_operationENS2_31betting_market_create_operationENS2_19bet_place_operationENS2_38betting_market_group_resolve_operationENS2_39betting_market_group_resolved_operationENS2_22bet_adjusted_operationENS2_52betting_market_group_cancel_unmatched_bets_operationENS2_21bet_matched_operationENS2_20bet_cancel_operationENS2_22bet_canceled_operationENS2_37betting_market_group_update_operationENS2_31betting_market_update_operationENS2_29event_update_status_operationENS2_22sport_delete_operationENS2_28event_group_delete_operationENS2_26affiliate_payout_operationENS2_35affiliate_referral_payout_operationENS2_30lottery_asset_create_operationENS2_25ticket_purchase_operationENS2_24lottery_reward_operationENS2_21lottery_end_operationENS2_30sweeps_vesting_claim_operationENS2_34custom_permission_create_operationENS2_34custom_permission_update_operationENS2_34custom_permission_delete_operationENS2_41custom_account_authority_create_operationENS2_41custom_account_authority_update_operationENS2_41custom_account_authority_delete_operationENS2_15offer_operationENS2_13bid_operationENS2_22cancel_offer_operationENS2_24finalize_offer_operationENS2_29nft_metadata_create_operationENS2_29nft_metadata_update_operationENS2_18nft_mint_operationENS2_32nft_safe_transfer_from_operationENS2_21nft_approve_operationENS2_34nft_set_approval_for_all_operationENS2_29account_role_create_operationENS2_29account_role_update_operationENS2_29account_role_delete_operationENS2_20son_create_operationENS2_20son_update_operationENS2_24son_deregister_operationENS2_23son_heartbeat_operationENS2_25son_report_down_operationENS2_25son_maintenance_operationENS2_29son_wallet_recreate_operationENS2_27son_wallet_update_operationENS2_35son_wallet_deposit_create_operationENS2_36son_wallet_deposit_process_operationENS2_36son_wallet_withdraw_create_operationENS2_37son_wallet_withdraw_process_operationENS2_31sidechain_address_add_operationENS2_34sidechain_address_update_operationENS2_34sidechain_address_delete_operationENS2_38sidechain_transaction_create_operationENS2_36sidechain_transaction_sign_operationENS2_36sidechain_transaction_send_operationENS2_38sidechain_transaction_settle_operationENS2_36nft_lottery_token_purchase_operationENS2_28nft_lottery_reward_operationENS2_25nft_lottery_end_operationENS2_29random_number_store_operationEEE5cleanEv (this=0x7fe5873fb0d0)
    at /home/peerplays/src/peerplays/libraries/fc/include/fc/static_variant.hpp:224
#10 0x0000561e0b218f24 in _ZN2fc14static_variantIJN8graphene5chain18transfer_operationENS2_28limit_order_create_operationENS2_28limit_order_cancel_operationENS2_27call_order_update_operationENS2_20fill_order_operationENS2_24account_create_operationENS2_24account_update_operationENS2_27account_whitelist_operationENS2_25account_upgrade_operationENS2_26account_transfer_operationENS2_22asset_create_operationENS2_22asset_update_operationENS2_31asset_update_bitasset_operationENS2_37asset_update_feed_producers_operationENS2_21asset_issue_operationENS2_23asset_reserve_operationENS2_29asset_fund_fee_pool_operationENS2_22asset_settle_operationENS2_29asset_global_settle_operationENS2_28asset_publish_feed_operationENS2_24witness_create_operationENS2_24witness_update_operationENS2_25proposal_create_operationENS2_25proposal_update_operationENS2_25proposal_delete_operationENS2_36withdraw_permission_create_operationENS2_36withdraw_permission_update_operationENS2_35withdraw_permission_claim_operationENS2_36withdraw_permission_delete_operationENS2_33committee_member_create_operationENS2_33committee_member_update_operationENS2_51committee_member_update_global_parameters_operationENS2_32vesting_balance_create_operationENS2_34vesting_balance_withdraw_operationENS2_23worker_create_operationENS2_16custom_operationENS2_16assert_operationENS2_23balance_claim_operationENS2_27override_transfer_operationENS2_27transfer_to_blind_operationENS2_24blind_transfer_operationENS2_29transfer_from_blind_operationENS2_29asset_settle_cancel_operationENS2_26asset_claim_fees_operationENS2_24fba_distribute_operationENS2_27tournament_create_operationENS2_25tournament_join_operationENS2_19game_move_operationENS2_31asset_update_dividend_operationENS2_37asset_dividend_distribution_operationENS2_27tournament_payout_operationENS2_26tournament_leave_operationENS2_22sport_create_operationENS2_22sport_update_operationENS2_28event_group_create_operationENS2_28event_group_update_operationENS2_22event_create_operationENS2_22event_update_operationENS2_37betting_market_rules_create_operationENS2_37betting_market_rules_update_operationENS2_37betting_market_group_create_operationENS2_31betting_market_create_operationENS2_19bet_place_operationENS2_38betting_market_group_resolve_operationENS2_39betting_market_group_resolved_operationENS2_22bet_adjusted_operationENS2_52betting_market_group_cancel_unmatched_bets_operationENS2_21bet_matched_operationENS2_20bet_cancel_operationENS2_22bet_canceled_operationENS2_37betting_market_group_update_operationENS2_31betting_market_update_operationENS2_29event_update_status_operationENS2_22sport_delete_operationENS2_28event_group_delete_operationENS2_26affiliate_payout_operationENS2_35affiliate_referral_payout_operationENS2_30lottery_asset_create_operationENS2_25ticket_purchase_operationENS2_24lottery_reward_operationENS2_21lottery_end_operationENS2_30sweeps_vesting_claim_operationENS2_34custom_permission_create_operationENS2_34custom_permission_update_operationENS2_34custom_permission_delete_operationENS2_41custom_account_authority_create_operationENS2_41custom_account_authority_update_operationENS2_41custom_account_authority_delete_operationENS2_15offer_operationENS2_13bid_operationENS2_22cancel_offer_operationENS2_24finalize_offer_operationENS2_29nft_metadata_create_operationENS2_29nft_metadata_update_operationENS2_18nft_mint_operationENS2_32nft_safe_transfer_from_operationENS2_21nft_approve_operationENS2_34nft_set_approval_for_all_operationENS2_29account_role_create_operationENS2_29account_role_update_operationENS2_29account_role_delete_operationENS2_20son_create_operationENS2_20son_update_operationENS2_24son_deregister_operationENS2_23son_heartbeat_operationENS2_25son_report_down_operationENS2_25son_maintenance_operationENS2_29son_wallet_recreate_operationENS2_27son_wallet_update_operationENS2_35son_wallet_deposit_create_operationENS2_36son_wallet_deposit_process_operationENS2_36son_wallet_withdraw_create_operationENS2_37son_wallet_withdraw_process_operationENS2_31sidechain_address_add_operationENS2_34sidechain_address_update_operationENS2_34sidechain_address_delete_operationENS2_38sidechain_transaction_create_operationENS2_36sidechain_transaction_sign_operationENS2_36sidechain_transaction_send_operationENS2_38sidechain_transaction_settle_operationENS2_36nft_lottery_token_purchase_operationENS2_28nft_lottery_reward_operationENS2_25nft_lottery_end_operationENS2_29random_number_store_operationEEED2Ev (this=0x7fe5873fb0d0, __in_chrg=<optimized out>)
    at /home/peerplays/src/peerplays/libraries/fc/include/fc/static_variant.hpp:259
#11 0x0000561e0b7dd19a in graphene::peerplays_sidechain::sidechain_net_handler::process_proposals (this=0x561e0f921d80) at /home/peerplays/src/peerplays/libraries/plugins/peerplays_sidechain/sidechain_net_handler.cpp:349
#12 0x0000561e0b79d61f in graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::process_proposals (this=0x561e0f8947b0, sidechain=graphene::chain::sidechain_type::hive)
    at /home/peerplays/src/peerplays/libraries/plugins/peerplays_sidechain/peerplays_sidechain_plugin.cpp:822
#13 0x0000561e0b798505 in graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::son_processing (this=0x561e0f8947b0, sidechain=graphene::chain::sidechain_type::hive)
    at /home/peerplays/src/peerplays/libraries/plugins/peerplays_sidechain/peerplays_sidechain_plugin.cpp:568
#14 0x0000561e0b79769c in graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::<lambda()>::operator()(void) const (__closure=0x561e0fbe0a58)
--Type <RET> for more, q to quit, c to continue without paging--
    at /home/peerplays/src/peerplays/libraries/plugins/peerplays_sidechain/peerplays_sidechain_plugin.cpp:524
#15 0x0000561e0b7a3755 in std::__invoke_impl<void, graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> >(std::__invoke_other, graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::<lambda()> &&) (__f=...) at /usr/include/c++/9/bits/invoke.h:60
#16 0x0000561e0b7a36f6 in std::__invoke<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> >(graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::<lambda()> &&) (__fn=...) at /usr/include/c++/9/bits/invoke.h:95
#17 0x0000561e0b7a3686 in std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x561e0fbe0a58)
    at /usr/include/c++/9/thread:244
#18 0x0000561e0b7a3613 in std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >::operator()(void) (this=0x561e0fbe0a58)
    at /usr/include/c++/9/thread:251
#19 0x0000561e0b7a3346 in std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<void>, std::__future_base::_Result_base::_Deleter>, std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >, void>::operator()(void) const (this=0x7fe5873fb910) at /usr/include/c++/9/future:1362
#20 0x0000561e0b7a3005 in std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter>(), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<void>, std::__future_base::_Result_base::_Deleter>, std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >, void> >::_M_invoke(const std::_Any_data &) (
    __functor=...) at /usr/include/c++/9/bits/std_function.h:286
#21 0x0000561e0b7a5ece in std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>::operator()() const (this=0x7fe5873fb910) at /usr/include/c++/9/bits/std_function.h:688
#22 0x0000561e0b7a4f53 in std::__future_base::_State_baseV2::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*) (this=0x561e0fbe0a20, __f=0x7fe5873fb910, 
    __did_set=0x7fe5873fb86f) at /usr/include/c++/9/future:561
#23 0x0000561e0b7ae5d5 in std::__invoke_impl<void, void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(std::__invoke_memfun_deref, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&) (__f=
    @0x7fe5873fb8b0: (void (std::__future_base::_State_baseV2::*)(class std::__future_base::_State_baseV2 * const, class std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter>()> *, bool *)) 0x561e0b7a4f18 <std::__future_base::_State_baseV2::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*)>, __t=@0x7fe5873fb880: 0x561e0fbe0a20)
    at /usr/include/c++/9/bits/invoke.h:73
#24 0x0000561e0b7a9680 in std::__invoke<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&) (__fn=
    @0x7fe5873fb8b0: (void (std::__future_base::_State_baseV2::*)(class std::__future_base::_State_baseV2 * const, class std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter>()> *, bool *)) 0x561e0b7a4f18 <std::__future_base::_State_baseV2::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*)>) at /usr/include/c++/9/bits/invoke.h:95
#25 0x0000561e0b7a5c82 in std::call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(std::once_flag&, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&)::{lambda()#1}::operator()() const (this=0x7fe5873fb800) at /usr/include/c++/9/mutex:671
#26 0x0000561e0b7a5cb1 in std::call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(std::once_flag&, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&)::{lambda()#2}::operator()() const (this=0x0) at /usr/include/c++/9/mutex:676
#27 0x0000561e0b7a5cc6 in std::call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(std::once_flag&, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&)::{lambda()#2}::_FUN() () at /usr/include/c++/9/mutex:676
#28 0x00007fe5d3d5d4df in __pthread_once_slow (once_control=0x561e0fbe0a38, init_routine=0x7fe5d3c39c20 <__once_proxy>) at pthread_once.c:116
#29 0x0000561e0b78d8a2 in __gthread_once (__once=0x561e0fbe0a38, __func=0x7fe5d3c39c20 <__once_proxy>) at /usr/include/x86_64-linux-gnu/c++/9/bits/gthr-default.h:700
#30 0x0000561e0b7a5d70 in std::call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(std::once_flag&, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&) (
    __once=..., __f=
    @0x7fe5873fb8b0: (void (std::__future_base::_State_baseV2::*)(class std::__future_base::_State_baseV2 * const, class std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter>()> *, bool *)) 0x561e0b7a4f18 <std::__future_base::_State_baseV2::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*)>) at /usr/include/c++/9/mutex:683
#31 0x0000561e0b7a4caf in std::__future_base::_State_baseV2::_M_set_result(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>, bool) (this=0x561e0fbe0a20, __res=..., 
    __ignore_failure=false) at /usr/include/c++/9/future:401
#32 0x0000561e0b7a2a7e in std::__future_base::_Async_state_impl<std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >, void>::<lambda()>::operator()(void) const (this=0x561e0fbe0a20) at /usr/include/c++/9/future:1662
#33 0x0000561e0b7a404d in std::__invoke_impl<void, std::__future_base::_Async_state_impl<_BoundFn, _Res>::_Async_state_impl(_BoundFn&&) [with _BoundFn = std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >; _Res = void]::<lambda()> >(std::__invoke_other, std::__future_base::_Async_state_impl<std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidecha--Type <RET> for more, q to quit, c to continue without paging--
in_plugin_impl::schedule_son_processing()::<lambda()> > >, void>::<lambda()> &&) (__f=...) at /usr/include/c++/9/bits/invoke.h:60
#34 0x0000561e0b7a3fee in std::__invoke<std::__future_base::_Async_state_impl<_BoundFn, _Res>::_Async_state_impl(_BoundFn&&) [with _BoundFn = std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >; _Res = void]::<lambda()> >(std::__future_base::_Async_state_impl<std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >, void>::<lambda()> &&) (__fn=...) at /usr/include/c++/9/bits/invoke.h:95
#35 0x0000561e0b7a3f30 in std::thread::_Invoker<std::tuple<std::__future_base::_Async_state_impl<_BoundFn, _Res>::_Async_state_impl(_BoundFn&&) [with _BoundFn = std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >; _Res = void]::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x561e0fbcc9b8) at /usr/include/c++/9/thread:244
#36 0x0000561e0b7a3e71 in std::thread::_Invoker<std::tuple<std::__future_base::_Async_state_impl<_BoundFn, _Res>::_Async_state_impl(_BoundFn&&) [with _BoundFn = std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >; _Res = void]::<lambda()> > >::operator()(void) (this=0x561e0fbcc9b8) at /usr/include/c++/9/thread:251
#37 0x0000561e0b7a39aa in std::thread::_State_impl<std::thread::_Invoker<std::tuple<std::__future_base::_Async_state_impl<_BoundFn, _Res>::_Async_state_impl(_BoundFn&&) [with _BoundFn = std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >; _Res = void]::<lambda()> > > >::_M_run(void) (this=0x561e0fbcc9b0) at /usr/include/c++/9/thread:195
#38 0x00007fe5d3c3ade4 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6
#39 0x00007fe5d3d54609 in start_thread (arg=<optimized out>) at pthread_create.c:477
#40 0x00007fe5d3927133 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
(gdb) 
(gdb)

@bobinson @serkixenos @vampik
@tai.sama

bitcoin deposit crash is reproducible, seems like similar trace as above: ``` kiran@warpspeed:/home$ sudo coredumpctl debug 1297952 PID: 1297952 (witness_node) UID: 0 (root) GID: 0 (root) Signal: 11 (SEGV) Timestamp: Thu 2023-02-23 00:48:25 +0530 (3min 55s ago) Command Line: ./witness_node -d ./witness_node_data_dir --replay-blockchain Executable: /home/peerplays/src/peerplays/build/programs/witness_node/witness_node Control Group: /docker/c1df8708ba774395860d359e695a45df55f08ee6b48998d7c96eef28844f338c Slice: -.slice Boot ID: 2467b5983e234846a5eb4d9cd930077a Machine ID: a388625228614d18b5373ad1be39a93f Hostname: warpspeed Storage: /var/lib/systemd/coredump/core.witness_node.0.2467b5983e234846a5eb4d9cd930077a.1297952.1677093505000000000000.lz4 Message: Process 1297952 (witness_node) of user 0 dumped core. Stack trace of thread 81784: #0 0x00007fe5d382a941 __GI_abort (libc.so.6 + 0x22941) #1 0x00007fe5d3c02911 n/a (libstdc++.so.6 + 0x9e911) #2 0x00007fe5d3c0e38c n/a (libstdc++.so.6 + 0xaa38c) #3 0x00007fe5d3c0d369 n/a (libstdc++.so.6 + 0xa9369) #4 0x00007fe5d3c0dd21 __gxx_personality_v0 (libstdc++.so.6 + 0xa9d21) #5 0x00007fe5d3a0abef n/a (libgcc_s.so.1 + 0x10bef) #6 0x00007fe5d3a0b281 _Unwind_RaiseException (libgcc_s.so.1 + 0x11281) #7 0x00007fe5d3c0e69c __cxa_throw (libstdc++.so.6 + 0xaa69c) #8 0x0000561e0d62ddfe n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6d7cdfe) #9 0x0000561e0b2222ec n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x49712ec) #10 0x0000561e0b218f24 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4967f24) #11 0x0000561e0b7dd19a n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4f2c19a) #12 0x0000561e0b79d61f n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4eec61f) #13 0x0000561e0b798505 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ee7505) #14 0x0000561e0b79769c n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ee669c) #15 0x0000561e0b7a3755 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2755) #16 0x0000561e0b7a36f6 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef26f6) #17 0x0000561e0b7a3686 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2686) #18 0x0000561e0b7a3613 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2613) #19 0x0000561e0b7a3346 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2346) #20 0x0000561e0b7a3005 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2005) #21 0x0000561e0b7a5ece n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef4ece) #22 0x0000561e0b7a4f53 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef3f53) #23 0x0000561e0b7ae5d5 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4efd5d5) #24 0x0000561e0b7a9680 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef8680) #25 0x0000561e0b7a5c82 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef4c82) #26 0x0000561e0b7a5cb1 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef4cb1) #27 0x0000561e0b7a5cc6 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef4cc6) #28 0x00007fe5d3d5d4df __pthread_once_slow (libpthread.so.0 + 0x114df) #29 0x0000561e0b78d8a2 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4edc8a2) #30 0x0000561e0b7a5d70 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef4d70) #31 0x0000561e0b7a4caf n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef3caf) #32 0x0000561e0b7a2a7e n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef1a7e) #33 0x0000561e0b7a404d n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef304d) #34 0x0000561e0b7a3fee n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2fee) #35 0x0000561e0b7a3f30 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2f30) #36 0x0000561e0b7a3e71 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef2e71) #37 0x0000561e0b7a39aa n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x4ef29aa) #38 0x00007fe5d3c3ade4 n/a (libstdc++.so.6 + 0xd6de4) #39 0x00007fe5d3d54609 start_thread (libpthread.so.0 + 0x8609) #40 0x00007fe5d3927133 __clone (libc.so.6 + 0x11f133) Stack trace of thread 16: #0 0x00007fe5d3d5b376 futex_wait_cancelable (libpthread.so.0 + 0xf376) #1 0x0000561e0bfa92e7 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56f82e7) #2 0x0000561e0bf99a28 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56e8a28) #3 0x0000561e0bf9cc87 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56ebc87) #4 0x0000561e0bf9c650 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56eb650) #5 0x0000561e0bfa0103 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56ef103) #6 0x0000561e0bff1780 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x5740780) #7 0x0000561e0bf99beb n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56e8beb) #8 0x00007fe5d3d54609 start_thread (libpthread.so.0 + 0x8609) #9 0x00007fe5d3927133 __clone (libc.so.6 + 0x11f133) Stack trace of thread 7: #0 0x00007fe5d3d5b376 futex_wait_cancelable (libpthread.so.0 + 0xf376) #1 0x0000561e0bfa92e7 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56f82e7) #2 0x0000561e0bf99a28 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56e8a28) #3 0x0000561e0bf9cc87 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56ebc87) #4 0x0000561e0bf9c650 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x56eb650) #5 0x0000561e0d816afc n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6f65afc) #6 0x0000561e0d816d60 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6f65d60) #7 0x0000561e0d823ed0 n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6f72ed0) #8 0x0000561e0d8a94bb n/a (/home/peerplays/src/peerplays/build/programs/witness_node/witness_node + 0x6ff84bb) GNU gdb (Ubuntu 9.2-0ubuntu1~20.04.1) 9.2 Copyright (C) 2020 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>. Find the GDB manual and other documentation resources online at: <http://www.gnu.org/software/gdb/documentation/>. For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /home/peerplays/src/peerplays/build/programs/witness_node/witness_node... [New LWP 81784] [New LWP 16] [New LWP 7] [New LWP 8] [New LWP 18] [New LWP 23] [New LWP 1] [New LWP 19] [New LWP 81782] [New LWP 21] [New LWP 24] [New LWP 20] [New LWP 11] [New LWP 14] [New LWP 9] [New LWP 13] [New LWP 15] [New LWP 22] [New LWP 10] [New LWP 12] [New LWP 17] warning: .dynamic section for "/usr/local/lib/libbitcoin-protocol.so.0" is not at the expected address (wrong library or version mismatch?) warning: .dynamic section for "/usr/local/lib/libbitcoin-client.so.0" is not at the expected address (wrong library or version mismatch?) warning: .dynamic section for "/usr/local/lib/libbitcoin-explorer.so.0" is not at the expected address (wrong library or version mismatch?) warning: .dynamic section for "/usr/local//lib/libsecp256k1.so.0" is not at the expected address (wrong library or version mismatch?) warning: .dynamic section for "/usr/local//lib/libbitcoin-network.so.0" is not at the expected address (wrong library or version mismatch?) [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `./witness_node -d ./witness_node_data_dir --replay-blockchain'. Program terminated with signal SIGSEGV, Segmentation fault. #0 __GI_abort () at abort.c:107 107 abort.c: No such file or directory. [Current thread is 1 (Thread 0x7fe5873fc700 (LWP 81784))] (gdb) bt #0 __GI_abort () at abort.c:107 #1 0x00007fe5d3c02911 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x00007fe5d3c0e38c in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 #3 0x00007fe5d3c0d369 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 #4 0x00007fe5d3c0dd21 in __gxx_personality_v0 () from /lib/x86_64-linux-gnu/libstdc++.so.6 #5 0x00007fe5d3a0abef in ?? () from /lib/x86_64-linux-gnu/libgcc_s.so.1 #6 0x00007fe5d3a0b281 in _Unwind_RaiseException () from /lib/x86_64-linux-gnu/libgcc_s.so.1 #7 0x00007fe5d3c0e69c in __cxa_throw () from /lib/x86_64-linux-gnu/libstdc++.so.6 #8 0x0000561e0d62ddfe in fc::impl::dynamic_storage::data (this=0x7fe5873fb0d8) at /home/peerplays/src/peerplays/libraries/fc/src/static_variant.cpp:15 #9 0x0000561e0b2222ec in _ZN2fc14static_variantIJN8graphene5chain18transfer_operationENS2_28limit_order_create_operationENS2_28limit_order_cancel_operationENS2_27call_order_update_operationENS2_20fill_order_operationENS2_24account_create_operationENS2_24account_update_operationENS2_27account_whitelist_operationENS2_25account_upgrade_operationENS2_26account_transfer_operationENS2_22asset_create_operationENS2_22asset_update_operationENS2_31asset_update_bitasset_operationENS2_37asset_update_feed_producers_operationENS2_21asset_issue_operationENS2_23asset_reserve_operationENS2_29asset_fund_fee_pool_operationENS2_22asset_settle_operationENS2_29asset_global_settle_operationENS2_28asset_publish_feed_operationENS2_24witness_create_operationENS2_24witness_update_operationENS2_25proposal_create_operationENS2_25proposal_update_operationENS2_25proposal_delete_operationENS2_36withdraw_permission_create_operationENS2_36withdraw_permission_update_operationENS2_35withdraw_permission_claim_operationENS2_36withdraw_permission_delete_operationENS2_33committee_member_create_operationENS2_33committee_member_update_operationENS2_51committee_member_update_global_parameters_operationENS2_32vesting_balance_create_operationENS2_34vesting_balance_withdraw_operationENS2_23worker_create_operationENS2_16custom_operationENS2_16assert_operationENS2_23balance_claim_operationENS2_27override_transfer_operationENS2_27transfer_to_blind_operationENS2_24blind_transfer_operationENS2_29transfer_from_blind_operationENS2_29asset_settle_cancel_operationENS2_26asset_claim_fees_operationENS2_24fba_distribute_operationENS2_27tournament_create_operationENS2_25tournament_join_operationENS2_19game_move_operationENS2_31asset_update_dividend_operationENS2_37asset_dividend_distribution_operationENS2_27tournament_payout_operationENS2_26tournament_leave_operationENS2_22sport_create_operationENS2_22sport_update_operationENS2_28event_group_create_operationENS2_28event_group_update_operationENS2_22event_create_operationENS2_22event_update_operationENS2_37betting_market_rules_create_operationENS2_37betting_market_rules_update_operationENS2_37betting_market_group_create_operationENS2_31betting_market_create_operationENS2_19bet_place_operationENS2_38betting_market_group_resolve_operationENS2_39betting_market_group_resolved_operationENS2_22bet_adjusted_operationENS2_52betting_market_group_cancel_unmatched_bets_operationENS2_21bet_matched_operationENS2_20bet_cancel_operationENS2_22bet_canceled_operationENS2_37betting_market_group_update_operationENS2_31betting_market_update_operationENS2_29event_update_status_operationENS2_22sport_delete_operationENS2_28event_group_delete_operationENS2_26affiliate_payout_operationENS2_35affiliate_referral_payout_operationENS2_30lottery_asset_create_operationENS2_25ticket_purchase_operationENS2_24lottery_reward_operationENS2_21lottery_end_operationENS2_30sweeps_vesting_claim_operationENS2_34custom_permission_create_operationENS2_34custom_permission_update_operationENS2_34custom_permission_delete_operationENS2_41custom_account_authority_create_operationENS2_41custom_account_authority_update_operationENS2_41custom_account_authority_delete_operationENS2_15offer_operationENS2_13bid_operationENS2_22cancel_offer_operationENS2_24finalize_offer_operationENS2_29nft_metadata_create_operationENS2_29nft_metadata_update_operationENS2_18nft_mint_operationENS2_32nft_safe_transfer_from_operationENS2_21nft_approve_operationENS2_34nft_set_approval_for_all_operationENS2_29account_role_create_operationENS2_29account_role_update_operationENS2_29account_role_delete_operationENS2_20son_create_operationENS2_20son_update_operationENS2_24son_deregister_operationENS2_23son_heartbeat_operationENS2_25son_report_down_operationENS2_25son_maintenance_operationENS2_29son_wallet_recreate_operationENS2_27son_wallet_update_operationENS2_35son_wallet_deposit_create_operationENS2_36son_wallet_deposit_process_operationENS2_36son_wallet_withdraw_create_operationENS2_37son_wallet_withdraw_process_operationENS2_31sidechain_address_add_operationENS2_34sidechain_address_update_operationENS2_34sidechain_address_delete_operationENS2_38sidechain_transaction_create_operationENS2_36sidechain_transaction_sign_operationENS2_36sidechain_transaction_send_operationENS2_38sidechain_transaction_settle_operationENS2_36nft_lottery_token_purchase_operationENS2_28nft_lottery_reward_operationENS2_25nft_lottery_end_operationENS2_29random_number_store_operationEEE5cleanEv (this=0x7fe5873fb0d0) at /home/peerplays/src/peerplays/libraries/fc/include/fc/static_variant.hpp:224 #10 0x0000561e0b218f24 in _ZN2fc14static_variantIJN8graphene5chain18transfer_operationENS2_28limit_order_create_operationENS2_28limit_order_cancel_operationENS2_27call_order_update_operationENS2_20fill_order_operationENS2_24account_create_operationENS2_24account_update_operationENS2_27account_whitelist_operationENS2_25account_upgrade_operationENS2_26account_transfer_operationENS2_22asset_create_operationENS2_22asset_update_operationENS2_31asset_update_bitasset_operationENS2_37asset_update_feed_producers_operationENS2_21asset_issue_operationENS2_23asset_reserve_operationENS2_29asset_fund_fee_pool_operationENS2_22asset_settle_operationENS2_29asset_global_settle_operationENS2_28asset_publish_feed_operationENS2_24witness_create_operationENS2_24witness_update_operationENS2_25proposal_create_operationENS2_25proposal_update_operationENS2_25proposal_delete_operationENS2_36withdraw_permission_create_operationENS2_36withdraw_permission_update_operationENS2_35withdraw_permission_claim_operationENS2_36withdraw_permission_delete_operationENS2_33committee_member_create_operationENS2_33committee_member_update_operationENS2_51committee_member_update_global_parameters_operationENS2_32vesting_balance_create_operationENS2_34vesting_balance_withdraw_operationENS2_23worker_create_operationENS2_16custom_operationENS2_16assert_operationENS2_23balance_claim_operationENS2_27override_transfer_operationENS2_27transfer_to_blind_operationENS2_24blind_transfer_operationENS2_29transfer_from_blind_operationENS2_29asset_settle_cancel_operationENS2_26asset_claim_fees_operationENS2_24fba_distribute_operationENS2_27tournament_create_operationENS2_25tournament_join_operationENS2_19game_move_operationENS2_31asset_update_dividend_operationENS2_37asset_dividend_distribution_operationENS2_27tournament_payout_operationENS2_26tournament_leave_operationENS2_22sport_create_operationENS2_22sport_update_operationENS2_28event_group_create_operationENS2_28event_group_update_operationENS2_22event_create_operationENS2_22event_update_operationENS2_37betting_market_rules_create_operationENS2_37betting_market_rules_update_operationENS2_37betting_market_group_create_operationENS2_31betting_market_create_operationENS2_19bet_place_operationENS2_38betting_market_group_resolve_operationENS2_39betting_market_group_resolved_operationENS2_22bet_adjusted_operationENS2_52betting_market_group_cancel_unmatched_bets_operationENS2_21bet_matched_operationENS2_20bet_cancel_operationENS2_22bet_canceled_operationENS2_37betting_market_group_update_operationENS2_31betting_market_update_operationENS2_29event_update_status_operationENS2_22sport_delete_operationENS2_28event_group_delete_operationENS2_26affiliate_payout_operationENS2_35affiliate_referral_payout_operationENS2_30lottery_asset_create_operationENS2_25ticket_purchase_operationENS2_24lottery_reward_operationENS2_21lottery_end_operationENS2_30sweeps_vesting_claim_operationENS2_34custom_permission_create_operationENS2_34custom_permission_update_operationENS2_34custom_permission_delete_operationENS2_41custom_account_authority_create_operationENS2_41custom_account_authority_update_operationENS2_41custom_account_authority_delete_operationENS2_15offer_operationENS2_13bid_operationENS2_22cancel_offer_operationENS2_24finalize_offer_operationENS2_29nft_metadata_create_operationENS2_29nft_metadata_update_operationENS2_18nft_mint_operationENS2_32nft_safe_transfer_from_operationENS2_21nft_approve_operationENS2_34nft_set_approval_for_all_operationENS2_29account_role_create_operationENS2_29account_role_update_operationENS2_29account_role_delete_operationENS2_20son_create_operationENS2_20son_update_operationENS2_24son_deregister_operationENS2_23son_heartbeat_operationENS2_25son_report_down_operationENS2_25son_maintenance_operationENS2_29son_wallet_recreate_operationENS2_27son_wallet_update_operationENS2_35son_wallet_deposit_create_operationENS2_36son_wallet_deposit_process_operationENS2_36son_wallet_withdraw_create_operationENS2_37son_wallet_withdraw_process_operationENS2_31sidechain_address_add_operationENS2_34sidechain_address_update_operationENS2_34sidechain_address_delete_operationENS2_38sidechain_transaction_create_operationENS2_36sidechain_transaction_sign_operationENS2_36sidechain_transaction_send_operationENS2_38sidechain_transaction_settle_operationENS2_36nft_lottery_token_purchase_operationENS2_28nft_lottery_reward_operationENS2_25nft_lottery_end_operationENS2_29random_number_store_operationEEED2Ev (this=0x7fe5873fb0d0, __in_chrg=<optimized out>) at /home/peerplays/src/peerplays/libraries/fc/include/fc/static_variant.hpp:259 #11 0x0000561e0b7dd19a in graphene::peerplays_sidechain::sidechain_net_handler::process_proposals (this=0x561e0f921d80) at /home/peerplays/src/peerplays/libraries/plugins/peerplays_sidechain/sidechain_net_handler.cpp:349 #12 0x0000561e0b79d61f in graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::process_proposals (this=0x561e0f8947b0, sidechain=graphene::chain::sidechain_type::hive) at /home/peerplays/src/peerplays/libraries/plugins/peerplays_sidechain/peerplays_sidechain_plugin.cpp:822 #13 0x0000561e0b798505 in graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::son_processing (this=0x561e0f8947b0, sidechain=graphene::chain::sidechain_type::hive) at /home/peerplays/src/peerplays/libraries/plugins/peerplays_sidechain/peerplays_sidechain_plugin.cpp:568 #14 0x0000561e0b79769c in graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::<lambda()>::operator()(void) const (__closure=0x561e0fbe0a58) --Type <RET> for more, q to quit, c to continue without paging-- at /home/peerplays/src/peerplays/libraries/plugins/peerplays_sidechain/peerplays_sidechain_plugin.cpp:524 #15 0x0000561e0b7a3755 in std::__invoke_impl<void, graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> >(std::__invoke_other, graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::<lambda()> &&) (__f=...) at /usr/include/c++/9/bits/invoke.h:60 #16 0x0000561e0b7a36f6 in std::__invoke<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> >(graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::<lambda()> &&) (__fn=...) at /usr/include/c++/9/bits/invoke.h:95 #17 0x0000561e0b7a3686 in std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x561e0fbe0a58) at /usr/include/c++/9/thread:244 #18 0x0000561e0b7a3613 in std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >::operator()(void) (this=0x561e0fbe0a58) at /usr/include/c++/9/thread:251 #19 0x0000561e0b7a3346 in std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<void>, std::__future_base::_Result_base::_Deleter>, std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >, void>::operator()(void) const (this=0x7fe5873fb910) at /usr/include/c++/9/future:1362 #20 0x0000561e0b7a3005 in std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter>(), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<void>, std::__future_base::_Result_base::_Deleter>, std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >, void> >::_M_invoke(const std::_Any_data &) ( __functor=...) at /usr/include/c++/9/bits/std_function.h:286 #21 0x0000561e0b7a5ece in std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>::operator()() const (this=0x7fe5873fb910) at /usr/include/c++/9/bits/std_function.h:688 #22 0x0000561e0b7a4f53 in std::__future_base::_State_baseV2::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*) (this=0x561e0fbe0a20, __f=0x7fe5873fb910, __did_set=0x7fe5873fb86f) at /usr/include/c++/9/future:561 #23 0x0000561e0b7ae5d5 in std::__invoke_impl<void, void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(std::__invoke_memfun_deref, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&) (__f= @0x7fe5873fb8b0: (void (std::__future_base::_State_baseV2::*)(class std::__future_base::_State_baseV2 * const, class std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter>()> *, bool *)) 0x561e0b7a4f18 <std::__future_base::_State_baseV2::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*)>, __t=@0x7fe5873fb880: 0x561e0fbe0a20) at /usr/include/c++/9/bits/invoke.h:73 #24 0x0000561e0b7a9680 in std::__invoke<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&) (__fn= @0x7fe5873fb8b0: (void (std::__future_base::_State_baseV2::*)(class std::__future_base::_State_baseV2 * const, class std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter>()> *, bool *)) 0x561e0b7a4f18 <std::__future_base::_State_baseV2::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*)>) at /usr/include/c++/9/bits/invoke.h:95 #25 0x0000561e0b7a5c82 in std::call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(std::once_flag&, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&)::{lambda()#1}::operator()() const (this=0x7fe5873fb800) at /usr/include/c++/9/mutex:671 #26 0x0000561e0b7a5cb1 in std::call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(std::once_flag&, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&)::{lambda()#2}::operator()() const (this=0x0) at /usr/include/c++/9/mutex:676 #27 0x0000561e0b7a5cc6 in std::call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(std::once_flag&, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&)::{lambda()#2}::_FUN() () at /usr/include/c++/9/mutex:676 #28 0x00007fe5d3d5d4df in __pthread_once_slow (once_control=0x561e0fbe0a38, init_routine=0x7fe5d3c39c20 <__once_proxy>) at pthread_once.c:116 #29 0x0000561e0b78d8a2 in __gthread_once (__once=0x561e0fbe0a38, __func=0x7fe5d3c39c20 <__once_proxy>) at /usr/include/x86_64-linux-gnu/c++/9/bits/gthr-default.h:700 #30 0x0000561e0b7a5d70 in std::call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(std::once_flag&, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&) ( __once=..., __f= @0x7fe5873fb8b0: (void (std::__future_base::_State_baseV2::*)(class std::__future_base::_State_baseV2 * const, class std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter>()> *, bool *)) 0x561e0b7a4f18 <std::__future_base::_State_baseV2::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*)>) at /usr/include/c++/9/mutex:683 #31 0x0000561e0b7a4caf in std::__future_base::_State_baseV2::_M_set_result(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>, bool) (this=0x561e0fbe0a20, __res=..., __ignore_failure=false) at /usr/include/c++/9/future:401 #32 0x0000561e0b7a2a7e in std::__future_base::_Async_state_impl<std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >, void>::<lambda()>::operator()(void) const (this=0x561e0fbe0a20) at /usr/include/c++/9/future:1662 #33 0x0000561e0b7a404d in std::__invoke_impl<void, std::__future_base::_Async_state_impl<_BoundFn, _Res>::_Async_state_impl(_BoundFn&&) [with _BoundFn = std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >; _Res = void]::<lambda()> >(std::__invoke_other, std::__future_base::_Async_state_impl<std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidecha--Type <RET> for more, q to quit, c to continue without paging-- in_plugin_impl::schedule_son_processing()::<lambda()> > >, void>::<lambda()> &&) (__f=...) at /usr/include/c++/9/bits/invoke.h:60 #34 0x0000561e0b7a3fee in std::__invoke<std::__future_base::_Async_state_impl<_BoundFn, _Res>::_Async_state_impl(_BoundFn&&) [with _BoundFn = std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >; _Res = void]::<lambda()> >(std::__future_base::_Async_state_impl<std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >, void>::<lambda()> &&) (__fn=...) at /usr/include/c++/9/bits/invoke.h:95 #35 0x0000561e0b7a3f30 in std::thread::_Invoker<std::tuple<std::__future_base::_Async_state_impl<_BoundFn, _Res>::_Async_state_impl(_BoundFn&&) [with _BoundFn = std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >; _Res = void]::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x561e0fbcc9b8) at /usr/include/c++/9/thread:244 #36 0x0000561e0b7a3e71 in std::thread::_Invoker<std::tuple<std::__future_base::_Async_state_impl<_BoundFn, _Res>::_Async_state_impl(_BoundFn&&) [with _BoundFn = std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >; _Res = void]::<lambda()> > >::operator()(void) (this=0x561e0fbcc9b8) at /usr/include/c++/9/thread:251 #37 0x0000561e0b7a39aa in std::thread::_State_impl<std::thread::_Invoker<std::tuple<std::__future_base::_Async_state_impl<_BoundFn, _Res>::_Async_state_impl(_BoundFn&&) [with _BoundFn = std::thread::_Invoker<std::tuple<graphene::peerplays_sidechain::detail::peerplays_sidechain_plugin_impl::schedule_son_processing()::<lambda()> > >; _Res = void]::<lambda()> > > >::_M_run(void) (this=0x561e0fbcc9b0) at /usr/include/c++/9/thread:195 #38 0x00007fe5d3c3ade4 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 #39 0x00007fe5d3d54609 in start_thread (arg=<optimized out>) at pthread_create.c:477 #40 0x00007fe5d3927133 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 (gdb) (gdb) ``` @bobinson @serkixenos @vampik @tai.sama