WIP implementation of protocol 1.6

docs/HOWTO.rst

@@ -28,17 +28,20 @@ functions. For example, `dash_hash`_ is required for DASH. Scrypt coins
 require a Python interpreter compiled and/or linked with OpenSSL 1.1.0
 or higher.
 
-You **must** be running a non-pruning bitcoin daemon with::
+You **must** be running a non-pruning bitcoin daemon.
+It is also recommended to have::
 
   txindex=1
 
-set in its configuration file.  If you have an existing installation
-of bitcoind and have not previously set this you will need to reindex
-the blockchain with::
+set in its configuration file, for better performance when serving many sessions.
+If you have an existing installation of bitcoind and have not previously set this
+but you do now, you will need to reindex the blockchain with::
 
   bitcoind -reindex
 
 which can take some time.
+If you intend to use a bitcoind without txindex enabled, you need to configure the
+`DAEMON_HAS_TXINDEX` :ref:`environment variable <environment>`.
 
 While not a requirement for running ElectrumX, it is intended to be
 run with supervisor software such as Daniel Bernstein's

docs/environment.rst

@@ -281,13 +281,14 @@ These environment variables are optional:
   version string. For example to drop versions from 1.0 to 1.2 use
   the regex ``1\.[0-2]\.\d+``.
 
-.. envvar:: DROP_CLIENT_UNKNOWN
+.. envvar:: DAEMON_HAS_TXINDEX
 
-  Set to anything non-empty to deny serving clients which do not
-  identify themselves first by issuing the server.version method
-  call with a non-empty client identifier. The connection is dropped 
-  on first actual method call. This might help to filter out simple 
-  robots. This behavior is off by default.
+  Set this environment variable to empty if the connected bitcoind
+  does not have txindex enabled. Defaults to True (has txindex).
+  We rely on bitcoind to lookup arbitrary txs, regardless of this setting.
+  Without txindex, tx lookup works as in `bitcoind PR #10275`_.
+  Note that performance when serving many sessions is better with txindex
+  (but initial sync should be unaffected).
 
 
 Resource Usage Limits
@@ -512,3 +513,4 @@ your available physical RAM:
 
 .. _lib/coins.py: https://github.com/spesmilo/electrumx/blob/master/src/electrumx/lib/coins.py
 .. _uvloop: https://pypi.python.org/pypi/uvloop
+.. _bitcoind PR #10275: https://github.com/bitcoin/bitcoin/pull/10275

pyproject.toml

@@ -37,7 +37,6 @@ Repository = "https://github.com/spesmilo/electrumx"
 [project.scripts]
 electrumx_server = "electrumx.cli.electrumx_server:main"
 electrumx_rpc = "electrumx.cli.electrumx_rpc:main"
-electrumx_compact_history = "electrumx.cli.electrumx_compact_history:main"
 
 [tool.setuptools.dynamic]
 version = { attr = 'electrumx.__version__' }

src/electrumx/lib/coins.py

@@ -233,7 +233,7 @@ def privkey_WIF(cls, privkey_bytes, compressed):
         return cls.ENCODE_CHECK(payload)
 
     @classmethod
-    def header_hash(cls, header):
+    def header_hash(cls, header: bytes) -> bytes:
         '''Given a header return hash'''
         return double_sha256(header)
 

src/electrumx/lib/tx.py

@@ -113,6 +113,13 @@ def serialize(self):
         ))
 
 
+@dataclass(kw_only=True, slots=True)
+class TXOSpendStatus:
+    prev_height: Optional[int]  # block height TXO is mined at. None if the outpoint never existed
+    spender_txhash: bytes = None
+    spender_height: int = None
+
+
 class Deserializer:
     '''Deserializes blocks into transactions.
 

src/electrumx/lib/util.py

@@ -35,7 +35,7 @@
 import sys
 from collections.abc import Container, Mapping
 from struct import Struct
-from typing import Set
+from typing import Set, Optional
 
 import aiorpcx
 
@@ -166,7 +166,7 @@ def chunks(items, size):
         yield items[i: i + size]
 
 
-def resolve_limit(limit):
+def resolve_limit(limit: Optional[int]) -> int:
     if limit is None or limit < 0:
         return -1
     assert isinstance(limit, int)
@@ -320,6 +320,7 @@ def protocol_version(client_req, min_tuple, max_tuple):
 struct_le_Q = Struct('<Q')
 struct_be_H = Struct('>H')
 struct_be_I = Struct('>I')
+struct_be_Q = Struct('>Q')
 structB = Struct('B')
 
 unpack_le_int32_from = struct_le_i.unpack_from
@@ -333,6 +334,7 @@ def protocol_version(client_req, min_tuple, max_tuple):
 unpack_le_uint32 = struct_le_I.unpack
 unpack_le_uint64 = struct_le_Q.unpack
 unpack_be_uint32 = struct_be_I.unpack
+unpack_be_uint64 = struct_be_Q.unpack
 
 pack_le_int32 = struct_le_i.pack
 pack_le_int64 = struct_le_q.pack
@@ -341,6 +343,7 @@ def protocol_version(client_req, min_tuple, max_tuple):
 pack_le_uint64 = struct_le_Q.pack
 pack_be_uint16 = struct_be_H.pack
 pack_be_uint32 = struct_be_I.pack
+pack_be_uint64 = struct_be_Q.pack
 pack_byte = structB.pack
 
 hex_to_bytes = bytes.fromhex