Merge pull request #719 from evoskuil/master

Store block size and weight both in txs, update qinterface.

Author: evoskuil

include/bitcoin/database/impl/query/archive_read.ipp

@@ -110,7 +110,7 @@ hashes CLASS::get_tx_keys(const header_link& link) const NOEXCEPT
 TEMPLATE
 size_t CLASS::get_tx_count(const header_link& link) const NOEXCEPT
 {
-    table::txs::get_tx_quantity txs{};
+    table::txs::get_tx_count txs{};
     if (!store_.txs.at(to_txs(link), txs))
         return {};
 
@@ -175,6 +175,15 @@ bool CLASS::get_tx_position(size_t& out, const tx_link& link) const NOEXCEPT
     return true;
 }
 
+TEMPLATE
+size_t CLASS::get_tx_size(const tx_link& link,
+    bool witness) const NOEXCEPT
+{
+    size_t light{}, heavy{};
+    return get_tx_sizes(light, heavy, link) ? (witness ? heavy : light) :
+        max_uint64;
+}
+
 TEMPLATE
 bool CLASS::get_tx_sizes(size_t& light, size_t& heavy,
     const tx_link& link) const NOEXCEPT
@@ -192,10 +201,25 @@ bool CLASS::get_tx_sizes(size_t& light, size_t& heavy,
 // ----------------------------------------------------------------------------
 
 TEMPLATE
-size_t CLASS::get_block_size(const header_link& link) const NOEXCEPT
+size_t CLASS::get_block_size(const header_link& link,
+    bool witness) const NOEXCEPT
+{
+    size_t light{}, heavy{};
+    return get_block_sizes(light, heavy, link) ? (witness ? heavy : light) :
+        max_uint64;
+}
+
+TEMPLATE
+bool CLASS::get_block_sizes(size_t& light, size_t& heavy,
+    const header_link& link) const NOEXCEPT
 {
-    table::txs::get_block_size txs{};
-    return store_.txs.at(to_txs(link), txs) ? txs.wire : zero;
+    table::txs::get_sizes sizes{};
+    if (!store_.txs.at(to_txs(link), sizes))
+        return false;
+
+    light = sizes.light;
+    heavy = sizes.heavy;
+    return true;
 }
 
 TEMPLATE

include/bitcoin/database/impl/query/archive_write.ipp

@@ -392,14 +392,18 @@ code CLASS::set_code(const block& block, const header_link& key,
         if ((ec = set_code(fk++, *tx, bypass)))
             return ec;
 
-    using bytes = linkage<schema::size>::integer;
+    // Optional hash, only has value on height intervals.
     auto interval = create_interval(key, height);
-    const auto size = block.serialized_size(true);
-    const auto wire = possible_narrow_cast<bytes>(size);
 
-    // Depth is only used for genesis (is_zero(tx_fks[0])).
+    // Depth is only set by writer for genesis (is_zero(tx_fks[0])).
     const auto depth = store_.interval_depth();
 
+    using bytes = linkage<schema::size>::integer;
+    const auto light = possible_narrow_cast<bytes>(
+        block.serialized_size(false));
+    const auto heavy = possible_narrow_cast<bytes>
+        (block.serialized_size(true));
+
     // ========================================================================
     const auto scope = store_.get_transactor();
     constexpr auto positive = true;
@@ -413,8 +417,9 @@ code CLASS::set_code(const block& block, const header_link& key,
     return store_.txs.put(to_txs(key), table::txs::put_group
     {
         {},
-        wire,
         count,
+        light,
+        heavy,
         tx_fks,
         std::move(interval),
         depth

include/bitcoin/database/impl/query/fees.ipp

@@ -27,14 +27,55 @@
 #include <utility>
 #include <bitcoin/database/define.hpp>
 
+#define SLOW_FEES
+
 namespace libbitcoin {
 namespace database {
 
-// TODO: optimize.
+// private
+TEMPLATE
+constexpr size_t CLASS::virtual_size(size_t light, size_t heavy) NOEXCEPT
+{
+    using namespace system;
+    using namespace system::chain;
+    constexpr auto scale = base_size_contribution + total_size_contribution;
+
+    const auto weight = ceilinged_add(
+        ceilinged_multiply(base_size_contribution, light),
+        ceilinged_multiply(total_size_contribution, heavy));
+
+    // Block weight is 3 * nominal size * + 1 * witness size [bip141].
+    return ceilinged_divide(weight, scale);
+}
+
+TEMPLATE
+bool CLASS::get_tx_virtual_size(uint64_t& out,
+    const tx_link& link) const NOEXCEPT
+{
+    size_t light{}, heavy{};
+    if (!get_tx_sizes(light, heavy, link))
+        return false;
+
+    out = virtual_size(light, heavy);
+    return true;
+}
+
+TEMPLATE
+bool CLASS::get_block_virtual_size(uint64_t& out,
+    const header_link& link) const NOEXCEPT
+{
+    size_t light{}, heavy{};
+    if (!get_block_sizes(light, heavy, link))
+        return false;
+
+    out = virtual_size(light, heavy);
+    return true;
+}
 
 TEMPLATE
 uint64_t CLASS::get_tx_fee(const tx_link& link) const NOEXCEPT
 {
+#if defined(SLOW_FEES)
     const auto tx = get_transaction(link, false);
     if (!tx)
         return max_uint64;
@@ -43,6 +84,12 @@ uint64_t CLASS::get_tx_fee(const tx_link& link) const NOEXCEPT
         return zero;
 
     return populate_without_metadata(*tx) ? tx->fee() : max_uint64;
+#else
+    // Get heavy and light sizes for the tx link and compute virtual size.
+    // Get and total value for each output of the tx link.
+    // Get all outputs links spent by the tx link (spends).
+    // Get and total value for each spend (todo: move value to outs).
+#endif
 }
 
 TEMPLATE

include/bitcoin/database/impl/query/height.ipp

@@ -38,7 +38,7 @@ size_t CLASS::get_candidate_size(size_t top) const NOEXCEPT
 {
     size_t wire{};
     for (auto height = zero; height <= top; ++height)
-        wire += get_block_size(to_candidate(height));
+        wire += get_block_size(to_candidate(height), true);
 
     return wire;
 }
@@ -55,7 +55,7 @@ size_t CLASS::get_confirmed_size(size_t top) const NOEXCEPT
 {
     size_t wire{};
     for (auto height = zero; height <= top; ++height)
-        wire += get_block_size(to_confirmed(height));
+        wire += get_block_size(to_confirmed(height), true);
 
     return wire;
 }

include/bitcoin/database/query.hpp

@@ -341,14 +341,19 @@ class query
     inline point_key get_point_key(const point_link& link) const NOEXCEPT;
     inline hash_digest get_point_hash(const point_link& link) const NOEXCEPT;
 
-    /// False implies not confirmed.
-    bool get_tx_height(size_t& out, const tx_link& link) const NOEXCEPT;
+    /// False position implies not confirmed (or fault).
     bool get_tx_position(size_t& out, const tx_link& link) const NOEXCEPT;
-    bool get_tx_sizes(size_t& light, size_t& is_locked,
+    bool get_tx_height(size_t& out, const tx_link& link) const NOEXCEPT;
+
+    /// Sizes.
+    size_t get_tx_size(const tx_link& link, bool witness) const NOEXCEPT;
+    size_t get_block_size(const header_link& link, bool witness) const NOEXCEPT;
+    bool get_block_sizes(size_t& light, size_t& heavy,
+        const header_link& link) const NOEXCEPT;
+    bool get_tx_sizes(size_t& light, size_t& heavy,
         const tx_link& link) const NOEXCEPT;
 
     /// Terminal implies not found, false implies fault.
-    size_t get_block_size(const header_link& link) const NOEXCEPT;
     height_link get_height(const hash_digest& key) const NOEXCEPT;
     height_link get_height(const header_link& link) const NOEXCEPT;
     bool get_height(size_t& out, const hash_digest& key) const NOEXCEPT;
@@ -401,6 +406,12 @@ class query
     /// Fees.
     /// -----------------------------------------------------------------------
 
+    /// Virtual size incorporates witnesses if archived.
+    bool get_tx_virtual_size(uint64_t& out,
+        const tx_link& link) const NOEXCEPT;
+    bool get_block_virtual_size(uint64_t& out,
+        const header_link& link) const NOEXCEPT;
+
     /// Total fee value by tx or block.
     uint64_t get_tx_fee(const tx_link& link) const NOEXCEPT;
     uint64_t get_block_fee(const header_link& link) const NOEXCEPT;
@@ -771,6 +782,7 @@ class query
     template <typename Functor>
     static inline code parallel_address_transform(std::atomic_bool& cancel,
         outpoints& out, const output_links& links, Functor&& functor) NOEXCEPT;
+    static constexpr size_t virtual_size(size_t light, size_t heavy) NOEXCEPT;
 
     // Not thread safe.
     size_t get_fork_() const NOEXCEPT;

include/bitcoin/database/tables/archives/transaction.hpp

@@ -29,6 +29,8 @@ namespace libbitcoin {
 namespace database {
 namespace table {
 
+// TODO: merge coinbase bit (saves about 1GB).
+
 /// Transaction is a canonical record hash table.
 struct transaction
   : public hash_map<schema::transaction>
@@ -146,11 +148,14 @@ struct transaction
         inline bool to_data(finalizer& sink) const NOEXCEPT
         {
             using namespace system;
+            const auto light = possible_narrow_cast<bytes::integer>(
+                tx.serialized_size(false));
+            const auto heavy = possible_narrow_cast<bytes::integer>
+                (tx.serialized_size(true));
+
             sink.write_byte(to_int<uint8_t>(tx.is_coinbase()));
-            sink.write_little_endian<bytes::integer, bytes::size>(
-                possible_narrow_cast<bytes::integer>(tx.serialized_size(false)));
-            sink.write_little_endian<bytes::integer, bytes::size>(
-                possible_narrow_cast<bytes::integer>(tx.serialized_size(true)));
+            sink.write_little_endian<bytes::integer, bytes::size>(light);
+            sink.write_little_endian<bytes::integer, bytes::size>(heavy);
             sink.write_little_endian<uint32_t>(tx.locktime());
             sink.write_little_endian<uint32_t>(tx.version());
             sink.write_little_endian<ix::integer, ix::size>(ins_count);