Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions libraries/chain/block_log.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,22 @@ namespace graphene { namespace chain {
return end_pos + sizeof(uint64_t);
}

/// Header only: unpack the reflected block_header and stop. The transactions that
/// follow it in the stored block are never parsed, which is the whole point —
/// unpacking them costs ~129 KB of heap per unique block and get_block_header
/// does not need them.
void read_block_header(uint64_t pos, block_header& header) const {
const auto file_size = get_mapped_size(block_mapped_file);
FC_ASSERT(file_size > pos);

const auto* ptr = block_mapped_file.data() + pos;
const auto available_size = file_size - pos;
const auto max_block_size = std::min<std::size_t>(available_size, CHAIN_BLOCK_SIZE);

fc::datastream<const char*> ds(ptr, max_block_size);
fc::raw::unpack(ds, header);
}

signed_block read_head() const {
auto pos = get_last_uint64(block_mapped_file);
signed_block block;
Expand Down Expand Up @@ -285,6 +301,26 @@ namespace graphene { namespace chain {
return result;
} FC_LOG_AND_RETHROW() }

optional<block_header> block_log::read_block_header_by_num(uint32_t block_num) const { try {
detail::read_lock lock(my->mutex);
optional<block_header> result;
uint64_t pos = my->get_block_pos(block_num);
if (pos != npos) {
block_header header;
my->read_block_header(pos, header);
// Same self-check as read_block_by_num: a header that names another height
// means the index and the file disagree, and a caller would get a header
// that silently belongs to a different block.
FC_ASSERT(
protocol::block_header::num_from_id(header.previous) + 1 == block_num,
"Wrong block was read from block log (${returned} != ${expected}).",
("returned", protocol::block_header::num_from_id(header.previous) + 1)
("expected", block_num));
result = std::move(header);
}
return result;
} FC_LOG_AND_RETHROW() }

uint64_t block_log::get_block_pos(uint32_t block_num) const {
detail::read_lock lock(my->mutex);
return my->get_block_pos(block_num);
Expand Down
41 changes: 41 additions & 0 deletions libraries/chain/database.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1521,6 +1521,47 @@ namespace graphene { namespace chain {
} FC_LOG_AND_RETHROW()
}

optional<block_header> database::fetch_block_header_by_number(uint32_t block_num) const {
try {
/* Same source order as fetch_block_by_number(): fork_db first (recent blocks
that may not be in the block log yet, including a rival branch that the
block_summary disambiguates), and only then the block log. The difference
is the block_log step: it unpacks the header alone instead of the whole
signed_block, because get_block_header() never looks at transactions and
unpacking them is what costs the node ~129 KB per unique block. */
optional<block_header> h;

auto results = _fork_db.fetch_block_by_number(block_num);
if (results.size() > 1) {
const auto *bs = find<block_summary_object, by_id>(block_summary_id_type(block_num & 0xFFFF));
if (bs && protocol::block_header::num_from_id(bs->block_id) == block_num)
for (const auto &r : results)
if (r->id == bs->block_id) { h = static_cast<const block_header&>(r->data); break; }
}
if (!h.valid() && results.size() == 1) {
h = static_cast<const block_header&>(results[0]->data);
} else if (!h.valid()) {
if (_fork_db.head()) {
auto fitem = _fork_db.walk_main_branch_to_num(block_num);
if (fitem && !fitem->anchor_only) {
h = static_cast<const block_header&>(fitem->data);
}
}
if (!h.valid()) {
h = _block_log.read_block_header_by_num(block_num);
// DLT rolling block_log fallback keeps the full-block path: the DLT log
// has no header-only read and is bounded to _dlt_block_log_max_blocks.
if (!h.valid()) {
auto full = _dlt_block_log.read_block_by_num(block_num);
if (full.valid()) h = static_cast<const block_header&>(*full);
}
}
}

return h;
} FC_LOG_AND_RETHROW()
}

const signed_transaction database::get_recent_transaction(const transaction_id_type &trx_id) const {
try {
auto &index = get_index<transaction_index>().indices().get<by_trx_id>();
Expand Down
13 changes: 13 additions & 0 deletions libraries/chain/include/graphene/chain/block_log.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,19 @@ namespace graphene {

optional <signed_block> read_block_by_num(uint32_t block_num) const;

/**
* Read only the header of a block, without deserializing its transactions.
*
* The on-wire layout is positional and the header fields come first
* (FC_REFLECT_DERIVED puts base members first), so the stored block can be
* unpacked as block_header and the parser stops right after the header. Callers
* that only need the header (get_block_header) must use this: unpacking a
* whole signed_block costs ~129 KB of heap per unique block, which is not
* returned to the OS and is enough to OOM a node that serves block
* enumeration.
*/
optional <block_header> read_block_header_by_num(uint32_t block_num) const;

/**
* Return offset of block in file, or block_log::npos if it does not exist.
*/
Expand Down
6 changes: 6 additions & 0 deletions libraries/chain/include/graphene/chain/database.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,12 @@ namespace graphene { namespace chain {

optional<signed_block> fetch_block_by_number(uint32_t num) const;

/// Same lookup as fetch_block_by_number(), but reads only the header from the
/// block log. Serving get_block_header through the full-block path costs
/// ~129 KB of heap per unique block (the transactions are unpacked and thrown
/// away), which a client enumerating blocks turns into gigabytes of RSS.
optional<block_header> fetch_block_header_by_number(uint32_t num) const;

/// Return only the applied-chain block at this height, or nothing if its identity is unknown.
optional<signed_block> fetch_applied_block_by_number(uint32_t num) const;

Expand Down
7 changes: 6 additions & 1 deletion plugins/database_api/api.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,12 @@ DEFINE_API(plugin, get_block_header) {
}

optional<block_header> plugin::api_impl::get_block_header(uint32_t block_num) const {
auto result = database().fetch_block_by_number(block_num);
/* Header-only lookup. This used to go through fetch_block_by_number(), which unpacks
the whole signed_block and returns its base as the header; the transactions that
were parsed and dropped cost ~129 KB of heap per unique block, and a client that
enumerates blocks turns that into gigabytes of RSS on a public node. Measured 19.11.2026: 245-249 MiB per 2000 fresh
blocks, identical with thread pool 8 and 256 and with prediction_market_api off. */
auto result = database().fetch_block_header_by_number(block_num);
if (result) {
return *result;
}
Expand Down
10 changes: 10 additions & 0 deletions tests/chain/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,13 @@ target_include_directories(chain_exit_threshold_tests PRIVATE
"${CMAKE_CURRENT_SOURCE_DIR}/../../libraries/protocol/include"
"${CMAKE_CURRENT_SOURCE_DIR}/../../libraries/chain/include")
add_test(NAME chain_exit_threshold COMMAND chain_exit_threshold_tests)

# Header-only unpack premise: pins that a stored block can be unpacked as block_header and
# that the parser stops before the transactions (the validity argument of the header-only
# get_block_header path). Needs fc for serialization, not the chain library.
add_executable(chain_block_header_unpack_tests block_header_unpack_test.cpp)
target_link_libraries(chain_block_header_unpack_tests PRIVATE graphene_protocol fc Boost::unit_test_framework)
target_include_directories(chain_block_header_unpack_tests PRIVATE
"${CMAKE_CURRENT_SOURCE_DIR}/../../libraries/protocol/include"
"${CMAKE_CURRENT_SOURCE_DIR}/../../libraries/chain/include")
add_test(NAME chain_block_header_unpack COMMAND chain_block_header_unpack_tests)
73 changes: 73 additions & 0 deletions tests/chain/block_header_unpack_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// Premise of the header-only block read used by database_api.get_block_header():
// a stored block can be unpacked as `block_header` and the parser stops right after the
// header, because FC_REFLECT_DERIVED puts the base members first on the wire.
//
// Why this test exists. Reading a whole signed_block to return its header costs ~129 KB
// of heap per unique block and is not returned to the OS (measured 19.11.2026: 245-249 MiB
// per 2000 fresh blocks, identical on thread pools of 8 and 256 and with the PM plugin off).
// A client enumerating blocks turned that into gigabytes of RSS on api.viz.world.
// The optimization is only valid while the wire order holds, and a change of the block
// layout would keep the code compiling while making get_block_header() return garbage for
// every field after the first divergence. Hence this pin.
#define BOOST_TEST_MODULE chain_block_header_unpack
#include <boost/test/unit_test.hpp>

#include <graphene/protocol/block.hpp>

#include <fc/io/raw.hpp>

using namespace graphene::protocol;

namespace {

/// A block whose three empty transactions form a tail the header-only read must NOT consume.
/// The height is irrelevant here (nothing looks at block_num), so `previous` stays zeroed.
signed_block make_block(uint32_t stamp, const std::string& validator) {
signed_block b;
b.previous = block_id_type();
b.timestamp = fc::time_point_sec(1700000000 + stamp);
b.validator = validator;
b.transactions.resize(3); // three empty transactions = a tail the header read must skip
return b;
}

} // namespace

BOOST_AUTO_TEST_CASE(header_only_unpack_reproduces_every_header_field) {
const signed_block b = make_block(1, "hub.viz.world");

const std::vector<char> wire = fc::raw::pack(b);
fc::datastream<const char*> ds(wire.data(), wire.size());
block_header header;
fc::raw::unpack(ds, header);

BOOST_CHECK(header.previous == b.previous);
BOOST_CHECK(header.timestamp == b.timestamp);
BOOST_CHECK_EQUAL(header.validator, b.validator);
BOOST_CHECK(header.transaction_merkle_root == b.transaction_merkle_root);
BOOST_CHECK_EQUAL(header.extensions.size(), b.extensions.size());
}

BOOST_AUTO_TEST_CASE(header_only_unpack_stops_before_the_transactions) {
// The point of the change: the parser must not walk the tail. If this ever stops
// holding, the "optimization" silently reads the whole block again and the 129 KB
// per block comes back — with the same green tests as before.
const signed_block empty = make_block(2, "a");
signed_block with_tail = make_block(3, "a");
with_tail.transactions.resize(50);

const std::vector<char> wire_empty = fc::raw::pack(empty);
const std::vector<char> wire_tail = fc::raw::pack(with_tail);
BOOST_REQUIRE(wire_tail.size() > wire_empty.size());

fc::datastream<const char*> ds_empty(wire_empty.data(), wire_empty.size());
fc::datastream<const char*> ds_tail(wire_tail.data(), wire_tail.size());
block_header h_empty, h_tail;
fc::raw::unpack(ds_empty, h_empty);
fc::raw::unpack(ds_tail, h_tail);

// Same header, different tails: the number of bytes consumed must be identical and
// must be strictly smaller than the whole block.
BOOST_CHECK_EQUAL(ds_empty.tellp(), ds_tail.tellp());
BOOST_CHECK(ds_tail.tellp() < (long)wire_tail.size());
}