diff --git a/nucleus/tile/Cache.h b/nucleus/tile/Cache.h index ab5c858f..1322d99d 100644 --- a/nucleus/tile/Cache.h +++ b/nucleus/tile/Cache.h @@ -1,6 +1,7 @@ /***************************************************************************** * AlpineMaps.org * Copyright (C) 2023 Adam Celarek + * Copyright (C) 2026 Gerald Kimmersdorfer * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -27,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -49,6 +51,11 @@ constexpr auto serialize(auto & archive, glm::vec<2, T> & vec) namespace nucleus::tile { +// The tile cache is written sequentially leaving dead bytes. But when the blob file +// contains disk_cache_compaction_threshold multiple of alive bytes we completely rewrite +// the file on disk +constexpr double disk_cache_compaction_threshold = 2.0; + /// This class is thread safe. be careful with the visit method as it writes the cache and therefore locks an internal mutex. template class Cache @@ -56,6 +63,8 @@ class Cache struct MetaData { uint64_t visited; uint64_t created; + uint64_t offset; // byte offset in the tile_cache file + uint64_t length; // length in bytes of this tile }; struct CacheObject { @@ -88,11 +97,7 @@ class Cache const VisitorFunction& functor, uint64_t visited_stamp); // must stay private or protected by mutex - static std::filesystem::path tile_path(const std::filesystem::path& base_path, const tile::Id& id) - { - std::string tile_name = std::to_string(id.zoom_level) + "_" + std::to_string(id.coords.x) + "_" + std::to_string(id.coords.y) + ".alp_tile"; - return base_path / tile_name; - } + static std::filesystem::path blob_path(const std::filesystem::path& base_path) { return base_path / "tile_cache.alp"; } static std::filesystem::path meta_info_path(const std::filesystem::path& base_path) { @@ -138,16 +143,19 @@ template std::expected Cache::write_to_disk(cons const auto unexpected_error = [](const auto& e) { return std::unexpected(QString::fromStdString(std::make_error_code(e).message())); }; static_assert(SerialisableTile); std::filesystem::create_directories(base_path); - std::unordered_map data; + // We create a vector instead of a map as its only ever iterated + std::vector> data; { auto locker = std::scoped_lock(m_data_mutex); - data = m_data; // copies only metadata and references to tiles + data.reserve(m_data.size()); + for (const auto& item : m_data) + data.emplace_back(item.first, item.second); // copies metadata and references to tiles } auto locker = std::scoped_lock(m_disk_cached_mutex); - const auto write = [](const auto& bytes, const auto& path) -> std::expected { + const auto write = [](const auto& bytes, const auto& path, QIODeviceBase::OpenMode mode) -> std::expected { QFile file(path); - const auto success = file.open(QIODeviceBase::WriteOnly); + const auto success = file.open(mode); if (!success) return std::unexpected(QString("Couldn't open file '%1' for writing!").arg(QString::fromStdString(path.string()))); file.write(bytes.data(), qint64(bytes.size())); @@ -158,65 +166,99 @@ template std::expected Cache::write_to_disk(cons std::swap(m_disk_cached, disk_cached_old); m_disk_cached.reserve(data.size()); - // removing disk cache items, that were removed or updated in ram - for (const auto& item : disk_cached_old) { - const tile::Id& id = item.first; - const MetaData& meta = item.second; - if (data.contains(id) && data.at(id).meta.created == meta.created) { - continue; - } - std::filesystem::remove(tile_path(base_path, id)); - } - - // write new or updated items to disk - for (const auto& item : data) { - const tile::Id& id = item.first; - const CacheObject& cache_object = item.second; - m_disk_cached[id] = cache_object.meta; - - if (disk_cached_old.contains(id) && disk_cached_old.at(id).created == cache_object.meta.created) - continue; - - std::vector bytes; - zpp::bits::out out(bytes); - const std::remove_cvref_t version = T::version_information; + // Decides on whether its time to compact the cache (deletes dead bytes) + const auto blob = blob_path(base_path); + const uint64_t old_blob_size = std::filesystem::exists(blob) ? uint64_t(std::filesystem::file_size(blob)) : 0; + uint64_t old_live_bytes = 0; + for (const auto& item : disk_cached_old) + old_live_bytes += item.second.length; + const uint64_t dead_bytes = old_blob_size > old_live_bytes ? old_blob_size - old_live_bytes : 0; + const bool compact = old_live_bytes > 0 && double(dead_bytes) > disk_cache_compaction_threshold * double(old_live_bytes); + + // We serialize all (updated) tiles into memory first to execute only one draw command + const auto write_tiles_to_blob + = [&](const std::vector>& items, uint64_t base_offset, QIODeviceBase::OpenMode mode) -> std::expected { + if (items.empty()) + return uint64_t(0); + std::vector buffer; + // IMPORTANT: zpp::bits::no_fit_size necessary, otherwise zpp_bits shrink the buffer for every single + // tile and we would have full reallocations for the tile count which destroys the whole + // gain we get from serializing them into memory first... + zpp::bits::out out(buffer, zpp::bits::no_fit_size {}); + for (const auto& [id, cache_object] : items) { + const uint64_t offset = base_offset + uint64_t(out.position()); { - const auto r = out(version); + const auto r = out(T::version_information); if (failure(r)) return unexpected_error(r); } { - const auto r = out(cache_object.data); + const auto r = out(cache_object->data); if (failure(r)) return unexpected_error(r); } - { - const auto r = write(bytes, tile_path(base_path, id)); - if (!r.has_value()) - return r; - } + m_disk_cached[id] = { cache_object->meta.visited, cache_object->meta.created, offset, base_offset + uint64_t(out.position()) - offset }; } + const uint64_t size = uint64_t(out.position()); + QFile blob_file(blob); + if (!blob_file.open(mode)) + return std::unexpected(QString("Couldn't open file '%1' for writing!").arg(QString::fromStdString(blob.string()))); + blob_file.write(buffer.data(), qint64(size)); + return size; + }; - std::vector bytes; - zpp::bits::out out(bytes); - const std::remove_cvref_t version = T::version_information; - { - const auto r = out(version); - if (failure(r)) - return unexpected_error(r); - } + if (compact) { + std::vector> items; + items.reserve(data.size()); + for (const auto& item : data) + items.emplace_back(item.first, &item.second); - { - const auto r = out(m_disk_cached); - if (failure(r)) - return unexpected_error(r); + const auto r = write_tiles_to_blob(items, 0, QIODeviceBase::WriteOnly); // WriteOnly truncates by default + if (!r.has_value()) + return std::unexpected(r.error()); + } else { + std::vector> items; + for (const auto& item : data) { + const tile::Id& id = item.first; + const CacheObject& cache_object = item.second; + + const auto old_entry = disk_cached_old.find(id); + if (old_entry != disk_cached_old.end() && old_entry->second.created == cache_object.meta.created) { + // unchanged payload: keep the location but refresh marker + auto entry = old_entry->second; + entry.visited = cache_object.meta.visited; + entry.created = cache_object.meta.created; + m_disk_cached.emplace(id, entry); + continue; + } + + items.emplace_back(id, &cache_object); } - const auto r = write(bytes, meta_info_path(base_path)); + const auto r = write_tiles_to_blob(items, old_blob_size, QIODeviceBase::Append); if (!r.has_value()) - return r; + return std::unexpected(r.error()); + } - return {}; + std::vector bytes; + zpp::bits::out out(bytes); + { + const auto r = out(T::version_information); + if (failure(r)) + return unexpected_error(r); + } + + { + const auto r = out(m_disk_cached); + if (failure(r)) + return unexpected_error(r); + } + + const auto r = write(bytes, meta_info_path(base_path), QIODeviceBase::WriteOnly); + if (!r.has_value()) + return r; + + return {}; } template std::expected Cache::read_from_disk(const std::filesystem::path& base_path) @@ -225,17 +267,16 @@ template std::expected Cache::read_from_disk(con auto locker = std::scoped_lock(m_data_mutex, m_disk_cached_mutex); Q_ASSERT(SerialisableTile); const auto check_version = [&unexpected_error](auto* in, const auto& path) -> std::expected { - std::remove_cvref_t version_info = {}; + auto version_info = T::version_information; { const auto r = (*in)(version_info); if (failure(r)) return unexpected_error(r); } if (version_info != T::version_information) { - version_info[version_info.size() - 1] = 0; // make sure that the string is 0 terminated. + version_info[version_info.size() - 1] = 0; // check 0 termination of string - return std::unexpected(QString("Cache file '%1' has incompatible version! Disk " - "version is '%2', but we expected '%3'.") + return std::unexpected(QString("File '%1' has incompatible version! ('%2', expected '%3')") .arg(QString::fromStdString(path.string())) .arg(version_info.data()) .arg(T::version_information.data())); @@ -279,19 +320,31 @@ template std::expected Cache::read_from_disk(con } } - for (const auto& entry : m_disk_cached) { - const tile::Id& id = entry.first; - const MetaData& meta = entry.second; + const auto blob = blob_path(base_path); + const auto blob_bytes = read_all(blob); + if (!blob_bytes.has_value()) { + clean_up(); + return std::unexpected(blob_bytes.error()); + } - const auto path = tile_path(base_path, id); - const auto bytes = read_all(path); - if (!bytes.has_value()) { + // Reading in blob order keeps the walk through the buffer sequential for better cache alignment + std::vector entries; + entries.reserve(m_disk_cached.size()); + for (const auto& entry : m_disk_cached) + entries.push_back(entry.second); + std::sort(entries.begin(), entries.end(), [](const MetaData& a, const MetaData& b) { return a.offset < b.offset; }); + m_data.reserve(entries.size()); + + for (const MetaData& meta : entries) { + if (meta.offset + meta.length > uint64_t(blob_bytes->size())) { clean_up(); - return std::unexpected(bytes.error()); + return std::unexpected(QString("Tile cache blob %1 is smaller than expected (corrupt or truncated).") + .arg(QString::fromStdString(blob.string()))); } - zpp::bits::in in(bytes.value()); + + zpp::bits::in in(std::span(blob_bytes->constData() + meta.offset, size_t(meta.length))); { - const auto r = check_version(&in, path); + const auto r = check_version(&in, blob); if (!r.has_value()) { clean_up(); return r; @@ -307,7 +360,8 @@ template std::expected Cache::read_from_disk(con } } d.meta = meta; - m_data[d.data.id] = d; + const auto id = d.data.id; + m_data.emplace(id, std::move(d)); } return {}; diff --git a/unittests/nucleus/tile_cache.cpp b/unittests/nucleus/tile_cache.cpp index 787a1fbf..a1b9ea39 100644 --- a/unittests/nucleus/tile_cache.cpp +++ b/unittests/nucleus/tile_cache.cpp @@ -1,6 +1,7 @@ /***************************************************************************** * AlpineMaps.org * Copyright (C) 2023 Adam Celarek + * Copyright (C) 2026 Gerald Kimmersdorfer * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -479,4 +480,42 @@ TEST_CASE("nucleus/tile/cache") } std::filesystem::remove_all(path); } + + SECTION("reading fails cleanly when the blob was truncated (e.g. by a crash mid-write)") + { + const auto path = std::filesystem::path(QStandardPaths::writableLocation(QStandardPaths::CacheLocation).toStdString()) / "test_tile_cache"; + std::filesystem::remove_all(path); + { + Cache cache; + cache.insert(create_test_tile({ 0, { 0, 0 } })); + cache.insert(create_test_tile({ 1, { 0, 0 } })); + CHECK(cache.write_to_disk(path).has_value()); + } + std::filesystem::resize_file(path / "tile_cache.alp", 1); // simulate a crash that cut the blob short + { + Cache cache; + CHECK(!cache.read_from_disk(path).has_value()); + CHECK(cache.n_cached_objects() == 0); + } + std::filesystem::remove_all(path); + } + + SECTION("reading fails cleanly when the index was truncated (e.g. by a crash mid-write)") + { + const auto path = std::filesystem::path(QStandardPaths::writableLocation(QStandardPaths::CacheLocation).toStdString()) / "test_tile_cache"; + std::filesystem::remove_all(path); + { + Cache cache; + cache.insert(create_test_tile({ 0, { 0, 0 } })); + cache.insert(create_test_tile({ 1, { 0, 0 } })); + CHECK(cache.write_to_disk(path).has_value()); + } + std::filesystem::resize_file(path / "meta_info.alp", 3); // simulate a crash mid-write + { + Cache cache; + CHECK(!cache.read_from_disk(path).has_value()); + CHECK(cache.n_cached_objects() == 0); + } + std::filesystem::remove_all(path); + } }