diff --git a/bench/Bench.hs b/bench/Bench.hs index b249879066..e5853478b6 100644 --- a/bench/Bench.hs +++ b/bench/Bench.hs @@ -12,6 +12,7 @@ module Main (main) where -- import Chainweb.Pact.Backend.Bench qualified as Checkpointer -- import Chainweb.Pact.Backend.ForkingBench qualified as ForkingBench import Chainweb.MempoolBench qualified as MempoolBench +import Chainweb.BlockHeaderDBBench qualified as BlockHeaderDBBench import Chainweb.CutDBBench qualified as CutDBBench -- import Chainweb.Pact.Backend.PactService qualified as PactService -- import Chainweb.Storage.Table.RocksDB (withTempRocksDb) @@ -24,6 +25,7 @@ main :: IO () main = defaultMain [ JSONEncoding.benchmarks , MempoolBench.bench + , BlockHeaderDBBench.bench , CutDBBench.bench ] diff --git a/bench/Chainweb/BlockHeaderDBBench.hs b/bench/Chainweb/BlockHeaderDBBench.hs new file mode 100644 index 0000000000..7d15a3bf7d --- /dev/null +++ b/bench/Chainweb/BlockHeaderDBBench.hs @@ -0,0 +1,142 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE ImportQualifiedPost #-} +{-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +-- | +-- Module: Chainweb.BlockHeaderDBBench +-- Copyright: Copyright © 2025 Kadena LLC. +-- License: MIT +-- +-- Benchmarks for 'BlockHeaderDb' queries, in particular the 'TreeDb' +-- branch traversal code paths that back header synchronization. +-- +module Chainweb.BlockHeaderDBBench (bench) where + +import Control.Exception +import Control.Monad +import Control.Monad.IO.Class (liftIO) +import Control.Monad.Trans.Resource (InternalState, ResourceT) +import Control.Monad.Trans.Resource qualified as ResourceT + +import Criterion.Main qualified as C + +import Data.HashSet qualified as HS + +import Numeric.Natural + +import Streaming.Prelude qualified as S + +-- internal modules + +import Chainweb.BlockHeader +import Chainweb.BlockHeaderDB +import Chainweb.Test.Utils +import Chainweb.TreeDB +import Chainweb.Utils +import Chainweb.Utils.Bench +import Chainweb.Version + +-- -------------------------------------------------------------------------- -- +-- Fixture + +-- | A linear trunk of this many blocks on top of genesis. +trunkLength :: Natural +trunkLength = 1_000 + +-- | The fork branches off from the trunk at this height. +forkStartHeight :: Natural +forkStartHeight = 950 + +-- | The fork extends this many blocks beyond the fork point, so the fork tip +-- has the same height as the trunk tip. +forkLength :: Natural +forkLength = 50 + +data BenchDb = BenchDb + { _bdbDb :: !BlockHeaderDb + , _bdbTrunkTip :: !BlockHeader + , _bdbForkPoint :: !BlockHeader + , _bdbForkTip :: !BlockHeader + } + +-- | Keep a 'ResourceT' allocation alive until Criterion invokes the matching +-- cleanup action. +acquire :: ResourceT IO a -> IO (NoopNFData (a, InternalState)) +acquire action = mask $ \restore -> do + state <- ResourceT.createInternalState + result <- restore (ResourceT.runInternalState action state) + `onException` ResourceT.closeInternalState state + return $ NoopNFData (result, state) + +release :: NoopNFData (a, InternalState) -> IO () +release (NoopNFData (_, state)) = ResourceT.closeInternalState state + +benchDbResource :: HasVersion => ResourceT IO BenchDb +benchDbResource = do + rocksDb <- withRocksResource + db <- withTestBlockHeaderDb rocksDb genesis + trunk <- liftIO $ insertN_ (Nonce 0) trunkLength genesis db + let trunkTip = last trunk + forkPoint = trunk !! (int forkStartHeight - 1) + fork <- liftIO $ insertN_ (Nonce 1) forkLength forkPoint db + return $ BenchDb db trunkTip forkPoint (last fork) + where + genesis = genesisBlockHeader toyChainId + +-- -------------------------------------------------------------------------- -- +-- Benchmarks + +bench :: C.Benchmark +bench = withVersion toyVersion + $ C.envWithCleanup (acquire benchDbResource) release + $ \ ~(NoopNFData (bdb, _)) -> benchmarks bdb + +benchmarks :: HasVersion => BenchDb -> C.Benchmark +benchmarks ~(BenchDb db trunkTip forkPoint forkTip) = C.bgroup "block-header-db" + [ C.bgroup "getBranch" + [ C.bench "ancestors/trunk-tip (1000 blocks)" + $ C.nfIO + $ S.length_ (ancestors db (key trunkTip)) + , C.bench "branchEntries/page (limit 250, maxRank=tip)" + $ C.nfIO + $ branchEntries db Nothing (Just 250) Nothing (Just tipRank) + mempty (upper trunkTip) + $ S.length_ . void + , C.bench "branchEntries/page (limit 250, maxRank=tip, lower=fork-point)" + $ C.nfIO + $ branchEntries db Nothing (Just 250) Nothing (Just tipRank) + (lower forkPoint) (upper trunkTip) + $ S.length_ . void + , C.bench "branchEntries/page (limit 250, maxRank=tip-500)" + $ C.nfIO + $ branchEntries db Nothing (Just 250) Nothing (Just midRank) + mempty (upper trunkTip) + $ S.length_ . void + ] + , C.bgroup "seekAncestor" + [ C.bench "unique rank (tip - 500)" + $ C.nfIO + $ seekAncestor db trunkTip (rank trunkTip - 500) + , C.bench "fork region (tip - 25)" + $ C.nfIO + $ seekAncestor db trunkTip (rank trunkTip - 25) + ] + , C.bgroup "fork" + [ C.bench "forkEntry (50 block fork)" + $ C.nfIO + $ forkEntry db trunkTip forkTip + , C.bench "branchDiff (50 block fork)" + $ C.nfIO + $ S.length_ (branchDiff db trunkTip forkTip) + , C.bench "getBranchIncreasing (last 100 blocks)" + $ C.nfIO + $ getBranchIncreasing db trunkTip (rank trunkTip - 100) S.length_ + ] + ] + where + tipRank = int (rank trunkTip) + midRank = int (rank trunkTip - 500) + upper = HS.singleton . UpperBound . key + lower = HS.singleton . LowerBound . key diff --git a/cabal.project b/cabal.project index b3ff1dbe2e..4111df51df 100644 --- a/cabal.project +++ b/cabal.project @@ -116,8 +116,8 @@ source-repository-package source-repository-package type: git location: https://github.com/kda-community/rocksdb-haskell.git - tag: cede9de2932a4ead1bd82fd7709b19ab7b19b33d - --sha256: 1dngd44va6h66vwpdpwmnj0zcky87m4vzykjwv49p2km12lwq9mf + tag: 177187da7941d1b85afebaa5b278fbd5d017e182 + --sha256: 037cr62rj86k037hs1pphdnpc4dg12nvdy53az9xndxk0msrl9ry source-repository-package type: git diff --git a/chainweb.cabal b/chainweb.cabal index 95bde3e314..eefd15dbc9 100644 --- a/chainweb.cabal +++ b/chainweb.cabal @@ -674,6 +674,7 @@ test-suite chainweb-tests Chainweb.Test.BlockHeader.Validation Chainweb.Test.BlockHeaderDB Chainweb.Test.BlockHeaderDB.PruneForks + Chainweb.Test.BlockHeaderDBBranch Chainweb.Test.Chainweb.Utils.Paging Chainweb.Test.CutDB Chainweb.Test.Difficulty @@ -930,6 +931,7 @@ benchmark bench main-is: Bench.hs type: exitcode-stdio-1.0 other-modules: + Chainweb.BlockHeaderDBBench Chainweb.CutDBBench Chainweb.MempoolBench -- Chainweb.Pact.Backend.ApplyCmd diff --git a/libs/chainweb-storage/src/Chainweb/Storage/DedupStore.hs b/libs/chainweb-storage/src/Chainweb/Storage/DedupStore.hs index 2d3ddffe21..11e01290e5 100644 --- a/libs/chainweb-storage/src/Chainweb/Storage/DedupStore.hs +++ b/libs/chainweb-storage/src/Chainweb/Storage/DedupStore.hs @@ -137,6 +137,7 @@ newDedupStore rdb vc kc n = DedupStore unless (B.length l == dedupHashSize) $ throwM $ DedupStoreCorruptedData "Internal key has wrong size" return (DedupHash l) + , _codecUnsafeIter = False } dedupChunkCodec = Codec @@ -148,6 +149,7 @@ newDedupStore rdb vc kc n = DedupStore | B.length c < dedupHashSize -> throwM $ DedupStoreCorruptedData "Got too short chunk" | otherwise -> return $ Chunk a (DedupHash c) d + , _codecUnsafeIter = False } -- | Insert a new items into a 'DedupStore'. diff --git a/libs/chainweb-storage/src/Chainweb/Storage/Table/RocksDB.hs b/libs/chainweb-storage/src/Chainweb/Storage/Table/RocksDB.hs index 7cd49aca0d..887162f694 100644 --- a/libs/chainweb-storage/src/Chainweb/Storage/Table/RocksDB.hs +++ b/libs/chainweb-storage/src/Chainweb/Storage/Table/RocksDB.hs @@ -297,6 +297,21 @@ data Codec a = Codec -- ^ encode a value. , _codecDecode :: !(forall m. MonadThrow m => B.ByteString -> m a) -- ^ decode a value. Throws an exception of decoding fails. + , _codecUnsafeIter :: !Bool + -- ^ Whether the decode function is compatible with the zero-copy + -- 'Database.RocksDB.Iterator.unsafeIterKey' and + -- 'Database.RocksDB.Iterator.unsafeIterValue' iterators from + -- rocksdb-haskell. + -- + -- Those iterators wrap the RocksDB-owned buffer directly instead of + -- copying it out and only force the result of the decode to weak head + -- normal form. Using them is therefore only sound when the decode + -- function never retains a reference to (never aliases) its input + -- 'B.ByteString' -- not even inside an unevaluated thunk. Only set this + -- to 'True' for codecs whose decode produces a value that is fully + -- independent of the input bytes (for instance decoders that copy the + -- input or build fresh values from it). When in doubt, leave it + -- 'False', in which case the (copying) iterators are used instead. } instance NoThunks (Codec a) where @@ -428,21 +443,46 @@ instance Iterator (RocksDbTableIter k v) k v where iterPrev = I.iterPrev . _rocksDbTableIter - iterEntry it = - I.iterEntry (_rocksDbTableIter it) >>= \case - Nothing -> return Nothing - Just (k, v) -> do - k' <- decIterKey it k - v' <- decIterVal it v - return $! Just $! Entry k' v' - - iterValue it = I.iterValue (_rocksDbTableIter it) >>= \case - Nothing -> return Nothing - Just v -> Just <$> (evaluate =<< decIterVal it v) - - iterKey it = I.iterKey (_rocksDbTableIter it) >>= \case - Nothing -> return Nothing - Just k -> Just <$> (evaluate =<< decIterKey it k) + iterEntry it + | _codecUnsafeIter (_rocksDbTableIterKeyCodec it) + , _codecUnsafeIter (_rocksDbTableIterValueCodec it) = + I.unsafeIterKey (_rocksDbTableIter it) (decIterKeyE it) >>= \case + Nothing -> return Nothing + Just (Left e) -> throwM e + Just (Right k) -> + I.unsafeIterValue (_rocksDbTableIter it) (decIterValE it) >>= \case + Nothing -> return Nothing + Just (Left e) -> throwM e + Just (Right v) -> return $! Just $! Entry k v + | otherwise = + I.iterEntry (_rocksDbTableIter it) >>= \case + Nothing -> return Nothing + Just (k, v) -> do + k' <- decIterKey it k + v' <- decIterVal it v + return $! Just $! Entry k' v' + + iterValue it + | _codecUnsafeIter (_rocksDbTableIterValueCodec it) = + I.unsafeIterValue (_rocksDbTableIter it) (decIterValE it) >>= \case + Nothing -> return Nothing + Just (Left e) -> throwM e + Just (Right v) -> Just <$> evaluate v + | otherwise = + I.iterValue (_rocksDbTableIter it) >>= \case + Nothing -> return Nothing + Just v -> Just <$> (evaluate =<< decIterVal it v) + + iterKey it + | _codecUnsafeIter (_rocksDbTableIterKeyCodec it) = + I.unsafeIterKey (_rocksDbTableIter it) (decIterKeyE it) >>= \case + Nothing -> return Nothing + Just (Left e) -> throwM e + Just (Right k) -> Just <$> evaluate k + | otherwise = + I.iterKey (_rocksDbTableIter it) >>= \case + Nothing -> return Nothing + Just k -> Just <$> (evaluate =<< decIterKey it k) instance IterableTable (RocksDbTable k v) (RocksDbTableIter k v) k v where withTableIterator db k = R.withReadOptions readOptions $ \opts_ptr -> @@ -647,6 +687,23 @@ decIterKey it k = where prefix = namespaceFirst $ _rocksDbTableIterNamespace it +-- | Pure variants of 'decIterKey' and 'decIterVal' that run the codec's decode +-- in the @Either SomeException@ monad. They are used as the (pure) continuation +-- for the zero-copy 'I.unsafeIterKey' / 'I.unsafeIterValue' iterators. +-- +-- These must only be used with codecs that set '_codecUnsafeIter' to 'True', +-- i.e. codecs whose decode never retains a reference to its input +-- 'B.ByteString'. +-- +decIterKeyE :: RocksDbTableIter k v -> B.ByteString -> Either SomeException k +decIterKeyE it k = + _codecDecode (_rocksDbTableIterKeyCodec it) (B.drop (B.length prefix) k) + where + prefix = namespaceFirst $ _rocksDbTableIterNamespace it + +decIterValE :: RocksDbTableIter k v -> B.ByteString -> Either SomeException v +decIterValE i = _codecDecode (_rocksDbTableIterValueCodec i) + checked :: HasCallStack => String -> (Ptr CString -> IO a) -> IO a checked whatWasIDoing act = alloca $ \errPtr -> do poke errPtr (nullPtr :: CString) diff --git a/libs/chainweb-storage/test/Main.hs b/libs/chainweb-storage/test/Main.hs index 8723a79bab..14a7e768a7 100644 --- a/libs/chainweb-storage/test/Main.hs +++ b/libs/chainweb-storage/test/Main.hs @@ -85,6 +85,7 @@ intCodec :: Codec Int intCodec = Codec (B8.pack . show) (either (throwM . CodecException) return . readEither @Int . B8.unpack) + True intTable :: RocksDb -> B8.ByteString -> RocksDbTable Int Int intTable db tableName = newTable db intCodec intCodec [tableName] diff --git a/src/Chainweb/BlockHeaderDB/Internal.hs b/src/Chainweb/BlockHeaderDB/Internal.hs index 8b520a3e70..0605991ef9 100644 --- a/src/Chainweb/BlockHeaderDB/Internal.hs +++ b/src/Chainweb/BlockHeaderDB/Internal.hs @@ -74,9 +74,12 @@ import Control.Monad.Trans.Resource hiding (throwM) import Data.Aeson import Data.Function +import Data.Functor.Of (Of) import Data.Hashable import Data.HashSet qualified as HS +import qualified Data.List as L import Data.Maybe +import Data.Semigroup (Max(..)) import qualified Data.Text.Encoding as T import GHC.Generics (Generic) @@ -104,6 +107,7 @@ import Chainweb.Storage.Table import Chainweb.Storage.Table.RocksDB import Numeric.Additive +import Numeric.Natural (Natural) -- -------------------------------------------------------------------------- -- -- | Configuration of the chain DB. @@ -286,14 +290,14 @@ mkBlockHeaderDb rdb cid = db where headerTable = newTable rdb - (Codec (runPutS . encodeRankedBlockHeader) (runGetS decodeRankedBlockHeader)) - (Codec (runPutS . encodeRankedBlockHash) (runGetS decodeRankedBlockHash)) + (Codec (runPutS . encodeRankedBlockHeader) (runGetS decodeRankedBlockHeader) True) + (Codec (runPutS . encodeRankedBlockHash) (runGetS decodeRankedBlockHash) True) ["BlockHeader", cidNs, "header"] rankTable = newTable rdb - (Codec (runPutS . encodeBlockHeight) (runGetS decodeBlockHeight)) - (Codec (runPutS . encodeBlockHash) (runGetS decodeBlockHash)) + (Codec (runPutS . encodeBlockHeight) (runGetS decodeBlockHeight) True) + (Codec (runPutS . encodeBlockHash) (runGetS decodeBlockHash) True) ["BlockHeader", cidNs, "rank"] cidNs = T.encodeUtf8 (toText cid) @@ -323,9 +327,6 @@ withBlockHeaderDb db cid = snd <$> allocate start closeBlockHeaderDb -- -------------------------------------------------------------------------- -- -- TreeDB instance --- | TODO provide more efficient branchEntries implementation that uses --- iterators. --- instance HasVersion => TreeDb BlockHeaderDb where type DbEntry BlockHeaderDb = BlockHeader @@ -346,16 +347,55 @@ instance HasVersion => TreeDb BlockHeaderDb where $ f . S.map _getRankedBlockHeader {-# INLINEABLE entries #-} - branchEntries db k l mir mar lower upper f = do - -- we use the ranked implementation as it is more efficient - -- TODO run these queries in parallel or batch them; usually it is a - -- relatively small number. - let mapSetM a = fmap HS.fromList . mapM a . HS.toList - rk <- mapM (mapM (getRankedKey db)) k - rlow <- mapSetM (mapM (getRankedKey db)) lower - rup <- mapSetM (mapM (getRankedKey db)) upper - chainBranchEntries (RankedBlockHeaderDb db) rk l mir mar rlow rup - $ f . S.map _getRankedBlockHeader + -- | This implementation of 'branchEntries' uses iterators over the ranked + -- block header table. + -- + -- The generic implementation ('chainBranchEntries') resolves the branch + -- bounds and then traverses the branches by looking up the parent of each + -- entry with an individual point lookup. Each of these lookups seeks the + -- table from its root and allocates a fresh set of read options. + -- + -- Since the ranked block header table is keyed by block height and block + -- hash, all entries of a block height are stored next to each other. The + -- implementation below uses a single iterator for the entire query and + -- resolves all entries of a block height with seeks into that iterator, + -- which avoids per-lookup seeking overhead. + -- + -- The traversal is equivalent to the one of 'getBranchResolved', which + -- backs the generic implementation. + -- + branchEntries db k l mir mar lower upper f + -- If the upper bounds are empty the result is empty, too. In that case + -- the lower bounds are not even resolved. + | HS.null upper = f (return (0, Eos True)) + | otherwise = do + -- Resolve the upper bounds. Keys that aren't in the database are + -- ignored. + uppers <- resolveBounds db _getUpperBound upper + if HS.null uppers + then f (return (0, Eos True)) + else do + lowers <- resolveBounds db _getLowerBound lower + + -- If a maximum rank is given, the upper bounds are reduced + -- to ancestors of that rank. This limits the depth of the + -- traversal and is essential for the performance of + -- queries that limit the result to a range of ranks that is + -- far below the upper bounds. + uppers' <- case mar of + Nothing -> return uppers + Just (MaxRank (Max m)) -> reduceUpperBounds db m uppers + + if HS.null uppers' + then f (return (0, Eos True)) + else do + let !startRank = int $ maximum $ map rank + $ HS.toList (uppers' <> lowers) + withTableIterator (_chainDbCas db) $ \it -> f $ + branchStream it startRank mempty mempty + (sortByRankDesc uppers') (sortByRankDesc lowers) + & applyRankDesc mir mar + & seekLimitStream key k l {-# INLINEABLE branchEntries #-} keys db k l mir mar f = do @@ -381,6 +421,119 @@ getRankedKey db h = do Just v -> return $ RankedBlockHash (int v) h {-# INLINE getRankedKey #-} +-- -------------------------------------------------------------------------- -- +-- Branch Traversal +-- +-- The following functions implement the 'branchEntries' method of the +-- 'BlockHeaderDb' instance of 'TreeDb'. They are the iterator based equivalent +-- of 'getBranchResolved' and 'chainBranchEntries'. + +-- | Resolve the bounds of a branch query to the corresponding block headers. +-- Keys that aren't in the database are ignored. +-- +resolveBounds + :: HasVersion + => BlockHeaderDb + -> (a -> BlockHash) + -- ^ Projector for the bound keys + -> HS.HashSet a + -> IO (HS.HashSet BlockHeader) +resolveBounds db f = fmap (HS.fromList . catMaybes) + . mapM (lookup db . f) . HS.toList +{-# INLINEABLE resolveBounds #-} + +-- | Replace the upper bounds of a branch query by ancestors of the given rank. +-- Bounds for which no ancestor of that rank exists are dropped. +-- +-- This limits the depth of the branch traversal and corresponds to the +-- optimization that 'chainBranchEntries' applies to the upper bounds. +-- +reduceUpperBounds + :: HasVersion + => BlockHeaderDb + -> Natural + -> HS.HashSet BlockHeader + -> IO (HS.HashSet BlockHeader) +reduceUpperBounds db m = fmap (HS.fromList . catMaybes) + . mapM (\e -> seekAncestor db e m) . HS.toList +{-# INLINEABLE reduceUpperBounds #-} + +-- | Sort block headers by descending rank. +-- +sortByRankDesc :: HasVersion => HS.HashSet BlockHeader -> [BlockHeader] +sortByRankDesc = L.sortBy (\a b -> compare (rank b) (rank a)) . HS.toList +{-# INLINEABLE sortByRankDesc #-} + +-- | Stream the ancestors of the given upper bounds that aren't descendents of +-- the given lower bounds. Entries are streamed in descending order by rank. +-- +-- @r@ is the rank of the entries that are streamed next. The inactive bounds +-- must be sorted by descending rank. Whenever the traversal descends to the +-- rank of a bound, the bound is moved into the corresponding active set. +-- +-- The iterator must remain valid for the lifetime of the returned stream. +-- +branchStream + :: HasVersion + => RocksDbTableIter RankedBlockHash RankedBlockHeader + -> Int + -- ^ Rank of the entries that are streamed next + -> HS.HashSet BlockHeader + -- ^ Upper bounds at the current rank + -> HS.HashSet BlockHeader + -- ^ Lower bounds at the current rank + -> [BlockHeader] + -- ^ Upper bounds of larger rank, sorted by descending rank + -> [BlockHeader] + -- ^ Lower bounds of larger rank, sorted by descending rank + -> S.Stream (Of BlockHeader) IO () +branchStream _ _ upAct _ upInact _ + | HS.null upAct && null upInact = return () +branchStream it r upAct loAct upInact loInact = do + let !r' = int r + (upNew, upInact') = span (\e -> rank e >= r') upInact + (loNew, loInact') = span (\e -> rank e >= r') loInact + upAct' = upAct <> HS.fromList upNew + loAct' = loAct <> HS.fromList loNew + + -- The entries of the upper branches that are not entries of the lower + -- branches. + yielded = upAct' `HS.difference` loAct' + + mapM_ S.yield (HS.toList yielded) + upNext <- liftIO $ branchParents it (pred r) yielded + loNext <- if HS.null loAct' + then return mempty + else liftIO $ branchParents it (pred r) loAct' + branchStream it (pred r) upNext loNext upInact' loInact' + +-- | Lookup the parents of the given block headers. Parents of genesis headers +-- are dropped. A 'TreeDbParentMissing' exception is thrown if a parent isn't in +-- the database. +-- +-- The entries are resolved by seeking the iterator to the key of the parent. +-- +branchParents + :: HasVersion + => RocksDbTableIter RankedBlockHash RankedBlockHeader + -> Int + -- ^ Rank of the parents + -> HS.HashSet BlockHeader + -> IO (HS.HashSet BlockHeader) +branchParents it r es + | null ps = return mempty + | otherwise = fmap HS.fromList $ forM ps $ \(h, e) -> do + iterSeek it (RankedBlockHash (int r) h) + iterValue it >>= \case + Just v + | view blockHash (_getRankedBlockHeader v) == h -> + return $! _getRankedBlockHeader v + _ -> throwM $ TreeDbParentMissing @BlockHeaderDb e + "Chainweb.BlockHeaderDB.Internal.branchParents" + where + ps = [ (p, e) | e <- HS.toList es, Just p <- [parent e] ] +{-# INLINEABLE branchParents #-} + -- -------------------------------------------------------------------------- -- -- TreeDB instance for RankedBlockHeaderDb diff --git a/src/Chainweb/CutDB.hs b/src/Chainweb/CutDB.hs index 57e7902eff..5b6b743d81 100644 --- a/src/Chainweb/CutDB.hs +++ b/src/Chainweb/CutDB.hs @@ -240,8 +240,8 @@ cutHashesTable rdb = Casify $ newTable rdb valueCodec keyCodec ["CutHashes"] where keyCodec = Codec (\(a,b,c) -> runPutS $ encodeCutHeightBe a >> encodeBlockWeightBe b >> encodeCutId c) - (runGetS $ (,,) <$> decodeCutHeightBe <*> decodeBlockWeightBe <*> decodeCutId) - valueCodec = Codec encodeToByteString decodeStrictOrThrow' + (runGetS $ (,,) <$> decodeCutHeightBe <*> decodeBlockWeightBe <*> decodeCutId) True + valueCodec = Codec encodeToByteString decodeStrictOrThrow' True -- -------------------------------------------------------------------------- -- -- Exceptions diff --git a/src/Chainweb/Pact/Payload/PayloadStore/RocksDB.hs b/src/Chainweb/Pact/Payload/PayloadStore/RocksDB.hs index 6e6c94c0d7..cd765f8aa6 100644 --- a/src/Chainweb/Pact/Payload/PayloadStore/RocksDB.hs +++ b/src/Chainweb/Pact/Payload/PayloadStore/RocksDB.hs @@ -39,36 +39,36 @@ newTransactionDb db = TransactionDb where newBlockPayloadHeightsStore :: RocksDbTable BlockPayloadHash BlockHeight newBlockPayloadHeightsStore = newTable db - (Codec (runPutS . encodeBlockHeight) (runGetS decodeBlockHeight)) - (Codec (runPutS . encodeBlockPayloadHash) (runGetS decodeBlockPayloadHash)) + (Codec (runPutS . encodeBlockHeight) (runGetS decodeBlockHeight) True) + (Codec (runPutS . encodeBlockPayloadHash) (runGetS decodeBlockPayloadHash) True) ["BlockPayloadIndex"] newBlockPayloadStore :: RocksDbTable (BlockHeight, BlockPayloadHash) BlockPayload newBlockPayloadStore = newTable db - (Codec encodeBlockPayloads decodeBlockPayloads) + (Codec encodeBlockPayloads decodeBlockPayloads True) (Codec (\(bh, bp) -> runPutS (encodeBlockHeight bh >> encodeBlockPayloadHash bp)) - (runGetS ((,) <$> decodeBlockHeight <*> decodeBlockPayloadHash))) + (runGetS ((,) <$> decodeBlockHeight <*> decodeBlockPayloadHash)) True) ["BlockPayload2"] newBlockTransactionsStore :: RocksDbTable (BlockHeight, BlockTransactionsHash) BlockTransactions newBlockTransactionsStore = newTable db - (Codec encodeBlockTransactions decodeBlockTransactions) + (Codec encodeBlockTransactions decodeBlockTransactions True) (Codec (\(h, hsh) -> runPutS $ encodeBlockHeight h >> encodeBlockTransactionsHash hsh) - (runGetS ((,) <$> decodeBlockHeight <*> decodeBlockTransactionsHash))) + (runGetS ((,) <$> decodeBlockHeight <*> decodeBlockTransactionsHash)) True) ["BlockTransactions2"] oldBlockPayloadStore :: Casify RocksDbTable BlockPayload oldBlockPayloadStore = Casify $ newTable db - (Codec encodeToByteString decodeStrictOrThrow') - (Codec (runPutS . encodeBlockPayloadHash) (runGetS decodeBlockPayloadHash)) + (Codec encodeToByteString decodeStrictOrThrow' True) + (Codec (runPutS . encodeBlockPayloadHash) (runGetS decodeBlockPayloadHash) True) ["BlockPayload"] oldBlockTransactionsStore :: Casify RocksDbTable BlockTransactions oldBlockTransactionsStore = Casify $ newTable db - (Codec encodeToByteString decodeStrictOrThrow') - (Codec (runPutS . encodeBlockTransactionsHash) (runGetS decodeBlockTransactionsHash)) + (Codec encodeToByteString decodeStrictOrThrow' True) + (Codec (runPutS . encodeBlockTransactionsHash) (runGetS decodeBlockTransactionsHash) True) ["BlockTransactions"] newPayloadDb :: RocksDb -> PayloadDb RocksDbTable @@ -82,16 +82,16 @@ newPayloadDb db = PayloadDb (newTransactionDb db) newPayloadCache where oldBlockOutputsTbl :: Casify RocksDbTable BlockOutputs oldBlockOutputsTbl = Casify $ newTable db - (Codec encodeToByteString decodeStrictOrThrow') - (Codec (runPutS . encodeBlockOutputsHash) (runGetS decodeBlockOutputsHash)) + (Codec encodeToByteString decodeStrictOrThrow' True) + (Codec (runPutS . encodeBlockOutputsHash) (runGetS decodeBlockOutputsHash) True) ["BlockOutputs"] newBlockOutputsTbl :: RocksDbTable (BlockHeight, BlockOutputsHash) BlockOutputs newBlockOutputsTbl = newTable db - (Codec encodeBlockOutputs decodeBlockOutputs) + (Codec encodeBlockOutputs decodeBlockOutputs True) (Codec (\(h, hsh) -> runPutS $ encodeBlockHeight h >> encodeBlockOutputsHash hsh) - (runGetS $ (,) <$> decodeBlockHeight <*> decodeBlockOutputsHash)) + (runGetS $ (,) <$> decodeBlockHeight <*> decodeBlockOutputsHash) True) ["BlockOutputs2"] transactionTreeStore :: TransactionTreeStore RocksDbTable @@ -99,16 +99,16 @@ newPayloadDb db = PayloadDb (newTransactionDb db) newPayloadCache where oldTransactionTreeTbl :: Casify RocksDbTable TransactionTree oldTransactionTreeTbl = Casify $ newTable db - (Codec encodeToByteString decodeStrictOrThrow') - (Codec (runPutS . encodeBlockTransactionsHash) (runGetS decodeBlockTransactionsHash)) + (Codec encodeToByteString decodeStrictOrThrow' True) + (Codec (runPutS . encodeBlockTransactionsHash) (runGetS decodeBlockTransactionsHash) True) ["TransactionTree"] newTransactionTreeTbl :: RocksDbTable (BlockHeight, BlockTransactionsHash) TransactionTree newTransactionTreeTbl = newTable db - (Codec encodeTransactionTree decodeTransactionTree) + (Codec encodeTransactionTree decodeTransactionTree True) (Codec (\(h, hsh) -> runPutS $ encodeBlockHeight h >> encodeBlockTransactionsHash hsh) - (runGetS $ (,) <$> decodeBlockHeight <*> decodeBlockTransactionsHash)) + (runGetS $ (,) <$> decodeBlockHeight <*> decodeBlockTransactionsHash) True) ["TransactionTree2"] outputTreeStore :: OutputTreeStore RocksDbTable @@ -116,14 +116,14 @@ newPayloadDb db = PayloadDb (newTransactionDb db) newPayloadCache where oldOutputTreeTbl :: Casify RocksDbTable OutputTree oldOutputTreeTbl = Casify $ newTable db - (Codec encodeToByteString decodeStrictOrThrow') - (Codec (runPutS . encodeBlockOutputsHash) (runGetS decodeBlockOutputsHash)) + (Codec encodeToByteString decodeStrictOrThrow' True) + (Codec (runPutS . encodeBlockOutputsHash) (runGetS decodeBlockOutputsHash) True) ["OutputTree"] newOutputTreeTbl :: RocksDbTable (BlockHeight, BlockOutputsHash) OutputTree newOutputTreeTbl = newTable db - (Codec encodeOutputTree decodeOutputTree) + (Codec encodeOutputTree decodeOutputTree True) (Codec (\(h, hsh) -> runPutS $ encodeBlockHeight h >> encodeBlockOutputsHash hsh) - (runGetS $ (,) <$> decodeBlockHeight <*> decodeBlockOutputsHash)) + (runGetS $ (,) <$> decodeBlockHeight <*> decodeBlockOutputsHash) True) ["OutputTree2"] diff --git a/src/Chainweb/PayloadProvider/EVM/HeaderDB.hs b/src/Chainweb/PayloadProvider/EVM/HeaderDB.hs index 4363d7c021..6169b86594 100644 --- a/src/Chainweb/PayloadProvider/EVM/HeaderDB.hs +++ b/src/Chainweb/PayloadProvider/EVM/HeaderDB.hs @@ -198,8 +198,8 @@ initHeaderDb config = do headerTable = newTable (_configRocksDb config) - (Codec encodeRankedHeader decodeRankedHeader) - (Codec (runPutS . encodeRankedBlockPayloadHash) (runGetS decodeRankedBlockPayloadHash)) + (Codec encodeRankedHeader decodeRankedHeader False) + (Codec (runPutS . encodeRankedBlockPayloadHash) (runGetS decodeRankedBlockPayloadHash) True) ["EvmHeader", cidNs, "header"] !db = HeaderDb diff --git a/src/Chainweb/PayloadProvider/EVM/PayloadDB.hs b/src/Chainweb/PayloadProvider/EVM/PayloadDB.hs index 67dbcf2068..7bfb3197f4 100644 --- a/src/Chainweb/PayloadProvider/EVM/PayloadDB.hs +++ b/src/Chainweb/PayloadProvider/EVM/PayloadDB.hs @@ -197,8 +197,8 @@ initPayloadDb config = do payloadTable = newTable (_configRocksDb config) - (Codec encodeRankedPayload decodeRankedPayload) - (Codec (runPutS . encodeRankedBlockPayloadHash) (runGetS decodeRankedBlockPayloadHash)) + (Codec encodeRankedPayload decodeRankedPayload False) + (Codec (runPutS . encodeRankedBlockPayloadHash) (runGetS decodeRankedBlockPayloadHash) True) ["EvmPayload", cidNs, "payload"] !db = PayloadDb diff --git a/src/Chainweb/PayloadProvider/Minimal/PayloadDB.hs b/src/Chainweb/PayloadProvider/Minimal/PayloadDB.hs index 57b168b4b1..966e474de5 100644 --- a/src/Chainweb/PayloadProvider/Minimal/PayloadDB.hs +++ b/src/Chainweb/PayloadProvider/Minimal/PayloadDB.hs @@ -174,8 +174,8 @@ initPayloadDb config = do payloadTable = newTable (_configRocksDb config) - (Chainweb.Storage.Table.RocksDB.Codec encodeRankedPayload decodeRankedPayload) - (Codec (runPutS . encodeRankedBlockPayloadHash) (runGetS decodeRankedBlockPayloadHash)) + (Chainweb.Storage.Table.RocksDB.Codec encodeRankedPayload decodeRankedPayload True) + (Codec (runPutS . encodeRankedBlockPayloadHash) (runGetS decodeRankedBlockPayloadHash) True) ["MinimalProvider", cidNs, "payload"] !db = PayloadDb diff --git a/src/Chainweb/TreeDB.hs b/src/Chainweb/TreeDB.hs index 29eac6632a..dbb1754884 100644 --- a/src/Chainweb/TreeDB.hs +++ b/src/Chainweb/TreeDB.hs @@ -58,6 +58,7 @@ module Chainweb.TreeDB , ancestors , defaultBranchEntries , chainBranchEntries +, applyRankDesc -- ** Lookups , lookupM @@ -532,16 +533,42 @@ getBranch -> HS.HashSet (UpperBound (DbKey db)) -> S.Stream (Of (DbEntry db)) IO () getBranch db lowerBounds upperBounds = do - lowers <- liftIO $ getEntriesHs _getLowerBound lowerBounds - uppers <- liftIO $ getEntriesHs _getUpperBound upperBounds + uppers <- liftIO $ getEntriesHs db _getUpperBound upperBounds + getBranchResolved db lowerBounds uppers - let mar = getMax $ fromJuste $ - foldMap' (foldMap' (Just . Max . rank)) [lowers, uppers] +-- | Resolve a set of bound keys to the corresponding entries. Missing keys +-- are ignored. +-- +getEntriesHs + :: TreeDb db + => db + -> (a -> DbKey db) + -> HS.HashSet a + -> IO (HS.HashSet (DbEntry db)) +getEntriesHs db f = streamToHashSet_ . lookupStream db . S.map f . S.each - go mar (activate mar lowers mempty) (activate mar uppers mempty) +-- | A variant of 'getBranch' where the upper bounds have already been +-- resolved to entries. +-- +getBranchResolved + :: forall db + . TreeDb db + => db + -> HS.HashSet (LowerBound (DbKey db)) + -> HS.HashSet (DbEntry db) + -> S.Stream (Of (DbEntry db)) IO () +getBranchResolved db lowerBounds uppers + -- If the upper bounds are empty the result is empty, too. In that case + -- we return early, before the lower bounds are resolved. + | HS.null uppers = return () + | otherwise = do + lowers <- liftIO $ getEntriesHs db _getLowerBound lowerBounds + + let mar = getMax $ fromJuste $ + foldMap' (foldMap' (Just . Max . rank)) [lowers, uppers] + + go mar (activate mar lowers mempty) (activate mar uppers mempty) where - getEntriesHs :: (a -> Key (DbEntry db)) -> HS.HashSet a -> IO (HS.HashSet (DbEntry db)) - getEntriesHs f = streamToHashSet_ . lookupStream db . S.map f . S.each getParentsHs = streamToHashSet_ . lookupParentStreamM GenesisParentNone db . S.each -- The newly active members of the set after advancing the highest ones @@ -553,9 +580,11 @@ getBranch db lowerBounds upperBounds = do -> HS.HashSet (DbEntry db) -> HS.HashSet (DbEntry db) -> (HS.HashSet (DbEntry db), HS.HashSet (DbEntry db)) - activate r inactive active = - let (stillInactive, newlyActive) = hsPartition (\x -> rank x < r) inactive - in (stillInactive, active <> newlyActive) + activate r inactive active + | HS.null inactive = (inactive, active) + | otherwise = + let (stillInactive, newlyActive) = hsPartition (\x -> rank x < r) inactive + in (stillInactive, active <> newlyActive) -- | The size of the input sets decreases monotonically. Space complexity is -- linear in the input and constant in the output size. @@ -581,7 +610,9 @@ getBranch db lowerBounds upperBounds = do => (a -> Bool) -> HS.HashSet a -> (HS.HashSet a, HS.HashSet a) - hsPartition p = (HS.fromList *** HS.fromList) . L.partition p . HS.toList + hsPartition p s = (yes, HS.difference s yes) + where + yes = HS.filter p s -- -------------------------------------------------------------------------- -- -- Limiting and Seeking a stream diff --git a/src/Chainweb/WebBlockHeaderDB.hs b/src/Chainweb/WebBlockHeaderDB.hs index 89e7948d27..bde95021ba 100644 --- a/src/Chainweb/WebBlockHeaderDB.hs +++ b/src/Chainweb/WebBlockHeaderDB.hs @@ -139,14 +139,14 @@ mkWebBlockHeaderDb db m = pruneJobTable = newTable db (Chainweb.Storage.Table.RocksDB.Codec (\(l, u) -> runPutS $ encodeBlockHeight l >> encodeBlockHeight u) - (runGetS $ (,) <$> decodeBlockHeight <*> decodeBlockHeight)) - (Chainweb.Storage.Table.RocksDB.Codec (\_ -> mempty) (runGetS (pure ()))) + (runGetS $ (,) <$> decodeBlockHeight <*> decodeBlockHeight) True) + (Chainweb.Storage.Table.RocksDB.Codec (\_ -> mempty) (runGetS (pure ())) True) ["BlockHeader", "prune-job"] highestPrunedTable = newTable db - (Chainweb.Storage.Table.RocksDB.Codec (runPutS . encodeBlockHeight) (runGetS decodeBlockHeight)) - (Chainweb.Storage.Table.RocksDB.Codec (\_ -> mempty) (runGetS (pure ()))) + (Chainweb.Storage.Table.RocksDB.Codec (runPutS . encodeBlockHeight) (runGetS decodeBlockHeight) True) + (Chainweb.Storage.Table.RocksDB.Codec (\_ -> mempty) (runGetS (pure ())) True) ["BlockHeader", "highest-pruned"] getWebBlockHeaderDb diff --git a/test/unit/Chainweb/Test/BlockHeaderDBBranch.hs b/test/unit/Chainweb/Test/BlockHeaderDBBranch.hs new file mode 100644 index 0000000000..af1a9cd347 --- /dev/null +++ b/test/unit/Chainweb/Test/BlockHeaderDBBranch.hs @@ -0,0 +1,272 @@ +{-# LANGUAGE ScopedTypeVariables #-} + +-- | +-- Module: Chainweb.Test.BlockHeaderDBBranch +-- Copyright: Copyright © 2026 Kadena LLC. +-- License: MIT +-- Stability: experimental +-- +-- Tests that the 'branchEntries' implementation of the 'BlockHeaderDb' instance +-- of 'TreeDb' agrees with the generic implementation. +-- +module Chainweb.Test.BlockHeaderDBBranch (tests) where + +import Control.Monad (filterM, forM) +import Control.Monad.IO.Class (liftIO) +import Control.Monad.Trans.Resource + +import Data.Foldable (toList, traverse_) +import qualified Data.HashSet as HS +import qualified Data.List as L +import Data.Semigroup (Max(..), Min(..)) +import Data.Tree (Tree(..)) + +import Numeric.Natural (Natural) + +import qualified Streaming.Prelude as S + +import Test.QuickCheck + ( Arbitrary(..) + , Gen + , Property + , arbitrary + , choose + , elements + , forAll + , frequency + , ioProperty + , resize + , shrinkList + , vectorOf + ) +import Test.Tasty +import Test.Tasty.HUnit +import Test.Tasty.QuickCheck (testProperty) + +-- internal modules + +import Chainweb.BlockHash (BlockHash, nullBlockHash) +import Chainweb.BlockHeader +import Chainweb.BlockHeaderDB +import Chainweb.BlockHeaderDB.Internal (unsafeInsertBlockHeaderDb) +import Chainweb.Test.Utils +import Chainweb.TreeDB +import Chainweb.Utils (int) +import Chainweb.Utils.Paging +import Chainweb.Version (HasVersion, withVersion) + +import Chainweb.Storage.Table.RocksDB + +-- | The parameters of a branch query +data Query = Query + { _qNext :: Maybe (NextItem BlockHash) + , _qLimit :: Maybe Limit + , _qMinRank :: Maybe MinRank + , _qMaxRank :: Maybe MaxRank + , _qLower :: HS.HashSet (LowerBound BlockHash) + , _qUpper :: HS.HashSet (UpperBound BlockHash) + } deriving (Show) + +-- | Generate a query for the headers of the given tree +genQuery :: HasVersion => Tree BlockHeader -> Gen Query +genQuery t = do + let hs = map key (toList t) + upper <- genSubset hs + lower <- genSubset hs + nxt <- frequency [(1, pure Nothing), (2, Just <$> (wrapNext =<< elements hs))] + Query nxt + <$> genMaybe (nat 12) + <*> genMaybe (minRank 20) + <*> genMaybe (maxRank 20) + <*> pure (HS.fromList (map LowerBound lower)) + <*> pure (HS.fromList (map UpperBound upper)) + where + nat :: Natural -> Gen Limit + nat n = (\i -> Limit (int i)) <$> choose (0, n) + minRank :: Natural -> Gen MinRank + minRank n = (\i -> MinRank (Min (int i))) <$> choose (0, n) + maxRank :: Natural -> Gen MaxRank + maxRank n = (\i -> MaxRank (Max (int i))) <$> choose (0, n) + wrapNext h = frequency [(1, pure (Inclusive h)), (1, pure (Exclusive h))] + genMaybe g = frequency [(1, pure Nothing), (2, Just <$> g)] + genSubset xs = filterM (const arbitrary) xs + +-- | Compare the result of 'branchEntries' with the result of the generic +-- implementation 'chainBranchEntries' for the same query. +-- +compareBranch :: HasVersion => BlockHeaderDb -> Query -> IO Bool +compareBranch db (Query k l mir mar lower upper) = do + (xs S.:> ra) <- branchEntries db k l mir mar lower upper S.toList + (ys S.:> rb) <- chainBranchEntries db k l mir mar lower upper S.toList + -- Within a rank the order of the streamed entries is unspecified, so the + -- results are compared as sorted lists. + let norm = L.sort . map key + return $ norm xs == norm ys && ra == rb + +-- | Compare both implementations for randomly generated queries on randomly +-- generated trees. +-- +prop_equiv :: HasVersion => RocksDb -> Property +prop_equiv rdb = forAll (resize 60 arbitrary) $ \(SparseTree t) -> + forAll (genQuery t) $ \q -> ioProperty $ + runResourceT $ do + db <- withTestBlockHeaderDb rdb (rootLabel t) + liftIO $ traverse_ (unsafeInsertBlockHeaderDb db) (toList t) + liftIO $ compareBranch db q + +-- | A chain that consists of a single trunk and a number of forks. +data Shape = Shape + { _shapeTrunkLen :: !Int + -- ^ Length of the trunk + , _shapeForks :: ![(Int, Int, Nonce)] + -- ^ (height of the fork point, length of the fork, nonce of the fork) + } deriving (Show) + +instance Arbitrary Shape where + arbitrary = do + n <- choose (20, 100) + k <- choose (0, 2) + fs <- vectorOf k $ do + at <- choose (0, n - 1) + l <- choose (1, 40) + nonce <- Nonce <$> arbitrary + return (at, l, nonce) + return (Shape n fs) + shrink (Shape n fs) = + [ Shape n' fs + | n' <- shrink n + , n' > 0 + ] ++ + [ Shape n fs' + | fs' <- shrinkList shrinkTuple fs + ] + where + shrinkTuple (at, l, nonce) = + [ (at', l', nonce') + | at' <- shrink at + , at' >= 0 + , at' < n + , l' <- shrink l + , l' > 0 + , nonce' <- [nonce] + ] + +-- | A branch query that is specified in terms of indexes into the headers of a +-- 'Shape'. This allows the query to be generated before the headers of the +-- shape are known. +data QuerySpec = QuerySpec + { _qsUpper :: [Int] + , _qsLower :: [Int] + , _qsCursor :: Maybe (Bool, Int) + -- ^ (is exclusive, index) + , _qsLimit :: Maybe Int + , _qsMinRank :: Maybe Int + , _qsMaxRank :: Maybe Int + , _qsBogusUpper :: Bool + , _qsBogusLower :: Bool + } deriving (Show) + +-- | The number of headers of a shape +shapeCount :: Shape -> Int +shapeCount (Shape n fs) = 1 + n + sum (map (\(_, l, _) -> l) fs) + +genQuerySpec :: Shape -> Gen QuerySpec +genQuerySpec shape = do + let count = shapeCount shape + QuerySpec + <$> genSubset [0 .. count - 1] + <*> genSubset [0 .. count - 1] + <*> genMaybe ((,) <$> arbitrary <*> choose (0, count - 1)) + <*> genMaybe (choose (0, count)) + <*> genMaybe (choose (0, count)) + <*> genMaybe (choose (0, count)) + <*> arbitrary + <*> arbitrary + where + genMaybe g = frequency [(1, pure Nothing), (2, Just <$> g)] + genSubset xs = filterM (const arbitrary) xs + +-- | Turn a 'QuerySpec' into a 'Query' for the given headers. The headers must +-- be the headers of the shape for which the spec was generated. +resolveQuery :: HasVersion => [BlockHeader] -> QuerySpec -> Query +resolveQuery hs (QuerySpec upper lower cursor lim mir mar bogusU bogusL) = Query + (fmap (\(ex, i) -> wrap (hs !! i) ex) cursor) + (fmap (Limit . int) lim) + (fmap (MinRank . Min . int) mir) + (fmap (MaxRank . Max . int) mar) + (addBogus bogusL LowerBound lower) + (addBogus bogusU UpperBound upper) + where + wrap h ex = if ex then Exclusive (key h) else Inclusive (key h) + addBogus False f is = HS.fromList (map (f . key . (hs !!)) is) + addBogus True f is = HS.insert (f nullBlockHash) + $ HS.fromList (map (f . key . (hs !!)) is) + +-- | Insert the headers of the given shape into the database and return them in +-- the canonical order that 'resolveQuery' expects. +insertShape :: HasVersion => BlockHeaderDb -> BlockHeader -> Shape -> IO [BlockHeader] +insertShape db gen (Shape n fs) = do + trunk <- insertN_ (Nonce 0) (int n) gen db + forks <- fmap concat $ forM fs $ \(at, l, nonce) -> + insertN_ nonce (int l) (trunk !! at) db + return (gen : trunk ++ forks) + +-- | Compare both implementations for randomly generated queries on chains that +-- consist of a long trunk and a number of forks of random length at random +-- heights. +-- +prop_deep :: HasVersion => RocksDb -> Property +prop_deep rdb = forAll arbitrary $ \shape -> + forAll (genQuerySpec shape) $ \qs -> ioProperty $ + runResourceT $ do + (gen, db) <- withToyDB rdb toyChainId + liftIO $ do + hs <- insertShape db gen shape + compareBranch db (resolveQuery hs qs) + +-- | Compare both implementations for a large number of queries on a chain that +-- consists of a long trunk and a long fork. +-- +deepBranchEquivalence :: HasVersion => RocksDb -> Assertion +deepBranchEquivalence rdb = runResourceT $ do + (gen, db) <- withToyDB rdb toyChainId + liftIO $ do + trunk <- insertN_ (Nonce 0) 200 gen db + let forkPoint = trunk !! 99 + fork <- insertN_ (Nonce 1) 150 forkPoint db + let trunkTip = last trunk + forkTip = last fork + mid = trunk !! 150 + bogus = nullBlockHash + both = HS.fromList [UpperBound (key trunkTip), UpperBound (key forkTip)] + queries = + [ Query nxt lim + (fmap (MinRank . Min) i) + (fmap (MaxRank . Max) j) + lower upper + | i <- [Nothing, Just 0, Just 100] + , j <- [Nothing, Just 50, Just 200] + , nxt <- [Nothing, Just (Inclusive (key mid)), Just (Exclusive (key mid))] + , lim <- [Nothing, Just 0, Just 7, Just 1000] + , lower <- + [ HS.empty + , HS.singleton (LowerBound (key forkPoint)) + , HS.singleton (LowerBound bogus) + ] + , upper <- + [ HS.singleton (UpperBound (key trunkTip)) + , HS.singleton (UpperBound (key forkTip)) + , both + ] + ] + results <- forM queries (compareBranch db) + assertBool "branchEntries doesn't match the generic implementation" (and results) + +tests :: RocksDb -> TestTree +tests rdb = withVersion toyVersion $ testGroup "branchEntries" + [ testProperty "matches the generic implementation" (prop_equiv rdb) + , testProperty "matches the generic implementation (trunk and forks)" + (prop_deep rdb) + , testCase "matches the generic implementation (trunk and fork)" + (deepBranchEquivalence rdb) + ] diff --git a/test/unit/ChainwebTests.hs b/test/unit/ChainwebTests.hs index 8fcaaf10d9..c275c5b301 100644 --- a/test/unit/ChainwebTests.hs +++ b/test/unit/ChainwebTests.hs @@ -25,6 +25,7 @@ import Chainweb.Test.BlockHeader.Genesis qualified (tests) import Chainweb.Test.BlockHeader.Validation qualified (tests) import Chainweb.Test.BlockHeaderDB qualified (tests) import Chainweb.Test.BlockHeaderDB.PruneForks qualified (tests) +import Chainweb.Test.BlockHeaderDBBranch qualified (tests) import Chainweb.Test.Chainweb.Utils.Paging qualified (properties) import Chainweb.Test.Cut qualified (properties) import Chainweb.Test.CutDB qualified (tests) @@ -98,6 +99,7 @@ suite rdb = [ testProperties "Chainweb.Test.Cut" (Chainweb.Test.Cut.properties rdb) , testGroup "BlockHeaderDb" [ Chainweb.Test.BlockHeaderDB.tests rdb + , Chainweb.Test.BlockHeaderDBBranch.tests rdb , Chainweb.Test.TreeDB.RemoteDB.tests , Chainweb.Test.BlockHeaderDB.PruneForks.tests rdb , testProperties "Chainweb.Test.TreeDB" Chainweb.Test.TreeDB.properties