diff --git a/.github/workflows/check-mainnet-config.yml b/.github/workflows/check-mainnet-config.yml index 9b3c7ebf98a..4f4eb657b9b 100644 --- a/.github/workflows/check-mainnet-config.yml +++ b/.github/workflows/check-mainnet-config.yml @@ -43,7 +43,6 @@ jobs: 'mainnet-byron-genesis.json' 'mainnet-checkpoints.json' 'mainnet-config.json' - 'mainnet-config-legacy.json' 'mainnet-peer-snapshot.json' 'mainnet-shelley-genesis.json' 'mainnet-topology.json' diff --git a/.gitignore b/.gitignore index 617e5c62f91..e4fe935faca 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,5 @@ cardano-tracer/cardano-tracer-test .codex .serena/ + +cardano-node/test/db-synthesizer/disk/chaindb \ No newline at end of file diff --git a/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/NodeToNode.hs b/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/NodeToNode.hs index f2030e39ae6..492f6c47f5c 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/NodeToNode.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/NodeToNode.hs @@ -40,6 +40,7 @@ import Ouroboros.Network.Mux (MiniProtocolCb (..), OuroborosApplicatio import Ouroboros.Network.PeerSelection.PeerSharing (PeerSharing (..)) import Ouroboros.Network.PeerSelection.PeerSharing.Codec (decodeRemoteAddress, encodeRemoteAddress) +import Ouroboros.Network.PerasSupport (PerasSupport (..)) import Ouroboros.Network.Protocol.BlockFetch.Client (BlockFetchClient (..), blockFetchClientPeer) import Ouroboros.Network.Protocol.Handshake.Version (simpleSingletonVersions) @@ -112,7 +113,7 @@ benchmarkConnectTxSubmit EnvConsts { .. } handshakeTracer submissionTracer codec supportedVers = supportedNodeToNodeVersions (Proxy @blk) myCodecs :: Codecs blk NtN.RemoteAddress DeserialiseFailure IO ByteString ByteString ByteString ByteString ByteString ByteString - ByteString + ByteString ByteString ByteString myCodecs = defaultCodecs codecConfig blkN2nVer encodeRemoteAddress decodeRemoteAddress n2nVer peerMultiplex :: NtN.Versions NodeToNodeVersion NtN.NodeToNodeVersionData @@ -129,10 +130,11 @@ benchmarkConnectTxSubmit EnvConsts { .. } handshakeTracer submissionTracer codec , NtN.diffusionMode = NtN.InitiatorOnlyDiffusionMode , NtN.peerSharing = ownPeerSharing , NtN.query = False + , NtN.perasSupport = PerasUnsupported }) $ \n2nData -> mkApp $ - NtN.nodeToNodeProtocols NtN.defaultMiniProtocolParameters + NtN.nodeToNodeProtocols mempty NtN.defaultMiniProtocolParameters NtN.NodeToNodeProtocols { NtN.chainSyncProtocol = InitiatorProtocolOnly $ MiniProtocolCb $ \_ctx channel -> runPeer @@ -160,6 +162,11 @@ benchmarkConnectTxSubmit EnvConsts { .. } handshakeTracer submissionTracer codec (cPeerSharingCodec myCodecs) channel (peerSharingClientPeer peerSharingClientNull) + -- TODO Peras is not supported here + , NtN.perasCertDiffusionProtocol = InitiatorProtocolOnly $ MiniProtocolCb $ \_ctx _channel -> + error "tx-generator: Peras cert diffusion is unsupported" + , NtN.perasVoteDiffusionProtocol = InitiatorProtocolOnly $ MiniProtocolCb $ \_ctx _channel -> + error "tx-generator: Peras vote diffusion is unsupported" } n2nVer n2nData diff --git a/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SizedMetadata.hs b/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SizedMetadata.hs index fa47d5bb538..75d58677a7c 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SizedMetadata.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SizedMetadata.hs @@ -4,11 +4,14 @@ {-# LANGUAGE GADTs #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} + module Cardano.Benchmarking.GeneratorTx.SizedMetadata where import Cardano.Api +import Cardano.Ledger.BaseTypes (maybeToStrictMaybe) +import qualified Cardano.Ledger.Core as L import Cardano.TxGenerator.Utils import Prelude @@ -17,6 +20,7 @@ import qualified Data.ByteString as BS import Data.Function ((&)) import qualified Data.Map.Strict as Map import Data.Word (Word64) +import Lens.Micro ((.~), (^.)) maxMapSize :: Int @@ -114,21 +118,25 @@ measureBSCosts era = map (metadataSize era . Just . bsMetadata) [0..maxBSSize] metadataSize :: forall era . IsShelleyBasedEra era => AsType era -> Maybe TxMetadata -> Int metadataSize p m = dummyTxSize p m - dummyTxSize p Nothing -dummyTxSizeInEra :: IsShelleyBasedEra era => TxMetadataInEra era -> Int -dummyTxSizeInEra metadata = case createTransactionBody shelleyBasedEra dummyTx of - Right b -> BS.length $ serialiseToCBOR b - Left err -> error $ "metaDataSize " ++ show err +dummyTxSizeInEra :: forall era. IsShelleyBasedEra era => TxMetadataInEra era -> Int +dummyTxSizeInEra metadata = + BS.length $ serialiseToCBOR dummyTx where - dummyTx = defaultTxBodyContent shelleyBasedEra - & setTxIns - [ ( mkTxIn "dbaff4e270cfb55612d9e2ac4658a27c79da4a5271c6f90853042d1403733810#0" - , BuildTxWith $ KeyWitness KeyWitnessForSpending - ) - ] - & setTxFee (mkTxFee 0) - & setTxValidityLowerBound TxValidityNoLowerBound - & setTxValidityUpperBound (mkTxValidityUpperBound 0) - & setTxMetadata metadata + sbe = shelleyBasedEra @era + txInputs = + [ ( mkTxIn "dbaff4e270cfb55612d9e2ac4658a27c79da4a5271c6f90853042d1403733810#0" + , BuildTxWith $ KeyWitness KeyWitnessForSpending + ) + ] + txAuxData = toAuxiliaryData sbe metadata TxAuxScriptsNone + ledgerTxBody = + mkCommonTxBody sbe txInputs [] (mkTxFee 0) TxWithdrawalsNone txAuxData + & invalidHereAfterTxBodyL sbe .~ convValidityUpperBound sbe (mkTxValidityUpperBound 0) + dummyTx :: Tx era + dummyTx = shelleyBasedEraConstraints sbe $ + ShelleyTx sbe $ + L.mkBasicTx (ledgerTxBody ^. txBodyL) + & L.auxDataTxL .~ maybeToStrictMaybe txAuxData dummyTxSize :: forall era . IsShelleyBasedEra era => AsType era -> Maybe TxMetadata -> Int dummyTxSize _p m = (dummyTxSizeInEra @era) $ metadataInEra m diff --git a/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SubmissionClient.hs b/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SubmissionClient.hs index 3efa975616e..3c3878aa75f 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SubmissionClient.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/GeneratorTx/SubmissionClient.hs @@ -24,7 +24,7 @@ module Cardano.Benchmarking.GeneratorTx.SubmissionClient , txSubmissionClient ) where -import Cardano.Api hiding (Active) +import Cardano.Api hiding (Active, CardanoBlock) import Cardano.Benchmarking.LogTypes import Cardano.Benchmarking.Types @@ -36,7 +36,6 @@ import qualified Ouroboros.Consensus.Cardano.Block as Block (TxId (GenTxIdAllegra, GenTxIdAlonzo, GenTxIdBabbage, GenTxIdConway, GenTxIdMary, GenTxIdShelley)) import Ouroboros.Consensus.Ledger.SupportsMempool (GenTxId) import qualified Ouroboros.Consensus.Ledger.SupportsMempool as Mempool -import Ouroboros.Consensus.Shelley.Eras (StandardCrypto) import qualified Ouroboros.Consensus.Shelley.Ledger.Mempool as Mempool (TxId (ShelleyTxId)) import Ouroboros.Network.Protocol.TxSubmission2.Client (ClientStIdle (..), ClientStTxIds (..), ClientStTxs (..), TxSubmissionClient (..)) @@ -102,7 +101,7 @@ txSubmissionClient tr bmtr initialTxSource endOfProtocolCallback = fail (T.unpack err) let (stillUnacked, acked) = L.splitAtEnd ack unAcked let newStats = stats { stsAcked = stsAcked stats + Ack ack } - traceWith bmtr $ SubmissionClientDiscardAcknowledged (getTxId . getTxBody <$> acked) + traceWith bmtr $ SubmissionClientDiscardAcknowledged (txIdFromTx <$> acked) return (txSource, UnAcked stillUnacked, newStats) queueNewTxs :: [Tx era] -> LocalState era -> LocalState era @@ -131,8 +130,8 @@ txSubmissionClient tr bmtr initialTxSource endOfProtocolCallback = let stateC@(_, UnAcked outs , stats) = queueNewTxs newTxs stateB traceWith tr $ idListTrace (ToAnnce newTxs) blocking - traceWith bmtr $ SubmissionClientReplyTxIds (getTxId . getTxBody <$> newTxs) - traceWith bmtr $ SubmissionClientUnAcked (getTxId . getTxBody <$> outs) + traceWith bmtr $ SubmissionClientReplyTxIds (txIdFromTx <$> newTxs) + traceWith bmtr $ SubmissionClientUnAcked (txIdFromTx <$> outs) case blocking of SingBlocking -> case NE.nonEmpty newTxs of @@ -156,12 +155,12 @@ txSubmissionClient tr bmtr initialTxSource endOfProtocolCallback = reqTxIds = fmap fromGenTxId txIds traceWith tr $ ReqTxs (length reqTxIds) let UnAcked ua = unAcked - uaIds = getTxId . getTxBody <$> ua - (toSend, _retained) = L.partition ((`L.elem` reqTxIds) . getTxId . getTxBody) ua + uaIds = txIdFromTx <$> ua + (toSend, _retained) = L.partition ((`L.elem` reqTxIds) . txIdFromTx) ua missIds = reqTxIds L.\\ uaIds traceWith tr $ TxList (length toSend) - traceWith bmtr $ SubmissionClientUnAcked (getTxId . getTxBody <$> ua) + traceWith bmtr $ SubmissionClientUnAcked (txIdFromTx <$> ua) traceWith bmtr $ TraceBenchTxSubServReq reqTxIds unless (L.null missIds) $ traceWith bmtr $ TraceBenchTxSubServUnav missIds @@ -191,6 +190,10 @@ txSubmissionClient tr bmtr initialTxSource endOfProtocolCallback = fromGenTxId (Block.GenTxIdConway (Mempool.ShelleyTxId i)) = fromShelleyTxId i fromGenTxId _ = error "TODO: fix incomplete match" + txIdFromTx :: Tx era -> TxId + txIdFromTx (ShelleyTx sbe tx) = + shelleyBasedEraConstraints sbe $ fromShelleyTxId $ Ledger.txIdTxBody (tx ^. Ledger.bodyTxL) + tokIsBlocking :: SingBlockingStyle a -> Bool tokIsBlocking = \case SingBlocking -> True diff --git a/bench/tx-generator/src/Cardano/Benchmarking/LogTypes.hs b/bench/tx-generator/src/Cardano/Benchmarking/LogTypes.hs index 6a67daea14d..0b7144fc4f4 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/LogTypes.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/LogTypes.hs @@ -22,7 +22,7 @@ module Cardano.Benchmarking.LogTypes , TraceBenchTxSubmit (..) ) where -import Cardano.Api +import Cardano.Api hiding (CardanoBlock) import Cardano.Benchmarking.OuroborosImports import Cardano.Benchmarking.Types diff --git a/bench/tx-generator/src/Cardano/Benchmarking/OuroborosImports.hs b/bench/tx-generator/src/Cardano/Benchmarking/OuroborosImports.hs index abd5a10c54f..5d31e39d4e6 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/OuroborosImports.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/OuroborosImports.hs @@ -1,6 +1,5 @@ {- HLINT ignore "Eta reduce" -} {-# LANGUAGE GADTs #-} -{-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE TypeApplications #-} module Cardano.Benchmarking.OuroborosImports @@ -38,21 +37,19 @@ import Prelude type CardanoBlock = Consensus.CardanoBlock StandardCrypto -toProtocolInfo :: SomeConsensusProtocol -> ProtocolInfo CardanoBlock -toProtocolInfo (SomeConsensusProtocol CardanoBlockType info) = fst $ protocolInfo @IO info +toProtocolInfo :: SomeConsensusProtocol -> IO (ProtocolInfo CardanoBlock) +toProtocolInfo (SomeConsensusProtocol CardanoBlockType info) = fst <$> protocolInfo @IO info toProtocolInfo _ = error "toProtocolInfo unknown protocol" -protocolToTopLevelConfig :: SomeConsensusProtocol -> TopLevelConfig CardanoBlock -protocolToTopLevelConfig ptcl = pInfoConfig - where - ProtocolInfo {pInfoConfig} = toProtocolInfo ptcl +protocolToTopLevelConfig :: SomeConsensusProtocol -> IO (TopLevelConfig CardanoBlock) +protocolToTopLevelConfig ptcl = pInfoConfig <$> toProtocolInfo ptcl -protocolToCodecConfig :: SomeConsensusProtocol -> CodecConfig CardanoBlock -protocolToCodecConfig = configCodec . protocolToTopLevelConfig +protocolToCodecConfig :: SomeConsensusProtocol -> IO (CodecConfig CardanoBlock) +protocolToCodecConfig = fmap configCodec . protocolToTopLevelConfig -protocolToNetworkId :: SomeConsensusProtocol -> NetworkId +protocolToNetworkId :: SomeConsensusProtocol -> IO NetworkId protocolToNetworkId ptcl - = Testnet $ getNetworkMagic $ configBlock $ protocolToTopLevelConfig ptcl + = Testnet . getNetworkMagic . configBlock <$> protocolToTopLevelConfig ptcl makeLocalConnectInfo :: NetworkId -> SocketPath -> LocalNodeConnectInfo makeLocalConnectInfo networkId socketPath diff --git a/bench/tx-generator/src/Cardano/Benchmarking/Script/Action.hs b/bench/tx-generator/src/Cardano/Benchmarking/Script/Action.hs index 3435fbddeb9..0c992938ee0 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/Script/Action.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/Script/Action.hs @@ -69,8 +69,8 @@ startProtocol configFile tracerSocket = do setEnvGenesis $ getGenesis protocol iomgr <- askIOManager + networkId <- liftIO $ protocolToNetworkId protocol let - networkId = protocolToNetworkId protocol tracerSocket' = (,,) iomgr networkId `fmap` tracerSocket setEnvNetworkId networkId diff --git a/bench/tx-generator/src/Cardano/Benchmarking/Script/Core.hs b/bench/tx-generator/src/Cardano/Benchmarking/Script/Core.hs index 592c4fed620..fdd9eefd3e6 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/Script/Core.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/Script/Core.hs @@ -9,7 +9,6 @@ {-# LANGUAGE LambdaCase #-} {-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE NumericUnderscores #-} -{-# LANGUAGE PackageImports #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ScopedTypeVariables #-} @@ -39,6 +38,7 @@ import Cardano.Benchmarking.Version as Version import Cardano.Benchmarking.Wallet as Wallet import qualified Cardano.Ledger.Coin as L import qualified Cardano.Ledger.Core as Ledger +import Cardano.Ledger.Tools (estimateMinFeeTx) import Cardano.Logging hiding (LocalSocket) import Cardano.TxGenerator.Fund as Fund import qualified Cardano.TxGenerator.FundQueue as FundQueue @@ -57,7 +57,6 @@ import Prelude import Control.Concurrent (threadDelay) import Control.Monad import Control.Monad.Trans.RWS.Strict (ask) -import "contra-tracer" Control.Tracer (Tracer (..)) import Data.ByteString.Lazy.Char8 as BSL (writeFile) import Data.Ratio ((%)) import qualified Data.Text as Text (unpack) @@ -137,11 +136,12 @@ getConnectClient = do protocol <- getEnvProtocol void $ return $ btSubmission2_ tracers envConsts <- lift ask + codecConfig <- liftIO $ protocolToCodecConfig protocol return $ benchmarkConnectTxSubmit envConsts - (Tracer $ traceWith (btConnect_ tracers)) + (mkTracer $ traceWith (btConnect_ tracers)) mempty -- (btSubmission2_ tracers) - (protocolToCodecConfig protocol) + codecConfig networkMagic waitBenchmark :: ActionM () waitBenchmark = do @@ -354,10 +354,12 @@ evalGenerator generator txParams@TxGenTxParams{txParamFee = fee} era = do Right tx -> do let txSize = txSizeInBytes tx - txFeeEstimate = case toLedgerPParams shelleyBasedEra protocolParameters of - Left{} -> Nothing - Right ledgerPParams -> Just $ - evaluateTransactionFee shelleyBasedEra ledgerPParams (getTxBody tx) (fromIntegral $ inputs + 1) 0 0 -- 1 key witness per tx input + 1 collateral + txFeeEstimate = case tx of + ShelleyTx sbe ledgerTx -> shelleyBasedEraConstraints sbe $ + case toLedgerPParams sbe protocolParameters of + Left{} -> Nothing + Right ledgerPParams -> Just $ + estimateMinFeeTx ledgerPParams ledgerTx (inputs + 1) 0 0 -- 1 key witness per tx input + 1 collateral traceDebug $ "Projected Tx size in bytes: " ++ show txSize traceDebug $ "Projected Tx fee in Coin: " ++ show txFeeEstimate -- TODO: possibly emit a warning when (Just txFeeEstimate) is lower than specified by config in TxGenTxParams.txFee diff --git a/bench/tx-generator/src/Cardano/Benchmarking/Tracer.hs b/bench/tx-generator/src/Cardano/Benchmarking/Tracer.hs index 17f66b3f290..08fd74ae013 100644 --- a/bench/tx-generator/src/Cardano/Benchmarking/Tracer.hs +++ b/bench/tx-generator/src/Cardano/Benchmarking/Tracer.hs @@ -96,22 +96,22 @@ initTxGenTracers mbForwarding = mdo confState <- emptyConfigReflection let - mkTracer :: (LogFormatting a, MetaTrace a) + mkConfiguredTracer :: (LogFormatting a, MetaTrace a) => Text -> Maybe (Trace IO FormattedMessage) -> Maybe (Trace IO FormattedMessage) -> IO (Trace IO a) - mkTracer namespace mbStdoutTracer' mbForwardingTracer' + mkConfiguredTracer namespace mbStdoutTracer' mbForwardingTracer' | isPrefixSilent namespace = pure mempty | otherwise = do tracer <- generatorTracer namespace mbStdoutTracer' mbForwardingTracer' configureTracers confState initialTraceConfig [tracer] pure tracer - benchTracer <- mkTracer TracerNameBench mbStdoutTracer mbForwardingTracer - n2nSubmitTracer <- mkTracer TracerNameSubmitN2N mbStdoutTracer mbForwardingTracer - connectTracer <- mkTracer TracerNameConnect mbStdoutTracer mbForwardingTracer - submitTracer <- mkTracer TracerNameSubmit mbStdoutTracer mbForwardingTracer + benchTracer <- mkConfiguredTracer TracerNameBench mbStdoutTracer mbForwardingTracer + n2nSubmitTracer <- mkConfiguredTracer TracerNameSubmitN2N mbStdoutTracer mbForwardingTracer + connectTracer <- mkConfiguredTracer TracerNameConnect mbStdoutTracer mbForwardingTracer + submitTracer <- mkConfiguredTracer TracerNameSubmit mbStdoutTracer mbForwardingTracer let tracers = BenchTracers diff --git a/bench/tx-generator/src/Cardano/TxGenerator/Genesis.hs b/bench/tx-generator/src/Cardano/TxGenerator/Genesis.hs index d15f80a3a38..0e3a9ba9b82 100644 --- a/bench/tx-generator/src/Cardano/TxGenerator/Genesis.hs +++ b/bench/tx-generator/src/Cardano/TxGenerator/Genesis.hs @@ -5,6 +5,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} + {- HLINT ignore "Use map with tuple-section" -} -- | This module provides means to secure funds that are given in genesis. @@ -22,16 +23,20 @@ where import Cardano.Api hiding (ShelleyGenesis) import qualified Cardano.Ledger.Coin as L +import qualified Cardano.Ledger.Core as Ledger +import Cardano.Ledger.Keys.WitVKey (WitVKey (WitVKey)) import Cardano.Ledger.Shelley.API (Addr (..)) import Cardano.TxGenerator.Fund import Cardano.TxGenerator.Types import Cardano.TxGenerator.Utils import Ouroboros.Consensus.Shelley.Node (validateGenesis) -import Data.Bifunctor (bimap, second) +import Data.Bifunctor (second) import Data.Function ((&)) import Data.List (find) import qualified Data.ListMap as ListMap (toList) +import qualified Data.Set as Set +import Lens.Micro ((.~), (^.)) genesisValidate :: ShelleyGenesis -> Either String () @@ -106,12 +111,16 @@ genesisExpenditure networkId inputKey addr value fee ttl outputKey pseudoTxIn = genesisTxInput networkId inputKey fund tx = FundInEra { - _fundTxIn = TxIn (getTxId $ getTxBody tx) (TxIx 0) + _fundTxIn = TxIn (txIdFromTx tx) (TxIx 0) , _fundWitness = KeyWitness KeyWitnessForSpending , _fundVal = value , _fundSigningKey = Just outputKey } + txIdFromTx :: Tx era -> TxId + txIdFromTx (ShelleyTx sbe' tx') = + shelleyBasedEraConstraints sbe' $ fromShelleyTxId $ Ledger.txIdTxBody (tx' ^. Ledger.bodyTxL) + mkGenesisTransaction :: forall era . IsShelleyBasedEra era => SigningKey GenesisUTxOKey @@ -120,18 +129,24 @@ mkGenesisTransaction :: forall era . -> [TxIn] -> [TxOut CtxTx era] -> Either TxGenError (Tx era) -mkGenesisTransaction key ttl fee txins txouts - = bimap - ApiError - (\b -> signShelleyTransaction (shelleyBasedEra @era) b [WitnessGenesisUTxOKey key]) - (createTransactionBody (shelleyBasedEra @era) txBodyContent) +mkGenesisTransaction key ttl fee txins txouts = + shelleyBasedEraConstraints sbe $ + let txInputs = zip txins $ repeat $ BuildTxWith $ KeyWitness KeyWitnessForSpending + ledgerTxBody = + mkCommonTxBody sbe txInputs txouts (mkTxFee fee) TxWithdrawalsNone Nothing + & invalidHereAfterTxBodyL sbe .~ convValidityUpperBound sbe (mkTxValidityUpperBound ttl) + rawBody = ledgerTxBody ^. txBodyL + unsignedLedgerTx = Ledger.mkBasicTx rawBody + txHash = Ledger.extractHash $ Ledger.hashAnnotated rawBody + shelleySigningKey = toShelleySigningKey (WitnessGenesisUTxOKey key) + witVKey = WitVKey + (getShelleyKeyWitnessVerificationKey shelleySigningKey) + (makeShelleySignature txHash shelleySigningKey) + signedLedgerTx = unsignedLedgerTx + & Ledger.witsTxL .~ (Ledger.mkBasicTxWits & Ledger.addrTxWitsL .~ Set.singleton witVKey) + in Right $ ShelleyTx sbe signedLedgerTx where - txBodyContent = defaultTxBodyContent shelleyBasedEra - & setTxIns (zip txins $ repeat $ BuildTxWith $ KeyWitness KeyWitnessForSpending) - & setTxOuts txouts - & setTxFee (mkTxFee fee) - & setTxValidityLowerBound TxValidityNoLowerBound - & setTxValidityUpperBound (mkTxValidityUpperBound ttl) + sbe = shelleyBasedEra @era castKey :: SigningKey PaymentKey -> SigningKey GenesisUTxOKey castKey (PaymentSigningKey skey) = GenesisUTxOSigningKey skey diff --git a/bench/tx-generator/src/Cardano/TxGenerator/ProtocolParameters.hs b/bench/tx-generator/src/Cardano/TxGenerator/ProtocolParameters.hs index 6c10ab8d2c5..bfb31bf7626 100644 --- a/bench/tx-generator/src/Cardano/TxGenerator/ProtocolParameters.hs +++ b/bench/tx-generator/src/Cardano/TxGenerator/ProtocolParameters.hs @@ -58,7 +58,7 @@ import Data.Int (Int64) import qualified Data.Map.Strict as Map import qualified Data.Scientific as Scientific import qualified Data.Text as Text -import Data.Word (Word16) +import Data.Word (Word16, Word32) import GHC.Generics import Lens.Micro ((&), (.~), (^.)) import Numeric.Natural (Natural) @@ -83,7 +83,7 @@ convertToLedgerProtocolParameters sbe pp = data ProtocolParameters = ProtocolParameters - { protocolParamProtocolVersion :: (Natural, Natural) + { protocolParamProtocolVersion :: (Natural, Word32) -- ^ Protocol version, major and minor. Updating the major version is -- used to trigger hard forks. -- (Major , Minor ) @@ -323,7 +323,7 @@ requireParam requireParam paramName = maybe (Left $ PpceMissingParameter paramName) -- Duplicated from "cardano-api" module "Cardano.Api.Internal.ProtocolParameters" -mkProtVer :: (Natural, Natural) -> Either ProtocolParametersConversionError Ledger.ProtVer +mkProtVer :: (Natural, Word32) -> Either ProtocolParametersConversionError Ledger.ProtVer mkProtVer (majorProtVer, minorProtVer) = maybeToRight (PpceVersionInvalid majorProtVer) $ (`Ledger.ProtVer` minorProtVer) <$> Ledger.mkVersion majorProtVer diff --git a/bench/tx-generator/src/Cardano/TxGenerator/Setup/NixService.hs b/bench/tx-generator/src/Cardano/TxGenerator/Setup/NixService.hs index 8b83528f49a..2e1c48c2852 100644 --- a/bench/tx-generator/src/Cardano/TxGenerator/Setup/NixService.hs +++ b/bench/tx-generator/src/Cardano/TxGenerator/Setup/NixService.hs @@ -27,7 +27,7 @@ import Cardano.Api (AnyCardanoEra, mapFile) import Cardano.CLI.Type.Common (FileDirection (..), SigningKeyFile) import qualified Cardano.Ledger.Coin as L import Cardano.Node.Configuration.NodeAddress (NodeAddress' (..), - NodeHostIPv4Address (..), NodeIPv4Address) + NodeIPv4Address) import Cardano.Node.Types (AdjustFilePaths (..)) import Cardano.TxGenerator.Internal.Orphans () import Cardano.TxGenerator.Types @@ -74,22 +74,20 @@ data NodeDescription = instance FromJSON NodeDescription where parseJSON = withObject "NodeDescription" \v -> do - unNodeHostIPv4Address + naHostAddress <- v .: "addr" Key "addr" naPort <- fmap toEnum $ v .: "port" Key "port" - let naHostAddress = NodeHostIPv4Address {..} - ndAddr = NodeAddress {..} + let ndAddr = NodeAddress {..} ndName <- v .:? "name" Key "name" .!= show ndAddr pure $ NodeDescription {..} instance ToJSON NodeDescription where toJSON NodeDescription {ndAddr, ndName} = object [ "name" .= ndName - , "addr" .= unNodeHostIPv4Address + , "addr" .= naHostAddress , "port" .= fromEnum naPort ] where _addr@NodeAddress {naHostAddress, naPort} = ndAddr - _hostAddr@NodeHostIPv4Address {unNodeHostIPv4Address} = naHostAddress -- Long GC pauses on target nodes can trigger spurious MVar deadlock diff --git a/bench/tx-generator/src/Cardano/TxGenerator/Setup/NodeConfig.hs b/bench/tx-generator/src/Cardano/TxGenerator/Setup/NodeConfig.hs index fab37d0fe8b..8e78199c68b 100644 --- a/bench/tx-generator/src/Cardano/TxGenerator/Setup/NodeConfig.hs +++ b/bench/tx-generator/src/Cardano/TxGenerator/Setup/NodeConfig.hs @@ -34,7 +34,7 @@ getGenesis :: SomeConsensusProtocol -> ShelleyGenesis getGenesis (SomeConsensusProtocol CardanoBlockType proto) = getConst $ Ledger.tcShelleyGenesisL Const transCfg where - ProtocolInfoArgsCardano Consensus.CardanoProtocolParams + ProtocolInfoArgsCardano _ Consensus.CardanoProtocolParams { Consensus.cardanoLedgerTransitionConfig = transCfg } = proto diff --git a/bench/tx-generator/src/Cardano/TxGenerator/Tx.hs b/bench/tx-generator/src/Cardano/TxGenerator/Tx.hs index 8358276732e..37c1642be87 100644 --- a/bench/tx-generator/src/Cardano/TxGenerator/Tx.hs +++ b/bench/tx-generator/src/Cardano/TxGenerator/Tx.hs @@ -2,7 +2,7 @@ {-# LANGUAGE GADTs #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} -{-# LANGUAGE TypeApplications #-} + module Cardano.TxGenerator.Tx (module Cardano.TxGenerator.Tx) @@ -10,15 +10,20 @@ module Cardano.TxGenerator.Tx import Cardano.Api hiding (txId) +import Cardano.Ledger.BaseTypes (maybeToStrictMaybe) import qualified Cardano.Ledger.Coin as L +import qualified Cardano.Ledger.Core as Ledger +import Cardano.Ledger.Keys.WitVKey (WitVKey (WitVKey)) import Cardano.TxGenerator.Fund import Cardano.TxGenerator.Types import Cardano.TxGenerator.UTxO (ToUTxOList) -import Data.Bifunctor (bimap, second) +import Data.Bifunctor (second) import qualified Data.ByteString as BS (length) import Data.Function ((&)) import Data.Maybe (mapMaybe) +import qualified Data.Set as Set +import Lens.Micro ((.~), (^.)) -- | 'CreateAndStore' is meant to represent building a transaction @@ -166,22 +171,33 @@ genTx :: forall era. () -> TxFee era -> TxMetadataInEra era -> TxGenerator era -genTx sbe ledgerParameters (collateral, collFunds) fee metadata inFunds outputs - = bimap - ApiError - (\b -> (signShelleyTransaction (shelleyBasedEra @era) b $ map WitnessPaymentKey allKeys, getTxId b)) - (createTransactionBody (shelleyBasedEra @era) txBodyContent) - where - allKeys = mapMaybe getFundKey $ inFunds ++ collFunds - txBodyContent = defaultTxBodyContent sbe - & setTxIns (map (\f -> (getFundTxIn f, BuildTxWith $ getFundWitness f)) inFunds) - & setTxInsCollateral collateral - & setTxOuts outputs - & setTxFee fee - & setTxValidityLowerBound TxValidityNoLowerBound - & setTxValidityUpperBound (defaultTxValidityUpperBound sbe) - & setTxMetadata metadata - & setTxProtocolParams (BuildTxWith (Just ledgerParameters)) +genTx sbe _ledgerParameters (collateral, collFunds) fee metadata inFunds outputs = + shelleyBasedEraConstraints sbe $ do + let allKeys = mapMaybe getFundKey $ inFunds ++ collFunds + setCollateral = case collateral of + TxInsCollateralNone -> id + TxInsCollateral eon _ -> collateralInputsTxBodyL eon .~ convCollateralTxIns collateral + txInputs = map (\f -> (getFundTxIn f, BuildTxWith $ getFundWitness f)) inFunds + txAuxData = toAuxiliaryData sbe metadata TxAuxScriptsNone + ledgerTxBody = + mkCommonTxBody sbe txInputs outputs fee TxWithdrawalsNone txAuxData + & invalidHereAfterTxBodyL sbe .~ convValidityUpperBound sbe (defaultTxValidityUpperBound sbe) + & setCollateral + rawBody = ledgerTxBody ^. txBodyL + unsignedLedgerTx = Ledger.mkBasicTx rawBody + txHash = Ledger.extractHash $ Ledger.hashAnnotated rawBody + witVKeys = Set.fromList + [ WitVKey + (getShelleyKeyWitnessVerificationKey sk) + (makeShelleySignature txHash sk) + | sk <- map (toShelleySigningKey . WitnessPaymentKey) allKeys + ] + signedLedgerTx = unsignedLedgerTx + & Ledger.witsTxL .~ (Ledger.mkBasicTxWits & Ledger.addrTxWitsL .~ witVKeys) + & Ledger.auxDataTxL .~ maybeToStrictMaybe txAuxData + tx = ShelleyTx sbe signedLedgerTx + txId = fromShelleyTxId $ Ledger.txIdTxBody rawBody + Right (tx, txId) txSizeInBytes :: forall era. IsShelleyBasedEra era => diff --git a/bench/tx-generator/tx-generator.cabal b/bench/tx-generator/tx-generator.cabal index 5ebadb43f08..acabbb3d6d7 100644 --- a/bench/tx-generator/tx-generator.cabal +++ b/bench/tx-generator/tx-generator.cabal @@ -113,28 +113,20 @@ library , cardano-binary , cardano-cli ^>= 11.1 , cardano-crypto-class - , cardano-crypto-wrapper , cardano-data , cardano-diffusion ^>= 1.0 , cardano-git-rev ^>= 0.2.2 - , cardano-ledger-alonzo , cardano-ledger-api - , cardano-ledger-byron , cardano-ledger-core , cardano-node , cardano-prelude - , cardano-strict-containers >=0.1 , contra-tracer , cborg >= 0.2.2 && < 0.3 , containers - , constraints-extras , directory , dlist , extra , filepath - , formatting - , generic-monoid - , ghc-prim , io-classes:{io-classes, strict-stm} , microlens , mtl @@ -142,7 +134,7 @@ library , network-mux , optparse-applicative , ouroboros-consensus:{ouroboros-consensus, cardano, diffusion} >= 3.0.1 - , ouroboros-network:{api, framework, framework-tracing, ouroboros-network, protocols} >= 1.1 + , ouroboros-network:{api, framework, tracing, ouroboros-network, protocols} >= 1.1 , plutus-ledger-api , plutus-tx , random @@ -158,7 +150,6 @@ library , trace-forward , transformers , transformers-except - , unordered-containers , yaml -- Needed by "Cardano.Api.Internal.ProtocolParameters" port. , either @@ -195,12 +186,12 @@ executable calibrate-script , aeson , aeson-pretty , bytestring + , cardano-api , containers , directory , extra , filepath , optparse-applicative - , cardano-api , text , transformers , transformers-except diff --git a/cabal.project b/cabal.project index 5ed9a449fe9..b9868d361ab 100644 --- a/cabal.project +++ b/cabal.project @@ -13,12 +13,8 @@ repository cardano-haskell-packages -- See CONTRIBUTING for information about these, including some Nix commands -- you need to run if you change them index-state: - , hackage.haskell.org 2026-04-17T09:20:55Z - , cardano-haskell-packages 2026-05-27T09:43:46Z - -active-repositories: - , :rest - , cardano-haskell-packages:override + , hackage.haskell.org 2026-06-29T22:49:53Z + , cardano-haskell-packages 2026-07-02T10:10:00Z constraints: -- haskell.nix patch does not work for 1.6.8 @@ -66,12 +62,18 @@ package cryptonite -- generation is dubious. Set the flag so we use /dev/urandom by default. flags: -support_rdrand +package snap-server + flags: -openssl + package bitvec flags: -simd +package cardano-diffusion + flags: +optparse-applicative-fork + -- required for haddocks to build successfully package plutus-scripts-bench - haddock-options: "--optghc=-fplugin-opt=PlutusTx.Plugin:defer-errors" + haddock-options: "--optghc=-fplugin-opt PlutusTx.Plugin:defer-errors" -- There is a suspected bug in `cabal` (https://github.com/haskell/cabal/issues/11663) -- that can be worked around with the following allow-newer stanzas @@ -82,3 +84,96 @@ allow-newer: -- IMPORTANT -- Do NOT add more source-repository-package stanzas here unless they are strictly -- temporary! Please read the section in CONTRIBUTING about updating dependencies. + + +source-repository-package + type: git + location: https://github.com/IntersectMBO/cardano-api.git + tag: 856e3c7f25ebf8b5f8083e24bd292676449ae4df + --sha256: sha256-KjA475iq66pFefawNjYd8gKqoJuzlWpkEhpfIYhh0n8= + subdir: + cardano-api + cardano-rpc + +source-repository-package + type: git + location: https://github.com/IntersectMBO/cardano-cli.git + tag: 3df8a98d42a906e0640716a89290d6a4a1c701d2 + --sha256: sha256-2Vf0K7FvI81KsHs3AktLNlK6syQDbAGy99aYS6dTC84= + subdir: + cardano-cli + +source-repository-package + type: git + location: https://github.com/IntersectMBO/cardano-ledger.git + tag: e9827fdc3def69c02fe826fdf46e2a412620ef98 + --sha256: sha256-bVR1ZFs1VDc7ag2QEpSf5rVj1mdBCB/Ag7nhj00UXQE= + subdir: + eras/allegra/impl + eras/alonzo/impl + eras/babbage/impl + eras/byron/chain/executable-spec + eras/byron/crypto + eras/byron/ledger/executable-spec + eras/byron/ledger/impl + eras/conway/impl + eras/dijkstra/impl + eras/mary/impl + eras/shelley-ma/test-suite + eras/shelley/impl + eras/shelley/test-suite + libs/cardano-data + libs/cardano-ledger-api + libs/cardano-ledger-binary + libs/cardano-ledger-core + libs/cardano-protocol + libs/cardano-protocol-tpraos + libs/non-integral + libs/small-steps + libs/vector-map + +source-repository-package + type: git + location: https://github.com/f-f/kes-agent.git + tag: 32c1ed675d22a30735d9f22f7afa436a3ef3e64a + --sha256: sha256-o7hFX1JnraS6Xq0WoXQwd9Z8GsPPv0Ls2DWvZ08o0ZU= + subdir: + kes-agent + kes-agent-crypto + +source-repository-package + type: git + location: https://github.com/IntersectMBO/cardano-config + tag: 146fae3b13ad13a72a72ca9792a8a44ef5076b52 + --sha256: sha256-M+LYeNr30ng261MnjwOURcWe1Gajn8T4YArAUf3XyRY= + +source-repository-package + type: git + location: https://github.com/IntersectMBO/ouroboros-consensus.git + tag: 97dac2a1f85f9dd8c5be14200b6ea4e756a3f07e + --sha256: sha256-47Po3W1Ut2hFA/LyZweg3orL2ZM3WVZEH65lOQziYRs= + +source-repository-package + type: git + location: https://github.com/IntersectMBO/ouroboros-network.git + tag: 7d5261376493afa83f9033e72717576b07bac251 + --sha256: sha256-gLphW8Pl9P7Du8WL8cQTkKx+gVyf5HEqSD4uFy3I+uE= + subdir: + ./cardano-diffusion + ./monoidal-synchronisation + ./network-mux + ./ouroboros-network + +source-repository-package + type: git + location: https://github.com/IntersectMBO/dmq-node.git + tag: 9bf9762d3e78b970ea2a3526a4c0b076382df86c + --sha256: sha256-pMU7fiWDs8mHb5F66hs7jVTbXEJYA2O20xQFGRk/WWw= + subdir: + dmq-node + +source-repository-package + type: git + location: https://github.com/f-f/ekg-forward + tag: b24b3aba2806ce223c62f8ce3e267ec92dcc52e2 + --sha256: sha256-s5Hxxm04HmFVmdBjAnFEsJEhTqr5Z/uiB4K1s2VaVwE= diff --git a/cardano-node-chairman/app/Cardano/Chairman.hs b/cardano-node-chairman/app/Cardano/Chairman.hs index b43842cf160..e1576f7b56d 100644 --- a/cardano-node-chairman/app/Cardano/Chairman.hs +++ b/cardano-node-chairman/app/Cardano/Chairman.hs @@ -15,7 +15,6 @@ import Cardano.Api import Cardano.Ledger.BaseTypes (unNonZero) import Ouroboros.Consensus.Block.Abstract -import Ouroboros.Consensus.Cardano.Block import Ouroboros.Consensus.Config.SecurityParam import Ouroboros.Network.AnchoredFragment (Anchor, AnchoredFragment) import qualified Ouroboros.Network.AnchoredFragment as AF @@ -272,7 +271,7 @@ runChairman tracer networkId runningTime socketPaths cModeParams secParam = do , localNodeSocketPath = socketPath } chairmanChainSyncClient = LocalChainSyncClient $ - chainSyncClient (showTracing tracer) socketPath chainsVar secParam + chainSyncClient (show >$< tracer) socketPath chainsVar secParam protocolsInMode = LocalNodeClientProtocols { localChainSyncClient = chairmanChainSyncClient , localTxSubmissionClient = Nothing diff --git a/cardano-node-chairman/app/Cardano/Chairman/Commands/Run.hs b/cardano-node-chairman/app/Cardano/Chairman/Commands/Run.hs index 66b645adf46..a7e68fdd855 100644 --- a/cardano-node-chairman/app/Cardano/Chairman/Commands/Run.hs +++ b/cardano-node-chairman/app/Cardano/Chairman/Commands/Run.hs @@ -24,7 +24,7 @@ import Ouroboros.Consensus.Config.SupportsNode import Ouroboros.Consensus.Node.ProtocolInfo import Control.Monad.Class.MonadTime.SI (DiffTime) -import Control.Tracer (Tracer (..), stdoutTracer) +import Control.Tracer (Tracer, mkTracer, stdoutTracer, traceWith) import Data.Monoid (Last (..)) import qualified Data.Time.Clock as DTC import Options.Applicative @@ -113,14 +113,14 @@ run RunOpts Left err -> putStrLn (docToString $ prettyError err) >> exitFailure Right p -> pure p - let (k , nId) = case p of - SomeConsensusProtocol _ runP -> - let ProtocolInfo { pInfoConfig } = fst $ Api.protocolInfo @IO runP - in ( Consensus.configSecurityParam pInfoConfig - , fromNetworkMagic . getNetworkMagic $ Consensus.configBlock pInfoConfig - ) + (k , nId) <- case p of + SomeConsensusProtocol _ runP -> do + ProtocolInfo { pInfoConfig } <- fst <$> Api.protocolInfo @IO runP + pure ( Consensus.configSecurityParam pInfoConfig + , fromNetworkMagic . getNetworkMagic $ Consensus.configBlock pInfoConfig + ) - consensusModeParams = getConsensusMode k ptclConfig + let consensusModeParams = getConsensusMode k ptclConfig chairmanTest (timed stdoutTracer) @@ -146,10 +146,10 @@ run RunOpts getLast pncProtocolConfig timed :: Tracer IO a -> Tracer IO a -timed (Tracer runTracer) = Tracer $ \a -> do +timed tr = mkTracer $ \a -> do ts <- DTC.getCurrentTime IO.putStr ("[" <> show ts <> "] ") - runTracer a + traceWith tr a cmdRun :: Mod CommandFields (IO ()) cmdRun = command "run" $ flip info idm $ run <$> parseRunOpts diff --git a/cardano-node-chairman/cardano-node-chairman.cabal b/cardano-node-chairman/cardano-node-chairman.cabal index fe0eaccea9c..e3b9ea28770 100644 --- a/cardano-node-chairman/cardano-node-chairman.cabal +++ b/cardano-node-chairman/cardano-node-chairman.cabal @@ -51,7 +51,7 @@ executable cardano-node-chairman , contra-tracer , io-classes:{io-classes, strict-stm, si-timers} ^>= 1.8 , optparse-applicative - , ouroboros-consensus:{ouroboros-consensus, cardano} + , ouroboros-consensus , ouroboros-network:{api, protocols} , text , time @@ -67,7 +67,7 @@ test-suite chairman-tests build-depends: , cardano-api , cardano-testnet - , cardano-crypto-class ^>=2.3 + , cardano-crypto-class ^>=2.5 , data-default-class , filepath , hedgehog diff --git a/cardano-node/app/DBSynthesizer/Parsers.hs b/cardano-node/app/DBSynthesizer/Parsers.hs new file mode 100644 index 00000000000..aac3ecdecfd --- /dev/null +++ b/cardano-node/app/DBSynthesizer/Parsers.hs @@ -0,0 +1,161 @@ +module DBSynthesizer.Parsers (parseCommandLine) where + +import Cardano.Node.Types (KESSource (..), ProtocolFilepaths (..)) +import Cardano.Tools.DBSynthesizer.Types +import Data.Word (Word64) +import Options.Applicative as Opt +import Ouroboros.Consensus.Block.Abstract (SlotNo (..)) + +parseCommandLine :: IO (FilePath, FilePath, ProtocolFilepaths, DBSynthesizerOptions) +parseCommandLine = + Opt.customExecParser p opts + where + p = Opt.prefs Opt.showHelpOnEmpty + opts = Opt.info parserCommandLine mempty + +parserCommandLine :: Parser (FilePath, FilePath, ProtocolFilepaths, DBSynthesizerOptions) +parserCommandLine = + (,,,) + <$> parseNodeConfigFilePath + <*> parseChainDBFilePath + <*> parseProtocolFilepaths + <*> parseDBSynthesizerOptions + +-- | The forging credentials, as file paths. Byron delegation credentials are +-- not wired up (the synthesizer forges Shelley-based blocks); the KES key path, +-- when given, is interpreted as a key file (not a KES agent socket). +parseProtocolFilepaths :: Parser ProtocolFilepaths +parseProtocolFilepaths = + mkFilepaths + <$> optional parseKesKeyFilePath + <*> optional parseVrfKeyFilePath + <*> optional parseOperationalCertFilePath + <*> optional parseBulkFilePath + where + mkFilepaths mKes mVrf mCert mBulk = + ProtocolFilepaths + { byronCertFile = Nothing + , byronKeyFile = Nothing + , shelleyKESSource = KESKeyFilePath <$> mKes + , shelleyVRFFile = mVrf + , shelleyCertFile = mCert + , shelleyBulkCredsFile = mBulk + } + +parseDBSynthesizerOptions :: Parser DBSynthesizerOptions +parseDBSynthesizerOptions = + DBSynthesizerOptions + <$> parseForgeOptions + <*> parseOpenMode + +parseForgeOptions :: Parser ForgeLimit +parseForgeOptions = + ForgeLimitSlot <$> parseSlotLimit + <|> ForgeLimitBlock <$> parseBlockLimit + <|> ForgeLimitEpoch <$> parseEpochLimit + +parseChainDBFilePath :: Parser FilePath +parseChainDBFilePath = + strOption + ( long "db" + <> metavar "PATH" + <> help "Path to the Chain DB" + <> completer (bashCompleter "directory") + ) + +parseNodeConfigFilePath :: Parser FilePath +parseNodeConfigFilePath = + strOption + ( long "config" + <> metavar "FILE" + <> help "Path to the node's config.json" + <> completer (bashCompleter "file") + ) + +parseOperationalCertFilePath :: Parser FilePath +parseOperationalCertFilePath = + strOption + ( long "shelley-operational-certificate" + <> metavar "FILE" + <> help "Path to the delegation certificate (in JSON TextEnvelope format)" + <> completer (bashCompleter "file") + ) + +parseKesKeyFilePath :: Parser FilePath +parseKesKeyFilePath = + strOption + ( long "shelley-kes-key" + <> metavar "FILE" + <> help "Path to the KES signing key (in JSON TextEnvelope format)" + <> completer (bashCompleter "file") + ) + +parseVrfKeyFilePath :: Parser FilePath +parseVrfKeyFilePath = + strOption + ( long "shelley-vrf-key" + <> metavar "FILE" + <> help "Path to the VRF signing key (in JSON TextEnvelope format)" + <> completer (bashCompleter "file") + ) + +parseBulkFilePath :: Parser FilePath +parseBulkFilePath = + strOption + ( long "bulk-credentials-file" + <> metavar "FILE" + <> help + "Path to the bulk credentials file (a JSON file containing an array of arrays containing 3 TextEnvelope objects for the opcert, VRF Signing key, KES signing key)" + <> completer (bashCompleter "file") + ) + +parseSlotLimit :: Parser SlotNo +parseSlotLimit = + SlotNo + <$> option + auto + ( short 's' + <> long "slots" + <> metavar "NUMBER" + <> help "Amount of slots to process" + ) + +parseBlockLimit :: Parser Word64 +parseBlockLimit = + option + auto + ( short 'b' + <> long "blocks" + <> metavar "NUMBER" + <> help "Amount of blocks to forge" + ) + +parseEpochLimit :: Parser Word64 +parseEpochLimit = + option + auto + ( short 'e' + <> long "epochs" + <> metavar "NUMBER" + <> help "Amount of epochs to process" + ) + +parseForce :: Parser Bool +parseForce = + switch + ( short 'f' + <> help "Force overwrite an existing Chain DB" + ) + +parseAppend :: Parser Bool +parseAppend = + switch + ( short 'a' + <> help "Append to an existing Chain DB" + ) + +parseOpenMode :: Parser DBSynthesizerOpenMode +parseOpenMode = + (parseForce *> pure OpenCreateForce) + <|> (parseAppend *> pure OpenAppend) + <|> pure OpenCreate diff --git a/cardano-node/app/cardano-node.hs b/cardano-node/app/cardano-node.hs index 563193bd652..b52c80289f5 100644 --- a/cardano-node/app/cardano-node.hs +++ b/cardano-node/app/cardano-node.hs @@ -4,21 +4,32 @@ {-# LANGUAGE RankNTypes #-} {-# LANGUAGE TemplateHaskell #-} +import qualified Cardano.Configuration as Cfg +import qualified Cardano.Configuration.CliArgs as CliArgs +import qualified Cardano.Configuration.Commands as Cmds import qualified Cardano.Crypto.Init as Crypto import Cardano.Git.Rev (gitRev) -import Cardano.Node.Configuration.POM (PartialNodeConfiguration (..)) +import Cardano.Node.Configuration.CardanoConfigAdapter + (cardanoConfigToNodeConfiguration) +import Cardano.Node.Configuration.CardanoConfigCompare + (compareConfigurations) +import Cardano.Node.Configuration.POM (NodeConfiguration (..), + PartialNodeConfiguration (..), defaultPartialNodeConfiguration, + makeNodeConfiguration, parseNodeConfigurationFP) import Cardano.Node.Handlers.TopLevel import Cardano.Node.Parsers (nodeCLIParser) import Cardano.Node.Run (runNode) import Cardano.Node.Tracing.Documentation (TraceDocumentationCmd (..), parseTraceDocumentationCmd, runTraceDocumentationCmd) +import Cardano.Node.Types (ConfigYamlFilePath (..)) -import Data.Monoid (Last (getLast)) +import Data.Monoid (Last (..)) import qualified Data.Text as Text import qualified Data.Text.IO as Text import Data.Version (showVersion) import Options.Applicative import qualified Options.Applicative as Opt +import System.Exit (exitFailure) import System.Info (arch, compilerName, compilerVersion, os) import System.IO (hPutStrLn, stderr) @@ -37,6 +48,7 @@ main = do runNode args TraceDocumentation tdc -> runTraceDocumentationCmd tdc VersionCmd -> runVersionCommand + ConfigCmd act -> act where p = Opt.prefs Opt.showHelpOnEmpty @@ -56,6 +68,7 @@ main = do Opt.info (fmap RunCmd nodeCLIParser <|> fmap TraceDocumentation parseTraceDocumentationCmd <|> parseVersionCmd + <|> fmap ConfigCmd configSubcommands <**> helper) ( Opt.fullDesc <> @@ -66,6 +79,7 @@ main = do data Command = RunCmd PartialNodeConfiguration | TraceDocumentation TraceDocumentationCmd | VersionCmd + | ConfigCmd (IO ()) -- Yes! A --version flag or version command. Either guess is right! parseVersionCmd :: Parser Command @@ -105,3 +119,84 @@ command' c descr p = [ command c (info (p <**> helper) $ mconcat [ progDesc descr ]) , metavar c ] + +-- cardano-config subcommands -------------------------------------------------- + +-- | The @migrate@, @schema@ and @resolve@ subcommands, spliced from the shared +-- @cardano-config:commands@ sublibrary. @migrate@ and @schema@ are +-- cardano-config's own commands, unchanged; @resolve@ is a node-specific variant +-- (see 'resolveDualCommand') that additionally cross-checks the node's own parser +-- against cardano-config's. +configSubcommands :: Parser (IO ()) +configSubcommands = + Opt.hsubparser + ( Opt.commandGroup "Configuration commands:" + <> Cmds.migrateCommand + <> Cmds.schemaCommand + <> resolveDualCommand + ) + +-- | A node-specific @resolve@: resolve the configuration with cardano-config +-- (printing the result as YAML, exactly like cardano-config's own @resolve@), +-- then re-resolve the same configuration with the node's own POM parser and +-- report any discrepancies between the two. Exits non-zero when they disagree, +-- so it doubles as a CI parity check while the node still has two parsers. +resolveDualCommand :: Mod CommandFields (IO ()) +resolveDualCommand = + command "resolve" + ( info + (runDualResolve <$> Cmds.resolveOptionsParser) + ( progDesc + ( "Resolve a cardano-node configuration (defaults + file + CLI) with both the " + <> "node and cardano-config parsers, print the result as YAML, and report any " + <> "discrepancies between the two parsers (exit non-zero if they disagree)." + ) + ) + ) + +runDualResolve :: Cmds.ResolveOptions -> IO () +runDualResolve resolveOpts@(Cmds.ResolveOptions cli _geneses) = do + -- Print the resolved configuration using cardano-config's own renderer (which + -- honours --with-geneses); this also terminates via 'die' if resolution fails. + Cmds.runResolveCommand resolveOpts + -- Cross-check: resolve the same inputs with the node's POM parser and diff. + discrepancies <- resolveDiscrepancies cli + case discrepancies of + [] -> + putStrLn "resolve: the node and cardano-config parsers agree on the resolved configuration." + ds -> do + hPutStrLn stderr $ + "resolve: " <> show (length ds) + <> " discrepancy(ies) between the node and cardano-config parsers:" + mapM_ (hPutStrLn stderr . (" - " <>)) ds + exitFailure + +-- | Resolve the configuration file (+ CLI) both ways and return the divergences. +-- The node (POM) side takes its CLI-supplied, file-absent fields (topology / +-- database / protocol files / socket) from the shared cardano-config resolution, +-- so the diff reflects how the two parsers read the configuration FILE (plus the +-- documented adapter gaps) rather than an independent — and necessarily +-- asymmetric — CLI reverse-mapping. +resolveDiscrepancies :: Cfg.CliArgs -> IO [String] +resolveDiscrepancies cli = do + (fileCfg, _warns) <- Cfg.parseConfigurationFiles configFp + case Cfg.resolveConfiguration cli fileCfg of + Left err -> pure ["cardano-config failed to resolve the configuration: " <> show err] + Right (cfgNc, _) -> + case cardanoConfigToNodeConfiguration cfgNc of + Left adaptErr -> pure ["cardano-config configuration could not be adapted: " <> adaptErr] + Right adaptedNc -> do + filePartial <- parseNodeConfigurationFP (Just (ConfigYamlFilePath configFp)) + let withCli = + (defaultPartialNodeConfiguration <> filePartial) + { pncConfigFile = Last (Just (ConfigYamlFilePath configFp)) + , pncTopologyFile = Last (Just (ncTopologyFile adaptedNc)) + , pncDatabaseFile = Last (Just (ncDatabaseFile adaptedNc)) + , pncProtocolFiles = Last (Just (ncProtocolFiles adaptedNc)) + , pncSocketConfig = Last (Just (ncSocketConfig adaptedNc)) + } + case makeNodeConfiguration withCli of + Left err -> pure ["node parser (makeNodeConfiguration) failed: " <> err] + Right pomNc -> pure (compareConfigurations pomNc adaptedNc) + where + configFp = CliArgs.configFilePath cli diff --git a/cardano-node/app/db-synthesizer.hs b/cardano-node/app/db-synthesizer.hs new file mode 100644 index 00000000000..d31782cd262 --- /dev/null +++ b/cardano-node/app/db-synthesizer.hs @@ -0,0 +1,25 @@ +-- | This tool synthesizes a valid ChainDB, replicating cardano-node's UX. +-- +-- Usage: db-synthesizer --config FILE --db PATH +-- [--shelley-operational-certificate FILE] +-- [--shelley-vrf-key FILE] [--shelley-kes-key FILE] +-- [--bulk-credentials-file FILE] +-- ((-s|--slots NUMBER) | (-b|--blocks NUMBER) | +-- (-e|--epochs NUMBER)) [-f | -a] +-- +-- The node configuration and forging credentials are turned into a Cardano +-- 'ProtocolInfo' and block forgers using cardano-node's own protocol-instantiation +-- machinery (see "Cardano.Node.Tools.DBSynthesizer"); the actual forging is done +-- by @ouroboros-consensus@'s @synthesize@. +module Main (main) where + +import Cardano.Crypto.Init (cryptoInit) +import Cardano.Node.Tools.DBSynthesizer (synthesizeFromConfig) +import DBSynthesizer.Parsers (parseCommandLine) + +main :: IO () +main = do + cryptoInit + (configFp, dbDir, protocolFiles, opts) <- parseCommandLine + result <- synthesizeFromConfig configFp protocolFiles opts dbDir + putStrLn $ "--> done; result: " ++ show result diff --git a/cardano-node/cardano-node.cabal b/cardano-node/cardano-node.cabal index 6f1441ca94e..bb642271cd7 100644 --- a/cardano-node/cardano-node.cabal +++ b/cardano-node/cardano-node.cabal @@ -59,7 +59,9 @@ library hs-source-dirs: src - exposed-modules: Cardano.Node.Configuration.NodeAddress + exposed-modules: Cardano.Node.Configuration.CardanoConfigAdapter + Cardano.Node.Configuration.CardanoConfigCompare + Cardano.Node.Configuration.NodeAddress Cardano.Node.Configuration.POM Cardano.Node.Configuration.LedgerDB Cardano.Node.Configuration.Socket @@ -113,6 +115,7 @@ library Cardano.Node.Tracing.Tracers.Shutdown Cardano.Node.Tracing.Tracers.HasIssuer Cardano.Node.Tracing.Tracers.Startup + Cardano.Node.Tools.DBSynthesizer Cardano.Node.Types other-modules: Paths_cardano_node @@ -124,8 +127,9 @@ library , base16-bytestring , bytestring , cardano-api ^>= 11.3 + , cardano-config , cardano-data - , cardano-crypto-class ^>=2.3 + , cardano-crypto-class ^>=2.5 , cardano-crypto-wrapper , cardano-git-rev ^>=0.2.2 , cardano-ledger-alonzo @@ -139,23 +143,25 @@ library , cardano-ledger-dijkstra , cardano-ledger-shelley , cardano-prelude + , cardano-protocol ^>= 0.1 , cardano-protocol-tpraos >= 1.4 , cardano-slotting >= 0.2 , cardano-rpc ^>= 11.0 , cborg ^>= 0.2.4 , containers - , contra-tracer + , contra-tracer >= 0.2.1 , data-default-class , deepseq , directory , dns , ekg-core , filepath + , fs-api , generic-data , hashable , hostname , io-classes:{io-classes,strict-stm,si-timers} ^>= 1.8 - , kes-agent ^>=1.2 + , kes-agent ^>=1.3 , microlens , mmap , network-mux @@ -164,8 +170,8 @@ library , network-mux >= 0.8 , nothunks , optparse-applicative - , ouroboros-consensus:{ouroboros-consensus, lmdb, lsm, cardano, diffusion, protocol} ^>= 3.0.1 - , ouroboros-network:{api, ouroboros-network, orphan-instances, framework, protocols, framework-tracing, tracing} ^>= 1.1 + , ouroboros-consensus:{ouroboros-consensus, lsm, cardano, diffusion, protocol, unstable-cardano-tools} ^>= 3.0.1 + , ouroboros-network:{api, ouroboros-network, orphan-instances, framework, protocols, tracing} ^>= 1.1 , cardano-diffusion:{api, cardano-diffusion, tracing, orphan-instances} ^>=1.0 , prettyprinter , prettyprinter-ansi-terminal @@ -181,11 +187,12 @@ library , sop-extras , text >= 2.0 , time - , trace-dispatcher ^>= 2.12.0 + , trace-dispatcher ^>= 2.13.0 , trace-forward ^>= 2.4.0 , trace-resources ^>= 0.2.4 , transformers , transformers-except + , tree-diff , typed-protocols:{typed-protocols, stateful} >= 1.2 , yaml @@ -205,12 +212,54 @@ executable cardano-node autogen-modules: Paths_cardano_node build-depends: base + , cardano-config + , cardano-config:commands , cardano-crypto-class , cardano-git-rev , cardano-node , optparse-applicative , text +executable db-synthesizer + import: project-config + hs-source-dirs: app + main-is: db-synthesizer.hs + ghc-options: -threaded + -rtsopts + + other-modules: DBSynthesizer.Parsers + + build-depends: base + , cardano-crypto-class + , cardano-node + , optparse-applicative + , ouroboros-consensus:{ouroboros-consensus, unstable-cardano-tools} ^>= 3.0.1 + +test-suite db-synthesizer-test + import: project-config + hs-source-dirs: test/db-synthesizer + main-is: Main.hs + type: exitcode-stdio-1.0 + + build-depends: base + , cardano-crypto-class + , cardano-node + , ouroboros-consensus:{ouroboros-consensus, cardano, unstable-cardano-tools} ^>= 3.0.1 + , tasty + , tasty-hunit + +test-suite cardano-config-compare-test + import: project-config + hs-source-dirs: test/cardano-config-compare + main-is: Main.hs + type: exitcode-stdio-1.0 + + build-depends: base + , cardano-config + , cardano-node + , tasty + , tasty-hunit + test-suite cardano-node-test import: project-config , maybe-unix @@ -231,6 +280,7 @@ test-suite cardano-node-test , contra-tracer , directory , filepath + , fs-api , hedgehog , hedgehog-corpus , hedgehog-extras ^>= 0.10 @@ -239,7 +289,6 @@ test-suite cardano-node-test , ouroboros-consensus:{ouroboros-consensus, diffusion} , ouroboros-network:{api, framework, ouroboros-network} , text - , trace-dispatcher , transformers , vector , yaml diff --git a/cardano-node/src/Cardano/Node/Configuration/CardanoConfigAdapter.hs b/cardano-node/src/Cardano/Node/Configuration/CardanoConfigAdapter.hs new file mode 100644 index 00000000000..ce8d698dc79 --- /dev/null +++ b/cardano-node/src/Cardano/Node/Configuration/CardanoConfigAdapter.hs @@ -0,0 +1,402 @@ +{-# LANGUAGE ScopedTypeVariables #-} + +-- | Adapter from @cardano-config@'s resolved configuration to the node's own +-- 'NodeConfiguration' (the POM one). +-- +-- The node currently runs both parsers and will eventually drop the legacy POM +-- parser, at which point @cardano-config@ must produce the 'NodeConfiguration' +-- that starts consensus and networking. This adapter is that eventual +-- replacement: it maps @cardano-config@'s resolved values onto a +-- 'PartialNodeConfiguration' and runs the node's own 'makeNodeConfiguration', so +-- fields cardano-config supplies come from cardano-config and the rest fall back +-- to the node defaults. Fields not yet mapped are listed in 'adapterGaps' — that +-- gap list is exactly what must be closed before POM can be dropped, and any +-- gap also shows up concretely as a divergence in +-- 'Cardano.Node.Configuration.CardanoConfigCompare.compareConfigurations'. +module Cardano.Node.Configuration.CardanoConfigAdapter + ( cardanoConfigToNodeConfiguration + , cardanoConfigToPartialNodeConfiguration + , nodeProtocolConfigurationFromCardanoConfig + , adapterGaps + ) where + +import Cardano.Api (File (..)) +import qualified Cardano.Configuration as Cfg +import Cardano.Crypto (RequiresNetworkMagic (..)) +import Cardano.Ledger.BaseTypes (strictMaybeToMaybe) +import Cardano.Ledger.BaseTypes.NonZero (nonZero) +import Cardano.Network.ConsensusMode (ConsensusMode (..)) +import Cardano.Network.NodeToNode (DiffusionMode (..)) +import Cardano.Network.PeerSelection (NumberOfBigLedgerPeers (..)) +import Cardano.Node.Configuration.LedgerDB (LedgerDbConfiguration (..), + LedgerDbSelectorFlag (..), noDeprecatedOptions) +import Cardano.Node.Configuration.POM (NodeConfiguration, + PartialNodeConfiguration (..), ResponderCoreAffinityPolicy (..), + defaultPartialNodeConfiguration, makeNodeConfiguration) +import Cardano.Node.Configuration.Socket (SocketConfig (..)) +import Cardano.Node.Handlers.Shutdown (ShutdownConfig (..), + ShutdownOn (..)) +import Cardano.Node.Types (CheckpointsFile (..), CheckpointsHash (..), + ConfigYamlFilePath (..), GenesisFile (..), + GenesisHash (..), KESSource (..), MaxConcurrencyBulkSync (..), + MaxConcurrencyDeadline (..), + NodeAlonzoProtocolConfiguration (..), + NodeByronProtocolConfiguration (..), + NodeCheckpointsConfiguration (..), + NodeConwayProtocolConfiguration (..), + NodeDijkstraProtocolConfiguration (..), + NodeHardForkProtocolConfiguration (..), + NodeProtocolConfiguration (..), + NodeShelleyProtocolConfiguration (..), ProtocolFilepaths (..), + TopologyFile (..)) +import Cardano.Slotting.Block (BlockNo (..)) +import Cardano.Slotting.Slot (EpochNo (..), SlotNo (..)) +import Cardano.Rpc.Server.Config (RpcConfigF (..)) +import Data.Functor.Identity (runIdentity) +import Data.Monoid (Last (..)) +import Data.Time.Clock (secondsToDiffTime) +import Ouroboros.Consensus.Node (NodeDatabasePaths (..)) +import Ouroboros.Consensus.Node.Genesis (GenesisConfigFlags (..), + defaultGenesisConfigFlags) +import Ouroboros.Consensus.Ledger.SupportsMempool (ByteSize32 (..)) +import Ouroboros.Consensus.Mempool (MempoolCapacityBytesOverride (..)) +import Ouroboros.Consensus.Storage.LedgerDB.Args (QueryBatchSize (..)) +import Ouroboros.Consensus.Storage.LedgerDB.Snapshots + (NumOfDiskSnapshots (..), SnapshotDelayRange (..), + SnapshotFrequency (..), SnapshotFrequencyArgs (..), + SnapshotPolicyArgs (..), defaultSnapshotPolicyArgs, + mithrilSnapshotPolicyArgs) +import Ouroboros.Consensus.Util.Args (OverrideOrDefault (..)) +import Ouroboros.Network.PeerSelection.PeerSharing (PeerSharing (..)) +import Ouroboros.Network.Server.RateLimiting (AcceptedConnectionsLimit (..)) +import Ouroboros.Network.TxSubmission.Inbound.V2.Types + (TxSubmissionInitDelay (..), TxSubmissionLogicVersion (..)) +import System.FilePath (takeDirectory, ()) + +-- | Build the node's 'NodeConfiguration' from a @cardano-config@-resolved +-- configuration, reusing the node's own 'makeNodeConfiguration'. Fields +-- cardano-config does not yet supply keep the node defaults (see 'adapterGaps'). +cardanoConfigToNodeConfiguration :: Cfg.NodeConfiguration -> Either String NodeConfiguration +cardanoConfigToNodeConfiguration = + makeNodeConfiguration . cardanoConfigToPartialNodeConfiguration + +-- | Map the @cardano-config@-resolved values onto a 'PartialNodeConfiguration', +-- overriding the node defaults for every field cardano-config supplies. +cardanoConfigToPartialNodeConfiguration :: Cfg.NodeConfiguration -> PartialNodeConfiguration +cardanoConfigToPartialNodeConfiguration cfg = + defaultPartialNodeConfiguration + { pncConfigFile = Last (Just (ConfigYamlFilePath (Cfg.configFilePath cfg))) + , pncTopologyFile = Last (Just (TopologyFile (Cfg.topologyFile cfg))) + , pncValidateDB = Last (Just (Cfg.validateDatabase cfg)) + , pncStartAsNonProducingNode = Last (Just (runIdentity (Cfg.startAsNonProducingNode protoCfg))) + , pncProtocolConfig = Last (Just (nodeProtocolConfigurationFromCardanoConfig cfg)) + , pncProtocolFiles = Last (Just (credentialsToProtocolFilepaths (Cfg.credentials cfg))) + , pncExperimentalProtocolsEnabled = Last (Just (runIdentity (Cfg.experimentalProtocolsEnabled netCfg))) + , pncMempoolTimeoutSoft = Last (Just (runIdentity (Cfg.mempoolTimeoutSoft mempCfg))) + , pncMempoolTimeoutHard = Last (Just (runIdentity (Cfg.mempoolTimeoutHard mempCfg))) + , pncMempoolTimeoutCapacity = Last (Just (runIdentity (Cfg.mempoolTimeoutCapacity mempCfg))) + , pncMinBigLedgerPeersForTrustedState = + Last (Just (NumberOfBigLedgerPeers (runIdentity (Cfg.minBigLedgerPeersForTrustedState netCfg)))) + , -- Peer-selection targets: deadline targets are optional (StrictMaybe), + -- sync targets are always resolved (Identity). Map them all. + pncDeadlineTargetOfRootPeers = + Last (strictMaybeToMaybe (Cfg.deadlineTargetOfRootPeers netCfg)) + , pncDeadlineTargetOfKnownPeers = + Last (strictMaybeToMaybe (Cfg.deadlineTargetOfKnownPeers netCfg)) + , pncDeadlineTargetOfEstablishedPeers = + Last (strictMaybeToMaybe (Cfg.deadlineTargetOfEstablishedPeers netCfg)) + , pncDeadlineTargetOfActivePeers = + Last (strictMaybeToMaybe (Cfg.deadlineTargetOfActivePeers netCfg)) + , pncDeadlineTargetOfKnownBigLedgerPeers = + Last (strictMaybeToMaybe (Cfg.deadlineTargetOfKnownBigLedgerPeers netCfg)) + , pncDeadlineTargetOfEstablishedBigLedgerPeers = + Last (strictMaybeToMaybe (Cfg.deadlineTargetOfEstablishedBigLedgerPeers netCfg)) + , pncDeadlineTargetOfActiveBigLedgerPeers = + Last (strictMaybeToMaybe (Cfg.deadlineTargetOfActiveBigLedgerPeers netCfg)) + , pncSyncTargetOfRootPeers = + Last (Just (runIdentity (Cfg.syncTargetOfRootPeers netCfg))) + , pncSyncTargetOfKnownPeers = + Last (Just (runIdentity (Cfg.syncTargetOfKnownPeers netCfg))) + , pncSyncTargetOfEstablishedPeers = + Last (Just (runIdentity (Cfg.syncTargetOfEstablishedPeers netCfg))) + , pncSyncTargetOfActivePeers = + Last (Just (runIdentity (Cfg.syncTargetOfActivePeers netCfg))) + , pncSyncTargetOfKnownBigLedgerPeers = + Last (Just (runIdentity (Cfg.syncTargetOfKnownBigLedgerPeers netCfg))) + , pncSyncTargetOfEstablishedBigLedgerPeers = + Last (Just (runIdentity (Cfg.syncTargetOfEstablishedBigLedgerPeers netCfg))) + , pncSyncTargetOfActiveBigLedgerPeers = + Last (Just (runIdentity (Cfg.syncTargetOfActiveBigLedgerPeers netCfg))) + , pncDatabaseFile = Last (Just (fromCfgDbPaths (runIdentity (Cfg.databasePath storeCfg)))) + , pncDiffusionMode = Last (Just (fromCfgDiffusionMode (runIdentity (Cfg.diffusionMode netCfg)))) + , pncMaxConcurrencyBulkSync = + Last (Just (MaxConcurrencyBulkSync (runIdentity (Cfg.maxConcurrencyBulkSync netCfg)))) + , pncMaxConcurrencyDeadline = + Last (Just (MaxConcurrencyDeadline (runIdentity (Cfg.maxConcurrencyDeadline netCfg)))) + , pncTxSubmissionInitDelay = + Last (Just (TxSubmissionInitDelay (runIdentity (Cfg.txSubmissionInitDelay netCfg)))) + , pncAcceptedConnectionsLimit = + Last (Just (fromCfgAcceptedConnLimit (runIdentity (Cfg.acceptedConnectionsLimit netCfg)))) + , pncConsensusMode = Last (Just (fromCfgConsensusMode consensusModeVal)) + , pncPeerSharing = + Last (fmap toPeerSharing (strictMaybeToMaybe (Cfg.peerSharing netCfg))) + , pncMaybeMempoolCapacityOverride = + Last (fmap (MempoolCapacityBytesOverride . ByteSize32 . fromIntegral) + (strictMaybeToMaybe (Cfg.mempoolCapacityOverride mempCfg))) + , pncShutdownConfig = + Last (Just (ShutdownConfig + (strictMaybeToMaybe (Cfg.shutdownIPC cfg)) + (fmap toNodeShutdownOn (strictMaybeToMaybe (Cfg.shutdownOnTarget cfg))))) + , pncResponderCoreAffinityPolicy = + Last (Just (fromCfgAffinity (runIdentity (Cfg.responderCoreAffinityPolicy netCfg)))) + , pncTxSubmissionLogicVersion = + Last (Just (fromCfgTxSubmissionLogic (runIdentity (Cfg.txSubmissionLogicVersion netCfg)))) + , -- The Genesis tuning flags only feed 'ncGenesisConfig' when the node runs + -- in Genesis mode (see 'makeNodeConfiguration'); in Praos mode the node + -- ignores them, so mirror POM and keep the defaults there. + pncGenesisConfigFlags = + Last (Just (case consensusModeVal of + Cfg.GenesisMode flags -> fromCfgGenesisFlags flags + Cfg.PraosMode -> defaultGenesisConfigFlags)) + , -- Only the local (IPC) socket path lives in the configuration file; the + -- node-to-node IPv4/IPv6/port bindings are CLI-only in the node, so they + -- stay empty here exactly as POM leaves them when parsing config alone. + pncSocketConfig = + Last (Just (SocketConfig mempty mempty mempty + (Last (fmap File (strictMaybeToMaybe (Cfg.socketPath lcc)))))) + , -- 'nodeSocketPath' (third field) is filled in by 'makeNodeConfiguration' + -- from the resolved socket config, so leave it empty here. + pncRpcConfig = + RpcConfig + (Last (Just (runIdentity (Cfg.enableGrpc lcc)))) + (Last (fmap File (strictMaybeToMaybe (Cfg.grpcSocketPath lcc)))) + mempty + , -- Backend selector, query batch size and snapshot policy are all mapped + -- from cardano-config. 'DeprecatedOptions' has no cardano-config + -- counterpart (they are the legacy top-level SnapshotInterval / + -- NumOfDiskSnapshots keys), so it keeps the node's empty default. + pncLedgerDbConfig = + Last (Just (LedgerDbConfiguration + (fromCfgSnapshotPolicy (strictMaybeToMaybe (Cfg.snapshots ledgerDbCfg))) + (maybe DefaultQueryBatchSize RequestedQueryBatchSize + (strictMaybeToMaybe (Cfg.queryBatchSize ledgerDbCfg))) + (maybe V2InMemory fromCfgBackend + (strictMaybeToMaybe (Cfg.backendSelector ledgerDbCfg))) + noDeprecatedOptions)) + } + where + protoCfg = Cfg.protocolConfiguration cfg + netCfg = Cfg.networkConfiguration cfg + mempCfg = Cfg.mempoolConfiguration cfg + storeCfg = Cfg.storageConfiguration cfg + lcc = Cfg.localConnectionsConfig cfg + ledgerDbCfg = runIdentity (Cfg.ledgerDbConfiguration storeCfg) + consensusModeVal = runIdentity (Cfg.getConsensusConfiguration (Cfg.consensusConfiguration cfg)) + + fromCfgDbPaths :: Cfg.NodeDatabasePaths -> NodeDatabasePaths + fromCfgDbPaths (Cfg.SingleDB p) = OnePathForAllDbs p + fromCfgDbPaths (Cfg.SplitDB imm vol) = MultipleDbPaths imm vol + + fromCfgDiffusionMode :: Cfg.DiffusionMode -> DiffusionMode + fromCfgDiffusionMode Cfg.InitiatorOnly = InitiatorOnlyDiffusionMode + fromCfgDiffusionMode Cfg.InitiatorAndResponder = InitiatorAndResponderDiffusionMode + + fromCfgAcceptedConnLimit :: Cfg.AcceptedConnectionsLimit -> AcceptedConnectionsLimit + fromCfgAcceptedConnLimit c = + AcceptedConnectionsLimit + { acceptedConnectionsHardLimit = Cfg.hardLimit c + , acceptedConnectionsSoftLimit = Cfg.softLimit c + , acceptedConnectionsDelay = Cfg.delayOnSoftLimit c + } + + fromCfgConsensusMode :: Cfg.ConsensusMode -> ConsensusMode + fromCfgConsensusMode Cfg.PraosMode = PraosMode + fromCfgConsensusMode (Cfg.GenesisMode _) = GenesisMode + + toPeerSharing :: Bool -> PeerSharing + toPeerSharing True = PeerSharingEnabled + toPeerSharing False = PeerSharingDisabled + + toNodeShutdownOn :: Cfg.ShutdownOn -> ShutdownOn + toNodeShutdownOn (Cfg.ShutdownAtSlot w) = ASlot (SlotNo w) + toNodeShutdownOn (Cfg.ShutdownAtBlock w) = ABlock (BlockNo w) + + fromCfgAffinity :: Cfg.ResponderCoreAffinityPolicy -> ResponderCoreAffinityPolicy + fromCfgAffinity Cfg.NoResponderCoreAffinity = NoResponderCoreAffinity + fromCfgAffinity Cfg.ResponderCoreAffinity = ResponderCoreAffinity + + fromCfgTxSubmissionLogic :: Cfg.TxSubmissionLogicVersion -> TxSubmissionLogicVersion + fromCfgTxSubmissionLogic Cfg.TxSubmissionLogicV1 = TxSubmissionLogicV1 + fromCfgTxSubmissionLogic Cfg.TxSubmissionLogicV2 = TxSubmissionLogicV2 + + -- Map cardano-config's snapshot policy onto the node's 'SnapshotPolicyArgs', + -- mirroring how POM's LedgerDB parser builds it: a named Mithril policy + -- selects the predefined 'mithrilSnapshotPolicyArgs', a custom policy is + -- mapped field-by-field, and absence keeps the node default. + fromCfgSnapshotPolicy :: Maybe Cfg.SnapshotPolicy -> SnapshotPolicyArgs + fromCfgSnapshotPolicy Nothing = defaultSnapshotPolicyArgs + fromCfgSnapshotPolicy (Just Cfg.MithrilSnapshotPolicy) = mithrilSnapshotPolicyArgs + fromCfgSnapshotPolicy (Just (Cfg.CustomSnapshotPolicy opts)) = + SnapshotPolicyArgs + (SnapshotFrequency SnapshotFrequencyArgs + { sfaInterval = + maybe UseDefault Override (strictMaybeToMaybe (Cfg.snapshotInterval opts) >>= nonZero) + , sfaOffset = + maybe UseDefault (Override . SlotNo) (strictMaybeToMaybe (Cfg.slotOffset opts)) + , sfaRateLimit = + maybe UseDefault (Override . secondsToDiffTime . fromIntegral) + (strictMaybeToMaybe (Cfg.snapshotRateLimit opts)) + , sfaDelaySnapshotRange = + case (strictMaybeToMaybe (Cfg.minDelay opts), strictMaybeToMaybe (Cfg.maxDelay opts)) of + (Just mn, Just mx) -> + Override (SnapshotDelayRange (secondsToDiffTime (fromIntegral mn)) + (secondsToDiffTime (fromIntegral mx))) + _ -> UseDefault + }) + (maybe UseDefault (Override . NumOfDiskSnapshots . fromIntegral) + (strictMaybeToMaybe (Cfg.numOfDiskSnapshots opts))) + + fromCfgBackend :: Cfg.LedgerDbBackendSelector -> LedgerDbSelectorFlag + fromCfgBackend Cfg.V2InMemory = V2InMemory + fromCfgBackend (Cfg.V2LSM dbPath exportPath) = + V2LSM (strictMaybeToMaybe dbPath) (strictMaybeToMaybe exportPath) + + -- cardano-config's 'GenesisConfigFlags' mirrors the node's field-for-field, + -- except 'gcfCSJJumpSize' is a raw 'Word64' there vs a 'SlotNo' here, and the + -- optional fields are 'StrictMaybe' vs 'Maybe'. + fromCfgGenesisFlags :: Cfg.GenesisConfigFlags -> GenesisConfigFlags + fromCfgGenesisFlags f = + GenesisConfigFlags + (Cfg.gcfEnableCSJ f) + (Cfg.gcfEnableLoEAndGDD f) + (Cfg.gcfEnableLoP f) + (strictMaybeToMaybe (Cfg.gcfBlockFetchGracePeriod f)) + (strictMaybeToMaybe (Cfg.gcfBucketCapacity f)) + (strictMaybeToMaybe (Cfg.gcfBucketRate f)) + (fmap SlotNo (strictMaybeToMaybe (Cfg.gcfCSJJumpSize f))) + (strictMaybeToMaybe (Cfg.gcfGDDRateLimit f)) + +-- | Map @cardano-config@ 'Cfg.Credentials' (file paths) onto the node's +-- 'ProtocolFilepaths'. +credentialsToProtocolFilepaths :: Cfg.Credentials -> ProtocolFilepaths +credentialsToProtocolFilepaths c = + ProtocolFilepaths + { byronCertFile = strictMaybeToMaybe (Cfg.byronDelegationCertificate c) + , byronKeyFile = strictMaybeToMaybe (Cfg.byronSigningKey c) + , shelleyKESSource = fmap fromCfgKES (strictMaybeToMaybe (Cfg.shelleyKES c)) + , shelleyVRFFile = strictMaybeToMaybe (Cfg.shelleyVRFKey c) + , shelleyCertFile = strictMaybeToMaybe (Cfg.shelleyOperationalCertificate c) + , shelleyBulkCredsFile = strictMaybeToMaybe (Cfg.bulkCredentialsFile c) + } + where + fromCfgKES (Cfg.KESKeyFilePath fp) = KESKeyFilePath fp + fromCfgKES (Cfg.KESAgentSocketPath fp) = KESAgentSocketPath fp + +-- | Build the node's 'NodeProtocolConfiguration' from a @cardano-config@-resolved +-- configuration. Genesis file paths are resolved relative to the configuration +-- file's directory (the way cardano-config resolves them at read time). +nodeProtocolConfigurationFromCardanoConfig :: + Cfg.NodeConfiguration -> NodeProtocolConfiguration +nodeProtocolConfigurationFromCardanoConfig cfg = + NodeProtocolConfigurationCardano + byronConfig + shelleyConfig + alonzoConfig + conwayConfig + dijkstraConfig + hardforkConfig + checkpointsConfig + where + protoCfg = Cfg.protocolConfiguration cfg + testCfg = Cfg.testingConfiguration cfg + configDir = takeDirectory (Cfg.configFilePath cfg) + + genFile :: Cfg.Hashed FilePath -> GenesisFile + genFile h = GenesisFile (configDir Cfg.hashed h) + + genHash :: Cfg.Hashed FilePath -> Maybe GenesisHash + genHash h = Just (GenesisHash (Cfg.hash h)) + + byronGen = Cfg.byronGenesis protoCfg + byronConfig = + NodeByronProtocolConfiguration + { npcByronGenesisFile = genFile (Cfg.byronGenesisFile byronGen) + , npcByronGenesisFileHash = genHash (Cfg.byronGenesisFile byronGen) + , npcByronReqNetworkMagic = + maybe RequiresNoMagic fromCfgReqNetworkMagic + (strictMaybeToMaybe (Cfg.byronReqNetworkMagic byronGen)) + , npcByronPbftSignatureThresh = Nothing + , -- cardano-config does not model the Byron software (block) version. The + -- Byron era is genesis-only for synthesis (the test configuration + -- hard-forks to a Shelley-based era at epoch 0), so a fixed default is + -- used. This surfaces as a divergence against POM (see 'adapterGaps'). + npcByronSupportedProtocolVersionMajor = 1 + , npcByronSupportedProtocolVersionMinor = 0 + , npcByronSupportedProtocolVersionAlt = 0 + } + + shelleyConfig = + NodeShelleyProtocolConfiguration + (genFile (Cfg.shelleyGenesis protoCfg)) + (genHash (Cfg.shelleyGenesis protoCfg)) + alonzoConfig = + NodeAlonzoProtocolConfiguration + (genFile (Cfg.alonzoGenesis protoCfg)) + (genHash (Cfg.alonzoGenesis protoCfg)) + conwayConfig = + NodeConwayProtocolConfiguration + (genFile (Cfg.conwayGenesis protoCfg)) + (genHash (Cfg.conwayGenesis protoCfg)) + dijkstraConfig = + fmap + (\h -> NodeDijkstraProtocolConfiguration (genFile h) (genHash h)) + (strictMaybeToMaybe (Cfg.experimentalGenesis testCfg)) + + hardforkConfig = + NodeHardForkProtocolConfiguration + { npcExperimentalHardForksEnabled = runIdentity (Cfg.experimentalHardForksEnabled testCfg) + , npcTestShelleyHardForkAtEpoch = epochOf (Cfg.testShelleyHardForkAtEpoch testCfg) + , npcTestShelleyHardForkAtVersion = strictMaybeToMaybe (Cfg.testShelleyHardForkAtVersion testCfg) + , npcTestAllegraHardForkAtEpoch = epochOf (Cfg.testAllegraHardForkAtEpoch testCfg) + , npcTestAllegraHardForkAtVersion = strictMaybeToMaybe (Cfg.testAllegraHardForkAtVersion testCfg) + , npcTestMaryHardForkAtEpoch = epochOf (Cfg.testMaryHardForkAtEpoch testCfg) + , npcTestMaryHardForkAtVersion = strictMaybeToMaybe (Cfg.testMaryHardForkAtVersion testCfg) + , npcTestAlonzoHardForkAtEpoch = epochOf (Cfg.testAlonzoHardForkAtEpoch testCfg) + , npcTestAlonzoHardForkAtVersion = strictMaybeToMaybe (Cfg.testAlonzoHardForkAtVersion testCfg) + , npcTestBabbageHardForkAtEpoch = epochOf (Cfg.testBabbageHardForkAtEpoch testCfg) + , npcTestBabbageHardForkAtVersion = strictMaybeToMaybe (Cfg.testBabbageHardForkAtVersion testCfg) + , npcTestConwayHardForkAtEpoch = epochOf (Cfg.testConwayHardForkAtEpoch testCfg) + , npcTestConwayHardForkAtVersion = strictMaybeToMaybe (Cfg.testConwayHardForkAtVersion testCfg) + , npcTestDijkstraHardForkAtEpoch = epochOf (Cfg.testDijkstraHardForkAtEpoch testCfg) + , npcTestDijkstraHardForkAtVersion = strictMaybeToMaybe (Cfg.testDijkstraHardForkAtVersion testCfg) + } + + -- Optional checkpoints file (and hash), path resolved relative to the config + -- directory like the genesis files above. + checkpointsConfig = + case strictMaybeToMaybe (Cfg.checkpointsFile protoCfg) of + Nothing -> NodeCheckpointsConfiguration Nothing Nothing + Just mh -> + NodeCheckpointsConfiguration + (Just (CheckpointsFile (configDir Cfg.maybeHashed mh))) + (fmap CheckpointsHash (strictMaybeToMaybe (Cfg.maybeHash mh))) + + epochOf = fmap EpochNo . strictMaybeToMaybe + + fromCfgReqNetworkMagic :: Cfg.RequiresNetworkMagic -> RequiresNetworkMagic + fromCfgReqNetworkMagic Cfg.RequiresNoMagic = RequiresNoMagic + fromCfgReqNetworkMagic Cfg.RequiresMagic = RequiresMagic + +-- | Node 'NodeConfiguration' fields the adapter does not yet populate from +-- @cardano-config@ (they keep the node defaults, so they show up as divergences +-- against POM). Closing these is the remaining work before POM can be dropped. +adapterGaps :: [String] +adapterGaps = + [ "ncProtocolConfig: Byron supported-protocol-version — genuinely not modelled by" + <> " cardano-config; hard-coded default (Byron era is genesis-only for synthesis)" + , "ncTraceForwardSocket — CLI-only in both the node and cardano-config (POM leaves it" + <> " empty when parsing the config file, filling it only from the command line;" + <> " cardano-config's tracerSocket is likewise a CLI argument). It is absent from" + <> " the resolved configuration file, so there is nothing to map and it stays at" + <> " the node default (empty), exactly as POM does when parsing config alone." + ] diff --git a/cardano-node/src/Cardano/Node/Configuration/CardanoConfigCompare.hs b/cardano-node/src/Cardano/Node/Configuration/CardanoConfigCompare.hs new file mode 100644 index 00000000000..699b518c1d3 --- /dev/null +++ b/cardano-node/src/Cardano/Node/Configuration/CardanoConfigCompare.hs @@ -0,0 +1,311 @@ +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE StandaloneDeriving #-} + +-- The 'ToExpr'/'Generic' instances below are orphans, used only for diffing here. +{-# OPTIONS_GHC -Wno-orphans #-} + +-- | Diff the node's POM-resolved 'NodeConfiguration' against the one produced by +-- the @cardano-config@ adapter. The composite fields (the per-era protocol +-- configuration records and the LedgerDB configuration) are diffed structurally +-- with @tree-diff@; scalar fields use a plain @node=… vs cardano-config=…@ line. +module Cardano.Node.Configuration.CardanoConfigCompare + ( compareConfigurations + , deprecatedFlagWarnings + ) where + +import Cardano.Node.Configuration.POM (NodeConfiguration (..)) +import Cardano.Node.Types (NodeProtocolConfiguration (..)) + +import Data.List (intercalate) + +import Cardano.Crypto (RequiresNetworkMagic) +import Cardano.Ledger.BaseTypes.NonZero (NonZero, unNonZero) +import Cardano.Node.Configuration.LedgerDB (DeprecatedOptions (..), + LedgerDbConfiguration (..), LedgerDbSelectorFlag (..)) +import Cardano.Node.Types (CheckpointsFile (..), CheckpointsHash, + GenesisFile (..), GenesisHash, + MaxConcurrencyBulkSync (..), MaxConcurrencyDeadline (..), + NodeAlonzoProtocolConfiguration (..), + NodeByronProtocolConfiguration (..), + NodeCheckpointsConfiguration (..), + NodeConwayProtocolConfiguration (..), + NodeDijkstraProtocolConfiguration (..), + NodeHardForkProtocolConfiguration (..), + NodeShelleyProtocolConfiguration (..)) +import Cardano.Slotting.Slot (EpochNo, SlotNo (..)) +import Data.Time.Clock (DiffTime, secondsToDiffTime) +import Data.TreeDiff (Expr (App, Rec), ToExpr (..), ediff, prettyEditExpr) +import qualified Data.TreeDiff.OMap as OMap +import GHC.Generics (Generic) +import Ouroboros.Consensus.Mempool (MempoolCapacityBytesOverride (..)) +import Ouroboros.Consensus.Storage.LedgerDB.Args (QueryBatchSize (..), + defaultQueryBatchSize) +import Ouroboros.Consensus.Storage.LedgerDB.Snapshots + (NumOfDiskSnapshots (..), SnapshotDelayRange (..), + SnapshotFrequency (..), SnapshotFrequencyArgs (..), + SnapshotPolicyArgs (..)) +import Ouroboros.Consensus.Util.Args (OverrideOrDefault (..)) + +-- | Guidance for deprecated node CLI flags that cardano-config's parser rejects: +-- the legacy aliases have a new spelling, and the mempool flags were removed +-- (mempool capacity is a config-file setting now). Pure, so it is unit-testable. +deprecatedFlagWarnings :: [String] -> [String] +deprecatedFlagWarnings = concatMap diagnose + where + -- Deprecated alias -> new (cardano-config-accepted) spelling. + renamed = + [ ("--delegation-certificate", "--byron-delegation-certificate") + , ("--signing-key", "--byron-signing-key") + , ("--non-producing-node", "--start-as-non-producing-node") + ] + removed = ["--mempool-capacity-override", "--no-mempool-capacity-override"] + + diagnose tok = + -- Accept both @--flag value@ and @--flag=value@ spellings. + let opt = takeWhile (/= '=') tok + in case lookup opt renamed of + Just new -> + [ "warning: deprecated CLI flag '" <> opt <> "'; use '" <> new + <> "' (required for cardano-config parsing / the upcoming config parser)" ] + Nothing + | opt `elem` removed -> + [ "warning: '" <> opt <> "' is deprecated and no longer supported; remove it" + <> " and set 'MempoolCapacityBytesOverride' in the configuration file instead" ] + | otherwise -> [] + +-- | Compare a POM-resolved configuration against the adapter-produced one, field +-- by field. Returns one entry per diverging field; empty means they agree. +compareConfigurations :: NodeConfiguration -> NodeConfiguration -> [String] +compareConfigurations pom adapted = + concat + [ compareProtocol (ncProtocolConfig pom) (ncProtocolConfig adapted) + , cmp "ValidateDB" ncValidateDB + , cmp "TopologyFile" ncTopologyFile + , cmp "DatabaseFile" ncDatabaseFile + , cmp "StartAsNonProducingNode" ncStartAsNonProducingNode + , cmp "ProtocolFiles" ncProtocolFiles + , cmp "ShutdownConfig" ncShutdownConfig + , cmp "SocketConfig" ncSocketConfig + , cmp "DiffusionMode" ncDiffusionMode + , cmp "ExperimentalProtocolsEnabled" ncExperimentalProtocolsEnabled + , cmp "MaxConcurrencyBulkSync" (normalizeMaxConcurrencyBulkSync . ncMaxConcurrencyBulkSync) + , cmp "MaxConcurrencyDeadline" (normalizeMaxConcurrencyDeadline . ncMaxConcurrencyDeadline) + , cmp "TraceForwardSocket" ncTraceForwardSocket + , cmp "MaybeMempoolCapacityOverride" (normalizeMempoolOverride . ncMaybeMempoolCapacityOverride) + , cmpTree "LedgerDbConfig" (normalizeLedgerDb . ncLedgerDbConfig) + , cmp "ProtocolIdleTimeout" ncProtocolIdleTimeout + , cmp "TimeWaitTimeout" ncTimeWaitTimeout + , cmp "EgressPollInterval" ncEgressPollInterval + , cmp "ChainSyncIdleTimeout" ncChainSyncIdleTimeout + , cmp "MempoolTimeoutSoft" ncMempoolTimeoutSoft + , cmp "MempoolTimeoutHard" ncMempoolTimeoutHard + , cmp "MempoolTimeoutCapacity" ncMempoolTimeoutCapacity + , cmp "AcceptedConnectionsLimit" ncAcceptedConnectionsLimit + , cmp "DeadlineTargetOfRootPeers" ncDeadlineTargetOfRootPeers + , cmp "DeadlineTargetOfKnownPeers" ncDeadlineTargetOfKnownPeers + , cmp "DeadlineTargetOfEstablishedPeers" ncDeadlineTargetOfEstablishedPeers + , cmp "DeadlineTargetOfActivePeers" ncDeadlineTargetOfActivePeers + , cmp "DeadlineTargetOfKnownBigLedgerPeers" ncDeadlineTargetOfKnownBigLedgerPeers + , cmp "DeadlineTargetOfEstablishedBigLedgerPeers" ncDeadlineTargetOfEstablishedBigLedgerPeers + , cmp "DeadlineTargetOfActiveBigLedgerPeers" ncDeadlineTargetOfActiveBigLedgerPeers + , cmp "SyncTargetOfRootPeers" ncSyncTargetOfRootPeers + , cmp "SyncTargetOfKnownPeers" ncSyncTargetOfKnownPeers + , cmp "SyncTargetOfEstablishedPeers" ncSyncTargetOfEstablishedPeers + , cmp "SyncTargetOfActivePeers" ncSyncTargetOfActivePeers + , cmp "SyncTargetOfKnownBigLedgerPeers" ncSyncTargetOfKnownBigLedgerPeers + , cmp "SyncTargetOfEstablishedBigLedgerPeers" ncSyncTargetOfEstablishedBigLedgerPeers + , cmp "SyncTargetOfActiveBigLedgerPeers" ncSyncTargetOfActiveBigLedgerPeers + , cmp "ConsensusMode" ncConsensusMode + , cmp "MinBigLedgerPeersForTrustedState" ncMinBigLedgerPeersForTrustedState + , cmp "PeerSharing" ncPeerSharing + , cmp "GenesisConfig" ncGenesisConfig + , cmp "ResponderCoreAffinityPolicy" ncResponderCoreAffinityPolicy + , cmp "RpcConfig" ncRpcConfig + , cmp "TxSubmissionLogicVersion" ncTxSubmissionLogicVersion + , cmp "TxSubmissionInitDelay" ncTxSubmissionInitDelay + ] + where + cmp :: (Eq a, Show a) => String -> (NodeConfiguration -> a) -> [String] + cmp label accessor = cmpValues label (accessor pom) (accessor adapted) + + cmpTree :: (Eq a, ToExpr a) => String -> (NodeConfiguration -> a) -> [String] + cmpTree label accessor = cmpValuesTree label (accessor pom) (accessor adapted) + +-- | Report a divergence between two scalar values. +cmpValues :: (Eq a, Show a) => String -> a -> a -> [String] +cmpValues label a b + | a == b = [] + | otherwise = [label <> ": node=" <> show a <> " vs cardano-config=" <> show b] + +-- | Report a divergence between two composite values as a @tree-diff@ structural +-- diff (@-@ is the node value, @+@ the cardano-config one), as one indented entry. +cmpValuesTree :: (Eq a, ToExpr a) => String -> a -> a -> [String] +cmpValuesTree label a b + | a == b = [] + | otherwise = + [ label <> ":\n" + <> intercalate "\n" + (map (" " <>) (lines (show (prettyEditExpr (ediff a b))))) ] + +-- | Compare the Cardano protocol configuration per era/component. +compareProtocol :: NodeProtocolConfiguration -> NodeProtocolConfiguration -> [String] +compareProtocol + (NodeProtocolConfigurationCardano b1 s1 a1 c1 d1 h1 k1) + (NodeProtocolConfigurationCardano b2 s2 a2 c2 d2 h2 k2) = + concat + [ cmpValuesTree "Byron protocol config" (normalizeByron b1) (normalizeByron b2) + , cmpValuesTree "Shelley protocol config" s1 s2 + , cmpValuesTree "Alonzo protocol config" a1 a2 + , cmpValuesTree "Conway protocol config" c1 c2 + , cmpValuesTree "Dijkstra protocol config" d1 d2 + , cmpValuesTree "HardFork protocol config" h1 h2 + , cmpValuesTree "Checkpoints protocol config" k1 k2 + ] + +-- --------------------------------------------------------------------------- +-- Normalization: hide representational-only divergences, where POM keeps a +-- "use the default" sentinel and cardano-config spells the default out. +-- --------------------------------------------------------------------------- + +-- | Normalize a 'LedgerDbConfiguration' before diffing (snapshot policy and +-- query batch size). +normalizeLedgerDb :: LedgerDbConfiguration -> LedgerDbConfiguration +normalizeLedgerDb (LedgerDbConfiguration spa qbs sel dep) = + LedgerDbConfiguration (normalizeSnapshotPolicy spa) (normalizeQueryBatchSize qbs) sel dep + +-- | 'DefaultQueryBatchSize' and an explicit request for the same size are +-- equivalent; collapse the latter. +normalizeQueryBatchSize :: QueryBatchSize -> QueryBatchSize +normalizeQueryBatchSize q + | defaultQueryBatchSize q == defaultQueryBatchSize DefaultQueryBatchSize = DefaultQueryBatchSize + | otherwise = q + +-- | Unset ('Nothing') resolves to the 'defaultBlockFetchConfiguration' value of +-- 1, which cardano-config spells out; treat @Just 1@ as 'Nothing'. +normalizeMaxConcurrencyBulkSync :: Maybe MaxConcurrencyBulkSync -> Maybe MaxConcurrencyBulkSync +normalizeMaxConcurrencyBulkSync (Just (MaxConcurrencyBulkSync 1)) = Nothing +normalizeMaxConcurrencyBulkSync x = x + +normalizeMaxConcurrencyDeadline :: Maybe MaxConcurrencyDeadline -> Maybe MaxConcurrencyDeadline +normalizeMaxConcurrencyDeadline (Just (MaxConcurrencyDeadline 1)) = Nothing +normalizeMaxConcurrencyDeadline x = x + +-- | @Just NoMempoolCapacityBytesOverride@ and 'Nothing' both mean "no override". +normalizeMempoolOverride :: Maybe MempoolCapacityBytesOverride -> Maybe MempoolCapacityBytesOverride +normalizeMempoolOverride (Just NoMempoolCapacityBytesOverride) = Nothing +normalizeMempoolOverride x = x + +-- | The Byron supported-protocol version is not modelled by cardano-config and +-- has no effect for a genesis-only Byron era; blank it on both sides. +normalizeByron :: NodeByronProtocolConfiguration -> NodeByronProtocolConfiguration +normalizeByron c = + c { npcByronSupportedProtocolVersionMajor = 0 + , npcByronSupportedProtocolVersionMinor = 0 + , npcByronSupportedProtocolVersionAlt = 0 + } + +-- | Collapse the snapshot overrides that merely restate the consensus default +-- (rate limit, delay range, count). The offset and interval genuinely differ and +-- stay flagged; the interval default is security-parameter dependent (@2*k@) and +-- cannot be reconstructed here. Constants mirror 'defaultSnapshotPolicy'. +normalizeSnapshotPolicy :: SnapshotPolicyArgs -> SnapshotPolicyArgs +normalizeSnapshotPolicy (SnapshotPolicyArgs freq num) = + SnapshotPolicyArgs (normalizeFrequency freq) (collapse defaultNumSnapshots num) + where + defaultNumSnapshots = NumOfDiskSnapshots 2 + defaultOffset = SlotNo 0 + defaultRateLimit = secondsToDiffTime (10 * 60) + defaultDelayRange = SnapshotDelayRange (secondsToDiffTime 300) (secondsToDiffTime 600) + + normalizeFrequency DisableSnapshots = DisableSnapshots + normalizeFrequency (SnapshotFrequency (SnapshotFrequencyArgs interval offset rateLimit delayRange)) = + SnapshotFrequency $ SnapshotFrequencyArgs + interval + (collapse defaultOffset offset) + (collapse defaultRateLimit rateLimit) + (collapse defaultDelayRange delayRange) + + collapse :: Eq a => a -> OverrideOrDefault a -> OverrideOrDefault a + collapse def (Override v) | v == def = UseDefault + collapse _ x = x + +-- --------------------------------------------------------------------------- +-- tree-diff instances for the composite records. Node-local records derive +-- 'Generic' here; opaque leaves (hashes, 'DiffTime') render via 'Show'; the +-- consensus snapshot-policy types lack 'Generic' and get explicit instances. +-- --------------------------------------------------------------------------- + +deriving instance Generic GenesisFile +deriving instance Generic CheckpointsFile +deriving instance Generic NodeByronProtocolConfiguration +deriving instance Generic NodeShelleyProtocolConfiguration +deriving instance Generic NodeAlonzoProtocolConfiguration +deriving instance Generic NodeConwayProtocolConfiguration +deriving instance Generic NodeDijkstraProtocolConfiguration +deriving instance Generic NodeHardForkProtocolConfiguration +deriving instance Generic NodeCheckpointsConfiguration +deriving instance Generic LedgerDbConfiguration +deriving instance Generic LedgerDbSelectorFlag +deriving instance Generic DeprecatedOptions + +instance ToExpr GenesisFile +instance ToExpr CheckpointsFile +instance ToExpr NodeByronProtocolConfiguration +instance ToExpr NodeShelleyProtocolConfiguration +instance ToExpr NodeAlonzoProtocolConfiguration +instance ToExpr NodeConwayProtocolConfiguration +instance ToExpr NodeDijkstraProtocolConfiguration +instance ToExpr NodeHardForkProtocolConfiguration +instance ToExpr NodeCheckpointsConfiguration +instance ToExpr LedgerDbConfiguration +instance ToExpr LedgerDbSelectorFlag +instance ToExpr DeprecatedOptions + +-- External types that already derive 'Generic'. +instance ToExpr RequiresNetworkMagic +instance ToExpr QueryBatchSize + +-- Opaque leaves rendered through 'Show'. +instance ToExpr GenesisHash where toExpr = exprViaShow +instance ToExpr CheckpointsHash where toExpr = exprViaShow +instance ToExpr EpochNo where toExpr = exprViaShow +instance ToExpr DiffTime where toExpr = exprViaShow + +-- The consensus snapshot-policy types (no usable 'Generic'). +instance ToExpr SnapshotPolicyArgs where + toExpr (SnapshotPolicyArgs freq num) = + Rec "SnapshotPolicyArgs" $ OMap.fromList + [ ("spaFrequency", toExpr freq) + , ("spaNum", toExpr num) + ] + +instance ToExpr SnapshotFrequency where + toExpr (SnapshotFrequency args) = App "SnapshotFrequency" [toExpr args] + toExpr DisableSnapshots = App "DisableSnapshots" [] + +instance ToExpr SnapshotFrequencyArgs where + toExpr (SnapshotFrequencyArgs interval offset rateLimit delayRange) = + Rec "SnapshotFrequencyArgs" $ OMap.fromList + [ ("sfaInterval", toExpr interval) + , ("sfaOffset", toExpr offset) + , ("sfaRateLimit", toExpr rateLimit) + , ("sfaDelaySnapshotRange", toExpr delayRange) + ] + +instance ToExpr a => ToExpr (OverrideOrDefault a) where + toExpr (Override a) = App "Override" [toExpr a] + toExpr UseDefault = App "UseDefault" [] + +-- 'NonZero' hides its constructor, so unwrap via 'unNonZero'. +instance ToExpr a => ToExpr (NonZero a) where + toExpr n = App "NonZero" [toExpr (unNonZero n)] + +instance ToExpr SlotNo where + toExpr (SlotNo w) = App "SlotNo" [toExpr w] + +instance ToExpr SnapshotDelayRange +instance ToExpr NumOfDiskSnapshots + +-- | Render a value as an opaque @tree-diff@ leaf via its 'Show' instance. +exprViaShow :: Show a => a -> Expr +exprViaShow x = App (show x) [] diff --git a/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs b/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs index 2c60b7e9d87..931a2db979e 100644 --- a/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs +++ b/cardano-node/src/Cardano/Node/Configuration/LedgerDB.hs @@ -6,6 +6,7 @@ {-# LANGUAGE TypeApplications #-} {-# OPTIONS_GHC -Wno-orphans #-} +{-# OPTIONS_GHC -Wno-unused-top-binds #-} module Cardano.Node.Configuration.LedgerDB ( DeprecatedOptions (..), @@ -20,8 +21,6 @@ import Ouroboros.Consensus.Ledger.SupportsProtocol import Ouroboros.Consensus.Storage.LedgerDB.API import Ouroboros.Consensus.Storage.LedgerDB.Args import Ouroboros.Consensus.Storage.LedgerDB.Snapshots -import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.Args as V1 -import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore.Impl.LMDB as LMDB import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.InMemory as InMemory import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.LSM as LSM @@ -31,6 +30,8 @@ import Data.Proxy import System.FilePath import System.Random (StdGen) +import Ouroboros.Consensus.Ledger.Basics (LedgerState) + -- | Choose the LedgerDB Backend -- -- As of UTxO-HD, the LedgerDB now uses either an in-memory backend or LMDB to @@ -43,22 +44,15 @@ import System.Random (StdGen) -- -- - 'V2LSM': Uses the LSM backend. data LedgerDbSelectorFlag = - V1LMDB - V1.FlushFrequency - -- ^ The frequency at which changes are flushed to the disk. - (Maybe FilePath) - -- ^ Path for the live tables. If not provided the default will be used - -- (@/lmdb@). - (Maybe Gigabytes) - -- ^ A map size can be specified, this is the maximum disk space the LMDB - -- database can fill. If not provided, the default of 16GB will be used. - (Maybe Int) - -- ^ An override to the max number of readers. - | V2InMemory + V2InMemory | V2LSM (Maybe FilePath) -- ^ Maybe a custom path to the LSM database. If not provided the default -- will be used (@/lsm@). + (Maybe FilePath) + -- ^ Maybe a path to which the LSM backend will export standalone + -- snapshots on every snapshot. If not provided, no standalone snapshots + -- are exported. deriving (Eq, Show) @@ -73,8 +67,7 @@ noDeprecatedOptions = DeprecatedOptions [] data LedgerDbConfiguration = LedgerDbConfiguration - NumOfDiskSnapshots - SnapshotInterval + SnapshotPolicyArgs QueryBatchSize LedgerDbSelectorFlag DeprecatedOptions @@ -89,63 +82,14 @@ newtype Gigabytes = Gigabytes Int toBytes :: Gigabytes -> Int toBytes (Gigabytes x) = x * 1024 * 1024 * 1024 --- | Recommended settings for the LMDB backing store. --- --- === @'lmdbMapSize'@ --- The default @'LMDBLimits'@ uses an @'lmdbMapSize'@ of @1024 * 1024 * 1024 * 16@ --- bytes, or 16 Gigabytes. @'lmdbMapSize'@ sets the size of the memory map --- that is used internally by the LMDB backing store, and is also the --- maximum size of the on-disk database. 16 GB should be sufficient for the --- medium term, i.e., it is sufficient until a more performant alternative to --- the LMDB backing store is implemented, which will probably replace the LMDB --- backing store altogether. --- --- Note(jdral): It is recommended not to set the @'lmdbMapSize'@ to a value --- that is much smaller than 16 GB through manual configuration: the node will --- die with a fatal error as soon as the database size exceeds the --- @'lmdbMapSize'@. If this fatal error were to occur, we would expect that --- the node can continue normal operation if it is restarted with a higher --- @'lmdbMapSize'@ configured. Nonetheless, this situation should be avoided. --- --- === @'lmdbMaxDatabases'@ --- The @'lmdbMaxDatabases'@ is set to 10, which means that the LMDB backing --- store will allow up @<= 10@ internal databases. We say /internal/ --- databases, since they are not exposed outside the backing store interface, --- such that from the outside view there is just one /logical/ database. --- Two of these internal databases are reserved for normal operation of the --- backing store, while the remaining databases will be used to store ledger --- tables. At the moment, there is at most one ledger table that will be --- stored in an internal database: the UTxO. Nonetheless, we set --- @'lmdbMaxDatabases'@ to @10@ in order to future-proof these limits. --- --- === @'lmdbMaxReaders'@ --- The @'lmdbMaxReaders'@ limit sets the maximum number of threads that can --- read from the LMDB database. Currently, there should only be a single reader --- active. Again, we set @'lmdbMaxReaders'@ to @16@ in order to future-proof --- these limits. --- --- === References --- For more information about LMDB limits, one should inspect: --- * The @lmdb-simple@ and @haskell-lmdb@ forked repositories. --- * The official LMDB API documentation at --- . -defaultLMDBLimits :: LMDB.LMDBLimits -defaultLMDBLimits = LMDB.LMDBLimits { - LMDB.lmdbMapSize = 16 * 1024 * 1024 * 1024 - , LMDB.lmdbMaxDatabases = 10 - , LMDB.lmdbMaxReaders = 16 - } - defaultLMDBPath :: FilePath -> FilePath defaultLMDBPath = ( "lmdb") -selectorToArgs :: forall blk. (LedgerSupportsProtocol blk, LedgerSupportsLedgerDB blk) => LedgerDbSelectorFlag -> FilePath -> StdGen -> (LedgerDbBackendArgs IO blk, StdGen) +selectorToArgs :: + forall blk. + ( LedgerSupportsProtocol blk + , LedgerDbSerialiseConstraints blk + , CanUpgradeLedgerTables LedgerState blk + ) => LedgerDbSelectorFlag -> FilePath -> StdGen -> (LedgerDbBackendArgs IO blk, StdGen) selectorToArgs V2InMemory _ = InMemory.mkInMemoryArgs -selectorToArgs (V1LMDB ff fp l mxReaders) fastStoragePath = - LMDB.mkLMDBArgs - ff - (fromMaybe (defaultLMDBPath fastStoragePath) fp) - ( maybe id (\overrideMaxReaders lim -> lim{LMDB.lmdbMaxReaders = overrideMaxReaders}) mxReaders $ - maybe id (\ll lim -> lim{LMDB.lmdbMapSize = toBytes ll}) l defaultLMDBLimits - ) -selectorToArgs (V2LSM fp) fastStoragePath = LSM.mkLSMArgsIO (Proxy @blk) (fromMaybe "lsm" fp) fastStoragePath +selectorToArgs (V2LSM fp fpExport) fastStoragePath = LSM.mkLSMArgsIO (Proxy @blk) (fromMaybe "lsm" fp) fpExport fastStoragePath diff --git a/cardano-node/src/Cardano/Node/Configuration/POM.hs b/cardano-node/src/Cardano/Node/Configuration/POM.hs index 72a7fc94ff3..f8c997958e3 100644 --- a/cardano-node/src/Cardano/Node/Configuration/POM.hs +++ b/cardano-node/src/Cardano/Node/Configuration/POM.hs @@ -28,6 +28,7 @@ module Cardano.Node.Configuration.POM where import Cardano.Crypto (RequiresNetworkMagic (..)) +import Cardano.Ledger.BaseTypes.NonZero (nonZero) import Cardano.Logging.Types import Cardano.Network.ConsensusMode (ConsensusMode (..), defaultConsensusMode) import qualified Cardano.Network.Diffusion.Configuration as Cardano @@ -46,8 +47,9 @@ import Ouroboros.Consensus.Node.Genesis (GenesisConfig, GenesisConfigF defaultGenesisConfigFlags, mkGenesisConfig) import Ouroboros.Consensus.Storage.LedgerDB.Args (QueryBatchSize (..)) import Ouroboros.Consensus.Storage.LedgerDB.Snapshots (NumOfDiskSnapshots (..), - SnapshotInterval (..)) -import Ouroboros.Consensus.Storage.LedgerDB.V1.Args (FlushFrequency (..)) + SnapshotDelayRange (..), SnapshotFrequency (..), SnapshotFrequencyArgs (..), + SnapshotPolicyArgs (..), defaultSnapshotPolicyArgs, mithrilSnapshotPolicyArgs) +import Ouroboros.Consensus.Util.Args (OverrideOrDefault (..)) import Ouroboros.Network.Diffusion.Configuration as Configuration import qualified Ouroboros.Network.Diffusion.Configuration as Ouroboros import qualified Ouroboros.Network.Mux as Mux @@ -64,6 +66,7 @@ import Data.Hashable (Hashable) import Data.Maybe import Data.Monoid (Last (..)) import Data.Text (Text) +import qualified Data.Text as Text import Data.Time.Clock (DiffTime, secondsToDiffTime) import Data.Yaml (decodeFileThrow) import GHC.Generics (Generic) @@ -484,8 +487,11 @@ instance FromJSON PartialNodeConfiguration where Nothing -> return Nothing parseLedgerDbConfig v = do - let snapInterval x = fmap (RequestedSnapshotInterval . secondsToDiffTime) <$> x .:? "SnapshotInterval" - snapNum x = fmap RequestedNumOfDiskSnapshots <$> x .:? "NumOfDiskSnapshots" + let snapInterval x = do + si <- x .:? "SnapshotInterval" + when (any (<= 0) si) $ fail $ "Non-positive SnapshotInterval: " <> show si + pure $ Override <$> (si >>= nonZero) + snapNum x = fmap (Override . NumOfDiskSnapshots) <$> x .:? "NumOfDiskSnapshots" mTopLevelSnapInterval <- snapInterval v mTopLevelSnapNum <- snapNum v @@ -499,27 +505,66 @@ instance FromJSON PartialNodeConfiguration where mLedgerDB <- v .:? "LedgerDB" case mLedgerDB of Nothing -> do - let si = fromMaybe DefaultSnapshotInterval mTopLevelSnapInterval - sn = fromMaybe DefaultNumOfDiskSnapshots mTopLevelSnapNum - return $ Just $ LedgerDbConfiguration sn si DefaultQueryBatchSize V2InMemory deprecatedOpts + let si = fromMaybe UseDefault mTopLevelSnapInterval + sn = fromMaybe UseDefault mTopLevelSnapNum + sf = SnapshotFrequencyArgs { + sfaInterval = si + , sfaOffset = UseDefault + , sfaRateLimit = UseDefault + , sfaDelaySnapshotRange = UseDefault + } + spArgs = SnapshotPolicyArgs (SnapshotFrequency sf) sn + return $ Just $ LedgerDbConfiguration spArgs DefaultQueryBatchSize V2InMemory deprecatedOpts + Just ledgerDB -> flip (withObject "LedgerDB") ledgerDB $ \o -> do - ldbSnapInterval <- (getLast . (Last mTopLevelSnapInterval <>) . Last <$> snapInterval o) .!= DefaultSnapshotInterval - ldbSnapNum <- (getLast . (Last mTopLevelSnapNum <>) . Last <$> snapNum o) .!= DefaultNumOfDiskSnapshots - qsize <- (fmap RequestedQueryBatchSize <$> o .:? "QueryBatchSize") .!= DefaultQueryBatchSize - backend <- o .:? "Backend" .!= "V2InMemory" - selector <- case backend of - "V1LMDB" -> do - flush <- (fmap RequestedFlushFrequency <$> o .:? "FlushFrequency") .!= DefaultFlushFrequency - mapSize :: Maybe Gigabytes <- o .:? "MapSize" - lmdbPath :: Maybe FilePath <- o .:? "LiveTablesPath" - mxReaders :: Maybe Int <- o .:? "MaxReaders" - return $ V1LMDB flush lmdbPath mapSize mxReaders + -- Parse snapshot options from an object, honouring any top-level + -- (deprecated) SnapshotInterval / NumOfDiskSnapshots overrides. + let parseSnapshotOpts s = do + sInterval <- (getLast . (Last mTopLevelSnapInterval <>) . Last <$> snapInterval s) .!= UseDefault + sNum <- (getLast . (Last mTopLevelSnapNum <>) . Last <$> snapNum s) .!= UseDefault + sOffset <- (fmap Override <$> s .:? "SlotOffset") .!= UseDefault + sRateLimit <- (fmap (Override . secondsToDiffTime) <$> s .:? "RateLimit") .!= UseDefault + sMinDelay <- s .:? "MinDelay" + sMaxDelay <- s .:? "MaxDelay" + sDelayRange <- + case (sMinDelay, sMaxDelay) of + (Just minDelay, Just maxDelay) -> + if minDelay <= maxDelay then + pure (Override (SnapshotDelayRange (secondsToDiffTime minDelay) (secondsToDiffTime maxDelay))) + else fail $ "Invalid ledger snapshot delay range, MinDelay > MaxDelay: " + <> show minDelay <> " > " <> show maxDelay + _ -> pure UseDefault + let sf = SnapshotFrequencyArgs { + sfaInterval = sInterval + , sfaOffset = sOffset + , sfaRateLimit = sRateLimit + , sfaDelaySnapshotRange = sDelayRange + } + pure $ SnapshotPolicyArgs (SnapshotFrequency sf) sNum + + qsize <- (fmap RequestedQueryBatchSize <$> o .:? "QueryBatchSize") .!= DefaultQueryBatchSize + backend <- o .:? "Backend" .!= "V2InMemory" + selector <- case backend of "V2InMemory" -> return V2InMemory "V2LSM" -> do lsmPath :: Maybe FilePath <- o .:? "LSMDatabasePath" - pure $ V2LSM lsmPath + lsmExportPath :: Maybe FilePath <- o .:? "LSMExportPath" + pure $ V2LSM lsmPath lsmExportPath _ -> fail $ "Malformed LedgerDB Backend: " <> backend - pure $ Just $ LedgerDbConfiguration ldbSnapNum ldbSnapInterval qsize selector deprecatedOpts + + -- A named policy (e.g. `Snapshots: Mithril`) selects a whole predefined + -- set of args; an object is parsed field-by-field; absence falls back to + -- the legacy top-level options for backward compatibility. + mSnapshotsVal <- o .:? "Snapshots" + spArgs <- case mSnapshotsVal of + Just (String name) -> case name of + "Mithril" -> pure mithrilSnapshotPolicyArgs + _ -> fail $ "Unknown named ledger snapshot policy: " <> Text.unpack name + <> ". Expected \"Mithril\" or an object with snapshot options." + Just sv -> withObject "Snapshots" parseSnapshotOpts sv + Nothing -> parseSnapshotOpts o + + pure $ Just $ LedgerDbConfiguration spArgs qsize selector deprecatedOpts parseByronProtocol v = do primary <- v .:? "ByronGenesisFile" @@ -683,8 +728,7 @@ defaultPartialNodeConfiguration = , pncLedgerDbConfig = Last $ Just $ LedgerDbConfiguration - DefaultNumOfDiskSnapshots - DefaultSnapshotInterval + defaultSnapshotPolicyArgs DefaultQueryBatchSize V2InMemory noDeprecatedOptions diff --git a/cardano-node/src/Cardano/Node/Protocol/Cardano.hs b/cardano-node/src/Cardano/Node/Protocol/Cardano.hs index 9e5598b6b50..0904cd4ff50 100644 --- a/cardano-node/src/Cardano/Node/Protocol/Cardano.hs +++ b/cardano-node/src/Cardano/Node/Protocol/Cardano.hs @@ -39,6 +39,10 @@ import Ouroboros.Consensus.HardFork.Combinator.Condense () import Prelude import Data.Function ((&)) +import System.FilePath (takeDirectory) +import System.FS.API (SomeHasFS (..)) +import System.FS.API.Types (MountPoint (MountPoint)) +import System.FS.IO (ioHasFS) ------------------------------------------------------------------------------ -- Real Cardano protocol @@ -147,8 +151,12 @@ mkSomeConsensusProtocolCardano NodeByronProtocolConfiguration { firstExceptT CardanoProtocolInstantiationCheckpointsReadError $ readCheckpointsMap checkpointsConfiguration + -- Filesystem rooted at the Shelley genesis directory, used by the ledger to + -- read initial funds/staking injected from genesis (testnets only). + let shelleyGenesisFS = SomeHasFS $ ioHasFS $ MountPoint $ takeDirectory $ unGenesisFile npcShelleyGenesisFile + return $! - SomeConsensusProtocol CardanoBlockType $ ProtocolInfoArgsCardano $ Consensus.CardanoProtocolParams { + SomeConsensusProtocol CardanoBlockType $ ProtocolInfoArgsCardano shelleyGenesisFS $ Consensus.CardanoProtocolParams { Consensus.byronProtocolParams = Consensus.ProtocolParamsByron { byronGenesis = byronGenesis, diff --git a/cardano-node/src/Cardano/Node/Protocol/Checkpoints.hs b/cardano-node/src/Cardano/Node/Protocol/Checkpoints.hs index 029d1decb04..780f23393b3 100644 --- a/cardano-node/src/Cardano/Node/Protocol/Checkpoints.hs +++ b/cardano-node/src/Cardano/Node/Protocol/Checkpoints.hs @@ -11,10 +11,8 @@ module Cardano.Node.Protocol.Checkpoints import Cardano.Api import qualified Cardano.Crypto.Hash.Class as Crypto -import Cardano.Protocol.Crypto (StandardCrypto) import Cardano.Node.Types import Ouroboros.Consensus.Block -import Ouroboros.Consensus.Cardano import Ouroboros.Consensus.Config (CheckpointsMap (..), emptyCheckpointsMap) import Control.Exception (IOException) @@ -105,10 +103,9 @@ instance Aeson.FromJSON WrapCheckpointsMap where -> Aeson.Parser (HeaderHash (CardanoBlock StandardCrypto)) parseCardanoHash = Aeson.withText "CheckpointHash" $ \t -> case B16.decode $ Text.encodeUtf8 t of - Right h -> do - when (BS.length h /= fromIntegral (hashSize p)) $ - fail $ "Invalid hash size for " <> Text.unpack t - pure $ fromRawHash p h + Right h -> + maybe (fail $ "Invalid hash size for " <> Text.unpack t) pure $ + fromRawHash p h Left e -> fail $ "Invalid base16 for " <> Text.unpack t <> ": " <> e where diff --git a/cardano-node/src/Cardano/Node/Protocol/Conway.hs b/cardano-node/src/Cardano/Node/Protocol/Conway.hs index ef75e1c0c49..c64f5981a52 100644 --- a/cardano-node/src/Cardano/Node/Protocol/Conway.hs +++ b/cardano-node/src/Cardano/Node/Protocol/Conway.hs @@ -98,6 +98,7 @@ emptyConwayGenesis cm = , cgCommittee = DefaultClass.def , cgDelegs = mempty , cgInitialDReps = mempty + , cgExtraConfig = SNothing } diff --git a/cardano-node/src/Cardano/Node/Protocol/Shelley.hs b/cardano-node/src/Cardano/Node/Protocol/Shelley.hs index a816b8e91ed..1bb1907a706 100644 --- a/cardano-node/src/Cardano/Node/Protocol/Shelley.hs +++ b/cardano-node/src/Cardano/Node/Protocol/Shelley.hs @@ -39,7 +39,6 @@ import Cardano.Node.Tracing.Era.Shelley () import Cardano.Node.Tracing.Formatting () import Cardano.Node.Tracing.Tracers.ChainDB () import Cardano.Node.Types -import Cardano.Protocol.Crypto (StandardCrypto) import qualified Ouroboros.Consensus.Cardano as Consensus import Ouroboros.Consensus.HardFork.Combinator.AcrossEras () import Ouroboros.Consensus.Protocol.Praos.Common (PraosCanBeLeader (..), @@ -53,6 +52,10 @@ import qualified Data.Aeson as Aeson import qualified Data.ByteString as BS import qualified Data.Text as T import System.Directory (getFileSize) +import System.FS.API (SomeHasFS (..)) +import System.FS.API.Types (MountPoint (MountPoint)) +import System.FS.IO (ioHasFS) +import System.FilePath (takeDirectory) import qualified System.IO.MMap as MMap @@ -82,7 +85,12 @@ mkSomeConsensusProtocolShelley NodeShelleyProtocolConfiguration { leaderCredentials <- firstExceptT PraosLeaderCredentialsError $ readLeaderCredentials files + -- Filesystem rooted at the Shelley genesis directory, used by the ledger to + -- read initial funds/staking injected from genesis (testnets only). + let shelleyGenesisFS = SomeHasFS $ ioHasFS $ MountPoint $ takeDirectory $ unGenesisFile npcShelleyGenesisFile + return $ SomeConsensusProtocol Api.ShelleyBlockType $ Api.ProtocolInfoArgsShelley + shelleyGenesisFS genesis Consensus.ProtocolParamsShelleyBased { shelleyBasedInitialNonce = genesisHashToPraosNonce genesisHash, diff --git a/cardano-node/src/Cardano/Node/Protocol/Types.hs b/cardano-node/src/Cardano/Node/Protocol/Types.hs index 1bf0d8860ca..42fa4d46ef2 100644 --- a/cardano-node/src/Cardano/Node/Protocol/Types.hs +++ b/cardano-node/src/Cardano/Node/Protocol/Types.hs @@ -50,5 +50,5 @@ data SomeConsensusProtocol where , Api.FromCBOR (HeaderHash blk) ) => Api.BlockType blk - -> Api.ProtocolInfoArgs blk + -> Api.ProtocolInfoArgs IO blk -> SomeConsensusProtocol diff --git a/cardano-node/src/Cardano/Node/Run.hs b/cardano-node/src/Cardano/Node/Run.hs index f6fe4e745d9..6335368ff9c 100644 --- a/cardano-node/src/Cardano/Node/Run.hs +++ b/cardano-node/src/Cardano/Node/Run.hs @@ -26,9 +26,15 @@ module Cardano.Node.Run import Cardano.Api (File (..), FileDirection (..)) import Cardano.Api.Error (displayError) import qualified Cardano.Api as Api +import qualified Cardano.Configuration as Cfg +import qualified Options.Applicative as Opt +import System.Environment (getArgs) import System.Random (randomIO) import qualified Cardano.Crypto.Init as Crypto +import Cardano.Node.Configuration.CardanoConfigAdapter (cardanoConfigToNodeConfiguration) +import Cardano.Node.Configuration.CardanoConfigCompare (compareConfigurations, + deprecatedFlagWarnings) import Cardano.Node.Configuration.LedgerDB import Cardano.Node.Configuration.NodeAddress import Cardano.Node.Configuration.POM (NodeConfiguration (..), @@ -50,6 +56,7 @@ import Cardano.Node.Protocol.Types import Cardano.Node.Queries import Cardano.Rpc.Server import Cardano.Rpc.Server.Config +import Data.IORef import Cardano.Node.Startup import Cardano.Node.TraceConstraints (TraceConstraints) import Cardano.Node.Tracing (Tracers (..)) @@ -140,6 +147,7 @@ import Data.Either (partitionEithers) import Data.Functor.Identity (Identity (..)) import Data.IP (toSockAddr) import Data.Map.Strict (Map) +import Data.List (isPrefixOf) import qualified Data.Map.Strict as Map import Data.Maybe (catMaybes, fromMaybe, mapMaybe) import Data.Monoid (Last (..)) @@ -187,6 +195,11 @@ runNode cmdPc = do let earlyTracer = stdoutTracer traceWith earlyTracer $ "Node configuration: " <> show nc + -- Also resolve the same configuration with the shared cardano-config parser + -- and warn (non-fatally) if it diverges from the node's own parser, so the two + -- can be reconciled before cardano-config becomes the sole parser. + compareWithCardanoConfig earlyTracer cmdPc nc + forM_ mShelleyVrfFile $ runThrowExceptT . checkVRFFilePermissions earlyTracer . File @@ -203,6 +216,71 @@ runNode cmdPc = do runThrowExceptT :: Exception e => ExceptT e IO a -> IO a runThrowExceptT act = runExceptT act >>= either Exception.throwIO pure +-- | Resolve the node configuration with the shared @cardano-config@ parser and +-- compare it against the node's own POM-resolved configuration, tracing a +-- non-fatal warning for each field that diverges. +-- +-- To keep the comparison fair, cardano-config resolves from the SAME two inputs +-- the node used: it parses the node's own command line with its own CLI parser +-- and combines that with the configuration file, so both sides are @file + CLI@. +-- If cardano-config cannot parse the argv (e.g. a node flag its CLI parser does +-- not model), that is itself a meaningful divergence signal: we warn and fall +-- back to a file-only cardano-config resolution so the rest is still checked. +-- Every parse\/resolve failure here is only ever warned about, never fatal. +compareWithCardanoConfig + :: Tracer IO String + -> PartialNodeConfiguration + -> NodeConfiguration + -> IO () +compareWithCardanoConfig tracer cmdPc nc = + case getLast (pncConfigFile cmdPc) of + Nothing -> pure () + Just (ConfigYamlFilePath cfgFp) -> do + -- Drop the leading @run@ subcommand token(s) to get the flag list, matching + -- cardano-config's flat 'parseCliArgs' (which has no @run@ subcommand). + argv <- getArgs + let flags = dropWhile (not . ("-" `isPrefixOf`)) argv + cliInfo = Opt.info Cfg.parseCliArgs mempty + cliArgs <- case Opt.execParserPure Opt.defaultPrefs cliInfo flags of + Opt.Success cli -> pure cli + Opt.Failure f -> do + let (msg, _exit) = Opt.renderFailure f "cardano-config" + -- A parse failure usually means a deprecated node flag alias; surface + -- actionable guidance for each before the generic parser error. + mapM_ (traceWith tracer . ("cardano-config: " <>)) (deprecatedFlagWarnings flags) + traceWith tracer $ + "cardano-config: could not parse the node CLI arguments (a node flag its" + <> " parser does not model?); comparing file-only. Parser error: " <> msg + pure (Cfg.defaultCliArgs cfgFp) + Opt.CompletionInvoked _ -> pure (Cfg.defaultCliArgs cfgFp) + -- Parse the same configuration file the node used and resolve it together + -- with the CLI arguments, exactly as the node's POM path does. + result <- try $ do + (fileCfg, _fileWarns) <- Cfg.parseConfigurationFiles cfgFp + Exception.evaluate (Cfg.resolveConfiguration cliArgs fileCfg) + case result of + Left (e :: Exception.SomeException) -> + traceWith tracer $ + "cardano-config: failed to parse node configuration (ignored): " <> show e + Right (Left err) -> + traceWith tracer $ + "cardano-config: failed to resolve node configuration (ignored): " <> show err + Right (Right (cfgNc, _warns)) -> + case cardanoConfigToNodeConfiguration cfgNc of + Left adaptErr -> + traceWith tracer $ + "cardano-config: could not adapt to node configuration (ignored): " <> adaptErr + Right adaptedNc -> + case compareConfigurations nc adaptedNc of + [] -> + traceWith tracer + "cardano-config: resolved configuration (file + CLI) agrees with the node parser." + divergences -> + traceWith tracer $ + unlines $ + "cardano-config: WARNING - resolved configuration (file + CLI) diverges from the node parser:" + : map (" - " <>) divergences + -- | Read node configuration from a file specified in 'PartialNodeConfiguration' buildNodeConfiguration :: HasCallStack => PartialNodeConfiguration -- ^ defaults @@ -238,15 +316,15 @@ handleNodeWithTracers -> SomeConsensusProtocol -> IO () handleNodeWithTracers cmdPc nc p@(SomeConsensusProtocol blockType runP) = do - let ProtocolInfo{pInfoConfig} = fst $ Api.protocolInfo @IO runP - networkMagic :: Api.NetworkMagic = getNetworkMagic $ Consensus.configBlock pInfoConfig + (ProtocolInfo{pInfoConfig}, mkBlockForging) <- Api.protocolInfo @IO runP + let networkMagic :: Api.NetworkMagic = getNetworkMagic $ Consensus.configBlock pInfoConfig -- This IORef contains node kernel structure which holds node kernel. -- Used for ledger queries and peer connection status. nodeKernelData <- mkNodeKernelData let fp = maybe "No file path found!" unConfigPath (getLast (pncConfigFile cmdPc)) - blockForging <- snd (Api.protocolInfo runP) nullTracer + blockForging <- mkBlockForging nullTracer tracers <- initTraceDispatcher nc @@ -302,7 +380,7 @@ handleSimpleNode ( Api.Protocol IO blk ) => Api.BlockType blk - -> Api.ProtocolInfoArgs blk + -> Api.ProtocolInfoArgs IO blk -> Tracers RemoteAddress LocalAddress blk IO -> NodeConfiguration -> PartialNodeConfiguration @@ -326,7 +404,7 @@ handleSimpleNode blockType runP tracers nc cmdPc networkMagic onKernel = do traceWith (startupTracer tracers) StartupDBValidation - let pInfo = fst $ Api.protocolInfo @IO runP + pInfo <- fst <$> Api.protocolInfo @IO runP (publicIPv4SocketOrAddr, publicIPv6SocketOrAddr, localSocketOrPath) <- do result <- runExceptT (gatherConfiguredSockets $ ncSocketConfig nc) @@ -406,7 +484,8 @@ handleSimpleNode blockType runP tracers nc cmdPc networkMagic onKernel = do } , rnNodeKernelHook = \registry nodeKernel -> do -- set the initial block forging - blockForging <- snd (Api.protocolInfo runP) (Consensus.kesAgentTracer $ consensusTracers tracers) + (_, mkBlockForging) <- Api.protocolInfo runP + blockForging <- mkBlockForging (Consensus.kesAgentTracer $ consensusTracers tracers) unless (ncStartAsNonProducingNode nc) $ setBlockForging nodeKernel blockForging @@ -449,6 +528,7 @@ handleSimpleNode blockType runP tracers nc cmdPc networkMagic onKernel = do #endif nForkPolicy <- getForkPolicy $ ncResponderCoreAffinityPolicy nc cForkPolicy <- getForkPolicy $ ncResponderCoreAffinityPolicy nc + nodeKernelAccessRef <- newIORef Nothing void $ let diffusionNodeArguments :: Cardano.Diffusion.CardanoNodeArguments IO diffusionNodeArguments = Cardano.Diffusion.CardanoNodeArguments { @@ -481,7 +561,7 @@ handleSimpleNode blockType runP tracers nc cmdPc networkMagic onKernel = do (readTVar ledgerPeerSnapshotVar) nc in - withAsync (rpcServerLoop (startupTracer tracers) (rpcTracer tracers) rpcConfigVar networkMagic) $ \_ -> + withAsync (rpcServerLoop (startupTracer tracers) (rpcTracer tracers) rpcConfigVar networkMagic nodeKernelAccessRef) $ \_ -> Node.run nodeArgs { rnNodeKernelHook = \registry nodeKernel -> do @@ -491,6 +571,8 @@ handleSimpleNode blockType runP tracers nc cmdPc networkMagic onKernel = do useBootstrapVar ledgerPeerSnapshotPathVar ledgerPeerSnapshotVar rpcConfigVar rnNodeKernelHook nodeArgs registry nodeKernel + mkNodeKernelAccess (contramap RpcUnsupportedBlockType (startupTracer tracers)) blockType (pInfoConfig pInfo) nodeKernel + >>= writeIORef nodeKernelAccessRef } StdRunNodeArgs { srnBfcMaxConcurrencyBulkSync = unMaxConcurrencyBulkSync <$> ncMaxConcurrencyBulkSync nc @@ -564,15 +646,11 @@ handleSimpleNode blockType runP tracers nc cmdPc networkMagic onKernel = do Just version_ -> Map.takeWhileAntitone (<= version_) LedgerDbConfiguration - snapInterval - numSnaps + snapshotPolicyArgs queryBatchSize ldbBackend deprecatedOpts = ncLedgerDbConfig nc - snapshotPolicyArgs :: SnapshotPolicyArgs - snapshotPolicyArgs = SnapshotPolicyArgs numSnaps snapInterval - -------------------------------------------------------------------------------- -- SIGHUP Handlers -------------------------------------------------------------------------------- @@ -642,7 +720,8 @@ updateBlockForging startupTracer kesAgentTracer blockType nodeKernel nc = do case Api.reflBlockType blockType blockType' of Just Refl -> do -- TODO: check if runP' has changed - blockForging <- snd (Api.protocolInfo runP') kesAgentTracer + (_, mkBlockForging) <- Api.protocolInfo runP' + blockForging <- mkBlockForging kesAgentTracer traceWith startupTracer (BlockForgingUpdate (if null blockForging then DisabledBlockForging @@ -767,8 +846,9 @@ rpcServerLoop :: Tracer IO (StartupTrace blk) -> Tracer IO TraceRpc -> StrictTVar IO RpcConfig -> NetworkMagic + -> IORef (Maybe NodeKernelAccess) -> IO () -rpcServerLoop startupTracer rpcTracer rpcConfigVar networkMagic = go +rpcServerLoop startupTracer rpcTracer rpcConfigVar networkMagic nodeKernelAccessRef = go where go = do config@RpcConfig{isEnabled = Identity enabled} <- readTVarIO rpcConfigVar @@ -776,7 +856,7 @@ rpcServerLoop startupTracer rpcTracer rpcConfigVar networkMagic = go then race_ (do - runRpcServer rpcTracer (config, networkMagic) + runRpcServer rpcTracer config networkMagic nodeKernelAccessRef traceWith startupTracer RpcForceDisabled disableRpcServer) (waitForRpcConfigChange config) diff --git a/cardano-node/src/Cardano/Node/Startup.hs b/cardano-node/src/Cardano/Node/Startup.hs index af9b9edee9f..33107409169 100644 --- a/cardano-node/src/Cardano/Node/Startup.hs +++ b/cardano-node/src/Cardano/Node/Startup.hs @@ -145,6 +145,8 @@ data StartupTrace blk = | RpcConfigUpdate Text -- | Log RPC configuration update error | RpcConfigUpdateError Text + -- | Log that node kernel access is not supported for the running block type. + | RpcUnsupportedBlockType Text -- | Log RPC is forcefully disabled after a RPC server crash. | RpcForceDisabled @@ -211,6 +213,27 @@ prepareNodeInfo -> IO NodeInfo prepareNodeInfo nc (SomeConsensusProtocol whichP pForInfo) tc nodeStartTime = do nodeName <- prepareNodeName + cfg <- pInfoConfig . fst <$> Api.protocolInfo @IO pForInfo + let getSystemStartByron = WCT.getSystemStart . getSystemStart . configBlock $ cfg + systemStartTime :: UTCTime + systemStartTime = + case whichP of + Api.ByronBlockType -> + getSystemStartByron + Api.ShelleyBlockType -> + let DegenLedgerConfig cfgShelley = configLedger cfg + in getSystemStartShelley cfgShelley + Api.CardanoBlockType -> + let CardanoLedgerConfig _ cfgShelley cfgAllegra cfgMary cfgAlonzo cfgBabbage cfgConway cfgDijkstra = configLedger cfg + in minimum [ getSystemStartByron + , getSystemStartShelley cfgShelley + , getSystemStartShelley cfgAllegra + , getSystemStartShelley cfgMary + , getSystemStartShelley cfgAlonzo + , getSystemStartShelley cfgBabbage + , getSystemStartShelley cfgConway + , getSystemStartShelley cfgDijkstra + ] return $ NodeInfo { niName = nodeName , niProtocol = pack . show . ncProtocol $ nc @@ -220,29 +243,6 @@ prepareNodeInfo nc (SomeConsensusProtocol whichP pForInfo) tc nodeStartTime = do , niSystemStartTime = systemStartTime } where - cfg = pInfoConfig $ fst $ Api.protocolInfo @IO pForInfo - - systemStartTime :: UTCTime - systemStartTime = - case whichP of - Api.ByronBlockType -> - getSystemStartByron - Api.ShelleyBlockType -> - let DegenLedgerConfig cfgShelley = configLedger cfg - in getSystemStartShelley cfgShelley - Api.CardanoBlockType -> - let CardanoLedgerConfig _ cfgShelley cfgAllegra cfgMary cfgAlonzo cfgBabbage cfgConway cfgDijkstra = configLedger cfg - in minimum [ getSystemStartByron - , getSystemStartShelley cfgShelley - , getSystemStartShelley cfgAllegra - , getSystemStartShelley cfgMary - , getSystemStartShelley cfgAlonzo - , getSystemStartShelley cfgBabbage - , getSystemStartShelley cfgConway - , getSystemStartShelley cfgDijkstra - ] - - getSystemStartByron = WCT.getSystemStart . getSystemStart . configBlock $ cfg getSystemStartShelley = sgSystemStart . shelleyLedgerGenesis . shelleyLedgerConfig prepareNodeName = diff --git a/cardano-node/src/Cardano/Node/Tools/DBSynthesizer.hs b/cardano-node/src/Cardano/Node/Tools/DBSynthesizer.hs new file mode 100644 index 00000000000..a23e12db559 --- /dev/null +++ b/cardano-node/src/Cardano/Node/Tools/DBSynthesizer.hs @@ -0,0 +1,127 @@ +{-# LANGUAGE GADTs #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +-- | Downstream home for the db-synthesizer's configuration and credential +-- machinery. +-- +-- The forging engine ('Consensus.synthesize') lives in consensus and consumes a +-- @('ProtocolInfo', block forgers)@ pair plus an 'EpochSize'. Constructing those +-- from a node configuration file and on-disk forging credentials is a node +-- concern, so it is done here. +-- +-- The node configuration file is parsed and resolved by the shared +-- @cardano-config@ package and then mapped to the node's own 'NodeConfiguration' +-- by the shared adapter ('cardanoConfigToNodeConfiguration'); its +-- 'ncProtocolConfig' is handed to cardano-node's protocol-instantiation +-- machinery ('Node.mkConsensusProtocol'), with the forging credentials supplied +-- separately (from the tool's CLI), and cardano-api's 'Api.protocolInfo' bridge +-- produces the forging @(ProtocolInfo, forgers)@ pair. +module Cardano.Node.Tools.DBSynthesizer + ( DBSynthesizerException (..) + , initializeProtocol + , synthesizeFromConfig + ) where + +import Cardano.Api (BlockType (..), ProtocolInfoArgs (..)) +import qualified Cardano.Api as Api (protocolInfo) + +import qualified Cardano.Configuration as Cfg (resolveConfigurationFromFile) +import qualified Cardano.Ledger.Api.Transition as Ledger (tcShelleyGenesisL) +import Cardano.Ledger.Shelley.Genesis (ShelleyGenesis, sgEpochLength) +import Cardano.Node.Configuration.CardanoConfigAdapter (cardanoConfigToNodeConfiguration) +import Cardano.Node.Configuration.POM (NodeConfiguration (..)) +import Cardano.Node.Protocol (ProtocolInstantiationError) +import qualified Cardano.Node.Protocol as Node (mkConsensusProtocol) +import Cardano.Node.Protocol.Types (SomeConsensusProtocol (..)) +import Cardano.Node.Types (ProtocolFilepaths) +import Cardano.Slotting.Slot (EpochSize) +import qualified Cardano.Tools.DBSynthesizer.Run as Consensus (synthesize) +import Cardano.Tools.DBSynthesizer.Types (DBSynthesizerOptions, ForgeResult) +import qualified Ouroboros.Consensus.Cardano.Node as Consensus + ( CardanoProtocolParams (..) + ) + +import Control.Applicative (Const (..)) +import Control.Exception (Exception (..), throwIO) +import Control.Monad.Trans.Except (runExceptT) + +import Control.Tracer (Tracer) +import Ouroboros.Consensus.Block.Forging (MkBlockForging) +import Ouroboros.Consensus.Cardano.Block (CardanoBlock, StandardCrypto) +import Ouroboros.Consensus.Node.ProtocolInfo (ProtocolInfo) +import Ouroboros.Consensus.Protocol.Praos.AgentClient (KESAgentClientTrace) + +-- | Something went wrong turning a node configuration (plus credentials) into a +-- forging-capable Cardano protocol. +data DBSynthesizerException + = -- | The configuration file could not be parsed/resolved by cardano-config, + -- or adapted to the node's 'NodeConfiguration'. + DBSynthesizerConfigError String + | -- | The protocol could not be instantiated from the configuration. + DBSynthesizerProtocolError ProtocolInstantiationError + | -- | The configuration resolved to a non-Cardano protocol, which the + -- synthesizer does not support. + DBSynthesizerNotCardano + deriving Show + +instance Exception DBSynthesizerException + +-- | Build the ready-made 'ProtocolInfo', block forgers and 'EpochSize' that +-- 'Consensus.synthesize' needs, from a node configuration file (parsed with +-- @cardano-config@, adapted to the node's 'NodeConfiguration') and forging +-- credentials (supplied separately, e.g. from the tool's CLI). +initializeProtocol :: + -- | Path to the node's @config.json@. + FilePath -> + -- | Forging credentials (KES\/VRF\/opcert or bulk creds). + ProtocolFilepaths -> + IO + ( ProtocolInfo (CardanoBlock StandardCrypto) + , Tracer IO KESAgentClientTrace -> + IO [MkBlockForging IO (CardanoBlock StandardCrypto)] + , EpochSize + ) +initializeProtocol configFp protocolFiles = do + cfgNc <- + Cfg.resolveConfigurationFromFile configFp >>= \case + Left err -> throwIO (DBSynthesizerConfigError (show err)) + Right (nc, _warnings) -> pure nc + nodeCfg <- + either (throwIO . DBSynthesizerConfigError) pure + (cardanoConfigToNodeConfiguration cfgNc) + someProto <- + either (throwIO . DBSynthesizerProtocolError) pure + =<< runExceptT (Node.mkConsensusProtocol (ncProtocolConfig nodeCfg) (Just protocolFiles)) + case someProto of + SomeConsensusProtocol CardanoBlockType runP -> do + (protoInfo, mkForgers) <- Api.protocolInfo @IO runP + pure (protoInfo, mkForgers, sgEpochLength (shelleyGenesisOf runP)) + SomeConsensusProtocol{} -> throwIO DBSynthesizerNotCardano + +-- | Forge a ChainDB from a node configuration file, credentials and forge +-- options — the whole @config -> protocol -> forge@ pipeline the standalone +-- @db-synthesizer@ executable runs. No transactions are injected. +synthesizeFromConfig :: + -- | Path to the node's @config.json@. + FilePath -> + -- | Forging credentials. + ProtocolFilepaths -> + DBSynthesizerOptions -> + -- | Directory of the ChainDB to forge into. + FilePath -> + IO ForgeResult +synthesizeFromConfig configFp protocolFiles opts dbDir = do + (protoInfo, mkForgers, epochSize) <- initializeProtocol configFp protocolFiles + Consensus.synthesize genTxs opts epochSize dbDir (protoInfo, mkForgers) + where + genTxs _ _ _ _ = pure [] + +-- | Extract the Shelley genesis from a Cardano protocol's transition config. +-- Total for the Cardano protocol; the caller has already matched +-- 'CardanoBlockType'. +shelleyGenesisOf :: ProtocolInfoArgs IO (CardanoBlock StandardCrypto) -> ShelleyGenesis +shelleyGenesisOf (ProtocolInfoArgsCardano _ Consensus.CardanoProtocolParams{Consensus.cardanoLedgerTransitionConfig = transCfg}) = + getConst $ Ledger.tcShelleyGenesisL Const transCfg diff --git a/cardano-node/src/Cardano/Node/Tracing/API.hs b/cardano-node/src/Cardano/Node/Tracing/API.hs index 4aa4793d466..024317833fe 100644 --- a/cardano-node/src/Cardano/Node/Tracing/API.hs +++ b/cardano-node/src/Cardano/Node/Tracing/API.hs @@ -41,9 +41,7 @@ import Prelude import Control.Concurrent.Async (link) import Control.DeepSeq (deepseq) import Control.Exception (SomeException (..)) -import "contra-tracer" Control.Tracer (traceWith) -import "trace-dispatcher" Control.Tracer (nullTracer) -import Data.Functor.Contravariant ((>$<)) +import "contra-tracer" Control.Tracer (nullTracer, traceWith) import qualified Data.Map.Strict as Map import Data.Maybe import Data.Time.Clock (getCurrentTime) @@ -71,7 +69,7 @@ initTraceDispatcher :: -> IO (Tracers RemoteAddress LocalAddress blk IO) initTraceDispatcher nc p networkMagic nodeKernel noBlockForging = do trConfig <- readConfigurationWithDefault - (unConfigPath $ ncConfigFile nc) + (FromFile (unConfigPath $ ncConfigFile nc)) defaultCardanoConfig (kickoffForwarder, kickoffPrometheusSimple, tracers) <- mkTracers trConfig diff --git a/cardano-node/src/Cardano/Node/Tracing/Consistency.hs b/cardano-node/src/Cardano/Node/Tracing/Consistency.hs index b611c07fc90..b56401d4e9f 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Consistency.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Consistency.hs @@ -14,6 +14,7 @@ module Cardano.Node.Tracing.Consistency import Cardano.Logging +import Cardano.Logging.DocuGenerator (dtWarnings) import Cardano.Logging.Resources import Cardano.Logging.Resources.Types () import Cardano.Network.NodeToNode (RemoteAddress) @@ -93,7 +94,6 @@ import qualified Ouroboros.Network.Protocol.LocalTxSubmission.Type as LTS import Ouroboros.Network.Protocol.TxSubmission2.Type (TxSubmission2) import qualified Ouroboros.Network.Server as Server (Trace (..)) import Ouroboros.Network.Snocket (LocalAddress (..)) -import Ouroboros.Network.Tracing.PeerSelection () import Ouroboros.Network.TxSubmission.Inbound.V2 (TraceTxSubmissionInbound) import Ouroboros.Network.TxSubmission.Outbound (TraceTxSubmissionOutbound) @@ -112,7 +112,7 @@ checkNodeTraceConfiguration :: -> IO NSWarnings checkNodeTraceConfiguration configFileName = do w1 <- checkTraceConfiguration - configFileName + (FromFile configFileName) defaultCardanoConfig getAllNamespaces (dt,_) <- docTracersFirstPhase Nothing diff --git a/cardano-node/src/Cardano/Node/Tracing/Documentation.hs b/cardano-node/src/Cardano/Node/Tracing/Documentation.hs index e5b0a998ad6..408c15318f8 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Documentation.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Documentation.hs @@ -20,13 +20,13 @@ module Cardano.Node.Tracing.Documentation , docTracersFirstPhase ) where -import Ouroboros.Network.Tracing.TxSubmission.Inbound () -import Ouroboros.Network.Tracing.TxSubmission.Outbound () -import Ouroboros.Network.Tracing.PeerSelection () import Cardano.Network.Tracing.PeerSelection () import Cardano.Network.Tracing.PeerSelectionCounters () import Cardano.Git.Rev (gitRev) import Cardano.Logging as Logging +import Cardano.Logging.DocuGenerator (DocTracer (..), docTracer, docTracerDatapoint, + documentTracer, docuResultsToText, docuResultsToMetricsHelptext, + docuResultsToNamespaces) import Cardano.Logging.Resources import Cardano.Logging.Resources.Types () import qualified Cardano.Network.PeerSelection.ExtraRootPeers as Cardano.PublicRootPeers @@ -209,7 +209,7 @@ docTracersFirstPhase :: forall blk peer remotePeer. -> IO (DocTracer, TraceConfig) docTracersFirstPhase condConfigFileName = do trConfig <- case condConfigFileName of - Just fn -> readConfigurationWithDefault fn defaultCardanoConfig + Just fn -> readConfigurationWithDefault (FromFile fn) defaultCardanoConfig Nothing -> pure defaultCardanoConfig let trBase :: Logging.Trace IO FormattedMessage = docTracer (Stdout MachineFormat) trForward :: Logging.Trace IO FormattedMessage = docTracer Forwarder diff --git a/cardano-node/src/Cardano/Node/Tracing/Era/Shelley.hs b/cardano-node/src/Cardano/Node/Tracing/Era/Shelley.hs index b048c027c0a..0e18516b3d4 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Era/Shelley.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Era/Shelley.hs @@ -47,14 +47,13 @@ import Cardano.Node.Tracing.Render (renderIncompleteWithdrawals, rende renderScriptHash, renderScriptIntegrityHash, renderTxId) import qualified Cardano.Protocol.Crypto as Ledger import Cardano.Protocol.TPraos.API (ChainTransitionError (ChainTransitionError)) -import Cardano.Protocol.TPraos.BHeader (LastAppliedBlock, labBlockNo) +import Cardano.Protocol.TPraos.BlockHeader (LastAppliedBlock, labBlockNo) import Cardano.Protocol.TPraos.OCert (KESPeriod (KESPeriod)) import Cardano.Protocol.TPraos.Rules.OCert import Cardano.Protocol.TPraos.Rules.Overlay import Cardano.Protocol.TPraos.Rules.Prtcl (PrtclPredicateFailure (OverlayFailure, UpdnFailure), PrtlSeqFailure (WrongBlockNoPrtclSeq, WrongBlockSequencePrtclSeq, WrongSlotIntervalPrtclSeq)) -import Cardano.Protocol.TPraos.Rules.Tickn (TicknPredicateFailure) import Cardano.Protocol.TPraos.Rules.Updn (UpdnPredicateFailure) import Cardano.Slotting.Block (BlockNo (..)) import Ouroboros.Consensus.Ledger.SupportsMempool (txId) @@ -64,11 +63,14 @@ import Ouroboros.Consensus.Protocol.TPraos (TPraosCannotForge (..)) import Ouroboros.Consensus.Shelley.Ledger hiding (TxId) import qualified Ouroboros.Consensus.Shelley.Ledger as Consensus import Ouroboros.Consensus.Shelley.Ledger.Inspect -import qualified Ouroboros.Consensus.Shelley.Protocol.Praos as Praos +import qualified Ouroboros.Consensus.Shelley.Protocol.EnvelopeChecks as Praos + (EnvelopeError (..)) import Ouroboros.Consensus.Util.Condense (condense) import Ouroboros.Network.Block (SlotNo (..), blockHash, blockNo, blockSlot) import Ouroboros.Network.Point (WithOrigin, withOriginToMaybe) +import Control.DeepSeq (NFData) + import Data.Aeson (ToJSON (..), Value (..), (.=)) import qualified Data.Aeson.Key as Aeson (fromText) import qualified Data.Aeson.Types as Aeson @@ -220,8 +222,8 @@ instance instance ( Consensus.ShelleyBasedEra era - , LogFormatting (PredicateFailure (ShelleyUTXO era)) - , LogFormatting (PredicateFailure (ShelleyUTXOW era)) + , LogFormatting (PredicateFailure (UTXO era)) + , LogFormatting (PredicateFailure (UTXOW era)) , LogFormatting (PredicateFailure (Ledger.EraRule "LEDGER" era)) , ToJSON (ApplyTxError era) ) => LogFormatting (ApplyTxError era) where @@ -249,9 +251,10 @@ instance instance ( Consensus.ShelleyBasedEra era - , LogFormatting (PredicateFailure (ShelleyUTXO era)) - , LogFormatting (PredicateFailure (ShelleyUTXOW era)) + , LogFormatting (PredicateFailure (UTXO era)) + , LogFormatting (PredicateFailure (UTXOW era)) , LogFormatting (PredicateFailure (Ledger.EraRule "BBODY" era)) + , NFData (PredicateFailure (Ledger.EraRule "BBODY" era)) ) => LogFormatting (BlockTransitionError era) where forMachine dtal (BlockTransitionError fs) = mconcat [ "kind" .= String "BlockTransitionError" @@ -320,8 +323,8 @@ instance LogFormatting PrtlSeqFailure where instance ( Consensus.ShelleyBasedEra era - , LogFormatting (PredicateFailure (ShelleyUTXO era)) - , LogFormatting (PredicateFailure (ShelleyUTXOW era)) + , LogFormatting (PredicateFailure (UTXO era)) + , LogFormatting (PredicateFailure (UTXOW era)) , LogFormatting (PredicateFailure (Ledger.EraRule "LEDGER" era)) , LogFormatting (PredicateFailure (Ledger.EraRule "LEDGERS" era)) ) => LogFormatting (ShelleyBbodyPredFailure era) where @@ -342,8 +345,8 @@ instance instance ( Consensus.ShelleyBasedEra era - , LogFormatting (PredicateFailure (ShelleyUTXO era)) - , LogFormatting (PredicateFailure (ShelleyUTXOW era)) + , LogFormatting (PredicateFailure (UTXO era)) + , LogFormatting (PredicateFailure (UTXOW era)) , LogFormatting (PredicateFailure (Ledger.EraRule "LEDGER" era)) ) => LogFormatting (ShelleyLedgersPredFailure era) where forMachine dtal (LedgerFailure f) = forMachine dtal f @@ -360,8 +363,8 @@ instance LogFormatting Withdrawals where instance ( Consensus.ShelleyBasedEra era - , LogFormatting (PredicateFailure (ShelleyUTXO era)) - , LogFormatting (PredicateFailure (ShelleyUTXOW era)) + , LogFormatting (PredicateFailure (UTXO era)) + , LogFormatting (PredicateFailure (UTXOW era)) , LogFormatting (PredicateFailure (Ledger.EraRule "DELEGS" era)) , LogFormatting (PredicateFailure (Ledger.EraRule "UTXOW" era)) ) => LogFormatting (ShelleyLedgerPredFailure era) where @@ -430,7 +433,7 @@ formatAsHex (Just bs) = show bs instance ( Consensus.ShelleyBasedEra era - , LogFormatting (PredicateFailure (ShelleyUTXO era)) + , LogFormatting (PredicateFailure (UTXO era)) , LogFormatting (PredicateFailure (Ledger.EraRule "UTXO" era)) ) => LogFormatting (ShelleyUtxowPredFailure era) where forMachine _dtal (InvalidWitnessesUTXOW wits') = @@ -767,10 +770,6 @@ instance LogFormatting (ShelleyPoolPredFailure era) where ] -instance LogFormatting TicknPredicateFailure where - forMachine _dtal x = case x of {} -- no constructors - - instance ( Ledger.Crypto crypto ) => LogFormatting (PrtclPredicateFailure crypto) where @@ -1325,7 +1324,7 @@ instance LogFormatting (Praos.PraosCannotForge crypto) where , "opCertStartingKesPeriod" .= kesPeriodValue startingKesPeriod ] -instance LogFormatting Praos.PraosEnvelopeError where +instance LogFormatting Praos.EnvelopeError where forMachine _ err' = case err' of Praos.ObsoleteNode maxPtclVersionFromPparams blkHeaderPtclVersion -> diff --git a/cardano-node/src/Cardano/Node/Tracing/Formatting.hs b/cardano-node/src/Cardano/Node/Tracing/Formatting.hs index 012eb0bf67a..a0c947aca64 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Formatting.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Formatting.hs @@ -2,6 +2,7 @@ {-# LANGUAGE LambdaCase #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} +{-# LANGUAGE TypeFamilies #-} {-# LANGUAGE UndecidableInstances #-} {-# OPTIONS_GHC -Wno-orphans #-} @@ -24,9 +25,9 @@ import Data.Void (Void) -- | Derives ConvertRawHash for Header blk from ConvertRawHash blk. -- Safe because HeaderHash (Header blk) = HeaderHash blk. instance ConvertRawHash blk => ConvertRawHash (Header blk) where - toShortRawHash _ = toShortRawHash (Proxy @blk) - fromShortRawHash _ = fromShortRawHash (Proxy @blk) - hashSize _ = hashSize (Proxy @blk) + type HashSize (Header blk) = HashSize blk + toShortRawHash _ = toShortRawHash (Proxy @blk) + unsafeFromShortRawHash _ = unsafeFromShortRawHash (Proxy @blk) -- | A bit of a weird one, but needed because some of the very general -- consensus interfaces are sometimes instantiated to 'Void', when there are diff --git a/cardano-node/src/Cardano/Node/Tracing/Render.hs b/cardano-node/src/Cardano/Node/Tracing/Render.hs index 3f3538457a6..b6b3576c200 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Render.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Render.hs @@ -238,7 +238,7 @@ renderAlonzoPlutusPurpose = \case Aeson.object ["spending" .= Api.fromShelleyTxIn txin] AlonzoMinting pid -> Aeson.object ["minting" .= Aeson.toJSON pid] - AlonzoRewarding (AsItem rwdAcct) -> + AlonzoWithdrawing (AsItem rwdAcct) -> Aeson.object ["rewarding" .= Aeson.String (Api.serialiseAddress $ Api.fromShelleyStakeAddr rwdAcct)] AlonzoCertifying cert -> Aeson.object ["certifying" .= Aeson.toJSON cert] @@ -252,7 +252,7 @@ renderConwayPlutusPurpose = \case Aeson.object ["spending" .= Api.fromShelleyTxIn txin] ConwayMinting pid -> Aeson.object ["minting" .= Aeson.toJSON pid] - ConwayRewarding (AsItem rwdAcct) -> + ConwayWithdrawing (AsItem rwdAcct) -> Aeson.object ["rewarding" .= Aeson.String (Api.serialiseAddress $ Api.fromShelleyStakeAddr rwdAcct)] ConwayCertifying cert -> Aeson.object ["certifying" .= Aeson.toJSON cert] diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers.hs index cbf985df114..1d6d1e82c98 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers.hs @@ -4,7 +4,6 @@ {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GADTs #-} {-# LANGUAGE LambdaCase #-} -{-# LANGUAGE PackageImports #-} {-# LANGUAGE QuantifiedConstraints #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} @@ -60,7 +59,6 @@ import qualified Ouroboros.Network.Diffusion as Diffusion import Codec.CBOR.Read (DeserialiseFailure) import Control.Monad (unless) -import "contra-tracer" Control.Tracer (Tracer (..)) import Cardano.Network.OrphanInstances () import Data.Aeson (ToJSON (..)) import Data.Proxy (Proxy (..)) @@ -175,26 +173,26 @@ mkDispatchTracers nodeKernel trBase trForward mbTrEKG trDataPoint trConfig p = d pure Tracers { - chainDBTracer = Tracer (traceWith chainDBTr') - <> Tracer (traceWith replayBlockTr') - <> Tracer (SR.traceNodeStateChainDB p nodeStateDP) + chainDBTracer = mkTracer (traceWith chainDBTr') + <> mkTracer (traceWith replayBlockTr') + <> mkTracer (SR.traceNodeStateChainDB p nodeStateDP) , consensusTracers = consensusTr - , churnModeTracer = Tracer (traceWith churnModeTr) + , churnModeTracer = mkTracer (traceWith churnModeTr) , nodeToClientTracers = nodeToClientTr , nodeToNodeTracers = nodeToNodeTr , diffusionTracers = diffusionTr - , startupTracer = Tracer (traceWith startupTr) - <> Tracer (SR.traceNodeStateStartup nodeStateDP) - , shutdownTracer = Tracer (traceWith shutdownTr) - <> Tracer (SR.traceNodeStateShutdown nodeStateDP) - , nodeInfoTracer = Tracer (traceWith nodeInfoDP) - , nodeStartupInfoTracer = Tracer (traceWith nodeStartupInfoDP) - , nodeStateTracer = Tracer (traceWith stateTr) - <> Tracer (traceWith nodeStateDP) - , nodeVersionTracer = Tracer (traceWith nodeVersionTr) - , resourcesTracer = Tracer (traceWith resourcesTr) - , ledgerMetricsTracer = Tracer (traceWith ledgerMetricsTr) - , rpcTracer = Tracer (traceWith rpcTr) + , startupTracer = mkTracer (traceWith startupTr) + <> mkTracer (SR.traceNodeStateStartup nodeStateDP) + , shutdownTracer = mkTracer (traceWith shutdownTr) + <> mkTracer (SR.traceNodeStateShutdown nodeStateDP) + , nodeInfoTracer = mkTracer (traceWith nodeInfoDP) + , nodeStartupInfoTracer = mkTracer (traceWith nodeStartupInfoDP) + , nodeStateTracer = mkTracer (traceWith stateTr) + <> mkTracer (traceWith nodeStateDP) + , nodeVersionTracer = mkTracer (traceWith nodeVersionTr) + , resourcesTracer = mkTracer (traceWith resourcesTr) + , ledgerMetricsTracer = mkTracer (traceWith ledgerMetricsTr) + , rpcTracer = mkTracer (traceWith rpcTr) } mkConsensusTracers :: forall blk. @@ -354,60 +352,71 @@ mkConsensusTracers configReflection trBase trForward mbTrEKG _trDataPoint trConf !txCountersTracer <- mkCardanoTracer trBase trForward mbTrEKG ["txCounters", "Remote"] + + !txPerasCertIn <- mkCardanoTracer trBase trForward mbTrEKG ["Peras", "Cert", "Inbound"] + !txPerasCertOut <- mkCardanoTracer trBase trForward mbTrEKG ["Peras", "Cert", "Outbound"] + !txPerasVoteIn <- mkCardanoTracer trBase trForward mbTrEKG ["Peras", "Vote", "Inbound"] + !txPerasVoteOut <- mkCardanoTracer trBase trForward mbTrEKG ["Peras", "Vote", "Outbound"] + + configureTracers configReflection trConfig [txCountersTracer] pure $ Consensus.Tracers - { Consensus.chainSyncClientTracer = Tracer $ + { Consensus.chainSyncClientTracer = mkTracer $ traceWith chainSyncClientTr - , Consensus.chainSyncServerHeaderTracer = Tracer $ + , Consensus.chainSyncServerHeaderTracer = mkTracer $ traceWith chainSyncServerHeaderTr <> traceWith chainSyncServerHeaderMetricsTr - , Consensus.chainSyncServerBlockTracer = Tracer $ + , Consensus.chainSyncServerBlockTracer = mkTracer $ traceWith chainSyncServerBlockTr - , Consensus.consensusSanityCheckTracer = Tracer $ + , Consensus.consensusSanityCheckTracer = mkTracer $ traceWith consensusSanityCheckTr - , Consensus.blockFetchDecisionTracer = Tracer $ + , Consensus.blockFetchDecisionTracer = mkTracer $ traceWith blockFetchDecisionTr - , Consensus.blockFetchClientTracer = Tracer $ + , Consensus.blockFetchClientTracer = mkTracer $ traceWith blockFetchClientTr <> traceWith blockFetchClientMetricsTr - , Consensus.blockFetchServerTracer = Tracer $ + , Consensus.blockFetchServerTracer = mkTracer $ traceWith blockFetchServerTr <> traceWith servedBlockLatestTr - , Consensus.forgeStateInfoTracer = Tracer $ + , Consensus.forgeStateInfoTracer = mkTracer $ traceWith (traceAsKESInfo (Proxy @blk) forgeKESInfoTr) - , Consensus.gddTracer = Tracer $ + , Consensus.gddTracer = mkTracer $ traceWith consensusGddTr - , Consensus.txInboundTracer = Tracer $ + , Consensus.txInboundTracer = mkTracer $ traceWith txInboundTr - , Consensus.txOutboundTracer = Tracer $ + , Consensus.txOutboundTracer = mkTracer $ traceWith txOutboundTr - , Consensus.localTxSubmissionServerTracer = Tracer $ + , Consensus.localTxSubmissionServerTracer = mkTracer $ traceWith localTxSubmissionServerTr - , Consensus.mempoolTracer = Tracer $ + , Consensus.mempoolTracer = mkTracer $ traceWith mempoolTr , Consensus.forgeTracer = - Tracer (\(Consensus.TraceLabelCreds _ x) -> traceWith forgeTr x) + mkTracer (\(Consensus.TraceLabelCreds _ x) -> traceWith forgeTr x) <> - Tracer (\(Consensus.TraceLabelCreds _ x) -> traceWith forgeStatsTr x) - , Consensus.blockchainTimeTracer = Tracer $ + mkTracer (\(Consensus.TraceLabelCreds _ x) -> traceWith forgeStatsTr x) + , Consensus.blockchainTimeTracer = mkTracer $ traceWith blockchainTimeTr - , Consensus.keepAliveClientTracer = Tracer $ + , Consensus.keepAliveClientTracer = mkTracer $ traceWith keepAliveClientTr - , Consensus.consensusErrorTracer = Tracer $ + , Consensus.consensusErrorTracer = mkTracer $ traceWith consensusStartupErrorTr . ConsensusStartupException - , Consensus.gsmTracer = Tracer $ + , Consensus.gsmTracer = mkTracer $ traceWith consensusGsmTr - , Consensus.csjTracer = Tracer $ + , Consensus.csjTracer = mkTracer $ traceWith consensusCsjTr - , Consensus.dbfTracer = Tracer $ + , Consensus.dbfTracer = mkTracer $ traceWith consensusDbfTr - , Consensus.kesAgentTracer = Tracer $ + , Consensus.kesAgentTracer = mkTracer $ traceWith consensusKesAgentTr - , Consensus.txLogicTracer = Tracer $ + , Consensus.txLogicTracer = mkTracer $ traceWith txLogicTracer - , Consensus.txCountersTracer = Tracer $ + , Consensus.txCountersTracer = mkTracer $ traceWith txCountersTracer + , Consensus.perasCertDiffusionInboundTracer = mkTracer $ traceWith txPerasCertIn + , Consensus.perasCertDiffusionOutboundTracer = mkTracer $ traceWith txPerasCertOut + , Consensus.perasVoteDiffusionInboundTracer = mkTracer $ traceWith txPerasVoteIn + , Consensus.perasVoteDiffusionOutboundTracer = mkTracer $ traceWith txPerasVoteOut } mkNodeToClientTracers :: forall blk. @@ -445,13 +454,13 @@ mkNodeToClientTracers configReflection trBase trForward mbTrEKG _trDataPoint trC configureTracers configReflection trConfig [stateQueryTr] pure $ NtC.Tracers - { NtC.tChainSyncTracer = Tracer $ + { NtC.tChainSyncTracer = mkTracer $ traceWith chainSyncTr - , NtC.tTxMonitorTracer = Tracer $ + , NtC.tTxMonitorTracer = mkTracer $ traceWith txMonitorTr - , NtC.tTxSubmissionTracer = Tracer $ + , NtC.tTxSubmissionTracer = mkTracer $ traceWith txSubmissionTr - , NtC.tStateQueryTracer = Tracer $ + , NtC.tStateQueryTracer = mkTracer $ traceWith stateQueryTr } @@ -502,28 +511,28 @@ mkNodeToNodeTracers configReflection trBase trForward mbTrEKG _trDataPoint trCon ["PeerSharing", "Remote"] configureTracers configReflection trConfig [peerSharingTracer] - !txLogicTracer <- mkCardanoTracer - trBase trForward mbTrEKG - ["txLogic", "Remote"] - configureTracers configReflection trConfig [txLogicTracer] + !txPerasCertDiffusion <- mkCardanoTracer trBase trForward mbTrEKG ["Peras", "Cert", "Inbound"] + !txPerasVoteDiffusion <- mkCardanoTracer trBase trForward mbTrEKG ["Peras", "Vote", "Inbound"] pure $ NtN.Tracers - { NtN.tChainSyncTracer = Tracer $ + { NtN.tChainSyncTracer = mkTracer $ traceWith chainSyncTracer - , NtN.tChainSyncSerialisedTracer = Tracer $ + , NtN.tChainSyncSerialisedTracer = mkTracer $ traceWith chainSyncSerialisedTr - , NtN.tBlockFetchTracer = Tracer $ + , NtN.tBlockFetchTracer = mkTracer $ traceWith blockFetchTr - , NtN.tBlockFetchSerialisedTracer = Tracer $ + , NtN.tBlockFetchSerialisedTracer = mkTracer $ traceWith blockFetchSerialisedTr - , NtN.tTxSubmission2Tracer = Tracer $ + , NtN.tTxSubmission2Tracer = mkTracer $ traceWith txSubmission2Tracer - , NtN.tKeepAliveTracer = Tracer $ + , NtN.tKeepAliveTracer = mkTracer $ traceWith keepAliveTracer - , NtN.tPeerSharingTracer = Tracer $ + , NtN.tPeerSharingTracer = mkTracer $ traceWith peerSharingTracer - , NtN.tTxLogicTracer = Tracer $ - traceWith txLogicTracer + , NtN.tPerasCertDiffusionTracer = mkTracer $ + traceWith txPerasCertDiffusion + , NtN.tPerasVoteDiffusionTracer = mkTracer $ + traceWith txPerasVoteDiffusion } mkDiffusionTracers :: @@ -668,54 +677,54 @@ mkDiffusionTracers configReflection trBase trForward mbTrEKG _trDataPoint trConf configureTracers configReflection trConfig [dtDnsTr] pure $ Diffusion.Tracers - { Diffusion.dtMuxTracer = Tracer $ + { Diffusion.dtMuxTracer = mkTracer $ traceWith dtMuxTr - , Diffusion.dtChannelTracer = Tracer $ + , Diffusion.dtChannelTracer = mkTracer $ traceWith dtChannelTracer - , Diffusion.dtBearerTracer = Tracer $ + , Diffusion.dtBearerTracer = mkTracer $ traceWith dtBearerTracer - , Diffusion.dtHandshakeTracer = Tracer $ + , Diffusion.dtHandshakeTracer = mkTracer $ traceWith dtHandshakeTracer - , Diffusion.dtLocalMuxTracer = Tracer $ + , Diffusion.dtLocalMuxTracer = mkTracer $ traceWith dtLocalMuxTr - , Diffusion.dtLocalChannelTracer = Tracer $ + , Diffusion.dtLocalChannelTracer = mkTracer $ traceWith dtLocalChannelTracer - , Diffusion.dtLocalBearerTracer = Tracer $ + , Diffusion.dtLocalBearerTracer = mkTracer $ traceWith dtLocalBearerTracer - , Diffusion.dtLocalHandshakeTracer = Tracer $ + , Diffusion.dtLocalHandshakeTracer = mkTracer $ traceWith dtLocalHandshakeTracer - , Diffusion.dtDiffusionTracer = Tracer $ + , Diffusion.dtDiffusionTracer = mkTracer $ traceWith dtDiffusionInitializationTr - , Diffusion.dtTraceLocalRootPeersTracer = Tracer $ + , Diffusion.dtTraceLocalRootPeersTracer = mkTracer $ traceWith localRootPeersTr - , Diffusion.dtTracePublicRootPeersTracer = Tracer $ + , Diffusion.dtTracePublicRootPeersTracer = mkTracer $ traceWith publicRootPeersTr - , Diffusion.dtTracePeerSelectionTracer = Tracer $ + , Diffusion.dtTracePeerSelectionTracer = mkTracer $ traceWith peerSelectionTr - , Diffusion.dtDebugPeerSelectionTracer = Tracer $ + , Diffusion.dtDebugPeerSelectionTracer = mkTracer $ traceWith debugPeerSelectionTr - , Diffusion.dtTracePeerSelectionCounters = Tracer $ + , Diffusion.dtTracePeerSelectionCounters = mkTracer $ traceWith peerSelectionCountersTr - , Diffusion.dtPeerSelectionActionsTracer = Tracer $ + , Diffusion.dtPeerSelectionActionsTracer = mkTracer $ traceWith peerSelectionActionsTr - , Diffusion.dtConnectionManagerTracer = Tracer $ + , Diffusion.dtConnectionManagerTracer = mkTracer $ traceWith connectionManagerTr - , Diffusion.dtConnectionManagerTransitionTracer = Tracer $ + , Diffusion.dtConnectionManagerTransitionTracer = mkTracer $ traceWith connectionManagerTransitionsTr - , Diffusion.dtServerTracer = Tracer $ + , Diffusion.dtServerTracer = mkTracer $ traceWith serverTr - , Diffusion.dtInboundGovernorTracer = Tracer $ + , Diffusion.dtInboundGovernorTracer = mkTracer $ traceWith inboundGovernorTr - , Diffusion.dtLocalInboundGovernorTracer = Tracer $ + , Diffusion.dtLocalInboundGovernorTracer = mkTracer $ traceWith localInboundGovernorTr - , Diffusion.dtInboundGovernorTransitionTracer = Tracer $ + , Diffusion.dtInboundGovernorTransitionTracer = mkTracer $ traceWith inboundGovernorTransitionsTr - , Diffusion.dtLocalConnectionManagerTracer = Tracer $ + , Diffusion.dtLocalConnectionManagerTracer = mkTracer $ traceWith localConnectionManagerTr - , Diffusion.dtLocalServerTracer = Tracer $ + , Diffusion.dtLocalServerTracer = mkTracer $ traceWith localServerTr - , Diffusion.dtTraceLedgerPeersTracer = Tracer $ + , Diffusion.dtTraceLedgerPeersTracer = mkTracer $ traceWith dtLedgerPeersTr - , Diffusion.dtDnsTracer = Tracer $ + , Diffusion.dtDnsTracer = mkTracer $ traceWith dtDnsTr } diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs index 1427af94e67..92ec34893cb 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs @@ -43,12 +43,11 @@ import Ouroboros.Consensus.Storage.ImmutableDB.Chunks.Internal (chunkN import qualified Ouroboros.Consensus.Storage.ImmutableDB.Impl.Types as ImmDB import qualified Ouroboros.Consensus.Storage.LedgerDB as LedgerDB import qualified Ouroboros.Consensus.Storage.LedgerDB.Snapshots as LedgerDB -import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore as V1 -import qualified Ouroboros.Consensus.Storage.LedgerDB.V1.BackingStore.Impl.LMDB as LMDB import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.Backend as V2 import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.InMemory as InMemory import qualified Ouroboros.Consensus.Storage.LedgerDB.V2.LSM as LSM -import qualified Ouroboros.Consensus.Storage.PerasCertDB.Impl as PerasCertDB +import qualified Ouroboros.Consensus.Storage.PerasCertDB as PerasCertDB +import qualified Ouroboros.Consensus.Storage.PerasVoteDB as PerasVoteDB import qualified Ouroboros.Consensus.Storage.VolatileDB as VolDB import Ouroboros.Consensus.TypeFamilyWrappers import Ouroboros.Consensus.Util.Condense (condense) @@ -59,6 +58,7 @@ import Ouroboros.Network.Block (MaxSlotNo (..)) import Data.Aeson (Object, ToJSON, Value (Object, String), object, toJSON, (.=)) import qualified Data.ByteString.Base16 as B16 import Data.Int (Int64) +import qualified Data.List.NonEmpty as NonEmpty import Data.SOP (All, K (..), hcmap, hcollapse) import Data.Text (Text) import qualified Data.Text as Text @@ -104,6 +104,7 @@ instance ( LogFormatting (Header blk) ) => LogFormatting (ChainDB.TraceEvent blk) where forHuman ChainDB.TraceLastShutdownUnclean = "ChainDB is not clean. Validating all immutable chunks" + forHuman (ChainDB.TracePerasVoteDbEvent v) = forHuman v forHuman (ChainDB.TraceAddBlockEvent v) = forHuman v forHuman (ChainDB.TraceFollowerEvent v) = forHuman v forHuman (ChainDB.TraceCopyToImmutableDBEvent v) = forHuman v @@ -130,6 +131,8 @@ instance ( LogFormatting (Header blk) RisingEdge -> "risingEdge" .= True FallingEdgeWith pt -> "fallingEdge" .= forMachine dtal pt ] + forMachine details (ChainDB.TracePerasVoteDbEvent v) = + forMachine details v forMachine details (ChainDB.TraceAddBlockEvent v) = forMachine details v forMachine details (ChainDB.TraceFollowerEvent v) = @@ -158,6 +161,7 @@ instance ( LogFormatting (Header blk) asMetrics ChainDB.TraceLastShutdownUnclean = [] asMetrics (ChainDB.TraceChainSelStarvationEvent _) = [] + asMetrics (ChainDB.TracePerasVoteDbEvent v) = asMetrics v asMetrics (ChainDB.TraceAddBlockEvent v) = asMetrics v asMetrics (ChainDB.TraceFollowerEvent v) = asMetrics v asMetrics (ChainDB.TraceCopyToImmutableDBEvent v) = asMetrics v @@ -177,6 +181,8 @@ instance MetaTrace (ChainDB.TraceEvent blk) where Namespace [] ["LastShutdownUnclean"] namespaceFor ChainDB.TraceChainSelStarvationEvent{} = Namespace [] ["ChainSelStarvationEvent"] + namespaceFor (ChainDB.TracePerasVoteDbEvent ev) = + nsPrependInner "PerasVoteDbEvent" (namespaceFor ev) namespaceFor (ChainDB.TraceAddBlockEvent ev) = nsPrependInner "AddBlockEvent" (namespaceFor ev) namespaceFor (ChainDB.TraceFollowerEvent ev) = @@ -1642,6 +1648,79 @@ instance MetaTrace (ChainDB.UnknownRange blk) where , Namespace [] ["ForkTooOld"] ] +-- -------------------------------------------------------------------------------- +-- -- Peras +-- -------------------------------------------------------------------------------- + +instance MetaTrace (PerasVoteDB.TraceEvent blk) where + namespaceFor (PerasVoteDB.AddVote {}) = Namespace [] ["AddVote"] + namespaceFor (PerasVoteDB.GarbageCollected {}) = Namespace [] ["GarbageCollected"] + allNamespaces = + [ Namespace [] ["AddVote"] + , Namespace [] ["GarbageCollected"] + ] + + severityFor _ _ = Just Info + privacyFor _ _ = Just Public + detailsFor _ _ = Just DNormal + + documentFor (Namespace _ ["AddVote"]) = Just "AddVote" + documentFor (Namespace _ ["GarbageCollected"]) = Just "GarbageCollected" + documentFor _ = Nothing + +instance StandardHash blk => LogFormatting (PerasVoteDB.TraceEvent blk) where + forHuman (PerasVoteDB.AddVote voteId _vote result) = + "Peras vote " <> Text.pack (show voteId) <> ": " <> Text.pack (show result) + forHuman (PerasVoteDB.GarbageCollected slotNo) = + "Peras vote DB garbage collected at slot " <> Text.pack (show slotNo) + + forMachine _dtal (PerasVoteDB.AddVote voteId _vote result) = + mconcat [ "kind" .= String "AddVote" + , "voteId" .= String (Text.pack $ show voteId) + , "result" .= String (Text.pack $ show result) + ] + forMachine _dtal (PerasVoteDB.GarbageCollected slotNo) = + mconcat [ "kind" .= String "GarbageCollected" + , "slot" .= String (Text.pack $ show slotNo) + ] + + asMetrics _ = [] + +instance MetaTrace (PerasCertDB.TraceEvent blk) where + namespaceFor (PerasCertDB.AddCert {}) = Namespace [] ["AddCert"] + namespaceFor (PerasCertDB.GarbageCollected _) = Namespace [] ["GarbageCollected"] + allNamespaces = + [ Namespace [] ["AddCert"] + , Namespace [] ["GarbageCollected"] + ] + + severityFor _ _ = Just Info + privacyFor _ _ = Just Public + detailsFor _ _ = Just DNormal + + documentFor (Namespace _ ["AddCert"]) = Just "AddCert" + documentFor (Namespace _ ["GarbageCollected"]) = Just "GarbageCollected" + documentFor _ = Nothing + +instance LogFormatting (PerasCertDB.TraceEvent blk) where + forHuman (PerasCertDB.AddCert roundNo _cert result) = + "Peras certificate for round " <> Text.pack (show roundNo) <> ": " <> Text.pack (show result) + forHuman (PerasCertDB.GarbageCollected slotNo) = + "Peras certificate DB garbage collected at slot " <> Text.pack (show slotNo) + + forMachine _dtal (PerasCertDB.AddCert roundNo _cert result) = + mconcat [ "kind" .= String "AddCert" + , "round" .= String (Text.pack $ show roundNo) + , "result" .= String (Text.pack $ show result) + ] + forMachine _dtal (PerasCertDB.GarbageCollected slotNo) = + mconcat [ "kind" .= String "GarbageCollected" + , "slot" .= String (Text.pack $ show slotNo) + ] + + asMetrics _ = [] + + -- -------------------------------------------------------------------------------- -- -- LedgerDB.TraceEvent -- -------------------------------------------------------------------------------- @@ -1712,6 +1791,13 @@ instance MetaTrace (LedgerDB.TraceEvent blk) where instance ( StandardHash blk , ConvertRawHash blk) => LogFormatting (LedgerDB.TraceSnapshotEvent blk) where + forHuman (LedgerDB.SnapshotRequestDelayed _snapshotRequestTime delayBeforeSnapshotting slots) = + Text.unwords [ "Scheduling to take ledger state snapshots at slots " + , showT (NonEmpty.toList slots) + , ", with a randomised delay of" + , showT delayBeforeSnapshotting + ] + forHuman LedgerDB.SnapshotRequestCompleted = "Completed taking a ledger state snapshot" forHuman (LedgerDB.TookSnapshot snap pt RisingEdge) = Text.unwords [ "Taking ledger snapshot" , showT snap @@ -1750,6 +1836,15 @@ instance ( StandardHash blk " Snapshot was created for a different backend. Convert it with `snapshot-converter`." _ -> "" + forMachine _dtals (LedgerDB.SnapshotRequestDelayed snapshotRequestTime delayBeforeSnapshotting slots) = + mconcat [ "kind" .= String "SnapshotRequestDelayed" + , "requestTime" .= show snapshotRequestTime + , "delayBeforeSnapshotting" .= show delayBeforeSnapshotting + , "slots" .= toJSON (NonEmpty.toList slots) + ] + forMachine _dtals LedgerDB.SnapshotRequestCompleted = + mconcat [ "kind" .= String "SnapshotRequestCompleted" + ] forMachine dtals (LedgerDB.TookSnapshot snap pt enclosedTiming) = mconcat [ "kind" .= String "TookSnapshot" , "snapshot" .= forMachine dtals snap @@ -1765,10 +1860,14 @@ instance ( StandardHash blk , "failure" .= show failure ] instance MetaTrace (LedgerDB.TraceSnapshotEvent blk) where + namespaceFor LedgerDB.SnapshotRequestDelayed {} = Namespace [] ["SnapshotRequestDelayed"] + namespaceFor LedgerDB.SnapshotRequestCompleted {} = Namespace [] ["SnapshotRequestCompleted"] namespaceFor LedgerDB.TookSnapshot {} = Namespace [] ["TookSnapshot"] namespaceFor LedgerDB.DeletedSnapshot {} = Namespace [] ["DeletedSnapshot"] namespaceFor LedgerDB.InvalidSnapshot {} = Namespace [] ["InvalidSnapshot"] + severityFor (Namespace _ ["SnapshotRequestDelayed"]) _ = Just Debug + severityFor (Namespace _ ["SnapshotRequestCompleted"]) _ = Just Debug severityFor (Namespace _ ["TookSnapshot"]) _ = Just Info severityFor (Namespace _ ["DeletedSnapshot"]) _ = Just Debug severityFor (Namespace _ ["InvalidSnapshot"]) _ = Just Error @@ -1786,12 +1885,18 @@ instance MetaTrace (LedgerDB.TraceSnapshotEvent blk) where , " seems to be from an old node or different backend, it will" , " be deleted" ] + documentFor (Namespace _ ["SnapshotRequestDelayed"]) = Just + "A delayed snapshot request was issued. The snapshot will be initiated at the specified timestamp, with the specified delay and for the specified slots" + documentFor (Namespace _ ["SnapshotRequestCompleted"]) = Just + "The delayed snapshot request was completed" documentFor _ = Nothing allNamespaces = [ Namespace [] ["TookSnapshot"] , Namespace [] ["DeletedSnapshot"] , Namespace [] ["InvalidSnapshot"] + , Namespace [] ["SnapshotRequestDelayed"] + , Namespace [] ["SnapshotRequestCompleted"] ] -------------------------------------------------------------------------------- @@ -2018,303 +2123,28 @@ instance MetaTrace LedgerDB.TraceForkerEvent where -------------------------------------------------------------------------------- instance LogFormatting LedgerDB.FlavorImplSpecificTrace where - forMachine dtal (LedgerDB.FlavorImplSpecificTraceV1 ev) = forMachine dtal ev forMachine dtal (LedgerDB.FlavorImplSpecificTraceV2 ev) = forMachine dtal ev - forHuman (LedgerDB.FlavorImplSpecificTraceV1 ev) = forHuman ev forHuman (LedgerDB.FlavorImplSpecificTraceV2 ev) = forHuman ev instance MetaTrace LedgerDB.FlavorImplSpecificTrace where - namespaceFor (LedgerDB.FlavorImplSpecificTraceV1 ev) = - nsPrependInner "V1" (namespaceFor ev) namespaceFor (LedgerDB.FlavorImplSpecificTraceV2 ev) = nsPrependInner "V2" (namespaceFor ev) - severityFor (Namespace out ("V1" : tl)) Nothing = - severityFor (Namespace out tl :: Namespace V1.SomeBackendTrace) Nothing - severityFor (Namespace out ("V1" : tl)) (Just (LedgerDB.FlavorImplSpecificTraceV1 ev)) = - severityFor (Namespace out tl :: Namespace V1.SomeBackendTrace) (Just ev) severityFor (Namespace out ("V2" : tl)) Nothing = severityFor (Namespace out tl :: Namespace V2.LedgerDBV2Trace) Nothing severityFor (Namespace out ("V2" : tl)) (Just (LedgerDB.FlavorImplSpecificTraceV2 ev)) = severityFor (Namespace out tl :: Namespace V2.LedgerDBV2Trace) (Just ev) severityFor _ _ = Nothing - documentFor (Namespace out ("V1" : tl)) = - documentFor (Namespace out tl :: Namespace V1.SomeBackendTrace) documentFor (Namespace out ("V2" : tl)) = documentFor (Namespace out tl :: Namespace V2.LedgerDBV2Trace) documentFor _ = Nothing allNamespaces = - map (nsPrependInner "V1") - (allNamespaces :: [Namespace V1.SomeBackendTrace]) - ++ map (nsPrependInner "V2") + map (nsPrependInner "V2") (allNamespaces :: [Namespace V2.LedgerDBV2Trace]) --------------------------------------------------------------------------------- --- V1 --------------------------------------------------------------------------------- - -unwrapV1Trace :: forall a backend. Typeable backend => (V1.Trace LMDB.LMDB -> a) -> V1.Trace backend -> a -unwrapV1Trace g ev = - case cast @(V1.Trace backend) @(V1.Trace LMDB.LMDB) ev of - Just t -> g t - _ -> error "blah" - -instance LogFormatting V1.SomeBackendTrace where - forMachine dtal (V1.SomeBackendTrace ev) = - unwrapV1Trace (forMachine dtal) ev - - forHuman (V1.SomeBackendTrace ev) = - unwrapV1Trace forHuman ev - -instance MetaTrace V1.SomeBackendTrace where - namespaceFor (V1.SomeBackendTrace ev) = - unwrapV1Trace (nsPrependInner "LMDB" . namespaceFor) ev - - severityFor (Namespace out ("LMDB" : tl)) (Just (V1.SomeBackendTrace ev)) = - unwrapV1Trace (severityFor (Namespace out tl :: Namespace (V1.Trace LMDB.LMDB)) . Just) ev - severityFor (Namespace _ ("LMDB" : _)) Nothing = - Just Debug - severityFor _ _ = Nothing - - documentFor (Namespace _ ("LMDB" : _)) = - Just "An LMDB trace" - documentFor _ = Nothing - - allNamespaces = - map (nsPrependInner "LMDB") - (allNamespaces :: [Namespace (V1.Trace LMDB.LMDB)]) - -instance LogFormatting (V1.Trace LMDB.LMDB) where - forMachine _dtal (LMDB.OnDiskBackingStoreInitialise limits) = - mconcat [ "kind" .= String "LMDBBackingStoreInitialise", "limits" .= showT limits ] - forMachine dtal (LMDB.OnDiskBackingStoreTrace ev) = forMachine dtal ev - - forHuman (LMDB.OnDiskBackingStoreInitialise limits) = "Initializing LMDB backing store with limits " <> showT limits - forHuman (LMDB.OnDiskBackingStoreTrace ev) = forHuman ev - -instance MetaTrace (V1.Trace LMDB.LMDB) where - namespaceFor LMDB.OnDiskBackingStoreInitialise{} = - Namespace [] ["Initialise"] - namespaceFor (LMDB.OnDiskBackingStoreTrace ev) = - nsPrependInner "BackingStoreEvent" (namespaceFor ev) - - severityFor (Namespace _ ("Initialise" : _)) _ = Just Debug - severityFor (Namespace out ("BackingStoreEvent" : tl)) Nothing = - severityFor (Namespace out tl :: Namespace V1.BackingStoreTrace) Nothing - severityFor (Namespace out ("BackingStoreEvent" : tl)) (Just (LMDB.OnDiskBackingStoreTrace ev)) = - severityFor (Namespace out tl :: Namespace V1.BackingStoreTrace) (Just ev) - severityFor _ _ = Nothing - - documentFor (Namespace _ ("Initialise" : _)) = Just - "Backing store is being initialised" - documentFor (Namespace out ("BackingStoreEvent" : tl)) = - documentFor (Namespace out tl :: Namespace V1.BackingStoreTrace) - documentFor _ = Nothing - - allNamespaces = - Namespace [] ["Initialise"] - : map (nsPrependInner "BackingStoreEvent") - (allNamespaces :: [Namespace V1.BackingStoreTrace]) - -instance LogFormatting V1.BackingStoreTrace where - forMachine _dtals V1.BSOpening = mempty - forMachine _dtals (V1.BSOpened p) = - maybe mempty (\p' -> mconcat [ "path" .= showT p' ]) p - forMachine _dtals (V1.BSInitialisingFromCopy p) = - mconcat [ "path" .= showT p ] - forMachine _dtals (V1.BSInitialisedFromCopy p) = - mconcat [ "path" .= showT p ] - forMachine _dtals (V1.BSInitialisingFromValues sl) = - mconcat [ "slot" .= showT sl ] - forMachine _dtals (V1.BSInitialisedFromValues sl) = - mconcat [ "slot" .= showT sl ] - forMachine _dtals V1.BSClosing = mempty - forMachine _dtals V1.BSAlreadyClosed = mempty - forMachine _dtals V1.BSClosed = mempty - forMachine _dtals (V1.BSCopying p) = - mconcat [ "path" .= showT p ] - forMachine _dtals (V1.BSCopied p) = - mconcat [ "path" .= showT p ] - forMachine _dtals V1.BSCreatingValueHandle = mempty - forMachine _dtals V1.BSCreatedValueHandle = mempty - forMachine _dtals (V1.BSWriting s) = - mconcat [ "slot" .= showT s ] - forMachine _dtals (V1.BSWritten s1 s2) = - mconcat [ "old" .= showT s1, "new" .= showT s2 ] - forMachine _dtals (V1.BSValueHandleTrace i _ev) = - maybe mempty (\i' -> mconcat ["idx" .= showT i']) i -instance LogFormatting V1.BackingStoreValueHandleTrace where - forMachine _dtals V1.BSVHClosing = mempty - forMachine _dtals V1.BSVHAlreadyClosed = mempty - forMachine _dtals V1.BSVHClosed = mempty - forMachine _dtals V1.BSVHRangeReading = mempty - forMachine _dtals V1.BSVHRangeRead = mempty - forMachine _dtals V1.BSVHReading = mempty - forMachine _dtals V1.BSVHRead = mempty - forMachine _dtals V1.BSVHStatting = mempty - forMachine _dtals V1.BSVHStatted = mempty - -instance MetaTrace V1.BackingStoreTrace where - namespaceFor V1.BSOpening = Namespace [] ["Opening"] - namespaceFor V1.BSOpened{} = Namespace [] ["Opened"] - namespaceFor V1.BSInitialisingFromCopy{} = - Namespace [] ["InitialisingFromCopy"] - namespaceFor V1.BSInitialisedFromCopy{} = - Namespace [] ["InitialisedFromCopy"] - namespaceFor V1.BSInitialisingFromValues{} = - Namespace [] ["InitialisingFromValues"] - namespaceFor V1.BSInitialisedFromValues{} = - Namespace [] ["InitialisedFromValues"] - namespaceFor V1.BSClosing = Namespace [] ["Closing"] - namespaceFor V1.BSAlreadyClosed = Namespace [] ["AlreadyClosed"] - namespaceFor V1.BSClosed = Namespace [] ["Closed"] - namespaceFor V1.BSCopying{} = Namespace [] ["Copying"] - namespaceFor V1.BSCopied{} = Namespace [] ["Copied"] - namespaceFor V1.BSCreatingValueHandle = Namespace [] ["CreatingValueHandle"] - namespaceFor V1.BSCreatedValueHandle = Namespace [] ["CreatedValueHandle"] - namespaceFor (V1.BSValueHandleTrace _ bsValueHandleTrace) = - nsPrependInner "ValueHandleTrace" (namespaceFor bsValueHandleTrace) - namespaceFor V1.BSWriting{} = Namespace [] ["Writing"] - namespaceFor V1.BSWritten{} = Namespace [] ["Written"] - - severityFor (Namespace _ ("Opening" : _)) _ = Just Debug - severityFor (Namespace _ ("Opened" : _)) _ = Just Debug - severityFor (Namespace _ ("InitialisingFromCopy" : _)) _ = Just Debug - severityFor (Namespace _ ("InitialisedFromCopy" : _)) _ = Just Debug - severityFor (Namespace _ ("InitialisingFromValues" : _)) _ = Just Debug - severityFor (Namespace _ ("InitialisedFromValues" : _)) _ = Just Debug - severityFor (Namespace _ ("Closing" : _)) _ = Just Debug - severityFor (Namespace _ ("AlreadyClosed" : _)) _ = Just Debug - severityFor (Namespace _ ("Closed" : _)) _ = Just Debug - severityFor (Namespace _ ("Copying" : _)) _ = Just Debug - severityFor (Namespace _ ("Copied" : _)) _ = Just Debug - severityFor (Namespace _ ("CreatingValueHandle" : _)) _ = Just Debug - severityFor (Namespace _ ("CreatedValueHandle" : _)) _ = Just Debug - severityFor (Namespace out ("ValueHandleTrace" : t1)) Nothing = - severityFor - (Namespace out t1 :: Namespace V1.BackingStoreValueHandleTrace) - Nothing - severityFor - (Namespace out ("ValueHandleTrace" : t1)) - (Just (V1.BSValueHandleTrace _ bsValueHandleTrace)) = - severityFor - (Namespace out t1 :: Namespace V1.BackingStoreValueHandleTrace) - (Just bsValueHandleTrace) - severityFor (Namespace _ ("Writing" : _)) _ = Just Debug - severityFor (Namespace _ ("Written" : _)) _ = Just Debug - severityFor _ _ = Nothing - - documentFor (Namespace _ ("Opening" : _ )) = Just - "Opening backing store" - documentFor (Namespace _ ("Opened" : _ )) = Just - "Backing store opened" - documentFor (Namespace _ ("InitialisingFromCopy" : _ )) = Just - "Initialising backing store from copy" - documentFor (Namespace _ ("InitialisedFromCopy" : _ )) = Just - "Backing store initialised from copy" - documentFor (Namespace _ ("InitialisingFromValues" : _ )) = Just - "Initialising backing store from values" - documentFor (Namespace _ ("InitialisedFromValues" : _ )) = Just - "Backing store initialised from values" - documentFor (Namespace _ ("Closing" : _ )) = Just - "Closing backing store" - documentFor (Namespace _ ("AlreadyClosed" : _ )) = Just - "Backing store is already closed" - documentFor (Namespace _ ("Closed" : _ )) = Just - "Backing store closed" - documentFor (Namespace _ ("Copying" : _ )) = Just - "Copying backing store" - documentFor (Namespace _ ("Copied" : _ )) = Just - "Backing store copied" - documentFor (Namespace _ ("CreatingValueHandle" : _ )) = Just - "Creating value handle for backing store" - documentFor (Namespace _ ("CreatedValueHandle" : _ )) = Just - "Value handle for backing store created" - documentFor (Namespace out ("ValueHandleTrace" : t1 )) = - documentFor (Namespace out t1 :: Namespace V1.BackingStoreValueHandleTrace) - documentFor (Namespace _ ("Writing" : _ )) = Just - "Writing backing store" - documentFor (Namespace _ ("Written" : _ )) = Just - "Backing store written" - documentFor _ = Nothing - - allNamespaces = - [ Namespace [] ["Opening"] - , Namespace [] ["Opened"] - , Namespace [] ["InitialisingFromCopy"] - , Namespace [] ["InitialisedFromCopy"] - , Namespace [] ["InitialisingFromValues"] - , Namespace [] ["InitialisedFromValues"] - , Namespace [] ["Closing"] - , Namespace [] ["AlreadyClosed"] - , Namespace [] ["Closed"] - , Namespace [] ["Copying"] - , Namespace [] ["Copied"] - , Namespace [] ["CreatingValueHandle"] - , Namespace [] ["CreatedValueHandle"] - , Namespace [] ["Writing"] - , Namespace [] ["Written"] - ] ++ map (nsPrependInner "ValueHandleTrace") - (allNamespaces :: [Namespace V1.BackingStoreValueHandleTrace]) - - -instance MetaTrace V1.BackingStoreValueHandleTrace where - namespaceFor V1.BSVHClosing = Namespace [] ["Closing"] - namespaceFor V1.BSVHAlreadyClosed = Namespace [] ["AlreadyClosed"] - namespaceFor V1.BSVHClosed = Namespace [] ["Closed"] - namespaceFor V1.BSVHRangeReading = Namespace [] ["RangeReading"] - namespaceFor V1.BSVHRangeRead = Namespace [] ["RangeRead"] - namespaceFor V1.BSVHReading = Namespace [] ["Reading"] - namespaceFor V1.BSVHRead = Namespace [] ["Read"] - namespaceFor V1.BSVHStatting = Namespace [] ["Statting"] - namespaceFor V1.BSVHStatted = Namespace [] ["Statted"] - - severityFor (Namespace _ ("Closing" : _ )) _ = Just Debug - severityFor (Namespace _ ("AlreadyClosed" : _ )) _ = Just Debug - severityFor (Namespace _ ("Closed" : _ )) _ = Just Debug - severityFor (Namespace _ ("RangeReading" : _ )) _ = Just Debug - severityFor (Namespace _ ("RangeRead" : _ )) _ = Just Debug - severityFor (Namespace _ ("Reading" : _ )) _ = Just Debug - severityFor (Namespace _ ("Read" : _ )) _ = Just Debug - severityFor (Namespace _ ("Statting" : _ )) _ = Just Debug - severityFor (Namespace _ ("Statted" : _ )) _ = Just Debug - severityFor _ _ = Nothing - - documentFor (Namespace _ ("Closing" : _ )) = Just - "Closing backing store value handle" - documentFor (Namespace _ ("AlreadyClosed" : _ )) = Just - "Backing store value handle already clsoed" - documentFor (Namespace _ ("Closed" : _ )) = Just - "Backing store value handle closed" - documentFor (Namespace _ ("RangeReading" : _ )) = Just - "Reading range for backing store value handle" - documentFor (Namespace _ ("RangeRead" : _ )) = Just - "Range for backing store value handle read" - documentFor (Namespace _ ("Reading" : _ )) = Just - "Reading backing store value handle" - documentFor (Namespace _ ("Read" : _ )) = Just - "Backing store value handle read" - documentFor (Namespace _ ("Statting" : _ )) = Just - "Statting backing store value handle" - documentFor (Namespace _ ("Statted" : _ )) = Just - "Backing store value handle statted" - documentFor _ = Nothing - - allNamespaces = - [ Namespace [] ["Closing"] - , Namespace [] ["AlreadyClosed"] - , Namespace [] ["Closed"] - , Namespace [] ["RangeReading"] - , Namespace [] ["RangeRead"] - , Namespace [] ["Reading"] - , Namespace [] ["Read"] - , Namespace [] ["Statting"] - , Namespace [] ["Statted"] - ] - {------------------------------------------------------------------------------- V2 -------------------------------------------------------------------------------} @@ -3088,30 +2918,6 @@ instance (Show (PBFT.PBftVerKeyHash c)) , "numForged" .= numForged ] --- PerasCertDB.TraceEvent instances -instance LogFormatting (PerasCertDB.TraceEvent blk) where - forHuman (PerasCertDB.AddedPerasCert _cert _peer) = "Added Peras certificate to database" - forHuman (PerasCertDB.IgnoredCertAlreadyInDB _cert _peer) = "Ignored Peras certificate already in database" - forHuman PerasCertDB.OpenedPerasCertDB = "Opened Peras certificate database" - forHuman PerasCertDB.ClosedPerasCertDB = "Closed Peras certificate database" - forHuman (PerasCertDB.AddingPerasCert _cert _peer) = "Adding Peras certificate to database" - - forMachine _dtal (PerasCertDB.AddedPerasCert cert _peer) = - mconcat ["kind" .= String "AddedPerasCert", - "cert" .= String (Text.pack $ show cert)] - forMachine _dtal (PerasCertDB.IgnoredCertAlreadyInDB cert _peer) = - mconcat ["kind" .= String "IgnoredCertAlreadyInDB", - "cert" .= String (Text.pack $ show cert)] - forMachine _dtal PerasCertDB.OpenedPerasCertDB = - mconcat ["kind" .= String "OpenedPerasCertDB"] - forMachine _dtal PerasCertDB.ClosedPerasCertDB = - mconcat ["kind" .= String "ClosedPerasCertDB"] - forMachine _dtal (PerasCertDB.AddingPerasCert cert _peer) = - mconcat ["kind" .= String "AddingPerasCert", - "cert" .= String (Text.pack $ show cert)] - - asMetrics _ = [] - -- ChainDB.TraceAddPerasCertEvent instances instance ConvertRawHash blk => LogFormatting (ChainDB.TraceAddPerasCertEvent blk) where forHuman (ChainDB.AddedPerasCertToQueue roundNo boostedBlock _queueSize) = @@ -3168,54 +2974,6 @@ instance ConvertRawHash blk => LogFormatting (ChainDB.TraceAddPerasCertEvent blk asMetrics _ = [] --- PerasCertDB.TraceEvent MetaTrace instance -instance MetaTrace (PerasCertDB.TraceEvent blk) where - namespaceFor (PerasCertDB.AddedPerasCert _ _) = - Namespace [] ["AddedPerasCert"] - namespaceFor (PerasCertDB.IgnoredCertAlreadyInDB _ _) = - Namespace [] ["IgnoredCertAlreadyInDB"] - namespaceFor PerasCertDB.OpenedPerasCertDB = - Namespace [] ["OpenedPerasCertDB"] - namespaceFor PerasCertDB.ClosedPerasCertDB = - Namespace [] ["ClosedPerasCertDB"] - namespaceFor (PerasCertDB.AddingPerasCert _ _) = - Namespace [] ["AddingPerasCert"] - - severityFor (Namespace _ ["AddedPerasCert"]) _ = Just Info - severityFor (Namespace _ ["IgnoredCertAlreadyInDB"]) _ = Just Info - severityFor (Namespace _ ["OpenedPerasCertDB"]) _ = Just Info - severityFor (Namespace _ ["ClosedPerasCertDB"]) _ = Just Info - severityFor (Namespace _ ["AddingPerasCert"]) _ = Just Debug - severityFor _ _ = Nothing - - privacyFor (Namespace _ ["AddedPerasCert"]) _ = Just Public - privacyFor (Namespace _ ["IgnoredCertAlreadyInDB"]) _ = Just Public - privacyFor (Namespace _ ["OpenedPerasCertDB"]) _ = Just Public - privacyFor (Namespace _ ["ClosedPerasCertDB"]) _ = Just Public - privacyFor (Namespace _ ["AddingPerasCert"]) _ = Just Public - privacyFor _ _ = Nothing - - detailsFor (Namespace _ ["AddedPerasCert"]) _ = Just DNormal - detailsFor (Namespace _ ["IgnoredCertAlreadyInDB"]) _ = Just DNormal - detailsFor (Namespace _ ["OpenedPerasCertDB"]) _ = Just DNormal - detailsFor (Namespace _ ["ClosedPerasCertDB"]) _ = Just DNormal - detailsFor (Namespace _ ["AddingPerasCert"]) _ = Just DDetailed - detailsFor _ _ = Nothing - - documentFor (Namespace _ ["AddedPerasCert"]) = Just "Certificate added to Peras certificate database" - documentFor (Namespace _ ["IgnoredCertAlreadyInDB"]) = Just "Certificate ignored as it was already in the database" - documentFor (Namespace _ ["OpenedPerasCertDB"]) = Just "Peras certificate database opened" - documentFor (Namespace _ ["ClosedPerasCertDB"]) = Just "Peras certificate database closed" - documentFor (Namespace _ ["AddingPerasCert"]) = Just "Adding certificate to Peras certificate database" - documentFor _ = Nothing - - allNamespaces = - [Namespace [] ["AddedPerasCert"], - Namespace [] ["IgnoredCertAlreadyInDB"], - Namespace [] ["OpenedPerasCertDB"], - Namespace [] ["ClosedPerasCertDB"], - Namespace [] ["AddingPerasCert"]] - -- ChainDB.TraceAddPerasCertEvent MetaTrace instance instance MetaTrace (ChainDB.TraceAddPerasCertEvent blk) where namespaceFor ChainDB.AddedPerasCertToQueue{} = Namespace [] ["AddedPerasCertToQueue"] diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/Consensus.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/Consensus.hs index 36ae550c0f2..edb2462b0d0 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/Consensus.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/Consensus.hs @@ -84,6 +84,8 @@ import qualified Data.Text as Text import Data.Time (NominalDiffTime) import Data.Word (Word32, Word64) import Network.TypedProtocol.Core +import Ouroboros.Consensus.MiniProtocol.ObjectDiffusion.Inbound (TraceObjectDiffusionInbound (..)) +import Ouroboros.Consensus.MiniProtocol.ObjectDiffusion.Outbound (TraceObjectDiffusionOutbound (..)) enclosingValue :: ToJSON a => Enclosing' a -> Value enclosingValue RisingEdge = object [ "edge" .= String "Starting" ] @@ -1040,6 +1042,7 @@ instance ( HasHeader blk instance MetaTrace SanityCheckIssue where namespaceFor InconsistentSecurityParam {} = Namespace [] ["SanityCheckIssue"] + namespaceFor _ = Namespace [] ["SanityCheckIssue"] severityFor (Namespace _ ["SanityCheckIssue"]) _ = Just Error severityFor _ _ = Nothing @@ -1054,8 +1057,12 @@ instance LogFormatting SanityCheckIssue where mconcat [ "kind" .= String "InconsistentSecurityParam" , "error" .= String (Text.pack $ show e) ] + forMachine _ _ = + mconcat [ "kind" .= String "SnapshotIssue" + ] forHuman (InconsistentSecurityParam e) = "Configuration contains multiple security parameters: " <> Text.pack (show e) + forHuman _ = "SnapshotIssue" -------------------------------------------------------------------------------- -- TxSubmissionServer Tracer @@ -2308,3 +2315,128 @@ instance MetaTrace KESAgentClientTrace where allNamespaces = Namespace [] ["KESAgentClientException"] : fmap nsCast (allNamespaces :: [Namespace Agent.ServiceClientTrace]) + +-------------------------------------------------------------------------------- +-- Peras +-------------------------------------------------------------------------------- + +-- TODO: Move this to a proper place. A lot of this is duplicated in the +-- ToObject instance. This is likely in an incorrect place. Fix +-- duplication. +instance LogFormatting (TraceObjectDiffusionInbound objectId object) where + forMachine _ (TraceObjectDiffusionInboundCollectedObjects payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionInboundCollectedObjects" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ (TraceObjectDiffusionInboundAddedObjects payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionInboundAddedObjects" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ (TraceObjectDiffusionInboundRecvControlMessage payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionInboundRecvControlMessage" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ (TraceObjectDiffusionInboundCanRequestMoreObjects payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionInboundCanRequestMoreObjects" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ (TraceObjectDiffusionInboundCannotRequestMoreObjects payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionInboundCannotRequestMoreObjects" + , "payload" .= String (Text.pack . show $ payload) + ] + +instance MetaTrace (TraceObjectDiffusionInbound objectId object) where + namespaceFor (TraceObjectDiffusionInboundCollectedObjects _) = + Namespace [] ["TraceObjectDiffusionInboundCollectedObjects"] + namespaceFor (TraceObjectDiffusionInboundAddedObjects _) = + Namespace [] ["TraceObjectDiffusionInboundAddedObjects"] + namespaceFor (TraceObjectDiffusionInboundRecvControlMessage _) = + Namespace [] ["TraceObjectDiffusionInboundRecvControlMessage"] + namespaceFor (TraceObjectDiffusionInboundCanRequestMoreObjects _) = + Namespace [] ["TraceObjectDiffusionInboundCanRequestMoreObjects"] + namespaceFor (TraceObjectDiffusionInboundCannotRequestMoreObjects _) = + Namespace [] ["TraceObjectDiffusionInboundCannotRequestMoreObjects"] + + severityFor (Namespace [] ["TraceObjectDiffusionInboundCollectedObjects"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionInboundAddedObjects"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionInboundRecvControlMessage"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionInboundCanRequestMoreObjects"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionInboundCannotRequestMoreObjects"]) _ = Just Info + severityFor _ _ = Nothing + + documentFor _ = Nothing + + allNamespaces = + [ Namespace [] ["TraceObjectDiffusionInboundCollectedObjects"] + , Namespace [] ["TraceObjectDiffusionInboundAddedObjects"] + , Namespace [] ["TraceObjectDiffusionInboundRecvControlMessage"] + , Namespace [] ["TraceObjectDiffusionInboundCanRequestMoreObjects"] + , Namespace [] ["TraceObjectDiffusionInboundCannotRequestMoreObjects"] + ] + +instance + ( Show objectId + , Show object + ) => + LogFormatting (TraceObjectDiffusionOutbound objectId object) + where + forMachine _ (TraceObjectDiffusionOutboundRecvMsgRequestObjectIds payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionOutboundRecvMsgRequestObjectIds" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ (TraceObjectDiffusionOutboundSendMsgReplyObjectIds payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionOutboundSendMsgReplyObjectIds" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ (TraceObjectDiffusionOutboundRecvMsgRequestObjects payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionOutboundRecvMsgRequestObjects" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ (TraceObjectDiffusionOutboundSendMsgReplyObjects payload) = + mconcat + [ "kind" .= String "TraceObjectDiffusionOutboundSendMsgReplyObjects" + , "payload" .= String (Text.pack . show $ payload) + ] + forMachine _ TraceObjectDiffusionOutboundTerminated = + mconcat + [ "kind" .= String "TraceObjectDiffusionOutboundTerminated" + ] + + +instance MetaTrace (TraceObjectDiffusionOutbound objectId object) where + namespaceFor (TraceObjectDiffusionOutboundRecvMsgRequestObjectIds _) = + Namespace [] ["TraceObjectDiffusionOutboundRecvMsgRequestObjectIds"] + namespaceFor (TraceObjectDiffusionOutboundSendMsgReplyObjectIds _) = + Namespace [] ["TraceObjectDiffusionOutboundSendMsgReplyObjectIds"] + namespaceFor (TraceObjectDiffusionOutboundRecvMsgRequestObjects _) = + Namespace [] ["TraceObjectDiffusionOutboundRecvMsgRequestObjects"] + namespaceFor (TraceObjectDiffusionOutboundSendMsgReplyObjects _) = + Namespace [] ["TraceObjectDiffusionOutboundSendMsgReplyObjects"] + namespaceFor TraceObjectDiffusionOutboundTerminated = + Namespace [] ["TraceObjectDiffusionOutboundTerminated"] + + + severityFor (Namespace [] ["TraceObjectDiffusionOutboundRecvMsgRequestObjectIds"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionOutboundSendMsgReplyObjectIds"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionOutboundRecvMsgRequestObjects"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionOutboundSendMsgReplyObjects"]) _ = Just Info + severityFor (Namespace [] ["TraceObjectDiffusionOutboundTerminated"]) _ = Just Info + severityFor _ _ = Nothing + + documentFor _ = Nothing + + allNamespaces = + [ Namespace [] ["TraceObjectDiffusionOutboundRecvMsgRequestObjectIds"] + , Namespace [] ["TraceObjectDiffusionOutboundSendMsgReplyObjectIds"] + , Namespace [] ["TraceObjectDiffusionOutboundRecvMsgRequestObjects"] + , Namespace [] ["TraceObjectDiffusionOutboundSendMsgReplyObjects"] + , Namespace [] ["TraceObjectDiffusionOutboundTerminated"] + ] diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/ConsensusStartupException.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/ConsensusStartupException.hs index 887f609633c..78c98bff879 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/ConsensusStartupException.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/ConsensusStartupException.hs @@ -27,7 +27,7 @@ instance LogFormatting ConsensusStartupException where instance MetaTrace ConsensusStartupException where namespaceFor ConsensusStartupException {} = Namespace [] ["ConsensusStartupException"] - severityFor (Namespace _ ["ConsensusStartupException"]) Nothing = Just Error + severityFor (Namespace _ ["ConsensusStartupException"]) _ = Just Error severityFor _ _ = Nothing documentFor (Namespace _ ["ConsensusStartupException"]) = Just diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/LedgerMetrics.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/LedgerMetrics.hs index 6f2e0820ff4..a94086dbc4c 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/LedgerMetrics.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/LedgerMetrics.hs @@ -40,7 +40,7 @@ import GHC.Conc (labelThread, myThreadId) startLedgerMetricsTracer :: forall blk - . IsLedger (LedgerState blk) + . IsLedger LedgerState blk => LedgerQueries blk => AF.HasHeader (Header blk) => AF.HasHeader blk @@ -93,7 +93,7 @@ data LedgerMetrics = } traceLedgerMetrics :: - ( IsLedger (LedgerState blk) + ( IsLedger LedgerState blk , LedgerQueries blk , AF.HasHeader blk , AF.HasHeader (Header blk)) diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToClient.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToClient.hs index ff105fbc036..e634e504d2f 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToClient.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToClient.hs @@ -17,6 +17,8 @@ import qualified Ouroboros.Network.Protocol.LocalStateQuery.Type as LSQ import qualified Ouroboros.Network.Protocol.LocalTxMonitor.Type as LTM import qualified Ouroboros.Network.Protocol.LocalTxSubmission.Type as LTS import Ouroboros.Network.Tracing () +import Ouroboros.Network.Protocol.ObjectDiffusion.Type (ObjectDiffusion) +import qualified Ouroboros.Network.Protocol.ObjectDiffusion.Type as OD import Data.Aeson (Value (String), (.=)) import Data.Text (Text, pack) @@ -466,3 +468,72 @@ instance MetaTrace (Stateful.AnyMessage (LSQ.LocalStateQuery blk pt (Query blk)) , Namespace [] ["ReAcquire"] , Namespace [] ["Done"] ] + +-- -------------------------------------------------------------------------------- +-- -- TObjectDiffusion Tracer +-- -------------------------------------------------------------------------------- + +instance LogFormatting (Simple.AnyMessage (ObjectDiffusion objectId object)) where + forMachine _dtal (Simple.AnyMessageAndAgency stok OD.MsgInit {}) = + mconcat [ "kind" .= String "MsgInit" + , "agency" .= String (pack $ show stok) + ] + forMachine _dtal (Simple.AnyMessageAndAgency stok OD.MsgRequestObjectIds {}) = + mconcat [ "kind" .= String "MsgRequestObjectIds" + , "agency" .= String (pack $ show stok) + ] + forMachine _dtal (Simple.AnyMessageAndAgency stok OD.MsgReplyObjectIds {}) = + mconcat [ "kind" .= String "MsgReplyObjectIds" + , "agency" .= String (pack $ show stok) + ] + forMachine _dtal (Simple.AnyMessageAndAgency stok OD.MsgRequestObjects {}) = + mconcat [ "kind" .= String "MsgRequestObjects" + , "agency" .= String (pack $ show stok) + ] + forMachine _dtal (Simple.AnyMessageAndAgency stok OD.MsgReplyObjects {}) = + mconcat [ "kind" .= String "MsgReplyObjects" + , "agency" .= String (pack $ show stok) + ] + forMachine _dtal (Simple.AnyMessageAndAgency stok OD.MsgDone {}) = + mconcat [ "kind" .= String "MsgDone" + , "agency" .= String (pack $ show stok) + ] + +instance MetaTrace (Simple.AnyMessage (ObjectDiffusion objectId object)) where + namespaceFor (Simple.AnyMessageAndAgency _agency OD.MsgInit {}) = + Namespace [] ["Init"] + namespaceFor (Simple.AnyMessageAndAgency _agency OD.MsgRequestObjectIds {}) = + Namespace [] ["RequestObjectIds"] + namespaceFor (Simple.AnyMessageAndAgency _agency OD.MsgReplyObjectIds {}) = + Namespace [] ["ReplyObjectIds"] + namespaceFor (Simple.AnyMessageAndAgency _agency OD.MsgRequestObjects {}) = + Namespace [] ["RequestObjects"] + namespaceFor (Simple.AnyMessageAndAgency _agency OD.MsgReplyObjects {}) = + Namespace [] ["ReplyObjects"] + namespaceFor (Simple.AnyMessageAndAgency _agency OD.MsgDone {}) = + Namespace [] ["Done"] + + severityFor (Namespace [] ["Init"]) _ = Just Info + severityFor (Namespace [] ["RequestObjectIds"]) _ = Just Info + severityFor (Namespace [] ["ReplyObjectIds"]) _ = Just Info + severityFor (Namespace [] ["RequestObjects"]) _ = Just Info + severityFor (Namespace [] ["ReplyObjects"]) _ = Just Info + severityFor (Namespace [] ["Done"]) _ = Just Info + severityFor _ _ = Nothing + + documentFor (Namespace [] ["Init"]) = Just "" + documentFor (Namespace [] ["RequestObjectIds"]) = Just "" + documentFor (Namespace [] ["ReplyObjectIds"]) = Just "" + documentFor (Namespace [] ["RequestObjects"]) = Just "" + documentFor (Namespace [] ["ReplyObjects"]) = Just "" + documentFor (Namespace [] ["Done"]) = Just "" + documentFor _ = Nothing + + allNamespaces = + [ Namespace [] ["Init"] + , Namespace [] ["RequestObjectIds"] + , Namespace [] ["ReplyObjectIds"] + , Namespace [] ["RequestObjects"] + , Namespace [] ["ReplyObjects"] + , Namespace [] ["Done"] + ] diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToNode.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToNode.hs index 1d38c3982cc..dcdd1187a1e 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToNode.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/NodeToNode.hs @@ -454,7 +454,7 @@ instance Show remotePeer => LogFormatting (TraceKeepAliveClient remotePeer) wher instance MetaTrace (TraceKeepAliveClient remotePeer) where namespaceFor AddSample {} = Namespace [] ["KeepAliveClient"] - severityFor (Namespace _ ["KeepAliveClient"]) Nothing = Just Info + severityFor (Namespace _ ["KeepAliveClient"]) _ = Just Info severityFor _ _ = Nothing documentFor (Namespace _ ["KeepAliveClient"]) = Just diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/Rpc.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/Rpc.hs index f9781f5efe6..73f52466b9b 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/Rpc.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/Rpc.hs @@ -13,7 +13,7 @@ import Cardano.Api.Pretty import Cardano.Logging hiding (nsInner) import Cardano.Rpc.Server (TraceRpc (..), TraceRpcQuery (..), TraceRpcSubmit (..), - TraceSpanEvent (..)) + TraceRpcSync (..), TraceSpanEvent (..)) import Data.Aeson (Object, Value (..), (.=)) @@ -48,6 +48,13 @@ instance LogFormatting TraceRpc where TraceRpcSubmitSpan s -> [spanToObject s] TraceRpcEvalTxDecodingError _ -> [] TraceRpcEvalTxSpan s -> [spanToObject s] + TraceRpcSync syncTrace -> + ["kind" .= String "SyncService"] + <> case syncTrace of + TraceRpcFetchBlockSpan s -> [spanToObject s] + TraceRpcFetchBlockNotFound _ -> [] + TraceRpcNodeKernelAccessUnavailable -> [] + TraceRpcForkerError _ -> [] forHuman = docToText . pretty @@ -59,6 +66,7 @@ instance LogFormatting TraceRpc where TraceRpcQuery (TraceRpcQuerySearchUtxosSpan (SpanBegin _)) -> [CounterM "rpc.request.QueryService.SearchUtxos" Nothing] TraceRpcSubmit (TraceRpcSubmitSpan (SpanBegin _)) -> [CounterM "rpc.request.SubmitService.SubmitTx" Nothing] TraceRpcSubmit (TraceRpcEvalTxSpan (SpanBegin _)) -> [CounterM "rpc.request.SubmitService.EvalTx" Nothing] + TraceRpcSync (TraceRpcFetchBlockSpan (SpanBegin _)) -> [CounterM "rpc.request.SyncService.FetchBlock" Nothing] _ -> [] instance MetaTrace TraceRpc where @@ -81,6 +89,13 @@ instance MetaTrace TraceRpc where TraceRpcSubmitSpan _ -> ["SubmitTx", "Span"] TraceRpcEvalTxDecodingError _ -> ["EvalTxDecodingError"] TraceRpcEvalTxSpan _ -> ["EvalTx", "Span"] + TraceRpcSync syncTrace -> + "SyncService" + : case syncTrace of + TraceRpcFetchBlockSpan _ -> ["FetchBlock", "Span"] + TraceRpcFetchBlockNotFound _ -> ["FetchBlockNotFound"] + TraceRpcNodeKernelAccessUnavailable -> ["NodeKernelAccessUnavailable"] + TraceRpcForkerError _ -> ["ForkerError"] severityFor (Namespace _ nsInner) _ = case nsInner of ["FatalError"] -> Just Error -- RPC server startup errors @@ -94,6 +109,10 @@ instance MetaTrace TraceRpc where ["SubmitService", "TxDecodingError"] -> Just Debug -- request error ["SubmitService", "TxValidationError"] -> Just Debug -- request error ["SubmitService", "EvalTxDecodingError"] -> Just Debug -- request error + ["SyncService", "FetchBlock", "Span"] -> Just Debug + ["SyncService", "FetchBlockNotFound"] -> Just Debug + ["SyncService", "NodeKernelAccessUnavailable"] -> Just Warning + ["SyncService", "ForkerError"] -> Just Warning _ -> Nothing documentFor (Namespace _ nsInner) = case nsInner of @@ -110,6 +129,10 @@ instance MetaTrace TraceRpc where ["SubmitService", "TxDecodingError"] -> Just "A regular request error, when submitted transaction decoding fails." ["SubmitService", "TxValidationError"] -> Just "A regular request error, when submitted transaction is invalid." ["SubmitService", "EvalTxDecodingError"] -> Just "A regular request error, when evalTx transaction decoding fails." + ["SyncService", "FetchBlock", "Span"] -> Just "Span for the FetchBlock SyncService method." + ["SyncService", "FetchBlockNotFound"] -> Just "Requested block was not found in ChainDB." + ["SyncService", "NodeKernelAccessUnavailable"] -> Just "Node kernel access not yet initialised. The node is still starting up." + ["SyncService", "ForkerError"] -> Just "Unexpected error from ledger forker." _ -> Nothing metricsDocFor (Namespace _ nsInner) = case nsInner of @@ -123,6 +146,8 @@ instance MetaTrace TraceRpc where [("rpc.request.SubmitService.SubmitTx", "Span for the SubmitTx UTXORPC method.")] ["SubmitService", "EvalTx", "Span"] -> [("rpc.request.SubmitService.EvalTx", "Span for the EvalTx UTXORPC method.")] + ["SyncService", "FetchBlock", "Span"] -> + [("rpc.request.SyncService.FetchBlock", "Span for the FetchBlock SyncService method.")] _ -> [] allNamespaces = @@ -138,6 +163,10 @@ instance MetaTrace TraceRpc where , ["SubmitService", "TxDecodingError"] , ["SubmitService", "TxValidationError"] , ["SubmitService", "EvalTxDecodingError"] + , ["SyncService", "FetchBlock", "Span"] + , ["SyncService", "FetchBlockNotFound"] + , ["SyncService", "NodeKernelAccessUnavailable"] + , ["SyncService", "ForkerError"] ] -- helper functions diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/Startup.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/Startup.hs index c8eefaef1cf..abe84e87b0b 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/Startup.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/Startup.hs @@ -67,8 +67,8 @@ getStartupInfo -> IO [StartupTrace blk] getStartupInfo nc (SomeConsensusProtocol whichP pForInfo) fp = do nodeStartTime <- getCurrentTime - let cfg = pInfoConfig $ fst $ Api.protocolInfo @IO pForInfo - basicInfoCommon = BICommon $ BasicInfoCommon { + cfg <- pInfoConfig . fst <$> Api.protocolInfo @IO pForInfo + let basicInfoCommon = BICommon $ BasicInfoCommon { biProtocol = pack . show $ ncProtocol nc , biVersion = pack . showVersion $ version , biCommit = $(gitRev) @@ -288,6 +288,9 @@ instance ( Show (BlockNodeToNodeVersion blk) forMachine _dtal (RpcConfigUpdateError err) = mconcat [ "kind" .= String "RpcConfigUpdateError" , "error" .= String ("Error while updating RPC configuration: " <> err) ] + forMachine _dtal (RpcUnsupportedBlockType blockType) = + mconcat [ "kind" .= String "RpcUnsupportedBlockType" + , "blockType" .= String blockType ] forMachine _dtal RpcForceDisabled = mconcat [ "kind" .= String "RpcForceDisabled" , "error" .= String (ppStartupInfoTrace RpcForceDisabled)] @@ -360,6 +363,8 @@ instance MetaTrace (StartupTrace blk) where Namespace [] ["RpcConfigUpdate"] namespaceFor RpcConfigUpdateError {} = Namespace [] ["RpcConfigUpdateError"] + namespaceFor RpcUnsupportedBlockType {} = + Namespace [] ["RpcUnsupportedBlockType"] namespaceFor RpcForceDisabled = Namespace [] ["RpcForceDisabled"] namespaceFor MovedTopLevelOption {} = @@ -376,6 +381,7 @@ instance MetaTrace (StartupTrace blk) where severityFor (Namespace _ ["WarningDevelopmentNodeToClientVersions"]) _ = Just Warning severityFor (Namespace _ ["RpcConfigUpdate"]) _ = Just Notice severityFor (Namespace _ ["RpcConfigUpdateError"]) _ = Just Error + severityFor (Namespace _ ["RpcUnsupportedBlockType"]) _ = Just Warning severityFor (Namespace _ ["RpcForceDisabled"]) _ = Just Error severityFor (Namespace _ ["BlockForgingUpdateError"]) _ = Just Error severityFor (Namespace _ ["BlockForgingBlockTypeMismatch"]) _ = Just Error @@ -407,6 +413,8 @@ instance MetaTrace (StartupTrace blk) where "" documentFor (Namespace [] ["RpcConfigUpdateError"]) = Just "" + documentFor (Namespace [] ["RpcUnsupportedBlockType"]) = Just + "" documentFor (Namespace [] ["RpcForceDisabled"]) = Just "" documentFor (Namespace [] ["NetworkConfigUpdate"]) = Just @@ -480,6 +488,7 @@ instance MetaTrace (StartupTrace blk) where , Namespace [] ["BlockForgingBlockTypeMismatch"] , Namespace [] ["RpcConfigUpdate"] , Namespace [] ["RpcConfigUpdateError"] + , Namespace [] ["RpcUnsupportedBlockType"] , Namespace [] ["RpcForceDisabled"] , Namespace [] ["NetworkConfigUpdate"] , Namespace [] ["NetworkConfigUpdateUnsupported"] @@ -512,6 +521,7 @@ nodeToNodeVersionToInt :: NodeToNodeVersion -> Int nodeToNodeVersionToInt = \case NodeToNodeV_14 -> 14 NodeToNodeV_15 -> 15 + NodeToNodeV_16 -> 16 -- | Pretty print 'StartupInfoTrace' -- @@ -605,6 +615,7 @@ ppStartupInfoTrace (LedgerPeerSnapshotLoaded slotNo) = ppStartupInfoTrace (RpcConfigUpdate config) = "Performing RPC configuration update: " <> config ppStartupInfoTrace (RpcConfigUpdateError err) = "Error while updating RPC configuration: " <> err +ppStartupInfoTrace (RpcUnsupportedBlockType blockType) = "RPC node kernel access is not supported for block type: " <> blockType ppStartupInfoTrace RpcForceDisabled = "RPC endpoint has crashed and because of that it got disabled. Enable gRPC endpoint and send SIGHUP to the node to reenable." ppStartupInfoTrace NonP2PWarning = nonP2PWarningMessage diff --git a/cardano-node/test/Test/Cardano/Config/Mainnet.hs b/cardano-node/test/Test/Cardano/Config/Mainnet.hs index 154f710d8cf..e9a02a56ac0 100644 --- a/cardano-node/test/Test/Cardano/Config/Mainnet.hs +++ b/cardano-node/test/Test/Cardano/Config/Mainnet.hs @@ -15,6 +15,9 @@ import qualified Data.Yaml as Y import qualified GHC.Stack as GHC import qualified System.Directory as IO import System.FilePath (()) +import System.FS.API (SomeHasFS (..)) +import System.FS.API.Types (MountPoint (MountPoint)) +import System.FS.IO (ioHasFS) import Hedgehog (Property, (===)) import qualified Hedgehog as H @@ -24,7 +27,8 @@ import qualified Hedgehog.Extras.Test.Process as H hprop_configMainnetHash :: Property hprop_configMainnetHash = H.propertyOnce $ do base <- H.note =<< H.evalIO . IO.canonicalizePath =<< H.getProjectBase - result <- H.evalIO $ runExceptT $ initialLedgerState $ File $ base "configuration/cardano/mainnet-config.json" + let fs = SomeHasFS (ioHasFS (MountPoint (base "configuration/cardano"))) + result <- H.evalIO $ runExceptT $ initialLedgerState fs $ File $ base "configuration/cardano/mainnet-config.json" case result of Right (_, _) -> return () Left e -> H.failWithCustom GHC.callStack Nothing (displayError e) diff --git a/cardano-node/test/Test/Cardano/Node/FilePermissions.hs b/cardano-node/test/Test/Cardano/Node/FilePermissions.hs index 0aa16e86453..6510b469ebb 100644 --- a/cardano-node/test/Test/Cardano/Node/FilePermissions.hs +++ b/cardano-node/test/Test/Cardano/Node/FilePermissions.hs @@ -14,43 +14,39 @@ module Test.Cardano.Node.FilePermissions ( tests ) where -import Control.Monad.Except -import "contra-tracer" Control.Tracer -import Control.Tracer.Arrow -import Data.Foldable -import Data.IORef -import System.Directory (removeFile) - import Cardano.Api + import Cardano.Node.Run (checkVRFFilePermissions) +import Cardano.Node.Types (VRFPrivateKeyFilePermissionError (..)) + +import Control.Exception (bracket) import Control.Monad (Monad (..)) +import Control.Monad.Except import Control.Monad.Except (runExceptT) import Control.Monad.IO.Class (MonadIO (liftIO)) +import "contra-tracer" Control.Tracer +import Control.Tracer.Arrow import Data.Bool (Bool, not) import Data.Either (Either (..)) import Data.Eq ((==)) +import Data.Foldable import Data.Foldable (foldl', length) import Data.Function (const, ($), (.)) +import Data.IORef import qualified Data.List as L import Data.Maybe (Maybe (..)) import Data.Semigroup (Semigroup (..)) -import Hedgehog -import Hedgehog.Internal.Property (Group (..), failWith) +import System.Directory (removeFile) import System.IO (FilePath, IO) -import Text.Show (Show (..)) -import Cardano.Node.Types (VRFPrivateKeyFilePermissionError (..)) -import Control.Exception (bracket) - -#ifdef UNIX - import System.Posix.Files import System.Posix.IO (closeFd, createFile) import System.Posix.Types (FileMode) +import Text.Show (Show (..)) import Hedgehog import qualified Hedgehog.Extras as H import qualified Hedgehog.Gen as Gen -#endif +import Hedgehog.Internal.Property (Group (..), failWith) {- HLINT ignore "Use fewer imports" -} @@ -166,7 +162,7 @@ mkCapturingTracer = do messages <- liftIO $ newIORef [] let registerMessage :: String -> IO () registerMessage msg = atomicModifyIORef messages (\msgs -> (msgs <> [msg], ())) - pure (Tracer registerMessage, messages) + pure (Tracer (emit registerMessage), messages) #endif -- ----------------------------------------------------------------------------- diff --git a/cardano-node/test/Test/Cardano/Node/POM.hs b/cardano-node/test/Test/Cardano/Node/POM.hs index 6a1cf3d6c0c..fd56c5248e5 100644 --- a/cardano-node/test/Test/Cardano/Node/POM.hs +++ b/cardano-node/test/Test/Cardano/Node/POM.hs @@ -1,6 +1,7 @@ {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE PatternSynonyms #-} +{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TemplateHaskell #-} module Test.Cardano.Node.POM @@ -24,19 +25,21 @@ import Cardano.Rpc.Server.Config (RpcConfigF (..), makeRpcConfig) import Ouroboros.Consensus.Node (NodeDatabasePaths (..)) import Ouroboros.Consensus.Node.Genesis (disableGenesisConfig) import Ouroboros.Consensus.Storage.LedgerDB.Args -import Ouroboros.Consensus.Storage.LedgerDB.Snapshots (NumOfDiskSnapshots (..), - SnapshotInterval (..)) +import Ouroboros.Consensus.Storage.LedgerDB.Snapshots (defaultSnapshotPolicyArgs, + mithrilSnapshotPolicyArgs) import Ouroboros.Network.Block (SlotNo (..)) import Ouroboros.Network.PeerSelection.PeerSharing (PeerSharing (..)) import Ouroboros.Network.TxSubmission.Inbound.V2.Types +import Data.Aeson (eitherDecode) import Data.Bifunctor (first) +import qualified Data.ByteString.Lazy as LBS import Data.Functor.Identity (Identity (..)) import Data.Monoid (Last (..)) import Data.String import Data.Text (Text) -import Hedgehog (Property, discover, (===)) +import Hedgehog (Property, discover, withTests, (===)) import qualified Hedgehog import qualified Hedgehog.Extras as H import Hedgehog.Internal.Property (evalEither, failWith) @@ -288,12 +291,62 @@ eExpectedConfig = do , ncConsensusMode = PraosMode , ncGenesisConfig = disableGenesisConfig , ncResponderCoreAffinityPolicy = NoResponderCoreAffinity - , ncLedgerDbConfig = LedgerDbConfiguration DefaultNumOfDiskSnapshots DefaultSnapshotInterval DefaultQueryBatchSize V2InMemory noDeprecatedOptions + , ncLedgerDbConfig = LedgerDbConfiguration defaultSnapshotPolicyArgs DefaultQueryBatchSize V2InMemory noDeprecatedOptions , ncRpcConfig , ncTxSubmissionLogicVersion = TxSubmissionLogicV1 , ncTxSubmissionInitDelay = defaultTxSubmissionInitDelay } +-- | Test that the legacy flat LedgerDB snapshot config format (options directly +-- under LedgerDB) parses identically to the new nested Snapshots format. +-- +-- TODO: this test could be removed once the old format is deprecated. +prop_legacySnapshotFormat_POM :: Property +prop_legacySnapshotFormat_POM = + withTests 1 . Hedgehog.property $ do + let legacyJson = "{ " <> dummyRequiredValues <> ", " + <> "\"LedgerDB\": {" + <> " \"Backend\": \"V2InMemory\"," + <> " \"SnapshotInterval\": 4320," + <> " \"NumOfDiskSnapshots\": 2" + <> "} }" + newJson = "{ " <> dummyRequiredValues <> ", " + <> "\"LedgerDB\": {" + <> " \"Backend\": \"V2InMemory\"," + <> " \"Snapshots\": {" + <> " \"SnapshotInterval\": 4320," + <> " \"NumOfDiskSnapshots\": 2" + <> " }" + <> "} }" + legacyConfig :: PartialNodeConfiguration <- evalEither $ eitherDecode legacyJson + newConfig :: PartialNodeConfiguration <- evalEither $ eitherDecode newJson + pncLedgerDbConfig legacyConfig === pncLedgerDbConfig newConfig + +-- | Test that the named \"Mithril\" snapshot policy selects +-- 'mithrilSnapshotPolicyArgs' as a whole. +prop_mithrilSnapshotPolicy_POM :: Property +prop_mithrilSnapshotPolicy_POM = + withTests 1 . Hedgehog.property $ do + let json = "{ " <> dummyRequiredValues <> ", " + <> "\"LedgerDB\": {" + <> " \"Backend\": \"V2InMemory\"," + <> " \"Snapshots\": \"Mithril\"" + <> "} }" + config :: PartialNodeConfiguration <- evalEither $ eitherDecode json + getLast (pncLedgerDbConfig config) === + Just (LedgerDbConfiguration mithrilSnapshotPolicyArgs DefaultQueryBatchSize V2InMemory noDeprecatedOptions) + +dummyRequiredValues :: LBS.ByteString +dummyRequiredValues = mconcat + [ "\"ByronGenesisFile\": \"x\"" + , ", \"ShelleyGenesisFile\": \"x\"" + , ", \"AlonzoGenesisFile\": \"x\"" + , ", \"ConwayGenesisFile\": \"x\"" + , ", \"LastKnownBlockVersion-Major\": 0" + , ", \"LastKnownBlockVersion-Minor\": 0" + , ", \"LastKnownBlockVersion-Alt\": 0" + ] + -- A socket config with a node socket path, needed for RPC-enabled tests -- because makeRpcConfig validates that a node socket exists when RPC is enabled. testSocketConfigWithPath :: Last SocketConfig diff --git a/cardano-node/test/cardano-config-compare/Main.hs b/cardano-node/test/cardano-config-compare/Main.hs new file mode 100644 index 00000000000..b833ac6b3ac --- /dev/null +++ b/cardano-node/test/cardano-config-compare/Main.hs @@ -0,0 +1,103 @@ +{-# LANGUAGE ScopedTypeVariables #-} + +-- | Tests for the dual-parse (node vs @cardano-config@) comparison: +-- 'deprecatedFlagWarnings' and 'compareConfigurations' run end-to-end on a +-- fixture resolved by both the POM parser and cardano-config + the adapter. +module Main (main) where + +import Data.List (isInfixOf, isPrefixOf) +import Data.Monoid (Last (..)) + +import qualified Cardano.Configuration as Cfg +import Cardano.Node.Configuration.CardanoConfigAdapter + (cardanoConfigToNodeConfiguration) +import Cardano.Node.Configuration.CardanoConfigCompare + (compareConfigurations, deprecatedFlagWarnings) +import Cardano.Node.Configuration.POM (NodeConfiguration (..), + PartialNodeConfiguration (..), defaultPartialNodeConfiguration, + makeNodeConfiguration, parseNodeConfigurationFP) +import Cardano.Node.Types (ConfigYamlFilePath (..)) + +import Test.Tasty +import Test.Tasty.HUnit + +-- | The db-synthesizer fixture configuration (test cwd is the repo root). +configPath :: FilePath +configPath = "cardano-node/test/db-synthesizer/disk/config/config.json" + +-- | Documented, expected divergences on this fixture — none: the node and +-- cardano-config are expected to agree on everything. +allowedResidualLabels :: [String] +allowedResidualLabels = [] + +main :: IO () +main = defaultMain tests + +tests :: TestTree +tests = testGroup "cardano-config dual-parse comparison" + [ testCase "deprecated CLI aliases yield migration guidance" testDeprecatedAliases + , testCase "removed mempool flags yield removal guidance" testRemovedMempoolFlags + , testCase "no guidance for accepted / unrelated flags" testNoFalsePositives + , testCase "compareConfigurations runs on the fixture (divergences ⊆ residuals)" + testFixtureComparison + ] + +testDeprecatedAliases :: Assertion +testDeprecatedAliases = do + let warnings = + deprecatedFlagWarnings + ["--delegation-certificate", "x", "--signing-key", "y", "--non-producing-node"] + suggests new = any (new `isInfixOf`) warnings + assertBool "suggests --byron-delegation-certificate" (suggests "--byron-delegation-certificate") + assertBool "suggests --byron-signing-key" (suggests "--byron-signing-key") + assertBool "suggests --start-as-non-producing-node" (suggests "--start-as-non-producing-node") + length warnings @?= 3 + +testRemovedMempoolFlags :: Assertion +testRemovedMempoolFlags = do + let warnings = deprecatedFlagWarnings ["--mempool-capacity-override", "100"] + length warnings @?= 1 + assertBool "says no longer supported" + (any ("no longer supported" `isInfixOf`) warnings) + assertBool "points to MempoolCapacityBytesOverride in the config file" + (any ("MempoolCapacityBytesOverride" `isInfixOf`) warnings) + +testNoFalsePositives :: Assertion +testNoFalsePositives = + deprecatedFlagWarnings ["--config", "c.json", "--topology", "t.json", "--database-path", "db"] + @?= [] + +testFixtureComparison :: Assertion +testFixtureComparison = do + -- cardano-config side: resolve from the file and adapt to a NodeConfiguration. + resolved <- Cfg.resolveConfigurationFromFile configPath + (cfgNc, _warns) <- either (assertFailure . (("cardano-config resolve failed: " <>) . show)) pure resolved + adapted <- either (assertFailure . ("adapter failed: " <>)) pure + (cardanoConfigToNodeConfiguration cfgNc) + + -- Node side: parse the same file with POM, mirroring the CLI-only fields + -- (topology / database / protocol files / socket) from the adapter output. + fileYaml <- parseNodeConfigurationFP (Just (ConfigYamlFilePath configPath)) + let withCli = + (defaultPartialNodeConfiguration <> fileYaml) + { pncConfigFile = Last (Just (ConfigYamlFilePath configPath)) + , pncTopologyFile = Last (Just (ncTopologyFile adapted)) + , pncDatabaseFile = Last (Just (ncDatabaseFile adapted)) + , pncProtocolFiles = Last (Just (ncProtocolFiles adapted)) + , pncSocketConfig = Last (Just (ncSocketConfig adapted)) + } + pomNc <- either (assertFailure . ("POM makeNodeConfiguration failed: " <>)) pure + (makeNodeConfiguration withCli) + + let divergences = compareConfigurations pomNc adapted + isAllowed d = any (`isPrefixOf` d) allowedResidualLabels + unexpected = filter (not . isAllowed) divergences + + -- Print what the comparison reports, so the run is legible even when it passes. + putStrLn $ " compareConfigurations reported " <> show (length divergences) + <> " divergence(s) on the fixture:" + mapM_ (putStrLn . (" - " <>)) divergences + + assertBool + ("divergences outside the documented residual set: " <> show unexpected) + (null unexpected) diff --git a/cardano-node/test/db-synthesizer/Main.hs b/cardano-node/test/db-synthesizer/Main.hs new file mode 100644 index 00000000000..068bfe5a3e0 --- /dev/null +++ b/cardano-node/test/db-synthesizer/Main.hs @@ -0,0 +1,142 @@ +{-# LANGUAGE TypeApplications #-} + +-- | End-to-end regression test for the downstream @db-synthesizer@: build a +-- Cardano 'ProtocolInfo' and block forgers from a node config file plus a +-- (bulk) forging-credentials fixture, synthesize a ChainDB, immutalise it, and +-- analyse it — checking the block count is preserved end to end. +-- +-- This is the config/credential-driven pipeline that used to live in +-- @ouroboros-consensus@'s @tools-test@ and moved downstream with the eject. It +-- deliberately uses real bulk credentials (a proper KES validity window) and the +-- original forge limits, exercising the path the standalone tool takes — not the +-- short-lived testlib credentials the in-repo synthesis-only test now uses. +module Main (main) where + +import Cardano.Crypto.Init (cryptoInit) +import Cardano.Node.Tools.DBSynthesizer (initializeProtocol) +import Cardano.Node.Types (KESSource, ProtocolFilepaths (..)) +import qualified Cardano.Tools.DBAnalyser.Block.Cardano as Cardano +import qualified Cardano.Tools.DBAnalyser.Run as DBAnalyser +import Cardano.Tools.DBAnalyser.Types +import qualified Cardano.Tools.DBImmutaliser.Run as DBImmutaliser +import qualified Cardano.Tools.DBSynthesizer.Run as DBSynthesizer +import Cardano.Tools.DBSynthesizer.Types +import Ouroboros.Consensus.Block (WithOrigin (Origin)) +import Ouroboros.Consensus.Cardano.Block (CardanoBlock, StandardCrypto) +import Test.Tasty +import Test.Tasty.HUnit + +-- | Fixtures live next to this test; paths are relative to the @cardano-node@ +-- package directory (cabal's working directory when running the test suite). +fixtureDir, nodeConfig, bulkCreds, chainDB :: FilePath +fixtureDir = "cardano-node/test/db-synthesizer/disk/config" +nodeConfig = fixtureDir <> "/config.json" +bulkCreds = fixtureDir <> "/bulk-creds-k2.json" +chainDB = "cardano-node/test/db-synthesizer/disk/chaindb" + +-- | Forging credentials for the test: only the bulk-credentials file, mirroring +-- how the tool is typically driven. Bulk creds carry a real KES validity window, +-- so the original (larger) forge limits below stay well within it. +testProtocolFiles :: ProtocolFilepaths +testProtocolFiles = + ProtocolFilepaths + { byronCertFile = Nothing + , byronKeyFile = Nothing + , shelleyKESSource = Nothing :: Maybe KESSource + , shelleyVRFFile = Nothing + , shelleyCertFile = Nothing + , shelleyBulkCredsFile = Just bulkCreds + } + +testSynthOptionsCreate :: DBSynthesizerOptions +testSynthOptionsCreate = + DBSynthesizerOptions + { synthLimit = ForgeLimitEpoch 1 + , synthOpenMode = OpenCreateForce + } + +testSynthOptionsAppend :: DBSynthesizerOptions +testSynthOptionsAppend = + DBSynthesizerOptions + { synthLimit = ForgeLimitSlot 8192 + , synthOpenMode = OpenAppend + } + +testImmutaliserConfig :: DBImmutaliser.Opts +testImmutaliserConfig = + DBImmutaliser.Opts + { DBImmutaliser.dbDirs = + DBImmutaliser.DBDirs + { DBImmutaliser.immDBDir = chainDB <> "/immutable" + , DBImmutaliser.volDBDir = chainDB <> "/volatile" + } + , DBImmutaliser.configFile = nodeConfig + , DBImmutaliser.verbose = False + , DBImmutaliser.dotOut = Nothing + , DBImmutaliser.dryRun = False + } + +testAnalyserConfig :: DBAnalyserConfig +testAnalyserConfig = + DBAnalyserConfig + { dbDir = chainDB + , ldbBackend = V2InMem + , verbose = False + , selectDB = SelectImmutableDB Origin + , validation = Just ValidateAllBlocks + , analysis = CountBlocks + , confLimit = Unlimited + } + +testBlockArgs :: Cardano.Args (CardanoBlock StandardCrypto) +testBlockArgs = Cardano.CardanoBlockArgs nodeConfig Nothing + +-- | 1. synthesize a ChainDB from scratch (create) and count blocks forged. +-- 2. append to it and count blocks forged. +-- 3. copy the VolatileDB into the ImmutableDB. +-- 4. analyse the ImmutableDB and confirm the total block count matches. +blockCountTest :: (String -> IO ()) -> Assertion +blockCountTest logStep = do + logStep "building protocol from config + bulk credentials" + (protocolInfo, mkForgers, epochSize) <- initializeProtocol nodeConfig testProtocolFiles + + logStep "running synthesis - create" + resultCreate <- + DBSynthesizer.synthesize genTxs testSynthOptionsCreate epochSize chainDB (protocolInfo, mkForgers) + let blockCountCreate = resultForged resultCreate + blockCountCreate > 0 @? "no blocks have been forged during create step" + + logStep "running synthesis - append" + resultAppend <- + DBSynthesizer.synthesize genTxs testSynthOptionsAppend epochSize chainDB (protocolInfo, mkForgers) + let blockCountAppend = resultForged resultAppend + blockCountAppend > 0 @? "no blocks have been forged during append step" + + logStep "copy volatile to immutable DB" + DBImmutaliser.run testImmutaliserConfig + + logStep "running analysis" + resultAnalysis <- DBAnalyser.analyse testAnalyserConfig testBlockArgs + + let blockCount = blockCountCreate + blockCountAppend + resultAnalysis == Just (ResultCountBlock blockCount) + @? "wrong number of blocks encountered during analysis \ + \ (counted: " + ++ show resultAnalysis + ++ "; expected: " + ++ show blockCount + ++ ")" + where + genTxs _ _ _ _ = pure [] + +tests :: TestTree +tests = + testGroup + "db-synthesizer" + [ testCaseSteps "synthesize (bulk creds) -> immutalise -> analyse: blockCount\n" blockCountTest + ] + +main :: IO () +main = do + cryptoInit + defaultMain tests diff --git a/cardano-node/test/db-synthesizer/disk/config/alonzo-genesis.json b/cardano-node/test/db-synthesizer/disk/config/alonzo-genesis.json new file mode 100644 index 00000000000..093071bb398 --- /dev/null +++ b/cardano-node/test/db-synthesizer/disk/config/alonzo-genesis.json @@ -0,0 +1,194 @@ +{ + "lovelacePerUTxOWord": 34482, + "executionPrices": { + "prSteps": { + "numerator": 721, + "denominator": 10000000 + }, + "prMem": { + "numerator": 577, + "denominator": 10000 + } + }, + "maxTxExUnits": { + "exUnitsMem": 14000000, + "exUnitsSteps": 10000000000 + }, + "maxBlockExUnits": { + "exUnitsMem": 56000000, + "exUnitsSteps": 40000000000 + }, + "maxValueSize": 5000, + "collateralPercentage": 150, + "maxCollateralInputs": 3, + "costModels": { + "PlutusV1": { + "sha2_256-memory-arguments": 4, + "equalsString-cpu-arguments-constant": 1000, + "cekDelayCost-exBudgetMemory": 100, + "lessThanEqualsByteString-cpu-arguments-intercept": 103599, + "divideInteger-memory-arguments-minimum": 1, + "appendByteString-cpu-arguments-slope": 621, + "blake2b-cpu-arguments-slope": 29175, + "iData-cpu-arguments": 150000, + "encodeUtf8-cpu-arguments-slope": 1000, + "unBData-cpu-arguments": 150000, + "multiplyInteger-cpu-arguments-intercept": 61516, + "cekConstCost-exBudgetMemory": 100, + "nullList-cpu-arguments": 150000, + "equalsString-cpu-arguments-intercept": 150000, + "trace-cpu-arguments": 150000, + "mkNilData-memory-arguments": 32, + "lengthOfByteString-cpu-arguments": 150000, + "cekBuiltinCost-exBudgetCPU": 29773, + "bData-cpu-arguments": 150000, + "subtractInteger-cpu-arguments-slope": 0, + "unIData-cpu-arguments": 150000, + "consByteString-memory-arguments-intercept": 0, + "divideInteger-memory-arguments-slope": 1, + "divideInteger-cpu-arguments-model-arguments-slope": 118, + "listData-cpu-arguments": 150000, + "headList-cpu-arguments": 150000, + "chooseData-memory-arguments": 32, + "equalsInteger-cpu-arguments-intercept": 136542, + "sha3_256-cpu-arguments-slope": 82363, + "sliceByteString-cpu-arguments-slope": 5000, + "unMapData-cpu-arguments": 150000, + "lessThanInteger-cpu-arguments-intercept": 179690, + "mkCons-cpu-arguments": 150000, + "appendString-memory-arguments-intercept": 0, + "modInteger-cpu-arguments-model-arguments-slope": 118, + "ifThenElse-cpu-arguments": 1, + "mkNilPairData-cpu-arguments": 150000, + "lessThanEqualsInteger-cpu-arguments-intercept": 145276, + "addInteger-memory-arguments-slope": 1, + "chooseList-memory-arguments": 32, + "constrData-memory-arguments": 32, + "decodeUtf8-cpu-arguments-intercept": 150000, + "equalsData-memory-arguments": 1, + "subtractInteger-memory-arguments-slope": 1, + "appendByteString-memory-arguments-intercept": 0, + "lengthOfByteString-memory-arguments": 4, + "headList-memory-arguments": 32, + "listData-memory-arguments": 32, + "consByteString-cpu-arguments-intercept": 150000, + "unIData-memory-arguments": 32, + "remainderInteger-memory-arguments-minimum": 1, + "bData-memory-arguments": 32, + "lessThanByteString-cpu-arguments-slope": 248, + "encodeUtf8-memory-arguments-intercept": 0, + "cekStartupCost-exBudgetCPU": 100, + "multiplyInteger-memory-arguments-intercept": 0, + "unListData-memory-arguments": 32, + "remainderInteger-cpu-arguments-model-arguments-slope": 118, + "cekVarCost-exBudgetCPU": 29773, + "remainderInteger-memory-arguments-slope": 1, + "cekForceCost-exBudgetCPU": 29773, + "sha2_256-cpu-arguments-slope": 29175, + "equalsInteger-memory-arguments": 1, + "indexByteString-memory-arguments": 1, + "addInteger-memory-arguments-intercept": 1, + "chooseUnit-cpu-arguments": 150000, + "sndPair-cpu-arguments": 150000, + "cekLamCost-exBudgetCPU": 29773, + "fstPair-cpu-arguments": 150000, + "quotientInteger-memory-arguments-minimum": 1, + "decodeUtf8-cpu-arguments-slope": 1000, + "lessThanInteger-memory-arguments": 1, + "lessThanEqualsInteger-cpu-arguments-slope": 1366, + "fstPair-memory-arguments": 32, + "modInteger-memory-arguments-intercept": 0, + "unConstrData-cpu-arguments": 150000, + "lessThanEqualsInteger-memory-arguments": 1, + "chooseUnit-memory-arguments": 32, + "sndPair-memory-arguments": 32, + "addInteger-cpu-arguments-intercept": 197209, + "decodeUtf8-memory-arguments-slope": 8, + "equalsData-cpu-arguments-intercept": 150000, + "mapData-cpu-arguments": 150000, + "mkPairData-cpu-arguments": 150000, + "quotientInteger-cpu-arguments-constant": 148000, + "consByteString-memory-arguments-slope": 1, + "cekVarCost-exBudgetMemory": 100, + "indexByteString-cpu-arguments": 150000, + "unListData-cpu-arguments": 150000, + "equalsInteger-cpu-arguments-slope": 1326, + "cekStartupCost-exBudgetMemory": 100, + "subtractInteger-cpu-arguments-intercept": 197209, + "divideInteger-cpu-arguments-model-arguments-intercept": 425507, + "divideInteger-memory-arguments-intercept": 0, + "cekForceCost-exBudgetMemory": 100, + "blake2b-cpu-arguments-intercept": 2477736, + "remainderInteger-cpu-arguments-constant": 148000, + "tailList-cpu-arguments": 150000, + "encodeUtf8-cpu-arguments-intercept": 150000, + "equalsString-cpu-arguments-slope": 1000, + "lessThanByteString-memory-arguments": 1, + "multiplyInteger-cpu-arguments-slope": 11218, + "appendByteString-cpu-arguments-intercept": 396231, + "lessThanEqualsByteString-cpu-arguments-slope": 248, + "modInteger-memory-arguments-slope": 1, + "addInteger-cpu-arguments-slope": 0, + "equalsData-cpu-arguments-slope": 10000, + "decodeUtf8-memory-arguments-intercept": 0, + "chooseList-cpu-arguments": 150000, + "constrData-cpu-arguments": 150000, + "equalsByteString-memory-arguments": 1, + "cekApplyCost-exBudgetCPU": 29773, + "quotientInteger-memory-arguments-slope": 1, + "verifySignature-cpu-arguments-intercept": 3345831, + "unMapData-memory-arguments": 32, + "mkCons-memory-arguments": 32, + "sliceByteString-memory-arguments-slope": 1, + "sha3_256-memory-arguments": 4, + "ifThenElse-memory-arguments": 1, + "mkNilPairData-memory-arguments": 32, + "equalsByteString-cpu-arguments-slope": 247, + "appendString-cpu-arguments-intercept": 150000, + "quotientInteger-cpu-arguments-model-arguments-slope": 118, + "cekApplyCost-exBudgetMemory": 100, + "equalsString-memory-arguments": 1, + "multiplyInteger-memory-arguments-slope": 1, + "cekBuiltinCost-exBudgetMemory": 100, + "remainderInteger-memory-arguments-intercept": 0, + "sha2_256-cpu-arguments-intercept": 2477736, + "remainderInteger-cpu-arguments-model-arguments-intercept": 425507, + "lessThanEqualsByteString-memory-arguments": 1, + "tailList-memory-arguments": 32, + "mkNilData-cpu-arguments": 150000, + "chooseData-cpu-arguments": 150000, + "unBData-memory-arguments": 32, + "blake2b-memory-arguments": 4, + "iData-memory-arguments": 32, + "nullList-memory-arguments": 32, + "cekDelayCost-exBudgetCPU": 29773, + "subtractInteger-memory-arguments-intercept": 1, + "lessThanByteString-cpu-arguments-intercept": 103599, + "consByteString-cpu-arguments-slope": 1000, + "appendByteString-memory-arguments-slope": 1, + "trace-memory-arguments": 32, + "divideInteger-cpu-arguments-constant": 148000, + "cekConstCost-exBudgetCPU": 29773, + "encodeUtf8-memory-arguments-slope": 8, + "quotientInteger-cpu-arguments-model-arguments-intercept": 425507, + "mapData-memory-arguments": 32, + "appendString-cpu-arguments-slope": 1000, + "modInteger-cpu-arguments-constant": 148000, + "verifySignature-cpu-arguments-slope": 1, + "unConstrData-memory-arguments": 32, + "quotientInteger-memory-arguments-intercept": 0, + "equalsByteString-cpu-arguments-constant": 150000, + "sliceByteString-memory-arguments-intercept": 0, + "mkPairData-memory-arguments": 32, + "equalsByteString-cpu-arguments-intercept": 112536, + "appendString-memory-arguments-slope": 1, + "lessThanInteger-cpu-arguments-slope": 497, + "modInteger-cpu-arguments-model-arguments-intercept": 425507, + "modInteger-memory-arguments-minimum": 1, + "sha3_256-cpu-arguments-intercept": 0, + "verifySignature-memory-arguments": 1, + "cekLamCost-exBudgetMemory": 100, + "sliceByteString-cpu-arguments-intercept": 150000 + } + } +} diff --git a/cardano-node/test/db-synthesizer/disk/config/bulk-creds-k2.json b/cardano-node/test/db-synthesizer/disk/config/bulk-creds-k2.json new file mode 100644 index 00000000000..fc64d6855b7 --- /dev/null +++ b/cardano-node/test/db-synthesizer/disk/config/bulk-creds-k2.json @@ -0,0 +1,34 @@ +[ + [ + { + "type": "NodeOperationalCertificate", + "description": "", + "cborHex": "82845820465dad8c08ecfe932f70bf287903d2d1973ac224f61cd0f9914ed052853f736b000058402cf9b1523a570f5a3333e1a602d3212e187b1e4b6b147b7cbc94657039de7e79e8ca6dc964cb7368b135c9607151e715d2ea9ccad9f3f550077b79fa3f64d1095820974aab238e812402dc9dbce33dd28203ae6df68616290a1b4aac347e881057bb" +} + , { + "type": "VrfSigningKey_PraosVRF", + "description": "VRF Signing Key", + "cborHex": "584040c0bd2dd8acfaded1d93c4844c2130058f86067af2e065dd3ae001e964a5f18b08644bf6ed9d404ba94c9ba9299a2ab53f36c57c02c38139f2138b6c71302c7" +} + , { + "type": "KesSigningKey_ed25519_kes_2^6", + "description": "KES Signing Key", + "cborHex": "5902606d23bd6e50df9416e52e9ee2cca23ac00f1ae78a62e50afcfc3cc8159b1e9ac888593015ae9c6124e33f143416b5c12195e3a2b947a00ef34e185f672b1047df6f5180047fffdaefea6337b2384087095873ba2d09ba74d1e826bbeec148e2db19ecb1db2e6d28748cf06cd36711d16fbced7fa2d5e0c1111832c36982196b417bd16ed77a4fd795fa22e2d394f3cb8940ca406431f4b105d6e9a47e5bcb4d5f86fa466b8228fcf17056f5e006ed522538c7ed32ad8724d3c63f5443907081f5f54f72868cb1475d05bb79d11a4c6abbed543c4898fc2f157aeb99adb27c31ca22ac195d04b13a0a1a3d118599d7ff8073d90063afcc87586e77b9795f73776e0f0bbf690440a243e729880cbcded7fc778f31cc873791296b1e43f87c869e197f1fd345fdf368136c936c53124caad8786379a194d3b348752b90dbfdd1199a3f8f8388940d5585825e2cffe7108b821d54351b6de2c9c4c8308d157b4b25070c77efc22a327e074e2ec01eac2bf9169a97d65cc826fbe827d0da045e5b680953b17a47b240b5e52653ad495d6ca90513f110d5a8353e92b416273a1bbc05e99050cd38dcb7a1f0e9d73aa0fbac201359fb26faa9235a851480b25dcf0ebe95cb2998b3f10f1baaabde842266c31ede1289ae1212cc9ae57a00262ada16dcd662f40c90ee1032e00dd4b6d1f17a0956517c8c38c354cb65b16bf6ace5d1d056205bd9f596020677ac06747335512dc9bafff75858a92cd6e947da98865ab364e6933d94a999afe22a0e8cbf3b8151e07073b343aa6632607f16d578a94e4f3b7050c2ee5e43a9279fd907e3deb75b244cb707423b06d71ab93b60b6fc23fa28" +} + ], [ + { + "type": "NodeOperationalCertificate", + "description": "", + "cborHex": "82845820a5ae7caf7a79b7f750d3d6da9a31d6523bdc0b99cc9dbfbdc11122e3ae07e8280000584071c1947b93fac5684a327a102f522d7b31daccfe8ef69ed0c36ed4618910245756bfe607b5a2bf7725045564b77ee18bfd7ed086b957d856a5491b51fbaedf065820e41015edc7b39489226d27c51dbe84c636466b3e29758a95445297614a8050bf" +} + , { + "type": "VrfSigningKey_PraosVRF", + "description": "VRF Signing Key", + "cborHex": "5840e2164474b17216bffb9494b8cdfc6d82f31f24e3f4dede8316221c11f616d75306a90f0597762346dd9eee0017623aca4745105f75b6d0d44355b26395372934" +} + , { + "type": "KesSigningKey_ed25519_kes_2^6", + "description": "KES Signing Key", + "cborHex": "59026076ae5c10752636ec89a8e9d25b74a7862f60d276246d13fa11bda92cfaf1417fe924137ee2a71629dcbbe950bb991dc8935033e3a4414b019510feb5f2c56d29de8e5249591afc25d214c024eac3c1186c26136a8719ca647c3c554aff75301df40a7243f0cea69d0da41b0edd95c35cc6644a433e1a59898f70a88b9578635c7f2a0dae07f48267c63e281eaeb4e9aad2e22f46229c4ee9f32e231f081a32c9b4ee2e7a940b2aa19d596f5b160abc0f83c66cd8c26d8f7226f4556d4a406e0b978df024d42a1a9236d58e8c64733aae1ee6e3258a27bfaf060b6c2913fc9babf1758bad0fe98819873bf34828f7ab5515d888f0c107f4c4010423f6ce7523de1cb543b0b471a48af5e367b75d856a36f5899c8019f91d321c22d012ee466e509d49ca12ed800448ad43ee1575de56abad60d0cd1d2bb9b541573504040c3b495d078e558c9ad0015d89e36515d4451c7adc87fdfe21cf21609684093e4d59c143c077e1127e25a0bb1fb8549c503b519f01f6092a3d3452341da2fb8687e07b340575532fe529cadd9701c300770930c4da09feed3a7f9b4d1253efe0fd1dc01122d7bf2324ff0779df1c65ef3886ca5196c4c5107c36dfc3dd292b9f90f2b55e380763e756b7f6b04d45d45e61aba849736babb9224adbf27a8880f1ecc23dd0bbe61a5b73fa269cc100bf3f6cbd17163f31d38aa22db320d37cbb767821de066a0f2b40f11bfc00404963d8418f54e9191f08d3c46319263b69cf51222c2ae4500627980856833e796e4435768172cb98b8b33ed5970a92ab3f046050c9f5aeeafa151f9d11b93c425b68cace42f87c51dee5f0a38071b3a8da23743d699c" +} + ]] diff --git a/cardano-node/test/db-synthesizer/disk/config/byron-genesis.json b/cardano-node/test/db-synthesizer/disk/config/byron-genesis.json new file mode 100644 index 00000000000..aec652492ff --- /dev/null +++ b/cardano-node/test/db-synthesizer/disk/config/byron-genesis.json @@ -0,0 +1,42 @@ +{ "bootStakeholders": + { "ce2950ee9b35c74336371a7393b2f1fa64d4af1831180076b106208d": 1 } +, "heavyDelegation": + { "ce2950ee9b35c74336371a7393b2f1fa64d4af1831180076b106208d": + { "omega": 0 + , "issuerPk": + "0Te/2OpdrE4IFuj6ZCSky8a/oeM9xE0phU0rQJE7v3Zg0wZop+bcZaKVe8qRk1zl0DM5vGIk5+XHZGhv7xh3rQ==" + , "delegatePk": + "ojky67+tV35+CmAFX7hkCCpPgz7EwrU8HCMDk7qEgoynuLaByN8S4ek4HjQXPq/b3vZFd08+Ip2BN/nC+e6Alg==" + , "cert": + "a8e1514662b6edd544f9e22d3bc8a961e6cfe5b1db35378188bb4fcd848e27c977952c8e73c29757b8b5b6b6c0b52254357383272c0b83e24cb91558907e8d04" + } } +, "startTime": 1655366659 +, "nonAvvmBalances": + { "2657WMsDfac5TVJguqJE11Z1tdx9HP72E9Roz32GVecUrX7oScFb1sXPzC43EnLUx": + "30000" + , "2657WMsDfac5V9qqEUfJm252BN5L81ni6CZyDS31cN7XZrAtsyqbz4yGr42bKG5B7": + "270000" + } +, "blockVersionData": + { "scriptVersion": 0 + , "slotDuration": "20000" + , "maxBlockSize": "641000" + , "maxHeaderSize": "200000" + , "maxTxSize": "4096" + , "maxProposalSize": "700" + , "mpcThd": "200000" + , "heavyDelThd": "300000" + , "updateVoteThd": "100000" + , "updateProposalThd": "100000" + , "updateImplicit": "10000" + , "softforkRule": + { "initThd": "900000" + , "minThd": "600000" + , "thdDecrement": "100000" + } + , "txFeePolicy": { "summand": "0" , "multiplier": "439460" } + , "unlockStakeEpoch": "184467" + } +, "protocolConsts": { "k": 2160 , "protocolMagic": 42 } +, "avvmDistr": {} +} \ No newline at end of file diff --git a/cardano-node/test/db-synthesizer/disk/config/config.json b/cardano-node/test/db-synthesizer/disk/config/config.json new file mode 100644 index 00000000000..70836962f78 --- /dev/null +++ b/cardano-node/test/db-synthesizer/disk/config/config.json @@ -0,0 +1,121 @@ +{ + "AcceptedConnectionsLimit": { + "delay": 5, + "hardLimit": 512, + "softLimit": 384 + }, + "AlonzoGenesisFile": "alonzo-genesis.json", + "AlonzoGenesisHash": "edb92321b614cfd7dcee3f49eedcde4f559ea9526194ff3cf42cd28a4bf0ad7b", + "ApplicationName": "cardano-sl", + "ApplicationVersion": 0, + "ByronGenesisFile": "byron-genesis.json", + "ByronGenesisHash": "6836b5d0ae3bb7250c318e8906ab2bf8e42e6acfd4483526be948752707ad435", + "ConwayGenesisFile": "conway-genesis.json", + "ConwayGenesisHash": "e4cbda3b0a0db8ea984330d0951d280ca490540c97cc6407ecd1011b174d983d", + "DijkstraGenesisFile": "dijkstra-genesis.json", + "DijkstraGenesisHash": "56c06ff0f668c584fc54fa3cee92dd5e121b67696924ac3b01b5aec9ecf95b78", + "EnableP2P": false, + "LastKnownBlockVersion-Alt": 0, + "LastKnownBlockVersion-Major": 3, + "LastKnownBlockVersion-Minor": 0, + "MaxKnownMajorProtocolVersion": 2, + "MempoolCapacityBytesOverride": "NoOverride", + "Protocol": "Cardano", + "ProtocolIdleTimeout": 5, + "RequiresNetworkMagic": "RequiresMagic", + "ShelleyGenesisFile": "shelley-genesis.json", + "ShelleyGenesisHash": "f6bb6e9d9b217681180754232470aa936716d62f8e11e944520b97490b100b7c", + "TargetNumberOfActivePeers": 20, + "TargetNumberOfEstablishedPeers": 50, + "TargetNumberOfKnownPeers": 100, + "TargetNumberOfRootPeers": 100, + "TestAllegraHardForkAtEpoch": 0, + "TestAlonzoHardForkAtEpoch": 0, + "TestBabbageHardForkAtEpoch": 0, + "ExperimentalHardForksEnabled": true, + "ExperimentalProtocolsEnabled": true, + "TestMaryHardForkAtEpoch": 0, + "TestShelleyHardForkAtEpoch": 0, + "TimeWaitTimeout": 60, + "TraceOptions": { + "": { + "backends": [ + "Stdout MachineFormat", + "EKGBackend", + "Forwarder" + ], + "severity": "Notice" + }, + "AcceptPolicy": { + "severity": "Info" + }, + "BlockFetchClient": { + "detail": "DMinimal", + "severity": "Info" + }, + "BlockFetchClient.CompletedBlockFetch": { + "maxFrequency": 2 + }, + "BlockFetchServer": { + "severity": "Info" + }, + "ChainDB": { + "severity": "Info" + }, + "ChainDB.AddBlockEvent.AddBlockValidation.ValidCandidate": { + "maxFrequency": 2 + }, + "ChainDB.AddBlockEvent.AddedBlockToQueue": { + "maxFrequency": 2 + }, + "ChainDB.AddBlockEvent.AddedBlockToVolatileDB": { + "maxFrequency": 2 + }, + "ChainDB.CopyToImmutableDBEvent.CopiedBlockToImmutableDB": { + "maxFrequency": 2 + }, + "ChainSyncClient": { + "detail": "DMinimal", + "severity": "Info" + }, + "ChainSyncServerBlock": { + "severity": "Info" + }, + "ChainSyncServerHeader": { + "severity": "Info" + }, + "DNSResolver": { + "severity": "Info" + }, + "DNSSubscription": { + "severity": "Info" + }, + "DiffusionInit": { + "severity": "Info" + }, + "ErrorPolicy": { + "severity": "Info" + }, + "Forge": { + "severity": "Info" + }, + "IpSubscription": { + "severity": "Info" + }, + "LocalErrorPolicy": { + "severity": "Info" + }, + "Mempool": { + "severity": "Info" + }, + "Resources": { + "severity": "Info" + }, + "TxSubmission2": { + "detail": "DMinimal" + } + }, + "TurnOnLogMetrics": true, + "TurnOnLogging": true, + "UseTraceDispatcher": true +} diff --git a/cardano-node/test/db-synthesizer/disk/config/conway-genesis.json b/cardano-node/test/db-synthesizer/disk/config/conway-genesis.json new file mode 100644 index 00000000000..08e1aed42a3 --- /dev/null +++ b/cardano-node/test/db-synthesizer/disk/config/conway-genesis.json @@ -0,0 +1,77 @@ +{ + "poolVotingThresholds": { + "committeeNormal": 0, + "committeeNoConfidence": 0, + "hardForkInitiation": 0, + "motionNoConfidence": 0, + "ppSecurityGroup": 0 + }, + "dRepVotingThresholds": { + "motionNoConfidence": 0, + "committeeNormal": 0, + "committeeNoConfidence": 0, + "updateToConstitution": 0, + "hardForkInitiation": 0, + "ppNetworkGroup": 0, + "ppEconomicGroup": 0, + "ppTechnicalGroup": 0, + "ppGovGroup": 0, + "treasuryWithdrawal": 0 + }, + "committeeMinSize": 0, + "committeeMaxTermLength": 0, + "govActionLifetime": 0, + "govActionDeposit": 0, + "dRepDeposit": 0, + "dRepActivity": 0, + "minFeeRefScriptCostPerByte": 0, + "plutusV3CostModel": [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0], + "constitution": { + "anchor": { + "url": "", + "dataHash": "0000000000000000000000000000000000000000000000000000000000000000" + } + }, + "committee": { + "members": { + "keyHash-4e88cc2d27c364aaf90648a87dfb95f8ee103ba67fa1f12f5e86c42a": 1, + "scriptHash-4e88cc2d27c364aaf90648a87dfb95f8ee103ba67fa1f12f5e86c42a": 2 + }, + "threshold": 0.5 + }, + "delegs": { + "keyHash-4e88cc2d27c364aaf90648a87dfb95f8ee103ba67fa1f12f5e86c42a": { + "dRep": "drep-alwaysAbstain" + }, + "keyHash-35bc5e86c42afbc593ab4cdd78301005df84ba67fa1f12f95f8ee103": { + "dRep": "drep-alwaysNoConfidence" + }, + "scriptHash-afbc5005df84ba5f8ee93ab435bc5e83067fa1f12f9c42cdd7110386": { + "dRep": "drep-keyHash-78301005df84ba67fa1f12f95f8ee10335bc5e86c42afbc593ab4cdd" + }, + "keyHash-df93ab435bc5eafbc500583067fa1f12f9110386c42cdd784ba5f8ee": { + "dRep": "drep-scriptHash-01305df84b078ac5e86c42afbc593ab4cdd67fa1f12f95f8ee10335b" + }, + "keyHash-5df84bcdd7a5f8ee93aafbc500b435bc5e83067fa1f12f9110386c42": { + "poolId": "0335bc5e86c42afbc578301005df84ba67fa1f12f95f8ee193ab4cdd" + }, + "keyHash-8ee93a5df84bc42cdd7a5fafbc500b435bc5e83067fa1f12f9110386": { + "poolId": "086c42afbc578301005df84ba67fa1f12f95f8ee193ab4cdd335bc5e", + "dRep": "drep-alwaysAbstain" + } + }, + "initialDReps": { + "keyHash-78301005df84ba67fa1f12f95f8ee10335bc5e86c42afbc593ab4cdd": { + "expiry": 1000, + "deposit": 5000 + }, + "scriptHash-01305df84b078ac5e86c42afbc593ab4cdd67fa1f12f95f8ee10335b": { + "expiry": 300, + "deposit": 6000, + "anchor": { + "url": "example.com", + "dataHash": "0000000000000000000000000000000000000000000000000000000000000000" + } + } + } +} diff --git a/cardano-node/test/db-synthesizer/disk/config/dijkstra-genesis.json b/cardano-node/test/db-synthesizer/disk/config/dijkstra-genesis.json new file mode 100644 index 00000000000..c33c6755721 --- /dev/null +++ b/cardano-node/test/db-synthesizer/disk/config/dijkstra-genesis.json @@ -0,0 +1,6 @@ +{ + "maxRefScriptSizePerBlock": 1048576, + "maxRefScriptSizePerTx": 204800, + "refScriptCostStride": 25600, + "refScriptCostMultiplier": 1.2 +} diff --git a/cardano-node/test/db-synthesizer/disk/config/shelley-genesis.json b/cardano-node/test/db-synthesizer/disk/config/shelley-genesis.json new file mode 100644 index 00000000000..7755bcd2242 --- /dev/null +++ b/cardano-node/test/db-synthesizer/disk/config/shelley-genesis.json @@ -0,0 +1,83 @@ +{ + "activeSlotsCoeff": 0.05, + "epochLength": 432000, + "genDelegs": {}, + "initialFunds": { + "0032635dc627da054f2a9e99559c56e16b02cf5f9237ba586ee1e648336b47a4e6e19ac5257fc97bd220bd9ae368aa2d775d86315ab7ef058f": 999500000000000, + "0064f4987ff07483636803f71f5f8442dad7f7fc46d83e2242d1548ad5d1f6f71a04ca856cc569fd068b069a555999c4776ad50b4cdd049d13": 999500000000000, + "602b43cb2b891e2dc9f5b07e051fb8d221a4a88ca161d95e859aa9ad8a": 9000000000000 + }, + "maxKESEvolutions": 60, + "maxLovelaceSupply": 2010000000000000, + "networkId": "Testnet", + "networkMagic": 42, + "protocolParams": { + "a0": 0.3, + "decentralisationParam": 0, + "eMax": 18, + "extraEntropy": { + "tag": "NeutralNonce" + }, + "keyDeposit": 400000, + "maxBlockBodySize": 81920, + "maxBlockHeaderSize": 1100, + "maxTxSize": 16384, + "minFeeA": 0, + "minFeeB": 0, + "minPoolCost": 0, + "minUTxOValue": 0, + "nOpt": 50, + "poolDeposit": 500000000, + "protocolVersion": { + "major": 5, + "minor": 0 + }, + "rho": 0.0022, + "tau": 0.05 + }, + "securityParam": 2160, + "slotLength": 1, + "slotsPerKESPeriod": 129600, + "staking": { + "pools": { + "1dc0a846ec816bdcc5288c0b57871a0400e86728ad0851132e43883f": { + "cost": 0, + "margin": 0, + "metadata": null, + "owners": [], + "pledge": 0, + "publicKey": "1dc0a846ec816bdcc5288c0b57871a0400e86728ad0851132e43883f", + "relays": [], + "rewardAccount": { + "credential": { + "key hash": "22c700325ec932f59048a7258e89b5f166604f184e0809d16a495550" + }, + "network": "Testnet" + }, + "vrf": "ee8fdadab21abed48fadee52492596841c561640920d1c022fa8ae51d206c714" + }, + "d3e16257b8c608ec4b2cb89621d7ddb60fc08839c1b491a598615fb5": { + "cost": 0, + "margin": 0, + "metadata": null, + "owners": [], + "pledge": 0, + "publicKey": "d3e16257b8c608ec4b2cb89621d7ddb60fc08839c1b491a598615fb5", + "relays": [], + "rewardAccount": { + "credential": { + "key hash": "3832f1051268ab7a7765142f21d695b7aaf40c576bf2d1a71894f0a4" + }, + "network": "Testnet" + }, + "vrf": "bbc57641002e501cf8bf98b776ba082c604e6af7c482f716934ed08d19c22733" + } + }, + "stake": { + "6b47a4e6e19ac5257fc97bd220bd9ae368aa2d775d86315ab7ef058f": "d3e16257b8c608ec4b2cb89621d7ddb60fc08839c1b491a598615fb5", + "d1f6f71a04ca856cc569fd068b069a555999c4776ad50b4cdd049d13": "1dc0a846ec816bdcc5288c0b57871a0400e86728ad0851132e43883f" + } + }, + "systemStart": "2022-06-16T08:04:19Z", + "updateQuorum": 5 +} diff --git a/cardano-submit-api/cardano-submit-api.cabal b/cardano-submit-api/cardano-submit-api.cabal index a7dbb612790..7a1be8a0ac1 100644 --- a/cardano-submit-api/cardano-submit-api.cabal +++ b/cardano-submit-api/cardano-submit-api.cabal @@ -42,7 +42,7 @@ library , cardano-api ^>= 11.3 , cardano-binary , cardano-cli ^>= 11.1 - , cardano-crypto-class ^>=2.3 + , cardano-crypto-class ^>=2.5 , containers , ekg-core , http-media @@ -99,4 +99,4 @@ test-suite unit main-is: test.hs hs-source-dirs: test build-depends: base - , cardano-crypto-class ^>=2.3 + , cardano-crypto-class ^>=2.5 diff --git a/cardano-submit-api/src/Cardano/TxSubmit.hs b/cardano-submit-api/src/Cardano/TxSubmit.hs index d19ff41fa89..bf706e3d510 100644 --- a/cardano-submit-api/src/Cardano/TxSubmit.hs +++ b/cardano-submit-api/src/Cardano/TxSubmit.hs @@ -9,6 +9,7 @@ module Cardano.TxSubmit ) where import Cardano.Logging (BackendConfig (..), ConfigOption (ConfBackend, ConfSeverity), + ConfigSource (FromFile), FormatLogging (HumanFormatColoured), SeverityF (SeverityF), SeverityS (Info), Trace, TraceConfig, configureTracers, ekgTracer, emptyConfigReflection, emptyTraceConfig, mkCardanoTracer, readConfigurationWithDefault, standardTracer, @@ -37,7 +38,7 @@ defaultTraceConfig = runTxSubmitWebapi :: TxSubmitNodeParams -> IO () runTxSubmitWebapi tsnp = do - tracingConfig <- readConfigurationWithDefault (unConfigFile tspConfigFile) defaultTraceConfig + tracingConfig <- readConfigurationWithDefault (FromFile (unConfigFile tspConfigFile)) defaultTraceConfig (trce, registrySample) <- mkTraceDispatcher tracingConfig Async.withAsync (runTxSubmitServer trce tspWebserverConfig tspProtocol tspNetworkId tspSocketPath) diff --git a/cardano-testnet/cardano-testnet.cabal b/cardano-testnet/cardano-testnet.cabal index 27ee0a8fc99..79d50ca87ce 100644 --- a/cardano-testnet/cardano-testnet.cabal +++ b/cardano-testnet/cardano-testnet.cabal @@ -43,7 +43,7 @@ library , bytestring , cardano-api ^>= 11.3 , cardano-cli:{cardano-cli, cardano-cli-test-lib} ^>= 11.1 - , cardano-crypto-class ^>=2.3 + , cardano-crypto-class ^>=2.5 , cardano-crypto-wrapper , cardano-git-rev ^>= 0.2.2 , cardano-ledger-alonzo @@ -57,13 +57,12 @@ library , cardano-ledger-dijkstra , cardano-ledger-shelley , cardano-node - , cardano-ping ^>= 0.10 + , cardano-diffusion:ping ^>= 1.0 , cardano-prelude , cardano-rpc , contra-tracer , containers , data-default-class - , cborg , containers , contra-tracer , data-default-class @@ -72,6 +71,7 @@ library , exceptions , extra , filepath + , fs-api ^>= 0.4 , hedgehog , hedgehog-extras ^>= 0.10 , http-conduit @@ -82,7 +82,6 @@ library , mono-traversable , mtl , network - , network-mux , optparse-applicative-fork , parsec , ouroboros-network:{api, framework, ouroboros-network} ^>= 1.1 @@ -155,7 +154,7 @@ executable cardano-testnet main-is: cardano-testnet.hs - build-depends: cardano-crypto-class ^>=2.3 + build-depends: cardano-crypto-class ^>=2.5 , cardano-cli , cardano-testnet , optparse-applicative-fork diff --git a/cardano-testnet/changelog.d/20260714_120000_fabrizio.ferrai_node_11_1_testnet_integration.md b/cardano-testnet/changelog.d/20260714_120000_fabrizio.ferrai_node_11_1_testnet_integration.md new file mode 100644 index 00000000000..96a6eecb8e3 --- /dev/null +++ b/cardano-testnet/changelog.d/20260714_120000_fabrizio.ferrai_node_11_1_testnet_integration.md @@ -0,0 +1,5 @@ +### Changed + +- Migrated ping support from `cardano-ping` to `cardano-diffusion:ping`: `Testnet.Ping` now wraps `Cardano.Network.Ping.pingClient` instead of the hand-rolled mux implementation. +- Threaded a filesystem handle (`SomeHasFS IO` from `fs-api`) through the `foldEpochState` call sites, via a new `mkNodeConfigFs` helper in `Testnet.Filepath`. +- Bumped `cardano-crypto-class` to `^>= 2.5`. diff --git a/cardano-testnet/src/Testnet/Blockfrost.hs b/cardano-testnet/src/Testnet/Blockfrost.hs index 16284428b4f..f81885eee9b 100644 --- a/cardano-testnet/src/Testnet/Blockfrost.hs +++ b/cardano-testnet/src/Testnet/Blockfrost.hs @@ -96,7 +96,7 @@ data BlockfrostParams = BlockfrostParams , bfgNOpt :: Word16 , bfgPoolDeposit :: Coin , bfgProtocolMajorVer :: Version - , bfgProtocolMinorVer :: Natural + , bfgProtocolMinorVer :: Word32 , bfgRho :: UnitInterval , bfgTau :: UnitInterval } deriving (Eq, Show) diff --git a/cardano-testnet/src/Testnet/ChainWatchdog.hs b/cardano-testnet/src/Testnet/ChainWatchdog.hs index a7b7292549f..7dcccece999 100644 --- a/cardano-testnet/src/Testnet/ChainWatchdog.hs +++ b/cardano-testnet/src/Testnet/ChainWatchdog.hs @@ -30,7 +30,7 @@ import Control.Exception (Exception (..), asyncExceptionFromException, asyncExceptionToException) import Control.Exception.Safe (SomeException, try) import Control.Monad (void, when) -import Control.Tracer (Tracer (..), traceWith) +import Control.Tracer (Tracer, mkTracer, traceWith) import Data.List.NonEmpty (NonEmpty) import Data.Maybe (isNothing) import Data.Text (Text) @@ -179,7 +179,7 @@ chainStallWatchdog tracer shelleyGenesis connectInfo nodeHandles testThread = do -- orchestration configuration and be used everywhere in the orchestration code -- instead of ad-hoc printing. stderrTracer :: Tracer IO Text -stderrTracer = Tracer $ \msg -> Text.hPutStrLn stderr msg >> hFlush stderr +stderrTracer = mkTracer $ \msg -> Text.hPutStrLn stderr msg >> hFlush stderr -- | Failure message explaining why a chain that stopped extending will never recover. -- See https://github.com/IntersectMBO/cardano-node/issues/5762 diff --git a/cardano-testnet/src/Testnet/Components/Query.hs b/cardano-testnet/src/Testnet/Components/Query.hs index 8f085bd662f..7b688fe17ca 100644 --- a/cardano-testnet/src/Testnet/Components/Query.hs +++ b/cardano-testnet/src/Testnet/Components/Query.hs @@ -80,6 +80,7 @@ import GHC.Exts (IsList (..)) import GHC.Stack import Lens.Micro (Lens', to, (^.)) +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Process.RunIO (liftIOAnnotated) import Testnet.Property.Assert import Testnet.Runtime @@ -102,9 +103,11 @@ waitUntilEpoch -> EpochNo -- ^ Desired epoch -> m EpochNo -- ^ The epoch number reached waitUntilEpoch nodeConfigFile socketPath desiredEpoch = withFrozenCallStack $ do - result <- H.evalIO . runExceptT $ - foldEpochState - nodeConfigFile socketPath QuickValidation desiredEpoch () (\_ _ _ -> pure ConditionNotMet) + result <- H.evalIO $ do + fs <- mkNodeConfigFs nodeConfigFile + runExceptT $ + foldEpochState + fs nodeConfigFile socketPath QuickValidation desiredEpoch () (\_ _ _ -> pure ConditionNotMet) case result of Left (FoldBlocksApplyBlockError (TerminationEpochReached epochNo)) -> pure epochNo @@ -388,7 +391,8 @@ getEpochStateView getEpochStateView nodeConfigFile socketPath = withFrozenCallStack $ do esv <- H.evalIO $ EpochStateView <$> newTVarIO (Left EpochStateNotInitialised) <*> newTVarIO 0 _ <- asyncRegister_ $ do - result <- runExceptT $ foldEpochState nodeConfigFile socketPath QuickValidation (EpochNo maxBound) () + fs <- mkNodeConfigFs nodeConfigFile + result <- runExceptT $ foldEpochState fs nodeConfigFile socketPath QuickValidation (EpochNo maxBound) () $ \epochState slotNumber blockNumber -> do liftIOAnnotated . atomically $ writeEpochStateView esv $ Right (epochState, slotNumber, blockNumber) pure ConditionNotMet diff --git a/cardano-testnet/src/Testnet/Defaults.hs b/cardano-testnet/src/Testnet/Defaults.hs index e6e743fb51c..dfddd000b2e 100644 --- a/cardano-testnet/src/Testnet/Defaults.hs +++ b/cardano-testnet/src/Testnet/Defaults.hs @@ -110,7 +110,7 @@ import Data.Scientific import Data.Text (Text) import qualified Data.Text as Text import Data.Time (UTCTime) -import Data.Word (Word64) +import Data.Word (Word32, Word64) import Lens.Micro import Numeric.Natural import Test.Cardano.Ledger.Core.Rational @@ -190,6 +190,7 @@ defaultConwayGenesis = do , cgCommittee = DefaultClass.def , cgDelegs = mempty , cgInitialDReps = mempty + , cgExtraConfig = SNothing } -- | The only era supported by cardano-testnet for the moment. @@ -417,7 +418,7 @@ eraToProtocolVersion = AnyShelleyBasedEra ShelleyBasedEraDijkstra -> mkProtVer (12, 0) -- TODO: Expose from cardano-api -mkProtVer :: (Natural, Natural) -> ProtVer +mkProtVer :: (Natural, Word32) -> ProtVer mkProtVer (majorProtVer, minorProtVer) = case (`ProtVer` minorProtVer) <$> Ledger.mkVersion majorProtVer of Just pVer -> pVer diff --git a/cardano-testnet/src/Testnet/Filepath.hs b/cardano-testnet/src/Testnet/Filepath.hs index 59c5771ab56..ab87ce9c1e1 100644 --- a/cardano-testnet/src/Testnet/Filepath.hs +++ b/cardano-testnet/src/Testnet/Filepath.hs @@ -9,19 +9,27 @@ module Testnet.Filepath , makeSocketDir , makeSprocket , makeTmpBaseAbsPath + , mkNodeConfigFs ) where import Prelude import Data.String (IsString (..)) +import System.Directory (makeAbsolute) import System.FilePath import Hedgehog.Extras.Stock.IO.Network.Sprocket (Sprocket (..)) import RIO (Display (..)) +import Cardano.Api (File (..)) + import Cardano.Node.Testnet.Paths (defaultSocketDir) +import System.FS.API (SomeHasFS (..)) +import System.FS.API.Types (MountPoint (MountPoint)) +import System.FS.IO (ioHasFS) + makeSprocket :: TmpAbsolutePath @@ -51,3 +59,8 @@ makeTmpBaseAbsPath (TmpAbsolutePath fp) = addTrailingPathSeparator $ takeDirecto makeLogDir :: TmpAbsolutePath -> FilePath makeLogDir (TmpAbsolutePath fp) = addTrailingPathSeparator $ fp "logs" + +mkNodeConfigFs :: File content direction -> IO (SomeHasFS IO) +mkNodeConfigFs configFile = do + configDir <- takeDirectory <$> makeAbsolute (unFile configFile) + pure $ SomeHasFS (ioHasFS (MountPoint configDir)) diff --git a/cardano-testnet/src/Testnet/Ping.hs b/cardano-testnet/src/Testnet/Ping.hs index b2d2824de09..1146dbed4ef 100644 --- a/cardano-testnet/src/Testnet/Ping.hs +++ b/cardano-testnet/src/Testnet/Ping.hs @@ -1,9 +1,4 @@ -{-# LANGUAGE BangPatterns #-} -{-# LANGUAGE LambdaCase #-} {-# LANGUAGE NamedFieldPuns #-} -{-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE RankNTypes #-} -{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} module Testnet.Ping @@ -12,36 +7,22 @@ module Testnet.Ping , waitForSprocket , waitForPortClosed , TestnetMagic - , PingClientError(..) + , CNP.PingClientException ) where -import Cardano.Api (Error (..)) +import qualified Cardano.Network.Ping as CNP -import Cardano.Network.Ping (HandshakeFailure, NodeVersion (..), handshakeDec, - handshakeReq, isSameVersionAndMagic, supportedNodeToClientVersions) - -import qualified Codec.CBOR.Read as CBOR import Control.Exception.Safe -import Control.Monad -import Control.Monad.Class.MonadTime.SI (Time) +import Control.Monad (when) import qualified Control.Monad.Class.MonadTimer.SI as MT import Control.Monad.IO.Class import qualified Control.Retry as R import Control.Tracer (nullTracer) -import qualified Data.ByteString.Lazy as LBS import Data.Either import Data.IORef -import qualified Data.List as L import Data.Word (Word32) -import qualified Network.Mux as Mux -import Network.Mux.Bearer (MakeBearer (..), makeSocketBearer) -import Network.Mux.Timeout (TimeoutFn, withTimeoutSerial) -import Network.Mux.Types (MiniProtocolDir (InitiatorDir), MiniProtocolNum (..), - RemoteClockModel (RemoteClockModel), SDU (..), SDUHeader (..)) -import qualified Network.Mux.Types as Mux -import Network.Socket (AddrInfo (..), PortNumber, StructLinger (..)) +import Network.Socket (AddrInfo (..), PortNumber) import qualified Network.Socket as Socket -import Prettyprinter import Testnet.Process.RunIO (liftIOAnnotated) @@ -50,93 +31,25 @@ import qualified Hedgehog.Extras.Stock.IO.Network.Sprocket as IO type TestnetMagic = Word32 --- | Mini protocol number. We're only sending ping, so 0. -handshakeNum :: MiniProtocolNum -handshakeNum = MiniProtocolNum 0 - --- | Timeout for reading a multiplexer service data unit, in seconds. -sduTimeout :: MT.DiffTime -sduTimeout = 30 - --- | Perform handshake query to obtain supported version numbers by node. -doHandshakeQuery :: Bool -doHandshakeQuery = True - -- | Ping the node once pingNode :: MonadIO m => TestnetMagic -- ^ testnet magic -> IO.Sprocket -- ^ node sprocket - -> m (Either PingClientError ()) -- ^ '()' means success -pingNode networkMagic sprocket = liftIOAnnotated $ bracket - (Socket.socket (Socket.addrFamily peer) Socket.Stream Socket.defaultProtocol) - Socket.close - (\sd -> handle (pure . Left . PceException) $ withTimeoutSerial $ \timeoutfn -> do - when (Socket.addrFamily peer /= Socket.AF_UNIX) $ do - Socket.setSocketOption sd Socket.NoDelay 1 - Socket.setSockOpt sd Socket.Linger - StructLinger - { sl_onoff = 1 - , sl_linger = 0 - } - - Socket.connect sd (Socket.addrAddress peer) - peerStr <- peerString - - bearer <- getBearer makeSocketBearer sduTimeout sd Nothing - - let versions = supportedNodeToClientVersions networkMagic - !_ <- Mux.write bearer nullTracer timeoutfn $ wrap handshakeNum InitiatorDir (handshakeReq versions doHandshakeQuery) - (msg, !_) <- nextMsg bearer timeoutfn handshakeNum - - pure $ case CBOR.deserialiseFromBytes handshakeDec msg of - Left err -> Left $ PceDecodingError peerStr err - Right (_, Left err) -> Left $ PceProtocolError peerStr err - Right (_, Right recVersions) - | areVersionsAccepted versions recVersions -> pure () - | otherwise -> Left $ PceVersionNegotiationError peerStr versions recVersions - ) + -> m (Either CNP.PingClientException ()) -- ^ 'Right ()' means success +pingNode networkMagic sprocket = + liftIOAnnotated $ + CNP.pingClient nullTracer nullTracer nullTracer nullTracer (pingOpts networkMagic) (sprocketToAddrInfo sprocket) where - peer = sprocketToAddrInfo sprocket :: AddrInfo - - -- | Wrap a message in a mux service data unit. - wrap :: MiniProtocolNum -> MiniProtocolDir -> LBS.ByteString -> SDU - wrap mhNum mhDir msBlob = SDU - { msHeader = SDUHeader - { mhTimestamp = RemoteClockModel 0 - , mhNum - , mhDir - , mhLength = fromIntegral $ LBS.length msBlob - } - , msBlob + pingOpts magic = CNP.PingOpts + { CNP.pingOptsCount = 1 + , CNP.pingOptsMagic = CNP.NetworkMagic magic + , CNP.pingOptsJson = CNP.AsText + , CNP.pingOptsQuiet = True + , CNP.pingOptsMode = CNP.PingMode + , CNP.pingOptsSRVPrefix = "_cardano._tcp" + , CNP.pingOptsColor = CNP.ColorAuto } - areVersionsAccepted :: [NodeVersion] -> [NodeVersion] -> Bool - areVersionsAccepted accVersions recVersions = - let intersects = L.intersectBy isSameVersionAndMagic recVersions accVersions in - not $ null intersects - - peerString :: IO String - peerString = - case Socket.addrFamily peer of - Socket.AF_UNIX -> pure . show $ Socket.addrAddress peer - _ -> do - (Just host, Just port) <- - Socket.getNameInfo - [Socket.NI_NUMERICHOST, Socket.NI_NUMERICSERV] - True True (Socket.addrAddress peer) - pure $ host <> ":" <> port - - -- | Fetch next message from mux bearer. Ignores messages not matching handshake protocol number. - nextMsg :: Mux.Bearer IO -- ^ a mux bearer - -> TimeoutFn IO -- ^ timeout function, for reading messages - -> MiniProtocolNum -- ^ handshake protocol number - -> IO (LBS.ByteString, Time) -- ^ raw message and timestamp - nextMsg bearer timeoutfn ptclNum = do - (sdu, t_e) <- Mux.read bearer nullTracer timeoutfn - if mhNum (msHeader sdu) == ptclNum - then pure (msBlob sdu, t_e) - else nextMsg bearer timeoutfn ptclNum - -- | Wait for 'sprocket' to become ready. Periodically tries to connect to 'sprocket', with the provided interval. -- If there was no success within 'timeout' period, return the last exception thrown during a connection -- attempt. @@ -185,29 +98,3 @@ waitForPortClosed timeout interval portNumber = liftIOAnnotated $ do let retryPolicy = R.constantDelay (round @Double $ realToFrac interval) <> R.limitRetries (ceiling $ toRational timeout / toRational interval) fmap not . R.retrying retryPolicy (const pure) $ \_ -> liftIOAnnotated (IO.isPortOpen (fromIntegral portNumber)) - -data PingClientError - = PceDecodingError - !String -- ^ peer string - !CBOR.DeserialiseFailure -- ^ deserialization exception - | PceProtocolError - !String -- ^ peer string - !HandshakeFailure -- ^ handshake exception - | PceVersionNegotiationError - !String -- ^ peer string - ![NodeVersion] -- ^ requested versions - ![NodeVersion] -- ^ received node versions - | PceException - !SomeException - -instance Error PingClientError where - prettyError = \case - PceDecodingError peerStr exception -> pretty peerStr <+> "Decoding error:" <+> pretty (displayException exception) - PceProtocolError peerStr exception -> pretty peerStr <+> "Protocol error:" <+> viaShow exception - PceVersionNegotiationError peerStr requestedVersions receivedVersions -> vsep - [ pretty peerStr <+> "Version negotiation error: No overlapping versions with" <+> viaShow requestedVersions - , "Received versions:" <+> viaShow receivedVersions - ] - PceException exception -> "An unknown exception occurred:" <+> pretty (displayException exception) - - diff --git a/cardano-testnet/src/Testnet/Process/Cli/SPO.hs b/cardano-testnet/src/Testnet/Process/Cli/SPO.hs index 6b976ff5a17..65fe2375332 100644 --- a/cardano-testnet/src/Testnet/Process/Cli/SPO.hs +++ b/cardano-testnet/src/Testnet/Process/Cli/SPO.hs @@ -101,7 +101,9 @@ checkStakeKeyRegistered tempAbsP nodeConfigFile sPath terminationEpoch execConfi sAddr <- case deserialiseAddress AsStakeAddress $ Text.pack stakeAddr of Just sAddr -> return sAddr Nothing -> H.failWithCustom GHC.callStack Nothing $ "Invalid stake address: " <> stakeAddr + fs <- liftIO $ mkNodeConfigFs nodeConfigFile result <- runExceptT $ foldEpochState + fs nodeConfigFile sPath QuickValidation diff --git a/cardano-testnet/src/Testnet/Runtime.hs b/cardano-testnet/src/Testnet/Runtime.hs index 7ee7cee16d0..2cf0dfa3326 100644 --- a/cardano-testnet/src/Testnet/Runtime.hs +++ b/cardano-testnet/src/Testnet/Runtime.hs @@ -491,8 +491,11 @@ startLedgerNewEpochStateLogging testnetRuntime tmpWorkspace = withFrozenCallStac let socketPath = H.sprocketSystemName . NEL.head $ testnetSprockets testnetRuntime + fs <- liftIOAnnotated $ mkNodeConfigFs (configurationFile testnetRuntime) + void $ asyncRegister_ . runExceptT $ foldEpochState + fs (configurationFile testnetRuntime) (Api.File socketPath) Api.QuickValidation @@ -507,9 +510,9 @@ startLedgerNewEpochStateLogging testnetRuntime tmpWorkspace = withFrozenCallStac -> SlotNo -> BlockNo -> StateT (Maybe AnyNewEpochState) IO ConditionResult - handler outputFp diffFp anes@(AnyNewEpochState !sbe !nes _) _ (BlockNo blockNo) = handleException $ do + handler outputFp diffFp anes@(AnyNewEpochState !sbe !nes _) _ (BlockNo blockNo') = handleException $ do let prettyNes = shelleyBasedEraConstraints sbe (encodePretty nes) - blockLabel = "#### BLOCK " <> show blockNo <> " ####" + blockLabel = "#### BLOCK " <> show blockNo' <> " ####" liftIOAnnotated . BSC.appendFile outputFp $ BSC.unlines [BSC.pack blockLabel, prettyNes, ""] -- store epoch state for logging of differences diff --git a/cardano-testnet/src/Testnet/Start/Cardano.hs b/cardano-testnet/src/Testnet/Start/Cardano.hs index 89d05c644a0..1f3226141af 100644 --- a/cardano-testnet/src/Testnet/Start/Cardano.hs +++ b/cardano-testnet/src/Testnet/Start/Cardano.hs @@ -454,16 +454,18 @@ cardanoTestnet -> TestnetNode -> m () waitForBlockThrow horizon timeoutSeconds nodeConfigFile node@TestnetNode{nodeName} = do + fs <- liftIO $ mkNodeConfigFs nodeConfigFile result <- timeout (timeoutSeconds * 1_000_000) $ runExceptT . foldEpochState + fs nodeConfigFile (nodeSocketPath node) QuickValidation (EpochNo maxBound) minBound - $ \_ slotNo blockNo -> do + $ \_ slotNo blockNo' -> do put slotNo - pure $ if blockNo >= 1 + pure $ if blockNo' >= 1 then ConditionMet -- we got one block else ConditionNotMet diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Api/TxReferenceInputDatum.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Api/TxReferenceInputDatum.hs index 97c46c57a91..46e023a88b1 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Api/TxReferenceInputDatum.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Api/TxReferenceInputDatum.hs @@ -156,7 +156,6 @@ hprop_tx_refin_datum = integrationRetryWorkspace 2 "api-tx-refin-dat" $ \tempAbs (unLedgerProtocolParameters pparams) mempty mempty - mempty ledgerUtxo content addr0 diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/FoldEpochState.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/FoldEpochState.hs index 990f4260ea3..122117d069f 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/FoldEpochState.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/FoldEpochState.hs @@ -17,6 +17,7 @@ import qualified Data.List.NonEmpty as NEL import qualified System.Directory as IO import System.FilePath (()) +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Property.Util (integrationRetryWorkspace) import Hedgehog ((===)) @@ -43,14 +44,16 @@ prop_foldEpochState = integrationRetryWorkspace 2 "foldEpochState" $ \tempAbsBas -> SlotNo -> BlockNo -> StateT [(SlotNo, BlockNo)] IO ConditionResult - handler _ slotNo blockNo = do - modify ((slotNo, blockNo):) + handler _ slotNo blockNo' = do + modify ((slotNo, blockNo'):) s <- get if length s >= 10 then pure ConditionMet else pure ConditionNotMet - (_, nums) <- H.leftFailM $ H.evalIO $ runExceptT $ - Api.foldEpochState configurationFile (Api.File socketPathAbs) Api.QuickValidation (EpochNo maxBound) [] handler + (_, nums) <- H.leftFailM $ H.evalIO $ do + fs <- mkNodeConfigFs configurationFile + runExceptT $ + Api.foldEpochState fs configurationFile (Api.File socketPathAbs) Api.QuickValidation (EpochNo maxBound) [] handler length nums === 10 diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/InfoAction.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/InfoAction.hs index e460d81a808..f6064960409 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/InfoAction.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/InfoAction.hs @@ -35,6 +35,7 @@ import System.FilePath (()) import Test.Cardano.CLI.Hash (serveFilesWhile) import Testnet.Components.Query import Testnet.Defaults +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Process.Cli.Keys import Testnet.Process.Cli.SPO (createStakeKeyRegistrationCertificate) import Testnet.Process.Cli.Transaction (retrieveTransactionId) @@ -247,8 +248,10 @@ hprop_ledger_events_info_action = integrationRetryWorkspace 2 "info-hash" $ \tem ] -- We check that info action was successfully ratified + fs <- evalIO $ mkNodeConfigFs configurationFile !meInfoRatified <- H.timeout 120_000_000 $ runExceptT $ foldBlocks + fs configurationFile socketPath FullValidation diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitution.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitution.hs index a9846c95997..3aec166bb4a 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitution.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitution.hs @@ -45,6 +45,7 @@ import Testnet.Components.Configuration import Testnet.Components.Query import Testnet.Defaults import Testnet.EpochStateProcessing (unsafeEraFromSbe, waitForGovActionVotes) +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Process.Cli.DRep import Testnet.Process.Cli.Keys import Testnet.Process.Cli.SPO (createStakeKeyRegistrationCertificate) @@ -429,14 +430,17 @@ hprop_ledger_events_propose_new_constitution = integrationRetryWorkspace 2 "prop stakePoolVotes === mempty -- We check that constitution was successfully ratified - void . H.leftFailM . H.evalIO . runExceptT $ - foldEpochState - configurationFile - socketPath - FullValidation - (EpochNo 20) - () - (\epochState _ _ -> foldBlocksCheckConstitutionWasRatified constitutionHash constitutionScriptHash epochState) + void . H.leftFailM . H.evalIO $ do + fs <- mkNodeConfigFs configurationFile + runExceptT $ + foldEpochState + fs + configurationFile + socketPath + FullValidation + (EpochNo 20) + () + (\epochState _ _ -> foldBlocksCheckConstitutionWasRatified constitutionHash constitutionScriptHash epochState) foldBlocksCheckConstitutionWasRatified :: String -- submitted constitution hash diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitutionSPO.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitutionSPO.hs index 260222d56f2..82cf467e17a 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitutionSPO.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/ProposeNewConstitutionSPO.hs @@ -31,6 +31,7 @@ import System.FilePath (()) import Testnet.Components.Query import Testnet.Defaults import Testnet.EpochStateProcessing (unsafeEraFromSbe) +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Process.Cli.DRep import Testnet.Process.Cli.Keys import qualified Testnet.Process.Cli.SPO as SPO @@ -183,7 +184,8 @@ getConstitutionProposal -> EpochNo -- ^ The termination epoch: the constitution proposal must be found *before* this epoch -> m (Maybe L.GovActionId) getConstitutionProposal nodeConfigFile socketPath maxEpoch = do - result <- H.evalIO . runExceptT $ foldEpochState nodeConfigFile socketPath QuickValidation maxEpoch Nothing + fs <- H.evalIO $ mkNodeConfigFs nodeConfigFile + result <- H.evalIO . runExceptT $ foldEpochState fs nodeConfigFile socketPath QuickValidation maxEpoch Nothing $ \(AnyNewEpochState actualEra newEpochState _) _slotNb _blockNb -> obtainCommonConstraints (unsafeEraFromSbe actualEra) $ do let proposals = newEpochState diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryGrowth.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryGrowth.hs index f0109900e5a..73526132f9a 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryGrowth.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryGrowth.hs @@ -24,6 +24,7 @@ import Lens.Micro ((^.)) import qualified System.Directory as IO import System.FilePath (()) +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Process.Run (execCli', mkExecConfig) import Testnet.Property.Util (integrationRetryWorkspace) import Testnet.Start.Types @@ -59,8 +60,10 @@ prop_check_if_treasury_is_growing = integrationRetryWorkspace 2 "growing-treasur execConfig <- mkExecConfig tempBaseAbsPath poolSprocket1 testnetMagic pure (execConfig, socketPathAbs) - (_condition, treasuryValues) <- H.leftFailM . H.evalIO . runExceptT $ - Api.foldEpochState configurationFile socketPathAbs Api.QuickValidation (EpochNo 10) M.empty handler + (_condition, treasuryValues) <- H.leftFailM . H.evalIO $ do + fs <- mkNodeConfigFs configurationFile + runExceptT $ + Api.foldEpochState fs configurationFile socketPathAbs Api.QuickValidation (EpochNo 10) M.empty handler H.note_ $ "treasury for last 5 epochs: " <> show treasuryValues let treasuriesSortedByEpoch = diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryWithdrawal.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryWithdrawal.hs index acb7cffee4d..882d20b9314 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryWithdrawal.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Gov/TreasuryWithdrawal.hs @@ -42,6 +42,7 @@ import Test.Cardano.CLI.Hash (serveFilesWhile) import Testnet.Components.Query import Testnet.Defaults import Testnet.EpochStateProcessing (unsafeEraFromSbe) +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Process.Cli.Keys (cliStakeAddressKeyGen) import Testnet.Process.Cli.SPO (createStakeKeyRegistrationCertificate) import Testnet.Process.Cli.Transaction (retrieveTransactionId) @@ -272,7 +273,8 @@ getAnyWithdrawals -> EpochNo -> m (Maybe (Map (Credential Staking) Coin)) getAnyWithdrawals nodeConfigFile socketPath maxEpoch = withFrozenCallStack $ do - fmap snd . H.leftFailM . evalIO . runExceptT $ foldEpochState nodeConfigFile socketPath FullValidation maxEpoch Nothing + fs <- evalIO $ mkNodeConfigFs nodeConfigFile + fmap snd . H.leftFailM . evalIO . runExceptT $ foldEpochState fs nodeConfigFile socketPath FullValidation maxEpoch Nothing $ \(AnyNewEpochState actualEra newEpochState _) _ _ -> obtainCommonConstraints (unsafeEraFromSbe actualEra) $ do let withdrawals = newEpochState @@ -297,7 +299,8 @@ getTreasuryWithdrawalProposal -> EpochNo -- ^ The termination epoch: the withdrawal proposal must be found *before* this epoch -> m (Maybe L.GovActionId) getTreasuryWithdrawalProposal nodeConfigFile socketPath maxEpoch = withFrozenCallStack $ do - fmap snd . H.leftFailM . evalIO . runExceptT $ foldEpochState nodeConfigFile socketPath QuickValidation maxEpoch Nothing + fs <- evalIO $ mkNodeConfigFs nodeConfigFile + fmap snd . H.leftFailM . evalIO . runExceptT $ foldEpochState fs nodeConfigFile socketPath QuickValidation maxEpoch Nothing $ \(AnyNewEpochState actualEra newEpochState _) _ _ -> obtainCommonConstraints (unsafeEraFromSbe actualEra) $ do let proposals = newEpochState diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Query.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Query.hs index ccf328fc442..5f9a7754376 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Query.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Rpc/Query.hs @@ -82,9 +82,9 @@ hprop_rpc_query_pparams = integrationRetryWorkspace 2 "rpc-query-pparams" $ \tem ---------- QueryTipLocalStateOutput{localStateChainTip} <- H.noteShowM $ execCliStdoutToJson execConfig [eraName, "query", "tip"] - (slot, blockHash, blockNo) <- case localStateChainTip of + (slot, blockHash, blockNo') <- case localStateChainTip of ChainTipAtGenesis -> H.failure -- impossible - ChainTip (SlotNo slot) (HeaderHash hash) (BlockNo blockNo) -> pure (slot, SBS.fromShort hash, blockNo) + ChainTip (SlotNo slot) (HeaderHash hash) (BlockNo blockNo') -> pure (slot, SBS.fromShort hash, blockNo') ----------------------------------- -- Compute expected tip timestamp @@ -122,7 +122,7 @@ hprop_rpc_query_pparams = integrationRetryWorkspace 2 "rpc-query-pparams" $ \tem --------------------------- pparamsResponse ^. U5c.ledgerTip . U5c.slot === slot pparamsResponse ^. U5c.ledgerTip . U5c.hash === blockHash - pparamsResponse ^. U5c.ledgerTip . U5c.height === blockNo + pparamsResponse ^. U5c.ledgerTip . U5c.height === blockNo' H.assertWithinTolerance (pparamsResponse ^. U5c.ledgerTip . U5c.timestamp) expectedTimestampMs 1000 -- https://docs.cardano.org/about-cardano/explore-more/parameter-guide diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/SanityCheck.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/SanityCheck.hs index 262b9d1bb51..cdeedddd6c7 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/SanityCheck.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/SanityCheck.hs @@ -24,6 +24,7 @@ import Data.Time.Clock import GHC.Conc (ThreadStatus (..), threadStatus) import GHC.Stack +import Testnet.Filepath (mkNodeConfigFs) import Testnet.Property.Util (integrationRetryWorkspace) import Testnet.Runtime import Testnet.Start.Types @@ -68,8 +69,10 @@ hprop_ledger_events_sanity_check = integrationRetryWorkspace 2 "ledger-events-sa H.note_ $ "Abs path: " <> tempAbsBasePath' H.note_ $ "Socketpath: " <> unFile socketPath + fs <- evalIO $ mkNodeConfigFs configurationFile !ret <- runExceptT $ handleIOExceptionsWith IOE $ evalIO $ runExceptT $ foldBlocks + fs configurationFile socketPath FullValidation diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Utils.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Utils.hs index c47eb86d43a..0f4421950f8 100644 --- a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Utils.hs +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Utils.hs @@ -63,9 +63,9 @@ nodesProduceBlocks envDir TestnetRuntime{testnetNodes, testnetMagic} = do case tip of ChainTipAtGenesis -> H.failure - ChainTip _ _ (BlockNo blockNo) -> + ChainTip _ _ (BlockNo blockNo') -> -- Blocks have been produced if the tip of the chain is > 0 - H.assertWith blockNo (> 0) + H.assertWith blockNo' (> 0) -- If everything went fine, terminate the node and exit with success exit <- H.evalIO $ do diff --git a/cardano-tracer/cardano-tracer.cabal b/cardano-tracer/cardano-tracer.cabal index 6c56dd9d3c1..7061329cd05 100644 --- a/cardano-tracer/cardano-tracer.cabal +++ b/cardano-tracer/cardano-tracer.cabal @@ -134,7 +134,7 @@ library , stm <2.5.2 || >=2.5.3 , text , time - , trace-dispatcher ^>= 2.12.0 + , trace-dispatcher ^>= 2.13.0 , trace-forward ^>= 2.4.0 , trace-resources ^>= 0.2.4 , vector diff --git a/cardano-tracer/src/Cardano/Tracer/Acceptors/Client.hs b/cardano-tracer/src/Cardano/Tracer/Acceptors/Client.hs index 5762e669497..2c3e5805180 100644 --- a/cardano-tracer/src/Cardano/Tracer/Acceptors/Client.hs +++ b/cardano-tracer/src/Cardano/Tracer/Acceptors/Client.hs @@ -20,7 +20,7 @@ import Ouroboros.Network.Mux (MiniProtocol (..), MiniProtocolLimits (. MiniProtocolNum (..), OuroborosApplication (..), OuroborosApplicationWithMinimalCtx, RunMiniProtocol (..), miniProtocolLimits, miniProtocolNum, miniProtocolRun) -import Ouroboros.Network.Protocol.Handshake.Codec (cborTermVersionDataCodec, +import Ouroboros.Network.Protocol.Handshake.Codec (mkVersionedCodecCBORTerm, codecHandshake, noTimeLimitsHandshake, timeLimitsHandshake) import Ouroboros.Network.Protocol.Handshake.Type (Handshake) import Ouroboros.Network.Protocol.Handshake.Version (acceptableVersion, queryVersion, @@ -140,7 +140,7 @@ doConnectToForwarderLocal snocket address netMagic timeLimits app = do args = ConnectToArgs { ctaHandshakeCodec = codecHandshake forwardingVersionCodec, ctaHandshakeTimeLimits = timeLimits, - ctaVersionDataCodec = cborTermVersionDataCodec forwardingCodecCBORTerm, + ctaVersionDataCodec = mkVersionedCodecCBORTerm forwardingCodecCBORTerm, ctaConnectTracers = nullNetworkConnectTracers, ctaHandshakeCallbacks = HandshakeCallbacks acceptableVersion queryVersion } @@ -174,7 +174,7 @@ doConnectToForwarderSocket snocket address netMagic timeLimits app = do args = ConnectToArgs { ctaHandshakeCodec = codecHandshake forwardingVersionCodec, ctaHandshakeTimeLimits = timeLimits, - ctaVersionDataCodec = cborTermVersionDataCodec forwardingCodecCBORTerm, + ctaVersionDataCodec = mkVersionedCodecCBORTerm forwardingCodecCBORTerm, ctaConnectTracers = nullNetworkConnectTracers, ctaHandshakeCallbacks = HandshakeCallbacks acceptableVersion queryVersion } diff --git a/cardano-tracer/src/Cardano/Tracer/Acceptors/Server.hs b/cardano-tracer/src/Cardano/Tracer/Acceptors/Server.hs index fa75a6170dd..486497107ce 100644 --- a/cardano-tracer/src/Cardano/Tracer/Acceptors/Server.hs +++ b/cardano-tracer/src/Cardano/Tracer/Acceptors/Server.hs @@ -126,7 +126,7 @@ doListenToForwarderLocal snocket address netMagic timeLimits app = do haHandshakeTracer = nullTracer, haBearerTracer = nullTracer, haHandshakeCodec = Handshake.codecHandshake forwardingVersionCodec, - haVersionDataCodec = Handshake.cborTermVersionDataCodec forwardingCodecCBORTerm, + haVersionDataCodec = Handshake.mkVersionedCodecCBORTerm forwardingCodecCBORTerm, haAcceptVersion = Handshake.acceptableVersion, haQueryVersion = Handshake.queryVersion, haTimeLimits = timeLimits @@ -158,7 +158,7 @@ doListenToForwarderSocket snocket address netMagic timeLimits app = do haHandshakeTracer = nullTracer, haBearerTracer = nullTracer, haHandshakeCodec = Handshake.codecHandshake forwardingVersionCodec, - haVersionDataCodec = Handshake.cborTermVersionDataCodec forwardingCodecCBORTerm, + haVersionDataCodec = Handshake.mkVersionedCodecCBORTerm forwardingCodecCBORTerm, haAcceptVersion = Handshake.acceptableVersion, haQueryVersion = Handshake.queryVersion, haTimeLimits = timeLimits diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Forwarder.hs b/cardano-tracer/test/Cardano/Tracer/Test/Forwarder.hs index 90a277c8683..be93092b328 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Forwarder.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Forwarder.hs @@ -30,7 +30,7 @@ import Ouroboros.Network.Mux (MiniProtocol (..), MiniProtocolLimits (. miniProtocolLimits, miniProtocolNum, miniProtocolRun) import Ouroboros.Network.Protocol.Handshake (Handshake, HandshakeArguments (..)) import qualified Ouroboros.Network.Protocol.Handshake as Handshake -import Ouroboros.Network.Protocol.Handshake.Codec (cborTermVersionDataCodec, +import Ouroboros.Network.Protocol.Handshake.Codec (mkVersionedCodecCBORTerm, codecHandshake, noTimeLimitsHandshake) import qualified Ouroboros.Network.Server.Simple as Server import Ouroboros.Network.Snocket (MakeBearer, Snocket, localAddressFromPath, localSnocket, @@ -232,7 +232,7 @@ doConnectToAcceptor TestSetup{..} snocket muxBearer address timeLimits (ekgConfi args = ConnectToArgs { ctaHandshakeCodec = codecHandshake forwardingVersionCodec, ctaHandshakeTimeLimits = timeLimits, - ctaVersionDataCodec = cborTermVersionDataCodec forwardingCodecCBORTerm, + ctaVersionDataCodec = mkVersionedCodecCBORTerm forwardingCodecCBORTerm, ctaConnectTracers = nullNetworkConnectTracers, ctaHandshakeCallbacks = HandshakeCallbacks Handshake.acceptableVersion Handshake.queryVersion } @@ -281,7 +281,7 @@ doListenToAcceptor TestSetup{..} haHandshakeTracer = nullTracer, haBearerTracer = nullTracer, haHandshakeCodec = codecHandshake forwardingVersionCodec, - haVersionDataCodec = cborTermVersionDataCodec forwardingCodecCBORTerm, + haVersionDataCodec = mkVersionedCodecCBORTerm forwardingCodecCBORTerm, haAcceptVersion = Handshake.acceptableVersion, haQueryVersion = Handshake.queryVersion, haTimeLimits = timeLimits diff --git a/configuration/cardano/mainnet-config-legacy.json b/configuration/cardano/mainnet-config-legacy.json deleted file mode 100644 index f4bc557037e..00000000000 --- a/configuration/cardano/mainnet-config-legacy.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "AlonzoGenesisFile": "mainnet-alonzo-genesis.json", - "AlonzoGenesisHash": "7e94a15f55d1e82d10f09203fa1d40f8eede58fd8066542cf6566008068ed874", - "ByronGenesisFile": "mainnet-byron-genesis.json", - "ByronGenesisHash": "5f20df933584822601f9e3f8c024eb5eb252fe8cefb24d1317dc3d432e940ebb", - "CheckpointsFile": "mainnet-checkpoints.json", - "CheckpointsFileHash": "3e6dee5bae7acc6d870187e72674b37c929be8c66e62a552cf6a876b1af31ade", - "ConsensusMode": "PraosMode", - "ConwayGenesisFile": "mainnet-conway-genesis.json", - "ConwayGenesisHash": "15a199f895e461ec0ffc6dd4e4028af28a492ab4e806d39cb674c88f7643ef62", - "LastKnownBlockVersion-Alt": 0, - "LastKnownBlockVersion-Major": 3, - "LastKnownBlockVersion-Minor": 0, - "LedgerDB": { - "Backend": "V2InMemory", - "NumOfDiskSnapshots": 2, - "QueryBatchSize": 100000, - "SnapshotInterval": 4320 - }, - "MaxKnownMajorProtocolVersion": 2, - "MinNodeVersion": "10.7.0", - "Protocol": "Cardano", - "RequiresNetworkMagic": "RequiresNoMagic", - "ShelleyGenesisFile": "mainnet-shelley-genesis.json", - "ShelleyGenesisHash": "1a3be38bcbb7911969283716ad7aa550250226b76a61fc51cc9a9a35d9276d81", - "TraceAcceptPolicy": true, - "TraceBlockFetchClient": false, - "TraceBlockFetchDecisions": false, - "TraceBlockFetchProtocol": false, - "TraceBlockFetchProtocolSerialised": false, - "TraceBlockFetchServer": false, - "TraceChainDb": true, - "TraceChainSyncBlockServer": false, - "TraceChainSyncClient": false, - "TraceChainSyncHeaderServer": false, - "TraceChainSyncProtocol": false, - "TraceConnectionManager": true, - "TraceDNSResolver": true, - "TraceDNSSubscription": true, - "TraceDiffusionInitialization": true, - "TraceErrorPolicy": true, - "TraceForge": true, - "TraceHandshake": true, - "TraceInboundGovernor": true, - "TraceIpSubscription": true, - "TraceLedgerPeers": true, - "TraceLocalChainSyncProtocol": false, - "TraceLocalConnectionManager": true, - "TraceLocalErrorPolicy": true, - "TraceLocalHandshake": true, - "TraceLocalRootPeers": true, - "TraceLocalTxSubmissionProtocol": false, - "TraceLocalTxSubmissionServer": false, - "TraceMempool": false, - "TraceMux": false, - "TracePeerSelection": true, - "TracePeerSelectionActions": true, - "TracePublicRootPeers": true, - "TraceServer": true, - "TraceTxInbound": false, - "TraceTxOutbound": false, - "TraceTxSubmissionProtocol": false, - "TracingVerbosity": "NormalVerbosity", - "TurnOnLogMetrics": true, - "TurnOnLogging": true, - "UseTraceDispatcher": false, - "defaultBackends": [ - "KatipBK" - ], - "defaultScribes": [ - [ - "StdoutSK", - "stdout" - ] - ], - "hasEKG": 12788, - "hasPrometheus": [ - "127.0.0.1", - 12798 - ], - "minSeverity": "Info", - "options": { - "mapBackends": { - "cardano.node.metrics": [ - "EKGViewBK" - ], - "cardano.node.resources": [ - "EKGViewBK" - ] - }, - "mapSubtrace": { - "cardano.node.metrics": { - "subtrace": "Neutral" - } - } - }, - "rotation": { - "rpKeepFilesNum": 10, - "rpLogLimitBytes": 5000000, - "rpMaxAgeHours": 24 - }, - "setupBackends": [ - "KatipBK" - ], - "setupScribes": [ - { - "scFormat": "ScText", - "scKind": "StdoutSK", - "scName": "stdout", - "scRotation": null - } - ] -} diff --git a/configuration/cardano/mainnet-config.json b/configuration/cardano/mainnet-config.json index b587e72e99d..5027e9dd27a 100644 --- a/configuration/cardano/mainnet-config.json +++ b/configuration/cardano/mainnet-config.json @@ -13,12 +13,14 @@ "LastKnownBlockVersion-Minor": 0, "LedgerDB": { "Backend": "V2InMemory", - "NumOfDiskSnapshots": 2, "QueryBatchSize": 100000, - "SnapshotInterval": 4320 + "Snapshots": { + "NumOfDiskSnapshots": 2, + "SnapshotInterval": 4320 + } }, "MaxKnownMajorProtocolVersion": 2, - "MinNodeVersion": "10.7.0", + "MinNodeVersion": "11.1.0", "Protocol": "Cardano", "RequiresNetworkMagic": "RequiresNoMagic", "ShelleyGenesisFile": "mainnet-shelley-genesis.json", @@ -107,14 +109,5 @@ "Startup.DiffusionInit": { "severity": "Info" } - }, - "TurnOnLogMetrics": true, - "TurnOnLogging": true, - "UseTraceDispatcher": true, - "defaultBackends": [], - "defaultScribes": [], - "minSeverity": "Critical", - "options": {}, - "setupBackends": [], - "setupScribes": [] + } } diff --git a/configuration/cardano/mainnet-config.yaml b/configuration/cardano/mainnet-config.yaml index 86f805d4e8e..51c99d46458 100644 --- a/configuration/cardano/mainnet-config.yaml +++ b/configuration/cardano/mainnet-config.yaml @@ -79,39 +79,44 @@ ConsensusMode: PraosMode # Additional configuration options can be found at: # https://ouroboros-consensus.cardano.intersectmbo.org/docs/for-developers/utxo-hd/migrating -LedgerDB: - # The time interval between snapshots, in seconds. - SnapshotInterval: 4320 - - # The number of disk snapshots to keep. - NumOfDiskSnapshots: 2 +LedgerDB: # When querying the store for a big range of UTxOs (such as with # QueryUTxOByAddress), the store will be read in batches of this size. QueryBatchSize: 100000 # The backend can either be in memory with `V2InMemory` or on disk with - # `V1LMDB`. + # `V2LSM`. Backend: V2InMemory -##### Version Information ##### + # Instead of an object with individual options, a predefined snapshot + # policy can be selected by name, e.g. `Snapshots: Mithril`. + Snapshots: + # The snapshot interval in slots. + SnapshotInterval: 4320 -# Min is currently 10.7.0 due to change of config bundled peer-snapshot -# version. -MinNodeVersion: "10.7.0" + # Start taking the snaphots at a slot offset. + # SlotOffset = 172800; -##### Logging configuration ##### + # A minimum duration between snapshots, in seconds (used to avoid excessive snapshots while syncing). + # Default is 10 minutes. + # RateLimit = 600; -# Enable or disable logging overall -TurnOnLogging: True + # Randomised snapshot delay range, in seconds. + # Both Min and Max need to be specified, otherwise the default delay of (5min, 10min) will be used. + # MinDelay = 300; + # MaxDelay = 600; -# Enable the collection of various OS metrics such as memory and CPU use. -# These metrics are traced in the context name: 'cardano.node.metrics' and can -# be directed to the logs or monitoring backends. -TurnOnLogMetrics: True + # The number of disk snapshots to keep. + NumOfDiskSnapshots: 2 -# Use the modern tracing system instead of the legacy tracing system. -UseTraceDispatcher: True +##### Version Information ##### + +# Min is currently 11.1.0 due to removal of legacy tracing system and +# introduction of deterministic snapshots. +MinNodeVersion: "11.1.0" + +##### Logging configuration ##### # Match the metrics prefix of the legacy tracing system to minimize breaking # changes. @@ -255,17 +260,6 @@ TraceOptions: Mempool.SyncNotNeeded: severity: Silence -# Required by the legacy tracing system, this key is still required for -# cardano-node to start. -minSeverity: Critical - -# Required by some legacy tests which may otherwise fail to start. -defaultBackends: [] -defaultScribes: [] -options: {} -setupBackends: [] -setupScribes: [] - # Set or unset the mempool capacity override in number of bytes. # # This is intended for testing, and for low-resource machines to run with a smaller mempool. diff --git a/configuration/cardano/testnet-template-config-legacy.json b/configuration/cardano/testnet-template-config-legacy.json deleted file mode 100644 index 1d58efae3f3..00000000000 --- a/configuration/cardano/testnet-template-config-legacy.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "AlonzoGenesisFile": "alonzo-genesis.json", - "ApplicationName": "cardano-sl", - "ApplicationVersion": 0, - "ByronGenesisFile": "byron-genesis.json", - "ConwayGenesisFile": "conway-genesis.json", - "DijkstraGenesisFile": "dijkstra-genesis.json", - "ExperimentalHardForksEnabled": false, - "ExperimentalProtocolsEnabled": true, - "LastKnownBlockVersion-Alt": 0, - "LastKnownBlockVersion-Major": 3, - "LastKnownBlockVersion-Minor": 1, - "LedgerDB": { - "Backend": "V2InMemory", - "NumOfDiskSnapshots": 2, - "QueryBatchSize": 100000, - "SnapshotInterval": 216 - }, - "MaxConcurrencyDeadline": 4, - "MaxKnownMajorProtocolVersion": 2, - "PBftSignatureThreshold": 1.1, - "Protocol": "Cardano", - "RequiresNetworkMagic": "RequiresMagic", - "ShelleyGenesisFile": "shelley-genesis.json", - "TestAllegraHardForkAtEpoch": 0, - "TestAlonzoHardForkAtEpoch": 0, - "TestMaryHardForkAtEpoch": 0, - "TestShelleyHardForkAtEpoch": 0, - "TraceAcceptPolicy": true, - "TraceBlockFetchClient": false, - "TraceBlockFetchDecisions": false, - "TraceBlockFetchProtocol": false, - "TraceBlockFetchProtocolSerialised": false, - "TraceBlockFetchServer": false, - "TraceChainDb": true, - "TraceChainSyncBlockServer": false, - "TraceChainSyncClient": false, - "TraceChainSyncHeaderServer": false, - "TraceChainSyncProtocol": false, - "TraceConnectionManager": true, - "TraceDNSResolver": true, - "TraceDNSSubscription": true, - "TraceDiffusionInitialization": true, - "TraceErrorPolicy": true, - "TraceForge": true, - "TraceHandshake": false, - "TraceInboundGovernor": true, - "TraceIpSubscription": true, - "TraceLedgerPeers": true, - "TraceLocalChainSyncProtocol": false, - "TraceLocalErrorPolicy": true, - "TraceLocalHandshake": false, - "TraceLocalRootPeers": true, - "TraceLocalTxSubmissionProtocol": false, - "TraceLocalTxSubmissionServer": false, - "TraceMempool": false, - "TraceMux": false, - "TracePeerSelection": true, - "TracePeerSelectionActions": true, - "TracePublicRootPeers": true, - "TraceServer": true, - "TraceTxInbound": false, - "TraceTxOutbound": false, - "TraceTxSubmissionProtocol": false, - "TracingVerbosity": "NormalVerbosity", - "TurnOnLogMetrics": true, - "TurnOnLogging": true, - "UseTraceDispatcher": false, - "defaultBackends": [ - "KatipBK" - ], - "defaultScribes": [ - [ - "StdoutSK", - "cardano" - ] - ], - "hasEKG": 12788, - "hasPrometheus": [ - "127.0.0.1", - 12798 - ], - "minSeverity": "Debug", - "options": { - "mapBackends": { - "cardano.node.metrics": [ - "EKGViewBK" - ], - "cardano.node.resources": [ - "EKGViewBK" - ] - }, - "mapSubtrace": { - "cardano.node.metrics": { - "subtrace": "Neutral" - } - } - }, - "rotation": { - "rpKeepFilesNum": 10, - "rpLogLimitBytes": 5000000, - "rpMaxAgeHours": 24 - }, - "setupBackends": [ - "KatipBK" - ], - "setupScribes": [ - { - "scFormat": "ScText", - "scKind": "StdoutSK", - "scName": "cardano" - } - ] -} diff --git a/configuration/cardano/testnet-template-config.json b/configuration/cardano/testnet-template-config.json index 0ab850d4bad..720f362803d 100644 --- a/configuration/cardano/testnet-template-config.json +++ b/configuration/cardano/testnet-template-config.json @@ -13,9 +13,11 @@ "LastKnownBlockVersion-Minor": 1, "LedgerDB": { "Backend": "V2InMemory", - "NumOfDiskSnapshots": 2, "QueryBatchSize": 100000, - "SnapshotInterval": 216 + "Snapshots": { + "NumOfDiskSnapshots": 2, + "SnapshotInterval": 216 + } }, "MaxConcurrencyDeadline": 4, "MaxKnownMajorProtocolVersion": 2, @@ -111,14 +113,5 @@ "Startup.DiffusionInit": { "severity": "Info" } - }, - "TurnOnLogMetrics": true, - "TurnOnLogging": true, - "UseTraceDispatcher": true, - "defaultBackends": [], - "defaultScribes": [], - "minSeverity": "Critical", - "options": {}, - "setupBackends": [], - "setupScribes": [] + } } diff --git a/configuration/cardano/update-config-files.sh b/configuration/cardano/update-config-files.sh index 28ca99ed46b..f7693918c8b 100755 --- a/configuration/cardano/update-config-files.sh +++ b/configuration/cardano/update-config-files.sh @@ -30,7 +30,6 @@ copyCfg "mainnet-alonzo-genesis.json" copyCfg "mainnet-byron-genesis.json" copyCfg "mainnet-checkpoints.json" copyCfg "mainnet-config.json" -copyCfg "mainnet-config-legacy.json" copyCfg "mainnet-conway-genesis.json" copyCfg "mainnet-peer-snapshot.json" copyCfg "mainnet-shelley-genesis.json" @@ -40,7 +39,6 @@ copyCfg "mainnet-topology.json" copyTmplCfg "alonzo.json" copyTmplCfg "byron.json" copyTmplCfg "config.json" -copyTmplCfg "config-legacy.json" copyTmplCfg "conway.json" copyTmplCfg "dijkstra.json" copyTmplCfg "shelley.json" diff --git a/flake.lock b/flake.lock index 2f70e21159d..a9ba5214204 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "CHaP": { "flake": false, "locked": { - "lastModified": 1779876270, - "narHash": "sha256-FA9E1EaQvPITpO/8weQyi7p3KHgyNb9GiwM6F96Aoeo=", + "lastModified": 1782990451, + "narHash": "sha256-U0o77JuGp6ADqym7TtGV3AwzRn5SFuiTRyCOR6qfxGA=", "owner": "intersectmbo", "repo": "cardano-haskell-packages", - "rev": "cb63b6483a5d6ce36fb07815736315bd4408162e", + "rev": "def6ba6e0324e451802f5a17b12a00bd64639e14", "type": "github" }, "original": { @@ -50,6 +50,23 @@ "type": "github" } }, + "cabal-32": { + "flake": false, + "locked": { + "lastModified": 1603716527, + "narHash": "sha256-X0TFfdD4KZpwl0Zr6x+PLxUt/VyKQfX7ylXHdmZIL+w=", + "owner": "haskell", + "repo": "cabal", + "rev": "48bf10787e27364730dd37a42b603cee8d6af7ee", + "type": "github" + }, + "original": { + "owner": "haskell", + "ref": "3.2", + "repo": "cabal", + "type": "github" + } + }, "cabal-34": { "flake": false, "locked": { @@ -125,6 +142,21 @@ "type": "github" } }, + "crane": { + "locked": { + "lastModified": 1776635034, + "narHash": "sha256-OEOJrT3ZfwbChzODfIH4GzlNTtOFuZFWPtW7jIeR8xU=", + "owner": "ipetkov", + "repo": "crane", + "rev": "dc7496d8ea6e526b1254b55d09b966e94673750f", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "customConfig": { "locked": { "lastModified": 1630400035, @@ -189,6 +221,24 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1775087534, + "narHash": "sha256-91qqW8lhL7TLwgQWijoGBbiD4t7/q75KTi8NxjVmSmA=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "3107b77cd68437b9a76194f0f7f9c55f2329ca5b", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "flake-utils": { "locked": { "lastModified": 1667395993, @@ -207,11 +257,11 @@ "hackage-for-stackage": { "flake": false, "locked": { - "lastModified": 1782348961, - "narHash": "sha256-9wVcuwjT+3GNmYt7JWBUuyfBXVNdp+jvhsjen+Zvcwk=", + "lastModified": 1762302430, + "narHash": "sha256-thtGuIGrodKEfZPh+Sv22m1BR2zxNQY8RCsGlBWroj4=", "owner": "input-output-hk", "repo": "hackage.nix", - "rev": "afa4af51502c5973e8a2529a35bda4a5e39a7ea8", + "rev": "c5dc9e01d45948892915b5394f23986277fb0ccb", "type": "github" }, "original": { @@ -256,11 +306,11 @@ "hackageNix_2": { "flake": false, "locked": { - "lastModified": 1778061448, - "narHash": "sha256-cPUF8+l1ej7x4UZcuuf6IDsxU1WWmGWC0vFBH+6jXZk=", + "lastModified": 1782826502, + "narHash": "sha256-G6bt7DeWkDXJWI/fHME467V/SOaUdfG6hk7bTRuWyKg=", "owner": "input-output-hk", "repo": "hackage.nix", - "rev": "ba6ab6f3b781c8f308cba4fa384eafa48033f3cc", + "rev": "6a87e2657c145eb81d5f0f53702d4f26c08f9cbf", "type": "github" }, "original": { @@ -272,6 +322,7 @@ "haskellNix": { "inputs": { "HTTP": "HTTP", + "cabal-32": "cabal-32", "cabal-34": "cabal-34", "cabal-36": "cabal-36", "cardano-shell": "cardano-shell", @@ -286,7 +337,6 @@ "hls-2.0": "hls-2.0", "hls-2.10": "hls-2.10", "hls-2.11": "hls-2.11", - "hls-2.12": "hls-2.12", "hls-2.2": "hls-2.2", "hls-2.3": "hls-2.3", "hls-2.4": "hls-2.4", @@ -305,17 +355,16 @@ "nixpkgs-2405": "nixpkgs-2405", "nixpkgs-2411": "nixpkgs-2411", "nixpkgs-2505": "nixpkgs-2505", - "nixpkgs-2511": "nixpkgs-2511", "nixpkgs-unstable": "nixpkgs-unstable", "old-ghc-nix": "old-ghc-nix", "stackage": "stackage" }, "locked": { - "lastModified": 1782375089, - "narHash": "sha256-icdawbfIVT+g6FshBQ/yCFsfzXF//0qjOQzhg0ANweE=", + "lastModified": 1762315551, + "narHash": "sha256-7uaB/UpiFn/+gf7s5NMpSTTUv5Ws30DjsmmqZry+1cY=", "owner": "input-output-hk", "repo": "haskell.nix", - "rev": "bfee90426aa5761fa67cbb4e5c39e525e34a3a9b", + "rev": "ef52c36b9835c77a255befe2a20075ba71e3bfab", "type": "github" }, "original": { @@ -408,23 +457,6 @@ "type": "github" } }, - "hls-2.12": { - "flake": false, - "locked": { - "lastModified": 1758709460, - "narHash": "sha256-xkI8MIIVEVARskfWbGAgP5sHG/lyeKnkm0LIOJ19X5w=", - "owner": "haskell", - "repo": "haskell-language-server", - "rev": "7d983de4fa7ff54369f6dd31444bdb9869aec83e", - "type": "github" - }, - "original": { - "owner": "haskell", - "ref": "2.12.0.0", - "repo": "haskell-language-server", - "type": "github" - } - }, "hls-2.2": { "flake": false, "locked": { @@ -605,15 +637,16 @@ "sodium": "sodium" }, "locked": { - "lastModified": 1777941182, - "narHash": "sha256-FX3+8GIrB2z4akmcYTStELDKVJWgqy9yFt0mxwpU3Qc=", + "lastModified": 1782080462, + "narHash": "sha256-jHlPlNk/3PKEx+A++IiWGX3jQtTYzsIM0iTub/VOn2g=", "owner": "input-output-hk", "repo": "iohk-nix", - "rev": "9de00113c11ba8cac908a63acf34b193cda7475b", + "rev": "d7af9d6d6cd3efc91a23772e52fc24a01117f798", "type": "github" }, "original": { "owner": "input-output-hk", + "ref": "node-11.1", "repo": "iohk-nix", "type": "github" } @@ -621,11 +654,11 @@ "iserv-proxy": { "flake": false, "locked": { - "lastModified": 1778457436, - "narHash": "sha256-bzZAHGzwcQGzBTipJuUs9tvMGO28kp0373zqnpn0g5A=", + "lastModified": 1755243078, + "narHash": "sha256-GLbl1YaohKdpzZVJFRdcI1O1oE3F3uBer4lFv3Yy0l8=", "owner": "stable-haskell", "repo": "iserv-proxy", - "rev": "8cdc446f8e2d91b120ecc075063e9475d387df52", + "rev": "150605195cb7183a6fb7bed82f23fedf37c6f52a", "type": "github" }, "original": { @@ -635,6 +668,29 @@ "type": "github" } }, + "mithril": { + "inputs": { + "crane": "crane", + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay", + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1776926918, + "narHash": "sha256-muV2LpheC4OZsU1ipL9j6TmjGufMpGrXzvon+eWahgg=", + "owner": "input-output-hk", + "repo": "mithril", + "rev": "2478748ea9771baed8181ef0938c78f79ed60760", + "type": "github" + }, + "original": { + "owner": "input-output-hk", + "ref": "refs/tags/2617.0", + "repo": "mithril", + "type": "github" + } + }, "nixlib": { "locked": { "lastModified": 1667696192, @@ -650,6 +706,22 @@ "type": "github" } }, + "nixpkgs": { + "locked": { + "lastModified": 1776329215, + "narHash": "sha256-a8BYi3mzoJ/AcJP8UldOx8emoPRLeWqALZWu4ZvjPXw=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "b86751bc4085f48661017fa226dee99fab6c651b", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs-2305": { "locked": { "lastModified": 1705033721, @@ -700,11 +772,11 @@ }, "nixpkgs-2411": { "locked": { - "lastModified": 1751290243, - "narHash": "sha256-kNf+obkpJZWar7HZymXZbW+Rlk3HTEIMlpc6FCNz0Ds=", + "lastModified": 1748037224, + "narHash": "sha256-92vihpZr6dwEMV6g98M5kHZIttrWahb9iRPBm1atcPk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5ab036a8d97cb9476fbe81b09076e6e91d15e1b6", + "rev": "f09dede81861f3a83f7f06641ead34f02f37597f", "type": "github" }, "original": { @@ -716,11 +788,11 @@ }, "nixpkgs-2505": { "locked": { - "lastModified": 1764560356, - "narHash": "sha256-M5aFEFPppI4UhdOxwdmceJ9bDJC4T6C6CzCK1E2FZyo=", + "lastModified": 1757716134, + "narHash": "sha256-OYoZLWvmCnCTCJQwaQlpK1IO5nkLnLLoUW8wwmPmrfU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6c8f0cca84510cc79e09ea99a299c9bc17d03cb6", + "rev": "e85b5aa112a98805a016bbf6291e726debbc448a", "type": "github" }, "original": { @@ -730,29 +802,28 @@ "type": "github" } }, - "nixpkgs-2511": { + "nixpkgs-lib": { "locked": { - "lastModified": 1775749320, - "narHash": "sha256-msT6frWJSQ2WR+0cpk+KPcZdLTLagUIsJwQwIX9JNSo=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "74b87959b2d16f59f54d8559cf3cf26b9d907949", + "lastModified": 1774748309, + "narHash": "sha256-+U7gF3qxzwD5TZuANzZPeJTZRHS29OFQgkQ2kiTJBIQ=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "333c4e0545a6da976206c74db8773a1645b5870a", "type": "github" }, "original": { - "owner": "NixOS", - "ref": "nixpkgs-25.11-darwin", - "repo": "nixpkgs", + "owner": "nix-community", + "repo": "nixpkgs.lib", "type": "github" } }, "nixpkgs-unstable": { "locked": { - "lastModified": 1775888245, - "narHash": "sha256-nwASzrRDD1JBEu/o8ekKYEXm/oJW6EMCzCRdrwcLe90=", + "lastModified": 1759070547, + "narHash": "sha256-JVZl8NaVRYb0+381nl7LvPE+A774/dRpif01FKLrYFQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "13043924aaa7375ce482ebe2494338e058282925", + "rev": "647e5c14cbd5067f44ac86b74f014962df460840", "type": "github" }, "original": { @@ -790,6 +861,7 @@ "haskellNix": "haskellNix", "incl": "incl", "iohkNix": "iohkNix", + "mithril": "mithril", "nixpkgs": [ "haskellNix", "nixpkgs-unstable" @@ -797,6 +869,27 @@ "utils": "utils" } }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "mithril", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1776654897, + "narHash": "sha256-Vqi4AiJVCcBGn/RmBtRCgyH5rCxqm/w0xV9diJWF1Ic=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "25d75be8139815a53560745fa060909777495105", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, "secp256k1": { "flake": false, "locked": { @@ -834,11 +927,11 @@ "stackage": { "flake": false, "locked": { - "lastModified": 1782175169, - "narHash": "sha256-x+fFnU8BP0ztG68OnvKNKpcwFbf8QxRh24dDfD7HCSc=", + "lastModified": 1762301584, + "narHash": "sha256-yLihKEbngbLV1EhuLJSencMCtrDM2sYGsVZkX8xlSK8=", "owner": "input-output-hk", "repo": "stackage.nix", - "rev": "7234ccacc6e8938830b19792a64d2ce0771ebcbc", + "rev": "ce12bd44df0b5488bdbbe8762d79379e2bc76d62", "type": "github" }, "original": { @@ -862,6 +955,27 @@ "type": "github" } }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "mithril", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1775636079, + "narHash": "sha256-pc20NRoMdiar8oPQceQT47UUZMBTiMdUuWrYu2obUP0=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "790751ff7fd3801feeaf96d7dc416a8d581265ba", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + }, "utils": { "inputs": { "systems": "systems" diff --git a/flake.nix b/flake.nix index 1d0202a5b89..a77aa6aa7ba 100644 --- a/flake.nix +++ b/flake.nix @@ -48,13 +48,17 @@ incl.url = "github:divnix/incl"; iohkNix = { - url = "github:input-output-hk/iohk-nix"; + url = "github:input-output-hk/iohk-nix/node-11.1"; inputs.nixpkgs.follows = "nixpkgs"; }; nixpkgs.follows = "haskellNix/nixpkgs-unstable"; utils.url = "github:numtide/flake-utils"; + + # Mithril signer is required as a release artifact constitutent. + # Use explicit ref tag path to ensure we get exactly what we expect. + mithril.url = "github:input-output-hk/mithril?ref=refs/tags/2617.0"; }; outputs = { @@ -63,6 +67,7 @@ haskellNix, incl, iohkNix, + mithril, nixpkgs, self, utils, @@ -70,7 +75,7 @@ } @ input: let inherit (builtins) elem match; inherit (nixpkgs) lib; - inherit (lib) collect getAttr genAttrs filterAttrs hasPrefix head isDerivation mapAttrs optionalAttrs optionals recursiveUpdate; + inherit (lib) collect getAttr genAttrs filterAttrs hasPrefix head isDerivation mapAttrs optionalAttrs optional optionals recursiveUpdate; inherit (utils.lib) eachSystem flattenTree; inherit (iohkNix.lib) prefixNamesWith; removeRecurse = lib.filterAttrsRecursive (n: _: n != "recurseForDerivations"); @@ -367,9 +372,11 @@ inherit pkgs; inherit (exes.cardano-node.identifier) version; platform = "linux"; - exes = collect isDerivation ( - filterAttrs (n: _: elem n releaseBins) projectExes - ); + exes = + collect isDerivation ( + filterAttrs (n: _: elem n releaseBins) projectExes + ) + ++ optional (system == "x86_64-linux") mithril.packages.${system}.mithril-signer; }; internal.roots.project = muslProject.roots; variants = mapAttrs (_: v: removeAttrs v.musl ["variants"]) ciJobsVariants; diff --git a/nix/binary-release.nix b/nix/binary-release.nix index 8a5e68bb5cc..5f4a7bbb79b 100644 --- a/nix/binary-release.nix +++ b/nix/binary-release.nix @@ -41,11 +41,6 @@ let (builtins.toJSON (env.nodeConfig // genesisAttrs)); - nodeConfigLegacy= pkgs.writeText - "config-legacy.json" - (builtins.toJSON - (env.nodeConfigLegacy // genesisAttrs)); - submitApiConfig = pkgs.writeText "submit-api-config.json" (builtins.toJSON env.submitApiConfig); @@ -68,7 +63,6 @@ let '' mkdir -p "share/${name}" jq . < "${nodeConfig}" > share/${name}/config.json - jq . < "${nodeConfigLegacy}" > share/${name}/config-legacy.json jq . < "${submitApiConfig}" > share/${name}/submit-api-config.json jq . < "${tracerConfig}" > share/${name}/tracer-config.json jq . < "${peerSnapshot}" > share/${name}/peer-snapshot.json diff --git a/nix/docker/README.md b/nix/docker/README.md index 902fe87ac80..ee55d9617b2 100644 --- a/nix/docker/README.md +++ b/nix/docker/README.md @@ -120,6 +120,16 @@ docker run \ ghcr.io/intersectmbo/cardano-node:dev ``` +The resulting merged config and topology are written to a private, +per-container runtime directory under `/tmp` (see +[Read-Only Root Filesystem](#read-only-root-filesystem) for the exact path) +as `config-merged.json` / `topology-merged.json` (node) or +`tracer-config-merged.json` (tracer), and used as the runtime configuration. +Relative file references (the config's `*File` keys and the topology's +`peerSnapshotFile`) are rewritten to absolute paths anchored at +`/opt/cardano/config/$NETWORK/` so they resolve from the new location. + + ## CLI Mode To run cardano-cli, leave the `NETWORK` env variable unset and provide entrypoint args starting with `cli` followed by cardano-cli command args. @@ -149,6 +159,108 @@ respectively. This makes bind mounting easier when switching between default state directory locations, `/{data,ipc,logs}`, will work for both modes. +## Read-Only Root Filesystem +Under a normal writable root filesystem no `/tmp` mount is needed. This holds +for every mode, run as root or as a non-root user, so existing deployments need +no change. + +A writable `/tmp` must be supplied explicitly only when the root filesystem is +made read-only. The image is compatible with `--read-only` (Docker/Podman) and +`securityContext.readOnlyRootFilesystem: true` (Kubernetes), provided the +runtime supplies writable storage for the state directories described above +(`/data`, `/ipc`, `/logs`) and a writable `/tmp` (tmpfs or `emptyDir`). Under +`--read-only` this applies to every run mode except `cli`, which does not use +`/tmp`. + +For example, scripts mode with a read-only root filesystem, a per-container +tmpfs at `/tmp`, and named volumes for the state directories: +``` +docker run \ + --read-only \ + --tmpfs /tmp \ + -v mainnet-data:/data \ + -v mainnet-ipc:/ipc \ + -v mainnet-logs:/logs \ + -e NETWORK=mainnet \ + ghcr.io/intersectmbo/cardano-node:dev +``` +All runtime-generated artifacts (the merged config/topology and the env +snapshot below) are written under a private, `0700` runtime directory that +the entrypoint creates in `/tmp`, at a fixed, predictable per-role path: + +``` +/tmp/cardano-node/ # node image: config-merged.json, topology-merged.json, env +/tmp/cardano-tracer/ # tracer image: tracer-config-merged.json, env +``` + +The path is fixed so operators and tooling can refer to the effective +config/topology dependably. The entrypoint creates the directory atomically +with `mkdir -m 700` and refuses to start if the path already exists and is +not a private directory it owns, so it never follows an attacker-planted +symlink. A consequence of the fixed name is that **a `/tmp` mount must not be +shared across containers of the same role** — a second node (or tracer) +container sharing one `/tmp` would refuse to start rather than collide. Use a +per-container `/tmp`. + +A resolved-configuration snapshot is written at runtime to `env` inside that +directory and can be `source`d for an interactive debug shell inside the +container. The path `/usr/local/bin/env` is preserved as a symlink to it for +backwards compatibility, so `source /usr/local/bin/env` keeps working. This +is the supported way to locate the effective configuration: sourcing it +exports `CARDANO_CONFIG`, `CARDANO_TOPOLOGY`, the socket path, etc. as the +node was actually launched with. + +In "scripts" mode GHC RTS output is directed to `/logs/` so the image keeps +working under a read-only root. The lightweight machine-readable RTS summary +(`/logs/cardano-node.stats`, written at process exit) is always produced; the +heavier profiling/eventlog outputs are produced only when profiling or eventlog +is enabled. In "custom" mode the operator chooses the RTS flags, so any such +output must similarly be directed to a writable mount, for example: +``` +... run \ + --config /opt/cardano/config/mainnet/config.json \ + ... \ + +RTS --machine-readable -t/logs/cardano-node.stats -po/logs/cardano-node -p -RTS +``` + +The read-only, non-root and private-`/tmp` behaviors of all three images are +exercised by the `nixosTests/cardanoOciReadonly` NixOS test +(`nix build .#checks..nixosTests/cardanoOciReadonly`). + + +## Non-Root User +The image can run as any non-root user (`docker run --user ` / +Kubernetes `securityContext.runAsUser`). None of the entrypoint or +`run-node` startup logic touches image-content directories at runtime; +all generated artifacts live under `/tmp`. + +The mount-point directories (`/data`, `/ipc`, `/logs`) are owned by +GID 0 and group-writable in the image, so non-root containers can write +to freshly-created Docker or Kubernetes volumes mounted at those paths +without an init container or pre-chown. To inherit the group-writable +perm, the non-root user needs to run with primary group 0 (the Kubernetes +default for `runAsUser`) or with supplementary group 0. In Kubernetes +you can also set `securityContext.fsGroup: 0` to have the kubelet chown +the volume on mount. For Docker, `--user :0` is the equivalent. + +For example, the read-only invocation above, run as a non-root UID in +group 0: +``` +docker run \ + --read-only \ + --tmpfs /tmp \ + --user 1000:0 \ + -v mainnet-data:/data \ + -v mainnet-ipc:/ipc \ + -v mainnet-logs:/logs \ + -e NETWORK=mainnet \ + ghcr.io/intersectmbo/cardano-node:dev +``` + +The image defaults to running as root; specify a UID explicitly +to opt into a non-root run. + + ## Cardano-node Socket Sharing To share a cardano-node socket with a different container, a volume can be made for establishing cross-container communication: @@ -216,13 +328,13 @@ state types as needed, without relying on host level tooling or full chain ledger replays. An example follows to convert preprod ledger state in a named docker volume -from a memory based ledger snapshot to an LMDB snapshot when node is not +from a memory based ledger snapshot to an LSM snapshot when node is not already running: ``` docker run -v preprod-data:/data --rm -it --entrypoint=bash ghcr.io/intersectmbo/cardano-node:dev -c ' mv /data/db/ledger /data/db/ledger-old \ - && mkdir -p /data/db/ledger \ - && snapshot-converter --mem-in /data/db/ledger-old/20807240 --lmdb-out /data/db/ledger/20807240 --config /opt/cardano/config/preprod/config.json + && mkdir -p /data/db/ledger /data/db/lsm \ + && snapshot-converter --input-mem /data/db/ledger-old/20807240 --output-lsm-snapshot /data/db/ledger/20807240 --output-lsm-database /data/db/lsm --config /opt/cardano/config/preprod/config.json ' ``` @@ -232,26 +344,6 @@ otherwise ledger replay from genesis will re-occur. For more info, see the [UTxO Migration Guide](https://ouroboros-consensus.cardano.intersectmbo.org/docs/references/miscellaneous/utxo-hd/migrating/). -## Legacy Tracing System -Cardano-node now defaults to using the new tracing system. The legacy tracing -system is deprecated and will be removed in a future node version. While still -available, the legacy tracing system can be used by following the example -above in "custom" mode whereby config is passed, and in this case, the config -passed is the legacy style configuration. - -Legacy default configuration files are also available within the image at paths: -`/opt/cardano/config/$NETWORK/config-legacy.json` - -An example of legacy tracing system usage is: -``` -docker run \ - -v preprod-data:/data \ - -e CARDANO_CONFIG="/opt/cardano/config/preprod/config-legacy.json" \ - -e CARDANO_TOPOLOGY="/opt/cardano/config/preprod/topology.json" \ - ghcr.io/intersectmbo/cardano-node:dev \ - run -``` - # Cardano Submit API Image Operation ## Scripts Mode diff --git a/nix/docker/context/node/bin/entrypoint b/nix/docker/context/node/bin/entrypoint index a160f641cb0..7acf6077ce8 100755 --- a/nix/docker/context/node/bin/entrypoint +++ b/nix/docker/context/node/bin/entrypoint @@ -3,6 +3,40 @@ set -euo pipefail [[ -n ${DEBUG:-} ]] && set -x +# Every mode except "cli" (which only execs cardano-cli) writes a +# resolved-config env snapshot and any merge-mode artifacts under a private +# runtime directory in /tmp. +if [[ ${1:-} != "cli" ]]; then + # Catch the common operator mistake of running with a read-only + # filesystem without mounting a writable /tmp. + if ! [[ -w /tmp ]]; then + echo "ERROR: /tmp is not writable." >&2 + echo "With a read-only filesystem, mount a tmpfs or emptyDir at /tmp." >&2 + exit 1 + fi + + # Generated artifacts go under a private, user-owned 0700 directory at a + # fixed, predictable per-role path, so operators and tooling can refer to + # the effective config/topology dependably. The effective file paths are + # also recorded in the env snapshot below; `source /usr/local/bin/env` + # exposes them as $CARDANO_CONFIG / $CARDANO_TOPOLOGY. + # + # This approach avoids a symlink/TOCTOU vector. Consequently a second node + # container sharing one /tmp mount refuses to start with a clear error rather + # than colliding; sharing a /tmp across same-role containers is unsupported. + # CARDANO_RUNTIME_DIR is exported for the run-* scripts. + export CARDANO_RUNTIME_DIR=/tmp/cardano-node + if ! mkdir -m 700 "$CARDANO_RUNTIME_DIR" 2>/dev/null; then + if [[ -L $CARDANO_RUNTIME_DIR || ! -d $CARDANO_RUNTIME_DIR || ! -O $CARDANO_RUNTIME_DIR ]]; then + echo "ERROR: $CARDANO_RUNTIME_DIR exists and is not a private directory owned by this user." >&2 + echo "Refusing to use it to avoid following an attacker-planted path." >&2 + exit 1 + fi + # Re-assert restrictive perms in case an earlier run left them looser. + chmod 700 "$CARDANO_RUNTIME_DIR" + fi +fi + # If the NETWORK env var is set to a valid cardano network, pre-defined # configuration will be used. if [[ -n ${NETWORK:-} ]]; then @@ -30,24 +64,55 @@ if [[ -n ${NETWORK:-} ]]; then # full replacement and null values persist. # # jq -S sorts output keys alphabetically for deterministic diffs. + # + # Merged files are written to /tmp so that the image can run as a + # non-root user ($CFG is image content and only writable by root) + # and under read-only root filesystems. + # + # cardano-node resolves relative file references in the config relative + # to the config file's own directory. Since the merged config now lives + # in /tmp instead of $CFG/$NETWORK, any relative reference would resolve + # against /tmp and fail. Rewrite each relative "*File" value to an + # absolute path anchored at the original config dir. + # + # The node config schema's file references are the "*File" keys + # (ByronGenesisFile, ShelleyGenesisFile, AlonzoGenesisFile, + # ConwayGenesisFile, CheckpointsFile, ...); matching the "File" suffix + # keeps this working if more are added. if [[ -n ${CARDANO_CONFIG_JSON_MERGE:-} ]]; then jq -S \ + --arg cfgDir "$CFG/$NETWORK" \ --argjson deepMerge "$CARDANO_CONFIG_JSON_MERGE" \ - '. * $deepMerge' \ + '. * $deepMerge + | with_entries( + if ((.key | endswith("File")) + and (.value | type == "string") + and (.value | startswith("/") | not)) + then .value = "\($cfgDir)/\(.value)" + else . + end + )' \ < "$CFG/$NETWORK/config.json" \ - > "$CFG/$NETWORK/config-merged.json" - export CARDANO_CONFIG="$CFG/$NETWORK/config-merged.json" + > "$CARDANO_RUNTIME_DIR/config-merged.json" + export CARDANO_CONFIG="$CARDANO_RUNTIME_DIR/config-merged.json" else export CARDANO_CONFIG="$CFG/$NETWORK/config.json" fi + # peerSnapshotFile is the only relative file reference in the topology + # schema; rewrite it to absolute for the same reason as the config above. if [[ -n ${CARDANO_TOPOLOGY_JSON_MERGE:-} ]]; then jq -S \ + --arg cfgDir "$CFG/$NETWORK" \ --argjson deepMerge "$CARDANO_TOPOLOGY_JSON_MERGE" \ - '. * $deepMerge' \ + '. * $deepMerge + | if (.peerSnapshotFile? | type) == "string" and (.peerSnapshotFile | startswith("/") | not) + then .peerSnapshotFile = "\($cfgDir)/\(.peerSnapshotFile)" + else . + end' \ < "$CFG/$NETWORK/topology.json" \ - > "$CFG/$NETWORK/topology-merged.json" - export CARDANO_TOPOLOGY="$CFG/$NETWORK/topology-merged.json" + > "$CARDANO_RUNTIME_DIR/topology-merged.json" + export CARDANO_TOPOLOGY="$CARDANO_RUNTIME_DIR/topology-merged.json" else export CARDANO_TOPOLOGY="$CFG/$NETWORK/topology.json" fi diff --git a/nix/docker/context/node/bin/run-node b/nix/docker/context/node/bin/run-node index 1229631f801..a59ccad7721 100755 --- a/nix/docker/context/node/bin/run-node +++ b/nix/docker/context/node/bin/run-node @@ -98,7 +98,7 @@ printRunEnv () { # writeRootEnv () { -cat << EOF > /usr/local/bin/env +cat << EOF > "$CARDANO_RUNTIME_DIR/env" #!/usr/bin/env bash # Docker run ENV vars @@ -106,30 +106,30 @@ EOF if [[ -n ${CARDANO_SHELLEY_KES_AGENT_SOCKET:-} ]]; then echo "CARDANO_SHELLEY_KES_AGENT_SOCKET=\"$CARDANO_SHELLEY_KES_AGENT_SOCKET\"" \ - >> /usr/local/bin/env + >> "$CARDANO_RUNTIME_DIR/env" fi if [[ -n ${CARDANO_TRACER_SOCKET_NETWORK_ACCEPT:-} ]]; then echo "CARDANO_TRACER_SOCKET_NETWORK_ACCEPT=\"$CARDANO_TRACER_SOCKET_NETWORK_ACCEPT\"" \ - >> /usr/local/bin/env + >> "$CARDANO_RUNTIME_DIR/env" fi if [[ -n ${CARDANO_TRACER_SOCKET_NETWORK_CONNECT:-} ]]; then echo "CARDANO_TRACER_SOCKET_NETWORK_CONNECT=\"$CARDANO_TRACER_SOCKET_NETWORK_CONNECT\"" \ - >> /usr/local/bin/env + >> "$CARDANO_RUNTIME_DIR/env" fi if [[ -n ${CARDANO_TRACER_SOCKET_PATH_ACCEPT:-} ]]; then echo "CARDANO_TRACER_SOCKET_PATH_ACCEPT=\"$CARDANO_TRACER_SOCKET_PATH_ACCEPT\"" \ - >> /usr/local/bin/env + >> "$CARDANO_RUNTIME_DIR/env" fi if [[ -n ${CARDANO_TRACER_SOCKET_PATH_CONNECT:-} ]]; then echo "CARDANO_TRACER_SOCKET_PATH_CONNECT=\"$CARDANO_TRACER_SOCKET_PATH_CONNECT\"" \ - >> /usr/local/bin/env + >> "$CARDANO_RUNTIME_DIR/env" fi -cat << EOF >> /usr/local/bin/env +cat << EOF >> "$CARDANO_RUNTIME_DIR/env" CARDANO_BIND_ADDR="$CARDANO_BIND_ADDR" CARDANO_BLOCK_PRODUCER=$CARDANO_BLOCK_PRODUCER CARDANO_CONFIG="$CARDANO_CONFIG" @@ -138,20 +138,6 @@ CARDANO_LOG_DIR="$CARDANO_LOG_DIR" CARDANO_PORT=$CARDANO_PORT CARDANO_SOCKET_PATH="$CARDANO_SOCKET_PATH" CARDANO_TOPOLOGY="$CARDANO_TOPOLOGY" - -CARDANO_PUBLIC_IP="${CARDANO_PUBLIC_IP:-}" -CARDANO_CUSTOM_PEERS="${CARDANO_CUSTOM_PEERS:-}" - -# Mapping for topologyUpdater -CNODE_HOSTNAME="${CARDANO_PUBLIC_IP:-}" -CNODE_PORT=$CARDANO_PORT -CUSTOM_PEERS="${CARDANO_CUSTOM_PEERS:-}" - -# Derived from CARDANO_CONFIG to support non-mainnet deployments -GENESIS_JSON="$(dirname "$CARDANO_CONFIG")/shelley-genesis.json" - -TOPOLOGY="$CARDANO_TOPOLOGY" -LOG_DIR="$CARDANO_LOG_DIR" EOF } diff --git a/nix/docker/context/tracer/bin/entrypoint b/nix/docker/context/tracer/bin/entrypoint index d6f9ff1cfc8..b06e80d8407 100755 --- a/nix/docker/context/tracer/bin/entrypoint +++ b/nix/docker/context/tracer/bin/entrypoint @@ -3,6 +3,39 @@ set -euo pipefail [[ -n ${DEBUG:-} ]] && set -x +# The image writes a resolved-config env snapshot and any merge-mode +# artifacts under a private runtime directory in /tmp. + +# Catch the common operator mistake of running with a read-only filesystem +# without mounting a writable /tmp. +if ! [[ -w /tmp ]]; then + echo "ERROR: /tmp is not writable." >&2 + echo "With a read-only filesystem, mount a tmpfs or emptyDir at /tmp." >&2 + exit 1 +fi + +# Generated artifacts go under a private, user-owned 0700 directory at a +# fixed, predictable per-role path, so operators and tooling can refer to the +# effective config dependably. The "-tracer" suffix keeps it distinct from +# the node image's dir when both share a pod's /tmp. The effective config +# path is also recorded in the env snapshot below; `source /usr/local/bin/env` +# exposes it as $CARDANO_CONFIG. +# +# This approach avoids a symlink/TOCTOU vector. Consequently a second tracer +# container sharing one /tmp mount refuses to start with a clear error rather +# than colliding; sharing a /tmp across same-role containers is unsupported. +# CARDANO_RUNTIME_DIR is exported for the run-* scripts. +export CARDANO_RUNTIME_DIR=/tmp/cardano-tracer +if ! mkdir -m 700 "$CARDANO_RUNTIME_DIR" 2>/dev/null; then + if [[ -L $CARDANO_RUNTIME_DIR || ! -d $CARDANO_RUNTIME_DIR || ! -O $CARDANO_RUNTIME_DIR ]]; then + echo "ERROR: $CARDANO_RUNTIME_DIR exists and is not a private directory owned by this user." >&2 + echo "Refusing to use it to avoid following an attacker-planted path." >&2 + exit 1 + fi + # Re-assert restrictive perms in case an earlier run left them looser. + chmod 700 "$CARDANO_RUNTIME_DIR" +fi + # If the NETWORK env var is set to a valid cardano network, pre-defined # configuration will be used. if [[ -n ${NETWORK:-} ]]; then @@ -30,13 +63,19 @@ if [[ -n ${NETWORK:-} ]]; then # full replacement and null values persist. # # jq -S sorts output keys alphabetically for deterministic diffs. + # + # Merged files are written to /tmp so that the image can run as a + # non-root user ($CFG is image content and only writable by root) + # and under read-only root filesystems. + # The base tracer config has no relative file + # references, so no path rewriting is needed. if [[ -n ${CARDANO_CONFIG_JSON_MERGE:-} ]]; then jq -S \ --argjson deepMerge "$CARDANO_CONFIG_JSON_MERGE" \ '. * $deepMerge' \ < "$CFG/$NETWORK/tracer-config.json" \ - > "$CFG/$NETWORK/tracer-config-merged.json" - export CARDANO_CONFIG="$CFG/$NETWORK/tracer-config-merged.json" + > "$CARDANO_RUNTIME_DIR/tracer-config-merged.json" + export CARDANO_CONFIG="$CARDANO_RUNTIME_DIR/tracer-config-merged.json" else export CARDANO_CONFIG="$CFG/$NETWORK/tracer-config.json" fi diff --git a/nix/docker/context/tracer/bin/run-tracer b/nix/docker/context/tracer/bin/run-tracer index 3c0bf3f48f7..f62c3cbdb31 100755 --- a/nix/docker/context/tracer/bin/run-tracer +++ b/nix/docker/context/tracer/bin/run-tracer @@ -25,7 +25,10 @@ printRunEnv () { echo "CARDANO_CONFIG=$CARDANO_CONFIG" echo "CARDANO_STATE_DIR=$CARDANO_STATE_DIR" - [[ -n ${CARDANO_MIN_LOG_SEVERITY:-} ]] && echo "CARDANO_MIN_LOG_SEVERITY=$CARDANO_MIN_LOG_SEVERITY" + + if [[ -n ${CARDANO_MIN_LOG_SEVERITY:-} ]]; then + echo "CARDANO_MIN_LOG_SEVERITY=$CARDANO_MIN_LOG_SEVERITY" + fi } ##################################################################### @@ -34,7 +37,7 @@ printRunEnv () { # writeRootEnv () { -cat << EOF > /usr/local/bin/env +cat << EOF > "$CARDANO_RUNTIME_DIR/env" #!/usr/bin/env bash # Docker run ENV vars @@ -44,7 +47,7 @@ EOF if [[ -n ${CARDANO_MIN_LOG_SEVERITY:-} ]]; then echo "CARDANO_MIN_LOG_SEVERITY=\"$CARDANO_MIN_LOG_SEVERITY\"" \ - >> /usr/local/bin/env + >> "$CARDANO_RUNTIME_DIR/env" fi } diff --git a/nix/docker/default.nix b/nix/docker/default.nix index 3805b4f85ce..f95f4e2b1cf 100644 --- a/nix/docker/default.nix +++ b/nix/docker/default.nix @@ -118,7 +118,7 @@ let done # Adjust genesis file, config refs - for i in config config-legacy db-sync-config; do + for i in config db-sync-config; do if [ -f "$out/config/$ENV/$i.json" ]; then sed -i "s|\"$ENV-|\"|g" "$out/config/$ENV/$i.json" fi @@ -152,6 +152,15 @@ in # Similarly, make a root level dir for logs: mkdir -p logs + # Make the mount-point directories group-writable. Group is already + # 0 (the build env writes files as 0:0). When a fresh Docker volume + # is first mounted at one of these paths, the perms propagate from + # the image, so non-root containers (running as a UID in group 0 — + # the K8s default for runAsUser — or with explicit fsGroup) can + # write to a freshly-created volume without an init container or + # pre-chown. + chmod g+w data ipc logs + # The "custom" operation mode of this image, when the NETWORK env is # unset and "run" is provided as an entrypoint arg, will use the # following default directories. To reduce confusion caused by default @@ -176,6 +185,12 @@ in ln -sv ${snapshot-converter}/bin/snapshot-converter usr/local/bin/snapshot-converter ln -sv ${jq}/bin/jq usr/local/bin/jq + # Backwards-compatible alias for the resolved-config env snapshot + # written by run-node at the fixed per-role path /tmp/cardano-node/env, + # so `source /usr/local/bin/env` keeps working while the image stays + # compatible with a read-only root filesystem. + ln -sv /tmp/cardano-node/env usr/local/bin/env + # Create iohk-nix network configs, organized by network directory. SRC="${genCfgs}" DST="opt/cardano" diff --git a/nix/docker/tracer.nix b/nix/docker/tracer.nix index db18fb0731f..42ce15c40b6 100644 --- a/nix/docker/tracer.nix +++ b/nix/docker/tracer.nix @@ -130,11 +130,21 @@ in # The "scripts" operation mode of this image, when the NETWORK env var is # set to a valid network, will use the following default directories # mounted at /: + mkdir -p data mkdir -p ipc # Similarly, make a root level dir for logs: mkdir -p logs + # Make the mount-point directories group-writable. Group is already + # 0 (the build env writes files as 0:0). When a fresh Docker volume + # is first mounted at one of these paths, the perms propagate from + # the image, so non-root containers (running as a UID in group 0 — + # the K8s default for runAsUser — or with explicit fsGroup) can + # write to a freshly-created volume without an init container or + # pre-chown. + chmod g+w data ipc logs + # The "custom" operation mode of this image, when the NETWORK env is # unset and "run" is provided as an entrypoint arg, will use the # following default directories. To reduce confusion caused by default @@ -143,6 +153,7 @@ in # permit use of volume mounts at the root directory location regardless # of which mode the image is operating in. mkdir -p opt/cardano + ln -sv /data opt/cardano/data ln -sv /ipc opt/cardano/ipc ln -sv /logs opt/cardano/logs @@ -153,6 +164,12 @@ in ln -sv ${cardano-tracer}/bin/cardano-tracer usr/local/bin/cardano-tracer ln -sv ${jq}/bin/jq usr/local/bin/jq + # Backwards-compatible alias for the resolved-config env snapshot + # written by run-tracer at the fixed per-role path /tmp/cardano-tracer/env, + # so `source /usr/local/bin/env` keeps working while the image stays + # compatible with a read-only root filesystem. + ln -sv /tmp/cardano-tracer/env usr/local/bin/env + # Create iohk-nix network configs, organized by network directory. SRC="${genCfgs}" DST="opt/cardano" diff --git a/nix/haskell.nix b/nix/haskell.nix index d086e60638f..13ba8ff8e2f 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -54,7 +54,6 @@ let # These programs will be available inside the nix-shell. nativeBuildInputs = with pkgs.pkgsBuildBuild; [ alejandra - lmdb nix-prefetch-git pkg-config git @@ -216,7 +215,6 @@ let mainnetConfigFiles = [ "configuration/cardano/mainnet-config.yaml" "configuration/cardano/mainnet-config.json" - "configuration/cardano/mainnet-config-legacy.json" "configuration/cardano/mainnet-byron-genesis.json" "configuration/cardano/mainnet-shelley-genesis.json" "configuration/cardano/mainnet-alonzo-genesis.json" diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix index 33860e28d54..48f0e5a8307 100644 --- a/nix/nixos/cardano-node-service.nix +++ b/nix/nixos/cardano-node-service.nix @@ -5,7 +5,7 @@ with lib; with builtins; let - inherit (types) attrs attrsOf bool either enum functionTo int listOf package nullOr str; + inherit (types) attrs attrsOf bool either enum functionTo int listOf package path nullOr str; cfg = config.services.cardano-node; envConfig = cfg.environments.${cfg.environment}; @@ -32,16 +32,6 @@ let peerSnapshotFile = cfg.peerSnapshotFile i; }; - oldTopology = i: { - Producers = concatMap (g: map (a: { - addr = a.address; - inherit (a) port; - valency = a.valency or 1; - }) g.accessPoints) ( - cfg.producers ++ (cfg.instanceProducers i) ++ cfg.publicProducers ++ (cfg.instancePublicProducers i) - ); - }; - assertNewTopology = i: let checkEval = tryEval ( @@ -58,7 +48,7 @@ let selectTopology = i: if cfg.topology != null then cfg.topology - else toFile "topology.json" (toJSON (if (cfg.useNewTopology != false) then assertNewTopology i else oldTopology i)); + else toFile "topology.json" (toJSON (assertNewTopology i)); topology = i: if cfg.useSystemdReload @@ -72,43 +62,27 @@ let // (mapAttrs' (era: epoch: nameValuePair "Test${era}HardForkAtEpoch" epoch ) cfg.forceHardForks) - // (optionalAttrs (cfg.useNewTopology != false) ( - { - MaxConcurrencyBulkSync = 2; - } // optionalAttrs (cfg.useNewTopology == true) { - # Starting with node 10.6.0, p2p is the only network - # operating mode and EnableP2P becomes a no-op and is not - # declared by default. - # - # Older node versions which still require an explicit - # declaration can set useNewTopology true. - EnableP2P = true; - } // optionalAttrs (cfg.targetNumberOfRootPeers != null) { - TargetNumberOfRootPeers = cfg.targetNumberOfRootPeers; - } // optionalAttrs (cfg.targetNumberOfKnownPeers != null) { - TargetNumberOfKnownPeers = cfg.targetNumberOfKnownPeers; - } // optionalAttrs (cfg.targetNumberOfEstablishedPeers != null) { - TargetNumberOfEstablishedPeers = cfg.targetNumberOfEstablishedPeers; - } // optionalAttrs (cfg.targetNumberOfActivePeers != null) { - TargetNumberOfActivePeers = cfg.targetNumberOfActivePeers; - }) - ) + // { + MaxConcurrencyBulkSync = 2; + } // optionalAttrs (cfg.targetNumberOfRootPeers != null) { + TargetNumberOfRootPeers = cfg.targetNumberOfRootPeers; + } // optionalAttrs (cfg.targetNumberOfKnownPeers != null) { + TargetNumberOfKnownPeers = cfg.targetNumberOfKnownPeers; + } // optionalAttrs (cfg.targetNumberOfEstablishedPeers != null) { + TargetNumberOfEstablishedPeers = cfg.targetNumberOfEstablishedPeers; + } // optionalAttrs (cfg.targetNumberOfActivePeers != null) { + TargetNumberOfActivePeers = cfg.targetNumberOfActivePeers; + } ) cfg.extraNodeConfig; baseInstanceConfig = i: - baseConfig - // optionalAttrs (cfg.withUtxoHdLsmt i){ - LedgerDB = { - Backend = "V2LSM"; - LSMDatabasePath = cfg.lsmDatabasePath i; - }; - } - // optionalAttrs (cfg.withUtxoHdLmdb i){ - LedgerDB = { - Backend = "V1LMDB"; - LiveTablesPath = cfg.lmdbDatabasePath i; - }; + recursiveUpdate + baseConfig (optionalAttrs (cfg.withUtxoHdLsmt i) { + LedgerDB = { + Backend = "V2LSM"; + LSMDatabasePath = cfg.lsmDatabasePath i; }; + }); in i: let instanceConfig = recursiveUpdate (baseInstanceConfig i) (cfg.extraNodeInstanceConfig i); nodeConfigFile = if (cfg.nodeConfigFile != null) then cfg.nodeConfigFile @@ -228,6 +202,12 @@ in { description = '' Haskell profiling types which are available and will be applied to the cardano-node binary if declared. + + Note: the default `profilingArgs` always include the lightweight + `--machine-readable -t...cardano-node.stats` RTS summary (written at + exit, useful in any build). The cost-centre/heap profiling flags and + the `-po` output stem are only added when this is not "none" or + `eventlog` is enabled. ''; }; @@ -413,16 +393,6 @@ in { description = ''The node database path, for each instance.''; }; - lmdbDatabasePath = mkOption { - type = funcToOr nullOrStr; - default = null; - apply = x : if lib.isFunction x then x else if x == null then _: null else _: x; - description = '' - A node UTxO-HD on-disk LMDB path for performant disk I/O, for each instance. - This could point to a direct-access SSD, with a specifically created journal-less file system and optimized mount options. - ''; - }; - lsmDatabasePath = mkOption { type = funcToOr nullOrStr; default = null; @@ -635,31 +605,6 @@ in { ''; }; - useNewTopology = mkOption { - type = nullOr bool; - default = cfg.nodeConfig.EnableP2P or null; - description = '' - Use new, p2p and ledger peers compatible topology. - - The useNewTopology option is deprecated and will be removed in the - future. As of cardano-node 10.6.0, this option should remain null. - For older node versions, a bool value can be set, but this will only - be supported until the Dijkstra hard fork at which point all - cardano-node versions will be compelled to upgrade and the - useNewTopology option will be removed. - - For node version < 10.6.0, useNewTopology will need to be explicitly - declared true or false to behave accordingly. If left null while - also using the auto-generated p2p topology, node will fail to start. - - For node version >= 10.6.0, useNewTopology should be left as null - until the option is removed after the Dijkstra hard fork. If - explicitly declared true, node will continue to work, but if declared - false while using the auto-generated legacy topology, node will fail to - start. - ''; - }; - useLegacyTracing = mkOption { type = bool; default = false; @@ -802,16 +747,6 @@ in { ''; }; - withUtxoHdLmdb = mkOption { - type = funcToOr bool; - default = false; - apply = x: if lib.isFunction x then x else _: x; - description = '' - On a UTxO-HD enabled node, the in-memory backend is the default. - This activates the on-disk backend (LMDB) instead. - ''; - }; - withUtxoHdLsmt = mkOption { type = funcToOr bool; default = false; @@ -843,13 +778,30 @@ in { description = ''Extra CLI args for cardano-node, to be surrounded by "+RTS"/"-RTS"''; }; + profilingOutputDir = mkOption { + type = nullOrStr; + default = null; + description = '' + Optional directory prefix for GHC RTS profiling output files + (cardano-node.stats, cardano-node.prof, cardano-node.hp, etc.). + When null, files are written relative to the working directory + (the systemd unit's WorkingDirectory for NixOS deployments, which + is cfg.stateDir). + ''; + }; + profilingArgs = mkOption { type = listOf str; - default = - [ "--machine-readable" - "-tcardano-node.stats" - "-pocardano-node" - ] + default = let + prefix = if cfg.profilingOutputDir == null then "" else "${cfg.profilingOutputDir}/"; + in + # Always emit the lightweight machine-readable RTS/GC summary at + # exit. It works in any build, costs nothing, and is useful + # telemetry. The OCI images use the profilingOutputDir option to + # ensure it lands on a writable mount under a read-only-root OCI + # image. + [ "--machine-readable" "-t${prefix}cardano-node.stats" ] + ++ optionals (cfg.profiling != "none" || cfg.eventlog) [ "-po${prefix}cardano-node" ] ++ optional (cfg.eventlog) "-l" ++ ( if cfg.profiling == "time" then ["-p"] @@ -877,12 +829,9 @@ in { # # Mainnet does not yet require it, but declaring it will also # facilitate testing. - if (cfg.useNewTopology != false) - then - if cfg.useSystemdReload - then "peer-snapshot-${toString i}.json" - else toFile "peer-snapshot.json" (toJSON (envConfig.peerSnapshot)) - else null; + if cfg.useSystemdReload + then "peer-snapshot-${toString i}.json" + else toFile "peer-snapshot.json" (toJSON (envConfig.peerSnapshot)); example = i: "/etc/cardano-node/peer-snapshot-${toString i}.json"; apply = x: if lib.isFunction x then x else _: x; description = '' @@ -906,7 +855,6 @@ in { }; config = mkIf cfg.enable ( let - lmdbPaths = filter (x: x != null) (map (e: cfg.lmdbDatabasePath e) (genList trivial.id cfg.instances)); lsmPaths = filter (x: x != null) (map (e: cfg.lsmDatabasePath e) (genList trivial.id cfg.instances)); genInstanceConf = f: listToAttrs (if cfg.instances > 1 then genList (i: let n = "cardano-node-${toString i}"; in nameValuePair n (f n i)) cfg.instances @@ -926,7 +874,7 @@ in { (acc: i: recursiveUpdate acc {"cardano-node/topology-${toString i}.json".source = selectTopology i;}) {} (range 0 (cfg.instances - 1))) ) - (mkIf ((cfg.useNewTopology != false) && cfg.useSystemdReload) + (mkIf cfg.useSystemdReload (foldl' (acc: i: recursiveUpdate acc ( optionalAttrs (cfg.peerSnapshotFile i != null) { @@ -950,12 +898,12 @@ in { wants = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; partOf = mkIf (cfg.instances > 1) ["cardano-node.service"]; - reloadTriggers = mkIf (cfg.useSystemdReload && (cfg.useNewTopology != false)) [ (selectTopology i) ]; + reloadTriggers = mkIf cfg.useSystemdReload [ (selectTopology i) ]; script = mkScript cfg i; serviceConfig = { User = "cardano-node"; Group = "cardano-node"; - ExecReload = mkIf (cfg.useSystemdReload && (cfg.useNewTopology != false)) "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; + ExecReload = mkIf cfg.useSystemdReload "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; Restart = "always"; RuntimeDirectory = mkIf (!cfg.systemdSocketActivation) (removePrefix cfg.runDirBase (runtimeDir i)); @@ -1033,21 +981,13 @@ in { ''; } { - assertion = !(cfg.systemdSocketActivation && (cfg.useNewTopology != false)); + assertion = !cfg.systemdSocketActivation; message = "Systemd socket activation cannot be used with p2p topology due to a systemd socket re-use issue."; } - { - assertion = (length lmdbPaths) == (length (lists.unique lmdbPaths)); - message = "When configuring multiple LMDB enabled nodes on one instance, lmdbDatabasePath must be unique."; - } { assertion = (length lsmPaths) == (length (lists.unique lsmPaths)); message = "When configuring multiple LSM enabled nodes on one instance, lsmDatabasePath must be unique."; } - { - assertion = all (i: !(cfg.withUtxoHdLmdb i && cfg.withUtxoHdLsmt i)) (genList trivial.id cfg.instances); - message = "Each instance can only declare either withUtxoHdLmdb or withUtxoHdLsmt"; - } { assertion = count (o: o != null) (with cfg; [ (tracerSocketPathAccept i) @@ -1058,13 +998,6 @@ in { message = "Only one option of services.cardano-node.tracerSocket(PathAccept|PathConnect|NetworkAccept|NetworkConnect) can be declared."; } ]; - - warnings = optional (cfg.useNewTopology != null) '' - The useNewTopology option is deprecated and will be removed in the future. As of cardano-node 10.6.0, this option should remain null. - For older node versions, a bool value can be set, but this will only be supported until the Dijkstra hard fork at which point all - cardano-node versions will be compelled to upgrade and the useNewTopology option will be removed. See the services.cardano-node.useNewTopology - option description for further details. - ''; } ]); } diff --git a/nix/nixos/cardano-submit-api-service.nix b/nix/nixos/cardano-submit-api-service.nix index fcebf68ca94..36c3bb83456 100644 --- a/nix/nixos/cardano-submit-api-service.nix +++ b/nix/nixos/cardano-submit-api-service.nix @@ -1,88 +1,138 @@ -{ config, lib, pkgs, ... }: +# This service exposes an http port, and connects to a cardano-node over a UNIX socket +{ + config, + lib, + pkgs, + ... +}: let + inherit (builtins) fromJSON readFile; + inherit (cfg.cardanoNodePackages) cardanoLib; -# notes: -# this service exposes an http port, and connects to a cardano-node over a UNIX socket -let cfg = config.services.cardano-submit-api; - inherit (cfg.cardanoNodePackages) cardanoLib; - envConfig = cfg.environment; in { options = { services.cardano-submit-api = { - enable = lib.mkEnableOption "enable the cardano-submit-api api"; - script = lib.mkOption { - internal = true; - type = lib.types.package; - }; - package = lib.mkOption { - type = lib.types.package; - default = cfg.cardanoNodePackages.cardano-submit-api; + enable = lib.mkEnableOption "Enable the cardano-submit-api api"; + + cardanoNodePackages = lib.mkOption { + type = lib.types.attrs; + default = pkgs.cardanoNodePackages or (import ../. {}).cardanoNodePackages; + defaultText = "cardano-node packages"; + description = '' + The cardano-node packages and library that should be used. + Main usage is sharing optimization to reduce eval time when services + are instantiated multiple times. + ''; }; - port = lib.mkOption { - type = lib.types.port; - default = 8090; + + config = lib.mkOption { + type = lib.types.nullOr lib.types.attrs; + default = cardanoLib.defaultSubmitApiConfig; + description = "Tracing configuration passed to submit-api."; }; - listenAddress = lib.mkOption { - type = lib.types.str; - default = "127.0.0.1"; + + environment = lib.mkOption { + type = lib.types.nullOr lib.types.attrs; + default = cfg.cardanoNodePackages.cardanoLib.environments.${cfg.network}; + description = "Cardano environment attrset for the selected network."; }; - socketPath = lib.mkOption { - type = lib.types.nullOr lib.types.path; + + group = lib.mkOption { + type = lib.types.nullOr lib.types.str; default = null; description = '' - cardano node socket path. If set, the entrypoint - takes this value over CARDANO_NODE_SOCKET_PATH env - variable. + Optional supplementary group added to the service's dynamic user, + typically the cardano-node socket group, so cardano-submit-api can + access the node socket at CARDANO_NODE_SOCKET_PATH. ''; }; - config = lib.mkOption { - type = lib.types.nullOr lib.types.attrs; - default = cardanoLib.defaultExplorerLogConfig; + + listenAddress = lib.mkOption { + type = lib.types.str; + default = "127.0.0.1"; + description = "Host address submit-api binds to."; }; + network = lib.mkOption { type = lib.types.nullOr lib.types.str; - description = "network name"; + description = "Network name."; default = null; }; - environment = lib.mkOption { - type = lib.types.nullOr lib.types.attrs; - default = cfg.cardanoNodePackages.cardanoLib.environments.${cfg.network}; + + package = lib.mkOption { + type = lib.types.package; + default = cfg.cardanoNodePackages.cardano-submit-api; + description = "The cardano-submit-api package to run."; }; - cardanoNodePackages = lib.mkOption { - type = lib.types.attrs; - default = pkgs.cardanoNodePackages or (import ../. {}).cardanoNodePackages; - defaultText = "cardano-node packages"; + + port = lib.mkOption { + type = lib.types.port; + default = 8090; + description = "HTTP port submit-api listens on."; + }; + + script = lib.mkOption { + internal = true; + type = lib.types.package; + description = "Generated cardano-submit-api launch script (internal)."; + }; + + socketPath = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; description = '' - The cardano-node packages and library that should be used. - Main usage is sharing optimization: - reduce eval time when service is instantiated multiple times. + The cardano node socket path. If set, the entrypoint takes this value + over CARDANO_NODE_SOCKET_PATH env variable. ''; }; }; }; config = let envNodeCfg = cfg.environment.nodeConfig; - shelleyGenesisParams = __fromJSON (__readFile envNodeCfg.ShelleyGenesisFile); - envFlag = if cfg.network == "mainnet" then "--mainnet" else "--testnet-magic ${toString shelleyGenesisParams.networkMagic}"; - in lib.mkIf cfg.enable { - services.cardano-submit-api.script = pkgs.writeShellScript "cardano-submit-api" '' - ${if (cfg.socketPath == null) then ''if [ -z "$CARDANO_NODE_SOCKET_PATH" ] - then - echo "You must set \$CARDANO_NODE_SOCKET_PATH" - exit 1 - fi'' else "export \"CARDANO_NODE_SOCKET_PATH=${cfg.socketPath}\""} - exec ${cfg.package}/bin/cardano-submit-api --socket-path "$CARDANO_NODE_SOCKET_PATH" ${envFlag} \ - --port ${toString cfg.port} \ - --listen-address ${cfg.listenAddress} \ - --config ${builtins.toFile "submit-api.json" (builtins.toJSON cfg.config)} - ''; - systemd.services.cardano-submit-api = { - serviceConfig = { - ExecStart = config.services.cardano-submit-api.script; - DynamicUser = true; + shelleyGenesisParams = fromJSON (readFile envNodeCfg.ShelleyGenesisFile); + envFlag = + if cfg.network == "mainnet" + then "--mainnet" + else "--testnet-magic ${toString shelleyGenesisParams.networkMagic}"; + in + lib.mkIf cfg.enable { + services.cardano-submit-api.script = pkgs.writeShellScript "cardano-submit-api" '' + ${ + if (cfg.socketPath == null) + then '' if [ -z "$CARDANO_NODE_SOCKET_PATH" ] + then + echo "You must set \$CARDANO_NODE_SOCKET_PATH" + exit 1 + fi'' + else "export \"CARDANO_NODE_SOCKET_PATH=${cfg.socketPath}\"" + } + exec ${cfg.package}/bin/cardano-submit-api --socket-path "$CARDANO_NODE_SOCKET_PATH" ${envFlag} \ + --port ${toString cfg.port} \ + --listen-address ${cfg.listenAddress} \ + --config ${builtins.toFile "submit-api.json" (builtins.toJSON cfg.config)} + ''; + systemd.services.cardano-submit-api = { + serviceConfig = + { + ExecStart = config.services.cardano-submit-api.script; + DynamicUser = true; + + # The api connects to the node over a UNIX socket that only becomes + # available once the node has started; `after` orders startup but does + # not wait for the socket. Default to restarting until it is reachable + # rather than failing permanently on a fresh boot. These are mkDefault + # so a consumer can impose a bounded restart + start-limit policy that + # lets persistent failures surface as a failed unit for alerting. + Restart = lib.mkDefault "always"; + RestartSec = lib.mkDefault 1; + } + // lib.optionalAttrs (cfg.group != null) { + # A DynamicUser is not otherwise a member of the node socket group, so + # without this it cannot open CARDANO_NODE_SOCKET_PATH. + SupplementaryGroups = [cfg.group]; + }; + wantedBy = ["multi-user.target"]; + after = ["cardano-node.service"]; }; - wantedBy = [ "multi-user.target" ]; - after = [ "cardano-node.service" ]; }; - }; } diff --git a/nix/nixos/cardano-tracer-service.nix b/nix/nixos/cardano-tracer-service.nix index 222c3eb627a..c2b2d42685c 100644 --- a/nix/nixos/cardano-tracer-service.nix +++ b/nix/nixos/cardano-tracer-service.nix @@ -492,16 +492,39 @@ in { description = '' Haskell profiling types which are available and will be applied to the cardano-tracer binary if declared. + + Note: the default `profilingArgs` always include the lightweight + `--machine-readable -t...cardano-tracer.stats` RTS summary (written + at exit, useful in any build). The cost-centre/heap profiling flags + and the `-po` output stem are only added when this is not "none" or + `eventlog` is enabled. + ''; + }; + + profilingOutputDir = mkOption { + type = nullOr str; + default = null; + description = '' + Optional directory prefix for GHC RTS profiling output files + (cardano-tracer.stats, cardano-tracer.prof, cardano-tracer.hp, etc.). + When null, files are written relative to the working directory + (the systemd unit's WorkingDirectory for NixOS deployments, which + is cfg.stateDir). ''; }; profilingArgs = mkOption { type = listOf str; - default = - [ "--machine-readable" - "-tcardano-node.stats" - "-pocardano-node" - ] + default = let + prefix = if cfg.profilingOutputDir == null then "" else "${cfg.profilingOutputDir}/"; + in + # Always emit the lightweight machine-readable RTS/GC summary at + # exit. It works in any build, costs nothing, and is useful + # telemetry. The OCI images use the profilingOutputDir option to + # ensure it lands on a writable mount under a read-only-root OCI + # image. + [ "--machine-readable" "-t${prefix}cardano-tracer.stats" ] + ++ optionals (cfg.profiling != "none" || cfg.eventlog) [ "-po${prefix}cardano-tracer" ] ++ optional (cfg.eventlog) "-l" ++ ( if cfg.profiling == "time" then ["-p"] diff --git a/nix/nixos/tests/cardano-oci-readonly.nix b/nix/nixos/tests/cardano-oci-readonly.nix new file mode 100644 index 00000000000..e01f495012c --- /dev/null +++ b/nix/nixos/tests/cardano-oci-readonly.nix @@ -0,0 +1,146 @@ +{ + pkgs, + dockerImage, + tracerDockerImage, + submitApiDockerImage, + ... +}: let + inherit (pkgs) lib; + + # Image refs as `docker load` will tag them (repoName:gitrev). + imageRef = "${dockerImage.imageName}:${dockerImage.imageTag}"; + tracerImageRef = "${tracerDockerImage.imageName}:${tracerDockerImage.imageTag}"; + submitApiImageRef = "${submitApiDockerImage.imageName}:${submitApiDockerImage.imageTag}"; + + network = "preview"; + + # Harmless merge keys, present only to exercise merge mode which is what + # triggers the /tmp runtime dir + relative-path rewriting. The double quotes + # are backslash-escaped because these are interpolated into double-quoted + # Python string literals in the testScript below. + configMerge = ''{\"MaxConcurrencyBulkSync\":2}''; + topologyMerge = ''{\"useLedgerAfterSlot\":1}''; + tracerConfigMerge = ''{\"verbosity\":\"Minimum\"}''; + + rt = "/tmp/cardano-node"; + tracerRt = "/tmp/cardano-tracer"; +in { + name = "cardano-oci-readonly-test"; + + nodes = { + machine = {...}: { + nixpkgs.pkgs = pkgs; + + # Room for loading three images (node, tracer, submit-api) + overlay + + # a starting node/tracer. + virtualisation.diskSize = 10240; + virtualisation.memorySize = 4096; + virtualisation.docker.enable = true; + }; + }; + + # Like the other tests here, this is sandboxed: the node cannot sync, but it + # starts and resolves its configuration, which is all these assertions need. + testScript = '' + start_all() + machine.wait_for_unit("multi-user.target") + machine.wait_until_succeeds("docker info", timeout=120) + machine.succeed("docker load -i ${dockerImage}") + + # Read-only root + non-root UID in group 0 + merge mode. Asserts the + # container starts as read-only and group-0 volume writes work, creates its + # fixed 0700 runtime dir, writes the merged config with absolute genesis + # paths, writes the env snapshot, and the /usr/local/bin/env alias sources + # it. + machine.succeed( + "docker run -d --name n1 --read-only --tmpfs /tmp --user 1000:0 " + "-v n1data:/data -v n1ipc:/ipc -v n1logs:/logs " + "-e NETWORK=${network} " + "-e CARDANO_CONFIG_JSON_MERGE='${configMerge}' " + "-e CARDANO_TOPOLOGY_JSON_MERGE='${topologyMerge}' " + "${imageRef}" + ) + + # The runtime artifacts are written by the entrypoint/run-node before the + # node execs, so they appear shortly after start. Waiting on the env + # snapshot also confirms the container did not crash under --read-only and + # --user since `docker exec` fails against an exited container. + machine.wait_until_succeeds("docker exec n1 test -f ${rt}/env", timeout=60) + machine.succeed("[ \"$(docker inspect -f '{{.State.Running}}' n1)\" = true ]") + + machine.succeed("docker exec n1 test -d ${rt}") + machine.succeed("docker exec n1 sh -c '[ \"$(stat -c %a ${rt})\" = 700 ]'") + machine.succeed("docker exec n1 test -f ${rt}/config-merged.json") + machine.succeed("docker exec n1 test -f ${rt}/topology-merged.json") + + # The merged config's relative "*File" references (ByronGenesisFile, + # ShelleyGenesisFile, ..., CheckpointsFile) must be rewritten to absolute + # paths so they resolve from /tmp. Mirror the entrypoint's predicate -- any + # top-level key ending in "File" with a string value -- and assert all are + # absolute. Use jq from the container. + machine.succeed( + "docker exec n1 jq -e " + "'[to_entries[]|select(.key|endswith(\"File\"))|.value|select(type==\"string\")] as $v " + "| ($v|length>0) and ($v|all(startswith(\"/\")))' " + "${rt}/config-merged.json" + ) + + # The merged topology's relative peerSnapshotFile must likewise be rewritten + # to an absolute path. + machine.succeed( + "docker exec n1 jq -e " + "'(.peerSnapshotFile|type==\"string\") and (.peerSnapshotFile|startswith(\"/\"))' " + "${rt}/topology-merged.json" + ) + + # The env snapshot is POSIX-sourceable; use sh and `.` to load it and + # confirm it populated the env. + machine.succeed( + "docker exec --user 1000:0 n1 sh -c " + "'. /usr/local/bin/env && [ -n \"$CARDANO_CONFIG\" ]'" + ) + machine.succeed("docker rm -f n1") + + # Read-only root WITHOUT a writable /tmp must fail fast with guidance. + status, out = machine.execute( + "docker run --rm --read-only -e NETWORK=${network} ${imageRef} 2>&1" + ) + assert status != 0, "expected non-zero exit when /tmp is not writable" + assert "/tmp is not writable" in out, f"missing actionable /tmp error; got: {out}" + + # Cli mode must NOT require a writable /tmp (read-only, no tmpfs). + machine.succeed("docker run --rm --read-only ${imageRef} cli version") + + # Tracer image: same read-only + non-root + merge-mode behavior as the + # node, minus the genesis/topology rewrites as tracer config has no + # relative file references. + machine.succeed("docker load -i ${tracerDockerImage}") + machine.succeed( + "docker run -d --name t1 --read-only --tmpfs /tmp --user 1000:0 " + "-v t1data:/data -v t1ipc:/ipc -v t1logs:/logs " + "-e NETWORK=${network} " + "-e CARDANO_CONFIG_JSON_MERGE='${tracerConfigMerge}' " + "${tracerImageRef}" + ) + # Give the entrypoint time to write artifacts and exec the tracer. If the + # container exits early, surface its logs rather than failing later with an + # opaque `docker exec` timeout against a dead container. + machine.sleep(10) + if machine.succeed("docker inspect -f '{{.State.Running}}' t1").strip() != "true": + _, logs = machine.execute("docker logs t1 2>&1") + raise Exception("tracer container exited early; docker logs:\n" + logs) + machine.succeed("docker exec t1 sh -c '[ \"$(stat -c %a ${tracerRt})\" = 700 ]'") + machine.succeed("docker exec t1 test -f ${tracerRt}/tracer-config-merged.json") + machine.succeed("docker exec t1 test -f ${tracerRt}/env") + machine.succeed( + "docker exec --user 1000:0 t1 sh -c " + "'. /usr/local/bin/env && [ -n \"$CARDANO_CONFIG\" ]'" + ) + machine.succeed("docker rm -f t1") + + # Submit-api image: stateless (no env snapshot / merge / state writes), + # so it only needs to execute under --read-only and as non-root. + machine.succeed("docker load -i ${submitApiDockerImage}") + machine.succeed("docker run --rm --read-only --user 1000:0 ${submitApiImageRef} --help >/dev/null") + ''; +} diff --git a/nix/nixos/tests/default.nix b/nix/nixos/tests/default.nix index c058a7d00e3..1ffa7813506 100644 --- a/nix/nixos/tests/default.nix +++ b/nix/nixos/tests/default.nix @@ -30,4 +30,14 @@ in { # Tests a mainnet edge node with submit-api using nixos service config. cardanoNodeEdge = callTest ./cardano-node-edge.nix {}; + + # Tests the OCI images (node, tracer, submit-api) under a read-only root, a + # non-root UID in group 0, and a private /tmp: container startup, the fixed + # 0700 runtime dir, merged-config path rewriting, the env snapshot/alias, the + # fail-fast on a missing writable /tmp, and that cli mode does not require /tmp. + cardanoOciReadonly = callTest ./cardano-oci-readonly.nix { + dockerImage = pkgs.dockerImage; + tracerDockerImage = pkgs.tracerDockerImage; + submitApiDockerImage = pkgs.submitApiDockerImage; + }; } diff --git a/nix/pkgs.nix b/nix/pkgs.nix index fa4fe6fab0e..be5d24bdb04 100644 --- a/nix/pkgs.nix +++ b/nix/pkgs.nix @@ -108,6 +108,15 @@ in with final; stateDir = "/data"; dbPrefix = "db"; socketPath = "/ipc/node.socket"; + # Direct GHC RTS output to /logs (a writable mount) so it stays off + # the container's read-only root. This matters even with + # profiling = "none": the lightweight `-t...cardano-node.stats` + # summary is emitted on every run, and without a prefix it would be + # written to the container's cwd (/), which fails under --read-only. + # Profiling/heap/eventlog output (when enabled) lands here too. + # Scoped to the image here rather than scripts.nix so bare + # `nix run .#/node` is unaffected. + profilingOutputDir = "/logs"; }; in callPackage ./docker { @@ -147,6 +156,13 @@ in with final; logFormat = "ForHuman"; } ]; + # As with the node image: direct GHC RTS output to /logs (a writable + # mount) so it stays off the read-only root. This matters even with + # profiling = "none" -- the always-on `-t...cardano-tracer.stats` + # summary would otherwise be written to the container cwd (/) and + # fail under --read-only. Profiling/heap/eventlog output lands here + # too when enabled. + profilingOutputDir = "/logs"; }; in callPackage ./docker/tracer.nix { diff --git a/nix/workbench/backend/nomad/cloud.sh b/nix/workbench/backend/nomad/cloud.sh index 38e1c4decb9..6fb63024d4e 100644 --- a/nix/workbench/backend/nomad/cloud.sh +++ b/nix/workbench/backend/nomad/cloud.sh @@ -701,13 +701,13 @@ allocate-run-nomadcloud() { read -p "Hit enter to continue ..." fi fi - # Clean, only producers, the "host_volumes" if being used for LMDB/LSMT. + # Clean, only producers, the "host_volumes" if being used for LSMT. # We do this for each producer instead of for all producer at once because # even if modules have the same name from a Nomad perspective, in each # client the real path is defined in Nomad's config file and may differ! # It's "slow" (fetches individual client configs), done only if necessary. - if test "${node_name}" != "explorer" \ - && jqtest '.node.utxo_lsmt' "${dir}"/profile.json \ + if test "${node_name}" != "explorer" \ + && jqtest '.node.utxo_lsmt' "${dir}"/profile.json \ && jqtest '(.cluster.nomad.host_volumes.producer | length) > 0' "${dir}"/profile.json then # Iterate over the profile's Nomad "host_volumes" array by key/index. diff --git a/nix/workbench/service/nodes.nix b/nix/workbench/service/nodes.nix index 0c457cbcd33..8472b4e04b6 100644 --- a/nix/workbench/service/nodes.nix +++ b/nix/workbench/service/nodes.nix @@ -121,10 +121,10 @@ with pkgs.lib; let topology = "topology.json"; nodeConfigFile = "config.json"; - # Allow for local clusters to have multiple LSMT directories in the same physical ssd_directory; + # Allow for local clusters to have multiple on-disk (LSM-tree) directories in the same physical ssd_directory; # non-block producers (like the explorer node) keep using the in-memory backend - withUtxoHdLsmt = profile.node.utxo_lsmt && isProducer; - lsmDatabasePath = liveTablesPath i; + withUtxoHdLsmt = profile.node.utxo_lsmt && isProducer; + lsmDatabasePath = liveTablesPath i; ## Combine: ## 0. baseNodeConfig (coming cardanoLib's testnet environ) @@ -141,51 +141,66 @@ with pkgs.lib; let } (recursiveUpdate (recursiveUpdate - (removeAttrs - baseNodeConfig - [ - ## Let the genesis hashes be auto-computed by the node: - "ByronGenesisHash" - "ShelleyGenesisHash" - "AlonzoGenesisHash" - "ConwayGenesisHash" - "DijkstraGenesisHash" - ] - // { - ExperimentalHardForksEnabled = true; - ExperimentalProtocolsEnabled = true; - TurnOnLogMetrics = true; - SnapshotInterval = 4230; - ChainSyncIdleTimeout = 0; - PeerSharing = false; + ( + recursiveUpdate + (removeAttrs + baseNodeConfig + [ + ## Let the genesis hashes be auto-computed by the node: + "ByronGenesisHash" + "ShelleyGenesisHash" + "AlonzoGenesisHash" + "ConwayGenesisHash" + "DijkstraGenesisHash" + ] + // { + ExperimentalHardForksEnabled = true; + ExperimentalProtocolsEnabled = true; + TurnOnLogMetrics = true; + ChainSyncIdleTimeout = 0; + PeerSharing = false; - ## defaults taken from: ouroboros-network/src/Ouroboros/Network/Diffusion/Configuration.hs - ## NB. the following inequality must hold: known >= established >= active >= 0 - SyncTargetNumberOfActivePeers = max 15 valency; # set to same value as TargetNumberOfActivePeers - SyncTargetNumberOfEstablishedPeers = max 40 valency; - TargetNumberOfActivePeers = max 15 valency; - TargetNumberOfEstablishedPeers = max 40 valency; + ## defaults taken from: ouroboros-network/src/Ouroboros/Network/Diffusion/Configuration.hs + ## NB. the following inequality must hold: known >= established >= active >= 0 + SyncTargetNumberOfActivePeers = max 15 valency; # set to same value as TargetNumberOfActivePeers + SyncTargetNumberOfEstablishedPeers = max 40 valency; + TargetNumberOfActivePeers = max 15 valency; + TargetNumberOfEstablishedPeers = max 40 valency; - ByronGenesisFile = "../genesis/genesis.byron.json"; - ShelleyGenesisFile = "../genesis/genesis.shelley.json"; - AlonzoGenesisFile = "../genesis/genesis.alonzo.json"; - ## ConwayGenesisFile / DijkstraGenesisFile are always declared - ## here even when the profile leaves .genesis.conway or - ## .genesis.dijkstra null: cardano-node's config parser - ## requires both keys unconditionally. - ## The referenced files are left for stub placeholders to be - ## generated by the genesis backend. - ## If the node never activates that era the stub is inert. - ConwayGenesisFile = "../genesis/genesis.conway.json"; - DijkstraGenesisFile = "../genesis/genesis.dijkstra.json"; - } - // optionalAttrs (profile.node.utxo_lsmt && isProducer) + ByronGenesisFile = "../genesis/genesis.byron.json"; + ShelleyGenesisFile = "../genesis/genesis.shelley.json"; + AlonzoGenesisFile = "../genesis/genesis.alonzo.json"; + ## ConwayGenesisFile / DijkstraGenesisFile are always declared + ## here even when the profile leaves .genesis.conway or + ## .genesis.dijkstra null: cardano-node's config parser + ## requires both keys unconditionally. + ## The referenced files are left for stub placeholders to be + ## generated by the genesis backend. + ## If the node never activates that era the stub is inert. + ConwayGenesisFile = "../genesis/genesis.conway.json"; + DijkstraGenesisFile = "../genesis/genesis.dijkstra.json"; + } + // optionalAttrs (profile.node.utxo_lsmt && isProducer) + { + LedgerDB = { + Backend = "V2LSM"; + LSMDatabasePath = liveTablesPath i; + }; + }) { + ## This LedgerDB attrset is defined regardless of backend choice. + ## It assumes the Backend default to be "V2InMemory"; it must not overwrite any of the above, more specfic choices. LedgerDB = { - Backend = "V2LSM"; - LSMDatabasePath = liveTablesPath i; + Snapshots = { + SnapshotInterval = 4230; + # Disable the randomised delay for kicking off snapshots: + # For benchmarks, exact timing needs to be reproducible, and identical for all nodes. + MinDelay = 0; + MaxDelay = 0; + }; }; - }) + } + ) ( if __hasAttr "preset" profile && profile.preset != null ## It's either an undisturbed preset, @@ -213,9 +228,9 @@ with pkgs.lib; let // optionalAttrs (profiling.eventlog or false) { # Add the `-l` RTS param with profiling. eventlog = true; + # Decide where the executable comes from: + ######################################### } - # Decide where the executable comes from: - ######################################### // optionalAttrs (!backend.useCabalRun) { package = workbenchNix.haskellProject.exes.cardano-node; } diff --git a/scripts/lite/mainnet-legacy-tracing.sh b/scripts/lite/mainnet-legacy-tracing.sh deleted file mode 100755 index 3fbe4b3d2ae..00000000000 --- a/scripts/lite/mainnet-legacy-tracing.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# This script connects a node to mainnet - -ROOT="$(realpath "$(dirname "$0")/../..")" -configuration="${ROOT}/configuration/cardano" - -data_dir=mainnetsingle -mkdir -p "${data_dir}" -db_dir="${data_dir}/db/node" -mkdir -p "${db_dir}" -socket_dir="${data_dir}/socket" -mkdir -p "${socket_dir}" - -# Launch a node -cabal run exe:cardano-node -- run \ - --config "${configuration}/mainnet-config-legacy.json" \ - --topology "${configuration}/mainnet-topology.json" \ - --database-path "${db_dir}" \ - --socket-path "${socket_dir}/node-1-socket" \ - --host-addr "0.0.0.0" \ - --port "3001" - - - -function cleanup() -{ - for child in $(jobs -p); do - echo kill "$child" && kill "$child" - done -} - -trap cleanup EXIT diff --git a/trace-forward/src/Trace/Forward/Forwarding.hs b/trace-forward/src/Trace/Forward/Forwarding.hs index 2960db6c85d..72198bb6f24 100644 --- a/trace-forward/src/Trace/Forward/Forwarding.hs +++ b/trace-forward/src/Trace/Forward/Forwarding.hs @@ -24,7 +24,7 @@ import Ouroboros.Network.Mux (MiniProtocol (..), MiniProtocolLimits (. MiniProtocolNum (..), OuroborosApplication (..), RunMiniProtocol (..), miniProtocolLimits, miniProtocolNum, miniProtocolRun) import Ouroboros.Network.Protocol.Handshake (HandshakeArguments (..)) -import Ouroboros.Network.Protocol.Handshake.Codec (cborTermVersionDataCodec, +import Ouroboros.Network.Protocol.Handshake.Codec (mkVersionedCodecCBORTerm, codecHandshake, noTimeLimitsHandshake, timeLimitsHandshake) import Ouroboros.Network.Protocol.Handshake.Type (Handshake) import Ouroboros.Network.Protocol.Handshake.Version (acceptableVersion, queryVersion, @@ -210,7 +210,7 @@ launchForwarders iomgr forwarding dpStore) (fromMaybe (const $ pure ()) initOnForwardInterruption) 1 - maxReconnectDelay + (fromIntegral maxReconnectDelay) launchForwardersViaLocalSocket :: IOManager @@ -288,7 +288,7 @@ doConnectToAcceptor magic snocket makeBearer configureSocket address timeLimits args = ConnectToArgs { ctaHandshakeCodec = codecHandshake forwardingVersionCodec, ctaHandshakeTimeLimits = timeLimits, - ctaVersionDataCodec = cborTermVersionDataCodec forwardingCodecCBORTerm, + ctaVersionDataCodec = mkVersionedCodecCBORTerm forwardingCodecCBORTerm, ctaConnectTracers = nullNetworkConnectTracers, ctaHandshakeCallbacks = HandshakeCallbacks acceptableVersion queryVersion } forwarderApp @@ -337,7 +337,7 @@ doListenToAcceptor magic snocket makeBearer configureSocket address timeLimits haBearerTracer = nullTracer, haHandshakeTracer = nullTracer, haHandshakeCodec = codecHandshake forwardingVersionCodec, - haVersionDataCodec = cborTermVersionDataCodec forwardingCodecCBORTerm, + haVersionDataCodec = mkVersionedCodecCBORTerm forwardingCodecCBORTerm, haAcceptVersion = acceptableVersion, haQueryVersion = queryVersion, haTimeLimits = timeLimits diff --git a/trace-forward/trace-forward.cabal b/trace-forward/trace-forward.cabal index 9f118e7cffb..6c0f013af49 100644 --- a/trace-forward/trace-forward.cabal +++ b/trace-forward/trace-forward.cabal @@ -76,7 +76,7 @@ library , serialise , stm , text - , trace-dispatcher ^>= 2.12 + , trace-dispatcher ^>= 2.13 , typed-protocols:{typed-protocols, cborg} ^>= 1.2 test-suite test