From 76a73d7cf619699e3d7a964e5ca2ba34ea291ed3 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 12 Jan 2026 12:56:46 +0900 Subject: [PATCH 01/87] feat: Support RecordBlockCreation (block reprocessing) --- .../ArbitrumRpcTestBlockchain.cs | 7 ++ .../ArbitrumTestBlockchainBase.cs | 8 ++ .../Rpc/ArbitrumRpcModuleTests.cs | 9 +- src/Nethermind.Arbitrum/ArbitrumPlugin.cs | 6 +- .../ArbitrumRpcModuleFactory.cs | 6 +- .../Data/DigestMessageParameters.cs | 5 + src/Nethermind.Arbitrum/Data/RecordResult.cs | 45 +++++++++ .../Execution/ArbitrumTransactionProcessor.cs | 2 +- .../Stateless/ArbitrumWitnessCollector.cs | 49 ++++++++++ ...trumWitnessGeneratingBlockProcessingEnv.cs | 92 +++++++++++++++++++ ...nessGeneratingBlockProcessingEnvFactory.cs | 57 ++++++++++++ .../Modules/ArbitrumRpcModule.cs | 37 ++++++++ .../ArbitrumRpcModuleWithComparison.cs | 4 +- .../Modules/IArbitrumRpcModule.cs | 5 +- 14 files changed, 325 insertions(+), 7 deletions(-) create mode 100644 src/Nethermind.Arbitrum/Data/RecordResult.cs create mode 100644 src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs create mode 100644 src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs create mode 100644 src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs index 42ca7869d..7fa82ffad 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs @@ -29,6 +29,7 @@ using Nethermind.State; using Nethermind.TxPool; using Nethermind.Wallet; +using Nethermind.Consensus.Stateless; namespace Nethermind.Arbitrum.Test.Infrastructure; @@ -253,6 +254,7 @@ private static ArbitrumRpcTestBlockchain CreateInternal(ArbitrumRpcTestBlockchai chain.Container.Resolve(), new Nethermind.Arbitrum.Config.VerifyBlockHashConfig(), // Disabled for tests new Nethermind.Serialization.Json.EthereumJsonSerializer(), + chain.Container.Resolve(), chain.Container.Resolve(), null) // No ProcessExitSource in tests .Create()); @@ -363,6 +365,11 @@ public ResultWrapper> FullSyncProgressMap() { return rpc.FullSyncProgressMap(); } + + public Task> RecordBlockCreation(RecordBlockCreationParameters parameters) + { + return rpc.RecordBlockCreation(parameters); + } } public class ScopedGlobalWorldStateAccessor(ArbitrumRpcTestBlockchain chain) diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs index aa5501b31..c8032466f 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs @@ -38,6 +38,7 @@ using Nethermind.State; using Nethermind.TxPool; using BlockchainProcessorOptions = Nethermind.Consensus.Processing.BlockchainProcessor.Options; +using Nethermind.Consensus.Stateless; namespace Nethermind.Arbitrum.Test.Infrastructure; @@ -85,11 +86,16 @@ public abstract class ArbitrumTestBlockchainBase(ChainSpec chainSpec, ArbitrumCo public ILogFinder LogFinder => Dependencies.LogFinder; public CachedL1PriceData CachedL1PriceData => Dependencies.CachedL1PriceData; + public IWitnessGeneratingBlockProcessingEnvFactory WitnessGeneratingBlockProcessingEnvFactory => Dependencies.WitnessGeneratingBlockProcessingEnvFactory; public ISpecProvider SpecProvider => Dependencies.SpecProvider; public IWasmDb WasmDB => Container.Resolve(); + // Maybe use Dependencies instead? for those 2 below + public IWasmStore WasmStore => Container.Resolve(); + public IArbosVersionProvider ArbosVersionProvider => Container.Resolve(); + public IDb CodeDB => Container.ResolveKeyed("code"); public IStylusTargetConfig StylusTargetConfig => Container.Resolve(); @@ -127,6 +133,7 @@ public static ArbitrumRpcTestBlockchain CreateTestBlockchainWithGenesis() return ArbitrumRpcTestBlockchain.CreateDefault(preConfigurer); } + protected virtual ArbitrumTestBlockchainBase Build(Action? configurer = null) { Timestamper = new ManualTimestamper(InitialTimestamp); @@ -307,6 +314,7 @@ protected record BlockchainContainerDependencies( IBlockProducerEnvFactory BlockProducerEnvFactory, ISealer Sealer, CachedL1PriceData CachedL1PriceData, + IWitnessGeneratingBlockProcessingEnvFactory WitnessGeneratingBlockProcessingEnvFactory, IArbitrumSpecHelper SpecHelper); private void InitializeArbitrumPluginSteps(IContainer container) diff --git a/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs b/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs index 662a827f4..e889eae02 100644 --- a/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs +++ b/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs @@ -22,6 +22,8 @@ using Nethermind.Arbitrum.Execution; using Nethermind.Consensus.Processing; using Nethermind.Core.Specs; +using Nethermind.Consensus.Stateless; +using Nethermind.Arbitrum.Execution.Stateless; namespace Nethermind.Arbitrum.Test.Rpc { @@ -43,7 +45,7 @@ public abstract class ArbitrumRpcModuleTests private IArbitrumConfig _arbitrumConfig = null!; private Mock _mainProcessingContextMock = null!; private ISpecProvider _specProvider = null!; - + private Mock _witnessGeneratingBlockProcessingEnvFactory = null!; [SetUp] public void Setup() { @@ -57,6 +59,7 @@ public void Setup() _specHelper = new Mock(); _blockProcessingQueue = new Mock(); _specProvider = FullChainSimulationChainSpecProvider.CreateDynamicSpecProvider(_chainSpec); + _witnessGeneratingBlockProcessingEnvFactory = new Mock(); _initializer = new ArbitrumBlockTreeInitializer( _chainSpec, @@ -85,6 +88,7 @@ public void Setup() cachedL1PriceData, _blockProcessingQueue.Object, _arbitrumConfig, + _witnessGeneratingBlockProcessingEnvFactory.Object, _blockConfig); } @@ -247,6 +251,7 @@ public async Task HeadMessageIndex_Always_ReturnsHeadMessageIndex() cachedL1PriceData, _blockProcessingQueue.Object, _arbitrumConfig, + _witnessGeneratingBlockProcessingEnvFactory.Object, _blockConfig); _specHelper.Setup(c => c.GenesisBlockNum).Returns((ulong)genesis.Number); @@ -278,6 +283,7 @@ public async Task HeadMessageIndex_HasNoBlocks_NoLatestHeaderFound() cachedL1PriceData, _blockProcessingQueue.Object, _arbitrumConfig, + _witnessGeneratingBlockProcessingEnvFactory.Object, _blockConfig); var result = await _rpcModule.HeadMessageIndex(); @@ -315,6 +321,7 @@ public async Task HeadMessageIndex_BlockNumberIsLowerThanGenesis_Fails() cachedL1PriceData, _blockProcessingQueue.Object, _arbitrumConfig, + _witnessGeneratingBlockProcessingEnvFactory.Object, _blockConfig); _specHelper.Setup(c => c.GenesisBlockNum).Returns(genesisBlockNum); diff --git a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs index 6ce74c00c..b51207c08 100644 --- a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs +++ b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs @@ -11,6 +11,7 @@ using Nethermind.Arbitrum.Core; using Nethermind.Arbitrum.Evm; using Nethermind.Arbitrum.Execution; +using Nethermind.Arbitrum.Execution.Stateless; using Nethermind.Arbitrum.Execution.Transactions; using Nethermind.Arbitrum.Genesis; using Nethermind.Arbitrum.Modules; @@ -21,6 +22,7 @@ using Nethermind.Consensus; using Nethermind.Consensus.Processing; using Nethermind.Consensus.Producers; +using Nethermind.Consensus.Stateless; using Nethermind.Core; using Nethermind.Core.Container; using Nethermind.Core.Specs; @@ -93,6 +95,7 @@ public Task InitRpcModules() _api.Config(), _api.Config(), _api.EthereumJsonSerializer, + _api.Context.Resolve(), _api.Config(), _api.ProcessExit ); @@ -221,7 +224,8 @@ protected override void Load(ContainerBuilder builder) // Rpcs .AddSingleton() - .Bind, ArbitrumEthModuleFactory>(); + .Bind, ArbitrumEthModuleFactory>() + .AddSingleton(); if (blocksConfig.BuildBlocksOnMainState) builder.AddSingleton(); diff --git a/src/Nethermind.Arbitrum/ArbitrumRpcModuleFactory.cs b/src/Nethermind.Arbitrum/ArbitrumRpcModuleFactory.cs index 734182bf1..756e62eb0 100644 --- a/src/Nethermind.Arbitrum/ArbitrumRpcModuleFactory.cs +++ b/src/Nethermind.Arbitrum/ArbitrumRpcModuleFactory.cs @@ -14,6 +14,7 @@ using Nethermind.Logging; using Nethermind.Serialization.Json; using Nethermind.Specs.ChainSpecStyle; +using Nethermind.Consensus.Stateless; namespace Nethermind.Arbitrum; @@ -30,6 +31,7 @@ public sealed class ArbitrumRpcModuleFactory( IArbitrumConfig arbitrumConfig, IVerifyBlockHashConfig verifyBlockHashConfig, IJsonSerializer jsonSerializer, + IWitnessGeneratingBlockProcessingEnvFactory witnessGeneratingBlockProcessingEnvFactory, IBlocksConfig blocksConfig, IProcessExitSource? processExitSource = null) : ModuleFactoryBase { @@ -38,7 +40,7 @@ public override IArbitrumRpcModule Create() if (!verifyBlockHashConfig.Enabled || string.IsNullOrWhiteSpace(verifyBlockHashConfig.ArbNodeRpcUrl)) return new ArbitrumRpcModule( initializer, blockTree, trigger, txSource, chainSpec, specHelper, - logManager, cachedL1PriceData, processingQueue, arbitrumConfig, blocksConfig); + logManager, cachedL1PriceData, processingQueue, arbitrumConfig, witnessGeneratingBlockProcessingEnvFactory, blocksConfig); ILogger logger = logManager.GetClassLogger(); if (logger.IsInfo) @@ -46,6 +48,6 @@ public override IArbitrumRpcModule Create() return new ArbitrumRpcModuleWithComparison( initializer, blockTree, trigger, txSource, chainSpec, specHelper, - logManager, cachedL1PriceData, processingQueue, arbitrumConfig, verifyBlockHashConfig, jsonSerializer, blocksConfig, processExitSource); + logManager, cachedL1PriceData, processingQueue, arbitrumConfig, verifyBlockHashConfig, jsonSerializer, witnessGeneratingBlockProcessingEnvFactory, blocksConfig, processExitSource); } } diff --git a/src/Nethermind.Arbitrum/Data/DigestMessageParameters.cs b/src/Nethermind.Arbitrum/Data/DigestMessageParameters.cs index 902c63ccf..d23e1b632 100644 --- a/src/Nethermind.Arbitrum/Data/DigestMessageParameters.cs +++ b/src/Nethermind.Arbitrum/Data/DigestMessageParameters.cs @@ -37,3 +37,8 @@ public record DigestInitMessage( [property: JsonPropertyName("initialL1BaseFee")] UInt256 InitialL1BaseFee, [property: JsonPropertyName("serializedChainConfig"), JsonConverter(typeof(Base64Converter))] byte[]? SerializedChainConfig ); + +public record RecordBlockCreationParameters( + [property: JsonPropertyName("index")] ulong Index, + [property: JsonPropertyName("message")] MessageWithMetadata Message +); \ No newline at end of file diff --git a/src/Nethermind.Arbitrum/Data/RecordResult.cs b/src/Nethermind.Arbitrum/Data/RecordResult.cs new file mode 100644 index 000000000..c4ce32755 --- /dev/null +++ b/src/Nethermind.Arbitrum/Data/RecordResult.cs @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Consensus.Stateless; +using Nethermind.Core.Crypto; + +using WasmTarget = string; + +namespace Nethermind.Arbitrum.Data +{ + // TODO: check type correctness (maybe class, record, etc.) + public sealed class RecordResult + { + public ulong Index { get; } + public Hash256 BlockHash { get; } + public Dictionary Preimages { get; } + public UserWasms? UserWasms { get; } + + public RecordResult(ulong messageIndex, Hash256 blockHash, Witness witness) + { + Index = messageIndex; + BlockHash = blockHash; + // UserWasms = new(new()); // TODO: add wasms + UserWasms = null!; // TODO: add wasms + + Preimages = new(); + foreach (byte[] code in witness.Codes) + Preimages.Add(Keccak.Compute(code), code); + foreach (byte[] state in witness.State) + Preimages.Add(Keccak.Compute(state), state); + foreach (byte[] header in witness.Headers) + Preimages.Add(Keccak.Compute(header), header); + // foreach (byte[] key in witness.Keys) + // Preimages.Add(Keccak.Compute(key), key); + } + } + + public sealed record class ActivatedWasm( + Dictionary Value + ); + + public sealed record class UserWasms( + Dictionary Value + ); +} diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs index da2638999..86f78b8fd 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs @@ -25,7 +25,6 @@ using Nethermind.Arbitrum.Precompiles.Abi; using Nethermind.Evm.GasPolicy; using Nethermind.Arbitrum.Stylus; -using Nethermind.Core.Attributes; namespace Nethermind.Arbitrum.Execution { @@ -437,6 +436,7 @@ private ArbitrumTransactionProcessorResult ProcessArbitrumInternalTransaction( ValueHash256 prevHash = ValueKeccak.Zero; if (blCtx.Header.Number > 0) { + // Can't we just do: blCtx.Header.ParentHash ? or else pass my witnessGeneratingHeaderFinder prevHash = blockTree.FindBlockHash(blCtx.Header.Number - 1); } diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs new file mode 100644 index 000000000..0a84b5a9b --- /dev/null +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Blockchain.Tracing; +using Nethermind.Consensus.Processing; +using Nethermind.Core; +using Nethermind.Core.Specs; +using Nethermind.Consensus.Stateless; +using Nethermind.Arbitrum.Arbos; +using Nethermind.Logging; +using Nethermind.Int256; + +namespace Nethermind.Arbitrum.Execution.Stateless; + +public class ArbitrumWitnessCollector( + WitnessGeneratingHeaderFinder headerFinder, + WitnessGeneratingWorldState worldState, + WitnessCapturingTrieStore trieStore, + IBlockProcessor blockProcessor, + ISpecProvider specProvider) : IWitnessCollector +{ + public Witness GetWitness(BlockHeader parentHeader, Block block) + { + using (worldState.BeginScope(parentHeader)) + { + // Get the chain ID, both to validate and because the replay binary also gets the chain ID, + // so we need to populate the recordingdb with preimages for retrieving the chain ID. + + ArbosState arbosState = ArbosState.OpenArbosState(worldState, new SystemBurner(), NullLogger.Instance); + + UInt256 chainId = arbosState.ChainId.Get(); + ulong genesisBlockNum = arbosState.GenesisBlockNum.Get(); + byte[] chainConfig = arbosState.ChainConfigStorage.Get(); + + (Block processed, TxReceipt[] receipts) = blockProcessor.ProcessOne(block, ProcessingOptions.ProducingBlock, + NullBlockTracer.Instance, specProvider.GetSpec(block.Header)); + + (byte[][] stateNodes, byte[][] codes, byte[][] keys) = worldState.GetWitness(parentHeader, trieStore.TouchedNodesRlp); + + return new Witness() + { + Headers = headerFinder.GetWitnessHeaders(parentHeader.Hash!), + Codes = codes, + State = stateNodes, + Keys = keys + }; + } + } +} diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs new file mode 100644 index 000000000..5cd09d5c3 --- /dev/null +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Arbitrum.Arbos; +using Nethermind.Arbitrum.Evm; +using Nethermind.Arbitrum.Precompiles; +using Nethermind.Arbitrum.Stylus; +using Nethermind.Blockchain; +using Nethermind.Blockchain.BeaconBlockRoot; +using Nethermind.Blockchain.Blocks; +using Nethermind.Blockchain.Headers; +using Nethermind.Blockchain.Receipts; +using Nethermind.Consensus; +using Nethermind.Consensus.ExecutionRequests; +using Nethermind.Consensus.Processing; +using Nethermind.Consensus.Rewards; +using Nethermind.Consensus.Stateless; +using Nethermind.Consensus.Validators; +using Nethermind.Consensus.Withdrawals; +using Nethermind.Core.Specs; +using Nethermind.Evm.State; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Logging; +using Nethermind.State; +using static Nethermind.Arbitrum.Execution.ArbitrumBlockProcessor; + +namespace Nethermind.Arbitrum.Execution.Stateless; + +public class ArbitrumWitnessGeneratingBlockProcessingEnv( + ISpecProvider specProvider, + WorldState baseWorldState, + IStateReader stateReader, + // ArbitrumBlockProductionTransactionsExecutor txExecutor, + WitnessCapturingTrieStore witnessCapturingTrieStore, + IReadOnlyBlockTree blockTree, + ISealValidator sealValidator, + IRewardCalculator rewardCalculator, + IHeaderStore headerStore, + IWasmStore wasmStore, + IArbosVersionProvider arbosVersionProvider, + ILogManager logManager) : IWitnessGeneratingBlockProcessingEnv +{ + private ITransactionProcessor CreateTransactionProcessor(IWorldState state, IHeaderFinder witnessGeneratingHeaderFinder) + { + BlockhashProvider blockhashProvider = new(new BlockhashCache(witnessGeneratingHeaderFinder, logManager), state, logManager); + // We don't give any l1BlockCache to the vm so that it forces querying the world state + ArbitrumVirtualMachine vm = new(blockhashProvider, wasmStore, specProvider, logManager); + + return new ArbitrumTransactionProcessor( + BlobBaseFeeCalculator.Instance, specProvider, state, wasmStore, + vm, blockTree, logManager, + new ArbitrumCodeInfoRepository(new EthereumCodeInfoRepository(state), arbosVersionProvider)); + } + + public IWitnessCollector CreateWitnessCollector() + { + Console.WriteLine("--- In Arb WitnessGeneratingBlockProcessingEnv.CreateWitnessCollector() ---"); + WitnessGeneratingWorldState state = new(baseWorldState, stateReader); + WitnessGeneratingHeaderFinder witnessGenHeaderFinder = new(headerStore); + ITransactionProcessor txProcessor = CreateTransactionProcessor(state, witnessGenHeaderFinder); + IBlockProcessor.IBlockTransactionsExecutor txExecutor = + new BlockProcessor.BlockValidationTransactionsExecutor( + new ExecuteTransactionProcessorAdapter(txProcessor), state); + + // TODO: Might have to create one to use custom/empty wasm store + // IBlockProcessor.IBlockTransactionsExecutor txExecutor = + // new ArbitrumBlockProductionTransactionsExecutor( + // txProcessor, state, wasmStore, null, logManager, specProvider); + + IHeaderValidator headerValidator = new HeaderValidator(blockTree, sealValidator, specProvider, logManager); + IBlockValidator blockValidator = new BlockValidator(new TxValidator(specProvider.ChainId), headerValidator, + new UnclesValidator(blockTree, headerValidator, logManager), specProvider, logManager); + + ArbitrumBlockProcessor blockProcessor = new( + specProvider, + blockValidator, + rewardCalculator, + txExecutor, + txProcessor, + new CachedL1PriceData(logManager), + state, + NullReceiptStorage.Instance, + new BlockhashStore(state), + wasmStore, + new BeaconBlockRootHandler(txProcessor, state), + logManager, + new WithdrawalProcessor(state, logManager), + new ExecutionRequestsProcessor(txProcessor)); + + return new ArbitrumWitnessCollector(witnessGenHeaderFinder, state, witnessCapturingTrieStore, blockProcessor, specProvider); + } +} diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs new file mode 100644 index 000000000..0d3d38ca4 --- /dev/null +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Autofac; +using Nethermind.Arbitrum.Arbos; +using Nethermind.Arbitrum.Stylus; +using Nethermind.Blockchain; +using Nethermind.Blockchain.Headers; +using Nethermind.Consensus; +using Nethermind.Consensus.Processing; +using Nethermind.Consensus.Rewards; +using Nethermind.Consensus.Stateless; +using Nethermind.Core; +using Nethermind.Core.Specs; +using Nethermind.Db; +using Nethermind.Evm.State; +using Nethermind.Logging; +using Nethermind.State; +using Nethermind.Trie.Pruning; +using static Nethermind.Arbitrum.Execution.ArbitrumBlockProcessor; + +namespace Nethermind.Arbitrum.Execution.Stateless; + +public class ArbitrumWitnessGeneratingBlockProcessingEnvFactory( + ILifetimeScope rootLifetimeScope, + IReadOnlyTrieStore readOnlyTrieStore, + IDbProvider dbProvider, + ILogManager logManager) : IWitnessGeneratingBlockProcessingEnvFactory +{ + public IWitnessGeneratingBlockProcessingEnvScope CreateScope() + { + IReadOnlyDbProvider readOnlyDbProvider = new ReadOnlyDbProvider(dbProvider, true); + WitnessCapturingTrieStore trieStore = new(readOnlyDbProvider.StateDb, readOnlyTrieStore); + IStateReader stateReader = new StateReader(trieStore, readOnlyDbProvider.CodeDb, logManager); + IWorldState worldState = new WorldState(new TrieStoreScopeProvider(trieStore, readOnlyDbProvider.CodeDb, logManager), logManager); + + ILifetimeScope envLifetimeScope = rootLifetimeScope.BeginLifetimeScope((builder) => builder + .AddScoped(stateReader) + .AddScoped(worldState) + .AddScoped(builder => + new ArbitrumWitnessGeneratingBlockProcessingEnv( + builder.Resolve(), + (builder.Resolve() as WorldState)!, + builder.Resolve(), + // (builder.Resolve() as ArbitrumBlockProductionTransactionsExecutor)!, + trieStore, + builder.Resolve(), + builder.Resolve(), + builder.Resolve(), + builder.Resolve(), + builder.Resolve(), + builder.Resolve(), + logManager))); + + return new ExecutionRecordingScope(envLifetimeScope); + } +} diff --git a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs index e14885cdc..c9b62323e 100644 --- a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs +++ b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs @@ -19,6 +19,7 @@ using Nethermind.JsonRpc; using Nethermind.Logging; using Nethermind.Specs.ChainSpecStyle; +using Nethermind.Consensus.Stateless; namespace Nethermind.Arbitrum.Modules; @@ -33,6 +34,7 @@ public class ArbitrumRpcModule( CachedL1PriceData cachedL1PriceData, IBlockProcessingQueue processingQueue, IArbitrumConfig arbitrumConfig, + IWitnessGeneratingBlockProcessingEnvFactory witnessGeneratingBlockProcessingEnvFactory, IBlocksConfig blocksConfig) : IArbitrumRpcModule { protected readonly SemaphoreSlim CreateBlocksSemaphore = new(1, 1); @@ -266,6 +268,41 @@ public ResultWrapper> FullSyncProgressMap() } } + public async Task> RecordBlockCreation(RecordBlockCreationParameters parameters) + { + // TODO + // - check nitro how it is implemented, whether it can execute a new block or + // if it just try fetching an existing block (if so, why pass all message data?) + + long blockNumber = (await MessageIndexToBlockNumber(parameters.Index)).Data; + Block? block = blockTree.FindBlock(blockNumber, BlockTreeLookupOptions.None); + if (block is null) + { + return ResultWrapper.Fail($"Unable to find block {blockNumber}"); + } + else if (block.Number == 0) + { + // Cannot generate witness for genesis block as the block itself does not contain any transaction + // responsible for the state setup. It is the weak subjectivity starting point to trust. + return ResultWrapper.Fail($"Cannot generate witness for genesis block"); + } + + BlockHeader? parent = blockTree.FindHeader(block.ParentHash!); + if (parent is null) + { + return ResultWrapper.Fail($"Unable to find parent for block {blockNumber}"); + } + // return ResultWrapper.Success( + // blockchainBridge.GenerateExecutionWitness(parent, block)); + IWitnessGeneratingBlockProcessingEnvScope scope = witnessGeneratingBlockProcessingEnvFactory.CreateScope(); + scope.Env.CreateWitnessCollector(); + IWitnessCollector witnessCollector = scope.Env.CreateWitnessCollector(); + Witness witness = witnessCollector.GetWitness(parent, block); + + RecordResult result = new(parameters.Index, block.Hash!, witness); + return ResultWrapper.Success(result); + } + protected async Task> ProduceBlockWhileLockedAsync(MessageWithMetadata messageWithMetadata, long blockNumber, BlockHeader? headBlockHeader) { ArbitrumPayloadAttributes payload = new() diff --git a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModuleWithComparison.cs b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModuleWithComparison.cs index 143f5cd6d..0c6cf1c54 100644 --- a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModuleWithComparison.cs +++ b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModuleWithComparison.cs @@ -15,6 +15,7 @@ using Nethermind.Logging; using Nethermind.Serialization.Json; using Nethermind.Specs.ChainSpecStyle; +using Nethermind.Consensus.Stateless; namespace Nethermind.Arbitrum.Modules; @@ -31,9 +32,10 @@ public sealed class ArbitrumRpcModuleWithComparison( IArbitrumConfig arbitrumConfig, IVerifyBlockHashConfig verifyBlockHashConfig, IJsonSerializer jsonSerializer, + IWitnessGeneratingBlockProcessingEnvFactory witnessGeneratingBlockProcessingEnvFactory, IBlocksConfig blocksConfig, IProcessExitSource? processExitSource = null) - : ArbitrumRpcModule(initializer, blockTree, trigger, txSource, chainSpec, specHelper, logManager, cachedL1PriceData, processingQueue, arbitrumConfig, blocksConfig) + : ArbitrumRpcModule(initializer, blockTree, trigger, txSource, chainSpec, specHelper, logManager, cachedL1PriceData, processingQueue, arbitrumConfig, witnessGeneratingBlockProcessingEnvFactory, blocksConfig) { private readonly ArbitrumComparisonRpcClient _comparisonRpcClient = new(verifyBlockHashConfig.ArbNodeRpcUrl!, jsonSerializer, logManager); private readonly long _verificationInterval = (long)verifyBlockHashConfig.VerifyEveryNBlocks; diff --git a/src/Nethermind.Arbitrum/Modules/IArbitrumRpcModule.cs b/src/Nethermind.Arbitrum/Modules/IArbitrumRpcModule.cs index e13e1b447..a5d788f96 100644 --- a/src/Nethermind.Arbitrum/Modules/IArbitrumRpcModule.cs +++ b/src/Nethermind.Arbitrum/Modules/IArbitrumRpcModule.cs @@ -16,7 +16,7 @@ public interface IArbitrumRpcModule : IRpcModule [JsonRpcMethod(IsSharable = false, IsImplemented = true)] Task> DigestMessage(DigestMessageParameters parameters); - Task> ResultAtMessageIndex(UInt64 messageIndex); + Task> ResultAtMessageIndex(ulong messageIndex); Task> HeadMessageIndex(); @@ -38,5 +38,8 @@ public interface IArbitrumRpcModule : IRpcModule [JsonRpcMethod(IsSharable = false, IsImplemented = true)] ResultWrapper> FullSyncProgressMap(); + + // Task> RecordBlockCreation(ulong Index, MessageWithMetadata message); + Task> RecordBlockCreation(RecordBlockCreationParameters parameters); } } From fd84bfb13950360691e5cbe9da82d915241187bb Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 20 Jan 2026 18:01:01 +0900 Subject: [PATCH 02/87] feat: Fix RecordBlockCreation with block production --- .../ArbitrumWitnessGenerationTests.cs | 547 ++++++++++++++++++ src/Nethermind.Arbitrum/Data/RecordResult.cs | 8 +- .../ArbitrumEvmInstructions.Environment.cs | 49 ++ .../Evm/ArbitrumVirtualMachine.cs | 8 +- src/Nethermind.Arbitrum/Evm/L1BlockCache.cs | 2 +- .../Execution/ArbitrumBlockProcessor.cs | 8 +- .../Execution/ArbitrumTransactionProcessor.cs | 2 +- .../ArbitrumStatelessBlockProcessingEnv.cs | 89 +++ .../Stateless/ArbitrumWitnessCollector.cs | 54 +- ...trumWitnessGeneratingBlockProcessingEnv.cs | 107 ++-- ...nessGeneratingBlockProcessingEnvFactory.cs | 105 +++- .../Modules/ArbitrumRpcModule.cs | 37 +- .../Precompiles/ArbitrumCodeInfoRepository.cs | 7 +- 13 files changed, 889 insertions(+), 134 deletions(-) create mode 100644 src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs create mode 100644 src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs new file mode 100644 index 000000000..d259829d9 --- /dev/null +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -0,0 +1,547 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Linq; +using System.Threading.Tasks; +using Nethermind.Arbitrum.Stylus; +using Nethermind.Arbitrum.Test.Infrastructure; +using Nethermind.Blockchain; +using Nethermind.Blockchain.BeaconBlockRoot; +using Nethermind.Blockchain.Blocks; +using Nethermind.Blockchain.Find; +using Nethermind.Blockchain.Headers; +using Nethermind.Blockchain.Receipts; +using Nethermind.Blockchain.Tracing; +using Nethermind.Config; +using Nethermind.Consensus; +using Nethermind.Consensus.ExecutionRequests; +using Nethermind.Consensus.Processing; +using Nethermind.Consensus.Producers; +using Nethermind.Consensus.Rewards; +using Nethermind.Consensus.Stateless; +using Nethermind.Consensus.Transactions; +using Nethermind.Consensus.Validators; +using Nethermind.Consensus.Withdrawals; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Specs; +using Nethermind.Core.Test; +using Nethermind.Core.Test.Blockchain; +using Nethermind.Core.Test.Builders; +using Nethermind.Core.Test.Db; +using Nethermind.Db; +using Nethermind.Evm; +using Nethermind.Evm.State; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Int256; +using Nethermind.JsonRpc; +using Nethermind.JsonRpc.Modules; +using Nethermind.Logging; +using Nethermind.Merge.Plugin; +using Nethermind.Specs; +using Nethermind.State; +using Nethermind.Trie; +using Nethermind.Trie.Pruning; +using NSubstitute; +using static Nethermind.Core.Block; +using Nethermind.Arbitrum.Data; +using Nethermind.Arbitrum.Execution.Stateless; + +namespace Nethermind.Arbitrum.Test.Execution; + +public class ArbitrumWitnessGenerationTests +{ + [TestCase(1ul)] + [TestCase(2ul)] + [TestCase(3ul)] + [TestCase(4ul)] + [TestCase(5ul)] + [TestCase(6ul)] + [TestCase(7ul)] + [TestCase(8ul)] + [TestCase(9ul)] + [TestCase(10ul)] + [TestCase(11ul)] + [TestCase(12ul)] + [TestCase(13ul)] + [TestCase(14ul)] + [TestCase(15ul)] + [TestCase(16ul)] + [TestCase(17ul)] + [TestCase(18ul)] + public async Task RecordBlockCreation_Witness_AllowsStatelessExecution(ulong messageIndex) + { + FullChainSimulationRecordingFile recording = new("./Recordings/1__arbos32_basefee92.jsonl"); + DigestMessageParameters digestMessage = recording.GetDigestMessages().First(m => m.Index == messageIndex); + + using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() + .WithRecording(recording) + .Build(); + + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation(new RecordBlockCreationParameters(digestMessage.Index, digestMessage.Message)); + RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestMessage.Index); + + Witness witness = recordResult.Witness; + + ISpecProvider specProvider = FullChainSimulationChainSpecProvider.CreateDynamicSpecProvider(); + ArbitrumStatelessBlockProcessingEnv blockProcessingEnv = + new(witness, specProvider, Always.Valid, chain.WasmStore, chain.ArbosVersionProvider, chain.LogManager); + + Block block = chain.BlockFinder.FindBlock(recordResult.BlockHash) + ?? throw new ArgumentException($"Unable to find block {recordResult.BlockHash}"); + BlockHeader parent = chain.BlockFinder.FindHeader(block.ParentHash!) + ?? throw new ArgumentException($"Unable to find parent for block {recordResult.BlockHash}"); + + using (blockProcessingEnv.WorldState.BeginScope(parent)) + { + (Block processed, TxReceipt[] _) = blockProcessingEnv.BlockProcessor.ProcessOne( + block, + ProcessingOptions.DoNotUpdateHead | ProcessingOptions.ReadOnlyChain, + NullBlockTracer.Instance, + specProvider.GetSpec(block.Header)); + + Assert.That(processed.Hash, Is.EqualTo(block.Hash)); + } + } + + private static T ThrowOnFailure(ResultWrapper result, ulong msgIndex) + { + if (result.Result != Result.Success) + throw new InvalidOperationException($"Failed to execute RPC method, message index {msgIndex}, code {result.ErrorCode}: {result.Result.Error}"); + + return result.Data; + } + + // [Test] + // public async Task SimpleArbitrumTest() + // { + // IDbProvider dbProvider = TestMemDbProvider.Init(); + // ILogManager logManager = LimboLogs.Instance; + + // PruningConfig pruningConfig = new(); + // TestFinalizedStateProvider finalizedStateProvider = new(pruningConfig.PruningBoundary); + // TrieStore store = new( + // new NodeStorage(dbProvider.StateDb), + // No.Pruning, + // Persist.EveryBlock, + // finalizedStateProvider, + // pruningConfig, + // LimboLogs.Instance); + // finalizedStateProvider.TrieStore = store; + + // // StateTree stateTree = new(store.GetTrieStore(null), LimboLogs.Instance); + + // long blockNumber = 0; + // // store.BeginBlockCommit(blockNumber); + // Address account123 = new("0x0000000000000000000000000000000000000123"); + // StateTree stateTree; + // using (IBlockCommitter committer = store.BeginBlockCommit(blockNumber)) + // { + // StorageTree storageTree = new(store.GetTrieStore(account123), LimboLogs.Instance); + // UInt256 key0 = 0; + // UInt256 value0 = 10; + // storageTree.Set(key0, value0.ToBigEndian()); + // UInt256 key1 = 1; + // UInt256 value1 = 15; + // storageTree.Set(key1, value1.ToBigEndian()); + // UInt256 key2 = 2; + // UInt256 value2 = 20; + // storageTree.Set(key2, value2.ToBigEndian()); + // storageTree.Commit(); + + // Account account = Build.An.Account.WithBalance(1.Ether()).WithStorageRoot(storageTree.RootHash).TestObject; + // stateTree = new(store.GetTrieStore(null), LimboLogs.Instance); + // stateTree.Set(account123, account); + // stateTree.Commit(); + // } + + // WorldState worldState = new(store, dbProvider.CodeDb, logManager); + // StateReader stateReader = new(store, dbProvider.CodeDb, logManager); + + // NullSealEngine sealer = NullSealEngine.Instance; + // MainnetSpecProvider specProvider = MainnetSpecProvider.Instance; + + + // Address sender = TestItem.AddressA; + // Hash256 stateRoot; + // using (var scope = worldState.BeginScope(new BlockHeader { StateRoot = stateTree.RootHash })) + // // using (var scope = worldState.BeginScope(null)) + // { + // // 0x60, 0x00, 0x60, 0x01, 0x55 = store value 0 at slot 1 in contract storage + // byte[] runtimeCode = Prepare.EvmCode.PushData(1).PushData(0).SSTORE().Done; + // worldState.InsertCode(account123, runtimeCode, specProvider.GenesisSpec); + + // worldState.CreateAccountIfNotExists(sender, 1.Ether()); + // // worldState.AddToBalance(sender, 1.Ether(), specProvider.GenesisSpec); + + // worldState.Commit(specProvider.GenesisSpec); + + // var bal = worldState.GetBalance(account123); + // Console.WriteLine($"--- {bal} ---"); + + // // TODO (Goal is to see why DumpState returns missing node) + // // 1. Check with current implem, i expect it might give an error when calling SetRootHash() in UpdateRootHash() + // // bc it'll try fetching it from the trie store but the root ref won't have been set there yet + // // |--> for that check the node shards after stateTree.Commit() and then continue from bulkWrite.Set() + // // ANSWER: the boolean is false and therefore RootRef is not fetched from trieStore and we keep the in-memory one, + // // so, it works fine but does not do what i want, need to call CommitTree() ! + // // 2. Use CommitTree() instead of RecalculateStateRoot() + // // worldState.RecalculateStateRoot(); // TODO: 1. just try keeping this and see what DumpState does + // worldState.CommitTree(blockNumber: 0); // TODO: 2. try this, this should make the DumpState work + // stateRoot = worldState.StateRoot; + // } + + // Console.WriteLine("--- print state 1 ---"); + // Console.WriteLine(stateReader.DumpState(stateRoot)); + + // using (var scope = worldState.BeginScope(null)) + // { + // // AddressA will be our sender + // // worldState.AddToBalanceAndCreateIfNotExists(new(accountAddr0), 1.Ether(), specProvider.GetSpec(new ForkActivation(0))); + // // // AddressB will be our receiver + // // worldState.CreateAccount(TestItem.AddressB, 0); + // // worldState.Commit(specProvider.GetSpec(new ForkActivation(0))); + // // worldState.RecalculateStateRoot(); + // } + + // // Create Genesis with the correct state root + // Block genesis = Build.A.Block.Genesis + // .WithStateRoot(stateRoot) + // .TestObject; + // // BlockTree blockTree = Build.A.BlockTree().TestObject; + // // Initialize BlockTree with Genesis (OfChainLength(1) ensures it's added) + // IHeaderStore headerStore = new HeaderStore(new MemDb(), new MemDb()); + + // BlockTreeBuilder blockTreeBuilder = Build.A.BlockTree(genesis); + // blockTreeBuilder.HeaderStore = headerStore; + // BlockTree blockTree = blockTreeBuilder + // .OfChainLength(1) + // .TestObject; + + // IncrementalTimestamper timestamper = new(); + // // var logManager = LimboLogs.Instance; + // BlocksConfig blocksConfig = new(); + // NoBlockRewards rewardCalculator = NoBlockRewards.Instance; + + // // new BlockhashCache(new HeaderStore(new MemDb(), new MemDb())) + // BlockhashProvider blockhashProvider = new(new BlockhashCache(headerStore, logManager), worldState, logManager); + // VirtualMachine vm = new(blockhashProvider, specProvider, logManager); + // TransactionProcessor txProcessor = new(new BlobBaseFeeCalculator(), specProvider, worldState, vm, new EthereumCodeInfoRepository(worldState), logManager); + + // IBlockProcessor.IBlockTransactionsExecutor txExecutor = + // new BlockProcessor.BlockValidationTransactionsExecutor( + // new ExecuteTransactionProcessorAdapter(txProcessor), worldState); + + // IHeaderValidator headerValidator = new HeaderValidator(blockTree, sealer, specProvider, logManager); + // IBlockValidator blockValidator = new BlockValidator(new TxValidator(specProvider.ChainId), headerValidator, + // new UnclesValidator(blockTree, headerValidator, logManager), specProvider, logManager); + + // BeaconBlockRootHandler beacon = new(txProcessor, worldState); + // IBlockProcessor blockProcessor = new BlockProcessor( + // specProvider, + // blockValidator, + // rewardCalculator, + // txExecutor, + // worldState, + // NullReceiptStorage.Instance, + // beacon, + // new BlockhashStore(worldState), + // logManager, + // new WithdrawalProcessor(worldState, logManager), + // new ExecutionRequestsProcessor(txProcessor)); + + // BranchProcessor branchProcessor = new( + // blockProcessor, + // specProvider, + // worldState, + // beacon, + // blockhashProvider, + // logManager + // ); + + // BlockchainProcessor blockchainProcessor = new( + // blockTree, + // branchProcessor, + // new MergeProcessingRecoveryStep(NoPoS.Instance), + // stateReader, + // logManager, + // BlockchainProcessor.Options.Default + // ); + + // // Tx to call contract + // var signedTx = Build.A.Transaction + // .WithTo(account123) + // // .WithData(new byte[] { 0x01 }) + // .WithNonce(0) + // .WithValue(0) + // .WithGasLimit(1_000_000) + // .WithGasPrice(10_0000_000) + // .WithSenderAddress(sender) + // .SignedAndResolved(TestItem.PrivateKeyA) + // .TestObject; + + // // Transaction transaction = Build.A.Transaction + // // .WithSenderAddress(sender) + // // .WithTo(to) + // // .WithGasLimit((long)gasLimit) + // // .WithMaxFeePerGas(baseFeePerGas * 2) + // // .WithValue(100) + // // .WithType(TxType.EIP1559) + // // .TestObject; + + + // var txSource = Substitute.For(); + // txSource.GetTransactions(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + // .Returns(new[] { signedTx }); + + // var producer = new TestBlockProducer( + // txSource, + // blockchainProcessor, + // worldState, + // sealer, + // blockTree, + // timestamper, + // specProvider, + // logManager, + // blocksConfig + // ); + + // // 4. Produce block + // var parent = blockTree.Head!.Header; + // var payloadAttributes = new PayloadAttributes + // { + // Timestamp = (ulong)parent.Timestamp + 1, + // PrevRandao = Hash256.Zero, + // SuggestedFeeRecipient = TestItem.AddressB, + // Withdrawals = Array.Empty(), + // ParentBeaconBlockRoot = Hash256.Zero + // }; + // var block = await producer.BuildBlock(payloadAttributes: payloadAttributes, token: CancellationToken.None); + // // Console.WriteLine($"Produced block {block!.ToString(Format.Full)}"); + + // Console.WriteLine("--- print state 2 ---"); + // Console.WriteLine(stateReader.DumpState(stateRoot)); + // } + + // [Test] + // public async Task SimpleTest() + // { + // // StateTree stateTree = TestItem.Tree.GetStateTree(); + // // TestItem.Tree.GetTrees(stateTree.TrieStore); + + // // ITrieStore store = new TestRawTrieStore(new MemDb()); + + // IDbProvider dbProvider = TestMemDbProvider.Init(); + // ILogManager logManager = LimboLogs.Instance; + + // PruningConfig pruningConfig = new(); + // TestFinalizedStateProvider finalizedStateProvider = new(pruningConfig.PruningBoundary); + // TrieStore store = new( + // new NodeStorage(dbProvider.StateDb), + // No.Pruning, + // Persist.EveryBlock, + // finalizedStateProvider, + // pruningConfig, + // LimboLogs.Instance); + // finalizedStateProvider.TrieStore = store; + + // StateTree oldStateTree = new(store.GetTrieStore(null), LimboLogs.Instance); + + // long blockNumber = 0; + // store.BeginBlockCommit(blockNumber); + + // TestItem.Tree.FillStateTreeWithTestAccounts(oldStateTree); + + // (StateTree stateTree, StorageTree _, Hash256 accountAddr0) = TestItem.Tree.GetTrees(store); + // WorldState worldState = new(store, dbProvider.CodeDb, logManager); + // StateReader stateReader = new(store, dbProvider.CodeDb, logManager); + // using (var scope = worldState.BeginScope(new BlockHeader { StateRoot = stateTree.RootHash })) + // { + // var bal = worldState.GetBalance(new(accountAddr0)); + // Console.WriteLine($"--- {bal} ---"); + // } + // NullSealEngine sealer = NullSealEngine.Instance; + // MainnetSpecProvider specProvider = MainnetSpecProvider.Instance; + + // Hash256 stateRoot; + // using (var scope = worldState.BeginScope(null)) + // { + // // AddressA will be our sender + // // worldState.AddToBalanceAndCreateIfNotExists(new(accountAddr0), 1.Ether(), specProvider.GetSpec(new ForkActivation(0))); + // // // AddressB will be our receiver + // // worldState.CreateAccount(TestItem.AddressB, 0); + // // worldState.Commit(specProvider.GetSpec(new ForkActivation(0))); + // // worldState.RecalculateStateRoot(); + // stateRoot = worldState.StateRoot; + // } + + // // Create Genesis with the correct state root + // Block genesis = Build.A.Block.Genesis + // .WithStateRoot(stateRoot) + // .TestObject; + // // BlockTree blockTree = Build.A.BlockTree().TestObject; + // // Initialize BlockTree with Genesis (OfChainLength(1) ensures it's added) + // IHeaderStore headerStore = new HeaderStore(new MemDb(), new MemDb()); + + // BlockTreeBuilder blockTreeBuilder = Build.A.BlockTree(genesis); + // blockTreeBuilder.HeaderStore = headerStore; + // BlockTree blockTree = blockTreeBuilder + // .OfChainLength(1) + // .TestObject; + // IncrementalTimestamper timestamper = new(); + // // var logManager = LimboLogs.Instance; + // BlocksConfig blocksConfig = new(); + // NoBlockRewards rewardCalculator = NoBlockRewards.Instance; + + // // BlockhashProvider blockhashProvider = new(blockTree, specProvider, worldState, logManager); + // // not blocktree ? + // BlockhashProvider blockhashProvider = new(new BlockhashCache(headerStore, logManager), worldState, logManager); + // VirtualMachine vm = new(blockhashProvider, specProvider, logManager); + // TransactionProcessor txProcessor = new(new BlobBaseFeeCalculator(), specProvider, worldState, vm, new EthereumCodeInfoRepository(worldState), logManager); + + // IBlockProcessor.IBlockTransactionsExecutor txExecutor = + // new BlockProcessor.BlockValidationTransactionsExecutor( + // new ExecuteTransactionProcessorAdapter(txProcessor), worldState); + + // IHeaderValidator headerValidator = new HeaderValidator(blockTree, sealer, specProvider, logManager); + // IBlockValidator blockValidator = new BlockValidator(new TxValidator(specProvider.ChainId), headerValidator, + // new UnclesValidator(blockTree, headerValidator, logManager), specProvider, logManager); + + // BeaconBlockRootHandler beacon = new(txProcessor, worldState); + // IBlockProcessor blockProcessor = new BlockProcessor( + // specProvider, + // blockValidator, + // rewardCalculator, + // txExecutor, + // worldState, + // NullReceiptStorage.Instance, + // beacon, + // new BlockhashStore(worldState), + // logManager, + // new WithdrawalProcessor(worldState, logManager), + // new ExecutionRequestsProcessor(txProcessor)); + + // BranchProcessor branchProcessor = new( + // blockProcessor, + // specProvider, + // worldState, + // beacon, + // blockhashProvider, + // logManager + // ); + + // BlockchainProcessor blockchainProcessor = new( + // blockTree, + // branchProcessor, + // new MergeProcessingRecoveryStep(NoPoS.Instance), + // stateReader, + // logManager, + // BlockchainProcessor.Options.Default + // ); + + // // 2. Transaction + // var signedTx = Build.A.Transaction + // .WithTo(TestItem.AddressB) + // // .WithData(new byte[] { 0x01 }) + // .WithNonce(0) + // .WithValue(100) + // .WithGasLimit(22000) + // .WithGasPrice(1000000000) + // .WithSenderAddress(new(accountAddr0)) + // // .SignedAndResolved() + // .TestObject; + + // // Transaction transaction = Build.A.Transaction + // // .WithSenderAddress(sender) + // // .WithTo(to) + // // .WithGasLimit((long)gasLimit) + // // .WithMaxFeePerGas(baseFeePerGas * 2) + // // .WithValue(100) + // // .WithType(TxType.EIP1559) + // // .TestObject; + + + // var txSource = Substitute.For(); + // txSource.GetTransactions(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + // .Returns(new[] { signedTx }); + + // var producer = new TestBlockProducer( + // txSource, + // blockchainProcessor, + // worldState, + // sealer, + // blockTree, + // timestamper, + // specProvider, + // logManager, + // blocksConfig + // ); + + // // 4. Produce block + // var parent = blockTree.Head!.Header; + // var payloadAttributes = new PayloadAttributes + // { + // Timestamp = (ulong)parent.Timestamp + 1, + // PrevRandao = Hash256.Zero, + // SuggestedFeeRecipient = TestItem.AddressB, + // Withdrawals = Array.Empty(), + // ParentBeaconBlockRoot = Hash256.Zero + // }; + // var block = await producer.BuildBlock(payloadAttributes: payloadAttributes, token: CancellationToken.None); + + // // worldState.CommitTree + + // // Assert.IsNotNull(block); + // Console.WriteLine($"Produced block {block?.Hash}"); + + + // // WitnessGeneratingBlockProcessingEnv env = new( + // // specProvider, + // // stateReader, + // // worldState, + // // new ReadOnlyBlockTree(blockTree), + // // sealer, + // // rewardCalculator, + // // logManager + // // ); + // } + + private async Task SaveWitnessToJsonFile(Witness witness) + { + var json = JsonSerializer.Serialize(witness, new JsonSerializerOptions { + WriteIndented = true, + IncludeFields = true, + Converters = { + new ByteArrayHexConverter() + } + }); + await File.WriteAllTextAsync("/Users/gugz/Documents/rpc_req/witness_from_test.json", + json + ); + } + + public class ByteArrayHexConverter : JsonConverter + { + // public override byte[] Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + // { + // string hex = reader.GetString()!; + // return Convert.FromHexString(hex.StartsWith("0x") ? hex.Substring(2) : hex); + // } + public override byte[] Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + string hex = reader.GetString()!; + + if (hex.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) + hex = hex.Substring(2); + + return Convert.FromHexString(hex); + } + + public override void Write(Utf8JsonWriter writer, byte[] value, JsonSerializerOptions options) + { + writer.WriteStringValue("0x" + Convert.ToHexString(value).ToLowerInvariant()); + // writer.WriteStringValue(Convert.ToHexString(value)); // uppercase hex + } + } + +} diff --git a/src/Nethermind.Arbitrum/Data/RecordResult.cs b/src/Nethermind.Arbitrum/Data/RecordResult.cs index c4ce32755..8238a9572 100644 --- a/src/Nethermind.Arbitrum/Data/RecordResult.cs +++ b/src/Nethermind.Arbitrum/Data/RecordResult.cs @@ -3,6 +3,7 @@ using Nethermind.Consensus.Stateless; using Nethermind.Core.Crypto; +using System.Text.Json.Serialization; using WasmTarget = string; @@ -16,11 +17,14 @@ public sealed class RecordResult public Dictionary Preimages { get; } public UserWasms? UserWasms { get; } + [JsonIgnore] + public Witness Witness { get; } + public RecordResult(ulong messageIndex, Hash256 blockHash, Witness witness) { Index = messageIndex; BlockHash = blockHash; - // UserWasms = new(new()); // TODO: add wasms + Witness = witness; UserWasms = null!; // TODO: add wasms Preimages = new(); @@ -30,8 +34,6 @@ public RecordResult(ulong messageIndex, Hash256 blockHash, Witness witness) Preimages.Add(Keccak.Compute(state), state); foreach (byte[] header in witness.Headers) Preimages.Add(Keccak.Compute(header), header); - // foreach (byte[] key in witness.Keys) - // Preimages.Add(Keccak.Compute(key), key); } } diff --git a/src/Nethermind.Arbitrum/Evm/ArbitrumEvmInstructions.Environment.cs b/src/Nethermind.Arbitrum/Evm/ArbitrumEvmInstructions.Environment.cs index 67bbf4301..2f2bec2a2 100644 --- a/src/Nethermind.Arbitrum/Evm/ArbitrumEvmInstructions.Environment.cs +++ b/src/Nethermind.Arbitrum/Evm/ArbitrumEvmInstructions.Environment.cs @@ -10,6 +10,8 @@ using Nethermind.Evm.GasPolicy; using Nethermind.Int256; using static Nethermind.Arbitrum.Evm.ArbitrumVirtualMachine; +using Nethermind.Evm.EvmObjectFormat; +using Nethermind.Core.Specs; namespace Nethermind.Arbitrum.Evm; @@ -143,6 +145,53 @@ public static EvmExceptionType InstructionBlockHash(VirtualMachine return EvmExceptionType.None; } + /// + /// Same as the base implementation but omits any optimization so that it always goes through + /// the world state to get and record the bytecode. Used for witness generation. + /// + [SkipLocalsInit] + public static EvmExceptionType InstructionExtCodeSize(VirtualMachine vm, + ref EvmStack stack, + ref TGasPolicy gas, + ref int programCounter) + where TGasPolicy : struct, IGasPolicy + where TTracingInst : struct, IFlag + { + IReleaseSpec spec = vm.Spec; + // Deduct the gas cost for external code access. + TGasPolicy.Consume(ref gas, spec.GetExtCodeCost()); + + // Pop the account address from the stack. + Address? address = stack.PopAddress(); + if (address is null) + goto StackUnderflow; + + // Charge gas for accessing the account's state. + if (!TGasPolicy.ConsumeAccountAccessGas(ref gas, spec, in vm.VmState.AccessTracker, vm.TxTracer.IsTracingAccess, address)) + goto OutOfGas; + + // No optimization applied: load the account's code from storage. + ReadOnlySpan accountCode = vm.CodeInfoRepository + .GetCachedCodeInfo(address, followDelegation: false, spec, out _) + .CodeSpan; + // If EOF is enabled and the code is an EOF contract, push a fixed size (2). + if (spec.IsEofEnabled && EofValidator.IsEof(accountCode, out _)) + { + stack.PushUInt32(2); + } + else + { + // Otherwise, push the actual code length. + stack.PushUInt32((uint)accountCode.Length); + } + return EvmExceptionType.None; + // Jump forward to be unpredicted by the branch predictor. + OutOfGas: + return EvmExceptionType.OutOfGas; + StackUnderflow: + return EvmExceptionType.StackUnderflow; + } + /// /// Safely downcasts to . /// In DEBUG builds, validates that the runtime type is correct. diff --git a/src/Nethermind.Arbitrum/Evm/ArbitrumVirtualMachine.cs b/src/Nethermind.Arbitrum/Evm/ArbitrumVirtualMachine.cs index e9997f8e6..7e6b83296 100644 --- a/src/Nethermind.Arbitrum/Evm/ArbitrumVirtualMachine.cs +++ b/src/Nethermind.Arbitrum/Evm/ArbitrumVirtualMachine.cs @@ -34,7 +34,8 @@ public sealed unsafe class ArbitrumVirtualMachine( IWasmStore wasmStore, ISpecProvider? specProvider, ILogManager? logManager, - IL1BlockCache? l1BlockCache = null + IL1BlockCache? l1BlockCache = null, + bool enableWitnessGeneration = false ) : VirtualMachine(blockHashProvider, specProvider, logManager), IVirtualMachine, IStylusVmHost { public IWasmStore WasmStore => wasmStore; @@ -380,6 +381,11 @@ protected override OpCode[] GenerateOpCodes(IReleaseSpec spec) opcodes[(int)Instruction.GASPRICE] = &ArbitrumEvmInstructions.InstructionBlkUInt256; opcodes[(int)Instruction.NUMBER] = &ArbitrumEvmInstructions.InstructionBlkUInt64; opcodes[(int)Instruction.BLOCKHASH] = &ArbitrumEvmInstructions.InstructionBlockHash; + // Opcode overrides specific for witness generation + if (enableWitnessGeneration) + { + opcodes[(int)Instruction.EXTCODESIZE] = &ArbitrumEvmInstructions.InstructionExtCodeSize; + } return opcodes; } diff --git a/src/Nethermind.Arbitrum/Evm/L1BlockCache.cs b/src/Nethermind.Arbitrum/Evm/L1BlockCache.cs index 1504d2169..21dc50e66 100644 --- a/src/Nethermind.Arbitrum/Evm/L1BlockCache.cs +++ b/src/Nethermind.Arbitrum/Evm/L1BlockCache.cs @@ -22,7 +22,7 @@ public sealed class L1BlockCache : IL1BlockCache /// 256 capacities match the BLOCKHASH opcode window (last 256 blocks). /// Thread-safe and shared across all transactions. /// - private static readonly ClockCache CachedL1BlockHashes = new(256); + private readonly ClockCache CachedL1BlockHashes = new(256); public ulong? GetCachedL1BlockNumber() { diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs index e7288f26b..9c241c7c0 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs @@ -424,12 +424,8 @@ private AddingTxEventArgs CanAddTransaction( ulong? blockGasLeft, int userTxsProcessed) { - // Skip gas limit check for non-user transactions - if (!IsUserTransaction(currentTx)) - return txPicker.CanAddTransaction(block, currentTx, transactionsInBlock, stateProvider); - - // Early check: reject if block gas is too low (unless this is the first user tx) - if (blockGasLeft < GasCostOf.Transaction && userTxsProcessed > 0) + // If we've done too much work in this block, discard the tx as early as possible + if (blockGasLeft < GasCostOf.Transaction && IsUserTransaction(currentTx)) { AddingTxEventArgs args = new(transactionsInBlock.Count, currentTx, block, transactionsInBlock); return args.Set(TxAction.Skip, TransactionResult.BlockGasLimitExceeded.ErrorDescription); diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs index 86f78b8fd..ec0f406e3 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs @@ -763,6 +763,7 @@ private static void TryReapOneRetryable( return; } + ulong windowsLeft = retryable.TimeoutWindowsLeft.Get(); if (timeout >= currentTimestamp) { // Not expired yet — return without popping @@ -771,7 +772,6 @@ private static void TryReapOneRetryable( // Expired — pop from queue _ = arbosState.RetryableState.TimeoutQueue.Pop(); - ulong windowsLeft = retryable.TimeoutWindowsLeft.Get(); if (windowsLeft == 0) { diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs new file mode 100644 index 000000000..0cb0ce848 --- /dev/null +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Arbitrum.Arbos; +using Nethermind.Arbitrum.Evm; +using Nethermind.Arbitrum.Execution; +using Nethermind.Arbitrum.Precompiles; +using Nethermind.Arbitrum.Stylus; +using Nethermind.Blockchain; +using Nethermind.Blockchain.BeaconBlockRoot; +using Nethermind.Blockchain.Blocks; +using Nethermind.Blockchain.Receipts; +using Nethermind.Consensus.ExecutionRequests; +using Nethermind.Consensus.Processing; +using Nethermind.Consensus.Rewards; +using Nethermind.Consensus.Validators; +using Nethermind.Consensus.Withdrawals; +using Nethermind.Core.Specs; +using Nethermind.Evm.State; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Logging; +using Nethermind.State; +using Nethermind.Trie; +using Nethermind.Consensus.Stateless; +using Nethermind.Consensus; + +namespace Nethermind.Arbitrum.Execution.Stateless; + +public class ArbitrumStatelessBlockProcessingEnv( + Witness witness, + ISpecProvider specProvider, + ISealValidator sealValidator, + IWasmStore wasmStore, + IArbosVersionProvider arbosVersionProvider, + ILogManager logManager) +{ + private IBlockProcessor? _blockProcessor; + public IBlockProcessor BlockProcessor + { + get => _blockProcessor ??= GetProcessor(); + } + + private IWorldState? _worldState; + public IWorldState WorldState + { + get => _worldState ??= new WorldState( + new TrieStoreScopeProvider(new RawTrieStore(witness.NodeStorage), + witness.CodeDb, logManager), logManager); + } + + private IBlockProcessor GetProcessor() + { + StatelessBlockTree statelessBlockTree = new(witness.DecodedHeaders); + ITransactionProcessor txProcessor = CreateTransactionProcessor(WorldState, statelessBlockTree); + IBlockProcessor.IBlockTransactionsExecutor txExecutor = + new BlockProcessor.BlockValidationTransactionsExecutor( + new ExecuteTransactionProcessorAdapter(txProcessor), + WorldState); + + IHeaderValidator headerValidator = new HeaderValidator(statelessBlockTree, sealValidator, specProvider, logManager); + IBlockValidator blockValidator = new BlockValidator(new TxValidator(specProvider.ChainId), headerValidator, + new UnclesValidator(statelessBlockTree, headerValidator, logManager), specProvider, logManager); + + return new ArbitrumBlockProcessor( + specProvider, + blockValidator, + NoBlockRewards.Instance, + txExecutor, + txProcessor, + new CachedL1PriceData(logManager), + WorldState, + NullReceiptStorage.Instance, + new BlockhashStore(WorldState), + wasmStore, + new BeaconBlockRootHandler(txProcessor, WorldState), + logManager, + new WithdrawalProcessor(WorldState, logManager), + new ExecutionRequestsProcessor(txProcessor) + ); + } + + + private ITransactionProcessor CreateTransactionProcessor(IWorldState state, StatelessBlockTree blockFinder) + { + BlockhashProvider blockhashProvider = new(blockFinder, state, logManager); + ArbitrumVirtualMachine vm = new(blockhashProvider, wasmStore, specProvider, logManager); + return new ArbitrumTransactionProcessor(BlobBaseFeeCalculator.Instance, specProvider, state, wasmStore, vm, blockFinder, logManager, new ArbitrumCodeInfoRepository(new EthereumCodeInfoRepository(state), arbosVersionProvider)); + } +} diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs index 0a84b5a9b..142eb8630 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs @@ -1,49 +1,63 @@ // SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using Nethermind.Blockchain.Tracing; -using Nethermind.Consensus.Processing; using Nethermind.Core; -using Nethermind.Core.Specs; using Nethermind.Consensus.Stateless; using Nethermind.Arbitrum.Arbos; +using Nethermind.Arbitrum.Config; using Nethermind.Logging; using Nethermind.Int256; +using Nethermind.Consensus; +using Nethermind.Consensus.Producers; +using Nethermind.Core.Specs; namespace Nethermind.Arbitrum.Execution.Stateless; +public interface IBlockBuildingWitnessCollector +{ + Task<(Block Block, Witness Witness)> BuildBlockAndGetWitness(BlockHeader parentHeader, PayloadAttributes payloadAttributes); +} + public class ArbitrumWitnessCollector( WitnessGeneratingHeaderFinder headerFinder, WitnessGeneratingWorldState worldState, WitnessCapturingTrieStore trieStore, - IBlockProcessor blockProcessor, - ISpecProvider specProvider) : IWitnessCollector + IBlockProducer blockProducer, + ISpecProvider specProvider, + IArbitrumSpecHelper specHelper) : IBlockBuildingWitnessCollector { - public Witness GetWitness(BlockHeader parentHeader, Block block) + public async Task<(Block Block, Witness Witness)> BuildBlockAndGetWitness(BlockHeader parentHeader, PayloadAttributes payloadAttributes) { using (worldState.BeginScope(parentHeader)) { - // Get the chain ID, both to validate and because the replay binary also gets the chain ID, - // so we need to populate the recordingdb with preimages for retrieving the chain ID. - ArbosState arbosState = ArbosState.OpenArbosState(worldState, new SystemBurner(), NullLogger.Instance); UInt256 chainId = arbosState.ChainId.Get(); ulong genesisBlockNum = arbosState.GenesisBlockNum.Get(); byte[] chainConfig = arbosState.ChainConfigStorage.Get(); - (Block processed, TxReceipt[] receipts) = blockProcessor.ProcessOne(block, ProcessingOptions.ProducingBlock, - NullBlockTracer.Instance, specProvider.GetSpec(block.Header)); - - (byte[][] stateNodes, byte[][] codes, byte[][] keys) = worldState.GetWitness(parentHeader, trieStore.TouchedNodesRlp); + if (chainId != specProvider.ChainId) + throw new InvalidOperationException($"ArbOS chainId mismatch. ArbOS={chainId}, local={specProvider.ChainId}."); - return new Witness() - { - Headers = headerFinder.GetWitnessHeaders(parentHeader.Hash!), - Codes = codes, - State = stateNodes, - Keys = keys - }; + if (genesisBlockNum != specHelper.GenesisBlockNum) + throw new InvalidOperationException($"ArbOS genesisBlockNum mismatch. ArbOS={genesisBlockNum}, local={specHelper.GenesisBlockNum}."); } + + + Block? producedBlock = await blockProducer.BuildBlock(parentHeader: parentHeader, payloadAttributes: payloadAttributes); + if (producedBlock?.Hash is null) + throw new Exception("Failed to build block or block has no hash."); + + (byte[][] stateNodes, byte[][] codes, byte[][] keys) = worldState.GetWitness(parentHeader, trieStore.TouchedNodesRlp); + + Witness witness = new() + { + Headers = headerFinder.GetWitnessHeaders(parentHeader.Hash!), + Codes = codes, + State = stateNodes, + Keys = keys + }; + + return (producedBlock, witness); } } diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs index 5cd09d5c3..a495b72f0 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs @@ -1,92 +1,59 @@ // SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using Nethermind.Arbitrum.Arbos; -using Nethermind.Arbitrum.Evm; -using Nethermind.Arbitrum.Precompiles; -using Nethermind.Arbitrum.Stylus; using Nethermind.Blockchain; -using Nethermind.Blockchain.BeaconBlockRoot; -using Nethermind.Blockchain.Blocks; -using Nethermind.Blockchain.Headers; -using Nethermind.Blockchain.Receipts; using Nethermind.Consensus; -using Nethermind.Consensus.ExecutionRequests; using Nethermind.Consensus.Processing; -using Nethermind.Consensus.Rewards; using Nethermind.Consensus.Stateless; -using Nethermind.Consensus.Validators; -using Nethermind.Consensus.Withdrawals; +using Nethermind.Arbitrum.Config; using Nethermind.Core.Specs; -using Nethermind.Evm.State; -using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; -using Nethermind.State; -using static Nethermind.Arbitrum.Execution.ArbitrumBlockProcessor; +using Nethermind.Core; +using Nethermind.Consensus.Transactions; +using Nethermind.Config; namespace Nethermind.Arbitrum.Execution.Stateless; +public interface IWitnessGeneratingPolyvalentEnv: IWitnessGeneratingBlockProcessingEnv +{ + IBlockBuildingWitnessCollector CreateBlockBuildingWitnessCollector(); +} + public class ArbitrumWitnessGeneratingBlockProcessingEnv( + ITxSource txSource, + IBlockchainProcessor chainProcessor, + IReadOnlyBlockTree blockTree, + WitnessGeneratingWorldState witnessGeneratingWorldState, + IBlocksConfig blocksConfig, ISpecProvider specProvider, - WorldState baseWorldState, - IStateReader stateReader, - // ArbitrumBlockProductionTransactionsExecutor txExecutor, + IArbitrumSpecHelper specHelper, + WitnessGeneratingHeaderFinder witnessGenHeaderFinder, WitnessCapturingTrieStore witnessCapturingTrieStore, - IReadOnlyBlockTree blockTree, - ISealValidator sealValidator, - IRewardCalculator rewardCalculator, - IHeaderStore headerStore, - IWasmStore wasmStore, - IArbosVersionProvider arbosVersionProvider, - ILogManager logManager) : IWitnessGeneratingBlockProcessingEnv + ILogManager logManager) : IWitnessGeneratingPolyvalentEnv { - private ITransactionProcessor CreateTransactionProcessor(IWorldState state, IHeaderFinder witnessGeneratingHeaderFinder) + public IExistingBlockWitnessCollector CreateExistingBlockWitnessCollector() { - BlockhashProvider blockhashProvider = new(new BlockhashCache(witnessGeneratingHeaderFinder, logManager), state, logManager); - // We don't give any l1BlockCache to the vm so that it forces querying the world state - ArbitrumVirtualMachine vm = new(blockhashProvider, wasmStore, specProvider, logManager); - - return new ArbitrumTransactionProcessor( - BlobBaseFeeCalculator.Instance, specProvider, state, wasmStore, - vm, blockTree, logManager, - new ArbitrumCodeInfoRepository(new EthereumCodeInfoRepository(state), arbosVersionProvider)); + // This is used for overriding the base NMC's debug_executionWitness implementation endpoint + // Not priority for now + throw new NotSupportedException($"{nameof(ArbitrumWitnessGeneratingBlockProcessingEnv)} does not support generating witnesses for already existing blocks."); } - public IWitnessCollector CreateWitnessCollector() + public IBlockBuildingWitnessCollector CreateBlockBuildingWitnessCollector() { - Console.WriteLine("--- In Arb WitnessGeneratingBlockProcessingEnv.CreateWitnessCollector() ---"); - WitnessGeneratingWorldState state = new(baseWorldState, stateReader); - WitnessGeneratingHeaderFinder witnessGenHeaderFinder = new(headerStore); - ITransactionProcessor txProcessor = CreateTransactionProcessor(state, witnessGenHeaderFinder); - IBlockProcessor.IBlockTransactionsExecutor txExecutor = - new BlockProcessor.BlockValidationTransactionsExecutor( - new ExecuteTransactionProcessorAdapter(txProcessor), state); - - // TODO: Might have to create one to use custom/empty wasm store - // IBlockProcessor.IBlockTransactionsExecutor txExecutor = - // new ArbitrumBlockProductionTransactionsExecutor( - // txProcessor, state, wasmStore, null, logManager, specProvider); - - IHeaderValidator headerValidator = new HeaderValidator(blockTree, sealValidator, specProvider, logManager); - IBlockValidator blockValidator = new BlockValidator(new TxValidator(specProvider.ChainId), headerValidator, - new UnclesValidator(blockTree, headerValidator, logManager), specProvider, logManager); - - ArbitrumBlockProcessor blockProcessor = new( - specProvider, - blockValidator, - rewardCalculator, - txExecutor, - txProcessor, - new CachedL1PriceData(logManager), - state, - NullReceiptStorage.Instance, - new BlockhashStore(state), - wasmStore, - new BeaconBlockRootHandler(txProcessor, state), - logManager, - new WithdrawalProcessor(state, logManager), - new ExecutionRequestsProcessor(txProcessor)); - - return new ArbitrumWitnessCollector(witnessGenHeaderFinder, state, witnessCapturingTrieStore, blockProcessor, specProvider); + Console.WriteLine("--- In Arb WitnessGeneratingBlockProcessingEnv.CreateBlockBuildingWitnessCollector() ---"); + + ArbitrumBlockProducer blockProducer = new( + txSource, + chainProcessor, + blockTree, + witnessGeneratingWorldState, + new ArbitrumGasLimitCalculator(), + NullSealEngine.Instance, + new ManualTimestamper(), + specProvider, + logManager, + blocksConfig); + + return new ArbitrumWitnessCollector(witnessGenHeaderFinder, witnessGeneratingWorldState, witnessCapturingTrieStore, blockProducer, specProvider, specHelper); } } diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index 0d3d38ca4..d3fe11b74 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -8,7 +8,6 @@ using Nethermind.Blockchain.Headers; using Nethermind.Consensus; using Nethermind.Consensus.Processing; -using Nethermind.Consensus.Rewards; using Nethermind.Consensus.Stateless; using Nethermind.Core; using Nethermind.Core.Specs; @@ -18,6 +17,16 @@ using Nethermind.State; using Nethermind.Trie.Pruning; using static Nethermind.Arbitrum.Execution.ArbitrumBlockProcessor; +using Nethermind.Arbitrum.Evm; +using Nethermind.Arbitrum.Precompiles; +using Nethermind.Blockchain.Receipts; +using Nethermind.Consensus.Withdrawals; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Consensus.Transactions; +using Nethermind.Consensus.Producers; +using Nethermind.Config; +using Nethermind.Evm; +using Nethermind.Arbitrum.Config; namespace Nethermind.Arbitrum.Execution.Stateless; @@ -27,29 +36,101 @@ public class ArbitrumWitnessGeneratingBlockProcessingEnvFactory( IDbProvider dbProvider, ILogManager logManager) : IWitnessGeneratingBlockProcessingEnvFactory { + // To force processing in BlockchainProcessor even though block is not better than head (already existing block) + private static BlocksConfig CreateWitnessBlocksConfig(IBlocksConfig blocksConfig) + { + return new BlocksConfig + { + TargetBlockGasLimit = blocksConfig.TargetBlockGasLimit, + MinGasPrice = blocksConfig.MinGasPrice, + RandomizedBlocks = blocksConfig.RandomizedBlocks, + ExtraData = blocksConfig.ExtraData, + SecondsPerSlot = blocksConfig.SecondsPerSlot, + SingleBlockImprovementOfSlot = blocksConfig.SingleBlockImprovementOfSlot, + PreWarmStateOnBlockProcessing = blocksConfig.PreWarmStateOnBlockProcessing, + CachePrecompilesOnBlockProcessing = blocksConfig.CachePrecompilesOnBlockProcessing, + PreWarmStateConcurrency = blocksConfig.PreWarmStateConcurrency, + BlockProductionTimeoutMs = blocksConfig.BlockProductionTimeoutMs, + GenesisTimeoutMs = blocksConfig.GenesisTimeoutMs, + BlockProductionMaxTxKilobytes = blocksConfig.BlockProductionMaxTxKilobytes, + GasToken = blocksConfig.GasToken, + BlockProductionBlobLimit = blocksConfig.BlockProductionBlobLimit, + BuildBlocksOnMainState = false, + }; + } + + private ITransactionProcessor CreateTransactionProcessor( + IWasmStore wasmStore, + ISpecProvider specProvider, + IBlockTree blockTree, + IArbosVersionProvider arbosVersionProvider, + IWorldState state, + IHeaderFinder witnessGeneratingHeaderFinder) + { + BlockhashProvider blockhashProvider = new(new BlockhashCache(witnessGeneratingHeaderFinder, logManager), state, logManager); + // We don't give any l1BlockCache to the vm so that it forces querying the world state + ArbitrumVirtualMachine vm = new(blockhashProvider, wasmStore, specProvider, logManager, enableWitnessGeneration: true); + + return new ArbitrumTransactionProcessor( + BlobBaseFeeCalculator.Instance, specProvider, state, wasmStore, + vm, blockTree, logManager, + new ArbitrumCodeInfoRepository(new CodeInfoRepository(state, new EthereumPrecompileProvider()), arbosVersionProvider, state as IWitnessBytecodeRecorder)); + } + public IWitnessGeneratingBlockProcessingEnvScope CreateScope() { + IBlocksConfig blocksConfig = rootLifetimeScope.Resolve(); IReadOnlyDbProvider readOnlyDbProvider = new ReadOnlyDbProvider(dbProvider, true); WitnessCapturingTrieStore trieStore = new(readOnlyDbProvider.StateDb, readOnlyTrieStore); IStateReader stateReader = new StateReader(trieStore, readOnlyDbProvider.CodeDb, logManager); - IWorldState worldState = new WorldState(new TrieStoreScopeProvider(trieStore, readOnlyDbProvider.CodeDb, logManager), logManager); + WorldState worldState = new(new TrieStoreScopeProvider(trieStore, readOnlyDbProvider.CodeDb, logManager), logManager); ILifetimeScope envLifetimeScope = rootLifetimeScope.BeginLifetimeScope((builder) => builder .AddScoped(stateReader) - .AddScoped(worldState) + .AddScoped(new WitnessGeneratingWorldState(worldState, stateReader)) + + .AddScoped(_ => CreateWitnessBlocksConfig(blocksConfig)) + + //TODO Create witness capturing wasm store somehow + .AddScoped(_ => new WasmDb(new MemDb())) + // new instance i think? to check but might have to create a new recording IWasmDb passed to it + .AddScoped(ctx => new WasmStore(ctx.Resolve(), ctx.Resolve(), cacheTag: 1)) + .AddScoped(builder => new WitnessGeneratingHeaderFinder(builder.Resolve())) + + .AddScoped(builder => CreateTransactionProcessor( + builder.Resolve(), + builder.Resolve(), + builder.Resolve(), + builder.Resolve(), + builder.Resolve(), + builder.Resolve())) + + // 1st: add the tx executor + .AddScoped() + + // 2nd: add block processor + .AddScoped(NullReceiptStorage.Instance) + .AddScoped(BlockchainProcessor.Options.NoReceipts) + .AddScoped() + + // 3rd: configure the builder for block production (like ArbitrumBlockProducerEnvFactory but with my own witness capturing world state) + .AddScoped(builder => builder.Resolve().Create()) + .AddScoped() + .AddDecorator() + .AddDecorator() + .AddScoped() + .AddScoped(builder => new ArbitrumWitnessGeneratingBlockProcessingEnv( + builder.Resolve(), + builder.Resolve(), + builder.Resolve(), + (builder.Resolve() as WitnessGeneratingWorldState)!, + builder.Resolve(), builder.Resolve(), - (builder.Resolve() as WorldState)!, - builder.Resolve(), - // (builder.Resolve() as ArbitrumBlockProductionTransactionsExecutor)!, + builder.Resolve(), + (builder.Resolve() as WitnessGeneratingHeaderFinder)!, trieStore, - builder.Resolve(), - builder.Resolve(), - builder.Resolve(), - builder.Resolve(), - builder.Resolve(), - builder.Resolve(), logManager))); return new ExecutionRecordingScope(envLifetimeScope); diff --git a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs index c9b62323e..c282d1efe 100644 --- a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs +++ b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs @@ -20,6 +20,7 @@ using Nethermind.Logging; using Nethermind.Specs.ChainSpecStyle; using Nethermind.Consensus.Stateless; +using Nethermind.Arbitrum.Execution.Stateless; namespace Nethermind.Arbitrum.Modules; @@ -270,36 +271,34 @@ public ResultWrapper> FullSyncProgressMap() public async Task> RecordBlockCreation(RecordBlockCreationParameters parameters) { - // TODO - // - check nitro how it is implemented, whether it can execute a new block or - // if it just try fetching an existing block (if so, why pass all message data?) - long blockNumber = (await MessageIndexToBlockNumber(parameters.Index)).Data; - Block? block = blockTree.FindBlock(blockNumber, BlockTreeLookupOptions.None); - if (block is null) - { - return ResultWrapper.Fail($"Unable to find block {blockNumber}"); - } - else if (block.Number == 0) + if (blockNumber == 0) { // Cannot generate witness for genesis block as the block itself does not contain any transaction // responsible for the state setup. It is the weak subjectivity starting point to trust. return ResultWrapper.Fail($"Cannot generate witness for genesis block"); } - BlockHeader? parent = blockTree.FindHeader(block.ParentHash!); + BlockHeader? parent = blockTree.FindHeader(blockNumber - 1); if (parent is null) { return ResultWrapper.Fail($"Unable to find parent for block {blockNumber}"); } - // return ResultWrapper.Success( - // blockchainBridge.GenerateExecutionWitness(parent, block)); - IWitnessGeneratingBlockProcessingEnvScope scope = witnessGeneratingBlockProcessingEnvFactory.CreateScope(); - scope.Env.CreateWitnessCollector(); - IWitnessCollector witnessCollector = scope.Env.CreateWitnessCollector(); - Witness witness = witnessCollector.GetWitness(parent, block); - - RecordResult result = new(parameters.Index, block.Hash!, witness); + + ArbitrumPayloadAttributes payload = new() + { + MessageWithMetadata = parameters.Message, + Number = blockNumber + }; + + using IWitnessGeneratingBlockProcessingEnvScope scope = witnessGeneratingBlockProcessingEnvFactory.CreateScope(); + IBlockBuildingWitnessCollector witnessCollector = ((IWitnessGeneratingPolyvalentEnv)scope.Env).CreateBlockBuildingWitnessCollector(); + (Block builtBlock, Witness witness) = await witnessCollector.BuildBlockAndGetWitness(parent, payload); + + if (builtBlock.Hash is null) + return ResultWrapper.Fail("Failed to build block or block has no hash."); + + RecordResult result = new(parameters.Index, builtBlock.Hash!, witness); return ResultWrapper.Success(result); } diff --git a/src/Nethermind.Arbitrum/Precompiles/ArbitrumCodeInfoRepository.cs b/src/Nethermind.Arbitrum/Precompiles/ArbitrumCodeInfoRepository.cs index ff8a6d966..1d9f7ddf2 100644 --- a/src/Nethermind.Arbitrum/Precompiles/ArbitrumCodeInfoRepository.cs +++ b/src/Nethermind.Arbitrum/Precompiles/ArbitrumCodeInfoRepository.cs @@ -9,10 +9,11 @@ using Nethermind.Core.Specs; using Nethermind.Evm; using Nethermind.Evm.CodeAnalysis; +using Nethermind.Consensus.Stateless; namespace Nethermind.Arbitrum.Precompiles; -public class ArbitrumCodeInfoRepository(ICodeInfoRepository codeInfoRepository, IArbosVersionProvider arbosVersionProvider) : ICodeInfoRepository +public class ArbitrumCodeInfoRepository(ICodeInfoRepository codeInfoRepository, IArbosVersionProvider arbosVersionProvider, IWitnessBytecodeRecorder? witnessBytecodeRecorder = null) : ICodeInfoRepository { private readonly Dictionary _arbitrumPrecompiles = InitializePrecompiledContracts(); @@ -63,6 +64,10 @@ delegationAddress is not null && return result; } + // Record precompile bytecode (0xFE) for witness generation as nitro records a trie node with invalid bytecode for it + // which kinda makes sense as there is no actual bytecode stored for precompiles + witnessBytecodeRecorder?.RecordBytecode([0xfe]); + // It's a precompile according to spec // Check if it's an Arbitrum precompile we handle delegationAddress = null; From 45df498124a61f80ac56ee0e9d24411543161ec5 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 20 Jan 2026 18:03:10 +0900 Subject: [PATCH 03/87] chore: Clean old experimental tests --- .../ArbitrumWitnessGenerationTests.cs | 471 ------------------ 1 file changed, 471 deletions(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index d259829d9..5159e98e8 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -1,49 +1,11 @@ -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Linq; -using System.Threading.Tasks; -using Nethermind.Arbitrum.Stylus; using Nethermind.Arbitrum.Test.Infrastructure; -using Nethermind.Blockchain; -using Nethermind.Blockchain.BeaconBlockRoot; -using Nethermind.Blockchain.Blocks; -using Nethermind.Blockchain.Find; -using Nethermind.Blockchain.Headers; -using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Tracing; -using Nethermind.Config; -using Nethermind.Consensus; -using Nethermind.Consensus.ExecutionRequests; using Nethermind.Consensus.Processing; -using Nethermind.Consensus.Producers; -using Nethermind.Consensus.Rewards; using Nethermind.Consensus.Stateless; -using Nethermind.Consensus.Transactions; using Nethermind.Consensus.Validators; -using Nethermind.Consensus.Withdrawals; using Nethermind.Core; -using Nethermind.Core.Crypto; -using Nethermind.Core.Extensions; using Nethermind.Core.Specs; -using Nethermind.Core.Test; -using Nethermind.Core.Test.Blockchain; -using Nethermind.Core.Test.Builders; -using Nethermind.Core.Test.Db; -using Nethermind.Db; -using Nethermind.Evm; -using Nethermind.Evm.State; -using Nethermind.Evm.TransactionProcessing; -using Nethermind.Int256; using Nethermind.JsonRpc; -using Nethermind.JsonRpc.Modules; -using Nethermind.Logging; -using Nethermind.Merge.Plugin; -using Nethermind.Specs; -using Nethermind.State; -using Nethermind.Trie; -using Nethermind.Trie.Pruning; -using NSubstitute; -using static Nethermind.Core.Block; using Nethermind.Arbitrum.Data; using Nethermind.Arbitrum.Execution.Stateless; @@ -111,437 +73,4 @@ private static T ThrowOnFailure(ResultWrapper result, ulong msgIndex) return result.Data; } - - // [Test] - // public async Task SimpleArbitrumTest() - // { - // IDbProvider dbProvider = TestMemDbProvider.Init(); - // ILogManager logManager = LimboLogs.Instance; - - // PruningConfig pruningConfig = new(); - // TestFinalizedStateProvider finalizedStateProvider = new(pruningConfig.PruningBoundary); - // TrieStore store = new( - // new NodeStorage(dbProvider.StateDb), - // No.Pruning, - // Persist.EveryBlock, - // finalizedStateProvider, - // pruningConfig, - // LimboLogs.Instance); - // finalizedStateProvider.TrieStore = store; - - // // StateTree stateTree = new(store.GetTrieStore(null), LimboLogs.Instance); - - // long blockNumber = 0; - // // store.BeginBlockCommit(blockNumber); - // Address account123 = new("0x0000000000000000000000000000000000000123"); - // StateTree stateTree; - // using (IBlockCommitter committer = store.BeginBlockCommit(blockNumber)) - // { - // StorageTree storageTree = new(store.GetTrieStore(account123), LimboLogs.Instance); - // UInt256 key0 = 0; - // UInt256 value0 = 10; - // storageTree.Set(key0, value0.ToBigEndian()); - // UInt256 key1 = 1; - // UInt256 value1 = 15; - // storageTree.Set(key1, value1.ToBigEndian()); - // UInt256 key2 = 2; - // UInt256 value2 = 20; - // storageTree.Set(key2, value2.ToBigEndian()); - // storageTree.Commit(); - - // Account account = Build.An.Account.WithBalance(1.Ether()).WithStorageRoot(storageTree.RootHash).TestObject; - // stateTree = new(store.GetTrieStore(null), LimboLogs.Instance); - // stateTree.Set(account123, account); - // stateTree.Commit(); - // } - - // WorldState worldState = new(store, dbProvider.CodeDb, logManager); - // StateReader stateReader = new(store, dbProvider.CodeDb, logManager); - - // NullSealEngine sealer = NullSealEngine.Instance; - // MainnetSpecProvider specProvider = MainnetSpecProvider.Instance; - - - // Address sender = TestItem.AddressA; - // Hash256 stateRoot; - // using (var scope = worldState.BeginScope(new BlockHeader { StateRoot = stateTree.RootHash })) - // // using (var scope = worldState.BeginScope(null)) - // { - // // 0x60, 0x00, 0x60, 0x01, 0x55 = store value 0 at slot 1 in contract storage - // byte[] runtimeCode = Prepare.EvmCode.PushData(1).PushData(0).SSTORE().Done; - // worldState.InsertCode(account123, runtimeCode, specProvider.GenesisSpec); - - // worldState.CreateAccountIfNotExists(sender, 1.Ether()); - // // worldState.AddToBalance(sender, 1.Ether(), specProvider.GenesisSpec); - - // worldState.Commit(specProvider.GenesisSpec); - - // var bal = worldState.GetBalance(account123); - // Console.WriteLine($"--- {bal} ---"); - - // // TODO (Goal is to see why DumpState returns missing node) - // // 1. Check with current implem, i expect it might give an error when calling SetRootHash() in UpdateRootHash() - // // bc it'll try fetching it from the trie store but the root ref won't have been set there yet - // // |--> for that check the node shards after stateTree.Commit() and then continue from bulkWrite.Set() - // // ANSWER: the boolean is false and therefore RootRef is not fetched from trieStore and we keep the in-memory one, - // // so, it works fine but does not do what i want, need to call CommitTree() ! - // // 2. Use CommitTree() instead of RecalculateStateRoot() - // // worldState.RecalculateStateRoot(); // TODO: 1. just try keeping this and see what DumpState does - // worldState.CommitTree(blockNumber: 0); // TODO: 2. try this, this should make the DumpState work - // stateRoot = worldState.StateRoot; - // } - - // Console.WriteLine("--- print state 1 ---"); - // Console.WriteLine(stateReader.DumpState(stateRoot)); - - // using (var scope = worldState.BeginScope(null)) - // { - // // AddressA will be our sender - // // worldState.AddToBalanceAndCreateIfNotExists(new(accountAddr0), 1.Ether(), specProvider.GetSpec(new ForkActivation(0))); - // // // AddressB will be our receiver - // // worldState.CreateAccount(TestItem.AddressB, 0); - // // worldState.Commit(specProvider.GetSpec(new ForkActivation(0))); - // // worldState.RecalculateStateRoot(); - // } - - // // Create Genesis with the correct state root - // Block genesis = Build.A.Block.Genesis - // .WithStateRoot(stateRoot) - // .TestObject; - // // BlockTree blockTree = Build.A.BlockTree().TestObject; - // // Initialize BlockTree with Genesis (OfChainLength(1) ensures it's added) - // IHeaderStore headerStore = new HeaderStore(new MemDb(), new MemDb()); - - // BlockTreeBuilder blockTreeBuilder = Build.A.BlockTree(genesis); - // blockTreeBuilder.HeaderStore = headerStore; - // BlockTree blockTree = blockTreeBuilder - // .OfChainLength(1) - // .TestObject; - - // IncrementalTimestamper timestamper = new(); - // // var logManager = LimboLogs.Instance; - // BlocksConfig blocksConfig = new(); - // NoBlockRewards rewardCalculator = NoBlockRewards.Instance; - - // // new BlockhashCache(new HeaderStore(new MemDb(), new MemDb())) - // BlockhashProvider blockhashProvider = new(new BlockhashCache(headerStore, logManager), worldState, logManager); - // VirtualMachine vm = new(blockhashProvider, specProvider, logManager); - // TransactionProcessor txProcessor = new(new BlobBaseFeeCalculator(), specProvider, worldState, vm, new EthereumCodeInfoRepository(worldState), logManager); - - // IBlockProcessor.IBlockTransactionsExecutor txExecutor = - // new BlockProcessor.BlockValidationTransactionsExecutor( - // new ExecuteTransactionProcessorAdapter(txProcessor), worldState); - - // IHeaderValidator headerValidator = new HeaderValidator(blockTree, sealer, specProvider, logManager); - // IBlockValidator blockValidator = new BlockValidator(new TxValidator(specProvider.ChainId), headerValidator, - // new UnclesValidator(blockTree, headerValidator, logManager), specProvider, logManager); - - // BeaconBlockRootHandler beacon = new(txProcessor, worldState); - // IBlockProcessor blockProcessor = new BlockProcessor( - // specProvider, - // blockValidator, - // rewardCalculator, - // txExecutor, - // worldState, - // NullReceiptStorage.Instance, - // beacon, - // new BlockhashStore(worldState), - // logManager, - // new WithdrawalProcessor(worldState, logManager), - // new ExecutionRequestsProcessor(txProcessor)); - - // BranchProcessor branchProcessor = new( - // blockProcessor, - // specProvider, - // worldState, - // beacon, - // blockhashProvider, - // logManager - // ); - - // BlockchainProcessor blockchainProcessor = new( - // blockTree, - // branchProcessor, - // new MergeProcessingRecoveryStep(NoPoS.Instance), - // stateReader, - // logManager, - // BlockchainProcessor.Options.Default - // ); - - // // Tx to call contract - // var signedTx = Build.A.Transaction - // .WithTo(account123) - // // .WithData(new byte[] { 0x01 }) - // .WithNonce(0) - // .WithValue(0) - // .WithGasLimit(1_000_000) - // .WithGasPrice(10_0000_000) - // .WithSenderAddress(sender) - // .SignedAndResolved(TestItem.PrivateKeyA) - // .TestObject; - - // // Transaction transaction = Build.A.Transaction - // // .WithSenderAddress(sender) - // // .WithTo(to) - // // .WithGasLimit((long)gasLimit) - // // .WithMaxFeePerGas(baseFeePerGas * 2) - // // .WithValue(100) - // // .WithType(TxType.EIP1559) - // // .TestObject; - - - // var txSource = Substitute.For(); - // txSource.GetTransactions(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - // .Returns(new[] { signedTx }); - - // var producer = new TestBlockProducer( - // txSource, - // blockchainProcessor, - // worldState, - // sealer, - // blockTree, - // timestamper, - // specProvider, - // logManager, - // blocksConfig - // ); - - // // 4. Produce block - // var parent = blockTree.Head!.Header; - // var payloadAttributes = new PayloadAttributes - // { - // Timestamp = (ulong)parent.Timestamp + 1, - // PrevRandao = Hash256.Zero, - // SuggestedFeeRecipient = TestItem.AddressB, - // Withdrawals = Array.Empty(), - // ParentBeaconBlockRoot = Hash256.Zero - // }; - // var block = await producer.BuildBlock(payloadAttributes: payloadAttributes, token: CancellationToken.None); - // // Console.WriteLine($"Produced block {block!.ToString(Format.Full)}"); - - // Console.WriteLine("--- print state 2 ---"); - // Console.WriteLine(stateReader.DumpState(stateRoot)); - // } - - // [Test] - // public async Task SimpleTest() - // { - // // StateTree stateTree = TestItem.Tree.GetStateTree(); - // // TestItem.Tree.GetTrees(stateTree.TrieStore); - - // // ITrieStore store = new TestRawTrieStore(new MemDb()); - - // IDbProvider dbProvider = TestMemDbProvider.Init(); - // ILogManager logManager = LimboLogs.Instance; - - // PruningConfig pruningConfig = new(); - // TestFinalizedStateProvider finalizedStateProvider = new(pruningConfig.PruningBoundary); - // TrieStore store = new( - // new NodeStorage(dbProvider.StateDb), - // No.Pruning, - // Persist.EveryBlock, - // finalizedStateProvider, - // pruningConfig, - // LimboLogs.Instance); - // finalizedStateProvider.TrieStore = store; - - // StateTree oldStateTree = new(store.GetTrieStore(null), LimboLogs.Instance); - - // long blockNumber = 0; - // store.BeginBlockCommit(blockNumber); - - // TestItem.Tree.FillStateTreeWithTestAccounts(oldStateTree); - - // (StateTree stateTree, StorageTree _, Hash256 accountAddr0) = TestItem.Tree.GetTrees(store); - // WorldState worldState = new(store, dbProvider.CodeDb, logManager); - // StateReader stateReader = new(store, dbProvider.CodeDb, logManager); - // using (var scope = worldState.BeginScope(new BlockHeader { StateRoot = stateTree.RootHash })) - // { - // var bal = worldState.GetBalance(new(accountAddr0)); - // Console.WriteLine($"--- {bal} ---"); - // } - // NullSealEngine sealer = NullSealEngine.Instance; - // MainnetSpecProvider specProvider = MainnetSpecProvider.Instance; - - // Hash256 stateRoot; - // using (var scope = worldState.BeginScope(null)) - // { - // // AddressA will be our sender - // // worldState.AddToBalanceAndCreateIfNotExists(new(accountAddr0), 1.Ether(), specProvider.GetSpec(new ForkActivation(0))); - // // // AddressB will be our receiver - // // worldState.CreateAccount(TestItem.AddressB, 0); - // // worldState.Commit(specProvider.GetSpec(new ForkActivation(0))); - // // worldState.RecalculateStateRoot(); - // stateRoot = worldState.StateRoot; - // } - - // // Create Genesis with the correct state root - // Block genesis = Build.A.Block.Genesis - // .WithStateRoot(stateRoot) - // .TestObject; - // // BlockTree blockTree = Build.A.BlockTree().TestObject; - // // Initialize BlockTree with Genesis (OfChainLength(1) ensures it's added) - // IHeaderStore headerStore = new HeaderStore(new MemDb(), new MemDb()); - - // BlockTreeBuilder blockTreeBuilder = Build.A.BlockTree(genesis); - // blockTreeBuilder.HeaderStore = headerStore; - // BlockTree blockTree = blockTreeBuilder - // .OfChainLength(1) - // .TestObject; - // IncrementalTimestamper timestamper = new(); - // // var logManager = LimboLogs.Instance; - // BlocksConfig blocksConfig = new(); - // NoBlockRewards rewardCalculator = NoBlockRewards.Instance; - - // // BlockhashProvider blockhashProvider = new(blockTree, specProvider, worldState, logManager); - // // not blocktree ? - // BlockhashProvider blockhashProvider = new(new BlockhashCache(headerStore, logManager), worldState, logManager); - // VirtualMachine vm = new(blockhashProvider, specProvider, logManager); - // TransactionProcessor txProcessor = new(new BlobBaseFeeCalculator(), specProvider, worldState, vm, new EthereumCodeInfoRepository(worldState), logManager); - - // IBlockProcessor.IBlockTransactionsExecutor txExecutor = - // new BlockProcessor.BlockValidationTransactionsExecutor( - // new ExecuteTransactionProcessorAdapter(txProcessor), worldState); - - // IHeaderValidator headerValidator = new HeaderValidator(blockTree, sealer, specProvider, logManager); - // IBlockValidator blockValidator = new BlockValidator(new TxValidator(specProvider.ChainId), headerValidator, - // new UnclesValidator(blockTree, headerValidator, logManager), specProvider, logManager); - - // BeaconBlockRootHandler beacon = new(txProcessor, worldState); - // IBlockProcessor blockProcessor = new BlockProcessor( - // specProvider, - // blockValidator, - // rewardCalculator, - // txExecutor, - // worldState, - // NullReceiptStorage.Instance, - // beacon, - // new BlockhashStore(worldState), - // logManager, - // new WithdrawalProcessor(worldState, logManager), - // new ExecutionRequestsProcessor(txProcessor)); - - // BranchProcessor branchProcessor = new( - // blockProcessor, - // specProvider, - // worldState, - // beacon, - // blockhashProvider, - // logManager - // ); - - // BlockchainProcessor blockchainProcessor = new( - // blockTree, - // branchProcessor, - // new MergeProcessingRecoveryStep(NoPoS.Instance), - // stateReader, - // logManager, - // BlockchainProcessor.Options.Default - // ); - - // // 2. Transaction - // var signedTx = Build.A.Transaction - // .WithTo(TestItem.AddressB) - // // .WithData(new byte[] { 0x01 }) - // .WithNonce(0) - // .WithValue(100) - // .WithGasLimit(22000) - // .WithGasPrice(1000000000) - // .WithSenderAddress(new(accountAddr0)) - // // .SignedAndResolved() - // .TestObject; - - // // Transaction transaction = Build.A.Transaction - // // .WithSenderAddress(sender) - // // .WithTo(to) - // // .WithGasLimit((long)gasLimit) - // // .WithMaxFeePerGas(baseFeePerGas * 2) - // // .WithValue(100) - // // .WithType(TxType.EIP1559) - // // .TestObject; - - - // var txSource = Substitute.For(); - // txSource.GetTransactions(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - // .Returns(new[] { signedTx }); - - // var producer = new TestBlockProducer( - // txSource, - // blockchainProcessor, - // worldState, - // sealer, - // blockTree, - // timestamper, - // specProvider, - // logManager, - // blocksConfig - // ); - - // // 4. Produce block - // var parent = blockTree.Head!.Header; - // var payloadAttributes = new PayloadAttributes - // { - // Timestamp = (ulong)parent.Timestamp + 1, - // PrevRandao = Hash256.Zero, - // SuggestedFeeRecipient = TestItem.AddressB, - // Withdrawals = Array.Empty(), - // ParentBeaconBlockRoot = Hash256.Zero - // }; - // var block = await producer.BuildBlock(payloadAttributes: payloadAttributes, token: CancellationToken.None); - - // // worldState.CommitTree - - // // Assert.IsNotNull(block); - // Console.WriteLine($"Produced block {block?.Hash}"); - - - // // WitnessGeneratingBlockProcessingEnv env = new( - // // specProvider, - // // stateReader, - // // worldState, - // // new ReadOnlyBlockTree(blockTree), - // // sealer, - // // rewardCalculator, - // // logManager - // // ); - // } - - private async Task SaveWitnessToJsonFile(Witness witness) - { - var json = JsonSerializer.Serialize(witness, new JsonSerializerOptions { - WriteIndented = true, - IncludeFields = true, - Converters = { - new ByteArrayHexConverter() - } - }); - await File.WriteAllTextAsync("/Users/gugz/Documents/rpc_req/witness_from_test.json", - json - ); - } - - public class ByteArrayHexConverter : JsonConverter - { - // public override byte[] Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - // { - // string hex = reader.GetString()!; - // return Convert.FromHexString(hex.StartsWith("0x") ? hex.Substring(2) : hex); - // } - public override byte[] Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - string hex = reader.GetString()!; - - if (hex.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) - hex = hex.Substring(2); - - return Convert.FromHexString(hex); - } - - public override void Write(Utf8JsonWriter writer, byte[] value, JsonSerializerOptions options) - { - writer.WriteStringValue("0x" + Convert.ToHexString(value).ToLowerInvariant()); - // writer.WriteStringValue(Convert.ToHexString(value)); // uppercase hex - } - } - } From 9211ade54b92160df3cccd76628327132934219f Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Wed, 21 Jan 2026 12:00:03 +0900 Subject: [PATCH 04/87] feat: Use latest NMC master --- src/Nethermind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind b/src/Nethermind index 9ccf900f3..ce4d6c379 160000 --- a/src/Nethermind +++ b/src/Nethermind @@ -1 +1 @@ -Subproject commit 9ccf900f3715b79f2c883ce6ae04eea96d10196f +Subproject commit ce4d6c37928c19378d546388d861560db81afeea From 9a3950bfaef61d603dddd0edbd38dc94f519a7a8 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Wed, 21 Jan 2026 12:03:21 +0900 Subject: [PATCH 05/87] fix format --- src/Nethermind.Arbitrum/Data/RecordResult.cs | 2 +- .../Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind.Arbitrum/Data/RecordResult.cs b/src/Nethermind.Arbitrum/Data/RecordResult.cs index ffea5b935..3ca14254b 100644 --- a/src/Nethermind.Arbitrum/Data/RecordResult.cs +++ b/src/Nethermind.Arbitrum/Data/RecordResult.cs @@ -11,7 +11,7 @@ namespace Nethermind.Arbitrum.Data { public sealed class RecordResult { - public ulong Index { get; } + public ulong Index { get; } public Hash256 BlockHash { get; } public Dictionary Preimages { get; } public UserWasms? UserWasms { get; } diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs index 1d0f3822f..5800f3ae4 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs @@ -14,7 +14,7 @@ namespace Nethermind.Arbitrum.Execution.Stateless; -public interface IWitnessGeneratingPolyvalentEnv: IWitnessGeneratingBlockProcessingEnv +public interface IWitnessGeneratingPolyvalentEnv : IWitnessGeneratingBlockProcessingEnv { IBlockBuildingWitnessCollector CreateBlockBuildingWitnessCollector(); } From e7c4ca55bffafc0a4efd282f9400dfbd85521d10 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Wed, 21 Jan 2026 12:23:24 +0900 Subject: [PATCH 06/87] fix: PR reviews --- src/Nethermind.Arbitrum/Data/RecordResult.cs | 64 +++++++++---------- .../Execution/ArbitrumTransactionProcessor.cs | 1 - .../ArbitrumStatelessBlockProcessingEnv.cs | 1 - .../Stateless/ArbitrumWitnessCollector.cs | 3 +- ...trumWitnessGeneratingBlockProcessingEnv.cs | 2 - .../Modules/ArbitrumRpcModule.cs | 4 ++ 6 files changed, 37 insertions(+), 38 deletions(-) diff --git a/src/Nethermind.Arbitrum/Data/RecordResult.cs b/src/Nethermind.Arbitrum/Data/RecordResult.cs index 3ca14254b..d71931401 100644 --- a/src/Nethermind.Arbitrum/Data/RecordResult.cs +++ b/src/Nethermind.Arbitrum/Data/RecordResult.cs @@ -7,40 +7,40 @@ using WasmTarget = string; -namespace Nethermind.Arbitrum.Data +namespace Nethermind.Arbitrum.Data; + +public sealed class RecordResult { - public sealed class RecordResult + public ulong Index { get; } + public Hash256 BlockHash { get; } + public Dictionary Preimages { get; } + public UserWasms? UserWasms { get; } + + [JsonIgnore] + public Witness Witness { get; } + + public RecordResult(ulong messageIndex, Hash256 blockHash, Witness witness) { - public ulong Index { get; } - public Hash256 BlockHash { get; } - public Dictionary Preimages { get; } - public UserWasms? UserWasms { get; } - - [JsonIgnore] - public Witness Witness { get; } - - public RecordResult(ulong messageIndex, Hash256 blockHash, Witness witness) - { - Index = messageIndex; - BlockHash = blockHash; - Witness = witness; - UserWasms = null!; // TODO: add wasms - - Preimages = new(); - foreach (byte[] code in witness.Codes) - Preimages.Add(Keccak.Compute(code), code); - foreach (byte[] state in witness.State) - Preimages.Add(Keccak.Compute(state), state); - foreach (byte[] header in witness.Headers) - Preimages.Add(Keccak.Compute(header), header); - } + Index = messageIndex; + BlockHash = blockHash; + Witness = witness; + UserWasms = null!; // TODO: add wasms + + Preimages = new(); + foreach (byte[] code in witness.Codes) + Preimages.Add(Keccak.Compute(code), code); + foreach (byte[] state in witness.State) + Preimages.Add(Keccak.Compute(state), state); + foreach (byte[] header in witness.Headers) + Preimages.Add(Keccak.Compute(header), header); } +} - public sealed record class ActivatedWasm( - Dictionary Value - ); +public sealed record class ActivatedWasm( + Dictionary Value +); + +public sealed record class UserWasms( + Dictionary Value +); - public sealed record class UserWasms( - Dictionary Value - ); -} diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs index 616b000b4..f4cb2ee64 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs @@ -467,7 +467,6 @@ private ArbitrumTransactionProcessorResult ProcessArbitrumInternalTransaction( ValueHash256 prevHash = ValueKeccak.Zero; if (blCtx.Header.Number > 0) { - // Can't we just do: blCtx.Header.ParentHash ? or else pass my witnessGeneratingHeaderFinder prevHash = blockTree.FindBlockHash(blCtx.Header.Number - 1); } diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs index 13c625218..c9ca32eff 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs @@ -81,7 +81,6 @@ private IBlockProcessor GetProcessor() ); } - private ITransactionProcessor CreateTransactionProcessor(IWorldState state, StatelessBlockTree blockFinder) { BlockhashProvider blockhashProvider = new(blockFinder, state, logManager); diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs index 142eb8630..ab73edfe2 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs @@ -43,10 +43,9 @@ public class ArbitrumWitnessCollector( throw new InvalidOperationException($"ArbOS genesisBlockNum mismatch. ArbOS={genesisBlockNum}, local={specHelper.GenesisBlockNum}."); } - Block? producedBlock = await blockProducer.BuildBlock(parentHeader: parentHeader, payloadAttributes: payloadAttributes); if (producedBlock?.Hash is null) - throw new Exception("Failed to build block or block has no hash."); + throw new NullReferenceException($"Failed to build block with parent header number: {parentHeader.Number} and hash: {parentHeader.Hash}"); (byte[][] stateNodes, byte[][] codes, byte[][] keys) = worldState.GetWitness(parentHeader, trieStore.TouchedNodesRlp); diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs index 5800f3ae4..2d187f0c6 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs @@ -40,8 +40,6 @@ public IExistingBlockWitnessCollector CreateExistingBlockWitnessCollector() public IBlockBuildingWitnessCollector CreateBlockBuildingWitnessCollector() { - Console.WriteLine("--- In Arb WitnessGeneratingBlockProcessingEnv.CreateBlockBuildingWitnessCollector() ---"); - ArbitrumBlockProducer blockProducer = new( txSource, chainProcessor, diff --git a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs index aed6b3c39..e4fd83adc 100644 --- a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs +++ b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs @@ -406,6 +406,10 @@ public async Task> RecordBlockCreation(RecordBlockCr if (builtBlock.Hash is null) return ResultWrapper.Fail("Failed to build block or block has no hash."); + Hash256? headerHash = blockTree.FindHash(blockNumber); + if (headerHash is null || headerHash != builtBlock.Hash) + return ResultWrapper.Fail($"Built block hash: {builtBlock.Hash} does not match canonical block header hash: {headerHash}"); + RecordResult result = new(parameters.Index, builtBlock.Hash!, witness); return ResultWrapper.Success(result); } From 79ed96203d06671c55df4c12b421f7614a664ed3 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Wed, 21 Jan 2026 12:50:32 +0900 Subject: [PATCH 07/87] chore: Use header.ParentHash instead of findBlockHash --- .../Execution/ArbitrumTransactionProcessorTests.cs | 14 -------------- .../Execution/ArbitrumTransactionProcessor.cs | 3 +-- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/ArbitrumTransactionProcessorTests.cs b/src/Nethermind.Arbitrum.Test/Execution/ArbitrumTransactionProcessorTests.cs index 0312431f7..75f097f88 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/ArbitrumTransactionProcessorTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/ArbitrumTransactionProcessorTests.cs @@ -72,7 +72,6 @@ public void ProcessArbitrumRetryTransaction_RetryableExists_ReturnsOkTransaction worldState, TestWasmStore.Create(), virtualMachine, - blockTree, _logManager, new EthereumCodeInfoRepository(worldState) ); @@ -159,7 +158,6 @@ public void ProcessArbitrumRetryTransaction_RetryableDoesNotExist_TracesErrorBut worldState, TestWasmStore.Create(), virtualMachine, - blockTree, _logManager, new EthereumCodeInfoRepository(worldState) ); @@ -225,7 +223,6 @@ public void ProcessArbitrumDepositTransaction_ValidTransaction_ReturnsOkTransact worldState, TestWasmStore.Create(), virtualMachine, - blockTree, _logManager, new EthereumCodeInfoRepository(worldState) ); @@ -289,7 +286,6 @@ public void ProcessArbitrumDepositTransaction_MalformedTx_TracesErrorButReturnsO worldState, TestWasmStore.Create(), virtualMachine, - blockTree, _logManager, new EthereumCodeInfoRepository(worldState) ); @@ -344,7 +340,6 @@ public void GasChargingHook_TxWithEnoughGas_TipsNetworkCorrectly() worldState, TestWasmStore.Create(), virtualMachine, - blockTree, _logManager, new EthereumCodeInfoRepository(worldState) ); @@ -1647,7 +1642,6 @@ public void ArbitrumTransaction_WithArbitrumBlockHeader_ProcessesCorrectly() worldState, TestWasmStore.Create(), virtualMachine, - blockTree, _logManager, new EthereumCodeInfoRepository(worldState) ); @@ -1734,7 +1728,6 @@ public void ArbitrumTransaction_WithoutNoBaseFee_UsesBlockBaseFee() worldState, TestWasmStore.Create(), virtualMachine, - blockTree, _logManager, new EthereumCodeInfoRepository(worldState) ); @@ -1816,7 +1809,6 @@ public void ArbitrumTransaction_WithArbitrumBlockHeader_UsesOriginalBaseFeeForGa worldState, TestWasmStore.Create(), virtualMachine, - blockTree, _logManager, new EthereumCodeInfoRepository(worldState) ); @@ -1931,7 +1923,6 @@ public void StartBlockTransaction_WhenQueueHasOnlyDeletedRetryables_ClearsQueueC worldState, TestWasmStore.Create(), virtualMachine, - blockTree, _logManager, new EthereumCodeInfoRepository(worldState) ); @@ -2015,7 +2006,6 @@ public void TryReapOneRetryable_WhenTimeoutIsZero_RemovesFromQueueAndReturnsEarl worldState, TestWasmStore.Create(), virtualMachine, - blockTree, _logManager, new EthereumCodeInfoRepository(worldState) ); @@ -2119,7 +2109,6 @@ public void TryReapOneRetryable_WhenBothRetryablesHaveTimeoutZero_RemovesBothFro worldState, TestWasmStore.Create(), virtualMachine, - blockTree, _logManager, new EthereumCodeInfoRepository(worldState) ); @@ -2204,7 +2193,6 @@ public void TryReapOneRetryable_WhenFirstRetryableDeletedSecondExpired_Processes worldState, TestWasmStore.Create(), virtualMachine, - blockTree, _logManager, new EthereumCodeInfoRepository(worldState) ); @@ -3129,7 +3117,6 @@ public void ArbitrumRetryTransaction_WhenValidationFailsInBuildUpMode_RevertsAll worldState, TestWasmStore.Create(), virtualMachine, - blockTree, _logManager, new EthereumCodeInfoRepository(worldState) ); @@ -3222,7 +3209,6 @@ public void ArbitrumRetryTransaction_WhenPreProcessingFailsInBuildUpMode_Reverts worldState, TestWasmStore.Create(), virtualMachine, - blockTree, _logManager, new EthereumCodeInfoRepository(worldState) ); diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs index c6a66be7e..1119e0d5b 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumTransactionProcessor.cs @@ -34,7 +34,6 @@ public class ArbitrumTransactionProcessor( IWorldState worldState, IWasmStore wasmStore, ArbitrumVirtualMachine virtualMachine, - IBlockTree blockTree, ILogManager logManager, ICodeInfoRepository? codeInfoRepository ) : TransactionProcessorBase(blobBaseFeeCalculator, specProvider, worldState, virtualMachine, codeInfoRepository, logManager) @@ -467,7 +466,7 @@ private ArbitrumTransactionProcessorResult ProcessArbitrumInternalTransaction( ValueHash256 prevHash = ValueKeccak.Zero; if (blCtx.Header.Number > 0) { - prevHash = blockTree.FindBlockHash(blCtx.Header.Number - 1); + prevHash = blCtx.Header.ParentHash!; } if (_arbosState!.CurrentArbosVersion >= ArbosVersion.ParentBlockHashSupport) From b25226a1e461d6925b93a347b2fdab92fb4a874d Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Wed, 21 Jan 2026 21:10:19 +0900 Subject: [PATCH 08/87] fix: Sync with NMC branch to return Witness object directly --- src/Nethermind | 2 +- .../Execution/Stateless/ArbitrumWitnessCollector.cs | 12 +----------- .../ArbitrumWitnessGeneratingBlockProcessingEnv.cs | 4 +--- ...trumWitnessGeneratingBlockProcessingEnvFactory.cs | 7 +++---- 4 files changed, 6 insertions(+), 19 deletions(-) diff --git a/src/Nethermind b/src/Nethermind index ce4d6c379..7945a5355 160000 --- a/src/Nethermind +++ b/src/Nethermind @@ -1 +1 @@ -Subproject commit ce4d6c37928c19378d546388d861560db81afeea +Subproject commit 7945a535536f086b4401a64074ec88821e9a1b58 diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs index ab73edfe2..b7dc10fab 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs @@ -19,9 +19,7 @@ public interface IBlockBuildingWitnessCollector } public class ArbitrumWitnessCollector( - WitnessGeneratingHeaderFinder headerFinder, WitnessGeneratingWorldState worldState, - WitnessCapturingTrieStore trieStore, IBlockProducer blockProducer, ISpecProvider specProvider, IArbitrumSpecHelper specHelper) : IBlockBuildingWitnessCollector @@ -47,15 +45,7 @@ public class ArbitrumWitnessCollector( if (producedBlock?.Hash is null) throw new NullReferenceException($"Failed to build block with parent header number: {parentHeader.Number} and hash: {parentHeader.Hash}"); - (byte[][] stateNodes, byte[][] codes, byte[][] keys) = worldState.GetWitness(parentHeader, trieStore.TouchedNodesRlp); - - Witness witness = new() - { - Headers = headerFinder.GetWitnessHeaders(parentHeader.Hash!), - Codes = codes, - State = stateNodes, - Keys = keys - }; + Witness witness = worldState.GetWitness(parentHeader); return (producedBlock, witness); } diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs index 2d187f0c6..e4f7941fb 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs @@ -27,8 +27,6 @@ public class ArbitrumWitnessGeneratingBlockProcessingEnv( IBlocksConfig blocksConfig, ISpecProvider specProvider, IArbitrumSpecHelper specHelper, - WitnessGeneratingHeaderFinder witnessGenHeaderFinder, - WitnessCapturingTrieStore witnessCapturingTrieStore, ILogManager logManager) : IWitnessGeneratingPolyvalentEnv { public IExistingBlockWitnessCollector CreateExistingBlockWitnessCollector() @@ -52,6 +50,6 @@ public IBlockBuildingWitnessCollector CreateBlockBuildingWitnessCollector() logManager, blocksConfig); - return new ArbitrumWitnessCollector(witnessGenHeaderFinder, witnessGeneratingWorldState, witnessCapturingTrieStore, blockProducer, specProvider, specHelper); + return new ArbitrumWitnessCollector(witnessGeneratingWorldState, blockProducer, specProvider, specHelper); } } diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index e7a44ec6e..d0322c2d7 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -87,7 +87,9 @@ public IWitnessGeneratingBlockProcessingEnvScope CreateScope() ILifetimeScope envLifetimeScope = rootLifetimeScope.BeginLifetimeScope((builder) => builder .AddScoped(stateReader) - .AddScoped(new WitnessGeneratingWorldState(worldState, stateReader)) + + .AddScoped(builder => new WitnessGeneratingHeaderFinder(builder.Resolve())) + .AddScoped(builder => new WitnessGeneratingWorldState(worldState, stateReader, trieStore, (builder.Resolve() as WitnessGeneratingHeaderFinder)!)) .AddScoped(_ => CreateWitnessBlocksConfig(blocksConfig)) @@ -95,7 +97,6 @@ public IWitnessGeneratingBlockProcessingEnvScope CreateScope() .AddScoped(_ => new WasmDb(new MemDb())) // new instance i think? to check but might have to create a new recording IWasmDb passed to it .AddScoped(ctx => new WasmStore(ctx.Resolve(), ctx.Resolve(), cacheTag: 1)) - .AddScoped(builder => new WitnessGeneratingHeaderFinder(builder.Resolve())) .AddScoped(builder => CreateTransactionProcessor( builder.Resolve(), @@ -129,8 +130,6 @@ public IWitnessGeneratingBlockProcessingEnvScope CreateScope() builder.Resolve(), builder.Resolve(), builder.Resolve(), - (builder.Resolve() as WitnessGeneratingHeaderFinder)!, - trieStore, logManager))); return new ExecutionRecordingScope(envLifetimeScope); From 764e860a1a40ec0176a13fcaf0a68259916080f6 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Wed, 21 Jan 2026 21:11:19 +0900 Subject: [PATCH 09/87] fix: Improve comparison between built block hash and canonical hash --- .../ArbitrumEvmInstructions.Environment.cs | 4 +-- .../Modules/ArbitrumRpcModule.cs | 27 ++++++++++++++++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/Nethermind.Arbitrum/Evm/ArbitrumEvmInstructions.Environment.cs b/src/Nethermind.Arbitrum/Evm/ArbitrumEvmInstructions.Environment.cs index 134a7865d..7dad729a9 100644 --- a/src/Nethermind.Arbitrum/Evm/ArbitrumEvmInstructions.Environment.cs +++ b/src/Nethermind.Arbitrum/Evm/ArbitrumEvmInstructions.Environment.cs @@ -145,10 +145,10 @@ public static EvmExceptionType InstructionBlockHash(VirtualMachine return EvmExceptionType.None; } - /// + /// /// Same as the base implementation but omits any optimization so that it always goes through /// the world state to get and record the bytecode. Used for witness generation. - /// + /// [SkipLocalsInit] public static EvmExceptionType InstructionExtCodeSize(VirtualMachine vm, ref EvmStack stack, diff --git a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs index e4fd83adc..c46d340ee 100644 --- a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs +++ b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Collections.Concurrent; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Text.Json; using Nethermind.Arbitrum.Config; @@ -404,11 +405,29 @@ public async Task> RecordBlockCreation(RecordBlockCr (Block builtBlock, Witness witness) = await witnessCollector.BuildBlockAndGetWitness(parent, payload); if (builtBlock.Hash is null) - return ResultWrapper.Fail("Failed to build block or block has no hash."); + return ResultWrapper.Fail($"Failed to build block {blockNumber} or block has no hash."); - Hash256? headerHash = blockTree.FindHash(blockNumber); - if (headerHash is null || headerHash != builtBlock.Hash) - return ResultWrapper.Fail($"Built block hash: {builtBlock.Hash} does not match canonical block header hash: {headerHash}"); + // Sometimes, it seems RecordBlockCreation is called slightly before the actual block is finalized/committed to the database. + // So we need to wait for the block to be available in the database. + Hash256? canonicalHash = null; + Stopwatch sw = Stopwatch.StartNew(); + while (sw.ElapsedMilliseconds <= arbitrumConfig.MessageLagMs) + { + canonicalHash = blockTree.FindCanonicalBlockInfo(blockNumber)?.BlockHash; + + if (canonicalHash is null) + { + await Task.Delay(10); + continue; + } + + break; + } + + if (canonicalHash is null) + return ResultWrapper.Fail(ArbitrumRpcErrors.BlockNotFound(blockNumber)); + else if (canonicalHash != builtBlock.Hash) + return ResultWrapper.Fail($"Built block hash: {builtBlock.Hash} does not match canonical block header hash: {canonicalHash}"); RecordResult result = new(parameters.Index, builtBlock.Hash!, witness); return ResultWrapper.Success(result); From 2a87f191308e6bafa6eae2174cec2ca74a17f3dd Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Wed, 21 Jan 2026 22:09:32 +0900 Subject: [PATCH 10/87] fix format --- src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs index c46d340ee..04af0a3ad 100644 --- a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs +++ b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs @@ -427,7 +427,7 @@ public async Task> RecordBlockCreation(RecordBlockCr if (canonicalHash is null) return ResultWrapper.Fail(ArbitrumRpcErrors.BlockNotFound(blockNumber)); else if (canonicalHash != builtBlock.Hash) - return ResultWrapper.Fail($"Built block hash: {builtBlock.Hash} does not match canonical block header hash: {canonicalHash}"); + return ResultWrapper.Fail($"Built block hash: {builtBlock.Hash} does not match canonical block header hash: {canonicalHash}"); RecordResult result = new(parameters.Index, builtBlock.Hash!, witness); return ResultWrapper.Success(result); From 7031beffeead0fb847e8c71af0a193ee4d5e56e4 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 23 Jan 2026 11:25:48 +0900 Subject: [PATCH 11/87] fix: Empty calldata should still set some state even if does not impact block hash --- src/Nethermind.Arbitrum/Arbos/Storage/ArbosStorage.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Nethermind.Arbitrum/Arbos/Storage/ArbosStorage.cs b/src/Nethermind.Arbitrum/Arbos/Storage/ArbosStorage.cs index 43ae069ac..c885de2d1 100644 --- a/src/Nethermind.Arbitrum/Arbos/Storage/ArbosStorage.cs +++ b/src/Nethermind.Arbitrum/Arbos/Storage/ArbosStorage.cs @@ -120,17 +120,14 @@ public void Set(byte[] value) ulong offset = 1; ReadOnlySpan span = value.AsSpan(); - while (span.Length > 32) + while (span.Length >= 32) { Set(offset, Hash256.FromBytesWithPadding(span[..32])); span = span[32..]; offset++; } - if (span.Length > 0) - { - Set(offset, Hash256.FromBytesWithPadding(span)); - } + Set(offset, Hash256.FromBytesWithPadding(span)); } public byte[] GetBytes() From 9eff0c4441971a29c11e183855c68e83c46acdbb Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Thu, 29 Jan 2026 13:04:23 +0900 Subject: [PATCH 12/87] feat: Capture user wasms --- .../Stylus/Infrastructure/TestStylusVmHost.cs | 8 ++ .../Arbos/Stylus/StylusNativeTests.cs | 38 +++++- .../ArbitrumWitnessGenerationTests.cs | 4 +- .../ArbitrumRpcTestBlockchain.cs | 4 +- .../Rpc/ArbitrumRpcModuleTests.cs | 5 +- src/Nethermind.Arbitrum/ArbitrumPlugin.cs | 12 +- .../ArbitrumRpcModuleFactory.cs | 4 +- .../Arbos/Programs/IStylusVmHost.cs | 4 + .../Arbos/Programs/StylusPrograms.cs | 9 +- .../Arbos/Stylus/StylusNative.cs | 19 ++- .../Data/DigestMessageParameters.cs | 3 +- src/Nethermind.Arbitrum/Data/RecordResult.cs | 31 ++--- .../Evm/ArbitrumVirtualMachine.cs | 14 +- .../ArbitrumStatelessBlockProcessingEnv.cs | 9 +- .../Stateless/ArbitrumUserWasmsRecorder.cs | 14 ++ .../Execution/Stateless/ArbitrumWitness.cs | 13 ++ .../Stateless/ArbitrumWitnessCollector.cs | 8 +- ...trumWitnessGeneratingBlockProcessingEnv.cs | 3 +- ...nessGeneratingBlockProcessingEnvFactory.cs | 123 ++++++++++-------- .../Modules/ArbitrumRpcModule.cs | 12 +- .../ArbitrumRpcModuleWithComparison.cs | 4 +- .../Stylus/StylusTargetConfig.cs | 6 +- 22 files changed, 229 insertions(+), 118 deletions(-) create mode 100644 src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumUserWasmsRecorder.cs create mode 100644 src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitness.cs diff --git a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/Infrastructure/TestStylusVmHost.cs b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/Infrastructure/TestStylusVmHost.cs index 7dce8e500..2b6774f79 100644 --- a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/Infrastructure/TestStylusVmHost.cs +++ b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/Infrastructure/TestStylusVmHost.cs @@ -6,6 +6,7 @@ using Nethermind.Arbitrum.Evm; using Nethermind.Arbitrum.Stylus; using Nethermind.Core; +using Nethermind.Core.Crypto; using Nethermind.Core.Specs; using Nethermind.Evm; using Nethermind.Evm.State; @@ -20,6 +21,7 @@ public class TestStylusVmHost( IWorldState worldState, IWasmStore wasmStore, IReleaseSpec spec, + bool isRecordingExecution = false, ulong currentArbosVersion = ArbosVersion.Forty) : IStylusVmHost { private readonly BlockExecutionContext _blockExecutionContext = blockExecutionContext; @@ -32,6 +34,7 @@ public class TestStylusVmHost( public VmState VmState { get; } = vmState; public IReleaseSpec Spec { get; } = spec; public ulong CurrentArbosVersion { get; } = currentArbosVersion; + public bool IsRecordingExecution => isRecordingExecution; public StylusEvmResult StylusCall(ExecutionType kind, Address to, ReadOnlyMemory input, ulong gasLeftReportedByRust, ulong gasRequestedByRust, in UInt256 value) { @@ -42,4 +45,9 @@ public StylusEvmResult StylusCreate(ReadOnlyMemory initCode, in UInt256 en { throw new NotImplementedException(); } + + public void RecordUserWasm(ValueHash256 moduleHash, IReadOnlyDictionary asmMap) + { + throw new NotImplementedException(); + } } diff --git a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs index b0ac20db7..6edaf0319 100644 --- a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs +++ b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs @@ -4,11 +4,13 @@ using System.Security.Cryptography; using System.Text; using FluentAssertions; +using Nethermind.Arbitrum.Arbos.Programs; using Nethermind.Arbitrum.Arbos.Stylus; using Nethermind.Arbitrum.Test.Arbos.Stylus.Infrastructure; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using StylusNative = Nethermind.Arbitrum.Arbos.Stylus.StylusNative; +using NSubstitute; namespace Nethermind.Arbitrum.Test.Arbos.Stylus; @@ -251,18 +253,26 @@ public static void Call_CounterContractSetsValue_UpdatesStorageThroughNativeApi( ulong gas = 1_000_000; uint arbosTag = 0; + ValueHash256 moduleHash = new(); + + IStylusVmHost vmHost = Substitute.For(); + vmHost.IsRecordingExecution.Returns(false); + // Get number (should be 0 initially) byte[] getNumberCalldata = CounterContractCallData.GetNumberCalldata(); - StylusNativeResult getNumberResult1 = StylusNative.Call(asmResult.Value!, getNumberCalldata, config, apiApi, evmData, true, arbosTag, ref gas); + vmHost.VmState.Env.InputData.Returns(getNumberCalldata); + StylusNativeResult getNumberResult1 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); getNumberResult1.Value.Should().BeEquivalentTo(new byte[32]); // Set number to 9 byte[] setNumberCalldata = CounterContractCallData.GetSetNumberCalldata(9); - StylusNativeResult setNumberResult = StylusNative.Call(asmResult.Value!, setNumberCalldata, config, apiApi, evmData, true, arbosTag, ref gas); + vmHost.VmState.Env.InputData.Returns(setNumberCalldata); + StylusNativeResult setNumberResult = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); setNumberResult.Value.Should().BeEmpty(); // Get number again (should now be 9) - StylusNativeResult getNumberResult2 = StylusNative.Call(asmResult.Value!, getNumberCalldata, config, apiApi, evmData, true, arbosTag, ref gas); + vmHost.VmState.Env.InputData.Returns(getNumberCalldata); + StylusNativeResult getNumberResult2 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); byte[] expected = new byte[32]; expected[^1] = 9; // Last byte should be 9 after setNumber(9) @@ -292,19 +302,27 @@ public static void Call_CounterContractIncrement_EmitsLogsAndUpdatesStorageThrou ulong gas = 1_000_000; uint arbosTag = 0; + ValueHash256 moduleHash = new(); + + IStylusVmHost vmHost = Substitute.For(); + vmHost.IsRecordingExecution.Returns(false); + // Get number (should be 0 initially) byte[] getNumberCalldata = CounterContractCallData.GetNumberCalldata(); - StylusNativeResult getNumberResult1 = StylusNative.Call(asmResult.Value!, getNumberCalldata, config, apiApi, evmData, true, arbosTag, ref gas); + vmHost.VmState.Env.InputData.Returns(getNumberCalldata); + StylusNativeResult getNumberResult1 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); getNumberResult1.Value.Should().BeEquivalentTo(new byte[32]); // Increment number from 0 to 1 byte[] incrementNumberCalldata = CounterContractCallData.GetIncrementCalldata(); + vmHost.VmState.Env.InputData.Returns(incrementNumberCalldata); StylusNativeResult incrementNumberResult = - StylusNative.Call(asmResult.Value!, incrementNumberCalldata, config, apiApi, evmData, true, arbosTag, ref gas); + StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); incrementNumberResult.IsSuccess.Should().BeTrue(); // Get number again (should now be 1) - StylusNativeResult getNumberResult2 = StylusNative.Call(asmResult.Value!, getNumberCalldata, config, apiApi, evmData, true, arbosTag, ref gas); + vmHost.VmState.Env.InputData.Returns(getNumberCalldata); + StylusNativeResult getNumberResult2 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); byte[] expected = new byte[32]; expected[^1] = 1; @@ -343,7 +361,13 @@ public static void Call_KeccakCalculation_ReturnsValidHash() ulong gas = 1_000_000; uint arbosTag = 0; - StylusNativeResult resultData = StylusNative.Call(asmResult.Value!, callDataBytes, config, apiApi, evmData, true, arbosTag, ref gas); + ValueHash256 moduleHash = new(); + + IStylusVmHost vmHost = Substitute.For(); + vmHost.IsRecordingExecution.Returns(false); + vmHost.VmState.Env.InputData.Returns(callDataBytes); + + StylusNativeResult resultData = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); resultData.Value.Should().BeEquivalentTo(hash); } diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index bc03a3dc3..eb9a85c62 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -40,10 +40,10 @@ public async Task RecordBlockCreation_Witness_AllowsStatelessExecution(ulong mes .WithRecording(recording) .Build(); - ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation(new RecordBlockCreationParameters(digestMessage.Index, digestMessage.Message)); + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation(new RecordBlockCreationParameters(digestMessage.Index, digestMessage.Message, WasmTargets: [])); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestMessage.Index); - Witness witness = recordResult.Witness; + ArbitrumWitness witness = recordResult.Witness; ISpecProvider specProvider = FullChainSimulationChainSpecProvider.CreateDynamicSpecProvider(); ArbitrumStatelessBlockProcessingEnv blockProcessingEnv = diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs index 78e72538a..dcf5014c3 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs @@ -30,7 +30,7 @@ using Nethermind.State; using Nethermind.TxPool; using Nethermind.Wallet; -using Nethermind.Consensus.Stateless; +using Nethermind.Arbitrum.Execution.Stateless; namespace Nethermind.Arbitrum.Test.Infrastructure; @@ -305,7 +305,7 @@ private static ArbitrumRpcTestBlockchain CreateInternal(ArbitrumRpcTestBlockchai chain.Container.Resolve(), new Nethermind.Arbitrum.Config.VerifyBlockHashConfig(), // Disabled for tests new Nethermind.Serialization.Json.EthereumJsonSerializer(), - chain.Container.Resolve(), + chain.Container.Resolve(), chain.Container.Resolve(), null) // No ProcessExitSource in tests .Create()); diff --git a/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs b/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs index cf1c3c2e2..12783067c 100644 --- a/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs +++ b/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs @@ -22,7 +22,6 @@ using Nethermind.Arbitrum.Execution; using Nethermind.Consensus.Processing; using Nethermind.Core.Specs; -using Nethermind.Consensus.Stateless; using Nethermind.Arbitrum.Execution.Stateless; namespace Nethermind.Arbitrum.Test.Rpc @@ -45,7 +44,7 @@ public abstract class ArbitrumRpcModuleTests private IArbitrumConfig _arbitrumConfig = null!; private Mock _mainProcessingContextMock = null!; private ISpecProvider _specProvider = null!; - private Mock _witnessGeneratingBlockProcessingEnvFactory = null!; + private Mock _witnessGeneratingBlockProcessingEnvFactory = null!; [SetUp] public void Setup() { @@ -59,7 +58,7 @@ public void Setup() _specHelper = new Mock(); _blockProcessingQueue = new Mock(); _specProvider = FullChainSimulationChainSpecProvider.CreateDynamicSpecProvider(_chainSpec); - _witnessGeneratingBlockProcessingEnvFactory = new Mock(); + _witnessGeneratingBlockProcessingEnvFactory = new Mock(); _initializer = new ArbitrumBlockTreeInitializer( _chainSpec, diff --git a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs index 7aea28558..fb80aac3f 100644 --- a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs +++ b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs @@ -106,7 +106,7 @@ public Task InitRpcModules() _api.Config(), _api.Config(), _api.EthereumJsonSerializer, - _api.Context.Resolve(), + _api.Context.Resolve(), _api.Config(), _api.ProcessExit ); @@ -203,13 +203,13 @@ protected override void Load(ContainerBuilder builder) .AddScoped() .AddSingleton() + .AddSingleton() .AddSingleton(context => { IWasmDb wasmDb = context.Resolve(); - return new WasmStore(wasmDb, new StylusTargetConfig(), cacheTag: 1); + IStylusTargetConfig stylusTargetConfig = context.Resolve(); + return new WasmStore(wasmDb, stylusTargetConfig, cacheTag: 1); }) - .AddSingleton() - .AddSingleton() @@ -243,7 +243,9 @@ protected override void Load(ContainerBuilder builder) // Rpcs .AddSingleton() .Bind, ArbitrumEthModuleFactory>() - .AddSingleton(); + + .AddSingleton() + .Bind(); if (blocksConfig.BuildBlocksOnMainState) builder.AddSingleton(); diff --git a/src/Nethermind.Arbitrum/ArbitrumRpcModuleFactory.cs b/src/Nethermind.Arbitrum/ArbitrumRpcModuleFactory.cs index 756e62eb0..96cc3e84b 100644 --- a/src/Nethermind.Arbitrum/ArbitrumRpcModuleFactory.cs +++ b/src/Nethermind.Arbitrum/ArbitrumRpcModuleFactory.cs @@ -14,7 +14,7 @@ using Nethermind.Logging; using Nethermind.Serialization.Json; using Nethermind.Specs.ChainSpecStyle; -using Nethermind.Consensus.Stateless; +using Nethermind.Arbitrum.Execution.Stateless; namespace Nethermind.Arbitrum; @@ -31,7 +31,7 @@ public sealed class ArbitrumRpcModuleFactory( IArbitrumConfig arbitrumConfig, IVerifyBlockHashConfig verifyBlockHashConfig, IJsonSerializer jsonSerializer, - IWitnessGeneratingBlockProcessingEnvFactory witnessGeneratingBlockProcessingEnvFactory, + IArbitrumWitnessGeneratingBlockProcessingEnvFactory witnessGeneratingBlockProcessingEnvFactory, IBlocksConfig blocksConfig, IProcessExitSource? processExitSource = null) : ModuleFactoryBase { diff --git a/src/Nethermind.Arbitrum/Arbos/Programs/IStylusVmHost.cs b/src/Nethermind.Arbitrum/Arbos/Programs/IStylusVmHost.cs index ad504afcc..7d42f9327 100644 --- a/src/Nethermind.Arbitrum/Arbos/Programs/IStylusVmHost.cs +++ b/src/Nethermind.Arbitrum/Arbos/Programs/IStylusVmHost.cs @@ -4,6 +4,7 @@ using Nethermind.Arbitrum.Evm; using Nethermind.Arbitrum.Stylus; using Nethermind.Core; +using Nethermind.Core.Crypto; using Nethermind.Core.Specs; using Nethermind.Evm; using Nethermind.Evm.State; @@ -20,9 +21,12 @@ public interface IStylusVmHost public VmState VmState { get; } public IReleaseSpec Spec { get; } ulong CurrentArbosVersion { get; } + bool IsRecordingExecution { get; } StylusEvmResult StylusCall(ExecutionType kind, Address to, ReadOnlyMemory input, ulong gasLeftReportedByRust, ulong gasRequestedByRust, in UInt256 value); StylusEvmResult StylusCreate(ReadOnlyMemory initCode, in UInt256 endowment, UInt256? salt, ulong gasLimit); + + void RecordUserWasm(ValueHash256 moduleHash, IReadOnlyDictionary asmMap); } diff --git a/src/Nethermind.Arbitrum/Arbos/Programs/StylusPrograms.cs b/src/Nethermind.Arbitrum/Arbos/Programs/StylusPrograms.cs index c7a839567..2967b45bb 100644 --- a/src/Nethermind.Arbitrum/Arbos/Programs/StylusPrograms.cs +++ b/src/Nethermind.Arbitrum/Arbos/Programs/StylusPrograms.cs @@ -203,8 +203,8 @@ public StylusOperationResult CallProgram(IStylusVmHost vmHost, TracingIn }; IStylusEvmApi evmApi = new StylusEvmApi(vmHost, vmHost.VmState.Env.ExecutingAccount, memoryModel); - StylusNativeResult callResult = StylusNative.Call(localAsm.Value, vmHost.VmState.Env.InputData.ToArray(), stylusConfig, evmApi, evmData, - debugMode, arbosTag, ref gasAvailable); + StylusNativeResult callResult = StylusNative.Call(localAsm.Value, stylusConfig, evmApi, evmData, + debugMode, vmHost, in moduleHash, arbosTag, ref gasAvailable); vmHost.VmState.Gas = ArbitrumGasPolicy.FromLong((long)gasAvailable); @@ -633,10 +633,11 @@ private static StylusOperationResult ActivateProgramInte // Native compilation tasks tasks.AddRange(nativeTargets.Select(target => Task.Run(async () => { - StylusNativeResult result = await CompileNativeWithTimeout(wasm, stylusVersion, debugMode, target, true, CraneliftActivationTimeout); + bool cranelift = false; + StylusNativeResult result = await CompileNativeWithTimeout(wasm, stylusVersion, debugMode, target, cranelift, CraneliftActivationTimeout); if (!result.IsSuccess) - result = await CompileNativeWithTimeout(wasm, stylusVersion, debugMode, target, false, CraneliftActivationTimeout); + result = await CompileNativeWithTimeout(wasm, stylusVersion, debugMode, target, !cranelift, CraneliftActivationTimeout); results.Add(result.IsSuccess ? new StylusActivateTaskResult(target, result.Value, null, StylusOperationResultType.Success) diff --git a/src/Nethermind.Arbitrum/Arbos/Stylus/StylusNative.cs b/src/Nethermind.Arbitrum/Arbos/Stylus/StylusNative.cs index d3b6fe8a8..99b26ff8b 100644 --- a/src/Nethermind.Arbitrum/Arbos/Stylus/StylusNative.cs +++ b/src/Nethermind.Arbitrum/Arbos/Stylus/StylusNative.cs @@ -4,6 +4,8 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.InteropServices; using System.Text; +using Nethermind.Arbitrum.Arbos.Programs; +using Nethermind.Core.Crypto; namespace Nethermind.Arbitrum.Arbos.Stylus; @@ -39,10 +41,11 @@ public static StylusNativeResult Failure(UserOutcomeKind status, string error public static unsafe partial class StylusNative { - public static StylusNativeResult Call(byte[] module, byte[] callData, StylusConfig config, IStylusEvmApi api, EvmData evmData, bool debug, - uint arbOsTag, ref ulong gas) + public static StylusNativeResult Call(byte[] module, StylusConfig config, IStylusEvmApi api, EvmData evmData, bool debug, + IStylusVmHost vmHost, in ValueHash256 moduleHash, uint arbOsTag, ref ulong gas) { using GoSliceHandle moduleSlice = GoSliceHandle.From(module); + byte[] callData = vmHost.VmState.Env.InputData.ToArray(); using GoSliceHandle callDataSlice = GoSliceHandle.From(callData); using StylusEnvApiRegistration registration = StylusEvmApiRegistry.Register(api); @@ -52,6 +55,18 @@ public static StylusNativeResult Call(byte[] module, byte[] callData, St Id = registration.Id }; + if (vmHost.IsRecordingExecution) + { + Dictionary asmMap = new(); + foreach (string target in vmHost.WasmStore.GetWasmTargets()) + { + if (!vmHost.WasmStore.TryGetActivatedAsm(target, moduleHash, out byte[]? asm)) + throw new InvalidOperationException($"Cannot find activated wasm, missing target: {target}"); + asmMap.Add(target, asm); + } + vmHost.RecordUserWasm(moduleHash, asmMap); + } + RustBytes output = new(); UserOutcomeKind status = stylus_call( moduleSlice.Data, diff --git a/src/Nethermind.Arbitrum/Data/DigestMessageParameters.cs b/src/Nethermind.Arbitrum/Data/DigestMessageParameters.cs index 996059129..6d3cebe73 100644 --- a/src/Nethermind.Arbitrum/Data/DigestMessageParameters.cs +++ b/src/Nethermind.Arbitrum/Data/DigestMessageParameters.cs @@ -58,5 +58,6 @@ public record ReorgParameters( public record RecordBlockCreationParameters( [property: JsonPropertyName("index")] ulong Index, - [property: JsonPropertyName("message")] MessageWithMetadata Message + [property: JsonPropertyName("message")] MessageWithMetadata Message, + [property: JsonPropertyName("wasmTargets")] string[] WasmTargets ); diff --git a/src/Nethermind.Arbitrum/Data/RecordResult.cs b/src/Nethermind.Arbitrum/Data/RecordResult.cs index d71931401..7e50acf4f 100644 --- a/src/Nethermind.Arbitrum/Data/RecordResult.cs +++ b/src/Nethermind.Arbitrum/Data/RecordResult.cs @@ -1,12 +1,10 @@ // SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using Nethermind.Consensus.Stateless; +using Nethermind.Arbitrum.Execution.Stateless; using Nethermind.Core.Crypto; using System.Text.Json.Serialization; -using WasmTarget = string; - namespace Nethermind.Arbitrum.Data; public sealed class RecordResult @@ -14,33 +12,26 @@ public sealed class RecordResult public ulong Index { get; } public Hash256 BlockHash { get; } public Dictionary Preimages { get; } - public UserWasms? UserWasms { get; } + public Dictionary>? UserWasms { get; } [JsonIgnore] - public Witness Witness { get; } + public ArbitrumWitness Witness { get; } - public RecordResult(ulong messageIndex, Hash256 blockHash, Witness witness) + public RecordResult(ulong messageIndex, Hash256 blockHash, ArbitrumWitness arbWitness) { Index = messageIndex; BlockHash = blockHash; - Witness = witness; - UserWasms = null!; // TODO: add wasms + Witness = arbWitness; + UserWasms = arbWitness.UserWasms?.ToDictionary( + kvp => kvp.Key.ToHash256(), + kvp => kvp.Value); Preimages = new(); - foreach (byte[] code in witness.Codes) + foreach (byte[] code in arbWitness.Witness.Codes) Preimages.Add(Keccak.Compute(code), code); - foreach (byte[] state in witness.State) + foreach (byte[] state in arbWitness.Witness.State) Preimages.Add(Keccak.Compute(state), state); - foreach (byte[] header in witness.Headers) + foreach (byte[] header in arbWitness.Witness.Headers) Preimages.Add(Keccak.Compute(header), header); } } - -public sealed record class ActivatedWasm( - Dictionary Value -); - -public sealed record class UserWasms( - Dictionary Value -); - diff --git a/src/Nethermind.Arbitrum/Evm/ArbitrumVirtualMachine.cs b/src/Nethermind.Arbitrum/Evm/ArbitrumVirtualMachine.cs index eb33f0966..39814add7 100644 --- a/src/Nethermind.Arbitrum/Evm/ArbitrumVirtualMachine.cs +++ b/src/Nethermind.Arbitrum/Evm/ArbitrumVirtualMachine.cs @@ -11,7 +11,6 @@ using Nethermind.Core.Specs; using Nethermind.Evm; using Nethermind.Evm.CodeAnalysis; -using Nethermind.Evm.GasPolicy; using Nethermind.Evm.State; using Nethermind.Logging; using Nethermind.Evm.Tracing; @@ -23,6 +22,9 @@ using System.Text.Json; using Nethermind.Arbitrum.Data; using Nethermind.Arbitrum.Math; +using Nethermind.Core.Crypto; +using Nethermind.Arbitrum.Execution.Stateless; +using System.Diagnostics; [assembly: InternalsVisibleTo("Nethermind.Arbitrum.Evm.Test")] namespace Nethermind.Arbitrum.Evm; @@ -35,7 +37,8 @@ public sealed unsafe class ArbitrumVirtualMachine( ISpecProvider? specProvider, ILogManager? logManager, IL1BlockCache? l1BlockCache = null, - bool enableWitnessGeneration = false + bool enableWitnessGeneration = false, + ArbitrumUserWasmsRecorder? wasmsRecorder = null ) : VirtualMachine(blockHashProvider, specProvider, logManager), IStylusVmHost { public IWasmStore WasmStore => wasmStore; @@ -43,6 +46,7 @@ public sealed unsafe class ArbitrumVirtualMachine( public ulong CurrentArbosVersion => FreeArbosState.CurrentArbosVersion; public ArbitrumTxExecutionContext ArbitrumTxExecutionContext { get; set; } = new(); public IL1BlockCache L1BlockCache { get; } = l1BlockCache ?? new L1BlockCache(); + public bool IsRecordingExecution => enableWitnessGeneration; private Dictionary Programs { get; } = new(); private SystemBurner _systemBurner = null!; private static readonly PrecompileExecutionFailureException PrecompileExecutionFailureException = new(); @@ -74,6 +78,12 @@ public override TransactionSubstate ExecuteTransaction( return result; } + public void RecordUserWasm(ValueHash256 moduleHash, IReadOnlyDictionary asmMap) + { + Debug.Assert(enableWitnessGeneration && wasmsRecorder is not null); + wasmsRecorder.RecordUserWasm(moduleHash, asmMap); + } + public StylusEvmResult StylusCall(ExecutionType kind, Address to, ReadOnlyMemory input, ulong gasLeftReportedByRust, ulong gasRequestedByRust, in UInt256 value) { ArbitrumGasPolicy gas = ArbitrumGasPolicy.FromLong((long)gasLeftReportedByRust); diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs index 4cd657479..981e2b28c 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs @@ -26,8 +26,9 @@ namespace Nethermind.Arbitrum.Execution.Stateless; +// TODO: use wasms (not priority for now) public class ArbitrumStatelessBlockProcessingEnv( - Witness witness, + ArbitrumWitness arbWitness, ISpecProvider specProvider, ISealValidator sealValidator, IWasmStore wasmStore, @@ -45,13 +46,13 @@ public IBlockProcessor BlockProcessor public IWorldState WorldState { get => _worldState ??= new WorldState( - new TrieStoreScopeProvider(new RawTrieStore(witness.NodeStorage), - witness.CodeDb, logManager), logManager); + new TrieStoreScopeProvider(new RawTrieStore(arbWitness.Witness.NodeStorage), + arbWitness.Witness.CodeDb, logManager), logManager); } private IBlockProcessor GetProcessor() { - StatelessBlockTree statelessBlockTree = new(witness.DecodedHeaders); + StatelessBlockTree statelessBlockTree = new(arbWitness.Witness.DecodedHeaders); ITransactionProcessor txProcessor = CreateTransactionProcessor(WorldState, statelessBlockTree); IBlockProcessor.IBlockTransactionsExecutor txExecutor = new BlockProcessor.BlockValidationTransactionsExecutor( diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumUserWasmsRecorder.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumUserWasmsRecorder.cs new file mode 100644 index 000000000..d2448196f --- /dev/null +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumUserWasmsRecorder.cs @@ -0,0 +1,14 @@ +using Nethermind.Core.Crypto; + +namespace Nethermind.Arbitrum.Execution.Stateless; + +public class ArbitrumUserWasmsRecorder +{ + public Dictionary>? UserWasms { get; private set; } + + public void RecordUserWasm(ValueHash256 moduleHash, IReadOnlyDictionary asmMap) + { + UserWasms ??= new(); + UserWasms[moduleHash] = asmMap; + } +} \ No newline at end of file diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitness.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitness.cs new file mode 100644 index 000000000..8443e7580 --- /dev/null +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitness.cs @@ -0,0 +1,13 @@ +using Nethermind.Consensus.Stateless; +using Nethermind.Core.Crypto; + +namespace Nethermind.Arbitrum.Execution.Stateless; + +public class ArbitrumWitness(Witness witness, Dictionary>? userWasms) +{ + private readonly Witness _witness = witness; + + public ref readonly Witness Witness => ref _witness; + + public Dictionary>? UserWasms => userWasms; +} \ No newline at end of file diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs index b7dc10fab..8cb3448a9 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs @@ -15,16 +15,17 @@ namespace Nethermind.Arbitrum.Execution.Stateless; public interface IBlockBuildingWitnessCollector { - Task<(Block Block, Witness Witness)> BuildBlockAndGetWitness(BlockHeader parentHeader, PayloadAttributes payloadAttributes); + Task<(Block Block, ArbitrumWitness Witness)> BuildBlockAndGetWitness(BlockHeader parentHeader, PayloadAttributes payloadAttributes); } public class ArbitrumWitnessCollector( WitnessGeneratingWorldState worldState, IBlockProducer blockProducer, + ArbitrumUserWasmsRecorder wasmsRecorder, ISpecProvider specProvider, IArbitrumSpecHelper specHelper) : IBlockBuildingWitnessCollector { - public async Task<(Block Block, Witness Witness)> BuildBlockAndGetWitness(BlockHeader parentHeader, PayloadAttributes payloadAttributes) + public async Task<(Block Block, ArbitrumWitness Witness)> BuildBlockAndGetWitness(BlockHeader parentHeader, PayloadAttributes payloadAttributes) { using (worldState.BeginScope(parentHeader)) { @@ -46,7 +47,8 @@ public class ArbitrumWitnessCollector( throw new NullReferenceException($"Failed to build block with parent header number: {parentHeader.Number} and hash: {parentHeader.Hash}"); Witness witness = worldState.GetWitness(parentHeader); + ArbitrumWitness arbitrumWitness = new(witness, wasmsRecorder.UserWasms); - return (producedBlock, witness); + return (producedBlock, arbitrumWitness); } } diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs index e4f7941fb..7c908b4c7 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs @@ -27,6 +27,7 @@ public class ArbitrumWitnessGeneratingBlockProcessingEnv( IBlocksConfig blocksConfig, ISpecProvider specProvider, IArbitrumSpecHelper specHelper, + ArbitrumUserWasmsRecorder wasmsRecorder, ILogManager logManager) : IWitnessGeneratingPolyvalentEnv { public IExistingBlockWitnessCollector CreateExistingBlockWitnessCollector() @@ -50,6 +51,6 @@ public IBlockBuildingWitnessCollector CreateBlockBuildingWitnessCollector() logManager, blocksConfig); - return new ArbitrumWitnessCollector(witnessGeneratingWorldState, blockProducer, specProvider, specHelper); + return new ArbitrumWitnessCollector(witnessGeneratingWorldState, blockProducer, wasmsRecorder, specProvider, specHelper); } } diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index d0322c2d7..c18be5fec 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -30,16 +30,21 @@ namespace Nethermind.Arbitrum.Execution.Stateless; + +public interface IArbitrumWitnessGeneratingBlockProcessingEnvFactory: IWitnessGeneratingBlockProcessingEnvFactory +{ + IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTargets); +} + public class ArbitrumWitnessGeneratingBlockProcessingEnvFactory( ILifetimeScope rootLifetimeScope, IReadOnlyTrieStore readOnlyTrieStore, IDbProvider dbProvider, - ILogManager logManager) : IWitnessGeneratingBlockProcessingEnvFactory + ILogManager logManager) : IArbitrumWitnessGeneratingBlockProcessingEnvFactory { // To force processing in BlockchainProcessor even though block is not better than head (already existing block) private static BlocksConfig CreateWitnessBlocksConfig(IBlocksConfig blocksConfig) - { - return new BlocksConfig + => new() { TargetBlockGasLimit = blocksConfig.TargetBlockGasLimit, MinGasPrice = blocksConfig.MinGasPrice, @@ -57,19 +62,18 @@ private static BlocksConfig CreateWitnessBlocksConfig(IBlocksConfig blocksConfig BlockProductionBlobLimit = blocksConfig.BlockProductionBlobLimit, BuildBlocksOnMainState = false, }; - } private ITransactionProcessor CreateTransactionProcessor( IWasmStore wasmStore, ISpecProvider specProvider, - IBlockTree blockTree, IArbosVersionProvider arbosVersionProvider, IWorldState state, - IHeaderFinder witnessGeneratingHeaderFinder) + IHeaderFinder witnessGeneratingHeaderFinder, + ArbitrumUserWasmsRecorder wasmsRecorder) { BlockhashProvider blockhashProvider = new(new BlockhashCache(witnessGeneratingHeaderFinder, logManager), state, logManager); // We don't give any l1BlockCache to the vm so that it forces querying the world state - ArbitrumVirtualMachine vm = new(blockhashProvider, wasmStore, specProvider, logManager, enableWitnessGeneration: true); + ArbitrumVirtualMachine vm = new(blockhashProvider, wasmStore, specProvider, logManager, enableWitnessGeneration: true, wasmsRecorder: wasmsRecorder); return new ArbitrumTransactionProcessor( BlobBaseFeeCalculator.Instance, specProvider, state, wasmStore, vm, logManager, @@ -77,60 +81,71 @@ private ITransactionProcessor CreateTransactionProcessor( arbosVersionProvider, state as IWitnessBytecodeRecorder)); } - public IWitnessGeneratingBlockProcessingEnvScope CreateScope() + // TODO: check debug endpoint exec later (compare with nitro) -- Not priority for now + public IWitnessGeneratingBlockProcessingEnvScope CreateScope() => CreateScope(null); + + public IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTargets) { - IBlocksConfig blocksConfig = rootLifetimeScope.Resolve(); IReadOnlyDbProvider readOnlyDbProvider = new ReadOnlyDbProvider(dbProvider, true); WitnessCapturingTrieStore trieStore = new(readOnlyDbProvider.StateDb, readOnlyTrieStore); IStateReader stateReader = new StateReader(trieStore, readOnlyDbProvider.CodeDb, logManager); WorldState worldState = new(new TrieStoreScopeProvider(trieStore, readOnlyDbProvider.CodeDb, logManager), logManager); - ILifetimeScope envLifetimeScope = rootLifetimeScope.BeginLifetimeScope((builder) => builder - .AddScoped(stateReader) - - .AddScoped(builder => new WitnessGeneratingHeaderFinder(builder.Resolve())) - .AddScoped(builder => new WitnessGeneratingWorldState(worldState, stateReader, trieStore, (builder.Resolve() as WitnessGeneratingHeaderFinder)!)) - - .AddScoped(_ => CreateWitnessBlocksConfig(blocksConfig)) - - //TODO Create witness capturing wasm store somehow - .AddScoped(_ => new WasmDb(new MemDb())) - // new instance i think? to check but might have to create a new recording IWasmDb passed to it - .AddScoped(ctx => new WasmStore(ctx.Resolve(), ctx.Resolve(), cacheTag: 1)) - - .AddScoped(builder => CreateTransactionProcessor( - builder.Resolve(), - builder.Resolve(), - builder.Resolve(), - builder.Resolve(), - builder.Resolve(), - builder.Resolve())) - - // 1st: add the tx executor - .AddScoped() - - // 2nd: add block processor - .AddScoped(NullReceiptStorage.Instance) - .AddScoped(BlockchainProcessor.Options.NoReceipts) - .AddScoped() - - // 3rd: configure the builder for block production (like ArbitrumBlockProducerEnvFactory but with my own witness capturing world state) - .AddScoped(builder => builder.Resolve().Create()) - .AddScoped() - .AddDecorator() - .AddDecorator() - .AddScoped() - - .AddScoped(builder => - new ArbitrumWitnessGeneratingBlockProcessingEnv( - builder.Resolve(), - builder.Resolve(), - builder.Resolve(), - (builder.Resolve() as WitnessGeneratingWorldState)!, - builder.Resolve(), + ArbitrumUserWasmsRecorder wasmsRecorder = new(); + IBlocksConfig blocksConfig = rootLifetimeScope.Resolve(); + + ILifetimeScope envLifetimeScope = rootLifetimeScope.BeginLifetimeScope((builder) => + { + if (wasmTargets is not null) + { + builder.AddScoped(_ => new StylusTargetConfig() { OverrideWasmTargets = wasmTargets }); + // Need to redeclare IWasmStore because it was originally declared as a singleton and therefore was cached with original IStylusTargetConfig + builder.AddScoped(ctx => new WasmStore(ctx.Resolve(), ctx.Resolve(), cacheTag: 1)); + } + + builder + .AddScoped(stateReader) + + .AddScoped(builder => new WitnessGeneratingHeaderFinder(builder.Resolve())) + .AddScoped(builder => new WitnessGeneratingWorldState(worldState, stateReader, trieStore, (builder.Resolve() as WitnessGeneratingHeaderFinder)!)) + + .AddScoped(_ => CreateWitnessBlocksConfig(blocksConfig)) + + .AddScoped(builder => CreateTransactionProcessor( + builder.Resolve(), builder.Resolve(), - builder.Resolve(), - logManager))); + builder.Resolve(), + builder.Resolve(), + builder.Resolve(), + wasmsRecorder)) + + // 1st: add the tx executor + .AddScoped() + + // 2nd: add block processor + .AddScoped(NullReceiptStorage.Instance) + .AddScoped(BlockchainProcessor.Options.NoReceipts) + .AddScoped() + + // 3rd: configure the builder for block production (like ArbitrumBlockProducerEnvFactory but with my own witness capturing world state) + .AddScoped(builder => builder.Resolve().Create()) + .AddScoped() + .AddDecorator() + .AddDecorator() + .AddScoped() + + .AddScoped(builder => + new ArbitrumWitnessGeneratingBlockProcessingEnv( + builder.Resolve(), + builder.Resolve(), + builder.Resolve(), + (builder.Resolve() as WitnessGeneratingWorldState)!, + builder.Resolve(), + builder.Resolve(), + builder.Resolve(), + wasmsRecorder, + logManager)); + }); return new ExecutionRecordingScope(envLifetimeScope); } diff --git a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs index 04af0a3ad..0a8d24faa 100644 --- a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs +++ b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs @@ -22,6 +22,7 @@ using Nethermind.Specs.ChainSpecStyle; using Nethermind.Consensus.Stateless; using Nethermind.Arbitrum.Execution.Stateless; +using Nethermind.Arbitrum.Arbos.Stylus; namespace Nethermind.Arbitrum.Modules; @@ -36,7 +37,7 @@ public class ArbitrumRpcModule( CachedL1PriceData cachedL1PriceData, IBlockProcessingQueue processingQueue, IArbitrumConfig arbitrumConfig, - IWitnessGeneratingBlockProcessingEnvFactory witnessGeneratingBlockProcessingEnvFactory, + IArbitrumWitnessGeneratingBlockProcessingEnvFactory witnessGeneratingBlockProcessingEnvFactory, IBlocksConfig blocksConfig) : IArbitrumRpcModule { protected readonly SemaphoreSlim CreateBlocksSemaphore = new(1, 1); @@ -400,9 +401,14 @@ public async Task> RecordBlockCreation(RecordBlockCr Number = blockNumber }; - using IWitnessGeneratingBlockProcessingEnvScope scope = witnessGeneratingBlockProcessingEnvFactory.CreateScope(); + string[] wasmTargets = parameters.WasmTargets; + string localTarget = StylusTargets.GetLocalTargetName(); + if (!wasmTargets.Contains(localTarget)) + wasmTargets = wasmTargets.Append(localTarget).ToArray(); + + using IWitnessGeneratingBlockProcessingEnvScope scope = witnessGeneratingBlockProcessingEnvFactory.CreateScope(wasmTargets); IBlockBuildingWitnessCollector witnessCollector = ((IWitnessGeneratingPolyvalentEnv)scope.Env).CreateBlockBuildingWitnessCollector(); - (Block builtBlock, Witness witness) = await witnessCollector.BuildBlockAndGetWitness(parent, payload); + (Block builtBlock, ArbitrumWitness witness) = await witnessCollector.BuildBlockAndGetWitness(parent, payload); if (builtBlock.Hash is null) return ResultWrapper.Fail($"Failed to build block {blockNumber} or block has no hash."); diff --git a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModuleWithComparison.cs b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModuleWithComparison.cs index 0c6cf1c54..9b4883baf 100644 --- a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModuleWithComparison.cs +++ b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModuleWithComparison.cs @@ -15,7 +15,7 @@ using Nethermind.Logging; using Nethermind.Serialization.Json; using Nethermind.Specs.ChainSpecStyle; -using Nethermind.Consensus.Stateless; +using Nethermind.Arbitrum.Execution.Stateless; namespace Nethermind.Arbitrum.Modules; @@ -32,7 +32,7 @@ public sealed class ArbitrumRpcModuleWithComparison( IArbitrumConfig arbitrumConfig, IVerifyBlockHashConfig verifyBlockHashConfig, IJsonSerializer jsonSerializer, - IWitnessGeneratingBlockProcessingEnvFactory witnessGeneratingBlockProcessingEnvFactory, + IArbitrumWitnessGeneratingBlockProcessingEnvFactory witnessGeneratingBlockProcessingEnvFactory, IBlocksConfig blocksConfig, IProcessExitSource? processExitSource = null) : ArbitrumRpcModule(initializer, blockTree, trigger, txSource, chainSpec, specHelper, logManager, cachedL1PriceData, processingQueue, arbitrumConfig, witnessGeneratingBlockProcessingEnvFactory, blocksConfig) diff --git a/src/Nethermind.Arbitrum/Stylus/StylusTargetConfig.cs b/src/Nethermind.Arbitrum/Stylus/StylusTargetConfig.cs index 19133fd0b..23e3dd0b8 100644 --- a/src/Nethermind.Arbitrum/Stylus/StylusTargetConfig.cs +++ b/src/Nethermind.Arbitrum/Stylus/StylusTargetConfig.cs @@ -24,9 +24,13 @@ public class StylusTargetConfig : IStylusTargetConfig public string Amd64 { get; set; } = StylusTargets.LinuxX64Descriptor; public string[] ExtraArchs { get; set; } = [StylusTargets.WavmTargetName]; public uint NativeLruCacheCapacityMb { get; set; } = 256; + public string[]? OverrideWasmTargets { get; init; } public IReadOnlyCollection GetWasmTargets() { + if (OverrideWasmTargets is not null) + return OverrideWasmTargets; + HashSet targets = [StylusTargets.GetLocalTargetName()]; foreach (string arch in ExtraArchs) { @@ -36,7 +40,7 @@ public IReadOnlyCollection GetWasmTargets() targets.Add(arch); } - // Ensure targets are always have the same order... from Nitro + // Ensure targets always have the same order... from Nitro return targets.OrderBy(t => t).ToArray(); } } From d8dd3661357f82a5ce45e63974efb02dc4db36d5 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 2 Feb 2026 19:05:33 +0900 Subject: [PATCH 13/87] fix: Forgotten isUserTx check --- src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs index 708f72b7e..435b48dd3 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs @@ -454,13 +454,14 @@ private AddingTxEventArgs CanAddTransaction( // Compute gas = total gas - data gas computeGas = currentTx.GasLimit - dataGas; - // Apply minimum gas floor + // Apply minimum gas floor (ensure at least TxGas is left in the pool before trying a state transition) if (computeGas < GasCostOf.Transaction) computeGas = GasCostOf.Transaction; // Check if compute gas fits in the block (only after first user tx) if (arbosState.CurrentArbosVersion < ArbosVersion.Fifty && computeGas > (long)blockGasLeft.Value + && IsUserTransaction(currentTx) && userTxsProcessed > 0) { AddingTxEventArgs args = new(transactionsInBlock.Count, currentTx, block, transactionsInBlock); From effbb2fb1bd10e3a05cabb90e669ff96c43e5cd3 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 2 Feb 2026 19:43:08 +0900 Subject: [PATCH 14/87] fix: Merge conflicts --- .../ArbitrumRpcTestBlockchain.cs | 2 +- src/Nethermind.Arbitrum/ArbitrumPlugin.cs | 3 - .../ArbitrumRpcModuleFactory.cs | 0 .../Execution/ArbitrumBlockProcessor.cs | 2 +- .../Execution/ArbitrumExecutionEngine.cs | 65 +++++++++++++++++++ .../ArbitrumExecutionEngineWithComparison.cs | 3 + .../Execution/IArbitrumExecutionEngine.cs | 1 + .../ArbitrumRpcModuleWithComparison.cs | 0 8 files changed, 71 insertions(+), 5 deletions(-) delete mode 100644 src/Nethermind.Arbitrum/ArbitrumRpcModuleFactory.cs delete mode 100644 src/Nethermind.Arbitrum/Modules/ArbitrumRpcModuleWithComparison.cs diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs index c0ce4e0a8..dcfb37738 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs @@ -303,9 +303,9 @@ private static ArbitrumRpcTestBlockchain CreateInternal(ArbitrumRpcTestBlockchai chain.Dependencies.CachedL1PriceData, chain.BlockProcessingQueue, chain.Container.Resolve(), + chain.Container.Resolve(), chain.Container.Resolve()); - // chain.Container.Resolve(), chain.ArbitrumRpcModule = new ArbitrumRpcModuleWrapper(chain, new ArbitrumRpcModule(engine)); chain.ArbitrumEthRpcModule = new ArbitrumEthRpcModule( diff --git a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs index 088cfda32..84d456a89 100644 --- a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs +++ b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs @@ -111,9 +111,6 @@ public Task InitRpcModules() _api.ProcessExit); } - // in ArbitrumRpcModuleFactory: - // _api.Context.Resolve(), - // Register Arbitrum RPC module IArbitrumRpcModule arbitrumRpcModule = new ArbitrumRpcModule(engine); _api.RpcModuleProvider.RegisterSingle(arbitrumRpcModule); diff --git a/src/Nethermind.Arbitrum/ArbitrumRpcModuleFactory.cs b/src/Nethermind.Arbitrum/ArbitrumRpcModuleFactory.cs deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs index 435b48dd3..a73449b5f 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs @@ -454,7 +454,7 @@ private AddingTxEventArgs CanAddTransaction( // Compute gas = total gas - data gas computeGas = currentTx.GasLimit - dataGas; - // Apply minimum gas floor (ensure at least TxGas is left in the pool before trying a state transition) + // Apply minimum gas floor if (computeGas < GasCostOf.Transaction) computeGas = GasCostOf.Transaction; diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs index 8c0942fb4..76c63d0bd 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs @@ -2,10 +2,13 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Collections.Concurrent; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Text.Json; +using Nethermind.Arbitrum.Arbos.Stylus; using Nethermind.Arbitrum.Config; using Nethermind.Arbitrum.Data; +using Nethermind.Arbitrum.Execution.Stateless; using Nethermind.Arbitrum.Genesis; using Nethermind.Arbitrum.Math; using Nethermind.Arbitrum.Modules; @@ -13,6 +16,7 @@ using Nethermind.Config; using Nethermind.Consensus.Processing; using Nethermind.Consensus.Producers; +using Nethermind.Consensus.Stateless; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.JsonRpc; @@ -34,6 +38,7 @@ public sealed class ArbitrumExecutionEngine( CachedL1PriceData cachedL1PriceData, IBlockProcessingQueue processingQueue, IArbitrumConfig arbitrumConfig, + IArbitrumWitnessGeneratingBlockProcessingEnvFactory witnessGeneratingBlockProcessingEnvFactory, IBlocksConfig blocksConfig) : IArbitrumExecutionEngine { @@ -512,6 +517,66 @@ public async Task> ProduceBlockWithoutWaitingOnProc } } + public async Task> RecordBlockCreation(RecordBlockCreationParameters parameters) + { + long blockNumber = MessageIndexToBlockNumber(parameters.Index).Data; + if (blockNumber == 0) + { + // Cannot generate witness for genesis block as the block itself does not contain any transaction + // responsible for the state setup. It is the weak subjectivity starting point to trust. + return ResultWrapper.Fail($"Cannot generate witness for genesis block"); + } + + BlockHeader? parent = BlockTree.FindHeader(blockNumber - 1); + if (parent is null) + { + return ResultWrapper.Fail($"Unable to find parent for block {blockNumber}"); + } + + ArbitrumPayloadAttributes payload = new() + { + MessageWithMetadata = parameters.Message, + Number = blockNumber + }; + + string[] wasmTargets = parameters.WasmTargets; + string localTarget = StylusTargets.GetLocalTargetName(); + if (!wasmTargets.Contains(localTarget)) + wasmTargets = wasmTargets.Append(localTarget).ToArray(); + + using IWitnessGeneratingBlockProcessingEnvScope scope = witnessGeneratingBlockProcessingEnvFactory.CreateScope(wasmTargets); + IBlockBuildingWitnessCollector witnessCollector = ((IWitnessGeneratingPolyvalentEnv)scope.Env).CreateBlockBuildingWitnessCollector(); + (Block builtBlock, ArbitrumWitness witness) = await witnessCollector.BuildBlockAndGetWitness(parent, payload); + + if (builtBlock.Hash is null) + return ResultWrapper.Fail($"Failed to build block {blockNumber} or block has no hash."); + + // Sometimes, it seems RecordBlockCreation is called slightly before the actual block is finalized/committed to the database. + // So we need to wait for the block to be available in the database. + Hash256? canonicalHash = null; + Stopwatch sw = Stopwatch.StartNew(); + while (sw.ElapsedMilliseconds <= arbitrumConfig.MessageLagMs) + { + canonicalHash = BlockTree.FindCanonicalBlockInfo(blockNumber)?.BlockHash; + + if (canonicalHash is null) + { + await Task.Delay(10); + continue; + } + + break; + } + + if (canonicalHash is null) + return ResultWrapper.Fail(ArbitrumRpcErrors.BlockNotFound(blockNumber)); + else if (canonicalHash != builtBlock.Hash) + return ResultWrapper.Fail($"Built block hash: {builtBlock.Hash} does not match canonical block header hash: {canonicalHash}"); + + RecordResult result = new(parameters.Index, builtBlock.Hash!, witness); + return ResultWrapper.Success(result); + } + private Hash256 GetSendRootFromBlock(Block block) { ArbitrumBlockHeaderInfo headerInfo = ArbitrumBlockHeaderInfo.Deserialize(block.Header, _logger); diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngineWithComparison.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngineWithComparison.cs index a19889604..10ee692af 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngineWithComparison.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngineWithComparison.cs @@ -279,4 +279,7 @@ private void TriggerGracefulShutdown(string reason) }); } } + + public Task> RecordBlockCreation(RecordBlockCreationParameters parameters) + => innerEngine.RecordBlockCreation(parameters); } diff --git a/src/Nethermind.Arbitrum/Execution/IArbitrumExecutionEngine.cs b/src/Nethermind.Arbitrum/Execution/IArbitrumExecutionEngine.cs index 91b385ded..ca1667951 100644 --- a/src/Nethermind.Arbitrum/Execution/IArbitrumExecutionEngine.cs +++ b/src/Nethermind.Arbitrum/Execution/IArbitrumExecutionEngine.cs @@ -27,4 +27,5 @@ public interface IArbitrumExecutionEngine ResultWrapper Synced(); ResultWrapper> FullSyncProgressMap(); Task> ArbOSVersionForMessageIndexAsync(ulong messageIndex); + Task> RecordBlockCreation(RecordBlockCreationParameters parameters); } diff --git a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModuleWithComparison.cs b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModuleWithComparison.cs deleted file mode 100644 index e69de29bb..000000000 From 6f5580d1a7068371ae2494968a327ff1904d5ffd Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 2 Feb 2026 20:10:37 +0900 Subject: [PATCH 15/87] Temporary files for docker builds --- .../Properties/configs/arbitrum-mainnet-archive.json | 4 +++- .../Properties/configs/arbitrum-mainnet.json | 4 +++- .../Properties/configs/arbitrum-sepolia-archive.json | 4 +++- .../Properties/configs/arbitrum-sepolia.json | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet-archive.json b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet-archive.json index c4a30fa0e..04a28a33d 100644 --- a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet-archive.json +++ b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet-archive.json @@ -15,6 +15,7 @@ "Enabled": true, "Port": 20545, "EnginePort": 20551, + "UnsecureDevNoRpcAuthentication": true, "EngineEnabledModules": ["Net", "Eth", "Subscribe", "Web3", "Arbitrum", "nitroexecution"], "AdditionalRpcUrls": [ "http://localhost:28551|http;ws|net;eth;subscribe;web3;client;debug" @@ -38,7 +39,8 @@ "Trace", "TxPool", "Vault", - "Web3" + "Web3", + "Arbitrum" ] }, "Pruning": { diff --git a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet.json b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet.json index ef67ab24d..655b2ad40 100644 --- a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet.json +++ b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet.json @@ -23,6 +23,7 @@ "Enabled": true, "Port": 20545, "EnginePort": 20551, + "UnsecureDevNoRpcAuthentication": true, "EngineEnabledModules": ["Net", "Eth", "Subscribe", "Web3", "Arbitrum", "nitroexecution"], "AdditionalRpcUrls": [ "http://localhost:28551|http;ws|net;eth;subscribe;web3;client;debug" @@ -46,7 +47,8 @@ "Trace", "TxPool", "Vault", - "Web3" + "Web3", + "Arbitrum" ] }, "Pruning": { diff --git a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia-archive.json b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia-archive.json index 8533bc57c..5b93caaec 100644 --- a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia-archive.json +++ b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia-archive.json @@ -15,6 +15,7 @@ "Enabled": true, "Port": 20545, "EnginePort": 20551, + "UnsecureDevNoRpcAuthentication": true, "EngineEnabledModules": ["Net", "Eth", "Subscribe", "Web3", "Arbitrum", "nitroexecution"], "AdditionalRpcUrls": [ "http://localhost:28551|http;ws|net;eth;subscribe;web3;client;debug" @@ -38,7 +39,8 @@ "Trace", "TxPool", "Vault", - "Web3" + "Web3", + "Arbitrum" ] }, "Pruning": { diff --git a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia.json b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia.json index 923993ba6..f8574150b 100644 --- a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia.json +++ b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia.json @@ -23,6 +23,7 @@ "Enabled": true, "Port": 20545, "EnginePort": 20551, + "UnsecureDevNoRpcAuthentication": true, "EngineEnabledModules": ["Net", "Eth", "Subscribe", "Web3", "Arbitrum", "nitroexecution"], "AdditionalRpcUrls": [ "http://localhost:28551|http;ws|net;eth;subscribe;web3;client;debug" @@ -46,7 +47,8 @@ "Trace", "TxPool", "Vault", - "Web3" + "Web3", + "Arbitrum" ] }, "Pruning": { From 40b7f1adb9a13bb35fb348104721c8da49b134be Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 2 Feb 2026 20:17:01 +0900 Subject: [PATCH 16/87] fix: Format check --- .../Execution/Stateless/ArbitrumUserWasmsRecorder.cs | 2 +- src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitness.cs | 2 +- .../ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumUserWasmsRecorder.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumUserWasmsRecorder.cs index d2448196f..b2c9243b8 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumUserWasmsRecorder.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumUserWasmsRecorder.cs @@ -11,4 +11,4 @@ public void RecordUserWasm(ValueHash256 moduleHash, IReadOnlyDictionary ref _witness; public Dictionary>? UserWasms => userWasms; -} \ No newline at end of file +} diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index c18be5fec..5e623d017 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -31,7 +31,7 @@ namespace Nethermind.Arbitrum.Execution.Stateless; -public interface IArbitrumWitnessGeneratingBlockProcessingEnvFactory: IWitnessGeneratingBlockProcessingEnvFactory +public interface IArbitrumWitnessGeneratingBlockProcessingEnvFactory : IWitnessGeneratingBlockProcessingEnvFactory { IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTargets); } From bccf26ca981749ea0898971c61740a06eaaa515a Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 2 Feb 2026 20:21:38 +0900 Subject: [PATCH 17/87] fix: Format check 2 --- src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs index 6edaf0319..6c52ec92b 100644 --- a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs +++ b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs @@ -322,7 +322,7 @@ public static void Call_CounterContractIncrement_EmitsLogsAndUpdatesStorageThrou // Get number again (should now be 1) vmHost.VmState.Env.InputData.Returns(getNumberCalldata); - StylusNativeResult getNumberResult2 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); + StylusNativeResult getNumberResult2 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); byte[] expected = new byte[32]; expected[^1] = 1; From 00f4cd389e38e5f6fe2214af535f85a23dcee697 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 3 Feb 2026 23:46:26 +0900 Subject: [PATCH 18/87] fix: Traverse state trie for precompile account even if bytecode is either invalid or empty --- ...ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs | 3 +-- .../Precompiles/ArbitrumCodeInfoRepository.cs | 9 +++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index 5e623d017..cccc64322 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -30,7 +30,6 @@ namespace Nethermind.Arbitrum.Execution.Stateless; - public interface IArbitrumWitnessGeneratingBlockProcessingEnvFactory : IWitnessGeneratingBlockProcessingEnvFactory { IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTargets); @@ -78,7 +77,7 @@ private ITransactionProcessor CreateTransactionProcessor( return new ArbitrumTransactionProcessor( BlobBaseFeeCalculator.Instance, specProvider, state, wasmStore, vm, logManager, new ArbitrumCodeInfoRepository(new CodeInfoRepository(state, new EthereumPrecompileProvider()), - arbosVersionProvider, state as IWitnessBytecodeRecorder)); + arbosVersionProvider, state as WitnessGeneratingWorldState)); } // TODO: check debug endpoint exec later (compare with nitro) -- Not priority for now diff --git a/src/Nethermind.Arbitrum/Precompiles/ArbitrumCodeInfoRepository.cs b/src/Nethermind.Arbitrum/Precompiles/ArbitrumCodeInfoRepository.cs index 1d9f7ddf2..932226135 100644 --- a/src/Nethermind.Arbitrum/Precompiles/ArbitrumCodeInfoRepository.cs +++ b/src/Nethermind.Arbitrum/Precompiles/ArbitrumCodeInfoRepository.cs @@ -13,7 +13,7 @@ namespace Nethermind.Arbitrum.Precompiles; -public class ArbitrumCodeInfoRepository(ICodeInfoRepository codeInfoRepository, IArbosVersionProvider arbosVersionProvider, IWitnessBytecodeRecorder? witnessBytecodeRecorder = null) : ICodeInfoRepository +public class ArbitrumCodeInfoRepository(ICodeInfoRepository codeInfoRepository, IArbosVersionProvider arbosVersionProvider, WitnessGeneratingWorldState? witnessGeneratingWorldState = null) : ICodeInfoRepository { private readonly Dictionary _arbitrumPrecompiles = InitializePrecompiledContracts(); @@ -64,9 +64,10 @@ delegationAddress is not null && return result; } - // Record precompile bytecode (0xFE) for witness generation as nitro records a trie node with invalid bytecode for it - // which kinda makes sense as there is no actual bytecode stored for precompiles - witnessBytecodeRecorder?.RecordBytecode([0xfe]); + // For witness generation, nitro traverses state trie (effectively capturing intermediate nodes) looking for precompile account for retrieving bytecode. + // Ensure trie traversal and recording of arbitrum precompile bytecode (0xfe stored) and nothing for ethereum precompiles (as no bytecode stored). + // Both arb and eth precompiles account exist but arb's have Arbos.Precompiles.InvalidCodeHash codehash while eth's have Keccak.OfAnEmptyString codehash. + witnessGeneratingWorldState?.GetCode(codeSource); // It's a precompile according to spec // Check if it's an Arbitrum precompile we handle From 314ebbea79680ab9dd2c793125b8b93f53b1dab7 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Thu, 5 Feb 2026 18:52:23 +0900 Subject: [PATCH 19/87] tests: Record user wasms --- .../Infrastructure/WasmGasTestHelper.cs | 4 +- .../Arbos/Stylus/StylusNativeTests.cs | 77 +++++++++-------- .../ArbitrumWitnessGenerationTests.cs | 82 ++++++++++++++----- .../ArbitrumTestBlockchainBase.cs | 7 +- .../ArbitrumStatelessBlockProcessingEnv.cs | 42 ++++++++-- 5 files changed, 143 insertions(+), 69 deletions(-) diff --git a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/Infrastructure/WasmGasTestHelper.cs b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/Infrastructure/WasmGasTestHelper.cs index d95b3af96..16f53e5bb 100644 --- a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/Infrastructure/WasmGasTestHelper.cs +++ b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/Infrastructure/WasmGasTestHelper.cs @@ -33,7 +33,7 @@ public class WasmGasTestHelper : IDisposable public IStylusVmHost VmHost => _vmHost; public IWorldState WorldState => _worldState; - public WasmGasTestHelper(long gasAvailable = 1_000_000, IReleaseSpec? spec = null) + public WasmGasTestHelper(long gasAvailable = 1_000_000, IReleaseSpec? spec = null, ReadOnlyMemory inputData = default) { spec ??= Cancun.Instance; _accessTracker = new StackAccessTracker(); @@ -48,7 +48,7 @@ public WasmGasTestHelper(long gasAvailable = 1_000_000, IReleaseSpec? spec = nul Address.Zero, Address.Zero, 0, 0, 0, - Array.Empty()); + inputData); _vmState = VmState.RentTopLevel( ArbitrumGasPolicy.FromLong(gasAvailable), diff --git a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs index 6c52ec92b..ba713b74a 100644 --- a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs +++ b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs @@ -10,7 +10,6 @@ using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using StylusNative = Nethermind.Arbitrum.Arbos.Stylus.StylusNative; -using NSubstitute; namespace Nethermind.Arbitrum.Test.Arbos.Stylus; @@ -255,29 +254,33 @@ public static void Call_CounterContractSetsValue_UpdatesStorageThroughNativeApi( ValueHash256 moduleHash = new(); - IStylusVmHost vmHost = Substitute.For(); - vmHost.IsRecordingExecution.Returns(false); + byte[] getNumberCalldata = CounterContractCallData.GetNumberCalldata(); + byte[] setNumberCalldata = CounterContractCallData.GetSetNumberCalldata(9); // Get number (should be 0 initially) - byte[] getNumberCalldata = CounterContractCallData.GetNumberCalldata(); - vmHost.VmState.Env.InputData.Returns(getNumberCalldata); - StylusNativeResult getNumberResult1 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); - getNumberResult1.Value.Should().BeEquivalentTo(new byte[32]); + using (WasmGasTestHelper helper = new(inputData: getNumberCalldata)) + { + StylusNativeResult getNumberResult1 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, helper.VmHost, moduleHash, arbosTag, ref gas); + getNumberResult1.Value.Should().BeEquivalentTo(new byte[32]); + } // Set number to 9 - byte[] setNumberCalldata = CounterContractCallData.GetSetNumberCalldata(9); - vmHost.VmState.Env.InputData.Returns(setNumberCalldata); - StylusNativeResult setNumberResult = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); - setNumberResult.Value.Should().BeEmpty(); + using (WasmGasTestHelper helper = new(inputData: setNumberCalldata)) + { + StylusNativeResult setNumberResult = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, helper.VmHost, moduleHash, arbosTag, ref gas); + setNumberResult.Value.Should().BeEmpty(); + } // Get number again (should now be 9) - vmHost.VmState.Env.InputData.Returns(getNumberCalldata); - StylusNativeResult getNumberResult2 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); + using (WasmGasTestHelper helper = new(inputData: getNumberCalldata)) + { + StylusNativeResult getNumberResult2 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, helper.VmHost, moduleHash, arbosTag, ref gas); - byte[] expected = new byte[32]; - expected[^1] = 9; // Last byte should be 9 after setNumber(9) + byte[] expected = new byte[32]; + expected[^1] = 9; // Last byte should be 9 after setNumber(9) - getNumberResult2.Value.Should().BeEquivalentTo(expected); + getNumberResult2.Value.Should().BeEquivalentTo(expected); + } } [Test] @@ -304,30 +307,34 @@ public static void Call_CounterContractIncrement_EmitsLogsAndUpdatesStorageThrou ValueHash256 moduleHash = new(); - IStylusVmHost vmHost = Substitute.For(); - vmHost.IsRecordingExecution.Returns(false); + byte[] getNumberCalldata = CounterContractCallData.GetNumberCalldata(); + byte[] incrementNumberCalldata = CounterContractCallData.GetIncrementCalldata(); // Get number (should be 0 initially) - byte[] getNumberCalldata = CounterContractCallData.GetNumberCalldata(); - vmHost.VmState.Env.InputData.Returns(getNumberCalldata); - StylusNativeResult getNumberResult1 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); - getNumberResult1.Value.Should().BeEquivalentTo(new byte[32]); + using (WasmGasTestHelper helper = new(inputData: getNumberCalldata)) + { + StylusNativeResult getNumberResult1 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, helper.VmHost, moduleHash, arbosTag, ref gas); + getNumberResult1.Value.Should().BeEquivalentTo(new byte[32]); + } // Increment number from 0 to 1 - byte[] incrementNumberCalldata = CounterContractCallData.GetIncrementCalldata(); - vmHost.VmState.Env.InputData.Returns(incrementNumberCalldata); - StylusNativeResult incrementNumberResult = - StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); - incrementNumberResult.IsSuccess.Should().BeTrue(); + using (WasmGasTestHelper helper = new(inputData: incrementNumberCalldata)) + { + StylusNativeResult incrementNumberResult = + StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, helper.VmHost, moduleHash, arbosTag, ref gas); + incrementNumberResult.IsSuccess.Should().BeTrue(); + } // Get number again (should now be 1) - vmHost.VmState.Env.InputData.Returns(getNumberCalldata); - StylusNativeResult getNumberResult2 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); + using (WasmGasTestHelper helper = new(inputData: getNumberCalldata)) + { + StylusNativeResult getNumberResult2 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, helper.VmHost, moduleHash, arbosTag, ref gas); - byte[] expected = new byte[32]; - expected[^1] = 1; + byte[] expected = new byte[32]; + expected[^1] = 1; - getNumberResult2.Value.Should().BeEquivalentTo(expected); + getNumberResult2.Value.Should().BeEquivalentTo(expected); + } } [Test] @@ -363,11 +370,9 @@ public static void Call_KeccakCalculation_ReturnsValidHash() ValueHash256 moduleHash = new(); - IStylusVmHost vmHost = Substitute.For(); - vmHost.IsRecordingExecution.Returns(false); - vmHost.VmState.Env.InputData.Returns(callDataBytes); + using WasmGasTestHelper helper = new(inputData: callDataBytes); - StylusNativeResult resultData = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, vmHost, moduleHash, arbosTag, ref gas); + StylusNativeResult resultData = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, helper.VmHost, moduleHash, arbosTag, ref gas); resultData.Value.Should().BeEquivalentTo(hash); } diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index eb9a85c62..0622b5987 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -1,10 +1,16 @@ +using FluentAssertions; +using Nethermind.Arbitrum.Precompiles; using Nethermind.Arbitrum.Test.Infrastructure; using Nethermind.Blockchain.Tracing; using Nethermind.Consensus.Processing; -using Nethermind.Consensus.Stateless; using Nethermind.Consensus.Validators; using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Core.Specs; +using Nethermind.Core.Test.Builders; +using Nethermind.Evm; +using Nethermind.Int256; using Nethermind.JsonRpc; using Nethermind.Arbitrum.Data; using Nethermind.Arbitrum.Execution.Stateless; @@ -13,25 +19,8 @@ namespace Nethermind.Arbitrum.Test.Execution; public class ArbitrumWitnessGenerationTests { - [TestCase(1ul)] - [TestCase(2ul)] - [TestCase(3ul)] - [TestCase(4ul)] - [TestCase(5ul)] - [TestCase(6ul)] - [TestCase(7ul)] - [TestCase(8ul)] - [TestCase(9ul)] - [TestCase(10ul)] - [TestCase(11ul)] - [TestCase(12ul)] - [TestCase(13ul)] - [TestCase(14ul)] - [TestCase(15ul)] - [TestCase(16ul)] - [TestCase(17ul)] - [TestCase(18ul)] - public async Task RecordBlockCreation_Witness_AllowsStatelessExecution(ulong messageIndex) + [TestCaseSource(nameof(ExecutionWitnessWithoutWasmsSource))] + public async Task RecordBlockCreation_WitnessWithoutUserWasms_StatelessExecutionIsSuccessful(ulong messageIndex) { FullChainSimulationRecordingFile recording = new("./Recordings/1__arbos32_basefee92.jsonl"); DigestMessageParameters digestMessage = recording.GetDigestMessages().First(m => m.Index == messageIndex); @@ -47,7 +36,7 @@ public async Task RecordBlockCreation_Witness_AllowsStatelessExecution(ulong mes ISpecProvider specProvider = FullChainSimulationChainSpecProvider.CreateDynamicSpecProvider(); ArbitrumStatelessBlockProcessingEnv blockProcessingEnv = - new(witness, specProvider, Always.Valid, chain.WasmStore, chain.ArbosVersionProvider, chain.LogManager, chain.ArbitrumConfig); + new(witness, specProvider, Always.Valid, chain.StylusTargetConfig, chain.ArbosVersionProvider, chain.LogManager, chain.ArbitrumConfig); Block block = chain.BlockFinder.FindBlock(recordResult.BlockHash) ?? throw new ArgumentException($"Unable to find block {recordResult.BlockHash}"); @@ -66,6 +55,57 @@ public async Task RecordBlockCreation_Witness_AllowsStatelessExecution(ulong mes } } + [TestCaseSource(nameof(ExecutionWitnessWithWasmsSource))] + public async Task RecordBlockCreation_WitnessWithUserWasms_StatelessExecutionIsSuccessful(ulong messageIndex) + { + FullChainSimulationRecordingFile recording = new("./Recordings/5__stylus.jsonl"); + DigestMessageParameters digestMessage = recording.GetDigestMessages().First(m => m.Index == messageIndex); + + using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() + .WithRecording(recording) + .Build(); + + string[] wasmTargets = chain.StylusTargetConfig.GetWasmTargets().ToArray(); + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation(new RecordBlockCreationParameters(digestMessage.Index, digestMessage.Message, WasmTargets: wasmTargets)); + RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestMessage.Index); + + ArbitrumWitness witness = recordResult.Witness; + + ISpecProvider specProvider = FullChainSimulationChainSpecProvider.CreateDynamicSpecProvider(); + ArbitrumStatelessBlockProcessingEnv blockProcessingEnv = + new(witness, specProvider, Always.Valid, chain.StylusTargetConfig, chain.ArbosVersionProvider, chain.LogManager, chain.ArbitrumConfig); + + Block block = chain.BlockFinder.FindBlock(recordResult.BlockHash) + ?? throw new ArgumentException($"Unable to find block {recordResult.BlockHash}"); + BlockHeader parent = chain.BlockFinder.FindHeader(block.ParentHash!) + ?? throw new ArgumentException($"Unable to find parent for block {recordResult.BlockHash}"); + + using (blockProcessingEnv.WorldState.BeginScope(parent)) + { + (Block processed, TxReceipt[] _) = blockProcessingEnv.BlockProcessor.ProcessOne( + block, + ProcessingOptions.DoNotUpdateHead | ProcessingOptions.ReadOnlyChain, + NullBlockTracer.Instance, + specProvider.GetSpec(block.Header)); + + Assert.That(processed.Hash, Is.EqualTo(block.Hash)); + } + } + + private static IEnumerable ExecutionWitnessWithoutWasmsSource() + { + // 18 blocks in the test where this test case source is used + for (ulong blockNumber = 1; blockNumber <= 18; blockNumber++) + yield return new TestCaseData(blockNumber); + } + + private static IEnumerable ExecutionWitnessWithWasmsSource() + { + // 47 blocks in the test where this test case source is used + for (ulong blockNumber = 1; blockNumber <= 47; blockNumber++) + yield return new TestCaseData(blockNumber); + } + private static T ThrowOnFailure(ResultWrapper result, ulong msgIndex) { if (result.Result != Result.Success) diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs index 7d4ce1a66..a38101fb3 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs @@ -92,9 +92,8 @@ public abstract class ArbitrumTestBlockchainBase(ChainSpec chainSpec, ArbitrumCo public IWasmDb WasmDB => Container.Resolve(); - // Maybe use Dependencies instead? for those 2 below - public IWasmStore WasmStore => Container.Resolve(); - public IArbosVersionProvider ArbosVersionProvider => Container.Resolve(); + public IWasmStore WasmStore => Dependencies.WasmStore; + public IArbosVersionProvider ArbosVersionProvider => Dependencies.ArbosVersionProvider; public IDb CodeDB => Container.ResolveKeyed("code"); @@ -317,6 +316,8 @@ protected record BlockchainContainerDependencies( IBlockProducerEnvFactory BlockProducerEnvFactory, ISealer Sealer, CachedL1PriceData CachedL1PriceData, + IWasmStore WasmStore, + IArbosVersionProvider ArbosVersionProvider, IArbitrumSpecHelper SpecHelper); private void InitializeArbitrumPluginSteps(IContainer container) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs index 981e2b28c..47b5ab0c2 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs @@ -14,7 +14,9 @@ using Nethermind.Consensus.Rewards; using Nethermind.Consensus.Validators; using Nethermind.Consensus.Withdrawals; +using Nethermind.Core.Crypto; using Nethermind.Core.Specs; +using Nethermind.Db; using Nethermind.Evm.State; using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; @@ -26,12 +28,11 @@ namespace Nethermind.Arbitrum.Execution.Stateless; -// TODO: use wasms (not priority for now) public class ArbitrumStatelessBlockProcessingEnv( ArbitrumWitness arbWitness, ISpecProvider specProvider, ISealValidator sealValidator, - IWasmStore wasmStore, + IStylusTargetConfig stylusTargetConfig, IArbosVersionProvider arbosVersionProvider, ILogManager logManager, IArbitrumConfig config) @@ -39,7 +40,7 @@ public class ArbitrumStatelessBlockProcessingEnv( private IBlockProcessor? _blockProcessor; public IBlockProcessor BlockProcessor { - get => _blockProcessor ??= GetProcessor(); + get => _blockProcessor ??= GetBlockProcessor(); } private IWorldState? _worldState; @@ -50,7 +51,34 @@ public IWorldState WorldState arbWitness.Witness.CodeDb, logManager), logManager); } - private IBlockProcessor GetProcessor() + private IWasmStore? _wasmStore; + public IWasmStore WasmStore + { + get => _wasmStore ??= CreateWasmStore(); + } + + private IWasmStore CreateWasmStore() + { + WasmDb wasmDb = new(new MemDb()); + WasmStore store = new(wasmDb, stylusTargetConfig, cacheTag: 1); + + // For info, pre-activation not even needed ! + // If we omit this, wasm store will lazily load wasms from codeDB and compile them to asms when needed during execution. + // + // Btw, that's what nitro's debug execution witness endpoint might be doing (didn't see wasms passed there when I last checked). + // To check but not priority for now. + if (arbWitness.UserWasms is not null) + { + foreach ((ValueHash256 moduleHash, IReadOnlyDictionary asmMap) in arbWitness.UserWasms) + { + store.ActivateWasm(in moduleHash, asmMap); + } + } + + return store; + } + + private IBlockProcessor GetBlockProcessor() { StatelessBlockTree statelessBlockTree = new(arbWitness.Witness.DecodedHeaders); ITransactionProcessor txProcessor = CreateTransactionProcessor(WorldState, statelessBlockTree); @@ -73,7 +101,7 @@ private IBlockProcessor GetProcessor() WorldState, NullReceiptStorage.Instance, new BlockhashStore(WorldState), - wasmStore, + WasmStore, new BeaconBlockRootHandler(txProcessor, WorldState), logManager, new WithdrawalProcessor(WorldState, logManager), @@ -85,7 +113,7 @@ private IBlockProcessor GetProcessor() private ITransactionProcessor CreateTransactionProcessor(IWorldState state, StatelessBlockTree blockFinder) { BlockhashProvider blockhashProvider = new(blockFinder, state, logManager); - ArbitrumVirtualMachine vm = new(blockhashProvider, wasmStore, specProvider, logManager); - return new ArbitrumTransactionProcessor(BlobBaseFeeCalculator.Instance, specProvider, state, wasmStore, vm, logManager, new ArbitrumCodeInfoRepository(new EthereumCodeInfoRepository(state), arbosVersionProvider)); + ArbitrumVirtualMachine vm = new(blockhashProvider, WasmStore, specProvider, logManager); + return new ArbitrumTransactionProcessor(BlobBaseFeeCalculator.Instance, specProvider, state, WasmStore, vm, logManager, new ArbitrumCodeInfoRepository(new EthereumCodeInfoRepository(state), arbosVersionProvider)); } } From fd168144a872729995ed9995421d7cf22d333be7 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Thu, 5 Feb 2026 18:55:08 +0900 Subject: [PATCH 20/87] tests: Always record bytecode for ExtCodeSize opcode --- .../ArbitrumWitnessGenerationTests.cs | 154 ++++++++++++++++++ .../ArbitrumRpcTestBlockchain.cs | 9 + 2 files changed, 163 insertions(+) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index 0622b5987..800e2c4e0 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -92,6 +92,160 @@ public async Task RecordBlockCreation_WitnessWithUserWasms_StatelessExecutionIsS } } + /// + /// Verifies that EXTCODESIZE correctly records the target contract's code in the witness, + /// even when followed by ISZERO (which would trigger a peephole optimization in base Nethermind). + /// Arbitrum overrides EXTCODESIZE to always access the contract bytecode for witness generation. + /// + [Test] + public async Task RecordBlockCreation_ExtCodeSizeFollowedByIsZero_StillRecordsTargetCodeInWitness() + { + UInt256 l1BaseFee = 92; + + using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() + .WithGenesisBlock(initialBaseFee: (ulong)l1BaseFee) + .Build(); + + Address sender = FullChainSimulationAccounts.Owner.Address; + + // Step 1: Fund the sender account with ETH deposit + TestEthDeposit deposit = new( + Keccak.Compute("deposit"), + l1BaseFee, + sender, + sender, + 100.Ether()); + ResultWrapper depositResult = await chain.Digest(deposit); + depositResult.Result.Should().Be(Result.Success); + + // Step 2: Deploy a target contract with some bytecode (does not matter what it does) + // Target contract runtime code: simply returns 42 when called + // PUSH1 42, PUSH1 0, MSTORE, PUSH1 32, PUSH1 0, RETURN + byte[] targetRuntimeCode = Prepare.EvmCode + .PushData(42) + .PushData(0) + .Op(Instruction.MSTORE) + .PushData(32) + .PushData(0) + .Op(Instruction.RETURN) + .Done; + + // Init code that deploys the runtime code + byte[] targetInitCode = Prepare.EvmCode + .ForInitOf(targetRuntimeCode) + .Done; + + Transaction deployTargetTx; + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + deployTargetTx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithTo(null) // Contract creation + .WithData(targetInitCode) + .WithMaxFeePerGas(10.GWei()) + .WithGasLimit(500_000) + .WithValue(0) + .WithNonce(chain.MainWorldState.GetNonce(sender)) + .SignedAndResolved(FullChainSimulationAccounts.Owner) + .TestObject; + } + + ResultWrapper deployTargetResult = await chain.Digest(new TestL2Transactions(l1BaseFee, sender, deployTargetTx)); + deployTargetResult.Result.Should().Be(Result.Success); + chain.LatestReceipts()[1].StatusCode.Should().Be(StatusCode.Success); + + // Compute the deployed target contract address + Address targetAddress = ContractAddress.From(sender, deployTargetTx.Nonce); + + // Step 3: Deploy a caller contract that uses EXTCODESIZE followed by ISZERO + // This pattern would trigger peephole optimization in base Nethermind consequently not fetching the contract bytecode, + // but Arbitrum should still record the full bytecode for witness generation. + // Caller contract: EXTCODESIZE(target), ISZERO, PUSH1 0, MSTORE, PUSH1 32, PUSH1 0, RETURN + byte[] callerRuntimeCode = Prepare.EvmCode + .EXTCODESIZE(targetAddress) + .Op(Instruction.ISZERO) // This triggers the peephole optimization pattern in base Nethermind + .PushData(0) + .Op(Instruction.MSTORE) + .PushData(32) + .PushData(0) + .Op(Instruction.RETURN) + .Done; + + byte[] callerInitCode = Prepare.EvmCode + .ForInitOf(callerRuntimeCode) + .Done; + + Transaction deployCallerTx; + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + deployCallerTx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithTo(null) // Contract creation + .WithData(callerInitCode) + .WithMaxFeePerGas(10.GWei()) + .WithGasLimit(500_000) + .WithValue(0) + .WithNonce(chain.MainWorldState.GetNonce(sender)) + .SignedAndResolved(FullChainSimulationAccounts.Owner) + .TestObject; + } + + ResultWrapper deployCallerResult = await chain.Digest(new TestL2Transactions(l1BaseFee, sender, deployCallerTx)); + deployCallerResult.Result.Should().Be(Result.Success); + chain.LatestReceipts()[1].StatusCode.Should().Be(StatusCode.Success); + + Address callerAddress = ContractAddress.From(sender, deployCallerTx.Nonce); + + // Step 4: Call the caller contract (triggers EXTCODESIZE on target) + Transaction callCallerTx; + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + callCallerTx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithTo(callerAddress) + .WithData([]) + .WithMaxFeePerGas(10.GWei()) + .WithGasLimit(500_000) + .WithValue(0) + .WithNonce(chain.MainWorldState.GetNonce(sender)) + .SignedAndResolved(FullChainSimulationAccounts.Owner) + .TestObject; + } + + // Use DigestAndGetParams to get the message parameters for RecordBlockCreation + (ResultWrapper callResult, DigestMessageParameters callParams) = + await chain.DigestAndGetParams(new TestL2Transactions(l1BaseFee, sender, callCallerTx)); + callResult.Result.Should().Be(Result.Success); + chain.LatestReceipts()[1].StatusCode.Should().Be(StatusCode.Success); + + // Step 5: Call RecordBlockCreation to generate the witness + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(callParams.Index, callParams.Message, WasmTargets: [])); + RecordResult recordResult = ThrowOnFailure(recordResultWrapper, callParams.Index); + + // Step 6: Verify the witness contains the target contract's code + ArbitrumWitness witness = recordResult.Witness; + byte[][] witnessCodes = witness.Witness.Codes; + + witnessCodes.Length.Should().Be(2, "Witness should contain both caller and target contract codes"); + + // The target contract's code should be in the witness + // (EXTCODESIZE should have triggered GetCode on the target) + Hash256 targetCodeHash = Keccak.Compute(targetRuntimeCode); + bool targetCodeInWitness = witnessCodes.Any(code => Keccak.Compute(code) == targetCodeHash); + + targetCodeInWitness.Should().BeTrue( + "Target contract's code should be recorded in witness when EXTCODESIZE is called, " + + "even if followed by ISZERO (peephole optimization pattern)"); + + // Also verify the caller contract's code is in the witness (since we executed it) + Hash256 callerCodeHash = Keccak.Compute(callerRuntimeCode); + bool callerCodeInWitness = witnessCodes.Any(code => Keccak.Compute(code) == callerCodeHash); + + callerCodeInWitness.Should().BeTrue( + "Caller contract's code should be recorded in witness since we executed it"); + } + private static IEnumerable ExecutionWitnessWithoutWasmsSource() { // 18 blocks in the test where this test case source is used diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs index dcfb37738..5352170dc 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs @@ -250,6 +250,15 @@ public async Task> Digest(TestL2Transactions messag return await ArbitrumRpcModule.DigestMessage(parameters); } + public async Task<(ResultWrapper Result, DigestMessageParameters Parameters)> DigestAndGetParams(TestL2Transactions message) + { + DigestMessageParameters parameters = CreateDigestMessage(ArbitrumL1MessageKind.L2Message, message.RequestId, message.L1BaseFee, + message.Sender, message.Transactions); + + ResultWrapper result = await ArbitrumRpcModule.DigestMessage(parameters); + return (result, parameters); + } + public void DumpBlocks() { List blocks = new(); From c871545c9b8bc1db6dc5db32518564f3254cdbb9 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Thu, 5 Feb 2026 18:55:59 +0900 Subject: [PATCH 21/87] tests: Record invalid bytecode for arb precompiles and nothing for eth precompiles --- .../ArbitrumWitnessGenerationTests.cs | 102 ++++++++++++++++++ .../ArbitrumBinaryTestWriter.cs | 2 +- 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index 800e2c4e0..81e15c646 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -246,6 +246,108 @@ public async Task RecordBlockCreation_ExtCodeSizeFollowedByIsZero_StillRecordsTa "Caller contract's code should be recorded in witness since we executed it"); } + /// + /// Verifies that witness generation correctly records precompile bytecodes: + /// - Arbitrum precompiles (e.g., ArbSys): should record 0xfe (INVALID opcode) + /// - Ethereum precompiles (e.g., ecrecover): should NOT record any bytecode (empty code) + /// + [Test] + public async Task RecordBlockCreation_PrecompileCalls_RecordsArbitrumPrecompileCodeButNotEthereumPrecompileCode() + { + UInt256 l1BaseFee = 92; + + using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() + .WithGenesisBlock(initialBaseFee: (ulong)l1BaseFee) + .Build(); + + Address sender = FullChainSimulationAccounts.Owner.Address; + + // Fund the sender account with ETH deposit + TestEthDeposit deposit = new( + Keccak.Compute("deposit"), + l1BaseFee, + sender, + sender, + 100.Ether()); + ResultWrapper depositResult = await chain.Digest(deposit); + depositResult.Result.Should().Be(Result.Success); + + // Create two transactions: + // 1. Call ArbSys (Arbitrum precompile at 0x64) - has 0xfe bytecode stored + // 2. Call ecrecover (Ethereum precompile at 0x01) - has no bytecode stored + + Address arbSysAddress = ArbSys.Address; // 0x64 + Address ecrecoverAddress = new("0x0000000000000000000000000000000000000001"); + + // ArbSys.arbBlockNumber() selector + byte[] arbBlockNumberCalldata = Keccak.Compute("arbBlockNumber()"u8).Bytes[..4].ToArray(); + + Transaction callArbSysTx; + Transaction callEcrecoverTx; + + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + // Transaction 1: Call ArbSys.arbBlockNumber() + callArbSysTx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithTo(arbSysAddress) + .WithData(arbBlockNumberCalldata) + .WithMaxFeePerGas(10.GWei()) + .WithGasLimit(100_000) + .WithValue(0) + .WithNonce(chain.MainWorldState.GetNonce(sender)) + .SignedAndResolved(FullChainSimulationAccounts.Owner) + .TestObject; + + // Transaction 2: Call ecrecover with dummy data (will fail but still executes precompile) + callEcrecoverTx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithTo(ecrecoverAddress) + .WithData(new byte[128]) // ecrecover expects 128 bytes (hash, v, r, s) + .WithMaxFeePerGas(10.GWei()) + .WithGasLimit(100_000) + .WithValue(0) + .WithNonce(chain.MainWorldState.GetNonce(sender) + 1) + .SignedAndResolved(FullChainSimulationAccounts.Owner) + .TestObject; + } + + // Digest both transactions in a single block + (ResultWrapper callResult, DigestMessageParameters callParams) = + await chain.DigestAndGetParams(new TestL2Transactions(l1BaseFee, sender, callArbSysTx, callEcrecoverTx)); + callResult.Result.Should().Be(Result.Success); + + // Both transactions should succeed (ecrecover returns empty on invalid input but doesn't revert) + TxReceipt[] receipts = chain.LatestReceipts(); + receipts[1].StatusCode.Should().Be(StatusCode.Success, "ArbSys call should succeed"); + receipts[2].StatusCode.Should().Be(StatusCode.Success, "ecrecover call should succeed"); + + // Call RecordBlockCreation to generate the witness + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(callParams.Index, callParams.Message, WasmTargets: [])); + RecordResult recordResult = ThrowOnFailure(recordResultWrapper, callParams.Index); + + // Verify the witness codes + ArbitrumWitness witness = recordResult.Witness; + byte[][] witnessCodes = witness.Witness.Codes; + + // Arbitrum precompile bytecode is 0xfe (INVALID opcode) + byte[] arbitrumPrecompileCode = Arbitrum.Arbos.Precompiles.InvalidCode; + + // The witness should contain the Arbitrum precompile's code (0xfe) + bool arbitrumPrecompileCodeInWitness = witnessCodes.Any(code => code.SequenceEqual(arbitrumPrecompileCode)); + arbitrumPrecompileCodeInWitness.Should().BeTrue( + "Arbitrum precompile bytecode (0xfe) should be recorded in witness when calling ArbSys"); + + // Verify that no empty code is recorded (Ethereum precompiles have no stored bytecode) + bool emptyCodeInWitness = witnessCodes.Any(code => code.Length == 0); + emptyCodeInWitness.Should().BeFalse("Ethereum precompiles empty bytecode should not be recorded in witness"); + + // The witness should have exactly 1 code: Arbitrum precompile (0xfe) + // No code from Ethereum precompile since it has empty bytecode + witnessCodes.Length.Should().Be(1, "Witness should contain only Arbitrum precompile code (0xfe)"); + } + private static IEnumerable ExecutionWitnessWithoutWasmsSource() { // 18 blocks in the test where this test case source is used diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumBinaryTestWriter.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumBinaryTestWriter.cs index 1c07cb65d..07516d192 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumBinaryTestWriter.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumBinaryTestWriter.cs @@ -41,7 +41,7 @@ public static void WriteHash256(BinaryWriter writer, Hash256 hash) public static void WriteByteString(BinaryWriter writer, byte[] data) { - WriteUInt256(writer, (ulong)data.Length); + WriteULongBigEndian(writer, (ulong)data.Length); writer.Write(data); } From 3fc5def0bc8ddd259a584b4547de20d6685cb638 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Thu, 5 Feb 2026 18:56:14 +0900 Subject: [PATCH 22/87] fix: Format --- src/Nethermind.Arbitrum/Stylus/WasmStoreRebuilder.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Nethermind.Arbitrum/Stylus/WasmStoreRebuilder.cs b/src/Nethermind.Arbitrum/Stylus/WasmStoreRebuilder.cs index 1bbb36fe0..1fe5f8187 100644 --- a/src/Nethermind.Arbitrum/Stylus/WasmStoreRebuilder.cs +++ b/src/Nethermind.Arbitrum/Stylus/WasmStoreRebuilder.cs @@ -11,12 +11,12 @@ public class WasmStoreRebuilder( ILogger logger) { public void RebuildWasmStore( - IDb codeDb, - Hash256 position, - ulong latestBlockTime, - ulong rebuildStartBlockTime, - bool debugMode, - CancellationToken cancellationToken) + IDb codeDb, + Hash256 position, + ulong latestBlockTime, + ulong rebuildStartBlockTime, + bool debugMode, + CancellationToken cancellationToken) { IReadOnlyCollection targets = targetConfig.GetWasmTargets(); DateTime lastStatusUpdate = DateTime.UtcNow; From 314b057851f73630ccb1dd73f780dc0e011b0552 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 6 Feb 2026 22:16:11 +0900 Subject: [PATCH 23/87] tests: User asms capture (not just stateless exec) --- .../ArbitrumWitnessGenerationTests.cs | 88 +++++++++++++++++-- .../ArbitrumStatelessBlockProcessingEnv.cs | 2 +- 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index 81e15c646..babbe573d 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -1,4 +1,5 @@ using FluentAssertions; +using Nethermind.Arbitrum.Arbos; using Nethermind.Arbitrum.Precompiles; using Nethermind.Arbitrum.Test.Infrastructure; using Nethermind.Blockchain.Tracing; @@ -12,6 +13,7 @@ using Nethermind.Evm; using Nethermind.Int256; using Nethermind.JsonRpc; +using Nethermind.Logging; using Nethermind.Arbitrum.Data; using Nethermind.Arbitrum.Execution.Stateless; @@ -19,7 +21,7 @@ namespace Nethermind.Arbitrum.Test.Execution; public class ArbitrumWitnessGenerationTests { - [TestCaseSource(nameof(ExecutionWitnessWithoutWasmsSource))] + [TestCaseSource(nameof(ExecutionWitnessWithoutStylusSource))] public async Task RecordBlockCreation_WitnessWithoutUserWasms_StatelessExecutionIsSuccessful(ulong messageIndex) { FullChainSimulationRecordingFile recording = new("./Recordings/1__arbos32_basefee92.jsonl"); @@ -55,8 +57,8 @@ public async Task RecordBlockCreation_WitnessWithoutUserWasms_StatelessExecution } } - [TestCaseSource(nameof(ExecutionWitnessWithWasmsSource))] - public async Task RecordBlockCreation_WitnessWithUserWasms_StatelessExecutionIsSuccessful(ulong messageIndex) + [TestCaseSource(nameof(ExecutionWitnessWithStylusSource))] + public async Task RecordBlockCreation_WitnessWithUserWasms_StatelessExecutionIsSuccessful(ulong messageIndex, Address[] _) { FullChainSimulationRecordingFile recording = new("./Recordings/5__stylus.jsonl"); DigestMessageParameters digestMessage = recording.GetDigestMessages().First(m => m.Index == messageIndex); @@ -92,6 +94,57 @@ public async Task RecordBlockCreation_WitnessWithUserWasms_StatelessExecutionIsS } } + [TestCaseSource(nameof(ExecutionWitnessWithStylusSource))] + public async Task RecordBlockCreation_WitnessWithUserWasms_CaptureAsms(ulong messageIndex, Address[] executedStylusContracts) + { + FullChainSimulationRecordingFile recording = new("./Recordings/5__stylus.jsonl"); + DigestMessageParameters digestMessage = recording.GetDigestMessages().First(m => m.Index == messageIndex); + + using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() + .WithRecording(recording) + .Build(); + + string[] wasmTargets = chain.StylusTargetConfig.GetWasmTargets().ToArray(); + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation(new RecordBlockCreationParameters(digestMessage.Index, digestMessage.Message, WasmTargets: wasmTargets)); + RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestMessage.Index); + + ArbitrumWitness witness = recordResult.Witness; + + // Build expected dictionary from chain state using the stylus contract addresses + Dictionary> expected = new(); + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head!.Header)) + { + ArbosState arbosState = ArbosState.OpenArbosState(chain.MainWorldState, new SystemBurner(), NullLogger.Instance); + + foreach (Address contract in executedStylusContracts) + { + ValueHash256 codeHash = chain.MainWorldState.GetCodeHash(contract); + ValueHash256 moduleHash = arbosState.Programs.ModuleHashesStorage.Get(codeHash); + + Dictionary expectedAsms = new(); + foreach (string target in wasmTargets) + { + chain.WasmStore.TryGetActivatedAsm(target, in moduleHash, out byte[]? asmBytes).Should().BeTrue( + $"WasmStore should contain ASM for module {moduleHash}, target '{target}'"); + expectedAsms[target] = asmBytes!; + } + + expected[moduleHash.ToHash256()] = expectedAsms; + } + } + + // Build actual dictionary from witness by hashing ASM byte arrays + Dictionary> actual = witness.UserWasms? + .ToDictionary( + kvp => kvp.Key.ToHash256(), + kvp => (IReadOnlyDictionary)kvp.Value.ToDictionary( + asm => asm.Key, + asm => asm.Value)) + ?? []; + + actual.Should().BeEquivalentTo(expected); + } + /// /// Verifies that EXTCODESIZE correctly records the target contract's code in the witness, /// even when followed by ISZERO (which would trigger a peephole optimization in base Nethermind). @@ -279,7 +332,6 @@ public async Task RecordBlockCreation_PrecompileCalls_RecordsArbitrumPrecompileC Address arbSysAddress = ArbSys.Address; // 0x64 Address ecrecoverAddress = new("0x0000000000000000000000000000000000000001"); - // ArbSys.arbBlockNumber() selector byte[] arbBlockNumberCalldata = Keccak.Compute("arbBlockNumber()"u8).Bytes[..4].ToArray(); Transaction callArbSysTx; @@ -348,18 +400,40 @@ public async Task RecordBlockCreation_PrecompileCalls_RecordsArbitrumPrecompileC witnessCodes.Length.Should().Be(1, "Witness should contain only Arbitrum precompile code (0xfe)"); } - private static IEnumerable ExecutionWitnessWithoutWasmsSource() + private static IEnumerable ExecutionWitnessWithoutStylusSource() { // 18 blocks in the test where this test case source is used for (ulong blockNumber = 1; blockNumber <= 18; blockNumber++) yield return new TestCaseData(blockNumber); } - private static IEnumerable ExecutionWitnessWithWasmsSource() + private static IEnumerable ExecutionWitnessWithStylusSource() { // 47 blocks in the test where this test case source is used + // Yield both the block number and the stylus contract addresses executed/called (activated ones should not be recorded) in that block for (ulong blockNumber = 1; blockNumber <= 47; blockNumber++) - yield return new TestCaseData(blockNumber); + { + if (blockNumber == 25) + yield return new TestCaseData((ulong)25, new[] { new Address("0x1294b86822ff4976bfe136cb06cf43ec7fcf2574") }); + else if (blockNumber == 26) + yield return new TestCaseData((ulong)26, new[] { new Address("0xe1080224b632a93951a7cfa33eeea9fd81558b5e") }); + else if (blockNumber == 32) + yield return new TestCaseData((ulong)32, new[] { new Address("0x1294b86822ff4976bfe136cb06cf43ec7fcf2574") }); + else if (blockNumber == 34) + yield return new TestCaseData((ulong)34, new[] { new Address("0x4af567288e68cad4aa93a272fe6139ca53859c70"), new Address("0x1294b86822ff4976bfe136cb06cf43ec7fcf2574") }); + else if (blockNumber == 38) + yield return new TestCaseData((ulong)38, new[] { new Address("0x1294b86822ff4976bfe136cb06cf43ec7fcf2574") }); + else if (blockNumber == 39) + yield return new TestCaseData((ulong)39, new[] { new Address("0x408da76e87511429485c32e4ad647dd14823fdc4"), new Address("0x1294b86822ff4976bfe136cb06cf43ec7fcf2574") }); + else if (blockNumber == 41) + yield return new TestCaseData((ulong)41, new[] { new Address("0x1294b86822ff4976bfe136cb06cf43ec7fcf2574") }); + else if (blockNumber == 42) + yield return new TestCaseData((ulong)42, new[] { new Address("0x408da76e87511429485c32e4ad647dd14823fdc4"), new Address("0x1294b86822ff4976bfe136cb06cf43ec7fcf2574") }); + else if (blockNumber == 47) + yield return new TestCaseData((ulong)47, new[] { new Address("0x841118047f42754332d0ad4db8a2893761dd7f5d"), new Address("0x1294b86822ff4976bfe136cb06cf43ec7fcf2574") }); + else + yield return new TestCaseData(blockNumber, Array.Empty
()); + } } private static T ThrowOnFailure(ResultWrapper result, ulong msgIndex) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs index 47b5ab0c2..7098dc605 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs @@ -62,7 +62,7 @@ private IWasmStore CreateWasmStore() WasmDb wasmDb = new(new MemDb()); WasmStore store = new(wasmDb, stylusTargetConfig, cacheTag: 1); - // For info, pre-activation not even needed ! + // For info, pre-activation is not even needed ! // If we omit this, wasm store will lazily load wasms from codeDB and compile them to asms when needed during execution. // // Btw, that's what nitro's debug execution witness endpoint might be doing (didn't see wasms passed there when I last checked). From 4b5d83da6228888a13758d8a22038f6945be977f Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 6 Feb 2026 22:24:49 +0900 Subject: [PATCH 24/87] tests: BLOCKHASH opcode in witness gen mode should not benefit from global cache --- .../ArbitrumWitnessGenerationTests.cs | 110 ++++++++++++++++++ src/Nethermind.Arbitrum/Evm/L1BlockCache.cs | 2 +- 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index babbe573d..9c55a2dd1 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -400,6 +400,116 @@ public async Task RecordBlockCreation_PrecompileCalls_RecordsArbitrumPrecompileC witnessCodes.Length.Should().Be(1, "Witness should contain only Arbitrum precompile code (0xfe)"); } + /// + /// Verifies that witness generation for BLOCKHASH always accesses storage (not the L1BlockCache). + /// The witness-generating VM has its own fresh cache and must access storage to get block hashes. + /// The storage access records the corresponding trie nodes in the witness. + /// Storage slot: 1 + l1BlockNumber % 256 in Blockhashes substorage. + /// + [Test] + public async Task RecordBlockCreation_BlockHashOpcode_RecordsStorageTrieNodeInWitness() + { + UInt256 l1BaseFee = 92; + + using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() + .WithGenesisBlock(initialBaseFee: (ulong)l1BaseFee) + .Build(); + + Address sender = FullChainSimulationAccounts.Owner.Address; + + // Fund the sender account with ETH deposit + TestEthDeposit deposit = new( + Keccak.Compute("deposit"), + l1BaseFee, + sender, + sender, + 100.Ether()); + ResultWrapper depositResult = await chain.Digest(deposit); + depositResult.Result.Should().Be(Result.Success); + + // Get the current L1 block number from the chain to compute a valid block number for BLOCKHASH + // BLOCKHASH returns the hash of the given L1 block number if it's within the last 256 blocks + ulong currentL1BlockNumber = chain.LatestL1BlockNumber; + + // Deploy a contract that uses BLOCKHASH opcode + // Contract: BLOCKHASH(currentL1BlockNumber - 1), PUSH1 0, MSTORE, PUSH1 32, PUSH1 0, RETURN + // This returns the hash of a previous L1 block + ulong targetL1BlockNumber = currentL1BlockNumber > 0 ? currentL1BlockNumber - 1 : 0; + + byte[] blockhashCallerCode = Prepare.EvmCode + .PushData(targetL1BlockNumber) + .Op(Instruction.BLOCKHASH) + .PushData(0) + .Op(Instruction.MSTORE) + .PushData(32) + .PushData(0) + .Op(Instruction.RETURN) + .Done; + + byte[] blockhashCallerInitCode = Prepare.EvmCode + .ForInitOf(blockhashCallerCode) + .Done; + + Transaction deployTx; + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + deployTx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithTo(null) // Contract creation + .WithData(blockhashCallerInitCode) + .WithMaxFeePerGas(10.GWei()) + .WithGasLimit(500_000) + .WithValue(0) + .WithNonce(chain.MainWorldState.GetNonce(sender)) + .SignedAndResolved(FullChainSimulationAccounts.Owner) + .TestObject; + } + + ResultWrapper deployResult = await chain.Digest(new TestL2Transactions(l1BaseFee, sender, deployTx)); + deployResult.Result.Should().Be(Result.Success); + chain.LatestReceipts()[1].StatusCode.Should().Be(StatusCode.Success); + + Address contractAddress = ContractAddress.From(sender, deployTx.Nonce); + + // Step 1: Call the contract (this populates the main VM's L1BlockCache) + Transaction callTx; + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + callTx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithTo(contractAddress) + .WithData([]) + .WithMaxFeePerGas(10.GWei()) + .WithGasLimit(100_000) + .WithValue(0) + .WithNonce(chain.MainWorldState.GetNonce(sender)) + .SignedAndResolved(FullChainSimulationAccounts.Owner) + .TestObject; + } + + (ResultWrapper call2Result, DigestMessageParameters call2Params) = + await chain.DigestAndGetParams(new TestL2Transactions(l1BaseFee, sender, callTx)); + call2Result.Result.Should().Be(Result.Success); + chain.LatestReceipts()[1].StatusCode.Should().Be(StatusCode.Success); + + // Step 2: Call the contract again (would use cached value from the block that just got built + // if cache were persisted into witness-generating env) + // Making sure witness-generating VM has its own empty cache and must access storage, recording the trie nodes. + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(call2Params.Index, call2Params.Message, WasmTargets: [])); + RecordResult recordResult = ThrowOnFailure(recordResultWrapper, call2Params.Index); + + ArbitrumWitness witness = recordResult.Witness; + + // The storage slot accessed is: 1 + l1BlockNumber % 256 in the Blockhashes substorage (see GetL1BlockHash) + // Too difficult to predict exact trie node hash here, so, using the hardcoded value (found during debugging) + witness.Witness.State.Any(node => Keccak.Compute(node) == new Hash256("0x30cfd2590e997a3c3bee0c89572aec183bae0976e06334354832b85514d0d37a")).Should().BeTrue( + "Witness state should contain leaf trie node for BLOCKHASH storage access"); + // Similarly, checking for an intermediate node capture when accessing the storage slot + witness.Witness.State.Any(node => Keccak.Compute(node) == new Hash256("0xad9a2d73baabd92487dd1840cd076a06a3eded05e8cbdebb930ddad669e51880")).Should().BeTrue( + "Witness state should contain intermediate trie node for BLOCKHASH storage access"); + } + private static IEnumerable ExecutionWitnessWithoutStylusSource() { // 18 blocks in the test where this test case source is used diff --git a/src/Nethermind.Arbitrum/Evm/L1BlockCache.cs b/src/Nethermind.Arbitrum/Evm/L1BlockCache.cs index 21dc50e66..ba9213562 100644 --- a/src/Nethermind.Arbitrum/Evm/L1BlockCache.cs +++ b/src/Nethermind.Arbitrum/Evm/L1BlockCache.cs @@ -18,7 +18,7 @@ public sealed class L1BlockCache : IL1BlockCache private ulong? _cachedL1BlockNumber; /// - /// Global LRU cache for L1 block hashes. + /// LRU cache for L1 block hashes. /// 256 capacities match the BLOCKHASH opcode window (last 256 blocks). /// Thread-safe and shared across all transactions. /// From bc394bcff51f1dd327609c3433dfb1ea75a1393d Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 6 Feb 2026 22:25:57 +0900 Subject: [PATCH 25/87] tests: Querying for a block hash should record all block headers from that block up to current block's parent header --- .../ArbitrumWitnessGenerationTests.cs | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index 9c55a2dd1..1b6679177 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -510,6 +510,71 @@ public async Task RecordBlockCreation_BlockHashOpcode_RecordsStorageTrieNodeInWi "Witness state should contain intermediate trie node for BLOCKHASH storage access"); } + /// + /// Verifies that calling ArbSys.ArbBlockHash records all headers between the requested block + /// and the current block's parent in the witness. + /// + [Test] + public async Task RecordBlockCreation_ArbBlockHash_RecordsHeadersInWitness() + { + FullChainSimulationRecordingFile recording = new("./Recordings/1__arbos32_basefee92.jsonl"); + + using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() + .WithRecording(recording) + .Build(); + + Address sender = FullChainSimulationAccounts.Owner.Address; + UInt256 l1BaseFee = 92; + + // After the recording, we have blocks 0 (genesis) through 18. We'll call ArbSys.arbBlockHash for block 7 + ulong targetBlockNumber = 7; + byte[] arbBlockHashCalldata = Keccak.Compute("arbBlockHash(uint256)"u8).Bytes[..4].ToArray(); + byte[] calldata = new byte[36]; + arbBlockHashCalldata.CopyTo(calldata, 0); + new UInt256(targetBlockNumber).ToBigEndian(calldata.AsSpan(4)); + + Transaction callTx; + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + callTx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithTo(ArbSys.Address) + .WithData(calldata) + .WithMaxFeePerGas(10.GWei()) + .WithGasLimit(100_000) + .WithValue(0) + .WithNonce(chain.MainWorldState.GetNonce(sender)) + .SignedAndResolved(FullChainSimulationAccounts.Owner) + .TestObject; + } + + (ResultWrapper callResult, DigestMessageParameters callParams) = + await chain.DigestAndGetParams(new TestL2Transactions(l1BaseFee, sender, callTx)); + callResult.Result.Should().Be(Result.Success); + chain.LatestReceipts()[1].StatusCode.Should().Be(StatusCode.Success, "ArbBlockHash call should succeed"); + + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(callParams.Index, callParams.Message, WasmTargets: [])); + RecordResult recordResult = ThrowOnFailure(recordResultWrapper, callParams.Index); + + ArbitrumWitness witness = recordResult.Witness; + + // The witness should contain all RLP-encoded headers from targetBlockNumber to parentBlockNumber (inclusive) + long parentBlockNumber = chain.BlockTree.Head!.Number - 1; + + HashSet expectedHeaderHashes = new(); + for (long blockNum = (long)targetBlockNumber; blockNum <= parentBlockNumber; blockNum++) + expectedHeaderHashes.Add(chain.BlockTree.FindBlock(blockNum)!.Hash!); + + // Witness headers are raw rlp-encoded headers; compute their hashes for comparison + HashSet actualHeaderHashes = witness.Witness.Headers + .Select(header => Keccak.Compute(header)) + .ToHashSet(); + + // Compare hashsets instead of lists to avoid ordering issues, as order in witness does not matter + actualHeaderHashes.Should().BeEquivalentTo(expectedHeaderHashes); + } + private static IEnumerable ExecutionWitnessWithoutStylusSource() { // 18 blocks in the test where this test case source is used From 9e95c13dfa7047a4d0c1c90d5f428f8ff4d204a6 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Sun, 8 Feb 2026 22:51:09 +0900 Subject: [PATCH 26/87] tests: ArbosStorage.Set still traverses trie even with empty bytes use case example: Submit retryable tx with empty calldata and that returns early --- .../ArbitrumWitnessGenerationTests.cs | 124 ++++++++++++++++++ .../ArbitrumRpcTestBlockchain.cs | 36 +++++ 2 files changed, 160 insertions(+) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index 1b6679177..f0e047fdb 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -1,7 +1,10 @@ using FluentAssertions; using Nethermind.Arbitrum.Arbos; +using Nethermind.Arbitrum.Arbos.Storage; +using Nethermind.Arbitrum.Execution.Transactions; using Nethermind.Arbitrum.Precompiles; using Nethermind.Arbitrum.Test.Infrastructure; +using Nethermind.Blockchain; using Nethermind.Blockchain.Tracing; using Nethermind.Consensus.Processing; using Nethermind.Consensus.Validators; @@ -10,12 +13,14 @@ using Nethermind.Core.Extensions; using Nethermind.Core.Specs; using Nethermind.Core.Test.Builders; +using Nethermind.Crypto; using Nethermind.Evm; using Nethermind.Int256; using Nethermind.JsonRpc; using Nethermind.Logging; using Nethermind.Arbitrum.Data; using Nethermind.Arbitrum.Execution.Stateless; +using Nethermind.Evm.Tracing; namespace Nethermind.Arbitrum.Test.Execution; @@ -575,6 +580,125 @@ public async Task RecordBlockCreation_ArbBlockHash_RecordsHeadersInWitness() actualHeaderHashes.Should().BeEquivalentTo(expectedHeaderHashes); } + /// + /// Verifies that submitting a retryable transaction with empty calldata still traverses the trie + /// for the calldata storage slot and records the trie nodes up that path in the witness. + /// This tests the fix in ArbosStorage.Set(byte[]) where "Set(offset, Hash256.FromBytesWithPadding(span));" + /// was previously surrounded by "if (span.Length > 0)" to ensure it is now always called. + /// + [Test] + public async Task RecordBlockCreation_SubmitRetryableWithEmptyCalldata_RecordsCalldataTrieNodeInWitness() + { + FullChainSimulationRecordingFile recording = new("./Recordings/1__arbos32_basefee92.jsonl"); + + using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() + .WithRecording(recording) + .Build(); + + Address sender = FullChainSimulationAccounts.Owner.Address; + Address receiver = TestItem.AddressA; + Address beneficiary = TestItem.AddressB; + UInt256 l1BaseFee = 92; + + // GasLimit is set to 0 to make submit retryable tx finish early (without emitting RedeemScheduledEvent event) + // Hence, no call to retryable.Calldata.Get() and therefore no trie node recorded without the fix. + TestSubmitRetryable retryable = new( + Hash256.FromBytesWithPadding([0x1]), + l1BaseFee, + sender, + receiver, + beneficiary, + DepositValue: 10.Ether(), + RetryValue: 1.Ether(), + GasFee: 1.GWei(), + GasLimit: 0, + MaxSubmissionFee: 128800); + + // Compute the tx hash to know which substorage path CreateRetryable will use + ArbitrumSubmitRetryableTransaction transaction = new() + { + SourceHash = retryable.RequestId, + Nonce = UInt256.Zero, + GasPrice = UInt256.Zero, + DecodedMaxFeePerGas = retryable.GasFee, + GasLimit = (long)retryable.GasLimit, + Value = 0, + Data = retryable.RetryData, + IsOPSystemTransaction = false, + Mint = retryable.DepositValue, + ChainId = chain.ChainSpec.ChainId, + RequestId = retryable.RequestId, + SenderAddress = retryable.Sender, + L1BaseFee = retryable.L1BaseFee, + DepositValue = retryable.DepositValue, + GasFeeCap = retryable.GasFee, + Gas = retryable.GasLimit, + RetryTo = retryable.Receiver, + RetryValue = retryable.RetryValue, + Beneficiary = retryable.Beneficiary, + MaxSubmissionFee = retryable.MaxSubmissionFee, + FeeRefundAddr = retryable.Beneficiary, + RetryData = retryable.RetryData + }; + Hash256 txHash = transaction.CalculateHash(); + + // Pre-populate the calldata substorage offset 1 with a non-zero value. + // This ensures a leaf trie node exists at that slot. When CreateRetryable calls + // Calldata.Set([]) with the fix, it writes zero to offset 1 (deleting the leaf), + // which captures the leaf trie node in the witness. Without the fix, offset 1 is never + // accessed and the trie nodes along that storage slot path are not captured. + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head!.Header)) + { + ArbosStorage calldataStorage = new ArbosStorage(chain.MainWorldState, new SystemBurner(), ArbosAddresses.ArbosSystemAccount) + .OpenSubStorage(ArbosSubspaceIDs.RetryablesSubspace) + .OpenSubStorage(txHash.BytesToArray()) + .OpenSubStorage(Retryable.CallDataKey); + // Just giving it some random non-zero value to create a leaf node in the trie for that storage slot + calldataStorage.Set(1, Hash256.FromBytesWithPadding([0x5])); + + chain.MainWorldState.Commit(chain.SpecProvider.GenesisSpec, NullTxTracer.Instance); + chain.MainWorldState.CommitTree(chain.BlockTree.Head!.Number + 1); + + // Create a fake block with the new state root so the next DigestMessage sees it. + // + // A bit of hack but without this, setting up the test is almost impossible / kinda random + // and hardly maintainable. Because, then you'd need to record an intermediate trie + // node instead of the leaf node, because regular scenarios won't let you have a leaf node there beforehand. + // And to do that, you'd need to influence the tx parameters to change its hash to change the calldata storage slot path, + // and pray that it accesses some intermediate trie node that is not already accessed elsewhere in the block, + // which is very fragile. And some slight future code change could easily change the trie structure + // and break the test without any code changes to the test or witness generation itself. + // Trust me, I spent too much time on this test. + BlockHeader newHeader = chain.BlockTree.Head!.Header.Clone(); + newHeader.ParentHash = chain.BlockTree.HeadHash; + newHeader.StateRoot = chain.MainWorldState.StateRoot; + newHeader.Number++; + newHeader.Hash = newHeader.CalculateHash(); + newHeader.TotalDifficulty = (newHeader.TotalDifficulty ?? 0) + 1; + Block newBlock = chain.BlockTree.Head!.WithReplacedHeader(newHeader); + chain.BlockTree.SuggestBlock(newBlock, BlockTreeSuggestOptions.ForceSetAsMain); + chain.BlockTree.UpdateMainChain([newBlock], true, true); + } + + // Advance the block index just as if DigestMessage had been called for creating the fake block + chain.AdvanceBlockNumber(1); + + (ResultWrapper result, DigestMessageParameters digestParams) = + await chain.DigestAndGetParams(retryable); + result.Result.Should().Be(Result.Success); + + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); + RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestParams.Index); + + ArbitrumWitness witness = recordResult.Witness; + + // Assert some trie node on the path to the calldata storage slot has been captured (not captured elsewhere during block recording ofc, otherwise test is useless) + // Here I assert the leaf node hash (found during debugging). + witness.Witness.State.Any(node => Keccak.Compute(node) == new Hash256("0xb2020a6fea12f86ace9de5bed3312ca953a2f8ae0730062fa9df4fc833c99782")).Should().BeTrue( + "Witness state should contain trie node for retryable empty calldata storage slot"); + } + private static IEnumerable ExecutionWitnessWithoutStylusSource() { // 18 blocks in the test where this test case source is used diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs index 5352170dc..065780537 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs @@ -259,6 +259,42 @@ public async Task> Digest(TestL2Transactions messag return (result, parameters); } + public async Task<(ResultWrapper Result, DigestMessageParameters Parameters)> DigestAndGetParams(TestSubmitRetryable retryable) + { + ArbitrumSubmitRetryableTransaction transaction = new() + { + SourceHash = retryable.RequestId, + Nonce = UInt256.Zero, + GasPrice = UInt256.Zero, + DecodedMaxFeePerGas = retryable.GasFee, + GasLimit = (long)retryable.GasLimit, + Value = 0, + Data = retryable.RetryData, + IsOPSystemTransaction = false, + Mint = retryable.DepositValue, + + ChainId = ChainSpec.ChainId, + RequestId = retryable.RequestId, + SenderAddress = retryable.Sender, + L1BaseFee = retryable.L1BaseFee, + DepositValue = retryable.DepositValue, + GasFeeCap = retryable.GasFee, + Gas = retryable.GasLimit, + RetryTo = retryable.Receiver, + RetryValue = retryable.RetryValue, + Beneficiary = retryable.Beneficiary, + MaxSubmissionFee = retryable.MaxSubmissionFee, + FeeRefundAddr = retryable.Beneficiary, + RetryData = retryable.RetryData + }; + + DigestMessageParameters parameters = CreateDigestMessage(ArbitrumL1MessageKind.SubmitRetryable, retryable.RequestId, retryable.L1BaseFee, + retryable.Sender, transaction); + + ResultWrapper result = await ArbitrumRpcModule.DigestMessage(parameters); + return (result, parameters); + } + public void DumpBlocks() { List blocks = new(); From 59c71769842755f002bc94ffa68524dde61d3212 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 9 Feb 2026 20:01:55 +0900 Subject: [PATCH 27/87] tests: ArbosStorage.Set still sets state even with default value --- .../Execution/ArbitrumTransactionProcessorTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/ArbitrumTransactionProcessorTests.cs b/src/Nethermind.Arbitrum.Test/Execution/ArbitrumTransactionProcessorTests.cs index 67a8fe261..2b434f4b1 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/ArbitrumTransactionProcessorTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/ArbitrumTransactionProcessorTests.cs @@ -1602,7 +1602,7 @@ public void ProcessTransactions_SubmitRetryable_TraceEntries() tracer.BeforeEvmTransfers.Count.Should().Be(0); tracer.AfterEvmTransfers.Count.Should().Be(0); GethLikeTxTrace trace = tracer.BuildResult(); - trace.Entries.Count.Should().Be(41); + trace.Entries.Count.Should().Be(42); } [Test] From 4991eba84dbe24874b6327e9d2b3069a4b4419e8 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 9 Feb 2026 18:26:03 +0900 Subject: [PATCH 28/87] tests: Reads and records TimeoutWindowsLeft at the right place in TryReapOneRetryable --- .../ArbitrumWitnessGenerationTests.cs | 157 ++++++++++++++---- .../FullChainSimulationAccounts.cs | 1 - 2 files changed, 126 insertions(+), 32 deletions(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index f0e047fdb..e29973a59 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -645,43 +645,28 @@ public async Task RecordBlockCreation_SubmitRetryableWithEmptyCalldata_RecordsCa // Pre-populate the calldata substorage offset 1 with a non-zero value. // This ensures a leaf trie node exists at that slot. When CreateRetryable calls // Calldata.Set([]) with the fix, it writes zero to offset 1 (deleting the leaf), - // which captures the leaf trie node in the witness. Without the fix, offset 1 is never - // accessed and the trie nodes along that storage slot path are not captured. - using (chain.MainWorldState.BeginScope(chain.BlockTree.Head!.Header)) + // which captures the leaf trie node (to know where to create the new leaf) in the witness. + // Without the fix, offset 1 is never accessed and the trie nodes along that storage slot path are not captured. + // + // Create a fake block with the new state root so the next DigestMessage sees it. + // + // A bit of hack but without this, setting up the test is almost impossible / kinda random + // and hardly maintainable. Because, then you'd need to record an intermediate trie + // node instead of the leaf node, because regular scenarios won't let you have a leaf node there beforehand. + // And to do that, you'd need to influence the tx parameters to change its hash to change the calldata storage slot path, + // and pray that it accesses some intermediate trie node that is not already accessed elsewhere in the block, + // which is very fragile. And some slight future code change could easily change the trie structure + // and break the test without any code changes to the test or witness generation itself. + // Trust me, I spent too much time on this test. + chain.AppendBlock(chain => { ArbosStorage calldataStorage = new ArbosStorage(chain.MainWorldState, new SystemBurner(), ArbosAddresses.ArbosSystemAccount) .OpenSubStorage(ArbosSubspaceIDs.RetryablesSubspace) .OpenSubStorage(txHash.BytesToArray()) .OpenSubStorage(Retryable.CallDataKey); - // Just giving it some random non-zero value to create a leaf node in the trie for that storage slot + // Just giving the offset 1 some random non-zero value to create a leaf node in the trie for that storage slot calldataStorage.Set(1, Hash256.FromBytesWithPadding([0x5])); - - chain.MainWorldState.Commit(chain.SpecProvider.GenesisSpec, NullTxTracer.Instance); - chain.MainWorldState.CommitTree(chain.BlockTree.Head!.Number + 1); - - // Create a fake block with the new state root so the next DigestMessage sees it. - // - // A bit of hack but without this, setting up the test is almost impossible / kinda random - // and hardly maintainable. Because, then you'd need to record an intermediate trie - // node instead of the leaf node, because regular scenarios won't let you have a leaf node there beforehand. - // And to do that, you'd need to influence the tx parameters to change its hash to change the calldata storage slot path, - // and pray that it accesses some intermediate trie node that is not already accessed elsewhere in the block, - // which is very fragile. And some slight future code change could easily change the trie structure - // and break the test without any code changes to the test or witness generation itself. - // Trust me, I spent too much time on this test. - BlockHeader newHeader = chain.BlockTree.Head!.Header.Clone(); - newHeader.ParentHash = chain.BlockTree.HeadHash; - newHeader.StateRoot = chain.MainWorldState.StateRoot; - newHeader.Number++; - newHeader.Hash = newHeader.CalculateHash(); - newHeader.TotalDifficulty = (newHeader.TotalDifficulty ?? 0) + 1; - Block newBlock = chain.BlockTree.Head!.WithReplacedHeader(newHeader); - chain.BlockTree.SuggestBlock(newBlock, BlockTreeSuggestOptions.ForceSetAsMain); - chain.BlockTree.UpdateMainChain([newBlock], true, true); - } - - // Advance the block index just as if DigestMessage had been called for creating the fake block - chain.AdvanceBlockNumber(1); + }); (ResultWrapper result, DigestMessageParameters digestParams) = await chain.DigestAndGetParams(retryable); @@ -699,6 +684,116 @@ public async Task RecordBlockCreation_SubmitRetryableWithEmptyCalldata_RecordsCa "Witness state should contain trie node for retryable empty calldata storage slot"); } + /// + /// Verifies that TryReapOneRetryable reads the TimeoutWindowsLeft storage slot (offset 6) + /// even when the retryable has not expired (early return path at timeout >= currentTimestamp). + /// This tests the fix where TimeoutWindowsLeft.Get() was moved before the expiration check, + /// matching Nitro's behavior and ensuring the storage slot is captured in the witness. + /// + [Test] + public async Task RecordBlockCreation_TryReapRetryableNotExpired_RecordsTimeoutWindowsLeftInWitness() + { + using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() + .WithGenesisBlock() + .Build(); + + Address sender = FullChainSimulationAccounts.Owner.Address; + UInt256 l1BaseFee = 92; + + // Step 1: Submit a retryable with GasLimit: 0 (no auto-redeem). + // This creates a retryable ticket and enqueues it in the timeout queue. + // Timeout is set to currentTimestamp + 1 week, so it won't expire in the next block. + TestSubmitRetryable retryable = new( + Hash256.FromBytesWithPadding([0x1]), + l1BaseFee, + sender, + TestItem.AddressA, + TestItem.AddressB, + DepositValue: 10.Ether(), + RetryValue: 1.Ether(), + GasFee: 1.GWei(), + GasLimit: 0, + MaxSubmissionFee: 128800); + + ResultWrapper retryableResult = await chain.Digest(retryable); + retryableResult.Result.Should().Be(Result.Success); + + // Compute the tx hash to know which substorage path CreateRetryable used + ArbitrumSubmitRetryableTransaction transaction = new() + { + SourceHash = retryable.RequestId, + Nonce = UInt256.Zero, + GasPrice = UInt256.Zero, + DecodedMaxFeePerGas = retryable.GasFee, + GasLimit = (long)retryable.GasLimit, + Value = 0, + Data = retryable.RetryData, + IsOPSystemTransaction = false, + Mint = retryable.DepositValue, + ChainId = chain.ChainSpec.ChainId, + RequestId = retryable.RequestId, + SenderAddress = retryable.Sender, + L1BaseFee = retryable.L1BaseFee, + DepositValue = retryable.DepositValue, + GasFeeCap = retryable.GasFee, + Gas = retryable.GasLimit, + RetryTo = retryable.Receiver, + RetryValue = retryable.RetryValue, + Beneficiary = retryable.Beneficiary, + MaxSubmissionFee = retryable.MaxSubmissionFee, + FeeRefundAddr = retryable.Beneficiary, + RetryData = retryable.RetryData + }; + Hash256 txHash = transaction.CalculateHash(); + + // CreateRetryable calls TimeoutWindowsLeft.Set(0) which stores empty bytes, deleting any leaf at that slot. + // Without a leaf, reading the slot only traverses shared intermediate nodes that might likely also be captured + // by the many other ArbOS storage accesses in the same block — making any assertion on those nodes unreliable. + // Pre-populating with a non-zero value after retryable creation creates a unique leaf, + // so the assertion targets something that only appears when TimeoutWindowsLeft.Get() is called. + // Same reason and hack as the previous test to create a fake block with the new state root. + chain.AppendBlock(chain => + { + ArbosStorage retryableStorage = new ArbosStorage(chain.MainWorldState, new SystemBurner(), ArbosAddresses.ArbosSystemAccount) + .OpenSubStorage(ArbosSubspaceIDs.RetryablesSubspace) + .OpenSubStorage(txHash.BytesToArray()); + retryableStorage.Set(Retryable.TimeoutWindowsLeftOffset, Hash256.FromBytesWithPadding([0x5])); + }); + + // Step 3: Create the next block to trigger a start tx, which calls TryReapOneRetryable. + // The retryable's timeout (~1 week from now) >= currentTimestamp, so it returns early. + // With the fix, TimeoutWindowsLeft.Get() is called before the check, capturing the storage slot. + Transaction transferTx; + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + transferTx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithTo(TestItem.AddressC) + .WithData([]) + .WithMaxFeePerGas(1.GWei()) + .WithGasLimit(21_000) + .WithValue(1) + .WithNonce(chain.MainWorldState.GetNonce(sender)) + .SignedAndResolved(FullChainSimulationAccounts.Owner) + .TestObject; + } + + (ResultWrapper result, DigestMessageParameters digestParams) = + await chain.DigestAndGetParams(new TestL2Transactions(l1BaseFee, sender, transferTx)); + result.Result.Should().Be(Result.Success); + + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); + RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestParams.Index); + + ArbitrumWitness witness = recordResult.Witness; + + // Assert the leaf trie node for TimeoutWindowsLeft (offset 6) has been captured. + // Trie node hash determined during debugging — without the fix, this node would NOT be in the witness. + witness.Witness.State.Any(node => Keccak.Compute(node) == new Hash256("0xb9b0e8140da26e36ad74be6f20e6dc5073cda81b1ed9c3c8d63388f69640f24e")).Should().BeTrue( + "Witness state should contain trie node for retryable TimeoutWindowsLeft storage slot"); + } + private static IEnumerable ExecutionWitnessWithoutStylusSource() { // 18 blocks in the test where this test case source is used diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/FullChainSimulationAccounts.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/FullChainSimulationAccounts.cs index 9c1e0cdb4..a5bfa66a3 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/FullChainSimulationAccounts.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/FullChainSimulationAccounts.cs @@ -1,4 +1,3 @@ -using Nethermind.Core.Test.Builders; using Nethermind.Crypto; namespace Nethermind.Arbitrum.Test.Infrastructure; From f1a2c45453a152e84f555ead6c602c814f368a42 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 9 Feb 2026 20:06:12 +0900 Subject: [PATCH 29/87] tests: Non user txs still go into gas related calculation and record brotli compression level storage slot trie node --- .../ArbitrumWitnessGenerationTests.cs | 39 +++++++++++++++++++ .../ArbitrumRpcTestBlockchain.cs | 10 +++++ .../NitroL2MessageSerializer.cs | 5 ++- 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index e29973a59..5325f2bec 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -794,6 +794,45 @@ public async Task RecordBlockCreation_TryReapRetryableNotExpired_RecordsTimeoutW "Witness state should contain trie node for retryable TimeoutWindowsLeft storage slot"); } + /// + /// Verifies that CanAddTransaction reads BrotliCompressionLevel (ArbOS root offset 7) for + /// non-user transactions (such as the StartBlock tx), matching Nitro's behavior where + /// the data gas calculation runs for all transactions regardless of type. + /// Previously, non-user txs returned early from CanAddTransaction before reaching the + /// BrotliCompressionLevel.Get() call, so the corresponding trie node was not captured. + /// + /// An EndOfBlock message produces a block containing only the StartBlock internal tx (no user + /// transactions). This isolates the test: BrotliCompressionLevel can only be captured by the + /// internal tx's CanAddTransaction path, not by any user tx execution or gas charging. + /// + [Test] + public async Task RecordBlockCreation_NonUserTransaction_RecordsBrotliCompressionLevelInWitness() + { + using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() + .WithGenesisBlock() + .Build(); + + UInt256 l1BaseFee = 92; + + // EndOfBlock message produces a block with only the StartBlock internal tx (no user txs). + // BrotliCompressionLevel can only be captured by the internal tx's CanAddTransaction path. + (ResultWrapper result, DigestMessageParameters digestParams) = + await chain.DigestAndGetParams(new TestEndOfBlock(l1BaseFee)); + result.Result.Should().Be(Result.Success); + + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); + RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestParams.Index); + + ArbitrumWitness witness = recordResult.Witness; + + // Assert the leaf trie node for BrotliCompressionLevel (offset 7) has been captured. + // Trie node hash determined during debugging — without the fix, this node would NOT be + // in the witness because non-user txs returned early from CanAddTransaction. + witness.Witness.State.Any(node => Keccak.Compute(node) == new Hash256("0x9bcf99179b305f1d54185508b47cc61fb0f8b804dd449a9b60ed068af7b1d62f")).Should().BeTrue( + "Witness state should contain trie node for BrotliCompressionLevel storage slot (offset 7)"); + } + private static IEnumerable ExecutionWitnessWithoutStylusSource() { // 18 blocks in the test where this test case source is used diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs index 065780537..6c56b5fdd 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs @@ -295,6 +295,14 @@ public async Task> Digest(TestL2Transactions messag return (result, parameters); } + public async Task<(ResultWrapper Result, DigestMessageParameters Parameters)> DigestAndGetParams(TestEndOfBlock message) + { + DigestMessageParameters parameters = CreateDigestMessage(ArbitrumL1MessageKind.EndOfBlock, Hash256.Zero, message.L1BaseFee, Address.Zero); + + ResultWrapper result = await ArbitrumRpcModule.DigestMessage(parameters); + return (result, parameters); + } + public void DumpBlocks() { List blocks = new(); @@ -560,3 +568,5 @@ public TestL2Transactions(UInt256 L1BaseFee, Address Sender, params Transaction[ } } + +public record TestEndOfBlock(UInt256 L1BaseFee); diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/NitroL2MessageSerializer.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/NitroL2MessageSerializer.cs index 32039d59e..11fe21d9b 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/NitroL2MessageSerializer.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/NitroL2MessageSerializer.cs @@ -15,7 +15,7 @@ public static class NitroL2MessageSerializer { public static byte[] SerializeTransactions(IReadOnlyList transactions, L1IncomingMessageHeader header) { - if (transactions.Count == 0) + if (transactions.Count == 0 && header.Kind != ArbitrumL1MessageKind.EndOfBlock) throw new ArgumentException("Transactions must be non-empty", nameof(transactions)); using MemoryStream stream = new(); @@ -52,6 +52,9 @@ public static byte[] SerializeTransactions(IReadOnlyList transactio throw new InvalidOperationException($"{ArbitrumL1MessageKind.BatchPostingReport} is not supported as {nameof(ArbitrumInternalTransaction)} " + $"can't be used to build proper {nameof(DigestMessageParameters)}. It lacks original BatchHash and ExtraGas properties."); + case ArbitrumL1MessageKind.EndOfBlock: + break; + default: throw new ArgumentException($"Unsupported L1 message kind: {header.Kind}"); } From 0c94213046a6d64623865ad4fc49d8f95b138242 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 9 Feb 2026 20:06:51 +0900 Subject: [PATCH 30/87] tests: Fix existing firstUserTx related tests --- .../Execution/ArbitrumBlockProcessorTests.cs | 100 ++++++++++++------ 1 file changed, 70 insertions(+), 30 deletions(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/ArbitrumBlockProcessorTests.cs b/src/Nethermind.Arbitrum.Test/Execution/ArbitrumBlockProcessorTests.cs index b20bb7ff6..611d1eb70 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/ArbitrumBlockProcessorTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/ArbitrumBlockProcessorTests.cs @@ -5,8 +5,10 @@ using FluentAssertions; using Nethermind.Arbitrum.Arbos; using Nethermind.Arbitrum.Config; +using Nethermind.Arbitrum.Data; using Nethermind.Arbitrum.Execution; using Nethermind.Arbitrum.Execution.Receipts; +using Nethermind.Arbitrum.Execution.Transactions; using Nethermind.Arbitrum.Test.Infrastructure; using Nethermind.Blockchain.Tracing; using Nethermind.Consensus.Processing; @@ -25,9 +27,9 @@ namespace Nethermind.Arbitrum.Test.Execution; public class ArbitrumBlockProcessorTests { [Test] - public void FirstUserTransaction_WhenBlockGasLimitExceeded_IsAlwaysIncluded() + public void FirstUserTx_WhenBlockGasLimitIsGreaterOrEqualTo21000AndIsExceededByTx_UserTxGetsBypassAndAlwaysGetsIncluded() { - TestContext ctx = new(blockGasLimit: 25_000); + TestContext ctx = new(blockGasLimit: GasCostOf.Transaction + 1); Transaction tx = ctx.CreateTransaction(gasLimit: 50_000, nonce: 0); @@ -39,61 +41,79 @@ public void FirstUserTransaction_WhenBlockGasLimitExceeded_IsAlwaysIncluded() } [Test] - public void SecondUserTransaction_WhenBlockGasLimitExceeded_IsRejected() + public void UserTxFirstOrNot_WhenBlockGasLimitIsLowerThan21000_IsNotIncluded() { - TestContext ctx = new(blockGasLimit: 50_000); + TestContext ctx = new(blockGasLimit: GasCostOf.Transaction - 1); - Transaction tx1 = ctx.CreateTransaction(gasLimit: 25_000, nonce: 0, to: TestItem.AddressB); - Transaction tx2 = ctx.CreateTransaction(gasLimit: 40_000, nonce: 1, to: TestItem.AddressC); + Transaction tx = ctx.CreateTransaction(gasLimit: 25_000, nonce: 0); - BlockToProduce block = ctx.ExecuteBlock(tx1, tx2); + BlockToProduce block = ctx.ExecuteBlock(tx); - block.Transactions.Count().Should().Be(1, - "only first user transaction should be included when second would exceed block gas limit"); - block.Transactions.First().Nonce.Should().Be(0, - "the included transaction should be the first one"); + block.Transactions.Count().Should().Be(0, + "if block gas left is lower than 21000, no user transaction must be included"); } [Test] - public void FirstUserTransaction_WhenInternalTransactionProcessedFirst_StillGetsFirstUserTxBypass() + public void FirstUserTx_WhenInternalTxProcessedFirstAndBlockGasLeftIsGreaterThanOrEqualTo21000ButLowerThanTxGas_StillGetsFirstUserTxBypass() { - TestContext ctx = new(blockGasLimit: 30_000); + // StartBlock tx costs 21k gas, so, leave >= 21k gas in block for user tx to trigger the bypass afterwards + // even though the user tx gas limit is higher than the remaining block gas + TestContext ctx = new(blockGasLimit: GasCostOf.Transaction * 2); Transaction userTx = ctx.CreateTransaction(gasLimit: 35_000, nonce: 0); - BlockToProduce block = ctx.ExecuteBlock(userTx); + BlockToProduce block = ctx.ExecuteBlock(withInternalTx: true, userTx); - block.Transactions.Count().Should().Be(1, + block.Transactions.Count().Should().Be(2, "user transaction should be included even though internal block start transaction " + "was processed first - internal transactions must not count toward user transaction counter"); } [Test] - public void UserTransactions_WhenMultipleWithinGasLimit_AreAllIncluded() + public void FirstUserTx_WhenInternalTxProcessedFirstAndBlockGasLeftIsLowerThan21000_IsNotIncluded() { - TestContext ctx = new(blockGasLimit: 100_000); + // StartBlock tx costs 21k gas, so, leave < 21k gas in block for user tx to not get included + TestContext ctx = new(blockGasLimit: GasCostOf.Transaction * 2 - 1); - Transaction tx1 = ctx.CreateTransaction(gasLimit: 30_000, nonce: 0, to: TestItem.AddressB); - Transaction tx2 = ctx.CreateTransaction(gasLimit: 30_000, nonce: 1, to: TestItem.AddressC); + Transaction userTx = ctx.CreateTransaction(gasLimit: 35_000, nonce: 0); - BlockToProduce block = ctx.ExecuteBlock(tx1, tx2); + BlockToProduce block = ctx.ExecuteBlock(withInternalTx: true, userTx); - block.Transactions.Count().Should().BeGreaterThanOrEqualTo(2, - "both user transactions should be included when there is sufficient block gas"); + block.Transactions.Count().Should().Be(1, + "user transaction should not get included as block gas left is not even >= 21k, independently of user tx counter"); + block.Transactions.First().Type.Should().Be((TxType)ArbitrumTxType.ArbitrumInternal); } [Test] - public void FirstUserTransaction_WhenZeroBlockGasLimit_IsStillIncluded() + public void SecondUserTx_WhenBlockGasLeftIsGreaterOrEqualTo21000ButLowerThanTxGas_DoesNotGetBypassAsNotFirstUserTxAndGetsRejected() { - TestContext ctx = new(blockGasLimit: 0); + TestContext ctx = new(blockGasLimit: 50_000); - Transaction tx = ctx.CreateTransaction(gasLimit: 25_000, nonce: 0); + Transaction tx1 = ctx.CreateTransaction(gasLimit: 25_000, nonce: 0, to: TestItem.AddressB); + // When tx2 is executed, block gas left is 29k (50-21 and not -25 because first tx real compute cost is 21k) + // So, first check of block gas left >= 21k passes, but second check tx computeGas > blockGasLeft also passes (skipping tx inclusion) + Transaction tx2 = ctx.CreateTransaction(gasLimit: 40_000, nonce: 1, to: TestItem.AddressC); - BlockToProduce block = ctx.ExecuteBlock(tx); + BlockToProduce block = ctx.ExecuteBlock(tx1, tx2); block.Transactions.Count().Should().Be(1, - "even with zero block gas limit, first user transaction must be included " + - "to guarantee block liveness and prevent empty blocks"); + "only first user transaction should be included when second would exceed block gas limit"); + block.Transactions.First().Nonce.Should().Be(0, + "the included transaction should be the first one"); + } + + [Test] + public void UserTransactions_WhenMultipleWithinGasLimit_AreAllIncluded() + { + TestContext ctx = new(blockGasLimit: 100_000); + + Transaction tx1 = ctx.CreateTransaction(gasLimit: 30_000, nonce: 0, to: TestItem.AddressB); + Transaction tx2 = ctx.CreateTransaction(gasLimit: 30_000, nonce: 1, to: TestItem.AddressC); + + BlockToProduce block = ctx.ExecuteBlock(tx1, tx2); + + block.Transactions.Count().Should().BeGreaterThanOrEqualTo(2, + "both user transactions should be included when there is sufficient block gas"); } [Test] @@ -289,7 +309,10 @@ public Transaction CreateTransaction(long gasLimit, UInt256 nonce, Address? to = .TestObject; } - public BlockToProduce ExecuteBlock(params Transaction[] transactions) + public BlockToProduce ExecuteBlock(params Transaction[] transactions) => + ExecuteBlock(withInternalTx: false, transactions); + + public BlockToProduce ExecuteBlock(bool withInternalTx, params Transaction[] transactions) { Block block = Build.A.Block .WithNumber(_chain.BlockTree.Head!.Number + 1) @@ -300,7 +323,24 @@ public BlockToProduce ExecuteBlock(params Transaction[] transactions) .WithTransactions(transactions) .TestObject; - BlockToProduce blockToProduce = new(block.Header, block.Transactions, block.Uncles); + Transaction[] allTransactions = block.Transactions; + if (withInternalTx) + { + L1IncomingMessageHeader l1Header = new( + ArbitrumL1MessageKind.L2Message, + Address.Zero, + BlockNumber: 0, + Timestamp: block.Timestamp, + RequestId: null, + BaseFeeL1: 0); + + Transaction internalTx = ArbitrumBlockProducer.CreateInternalTransaction( + l1Header, block.Header, _chain.BlockTree.Head!.Header, _chain.SpecProvider); + + allTransactions = allTransactions.Prepend(internalTx).ToArray(); + } + + BlockToProduce blockToProduce = new(block.Header, allTransactions, block.Uncles); ArbitrumChainSpecEngineParameters chainSpecParams = _chain.ChainSpec .EngineChainSpecParametersProvider From 81834bfbea81b3b0361fd113cac726ab7a46c6b3 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 9 Feb 2026 20:07:07 +0900 Subject: [PATCH 31/87] Fix underflow bug --- src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs index a73449b5f..1f5f5636a 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumBlockProcessor.cs @@ -332,7 +332,7 @@ private ulong CalculateAndUpdateBlockGasLimit(long txGasUsed, long dataGas, ulon computeUsed = GasCostOf.Transaction; } - return System.Math.Max(0, blockGasLeft - (ulong)computeUsed); + return Utils.SaturateSub(blockGasLeft, (ulong)computeUsed); } private void UpdateArbitrumBlockHeader(BlockHeader header, IWorldState stateProvider) From 6ceb9a22219a8874221516ec98e4c313bdb17d1d Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Wed, 11 Feb 2026 16:13:27 +0900 Subject: [PATCH 32/87] Update to latest NMC wit gen branch --- src/Nethermind | 2 +- src/Nethermind.Arbitrum.Test/Nethermind.Arbitrum.Test.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind b/src/Nethermind index 7945a5355..f7305bdd2 160000 --- a/src/Nethermind +++ b/src/Nethermind @@ -1 +1 @@ -Subproject commit 7945a535536f086b4401a64074ec88821e9a1b58 +Subproject commit f7305bdd2e9f498bcf8af5d93b526424b00cbbc9 diff --git a/src/Nethermind.Arbitrum.Test/Nethermind.Arbitrum.Test.csproj b/src/Nethermind.Arbitrum.Test/Nethermind.Arbitrum.Test.csproj index 4ddcd1527..a4554d159 100644 --- a/src/Nethermind.Arbitrum.Test/Nethermind.Arbitrum.Test.csproj +++ b/src/Nethermind.Arbitrum.Test/Nethermind.Arbitrum.Test.csproj @@ -13,7 +13,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive From 2f88e336149c4042f62c9f7a1afd1a2edc39e2a3 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Wed, 11 Feb 2026 16:14:32 +0900 Subject: [PATCH 33/87] fix: Add serializedChainConfig and initialL1BaseFee to full chain sim chainspec --- .../Properties/chainspec/arbitrum-local.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum/Properties/chainspec/arbitrum-local.json b/src/Nethermind.Arbitrum/Properties/chainspec/arbitrum-local.json index 3c8e4a0ec..f2878fc81 100644 --- a/src/Nethermind.Arbitrum/Properties/chainspec/arbitrum-local.json +++ b/src/Nethermind.Arbitrum/Properties/chainspec/arbitrum-local.json @@ -6,9 +6,11 @@ "initialArbOSVersion": 32, "initialChainOwner": "0x5E1497dD1f08C87b2d8FE23e9AAB6c1De833D927", "genesisBlockNum": 0, + "initialL1BaseFee": "70", "enableArbOS": true, "allowDebugPrecompiles": true, - "dataAvailabilityCommittee": false + "dataAvailabilityCommittee": false, + "serializedChainConfig": "eyJjaGFpbklkIjo0MTIzNDYsImhvbWVzdGVhZEJsb2NrIjowLCJkYW9Gb3JrU3VwcG9ydCI6dHJ1ZSwiZWlwMTUwQmxvY2siOjAsImVpcDE1MEhhc2giOiIweDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLCJlaXAxNTVCbG9jayI6MCwiZWlwMTU4QmxvY2siOjAsImJ5emFudGl1bUJsb2NrIjowLCJjb25zdGFudGlub3BsZUJsb2NrIjowLCJwZXRlcnNidXJnQmxvY2siOjAsImlzdGFuYnVsQmxvY2siOjAsIm11aXJHbGFjaWVyQmxvY2siOjAsImJlcmxpbkJsb2NrIjowLCJsb25kb25CbG9jayI6MCwiY2xpcXVlIjp7InBlcmlvZCI6MCwiZXBvY2giOjB9LCJhcmJpdHJ1bSI6eyJFbmFibGVBcmJPUyI6dHJ1ZSwiQWxsb3dEZWJ1Z1ByZWNvbXBpbGVzIjp0cnVlLCJEYXRhQXZhaWxhYmlsaXR5Q29tbWl0dGVlIjpmYWxzZSwiSW5pdGlhbEFyYk9TVmVyc2lvbiI6MzIsIkluaXRpYWxDaGFpbk93bmVyIjoiMHg1RTE0OTdkRDFmMDhDODdiMmQ4RkUyM2U5QUFCNmMxRGU4MzNEOTI3IiwiR2VuZXNpc0Jsb2NrTnVtIjowfX0=" } }, "params": { From 91c146afd035fd7e88a686cd963efbcad2f2fe1f Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Wed, 11 Feb 2026 16:15:36 +0900 Subject: [PATCH 34/87] fix: Forgotten SpecHelper assignment in VM bug --- src/Nethermind.Arbitrum/Config/ArbitrumSpecHelper.cs | 2 +- src/Nethermind.Arbitrum/Evm/ArbitrumVirtualMachine.cs | 1 + .../Precompiles/ArbitrumPrecompileExecutionContext.cs | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum/Config/ArbitrumSpecHelper.cs b/src/Nethermind.Arbitrum/Config/ArbitrumSpecHelper.cs index 0d05a1d59..75fe7f15d 100644 --- a/src/Nethermind.Arbitrum/Config/ArbitrumSpecHelper.cs +++ b/src/Nethermind.Arbitrum/Config/ArbitrumSpecHelper.cs @@ -50,7 +50,7 @@ public class ArbitrumSpecHelper(ArbitrumChainSpecEngineParameters parameters) : public ulong GenesisBlockNum => parameters.GenesisBlockNum ?? 0; public UInt256 InitialL1BaseFee => parameters.InitialL1BaseFee ?? DefaultInitialL1BaseFee; public bool EnableArbOS => parameters.EnableArbOS ?? true; - public bool AllowDebugPrecompiles => parameters.AllowDebugPrecompiles ?? true; + public bool AllowDebugPrecompiles => parameters.AllowDebugPrecompiles ?? false; public bool DataAvailabilityCommittee => parameters.DataAvailabilityCommittee ?? false; public ulong? MaxCodeSize => parameters.MaxCodeSize; public ulong? MaxInitCodeSize => parameters.MaxInitCodeSize; diff --git a/src/Nethermind.Arbitrum/Evm/ArbitrumVirtualMachine.cs b/src/Nethermind.Arbitrum/Evm/ArbitrumVirtualMachine.cs index 8b3f31373..d23dc169a 100644 --- a/src/Nethermind.Arbitrum/Evm/ArbitrumVirtualMachine.cs +++ b/src/Nethermind.Arbitrum/Evm/ArbitrumVirtualMachine.cs @@ -521,6 +521,7 @@ private CallResult RunPrecompile(VmState state, PrecompileInf CurrentRefundTo = ArbitrumTxExecutionContext.CurrentRefundTo, PosterFee = ArbitrumTxExecutionContext.PosterFee, ExecutingAccount = state.Env.ExecutingAccount, + SpecHelper = specHelper }; return precompile.IsDebug diff --git a/src/Nethermind.Arbitrum/Precompiles/ArbitrumPrecompileExecutionContext.cs b/src/Nethermind.Arbitrum/Precompiles/ArbitrumPrecompileExecutionContext.cs index 71b0e0e8d..8496516cc 100644 --- a/src/Nethermind.Arbitrum/Precompiles/ArbitrumPrecompileExecutionContext.cs +++ b/src/Nethermind.Arbitrum/Precompiles/ArbitrumPrecompileExecutionContext.cs @@ -68,6 +68,7 @@ public record ArbitrumPrecompileExecutionContext( public bool IsMethodCalledPure { get; set; } public ulong Burned => GasSupplied - GasLeft; + public IArbitrumSpecHelper? SpecHelper { get; init; } private ulong _gasLeft = GasSupplied; From 443ebd4ce025645c6556d43a545f5fd3fddcce69 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Wed, 11 Feb 2026 16:34:05 +0900 Subject: [PATCH 35/87] feat: Update to latest base NMC wit gen branch (without account proof collection tree visitor) --- src/Nethermind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind b/src/Nethermind index f7305bdd2..becf56c7d 160000 --- a/src/Nethermind +++ b/src/Nethermind @@ -1 +1 @@ -Subproject commit f7305bdd2e9f498bcf8af5d93b526424b00cbbc9 +Subproject commit becf56c7d766820dfcd94cacea0ccf5b3baac0a7 From 4486385f3ff079d7766c62637c11a9067519a2a5 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Wed, 11 Feb 2026 16:40:04 +0900 Subject: [PATCH 36/87] tests: Fix AllowDebugPrecompiles to default to false --- .../Config/ArbitrumChainSpecEngineParametersTests.cs | 2 +- .../Rpc/DigestMessage/NitroL2MessageParserTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind.Arbitrum.Test/Config/ArbitrumChainSpecEngineParametersTests.cs b/src/Nethermind.Arbitrum.Test/Config/ArbitrumChainSpecEngineParametersTests.cs index 1fda91cc1..9e768319b 100644 --- a/src/Nethermind.Arbitrum.Test/Config/ArbitrumChainSpecEngineParametersTests.cs +++ b/src/Nethermind.Arbitrum.Test/Config/ArbitrumChainSpecEngineParametersTests.cs @@ -129,7 +129,7 @@ public void Create_WithNullParameters_UsesDefaultValues() InitialChainOwner = new Address("0x5E1497dD1f08C87b2d8FE23e9AAB6c1De833D927"), GenesisBlockNum = 0, EnableArbOS = true, - AllowDebugPrecompiles = true, + AllowDebugPrecompiles = false, DataAvailabilityCommittee = false, MaxCodeSize = null, MaxInitCodeSize = null diff --git a/src/Nethermind.Arbitrum.Test/Rpc/DigestMessage/NitroL2MessageParserTests.cs b/src/Nethermind.Arbitrum.Test/Rpc/DigestMessage/NitroL2MessageParserTests.cs index f2febb667..406a6757c 100644 --- a/src/Nethermind.Arbitrum.Test/Rpc/DigestMessage/NitroL2MessageParserTests.cs +++ b/src/Nethermind.Arbitrum.Test/Rpc/DigestMessage/NitroL2MessageParserTests.cs @@ -476,7 +476,7 @@ public void GetCanonicalArbitrumParameters_WhenL1ConfigIsUnavailable_ReturnsFall InitialArbOSVersion = 10, InitialChainOwner = Address.Zero, GenesisBlockNum = 100, - AllowDebugPrecompiles = true, + AllowDebugPrecompiles = false, DataAvailabilityCommittee = false, MaxCodeSize = null, MaxInitCodeSize = null, From b0fe887b0c160468867f184a02cef790f65762ee Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Wed, 11 Feb 2026 16:53:24 +0900 Subject: [PATCH 37/87] fix: Removed stateReader parameter to wit gen worldstate constructor --- .../ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index 1636b73a6..92062e6b2 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -107,7 +107,7 @@ public IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTarge .AddScoped(stateReader) .AddScoped(builder => new WitnessGeneratingHeaderFinder(builder.Resolve())) - .AddScoped(builder => new WitnessGeneratingWorldState(worldState, stateReader, trieStore, (builder.Resolve() as WitnessGeneratingHeaderFinder)!)) + .AddScoped(builder => new WitnessGeneratingWorldState(worldState, trieStore, (builder.Resolve() as WitnessGeneratingHeaderFinder)!)) .AddScoped(_ => CreateWitnessBlocksConfig(blocksConfig)) From 32f62aa58adb5a2e80ea01156db8dd42b6fb71fe Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Thu, 12 Feb 2026 17:17:20 +0900 Subject: [PATCH 38/87] feat: Update to latest base NMC wit gen branch --- src/Nethermind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind b/src/Nethermind index becf56c7d..0c66d8b91 160000 --- a/src/Nethermind +++ b/src/Nethermind @@ -1 +1 @@ -Subproject commit becf56c7d766820dfcd94cacea0ccf5b3baac0a7 +Subproject commit 0c66d8b918bec8015fd0c5d7fdc8b223d6991882 From e044271d80f5e6cfc826ec5d5f5f70a76da0fec3 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Thu, 12 Feb 2026 17:21:50 +0900 Subject: [PATCH 39/87] fix: Add stateReader parameter to wit gen worldstate constructor --- .../ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index 92062e6b2..1636b73a6 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -107,7 +107,7 @@ public IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTarge .AddScoped(stateReader) .AddScoped(builder => new WitnessGeneratingHeaderFinder(builder.Resolve())) - .AddScoped(builder => new WitnessGeneratingWorldState(worldState, trieStore, (builder.Resolve() as WitnessGeneratingHeaderFinder)!)) + .AddScoped(builder => new WitnessGeneratingWorldState(worldState, stateReader, trieStore, (builder.Resolve() as WitnessGeneratingHeaderFinder)!)) .AddScoped(_ => CreateWitnessBlocksConfig(blocksConfig)) From f7a25bf9cf904c8c4a08a3891b8e69130189ebd4 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 13 Feb 2026 18:26:34 +0900 Subject: [PATCH 40/87] feat: Update to latest base NMC wit gen branch --- src/Nethermind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind b/src/Nethermind index 0c66d8b91..90dee7207 160000 --- a/src/Nethermind +++ b/src/Nethermind @@ -1 +1 @@ -Subproject commit 0c66d8b918bec8015fd0c5d7fdc8b223d6991882 +Subproject commit 90dee72072d32e1195d4f1a80f6aa7d646779bc3 From 952671b128d64d67c85568b80ca2d2313fa45040 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 13 Feb 2026 18:27:27 +0900 Subject: [PATCH 41/87] tests: Reverted txs and set-then-reset storage slots still records trie nodes --- .../ArbitrumWitnessGenerationTests.cs | 443 +++++++++++++++++- 1 file changed, 440 insertions(+), 3 deletions(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index a179056d8..1742de769 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -4,7 +4,6 @@ using Nethermind.Arbitrum.Execution.Transactions; using Nethermind.Arbitrum.Precompiles; using Nethermind.Arbitrum.Test.Infrastructure; -using Nethermind.Blockchain; using Nethermind.Blockchain.Tracing; using Nethermind.Consensus.Processing; using Nethermind.Consensus.Validators; @@ -20,7 +19,7 @@ using Nethermind.Logging; using Nethermind.Arbitrum.Data; using Nethermind.Arbitrum.Execution.Stateless; -using Nethermind.Evm.Tracing; +using Nethermind.State.Proofs; namespace Nethermind.Arbitrum.Test.Execution; @@ -803,7 +802,7 @@ public async Task RecordBlockCreation_TryReapRetryableNotExpired_RecordsTimeoutW /// /// An EndOfBlock message produces a block containing only the StartBlock internal tx (no user /// transactions). This isolates the test: BrotliCompressionLevel can only be captured by the - /// internal tx's CanAddTransaction path, not by any user tx execution or gas charging. + /// internal tx's CanAddTransaction path, not by any user tx execution or gas charging hook. ///
[Test] public async Task RecordBlockCreation_NonUserTransaction_RecordsBrotliCompressionLevelInWitness() @@ -833,6 +832,422 @@ public async Task RecordBlockCreation_NonUserTransaction_RecordsBrotliCompressio "Witness state should contain trie node for BrotliCompressionLevel storage slot (offset 7)"); } + /// + /// Verifies that when a storage slot is modified by one transaction and reset to its original + /// value through SSTORE opcode within the same block, the witness still captures the storage + /// trie nodes. + /// + /// Even if the final net change is zero, the storage slot is anyway accessed (read) during SSTORE execution + /// and therefore trie nodes should be contained in the witness. + /// + [Test] + public async Task RecordBlockCreation_WhenStorageSlotModifiedAndResetInSameBlockThroughSStoreOpcode_StillRecordsStorageTrieNodes() + { + UInt256 l1BaseFee = 92; + + using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() + .WithGenesisBlock(initialBaseFee: (ulong)l1BaseFee) + .Build(); + + Address sender = FullChainSimulationAccounts.Owner.Address; + + // Fund the sender account + ResultWrapper depositResult = await chain.Digest(new TestEthDeposit( + Keccak.Compute("deposit"), l1BaseFee, sender, sender, 100.Ether())); + depositResult.Result.Should().Be(Result.Success); + + // Deploy a simple setter contract: SSTORE(slot=0, value=CALLDATALOAD(0)) + // Constructor also initializes slot 0 to value 1. + UInt256 storageSlot = 0; + UInt256 initialValue = 1; + byte[] setterRuntimeCode = Prepare.EvmCode + .PushData(0) // offset for CALLDATALOAD + .Op(Instruction.CALLDATALOAD) // load 32 bytes from calldata + .PushData(storageSlot) // storage slot + .Op(Instruction.SSTORE) // store + .Op(Instruction.STOP) + .Done; + + byte[] setterInitCode = Prepare.EvmCode + .PushData(initialValue) + .PushData(0) // storage slot 0 + .Op(Instruction.SSTORE) // initialize slot 0 with initial value + .ForInitOf(setterRuntimeCode) // 3 instructions above correspond to constructor + .Done; + + Transaction deployTx; + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + deployTx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithTo(null) // contract creation + .WithData(setterInitCode) + .WithMaxFeePerGas(10.GWei()) + .WithGasLimit(500_000) + .WithValue(0) + .WithNonce(chain.MainWorldState.GetNonce(sender)) + .SignedAndResolved(FullChainSimulationAccounts.Owner) + .TestObject; + } + + ResultWrapper deployResult = await chain.Digest(new TestL2Transactions(l1BaseFee, sender, deployTx)); + deployResult.Result.Should().Be(Result.Success); + chain.LatestReceipts()[1].StatusCode.Should().Be(StatusCode.Success, "contract deployment should succeed"); + + Address contractAddress = ContractAddress.From(sender, deployTx.Nonce); + + // Parent header is the state BEFORE the modify/reset block + BlockHeader parentHeader = chain.BlockTree.Head!.Header; + + // Create two transactions in the same block: + // TX1: set slot 0 to value 2 (modifies storage) + // TX2: set slot 0 back to value 1 (resets to original) + byte[] setTo2 = new UInt256(2).ToBigEndian(); + byte[] setToInitialValue = initialValue.ToBigEndian(); + + Transaction tx1; + Transaction tx2; + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + UInt256 nonce = chain.MainWorldState.GetNonce(sender); + + tx1 = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithTo(contractAddress) + .WithData(setTo2) + .WithMaxFeePerGas(10.GWei()) + .WithGasLimit(100_000) + .WithValue(0) + .WithNonce(nonce) + .SignedAndResolved(FullChainSimulationAccounts.Owner) + .TestObject; + + tx2 = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithTo(contractAddress) + .WithData(setToInitialValue) + .WithMaxFeePerGas(10.GWei()) + .WithGasLimit(100_000) + .WithValue(0) + .WithNonce(nonce + 1) + .SignedAndResolved(FullChainSimulationAccounts.Owner) + .TestObject; + } + + (ResultWrapper result, DigestMessageParameters digestParams) = + await chain.DigestAndGetParams(new TestL2Transactions(l1BaseFee, sender, tx1, tx2)); + result.Result.Should().Be(Result.Success); + chain.LatestReceipts()[1].StatusCode.Should().Be(StatusCode.Success, "TX1 (set to 2) should succeed"); + chain.LatestReceipts()[2].StatusCode.Should().Be(StatusCode.Success, "TX2 (reset to 1) should succeed"); + + // Record block creation and generate witness + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); + RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestParams.Index); + + // Assert the storage slot still has its original value + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + chain.MainWorldState.Get(new(contractAddress, storageSlot)).ToArray().Should().BeEquivalentTo( + initialValue.ToBigEndian().WithoutLeadingZeros().ToArray()); + } + + ArbitrumWitness witness = recordResult.Witness; + + // Collect the expected storage proof from the parent state. + AccountProofCollector collector = new(contractAddress, [storageSlot]); + chain.StateReader.RunTreeVisitor(collector, parentHeader); + AccountProof accountProof = collector.BuildResult(); + + byte[][] storageProofNodes = accountProof.StorageProofs! + .SelectMany(sp => sp.Proof!) + .ToArray(); + + storageProofNodes.Should().NotBeEmpty( + "the contract should have a non-empty storage proof for slot 0 in the parent state"); + + HashSet witnessNodeHashes = witness.Witness.State + .Select(Keccak.Compute) + .ToHashSet(); + + foreach (byte[] proofNode in storageProofNodes) + { + witnessNodeHashes.Should().Contain(Keccak.Compute(proofNode), + "witness should contain storage trie proof node even when the net storage change " + + "is zero (slot was modified by TX1 then reset to original value by TX2)"); + } + } + + /// + /// Similar to the above (storage slot set then reset) but instead of using SSTORE, we modify the + /// state directly through the WorldState, and therefore the storage slot written to has not been read before. + /// + /// Since Nethermind caches writes and commits storage changes per-block, the net change is zero and + /// but the trie nodes are still traversed during commit. This makes sense because even if the value has been reset, + /// we do not know its original value. + /// + [Test] + public async Task RecordBlockCreation_WhenStateModifiedAndResetDirectlyViaWorldStateNotSStoreOpcode_StillRecordsStorageTrieNodes() + { + UInt256 l1BaseFee = 92; + + using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() + .WithGenesisBlock(initialBaseFee: (ulong)l1BaseFee) + .Build(); + + Address sender = FullChainSimulationAccounts.Owner.Address; + + // Fund the sender account + TestEthDeposit deposit = new( + Keccak.Compute("deposit"), + l1BaseFee, + sender, + sender, + 100.Ether()); + ResultWrapper depositResult = await chain.Digest(deposit); + depositResult.Result.Should().Be(Result.Success); + + // Pre-populate NetworkFeeAccount (ArbOS root storage, offset 3) with a known original value. + // This creates a leaf trie node at that storage path so the assertion targets something unique. + Address originalFeeAccount = TestItem.AddressB; + chain.AppendBlock(chain => + { + ArbosState arbosState = ArbosState.OpenArbosState(chain.MainWorldState, new SystemBurner(), NullLogger.Instance); + arbosState.NetworkFeeAccount.Set(originalFeeAccount); + }); + + BlockHeader parentHeader = chain.BlockTree.Head!.Header; + + byte[] selector = Keccak.Compute("setNetworkFeeAccount(address)"u8).Bytes[..4].ToArray(); + + Address newFeeAccount = TestItem.AddressC; + + byte[] setToNewCalldata = new byte[36]; + selector.CopyTo(setToNewCalldata, 0); + newFeeAccount.Bytes.CopyTo(setToNewCalldata.AsSpan(16)); + + byte[] resetToOriginalCalldata = new byte[36]; + selector.CopyTo(resetToOriginalCalldata, 0); + originalFeeAccount.Bytes.CopyTo(resetToOriginalCalldata.AsSpan(16)); + + // TX1: Set NetworkFeeAccount to newFeeAccount + // TX2: Reset NetworkFeeAccount back to originalFeeAccount + Transaction tx1; + Transaction tx2; + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + UInt256 nonce = chain.MainWorldState.GetNonce(sender); + + tx1 = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithTo(ArbosAddresses.ArbOwnerAddress) + .WithData(setToNewCalldata) + .WithMaxFeePerGas(10.GWei()) + .WithGasLimit(500_000) + .WithValue(0) + .WithNonce(nonce) + .SignedAndResolved(FullChainSimulationAccounts.Owner) + .TestObject; + + tx2 = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithTo(ArbosAddresses.ArbOwnerAddress) + .WithData(resetToOriginalCalldata) + .WithMaxFeePerGas(10.GWei()) + .WithGasLimit(500_000) + .WithValue(0) + .WithNonce(nonce + 1) + .SignedAndResolved(FullChainSimulationAccounts.Owner) + .TestObject; + } + + (ResultWrapper result, DigestMessageParameters digestParams) = + await chain.DigestAndGetParams(new TestL2Transactions(l1BaseFee, sender, tx1, tx2)); + result.Result.Should().Be(Result.Success); + + TxReceipt[] receipts = chain.LatestReceipts(); + receipts[1].StatusCode.Should().Be(StatusCode.Success, "TX1 (set to new) should succeed"); + receipts[2].StatusCode.Should().Be(StatusCode.Success, "TX2 (reset to original) should succeed"); + + // Assert NetworkFeeAccount still has its original value + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + ArbosState arbosState = ArbosState.OpenArbosState(chain.MainWorldState, new SystemBurner(), NullLogger.Instance); + arbosState.NetworkFeeAccount.Get().Should().Be(originalFeeAccount, + "NetworkFeeAccount should retain its original value after modify + reset in the same block"); + } + + // Record block creation and generate witness + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); + RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestParams.Index); + + // Assert the network fee account is indeed the original one (changed then reset) + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + ArbosState arbosState = ArbosState.OpenArbosState(chain.MainWorldState, new SystemBurner(), NullLogger.Instance); + arbosState.NetworkFeeAccount.Get().Should().Be(originalFeeAccount); + } + + ArbitrumWitness witness = recordResult.Witness; + + // NetworkFeeAccount is at root BackingStorage (empty storageKey), offset 3 + UInt256 networkFeeAccountSlot = ComputeMappedStorageSlot([], ArbosStateOffsets.NetworkFeeAccountOffset); + + // Collect expected storage proof from the parent state + AccountProofCollector collector = new(ArbosAddresses.ArbosSystemAccount, [networkFeeAccountSlot]); + chain.StateReader.RunTreeVisitor(collector, parentHeader); + AccountProof accountProof = collector.BuildResult(); + + byte[][] storageProofNodes = accountProof.StorageProofs! + .SelectMany(sp => sp.Proof!) + .ToArray(); + + storageProofNodes.Should().NotBeEmpty( + "NetworkFeeAccount slot should have a non-empty storage proof in the parent state"); + + HashSet witnessNodeHashes = witness.Witness.State + .Select(Keccak.Compute) + .ToHashSet(); + + foreach (byte[] proofNode in storageProofNodes) + { + witnessNodeHashes.Should().Contain(Keccak.Compute(proofNode), + "witness should contain storage trie proof node for NetworkFeeAccount " + + "even when the net storage change is zero (modified by TX1 then reset by TX2)"); + } + } + + /// + /// Verifies that when a transaction reverts, the witness still captures the storage trie nodes + /// for the storage slots written during execution. + /// In Nethermind, storage writes are cached and only applied to the trie during the commit phase. + /// A revert discards the cached writes, so the trie is never traversed for those paths. The + /// AccountProofCollector pass in GetWitness compensates by explicitly collecting proofs for all + /// tracked storage slots, matching Nitro's behavior where trie nodes are captured regardless of reverts. + /// + [Test] + public async Task RecordBlockCreation_TransactionSetsSomeStateButReverts_StillRecordsStorageTrieNodes() + { + UInt256 l1BaseFee = 92; + + using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() + .WithGenesisBlock(initialBaseFee: (ulong)l1BaseFee) + .Build(); + + Address sender = FullChainSimulationAccounts.Owner.Address; + + // Fund the sender account + TestEthDeposit deposit = new( + Keccak.Compute("deposit"), + l1BaseFee, + sender, + sender, + 100.Ether()); + ResultWrapper depositResult = await chain.Digest(deposit); + depositResult.Result.Should().Be(Result.Success); + + // Pre-populate AddressTable._backingStorage at offset 1 with a non-zero value to create + // a unique leaf node. When Register is called for the first time on a new address, it + // increments numItems from 0 to 1 and writes to _backingStorage at offset 1. Pre-populating + // ensures a leaf trie node exists at that path, so the assertion targets a node that only + // appears when this specific storage slot is accessed — not captured by other ArbOS operations. + // Just a hack to make test deterministic and reliable. + chain.AppendBlock(chain => + { + ArbosStorage backingStorage = new ArbosStorage(chain.MainWorldState, new SystemBurner(), ArbosAddresses.ArbosSystemAccount) + .OpenSubStorage(ArbosSubspaceIDs.AddressTableSubspace); + backingStorage.Set(1, Hash256.FromBytesWithPadding([0x5])); + }); + + BlockHeader parentHeader = chain.BlockTree.Head!.Header; + + // Build calldata for register(address) + Address addressToRegister = TestItem.AddressA; + byte[] registerSelector = Keccak.Compute("register(address)"u8).Bytes[..4].ToArray(); + byte[] calldata = new byte[36]; + registerSelector.CopyTo(calldata, 0); + addressToRegister.Bytes.CopyTo(calldata.AsSpan(16)); + + // Gas limit must be high enough for ArbAddressTable.Register to execute _backingStorage.Set(1, ...) — the + // pure write whose trie traversal we want to verify — but low enough that the transaction + // ultimately reverts + Transaction registerTx; + + // intrinsic cost for transaction with 36 bytes of data + long intrinsicGasCost = 21_432; + // gas cost for precompile input data (32 calldata bytes excluding 4-bytes selector) + opening arbos as non-pure method + long precompileInputAndOpeningArbosGasCost = 3 + (long)ArbosStorage.StorageReadCost; + // gas cost for precompile execution (2 reads, 3 writes) + ulong precompileExecGasCost = 2 * ArbosStorage.StorageReadCost + 3 * ArbosStorage.StorageWriteCost; + long precompileOutputGasCost = 3; + // gasLimit does not contain enough gas for paying for output data causing revert + long gasLimit = intrinsicGasCost + precompileInputAndOpeningArbosGasCost + (long)precompileExecGasCost + precompileOutputGasCost - 1; + + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + registerTx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithTo(ArbosAddresses.ArbAddressTableAddress) + .WithData(calldata) + .WithMaxFeePerGas(10.GWei()) + .WithGasLimit(gasLimit) // Not enough gas, causing revert + .WithValue(0) + .WithNonce(chain.MainWorldState.GetNonce(sender)) + .SignedAndResolved(FullChainSimulationAccounts.Owner) + .TestObject; + } + + (ResultWrapper result, DigestMessageParameters digestParams) = + await chain.DigestAndGetParams(new TestL2Transactions(l1BaseFee, sender, registerTx)); + result.Result.Should().Be(Result.Success); + chain.LatestReceipts()[1].StatusCode.Should().Be(StatusCode.Failure, + "Register should revert due to insufficient gas"); + + // Assert the address was not registered (state changes were reverted) + using (chain.MainWorldState.BeginScope(chain.BlockTree.Head?.Header)) + { + ArbosState arbosState = ArbosState.OpenArbosState(chain.MainWorldState, new SystemBurner(), NullLogger.Instance); + arbosState.AddressTable.AddressExists(addressToRegister).Should().BeFalse( + "address should not be registered since the transaction reverted"); + } + + // Record the block and generate the witness + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); + RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestParams.Index); + + ArbitrumWitness witness = recordResult.Witness; + + // Compute the mapped Ethereum storage slot for AddressTable._backingStorage at offset 1. + // This replicates ArbosStorage.MapAddress to determine the actual storage trie key. + byte[] addressTableStorageKey = Keccak.Compute(ArbosSubspaceIDs.AddressTableSubspace).BytesToArray(); + UInt256 backingStorageSlot = ComputeMappedStorageSlot(addressTableStorageKey, 1); + + // Collect expected storage proof from the parent state + AccountProofCollector collector = new(ArbosAddresses.ArbosSystemAccount, [backingStorageSlot]); + chain.StateReader.RunTreeVisitor(collector, parentHeader); + AccountProof accountProof = collector.BuildResult(); + + byte[][] storageProofNodes = accountProof.StorageProofs! + .SelectMany(sp => sp.Proof!) + .ToArray(); + + storageProofNodes.Should().NotBeEmpty( + "pre-populated slot should have a non-empty storage proof in the parent state"); + + HashSet witnessNodeHashes = witness.Witness.State + .Select(Keccak.Compute) + .ToHashSet(); + + foreach (byte[] proofNode in storageProofNodes) + { + witnessNodeHashes.Should().Contain(Keccak.Compute(proofNode), + "witness should contain storage trie proof node for AddressTable._backingStorage " + + "even when the transaction reverted"); + } + } + private static IEnumerable ExecutionWitnessWithoutStylusSource() { // 18 blocks in the test where this test case source is used @@ -869,6 +1284,28 @@ private static IEnumerable ExecutionWitnessWithStylusSource() } } + /// + /// Replicates ArbosStorage.MapAddress to compute the Ethereum storage slot + /// from a subspace storage key and a logical offset. + /// + private static UInt256 ComputeMappedStorageSlot(byte[] storageKey, ulong offset) + { + byte[] keyBytes = new byte[32]; + new UInt256(offset).ToBigEndian(keyBytes); + + const int boundary = 31; + byte[] keccakInput = new byte[storageKey.Length + boundary]; + storageKey.CopyTo(keccakInput, 0); + Array.Copy(keyBytes, 0, keccakInput, storageKey.Length, boundary); + + byte[] hash = Keccak.Compute(keccakInput).BytesToArray(); + byte[] mappedKey = new byte[32]; + Array.Copy(hash, 0, mappedKey, 0, boundary); + mappedKey[boundary] = keyBytes[boundary]; + + return new UInt256(mappedKey, isBigEndian: true); + } + private static T ThrowOnFailure(ResultWrapper result, ulong msgIndex) { if (result.Result != Result.Success) From e394cd828cd183b3d53137572b3b699f55b9722d Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 17 Feb 2026 16:07:07 +0900 Subject: [PATCH 42/87] feat: Update to latest base NMC wit gen branch --- src/Nethermind | 2 +- src/Nethermind.Arbitrum/Data/RecordResult.cs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Nethermind b/src/Nethermind index 6941269f1..fbad7a2f2 160000 --- a/src/Nethermind +++ b/src/Nethermind @@ -1 +1 @@ -Subproject commit 6941269f17c6313f1e64699551770e3e14ae019e +Subproject commit fbad7a2f2f8eef577e32d37c79b4e515801b9755 diff --git a/src/Nethermind.Arbitrum/Data/RecordResult.cs b/src/Nethermind.Arbitrum/Data/RecordResult.cs index 7e50acf4f..0dc979e0f 100644 --- a/src/Nethermind.Arbitrum/Data/RecordResult.cs +++ b/src/Nethermind.Arbitrum/Data/RecordResult.cs @@ -26,6 +26,7 @@ public RecordResult(ulong messageIndex, Hash256 blockHash, ArbitrumWitness arbWi kvp => kvp.Key.ToHash256(), kvp => kvp.Value); + // Witness codes, states and headers should all be unique, so, using Add() is safe here Preimages = new(); foreach (byte[] code in arbWitness.Witness.Codes) Preimages.Add(Keccak.Compute(code), code); From 0d11609e3fa4933f0c3de88daf31f42f56170885 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 17 Feb 2026 17:50:44 +0900 Subject: [PATCH 43/87] chore: Add explanatory comment --- .../Execution/Stateless/ArbitrumWitnessCollector.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs index 8cb3448a9..c7951859f 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs @@ -33,7 +33,8 @@ public class ArbitrumWitnessCollector( UInt256 chainId = arbosState.ChainId.Get(); ulong genesisBlockNum = arbosState.GenesisBlockNum.Get(); - byte[] chainConfig = arbosState.ChainConfigStorage.Get(); + // Chain config not used but still necessary to read to ensure they are included in the witness + byte[] _ = arbosState.ChainConfigStorage.Get(); if (chainId != specProvider.ChainId) throw new InvalidOperationException($"ArbOS chainId mismatch. ArbOS={chainId}, local={specProvider.ChainId}."); From 89dbea9409f2d03377fb52984232b92ba20b6df5 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 17 Feb 2026 17:51:46 +0900 Subject: [PATCH 44/87] Revert "Temporary files for docker builds" This reverts commit 6f5580d1a7068371ae2494968a327ff1904d5ffd. --- .../Properties/configs/arbitrum-mainnet-archive.json | 4 +--- .../Properties/configs/arbitrum-mainnet.json | 4 +--- .../Properties/configs/arbitrum-sepolia-archive.json | 4 +--- .../Properties/configs/arbitrum-sepolia.json | 4 +--- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet-archive.json b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet-archive.json index 03134290c..ad06b6189 100644 --- a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet-archive.json +++ b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet-archive.json @@ -15,7 +15,6 @@ "Enabled": true, "Port": 20545, "EnginePort": 20551, - "UnsecureDevNoRpcAuthentication": true, "EngineEnabledModules": ["Net", "Eth", "Subscribe", "Web3", "Arbitrum", "nitroexecution"], "AdditionalRpcUrls": [ "http://localhost:28551|http;ws|net;eth;subscribe;web3;client;debug" @@ -39,8 +38,7 @@ "Trace", "TxPool", "Vault", - "Web3", - "Arbitrum" + "Web3" ] }, "Pruning": { diff --git a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet.json b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet.json index 4df8ce569..1070cf345 100644 --- a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet.json +++ b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet.json @@ -23,7 +23,6 @@ "Enabled": true, "Port": 20545, "EnginePort": 20551, - "UnsecureDevNoRpcAuthentication": true, "EngineEnabledModules": ["Net", "Eth", "Subscribe", "Web3", "Arbitrum", "nitroexecution"], "AdditionalRpcUrls": [ "http://localhost:28551|http;ws|net;eth;subscribe;web3;client;debug" @@ -47,8 +46,7 @@ "Trace", "TxPool", "Vault", - "Web3", - "Arbitrum" + "Web3" ] }, "Pruning": { diff --git a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia-archive.json b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia-archive.json index 5b93caaec..8533bc57c 100644 --- a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia-archive.json +++ b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia-archive.json @@ -15,7 +15,6 @@ "Enabled": true, "Port": 20545, "EnginePort": 20551, - "UnsecureDevNoRpcAuthentication": true, "EngineEnabledModules": ["Net", "Eth", "Subscribe", "Web3", "Arbitrum", "nitroexecution"], "AdditionalRpcUrls": [ "http://localhost:28551|http;ws|net;eth;subscribe;web3;client;debug" @@ -39,8 +38,7 @@ "Trace", "TxPool", "Vault", - "Web3", - "Arbitrum" + "Web3" ] }, "Pruning": { diff --git a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia.json b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia.json index f8574150b..923993ba6 100644 --- a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia.json +++ b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia.json @@ -23,7 +23,6 @@ "Enabled": true, "Port": 20545, "EnginePort": 20551, - "UnsecureDevNoRpcAuthentication": true, "EngineEnabledModules": ["Net", "Eth", "Subscribe", "Web3", "Arbitrum", "nitroexecution"], "AdditionalRpcUrls": [ "http://localhost:28551|http;ws|net;eth;subscribe;web3;client;debug" @@ -47,8 +46,7 @@ "Trace", "TxPool", "Vault", - "Web3", - "Arbitrum" + "Web3" ] }, "Pruning": { From c3942329822f58612aaaa9431d11b55e1aed90ce Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 23 Feb 2026 17:33:31 +0900 Subject: [PATCH 45/87] feat: Update to latest base wit gen branch and fix build errors --- src/Nethermind | 2 +- .../Execution/Stateless/ArbitrumWitnessGenerationTests.cs | 4 ++-- .../Infrastructure/ArbitrumRpcTestBlockchain.cs | 2 ++ .../Stateless/ArbitrumStatelessBlockProcessingEnv.cs | 6 ++---- src/Nethermind.Arbitrum/Modules/ArbitrumEthModuleFactory.cs | 3 +++ src/Nethermind.Arbitrum/Modules/ArbitrumEthRpcModule.cs | 4 +++- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/Nethermind b/src/Nethermind index fbad7a2f2..0b0ba7658 160000 --- a/src/Nethermind +++ b/src/Nethermind @@ -1 +1 @@ -Subproject commit fbad7a2f2f8eef577e32d37c79b4e515801b9755 +Subproject commit 0b0ba765869fa9dee6a1965228e5542ffc0e5e4a diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index 1742de769..589a865bb 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -282,7 +282,7 @@ public async Task RecordBlockCreation_ExtCodeSizeFollowedByIsZero_StillRecordsTa // Step 6: Verify the witness contains the target contract's code ArbitrumWitness witness = recordResult.Witness; - byte[][] witnessCodes = witness.Witness.Codes; + byte[][] witnessCodes = witness.Witness.Codes.ToArray(); witnessCodes.Length.Should().Be(2, "Witness should contain both caller and target contract codes"); @@ -385,7 +385,7 @@ public async Task RecordBlockCreation_PrecompileCalls_RecordsArbitrumPrecompileC // Verify the witness codes ArbitrumWitness witness = recordResult.Witness; - byte[][] witnessCodes = witness.Witness.Codes; + byte[][] witnessCodes = witness.Witness.Codes.ToArray(); // Arbitrum precompile bytecode is 0xfe (INVALID opcode) byte[] arbitrumPrecompileCode = Arbitrum.Arbos.Precompiles.InvalidCode; diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs index 3faca8fd2..9c2d8c151 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs @@ -32,6 +32,7 @@ using Nethermind.TxPool; using Nethermind.Wallet; using Nethermind.Arbitrum.Execution.Stateless; +using Nethermind.Db.LogIndex; namespace Nethermind.Arbitrum.Test.Infrastructure; @@ -377,6 +378,7 @@ private static ArbitrumRpcTestBlockchain CreateInternal(ArbitrumRpcTestBlockchai chain.Container.Resolve(), chain.Container.Resolve(), chain.Container.Resolve(), + chain.Container.Resolve(), chain.Container.Resolve().SecondsPerSlot, chain.Container.Resolve() ); diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs index d39ac62a4..50eef2c2e 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs @@ -47,9 +47,7 @@ public IBlockProcessor BlockProcessor private IWorldState? _worldState; public IWorldState WorldState { - get => _worldState ??= new WorldState( - new TrieStoreScopeProvider(new RawTrieStore(arbWitness.Witness.NodeStorage), - arbWitness.Witness.CodeDb, logManager), logManager); + get => _worldState ??= new WorldState(new TrieStoreScopeProvider(new RawTrieStore(arbWitness.Witness.CreateNodeStorage()), arbWitness.Witness.CreateCodeDb(), logManager), logManager); } private IWasmStore? _wasmStore; @@ -81,7 +79,7 @@ private IWasmStore CreateWasmStore() private IBlockProcessor GetBlockProcessor() { - StatelessBlockTree statelessBlockTree = new(arbWitness.Witness.DecodedHeaders); + StatelessBlockTree statelessBlockTree = new(arbWitness.Witness.DecodeHeaders()); ITransactionProcessor txProcessor = CreateTransactionProcessor(WorldState, statelessBlockTree); IBlockProcessor.IBlockTransactionsExecutor txExecutor = new BlockProcessor.BlockValidationTransactionsExecutor( diff --git a/src/Nethermind.Arbitrum/Modules/ArbitrumEthModuleFactory.cs b/src/Nethermind.Arbitrum/Modules/ArbitrumEthModuleFactory.cs index 010c2a21b..66c513a2a 100644 --- a/src/Nethermind.Arbitrum/Modules/ArbitrumEthModuleFactory.cs +++ b/src/Nethermind.Arbitrum/Modules/ArbitrumEthModuleFactory.cs @@ -13,6 +13,7 @@ using Nethermind.JsonRpc.Modules.Eth; using Nethermind.JsonRpc.Modules.Eth.FeeHistory; using Nethermind.JsonRpc.Modules.Eth.GasPrice; +using Nethermind.Db.LogIndex; using Nethermind.Logging; using Nethermind.Network; using Nethermind.State; @@ -37,6 +38,7 @@ public class ArbitrumEthModuleFactory( IFeeHistoryOracle feeHistoryOracle, IProtocolsManager protocolsManager, IForkInfo forkInfo, + ILogIndexConfig? logIndexConfig, IBlocksConfig blocksConfig, ArbitrumChainSpecEngineParameters chainSpecParams) : ModuleFactoryBase { @@ -58,6 +60,7 @@ public override IEthRpcModule Create() feeHistoryOracle, protocolsManager, forkInfo, + logIndexConfig, blocksConfig.SecondsPerSlot, chainSpecParams); } diff --git a/src/Nethermind.Arbitrum/Modules/ArbitrumEthRpcModule.cs b/src/Nethermind.Arbitrum/Modules/ArbitrumEthRpcModule.cs index 213e244fd..5779bfc2d 100644 --- a/src/Nethermind.Arbitrum/Modules/ArbitrumEthRpcModule.cs +++ b/src/Nethermind.Arbitrum/Modules/ArbitrumEthRpcModule.cs @@ -21,6 +21,7 @@ using Nethermind.JsonRpc.Modules.Eth; using Nethermind.JsonRpc.Modules.Eth.FeeHistory; using Nethermind.JsonRpc.Modules.Eth.GasPrice; +using Nethermind.Db.LogIndex; using Nethermind.Logging; using Nethermind.Network; using Nethermind.Specs.Forks; @@ -51,9 +52,10 @@ public ArbitrumEthRpcModule( IFeeHistoryOracle feeHistoryOracle, IProtocolsManager protocolsManager, IForkInfo forkInfo, + ILogIndexConfig? logIndexConfig, ulong? secondsPerSlot, ArbitrumChainSpecEngineParameters chainSpecParams) - : base(rpcConfig, blockchainBridge, blockFinder, receiptFinder, stateReader, txPool, txSender, wallet, logManager, specProvider, gasPriceOracle, ethSyncingInfo, feeHistoryOracle, protocolsManager, forkInfo, secondsPerSlot) + : base(rpcConfig, blockchainBridge, blockFinder, receiptFinder, stateReader, txPool, txSender, wallet, logManager, specProvider, gasPriceOracle, ethSyncingInfo, feeHistoryOracle, protocolsManager, forkInfo, logIndexConfig, secondsPerSlot) { _chainSpecParams = chainSpecParams; } From 3ce26287a4cde10448ddca6eb18292bcc38add13 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 23 Feb 2026 22:30:23 +0900 Subject: [PATCH 46/87] fix: Dispose of pool array list --- .../ArbitrumWitnessGenerationTests.cs | 98 +++++++------------ .../ArbitrumRpcTestBlockchain.cs | 28 ++++++ src/Nethermind.Arbitrum/Data/RecordResult.cs | 5 - .../Execution/ArbitrumExecutionEngine.cs | 45 +++++---- .../Execution/Stateless/ArbitrumWitness.cs | 8 +- 5 files changed, 91 insertions(+), 93 deletions(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index 589a865bb..54b6f39ea 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -38,7 +38,9 @@ public async Task RecordBlockCreation_WitnessWithoutUserWasms_StatelessExecution ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation(new RecordBlockCreationParameters(digestMessage.Index, digestMessage.Message, WasmTargets: [])); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestMessage.Index); - ArbitrumWitness witness = recordResult.Witness; + // Just to get witness because RecordResult just contains the whole mixed preimages dictionary + using ArbitrumWitness witness = await chain.BuildBlockWitness(new RecordBlockCreationParameters(digestMessage.Index, digestMessage.Message, WasmTargets: [])); + AssertWitnessMatchesRecordResult(witness, recordResult); ISpecProvider specProvider = FullChainSimulationChainSpecProvider.CreateDynamicSpecProvider(); ArbitrumStatelessBlockProcessingEnv blockProcessingEnv = @@ -75,7 +77,8 @@ public async Task RecordBlockCreation_WitnessWithUserWasms_StatelessExecutionIsS ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation(new RecordBlockCreationParameters(digestMessage.Index, digestMessage.Message, WasmTargets: wasmTargets)); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestMessage.Index); - ArbitrumWitness witness = recordResult.Witness; + using ArbitrumWitness witness = await chain.BuildBlockWitness(new RecordBlockCreationParameters(digestMessage.Index, digestMessage.Message, WasmTargets: wasmTargets)); + AssertWitnessMatchesRecordResult(witness, recordResult); ISpecProvider specProvider = FullChainSimulationChainSpecProvider.CreateDynamicSpecProvider(); ArbitrumStatelessBlockProcessingEnv blockProcessingEnv = @@ -112,8 +115,6 @@ public async Task RecordBlockCreation_WitnessWithUserWasms_CaptureAsms(ulong mes ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation(new RecordBlockCreationParameters(digestMessage.Index, digestMessage.Message, WasmTargets: wasmTargets)); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestMessage.Index); - ArbitrumWitness witness = recordResult.Witness; - // Build expected dictionary from chain state using the stylus contract addresses Dictionary> expected = new(); using (chain.MainWorldState.BeginScope(chain.BlockTree.Head!.Header)) @@ -137,14 +138,7 @@ public async Task RecordBlockCreation_WitnessWithUserWasms_CaptureAsms(ulong mes } } - // Build actual dictionary from witness by hashing ASM byte arrays - Dictionary> actual = witness.UserWasms? - .ToDictionary( - kvp => kvp.Key.ToHash256(), - kvp => (IReadOnlyDictionary)kvp.Value.ToDictionary( - asm => asm.Key, - asm => asm.Value)) - ?? []; + Dictionary> actual = recordResult.UserWasms ?? []; actual.Should().BeEquivalentTo(expected); } @@ -280,8 +274,11 @@ public async Task RecordBlockCreation_ExtCodeSizeFollowedByIsZero_StillRecordsTa new RecordBlockCreationParameters(callParams.Index, callParams.Message, WasmTargets: [])); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, callParams.Index); + using ArbitrumWitness witness = await chain.BuildBlockWitness( + new RecordBlockCreationParameters(callParams.Index, callParams.Message, WasmTargets: [])); + AssertWitnessMatchesRecordResult(witness, recordResult); + // Step 6: Verify the witness contains the target contract's code - ArbitrumWitness witness = recordResult.Witness; byte[][] witnessCodes = witness.Witness.Codes.ToArray(); witnessCodes.Length.Should().Be(2, "Witness should contain both caller and target contract codes"); @@ -289,17 +286,13 @@ public async Task RecordBlockCreation_ExtCodeSizeFollowedByIsZero_StillRecordsTa // The target contract's code should be in the witness // (EXTCODESIZE should have triggered GetCode on the target) Hash256 targetCodeHash = Keccak.Compute(targetRuntimeCode); - bool targetCodeInWitness = witnessCodes.Any(code => Keccak.Compute(code) == targetCodeHash); - - targetCodeInWitness.Should().BeTrue( + witnessCodes.Any(code => Keccak.Compute(code) == targetCodeHash).Should().BeTrue( "Target contract's code should be recorded in witness when EXTCODESIZE is called, " + "even if followed by ISZERO (peephole optimization pattern)"); // Also verify the caller contract's code is in the witness (since we executed it) Hash256 callerCodeHash = Keccak.Compute(callerRuntimeCode); - bool callerCodeInWitness = witnessCodes.Any(code => Keccak.Compute(code) == callerCodeHash); - - callerCodeInWitness.Should().BeTrue( + witnessCodes.Any(code => Keccak.Compute(code) == callerCodeHash).Should().BeTrue( "Caller contract's code should be recorded in witness since we executed it"); } @@ -383,21 +376,22 @@ public async Task RecordBlockCreation_PrecompileCalls_RecordsArbitrumPrecompileC new RecordBlockCreationParameters(callParams.Index, callParams.Message, WasmTargets: [])); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, callParams.Index); - // Verify the witness codes - ArbitrumWitness witness = recordResult.Witness; + using ArbitrumWitness witness = await chain.BuildBlockWitness( + new RecordBlockCreationParameters(callParams.Index, callParams.Message, WasmTargets: [])); + AssertWitnessMatchesRecordResult(witness, recordResult); + byte[][] witnessCodes = witness.Witness.Codes.ToArray(); // Arbitrum precompile bytecode is 0xfe (INVALID opcode) byte[] arbitrumPrecompileCode = Arbitrum.Arbos.Precompiles.InvalidCode; // The witness should contain the Arbitrum precompile's code (0xfe) - bool arbitrumPrecompileCodeInWitness = witnessCodes.Any(code => code.SequenceEqual(arbitrumPrecompileCode)); - arbitrumPrecompileCodeInWitness.Should().BeTrue( + witnessCodes.Any(code => code.SequenceEqual(arbitrumPrecompileCode)).Should().BeTrue( "Arbitrum precompile bytecode (0xfe) should be recorded in witness when calling ArbSys"); // Verify that no empty code is recorded (Ethereum precompiles have no stored bytecode) - bool emptyCodeInWitness = witnessCodes.Any(code => code.Length == 0); - emptyCodeInWitness.Should().BeFalse("Ethereum precompiles empty bytecode should not be recorded in witness"); + witnessCodes.Any(code => code.Length == 0).Should().BeFalse( + "Ethereum precompiles empty bytecode should not be recorded in witness"); // The witness should have exactly 1 code: Arbitrum precompile (0xfe) // No code from Ethereum precompile since it has empty bytecode @@ -503,14 +497,12 @@ public async Task RecordBlockCreation_BlockHashOpcode_RecordsStorageTrieNodeInWi new RecordBlockCreationParameters(call2Params.Index, call2Params.Message, WasmTargets: [])); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, call2Params.Index); - ArbitrumWitness witness = recordResult.Witness; - // The storage slot accessed is: 1 + l1BlockNumber % 256 in the Blockhashes substorage (see GetL1BlockHash) // Too difficult to predict exact trie node hash here, so, using the hardcoded value (found during debugging) - witness.Witness.State.Any(node => Keccak.Compute(node) == new Hash256("0x30cfd2590e997a3c3bee0c89572aec183bae0976e06334354832b85514d0d37a")).Should().BeTrue( + recordResult.Preimages.ContainsKey(new Hash256("0x30cfd2590e997a3c3bee0c89572aec183bae0976e06334354832b85514d0d37a")).Should().BeTrue( "Witness state should contain leaf trie node for BLOCKHASH storage access"); // Similarly, checking for an intermediate node capture when accessing the storage slot - witness.Witness.State.Any(node => Keccak.Compute(node) == new Hash256("0xad9a2d73baabd92487dd1840cd076a06a3eded05e8cbdebb930ddad669e51880")).Should().BeTrue( + recordResult.Preimages.ContainsKey(new Hash256("0xad9a2d73baabd92487dd1840cd076a06a3eded05e8cbdebb930ddad669e51880")).Should().BeTrue( "Witness state should contain intermediate trie node for BLOCKHASH storage access"); } @@ -561,7 +553,9 @@ public async Task RecordBlockCreation_ArbBlockHash_RecordsHeadersInWitness() new RecordBlockCreationParameters(callParams.Index, callParams.Message, WasmTargets: [])); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, callParams.Index); - ArbitrumWitness witness = recordResult.Witness; + using ArbitrumWitness witness = await chain.BuildBlockWitness( + new RecordBlockCreationParameters(callParams.Index, callParams.Message, WasmTargets: [])); + AssertWitnessMatchesRecordResult(witness, recordResult); // The witness should contain all RLP-encoded headers from targetBlockNumber to parentBlockNumber (inclusive) long parentBlockNumber = chain.BlockTree.Head!.Number - 1; @@ -649,7 +643,7 @@ public async Task RecordBlockCreation_SubmitRetryableWithEmptyCalldata_RecordsCa // // Create a fake block with the new state root so the next DigestMessage sees it. // - // A bit of hack but without this, setting up the test is almost impossible / kinda random + // A bit of a hack but without this, setting up the test is almost impossible / kinda random // and hardly maintainable. Because, then you'd need to record an intermediate trie // node instead of the leaf node, because regular scenarios won't let you have a leaf node there beforehand. // And to do that, you'd need to influence the tx parameters to change its hash to change the calldata storage slot path, @@ -675,11 +669,9 @@ public async Task RecordBlockCreation_SubmitRetryableWithEmptyCalldata_RecordsCa new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestParams.Index); - ArbitrumWitness witness = recordResult.Witness; - // Assert some trie node on the path to the calldata storage slot has been captured (not captured elsewhere during block recording ofc, otherwise test is useless) // Here I assert the leaf node hash (found during debugging). - witness.Witness.State.Any(node => Keccak.Compute(node) == new Hash256("0xb2020a6fea12f86ace9de5bed3312ca953a2f8ae0730062fa9df4fc833c99782")).Should().BeTrue( + recordResult.Preimages.ContainsKey(new Hash256("0xb2020a6fea12f86ace9de5bed3312ca953a2f8ae0730062fa9df4fc833c99782")).Should().BeTrue( "Witness state should contain trie node for retryable empty calldata storage slot"); } @@ -785,11 +777,9 @@ public async Task RecordBlockCreation_TryReapRetryableNotExpired_RecordsTimeoutW new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestParams.Index); - ArbitrumWitness witness = recordResult.Witness; - // Assert the leaf trie node for TimeoutWindowsLeft (offset 6) has been captured. // Trie node hash determined during debugging — without the fix, this node would NOT be in the witness. - witness.Witness.State.Any(node => Keccak.Compute(node) == new Hash256("0xb9b0e8140da26e36ad74be6f20e6dc5073cda81b1ed9c3c8d63388f69640f24e")).Should().BeTrue( + recordResult.Preimages.ContainsKey(new Hash256("0xb9b0e8140da26e36ad74be6f20e6dc5073cda81b1ed9c3c8d63388f69640f24e")).Should().BeTrue( "Witness state should contain trie node for retryable TimeoutWindowsLeft storage slot"); } @@ -823,12 +813,10 @@ public async Task RecordBlockCreation_NonUserTransaction_RecordsBrotliCompressio new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestParams.Index); - ArbitrumWitness witness = recordResult.Witness; - // Assert the leaf trie node for BrotliCompressionLevel (offset 7) has been captured. // Trie node hash determined during debugging — without the fix, this node would NOT be // in the witness because non-user txs returned early from CanAddTransaction. - witness.Witness.State.Any(node => Keccak.Compute(node) == new Hash256("0x9bcf99179b305f1d54185508b47cc61fb0f8b804dd449a9b60ed068af7b1d62f")).Should().BeTrue( + recordResult.Preimages.ContainsKey(new Hash256("0x9bcf99179b305f1d54185508b47cc61fb0f8b804dd449a9b60ed068af7b1d62f")).Should().BeTrue( "Witness state should contain trie node for BrotliCompressionLevel storage slot (offset 7)"); } @@ -952,8 +940,6 @@ public async Task RecordBlockCreation_WhenStorageSlotModifiedAndResetInSameBlock initialValue.ToBigEndian().WithoutLeadingZeros().ToArray()); } - ArbitrumWitness witness = recordResult.Witness; - // Collect the expected storage proof from the parent state. AccountProofCollector collector = new(contractAddress, [storageSlot]); chain.StateReader.RunTreeVisitor(collector, parentHeader); @@ -966,13 +952,9 @@ public async Task RecordBlockCreation_WhenStorageSlotModifiedAndResetInSameBlock storageProofNodes.Should().NotBeEmpty( "the contract should have a non-empty storage proof for slot 0 in the parent state"); - HashSet witnessNodeHashes = witness.Witness.State - .Select(Keccak.Compute) - .ToHashSet(); - foreach (byte[] proofNode in storageProofNodes) { - witnessNodeHashes.Should().Contain(Keccak.Compute(proofNode), + recordResult.Preimages.ContainsKey(Keccak.Compute(proofNode)).Should().BeTrue( "witness should contain storage trie proof node even when the net storage change " + "is zero (slot was modified by TX1 then reset to original value by TX2)"); } @@ -1089,8 +1071,6 @@ public async Task RecordBlockCreation_WhenStateModifiedAndResetDirectlyViaWorldS arbosState.NetworkFeeAccount.Get().Should().Be(originalFeeAccount); } - ArbitrumWitness witness = recordResult.Witness; - // NetworkFeeAccount is at root BackingStorage (empty storageKey), offset 3 UInt256 networkFeeAccountSlot = ComputeMappedStorageSlot([], ArbosStateOffsets.NetworkFeeAccountOffset); @@ -1106,13 +1086,9 @@ public async Task RecordBlockCreation_WhenStateModifiedAndResetDirectlyViaWorldS storageProofNodes.Should().NotBeEmpty( "NetworkFeeAccount slot should have a non-empty storage proof in the parent state"); - HashSet witnessNodeHashes = witness.Witness.State - .Select(Keccak.Compute) - .ToHashSet(); - foreach (byte[] proofNode in storageProofNodes) { - witnessNodeHashes.Should().Contain(Keccak.Compute(proofNode), + recordResult.Preimages.ContainsKey(Keccak.Compute(proofNode)).Should().BeTrue( "witness should contain storage trie proof node for NetworkFeeAccount " + "even when the net storage change is zero (modified by TX1 then reset by TX2)"); } @@ -1217,8 +1193,6 @@ public async Task RecordBlockCreation_TransactionSetsSomeStateButReverts_StillRe new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestParams.Index); - ArbitrumWitness witness = recordResult.Witness; - // Compute the mapped Ethereum storage slot for AddressTable._backingStorage at offset 1. // This replicates ArbosStorage.MapAddress to determine the actual storage trie key. byte[] addressTableStorageKey = Keccak.Compute(ArbosSubspaceIDs.AddressTableSubspace).BytesToArray(); @@ -1236,13 +1210,9 @@ public async Task RecordBlockCreation_TransactionSetsSomeStateButReverts_StillRe storageProofNodes.Should().NotBeEmpty( "pre-populated slot should have a non-empty storage proof in the parent state"); - HashSet witnessNodeHashes = witness.Witness.State - .Select(Keccak.Compute) - .ToHashSet(); - foreach (byte[] proofNode in storageProofNodes) { - witnessNodeHashes.Should().Contain(Keccak.Compute(proofNode), + recordResult.Preimages.ContainsKey(Keccak.Compute(proofNode)).Should().BeTrue( "witness should contain storage trie proof node for AddressTable._backingStorage " + "even when the transaction reverted"); } @@ -1306,6 +1276,12 @@ private static UInt256 ComputeMappedStorageSlot(byte[] storageKey, ulong offset) return new UInt256(mappedKey, isBigEndian: true); } + private static void AssertWitnessMatchesRecordResult(ArbitrumWitness witness, RecordResult recordResult) + { + RecordResult fromWitness = new(recordResult.Index, recordResult.BlockHash, witness); + fromWitness.Should().BeEquivalentTo(recordResult); + } + private static T ThrowOnFailure(ResultWrapper result, ulong msgIndex) { if (result.Result != Result.Success) diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs index 9c2d8c151..3767d50d6 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs @@ -7,6 +7,7 @@ using Autofac; using Nethermind.Arbitrum.Arbos; using Nethermind.Arbitrum.Arbos.Storage; +using Nethermind.Arbitrum.Arbos.Stylus; using Nethermind.Arbitrum.Genesis; using Nethermind.Arbitrum.Modules; using Nethermind.Arbitrum.Config; @@ -32,6 +33,8 @@ using Nethermind.TxPool; using Nethermind.Wallet; using Nethermind.Arbitrum.Execution.Stateless; +using Nethermind.Arbitrum.Math; +using Nethermind.Consensus.Stateless; using Nethermind.Db.LogIndex; namespace Nethermind.Arbitrum.Test.Infrastructure; @@ -304,6 +307,31 @@ public async Task> Digest(TestL2Transactions messag return (result, parameters); } + // Helper function to return the witness because RecordBlockCreation returns the accumulated preimages altogether + public async Task BuildBlockWitness(RecordBlockCreationParameters parameters) + { + long blockNumber = MessageBlockConverter.MessageIndexToBlockNumber(parameters.Index, Dependencies.SpecHelper); + BlockHeader parent = BlockTree.FindHeader(blockNumber - 1) + ?? throw new ArgumentException($"Unable to find parent for block {blockNumber}"); + + ArbitrumPayloadAttributes payload = new() + { + MessageWithMetadata = parameters.Message, + Number = blockNumber + }; + + string[] wasmTargets = parameters.WasmTargets; + string localTarget = StylusTargets.GetLocalTargetName(); + if (!wasmTargets.Contains(localTarget)) + wasmTargets = wasmTargets.Append(localTarget).ToArray(); + + IArbitrumWitnessGeneratingBlockProcessingEnvFactory factory = Container.Resolve(); + using IWitnessGeneratingBlockProcessingEnvScope scope = factory.CreateScope(wasmTargets); + IBlockBuildingWitnessCollector witnessCollector = ((IWitnessGeneratingPolyvalentEnv)scope.Env).CreateBlockBuildingWitnessCollector(); + (Block _, ArbitrumWitness witness) = await witnessCollector.BuildBlockAndGetWitness(parent, payload); + return witness; + } + public void DumpBlocks() { List blocks = new(); diff --git a/src/Nethermind.Arbitrum/Data/RecordResult.cs b/src/Nethermind.Arbitrum/Data/RecordResult.cs index 0dc979e0f..760017006 100644 --- a/src/Nethermind.Arbitrum/Data/RecordResult.cs +++ b/src/Nethermind.Arbitrum/Data/RecordResult.cs @@ -3,7 +3,6 @@ using Nethermind.Arbitrum.Execution.Stateless; using Nethermind.Core.Crypto; -using System.Text.Json.Serialization; namespace Nethermind.Arbitrum.Data; @@ -14,14 +13,10 @@ public sealed class RecordResult public Dictionary Preimages { get; } public Dictionary>? UserWasms { get; } - [JsonIgnore] - public ArbitrumWitness Witness { get; } - public RecordResult(ulong messageIndex, Hash256 blockHash, ArbitrumWitness arbWitness) { Index = messageIndex; BlockHash = blockHash; - Witness = arbWitness; UserWasms = arbWitness.UserWasms?.ToDictionary( kvp => kvp.Key.ToHash256(), kvp => kvp.Value); diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs index 21d216ba4..661731c0b 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs @@ -548,33 +548,36 @@ public async Task> RecordBlockCreation(RecordBlockCr IBlockBuildingWitnessCollector witnessCollector = ((IWitnessGeneratingPolyvalentEnv)scope.Env).CreateBlockBuildingWitnessCollector(); (Block builtBlock, ArbitrumWitness witness) = await witnessCollector.BuildBlockAndGetWitness(parent, payload); - if (builtBlock.Hash is null) - return ResultWrapper.Fail($"Failed to build block {blockNumber} or block has no hash."); - - // Sometimes, it seems RecordBlockCreation is called slightly before the actual block is finalized/committed to the database. - // So we need to wait for the block to be available in the database. - Hash256? canonicalHash = null; - Stopwatch sw = Stopwatch.StartNew(); - while (sw.ElapsedMilliseconds <= arbitrumConfig.MessageLagMs) + using (witness) { - canonicalHash = BlockTree.FindCanonicalBlockInfo(blockNumber)?.BlockHash; + if (builtBlock.Hash is null) + return ResultWrapper.Fail($"Failed to build block {blockNumber} or block has no hash."); - if (canonicalHash is null) + // Sometimes, it seems RecordBlockCreation is called slightly before the actual block is finalized/committed to the database. + // So we need to wait for the block to be available in the database. + Hash256? canonicalHash = null; + Stopwatch sw = Stopwatch.StartNew(); + while (sw.ElapsedMilliseconds <= arbitrumConfig.MessageLagMs) { - await Task.Delay(10); - continue; - } + canonicalHash = BlockTree.FindCanonicalBlockInfo(blockNumber)?.BlockHash; - break; - } + if (canonicalHash is null) + { + await Task.Delay(10); + continue; + } - if (canonicalHash is null) - return ResultWrapper.Fail(ArbitrumRpcErrors.BlockNotFound(blockNumber)); - else if (canonicalHash != builtBlock.Hash) - return ResultWrapper.Fail($"Built block hash: {builtBlock.Hash} does not match canonical block header hash: {canonicalHash}"); + break; + } + + if (canonicalHash is null) + return ResultWrapper.Fail(ArbitrumRpcErrors.BlockNotFound(blockNumber)); + else if (canonicalHash != builtBlock.Hash) + return ResultWrapper.Fail($"Built block hash: {builtBlock.Hash} does not match canonical block header hash: {canonicalHash}"); - RecordResult result = new(parameters.Index, builtBlock.Hash!, witness); - return ResultWrapper.Success(result); + RecordResult result = new(parameters.Index, builtBlock.Hash!, witness); + return ResultWrapper.Success(result); + } } private Hash256 GetSendRootFromBlock(Block block) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitness.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitness.cs index 610d73672..30fd4a457 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitness.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitness.cs @@ -3,11 +3,7 @@ namespace Nethermind.Arbitrum.Execution.Stateless; -public class ArbitrumWitness(Witness witness, Dictionary>? userWasms) +public record class ArbitrumWitness(Witness Witness, Dictionary>? UserWasms) : IDisposable { - private readonly Witness _witness = witness; - - public ref readonly Witness Witness => ref _witness; - - public Dictionary>? UserWasms => userWasms; + public void Dispose() => Witness.Dispose(); } From e4a030e837afd237dc8f7178eb459ade8d152710 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Thu, 26 Feb 2026 15:05:08 +0800 Subject: [PATCH 47/87] feat: Update to latest base nmc wit gen branch --- src/Nethermind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind b/src/Nethermind index c70ede728..0b5ee27e9 160000 --- a/src/Nethermind +++ b/src/Nethermind @@ -1 +1 @@ -Subproject commit c70ede728beb4f72fb263cc533343e68f663657e +Subproject commit 0b5ee27e90a58afb6bcf85eae9dae1d9dc78d964 From 720c07a7abc051d02a378dc1b05e483599b970b6 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Thu, 26 Feb 2026 15:39:34 +0800 Subject: [PATCH 48/87] fix: merge conflicts --- .../Infrastructure/ArbitrumRpcTestBlockchain.cs | 1 - .../Config/ArbitrumBlockProducerEnvFactory.cs | 2 +- src/Nethermind.Arbitrum/Modules/ArbitrumEthModuleFactory.cs | 4 +--- src/Nethermind.Arbitrum/Modules/ArbitrumEthRpcModule.cs | 1 - 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs index e81bf25bc..f909058d4 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs @@ -36,7 +36,6 @@ using Nethermind.Arbitrum.Execution.Stateless; using Nethermind.Arbitrum.Math; using Nethermind.Consensus.Stateless; -using Nethermind.Db.LogIndex; namespace Nethermind.Arbitrum.Test.Infrastructure; diff --git a/src/Nethermind.Arbitrum/Config/ArbitrumBlockProducerEnvFactory.cs b/src/Nethermind.Arbitrum/Config/ArbitrumBlockProducerEnvFactory.cs index 2604db11e..f0250add5 100644 --- a/src/Nethermind.Arbitrum/Config/ArbitrumBlockProducerEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Config/ArbitrumBlockProducerEnvFactory.cs @@ -75,7 +75,7 @@ protected override ContainerBuilder ConfigureBuilder(ContainerBuilder builder) PreBlockCaches preBlockCaches = ctx.Resolve(); IPrecompileProvider precompileProvider = ctx.Resolve(); // Note: The use of FrozenDictionary means that this cannot be used for other processing env also due to risk of memory leak. - return new CachedCodeInfoRepository(precompileProvider, originalCodeInfoRepository, + return new PrecompileCachedCodeInfoRepository(precompileProvider, originalCodeInfoRepository, blocksConfig.CachePrecompilesOnBlockProcessing ? preBlockCaches?.PrecompileCache : null); }); } diff --git a/src/Nethermind.Arbitrum/Modules/ArbitrumEthModuleFactory.cs b/src/Nethermind.Arbitrum/Modules/ArbitrumEthModuleFactory.cs index ccffc1167..2f75db0cb 100644 --- a/src/Nethermind.Arbitrum/Modules/ArbitrumEthModuleFactory.cs +++ b/src/Nethermind.Arbitrum/Modules/ArbitrumEthModuleFactory.cs @@ -14,7 +14,6 @@ using Nethermind.JsonRpc.Modules.Eth; using Nethermind.JsonRpc.Modules.Eth.FeeHistory; using Nethermind.JsonRpc.Modules.Eth.GasPrice; -using Nethermind.Db.LogIndex; using Nethermind.Logging; using Nethermind.Network; using Nethermind.State; @@ -39,9 +38,8 @@ public class ArbitrumEthModuleFactory( IFeeHistoryOracle feeHistoryOracle, IProtocolsManager protocolsManager, IForkInfo forkInfo, - ILogIndexConfig? logIndexConfig, IBlocksConfig blocksConfig, - ILogIndexConfig logIndexConfig, + ILogIndexConfig? logIndexConfig, ArbitrumChainSpecEngineParameters chainSpecParams) : ModuleFactoryBase { public override IEthRpcModule Create() diff --git a/src/Nethermind.Arbitrum/Modules/ArbitrumEthRpcModule.cs b/src/Nethermind.Arbitrum/Modules/ArbitrumEthRpcModule.cs index 8117655e0..8374636e7 100644 --- a/src/Nethermind.Arbitrum/Modules/ArbitrumEthRpcModule.cs +++ b/src/Nethermind.Arbitrum/Modules/ArbitrumEthRpcModule.cs @@ -22,7 +22,6 @@ using Nethermind.JsonRpc.Modules.Eth; using Nethermind.JsonRpc.Modules.Eth.FeeHistory; using Nethermind.JsonRpc.Modules.Eth.GasPrice; -using Nethermind.Db.LogIndex; using Nethermind.Logging; using Nethermind.Network; using Nethermind.Specs.Forks; From 0abeb975d368d51c17e910e6c40d532e50049d5c Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 3 Mar 2026 19:00:24 +0800 Subject: [PATCH 49/87] fix: Merge conflicts --- .../Infrastructure/WasmGasTestHelper.cs | 4 +- .../Arbos/Stylus/StylusNativeTests.cs | 73 ++---- .../Arbos/Programs/StylusPrograms.cs | 16 +- .../Arbos/Stylus/StylusNative.cs | 239 ------------------ 4 files changed, 38 insertions(+), 294 deletions(-) delete mode 100644 src/Nethermind.Arbitrum/Arbos/Stylus/StylusNative.cs diff --git a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/Infrastructure/WasmGasTestHelper.cs b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/Infrastructure/WasmGasTestHelper.cs index 90d94ba81..ad1de9932 100644 --- a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/Infrastructure/WasmGasTestHelper.cs +++ b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/Infrastructure/WasmGasTestHelper.cs @@ -33,7 +33,7 @@ public class WasmGasTestHelper : IDisposable public IStylusVmHost VmHost => _vmHost; public IWorldState WorldState => _worldState; - public WasmGasTestHelper(long gasAvailable = 1_000_000, IReleaseSpec? spec = null, ReadOnlyMemory inputData = default) + public WasmGasTestHelper(long gasAvailable = 1_000_000, IReleaseSpec? spec = null) { spec ??= Cancun.Instance; _accessTracker = new StackAccessTracker(); @@ -48,7 +48,7 @@ public WasmGasTestHelper(long gasAvailable = 1_000_000, IReleaseSpec? spec = nul Address.Zero, Address.Zero, 0, 0, 0, - inputData); + Array.Empty()); _vmState = VmState.RentTopLevel( ArbitrumGasPolicy.FromLong(gasAvailable), diff --git a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs index ca2ad1c74..35578b64c 100644 --- a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs +++ b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs @@ -4,7 +4,6 @@ using System.Security.Cryptography; using System.Text; using FluentAssertions; -using Nethermind.Arbitrum.Programs; using Nethermind.Arbitrum.Stylus; using Nethermind.Arbitrum.Test.Arbos.Stylus.Infrastructure; using Nethermind.Core.Crypto; @@ -251,35 +250,23 @@ public static void Call_CounterContractSetsValue_UpdatesStorageThroughNativeApi( ulong gas = 1_000_000; uint arbosTag = 0; - ValueHash256 moduleHash = new(); - - byte[] getNumberCalldata = CounterContractCallData.GetNumberCalldata(); - byte[] setNumberCalldata = CounterContractCallData.GetSetNumberCalldata(9); - // Get number (should be 0 initially) - using (WasmGasTestHelper helper = new(inputData: getNumberCalldata)) - { - StylusNativeResult getNumberResult1 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, helper.VmHost, moduleHash, arbosTag, ref gas); - getNumberResult1.Value.Should().BeEquivalentTo(new byte[32]); - } + byte[] getNumberCalldata = CounterContractCallData.GetNumberCalldata(); + StylusNativeResult getNumberResult1 = StylusNative.Call(asmResult.Value!, getNumberCalldata, config, apiApi, evmData, true, arbosTag, ref gas); + getNumberResult1.Value.Should().BeEquivalentTo(new byte[32]); // Set number to 9 - using (WasmGasTestHelper helper = new(inputData: setNumberCalldata)) - { - StylusNativeResult setNumberResult = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, helper.VmHost, moduleHash, arbosTag, ref gas); - setNumberResult.Value.Should().BeEmpty(); - } + byte[] setNumberCalldata = CounterContractCallData.GetSetNumberCalldata(9); + StylusNativeResult setNumberResult = StylusNative.Call(asmResult.Value!, setNumberCalldata, config, apiApi, evmData, true, arbosTag, ref gas); + setNumberResult.Value.Should().BeEmpty(); // Get number again (should now be 9) - using (WasmGasTestHelper helper = new(inputData: getNumberCalldata)) - { - StylusNativeResult getNumberResult2 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, helper.VmHost, moduleHash, arbosTag, ref gas); + StylusNativeResult getNumberResult2 = StylusNative.Call(asmResult.Value!, getNumberCalldata, config, apiApi, evmData, true, arbosTag, ref gas); - byte[] expected = new byte[32]; - expected[^1] = 9; // Last byte should be 9 after setNumber(9) + byte[] expected = new byte[32]; + expected[^1] = 9; // Last byte should be 9 after setNumber(9) - getNumberResult2.Value.Should().BeEquivalentTo(expected); - } + getNumberResult2.Value.Should().BeEquivalentTo(expected); } [Test] @@ -304,36 +291,24 @@ public static void Call_CounterContractIncrement_EmitsLogsAndUpdatesStorageThrou ulong gas = 1_000_000; uint arbosTag = 0; - ValueHash256 moduleHash = new(); - + // Get number (should be 0 initially) byte[] getNumberCalldata = CounterContractCallData.GetNumberCalldata(); - byte[] incrementNumberCalldata = CounterContractCallData.GetIncrementCalldata(); - - // Get number (should be 0 initially) - using (WasmGasTestHelper helper = new(inputData: getNumberCalldata)) - { - StylusNativeResult getNumberResult1 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, helper.VmHost, moduleHash, arbosTag, ref gas); - getNumberResult1.Value.Should().BeEquivalentTo(new byte[32]); - } + StylusNativeResult getNumberResult1 = StylusNative.Call(asmResult.Value!, getNumberCalldata, config, apiApi, evmData, true, arbosTag, ref gas); + getNumberResult1.Value.Should().BeEquivalentTo(new byte[32]); // Increment number from 0 to 1 - using (WasmGasTestHelper helper = new(inputData: incrementNumberCalldata)) - { - StylusNativeResult incrementNumberResult = - StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, helper.VmHost, moduleHash, arbosTag, ref gas); - incrementNumberResult.IsSuccess.Should().BeTrue(); - } + byte[] incrementNumberCalldata = CounterContractCallData.GetIncrementCalldata(); + StylusNativeResult incrementNumberResult = + StylusNative.Call(asmResult.Value!, incrementNumberCalldata, config, apiApi, evmData, true, arbosTag, ref gas); + incrementNumberResult.IsSuccess.Should().BeTrue(); // Get number again (should now be 1) - using (WasmGasTestHelper helper = new(inputData: getNumberCalldata)) - { - StylusNativeResult getNumberResult2 = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, helper.VmHost, moduleHash, arbosTag, ref gas); + StylusNativeResult getNumberResult2 = StylusNative.Call(asmResult.Value!, getNumberCalldata, config, apiApi, evmData, true, arbosTag, ref gas); - byte[] expected = new byte[32]; - expected[^1] = 1; + byte[] expected = new byte[32]; + expected[^1] = 1; - getNumberResult2.Value.Should().BeEquivalentTo(expected); - } + getNumberResult2.Value.Should().BeEquivalentTo(expected); } [Test] @@ -367,11 +342,7 @@ public static void Call_KeccakCalculation_ReturnsValidHash() ulong gas = 1_000_000; uint arbosTag = 0; - ValueHash256 moduleHash = new(); - - using WasmGasTestHelper helper = new(inputData: callDataBytes); - - StylusNativeResult resultData = StylusNative.Call(asmResult.Value!, config, apiApi, evmData, true, helper.VmHost, moduleHash, arbosTag, ref gas); + StylusNativeResult resultData = StylusNative.Call(asmResult.Value!, callDataBytes, config, apiApi, evmData, true, arbosTag, ref gas); resultData.Value.Should().BeEquivalentTo(hash); } diff --git a/src/Nethermind.Arbitrum/Arbos/Programs/StylusPrograms.cs b/src/Nethermind.Arbitrum/Arbos/Programs/StylusPrograms.cs index 5d64ecf32..fc2bb8379 100644 --- a/src/Nethermind.Arbitrum/Arbos/Programs/StylusPrograms.cs +++ b/src/Nethermind.Arbitrum/Arbos/Programs/StylusPrograms.cs @@ -202,9 +202,21 @@ public StylusOperationResult CallProgram(IStylusVmHost vmHost, TracingIn using IStylusEvmApi evmApi = new StylusEvmApi(vmHost, vmHost.VmState.Env.ExecutingAccount, memoryModel); + if (vmHost.IsRecordingExecution) + { + Dictionary asmMap = new(); + foreach (string target in vmHost.WasmStore.GetWasmTargets()) + { + if (!vmHost.WasmStore.TryGetActivatedAsm(target, moduleHash, out byte[]? asm)) + throw new InvalidOperationException($"Cannot find activated wasm, missing target: {target}"); + asmMap.Add(target, asm); + } + vmHost.RecordUserWasm(moduleHash, asmMap); + } + long startTimestamp = Stopwatch.GetTimestamp(); - StylusNativeResult callResult = StylusNative.Call(localAsm.Value, stylusConfig, evmApi, evmData, - debugMode, vmHost, in moduleHash, arbosTag, ref gasAvailable); + StylusNativeResult callResult = StylusNative.Call(localAsm.Value, vmHost.VmState.Env.InputData.ToArray(), + stylusConfig, evmApi, evmData, debugMode, arbosTag, ref gasAvailable); long elapsedMicroseconds = (long)Stopwatch.GetElapsedTime(startTimestamp).TotalMicroseconds; vmHost.VmState.Gas = ArbitrumGasPolicy.FromLong((long)gasAvailable); diff --git a/src/Nethermind.Arbitrum/Arbos/Stylus/StylusNative.cs b/src/Nethermind.Arbitrum/Arbos/Stylus/StylusNative.cs deleted file mode 100644 index 1961377b3..000000000 --- a/src/Nethermind.Arbitrum/Arbos/Stylus/StylusNative.cs +++ /dev/null @@ -1,239 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 -// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md - -using System.Diagnostics.CodeAnalysis; -using System.Runtime.InteropServices; -using System.Text; -using Nethermind.Arbitrum.Arbos.Programs; -using Nethermind.Core.Crypto; - -namespace Nethermind.Arbitrum.Arbos.Stylus; - -public readonly record struct StylusNativeResult(UserOutcomeKind Status, string Error, T? Value) -{ - [MemberNotNullWhen(true, nameof(Value))] - public bool IsSuccess => Status == UserOutcomeKind.Success; - - public void Deconstruct(out UserOutcomeKind status, out string error, out T? value) - { - status = Status; - error = Error; - value = Value; - } - - public static StylusNativeResult Success(T value) - { - return new StylusNativeResult(UserOutcomeKind.Success, string.Empty, value); - } - - public static StylusNativeResult Failure(UserOutcomeKind status, string error) - { - return new StylusNativeResult(status, error, default); - } - - public static StylusNativeResult Failure(UserOutcomeKind status, string error, T data) - { - return new StylusNativeResult(status, error, data); - } -} - -public readonly record struct ActivateResult(Bytes32 ModuleHash, StylusData ActivationInfo, byte[] WavmModule); - -public static unsafe partial class StylusNative -{ - public static StylusNativeResult Call(byte[] module, StylusConfig config, IStylusEvmApi api, EvmData evmData, bool debug, - IStylusVmHost vmHost, in ValueHash256 moduleHash, uint arbOsTag, ref ulong gas) - { - using GoSliceHandle moduleSlice = GoSliceHandle.From(module); - byte[] callData = vmHost.VmState.Env.InputData.ToArray(); - using GoSliceHandle callDataSlice = GoSliceHandle.From(callData); - using StylusEnvApiRegistration registration = StylusEvmApiRegistry.Register(api); - - NativeRequestHandler handler = new() - { - HandleRequestFptr = &StylusEvmApiRegistry.HandleStylusEnvApiRequest, - Id = registration.Id - }; - - if (vmHost.IsRecordingExecution) - { - Dictionary asmMap = new(); - foreach (string target in vmHost.WasmStore.GetWasmTargets()) - { - if (!vmHost.WasmStore.TryGetActivatedAsm(target, moduleHash, out byte[]? asm)) - throw new InvalidOperationException($"Cannot find activated wasm, missing target: {target}"); - asmMap.Add(target, asm); - } - vmHost.RecordUserWasm(moduleHash, asmMap); - } - - RustBytes output = new(); - UserOutcomeKind status = stylus_call( - moduleSlice.Data, - callDataSlice.Data, - config, - handler, - evmData, - debug, - ref output, - ref gas, - arbOsTag); - - byte[] resultBytes = ReadAndFreeRustBytes(output); - - return status switch - { - UserOutcomeKind.Success => StylusNativeResult.Success(resultBytes), - UserOutcomeKind.Revert => StylusNativeResult.Failure(status, Encoding.UTF8.GetString(resultBytes), resultBytes), - UserOutcomeKind.Failure => StylusNativeResult.Failure(status, Encoding.UTF8.GetString(resultBytes)), - UserOutcomeKind.OutOfInk => StylusNativeResult.Failure(status, "max call depth exceeded"), - UserOutcomeKind.OutOfStack => StylusNativeResult.Failure(status, "out of gas"), - _ => StylusNativeResult.Failure(status, "Unknown error during Stylus call", resultBytes) - }; - } - - public static StylusNativeResult Compile(byte[] wasm, ushort version, bool debug, string targetName, bool cranelift) - { - using GoSliceHandle wasmSlice = GoSliceHandle.From(wasm); - using GoSliceHandle targetSlice = GoSliceHandle.From(targetName); - - RustBytes output = new(); - UserOutcomeKind status = stylus_compile(wasmSlice.Data, version, debug, targetSlice.Data, cranelift, ref output); - byte[] resultBytes = ReadAndFreeRustBytes(output); - - return status != UserOutcomeKind.Success - ? StylusNativeResult.Failure(status, Encoding.UTF8.GetString(resultBytes)) - : StylusNativeResult.Success(resultBytes); - } - - public static StylusNativeResult SetTarget(string name, string descriptor, bool isNative) - { - using GoSliceHandle nameSlice = GoSliceHandle.From(name); - using GoSliceHandle descriptorSlice = GoSliceHandle.From(descriptor); - - RustBytes output = new(); - UserOutcomeKind status = stylus_target_set( - nameSlice.Data, - descriptorSlice.Data, - ref output, - isNative); - - byte[] resultBytes = ReadAndFreeRustBytes(output); - - return status != UserOutcomeKind.Success - ? StylusNativeResult.Failure(status, Encoding.UTF8.GetString(resultBytes)) - : StylusNativeResult.Success(resultBytes); - } - - public static StylusNativeResult WatToWasm(byte[] wat) - { - using GoSliceHandle watSlice = GoSliceHandle.From(wat); - - RustBytes output = new(); - UserOutcomeKind watStatus = wat_to_wasm(watSlice.Data, ref output); - - byte[] resultBytes = ReadAndFreeRustBytes(output); - - return watStatus == UserOutcomeKind.Success - ? StylusNativeResult.Success(resultBytes) - : StylusNativeResult.Failure(watStatus, Encoding.UTF8.GetString(resultBytes)); - } - - public static StylusNativeResult Activate(byte[] wasm, ushort pageLimit, ushort stylusVersion, ulong arbosVersionForGas, bool debug, - Bytes32 codeHash, ref ulong gas) - { - using GoSliceHandle wasmSlice = GoSliceHandle.From(wasm); - - RustBytes output = new(); - UserOutcomeKind status = stylus_activate( - wasmSlice.Data, - pageLimit, - stylusVersion, - arbosVersionForGas, - debug, - ref output, - ref codeHash, - out Bytes32 moduleHash, - out StylusData stylusData, - ref gas); - - byte[] resultBytes = ReadAndFreeRustBytes(output); - - return status != UserOutcomeKind.Success - ? StylusNativeResult.Failure(status, Encoding.UTF8.GetString(resultBytes)) - : StylusNativeResult.Success(new ActivateResult(moduleHash, stylusData, resultBytes)); - } - - public static BrotliStatus BrotliCompress(ReadOnlySpan input, Span output, uint level, BrotliDictionary dictionary, out int bytesWritten) - { - ReadOnlySpan nonEmptyInput = EnsureBrotliNonEmpty(input); - - fixed (byte* inputPtr = nonEmptyInput) - fixed (byte* outputPtr = output) - { - nuint inputLen = (nuint)nonEmptyInput.Length; - nuint outputLen = (nuint)output.Length; - - BrotliBuffer inputBuffer = new() { Ptr = inputPtr, Len = &inputLen }; - BrotliBuffer outputBuffer = new() { Ptr = outputPtr, Len = &outputLen }; - - BrotliStatus status = brotli_compress(inputBuffer, outputBuffer, dictionary, level); - bytesWritten = (int)outputLen; - - return status; - } - } - - public static BrotliStatus BrotliDecompress(ReadOnlySpan input, Span output, BrotliDictionary dictionary, out int bytesWritten) - { - ReadOnlySpan nonEmptyInput = EnsureBrotliNonEmpty(input); - - fixed (byte* inputPtr = nonEmptyInput) - fixed (byte* outputPtr = output) - { - nuint inputLen = (nuint)nonEmptyInput.Length; - nuint outputLen = (nuint)output.Length; - - BrotliBuffer inputBuffer = new() { Ptr = inputPtr, Len = &inputLen }; - BrotliBuffer outputBuffer = new() { Ptr = outputPtr, Len = &outputLen }; - - BrotliStatus status = brotli_decompress(inputBuffer, outputBuffer, dictionary); - bytesWritten = (int)outputLen; - - return status; - } - } - - public static void SetWasmLruCacheCapacity(ulong capacity) - { - stylus_set_cache_lru_capacity(capacity); - } - - public static int GetCompressedBufferSize(int inputSize) - { - // This matches the typical brotli worst-case compression bound - return inputSize + (inputSize >> 10) * 8 + 64; - } - - private static byte[] ReadAndFreeRustBytes(RustBytes output) - { - if (output.Len == 0) - { - free_rust_bytes(output); - return []; - } - - byte[] buffer = new byte[(int)output.Len]; - Marshal.Copy(output.Ptr, buffer, 0, buffer.Length); - - free_rust_bytes(output); - - return buffer; - } - - private static ReadOnlySpan EnsureBrotliNonEmpty(ReadOnlySpan input) - { - // Nitro: Ensures pointer is not null (shouldn't be necessary, but brotli docs are picky about NULL) - return input.Length > 0 ? input : [0x00]; - } -} From 05d9cf6ff2295c7a3730a5d9de1c4547256b6c2c Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 3 Mar 2026 19:14:42 +0800 Subject: [PATCH 50/87] fix: Update to latest nmc (with witness gen) --- src/Nethermind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind b/src/Nethermind index cdae536d9..e98856ba6 160000 --- a/src/Nethermind +++ b/src/Nethermind @@ -1 +1 @@ -Subproject commit cdae536d9ddf6e4995ac11e6d1ed602e836295be +Subproject commit e98856ba66d7c32e3619d8b93206547ac597c737 From 0dd006fc526e3ffb12c7a013fb07ae64769c4066 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 3 Mar 2026 19:48:34 +0800 Subject: [PATCH 51/87] fix: Format --- src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs index 35578b64c..9f611258d 100644 --- a/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs +++ b/src/Nethermind.Arbitrum.Test/Arbos/Stylus/StylusNativeTests.cs @@ -291,7 +291,7 @@ public static void Call_CounterContractIncrement_EmitsLogsAndUpdatesStorageThrou ulong gas = 1_000_000; uint arbosTag = 0; - // Get number (should be 0 initially) + // Get number (should be 0 initially) byte[] getNumberCalldata = CounterContractCallData.GetNumberCalldata(); StylusNativeResult getNumberResult1 = StylusNative.Call(asmResult.Value!, getNumberCalldata, config, apiApi, evmData, true, arbosTag, ref gas); getNumberResult1.Value.Should().BeEquivalentTo(new byte[32]); From 415ae888ce11a1a6ad6d512506681a49a74842f3 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 3 Mar 2026 19:50:12 +0800 Subject: [PATCH 52/87] feat: Use NoOp L1BlockCache instead of removing static from L1BlockCache.CachedL1BlockHashes --- src/Nethermind.Arbitrum/Evm/L1BlockCache.cs | 2 +- ...nessGeneratingBlockProcessingEnvFactory.cs | 4 +- .../Execution/Stateless/NoOpL1BlockCache.cs | 37 +++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 src/Nethermind.Arbitrum/Execution/Stateless/NoOpL1BlockCache.cs diff --git a/src/Nethermind.Arbitrum/Evm/L1BlockCache.cs b/src/Nethermind.Arbitrum/Evm/L1BlockCache.cs index 1fd8f209b..2ad929fed 100644 --- a/src/Nethermind.Arbitrum/Evm/L1BlockCache.cs +++ b/src/Nethermind.Arbitrum/Evm/L1BlockCache.cs @@ -22,7 +22,7 @@ public sealed class L1BlockCache : IL1BlockCache /// 256 capacities match the BLOCKHASH opcode window (last 256 blocks). /// Thread-safe and shared across all transactions. /// - private readonly ClockCache CachedL1BlockHashes = new(256); + private static readonly ClockCache CachedL1BlockHashes = new(256); public ulong? GetCachedL1BlockNumber() { diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index 1636b73a6..26c6ead6a 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -72,8 +72,8 @@ private ITransactionProcessor CreateTransactionProcessor( ArbitrumUserWasmsRecorder wasmsRecorder) { BlockhashProvider blockhashProvider = new(new BlockhashCache(witnessGeneratingHeaderFinder, logManager), state, logManager); - // We don't give any l1BlockCache to the vm so that it forces querying the world state - ArbitrumVirtualMachine vm = new(arbitrumSpecHelper, blockhashProvider, wasmStore, specProvider, logManager, enableWitnessGeneration: true, wasmsRecorder: wasmsRecorder); + // We don't give a NoOp l1BlockCache to the vm so that it forces querying the world state + ArbitrumVirtualMachine vm = new(arbitrumSpecHelper, blockhashProvider, wasmStore, specProvider, logManager, new NoOpL1BlockCache(), enableWitnessGeneration: true, wasmsRecorder: wasmsRecorder); return new ArbitrumTransactionProcessor( BlobBaseFeeCalculator.Instance, specProvider, state, wasmStore, vm, logManager, diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/NoOpL1BlockCache.cs b/src/Nethermind.Arbitrum/Execution/Stateless/NoOpL1BlockCache.cs new file mode 100644 index 000000000..b153b4204 --- /dev/null +++ b/src/Nethermind.Arbitrum/Execution/Stateless/NoOpL1BlockCache.cs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md + +using Nethermind.Core.Crypto; + +namespace Nethermind.Arbitrum.Evm; + +/// +/// No-op implementation of L1 block caching. +/// Does not cache any block numbers or hashes so that validator +/// always goes through the world state and records state accesses. +/// +public sealed class NoOpL1BlockCache : IL1BlockCache +{ + public ulong? GetCachedL1BlockNumber() + { + return null; + } + + public void SetCachedL1BlockNumber(ulong blockNumber) + { + } + + public void ClearL1BlockNumberCache() + { + } + + public bool TryGetL1BlockHash(ulong l1BlockNumber, out Hash256 hash) + { + hash = Hash256.Zero; + return false; + } + + public void SetL1BlockHash(ulong l1BlockNumber, Hash256 hash) + { + } +} From 0bd26a0aedcc8c2a58450f953cbc3b17b571d0be Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 3 Mar 2026 20:06:35 +0800 Subject: [PATCH 53/87] fix build --- .../Infrastructure/ArbitrumRpcTestBlockchain.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs index f909058d4..3aed68a47 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs @@ -7,7 +7,6 @@ using Autofac; using Nethermind.Arbitrum.Arbos; using Nethermind.Arbitrum.Arbos.Storage; -using Nethermind.Arbitrum.Arbos.Stylus; using Nethermind.Arbitrum.Genesis; using Nethermind.Arbitrum.Modules; using Nethermind.Arbitrum.Config; @@ -36,6 +35,7 @@ using Nethermind.Arbitrum.Execution.Stateless; using Nethermind.Arbitrum.Math; using Nethermind.Consensus.Stateless; +using Nethermind.Arbitrum.Stylus; namespace Nethermind.Arbitrum.Test.Infrastructure; From ec724ddbab5ba5a34c780e874d5da905d023d44c Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 6 Mar 2026 19:32:49 +0800 Subject: [PATCH 54/87] fix: Resolve IWasmStore more automatically --- src/Nethermind.Arbitrum/ArbitrumPlugin.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs index 0fef141fe..f72d0787d 100644 --- a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs +++ b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs @@ -253,12 +253,7 @@ protected override void Load(ContainerBuilder builder) .AddSingleton() .AddSingleton() - .AddSingleton(context => - { - IWasmDb wasmDb = context.Resolve(); - IStylusTargetConfig stylusTargetConfig = context.Resolve(); - return new WasmStore(wasmDb, stylusTargetConfig, cacheTag: 1); - }) + .AddSingleton((db, config) => new WasmStore(db, config, cacheTag: 1)) .AddSingleton() From 8d63b9d082bde37ec0c05782782d3888fa3db3f2 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 6 Mar 2026 19:33:12 +0800 Subject: [PATCH 55/87] fix: Add correct licensing info --- .../Execution/Stateless/ArbitrumWitnessGenerationTests.cs | 3 +++ src/Nethermind.Arbitrum/Data/RecordResult.cs | 4 ++-- .../Stateless/ArbitrumStatelessBlockProcessingEnv.cs | 4 ++-- .../Execution/Stateless/ArbitrumUserWasmsRecorder.cs | 3 +++ .../Execution/Stateless/ArbitrumWitness.cs | 3 +++ .../Execution/Stateless/ArbitrumWitnessCollector.cs | 4 ++-- .../Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs | 4 ++-- .../ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs | 4 ++-- 8 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index 54b6f39ea..c56450c69 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md + using FluentAssertions; using Nethermind.Arbitrum.Arbos; using Nethermind.Arbitrum.Arbos.Storage; diff --git a/src/Nethermind.Arbitrum/Data/RecordResult.cs b/src/Nethermind.Arbitrum/Data/RecordResult.cs index 760017006..19bc5820f 100644 --- a/src/Nethermind.Arbitrum/Data/RecordResult.cs +++ b/src/Nethermind.Arbitrum/Data/RecordResult.cs @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md using Nethermind.Arbitrum.Execution.Stateless; using Nethermind.Core.Crypto; diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs index 50eef2c2e..284fa47de 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md using Nethermind.Arbitrum.Arbos; using Nethermind.Arbitrum.Evm; diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumUserWasmsRecorder.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumUserWasmsRecorder.cs index b2c9243b8..c3cae426e 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumUserWasmsRecorder.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumUserWasmsRecorder.cs @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md + using Nethermind.Core.Crypto; namespace Nethermind.Arbitrum.Execution.Stateless; diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitness.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitness.cs index 30fd4a457..5f65ac776 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitness.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitness.cs @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md + using Nethermind.Consensus.Stateless; using Nethermind.Core.Crypto; diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs index c7951859f..23abe8f01 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessCollector.cs @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md using Nethermind.Core; using Nethermind.Consensus.Stateless; diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs index 7c908b4c7..23af1a153 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnv.cs @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md using Nethermind.Blockchain; using Nethermind.Consensus; diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index 26c6ead6a..3d726577c 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md using Autofac; using Nethermind.Arbitrum.Arbos; From 5906b904ebf294e092229c9dc0de7094310e00ba Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 6 Mar 2026 19:35:48 +0800 Subject: [PATCH 56/87] fix: Typo --- .../ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index 3d726577c..366053186 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -72,7 +72,7 @@ private ITransactionProcessor CreateTransactionProcessor( ArbitrumUserWasmsRecorder wasmsRecorder) { BlockhashProvider blockhashProvider = new(new BlockhashCache(witnessGeneratingHeaderFinder, logManager), state, logManager); - // We don't give a NoOp l1BlockCache to the vm so that it forces querying the world state + // We give a NoOp l1BlockCache to the vm so that it forces querying the world state ArbitrumVirtualMachine vm = new(arbitrumSpecHelper, blockhashProvider, wasmStore, specProvider, logManager, new NoOpL1BlockCache(), enableWitnessGeneration: true, wasmsRecorder: wasmsRecorder); return new ArbitrumTransactionProcessor( From a425c6787d736b7ffd2e21512d806ebb74de1c6f Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 6 Mar 2026 20:20:17 +0800 Subject: [PATCH 57/87] fix: Use simplified DI registration syntax --- .../ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index 366053186..dd2f85dea 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -100,7 +100,7 @@ public IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTarge { builder.AddScoped(_ => new StylusTargetConfig() { OverrideWasmTargets = wasmTargets }); // Need to redeclare IWasmStore because it was originally declared as a singleton and therefore was cached with original IStylusTargetConfig - builder.AddScoped(ctx => new WasmStore(ctx.Resolve(), ctx.Resolve(), cacheTag: 1)); + builder.AddScoped((db, config) => new WasmStore(db, config, cacheTag: 1)); } builder From 70d934c38b41e99388f6283eb2b002524aee16f3 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 6 Mar 2026 20:21:08 +0800 Subject: [PATCH 58/87] feat: Subscribe to BlockAddedToMain event instead of manual active waiting loop --- .../Execution/ArbitrumExecutionEngine.cs | 43 +++++++++++-------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs index 14fe191c9..f59824b38 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md using System.Collections.Concurrent; -using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Text.Json; using Nethermind.Arbitrum.Config; @@ -549,30 +548,40 @@ public async Task> RecordBlockCreation(RecordBlockCr if (builtBlock.Hash is null) return ResultWrapper.Fail($"Failed to build block {blockNumber} or block has no hash."); - // Sometimes, it seems RecordBlockCreation is called slightly before the actual block is finalized/committed to the database. - // So we need to wait for the block to be available in the database. - Hash256? canonicalHash = null; - Stopwatch sw = Stopwatch.StartNew(); - while (sw.ElapsedMilliseconds <= arbitrumConfig.MessageLagMs) + TaskCompletionSource blockAddedTcs = new(); + + void OnBlockAddedToMain(object? sender, BlockReplacementEventArgs e) { - canonicalHash = BlockTree.FindCanonicalBlockInfo(blockNumber)?.BlockHash; + if (e.Block.Number == blockNumber) + blockAddedTcs.TrySetResult(e.Block.Hash!); + } + + BlockTree.BlockAddedToMain += OnBlockAddedToMain; + try + { + // Check immediately in case the block was committed before we subscribed + Hash256? canonicalHash = BlockTree.FindCanonicalBlockInfo(blockNumber)?.BlockHash; if (canonicalHash is null) { - await Task.Delay(10); - continue; + using CancellationTokenSource cts = arbitrumConfig.BuildProcessingTimeoutTokenSource(); + canonicalHash = await blockAddedTcs.Task.WaitAsync(cts.Token); } - break; - } + if (canonicalHash != builtBlock.Hash) + return ResultWrapper.Fail($"Built block hash: {builtBlock.Hash} does not match canonical block header hash: {canonicalHash}"); - if (canonicalHash is null) + RecordResult result = new(parameters.Index, builtBlock.Hash!, witness); + return ResultWrapper.Success(result); + } + catch (OperationCanceledException) + { return ResultWrapper.Fail(ArbitrumRpcErrors.BlockNotFound(blockNumber)); - else if (canonicalHash != builtBlock.Hash) - return ResultWrapper.Fail($"Built block hash: {builtBlock.Hash} does not match canonical block header hash: {canonicalHash}"); - - RecordResult result = new(parameters.Index, builtBlock.Hash!, witness); - return ResultWrapper.Success(result); + } + finally + { + BlockTree.BlockAddedToMain -= OnBlockAddedToMain; + } } } From 459c55bb519af54305346cd5efc729e836f0dc74 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 9 Mar 2026 18:31:07 +0800 Subject: [PATCH 59/87] fix: Use autofac as much as possible for ArbitrumWitnessGeneratingBlockProcessingEnvFactory --- ...nessGeneratingBlockProcessingEnvFactory.cs | 82 +++++++++---------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index dd2f85dea..61a5be85b 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -62,25 +62,6 @@ private static BlocksConfig CreateWitnessBlocksConfig(IBlocksConfig blocksConfig BuildBlocksOnMainState = false, }; - private ITransactionProcessor CreateTransactionProcessor( - IArbitrumSpecHelper arbitrumSpecHelper, - IWasmStore wasmStore, - ISpecProvider specProvider, - IArbosVersionProvider arbosVersionProvider, - IWorldState state, - IHeaderFinder witnessGeneratingHeaderFinder, - ArbitrumUserWasmsRecorder wasmsRecorder) - { - BlockhashProvider blockhashProvider = new(new BlockhashCache(witnessGeneratingHeaderFinder, logManager), state, logManager); - // We give a NoOp l1BlockCache to the vm so that it forces querying the world state - ArbitrumVirtualMachine vm = new(arbitrumSpecHelper, blockhashProvider, wasmStore, specProvider, logManager, new NoOpL1BlockCache(), enableWitnessGeneration: true, wasmsRecorder: wasmsRecorder); - - return new ArbitrumTransactionProcessor( - BlobBaseFeeCalculator.Instance, specProvider, state, wasmStore, vm, logManager, - new ArbitrumCodeInfoRepository(new CodeInfoRepository(state, new EthereumPrecompileProvider()), - arbosVersionProvider, state as WitnessGeneratingWorldState)); - } - // TODO: check debug endpoint exec later (compare with nitro) -- Not priority for now public IWitnessGeneratingBlockProcessingEnvScope CreateScope() => CreateScope(null); @@ -91,7 +72,6 @@ public IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTarge IStateReader stateReader = new StateReader(trieStore, readOnlyDbProvider.CodeDb, logManager); WorldState worldState = new(new TrieStoreScopeProvider(trieStore, readOnlyDbProvider.CodeDb, logManager), logManager); - ArbitrumUserWasmsRecorder wasmsRecorder = new(); IBlocksConfig blocksConfig = rootLifetimeScope.Resolve(); ILifetimeScope envLifetimeScope = rootLifetimeScope.BeginLifetimeScope((builder) => @@ -105,20 +85,48 @@ public IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTarge builder .AddScoped(stateReader) + .AddScoped() + + .AddScoped(headerStore => new WitnessGeneratingHeaderFinder(headerStore)) + .BindScoped() - .AddScoped(builder => new WitnessGeneratingHeaderFinder(builder.Resolve())) - .AddScoped(builder => new WitnessGeneratingWorldState(worldState, stateReader, trieStore, (builder.Resolve() as WitnessGeneratingHeaderFinder)!)) + .AddScoped(headerFinder => + new WitnessGeneratingWorldState(worldState, stateReader, trieStore, headerFinder)) + .BindScoped() .AddScoped(_ => CreateWitnessBlocksConfig(blocksConfig)) - .AddScoped(builder => CreateTransactionProcessor( - builder.Resolve(), - builder.Resolve(), - builder.Resolve(), - builder.Resolve(), - builder.Resolve(), - builder.Resolve(), - wasmsRecorder)) + // We give a NoOp l1BlockCache to the vm so that it forces querying + // the world state to record state accesses. + // The VM gets its own private BlockhashProvider backed by WitnessGeneratingHeaderFinder so that + // blockhash lookups are recorded in the witness. We do NOT register IBlockhashProvider/IBlockhashCache + // in the child scope so that BranchProcessor (which is AddScoped and calls Prefetch()) falls back to + // the root scope's unrecorded provider and does not pollute the witness with prefetch header lookups. + .AddScoped(ctx => + { + ILogManager log = ctx.Resolve(); + BlockhashCache recordingCache = new(ctx.Resolve(), log); + BlockhashProvider recordingProvider = new(recordingCache, ctx.Resolve(), log); + return new ArbitrumVirtualMachine( + ctx.Resolve(), + recordingProvider, + ctx.Resolve(), + ctx.Resolve(), + log, + new NoOpL1BlockCache(), + enableWitnessGeneration: true, + wasmsRecorder: ctx.Resolve()); + }) + + // Pass CodeInfoRepository, which does not cache anything, forcing querying the + // the world state to record state accesses. + .AddScoped((state, versionProvider) => + new ArbitrumCodeInfoRepository( + new CodeInfoRepository(state, new EthereumPrecompileProvider()), + versionProvider, + state as WitnessGeneratingWorldState)) + + .AddScoped() // 1st: add the tx executor .AddScoped() @@ -129,23 +137,13 @@ public IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTarge .AddScoped() // 3rd: configure the builder for block production (like ArbitrumBlockProducerEnvFactory but with my own witness capturing world state) - .AddScoped(builder => builder.Resolve().Create()) + .AddScoped(factory => factory.Create()) .AddScoped() .AddDecorator() .AddDecorator() .AddScoped() - .AddScoped(builder => - new ArbitrumWitnessGeneratingBlockProcessingEnv( - builder.Resolve(), - builder.Resolve(), - builder.Resolve(), - (builder.Resolve() as WitnessGeneratingWorldState)!, - builder.Resolve(), - builder.Resolve(), - builder.Resolve(), - wasmsRecorder, - logManager)); + .AddScoped(); }); return new ExecutionRecordingScope(envLifetimeScope); From 243a2fea39898960ca07e1ceb1cfa4db83a6e1f2 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 9 Mar 2026 18:31:54 +0800 Subject: [PATCH 60/87] fix: Use autofac as much as possible for ArbitrumStatelessBlockProcessingEnvScope --- .../ArbitrumWitnessGenerationTests.cs | 14 +- src/Nethermind.Arbitrum/ArbitrumPlugin.cs | 4 +- .../ArbitrumStatelessBlockProcessingEnv.cs | 118 ---------------- ...itrumStatelessBlockProcessingEnvFactory.cs | 131 ++++++++++++++++++ 4 files changed, 141 insertions(+), 126 deletions(-) delete mode 100644 src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs create mode 100644 src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnvFactory.cs diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index c56450c69..ab1b14060 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -9,7 +9,7 @@ using Nethermind.Arbitrum.Test.Infrastructure; using Nethermind.Blockchain.Tracing; using Nethermind.Consensus.Processing; -using Nethermind.Consensus.Validators; +using Autofac; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; @@ -45,9 +45,9 @@ public async Task RecordBlockCreation_WitnessWithoutUserWasms_StatelessExecution using ArbitrumWitness witness = await chain.BuildBlockWitness(new RecordBlockCreationParameters(digestMessage.Index, digestMessage.Message, WasmTargets: [])); AssertWitnessMatchesRecordResult(witness, recordResult); - ISpecProvider specProvider = FullChainSimulationChainSpecProvider.CreateDynamicSpecProvider(); - ArbitrumStatelessBlockProcessingEnv blockProcessingEnv = - new(witness, chain.SpecHelper, specProvider, Always.Valid, chain.StylusTargetConfig, chain.ArbosVersionProvider, chain.LogManager, chain.ArbitrumConfig); + ISpecProvider specProvider = chain.Container.Resolve(); + using IArbitrumStatelessBlockProcessingEnvScope blockProcessingEnv = + chain.Container.Resolve().CreateScope(witness); Block block = chain.BlockFinder.FindBlock(recordResult.BlockHash) ?? throw new ArgumentException($"Unable to find block {recordResult.BlockHash}"); @@ -83,9 +83,9 @@ public async Task RecordBlockCreation_WitnessWithUserWasms_StatelessExecutionIsS using ArbitrumWitness witness = await chain.BuildBlockWitness(new RecordBlockCreationParameters(digestMessage.Index, digestMessage.Message, WasmTargets: wasmTargets)); AssertWitnessMatchesRecordResult(witness, recordResult); - ISpecProvider specProvider = FullChainSimulationChainSpecProvider.CreateDynamicSpecProvider(); - ArbitrumStatelessBlockProcessingEnv blockProcessingEnv = - new(witness, chain.SpecHelper, specProvider, Always.Valid, chain.StylusTargetConfig, chain.ArbosVersionProvider, chain.LogManager, chain.ArbitrumConfig); + ISpecProvider specProvider = chain.Container.Resolve(); + using IArbitrumStatelessBlockProcessingEnvScope blockProcessingEnv = + chain.Container.Resolve().CreateScope(witness); Block block = chain.BlockFinder.FindBlock(recordResult.BlockHash) ?? throw new ArgumentException($"Unable to find block {recordResult.BlockHash}"); diff --git a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs index f72d0787d..9c0fa6170 100644 --- a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs +++ b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs @@ -295,7 +295,9 @@ protected override void Load(ContainerBuilder builder) .Bind, ArbitrumEthModuleFactory>() .AddSingleton() - .Bind(); + .Bind() + + .AddSingleton(); if (blocksConfig.BuildBlocksOnMainState) builder.AddSingleton(); diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs deleted file mode 100644 index 284fa47de..000000000 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnv.cs +++ /dev/null @@ -1,118 +0,0 @@ -// SPDX-License-Identifier: BUSL-1.1 -// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md - -using Nethermind.Arbitrum.Arbos; -using Nethermind.Arbitrum.Evm; -using Nethermind.Arbitrum.Precompiles; -using Nethermind.Arbitrum.Stylus; -using Nethermind.Blockchain; -using Nethermind.Blockchain.BeaconBlockRoot; -using Nethermind.Blockchain.Blocks; -using Nethermind.Blockchain.Receipts; -using Nethermind.Consensus.ExecutionRequests; -using Nethermind.Consensus.Processing; -using Nethermind.Consensus.Rewards; -using Nethermind.Consensus.Validators; -using Nethermind.Consensus.Withdrawals; -using Nethermind.Core.Crypto; -using Nethermind.Core.Specs; -using Nethermind.Db; -using Nethermind.Evm.State; -using Nethermind.Evm.TransactionProcessing; -using Nethermind.Logging; -using Nethermind.State; -using Nethermind.Trie; -using Nethermind.Consensus.Stateless; -using Nethermind.Consensus; -using Nethermind.Arbitrum.Config; - -namespace Nethermind.Arbitrum.Execution.Stateless; - -public class ArbitrumStatelessBlockProcessingEnv( - ArbitrumWitness arbWitness, - IArbitrumSpecHelper arbitrumSpecHelper, - ISpecProvider specProvider, - ISealValidator sealValidator, - IStylusTargetConfig stylusTargetConfig, - IArbosVersionProvider arbosVersionProvider, - ILogManager logManager, - IArbitrumConfig config) -{ - private IBlockProcessor? _blockProcessor; - public IBlockProcessor BlockProcessor - { - get => _blockProcessor ??= GetBlockProcessor(); - } - - private IWorldState? _worldState; - public IWorldState WorldState - { - get => _worldState ??= new WorldState(new TrieStoreScopeProvider(new RawTrieStore(arbWitness.Witness.CreateNodeStorage()), arbWitness.Witness.CreateCodeDb(), logManager), logManager); - } - - private IWasmStore? _wasmStore; - public IWasmStore WasmStore - { - get => _wasmStore ??= CreateWasmStore(); - } - - private IWasmStore CreateWasmStore() - { - WasmDb wasmDb = new(new MemDb()); - WasmStore store = new(wasmDb, stylusTargetConfig, cacheTag: 1); - - // For info, pre-activation is not even needed ! - // If we omit this, wasm store will lazily load wasms from codeDB and compile them to asms when needed during execution. - // - // Btw, that's what nitro's debug execution witness endpoint might be doing (didn't see wasms passed there when I last checked). - // To check but not priority for now. - if (arbWitness.UserWasms is not null) - { - foreach ((ValueHash256 moduleHash, IReadOnlyDictionary asmMap) in arbWitness.UserWasms) - { - store.ActivateWasm(in moduleHash, asmMap); - } - } - - return store; - } - - private IBlockProcessor GetBlockProcessor() - { - StatelessBlockTree statelessBlockTree = new(arbWitness.Witness.DecodeHeaders()); - ITransactionProcessor txProcessor = CreateTransactionProcessor(WorldState, statelessBlockTree); - IBlockProcessor.IBlockTransactionsExecutor txExecutor = - new BlockProcessor.BlockValidationTransactionsExecutor( - new ExecuteTransactionProcessorAdapter(txProcessor), - WorldState); - - IHeaderValidator headerValidator = new HeaderValidator(statelessBlockTree, sealValidator, specProvider, logManager); - IBlockValidator blockValidator = new BlockValidator(new TxValidator(specProvider.ChainId), headerValidator, - new UnclesValidator(statelessBlockTree, headerValidator, logManager), specProvider, logManager); - - return new ArbitrumBlockProcessor( - specProvider, - blockValidator, - NoBlockRewards.Instance, - txExecutor, - txProcessor, - new CachedL1PriceData(logManager), - WorldState, - NullReceiptStorage.Instance, - new BlockhashStore(WorldState), - WasmStore, - new BeaconBlockRootHandler(txProcessor, WorldState), - logManager, - new WithdrawalProcessor(WorldState, logManager), - new ExecutionRequestsProcessor(txProcessor), - config - ); - } - - private ITransactionProcessor CreateTransactionProcessor(IWorldState state, StatelessBlockTree blockFinder) - { - BlockhashProvider blockhashProvider = new(blockFinder, state, logManager); - ArbitrumVirtualMachine vm = new(arbitrumSpecHelper, blockhashProvider, WasmStore, specProvider, logManager); - return new ArbitrumTransactionProcessor(BlobBaseFeeCalculator.Instance, specProvider, state, WasmStore, vm, logManager, new ArbitrumCodeInfoRepository(new EthereumCodeInfoRepository(state), arbosVersionProvider)); - } -} diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnvFactory.cs new file mode 100644 index 000000000..da23a641d --- /dev/null +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumStatelessBlockProcessingEnvFactory.cs @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md + +using Autofac; +using Nethermind.Arbitrum.Arbos; +using Nethermind.Arbitrum.Evm; +using Nethermind.Arbitrum.Precompiles; +using Nethermind.Arbitrum.Stylus; +using Nethermind.Blockchain; +using Nethermind.Blockchain.BeaconBlockRoot; +using Nethermind.Blockchain.Blocks; +using Nethermind.Blockchain.Receipts; +using Nethermind.Consensus; +using Nethermind.Consensus.ExecutionRequests; +using Nethermind.Consensus.Processing; +using Nethermind.Consensus.Rewards; +using Nethermind.Consensus.Stateless; +using Nethermind.Consensus.Validators; +using Nethermind.Consensus.Withdrawals; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.Db; +using Nethermind.Evm; +using Nethermind.Evm.State; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Logging; +using Nethermind.State; +using Nethermind.Trie; + +namespace Nethermind.Arbitrum.Execution.Stateless; + +public interface IArbitrumStatelessBlockProcessingEnvScope : IDisposable +{ + IWorldState WorldState { get; } + IBlockProcessor BlockProcessor { get; } +} + +public sealed class ArbitrumStatelessBlockProcessingEnvScope(ILifetimeScope scope) : IArbitrumStatelessBlockProcessingEnvScope +{ + public IWorldState WorldState { get; } = scope.Resolve(); + public IBlockProcessor BlockProcessor { get; } = scope.Resolve(); + + public void Dispose() => scope.Dispose(); +} + +public class ArbitrumStatelessBlockProcessingEnvFactory(ILifetimeScope rootLifetimeScope) +{ + public IArbitrumStatelessBlockProcessingEnvScope CreateScope(ArbitrumWitness witness) + { + StatelessBlockTree statelessBlockTree = new(witness.Witness.DecodeHeaders()); + + ILifetimeScope scope = rootLifetimeScope.BeginLifetimeScope(builder => + { + builder + .AddScoped(_ => statelessBlockTree) + .BindScoped() + .BindScoped() + + .AddScoped(ctx => new WorldState( + new TrieStoreScopeProvider( + new RawTrieStore(witness.Witness.CreateNodeStorage()), + witness.Witness.CreateCodeDb(), + ctx.Resolve()), + ctx.Resolve())) + + .AddScoped(stylusTargetConfig => + { + WasmDb wasmDb = new(new MemDb()); + WasmStore store = new(wasmDb, stylusTargetConfig, cacheTag: 1); + + // For info, pre-activation is not even needed ! + // If we omit this, wasm store will load wasms from codeDB and lazily compile them to asms when needed during execution. + if (witness.UserWasms is not null) + { + foreach ((ValueHash256 moduleHash, IReadOnlyDictionary asmMap) in witness.UserWasms) + store.ActivateWasm(in moduleHash, asmMap); + } + + return store; + }) + + .AddScoped() + + // Use NoOpL1BlockCache so that L1 hash lookups always go through the witness world state, + // ensuring the witness must be complete for stateless execution to succeed. + .AddScoped(_ => new NoOpL1BlockCache()) + .AddScoped() + + .AddScoped((state, versionProvider) => + new ArbitrumCodeInfoRepository( + new CodeInfoRepository(state, new EthereumPrecompileProvider()), + versionProvider)) + + .AddScoped() + + .AddScoped( + (blockTree, sealValidator, specProvider, logManager) => new HeaderValidator(blockTree, sealValidator, specProvider, logManager)) + + .AddScoped( + (blockTree, headerValidator, logManager) => new UnclesValidator(blockTree, headerValidator, logManager)) + + .AddScoped( + (headerValidator, unclesValidator, specProvider, logManager) => + new BlockValidator(new TxValidator(specProvider.ChainId), headerValidator, unclesValidator, specProvider, logManager)) + + .AddScoped(_ => NoBlockRewards.Instance) + .AddScoped(_ => NullReceiptStorage.Instance) + .AddScoped(logManager => new CachedL1PriceData(logManager)) + .AddScoped(worldState => new BlockhashStore(worldState)) + + .AddScoped( + (txProcessor, worldState) => new BeaconBlockRootHandler(txProcessor, worldState)) + + .AddScoped( + (worldState, logManager) => new WithdrawalProcessor(worldState, logManager)) + + .AddScoped( + txProcessor => new ExecutionRequestsProcessor(txProcessor)) + + .AddScoped( + (txProcessor, worldState) => new BlockProcessor.BlockValidationTransactionsExecutor( + new ExecuteTransactionProcessorAdapter(txProcessor), + worldState)) + + .AddScoped(); + }); + + return new ArbitrumStatelessBlockProcessingEnvScope(scope); + } +} From 9b39076158df2ad8729cd039958f3ab4df0744d1 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 9 Mar 2026 18:44:01 +0800 Subject: [PATCH 61/87] fix: Remove useless declaration --- .../ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index 61a5be85b..a7b8d059e 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -77,11 +77,9 @@ public IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTarge ILifetimeScope envLifetimeScope = rootLifetimeScope.BeginLifetimeScope((builder) => { if (wasmTargets is not null) - { + // No need to redeclare IWasmStore because it is now declared as scoped + // and therefore a new instance will be created in this child scope with the correct dependencies builder.AddScoped(_ => new StylusTargetConfig() { OverrideWasmTargets = wasmTargets }); - // Need to redeclare IWasmStore because it was originally declared as a singleton and therefore was cached with original IStylusTargetConfig - builder.AddScoped((db, config) => new WasmStore(db, config, cacheTag: 1)); - } builder .AddScoped(stateReader) From be46bc23af001f00a2006a9b66c83250681cd365 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 9 Mar 2026 18:53:55 +0800 Subject: [PATCH 62/87] fix: Do not update branch --- src/Nethermind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind b/src/Nethermind index e98856ba6..e7d864787 160000 --- a/src/Nethermind +++ b/src/Nethermind @@ -1 +1 @@ -Subproject commit e98856ba66d7c32e3619d8b93206547ac597c737 +Subproject commit e7d8647878000b609daeb2948af3f00b62ca8de1 From c7c9607938ebeeca01c2832a68aebacc5043cc55 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 9 Mar 2026 20:08:17 +0800 Subject: [PATCH 63/87] fix Build --- .../Evm/ArbitrumEvmInstructions.Environment.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum/Evm/ArbitrumEvmInstructions.Environment.cs b/src/Nethermind.Arbitrum/Evm/ArbitrumEvmInstructions.Environment.cs index 24736ac8c..549c41160 100644 --- a/src/Nethermind.Arbitrum/Evm/ArbitrumEvmInstructions.Environment.cs +++ b/src/Nethermind.Arbitrum/Evm/ArbitrumEvmInstructions.Environment.cs @@ -159,7 +159,7 @@ public static EvmExceptionType InstructionExtCodeSize( { IReleaseSpec spec = vm.Spec; // Deduct the gas cost for external code access. - TGasPolicy.Consume(ref gas, spec.GetExtCodeCost()); + TGasPolicy.Consume(ref gas, spec.GasCosts.ExtCodeCost); // Pop the account address from the stack. Address? address = stack.PopAddress(); From 35d12a82e33b203e16168c6017b9c0ba23f8a90b Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 10 Mar 2026 13:17:06 +0800 Subject: [PATCH 64/87] fix tests --- .../ArbitrumWitnessGenerationTests.cs | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index ab1b14060..4dbc80498 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -168,7 +168,7 @@ public async Task RecordBlockCreation_ExtCodeSizeFollowedByIsZero_StillRecordsTa l1BaseFee, sender, sender, - 100.Ether()); + 100.Ether); ResultWrapper depositResult = await chain.Digest(deposit); depositResult.Result.Should().Be(Result.Success); @@ -196,7 +196,7 @@ public async Task RecordBlockCreation_ExtCodeSizeFollowedByIsZero_StillRecordsTa .WithType(TxType.EIP1559) .WithTo(null) // Contract creation .WithData(targetInitCode) - .WithMaxFeePerGas(10.GWei()) + .WithMaxFeePerGas(10.GWei) .WithGasLimit(500_000) .WithValue(0) .WithNonce(chain.MainWorldState.GetNonce(sender)) @@ -236,7 +236,7 @@ public async Task RecordBlockCreation_ExtCodeSizeFollowedByIsZero_StillRecordsTa .WithType(TxType.EIP1559) .WithTo(null) // Contract creation .WithData(callerInitCode) - .WithMaxFeePerGas(10.GWei()) + .WithMaxFeePerGas(10.GWei) .WithGasLimit(500_000) .WithValue(0) .WithNonce(chain.MainWorldState.GetNonce(sender)) @@ -258,7 +258,7 @@ public async Task RecordBlockCreation_ExtCodeSizeFollowedByIsZero_StillRecordsTa .WithType(TxType.EIP1559) .WithTo(callerAddress) .WithData([]) - .WithMaxFeePerGas(10.GWei()) + .WithMaxFeePerGas(10.GWei) .WithGasLimit(500_000) .WithValue(0) .WithNonce(chain.MainWorldState.GetNonce(sender)) @@ -321,7 +321,7 @@ public async Task RecordBlockCreation_PrecompileCalls_RecordsArbitrumPrecompileC l1BaseFee, sender, sender, - 100.Ether()); + 100.Ether); ResultWrapper depositResult = await chain.Digest(deposit); depositResult.Result.Should().Be(Result.Success); @@ -344,7 +344,7 @@ public async Task RecordBlockCreation_PrecompileCalls_RecordsArbitrumPrecompileC .WithType(TxType.EIP1559) .WithTo(arbSysAddress) .WithData(arbBlockNumberCalldata) - .WithMaxFeePerGas(10.GWei()) + .WithMaxFeePerGas(10.GWei) .WithGasLimit(100_000) .WithValue(0) .WithNonce(chain.MainWorldState.GetNonce(sender)) @@ -356,7 +356,7 @@ public async Task RecordBlockCreation_PrecompileCalls_RecordsArbitrumPrecompileC .WithType(TxType.EIP1559) .WithTo(ecrecoverAddress) .WithData(new byte[128]) // ecrecover expects 128 bytes (hash, v, r, s) - .WithMaxFeePerGas(10.GWei()) + .WithMaxFeePerGas(10.GWei) .WithGasLimit(100_000) .WithValue(0) .WithNonce(chain.MainWorldState.GetNonce(sender) + 1) @@ -424,7 +424,7 @@ public async Task RecordBlockCreation_BlockHashOpcode_RecordsStorageTrieNodeInWi l1BaseFee, sender, sender, - 100.Ether()); + 100.Ether); ResultWrapper depositResult = await chain.Digest(deposit); depositResult.Result.Should().Be(Result.Success); @@ -458,7 +458,7 @@ public async Task RecordBlockCreation_BlockHashOpcode_RecordsStorageTrieNodeInWi .WithType(TxType.EIP1559) .WithTo(null) // Contract creation .WithData(blockhashCallerInitCode) - .WithMaxFeePerGas(10.GWei()) + .WithMaxFeePerGas(10.GWei) .WithGasLimit(500_000) .WithValue(0) .WithNonce(chain.MainWorldState.GetNonce(sender)) @@ -480,7 +480,7 @@ public async Task RecordBlockCreation_BlockHashOpcode_RecordsStorageTrieNodeInWi .WithType(TxType.EIP1559) .WithTo(contractAddress) .WithData([]) - .WithMaxFeePerGas(10.GWei()) + .WithMaxFeePerGas(10.GWei) .WithGasLimit(100_000) .WithValue(0) .WithNonce(chain.MainWorldState.GetNonce(sender)) @@ -539,7 +539,7 @@ public async Task RecordBlockCreation_ArbBlockHash_RecordsHeadersInWitness() .WithType(TxType.EIP1559) .WithTo(ArbSys.Address) .WithData(calldata) - .WithMaxFeePerGas(10.GWei()) + .WithMaxFeePerGas(10.GWei) .WithGasLimit(100_000) .WithValue(0) .WithNonce(chain.MainWorldState.GetNonce(sender)) @@ -604,9 +604,9 @@ public async Task RecordBlockCreation_SubmitRetryableWithEmptyCalldata_RecordsCa sender, receiver, beneficiary, - DepositValue: 10.Ether(), - RetryValue: 1.Ether(), - GasFee: 1.GWei(), + DepositValue: 10.Ether, + RetryValue: 1.Ether, + GasFee: 1.GWei, GasLimit: 0, MaxSubmissionFee: 128800); @@ -703,9 +703,9 @@ public async Task RecordBlockCreation_TryReapRetryableNotExpired_RecordsTimeoutW sender, TestItem.AddressA, TestItem.AddressB, - DepositValue: 10.Ether(), - RetryValue: 1.Ether(), - GasFee: 1.GWei(), + DepositValue: 10.Ether, + RetryValue: 1.Ether, + GasFee: 1.GWei, GasLimit: 0, MaxSubmissionFee: 128800); @@ -764,7 +764,7 @@ public async Task RecordBlockCreation_TryReapRetryableNotExpired_RecordsTimeoutW .WithType(TxType.EIP1559) .WithTo(TestItem.AddressC) .WithData([]) - .WithMaxFeePerGas(1.GWei()) + .WithMaxFeePerGas(1.GWei) .WithGasLimit(21_000) .WithValue(1) .WithNonce(chain.MainWorldState.GetNonce(sender)) @@ -844,7 +844,7 @@ public async Task RecordBlockCreation_WhenStorageSlotModifiedAndResetInSameBlock // Fund the sender account ResultWrapper depositResult = await chain.Digest(new TestEthDeposit( - Keccak.Compute("deposit"), l1BaseFee, sender, sender, 100.Ether())); + Keccak.Compute("deposit"), l1BaseFee, sender, sender, 100.Ether)); depositResult.Result.Should().Be(Result.Success); // Deploy a simple setter contract: SSTORE(slot=0, value=CALLDATALOAD(0)) @@ -873,7 +873,7 @@ public async Task RecordBlockCreation_WhenStorageSlotModifiedAndResetInSameBlock .WithType(TxType.EIP1559) .WithTo(null) // contract creation .WithData(setterInitCode) - .WithMaxFeePerGas(10.GWei()) + .WithMaxFeePerGas(10.GWei) .WithGasLimit(500_000) .WithValue(0) .WithNonce(chain.MainWorldState.GetNonce(sender)) @@ -906,7 +906,7 @@ public async Task RecordBlockCreation_WhenStorageSlotModifiedAndResetInSameBlock .WithType(TxType.EIP1559) .WithTo(contractAddress) .WithData(setTo2) - .WithMaxFeePerGas(10.GWei()) + .WithMaxFeePerGas(10.GWei) .WithGasLimit(100_000) .WithValue(0) .WithNonce(nonce) @@ -917,7 +917,7 @@ public async Task RecordBlockCreation_WhenStorageSlotModifiedAndResetInSameBlock .WithType(TxType.EIP1559) .WithTo(contractAddress) .WithData(setToInitialValue) - .WithMaxFeePerGas(10.GWei()) + .WithMaxFeePerGas(10.GWei) .WithGasLimit(100_000) .WithValue(0) .WithNonce(nonce + 1) @@ -988,7 +988,7 @@ public async Task RecordBlockCreation_WhenStateModifiedAndResetDirectlyViaWorldS l1BaseFee, sender, sender, - 100.Ether()); + 100.Ether); ResultWrapper depositResult = await chain.Digest(deposit); depositResult.Result.Should().Be(Result.Success); @@ -1027,7 +1027,7 @@ public async Task RecordBlockCreation_WhenStateModifiedAndResetDirectlyViaWorldS .WithType(TxType.EIP1559) .WithTo(ArbosAddresses.ArbOwnerAddress) .WithData(setToNewCalldata) - .WithMaxFeePerGas(10.GWei()) + .WithMaxFeePerGas(10.GWei) .WithGasLimit(500_000) .WithValue(0) .WithNonce(nonce) @@ -1038,7 +1038,7 @@ public async Task RecordBlockCreation_WhenStateModifiedAndResetDirectlyViaWorldS .WithType(TxType.EIP1559) .WithTo(ArbosAddresses.ArbOwnerAddress) .WithData(resetToOriginalCalldata) - .WithMaxFeePerGas(10.GWei()) + .WithMaxFeePerGas(10.GWei) .WithGasLimit(500_000) .WithValue(0) .WithNonce(nonce + 1) @@ -1122,7 +1122,7 @@ public async Task RecordBlockCreation_TransactionSetsSomeStateButReverts_StillRe l1BaseFee, sender, sender, - 100.Ether()); + 100.Ether); ResultWrapper depositResult = await chain.Digest(deposit); depositResult.Result.Should().Be(Result.Success); @@ -1169,7 +1169,7 @@ public async Task RecordBlockCreation_TransactionSetsSomeStateButReverts_StillRe .WithType(TxType.EIP1559) .WithTo(ArbosAddresses.ArbAddressTableAddress) .WithData(calldata) - .WithMaxFeePerGas(10.GWei()) + .WithMaxFeePerGas(10.GWei) .WithGasLimit(gasLimit) // Not enough gas, causing revert .WithValue(0) .WithNonce(chain.MainWorldState.GetNonce(sender)) From 9817438ca0401ce995c3bf02aec1e76cc5670ea4 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Thu, 19 Feb 2026 13:19:09 +0900 Subject: [PATCH 65/87] feat: Support state reconstruction --- src/Nethermind.Arbitrum/ArbitrumPlugin.cs | 4 + .../Execution/ArbitrumExecutionEngine.cs | 3 + ...nessGeneratingBlockProcessingEnvFactory.cs | 7 +- .../Stateless/ReconstructedStateTrieStore.cs | 52 +++++ .../Execution/Stateless/StateReconstructor.cs | 208 ++++++++++++++++++ 5 files changed, 270 insertions(+), 4 deletions(-) create mode 100644 src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs create mode 100644 src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs diff --git a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs index 1b42f77b9..57c52fe43 100644 --- a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs +++ b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs @@ -47,6 +47,7 @@ using Nethermind.Arbitrum.Tracing; using Nethermind.Blockchain.Tracing.GethStyle.Custom.Native; using Nethermind.State; +using Nethermind.Trie.Pruning; namespace Nethermind.Arbitrum; @@ -294,6 +295,9 @@ protected override void Load(ContainerBuilder builder) .AddSingleton() .Bind, ArbitrumEthModuleFactory>() + // Execution recording (state reconstruction + witness generation) + .AddSingleton(ctx => new ReconstructedStateTrieStore(new Db.MemDb(), ctx.Resolve())) + .AddSingleton() .AddSingleton() .Bind() diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs index f59824b38..79f5077d8 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs @@ -40,6 +40,7 @@ public sealed class ArbitrumExecutionEngine( IBlockProcessingQueue processingQueue, IArbitrumConfig arbitrumConfig, IArbitrumWitnessGeneratingBlockProcessingEnvFactory witnessGeneratingBlockProcessingEnvFactory, + StateReconstructor stateReconstructor, IBlocksConfig blocksConfig) : IArbitrumExecutionEngine { @@ -534,6 +535,8 @@ public async Task> RecordBlockCreation(RecordBlockCr Number = blockNumber }; + stateReconstructor.EnsureStateAvailable(parent); + string[] wasmTargets = parameters.WasmTargets; string localTarget = StylusTargets.GetLocalTargetName(); if (!wasmTargets.Contains(localTarget)) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index a7b8d059e..2be4aa374 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -15,7 +15,6 @@ using Nethermind.Evm.State; using Nethermind.Logging; using Nethermind.State; -using Nethermind.Trie.Pruning; using static Nethermind.Arbitrum.Execution.ArbitrumBlockProcessor; using Nethermind.Arbitrum.Evm; using Nethermind.Arbitrum.Precompiles; @@ -37,7 +36,7 @@ public interface IArbitrumWitnessGeneratingBlockProcessingEnvFactory : IWitnessG public class ArbitrumWitnessGeneratingBlockProcessingEnvFactory( ILifetimeScope rootLifetimeScope, - IReadOnlyTrieStore readOnlyTrieStore, + ReconstructedStateTrieStore reconstructedStateTrieStore, IDbProvider dbProvider, ILogManager logManager) : IArbitrumWitnessGeneratingBlockProcessingEnvFactory { @@ -68,7 +67,7 @@ private static BlocksConfig CreateWitnessBlocksConfig(IBlocksConfig blocksConfig public IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTargets) { IReadOnlyDbProvider readOnlyDbProvider = new ReadOnlyDbProvider(dbProvider, true); - WitnessCapturingTrieStore trieStore = new(readOnlyDbProvider.StateDb, readOnlyTrieStore); + WitnessCapturingTrieStore trieStore = new(reconstructedStateTrieStore); IStateReader stateReader = new StateReader(trieStore, readOnlyDbProvider.CodeDb, logManager); WorldState worldState = new(new TrieStoreScopeProvider(trieStore, readOnlyDbProvider.CodeDb, logManager), logManager); @@ -127,6 +126,7 @@ public IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTarge .AddScoped() // 1st: add the tx executor + .AddScoped() .AddScoped() // 2nd: add block processor @@ -136,7 +136,6 @@ public IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTarge // 3rd: configure the builder for block production (like ArbitrumBlockProducerEnvFactory but with my own witness capturing world state) .AddScoped(factory => factory.Create()) - .AddScoped() .AddDecorator() .AddDecorator() .AddScoped() diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs new file mode 100644 index 000000000..e22c115f1 --- /dev/null +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs @@ -0,0 +1,52 @@ +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Trie; +using Nethermind.Trie.Pruning; + +namespace Nethermind.Arbitrum.Execution.Stateless; + +/// +/// Overlay trie store for state reconstruction. Reconstructed trie nodes are stored in a MemDb overlay +/// and fall back to the base store (main TrieStore dirty cache + disk) for reads. +/// BeginScope is a no-op to avoid acquiring the main TrieStore's scope/pruning locks during +/// potentially long-running state reconstruction. +/// +public class ReconstructedStateTrieStore(IKeyValueStoreWithBatching keyValueStore, IReadOnlyTrieStore baseStore) : ITrieStore, IReadOnlyTrieStore +{ + private readonly INodeStorage _nodeStorage = new NodeStorage(keyValueStore); + + public void Dispose() + { + } + + public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 hash) + => baseStore.FindCachedOrUnknown(address, in path, hash); + + public byte[]? LoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) + { + byte[]? rlp = TryLoadRlp(address, in path, hash, flags); + if (rlp is null) + throw new MissingTrieNodeException("Missing RLP node", address, path, hash); + return rlp; + } + + public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) + => _nodeStorage.Get(address, in path, hash, flags) ?? baseStore.TryLoadRlp(address, in path, hash, flags); + + public bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak) + => _nodeStorage.Get(address, in path, in keccak) is not null || baseStore.IsPersisted(address, in path, in keccak); + + public bool HasRoot(Hash256 stateRoot) + => _nodeStorage.Get(null, TreePath.Empty, stateRoot) is not null || baseStore.HasRoot(stateRoot); + + public IDisposable BeginScope(BlockHeader? baseBlock) => new Reactive.AnonymousDisposable(() => { }); + + public IScopedTrieStore GetTrieStore(Hash256? address) => new ScopedTrieStore(this, address); + + public INodeStorage.KeyScheme Scheme => baseStore.Scheme; + + public IBlockCommitter BeginBlockCommit(long blockNumber) => NullCommitter.Instance; + + public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) + => new RawScopedTrieStore.Committer(_nodeStorage, address, writeFlags); +} diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs b/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs new file mode 100644 index 000000000..ea8bf07d7 --- /dev/null +++ b/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs @@ -0,0 +1,208 @@ +using Autofac; +using Nethermind.Arbitrum.Arbos; +using Nethermind.Arbitrum.Config; +using Nethermind.Arbitrum.Evm; +using Nethermind.Arbitrum.Precompiles; +using Nethermind.Arbitrum.Stylus; +using Nethermind.Blockchain; +using Nethermind.Blockchain.Headers; +using Nethermind.Blockchain.Receipts; +using Nethermind.Blockchain.Tracing; +using Nethermind.Config; +using Nethermind.Consensus.Processing; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.Db; +using Nethermind.Evm; +using Nethermind.Evm.State; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Logging; +using Nethermind.State; + +namespace Nethermind.Arbitrum.Execution.Stateless; + +public class StateReconstructor +{ + private readonly ReconstructedStateTrieStore _trieStore; + private readonly IBlockTree _blockTree; + private readonly ILifetimeScope _rootLifetimeScope; + private readonly ILogManager _logManager; + private readonly ILogger _logger; + private readonly long _genesisBlockNumber; + + public StateReconstructor( + ReconstructedStateTrieStore trieStore, + IBlockTree blockTree, + ILifetimeScope rootLifetimeScope, + IArbitrumSpecHelper specHelper, + ILogManager logManager) + { + _trieStore = trieStore; + _blockTree = blockTree; + _rootLifetimeScope = rootLifetimeScope; + _logManager = logManager; + _logger = logManager.GetClassLogger(); + _genesisBlockNumber = (long)specHelper.GenesisBlockNum; + } + + /// + /// Ensures the state for the given parent header is available in the ReconstructedStateTrieStore. + /// If unavailable, walks backward to find the nearest available state and re-executes blocks forward. + /// + public void EnsureStateAvailable(BlockHeader targetParent) + { + if (_trieStore.HasRoot(targetParent.StateRoot!)) + { + if (_logger.IsDebug) + _logger.Debug($"State already available for block {targetParent.Number} (root {targetParent.StateRoot})"); + return; + } + + if (_logger.IsInfo) + _logger.Info($"State not available for block {targetParent.Number} (root {targetParent.StateRoot}), reconstructing..."); + + BlockHeader lastAvailable = FindLastAvailableState(targetParent); + + if (_logger.IsInfo) + _logger.Info($"Found available state at block {lastAvailable.Number} (root {lastAvailable.StateRoot}), re-executing {targetParent.Number - lastAvailable.Number} blocks forward"); + + ReExecuteBlocks(lastAvailable, targetParent); + + if (!_trieStore.HasRoot(targetParent.StateRoot!)) + throw new InvalidOperationException($"State reconstruction failed: root {targetParent.StateRoot} not available after re-execution"); + } + + /// + // For recreating state, this method walks backwards from the target header until it finds a header + // whose state root is available in the RecordingTrieStore or otherwise reaches genesis and throws there. + /// + private BlockHeader FindLastAvailableState(BlockHeader target) + { + BlockHeader current = target; + + while (true) + { + if (_trieStore.HasRoot(current.StateRoot!)) + return current; + + if (current.Number <= _genesisBlockNumber) + throw new InvalidOperationException($"Reached genesis (block {_genesisBlockNumber}) without finding available state while looking for block {target.Number}"); + + BlockHeader? parent = _blockTree.FindHeader(current.ParentHash!, BlockTreeLookupOptions.RequireCanonical, current.Number - 1); + if (parent is null) + throw new InvalidOperationException($"Cannot find header for block {current.Number - 1} during state reconstruction"); + + current = parent; + } + } + + private void ReExecuteBlocks(BlockHeader lastAvailable, BlockHeader targetParent) + { + long startBlock = lastAvailable.Number + 1; + long endBlock = targetParent.Number; + + IBlocksConfig blocksConfig = _rootLifetimeScope.Resolve(); + // Not necessary to write codeDB in read only given writes to it are idempotent + WorldState worldState = new( + new TrieStoreScopeProvider(_trieStore, _rootLifetimeScope.Resolve().CodeDb, _logManager), + _logManager); + + using ILifetimeScope scope = _rootLifetimeScope.BeginLifetimeScope(builder => + { + builder + .AddScoped(_ => worldState) + .AddScoped(_ => CreateReconstructionBlocksConfig(blocksConfig)) + + .AddScoped(ctx => CreateTransactionProcessor( + ctx.Resolve(), + ctx.Resolve(), + ctx.Resolve(), + ctx.Resolve(), + worldState, + ctx.Resolve())) + + .AddScoped(ctx => new BlockProcessor.BlockValidationTransactionsExecutor( + new BuildUpTransactionProcessorAdapter(ctx.Resolve()), + worldState)) + + .AddScoped(NullReceiptStorage.Instance) + .AddScoped(BlockchainProcessor.Options.NoReceipts) + .AddScoped(); + }); + + IBlockProcessor blockProcessor = scope.Resolve(); + ISpecProvider specProvider = scope.Resolve(); + + using (worldState.BeginScope(lastAvailable)) + { + Hash256 expectedParentHash = lastAvailable.Hash!; + + for (long blockNumber = startBlock; blockNumber <= endBlock; blockNumber++) + { + Block? block = _blockTree.FindBlock(blockNumber, BlockTreeLookupOptions.RequireCanonical); + if (block is null) + throw new InvalidOperationException($"Cannot find block {blockNumber} during state reconstruction"); + + if (block.ParentHash != expectedParentHash) + throw new InvalidOperationException( + $"Parent hash mismatch at block {blockNumber}: expected {expectedParentHash}, got {block.ParentHash}"); + + Hash256 expectedBlockHash = block.Hash!; + IReleaseSpec spec = specProvider.GetSpec(block.Header); + (Block processedBlock, _) = blockProcessor.ProcessOne(block, ProcessingOptions.ForceProcessing, NullBlockTracer.Instance, spec); + + if (processedBlock.Hash != expectedBlockHash) + throw new InvalidOperationException( + $"Block hash mismatch after re-execution of block {blockNumber}: expected {expectedBlockHash}, got {processedBlock.Hash}"); + + worldState.CommitTree(block.Number); + worldState.Reset(); + + expectedParentHash = processedBlock.Hash!; + + if (_logger.IsDebug && blockNumber % 100 == 0) + _logger.Debug($"State reconstruction progress: {blockNumber - startBlock + 1}/{endBlock - startBlock + 1} blocks"); + } + } + + if (_logger.IsInfo) + _logger.Info($"State reconstruction complete: re-executed {endBlock - startBlock + 1} blocks ({startBlock} to {endBlock})"); + } + + private ITransactionProcessor CreateTransactionProcessor( + IArbitrumSpecHelper arbitrumSpecHelper, + IWasmStore wasmStore, + ISpecProvider specProvider, + IArbosVersionProvider arbosVersionProvider, + IWorldState state, + IHeaderFinder headerFinder) + { + BlockhashProvider blockhashProvider = new(new BlockhashCache(headerFinder, _logManager), state, _logManager); + ArbitrumVirtualMachine vm = new(arbitrumSpecHelper, blockhashProvider, wasmStore, specProvider, _logManager); + + return new ArbitrumTransactionProcessor( + BlobBaseFeeCalculator.Instance, specProvider, state, wasmStore, vm, _logManager, + new ArbitrumCodeInfoRepository(new CodeInfoRepository(state, new EthereumPrecompileProvider()), arbosVersionProvider)); + } + + private static BlocksConfig CreateReconstructionBlocksConfig(IBlocksConfig blocksConfig) + => new() + { + TargetBlockGasLimit = blocksConfig.TargetBlockGasLimit, + MinGasPrice = blocksConfig.MinGasPrice, + RandomizedBlocks = blocksConfig.RandomizedBlocks, + ExtraData = blocksConfig.ExtraData, + SecondsPerSlot = blocksConfig.SecondsPerSlot, + SingleBlockImprovementOfSlot = blocksConfig.SingleBlockImprovementOfSlot, + PreWarmStateOnBlockProcessing = false, + CachePrecompilesOnBlockProcessing = blocksConfig.CachePrecompilesOnBlockProcessing, + PreWarmStateConcurrency = blocksConfig.PreWarmStateConcurrency, + BlockProductionTimeoutMs = blocksConfig.BlockProductionTimeoutMs, + GenesisTimeoutMs = blocksConfig.GenesisTimeoutMs, + BlockProductionMaxTxKilobytes = blocksConfig.BlockProductionMaxTxKilobytes, + GasToken = blocksConfig.GasToken, + BlockProductionBlobLimit = blocksConfig.BlockProductionBlobLimit, + BuildBlocksOnMainState = false, + }; +} From 3de943f8b29606ca3279d2eeba363c025eb44f91 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Thu, 19 Feb 2026 13:20:09 +0900 Subject: [PATCH 66/87] feat: Support PrepareForRecord --- .../Data/DigestMessageParameters.cs | 5 +++ .../Execution/ArbitrumExecutionEngine.cs | 36 +++++++++++++++++++ .../ArbitrumExecutionEngineWithComparison.cs | 3 ++ .../Execution/IArbitrumExecutionEngine.cs | 1 + .../Modules/ArbitrumRpcModule.cs | 3 ++ .../Modules/IArbitrumRpcModule.cs | 3 ++ 6 files changed, 51 insertions(+) diff --git a/src/Nethermind.Arbitrum/Data/DigestMessageParameters.cs b/src/Nethermind.Arbitrum/Data/DigestMessageParameters.cs index 2b694092e..bc6a480f9 100644 --- a/src/Nethermind.Arbitrum/Data/DigestMessageParameters.cs +++ b/src/Nethermind.Arbitrum/Data/DigestMessageParameters.cs @@ -64,3 +64,8 @@ public record RecordBlockCreationParameters( [property: JsonPropertyName("message")] MessageWithMetadata Message, [property: JsonPropertyName("wasmTargets")] string[] WasmTargets ); + +public record PrepareForRecordParameters( + [property: JsonPropertyName("start")] ulong Start, + [property: JsonPropertyName("end")] ulong End +); diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs index 79f5077d8..c31a65ffa 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs @@ -588,6 +588,42 @@ void OnBlockAddedToMain(object? sender, BlockReplacementEventArgs e) } } + public ResultWrapper PrepareForRecord(PrepareForRecordParameters parameters) + { + if (parameters.End < parameters.Start) + return ResultWrapper.Fail($"Invalid range: start {parameters.Start} > end {parameters.End}"); + + ulong numOfBlocks = parameters.End + 1 - parameters.Start; + long headerNum = MessageIndexToBlockNumber(parameters.Start).Data; + if (parameters.Start > 0) + headerNum--; // need to get previous as RecordBlockCreation executes from the parent block's state + else + numOfBlocks--; // genesis block doesn't need preparation, so recording one less block + + long lastHeaderNum = headerNum + (long)numOfBlocks; + for (long current = headerNum; current <= lastHeaderNum; current++) + { + BlockHeader? header = BlockTree.FindHeader(current); + if (header is null) + { + _logger.Warn($"PrepareForRecord: header not found for block {current}"); + break; + } + + try + { + stateReconstructor.EnsureStateAvailable(header); + } + catch (Exception ex) + { + _logger.Warn($"PrepareForRecord: failed to ensure state for block {current}: {ex.Message}"); + break; + } + } + + return ResultWrapper.Success(default); + } + private Hash256 GetSendRootFromBlock(Block block) { ArbitrumBlockHeaderInfo headerInfo = ArbitrumBlockHeaderInfo.Deserialize(block.Header, _logger); diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngineWithComparison.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngineWithComparison.cs index 9b4ad5b9e..3249b7938 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngineWithComparison.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngineWithComparison.cs @@ -282,4 +282,7 @@ private void TriggerGracefulShutdown(string reason) public Task> RecordBlockCreation(RecordBlockCreationParameters parameters) => innerEngine.RecordBlockCreation(parameters); + + public ResultWrapper PrepareForRecord(PrepareForRecordParameters parameters) + => innerEngine.PrepareForRecord(parameters); } diff --git a/src/Nethermind.Arbitrum/Execution/IArbitrumExecutionEngine.cs b/src/Nethermind.Arbitrum/Execution/IArbitrumExecutionEngine.cs index 37d030d8b..cc4da3038 100644 --- a/src/Nethermind.Arbitrum/Execution/IArbitrumExecutionEngine.cs +++ b/src/Nethermind.Arbitrum/Execution/IArbitrumExecutionEngine.cs @@ -28,4 +28,5 @@ public interface IArbitrumExecutionEngine ResultWrapper> FullSyncProgressMap(); Task> ArbOSVersionForMessageIndexAsync(ulong messageIndex); Task> RecordBlockCreation(RecordBlockCreationParameters parameters); + ResultWrapper PrepareForRecord(PrepareForRecordParameters parameters); } diff --git a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs index 1f30c18b4..c583eab88 100644 --- a/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs +++ b/src/Nethermind.Arbitrum/Modules/ArbitrumRpcModule.cs @@ -79,4 +79,7 @@ public Task> MaintenanceStatus() public Task> RecordBlockCreation(RecordBlockCreationParameters parameters) => engine.RecordBlockCreation(parameters); + + public ResultWrapper PrepareForRecord(PrepareForRecordParameters parameters) + => engine.PrepareForRecord(parameters); } diff --git a/src/Nethermind.Arbitrum/Modules/IArbitrumRpcModule.cs b/src/Nethermind.Arbitrum/Modules/IArbitrumRpcModule.cs index a49bea7fd..6c9779820 100644 --- a/src/Nethermind.Arbitrum/Modules/IArbitrumRpcModule.cs +++ b/src/Nethermind.Arbitrum/Modules/IArbitrumRpcModule.cs @@ -58,5 +58,8 @@ public interface IArbitrumRpcModule : IRpcModule [JsonRpcMethod(IsSharable = false, IsImplemented = true)] Task> RecordBlockCreation(RecordBlockCreationParameters parameters); + + [JsonRpcMethod(IsSharable = false, IsImplemented = true)] + ResultWrapper PrepareForRecord(PrepareForRecordParameters parameters); } } From 3bacaaf0870c923aa55afa999409c9c31396f06c Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Thu, 19 Feb 2026 13:22:04 +0900 Subject: [PATCH 67/87] tests: RecordBlockCreation with state reconstruction and PrepareForRecord --- .../Stateless/StateReconstructorTests.cs | 329 ++++++++++++++++++ .../ArbitrumRpcTestBlockchain.cs | 6 + .../ArbitrumTestBlockchainBase.cs | 2 + .../ArbitrumTestBlockchainBuilder.cs | 10 +- .../RecordingTests.cs | 4 +- .../ArbitrumRpcModuleTests.DigestMessage.cs | 2 +- .../Rpc/ArbitrumRpcModuleTests.cs | 6 + 7 files changed, 356 insertions(+), 3 deletions(-) create mode 100644 src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs new file mode 100644 index 000000000..c2452ea44 --- /dev/null +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs @@ -0,0 +1,329 @@ +using Autofac; +using FluentAssertions; +using Nethermind.Arbitrum.Data; +using Nethermind.Arbitrum.Execution.Stateless; +using Nethermind.Arbitrum.Test.Infrastructure; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Db; +using Nethermind.JsonRpc; +using Nethermind.Trie; +using Nethermind.Trie.Pruning; + +namespace Nethermind.Arbitrum.Test.Execution.Stateless; + +public class StateReconstructorTests +{ + private const string RecordingPath = "./Recordings/1__arbos32_basefee92.jsonl"; + + [Test] + public async Task RecordBlockCreation_WithFullyPrunedState_ReconstructsStateFromGenesis() + { + SwitchableReadOnlyTrieStore switchableStore = new(); + using ArbitrumRpcTestBlockchain chain = BuildChainWithRecording(switchableStore); + + DigestMessageParameters lastDigestMessage = GetLastDigestedMessage(); + long headNumber = (long)lastDigestMessage.Index; + + // Switch to pruned mode — only genesis root is "available" + Hash256 genesisStateRoot = chain.BlockTree.FindHeader((long)chain.GenesisBlockNumber)!.StateRoot!; + switchableStore.EnablePruning(new HashSet { genesisStateRoot }); + + // Verify ALL non-genesis state roots are NOT available before reconstruction + ReconstructedStateTrieStore trieStore = chain.Container.Resolve(); + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= headNumber; blockNum++) + { + BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; + if (blockNum == (long)chain.GenesisBlockNumber) + trieStore.HasRoot(header.StateRoot!).Should().BeTrue( + $"genesis state root should be available before reconstruction"); + else + trieStore.HasRoot(header.StateRoot!).Should().BeFalse( + $"state root for block {blockNum} should not be available before reconstruction"); + } + + // RecordBlockCreation triggers state reconstruction from the nearest available state (genesis, in this case) + ResultWrapper recordResult = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(lastDigestMessage.Index, lastDigestMessage.Message, WasmTargets: [])); + + recordResult.Result.Should().Be(Result.Success); + recordResult.Data.BlockHash.Should().Be(new Hash256(RecordingTests.Block18Hash)); + recordResult.Data.Witness.Should().NotBeNull(); + + // ALL state roots from genesis to head should now be available + // StateReconstructor reconstructed until head-1, and RecordBlockCreation should have reconstructed the head block's state as well + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= headNumber; blockNum++) + { + BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; + trieStore.HasRoot(header.StateRoot!).Should().BeTrue( + $"state root for block {blockNum} should be available after reconstruction"); + } + } + + [Test] + public async Task RecordBlockCreation_WithPartiallyPrunedState_ReconstructsStateFromNearestAvailable() + { + SwitchableReadOnlyTrieStore switchableStore = new(); + using ArbitrumRpcTestBlockchain chain = BuildChainWithRecording(switchableStore); + + DigestMessageParameters lastDigestMessage = GetLastDigestedMessage(); + long headNumber = (long)lastDigestMessage.Index; + + // Switch to pruned mode — genesis and an intermediate block are available + Hash256 genesisStateRoot = chain.BlockTree.FindHeader((long)chain.GenesisBlockNumber)!.StateRoot!; + long intermediateBlockNumber = (long)chain.GenesisBlockNumber + 7; + Hash256 intermediateStateRoot = chain.BlockTree.FindHeader(intermediateBlockNumber)!.StateRoot!; + switchableStore.EnablePruning(new HashSet { genesisStateRoot, intermediateStateRoot }); + + // Verify state roots except for genesis and the intermediate block are NOT available before reconstruction + ReconstructedStateTrieStore trieStore = chain.Container.Resolve(); + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= headNumber; blockNum++) + { + BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; + if (blockNum == (long)chain.GenesisBlockNumber || blockNum == intermediateBlockNumber) + trieStore.HasRoot(header.StateRoot!).Should().BeTrue( + $"genesis and intermediate state roots only should be available before reconstruction"); + else + trieStore.HasRoot(header.StateRoot!).Should().BeFalse( + $"state root for block {blockNum} should not be available before reconstruction"); + } + + // RecordBlockCreation should reconstruct from the intermediate block, not genesis + ResultWrapper recordResult = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(lastDigestMessage.Index, lastDigestMessage.Message, WasmTargets: [])); + + recordResult.Result.Should().Be(Result.Success); + recordResult.Data.BlockHash.Should().Be(new Hash256(RecordingTests.Block18Hash)); + recordResult.Data.Witness.Should().NotBeNull(); + + // State roots AFTER the intermediate block should now be available + // StateReconstructor reconstructed until head-1, and RecordBlockCreation should have reconstructed the head block's state as well + for (long blockNum = intermediateBlockNumber; blockNum <= headNumber; blockNum++) + { + BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; + trieStore.HasRoot(header.StateRoot!).Should().BeTrue( + $"state root for block {blockNum} should be available after reconstruction from intermediate block"); + } + + // State roots BEFORE the intermediate block should NOT have been reconstructed + for (long blockNum = (long)chain.GenesisBlockNumber + 1; blockNum < intermediateBlockNumber; blockNum++) + { + BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; + trieStore.HasRoot(header.StateRoot!).Should().BeFalse( + $"state root for block {blockNum} should not be reconstructed (before nearest available state)"); + } + } + + [Test] + public async Task RecordBlockCreation_StateAlreadyAvailable_SkipsReconstruction() + { + using ArbitrumRpcTestBlockchain chain = BuildChainWithRecording(); + + ReconstructedStateTrieStore trieStore = chain.Container.Resolve(); + trieStore.HasRoot(chain.BlockTree.Head!.StateRoot!).Should().BeTrue( + "head state root should already be available before RecordBlockCreation"); + + // In archive mode, state is always available — EnsureStateAvailable is a no-op + DigestMessageParameters lastDigestMessage = GetLastDigestedMessage(); + ResultWrapper recordResult = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(lastDigestMessage.Index, lastDigestMessage.Message, WasmTargets: [])); + + recordResult.Result.Should().Be(Result.Success); + recordResult.Data.BlockHash.Should().Be(new Hash256(RecordingTests.Block18Hash)); + recordResult.Data.Witness.Should().NotBeNull(); + } + + [Test] + public void PrepareForRecord_WithFullyPrunedState_ReconstructsAllStatesInRange() + { + SwitchableReadOnlyTrieStore switchableStore = new(); + using ArbitrumRpcTestBlockchain chain = BuildChainWithRecording(switchableStore); + + long headNumber = chain.BlockTree.Head!.Number; + + // Switch to pruned mode — only genesis root is "available" + Hash256 genesisStateRoot = chain.BlockTree.FindHeader((long)chain.GenesisBlockNumber)!.StateRoot!; + switchableStore.EnablePruning(new HashSet { genesisStateRoot }); + + // Verify state roots are NOT available before PrepareForRecord + ReconstructedStateTrieStore trieStore = chain.Container.Resolve(); + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= headNumber; blockNum++) + { + BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; + if (blockNum == (long)chain.GenesisBlockNumber) + trieStore.HasRoot(header.StateRoot!).Should().BeTrue( + $"genesis state root should be available before PrepareForRecord"); + else + trieStore.HasRoot(header.StateRoot!).Should().BeFalse( + $"state root for block {blockNum} should not be available before PrepareForRecord"); + } + + ulong end = 10; + ResultWrapper result = chain.ArbitrumRpcModule.PrepareForRecord( + new PrepareForRecordParameters(Start: 5, end)); + result.Result.Should().Be(Result.Success); + + // State roots for all blocks in the range should now be available. + // StateReconstructor also reconstructed the blocks before the start block (from nearest available, + // here genesis) in order to reconstruct the blocks in the range + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= headNumber; blockNum++) + { + BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; + if (blockNum >= (long)chain.GenesisBlockNumber && blockNum <= (long)end) + trieStore.HasRoot(header.StateRoot!).Should().BeTrue( + $"state root for block {blockNum} should be available after PrepareForRecord"); + else + trieStore.HasRoot(header.StateRoot!).Should().BeFalse( + $"state root for block {blockNum} should not be available after PrepareForRecord"); + } + } + + [Test] + public void PrepareForRecord_WithPartiallyPrunedState_ReconstructsFromNearestAvailable() + { + SwitchableReadOnlyTrieStore switchableStore = new(); + using ArbitrumRpcTestBlockchain chain = BuildChainWithRecording(switchableStore); + + long headNumber = chain.BlockTree.Head!.Number; + + // Switch to pruned mode — genesis and an intermediate block are available + Hash256 genesisStateRoot = chain.BlockTree.FindHeader((long)chain.GenesisBlockNumber)!.StateRoot!; + long intermediateBlockNumber = (long)chain.GenesisBlockNumber + 11; + Hash256 intermediateStateRoot = chain.BlockTree.FindHeader(intermediateBlockNumber)!.StateRoot!; + switchableStore.EnablePruning(new HashSet { genesisStateRoot, intermediateStateRoot }); + + // Verify state roots after the intermediate block are NOT available + ReconstructedStateTrieStore trieStore = chain.Container.Resolve(); + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= headNumber; blockNum++) + { + BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; + if (blockNum == intermediateBlockNumber || blockNum == (long)chain.GenesisBlockNumber) + trieStore.HasRoot(header.StateRoot!).Should().BeTrue( + $"state root for block {blockNum} should be available before PrepareForRecord"); + else + trieStore.HasRoot(header.StateRoot!).Should().BeFalse( + $"state root for block {blockNum} should not be available before PrepareForRecord"); + } + + ulong end = 17; + ResultWrapper result = chain.ArbitrumRpcModule.PrepareForRecord( + new PrepareForRecordParameters(Start: 13, end)); + result.Result.Should().Be(Result.Success); + + // State roots for all blocks in the range should now be available + // StateReconstructor also reconstructed the blocks before the start block (from nearest available, + // here the intermediate block) in order to reconstruct the blocks in the range + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= headNumber; blockNum++) + { + BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; + if (blockNum == (long)chain.GenesisBlockNumber || (blockNum >= intermediateBlockNumber && blockNum <= (long)end)) + trieStore.HasRoot(header.StateRoot!).Should().BeTrue( + $"state root for block {blockNum} should be available after PrepareForRecord"); + else + trieStore.HasRoot(header.StateRoot!).Should().BeFalse( + $"state root for block {blockNum} should not be available after PrepareForRecord"); + } + } + + [Test] + public void PrepareForRecord_StateAlreadyAvailable_SkipsReconstruction() + { + using ArbitrumRpcTestBlockchain chain = BuildChainWithRecording(); + + // All state roots should already be available before PrepareForRecord + ReconstructedStateTrieStore trieStore = chain.Container.Resolve(); + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= (long)chain.LatestL2BlockIndex; blockNum++) + { + BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; + trieStore.HasRoot(header.StateRoot!).Should().BeTrue( + $"state root for block {blockNum} should be available before PrepareForRecord"); + } + + // In archive mode, state is always available — PrepareForRecord is a no-op + ResultWrapper result = chain.ArbitrumRpcModule.PrepareForRecord( + new PrepareForRecordParameters(Start: 10, End: 15)); + + result.Result.Should().Be(Result.Success); + } + + [Test] + public void PrepareForRecord_InvalidRange_ReturnsError() + { + using ArbitrumRpcTestBlockchain chain = BuildChainWithRecording(); + + ulong start = 10; + ulong end = 5; + ResultWrapper result = chain.ArbitrumRpcModule.PrepareForRecord( + new PrepareForRecordParameters(start, end)); + + result.Result.Should().NotBe(Result.Success); + result.Result.Error.Should().Be($"Invalid range: start {start} > end {end}"); + } + + private static ArbitrumRpcTestBlockchain BuildChainWithRecording(SwitchableReadOnlyTrieStore? switchableStore = null) + { + FullChainSimulationRecordingFile recording = new(RecordingPath); + + ArbitrumTestBlockchainBuilder builder = new ArbitrumTestBlockchainBuilder() + .WithRecording(recording); + + if (switchableStore is not null) + builder.WithContainerConfigurer(b => b.AddSingleton(ctx => + new ReconstructedStateTrieStore(new MemDb(), switchableStore.Wrap(ctx.Resolve())))); + + return builder.Build(); + } + + private static DigestMessageParameters GetLastDigestedMessage() + { + FullChainSimulationRecordingFile recording = new(RecordingPath); + return recording.GetDigestMessages().Last(); + } + + /// + /// A controller that wraps an IReadOnlyTrieStore with switchable HasRoot behavior. + /// Initially passes through all calls (including HasRoot) to the real store. + /// After EnablePruning() is called, HasRoot returns false for roots not in the allowed set, + /// while all other operations still delegate to the real store. + /// This simulates pruning mode where the few available trie nodes are "on disk" while all the others have been evicted from the dirty cache. + /// + private class SwitchableReadOnlyTrieStore + { + private HashSet? _availableRoots; + + public IReadOnlyTrieStore Wrap(IReadOnlyTrieStore inner) => new Wrapper(inner, this); + + public void EnablePruning(HashSet availableRoots) => _availableRoots = availableRoots; + + private class Wrapper(IReadOnlyTrieStore inner, SwitchableReadOnlyTrieStore controller) : IReadOnlyTrieStore + { + public void Dispose() { } + + public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 hash) + => inner.FindCachedOrUnknown(address, in path, hash); + + public byte[]? LoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) + => inner.LoadRlp(address, in path, hash, flags); + + public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) + => inner.TryLoadRlp(address, in path, hash, flags); + + public bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak) + => inner.IsPersisted(address, in path, in keccak); + + public INodeStorage.KeyScheme Scheme => inner.Scheme; + + public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) + => inner.BeginCommit(address, root, writeFlags); + + public bool HasRoot(Hash256 stateRoot) + => controller._availableRoots?.Contains(stateRoot) ?? inner.HasRoot(stateRoot); + + public IDisposable BeginScope(BlockHeader? baseBlock) => inner.BeginScope(baseBlock); + + public IScopedTrieStore GetTrieStore(Hash256? address) => inner.GetTrieStore(address); + + public IBlockCommitter BeginBlockCommit(long blockNumber) => inner.BeginBlockCommit(blockNumber); + } + } +} diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs index 3aed68a47..902142380 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumRpcTestBlockchain.cs @@ -386,6 +386,7 @@ private static ArbitrumRpcTestBlockchain CreateInternal(ArbitrumRpcTestBlockchai chain.BlockProcessingQueue, chain.Container.Resolve(), chain.Container.Resolve(), + chain.Dependencies.StateReconstructor, chain.Container.Resolve()); chain.ArbitrumRpcModule = new ArbitrumRpcModuleWrapper(chain, new ArbitrumRpcModule(engine)); @@ -554,6 +555,11 @@ public Task> RecordBlockCreation(RecordBlockCreation { return rpc.RecordBlockCreation(parameters); } + + public ResultWrapper PrepareForRecord(PrepareForRecordParameters parameters) + { + return rpc.PrepareForRecord(parameters); + } } public class ScopedGlobalWorldStateAccessor(ArbitrumRpcTestBlockchain chain) diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs index bceebc2da..4be6bbd54 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs @@ -42,6 +42,7 @@ using Nethermind.TxPool; using NSubstitute; using BlockchainProcessorOptions = Nethermind.Consensus.Processing.BlockchainProcessor.Options; +using Nethermind.Arbitrum.Execution.Stateless; namespace Nethermind.Arbitrum.Test.Infrastructure; @@ -323,6 +324,7 @@ protected record BlockchainContainerDependencies( CachedL1PriceData CachedL1PriceData, IWasmStore WasmStore, IArbosVersionProvider ArbosVersionProvider, + StateReconstructor StateReconstructor, IArbitrumSpecHelper SpecHelper); private void InitializeArbitrumPluginSteps(IContainer container) diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBuilder.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBuilder.cs index 3d2474077..420b1bce9 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBuilder.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBuilder.cs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BUSL-1.1 // SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md +using Autofac; using Nethermind.Arbitrum.Config; using Nethermind.Arbitrum.Data; using Nethermind.Core; @@ -15,6 +16,7 @@ public class ArbitrumTestBlockchainBuilder private readonly List> _configurations = new(); private ChainSpec _chainSpec = FullChainSimulationChainSpecProvider.Create(); private Action? _configureArbitrum; + private Action? _configurer; public ArbitrumTestBlockchainBuilder WithChainSpec(ChainSpec chainSpec) { @@ -28,6 +30,12 @@ public ArbitrumTestBlockchainBuilder WithArbitrumConfig(Action c return this; } + public ArbitrumTestBlockchainBuilder WithContainerConfigurer(Action configurer) + { + _configurer = configurer; + return this; + } + public ArbitrumTestBlockchainBuilder WithGenesisBlock(ulong initialBaseFee = 92, ulong arbosVersion = 32) { _chainSpec = FullChainSimulationChainSpecProvider.Create(arbosVersion); @@ -76,7 +84,7 @@ public ArbitrumTestBlockchainBuilder WithRecording(IFullChainSimulationRecording public ArbitrumRpcTestBlockchain Build() { - ArbitrumRpcTestBlockchain chain = ArbitrumRpcTestBlockchain.CreateDefault(chainSpec: _chainSpec, configureArbitrum: _configureArbitrum); + ArbitrumRpcTestBlockchain chain = ArbitrumRpcTestBlockchain.CreateDefault(configurer: _configurer, chainSpec: _chainSpec, configureArbitrum: _configureArbitrum); foreach (Action configuration in _configurations) configuration(chain); diff --git a/src/Nethermind.Arbitrum.Test/RecordingTests.cs b/src/Nethermind.Arbitrum.Test/RecordingTests.cs index 449aa2ce2..17e19d42e 100644 --- a/src/Nethermind.Arbitrum.Test/RecordingTests.cs +++ b/src/Nethermind.Arbitrum.Test/RecordingTests.cs @@ -10,7 +10,9 @@ namespace Nethermind.Arbitrum.Test; public class RecordingTests { - [TestCase("./Recordings/1__arbos32_basefee92.jsonl", 18, "0x131320467d82b8bfd1fc6504ed4e13802b7e427b1c3d1ff3c367737d4fc18fa9")] + public const string Block18Hash = "0x131320467d82b8bfd1fc6504ed4e13802b7e427b1c3d1ff3c367737d4fc18fa9"; + + [TestCase("./Recordings/1__arbos32_basefee92.jsonl", 18, Block18Hash)] [TestCase("./Recordings/2__stylus.jsonl", 18, "0x13acf142e2463eaf5049f9fe1b64f0bf5d8c6ea7ebfd950335582e7a63746ced")] [TestCase("./Recordings/2__stylus.jsonl", 19, "0xe869b42547c1c017efb9043524612975f2404412f10878a8d1f273ba11c3df83")] // Solidity Counter [TestCase("./Recordings/2__stylus.jsonl", 20, "0x2e8e1d1e4e868a5657e36383d586df9eaee84814eea51fe1b1709d976c65e820")] // Solidity Call diff --git a/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.DigestMessage.cs b/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.DigestMessage.cs index 286b51f63..cfae24e6f 100644 --- a/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.DigestMessage.cs +++ b/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.DigestMessage.cs @@ -189,7 +189,7 @@ public async Task DigestMessage_L2MessageCallContract_CallsContract() } [Test] - [TestCase(18UL, "0x131320467d82b8bfd1fc6504ed4e13802b7e427b1c3d1ff3c367737d4fc18fa9")] + [TestCase(18UL, RecordingTests.Block18Hash)] [TestCase(12UL, "0x370d29c3638d32f7d8d142feb177362ad56c9ebb34ac7fb6a629fa1aa4ea6a89")] [TestCase(0UL, "0xbd9f2163899efb7c39f945c9a7744b2c3ff12cfa00fe573dcb480a436c0803a8")] public async Task DigestMessage_BlockAlreadyExists_ReturnsExistingBlockHash(ulong blockNumber, string blockHashStr) diff --git a/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs b/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs index c9b2a1906..64d248bc3 100644 --- a/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs +++ b/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs @@ -43,6 +43,7 @@ public abstract class ArbitrumRpcModuleTests private Mock _mainProcessingContextMock = null!; private ISpecProvider _specProvider = null!; private Mock _witnessGeneratingBlockProcessingEnvFactory = null!; + private Mock _stateReconstructor = null!; [SetUp] public void Setup() { @@ -57,6 +58,7 @@ public void Setup() _blockProcessingQueue = new Mock(); _specProvider = FullChainSimulationChainSpecProvider.CreateDynamicSpecProvider(_chainSpec); _witnessGeneratingBlockProcessingEnvFactory = new Mock(); + _stateReconstructor = new Mock(); ArbitrumChainSpecEngineParameters parameters = _chainSpec.EngineChainSpecParametersProvider .GetChainSpecParameters(); @@ -90,6 +92,7 @@ public void Setup() _blockProcessingQueue.Object, _arbitrumConfig, _witnessGeneratingBlockProcessingEnvFactory.Object, + _stateReconstructor.Object, _blockConfig); _rpcModule = new ArbitrumRpcModule(engine); @@ -254,6 +257,7 @@ public async Task HeadMessageIndex_Always_ReturnsHeadMessageIndex() _blockProcessingQueue.Object, _arbitrumConfig, _witnessGeneratingBlockProcessingEnvFactory.Object, + _stateReconstructor.Object, _blockConfig); _rpcModule = new ArbitrumRpcModule(engine); @@ -287,6 +291,7 @@ public async Task HeadMessageIndex_HasNoBlocks_NoLatestHeaderFound() _blockProcessingQueue.Object, _arbitrumConfig, _witnessGeneratingBlockProcessingEnvFactory.Object, + _stateReconstructor.Object, _blockConfig); _rpcModule = new ArbitrumRpcModule(engine); @@ -326,6 +331,7 @@ public async Task HeadMessageIndex_BlockNumberIsLowerThanGenesis_Fails() _blockProcessingQueue.Object, _arbitrumConfig, _witnessGeneratingBlockProcessingEnvFactory.Object, + _stateReconstructor.Object, _blockConfig); _rpcModule = new ArbitrumRpcModule(engine); From 814f1cdf6af3e2322ec7fa6f0c39545a3434097b Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 23 Feb 2026 17:00:05 +0900 Subject: [PATCH 68/87] fix: HasRoot only check overlay memDB or database Avoid bug with state initially present in dirty nodes cache before getting evicted --- .../Stateless/ReconstructedStateTrieStore.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs index e22c115f1..2ab9ad3de 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs @@ -36,8 +36,17 @@ public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 public bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak) => _nodeStorage.Get(address, in path, in keccak) is not null || baseStore.IsPersisted(address, in path, in keccak); + /// + /// Checks the local overlay first, then falls back to reading from the base store's persistent + /// node storage (disk). We intentionally avoid because + /// it checks the dirty node cache first, which is volatile: a TOCTOU race can occur where HasRoot + /// returns true (state in dirty cache) but pruning evicts those nodes before reconstruction reads + /// them. bypasses the dirty cache and reads directly + /// from the underlying persistent storage, so it is stable. + /// public bool HasRoot(Hash256 stateRoot) - => _nodeStorage.Get(null, TreePath.Empty, stateRoot) is not null || baseStore.HasRoot(stateRoot); + => _nodeStorage.Get(null, TreePath.Empty, stateRoot) is not null + || baseStore.TryLoadRlp(null, TreePath.Empty, stateRoot) is not null; public IDisposable BeginScope(BlockHeader? baseBlock) => new Reactive.AnonymousDisposable(() => { }); From fe474b3f98609e9b86f50d03a9454bf4957f9461 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 23 Feb 2026 17:01:17 +0900 Subject: [PATCH 69/87] fix: Recover txs SenderAddress before re-executing blocks --- .../Execution/Stateless/StateReconstructor.cs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs b/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs index ea8bf07d7..9b51f3bf2 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs @@ -13,6 +13,7 @@ using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Specs; +using Nethermind.Crypto; using Nethermind.Db; using Nethermind.Evm; using Nethermind.Evm.State; @@ -27,6 +28,8 @@ public class StateReconstructor private readonly ReconstructedStateTrieStore _trieStore; private readonly IBlockTree _blockTree; private readonly ILifetimeScope _rootLifetimeScope; + private readonly IReceiptStorage _receiptStorage; + private readonly IEthereumEcdsa _ecdsa; private readonly ILogManager _logManager; private readonly ILogger _logger; private readonly long _genesisBlockNumber; @@ -35,12 +38,16 @@ public StateReconstructor( ReconstructedStateTrieStore trieStore, IBlockTree blockTree, ILifetimeScope rootLifetimeScope, + IReceiptStorage receiptStorage, + IEthereumEcdsa ecdsa, IArbitrumSpecHelper specHelper, ILogManager logManager) { _trieStore = trieStore; _blockTree = blockTree; _rootLifetimeScope = rootLifetimeScope; + _receiptStorage = receiptStorage; + _ecdsa = ecdsa; _logManager = logManager; _logger = logManager.GetClassLogger(); _genesisBlockNumber = (long)specHelper.GenesisBlockNum; @@ -148,6 +155,10 @@ private void ReExecuteBlocks(BlockHeader lastAvailable, BlockHeader targetParent throw new InvalidOperationException( $"Parent hash mismatch at block {blockNumber}: expected {expectedParentHash}, got {block.ParentHash}"); + // SenderAddress is not persisted in block RLP — recover from receipts (fast path for + // Arbitrum internal txs which have no ECDSA signature) or from ECDSA signature. + RecoverTxSenders(block); + Hash256 expectedBlockHash = block.Hash!; IReleaseSpec spec = specProvider.GetSpec(block.Header); (Block processedBlock, _) = blockProcessor.ProcessOne(block, ProcessingOptions.ForceProcessing, NullBlockTracer.Instance, spec); @@ -170,6 +181,21 @@ private void ReExecuteBlocks(BlockHeader lastAvailable, BlockHeader targetParent _logger.Info($"State reconstruction complete: re-executed {endBlock - startBlock + 1} blocks ({startBlock} to {endBlock})"); } + private void RecoverTxSenders(Block block) + { + TxReceipt[] receipts = _receiptStorage.Get(block); + if (block.Transactions.Length == receipts.Length) + { + for (int i = 0; i < block.Transactions.Length; i++) + block.Transactions[i].SenderAddress ??= receipts[i].Sender ?? _ecdsa.RecoverAddress(block.Transactions[i]); + } + else + { + for (int i = 0; i < block.Transactions.Length; i++) + block.Transactions[i].SenderAddress ??= _ecdsa.RecoverAddress(block.Transactions[i]); + } + } + private ITransactionProcessor CreateTransactionProcessor( IArbitrumSpecHelper arbitrumSpecHelper, IWasmStore wasmStore, From 0d1b5932f620cf1f26ca7d096b25ee39eaa44ab4 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Mon, 23 Feb 2026 17:03:40 +0900 Subject: [PATCH 70/87] fix: Use lock to recreate every needed state only once --- .../Execution/Stateless/StateReconstructor.cs | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs b/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs index 9b51f3bf2..599ad0aca 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs @@ -33,6 +33,7 @@ public class StateReconstructor private readonly ILogManager _logManager; private readonly ILogger _logger; private readonly long _genesisBlockNumber; + private readonly object _reconstructionLock = new(); public StateReconstructor( ReconstructedStateTrieStore trieStore, @@ -59,6 +60,7 @@ public StateReconstructor( /// public void EnsureStateAvailable(BlockHeader targetParent) { + // Fast path: avoid lock acquisition when state is already in the overlay. if (_trieStore.HasRoot(targetParent.StateRoot!)) { if (_logger.IsDebug) @@ -66,18 +68,26 @@ public void EnsureStateAvailable(BlockHeader targetParent) return; } - if (_logger.IsInfo) - _logger.Info($"State not available for block {targetParent.Number} (root {targetParent.StateRoot}), reconstructing..."); + lock (_reconstructionLock) + { + // Re-check after acquiring the lock: another thread may have reconstructed while we waited. + if (_trieStore.HasRoot(targetParent.StateRoot!)) + return; - BlockHeader lastAvailable = FindLastAvailableState(targetParent); - if (_logger.IsInfo) - _logger.Info($"Found available state at block {lastAvailable.Number} (root {lastAvailable.StateRoot}), re-executing {targetParent.Number - lastAvailable.Number} blocks forward"); + if (_logger.IsInfo) + _logger.Info($"State not available for block {targetParent.Number} (root {targetParent.StateRoot}), reconstructing..."); - ReExecuteBlocks(lastAvailable, targetParent); + BlockHeader lastAvailable = FindLastAvailableState(targetParent); - if (!_trieStore.HasRoot(targetParent.StateRoot!)) - throw new InvalidOperationException($"State reconstruction failed: root {targetParent.StateRoot} not available after re-execution"); + if (_logger.IsInfo) + _logger.Info($"Found available state at block {lastAvailable.Number} (root {lastAvailable.StateRoot}), re-executing {targetParent.Number - lastAvailable.Number} blocks forward"); + + ReExecuteBlocks(lastAvailable, targetParent); + + if (!_trieStore.HasRoot(targetParent.StateRoot!)) + throw new InvalidOperationException($"State reconstruction failed: root {targetParent.StateRoot} not available after re-execution"); + } } /// From 6dfcc89f8dfde2e735e71fabf0befdbfc7c76f0c Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 3 Mar 2026 14:38:10 +0800 Subject: [PATCH 71/87] fix Build --- .../Execution/Stateless/StateReconstructorTests.cs | 9 +++------ .../Execution/Stateless/ReconstructedStateTrieStore.cs | 3 --- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs index c2452ea44..0b4deb9a8 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs @@ -48,7 +48,7 @@ public async Task RecordBlockCreation_WithFullyPrunedState_ReconstructsStateFrom recordResult.Result.Should().Be(Result.Success); recordResult.Data.BlockHash.Should().Be(new Hash256(RecordingTests.Block18Hash)); - recordResult.Data.Witness.Should().NotBeNull(); + recordResult.Data.Preimages.Should().NotBeEmpty(); // ALL state roots from genesis to head should now be available // StateReconstructor reconstructed until head-1, and RecordBlockCreation should have reconstructed the head block's state as well @@ -94,7 +94,7 @@ public async Task RecordBlockCreation_WithPartiallyPrunedState_ReconstructsState recordResult.Result.Should().Be(Result.Success); recordResult.Data.BlockHash.Should().Be(new Hash256(RecordingTests.Block18Hash)); - recordResult.Data.Witness.Should().NotBeNull(); + recordResult.Data.Preimages.Should().NotBeEmpty(); // State roots AFTER the intermediate block should now be available // StateReconstructor reconstructed until head-1, and RecordBlockCreation should have reconstructed the head block's state as well @@ -130,7 +130,7 @@ public async Task RecordBlockCreation_StateAlreadyAvailable_SkipsReconstruction( recordResult.Result.Should().Be(Result.Success); recordResult.Data.BlockHash.Should().Be(new Hash256(RecordingTests.Block18Hash)); - recordResult.Data.Witness.Should().NotBeNull(); + recordResult.Data.Preimages.Should().NotBeEmpty(); } [Test] @@ -308,9 +308,6 @@ public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => inner.TryLoadRlp(address, in path, hash, flags); - public bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak) - => inner.IsPersisted(address, in path, in keccak); - public INodeStorage.KeyScheme Scheme => inner.Scheme; public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs index 2ab9ad3de..25f2e0fe5 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs @@ -33,9 +33,6 @@ public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => _nodeStorage.Get(address, in path, hash, flags) ?? baseStore.TryLoadRlp(address, in path, hash, flags); - public bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak) - => _nodeStorage.Get(address, in path, in keccak) is not null || baseStore.IsPersisted(address, in path, in keccak); - /// /// Checks the local overlay first, then falls back to reading from the base store's persistent /// node storage (disk). We intentionally avoid because From 21857c03952a3d8e244a4bd1edf89b34c63f2a82 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 3 Mar 2026 15:09:33 +0800 Subject: [PATCH 72/87] feat: Make witness generation read only wrt reconstructed state trie store --- ...nessGeneratingBlockProcessingEnvFactory.cs | 3 +- .../Stateless/ReconstructedStateTrieStore.cs | 34 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index 2be4aa374..0ba0266d9 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -26,6 +26,7 @@ using Nethermind.Config; using Nethermind.Evm; using Nethermind.Arbitrum.Config; +using Nethermind.Trie.Pruning; namespace Nethermind.Arbitrum.Execution.Stateless; @@ -67,7 +68,7 @@ private static BlocksConfig CreateWitnessBlocksConfig(IBlocksConfig blocksConfig public IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTargets) { IReadOnlyDbProvider readOnlyDbProvider = new ReadOnlyDbProvider(dbProvider, true); - WitnessCapturingTrieStore trieStore = new(reconstructedStateTrieStore); + WitnessCapturingTrieStore trieStore = new(new ReadOnlyReconstructedStateTrieStore(reconstructedStateTrieStore)); IStateReader stateReader = new StateReader(trieStore, readOnlyDbProvider.CodeDb, logManager); WorldState worldState = new(new TrieStoreScopeProvider(trieStore, readOnlyDbProvider.CodeDb, logManager), logManager); diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs index 25f2e0fe5..5d2f7086d 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs @@ -5,6 +5,40 @@ namespace Nethermind.Arbitrum.Execution.Stateless; +/// +/// Wraps a ReconstructedStateTrieStore for read-only access during witness generation. +/// BeginCommit returns a no-op committer so witness execution doesn't write to the overlay. +/// Only PrepareForRecord should write to the overlay to reconstruct the needed state. +/// +internal sealed class ReadOnlyReconstructedStateTrieStore(ReconstructedStateTrieStore inner) + : ITrieStore, IReadOnlyTrieStore +{ + public INodeStorage.KeyScheme Scheme => inner.Scheme; + + public bool HasRoot(Hash256 stateRoot) => inner.HasRoot(stateRoot); + + public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 hash) + => inner.FindCachedOrUnknown(address, in path, hash); + + public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) + => inner.TryLoadRlp(address, in path, hash, flags); + + public byte[]? LoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) + => inner.LoadRlp(address, in path, hash, flags); + + public IScopedTrieStore GetTrieStore(Hash256? address) => new ScopedTrieStore(this, address); + + public IDisposable BeginScope(BlockHeader? baseBlock) => inner.BeginScope(baseBlock); + + public IBlockCommitter BeginBlockCommit(long blockNumber) => NullCommitter.Instance; + + // Prevent writes from witness generation: + public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) + => NullCommitter.Instance; + + public void Dispose() { } +} + /// /// Overlay trie store for state reconstruction. Reconstructed trie nodes are stored in a MemDb overlay /// and fall back to the base store (main TrieStore dirty cache + disk) for reads. From 358cb303324f7e0f8152ea9d580e6503a05e1789 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 3 Mar 2026 15:19:27 +0800 Subject: [PATCH 73/87] feat: Prune reconstructed state that then becomes useless --- .../Config/ArbitrumConfig.cs | 2 +- .../Config/IArbitrumConfig.cs | 3 + .../Execution/ArbitrumExecutionEngine.cs | 9 ++ .../Stateless/ReconstructedStateTrieStore.cs | 137 +++++++++++++++++- .../Execution/Stateless/StateReconstructor.cs | 77 ++++++++-- 5 files changed, 213 insertions(+), 15 deletions(-) diff --git a/src/Nethermind.Arbitrum/Config/ArbitrumConfig.cs b/src/Nethermind.Arbitrum/Config/ArbitrumConfig.cs index 82797d09b..331e7a573 100644 --- a/src/Nethermind.Arbitrum/Config/ArbitrumConfig.cs +++ b/src/Nethermind.Arbitrum/Config/ArbitrumConfig.cs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md using System.Diagnostics; -using Nethermind.Arbitrum.Stylus; namespace Nethermind.Arbitrum.Config; @@ -14,6 +13,7 @@ public class ArbitrumConfig : IArbitrumConfig public WasmRebuildMode RebuildLocalWasm { get; set; } = WasmRebuildMode.Auto; public int MessageLagMs { get; set; } = 1000; public bool ExposeMultiGas { get; set; } = false; + public int ValidatorMaxStatesPrepared { get; set; } = 1000; } public static class ArbitrumConfigExtensions diff --git a/src/Nethermind.Arbitrum/Config/IArbitrumConfig.cs b/src/Nethermind.Arbitrum/Config/IArbitrumConfig.cs index 4bdff1844..3c4ca364d 100644 --- a/src/Nethermind.Arbitrum/Config/IArbitrumConfig.cs +++ b/src/Nethermind.Arbitrum/Config/IArbitrumConfig.cs @@ -25,4 +25,7 @@ public interface IArbitrumConfig : IConfig [ConfigItem(Description = "Experimental: Expose multi-dimensional gas in transaction receipts", DefaultValue = "false")] bool ExposeMultiGas { get; set; } + + [ConfigItem(Description = "Maximum number of state roots to keep pinned in the MemDb overlay simultaneously", DefaultValue = "1000")] + int ValidatorMaxStatesPrepared { get; set; } } diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs index c31a65ffa..a65c10440 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs @@ -535,6 +535,7 @@ public async Task> RecordBlockCreation(RecordBlockCr Number = blockNumber }; + // temporary reference to parent trie stateReconstructor.EnsureStateAvailable(parent); string[] wasmTargets = parameters.WasmTargets; @@ -546,6 +547,9 @@ public async Task> RecordBlockCreation(RecordBlockCr IBlockBuildingWitnessCollector witnessCollector = ((IWitnessGeneratingPolyvalentEnv)scope.Env).CreateBlockBuildingWitnessCollector(); (Block builtBlock, ArbitrumWitness witness) = await witnessCollector.BuildBlockAndGetWitness(parent, payload); + // references to parent trie are now removed + stateReconstructor.DereferenceRoot(parent.StateRoot!); + using (witness) { if (builtBlock.Hash is null) @@ -601,6 +605,8 @@ public ResultWrapper PrepareForRecord(PrepareForRecordParameters numOfBlocks--; // genesis block doesn't need preparation, so recording one less block long lastHeaderNum = headerNum + (long)numOfBlocks; + List referencedStateRoots = new List((int)numOfBlocks); + for (long current = headerNum; current <= lastHeaderNum; current++) { BlockHeader? header = BlockTree.FindHeader(current); @@ -613,6 +619,7 @@ public ResultWrapper PrepareForRecord(PrepareForRecordParameters try { stateReconstructor.EnsureStateAvailable(header); + referencedStateRoots.Add(header.StateRoot!); } catch (Exception ex) { @@ -621,6 +628,8 @@ public ResultWrapper PrepareForRecord(PrepareForRecordParameters } } + stateReconstructor.PreparedAddTrim(referencedStateRoots); + return ResultWrapper.Success(default); } diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs index 5d2f7086d..8fdf0f4df 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs @@ -1,5 +1,10 @@ +using System.Collections.Concurrent; using Nethermind.Core; +using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Db; +using Nethermind.Serialization.Rlp; using Nethermind.Trie; using Nethermind.Trie.Pruning; @@ -45,9 +50,15 @@ public void Dispose() { } /// BeginScope is a no-op to avoid acquiring the main TrieStore's scope/pruning locks during /// potentially long-running state reconstruction. /// -public class ReconstructedStateTrieStore(IKeyValueStoreWithBatching keyValueStore, IReadOnlyTrieStore baseStore) : ITrieStore, IReadOnlyTrieStore +public class ReconstructedStateTrieStore(MemDb memDb, IReadOnlyTrieStore baseStore) : ITrieStore, IReadOnlyTrieStore { - private readonly INodeStorage _nodeStorage = new NodeStorage(keyValueStore); + private readonly INodeStorage _nodeStorage = new NodeStorage(memDb); + private readonly MemDb _memDb = memDb; + + /// Per-MemDb-key reference counts for tracking which nodes are still needed by at least one alive state root. + private readonly ConcurrentDictionary _refCounts = new(Bytes.EqualityComparer); + + private static readonly AccountDecoder _accountDecoder = AccountDecoder.Instance; public void Dispose() { @@ -89,4 +100,126 @@ public bool HasRoot(Hash256 stateRoot) public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) => new RawScopedTrieStore.Committer(_nodeStorage, address, writeFlags); + + /// + /// Traverses all MemDb-resident trie nodes reachable from the given state root and increments + /// their reference counts. Call when adding a state root to the alive set. + /// + public void Reference(Hash256 stateRoot) + { + Traverse(null, TreePath.Empty, stateRoot, key => + { + _refCounts[key] = _refCounts.TryGetValue(key, out int count) ? count + 1 : 1; + }); + } + + /// + /// Traverses all MemDb-resident trie nodes reachable from the given state root and decrements + /// their reference counts. Nodes whose count reaches zero are evicted from the MemDb. + /// Call when removing a state root from the alive set. + /// + public void Dereference(Hash256 stateRoot) + { + Traverse(null, TreePath.Empty, stateRoot, key => + { + if (!_refCounts.TryGetValue(key, out int count)) + return; + + if (count <= 1) + { + _refCounts.Remove(key, out _); + _memDb.Remove(key); + } + else + { + _refCounts[key] = count - 1; + } + }); + } + + private void Traverse(Hash256? address, TreePath path, Hash256 hash, Action onKey) + { + Stack<(Hash256? address, TreePath path, Hash256 hash)> stack = new(); + stack.Push((address, path, hash)); + + while (stack.TryPop(out (Hash256? addr, TreePath p, Hash256 h) item)) + { + byte[] key = NodeStorage.GetHalfPathNodeStoragePath(item.addr, item.p, item.h); + byte[]? rlp = _memDb[key]; + // If the node is not in memDB, neither are its children. Then no need to reference them. + if (rlp is null) + continue; + + // Push children to stack BEFORE calling onKey (which during Dereference may delete this node). + // Since this is a tree traversal (no intra-trie node sharing under HalfPath scheme), each key + // is visited at most once. + PushChildren(rlp, item.addr, item.p, stack); + onKey(key); + } + } + + private static void PushChildren( + byte[] rlp, + Hash256? address, + TreePath path, + Stack<(Hash256? address, TreePath path, Hash256 hash)> stack) + { + SpanSource span = new SpanSource(rlp); + ValueRlpStream stream = new ValueRlpStream(span); + stream.ReadSequenceLength(); + int items = stream.PeekNumberOfItemsRemaining(null, 3); + + if (items > 2) + { + // Branch node: up to 16 hash-referenced children + for (int i = 0; i < 16; i++) + { + (int _, int contentLength) = stream.PeekPrefixAndContentLength(); + if (contentLength == 32) + stack.Push((address, path.Append(i), stream.DecodeKeccak()!)); + else + stream.SkipItem(); + } + // Branch value slot (index 16) is not a trie node; skip it. + } + else if (items == 2) + { + ReadOnlySpan encodedPath = stream.DecodeByteArraySpan(); + (byte[] pathNibbles, bool isLeaf) = HexPrefix.FromBytes(encodedPath); + + if (isLeaf) + { + // State trie account leaf: decode account to follow the storage trie if non-empty. + if (address is null) + { + ReadOnlySpan accountRlp = stream.DecodeByteArraySpan(); + Hash256? storageRoot = DecodeAccountStorageRoot(accountRlp); + if (storageRoot is not null) + { + // The full 64-nibble path (root → this leaf) equals Keccak(accountAddress), + // which is the address key used by NodeStorage for storage trie nodes. + TreePath fullPath = path.Append(pathNibbles); + Hash256 addressHash = new Hash256(in fullPath.Path); + stack.Push((addressHash, TreePath.Empty, storageRoot)); + } + } + // Storage trie leaf: value is a storage slot — no child nodes. + } + else + { + // Extension node: single hash-referenced child, path extended by pathNibbles. + (int _, int contentLength) = stream.PeekPrefixAndContentLength(); + if (contentLength == 32) + stack.Push((address, path.Append(pathNibbles), stream.DecodeKeccak()!)); + // Inline child (< 32 bytes) is embedded in the parent — not a separate MemDb entry. + } + } + } + + private static Hash256? DecodeAccountStorageRoot(ReadOnlySpan accountRlp) + { + Rlp.ValueDecoderContext ctx = new Rlp.ValueDecoderContext(accountRlp); + Hash256 storageRoot = _accountDecoder.DecodeStorageRootOnly(ref ctx); + return storageRoot == Keccak.EmptyTreeHash ? null : storageRoot; + } } diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs b/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs index 599ad0aca..705efadf0 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using Autofac; using Nethermind.Arbitrum.Arbos; using Nethermind.Arbitrum.Config; @@ -35,6 +36,15 @@ public class StateReconstructor private readonly long _genesisBlockNumber; private readonly object _reconstructionLock = new(); + /// + /// Maximum number of state roots to keep pinned in the MemDb overlay simultaneously. + /// When exceeded, the oldest entries are evicted (their nodes dereferenced and potentially deleted). + /// + private readonly int _maxStatesPrepared; + + /// FIFO queue of pinned state roots; oldest entries are evicted when the queue exceeds . + private readonly ConcurrentQueue _preparedQueue = new(); + public StateReconstructor( ReconstructedStateTrieStore trieStore, IBlockTree blockTree, @@ -42,6 +52,7 @@ public StateReconstructor( IReceiptStorage receiptStorage, IEthereumEcdsa ecdsa, IArbitrumSpecHelper specHelper, + IArbitrumConfig arbitrumConfig, ILogManager logManager) { _trieStore = trieStore; @@ -52,41 +63,47 @@ public StateReconstructor( _logManager = logManager; _logger = logManager.GetClassLogger(); _genesisBlockNumber = (long)specHelper.GenesisBlockNum; + _maxStatesPrepared = arbitrumConfig.ValidatorMaxStatesPrepared; } /// /// Ensures the state for the given parent header is available in the ReconstructedStateTrieStore. /// If unavailable, walks backward to find the nearest available state and re-executes blocks forward. + /// After this call, the state root is pinned in the prepared queue (if MemDb-resident) and + /// will be kept alive until evicted by later calls. /// public void EnsureStateAvailable(BlockHeader targetParent) { - // Fast path: avoid lock acquisition when state is already in the overlay. - if (_trieStore.HasRoot(targetParent.StateRoot!)) - { - if (_logger.IsDebug) - _logger.Debug($"State already available for block {targetParent.Number} (root {targetParent.StateRoot})"); - return; - } + Hash256 stateRoot = targetParent.StateRoot!; lock (_reconstructionLock) { // Re-check after acquiring the lock: another thread may have reconstructed while we waited. - if (_trieStore.HasRoot(targetParent.StateRoot!)) - return; + if (_trieStore.HasRoot(stateRoot)) + { + // Pin the state root if it lives in the MemDb overlay + _trieStore.Reference(stateRoot); + + if (_logger.IsDebug) + _logger.Debug($"State already available for block {targetParent.Number} (root {stateRoot})"); + return; + } if (_logger.IsInfo) - _logger.Info($"State not available for block {targetParent.Number} (root {targetParent.StateRoot}), reconstructing..."); + _logger.Info($"State not available for block {targetParent.Number} (root {stateRoot}), reconstructing..."); BlockHeader lastAvailable = FindLastAvailableState(targetParent); + // Pin the lastAvailable's state root if it lives in the MemDb overlay + _trieStore.Reference(lastAvailable.StateRoot!); if (_logger.IsInfo) _logger.Info($"Found available state at block {lastAvailable.Number} (root {lastAvailable.StateRoot}), re-executing {targetParent.Number - lastAvailable.Number} blocks forward"); ReExecuteBlocks(lastAvailable, targetParent); - if (!_trieStore.HasRoot(targetParent.StateRoot!)) - throw new InvalidOperationException($"State reconstruction failed: root {targetParent.StateRoot} not available after re-execution"); + if (!_trieStore.HasRoot(stateRoot)) + throw new InvalidOperationException($"State reconstruction failed: root {stateRoot} not available after re-execution"); } } @@ -154,6 +171,7 @@ private void ReExecuteBlocks(BlockHeader lastAvailable, BlockHeader targetParent using (worldState.BeginScope(lastAvailable)) { Hash256 expectedParentHash = lastAvailable.Hash!; + Hash256 prevStateRoot = lastAvailable.StateRoot!; for (long blockNumber = startBlock; blockNumber <= endBlock; blockNumber++) { @@ -178,6 +196,16 @@ private void ReExecuteBlocks(BlockHeader lastAvailable, BlockHeader targetParent $"Block hash mismatch after re-execution of block {blockNumber}: expected {expectedBlockHash}, got {processedBlock.Hash}"); worldState.CommitTree(block.Number); + + Hash256 currentStateRoot = processedBlock.Header.StateRoot!; + + // Pin the newly reconstructed state + _trieStore.Reference(currentStateRoot); + // Dereference the previous block's state (temporary reference only) + _trieStore.Dereference(prevStateRoot); + + prevStateRoot = currentStateRoot; + worldState.Reset(); expectedParentHash = processedBlock.Hash!; @@ -191,6 +219,31 @@ private void ReExecuteBlocks(BlockHeader lastAvailable, BlockHeader targetParent _logger.Info($"State reconstruction complete: re-executed {endBlock - startBlock + 1} blocks ({startBlock} to {endBlock})"); } + public void DereferenceRoot(Hash256 parentStateRoot) + { + lock (_reconstructionLock) + _trieStore.Dereference(parentStateRoot); + } + + public void PreparedAddTrim(List stateRoots) + { + lock (_reconstructionLock) + { + foreach (Hash256 stateRoot in stateRoots) + _preparedQueue.Enqueue(stateRoot); + + if (_preparedQueue.Count > _maxStatesPrepared) + { + int toEvict = _preparedQueue.Count - _maxStatesPrepared; + for (int i = 0; i < toEvict; i++) + { + if (_preparedQueue.TryDequeue(out Hash256? oldStateRoot)) + _trieStore.Dereference(oldStateRoot); + } + } + } + } + private void RecoverTxSenders(Block block) { TxReceipt[] receipts = _receiptStorage.Get(block); From b717a753bd3abada71a79c2a8afcddd4821e048d Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 3 Mar 2026 15:34:11 +0800 Subject: [PATCH 74/87] feat: Flushes whole dirty cache on shutdown (commented) --- src/Nethermind.Arbitrum/ArbitrumPlugin.cs | 27 +++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs index 57c52fe43..b1115b73d 100644 --- a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs +++ b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs @@ -159,6 +159,33 @@ public Task InitRpcModules() _api.RpcModuleProvider.RegisterSingle(debugModule); } + // Flush the trie dirty cache to disk on shutdown so that the latest processed blocks' + // state is persisted. Without this, Nethermind only persists up to head - PruningBoundary, + // while Nitro persists up to head. On restart, the mismatch causes Nitro to detect an error. + // _api.ProcessExit!.Token.Register(() => + // { + // ILogger logger = _api.LogManager.GetClassLogger(); + // if (logger.IsInfo) + // logger.Info("Flushing trie cache to disk before shutdown..."); + + // _api.WorldStateManager?.FlushCache(CancellationToken.None); + + // // FlushCache persists state but does not update BestPersistedState because + // // TrieStore.AnnounceReorgBoundaries() requires LatestCommittedBlockNumber >= LastPersistedBlockNumber + maxDepth, + // // which is never met when persisting up to the head block itself. + // // Set it directly so that on restart the node head matches the last processed block. + // long? headNumber = _api.BlockTree!.Head?.Number; + // if (headNumber.HasValue) + // { + // if (logger.IsInfo) + // logger.Info($"Setting BestPersistedState to {headNumber.Value}."); + // _api.BlockTree!.BestPersistedState = headNumber.Value; + // } + + // if (logger.IsInfo) + // logger.Info("Trie cache flushed."); + // }); + return Task.CompletedTask; } From 5173e81e6415f77c5543a937cd0c9dc9497af172 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 3 Mar 2026 15:39:42 +0800 Subject: [PATCH 75/87] feat: Add special configs for validation with memory pruning enabled --- .../arbitrum-mainnet-with-validation.json | 79 ++++++++++++++++++ .../arbitrum-sepolia-with-validation.json | 81 +++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet-with-validation.json create mode 100644 src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia-with-validation.json diff --git a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet-with-validation.json b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet-with-validation.json new file mode 100644 index 000000000..ba462e8ec --- /dev/null +++ b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-mainnet-with-validation.json @@ -0,0 +1,79 @@ +{ + "$schema": "https://raw.githubusercontent.com/NethermindEth/core-scripts/refs/heads/main/schemas/config.json", + "Init": { + "ChainSpecPath": "chainspec/arbitrum-mainnet.json", + "BaseDbPath": "nethermind_db/arbitrum-mainnet", + "LogFileName": "arbitrum-mainnet.log" + }, + "TxPool": { + "BlobsSupport": "Disabled" + }, + "Sync": { + "NetworkingEnabled": false, + "FastSync": true, + "SnapSync": true, + "FastSyncCatchUpHeightDelta": "10000000000", + "PivotNumber": 260000000, + "PivotHash": "0x0000000000000000000000000000000000000000000000000000000000000000" + }, + "Discovery": { + "DiscoveryVersion": "V5" + }, + "JsonRpc": { + "Enabled": true, + "Port": 20545, + "EnginePort": 20551, + "UnsecureDevNoRpcAuthentication": true, + "EngineEnabledModules": ["Net", "Eth", "Subscribe", "Web3", "Arbitrum", "nitroexecution"], + "AdditionalRpcUrls": [ + "http://localhost:28551|http;ws|net;eth;subscribe;web3;client;debug" + ], + "EnabledModules": [ + "Admin", + "Clique", + "Consensus", + "Db", + "Debug", + "Deposit", + "Erc20", + "Eth", + "Evm", + "Net", + "Nft", + "Parity", + "Personal", + "Proof", + "Subscribe", + "Trace", + "TxPool", + "Vault", + "Web3", + "Arbitrum" + ] + }, + "Pruning": { + "PruningBoundary": 192, + "MaxUnpersistedBlockCount": 1000, + "TrackPastKeys": false, + "PersistHeadOnShutdown": true + }, + "Blocks": { + "SecondsPerSlot": 2, + "PreWarmStateOnBlockProcessing": false, + "BuildBlocksOnMainState": true + }, + "Metrics": { + "NodeName": "Nethermind Arbitrum One" + }, + "Merge": { + "Enabled": true + }, + "Arbitrum": { + "ValidatorMaxStatesPrepared": 1000 + }, + "Snapshot": { + "Enabled": true, + "SnapshotDirectory": "snapshot/mainnet", + "DownloadUrl": "https://arb-snapshot.nethermind.dev/arbitrum-snapshot/snapshot.zip" + } +} diff --git a/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia-with-validation.json b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia-with-validation.json new file mode 100644 index 000000000..6d7f05c2d --- /dev/null +++ b/src/Nethermind.Arbitrum/Properties/configs/arbitrum-sepolia-with-validation.json @@ -0,0 +1,81 @@ +{ + "$schema": "https://raw.githubusercontent.com/NethermindEth/core-scripts/refs/heads/main/schemas/config.json", + "Init": { + "ChainSpecPath": "chainspec/arbitrum-sepolia.json", + "BaseDbPath": "nethermind_db/arbitrum-sepolia", + "LogFileName": "arbitrum-sepolia.log" + }, + "TxPool": { + "BlobsSupport": "Disabled" + }, + "Sync": { + "NetworkingEnabled": false, + "FastSync": true, + "SnapSync": true, + "FastSyncCatchUpHeightDelta": "10000000000", + "PivotNumber": 26340000, + "PivotHash": "0x4aa85d310d133f30b102379cd5fbc5d5e812d962f4f1c84b3cd046cbf7ea932d" + }, + "Discovery": { + "DiscoveryVersion": "V5" + }, + "JsonRpc": { + "Enabled": true, + "Port": 20545, + "EnginePort": 20551, + "UnsecureDevNoRpcAuthentication": true, + "EngineEnabledModules": ["Net", "Eth", "Subscribe", "Web3", "Arbitrum", "nitroexecution"], + "AdditionalRpcUrls": [ + "http://localhost:28551|http;ws|net;eth;subscribe;web3;client;debug" + ], + "EnabledModules": [ + "Admin", + "Clique", + "Consensus", + "Db", + "Debug", + "Deposit", + "Erc20", + "Eth", + "Evm", + "Net", + "Nft", + "Parity", + "Personal", + "Proof", + "Subscribe", + "Trace", + "TxPool", + "Vault", + "Web3", + "Arbitrum" + ] + }, + "Pruning": { + "PruningBoundary": 192, + "MaxUnpersistedBlockCount": 1000, + "TrackPastKeys": false, + "PersistHeadOnShutdown": true + }, + "Blocks": { + "SecondsPerSlot": 2, + "PreWarmStateOnBlockProcessing": false, + "BuildBlocksOnMainState": true + }, + "Metrics": { + "NodeName": "Nethermind Arbitrum Sepolia" + }, + "Merge": { + "Enabled": true + }, + "Arbitrum": { + "BlockProcessingTimeout": 10000, + "RebuildLocalWasm": "auto", + "ValidatorMaxStatesPrepared": 1000 + }, + "VerifyBlockHash": { + "Enabled": false, + "VerifyEveryNBlocks": 10000, + "ArbNodeRpcUrl": "https://sepolia-rollup.arbitrum.io/rpc" + } +} From 21b646292f14d9d59c6c12c862cd16bf4f8ebb4d Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Wed, 4 Mar 2026 16:26:16 +0800 Subject: [PATCH 76/87] fix: Conflicts with latest master --- src/Nethermind.Arbitrum/Arbos/Programs/StylusParams.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum/Arbos/Programs/StylusParams.cs b/src/Nethermind.Arbitrum/Arbos/Programs/StylusParams.cs index f8d8e7572..357468484 100644 --- a/src/Nethermind.Arbitrum/Arbos/Programs/StylusParams.cs +++ b/src/Nethermind.Arbitrum/Arbos/Programs/StylusParams.cs @@ -5,9 +5,9 @@ using System.Diagnostics.CodeAnalysis; using Nethermind.Arbitrum.Arbos.Storage; using Nethermind.Arbitrum.Data.Transactions; +using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; -using Nethermind.Core; using Nethermind.Int256; namespace Nethermind.Arbitrum.Arbos.Programs; From 61839ce92435e39ac1bb84a49da7fff2fd71a2a3 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 6 Mar 2026 18:31:40 +0800 Subject: [PATCH 77/87] fix existing tests --- .../ArbitrumWitnessGenerationTests.cs | 43 +++++- .../Stateless/StateReconstructorTests.cs | 126 +++++++++++++----- .../ArbitrumTestBlockchainBuilder.cs | 4 +- 3 files changed, 131 insertions(+), 42 deletions(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs index 4dbc80498..1f1be777e 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/ArbitrumWitnessGenerationTests.cs @@ -36,7 +36,8 @@ public async Task RecordBlockCreation_WitnessWithoutUserWasms_StatelessExecution using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() .WithRecording(recording) - .Build(); + // Flush trie nodes to underlying nodeStorage to make state roots accessible for ReconstructedStateTrieStore during witness generation + .Build(chain => chain.WorldStateManager.FlushCache(CancellationToken.None)); ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation(new RecordBlockCreationParameters(digestMessage.Index, digestMessage.Message, WasmTargets: [])); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestMessage.Index); @@ -74,7 +75,8 @@ public async Task RecordBlockCreation_WitnessWithUserWasms_StatelessExecutionIsS using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() .WithRecording(recording) - .Build(); + // Flush trie nodes to underlying nodeStorage to make state roots accessible for ReconstructedStateTrieStore during witness generation + .Build(chain => chain.WorldStateManager.FlushCache(CancellationToken.None)); string[] wasmTargets = chain.StylusTargetConfig.GetWasmTargets().ToArray(); ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation(new RecordBlockCreationParameters(digestMessage.Index, digestMessage.Message, WasmTargets: wasmTargets)); @@ -112,7 +114,8 @@ public async Task RecordBlockCreation_WitnessWithUserWasms_CaptureAsms(ulong mes using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() .WithRecording(recording) - .Build(); + // Flush trie nodes to underlying nodeStorage to make state roots accessible for ReconstructedStateTrieStore during witness generation + .Build(chain => chain.WorldStateManager.FlushCache(CancellationToken.None)); string[] wasmTargets = chain.StylusTargetConfig.GetWasmTargets().ToArray(); ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation(new RecordBlockCreationParameters(digestMessage.Index, digestMessage.Message, WasmTargets: wasmTargets)); @@ -158,7 +161,7 @@ public async Task RecordBlockCreation_ExtCodeSizeFollowedByIsZero_StillRecordsTa using ArbitrumRpcTestBlockchain chain = new ArbitrumTestBlockchainBuilder() .WithGenesisBlock(initialBaseFee: (ulong)l1BaseFee) - .Build(); + .Build(); // no need to flush trie here, do it before RecordBlockCreation call Address sender = FullChainSimulationAccounts.Owner.Address; @@ -272,6 +275,9 @@ public async Task RecordBlockCreation_ExtCodeSizeFollowedByIsZero_StillRecordsTa callResult.Result.Should().Be(Result.Success); chain.LatestReceipts()[1].StatusCode.Should().Be(StatusCode.Success); + // Flush trie nodes to underlying nodeStorage to make state roots accessible for ReconstructedStateTrieStore during witness generation + chain.WorldStateManager.FlushCache(CancellationToken.None); + // Step 5: Call RecordBlockCreation to generate the witness ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( new RecordBlockCreationParameters(callParams.Index, callParams.Message, WasmTargets: [])); @@ -374,6 +380,9 @@ public async Task RecordBlockCreation_PrecompileCalls_RecordsArbitrumPrecompileC receipts[1].StatusCode.Should().Be(StatusCode.Success, "ArbSys call should succeed"); receipts[2].StatusCode.Should().Be(StatusCode.Success, "ecrecover call should succeed"); + // Flush trie nodes to underlying nodeStorage to make state roots accessible for ReconstructedStateTrieStore during witness generation + chain.WorldStateManager.FlushCache(CancellationToken.None); + // Call RecordBlockCreation to generate the witness ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( new RecordBlockCreationParameters(callParams.Index, callParams.Message, WasmTargets: [])); @@ -493,6 +502,9 @@ public async Task RecordBlockCreation_BlockHashOpcode_RecordsStorageTrieNodeInWi call2Result.Result.Should().Be(Result.Success); chain.LatestReceipts()[1].StatusCode.Should().Be(StatusCode.Success); + // Flush trie nodes to underlying nodeStorage to make state roots accessible for ReconstructedStateTrieStore during witness generation + chain.WorldStateManager.FlushCache(CancellationToken.None); + // Step 2: Call the contract again (would use cached value from the block that just got built // if cache were persisted into witness-generating env) // Making sure witness-generating VM has its own empty cache and must access storage, recording the trie nodes. @@ -552,6 +564,9 @@ public async Task RecordBlockCreation_ArbBlockHash_RecordsHeadersInWitness() callResult.Result.Should().Be(Result.Success); chain.LatestReceipts()[1].StatusCode.Should().Be(StatusCode.Success, "ArbBlockHash call should succeed"); + // Flush trie nodes to underlying nodeStorage to make state roots accessible for ReconstructedStateTrieStore during witness generation + chain.WorldStateManager.FlushCache(CancellationToken.None); + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( new RecordBlockCreationParameters(callParams.Index, callParams.Message, WasmTargets: [])); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, callParams.Index); @@ -668,6 +683,9 @@ public async Task RecordBlockCreation_SubmitRetryableWithEmptyCalldata_RecordsCa await chain.DigestAndGetParams(retryable); result.Result.Should().Be(Result.Success); + // Flush trie nodes to underlying nodeStorage to make state roots accessible for ReconstructedStateTrieStore during witness generation + chain.WorldStateManager.FlushCache(CancellationToken.None); + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestParams.Index); @@ -776,6 +794,9 @@ public async Task RecordBlockCreation_TryReapRetryableNotExpired_RecordsTimeoutW await chain.DigestAndGetParams(new TestL2Transactions(l1BaseFee, sender, transferTx)); result.Result.Should().Be(Result.Success); + // Flush trie nodes to underlying nodeStorage to make state roots accessible for ReconstructedStateTrieStore during witness generation + chain.WorldStateManager.FlushCache(CancellationToken.None); + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestParams.Index); @@ -812,6 +833,9 @@ public async Task RecordBlockCreation_NonUserTransaction_RecordsBrotliCompressio await chain.DigestAndGetParams(new TestEndOfBlock(l1BaseFee)); result.Result.Should().Be(Result.Success); + // Flush trie nodes to underlying nodeStorage to make state roots accessible for ReconstructedStateTrieStore during witness generation + chain.WorldStateManager.FlushCache(CancellationToken.None); + ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); RecordResult recordResult = ThrowOnFailure(recordResultWrapper, digestParams.Index); @@ -931,6 +955,9 @@ public async Task RecordBlockCreation_WhenStorageSlotModifiedAndResetInSameBlock chain.LatestReceipts()[1].StatusCode.Should().Be(StatusCode.Success, "TX1 (set to 2) should succeed"); chain.LatestReceipts()[2].StatusCode.Should().Be(StatusCode.Success, "TX2 (reset to 1) should succeed"); + // Flush trie nodes to underlying nodeStorage to make state roots accessible for ReconstructedStateTrieStore during witness generation + chain.WorldStateManager.FlushCache(CancellationToken.None); + // Record block creation and generate witness ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); @@ -993,7 +1020,7 @@ public async Task RecordBlockCreation_WhenStateModifiedAndResetDirectlyViaWorldS depositResult.Result.Should().Be(Result.Success); // Pre-populate NetworkFeeAccount (ArbOS root storage, offset 3) with a known original value. - // This creates a leaf trie node at that storage path so the assertion targets something unique. + // This creates a leaf trie node at that storage path so the assertion targets a unique trie node. Address originalFeeAccount = TestItem.AddressB; chain.AppendBlock(chain => { @@ -1062,6 +1089,9 @@ public async Task RecordBlockCreation_WhenStateModifiedAndResetDirectlyViaWorldS "NetworkFeeAccount should retain its original value after modify + reset in the same block"); } + // Flush trie nodes to underlying nodeStorage to make state roots accessible for ReconstructedStateTrieStore during witness generation + chain.WorldStateManager.FlushCache(CancellationToken.None); + // Record block creation and generate witness ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); @@ -1191,6 +1221,9 @@ public async Task RecordBlockCreation_TransactionSetsSomeStateButReverts_StillRe "address should not be registered since the transaction reverted"); } + // Flush trie nodes to underlying nodeStorage to make state roots accessible for ReconstructedStateTrieStore during witness generation + chain.WorldStateManager.FlushCache(CancellationToken.None); + // Record the block and generate the witness ResultWrapper recordResultWrapper = await chain.ArbitrumRpcModule.RecordBlockCreation( new RecordBlockCreationParameters(digestParams.Index, digestParams.Message, WasmTargets: [])); diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs index 0b4deb9a8..6355e9e03 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs @@ -50,13 +50,18 @@ public async Task RecordBlockCreation_WithFullyPrunedState_ReconstructsStateFrom recordResult.Data.BlockHash.Should().Be(new Hash256(RecordingTests.Block18Hash)); recordResult.Data.Preimages.Should().NotBeEmpty(); - // ALL state roots from genesis to head should now be available - // StateReconstructor reconstructed until head-1, and RecordBlockCreation should have reconstructed the head block's state as well + // All state roots got reconstructed from genesis to head - 1 (RecordBlockCreation is read only!), + // but due to reconstructed state pruning, all these intermediate state root reconstructions got pruned! + // So, we get the same state as before the call to RecordBlockCreation. for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= headNumber; blockNum++) { BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; - trieStore.HasRoot(header.StateRoot!).Should().BeTrue( - $"state root for block {blockNum} should be available after reconstruction"); + if (blockNum == (long)chain.GenesisBlockNumber) + trieStore.HasRoot(header.StateRoot!).Should().BeTrue( + $"state root for block {blockNum} should be available after reconstruction"); + else + trieStore.HasRoot(header.StateRoot!).Should().BeFalse( + $"state root for block {blockNum} should not be available after reconstruction"); } } @@ -96,41 +101,65 @@ public async Task RecordBlockCreation_WithPartiallyPrunedState_ReconstructsState recordResult.Data.BlockHash.Should().Be(new Hash256(RecordingTests.Block18Hash)); recordResult.Data.Preimages.Should().NotBeEmpty(); - // State roots AFTER the intermediate block should now be available - // StateReconstructor reconstructed until head-1, and RecordBlockCreation should have reconstructed the head block's state as well - for (long blockNum = intermediateBlockNumber; blockNum <= headNumber; blockNum++) - { - BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; - trieStore.HasRoot(header.StateRoot!).Should().BeTrue( - $"state root for block {blockNum} should be available after reconstruction from intermediate block"); - } - - // State roots BEFORE the intermediate block should NOT have been reconstructed - for (long blockNum = (long)chain.GenesisBlockNumber + 1; blockNum < intermediateBlockNumber; blockNum++) + // All state roots got reconstructed from the intermediate block to head - 1 (RecordBlockCreation is read only!), + // but due to reconstructed state pruning, all these intermediate state root reconstructions got pruned! + // So, we get the same state as before the call to RecordBlockCreation. + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= headNumber; blockNum++) { BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; - trieStore.HasRoot(header.StateRoot!).Should().BeFalse( - $"state root for block {blockNum} should not be reconstructed (before nearest available state)"); + if (blockNum == (long)chain.GenesisBlockNumber || blockNum == intermediateBlockNumber) + trieStore.HasRoot(header.StateRoot!).Should().BeTrue( + $"genesis and intermediate state roots only should be available before reconstruction"); + else + trieStore.HasRoot(header.StateRoot!).Should().BeFalse( + $"state root for block {blockNum} should not be available before reconstruction"); } } [Test] public async Task RecordBlockCreation_StateAlreadyAvailable_SkipsReconstruction() { - using ArbitrumRpcTestBlockchain chain = BuildChainWithRecording(); + SwitchableReadOnlyTrieStore switchableStore = new(); + using ArbitrumRpcTestBlockchain chain = BuildChainWithRecording(switchableStore); + + DigestMessageParameters lastDigestMessage = GetLastDigestedMessage(); + + // Make target's parent's state root available from the start + long targetParentBlockNumber = (long)lastDigestMessage.Index - 1; + Hash256 targetParentStateRoot = chain.BlockTree.FindHeader(targetParentBlockNumber)!.StateRoot!; + switchableStore.EnablePruning(new HashSet { targetParentStateRoot }); ReconstructedStateTrieStore trieStore = chain.Container.Resolve(); - trieStore.HasRoot(chain.BlockTree.Head!.StateRoot!).Should().BeTrue( - "head state root should already be available before RecordBlockCreation"); + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= (long)chain.LatestL2BlockIndex; blockNum++) + { + BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; + if (blockNum == targetParentBlockNumber) + trieStore.HasRoot(header.StateRoot!).Should().BeTrue( + $"parent state root for block {blockNum} should be available before RecordBlockCreation"); + else + trieStore.HasRoot(header.StateRoot!).Should().BeFalse( + $"state root for block {blockNum} should not be available before RecordBlockCreation"); + } - // In archive mode, state is always available — EnsureStateAvailable is a no-op - DigestMessageParameters lastDigestMessage = GetLastDigestedMessage(); ResultWrapper recordResult = await chain.ArbitrumRpcModule.RecordBlockCreation( new RecordBlockCreationParameters(lastDigestMessage.Index, lastDigestMessage.Message, WasmTargets: [])); recordResult.Result.Should().Be(Result.Success); recordResult.Data.BlockHash.Should().Be(new Hash256(RecordingTests.Block18Hash)); recordResult.Data.Preimages.Should().NotBeEmpty(); + + // As target's parent's state root is already available, EnsureStateAvailable is a no-op + // and RecordBlockCreation is read only, so state roots availability should be unchanged after the call. + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= (long)chain.LatestL2BlockIndex; blockNum++) + { + BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; + if (blockNum == targetParentBlockNumber) + trieStore.HasRoot(header.StateRoot!).Should().BeTrue( + $"parent state root for block {blockNum} should be available after RecordBlockCreation"); + else + trieStore.HasRoot(header.StateRoot!).Should().BeFalse( + $"state root for block {blockNum} should not be available after RecordBlockCreation"); + } } [Test] @@ -158,18 +187,20 @@ public void PrepareForRecord_WithFullyPrunedState_ReconstructsAllStatesInRange() $"state root for block {blockNum} should not be available before PrepareForRecord"); } + ulong start = 5; ulong end = 10; ResultWrapper result = chain.ArbitrumRpcModule.PrepareForRecord( - new PrepareForRecordParameters(Start: 5, end)); + new PrepareForRecordParameters(start, end)); result.Result.Should().Be(Result.Success); - // State roots for all blocks in the range should now be available. + // State roots for all blocks in the range [start-1, end] (in addition to genesis) should now be available. // StateReconstructor also reconstructed the blocks before the start block (from nearest available, - // here genesis) in order to reconstruct the blocks in the range + // here genesis) in order to reconstruct the blocks in the range, but those ones then got pruned + // as not pinned/referenced (for future use). for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= headNumber; blockNum++) { BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; - if (blockNum >= (long)chain.GenesisBlockNumber && blockNum <= (long)end) + if (blockNum == (long)chain.GenesisBlockNumber || blockNum >= (long)start - 1 && blockNum <= (long)end) trieStore.HasRoot(header.StateRoot!).Should().BeTrue( $"state root for block {blockNum} should be available after PrepareForRecord"); else @@ -188,7 +219,7 @@ public void PrepareForRecord_WithPartiallyPrunedState_ReconstructsFromNearestAva // Switch to pruned mode — genesis and an intermediate block are available Hash256 genesisStateRoot = chain.BlockTree.FindHeader((long)chain.GenesisBlockNumber)!.StateRoot!; - long intermediateBlockNumber = (long)chain.GenesisBlockNumber + 11; + long intermediateBlockNumber = (long)chain.GenesisBlockNumber + 9; Hash256 intermediateStateRoot = chain.BlockTree.FindHeader(intermediateBlockNumber)!.StateRoot!; switchableStore.EnablePruning(new HashSet { genesisStateRoot, intermediateStateRoot }); @@ -205,18 +236,21 @@ public void PrepareForRecord_WithPartiallyPrunedState_ReconstructsFromNearestAva $"state root for block {blockNum} should not be available before PrepareForRecord"); } + ulong start = 13; ulong end = 17; ResultWrapper result = chain.ArbitrumRpcModule.PrepareForRecord( - new PrepareForRecordParameters(Start: 13, end)); + new PrepareForRecordParameters(start, end)); result.Result.Should().Be(Result.Success); - // State roots for all blocks in the range should now be available + // State roots for all blocks in the range [start-1, end] (in addition to the already available + // genesis and the intermediate block) should now be available. // StateReconstructor also reconstructed the blocks before the start block (from nearest available, - // here the intermediate block) in order to reconstruct the blocks in the range + // here the intermediate block) in order to reconstruct the blocks in the range, + // but those ones then got pruned as not pinned/referenced (for future use). for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= headNumber; blockNum++) { BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; - if (blockNum == (long)chain.GenesisBlockNumber || (blockNum >= intermediateBlockNumber && blockNum <= (long)end)) + if (blockNum == (long)chain.GenesisBlockNumber || blockNum == intermediateBlockNumber || blockNum >= (long)start - 1 && blockNum <= (long)end) trieStore.HasRoot(header.StateRoot!).Should().BeTrue( $"state root for block {blockNum} should be available after PrepareForRecord"); else @@ -228,6 +262,7 @@ public void PrepareForRecord_WithPartiallyPrunedState_ReconstructsFromNearestAva [Test] public void PrepareForRecord_StateAlreadyAvailable_SkipsReconstruction() { + // No state root controller passed to the chain, all state roots are available from the start using ArbitrumRpcTestBlockchain chain = BuildChainWithRecording(); // All state roots should already be available before PrepareForRecord @@ -235,8 +270,8 @@ public void PrepareForRecord_StateAlreadyAvailable_SkipsReconstruction() for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= (long)chain.LatestL2BlockIndex; blockNum++) { BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; - trieStore.HasRoot(header.StateRoot!).Should().BeTrue( - $"state root for block {blockNum} should be available before PrepareForRecord"); + trieStore.HasRoot(header.StateRoot!).Should().BeTrue( + $"state root for block {blockNum} should be available before PrepareForRecord"); } // In archive mode, state is always available — PrepareForRecord is a no-op @@ -244,6 +279,13 @@ public void PrepareForRecord_StateAlreadyAvailable_SkipsReconstruction() new PrepareForRecordParameters(Start: 10, End: 15)); result.Result.Should().Be(Result.Success); + + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= (long)chain.LatestL2BlockIndex; blockNum++) + { + BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; + trieStore.HasRoot(header.StateRoot!).Should().BeTrue( + $"state root for block {blockNum} should be available after PrepareForRecord"); + } } [Test] @@ -271,7 +313,8 @@ private static ArbitrumRpcTestBlockchain BuildChainWithRecording(SwitchableReadO builder.WithContainerConfigurer(b => b.AddSingleton(ctx => new ReconstructedStateTrieStore(new MemDb(), switchableStore.Wrap(ctx.Resolve())))); - return builder.Build(); + // Flush trie nodes to underlying nodeStorage to make state roots accessible for ReconstructedStateTrieStore + return builder.Build(chain => chain.WorldStateManager.FlushCache(CancellationToken.None)); } private static DigestMessageParameters GetLastDigestedMessage() @@ -305,16 +348,27 @@ public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 public byte[]? LoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => inner.LoadRlp(address, in path, hash, flags); + // ReconstructedStateTrieStore.HasRoot() will call this method to check if a state root is available, + // so we override it to implement the pruning behavior. + // But we also need to make sure it does not impact regular calls to it when fetching nodes outside of state roots. public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) - => inner.TryLoadRlp(address, in path, hash, flags); + { + // If the controller overrides available roots and we are loading a state root (address is null and path is empty), + // only return the node if it's in the available set + if (controller._availableRoots is not null && address is null && path.Length == 0) + return controller._availableRoots.Contains(hash) ? inner.TryLoadRlp(address, in path, hash, flags) : null; + + return inner.TryLoadRlp(address, in path, hash, flags); + } public INodeStorage.KeyScheme Scheme => inner.Scheme; public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) => inner.BeginCommit(address, root, writeFlags); + // This method should not even be called as ReconstructedStateTrieStore.HasRoot() will call this.TryLoadRlp() directly! public bool HasRoot(Hash256 stateRoot) - => controller._availableRoots?.Contains(stateRoot) ?? inner.HasRoot(stateRoot); + => throw new UnauthorizedAccessException("Method HasRoot should not be called"); public IDisposable BeginScope(BlockHeader? baseBlock) => inner.BeginScope(baseBlock); diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBuilder.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBuilder.cs index 420b1bce9..8209fc14a 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBuilder.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBuilder.cs @@ -82,13 +82,15 @@ public ArbitrumTestBlockchainBuilder WithRecording(IFullChainSimulationRecording return this; } - public ArbitrumRpcTestBlockchain Build() + public ArbitrumRpcTestBlockchain Build(Action? afterBuild = null) { ArbitrumRpcTestBlockchain chain = ArbitrumRpcTestBlockchain.CreateDefault(configurer: _configurer, chainSpec: _chainSpec, configureArbitrum: _configureArbitrum); foreach (Action configuration in _configurations) configuration(chain); + afterBuild?.Invoke(chain); + return chain; } From b40e0916adf858ac631e604d6f5dd83e81628d76 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 6 Mar 2026 18:33:14 +0800 Subject: [PATCH 78/87] fix: Create IStateReconstructor --- .../Rpc/ArbitrumRpcModuleTests.cs | 4 ++-- src/Nethermind.Arbitrum/ArbitrumPlugin.cs | 2 +- .../Execution/ArbitrumExecutionEngine.cs | 2 +- .../Execution/Stateless/IStateReconstructor.cs | 11 +++++++++++ .../Execution/Stateless/StateReconstructor.cs | 2 +- 5 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 src/Nethermind.Arbitrum/Execution/Stateless/IStateReconstructor.cs diff --git a/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs b/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs index 64d248bc3..3f4c04cd1 100644 --- a/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs +++ b/src/Nethermind.Arbitrum.Test/Rpc/ArbitrumRpcModuleTests.cs @@ -43,7 +43,7 @@ public abstract class ArbitrumRpcModuleTests private Mock _mainProcessingContextMock = null!; private ISpecProvider _specProvider = null!; private Mock _witnessGeneratingBlockProcessingEnvFactory = null!; - private Mock _stateReconstructor = null!; + private Mock _stateReconstructor = null!; [SetUp] public void Setup() { @@ -58,7 +58,7 @@ public void Setup() _blockProcessingQueue = new Mock(); _specProvider = FullChainSimulationChainSpecProvider.CreateDynamicSpecProvider(_chainSpec); _witnessGeneratingBlockProcessingEnvFactory = new Mock(); - _stateReconstructor = new Mock(); + _stateReconstructor = new Mock(); ArbitrumChainSpecEngineParameters parameters = _chainSpec.EngineChainSpecParametersProvider .GetChainSpecParameters(); diff --git a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs index b1115b73d..464391328 100644 --- a/src/Nethermind.Arbitrum/ArbitrumPlugin.cs +++ b/src/Nethermind.Arbitrum/ArbitrumPlugin.cs @@ -324,7 +324,7 @@ protected override void Load(ContainerBuilder builder) // Execution recording (state reconstruction + witness generation) .AddSingleton(ctx => new ReconstructedStateTrieStore(new Db.MemDb(), ctx.Resolve())) - .AddSingleton() + .AddSingleton() .AddSingleton() .Bind() diff --git a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs index a65c10440..53f331c1c 100644 --- a/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs +++ b/src/Nethermind.Arbitrum/Execution/ArbitrumExecutionEngine.cs @@ -40,7 +40,7 @@ public sealed class ArbitrumExecutionEngine( IBlockProcessingQueue processingQueue, IArbitrumConfig arbitrumConfig, IArbitrumWitnessGeneratingBlockProcessingEnvFactory witnessGeneratingBlockProcessingEnvFactory, - StateReconstructor stateReconstructor, + IStateReconstructor stateReconstructor, IBlocksConfig blocksConfig) : IArbitrumExecutionEngine { diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/IStateReconstructor.cs b/src/Nethermind.Arbitrum/Execution/Stateless/IStateReconstructor.cs new file mode 100644 index 000000000..19bb11724 --- /dev/null +++ b/src/Nethermind.Arbitrum/Execution/Stateless/IStateReconstructor.cs @@ -0,0 +1,11 @@ +using Nethermind.Core; +using Nethermind.Core.Crypto; + +namespace Nethermind.Arbitrum.Execution.Stateless; + +public interface IStateReconstructor +{ + void EnsureStateAvailable(BlockHeader targetParent); + void DereferenceRoot(Hash256 parentStateRoot); + void PreparedAddTrim(List<(Hash256, ulong)> stateRoots); +} diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs b/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs index 705efadf0..a900a35ee 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs @@ -24,7 +24,7 @@ namespace Nethermind.Arbitrum.Execution.Stateless; -public class StateReconstructor +public class StateReconstructor : IStateReconstructor { private readonly ReconstructedStateTrieStore _trieStore; private readonly IBlockTree _blockTree; From 40895ce4e7992676d51ad66a8dd174eae4524345 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 6 Mar 2026 18:35:11 +0800 Subject: [PATCH 79/87] tests: ReconstructedState pruning and integration tests --- .../Stateless/StateReconstructorTests.cs | 189 +++++++++++++++++- 1 file changed, 188 insertions(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs index 6355e9e03..219b2cef1 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs @@ -302,7 +302,185 @@ public void PrepareForRecord_InvalidRange_ReturnsError() result.Result.Error.Should().Be($"Invalid range: start {start} > end {end}"); } - private static ArbitrumRpcTestBlockchain BuildChainWithRecording(SwitchableReadOnlyTrieStore? switchableStore = null) + [Test] + public async Task PrepareForRecord_ThenRecordBlockCreation_PreparedStateRemainsAvailable() + { + SwitchableReadOnlyTrieStore switchableStore = new(); + using ArbitrumRpcTestBlockchain chain = BuildChainWithRecording(switchableStore); + + Hash256 genesisStateRoot = chain.BlockTree.FindHeader((long)chain.GenesisBlockNumber)!.StateRoot!; + switchableStore.EnablePruning(new HashSet { genesisStateRoot }); + + ulong prepareStart = 14; + ulong prepareEnd = 17; + ResultWrapper prepareResult = chain.ArbitrumRpcModule.PrepareForRecord( + new PrepareForRecordParameters(prepareStart, prepareEnd)); + prepareResult.Result.Should().Be(Result.Success); + + // PrepareForRecord also includes the parent state (prepareStart-1) so RecordBlockCreation can access it + long overlayStart = (long)prepareStart - 1; + ReconstructedStateTrieStore trieStore = chain.Container.Resolve(); + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= chain.BlockTree.Head!.Number; blockNum++) + { + BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; + + bool shouldBeAvailable = blockNum == (long)chain.GenesisBlockNumber + || (blockNum >= overlayStart && blockNum <= (long)prepareEnd); + + trieStore.HasRoot(header.StateRoot!).Should().Be(shouldBeAvailable, + $"block {blockNum} state should {(shouldBeAvailable ? "" : "not ")}be available after PrepareForRecord"); + } + + // RecordBlockCreation for block 18 uses block 17's already-prepared state — no reconstruction needed + DigestMessageParameters lastMessage = GetLastDigestedMessage(); + ResultWrapper recordResult = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(lastMessage.Index, lastMessage.Message, WasmTargets: [])); + + recordResult.Result.Should().Be(Result.Success); + recordResult.Data.Preimages.Should().NotBeEmpty(); + + // PrepareForRecord-pinned states are unaffected by the RecordBlockCreation + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= chain.BlockTree.Head!.Number; blockNum++) + { + BlockHeader header = chain.BlockTree.FindHeader(blockNum)!; + + bool shouldBeAvailable = blockNum == (long)chain.GenesisBlockNumber + || (blockNum >= overlayStart && blockNum <= (long)prepareEnd); + + trieStore.HasRoot(header.StateRoot!).Should().Be(shouldBeAvailable, + $"block {blockNum} state should {(shouldBeAvailable ? "" : "not ")}be available after RecordBlockCreation"); + } + } + + [Test] + public void PrepareForRecord_WithSmallMaxStatesPrepared_EvictsOldStates() + { + SwitchableReadOnlyTrieStore switchableStore = new(); + using ArbitrumRpcTestBlockchain chain = BuildChainWithRecording(switchableStore, maxStatesPrepared: 3); + + Hash256 genesisStateRoot = chain.BlockTree.FindHeader((long)chain.GenesisBlockNumber)!.StateRoot!; + switchableStore.EnablePruning(new HashSet { genesisStateRoot }); + + ReconstructedStateTrieStore trieStore = chain.Container.Resolve(); + + // First PrepareForRecord: 4 states [4,5,6,7] prepared but max=3 → block 4 immediately evicted + ResultWrapper firstResult = chain.ArbitrumRpcModule.PrepareForRecord( + new PrepareForRecordParameters(Start: 5, End: 7)); + firstResult.Result.Should().Be(Result.Success); + + trieStore.HasRoot(chain.BlockTree.FindHeader(4)!.StateRoot!).Should().BeFalse( + "block 4 was the oldest in the queue and should have been evicted when max was exceeded"); + trieStore.HasRoot(chain.BlockTree.FindHeader(5)!.StateRoot!).Should().BeTrue("block 5 should be available"); + trieStore.HasRoot(chain.BlockTree.FindHeader(6)!.StateRoot!).Should().BeTrue("block 6 should be available"); + trieStore.HasRoot(chain.BlockTree.FindHeader(7)!.StateRoot!).Should().BeTrue("block 7 should be available"); + + // Second PrepareForRecord: 4 more states [9,10,11,12] added → queue [5,6,7,9,10,11,12], keep 3 most recent [10,11,12] + ResultWrapper secondResult = chain.ArbitrumRpcModule.PrepareForRecord( + new PrepareForRecordParameters(Start: 10, End: 12)); + secondResult.Result.Should().Be(Result.Success); + + trieStore.HasRoot(chain.BlockTree.FindHeader(5)!.StateRoot!).Should().BeFalse("block 5 should be evicted"); + trieStore.HasRoot(chain.BlockTree.FindHeader(6)!.StateRoot!).Should().BeFalse("block 6 should be evicted"); + trieStore.HasRoot(chain.BlockTree.FindHeader(7)!.StateRoot!).Should().BeFalse("block 7 should be evicted"); + // Block 9 was reconstructed as an intermediate but wasn't kept: it was enqueued then immediately evicted + trieStore.HasRoot(chain.BlockTree.FindHeader(9)!.StateRoot!).Should().BeFalse( + "block 9 was evicted as the oldest remaining state after second PrepareForRecord"); + trieStore.HasRoot(chain.BlockTree.FindHeader(10)!.StateRoot!).Should().BeTrue("block 10 should be available"); + trieStore.HasRoot(chain.BlockTree.FindHeader(11)!.StateRoot!).Should().BeTrue("block 11 should be available"); + trieStore.HasRoot(chain.BlockTree.FindHeader(12)!.StateRoot!).Should().BeTrue("block 12 should be available"); + } + + [Test] + public async Task PrepareForRecord_InterleavedWithRecordBlockCreation_MaintainsCorrectAvailability() + { + SwitchableReadOnlyTrieStore switchableStore = new(); + // max=5 so the first PrepareForRecord [3,4,5,6,7] fits exactly without eviction + using ArbitrumRpcTestBlockchain chain = BuildChainWithRecording(switchableStore, maxStatesPrepared: 5); + + Hash256 genesisStateRoot = chain.BlockTree.FindHeader((long)chain.GenesisBlockNumber)!.StateRoot!; + switchableStore.EnablePruning(new HashSet { genesisStateRoot }); + + ReconstructedStateTrieStore trieStore = chain.Container.Resolve(); + + // Phase 1: prepare states for blocks 3-7 (PrepareForRecord includes start-1=3) + ulong start1 = 4; + ulong end1 = 7; + ResultWrapper firstPrepare = chain.ArbitrumRpcModule.PrepareForRecord( + new PrepareForRecordParameters(start1, end1)); + firstPrepare.Result.Should().Be(Result.Success); + + DigestMessageParameters lastDigestMsg = GetLastDigestedMessage(); + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= (long)lastDigestMsg.Index; blockNum++) + { + if (blockNum == (long)chain.GenesisBlockNumber || (blockNum >= (long)start1 - 1 && blockNum <= (long)end1)) + trieStore.HasRoot(chain.BlockTree.FindHeader(blockNum)!.StateRoot!).Should().BeTrue( + $"block {blockNum} state should be available after first PrepareForRecord"); + else + trieStore.HasRoot(chain.BlockTree.FindHeader(blockNum)!.StateRoot!).Should().BeFalse( + $"block {blockNum} state should not be available after first PrepareForRecord"); + } + + // Phase 2: (Unordered) RecordBlockCreation calls reuse prepared states — no reconstruction, prepared states unaffected + DigestMessageParameters msg8 = GetDigestedMessage(8); + ResultWrapper record8 = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(msg8.Index, msg8.Message, WasmTargets: [])); + record8.Result.Should().Be(Result.Success); + record8.Data.Preimages.Should().NotBeEmpty(); + + DigestMessageParameters msg6 = GetDigestedMessage(6); + ResultWrapper record6 = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(msg6.Index, msg6.Message, WasmTargets: [])); + record6.Result.Should().Be(Result.Success); + record6.Data.Preimages.Should().NotBeEmpty(); + + // Prepared states are unchanged after read-only RecordBlockCreations + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= (long)lastDigestMsg.Index; blockNum++) + { + if (blockNum == (long)chain.GenesisBlockNumber || (blockNum >= (long)start1 - 1 && blockNum <= (long)end1)) + trieStore.HasRoot(chain.BlockTree.FindHeader(blockNum)!.StateRoot!).Should().BeTrue( + $"block {blockNum} state should be available after first RecordBlockCreation"); + else + trieStore.HasRoot(chain.BlockTree.FindHeader(blockNum)!.StateRoot!).Should().BeFalse( + $"block {blockNum} state should not be available after first RecordBlockCreation"); + } + + // Phase 3: second PrepareForRecord prepares [11,12,13,14] → queue [3,4,5,6,7,11,12,13,14], evict 4 oldest [3,4,5,6] + ResultWrapper secondPrepare = chain.ArbitrumRpcModule.PrepareForRecord( + new PrepareForRecordParameters(Start: 12, End: 14)); + secondPrepare.Result.Should().Be(Result.Success); + + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= (long)lastDigestMsg.Index; blockNum++) + { + if (blockNum == (long)chain.GenesisBlockNumber || blockNum == 7 || blockNum == 11 || blockNum == 12 || blockNum == 13 || blockNum == 14) + trieStore.HasRoot(chain.BlockTree.FindHeader(blockNum)!.StateRoot!).Should().BeTrue( + $"block {blockNum} state should be available after second PrepareForRecord"); + else + trieStore.HasRoot(chain.BlockTree.FindHeader(blockNum)!.StateRoot!).Should().BeFalse( + $"block {blockNum} state should not be available after second PrepareForRecord"); + } + + // Phase 4: one last RecordBlockCreation call that does not find parent state: reconstructs it temporarily + // and evicts it before the call returns. The prepared states remain unaffected. + DigestMessageParameters msg7 = GetDigestedMessage(7); + ResultWrapper record7 = await chain.ArbitrumRpcModule.RecordBlockCreation( + new RecordBlockCreationParameters(msg7.Index, msg7.Message, WasmTargets: [])); + record7.Result.Should().Be(Result.Success); + record7.Data.Preimages.Should().NotBeEmpty(); + + for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= (long)lastDigestMsg.Index; blockNum++) + { + if (blockNum == (long)chain.GenesisBlockNumber || blockNum == 7 || blockNum == 11 || blockNum == 12 || blockNum == 13 || blockNum == 14) + trieStore.HasRoot(chain.BlockTree.FindHeader(blockNum)!.StateRoot!).Should().BeTrue( + $"block {blockNum} state should be available after second RecordBlockCreation"); + else + trieStore.HasRoot(chain.BlockTree.FindHeader(blockNum)!.StateRoot!).Should().BeFalse( + $"block {blockNum} state should not be available after second RecordBlockCreation"); + } + } + + private static ArbitrumRpcTestBlockchain BuildChainWithRecording( + SwitchableReadOnlyTrieStore? switchableStore = null, + int? maxStatesPrepared = null) { FullChainSimulationRecordingFile recording = new(RecordingPath); @@ -313,6 +491,9 @@ private static ArbitrumRpcTestBlockchain BuildChainWithRecording(SwitchableReadO builder.WithContainerConfigurer(b => b.AddSingleton(ctx => new ReconstructedStateTrieStore(new MemDb(), switchableStore.Wrap(ctx.Resolve())))); + if (maxStatesPrepared.HasValue) + builder.WithArbitrumConfig(cfg => cfg.ValidatorMaxStatesPrepared = maxStatesPrepared.Value); + // Flush trie nodes to underlying nodeStorage to make state roots accessible for ReconstructedStateTrieStore return builder.Build(chain => chain.WorldStateManager.FlushCache(CancellationToken.None)); } @@ -323,6 +504,12 @@ private static DigestMessageParameters GetLastDigestedMessage() return recording.GetDigestMessages().Last(); } + private static DigestMessageParameters GetDigestedMessage(ulong index) + { + FullChainSimulationRecordingFile recording = new(RecordingPath); + return recording.GetDigestMessages().Single(m => m.Index == index); + } + /// /// A controller that wraps an IReadOnlyTrieStore with switchable HasRoot behavior. /// Initially passes through all calls (including HasRoot) to the real store. From 4653a70ff39ee82a1d41c09e75cf16e47f6ec3ef Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 10 Mar 2026 13:40:01 +0800 Subject: [PATCH 80/87] fix: update to latest base wit gen branch --- src/Nethermind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind b/src/Nethermind index 256bc8fc1..413e5c8e9 160000 --- a/src/Nethermind +++ b/src/Nethermind @@ -1 +1 @@ -Subproject commit 256bc8fc12a00e981b5827367816cd24e9499c7b +Subproject commit 413e5c8e91f3956df48cee93b99b75f7ab223c66 From 8a47185fe331f38c21423b26f6bc19aaf0e45295 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Tue, 10 Mar 2026 20:12:08 +0800 Subject: [PATCH 81/87] fix IStateReconstructor --- .../Execution/Stateless/IStateReconstructor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/IStateReconstructor.cs b/src/Nethermind.Arbitrum/Execution/Stateless/IStateReconstructor.cs index 19bb11724..7e608b85c 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/IStateReconstructor.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/IStateReconstructor.cs @@ -7,5 +7,5 @@ public interface IStateReconstructor { void EnsureStateAvailable(BlockHeader targetParent); void DereferenceRoot(Hash256 parentStateRoot); - void PreparedAddTrim(List<(Hash256, ulong)> stateRoots); + void PreparedAddTrim(List stateRoots); } From 97c7381e049d4ec145b49419851402c7ede6c9d4 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Wed, 11 Mar 2026 17:12:47 +0800 Subject: [PATCH 82/87] fix: Update to latest wit gen v2 branch --- src/Nethermind | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind b/src/Nethermind index 413e5c8e9..b52d305ed 160000 --- a/src/Nethermind +++ b/src/Nethermind @@ -1 +1 @@ -Subproject commit 413e5c8e91f3956df48cee93b99b75f7ab223c66 +Subproject commit b52d305ed3bf25bd3ee6b214205c6c65ab88f785 From 93a45a485f1d5bda5fe86e80cc12fcf743b2cf6c Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 13 Mar 2026 10:22:49 +0800 Subject: [PATCH 83/87] fix: Use interface instead of concrete type --- .../Infrastructure/ArbitrumTestBlockchainBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs index 4be6bbd54..389267253 100644 --- a/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs +++ b/src/Nethermind.Arbitrum.Test/Infrastructure/ArbitrumTestBlockchainBase.cs @@ -324,7 +324,7 @@ protected record BlockchainContainerDependencies( CachedL1PriceData CachedL1PriceData, IWasmStore WasmStore, IArbosVersionProvider ArbosVersionProvider, - StateReconstructor StateReconstructor, + IStateReconstructor StateReconstructor, IArbitrumSpecHelper SpecHelper); private void InitializeArbitrumPluginSteps(IContainer container) From 85fc5070705072286b138f40abf3792b1fc8c22c Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 13 Mar 2026 13:54:48 +0800 Subject: [PATCH 84/87] feat: Remove useless ReadOnlyReconstructedStateTrieStore --- src/Nethermind | 2 +- ...nessGeneratingBlockProcessingEnvFactory.cs | 2 +- .../Stateless/ReconstructedStateTrieStore.cs | 38 ++----------------- 3 files changed, 6 insertions(+), 36 deletions(-) diff --git a/src/Nethermind b/src/Nethermind index b52d305ed..483dbf664 160000 --- a/src/Nethermind +++ b/src/Nethermind @@ -1 +1 @@ -Subproject commit b52d305ed3bf25bd3ee6b214205c6c65ab88f785 +Subproject commit 483dbf6647865181ad95a02a48e2dfbeadc3f082 diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs index 0ba0266d9..eb37fc8ff 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ArbitrumWitnessGeneratingBlockProcessingEnvFactory.cs @@ -68,7 +68,7 @@ private static BlocksConfig CreateWitnessBlocksConfig(IBlocksConfig blocksConfig public IWitnessGeneratingBlockProcessingEnvScope CreateScope(string[]? wasmTargets) { IReadOnlyDbProvider readOnlyDbProvider = new ReadOnlyDbProvider(dbProvider, true); - WitnessCapturingTrieStore trieStore = new(new ReadOnlyReconstructedStateTrieStore(reconstructedStateTrieStore)); + WitnessCapturingTrieStore trieStore = new(reconstructedStateTrieStore); IStateReader stateReader = new StateReader(trieStore, readOnlyDbProvider.CodeDb, logManager); WorldState worldState = new(new TrieStoreScopeProvider(trieStore, readOnlyDbProvider.CodeDb, logManager), logManager); diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs index 8fdf0f4df..b2ebc998c 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs @@ -10,45 +10,15 @@ namespace Nethermind.Arbitrum.Execution.Stateless; -/// -/// Wraps a ReconstructedStateTrieStore for read-only access during witness generation. -/// BeginCommit returns a no-op committer so witness execution doesn't write to the overlay. -/// Only PrepareForRecord should write to the overlay to reconstruct the needed state. -/// -internal sealed class ReadOnlyReconstructedStateTrieStore(ReconstructedStateTrieStore inner) - : ITrieStore, IReadOnlyTrieStore -{ - public INodeStorage.KeyScheme Scheme => inner.Scheme; - - public bool HasRoot(Hash256 stateRoot) => inner.HasRoot(stateRoot); - - public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 hash) - => inner.FindCachedOrUnknown(address, in path, hash); - - public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) - => inner.TryLoadRlp(address, in path, hash, flags); - - public byte[]? LoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) - => inner.LoadRlp(address, in path, hash, flags); - - public IScopedTrieStore GetTrieStore(Hash256? address) => new ScopedTrieStore(this, address); - - public IDisposable BeginScope(BlockHeader? baseBlock) => inner.BeginScope(baseBlock); - - public IBlockCommitter BeginBlockCommit(long blockNumber) => NullCommitter.Instance; - - // Prevent writes from witness generation: - public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) - => NullCommitter.Instance; - - public void Dispose() { } -} - /// /// Overlay trie store for state reconstruction. Reconstructed trie nodes are stored in a MemDb overlay /// and fall back to the base store (main TrieStore dirty cache + disk) for reads. /// BeginScope is a no-op to avoid acquiring the main TrieStore's scope/pruning locks during /// potentially long-running state reconstruction. +/// +/// Additional notes: +/// - Only PrepareForRecord should write to the overlay to reconstruct the needed state, +/// witness generation is read only against the overlay. /// public class ReconstructedStateTrieStore(MemDb memDb, IReadOnlyTrieStore baseStore) : ITrieStore, IReadOnlyTrieStore { From ed35a451ee43568da34fc47bf9db6f29591ec846 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 13 Mar 2026 13:57:42 +0800 Subject: [PATCH 85/87] fix: Remove now non-existent SpanSource --- .../Execution/Stateless/ReconstructedStateTrieStore.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs index b2ebc998c..4af71f4c6 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs @@ -134,8 +134,7 @@ private static void PushChildren( TreePath path, Stack<(Hash256? address, TreePath path, Hash256 hash)> stack) { - SpanSource span = new SpanSource(rlp); - ValueRlpStream stream = new ValueRlpStream(span); + ValueRlpStream stream = new ValueRlpStream(rlp); stream.ReadSequenceLength(); int items = stream.PeekNumberOfItemsRemaining(null, 3); From 10a7e0850cbb67d8aba74e3dfd671e86603aa5ca Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 13 Mar 2026 14:13:42 +0800 Subject: [PATCH 86/87] fix: Format --- .../Execution/Stateless/StateReconstructorTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs index 219b2cef1..b78013d71 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs @@ -409,7 +409,7 @@ public async Task PrepareForRecord_InterleavedWithRecordBlockCreation_MaintainsC new PrepareForRecordParameters(start1, end1)); firstPrepare.Result.Should().Be(Result.Success); - DigestMessageParameters lastDigestMsg = GetLastDigestedMessage(); + DigestMessageParameters lastDigestMsg = GetLastDigestedMessage(); for (long blockNum = (long)chain.GenesisBlockNumber; blockNum <= (long)lastDigestMsg.Index; blockNum++) { if (blockNum == (long)chain.GenesisBlockNumber || (blockNum >= (long)start1 - 1 && blockNum <= (long)end1)) From 4a0af7f5530e1ea93c7c4f8ffd185089ce563a53 Mon Sep 17 00:00:00 2001 From: Hugo Demeyere Date: Fri, 13 Mar 2026 14:22:01 +0800 Subject: [PATCH 87/87] feat: Add license info on new files --- .../Execution/Stateless/StateReconstructorTests.cs | 3 +++ .../Execution/Stateless/IStateReconstructor.cs | 3 +++ .../Stateless/ReconstructedStateTrieStore.cs | 11 +++++++---- .../Execution/Stateless/StateReconstructor.cs | 3 +++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs b/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs index b78013d71..d2db40959 100644 --- a/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs +++ b/src/Nethermind.Arbitrum.Test/Execution/Stateless/StateReconstructorTests.cs @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md + using Autofac; using FluentAssertions; using Nethermind.Arbitrum.Data; diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/IStateReconstructor.cs b/src/Nethermind.Arbitrum/Execution/Stateless/IStateReconstructor.cs index 7e608b85c..8e4af1d1b 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/IStateReconstructor.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/IStateReconstructor.cs @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md + using Nethermind.Core; using Nethermind.Core.Crypto; diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs index 4af71f4c6..37f8bdb4a 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/ReconstructedStateTrieStore.cs @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md + using System.Collections.Concurrent; using Nethermind.Core; using Nethermind.Core.Buffers; @@ -15,11 +18,11 @@ namespace Nethermind.Arbitrum.Execution.Stateless; /// and fall back to the base store (main TrieStore dirty cache + disk) for reads. /// BeginScope is a no-op to avoid acquiring the main TrieStore's scope/pruning locks during /// potentially long-running state reconstruction. -/// -/// Additional notes: -/// - Only PrepareForRecord should write to the overlay to reconstruct the needed state, -/// witness generation is read only against the overlay. /// +/// +/// Only PrepareForRecord should write to the overlay to reconstruct the needed state, +/// witness generation is read only against the overlay. +/// public class ReconstructedStateTrieStore(MemDb memDb, IReadOnlyTrieStore baseStore) : ITrieStore, IReadOnlyTrieStore { private readonly INodeStorage _nodeStorage = new NodeStorage(memDb); diff --git a/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs b/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs index a900a35ee..19c483716 100644 --- a/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs +++ b/src/Nethermind.Arbitrum/Execution/Stateless/StateReconstructor.cs @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: BUSL-1.1 +// SPDX-FileCopyrightText: https://github.com/NethermindEth/nethermind-arbitrum/blob/main/LICENSE.md + using System.Collections.Concurrent; using Autofac; using Nethermind.Arbitrum.Arbos;