10311 gloas da

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/BlockFactoryFuluTest.java

@@ -165,7 +165,7 @@ void shouldCreateValidDataColumnSidecarsForBlockContents() {
                                       .getProofs()
                                       .get(blobIndex * CELLS_PER_EXT_BLOB + index))
                           .toList());
-              assertThat(dataColumnSidecar.getKzgCommitments()).isEqualTo(expectedCommitments);
+              assertThat(dataColumnSidecar.getMaybeKzgCommitments()).contains(expectedCommitments);
             });
   }
 

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactoryFuluTest.java

@@ -321,7 +321,7 @@ void shouldCreateDataColumnSidecarsForBlockContents() {
                               blobIndex ->
                                   expectedProofs.get(blobIndex * CELLS_PER_EXT_BLOB + index))
                           .toList());
-              assertThat(dataColumnSidecar.getKzgCommitments()).isEqualTo(expectedCommitments);
+              assertThat(dataColumnSidecar.getMaybeKzgCommitments()).contains(expectedCommitments);
               // verify the merkle proof
               assertThat(miscHelpersFulu.verifyDataColumnSidecarInclusionProof(dataColumnSidecar))
                   .isTrue();
@@ -383,7 +383,7 @@ void shouldCreateDataColumnSidecarsForBlindedBlock_ForLocalFallback() {
                               blobIndex ->
                                   expectedProofs.get(blobIndex * CELLS_PER_EXT_BLOB + index))
                           .toList());
-              assertThat(dataColumnSidecar.getKzgCommitments()).isEqualTo(expectedCommitments);
+              assertThat(dataColumnSidecar.getMaybeKzgCommitments()).contains(expectedCommitments);
               // verify the merkle proof
               assertThat(miscHelpersFulu.verifyDataColumnSidecarInclusionProof(dataColumnSidecar))
                   .isTrue();

ethereum/spec/src/main/java/tech/pegasys/teku/spec/Spec.java

@@ -1310,6 +1310,10 @@ public Optional<Integer> getNumberOfDataColumnSubnets() {
     return getSpecConfigFulu().map(SpecConfigFulu::getDataColumnSidecarSubnetCount);
   }
 
+  public int getNumberOfCustodyGroups(final UInt64 slot) {
+    return SpecConfigFulu.required(atSlot(slot).getConfig()).getNumberOfCustodyGroups();
+  }
+
   public boolean isAvailabilityOfDataColumnSidecarsRequiredAtEpoch(
       final ReadOnlyStore store, final UInt64 epoch) {
     if (getSpecConfigFulu().isEmpty()) {

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/blobs/DataColumnSidecar.java

@@ -27,17 +27,14 @@
 
 public interface DataColumnSidecar extends SszContainer {
 
+  @Override
+  DataColumnSidecarSchema<? extends DataColumnSidecar> getSchema();
+
   UInt64 getIndex();
 
   DataColumn getColumn();
 
-  default Optional<SszList<SszKZGCommitment>> getMaybeKzgCommitments() {
-    return Optional.of(getKzgCommitments());
-  }
-
-  // TODO-GLOAS: https://github.com/Consensys/teku/issues/10311 need to verify this is not called
-  // for Gloas or use everywhere getMaybeKzgCommitments() alternatively
-  SszList<SszKZGCommitment> getKzgCommitments();
+  Optional<SszList<SszKZGCommitment>> getMaybeKzgCommitments();
 
   SszList<SszKZGProof> getKzgProofs();
 
@@ -57,7 +54,6 @@ default String toLogString() {
         getBeaconBlockRoot(),
         getIndex(),
         getColumn().toBriefString(),
-        getKzgCommitments().size(),
         getKzgProofs().size());
   }
 }

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/blobs/versions/fulu/DataColumnSidecarFulu.java

@@ -74,6 +74,11 @@ public static DataColumnSidecarFulu required(final DataColumnSidecar dataColumnS
                 kzgCommitmentsInclusionProof.stream().map(SszBytes32::of).toList()));
   }
 
+  @Override
+  public DataColumnSidecarSchemaFulu getSchema() {
+    return (DataColumnSidecarSchemaFulu) super.getSchema();
+  }
+
   @Override
   public UInt64 getIndex() {
     return getField0().get();
@@ -85,6 +90,10 @@ public DataColumn getColumn() {
   }
 
   @Override
+  public Optional<SszList<SszKZGCommitment>> getMaybeKzgCommitments() {
+    return Optional.of(getField2());
+  }
+
   public SszList<SszKZGCommitment> getKzgCommitments() {
     return getField2();
   }

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/blobs/versions/gloas/DataColumnSidecarGloas.java

@@ -52,6 +52,11 @@ public class DataColumnSidecarGloas
         SszBytes32.of(beaconBlockRoot));
   }
 
+  @Override
+  public DataColumnSidecarSchemaGloas getSchema() {
+    return (DataColumnSidecarSchemaGloas) super.getSchema();
+  }
+
   @Override
   public UInt64 getIndex() {
     return getField0().get();
@@ -67,11 +72,6 @@ public Optional<SszList<SszKZGCommitment>> getMaybeKzgCommitments() {
     return Optional.empty();
   }
 
-  @Override
-  public SszList<SszKZGCommitment> getKzgCommitments() {
-    throw new UnsupportedOperationException("kzg_commitments field was removed in Gloas");
-  }
-
   @Override
   public SszList<SszKZGProof> getKzgProofs() {
     return getField2();

ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/util/DataColumnSidecarUtil.java

@@ -13,6 +13,7 @@
 
 package tech.pegasys.teku.spec.logic.common.util;
 
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -23,12 +24,19 @@
 import org.apache.tuweni.bytes.Bytes32;
 import tech.pegasys.teku.bls.BLSSignature;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
+import tech.pegasys.teku.infrastructure.ssz.SszList;
+import tech.pegasys.teku.infrastructure.ssz.collections.SszBytes32Vector;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.Spec;
 import tech.pegasys.teku.spec.datastructures.blobs.DataColumnSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock;
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockHeader;
+import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlockHeader;
+import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot;
+import tech.pegasys.teku.spec.datastructures.blocks.blockbody.BeaconBlockBody;
+import tech.pegasys.teku.spec.datastructures.execution.BlobAndCellProofs;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
+import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment;
 import tech.pegasys.teku.spec.logic.common.statetransition.results.BlockImportResult;
 
 /**
@@ -81,7 +89,7 @@ SafeFuture<Optional<DataColumnSidecarValidationError>> validateWithState(
   DataColumnSidecarTrackingKey extractTrackingKey(DataColumnSidecar dataColumnSidecar);
 
   DataColumnSidecarTrackingKey extractTrackingKeyFromHeader(
-      BeaconBlockHeader header, DataColumnSidecar dataColumnSidecar);
+      Optional<BeaconBlockHeader> maybeBeaconBlockHeader, DataColumnSidecar dataColumnSidecar);
 
   boolean verifyDataColumnSidecarStructure(DataColumnSidecar dataColumnSidecar);
 
@@ -95,6 +103,24 @@ void cacheValidatedInfo(
       Set<Bytes32> validSignedBlockHeaders,
       Set<InclusionProofInfo> validInclusionProofInfoSet);
 
+  SszList<SszKZGCommitment> getKzgCommitments(BeaconBlock block);
+
+  List<DataColumnSidecar> constructDataColumnSidecars(
+      Optional<SignedBeaconBlockHeader> maybeSignedBeaconBlockHeader,
+      SlotAndBlockRoot slotAndBlockRoot,
+      Optional<SszList<SszKZGCommitment>> maybeSszKZGCommitments,
+      Optional<List<Bytes32>> maybeKzgCommitmentsInclusionProof,
+      List<BlobAndCellProofs> blobAndCellProofsList);
+
+  Optional<List<Bytes32>> computeDataColumnKzgCommitmentsInclusionProof(
+      BeaconBlockBody beaconBlockBody);
+
+  Optional<SszBytes32Vector> getMaybeKzgCommitmentsInclusionProof(
+      DataColumnSidecar dataColumnSidecar);
+
+  List<DataColumnSidecar> reconstructAllDataColumnSidecars(
+      List<DataColumnSidecar> dataColumnSidecars);
+
   record InclusionProofInfo(
       Bytes32 commitmentsRoot, Bytes32 inclusionProofRoot, Bytes32 bodyRoot) {}
 

ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/fulu/helpers/MiscHelpersFulu.java

@@ -60,6 +60,7 @@
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlockHeader;
+import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot;
 import tech.pegasys.teku.spec.datastructures.blocks.blockbody.BeaconBlockBody;
 import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.BeaconBlockBodyDeneb;
 import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.BlindedBeaconBlockBodyDeneb;
@@ -240,32 +241,34 @@ public UInt64 getValidatorsCustodyRequirement(
 
   public boolean verifyDataColumnSidecar(final DataColumnSidecar dataColumnSidecar) {
     final int numberOfColumns = specConfigFulu.getNumberOfColumns();
-    final UInt64 epoch = computeEpochAtSlot(dataColumnSidecar.getSlot());
+    final DataColumnSidecarFulu dataColumnSidecarFulu =
+        DataColumnSidecarFulu.required(dataColumnSidecar);
+    final UInt64 epoch = computeEpochAtSlot(dataColumnSidecarFulu.getSlot());
 
-    if (!dataColumnSidecar.getIndex().isLessThan(numberOfColumns)) {
+    if (!dataColumnSidecarFulu.getIndex().isLessThan(numberOfColumns)) {
       LOG.trace(
           "DataColumnSidecar has invalid index {}. Should be less than {}",
-          dataColumnSidecar.getIndex(),
+          dataColumnSidecarFulu.getIndex(),
           numberOfColumns);
       return false;
     }
-    if (dataColumnSidecar.getKzgCommitments().isEmpty()) {
+    if (dataColumnSidecarFulu.getKzgCommitments().isEmpty()) {
       LOG.trace("DataColumnSidecar has no kzg commitments");
       return false;
     }
 
-    if (dataColumnSidecar.getKzgCommitments().size()
-        > getBlobParameters(epoch).maxBlobsPerBlock()) {
+    final int kzgCommitmentsSize = dataColumnSidecarFulu.getKzgCommitments().size();
+    if (kzgCommitmentsSize > getBlobParameters(epoch).maxBlobsPerBlock()) {
       LOG.trace(
           "DataColumnSidecar has too many commitments when compared to the BPO for epoch {}",
           epoch);
       return false;
     }
-    if (dataColumnSidecar.getColumn().size() != dataColumnSidecar.getKzgCommitments().size()) {
+    if (dataColumnSidecar.getColumn().size() != kzgCommitmentsSize) {
       LOG.trace(
           "DataColumnSidecar has unequal data column ({}) and kzg commitments ({}) sizes",
           dataColumnSidecar.getColumn().size(),
-          dataColumnSidecar.getKzgCommitments().size());
+          kzgCommitmentsSize);
       return false;
     }
     if (dataColumnSidecar.getColumn().size() != dataColumnSidecar.getKzgProofs().size()) {
@@ -290,7 +293,7 @@ public boolean verifyDataColumnSidecarKzgProofs(final DataColumnSidecar dataColu
             .collect(Collectors.toList());
     return getKzg()
         .verifyCellProofBatch(
-            dataColumnSidecar.getKzgCommitments().stream()
+            DataColumnSidecarFulu.required(dataColumnSidecar).getKzgCommitments().stream()
                 .map(SszKZGCommitment::getKZGCommitment)
                 .toList(),
             cellWithIds,
@@ -315,7 +318,8 @@ public boolean verifyDataColumnSidecarKzgProofsBatch(
     return getKzg()
         .verifyCellProofBatch(
             dataColumnSidecars.stream()
-                .flatMap(sidecar -> sidecar.getKzgCommitments().stream())
+                .flatMap(
+                    sidecar -> DataColumnSidecarFulu.required(sidecar).getKzgCommitments().stream())
                 .map(SszKZGCommitment::getKZGCommitment)
                 .toList(),
             cellWithIds,
@@ -326,15 +330,17 @@ public boolean verifyDataColumnSidecarKzgProofsBatch(
   }
 
   public boolean verifyDataColumnSidecarInclusionProof(final DataColumnSidecar dataColumnSidecar) {
-    if (dataColumnSidecar.getKzgCommitments().isEmpty()) {
+    final DataColumnSidecarFulu dataColumnSidecarFulu =
+        DataColumnSidecarFulu.required(dataColumnSidecar);
+    if (dataColumnSidecarFulu.getKzgCommitments().isEmpty()) {
       return false;
     }
     return predicates.isValidMerkleBranch(
-        dataColumnSidecar.getKzgCommitments().hashTreeRoot(),
-        DataColumnSidecarFulu.required(dataColumnSidecar).getKzgCommitmentsInclusionProof(),
+        dataColumnSidecarFulu.getKzgCommitments().hashTreeRoot(),
+        dataColumnSidecarFulu.getKzgCommitmentsInclusionProof(),
         specConfigFulu.getKzgCommitmentsInclusionProofDepth().intValue(),
         getBlockBodyKzgCommitmentsGeneralizedIndex(),
-        DataColumnSidecarFulu.required(dataColumnSidecar).getBlockBodyRoot());
+        dataColumnSidecarFulu.getBlockBodyRoot());
   }
 
   public int getBlockBodyKzgCommitmentsGeneralizedIndex() {
@@ -369,17 +375,18 @@ public List<DataColumnSidecar> constructDataColumnSidecars(
   }
 
   public List<DataColumnSidecar> constructDataColumnSidecars(
-      final SignedBeaconBlockHeader signedBeaconBlockHeader,
-      final SszList<SszKZGCommitment> sszKZGCommitments,
-      final List<Bytes32> kzgCommitmentsInclusionProof,
+      final Optional<SignedBeaconBlockHeader> maybeSignedBeaconBlockHeader,
+      final SlotAndBlockRoot slotAndBlockRoot,
+      final Optional<SszList<SszKZGCommitment>> maybeSszKZGCommitments,
+      final Optional<List<Bytes32>> maybeKzgCommitmentsInclusionProof,
       final List<BlobAndCellProofs> blobAndCellProofsList) {
     final List<List<MatrixEntry>> extendedMatrix = computeExtendedMatrix(blobAndCellProofsList);
     return constructDataColumnSidecarsInternal(
         builder ->
             builder
-                .kzgCommitments(sszKZGCommitments)
-                .signedBlockHeader(signedBeaconBlockHeader)
-                .kzgCommitmentsInclusionProof(kzgCommitmentsInclusionProof),
+                .kzgCommitments(maybeSszKZGCommitments.orElseThrow())
+                .signedBlockHeader(maybeSignedBeaconBlockHeader.orElseThrow())
+                .kzgCommitmentsInclusionProof(maybeKzgCommitmentsInclusionProof.orElseThrow()),
         extendedMatrix);
   }
 
@@ -546,19 +553,15 @@ public List<DataColumnSidecar> reconstructAllDataColumnSidecars(
             .toList();
     final List<List<MatrixEntry>> blobColumnEntries = transpose(columnBlobEntries);
     final List<List<MatrixEntry>> extendedMatrix = recoverMatrix(blobColumnEntries);
-    final DataColumnSidecar anyExistingSidecar =
-        existingSidecars.stream().findFirst().orElseThrow();
-    final SignedBeaconBlockHeader signedBeaconBlockHeader =
-        DataColumnSidecarFulu.required(anyExistingSidecar).getSignedBlockHeader();
+    final DataColumnSidecarFulu anyExistingSidecar =
+        DataColumnSidecarFulu.required(existingSidecars.stream().findFirst().orElseThrow());
     return constructDataColumnSidecarsInternal(
         builder ->
             builder
                 .kzgCommitments(anyExistingSidecar.getKzgCommitments())
-                .signedBlockHeader(signedBeaconBlockHeader)
+                .signedBlockHeader(anyExistingSidecar.getSignedBlockHeader())
                 .kzgCommitmentsInclusionProof(
-                    DataColumnSidecarFulu.required(anyExistingSidecar)
-                        .getKzgCommitmentsInclusionProof()
-                        .asListUnboxed()),
+                    anyExistingSidecar.getKzgCommitmentsInclusionProof().asListUnboxed()),
         extendedMatrix);
   }
 
@@ -569,7 +572,7 @@ public List<DataColumnSidecar> reconstructAllDataColumnSidecars(
    *
    * <p>The data structure for storing cells is implementation-dependent.
    */
-  private List<List<MatrixEntry>> recoverMatrix(final List<List<MatrixEntry>> partialMatrix) {
+  public List<List<MatrixEntry>> recoverMatrix(final List<List<MatrixEntry>> partialMatrix) {
     return IntStream.range(0, partialMatrix.size())
         .parallel()
         .mapToObj(
@@ -604,7 +607,7 @@ public int getSamplingGroupCount(final int custodyRequirement) {
     return Math.max(custodyRequirement, specConfigFulu.getSamplesPerSlot());
   }
 
-  private static <T> List<List<T>> transpose(final List<List<T>> matrix) {
+  public static <T> List<List<T>> transpose(final List<List<T>> matrix) {
     final int rowCount = matrix.size();
     final int colCount = matrix.getFirst().size();
     final List<List<T>> ret =