Besu fix sync tests withdrawals

clients/besu/besu.sh

@@ -99,16 +99,18 @@ if [ -d /blocks ]; then
     IMPORTFLAGS="$IMPORTFLAGS $blocks"
 fi
 
-
-# Configure mining.
-if [ "$HIVE_MINER" != "" ]; then
-    FLAGS="$FLAGS --miner-enabled --miner-coinbase=$HIVE_MINER"
-    # For clique mining, besu uses the node key as the block signing key.
-    if [ "$HIVE_CLIQUE_PRIVATEKEY" != "" ]; then
-        echo "Importing clique signing key as node key..."
-        echo "$HIVE_CLIQUE_PRIVATEKEY" > /opt/besu/key
+if [ "$HIVE_BOOTNODE" != "" ]; then
+    # Configure mining.
+    if [ "$HIVE_MINER" != "" ]; then
+        FLAGS="$FLAGS --miner-enabled --miner-coinbase=$HIVE_MINER"
+        # For clique mining, besu uses the node key as the block signing key.
+        if [ "$HIVE_CLIQUE_PRIVATEKEY" != "" ]; then
+            echo "Importing clique signing key as node key..."
+            echo "$HIVE_CLIQUE_PRIVATEKEY" > /opt/besu/key
+        fi
     fi
 fi
+
 if [ "$HIVE_MINER_EXTRA" != "" ]; then
     FLAGS="$FLAGS --miner-extra-data=$HIVE_MINER_EXTRA"
 fi

clients/erigon/Dockerfile

@@ -1,4 +1,4 @@
-ARG branch=devel
+ARG branch=docker_withdrawals
 FROM thorax/erigon:$branch
 
 # The upstream erigon container uses a non-root user, but we need

clients/erigon/mapper.jq

@@ -51,5 +51,6 @@ def to_bool:
     "berlinBlock": env.HIVE_FORK_BERLIN|to_int,
     "londonBlock": env.HIVE_FORK_LONDON|to_int,
     "terminalTotalDifficulty": env.HIVE_TERMINAL_TOTAL_DIFFICULTY|to_int,
+    "shanghaiTime": env.HIVE_SHANGHAI_TIMESTAMP|to_int,
   }|remove_empty
 }

clients/go-ethereum/Dockerfile

@@ -4,29 +4,30 @@
 # produce a very minimalistic container that can be reused many times without
 # needing to constantly rebuild.
 
-ARG branch=latest
-FROM ethereum/client-go:$branch
+ARG branch=withdrawals-timestamp
+# FROM ethereum/client-go:$branch
 
-RUN apk add --update bash curl jq
+# RUN apk add --update bash curl jq
 
-# FROM alpine:latest
-# ARG branch=master
+FROM alpine:latest
+ARG branch=master
 
 # Build go-ethereum on the fly and delete all build tools afterwards
-# RUN \
-#   apk add --update bash curl jq go git make gcc musl-dev              \
-#         ca-certificates linux-headers                           && \
-#   git clone --depth 1 --branch $branch  https://github.com/ethereum/go-ethereum && \
-#   (cd go-ethereum && make geth)                               && \
-#   (cd go-ethereum                                             && \
-#   echo "{}"                                                      \
-#   | jq ".+ {\"repo\":\"$(git config --get remote.origin.url)\"}" \
-#   | jq ".+ {\"branch\":\"$(git rev-parse --abbrev-ref HEAD)\"}"  \
-#   | jq ".+ {\"commit\":\"$(git rev-parse HEAD)\"}"               \
-#   > /version.json)                                            && \
-#   cp go-ethereum/build/bin/geth /geth                         && \
-#   apk del go git make gcc musl-dev linux-headers              && \
-#   rm -rf /go-ethereum && rm -rf /var/cache/apk/*
+RUN echo https://dl-cdn.alpinelinux.org/alpine/edge/testing >> /etc/apk/repositories
+RUN \
+  apk add --update bash curl jq go delve git make gcc musl-dev              \
+        ca-certificates linux-headers                           && \
+  git clone --depth 1 --branch withdrawals-timestamp  https://github.com/lightclient/go-ethereum && \
+  (cd go-ethereum && make geth)                               && \
+  (cd go-ethereum                                             && \
+  echo "{}"                                                      \
+  | jq ".+ {\"repo\":\"$(git config --get remote.origin.url)\"}" \
+  | jq ".+ {\"branch\":\"$(git rev-parse --abbrev-ref HEAD)\"}"  \
+  | jq ".+ {\"commit\":\"$(git rev-parse HEAD)\"}"               \
+  > /version.json)                                            && \
+  cp go-ethereum/build/bin/geth /usr/local/bin/geth           && \
+  apk del go git make gcc musl-dev linux-headers              && \
+  rm -rf /var/cache/apk/*
 
 RUN /usr/local/bin/geth console --exec 'console.log(admin.nodeInfo.name)' --maxpeers=0 --nodiscover --dev 2>/dev/null | head -1 > /version.txt
 
@@ -43,7 +44,7 @@ RUN chmod +x /hive-bin/enode.sh
 ADD genesis.json /genesis.json
 
 # Export the usual networking ports to allow outside access to the node
-EXPOSE 8545 8546 8547 8551 30303 30303/udp
+EXPOSE 8545 8546 8547 8551 30303 30303/udp 40000
 
 # Generate the ethash verification caches
 RUN \

clients/go-ethereum/geth.sh

@@ -173,5 +173,10 @@ fi
 
 # Run the go-ethereum implementation with the requested flags.
 FLAGS="$FLAGS --nat=none"
-echo "Running go-ethereum with flags $FLAGS"
-$geth $FLAGS
+if [ "$HIVE_CLIENT_DEBUGGING" != "" ]; then
+    echo "Running go-ethereum in debugger mode with flags $FLAGS"
+    "dlv" "--listen=:40000" "--headless=true" "--api-version=2" "--accept-multiclient" "--allow-non-terminal-interactive" "exec" $geth -- $FLAGS
+else
+    echo "Running go-ethereum with flags $FLAGS"
+    $geth $FLAGS
+fi

clients/go-ethereum/mapper.jq

@@ -54,5 +54,6 @@ def to_bool:
     "londonBlock": env.HIVE_FORK_LONDON|to_int,
     "mergeForkBlock": env.HIVE_MERGE_BLOCK_ID|to_int,
     "terminalTotalDifficulty": env.HIVE_TERMINAL_TOTAL_DIFFICULTY|to_int,
+    "shanghaiTime": env.HIVE_SHANGHAI_TIMESTAMP|to_int,
   }|remove_empty
 }

clients/nethermind/Dockerfile

@@ -1,5 +1,5 @@
 ARG branch=latest
-FROM nethermindeth/hive:$branch
+FROM nethermindeth/nethermind:withdrawals_yolo
 
 RUN apt-get update && apt-get install -y wget
 RUN wget https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -O /usr/local/bin/jq && \

clients/nethermind/mapper.jq

@@ -134,7 +134,10 @@ def clique_engine:
     "eip3198Transition": env.HIVE_FORK_LONDON|to_hex,
 
     # Merge
-    "MergeForkIdTransition": env.HIVE_MERGE_BLOCK_ID|to_hex,
+    # "MergeForkIdTransition": env.HIVE_MERGE_BLOCK_ID|to_hex,
+
+    # Shanghai
+    "eip4895TransitionTimestamp": env.HIVE_SHANGHAI_TIMESTAMP|to_hex,
 
     # Other chain parameters
     "networkID": env.HIVE_NETWORK_ID|to_hex,

go.work.sum

@@ -1,3 +1,46 @@
+cloud.google.com/go/bigtable v1.2.0 h1:F4cCmA4nuV84V5zYQ3MKY+M1Cw1avHDuf3S/LcZPA9c=
+github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
+github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
+github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
+github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db h1:nxAtV4VajJDhKysp2kdcJZsq8Ss1xSA0vZTkVHHJd0E=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/c-bata/go-prompt v0.2.2 h1:uyKRz6Z6DUyj49QVijyM339UJV9yhbr70gESwbNU3e0=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI=
+github.com/eclipse/paho.mqtt.golang v1.2.0 h1:1F8mhG9+aO5/xpdtFkW4SxOJB67ukuDC3t2y2qayIX0=
+github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 h1:IZqZOB2fydHte3kUgxrzK5E1fW7RQGeDwE8F/ZZnUYc=
+github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
+github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
+github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
+github.com/golang/geo v0.0.0-20190916061304-5b978397cfec h1:lJwO/92dFXWeXOZdoGXgptLmNLwynMSHUmU6besqtiw=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A=
+github.com/influxdata/flux v0.65.1 h1:77BcVUCzvN5HMm8+j9PRBQ4iZcu98Dl4Y9rf+J5vhnc=
+github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385 h1:ED4e5Cc3z5vSN2Tz2GkOHN7vs4Sxe2yds6CXvDnvZFE=
+github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9 h1:MHTrDWmQpHq/hkq+7cw9oYAt2PqUw52TZazRA0N7PGE=
 github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio=
+github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
+github.com/marioevz/go-ethereum v1.10.14-0.20230113164816-9cfc6ec5a607 h1:3RAUX8WNcojZXfBGkqmJ0UdjASXl0uz+H5qSGDPNik0=
+github.com/marioevz/go-ethereum v1.10.14-0.20230113164816-9cfc6ec5a607/go.mod h1:n7VlOgCwYheLB/mi+V8ni2yf8K2qM3N9WAmalxkhk+c=
+github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
+github.com/onsi/ginkgo/v2 v2.1.3 h1:e/3Cwtogj0HA+25nMP1jCMDIf8RtRYbGwGGuBIFztkc=
+github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
+github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
+github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5 h1:tFwafIEMf0B7NlcxV/zJ6leBIa81D3hgGSgsE5hCkOQ=
+github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/common v0.30.0 h1:JEkYlQnpzrzQFxi6gnukFPdQ+ac82oRhzMcIduJu/Ug=
+github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
+github.com/segmentio/kafka-go v0.2.0 h1:HtCSf6B4gN/87yc5qTl7WsxPKQIIGXLPPM1bMCPOsoY=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
 github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo=
+github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6 h1:YdYsPAZ2pC6Tow/nPZOPQ96O3hm/ToAkGsPLzedXERk=
+golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA=
+golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM=
+golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618=
+gonum.org/v1/gonum v0.6.0 h1:DJy6UzXbahnGUf1ujUNkh/NEtK14qMo2nvlBPs4U5yw=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=

simulators/ethereum/engine/README.md

@@ -455,4 +455,7 @@ Invalid token error is expected.
 
 - Positive time drift, within limit, correct secret:  
 Engine API call where the `iat` claim contains a positive time drift smaller than the maximum threshold, and the secret to calculate the token is correct.
-No error is expected.
+No error is expected.
+
+## Engine API Shanghai Upgrade Tests:
+See [withdrawals](suites/withdrawals/README.md).

simulators/ethereum/engine/client/engine.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"math/big"
 
+	"github.com/ethereum/go-ethereum/core"
 	api "github.com/ethereum/go-ethereum/core/beacon"
 	"github.com/ethereum/hive/hivesim"
 
@@ -24,12 +25,16 @@ type Eth interface {
 }
 
 type Engine interface {
-	ForkchoiceUpdatedV1(ctx context.Context, fcState *api.ForkchoiceStateV1, pAttributes *api.PayloadAttributesV1) (api.ForkChoiceResponse, error)
-	GetPayloadV1(ctx context.Context, payloadId *api.PayloadID) (api.ExecutableDataV1, error)
-	NewPayloadV1(ctx context.Context, payload *api.ExecutableDataV1) (api.PayloadStatusV1, error)
+	ForkchoiceUpdatedV1(ctx context.Context, fcState *api.ForkchoiceStateV1, pAttributes *api.PayloadAttributes) (api.ForkChoiceResponse, error)
+	ForkchoiceUpdatedV2(ctx context.Context, fcState *api.ForkchoiceStateV1, pAttributes *api.PayloadAttributes) (api.ForkChoiceResponse, error)
 
-	LatestForkchoiceSent() (fcState *api.ForkchoiceStateV1, pAttributes *api.PayloadAttributesV1)
-	LatestNewPayloadSent() (payload *api.ExecutableDataV1)
+	GetPayloadV1(ctx context.Context, payloadId *api.PayloadID) (api.ExecutableData, error)
+	GetPayloadV2(ctx context.Context, payloadId *api.PayloadID) (api.ExecutableData, *big.Int, error)
+	NewPayloadV1(ctx context.Context, payload *api.ExecutableData) (api.PayloadStatusV1, error)
+	NewPayloadV2(ctx context.Context, payload *api.ExecutableData) (api.PayloadStatusV1, error)
+
+	LatestForkchoiceSent() (fcState *api.ForkchoiceStateV1, pAttributes *api.PayloadAttributes)
+	LatestNewPayloadSent() (payload *api.ExecutableData)
 
 	LatestForkchoiceResponse() (fcuResponse *api.ForkChoiceResponse)
 	LatestNewPayloadResponse() (payloadResponse *api.PayloadStatusV1)
@@ -57,7 +62,7 @@ type EngineClient interface {
 }
 
 type EngineStarter interface {
-	StartClient(T *hivesim.T, testContext context.Context, ClientParams hivesim.Params, ClientFiles hivesim.Params, bootClients ...EngineClient) (EngineClient, error)
+	StartClient(T *hivesim.T, testContext context.Context, genesis *core.Genesis, ClientParams hivesim.Params, ClientFiles hivesim.Params, bootClients ...EngineClient) (EngineClient, error)
 }
 
 var (