feat(t8n): Support evmone's --opcode.count option

pyproject.toml

@@ -113,6 +113,7 @@ groupstats = "cli.show_pre_alloc_group_stats:main"
 extract_config = "cli.extract_config:extract_config"
 compare_fixtures = "cli.compare_fixtures:main"
 modify_static_test_gas_limits = "cli.modify_static_test_gas_limits:main"
+diff_opcode_counts = "cli.diff_opcode_counts:main"
 
 [tool.setuptools.packages.find]
 where = ["src"]

src/cli/diff_opcode_counts.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+"""
+Compare opcode counts between two folders of JSON fixtures.
+
+This script crawls two folders for JSON files, parses them using the Fixtures
+model, and compares the opcode_count field from the info section between
+fixtures with the same name.
+"""
+
+import sys
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import click
+
+from ethereum_clis.cli_types import OpcodeCount
+from ethereum_test_fixtures.file import Fixtures
+
+
+def find_json_files(directory: Path) -> List[Path]:
+    """Find all JSON files in a directory, excluding index.json files."""
+    json_files = []
+    if directory.is_dir():
+        for file_path in directory.rglob("*.json"):
+            if file_path.name != "index.json":
+                json_files.append(file_path)
+    return json_files
+
+
+def load_fixtures_from_file(
+    file_path: Path, remove_from_fixture_names: List[str]
+) -> Optional[Fixtures]:
+    """Load fixtures from a JSON file using the Fixtures model."""
+    try:
+        fixtures = Fixtures.model_validate_json(file_path.read_text())
+        renames = []
+        for k in fixtures.root:
+            new_name = None
+            for s in remove_from_fixture_names:
+                if s in k:
+                    if new_name is None:
+                        new_name = k.replace(s, "")
+                    else:
+                        new_name = new_name.replace(s, "")
+            if new_name is not None:
+                renames.append((k, new_name))
+        for old_name, new_name in renames:
+            fixtures.root[new_name] = fixtures.root.pop(old_name)
+        return fixtures
+    except Exception as e:
+        print(f"Error loading {file_path}: {e}", file=sys.stderr)
+        return None
+
+
+def extract_opcode_counts_from_fixtures(fixtures: Fixtures) -> Dict[str, OpcodeCount]:
+    """Extract opcode_count from info field for each fixture."""
+    opcode_counts = {}
+    for fixture_name, fixture in fixtures.items():
+        if hasattr(fixture, "info") and fixture.info and "opcode_count" in fixture.info:
+            try:
+                opcode_count = OpcodeCount.model_validate(fixture.info["opcode_count"])
+                opcode_counts[fixture_name] = opcode_count
+            except Exception as e:
+                print(f"Error parsing opcode_count for {fixture_name}: {e}", file=sys.stderr)
+    return opcode_counts
+
+
+def load_all_opcode_counts(
+    directory: Path, remove_from_fixture_names: List[str]
+) -> Dict[str, OpcodeCount]:
+    """Load all opcode counts from all JSON files in a directory."""
+    all_opcode_counts = {}
+    json_files = find_json_files(directory)
+
+    for json_file in json_files:
+        fixtures = load_fixtures_from_file(
+            json_file, remove_from_fixture_names=remove_from_fixture_names
+        )
+        if fixtures:
+            file_opcode_counts = extract_opcode_counts_from_fixtures(fixtures)
+            # Use fixture name as key, if there are conflicts, choose the last
+            all_opcode_counts.update(file_opcode_counts)
+
+    return all_opcode_counts
+
+
+def compare_opcode_counts(count1: OpcodeCount, count2: OpcodeCount) -> Dict[str, int]:
+    """Compare two opcode counts and return the differences."""
+    differences = {}
+
+    # Get all unique opcodes from both counts
+    all_opcodes = set(count1.root.keys()) | set(count2.root.keys())
+
+    for opcode in all_opcodes:
+        val1 = count1.root.get(opcode, 0)
+        val2 = count2.root.get(opcode, 0)
+        diff = val2 - val1
+        if diff != 0:
+            differences[str(opcode)] = diff
+
+    return differences
+
+
+@click.command()
+@click.argument("base", type=click.Path(exists=True, file_okay=False, path_type=Path))
+@click.argument("patch", type=click.Path(exists=True, file_okay=False, path_type=Path))
+@click.option(
+    "--show-common",
+    is_flag=True,
+    help="Print fixtures that contain identical opcode counts.",
+)
+@click.option(
+    "--show-missing",
+    is_flag=True,
+    help="Print fixtures only found in one of the folders.",
+)
+@click.option(
+    "--remove-from-fixture-names",
+    "-r",
+    multiple=True,
+    help="String to be removed from the fixture name, in case the fixture names have changed, "
+    "in order to make the comparison easier. "
+    "Can be specified multiple times.",
+)
+def main(
+    base: Path,
+    patch: Path,
+    show_common: bool,
+    show_missing: bool,
+    remove_from_fixture_names: List[str],
+):
+    """Crawl two folders, compare and print the opcode count diffs."""
+    print(f"Loading opcode counts from {base}...")
+    opcode_counts1 = load_all_opcode_counts(base, remove_from_fixture_names)
+    print(f"Found {len(opcode_counts1)} fixtures with opcode counts")
+
+    print(f"Loading opcode counts from {patch}...")
+    opcode_counts2 = load_all_opcode_counts(patch, remove_from_fixture_names)
+    print(f"Found {len(opcode_counts2)} fixtures with opcode counts")
+
+    # Find common fixture names
+    common_names = set(opcode_counts1.keys()) & set(opcode_counts2.keys())
+    only_in_1 = set(opcode_counts1.keys()) - set(opcode_counts2.keys())
+    only_in_2 = set(opcode_counts2.keys()) - set(opcode_counts1.keys())
+
+    print("\nSummary:")
+    print(f"  Common fixtures: {len(common_names)}")
+    print(f"  Only in {base.name}: {len(only_in_1)}")
+    print(f"  Only in {patch.name}: {len(only_in_2)}")
+
+    # Show missing fixtures if requested
+    if show_missing:
+        if only_in_1:
+            print(f"\nFixtures only in {base.name}:")
+            for name in sorted(only_in_1):
+                print(f"  {name}")
+
+        if only_in_2:
+            print(f"\nFixtures only in {patch.name}:")
+            for name in sorted(only_in_2):
+                print(f"  {name}")
+
+    # Compare common fixtures
+    differences_found = False
+    common_with_same_counts = 0
+
+    for fixture_name in sorted(common_names):
+        count1 = opcode_counts1[fixture_name]
+        count2 = opcode_counts2[fixture_name]
+
+        differences = compare_opcode_counts(count1, count2)
+
+        if differences:
+            differences_found = True
+            print(f"\n{fixture_name}:")
+            for opcode, diff in sorted(differences.items()):
+                if diff > 0:
+                    print(f"  +{diff} {opcode}")
+                else:
+                    print(f"  {diff} {opcode}")
+        elif show_common:
+            print(f"\n{fixture_name}: No differences")
+            common_with_same_counts += 1
+
+    if not differences_found:
+        print("\nNo differences found in opcode counts between common fixtures!")
+    elif show_common:
+        print(f"\n{common_with_same_counts} fixtures have identical opcode counts")
+
+
+if __name__ == "__main__":
+    main()

src/ethereum_clis/cli_types.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 from typing import Annotated, Any, Dict, List, Self
 
-from pydantic import Field
+from pydantic import Field, PlainSerializer, PlainValidator
 
 from ethereum_test_base_types import (
     BlobSchedule,
@@ -29,6 +29,7 @@
     Transaction,
     TransactionReceipt,
 )
+from ethereum_test_vm import Opcode, Opcodes
 from pytest_plugins.custom_logging import get_logger
 
 logger = get_logger(__name__)
@@ -175,6 +176,40 @@ def print(self):
             tx.print()
 
 
+_opcode_synonyms = {
+    "KECCAK256": "SHA3",
+}
+
+
+def validate_opcode(obj: Any) -> Opcodes | Opcode:
+    """Validate an opcode from a string."""
+    if isinstance(obj, Opcode) or isinstance(obj, Opcodes):
+        return obj
+    if isinstance(obj, str):
+        if obj in _opcode_synonyms:
+            obj = _opcode_synonyms[obj]
+        for op in Opcodes:
+            if str(op) == obj:
+                return op
+    raise Exception(f"Unable to validate {obj} (type={type(obj)})")
+
+
+class OpcodeCount(EthereumTestRootModel):
+    """Opcode count returned from the evm tool."""
+
+    root: Dict[
+        Annotated[Opcodes, PlainValidator(validate_opcode), PlainSerializer(lambda o: str(o))], int
+    ]
+
+    def __add__(self, other: Self) -> Self:
+        """Add two instances of opcode count dictionaries."""
+        assert isinstance(other, OpcodeCount), f"Incompatible type {type(other)}"
+        new_dict = self.model_dump() | other.model_dump()
+        for match_key in self.root.keys() & other.root.keys():
+            new_dict[match_key] = self.root[match_key] + other.root[match_key]
+        return self.__class__(new_dict)
+
+
 class Result(CamelModel):
     """Result of a transition tool output."""
 
@@ -202,6 +237,7 @@ class Result(CamelModel):
         BlockExceptionWithMessage | UndefinedException | None, ExceptionMapperValidator
     ] = None
     traces: Traces | None = None
+    opcode_count: OpcodeCount | None = None
 
 
 class TransitionToolInput(CamelModel):

src/ethereum_clis/clis/evmone.py

@@ -25,6 +25,7 @@ class EvmOneTransitionTool(TransitionTool):
     binary: Path
     cached_version: Optional[str] = None
     trace: bool
+    supports_opcode_count: ClassVar[bool] = True
 
     def __init__(
         self,

src/ethereum_clis/transition_tool.py

@@ -25,6 +25,7 @@
 from ethereum_test_types import Alloc, Environment, Transaction
 
 from .cli_types import (
+    OpcodeCount,
     Traces,
     TransactionReceipt,
     TransactionTraces,
@@ -71,6 +72,7 @@ class TransitionTool(EthereumCLI):
     t8n_use_server: bool = False
     server_url: str | None = None
     process: Optional[subprocess.Popen] = None
+    supports_opcode_count: ClassVar[bool] = False
 
     supports_xdist: ClassVar[bool] = True
 
@@ -248,6 +250,13 @@ def _evaluate_filesystem(
             "--state.chainid",
             str(t8n_data.chain_id),
         ]
+        if self.supports_opcode_count:
+            args.extend(
+                [
+                    "--opcode.count",
+                    "opcodes.json",
+                ]
+            )
 
         if self.trace:
             args.append("--trace")
@@ -308,6 +317,20 @@ def _evaluate_filesystem(
         output = TransitionToolOutput.model_validate(
             output_contents, context={"exception_mapper": self.exception_mapper}
         )
+        if self.supports_opcode_count:
+            opcode_count_file_path = Path(temp_dir.name) / "opcodes.json"
+            if opcode_count_file_path.exists():
+                opcode_count = OpcodeCount.model_validate_json(opcode_count_file_path.read_text())
+                output.result.opcode_count = opcode_count
+
+                if debug_output_path:
+                    dump_files_to_directory(
+                        debug_output_path,
+                        {
+                            "opcodes.json": opcode_count.model_dump(),
+                        },
+                    )
+
         if self.trace:
             output.result.traces = self.collect_traces(
                 output.result.receipts, temp_dir, debug_output_path

src/ethereum_test_specs/base.py

@@ -15,6 +15,7 @@
 from typing_extensions import Self
 
 from ethereum_clis import Result, TransitionTool
+from ethereum_clis.cli_types import OpcodeCount
 from ethereum_test_base_types import to_hex
 from ethereum_test_execution import BaseExecute, ExecuteFormat, LabeledExecuteFormat
 from ethereum_test_fixtures import (
@@ -75,6 +76,7 @@ class BaseTest(BaseModel):
     _operation_mode: OpMode | None = PrivateAttr(None)
     _gas_optimization: int | None = PrivateAttr(None)
     _gas_optimization_max_gas_limit: int | None = PrivateAttr(None)
+    _opcode_count: OpcodeCount | None = PrivateAttr(None)
 
     expected_benchmark_gas_used: int | None = None
     skip_gas_used_validation: bool = False
@@ -130,6 +132,7 @@ def from_test(
         )
         new_instance._request = base_test._request
         new_instance._operation_mode = base_test._operation_mode
+        new_instance._opcode_count = base_test._opcode_count
         return new_instance
 
     @classmethod

src/ethereum_test_specs/blockchain.py

@@ -531,6 +531,12 @@ def generate_block_data(
             slow_request=self.is_tx_gas_heavy_test(),
         )
 
+        if transition_tool_output.result.opcode_count is not None:
+            if self._opcode_count is None:
+                self._opcode_count = transition_tool_output.result.opcode_count
+            else:
+                self._opcode_count += transition_tool_output.result.opcode_count
+
         # One special case of the invalid transactions is the blob gas used,
         # since this value is not included in the transition tool result, but
         # it is included in the block header, and some clients check it before
@@ -746,6 +752,9 @@ def make_fixture(
                 )
         self.check_exception_test(exception=invalid_blocks > 0)
         self.verify_post_state(t8n, t8n_state=alloc)
+        info = {}
+        if self._opcode_count is not None:
+            info["opcode_count"] = self._opcode_count.model_dump()
         return BlockchainFixture(
             fork=fork,
             genesis=genesis.header,
@@ -760,6 +769,7 @@ def make_fixture(
                 blob_schedule=FixtureBlobSchedule.from_blob_schedule(fork.blob_schedule()),
                 chain_id=self.chain_id,
             ),
+            info=info,
         )
 
     def make_hive_fixture(
@@ -812,6 +822,9 @@ def make_hive_fixture(
         self.verify_post_state(t8n, t8n_state=alloc)
 
         # Create base fixture data, common to all fixture formats
+        info = {}
+        if self._opcode_count is not None:
+            info["opcode_count"] = self._opcode_count.model_dump()
         fixture_data = {
             "fork": fork,
             "genesis": genesis.header,
@@ -825,6 +838,7 @@ def make_hive_fixture(
                 chain_id=self.chain_id,
                 blob_schedule=FixtureBlobSchedule.from_blob_schedule(fork.blob_schedule()),
             ),
+            "info": info,
         }
 
         # Add format-specific fields