Skip to content

Commit ab4f5f2

Browse files
r0qscameel
r0qs
and
cameel
committed
Initial prototype of prb-math external tests using foundry rewritten in python
Co-authored-by: Kamil Śliwak <[email protected]>
1 parent ecd56e6 commit ab4f5f2

File tree

6 files changed

+491
-3
lines changed

6 files changed

+491
-3
lines changed

‎.circleci/config.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -691,7 +691,7 @@ defaults:
691691
name: t_native_test_ext_prb_math
692692
project: prb-math
693693
binary_type: native
694-
image: cimg/node:18.16
694+
image: cimg/rust:1.70
695695

696696
- job_native_test_ext_elementfi: &job_native_test_ext_elementfi
697697
<<: *requires_b_ubu_static
@@ -1724,11 +1724,13 @@ workflows:
17241724
- t_native_test_ext_yield_liquidator
17251725
- t_native_test_ext_perpetual_pools
17261726
- t_native_test_ext_uniswap
1727-
- t_native_test_ext_prb_math
17281727
- t_native_test_ext_elementfi
17291728
- t_native_test_ext_brink
17301729
# NOTE: We are disabling gp2 tests due to constant failures.
17311730
#- t_native_test_ext_gp2
1731+
# TODO: Dropping prb-math from the benchmarks since it is not implemented yet
1732+
# in the new Foundry external testing infrastructure.
1733+
# - t_native_test_ext_prb_math
17321734
# NOTE: The external tests below were commented because they
17331735
# depend on a specific version of hardhat which does not support shanghai EVM.
17341736
#- t_native_test_ext_trident

‎scripts/externalTests/runners/base.py

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
#!/usr/bin/env python3
2+
3+
# ------------------------------------------------------------------------------
4+
# This file is part of solidity.
5+
#
6+
# solidity is free software: you can redistribute it and/or modify
7+
# it under the terms of the GNU General Public License as published by
8+
# the Free Software Foundation, either version 3 of the License, or
9+
# (at your option) any later version.
10+
#
11+
# solidity is distributed in the hope that it will be useful,
12+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
# GNU General Public License for more details.
15+
#
16+
# You should have received a copy of the GNU General Public License
17+
# along with solidity. If not, see <http://www.gnu.org/licenses/>
18+
#
19+
# (c) 2023 solidity contributors.
20+
# ------------------------------------------------------------------------------
21+
22+
import os
23+
import subprocess
24+
from abc import ABCMeta
25+
from abc import abstractmethod
26+
from dataclasses import dataclass
27+
from dataclasses import field
28+
from pathlib import Path
29+
from shutil import rmtree
30+
from tempfile import mkdtemp
31+
from textwrap import dedent
32+
from typing import List
33+
from typing import Set
34+
35+
from test_helpers import download_project
36+
from test_helpers import get_solc_short_version
37+
from test_helpers import parse_command_line
38+
from test_helpers import parse_custom_presets
39+
from test_helpers import parse_solc_version
40+
from test_helpers import replace_version_pragmas
41+
from test_helpers import settings_from_preset
42+
from test_helpers import SettingsPreset
43+
44+
CURRENT_EVM_VERSION: str = "shanghai"
45+
46+
@dataclass
47+
class TestConfig:
48+
name: str
49+
repo_url: str
50+
ref_type: str
51+
ref: str
52+
compile_only_presets: List[SettingsPreset] = field(default_factory=list)
53+
settings_presets: List[SettingsPreset] = field(default_factory=lambda: list(SettingsPreset))
54+
evm_version: str = field(default=CURRENT_EVM_VERSION)
55+
56+
def selected_presets(self) -> Set[SettingsPreset]:
57+
return set(self.compile_only_presets + self.settings_presets)
58+
59+
60+
class BaseRunner(metaclass=ABCMeta):
61+
config: TestConfig
62+
solc_binary_type: str
63+
solc_binary_path: Path
64+
presets: Set[SettingsPreset]
65+
66+
def __init__(self, argv, config: TestConfig):
67+
args = parse_command_line(f"{config.name} external tests", argv)
68+
self.config = config
69+
self.solc_binary_type = args.solc_binary_type
70+
self.solc_binary_path = args.solc_binary_path
71+
self.presets = parse_custom_presets(args.selected_presets) if args.selected_presets else config.selected_presets()
72+
self.env = os.environ.copy()
73+
self.tmp_dir = mkdtemp(prefix=f"ext-test-{config.name}-")
74+
self.test_dir = Path(self.tmp_dir) / "ext"
75+
76+
def setup_solc(self) -> str:
77+
if self.solc_binary_type == "solcjs":
78+
# TODO: add support to solc-js
79+
raise NotImplementedError()
80+
print("Setting up solc...")
81+
solc_version_output = subprocess.check_output(
82+
[self.solc_binary_path, "--version"],
83+
shell=False,
84+
encoding="utf-8"
85+
).split(":")[1]
86+
return parse_solc_version(solc_version_output)
87+
88+
@staticmethod
89+
def enter_test_dir(fn):
90+
"""Run a function inside the test directory"""
91+
92+
previous_dir = os.getcwd()
93+
def f(self, *args, **kwargs):
94+
try:
95+
assert self.test_dir is not None
96+
os.chdir(self.test_dir)
97+
return fn(self, *args, **kwargs)
98+
finally:
99+
# Restore the previous directory after execute fn
100+
os.chdir(previous_dir)
101+
return f
102+
103+
def setup_environment(self):
104+
"""Configure the project build environment"""
105+
print("Configuring Runner building environment...")
106+
replace_version_pragmas(self.test_dir)
107+
108+
@enter_test_dir
109+
def clean(self):
110+
"""Clean temporary directories"""
111+
rmtree(self.tmp_dir)
112+
113+
@enter_test_dir
114+
@abstractmethod
115+
def configure(self):
116+
raise NotImplementedError()
117+
118+
@enter_test_dir
119+
@abstractmethod
120+
def compile(self, preset: SettingsPreset):
121+
raise NotImplementedError()
122+
123+
@enter_test_dir
124+
@abstractmethod
125+
def run_test(self):
126+
raise NotImplementedError()
127+
128+
def run_test(runner: BaseRunner):
129+
print(f"Testing {runner.config.name}...\n===========================")
130+
print(f"Selected settings presets: {' '.join(p.value for p in runner.presets)}")
131+
132+
# Configure solc compiler
133+
solc_version = runner.setup_solc()
134+
print(f"Using compiler version {solc_version}")
135+
136+
# Download project
137+
download_project(runner.test_dir, runner.config.repo_url, runner.config.ref_type, runner.config.ref)
138+
139+
# Configure run environment
140+
runner.setup_environment()
141+
142+
# Configure TestRunner instance
143+
print(dedent(f"""\
144+
Configuring runner's profiles with:
145+
-------------------------------------
146+
Binary type: {runner.solc_binary_type}
147+
Compiler path: {runner.solc_binary_path}
148+
-------------------------------------
149+
"""))
150+
runner.configure()
151+
for preset in runner.presets:
152+
print("Running compile function...")
153+
settings = settings_from_preset(preset, runner.config.evm_version)
154+
print(dedent(f"""\
155+
-------------------------------------
156+
Settings preset: {preset.value}
157+
Settings: {settings}
158+
EVM version: {runner.config.evm_version}
159+
Compiler version: {get_solc_short_version(solc_version)}
160+
Compiler version (full): {solc_version}
161+
-------------------------------------
162+
"""))
163+
runner.compile(preset)
164+
# TODO: COMPILE_ONLY should be a command-line option
165+
if os.environ.get("COMPILE_ONLY") == "1" or preset in runner.config.compile_only_presets:
166+
print("Skipping test function...")
167+
else:
168+
print("Running test function...")
169+
runner.run_test()
170+
# TODO: store_benchmark_report
171+
runner.clean()
172+
print("Done.")

‎scripts/externalTests/runners/foundry.py

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
#!/usr/bin/env python3
2+
3+
# ------------------------------------------------------------------------------
4+
# This file is part of solidity.
5+
#
6+
# solidity is free software: you can redistribute it and/or modify
7+
# it under the terms of the GNU General Public License as published by
8+
# the Free Software Foundation, either version 3 of the License, or
9+
# (at your option) any later version.
10+
#
11+
# solidity is distributed in the hope that it will be useful,
12+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
# GNU General Public License for more details.
15+
#
16+
# You should have received a copy of the GNU General Public License
17+
# along with solidity. If not, see <http://www.gnu.org/licenses/>
18+
#
19+
# (c) 2023 solidity contributors.
20+
# ------------------------------------------------------------------------------
21+
22+
import os
23+
import re
24+
import subprocess
25+
from shutil import which
26+
from textwrap import dedent
27+
from typing import Optional
28+
29+
from runners.base import BaseRunner
30+
from test_helpers import SettingsPreset
31+
from test_helpers import settings_from_preset
32+
33+
def run_forge_command(command: str, env: Optional[dict] = None):
34+
subprocess.run(
35+
command.split(),
36+
env=env if env is not None else os.environ.copy(),
37+
check=True
38+
)
39+
40+
41+
class FoundryRunner(BaseRunner):
42+
"""Configure and run Foundry-based projects"""
43+
44+
FOUNDRY_CONFIG_FILE = "foundry.toml"
45+
46+
def setup_environment(self):
47+
super().setup_environment()
48+
if which("forge") is None:
49+
raise RuntimeError("Forge not found.")
50+
51+
@staticmethod
52+
def profile_name(preset: SettingsPreset):
53+
"""Returns foundry profile name"""
54+
# Replace - or + by underscore to avoid invalid toml syntax
55+
return re.sub(r"(\-|\+)+", "_", preset.value)
56+
57+
@staticmethod
58+
def profile_section(profile_fields: dict) -> str:
59+
return dedent("""\
60+
[profile.{name}]
61+
gas_reports = ["*"]
62+
auto_detect_solc = false
63+
solc = "{solc}"
64+
evm_version = "{evm_version}"
65+
optimizer = {optimizer}
66+
via_ir = {via_ir}
67+
68+
[profile.{name}.optimizer_details]
69+
yul = {yul}
70+
""").format(**profile_fields)
71+
72+
@BaseRunner.enter_test_dir
73+
def configure(self):
74+
"""Configure forge tests profiles"""
75+
76+
profiles = []
77+
for preset in self.presets:
78+
settings = settings_from_preset(preset, self.config.evm_version)
79+
profiles.append(self.profile_section({
80+
"name": self.profile_name(preset),
81+
"solc": self.solc_binary_path,
82+
"evm_version": self.config.evm_version,
83+
"optimizer": str(settings["optimizer"]["enabled"]).lower(),
84+
"via_ir": str(settings["viaIR"]).lower(),
85+
"yul": str(settings["optimizer"]["details"]["yul"]).lower(),
86+
}))
87+
88+
with open(
89+
file=self.test_dir / self.FOUNDRY_CONFIG_FILE,
90+
mode="a",
91+
encoding="utf-8",
92+
) as f:
93+
for profile in profiles:
94+
f.write(profile)
95+
96+
run_forge_command("forge install", self.env)
97+
98+
@BaseRunner.enter_test_dir
99+
def compile(self, preset: SettingsPreset):
100+
"""Compile project"""
101+
102+
# Set the Foundry profile environment variable
103+
self.env.update({"FOUNDRY_PROFILE": self.profile_name(preset)})
104+
run_forge_command("forge build", self.env)
105+
106+
@BaseRunner.enter_test_dir
107+
def run_test(self):
108+
"""Run project tests"""
109+
110+
run_forge_command("forge test --gas-report", self.env)

0 commit comments

Comments
 (0)