diff --git a/.github/workflows/codeflash-optimize.yaml b/.github/workflows/codeflash-optimize.yaml index b3a6ecb6a..907f37788 100644 --- a/.github/workflows/codeflash-optimize.yaml +++ b/.github/workflows/codeflash-optimize.yaml @@ -20,7 +20,6 @@ jobs: CODEFLASH_AIS_SERVER: prod POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} CODEFLASH_API_KEY: ${{ secrets.CODEFLASH_API_KEY }} - CODEFLASH_PR_NUMBER: ${{ github.event.number }} COLUMNS: 110 steps: - name: 🛎️ Checkout diff --git a/codeflash/api/cfapi.py b/codeflash/api/cfapi.py index ca1cf7e4e..0f1fbf654 100644 --- a/codeflash/api/cfapi.py +++ b/codeflash/api/cfapi.py @@ -205,9 +205,10 @@ def get_blocklisted_functions() -> dict[str, set[str]] | dict[str, Any]: if pr_number is None: return {} - owner, repo = get_repo_owner_and_name() - information = {"pr_number": pr_number, "repo_owner": owner, "repo_name": repo} try: + owner, repo = get_repo_owner_and_name() + information = {"pr_number": pr_number, "repo_owner": owner, "repo_name": repo} + req = make_cfapi_request(endpoint="/verify-existing-optimizations", method="POST", payload=information) req.raise_for_status() content: dict[str, list[str]] = req.json() diff --git a/codeflash/cli_cmds/workflows/codeflash-optimize.yaml b/codeflash/cli_cmds/workflows/codeflash-optimize.yaml index ccf2ec382..a080d53d4 100644 --- a/codeflash/cli_cmds/workflows/codeflash-optimize.yaml +++ b/codeflash/cli_cmds/workflows/codeflash-optimize.yaml @@ -21,7 +21,6 @@ jobs: runs-on: ubuntu-latest env: CODEFLASH_API_KEY: ${{ secrets.CODEFLASH_API_KEY }} - CODEFLASH_PR_NUMBER: ${{ github.event.number }} {{ working_directory }} steps: - name: 🛎️ Checkout diff --git a/codeflash/code_utils/env_utils.py b/codeflash/code_utils/env_utils.py index 3def52e38..2c8ca22a1 100644 --- a/codeflash/code_utils/env_utils.py +++ b/codeflash/code_utils/env_utils.py @@ -74,17 +74,22 @@ def ensure_codeflash_api_key() -> bool: @lru_cache(maxsize=1) def get_pr_number() -> Optional[int]: + event_data = get_cached_gh_event_data() + pr_number = event_data.get("number") + if pr_number: + return int(pr_number) + pr_number = os.environ.get("CODEFLASH_PR_NUMBER") - if not pr_number: - return None - return int(pr_number) + if pr_number: + return int(pr_number) + return None def ensure_pr_number() -> bool: if not get_pr_number(): msg = ( - "CODEFLASH_PR_NUMBER not found in environment variables; make sure the Github Action is setting this so " - "Codeflash can comment on the right PR" + "Codeflash couldn't detect your pull request number. Are you running Codeflash within a GitHub Action?" + "If not, please set the CODEFLASH_PR_NUMBER environment variable to ensure Codeflash can comment on the correct PR." ) raise OSError(msg) return True @@ -96,22 +101,19 @@ def is_end_to_end() -> bool: @lru_cache(maxsize=1) -def is_repo_a_fork() -> bool: - event = get_cached_gh_event_data() - if event is None: - return False - return bool(event["repository"]["fork"]) - - -@lru_cache(maxsize=1) -def get_cached_gh_event_data() -> dict[str, Any] | None: +def get_cached_gh_event_data() -> dict[str, Any]: event_path = os.getenv("GITHUB_EVENT_PATH") if not event_path: - return None + return {} with Path(event_path).open() as f: return json.load(f) # type: ignore # noqa +def is_repo_a_fork() -> bool: + event = get_cached_gh_event_data() + return bool(event.get("repository", {}).get("fork", False)) + + @lru_cache(maxsize=1) def is_ci() -> bool: """Check if running in a CI environment.""" @@ -125,14 +127,5 @@ def is_LSP_enabled() -> bool: def is_pr_draft() -> bool: """Check if the PR is draft. in the github action context.""" - try: - event_path = os.getenv("GITHUB_EVENT_PATH") - pr_number = get_pr_number() - if pr_number is not None and event_path: - with Path(event_path).open() as f: - event_data = json.load(f) - return bool(event_data["pull_request"]["draft"]) - return False # noqa - except Exception as e: - logger.warning(f"Error checking if PR is draft: {e}") - return False + event = get_cached_gh_event_data() + return bool(event.get("pull_request", {}).get("draft", False)) diff --git a/codeflash/result/critic.py b/codeflash/result/critic.py index 6702b7bb4..dd8b5a564 100644 --- a/codeflash/result/critic.py +++ b/codeflash/result/critic.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from codeflash.cli_cmds.console import logger from codeflash.code_utils import env_utils @@ -26,7 +26,10 @@ def performance_gain(*, original_runtime_ns: int, optimized_runtime_ns: int) -> def speedup_critic( - candidate_result: OptimizedCandidateResult, original_code_runtime: int, best_runtime_until_now: int + candidate_result: OptimizedCandidateResult, + original_code_runtime: int, + best_runtime_until_now: int, + disable_gh_action_noise: Optional[bool] = None, ) -> bool: """Take in a correct optimized Test Result and decide if the optimization should actually be surfaced to the user. @@ -35,10 +38,11 @@ def speedup_critic( when the original runtime is less than 10 microseconds, and becomes MIN_IMPROVEMENT_THRESHOLD for any higher runtime. The noise floor is doubled when benchmarking on a (noisy) GitHub Action virtual instance, also we want to be more confident there. """ - in_github_actions_mode = bool(env_utils.get_pr_number()) noise_floor = 3 * MIN_IMPROVEMENT_THRESHOLD if original_code_runtime < 10000 else MIN_IMPROVEMENT_THRESHOLD - if in_github_actions_mode: - noise_floor = noise_floor * 2 # Increase the noise floor in GitHub Actions mode + if not disable_gh_action_noise: + in_github_actions_mode = bool(env_utils.get_pr_number()) + if in_github_actions_mode: + noise_floor = noise_floor * 2 # Increase the noise floor in GitHub Actions mode perf_gain = performance_gain( original_runtime_ns=original_code_runtime, optimized_runtime_ns=candidate_result.best_test_runtime diff --git a/docs/docs/getting-started/codeflash-github-actions.md b/docs/docs/getting-started/codeflash-github-actions.md index 7f9574dfb..177979ed8 100644 --- a/docs/docs/getting-started/codeflash-github-actions.md +++ b/docs/docs/getting-started/codeflash-github-actions.md @@ -57,7 +57,6 @@ jobs: runs-on: ubuntu-latest env: CODEFLASH_API_KEY: ${{ secrets.CODEFLASH_API_KEY }} - CODEFLASH_PR_NUMBER: ${{ github.event.number }} steps: - uses: actions/checkout@v4 with: diff --git a/tests/test_critic.py b/tests/test_critic.py index 569c2badb..49dfdfd64 100644 --- a/tests/test_critic.py +++ b/tests/test_critic.py @@ -41,7 +41,7 @@ def test_speedup_critic() -> None: total_candidate_timing=12, ) - assert speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now) # 20% improvement + assert speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now, True) # 20% improvement candidate_result = OptimizedCandidateResult( max_loop_count=5, @@ -52,7 +52,7 @@ def test_speedup_critic() -> None: optimization_candidate_index=0, ) - assert not speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now) # 6% improvement + assert not speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now, True) # 6% improvement original_code_runtime = 100000 best_runtime_until_now = 100000 @@ -66,7 +66,7 @@ def test_speedup_critic() -> None: optimization_candidate_index=0, ) - assert speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now) # 6% improvement + assert speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now, True) # 6% improvement def test_generated_test_critic() -> None: