Merge pull request #1254 from KRRT7/cf-e2e

add benchmark configuration and case for codeflash

Author: PawelPeczek-Roboflow

.github/workflows/codeflash.yml

@@ -39,6 +39,6 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install --upgrade setuptools
-          pip install -r requirements/_requirements.txt -r requirements/requirements.cpu.txt -r requirements/requirements.sdk.http.txt -r requirements/requirements.test.unit.txt -r requirements/requirements.http.txt -r requirements/requirements.transformers.txt -r requirements/requirements.code_analysis.txt
+          pip install -r requirements/_requirements.txt -r requirements/requirements.cpu.txt -r requirements/requirements.sdk.http.txt -r requirements/requirements.test.unit.txt -r requirements/requirements.http.txt -r requirements/requirements.transformers.txt -r requirements/requirements.code_analysis.txt -r requirements/requirements.cli.txt -r requirements/requirements.test.integration.txt
       - name: ⚡️Codeflash Optimization
-        run: codeflash
+        run: codeflash --benchmark

pyproject.toml

@@ -2,6 +2,7 @@
 # All paths are relative to this pyproject.toml's directory.
 module-root = "inference"
 tests-root = "tests/inference/unit_tests"
+benchmarks-root = "tests/benchmarks"
 test-framework = "pytest"
 ignore-paths = []
 formatter-cmds = ["black $file"]

tests/benchmarks/core/test_speed_benchmark.py

@@ -0,0 +1,45 @@
+import numpy as np
+import pytest
+from inference import get_model
+from inference_cli.lib.benchmark.dataset import load_dataset_images
+
+
+@pytest.fixture
+def dataset_reference() -> tuple[list[np.ndarray], set[tuple[int, int]]]:
+    dataset_images = load_dataset_images(
+        dataset_reference="coco",
+    )
+    return dataset_images, {i.shape[:2] for i in dataset_images}
+
+
+# args of inference benchmark python-package-speed -m yolov8n-seg-640 -bi 10000 command
+args = {
+   "dataset_reference": "coco",
+    "warm_up_inferences": 10,
+    "benchmark_inferences": 10000,
+    "batch_size": 1,
+    "api_key": None,
+    "model_configuration": None,
+    "output_location": None,
+}
+
+def test_benchmark_equivalent_rfdetr(benchmark, dataset_reference):
+    images, image_sizes = dataset_reference
+
+    model = get_model(model_id="rfdetr-base", api_key=None)
+
+    benchmark(model.infer, images)
+
+def test_benchmark_equivalent_yolov8n_seg(benchmark, dataset_reference):
+    images, image_sizes = dataset_reference
+
+    model = get_model(model_id="yolov8n-seg-640", api_key=None)
+
+    benchmark(model.infer, images)
+
+def test_benchmark_equivalent_yolov8n(benchmark, dataset_reference):
+    images, image_sizes = dataset_reference
+
+    model = get_model(model_id="yolov8n-640", api_key=None)
+
+    benchmark(model.infer, images)