Skip to content

Commit 1c94966

Browse files
vjatoth-qtivjatoth-qti
committed
Add evaluators to the recipes
1 parent abf5622 commit 1c94966

File tree

18 files changed

+774
-71
lines changed

18 files changed

+774
-71
lines changed

‎google-bert-bert-base-multilingual-cased/QNN/README.md‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,10 @@
22

33
### QNN-GPU:
44

5-
Running QNN-GPU configs requires features and fixes that are not available in the released Olive version 0.9.3.
6-
To ensure compatibility, please install Olive directly from the source at the required commit:
5+
Please install Olive directly using:
76

87
```bash
9-
pip install git+https://github.com/microsoft/Olive.git@da24463e14ed976503dc5871572b285bc5ddc4b2
10-
```
11-
12-
If you previously installed Olive via PyPI or pinned it to version 0.9.3, please uninstall it first and then use the above
13-
commit to install:
14-
15-
```bash
16-
pip uninstall olive-ai
8+
pip install olive-ai
179
```
1810

1911
To run the config:

‎google-bert-bert-base-multilingual-cased/QNN/config_gpu_fp32.json‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,71 @@
4747
]
4848
}
4949
},
50+
"data_configs": [
51+
{
52+
"name": "evaluation_data_config",
53+
"type": "HuggingfaceContainer",
54+
"load_dataset_config": {
55+
"data_name": "facebook/xnli",
56+
"subset": "en",
57+
"split": "validation"
58+
},
59+
"pre_process_data_config": {
60+
"input_cols": [
61+
"premise"
62+
],
63+
"padding": "max_length",
64+
"max_length": 128,
65+
"max_samples": 10
66+
},
67+
"dataloader_config": {
68+
"batch_size": 1
69+
}
70+
}
71+
],
72+
"evaluators": {
73+
"common_evaluator": {
74+
"metrics": [
75+
{
76+
"name": "latency",
77+
"type": "latency",
78+
"data_config": "evaluation_data_config",
79+
"sub_types": [
80+
{
81+
"name": "avg",
82+
"priority": 1,
83+
"goal": {
84+
"type": "percent-min-improvement",
85+
"value": 0.1
86+
}
87+
},
88+
{
89+
"name": "max"
90+
},
91+
{
92+
"name": "min"
93+
}
94+
]
95+
},
96+
{
97+
"name": "throughput",
98+
"type": "throughput",
99+
"data_config": "evaluation_data_config",
100+
"sub_types": [
101+
{
102+
"name": "avg"
103+
},
104+
{
105+
"name": "max"
106+
},
107+
{
108+
"name": "min"
109+
}
110+
]
111+
}
112+
]
113+
}
114+
},
50115
"passes": {
51116
"conversion": {
52117
"type": "OnnxConversion",
@@ -66,6 +131,7 @@
66131
}
67132
},
68133
"target": "target_system",
134+
"evaluator": "common_evaluator",
69135
"cache_dir": "cache",
70136
"output_dir": "output",
71137
"evaluate_input_model": false,

‎google-vit-base-patch16-224/QNN/README.md‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,10 @@ python .\val_tiny_imagenet.py
1919

2020
### QNN-GPU:
2121

22-
Running QNN-GPU configs requires features and fixes that are not available in the released Olive version 0.9.3.
23-
To ensure compatibility, please install Olive directly from the source at the required commit:
22+
Please install Olive directly using:
2423

2524
```bash
26-
pip install git+https://github.com/microsoft/Olive.git@da24463e14ed976503dc5871572b285bc5ddc4b2
27-
```
28-
29-
If you previously installed Olive via PyPI or pinned it to version 0.9.3, please uninstall it first and then use the above
30-
commit to install:
31-
32-
```bash
33-
pip uninstall olive-ai
25+
pip install olive-ai
3426
```
3527

3628
To run the config:

‎google-vit-base-patch16-224/QNN/config_gpu_fp32.json‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,59 @@
3333
]
3434
}
3535
},
36+
"data_configs": [
37+
{
38+
"name": "evaluation_data_config",
39+
"type": "HuggingfaceContainer",
40+
"user_script": "vit-base-patch16-224.py",
41+
"load_dataset_config": {
42+
"data_name": "timm/mini-imagenet",
43+
"split": "validation",
44+
"streaming": true,
45+
"trust_remote_code": true
46+
},
47+
"pre_process_data_config": {
48+
"type": "dataset_pre_process",
49+
"size": 1000,
50+
"cache_key": "imagedata_evaluation"
51+
},
52+
"post_process_data_config": {
53+
"type": "dataset_post_process"
54+
}
55+
}
56+
],
57+
"evaluators": {
58+
"common_evaluator": {
59+
"metrics": [
60+
{
61+
"name": "accuracy",
62+
"type": "accuracy",
63+
"data_config": "evaluation_data_config",
64+
"sub_types": [
65+
{
66+
"name": "accuracy_score",
67+
"priority": 1,
68+
"metric_config": {
69+
"task": "multiclass",
70+
"num_classes": 1000
71+
}
72+
}
73+
]
74+
},
75+
{
76+
"name": "latency",
77+
"type": "latency",
78+
"data_config": "evaluation_data_config",
79+
"sub_types": [
80+
{
81+
"name": "avg",
82+
"priority": 2
83+
}
84+
]
85+
}
86+
]
87+
}
88+
},
3689
"passes": {
3790
"conversion": {
3891
"type": "OnnxConversion",
@@ -49,6 +102,7 @@
49102
}
50103
},
51104
"target": "target_system",
105+
"evaluator": "common_evaluator",
52106
"cache_dir": "cache",
53107
"output_dir": "output",
54108
"evaluate_input_model": false,

‎google-vit-base-patch16-224/QNN/vit-base-patch16-224.py‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
# -------------------------------------------------------------------------
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# Licensed under the MIT License.
4+
# --------------------------------------------------------------------------
5+
from logging import getLogger
6+
from pathlib import Path
7+
8+
import numpy as np
9+
import torchvision.transforms as transforms
10+
import transformers
11+
from torch import from_numpy
12+
from torch.utils.data import Dataset
13+
14+
from olive.data.registry import Registry
15+
16+
logger = getLogger(__name__)
17+
18+
def get_imagenet_label_map():
19+
import json
20+
cache_file = Path(f"./cache/data/imagenet_class_index.json")
21+
if not cache_file.exists():
22+
import requests
23+
imagenet_class_index_url = (
24+
"https://raw.githubusercontent.com/pytorch/vision/main/gallery/assets/imagenet_class_index.json"
25+
)
26+
response = requests.get(imagenet_class_index_url)
27+
response.raise_for_status() # Ensure the request was successful
28+
content = response.json()
29+
cache_file.parent.resolve().mkdir(parents=True, exist_ok=True)
30+
with open(cache_file, "w") as f:
31+
json.dump(content, f)
32+
else:
33+
with open(cache_file) as f:
34+
content = json.loads(f.read())
35+
36+
return {v[0]: int(k) for k, v in content.items()}
37+
38+
def adapt_label_for_mini_imagenet(labels: list, label_names: list):
39+
label_map = get_imagenet_label_map()
40+
return [label_map[label_names[x]] for x in labels]
41+
42+
class ImagenetDataset(Dataset):
43+
def __init__(self, data):
44+
self.images = from_numpy(data["images"])
45+
self.labels = from_numpy(data["labels"])
46+
47+
def __len__(self):
48+
return min(len(self.images), len(self.labels))
49+
50+
def __getitem__(self, idx):
51+
return {"pixel_values": self.images[idx]}, self.labels[idx]
52+
53+
54+
@Registry.register_post_process()
55+
def dataset_post_process(output):
56+
return (
57+
output.logits.argmax(axis=1)
58+
if isinstance(output, transformers.modeling_outputs.ModelOutput)
59+
else output.argmax(axis=1)
60+
)
61+
62+
from transformers import AutoImageProcessor
63+
processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224", use_fast=True)
64+
65+
@Registry.register_pre_process()
66+
def dataset_pre_process(output_data, **kwargs):
67+
shuffle = kwargs.get("shuffle", True)
68+
if shuffle:
69+
seed = kwargs.get("seed", 42)
70+
output_data = output_data.shuffle(seed=seed)
71+
cache_key = kwargs.get("cache_key")
72+
size = kwargs.get("size", 256)
73+
cache_file = None
74+
if cache_key:
75+
cache_file = Path(f"./cache/data/{cache_key}_{output_data.info.dataset_name}_{size}.npz")
76+
if cache_file.exists():
77+
with np.load(Path(cache_file)) as data:
78+
return ImagenetDataset(data)
79+
80+
labels = []
81+
images = []
82+
for i, sample in enumerate(output_data):
83+
if i >= size:
84+
break
85+
image = sample["image"]
86+
label = sample["label"]
87+
image = image.convert("RGB")
88+
image = processor(image)["pixel_values"][0]
89+
images.append(image)
90+
labels.append(label)
91+
92+
if(output_data.info.dataset_name == "mini-imagenet"):
93+
labels = adapt_label_for_mini_imagenet(labels, output_data.features["label"].names)
94+
result_data = ImagenetDataset({"images": np.array(images), "labels": np.array(labels)})
95+
96+
if cache_file:
97+
cache_file.parent.resolve().mkdir(parents=True, exist_ok=True)
98+
np.savez(cache_file, images=np.array(images), labels=np.array(labels))
99+
100+
return result_data

‎intel-bert-base-uncased-mrpc/QNN/README.md‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,10 @@
22

33
### QNN-GPU:
44

5-
Running QNN-GPU configs requires features and fixes that are not available in the released Olive version 0.9.3.
6-
To ensure compatibility, please install Olive directly from the source at the required commit:
5+
Please install Olive directly using:
76

87
```bash
9-
pip install git+https://github.com/microsoft/Olive.git@da24463e14ed976503dc5871572b285bc5ddc4b2
10-
```
11-
12-
If you previously installed Olive via PyPI or pinned it to version 0.9.3, please uninstall it first and then use the above
13-
commit to install:
14-
15-
```bash
16-
pip uninstall olive-ai
8+
pip install olive-ai
179
```
1810

1911
To run the config:

0 commit comments

Comments
 (0)