add reader_onnx

Author: Semyon1104

app/Converters/CMakeLists.txt

@@ -1,6 +1,6 @@
 add_executable(Reader_weights reader_weights_sample.cpp)
-
+add_executable(Reader_weights_onnx reader_weights_sample_onnx.cpp)
 target_link_libraries(Reader_weights PUBLIC perf_lib layers_lib reader_lib)
-
+target_link_libraries(Reader_weights_onnx PUBLIC perf_lib layers_lib reader_lib)
 add_definitions(-DMODEL_PATH_H5="${CMAKE_SOURCE_DIR}/docs/jsons/model_data_alexnet_1.json")
 add_definitions(-DMODEL_PATH_GOOGLENET_ONNX="${CMAKE_SOURCE_DIR}/docs/jsons/googlenet_onnx_model.json")

app/Converters/parser.py

@@ -17,8 +17,8 @@ def __init__(self, **kwargs):
 
 # Пути к модели и JSON файлу
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-MODEL_PATH = os.path.join(BASE_DIR, 'docs', 'AlexNet-model.h5')
-MODEL_DATA_PATH = os.path.join(BASE_DIR, 'docs', 'model_data_alexnet_1.json')
+MODEL_PATH = os.path.join(BASE_DIR, 'docs\\models', 'AlexNet-model.h5')
+MODEL_DATA_PATH = os.path.join(BASE_DIR, 'docs\\jsons', 'model_data_alexnet_1.json')
 
 # Загрузка модели
 model = load_model(MODEL_PATH, custom_objects={'GlorotUniform': CustomGlorotUniform, 'Zeros': CustomZeros})

app/Converters/parser_onnx.py

@@ -7,11 +7,9 @@
 def onnx_to_json(model_path, output_json_path):
     # Загрузка модели
     model = onnx.load(model_path)
-
-    # Проверка валидности модели
     onnx.checker.check_model(model)
 
-    # Создаем словарь для быстрого доступа к инициализаторам по их именам
+    # Словарь инициализаторов
     initializers_dict = {
         init.name: {
             "data_type": init.data_type,
@@ -21,10 +19,7 @@ def onnx_to_json(model_path, output_json_path):
         for init in model.graph.initializer
     }
 
-    # Создаем список слоев в формате Keras
     layer_info = []
-
-    # Обрабатываем входные данные как первый слой
     input_layer = {
         "index": 0,
         "name": "input_1",
@@ -34,56 +29,64 @@ def onnx_to_json(model_path, output_json_path):
     }
     layer_info.append(input_layer)
 
-    # Обработка узлов (операций) как слоев
     for node in model.graph.node:
-        # Создаем запись слоя
         layer_data = {
             "index": len(layer_info),
             "name": node.name.replace('/', '_'),
             "type": node.op_type,
-            "weights": [],
-            "attributes": {}  # Сохраняем все атрибуты здесь
+            "attributes": {}
         }
 
-        # Обрабатываем все атрибуты узла
+        # Обработка атрибутов
         for attr in node.attribute:
             attr_value = helper.get_attribute_value(attr)
-
-            # Преобразуем разные типы атрибутов
             if isinstance(attr_value, bytes):
                 attr_value = attr_value.decode('utf-8', errors='ignore')
             elif hasattr(attr_value, 'tolist'):
                 attr_value = attr_value.tolist()
             elif str(type(attr_value)).endswith("RepeatedScalarContainer'>"):
                 attr_value = list(attr_value)
-
-            # Сохраняем атрибут
             layer_data["attributes"][attr.name] = attr_value
 
-            # Специальная обработка для удобства (можно использовать или игнорировать)
             if attr.name == "pads":
                 layer_data["padding"] = "same" if any(p > 0 for p in attr_value) else "valid"
             elif attr.name == "kernel_shape":
                 layer_data["kernel_size"] = attr_value
             elif attr.name == "strides":
                 layer_data["strides"] = attr_value
 
-        # Добавляем веса в формате Keras (один список с ядрами и bias)
-        layer_weights = []
+        # Собираем все initializers для этого узла
+        node_init = []
         for input_name in node.input:
             if input_name in initializers_dict:
-                init = initializers_dict[input_name]
-                if len(init["dims"]) > 1:  # Ядра свертки/матрицы весов
-                    layer_weights.extend(init["values"])
-                else:  # Bias
-                    layer_weights.append(init["values"])
-
-        if layer_weights:
-            layer_data["weights"] = layer_weights
+                node_init.append(initializers_dict[input_name])
+
+        # Новая логика: разделяем weights/value/bias
+        if len(node_init) == 1:
+            init = node_init[0]
+            if len(init["dims"]) == 0 or (len(init["dims"]) == 1 and init["dims"][0] == 1):
+                # Скалярное значение или массив из одного элемента
+                layer_data["value"] = init["values"] if len(init["dims"]) == 0 else init["values"][0]
+            else:
+                # Многомерные данные
+                layer_data["weights"] = init["values"]
+        elif len(node_init) > 1:
+            # Для нескольких инициализаторов: weights + bias
+            weights = []
+            for init in node_init[:-1]:
+                if len(init["dims"]) > 0:
+                    weights.extend(init["values"]) if isinstance(init["values"][0], list) else weights.append(
+                        init["values"])
+
+            if weights:
+                layer_data["weights"] = weights
+
+            # Последний инициализатор - bias (если одномерный)
+            if len(node_init[-1]["dims"]) == 1:
+                layer_data["bias"] = node_init[-1]["values"]
 
         layer_info.append(layer_data)
 
-    # Custom JSON encoder
     class CustomEncoder(json.JSONEncoder):
         def default(self, obj):
             if hasattr(obj, 'tolist'):
@@ -92,7 +95,6 @@ def default(self, obj):
                 return list(obj)
             return super().default(obj)
 
-    # Сохранение в JSON файл
     with open(output_json_path, 'w') as f:
         json.dump(layer_info, f, indent=2, cls=CustomEncoder)
 

app/Converters/reader_weights_sample_onnx.cpp

@@ -0,0 +1,81 @@
+#include <iostream>
+
+#include "Weights_Reader/reader_weights_onnx.hpp"
+
+int main() {
+  std::string json_file = MODEL_PATH_GOOGLENET_ONNX;
+  json model_data = read_json_onnx(json_file);
+
+  std::cout << "Model contains " << model_data.size()
+            << " layers:" << std::endl;
+  std::cout << "--------------------------------------------------"
+            << std::endl;
+
+  for (const auto& layer_data : model_data) {
+    int layer_index = layer_data["index"];
+    std::string layer_name = layer_data["name"];
+    std::string layer_type = layer_data["type"];
+    bool has_weights =
+        layer_data.contains("weights") && !layer_data["weights"].empty();
+    bool has_value = layer_data.contains("value");
+
+    std::cout << "Layer " << layer_index << ": " << layer_name << " ("
+              << layer_type << ")" << std::endl;
+
+    // Вывод атрибутов
+    if (layer_data.contains("attributes") &&
+        !layer_data["attributes"].empty()) {
+      std::cout << "  Attributes:" << std::endl;
+      for (const auto& [key, value] : layer_data["attributes"].items()) {
+        std::cout << "    " << key << ": ";
+        if (value.is_array()) {
+          std::cout << "[";
+          for (const auto& v : value) {
+            if (v.is_number())
+              std::cout << v.get<float>() << " ";
+            else if (v.is_string())
+              std::cout << v.get<std::string>() << " ";
+          }
+          std::cout << "]";
+        } else if (value.is_number()) {
+          std::cout << value.get<float>();
+        } else if (value.is_string()) {
+          std::cout << value.get<std::string>();
+        }
+        std::cout << std::endl;
+      }
+    }
+
+    // Обработка value (скалярных значений)
+    if (has_value) {
+      try {
+        float value = layer_data["value"].get<float>();
+        std::cout << "  Value: " << value << std::endl;
+      } catch (const std::exception& e) {
+        std::cerr << "  Error processing value: " << e.what() << std::endl;
+      }
+    }
+
+    // Обработка весов
+    if (has_weights) {
+      try {
+        Tensor tensor = create_tensor_from_json_onnx(layer_data, Type::kFloat);
+
+        std::cout << "  Weights shape: " << tensor.get_shape() << std::endl;
+
+        if (!tensor.get_bias().empty()) {
+          std::cout << "  Bias size: " << tensor.get_bias().size() << std::endl;
+        }
+      } catch (const std::exception& e) {
+        std::cerr << "  Error processing weights: " << e.what() << std::endl;
+      }
+    } else if (!has_value) {
+      std::cout << "  No weights or value" << std::endl;
+    }
+
+    std::cout << "--------------------------------------------------"
+              << std::endl;
+  }
+
+  return 0;
+}

include/Weights_Reader/reader_weights_onnx.hpp

@@ -0,0 +1,17 @@
+#include <nlohmann/json.hpp>
+#include <string>
+#include <vector>
+
+#include "layers/Tensor.hpp"
+
+using json = nlohmann::json;
+using namespace itlab_2023;
+
+json read_json_onnx(const std::string& filename);
+void extract_values_from_json_onnx(const json& j, std::vector<float>& values);
+void parse_json_shape_onnx(const json& j, std::vector<size_t>& shape,
+                           size_t dim);
+Tensor create_tensor_from_json_onnx(const json& j, Type type);
+
+void parse_onnx_weights(const json& j, std::vector<float>& weights,
+                        std::vector<float>& bias);

include/layers/Tensor.hpp

@@ -83,10 +83,6 @@ class Tensor {
       throw std::invalid_argument("Incorrect vector size given to Tensor");
     }
     values_ = a;
-    if (bias.size() != shape_[shape_.dims() - 1]) {
-      throw std::invalid_argument(
-          "Bias size does not match the last dimension of the shape");
-    }
   }
 
   Tensor(const Tensor& t) = default;

src/Weights_Reader/reader_weights_onnx.cpp

@@ -0,0 +1,82 @@
+#include "Weights_Reader/reader_weights_onnx.hpp"
+
+#include <fstream>
+#include <iostream>
+#include <nlohmann/json.hpp>
+#include <stdexcept>
+#include <vector>
+
+using json = nlohmann::json;
+
+json read_json_onnx(const std::string& filename) {
+  std::ifstream ifs(filename);
+  if (!ifs.is_open()) {
+    throw std::runtime_error("Failed to open JSON file: " + filename);
+  }
+
+  json model_data;
+  try {
+    ifs >> model_data;
+  } catch (const json::parse_error& e) {
+    throw std::runtime_error("JSON parse error: " + std::string(e.what()));
+  }
+
+  return model_data;
+}
+
+void extract_values_from_json_onnx(const json& j, std::vector<float>& values) {
+  if (j.is_array()) {
+    for (const auto& item : j) {
+      extract_values_from_json_onnx(item, values);
+    }
+  } else if (j.is_number()) {
+    values.push_back(j.get<float>());
+  }
+}
+
+void parse_json_shape_onnx(const json& j, std::vector<size_t>& shape,
+                           size_t dim = 0) {
+  if (!j.is_array()) {
+    if (dim == 0) shape.push_back(0);
+    return;
+  }
+
+  if (shape.size() <= dim) {
+    shape.push_back(j.size());
+  }
+
+  if (!j.empty()) {
+    parse_json_shape_onnx(j[0], shape, dim + 1);
+  }
+}
+
+Tensor create_tensor_from_json_onnx(const json& layer_data, Type type) {
+  if (type != Type::kFloat) {
+    throw std::invalid_argument("Only float type is supported");
+  }
+
+  std::vector<float> weights;
+  if (layer_data.contains("weights") && !layer_data["weights"].empty()) {
+    extract_values_from_json_onnx(layer_data["weights"], weights);
+  }
+
+  // Извлекаем bias (если есть)
+  std::vector<float> bias;
+  if (layer_data.contains("bias") && !layer_data["bias"].empty()) {
+    extract_values_from_json_onnx(layer_data["bias"], bias);
+  }
+
+  // Определяем shape
+  std::vector<size_t> shape;
+  if (layer_data.contains("weights")) {
+    parse_json_shape_onnx(layer_data["weights"], shape);
+  }
+
+  std::cout << "Extracted weights size: " << weights.size() << std::endl;
+  std::cout << "Shape: ";
+  for (auto dim : shape) std::cout << dim << " ";
+  std::cout << std::endl;
+  std::cout << "Extracted bias size: " << bias.size() << std::endl;
+
+  return make_tensor<float>(weights, Shape(shape), bias);
+}