From 46f4a4ec7edec3cb5e3f1cf26d8c5f26f23a7bc8 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 16 Jul 2025 11:35:07 -0700 Subject: [PATCH 01/14] enable free dimension override for graph optimization level 0 --- .../core/optimizer/graph_transformer_utils.cc | 4 ++++ onnxruntime/core/session/inference_session.cc | 12 +++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc index 4edf804e48aaa..42746a000a39e 100644 --- a/onnxruntime/core/optimizer/graph_transformer_utils.cc +++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc @@ -220,6 +220,10 @@ InlinedVector> GenerateTransformers( AllocatorPtr cpu_allocator = CPUAllocator::DefaultInstance(); switch (level) { + case TransformerLevel::Default: { + transformers.emplace_back(std::make_unique( + session_options.free_dimension_overrides)); + } break; case TransformerLevel::Level1: { // RewriteRule optimizations are the simplest (they generally remove unnecessary nodes and are cheap to run) // so run them first so there is potentially less for the more intensive optimizations like ConstantFolding, diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index f147242da668f..ff11ea8e34bc4 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -3645,8 +3645,18 @@ common::Status InferenceSession::AddPredefinedTransformers( RecordRuntimeOptimizationProducedNodeOpSchemaFn record_runtime_optimization_produced_op_schema_fn, const logging::Logger& logger) const { const auto& cpu_ep = *execution_providers_.Get(onnxruntime::kCpuExecutionProvider); - for (int i = static_cast(TransformerLevel::Level1); i <= static_cast(TransformerLevel::MaxLevel); i++) { + for (int i = static_cast(TransformerLevel::Default); i <= static_cast(TransformerLevel::MaxLevel); i++) { TransformerLevel level = static_cast(i); + + // Still enable free dimension override even if graph optimization level is 0 + if ((graph_optimization_level == TransformerLevel::Default) && (level == TransformerLevel::Default)) { + auto transformers_to_register = [&]() { + return optimizer_utils::GenerateTransformers(level, session_options_, cpu_ep, logger, + optimizers_to_disable_, + GetIntraOpThreadPoolToUse()); + }(); + } + if (graph_optimization_level >= level) { // Generate and register transformers for level auto transformers_to_register = [&]() { From 4dd6b380017d641009db25778b9675d484dd7cd3 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 16 Jul 2025 11:42:03 -0700 Subject: [PATCH 02/14] update comment --- onnxruntime/core/session/inference_session.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index ff11ea8e34bc4..75d3fc54cb0b6 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -3648,7 +3648,8 @@ common::Status InferenceSession::AddPredefinedTransformers( for (int i = static_cast(TransformerLevel::Default); i <= static_cast(TransformerLevel::MaxLevel); i++) { TransformerLevel level = static_cast(i); - // Still enable free dimension override even if graph optimization level is 0 + // Enable free dimension override even when the graph optimization level is 0. + // If the optimization level is above 0, the override will be applied during level 1 optimization. if ((graph_optimization_level == TransformerLevel::Default) && (level == TransformerLevel::Default)) { auto transformers_to_register = [&]() { return optimizer_utils::GenerateTransformers(level, session_options_, cpu_ep, logger, From f1b5e54f38166625180b7db46066dbab2dc4d266 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 16 Jul 2025 12:11:14 -0700 Subject: [PATCH 03/14] update --- onnxruntime/core/session/inference_session.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 75d3fc54cb0b6..9093733db45f0 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -3647,11 +3647,12 @@ common::Status InferenceSession::AddPredefinedTransformers( const auto& cpu_ep = *execution_providers_.Get(onnxruntime::kCpuExecutionProvider); for (int i = static_cast(TransformerLevel::Default); i <= static_cast(TransformerLevel::MaxLevel); i++) { TransformerLevel level = static_cast(i); + onnxruntime::InlinedVector> transformers_to_register; // Enable free dimension override even when the graph optimization level is 0. // If the optimization level is above 0, the override will be applied during level 1 optimization. if ((graph_optimization_level == TransformerLevel::Default) && (level == TransformerLevel::Default)) { - auto transformers_to_register = [&]() { + transformers_to_register = [&]() { return optimizer_utils::GenerateTransformers(level, session_options_, cpu_ep, logger, optimizers_to_disable_, GetIntraOpThreadPoolToUse()); @@ -3660,7 +3661,7 @@ common::Status InferenceSession::AddPredefinedTransformers( if (graph_optimization_level >= level) { // Generate and register transformers for level - auto transformers_to_register = [&]() { + transformers_to_register = [&]() { const bool use_full_build_optimizations = level == TransformerLevel::Level1 || minimal_build_optimization_handling == MinimalBuildOptimizationHandling::ApplyFullBuildOptimizations; @@ -3682,10 +3683,10 @@ common::Status InferenceSession::AddPredefinedTransformers( GetIntraOpThreadPoolToUse()); } }(); + } - for (auto& entry : transformers_to_register) { - ORT_RETURN_IF_ERROR(transformer_manager.Register(std::move(entry), level)); - } + for (auto& entry : transformers_to_register) { + ORT_RETURN_IF_ERROR(transformer_manager.Register(std::move(entry), level)); } } return Status::OK(); From 31f5b5e73d14ddb068649830832661784fc7f725 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 16 Jul 2025 13:55:15 -0700 Subject: [PATCH 04/14] fix issue --- onnxruntime/core/session/inference_session.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 9093733db45f0..dde2031e8bbee 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -3647,7 +3647,7 @@ common::Status InferenceSession::AddPredefinedTransformers( const auto& cpu_ep = *execution_providers_.Get(onnxruntime::kCpuExecutionProvider); for (int i = static_cast(TransformerLevel::Default); i <= static_cast(TransformerLevel::MaxLevel); i++) { TransformerLevel level = static_cast(i); - onnxruntime::InlinedVector> transformers_to_register; + std::function>()> transformers_to_register; // Enable free dimension override even when the graph optimization level is 0. // If the optimization level is above 0, the override will be applied during level 1 optimization. @@ -3656,7 +3656,7 @@ common::Status InferenceSession::AddPredefinedTransformers( return optimizer_utils::GenerateTransformers(level, session_options_, cpu_ep, logger, optimizers_to_disable_, GetIntraOpThreadPoolToUse()); - }(); + }; } if (graph_optimization_level >= level) { @@ -3682,10 +3682,10 @@ common::Status InferenceSession::AddPredefinedTransformers( optimizers_to_disable_, GetIntraOpThreadPoolToUse()); } - }(); + }; } - for (auto& entry : transformers_to_register) { + for (auto& entry : transformers_to_register()) { ORT_RETURN_IF_ERROR(transformer_manager.Register(std::move(entry), level)); } } From d1c13be37af665f0318e7e9a12b567c1225dde96 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 16 Jul 2025 14:02:09 -0700 Subject: [PATCH 05/14] fix issue --- onnxruntime/core/session/inference_session.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index dde2031e8bbee..e3da7edaaaa99 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -3657,9 +3657,7 @@ common::Status InferenceSession::AddPredefinedTransformers( optimizers_to_disable_, GetIntraOpThreadPoolToUse()); }; - } - - if (graph_optimization_level >= level) { + } else if (graph_optimization_level >= level) { // Generate and register transformers for level transformers_to_register = [&]() { const bool use_full_build_optimizations = From 6fa3ff759d76a954dfa5c511842c37f5afed6ef6 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 16 Jul 2025 14:23:06 -0700 Subject: [PATCH 06/14] fix checking logic --- onnxruntime/core/session/inference_session.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index e3da7edaaaa99..2d6cda294f52b 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -3657,7 +3657,7 @@ common::Status InferenceSession::AddPredefinedTransformers( optimizers_to_disable_, GetIntraOpThreadPoolToUse()); }; - } else if (graph_optimization_level >= level) { + } else if ((graph_optimization_level != TransformerLevel::Default) && graph_optimization_level >= level) { // Generate and register transformers for level transformers_to_register = [&]() { const bool use_full_build_optimizations = @@ -3683,8 +3683,10 @@ common::Status InferenceSession::AddPredefinedTransformers( }; } - for (auto& entry : transformers_to_register()) { - ORT_RETURN_IF_ERROR(transformer_manager.Register(std::move(entry), level)); + if (transformers_to_register) { // Ensure the lambda is initialized before invoking it + for (auto& entry : transformers_to_register()) { + ORT_RETURN_IF_ERROR(transformer_manager.Register(std::move(entry), level)); + } } } return Status::OK(); From b387056a36d4c672f2d8126a6f2c2cd72f5ba174 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 16 Jul 2025 14:28:35 -0700 Subject: [PATCH 07/14] fix checking logic --- onnxruntime/core/session/inference_session.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 2d6cda294f52b..6303483ebcfda 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -3651,13 +3651,17 @@ common::Status InferenceSession::AddPredefinedTransformers( // Enable free dimension override even when the graph optimization level is 0. // If the optimization level is above 0, the override will be applied during level 1 optimization. - if ((graph_optimization_level == TransformerLevel::Default) && (level == TransformerLevel::Default)) { + if ((level == TransformerLevel::Default) && (graph_optimization_level == TransformerLevel::Default)) { transformers_to_register = [&]() { return optimizer_utils::GenerateTransformers(level, session_options_, cpu_ep, logger, optimizers_to_disable_, GetIntraOpThreadPoolToUse()); }; - } else if ((graph_optimization_level != TransformerLevel::Default) && graph_optimization_level >= level) { + } else if (level == TransformerLevel::Default) { + continue; + } + + if (graph_optimization_level >= level) { // Generate and register transformers for level transformers_to_register = [&]() { const bool use_full_build_optimizations = From f3f35786af164c629a34b88d443854913aaed9b9 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 16 Jul 2025 14:29:45 -0700 Subject: [PATCH 08/14] address lintrunner issue --- onnxruntime/core/session/inference_session.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 6303483ebcfda..c4e7f43cbbec6 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -3659,7 +3659,7 @@ common::Status InferenceSession::AddPredefinedTransformers( }; } else if (level == TransformerLevel::Default) { continue; - } + } if (graph_optimization_level >= level) { // Generate and register transformers for level From 1fd8e63ba5865f627465207ffaf19d0c21eece86 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 16 Jul 2025 14:49:17 -0700 Subject: [PATCH 09/14] fix checking logic --- onnxruntime/core/session/inference_session.cc | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index c4e7f43cbbec6..052eb532208a2 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -3651,40 +3651,40 @@ common::Status InferenceSession::AddPredefinedTransformers( // Enable free dimension override even when the graph optimization level is 0. // If the optimization level is above 0, the override will be applied during level 1 optimization. - if ((level == TransformerLevel::Default) && (graph_optimization_level == TransformerLevel::Default)) { - transformers_to_register = [&]() { - return optimizer_utils::GenerateTransformers(level, session_options_, cpu_ep, logger, - optimizers_to_disable_, - GetIntraOpThreadPoolToUse()); - }; - } else if (level == TransformerLevel::Default) { - continue; - } - - if (graph_optimization_level >= level) { - // Generate and register transformers for level - transformers_to_register = [&]() { - const bool use_full_build_optimizations = - level == TransformerLevel::Level1 || - minimal_build_optimization_handling == MinimalBuildOptimizationHandling::ApplyFullBuildOptimizations; - - if (use_full_build_optimizations) { + if ((level == TransformerLevel::Default)) { + if (graph_optimization_level == TransformerLevel::Default) { + transformers_to_register = [&]() { return optimizer_utils::GenerateTransformers(level, session_options_, cpu_ep, logger, optimizers_to_disable_, GetIntraOpThreadPoolToUse()); - } else { - const auto sat_context = - minimal_build_optimization_handling == - MinimalBuildOptimizationHandling::SaveMinimalBuildRuntimeOptimizations - ? SatApplyContextVariant{SatRuntimeOptimizationSaveContext{ - record_runtime_optimization_produced_op_schema_fn}} - : SatApplyContextVariant{SatDirectApplicationContext{}}; - return optimizer_utils::GenerateTransformersForMinimalBuild(level, session_options_, sat_context, cpu_ep, - logger, - optimizers_to_disable_, - GetIntraOpThreadPoolToUse()); - } - }; + }; + } + } else { + if (graph_optimization_level >= level) { + // Generate and register transformers for level + transformers_to_register = [&]() { + const bool use_full_build_optimizations = + level == TransformerLevel::Level1 || + minimal_build_optimization_handling == MinimalBuildOptimizationHandling::ApplyFullBuildOptimizations; + + if (use_full_build_optimizations) { + return optimizer_utils::GenerateTransformers(level, session_options_, cpu_ep, logger, + optimizers_to_disable_, + GetIntraOpThreadPoolToUse()); + } else { + const auto sat_context = + minimal_build_optimization_handling == + MinimalBuildOptimizationHandling::SaveMinimalBuildRuntimeOptimizations + ? SatApplyContextVariant{SatRuntimeOptimizationSaveContext{ + record_runtime_optimization_produced_op_schema_fn}} + : SatApplyContextVariant{SatDirectApplicationContext{}}; + return optimizer_utils::GenerateTransformersForMinimalBuild(level, session_options_, sat_context, cpu_ep, + logger, + optimizers_to_disable_, + GetIntraOpThreadPoolToUse()); + } + }; + } } if (transformers_to_register) { // Ensure the lambda is initialized before invoking it From 9f925279a24baf30c29367fe8f8a963fe5b4c497 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 16 Jul 2025 15:12:56 -0700 Subject: [PATCH 10/14] fix warning --- onnxruntime/core/session/inference_session.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 052eb532208a2..be0be5874fb18 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -3651,7 +3651,7 @@ common::Status InferenceSession::AddPredefinedTransformers( // Enable free dimension override even when the graph optimization level is 0. // If the optimization level is above 0, the override will be applied during level 1 optimization. - if ((level == TransformerLevel::Default)) { + if (level == TransformerLevel::Default) { if (graph_optimization_level == TransformerLevel::Default) { transformers_to_register = [&]() { return optimizer_utils::GenerateTransformers(level, session_options_, cpu_ep, logger, From 5153c73543b20ae21a9b678d473a1496eec6f886 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 16 Jul 2025 16:59:39 -0700 Subject: [PATCH 11/14] Add unit tests --- onnxruntime/core/session/inference_session.cc | 3 +- .../optimizer/free_dimension_override_test.cc | 12 +++-- .../graph_runtime_optimization_test.cc | 49 +++++++++++++++++++ 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index be0be5874fb18..15dc6f377766a 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -1349,7 +1349,8 @@ common::Status InferenceSession::TransformGraph(onnxruntime::Graph& graph, bool ORT_RETURN_IF_ERROR_SESSIONID_(apply_transformer_once(ensure_unique_dq_for_node_unit, *session_logger_, graph)); } - // apply execution provider independent level 1 graph optimizations. + // apply execution provider independent level 0 and 1 graph optimizations. + ORT_RETURN_IF_ERROR_SESSIONID_(graph_transformer_mgr_.ApplyTransformers(graph, TransformerLevel::Default, *session_logger_)); ORT_RETURN_IF_ERROR_SESSIONID_(graph_transformer_mgr_.ApplyTransformers(graph, TransformerLevel::Level1, *session_logger_)); // if saving model to ORT format we only assign nodes a custom EP can handle and don't compile them. diff --git a/onnxruntime/test/optimizer/free_dimension_override_test.cc b/onnxruntime/test/optimizer/free_dimension_override_test.cc index ce778cddb45a3..08f7ebf1c42fc 100644 --- a/onnxruntime/test/optimizer/free_dimension_override_test.cc +++ b/onnxruntime/test/optimizer/free_dimension_override_test.cc @@ -18,7 +18,7 @@ using namespace ONNX_NAMESPACE; namespace onnxruntime { namespace test { -void TestFreeDimensions(FreeDimensionOverrideType overrideType) { +void TestFreeDimensions(FreeDimensionOverrideType overrideType, TransformerLevel level) { auto model_uri = ORT_TSTR("testdata/abs_free_dimensions.onnx"); std::shared_ptr model; @@ -43,9 +43,9 @@ void TestFreeDimensions(FreeDimensionOverrideType overrideType) { auto graph_transformer = std::make_unique(overrides); onnxruntime::GraphTransformerManager graph_transformation_mgr(5); - ASSERT_STATUS_OK(graph_transformation_mgr.Register(std::move(graph_transformer), TransformerLevel::Level1)); + ASSERT_STATUS_OK(graph_transformation_mgr.Register(std::move(graph_transformer), level)); - ASSERT_STATUS_OK(graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level1, + ASSERT_STATUS_OK(graph_transformation_mgr.ApplyTransformers(graph, level, DefaultLoggingManager().DefaultLogger())); // Verify that the shape of the input graph has the correct values @@ -73,8 +73,10 @@ void TestFreeDimensions(FreeDimensionOverrideType overrideType) { } TEST(FreeDimensionOverrideDenotationTransformerTest, Test) { - TestFreeDimensions(FreeDimensionOverrideType::Denotation); - TestFreeDimensions(FreeDimensionOverrideType::Name); + TestFreeDimensions(FreeDimensionOverrideType::Denotation, TransformerLevel::Level1); + TestFreeDimensions(FreeDimensionOverrideType::Name, TransformerLevel::Level1); + TestFreeDimensions(FreeDimensionOverrideType::Denotation, TransformerLevel::Default); + TestFreeDimensions(FreeDimensionOverrideType::Name, TransformerLevel::Default); } } // namespace test diff --git a/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc b/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc index d8d7cd58d50e2..e7f4b04f82237 100644 --- a/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc +++ b/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc @@ -280,6 +280,48 @@ void CheckNhwcTransformerIsApplied(const PathString& ort_model_path, graph_op_counts_checker, graph_checker)); }; + +// if level 0 optimization is enabled the free dimension override should be enabled. +void CheckFreeDimensionOverrideIsApplied(const PathString& model_path, + TransformerLevel level, + FreeDimensionOverrideType overrideType) { + SessionOptions so{}; + so.graph_optimization_level = level; + if (overrideType == FreeDimensionOverrideType::Denotation) { + so.free_dimension_overrides.push_back( + onnxruntime::FreeDimensionOverride{onnx::DATA_BATCH, overrideType, 1}); + so.free_dimension_overrides.push_back( + onnxruntime::FreeDimensionOverride{onnx::DATA_CHANNEL, overrideType, 42}); + } else { + so.free_dimension_overrides.push_back( + onnxruntime::FreeDimensionOverride{"Dim1", overrideType, 1}); + so.free_dimension_overrides.push_back( + onnxruntime::FreeDimensionOverride{"Dim2", overrideType, 42}); + } + + GraphCheckerFn graph_checker = [](const Graph& graph) { + // Verify that the shape of the input graph has the correct values + + const auto& graph_inputs = graph.GetInputs(); + ASSERT_TRUE(graph_inputs.size() == 1); // This model only has a single input ('x') + + const auto* input_shape = graph_inputs[0]->Shape(); + ASSERT_TRUE(input_shape->dim_size() == 3); // Model takes a 3D tensor as input; two of those dimensions are (were) free dimensions + + ASSERT_TRUE(input_shape->dim(0).denotation() == onnx::DATA_BATCH); + ASSERT_TRUE(input_shape->dim(0).has_dim_value()); + ASSERT_TRUE(input_shape->dim(0).dim_value() == 1); + + ASSERT_TRUE(input_shape->dim(1).denotation() == onnx::DATA_CHANNEL); + ASSERT_TRUE(input_shape->dim(1).has_dim_value()); + ASSERT_TRUE(input_shape->dim(1).dim_value() == 42); + }; + + ASSERT_NO_FATAL_FAILURE(LoadAndInitializeSession( + so, model_path, + nullptr, + graph_checker)); +}; } // namespace TEST(GraphRuntimeOptimizationTest, QDQConv) { @@ -375,6 +417,13 @@ TEST(GraphRuntimeOptimizationTest, TestNhwcTransformerDirectlyUpdatesQLinearConv }); } +TEST(GraphRuntimeOptimizationTest, TestFreeDimensionOverride) { + CheckFreeDimensionOverrideIsApplied(ORT_TSTR("testdata/abs_free_dimensions.onnx"), TransformerLevel::Default, FreeDimensionOverrideType::Denotation); + CheckFreeDimensionOverrideIsApplied(ORT_TSTR("testdata/abs_free_dimensions.onnx"), TransformerLevel::Default, FreeDimensionOverrideType::Name); + CheckFreeDimensionOverrideIsApplied(ORT_TSTR("testdata/abs_free_dimensions.onnx"), TransformerLevel::Level1, FreeDimensionOverrideType::Denotation); + CheckFreeDimensionOverrideIsApplied(ORT_TSTR("testdata/abs_free_dimensions.onnx"), TransformerLevel::Level1, FreeDimensionOverrideType::Name); +} + #if !defined(ORT_MINIMAL_BUILD) TEST(GraphRuntimeOptimizationTest, TestOnlyApplyMinimalBuildOptimizations) { // This test assumes that AttentionFusion is not included in the minimal build optimizations. From b10c685348719f41c550de6b3ae9c99d197028a0 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 16 Jul 2025 17:49:27 -0700 Subject: [PATCH 12/14] fix issue for minimal build --- .../graph_runtime_optimization_test.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc b/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc index e7f4b04f82237..1a67eb3bbc226 100644 --- a/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc +++ b/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc @@ -289,9 +289,9 @@ void CheckFreeDimensionOverrideIsApplied(const PathString& model_path, so.graph_optimization_level = level; if (overrideType == FreeDimensionOverrideType::Denotation) { so.free_dimension_overrides.push_back( - onnxruntime::FreeDimensionOverride{onnx::DATA_BATCH, overrideType, 1}); + onnxruntime::FreeDimensionOverride{"DATA_BATCH", overrideType, 1}); so.free_dimension_overrides.push_back( - onnxruntime::FreeDimensionOverride{onnx::DATA_CHANNEL, overrideType, 42}); + onnxruntime::FreeDimensionOverride{"DATA_CHANNEL", overrideType, 42}); } else { so.free_dimension_overrides.push_back( onnxruntime::FreeDimensionOverride{"Dim1", overrideType, 1}); @@ -308,11 +308,11 @@ void CheckFreeDimensionOverrideIsApplied(const PathString& model_path, const auto* input_shape = graph_inputs[0]->Shape(); ASSERT_TRUE(input_shape->dim_size() == 3); // Model takes a 3D tensor as input; two of those dimensions are (were) free dimensions - ASSERT_TRUE(input_shape->dim(0).denotation() == onnx::DATA_BATCH); + ASSERT_TRUE(input_shape->dim(0).denotation() == "DATA_BATCH"); ASSERT_TRUE(input_shape->dim(0).has_dim_value()); ASSERT_TRUE(input_shape->dim(0).dim_value() == 1); - ASSERT_TRUE(input_shape->dim(1).denotation() == onnx::DATA_CHANNEL); + ASSERT_TRUE(input_shape->dim(1).denotation() == "DATA_CHANNEL"); ASSERT_TRUE(input_shape->dim(1).has_dim_value()); ASSERT_TRUE(input_shape->dim(1).dim_value() == 42); }; From 178b4211c4f1b0f5d5838841fd4a9106bb758419 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 16 Jul 2025 18:26:20 -0700 Subject: [PATCH 13/14] fix issue for minimal build --- .../runtime_optimization/graph_runtime_optimization_test.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc b/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc index 1a67eb3bbc226..5711c0cdbdb61 100644 --- a/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc +++ b/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc @@ -416,7 +416,7 @@ TEST(GraphRuntimeOptimizationTest, TestNhwcTransformerDirectlyUpdatesQLinearConv {"com.microsoft.QLinearConv", n}})); }); } - +#if !defined(ORT_MINIMAL_BUILD) TEST(GraphRuntimeOptimizationTest, TestFreeDimensionOverride) { CheckFreeDimensionOverrideIsApplied(ORT_TSTR("testdata/abs_free_dimensions.onnx"), TransformerLevel::Default, FreeDimensionOverrideType::Denotation); CheckFreeDimensionOverrideIsApplied(ORT_TSTR("testdata/abs_free_dimensions.onnx"), TransformerLevel::Default, FreeDimensionOverrideType::Name); @@ -424,7 +424,6 @@ TEST(GraphRuntimeOptimizationTest, TestFreeDimensionOverride) { CheckFreeDimensionOverrideIsApplied(ORT_TSTR("testdata/abs_free_dimensions.onnx"), TransformerLevel::Level1, FreeDimensionOverrideType::Name); } -#if !defined(ORT_MINIMAL_BUILD) TEST(GraphRuntimeOptimizationTest, TestOnlyApplyMinimalBuildOptimizations) { // This test assumes that AttentionFusion is not included in the minimal build optimizations. // Update it if that changes. From fe16823431207ced6b28b4dd518368159c90941c Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 16 Jul 2025 19:00:44 -0700 Subject: [PATCH 14/14] address reviewer's comment --- onnxruntime/core/optimizer/graph_transformer_utils.cc | 6 ++++-- .../runtime_optimization/graph_runtime_optimization_test.cc | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc index 42746a000a39e..dbf86e2bb7fc7 100644 --- a/onnxruntime/core/optimizer/graph_transformer_utils.cc +++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc @@ -221,8 +221,10 @@ InlinedVector> GenerateTransformers( switch (level) { case TransformerLevel::Default: { - transformers.emplace_back(std::make_unique( - session_options.free_dimension_overrides)); + if (!session_options.free_dimension_overrides.empty()) { + transformers.emplace_back(std::make_unique( + session_options.free_dimension_overrides)); + } } break; case TransformerLevel::Level1: { // RewriteRule optimizations are the simplest (they generally remove unnecessary nodes and are cheap to run) diff --git a/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc b/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc index 5711c0cdbdb61..a59ce60d65136 100644 --- a/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc +++ b/onnxruntime/test/optimizer/runtime_optimization/graph_runtime_optimization_test.cc @@ -281,6 +281,7 @@ void CheckNhwcTransformerIsApplied(const PathString& ort_model_path, graph_checker)); }; +#if !defined(ORT_MINIMAL_BUILD) // if level 0 optimization is enabled the free dimension override should be enabled. void CheckFreeDimensionOverrideIsApplied(const PathString& model_path, TransformerLevel level, @@ -322,6 +323,7 @@ void CheckFreeDimensionOverrideIsApplied(const PathString& model_path, nullptr, graph_checker)); }; +#endif // !defined(ORT_MINIMAL_BUILD) } // namespace TEST(GraphRuntimeOptimizationTest, QDQConv) {