diff --git a/devtools/etrecord/_etrecord.py b/devtools/etrecord/_etrecord.py index ffb81a8e41a..014148f2a13 100644 --- a/devtools/etrecord/_etrecord.py +++ b/devtools/etrecord/_etrecord.py @@ -45,6 +45,8 @@ class StrEnum(str, Enum): class ETRecordReservedFileNames(StrEnum): ETRECORD_IDENTIFIER = "ETRECORD_V0" + EXPORTED_PROGRAM = "exported_program" + EXPORT_GRAPH_ID = "export_graph_id" EDGE_DIALECT_EXPORTED_PROGRAM = "edge_dialect_exported_program" ET_DIALECT_GRAPH_MODULE = "et_dialect_graph_module" DEBUG_HANDLE_MAP_NAME = "debug_handle_map" @@ -55,6 +57,8 @@ class ETRecordReservedFileNames(StrEnum): @dataclass class ETRecord: + exported_program: Optional[ExportedProgram] = None + export_graph_id: Optional[int] = None edge_dialect_program: Optional[ExportedProgram] = None graph_map: Optional[Dict[str, ExportedProgram]] = None _debug_handle_map: Optional[Dict[int, Union[int, List[int]]]] = None @@ -71,17 +75,20 @@ def _handle_exported_program( assert isinstance(ep, ExportedProgram) serialized_artifact = serialize(ep) assert isinstance(serialized_artifact.exported_program, bytes) + + method_name = f"/{method_name}" if method_name != "" else "" + etrecord_zip.writestr( - f"{module_name}/{method_name}", serialized_artifact.exported_program + f"{module_name}{method_name}", serialized_artifact.exported_program ) etrecord_zip.writestr( - f"{module_name}/{method_name}_state_dict", serialized_artifact.state_dict + f"{module_name}{method_name}_state_dict", serialized_artifact.state_dict ) etrecord_zip.writestr( - f"{module_name}/{method_name}_constants", serialized_artifact.constants + f"{module_name}{method_name}_constants", serialized_artifact.constants ) etrecord_zip.writestr( - f"{module_name}/{method_name}_example_inputs", + f"{module_name}{method_name}_example_inputs", serialized_artifact.example_inputs, ) @@ -188,7 +195,10 @@ def generate_etrecord( ExecutorchProgramManager, BundledProgram, ], - export_modules: Optional[ + exported_program: Optional[ + Union[ExportedProgram, Dict[str, ExportedProgram]] + ] = None, + extra_recorded_export_modules: Optional[ Dict[ str, Union[ @@ -202,7 +212,7 @@ def generate_etrecord( """ Generates an `ETRecord` from the given objects, serializes it and saves it to the given path. The objects that will be serialized to an `ETRecord` are all the graph modules present - in the `export_modules` dict, the graph module present in the edge dialect program object, + in the `extra_recorded_export_modules` dict, the graph module present in the edge dialect program object, and also the graph module present in the ExecuTorch program object, which is the closest graph module representation of what is eventually run on the device. In addition to all the graph modules, we also serialize the program buffer, which the users @@ -213,7 +223,8 @@ def generate_etrecord( et_record: Path to where the `ETRecord` file will be saved to. edge_dialect_program: `EdgeProgramManager` for this model returned by the call to to_edge() executorch_program: The ExecuTorch program for this model returned by the call to `to_executorch()` or the `BundledProgram` of this model - export_modules [Optional]: **Should be ignored by OSS users**. A dictionary of graph modules with the key being the user provided name and the + exported_program: Optional graph module for this model returned by the call to `torch.export` from nn.Module. + extra_recorded_export_modules [Optional]: **Should be ignored by OSS users**. A dictionary of graph modules with the key being the user provided name and the value being the corresponding exported module. The exported graph modules can be either the output of `torch.export()` or `exir.to_edge()`. @@ -229,15 +240,32 @@ def generate_etrecord( # is an etrecord when it's used later in the Developer Tools. etrecord_zip.writestr(ETRecordReservedFileNames.ETRECORD_IDENTIFIER, "") - if export_modules is not None: - for module_name, export_module in export_modules.items(): + # Calculate export_graph_id before modifying exported_program + export_graph_id = 0 + + if exported_program is not None: + # If multiple exported programs are provided, only save forward method + if isinstance(exported_program, dict) and "forward" in exported_program: + exported_program = exported_program["forward"] + + if isinstance(exported_program, ExportedProgram): + export_graph_id = id(exported_program.graph) + _handle_exported_program( + etrecord_zip, + ETRecordReservedFileNames.EXPORTED_PROGRAM, + "", + exported_program, + ) + + if extra_recorded_export_modules is not None: + for module_name, export_module in extra_recorded_export_modules.items(): contains_reserved_name = any( reserved_name in module_name for reserved_name in ETRecordReservedFileNames ) if contains_reserved_name: raise RuntimeError( - f"The name {module_name} provided in the export_modules dict is a reserved name in the ETRecord namespace." + f"The name {module_name} provided in the extra_recorded_export_modules dict is a reserved name in the ETRecord namespace." ) _handle_export_module(etrecord_zip, export_module, module_name) @@ -286,6 +314,11 @@ def generate_etrecord( json.dumps(executorch_program.delegate_map), ) + etrecord_zip.writestr( + ETRecordReservedFileNames.EXPORT_GRAPH_ID, + json.dumps(export_graph_id), + ) + def parse_etrecord(etrecord_path: str) -> ETRecord: # noqa: C901 """ @@ -318,9 +351,11 @@ def parse_etrecord(etrecord_path: str) -> ETRecord: # noqa: C901 graph_map: Dict[str, ExportedProgram] = {} debug_handle_map = None delegate_map = None + exported_program = None edge_dialect_program = None reference_outputs = None representative_inputs = None + export_graph_id = 0 serialized_exported_program_files = set() serialized_state_dict_files = set() @@ -347,6 +382,14 @@ def parse_etrecord(etrecord_path: str) -> ETRecord: # noqa: C901 etrecord_zip.read(f"{entry}_example_inputs"), ) edge_dialect_program = deserialize(serialized_artifact) + elif entry == ETRecordReservedFileNames.EXPORTED_PROGRAM: + serialized_artifact = SerializedArtifact( + etrecord_zip.read(ETRecordReservedFileNames.EXPORTED_PROGRAM), + etrecord_zip.read(f"{entry}_state_dict"), + etrecord_zip.read(f"{entry}_constants"), + etrecord_zip.read(f"{entry}_example_inputs"), + ) + exported_program = deserialize(serialized_artifact) elif entry == ETRecordReservedFileNames.REFERENCE_OUTPUTS: # @lint-ignore PYTHONPICKLEISBAD reference_outputs = pickle.loads( @@ -357,6 +400,10 @@ def parse_etrecord(etrecord_path: str) -> ETRecord: # noqa: C901 representative_inputs = pickle.loads( etrecord_zip.read(ETRecordReservedFileNames.REPRESENTATIVE_INPUTS) ) + elif entry == ETRecordReservedFileNames.EXPORT_GRAPH_ID: + export_graph_id = json.loads( + etrecord_zip.read(ETRecordReservedFileNames.EXPORT_GRAPH_ID) + ) else: if entry.endswith("state_dict"): serialized_state_dict_files.add(entry) @@ -383,10 +430,12 @@ def parse_etrecord(etrecord_path: str) -> ETRecord: # noqa: C901 graph_map[serialized_file] = deserialize(serialized_artifact) return ETRecord( + exported_program=exported_program, edge_dialect_program=edge_dialect_program, graph_map=graph_map, _debug_handle_map=debug_handle_map, _delegate_map=delegate_map, _reference_outputs=reference_outputs, _representative_inputs=representative_inputs, + export_graph_id=export_graph_id, ) diff --git a/devtools/etrecord/tests/etrecord_test.py b/devtools/etrecord/tests/etrecord_test.py index dd1d40e0292..85d19c5e952 100644 --- a/devtools/etrecord/tests/etrecord_test.py +++ b/devtools/etrecord/tests/etrecord_test.py @@ -100,12 +100,13 @@ def test_etrecord_generation(self): tmpdirname + "/etrecord.bin", edge_output, et_output, - { + extra_recorded_export_modules={ "aten_dialect_output": captured_output, }, ) etrecord = parse_etrecord(tmpdirname + "/etrecord.bin") + self.check_graph_closeness( etrecord.graph_map["aten_dialect_output/forward"], captured_output.exported_program.graph_module, @@ -184,7 +185,7 @@ def test_etrecord_invalid_input(self): tmpdirname + "/etrecord.bin", edge_output, et_output, - {"fail_test_case": et_output}, + extra_recorded_export_modules={"fail_test_case": et_output}, ) def test_etrecord_reserved_name(self): @@ -196,5 +197,84 @@ def test_etrecord_reserved_name(self): tmpdirname + "/etrecord.bin", edge_output, et_output, - {reserved_name: captured_output.exported_program.graph_module}, + extra_recorded_export_modules={ + reserved_name: captured_output.exported_program.graph_module + }, ) + + def test_etrecord_generation_with_exported_program(self): + """Test that exported program can be recorded and parsed back correctly.""" + captured_output, edge_output, et_output = self.get_test_model() + original_exported_program = captured_output.exported_program + expected_graph_id = id(original_exported_program.graph) + + with tempfile.TemporaryDirectory() as tmpdirname: + # Generate ETRecord with exported program + generate_etrecord( + tmpdirname + "/etrecord.bin", + edge_output, + et_output, + exported_program=original_exported_program, + ) + + # Parse ETRecord back + etrecord = parse_etrecord(tmpdirname + "/etrecord.bin") + + # Validate that the parsed exported program matches the original + self.assertIsNotNone(etrecord.exported_program) + self.check_graph_closeness( + etrecord.exported_program, + original_exported_program.graph_module, + ) + + # Validate other components are still present + self.check_graph_closeness( + etrecord.edge_dialect_program, + edge_output.exported_program.graph_module, + ) + self.assertEqual( + etrecord._debug_handle_map, + json.loads(json.dumps(et_output.debug_handle_map)), + ) + + # Validate that export_graph_id matches the expected value + self.assertEqual(etrecord.export_graph_id, expected_graph_id) + + def test_etrecord_generation_with_exported_program_dict(self): + """Test that exported program dictionary can be recorded and parsed back correctly.""" + captured_output, edge_output, et_output = self.get_test_model() + original_exported_program = captured_output.exported_program + exported_program_dict = {"forward": original_exported_program} + expected_graph_id = id(original_exported_program.graph) + + with tempfile.TemporaryDirectory() as tmpdirname: + # Generate ETRecord with exported program dictionary + generate_etrecord( + tmpdirname + "/etrecord.bin", + edge_output, + et_output, + exported_program=exported_program_dict, + ) + + # Parse ETRecord back + etrecord = parse_etrecord(tmpdirname + "/etrecord.bin") + + # Validate that the parsed exported program matches the original + self.assertIsNotNone(etrecord.exported_program) + self.check_graph_closeness( + etrecord.exported_program, + original_exported_program.graph_module, + ) + + # Validate other components are still present + self.check_graph_closeness( + etrecord.edge_dialect_program, + edge_output.exported_program.graph_module, + ) + self.assertEqual( + etrecord._debug_handle_map, + json.loads(json.dumps(et_output.debug_handle_map)), + ) + + # Validate that export_graph_id matches the expected value + self.assertEqual(etrecord.export_graph_id, expected_graph_id) diff --git a/devtools/inspector/_inspector_utils.py b/devtools/inspector/_inspector_utils.py index 32a46ab0276..d49ce3959a6 100644 --- a/devtools/inspector/_inspector_utils.py +++ b/devtools/inspector/_inspector_utils.py @@ -35,8 +35,17 @@ from executorch.devtools.etdump.serialize import deserialize_from_etdump_flatcc from executorch.devtools.etrecord import ETRecord +from executorch.exir.debug_handle_utils import ( + DEBUG_HANDLE_KEY, + get_greatest_ancestor_node_identifier, +) + +from executorch.exir.graph_module import bfs_trace_with_node_process + from tabulate import tabulate +from torch.export import ExportedProgram + FORWARD = "forward" EDGE_DIALECT_GRAPH_KEY = "edge_dialect_graph_module" @@ -888,3 +897,71 @@ def compare_intermediate_outputs(a: Any, b: Any, comparator) -> List[float]: else: # Raise an error if one is a sequence and the other is not raise ValueError("Both inputs must be sequences or both must be non-sequences.") + + +def propagate_back_debug_handle( + exported_program: ExportedProgram, + exported_program_graph_id: int, + edge_dialect_program: ExportedProgram, +) -> bool: + """ + Propagate debug handle from edge dialect program back to the exported program while maintain the correctness + of operator tracing. + + e.g. + export program: op1 -> op2 -> op3 + edge dialect program: op1_0 -> op3_0 -> op3_1 + where op1_0 is from op1, op3_0 and op3_1 are from op3, op2 is removed by to_edge pipeline (e.g. RemoveNoopPass). + + Then debug handle of op1 should be same as op1_0, and debug handle of op3 should be same as op3_0 and op3_1. + The debug handle of op2 will be a non-existing debug handle in edge dialect program for further skipping. + + Return: True if: + a. every debug handle in the edge dialect program has a corresponding node in the exported program + b. the exported program is the greatest ancestor of the edge dialect program + + Otherwise, return False. + """ + + # 1. set up a mapping from debug handle to identifier of export program's node + # using edge dialect program nodes' debug handles and from_node info + export_graph_node_id_to_debug_handle = { + get_greatest_ancestor_node_identifier(node): node.meta[DEBUG_HANDLE_KEY] + for node in edge_dialect_program.graph.nodes + if node.op not in ("placeholder", "output") + } + + # 2. equip debug handle to the exported program's nodes using the mapping + # number of nodes in the exported program that have matched entry in export_graph_node_id_to_debug_handle + n_matched_node = 0 + + # debug handle for the node in the exported program but not in the edge dialect program + debug_handle_for_removed_node = ( + max(export_graph_node_id_to_debug_handle.values()) + 1 + ) + + def _find_n_match_node(node: torch.fx.Node) -> None: + nonlocal n_matched_node + if node.name in ("output", "placeholder"): + return + node_id = f"{node.name}.{exported_program_graph_id}" + if node_id in export_graph_node_id_to_debug_handle: + n_matched_node += 1 + + def _equip_debug_handle(node: torch.fx.Node) -> None: + if node.name in ("output", "placeholder"): + return + node_id = f"{node.name}.{exported_program_graph_id}" + if node_id in export_graph_node_id_to_debug_handle: + node.meta[DEBUG_HANDLE_KEY] = export_graph_node_id_to_debug_handle[node_id] + else: + node.meta[DEBUG_HANDLE_KEY] = debug_handle_for_removed_node + + bfs_trace_with_node_process(exported_program.graph_module, _find_n_match_node) + + # if any node in the edge dialect program has no corresponding node in the exported program, match failed + if n_matched_node != len(export_graph_node_id_to_debug_handle): + return False + + bfs_trace_with_node_process(exported_program.graph_module, _equip_debug_handle) + return True diff --git a/devtools/inspector/tests/inspector_test.py b/devtools/inspector/tests/inspector_test.py index 7246dd3bac0..1b4051cb813 100644 --- a/devtools/inspector/tests/inspector_test.py +++ b/devtools/inspector/tests/inspector_test.py @@ -327,7 +327,7 @@ def test_inspector_get_exported_program(self): tmpdirname + "/etrecord.bin", edge_output, et_output, - { + extra_recorded_export_modules={ "aten_dialect_output": captured_output, }, ) diff --git a/devtools/inspector/tests/inspector_utils_test.py b/devtools/inspector/tests/inspector_utils_test.py index 9c0b5fc7fc5..a77d541cb06 100644 --- a/devtools/inspector/tests/inspector_utils_test.py +++ b/devtools/inspector/tests/inspector_utils_test.py @@ -10,8 +10,9 @@ import unittest from typing import Dict, Tuple -import torch +import executorch.exir.tests.models as models +import torch from executorch.devtools import generate_etrecord, parse_etrecord from executorch.devtools.debug_format.base_schema import ( @@ -41,9 +42,13 @@ map_runtime_aot_intermediate_outputs, merge_runtime_overlapping_debug_handles, NodeFilter, + propagate_back_debug_handle, TimeScale, ) from executorch.devtools.inspector.numerical_comparator import L1Comparator +from executorch.exir import to_edge +from executorch.exir.debug_handle_utils import DEBUG_HANDLE_KEY +from torch.export import export class TestInspectorUtils(unittest.TestCase): @@ -54,7 +59,7 @@ def test_gen_graphs_from_etrecord(self): tmpdirname + "/etrecord.bin", edge_output, et_output, - { + extra_recorded_export_modules={ "aten_dialect_output": captured_output, }, ) @@ -583,6 +588,113 @@ def test_compare_intermediate_outputs_sequence_and_non_sequence(self): with self.assertRaises(ValueError): compare_intermediate_outputs(a, b, L1Comparator()) + def test_equip_debug_handle_to_export_program_success(self): + """Test that propagate_back_debug_handle returns True and properly equips debug handles.""" + # Create a test model + model = models.FeedForwardBlock(5, 10) + inputs = (torch.rand(5, 5),) + + # Export the model + exported_program = export(model, inputs) + export_graph_id = id(exported_program.graph) + + # Convert to edge dialect + edge_dialect_program = to_edge(exported_program).exported_program() + + # Call propagate_back_debug_handle + result = propagate_back_debug_handle( + exported_program, export_graph_id, edge_dialect_program + ) + + self.assertTrue(result) + + # Check that debug handles are properly equipped in the exported program + exported_program_debug_handles = [] + for node in exported_program.graph.nodes: + if node.op not in ("placeholder", "output"): + self.assertIn(DEBUG_HANDLE_KEY, node.meta) + self.assertIsNotNone(node.meta[DEBUG_HANDLE_KEY]) + exported_program_debug_handles.append(node.meta[DEBUG_HANDLE_KEY]) + + edge_dialect_program_debug_handles = [] + for node in edge_dialect_program.graph.nodes: + if node.op not in ("placeholder", "output"): + self.assertIn(DEBUG_HANDLE_KEY, node.meta) + self.assertIsNotNone(node.meta[DEBUG_HANDLE_KEY]) + edge_dialect_program_debug_handles.append(node.meta[DEBUG_HANDLE_KEY]) + + # The 0th operator in the exported program (layer_norm) has been decomposed into 0th and 1st ops in edge dialect graph (native_layer_norm and getitem) + # So they should have the same debug handle + self.assertEqual( + exported_program_debug_handles[0], edge_dialect_program_debug_handles[0] + ) + self.assertEqual( + exported_program_debug_handles[0], edge_dialect_program_debug_handles[1] + ) + + def test_equip_debug_handle_to_export_program_failure(self): + """Test that propagate_back_debug_handle returns False when there's a mismatch.""" + # Create a test model + model = models.FeedForwardBlock(5, 10) + inputs = (torch.rand(5, 5),) + + exported_program = export(model, inputs) + edge_dialect_program = to_edge(exported_program).exported_program() + + # Create a different exported program (reexport) to cause mismatch + reexported_program = export(model, inputs) + reexport_graph_id = id(reexported_program.graph) + + # Call propagate_back_debug_handle with mismatched programs + # This should return False because the reexported program has different node identifiers + result = propagate_back_debug_handle( + reexported_program, reexport_graph_id, edge_dialect_program + ) + + # Check that it returns False due to mismatch + self.assertFalse(result) + + def test_equip_debug_handle_to_export_program_op_to_be_removed_in_to_edge(self): + """Test that propagate_back_debug_handle returns True and properly equips debug handles when an op is removed in to_edge""" + + class M(torch.nn.Module): + """ + Simple model with ops that will be removed in to_edge + """ + + def __init__(self) -> None: + super().__init__() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x + 1 + x = x.to(x.dtype) + x = x + 1 + return x + + inputs = (torch.rand(5, 5),) + exported_program = torch.export.export(M(), inputs) + export_graph_id = id(exported_program.graph) + edge_dialect_program = to_edge(exported_program).exported_program() + + self.assertTrue( + propagate_back_debug_handle( + exported_program, export_graph_id, edge_dialect_program + ) + ) + + # only two add ops in the exported program will keep in edge dialect program, so the debug handles for removed op will be three + debug_handle_for_removed_node = 3 + + for node in exported_program.graph.nodes: + if node.name == "add": + self.assertEqual(node.meta[DEBUG_HANDLE_KEY], 1) + elif node.name == "add_1": + self.assertEqual(node.meta[DEBUG_HANDLE_KEY], 2) + elif node.op not in ("placeholder", "output"): + self.assertEqual( + node.meta[DEBUG_HANDLE_KEY], debug_handle_for_removed_node + ) + def gen_mock_operator_graph_with_expected_map() -> ( Tuple[OperatorGraph, Dict[int, OperatorNode]] diff --git a/examples/devtools/scripts/gen_sample_etrecord.py b/examples/devtools/scripts/gen_sample_etrecord.py index a6b3d487251..e5b46cdede5 100644 --- a/examples/devtools/scripts/gen_sample_etrecord.py +++ b/examples/devtools/scripts/gen_sample_etrecord.py @@ -41,7 +41,7 @@ def gen_etrecord(model: torch.nn.Module, inputs: Any, output_path=None): (DEFAULT_OUTPUT_PATH if not output_path else output_path), edge_dialect_program=edge_program, executorch_program=et_program, - export_modules={ + extra_recorded_export_modules={ "aten_dialect_output": aten_dialect, }, ) diff --git a/exir/TARGETS b/exir/TARGETS index 7916cec29fb..cda57de7f80 100644 --- a/exir/TARGETS +++ b/exir/TARGETS @@ -277,3 +277,11 @@ python_library( "fbsource//third-party/pypi/typing-extensions:typing-extensions", ], ) + +python_library( + name = "debug_handle_utils", + srcs = ["debug_handle_utils.py"], + deps = [ + "//caffe2:torch", + ], +) diff --git a/exir/debug_handle_utils.py b/exir/debug_handle_utils.py new file mode 100644 index 00000000000..d1a70fcd213 --- /dev/null +++ b/exir/debug_handle_utils.py @@ -0,0 +1,27 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from torch.fx import Node + +FROM_NODE_KEY = "from_node" +DEBUG_HANDLE_KEY = "debug_handle" + + +def get_greatest_ancestor_node_identifier(node: Node) -> str: + """Get the identifier of the greatest ancestor node of the given node. + + The identifier is the concatenation of the node name and graph id of the + greatest ancestor node, where the graph id is the unique id for every graph + module in the export flow and node name is unique within the same graph module. + """ + + node_source = node.meta[FROM_NODE_KEY] + node_source = node_source[-1] + + while len(node_source.from_node) > 0: + node_source = node_source.from_node[-1] + + return f"{node_source.name}.{str(node_source.graph_id)}" diff --git a/exir/passes/TARGETS b/exir/passes/TARGETS index 8699fe2fd02..0a1f5117f20 100644 --- a/exir/passes/TARGETS +++ b/exir/passes/TARGETS @@ -342,6 +342,7 @@ python_library( ], deps = [ "//caffe2:torch", + "//executorch/exir:debug_handle_utils", "//executorch/exir:graph_module", "//executorch/exir:pass_base", ], diff --git a/exir/passes/debug_handle_generator_pass.py b/exir/passes/debug_handle_generator_pass.py index 425558664b3..fe705273a51 100644 --- a/exir/passes/debug_handle_generator_pass.py +++ b/exir/passes/debug_handle_generator_pass.py @@ -6,6 +6,11 @@ from typing import Dict +from executorch.exir.debug_handle_utils import ( + DEBUG_HANDLE_KEY, + FROM_NODE_KEY, + get_greatest_ancestor_node_identifier, +) from executorch.exir.graph_module import bfs_trace_with_node_process from executorch.exir.pass_base import ExportPass from torch.export import ExportedProgram @@ -21,27 +26,8 @@ def call(self, graph_module: GraphModule) -> PassResult: greatest ancestor node in the export flow. """ - FROM_NODE_KEY = "from_node" - DEBUG_HANDLE_KEY = "debug_handle" - source_node_id_to_debug_handle: Dict[str, int] = {} - def _get_greatest_ancestor_node_identifier(node: Node) -> str: - """Get the identifier of the greatest ancestor node of the given node. - - The identifier is the concatenation of the node name and graph id of the - greatest ancestor node, where the graph id is the unique id for every graph - module in the export flow and node name is unique within the same graph module. - """ - - node_source = node.meta[FROM_NODE_KEY] - node_source = node_source[-1] - - while len(node_source.from_node) > 0: - node_source = node_source.from_node[-1] - - return node_source.name + str(node_source.graph_id) - def _extract_debug_handles_from_node(node: Node) -> None: """ Generate a debug handle based on node's oldest ancestor node's name @@ -56,7 +42,7 @@ def _extract_debug_handles_from_node(node: Node) -> None: FROM_NODE_KEY in node.meta ), f"Node {node} does not have meta key {FROM_NODE_KEY}" - greatest_ancestor_node_id = _get_greatest_ancestor_node_identifier(node) + greatest_ancestor_node_id = get_greatest_ancestor_node_identifier(node) debug_handle = ( len(source_node_id_to_debug_handle) + 1