fix bugs in cpp runtime

Author: zewenli98

examples/dynamo/autocast_example.py

@@ -61,7 +61,7 @@ def forward(self, x):
             enable_autocast=True,
             autocast_low_precision_type=torch.float16,
             autocast_excluded_nodes={"^conv1$", "relu"},
-            autocast_excluded_ops={torch.ops.aten.flatten.using_ints},
+            autocast_excluded_ops={"torch.ops.aten.flatten.using_ints"},
             autocast_max_output_threshold=512,
             autocast_max_depth_of_reduction=None,
             autocast_calibration_dataloader=calibration_dataloader,

py/torch_tensorrt/dynamo/lowering/passes/nodeclassifier.py

@@ -6,6 +6,9 @@
 from typing import Collection, Optional
 
 import torch
+from torch_tensorrt.dynamo.conversion._ConverterRegistry import (
+    ConverterRegistry,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -78,7 +81,8 @@ def __init__(self, excluded_ops):
         self.excluded_ops = excluded_ops
 
     def _check_inner(self, node):
-        return node.target in self.excluded_ops
+        node_name = ConverterRegistry.qualified_name_or_str(node.target)
+        return node_name in self.excluded_ops
 
 
 class IORangeRule(NodeRuleBase):

py/torch_tensorrt/dynamo/lowering/passes/rule_based_autocast.py

@@ -41,6 +41,7 @@ def rule_based_autocast(
     reference_data: dict[str, torch.Tensor] = (
         settings.autocast_intermediate_node_outputs
     )
+    del settings.autocast_intermediate_node_outputs
 
     node_classifier = NodeClassifier(
         gm.graph.nodes,

tests/py/dynamo/models/test_autocast.py

@@ -56,7 +56,7 @@ def forward(self, x):
             enable_autocast=True,
             autocast_low_precision_type=torch.float16,
             autocast_excluded_nodes={"^conv1$", "relu"},
-            autocast_excluded_ops={torch.ops.aten.flatten.using_ints},
+            autocast_excluded_ops={"torch.ops.aten.flatten.using_ints"},
             autocast_max_output_threshold=512,
             autocast_max_depth_of_reduction=None,
             autocast_calibration_dataloader=calibration_dataloader,
@@ -147,7 +147,7 @@ def forward(self, x):
             enable_autocast=True,
             autocast_low_precision_type=torch.bfloat16,
             autocast_excluded_nodes={"^conv1$", "relu"},
-            autocast_excluded_ops={torch.ops.aten.flatten.using_ints},
+            autocast_excluded_ops={"torch.ops.aten.flatten.using_ints"},
             autocast_max_output_threshold=512,
             autocast_max_depth_of_reduction=None,
             autocast_calibration_dataloader=calibration_dataloader,
@@ -220,13 +220,13 @@ def forward(self, x):
             ep.module(),
             arg_inputs=inputs,
             min_block_size=1,
-            use_python_runtime=True,
+            use_python_runtime=False,
             use_explicit_typing=True,
             # Torch-TensorRT's autocast doesn't affect layers inside Pytorch autocast
             enable_autocast=True,
             autocast_low_precision_type=torch.bfloat16,
             autocast_excluded_nodes={"^conv1$", "relu"},
-            autocast_excluded_ops={torch.ops.aten.flatten.using_ints},
+            autocast_excluded_ops={"torch.ops.aten.flatten.using_ints"},
             autocast_max_output_threshold=512,
             autocast_max_depth_of_reduction=None,
             autocast_calibration_dataloader=calibration_dataloader,
@@ -328,13 +328,13 @@ def forward(self, x, y):
             ep,
             arg_inputs=inputs,
             min_block_size=1,
-            use_python_runtime=True,
+            use_python_runtime=False,
             use_explicit_typing=True,
             # Torch-TensorRT's autocast doesn't affect layers inside Pytorch autocast
             enable_autocast=True,
             autocast_low_precision_type=torch.bfloat16,
             autocast_excluded_nodes={"^conv1$", "relu"},
-            autocast_excluded_ops={torch.ops.aten.flatten.using_ints},
+            autocast_excluded_ops={"torch.ops.aten.flatten.using_ints"},
             autocast_max_output_threshold=512,
             autocast_max_depth_of_reduction=None,
             autocast_calibration_dataloader=calibration_dataloader,