diff --git a/newrelic/api/opentelemetry.py b/newrelic/api/opentelemetry.py index d1b59a619..be9f7df75 100644 --- a/newrelic/api/opentelemetry.py +++ b/newrelic/api/opentelemetry.py @@ -17,6 +17,10 @@ from contextlib import contextmanager from opentelemetry import trace as otel_api_trace +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from opentelemetry.baggage.propagation import W3CBaggagePropagator +from opentelemetry.propagators.composite import CompositePropagator +from opentelemetry.propagate import set_global_textmap from newrelic.api.application import application_instance from newrelic.api.background_task import BackgroundTask @@ -28,11 +32,137 @@ from newrelic.api.time_trace import current_trace, notice_error from newrelic.api.transaction import Sentinel, current_transaction from newrelic.api.web_transaction import WebTransaction + +from newrelic.common.encoding_utils import ( + W3CTraceState, + NrTraceState, +) from newrelic.core.otlp_utils import create_resource _logger = logging.getLogger(__name__) +class NRTraceContextPropagator(TraceContextTextMapPropagator): + LIST_OF_TRACEPARENT_KEYS = ("traceparent", "HTTP_TRACEPARENT") + LIST_OF_TRACESTATE_KEYS = ("tracestate", "HTTP_TRACESTATE") + + def _convert_nr_to_otel(self, tracestate): + application_settings = application_instance(activate=False).settings + vendors = W3CTraceState.decode(tracestate) + trusted_account_key = application_settings.trusted_account_key or ( + application_settings.serverless_mode.enabled and application_settings.account_id + ) + payload = vendors.pop(f"{trusted_account_key}@nr", "") + + otel_tracestate = W3CTraceState(NrTraceState.decode(payload, trusted_account_key)).text() + return otel_tracestate + + def _convert_otel_to_nr(self, tracestate): + tracestate_dict = W3CTraceState.decode(tracestate) + # Convert sampled, priority, and timestamp data types + tracestate_dict["sa"] = True if tracestate_dict.get("sa").upper() == "TRUE" else False + tracestate_dict["pr"] = float(tracestate_dict.get("pr")) + tracestate_dict["ti"] = int(tracestate_dict.get("ti")) + + nr_tracestate = NrTraceState(tracestate_dict).text() + return nr_tracestate + + def extract(self, carrier, context=None, getter=None): + # We need to make sure that the carrier goes out + # in OTel format. However, we want to convert this to + # NR to use the `accept_distributed_trace_headers` API + transaction = current_transaction() + tracestate_key = None + tracestate_headers = None + for key in self.LIST_OF_TRACESTATE_KEYS: + if key in carrier: + tracestate_key = key + tracestate_headers = carrier[tracestate_key] + break + # If we are passing into New Relic, traceparent and/or tracestate's keys also need to be NR compatible. + if tracestate_headers: + # Check to see if in NR or OTel format + if "@nr=" in tracestate_headers: + # NR format + # Reformatting DT keys in case they are in the HTTP_* format: + nr_headers = carrier.copy() + for header_type in ("traceparent", "tracestate", "newrelic"): + if (header_type not in nr_headers) and (f"HTTP_{header_type.upper()}" in nr_headers): + nr_headers[header_type] = nr_headers.pop(f"HTTP_{header_type.upper()}") + transaction.accept_distributed_trace_headers(nr_headers) + # Convert NR format to OTel format for OTel extract function + tracestate = self._convert_nr_to_otel(tracestate_headers) + carrier[tracestate_key] = tracestate + else: + # OTel format + if transaction: + # Convert to NR format to use the + # `accept_distributed_trace_headers` API + nr_tracestate = self._convert_otel_to_nr(tracestate_headers) + nr_headers = {key: value for key, value in carrier.items()} + nr_headers.pop("HTTP_TRACESTATE", None) + nr_headers["tracestate"] = nr_tracestate + for header_type in ("traceparent", "newrelic"): + if header_type not in nr_headers: + nr_headers[header_type] = nr_headers.pop(f"HTTP_{header_type.upper()}", None) + transaction.accept_distributed_trace_headers(nr_headers) + elif ("traceparent" in carrier) and transaction: + transaction.accept_distributed_trace_headers(carrier) + + return super().extract(carrier=carrier, context=context, getter=getter) + + + def inject(self, carrier, context=None, setter=None): + transaction = current_transaction() + # Only insert headers if we have not done so already this transaction + # Distributed Trace State will have the following states: + # 0 if not set + # 1 if already accepted + # 2 if inserted but not accepted + + if transaction and not transaction._distributed_trace_state: + try: + nr_headers = [(key, value) for key, value in carrier.items()] + transaction.insert_distributed_trace_headers(nr_headers) + # Convert back, now with new headers + carrier.update(dict(nr_headers)) + carrier["tracestate"] = self._convert_nr_to_otel(carrier["tracestate"]) + + except AttributeError: + # Already in list form. + transaction.insert_distributed_trace_headers(carrier) + + # If it came in list form, we likely want to keep it in that format. + # Convert to dict to modify NR format of tracestate to Otel's format + # and then convert back to the list of tuples. + otel_headers = dict(carrier) + otel_headers["tracestate"] = self._convert_nr_to_otel(otel_headers["tracestate"]) + + # This is done instead of assigning the result of a list + # comprehension to preserve the ID of the carrier in + # order to allow propagation. + for header in otel_headers.items(): + if header not in carrier: + carrier.append(header) + + elif not transaction: + # Convert carrier's tracestate to Otel format if not already + # This assumes that carrier is a dict but tracestate is in NR format. + if ("tracestate" in carrier) and ("@nr=" in carrier["tracestate"]): + # Needs to be converted to OTel before running original function + carrier["tracestate"] = self._convert_nr_to_otel(carrier["tracestate"]) + return super().inject(carrier=carrier, context=context, setter=setter) + + +# Context and Context Propagator Setup +otel_context_propagator = CompositePropagator( + propagators=[ + NRTraceContextPropagator(), + W3CBaggagePropagator(), + ] +) +set_global_textmap(otel_context_propagator) + # ---------------------------------------------- # Custom OTel Spans and Traces # ---------------------------------------------- @@ -168,7 +298,17 @@ def get_span_context(self): if not getattr(self, "nr_trace", False): return otel_api_trace.INVALID_SPAN_CONTEXT - otel_tracestate_headers = None + if self.nr_transaction.settings.distributed_tracing.enabled: + nr_tracestate_headers = ( + self.nr_transaction._create_distributed_trace_data() + ) + + nr_tracestate_headers["sa"] = self._sampled() + otel_tracestate_headers = [ + (key, str(value)) for key, value in nr_tracestate_headers.items() + ] + else: + otel_tracestate_headers = None return otel_api_trace.SpanContext( trace_id=int(self.nr_transaction.trace_id, 16), @@ -287,7 +427,17 @@ def start_span( if parent_span_context is None or not parent_span_context.is_valid: parent_span_context = None + # If parent_span_context exists, we can create traceparent + # and tracestate headers + _headers = {} + if parent_span_context and self.nr_application.settings.distributed_tracing.enabled: + parent_span_trace_id = parent_span_context.trace_id + parent_span_span_id = parent_span_context.span_id + parent_span_trace_flags = parent_span_context.trace_flags + + # If remote_parent, transaction must be created, regardless of kind type + # Make sure we transfer DT headers when we are here, if DT is enabled if parent_span_context and parent_span_context.is_remote: if kind in (otel_api_trace.SpanKind.SERVER, otel_api_trace.SpanKind.CLIENT): # This is a web request @@ -297,6 +447,11 @@ def start_span( port = self.attributes.get("net.host.port") request_method = self.attributes.get("http.method") request_path = self.attributes.get("http.route") + + if not headers: + headers = _headers + update_sampled_flag = True + transaction = WebTransaction( self.nr_application, name=name, @@ -307,7 +462,13 @@ def start_span( request_path=request_path, headers=headers, ) - elif kind in (otel_api_trace.SpanKind.PRODUCER, otel_api_trace.SpanKind.INTERNAL): + + if update_sampled_flag and parent_span_context: + transaction._sampled = bool(parent_span_trace_flags) + elif kind in ( + otel_api_trace.SpanKind.PRODUCER, + otel_api_trace.SpanKind.INTERNAL, + ): transaction = BackgroundTask(self.nr_application, name=name) elif kind == otel_api_trace.SpanKind.CONSUMER: transaction = MessageTransaction( @@ -339,6 +500,10 @@ def start_span( request_method = self.attributes.get("http.method") request_path = self.attributes.get("http.route") + if not headers: + headers = _headers + update_GUID_flag = True + transaction = WebTransaction( self.nr_application, name=name, @@ -349,6 +514,11 @@ def start_span( request_path=request_path, headers=headers, ) + + if update_GUID_flag and parent_span_context: + guid = parent_span_trace_id >> 64 + transaction.guid = f"{guid:x}" + transaction.__enter__() elif kind == otel_api_trace.SpanKind.INTERNAL: if transaction: @@ -436,4 +606,4 @@ def get_tracer( *args, **kwargs, ): - return Tracer(resource=self._resource, instrumentation_library=instrumenting_module_name, *args, **kwargs) \ No newline at end of file + return Tracer(resource=self._resource, instrumentation_library=instrumenting_module_name, *args, **kwargs) diff --git a/newrelic/api/transaction.py b/newrelic/api/transaction.py index 6eb94cbcf..0f73f2df9 100644 --- a/newrelic/api/transaction.py +++ b/newrelic/api/transaction.py @@ -1427,6 +1427,18 @@ def accept_distributed_trace_headers(self, headers, transport_type="HTTP"): data.update(tracestate_data) else: self._record_supportability("Supportability/TraceContext/TraceState/InvalidNrEntry") + elif not payload and (tracestate == self.tracestate): + self._record_supportability("Supportability/TraceContext/TraceState/NoNrEntry") + self._record_supportability("Supportability/TraceContext/TraceState/OtelEntry") + try: + vendors["sa"] = True if vendors.get("sa").lower() == "true" else False + vendors["pr"] = float(vendors.get("pr")) + vendors["ti"] = int(vendors.get("ti")) + + self.trusted_parent_span = vendors.pop("id", None) + data.update(vendors) + except: + pass else: self._record_supportability("Supportability/TraceContext/TraceState/NoNrEntry") diff --git a/newrelic/config.py b/newrelic/config.py index cba1f3938..e05396a8a 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -4370,11 +4370,27 @@ def _process_module_builtin_defaults(): # Hybrid Agent Hooks _process_module_definition( - "opentelemetry.trace", "newrelic.hooks.hybridagent_opentelemetry", "instrument_trace_api" + "opentelemetry.context", + "newrelic.hooks.hybridagent_opentelemetry", + "instrument_context_api", ) _process_module_definition( - "opentelemetry.instrumentation.utils", "newrelic.hooks.hybridagent_opentelemetry", "instrument_utils" + "opentelemetry.instrumentation.propagators", + "newrelic.hooks.hybridagent_opentelemetry", + "instrument_global_propagators_api", + ) + + _process_module_definition( + "opentelemetry.trace", + "newrelic.hooks.hybridagent_opentelemetry", + "instrument_trace_api", + ) + + _process_module_definition( + "opentelemetry.instrumentation.utils", + "newrelic.hooks.hybridagent_opentelemetry", + "instrument_utils", ) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 1923424af..c592338af 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -1272,7 +1272,9 @@ def default_otlp_host(host): _settings.azure_operator.enabled = _environ_as_bool("NEW_RELIC_AZURE_OPERATOR_ENABLED", default=False) _settings.package_reporting.enabled = _environ_as_bool("NEW_RELIC_PACKAGE_REPORTING_ENABLED", default=True) _settings.ml_insights_events.enabled = _environ_as_bool("NEW_RELIC_ML_INSIGHTS_EVENTS_ENABLED", default=False) -_settings.otel_bridge.enabled = _environ_as_bool("NEW_RELIC_OTEL_BRIDGE_ENABLED", default=False) +_settings.otel_bridge.enabled = _environ_as_bool( + "NEW_RELIC_OTEL_BRIDGE_ENABLED", default=False +) def global_settings(): diff --git a/newrelic/hooks/hybridagent_opentelemetry.py b/newrelic/hooks/hybridagent_opentelemetry.py index 4c5538d09..4a6bc1516 100644 --- a/newrelic/hooks/hybridagent_opentelemetry.py +++ b/newrelic/hooks/hybridagent_opentelemetry.py @@ -13,17 +13,68 @@ # limitations under the License. import logging +import os from newrelic.api.application import application_instance from newrelic.api.time_trace import add_custom_span_attribute, current_trace from newrelic.api.transaction import Sentinel, current_transaction from newrelic.common.object_wrapper import wrap_function_wrapper from newrelic.common.signature import bind_args +from newrelic.common.encoding_utils import PARENT_TYPE, NrTraceState, W3CTraceState from newrelic.core.config import global_settings _logger = logging.getLogger(__name__) _TRACER_PROVIDER = None +# Enable OpenTelemetry Bridge to capture HTTP +# request/response headers as span attributes: +os.environ["OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST"] = ".*" +os.environ["OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE"] = ".*" + + +########################################### +# Context Instrumentation +########################################### + +def wrap__load_runtime_context(wrapped, instance, args, kwargs): + settings = global_settings() + + if not settings.otel_bridge.enabled: + return wrapped(*args, **kwargs) + + from opentelemetry.context.contextvars_context import ContextVarsRuntimeContext + + context = ContextVarsRuntimeContext() + return context + + +def wrap_get_global_response_propagator(wrapped, instance, args, kwargs): + settings = global_settings() + + if not settings.otel_bridge.enabled: + return wrapped(*args, **kwargs) + + from newrelic.api.opentelemetry import otel_context_propagator + from opentelemetry.instrumentation.propagators import set_global_response_propagator + + set_global_response_propagator(otel_context_propagator) + + return otel_context_propagator + + +def instrument_context_api(module): + + if hasattr(module, "_load_runtime_context"): + wrap_function_wrapper(module, "_load_runtime_context", wrap__load_runtime_context) + + +def instrument_global_propagators_api(module): + # Need to disable this instrumentation if settings.otel_bridge is disabled + + if hasattr(module, "get_global_response_propagator"): + wrap_function_wrapper(module, "get_global_response_propagator", wrap_get_global_response_propagator) + + ########################################### # Trace Instrumentation ########################################### @@ -31,7 +82,6 @@ def wrap_set_tracer_provider(wrapped, instance, args, kwargs): settings = global_settings() - if not settings.otel_bridge.enabled: return wrapped(*args, **kwargs) @@ -112,7 +162,21 @@ def set_attributes(self, attributes): for key, value in attributes.items(): add_custom_span_attribute(key, value) - otel_tracestate_headers = None + if transaction.settings.distributed_tracing.enabled: + transaction._trace_id = f'{span.get_span_context().trace_id:x}' + guid = span.get_span_context().trace_id >> 64 + transaction.guid = f'{guid:x}' + + nr_tracestate_headers = ( + transaction._create_distributed_trace_data() + ) + nr_tracestate_headers["sa"] = bool(span.get_span_context().trace_flags) + + otel_tracestate_headers = [ + (key, str(value)) for key, value in nr_tracestate_headers.items() + ] + else: + otel_tracestate_headers = None span_context = otel_api_trace.SpanContext( trace_id=int(transaction.trace_id, 16), @@ -149,6 +213,19 @@ def wrap_start_internal_or_server_span(wrapped, instance, args, kwargs): else: nr_headers = context_carrier.copy() + if ("tracestate" in nr_headers) and ("@nr=" not in nr_headers["tracestate"]): + # The tracestate has been generated by OTel and is in OTel format + otel_tracestate = nr_headers.pop("tracestate", None) + tracestate_dict = W3CTraceState.decode(otel_tracestate) + + # Convert sampled, priority, and timestamp data types + tracestate_dict["sa"] = bool(tracestate_dict.get("sa")) + tracestate_dict["pr"] = float(tracestate_dict.get("pr")) + tracestate_dict["ti"] = int(tracestate_dict.get("ti")) + + nr_tracestate = NrTraceState(tracestate_dict).text() + nr_headers["tracestate"] = nr_tracestate + attributes["nr.nonhttp.headers"] = nr_headers bound_args["attributes"] = attributes diff --git a/tests/hybridagent_opentelemetry/cross_agent/TestCaseDefinitions.json b/tests/hybridagent_opentelemetry/cross_agent/TestCaseDefinitions.json new file mode 100644 index 000000000..ff91707c8 --- /dev/null +++ b/tests/hybridagent_opentelemetry/cross_agent/TestCaseDefinitions.json @@ -0,0 +1,641 @@ +[ + { + "testDescription": "Does not create segment without a transaction", + + "operations": [ + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "Bar", + "spanKind": "Internal" + }, + "assertions": [ + { + "description": "The OpenTelmetry span should not be created", + "rule": { + "operator": "NotValid", + "parameters": { + "object": "currentOTelSpan" + } + } + }, + { + "description": "There should be no transaction", + "rule": { + "operator": "NotValid", + "parameters": { + "object": "currentTransaction" + } + } + } + ] + } + ], + + "agentOutput": { + "transactions": [], + "spans": [] + } + }, + + { + "testDescription": "Creates OpenTelemetry segment in a transaction", + + "operations": [ + { + "command": "DoWorkInTransaction", + "parameters": { + "transactionName": "Foo" + }, + "childOperations": [ + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "Bar", + "spanKind": "Internal" + }, + "assertions": [ + { + "description": "OpenTelemetry API and New Relic API report the same traceId", + "rule": { + "operator": "Equals", + "parameters": { + "left": "currentOTelSpan.traceId", + "right": "currentTransaction.traceId" + } + } + }, + { + "description": "OpenTelemetry API and New Relic API report the same spanId", + "rule": { + "operator": "Equals", + "parameters": { + "left": "currentOTelSpan.spanId", + "right": "currentSegment.spanId" + } + } + } + ] + } + ] + } + ], + + "agentOutput": { + "transactions": [ + { + "name": "Foo" + } + ], + "spans": [ + { + "name": "Bar", + "category": "generic", + "parentName": "Foo" + }, + { + "name": "Foo", + "category": "generic", + "entryPoint": true + } + ] + } + }, + + { + "testDescription": "Creates New Relic span as child of OpenTelemetry span", + + "operations": [ + { + "command": "DoWorkInTransaction", + "parameters": { + "transactionName": "Foo" + }, + "childOperations": [ + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "Bar", + "spanKind": "Internal" + }, + "childOperations": [ + { + "command": "DoWorkInSegment", + "parameters": { + "segmentName": "Baz" + }, + "assertions": [ + { + "description": "OpenTelemetry API and New Relic API report the same traceId", + "rule": { + "operator": "Equals", + "parameters": { + "left": "currentOTelSpan.traceId", + "right": "currentTransaction.traceId" + } + } + }, + { + "description": "OpenTelemetry API and New Relic API report the same spanId", + "rule": { + "operator": "Equals", + "parameters": { + "left": "currentOTelSpan.spanId", + "right": "currentSegment.spanId" + } + } + } + ] + } + ] + } + ] + } + ], + + "agentOutput": { + "transactions": [ + { + "name": "Foo" + } + ], + "spans": [ + { + "name": "Baz", + "category": "generic", + "parentName": "Bar" + }, + { + "name": "Bar", + "category": "generic", + "parentName": "Foo" + }, + { + "name": "Foo", + "category": "generic", + "entryPoint": true + } + ] + } + }, + + { + "testDescription": "OpenTelemetry API can add custom attributes to spans", + + "operations": [ + { + "command": "DoWorkInTransaction", + "parameters": { + "transactionName": "Foo" + }, + "childOperations": [ + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "Bar", + "spanKind": "Internal" + }, + "childOperations": [ + { + "command": "DoWorkInSegment", + "parameters": { + "segmentName": "Baz" + }, + "childOperations": [ + { + "command": "AddOTelAttribute", + "parameters": { + "name": "spanNumber", + "value": 2 + } + } + ] + }, + { + "command": "AddOTelAttribute", + "parameters": { + "name": "spanNumber", + "value": 1 + } + } + ] + } + ] + } + ], + + "agentOutput": { + "transactions": [ + { + "name": "Foo" + } + ], + "spans": [ + { + "name": "Baz", + "attributes": { + "spanNumber": 2 + } + }, + { + "name": "Bar", + "attributes": { + "spanNumber": 1 + } + } + ] + } + }, + + { + "testDescription": "OpenTelemetry API can record errors", + + "operations": [ + { + "command": "DoWorkInTransaction", + "parameters": { + "transactionName": "Foo" + }, + "childOperations": [ + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "Bar", + "spanKind": "Internal" + }, + "childOperations": [ + { + "command": "RecordExceptionOnSpan", + "parameters": { + "errorMessage": "Test exception message" + } + } + ] + } + ] + } + ], + + "agentOutput": { + "transactions": [ + { + "name": "Foo" + } + ], + "spans": [ + { + "name": "Bar", + "attributes": { + "error.message": "Test exception message" + } + } + ] + } + }, + + { + "testDescription": "OpenTelemetry API and New Relic API can inject outbound trace context", + + "operations": [ + { + "command": "DoWorkInTransaction", + "parameters": { + "transactionName": "Foo" + }, + "childOperations": [ + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "OTelSpan1", + "spanKind": "Client" + }, + "childOperations": [ + { + "command": "SimulateExternalCall", + "parameters": { + "url": "url1" + }, + "childOperations": [ + { + "command": "OTelInjectHeaders", + "assertions": [ + { + "description": "Correct traceId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.traceId", + "right": "currentTransaction.traceId" + } + } + }, + { + "description": "Correct spanId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.spanId", + "right": "currentSegment.spanId" + } + } + }, + { + "description": "Correct sampled flag was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.sampled", + "right": "currentTransaction.sampled" + } + } + } + ] + } + ] + } + ] + }, + { + "command": "DoWorkInSegment", + "parameters": { + "segmentName": "segment1" + }, + "childOperations": [ + { + "command": "SimulateExternalCall", + "parameters": { + "url": "url2" + }, + "childOperations": [ + { + "command": "OTelInjectHeaders", + "assertions": [ + { + "description": "Correct traceId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.traceId", + "right": "currentTransaction.traceId" + } + } + }, + { + "description": "Correct spanId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.spanId", + "right": "currentSegment.spanId" + } + } + }, + { + "description": "Correct sampled flag was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.sampled", + "right": "currentTransaction.sampled" + } + } + } + ] + } + ] + } + ] + }, + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "OTelSpan2", + "spanKind": "Client" + }, + "childOperations": [ + { + "command": "SimulateExternalCall", + "parameters": { + "url": "url3" + }, + "childOperations": [ + { + "command": "NRInjectHeaders", + "assertions": [ + { + "description": "Correct traceId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.traceId", + "right": "currentTransaction.traceId" + } + } + }, + { + "description": "Correct spanId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.spanId", + "right": "currentSegment.spanId" + } + } + }, + { + "description": "Correct sampled flag was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.sampled", + "right": "currentTransaction.sampled" + } + } + } + ] + } + ] + } + ] + }, + { + "command": "DoWorkInSegment", + "parameters": { + "segmentName": "segment2" + }, + "childOperations": [ + { + "command": "SimulateExternalCall", + "parameters": { + "url": "url4" + }, + "childOperations": [ + { + "command": "NRInjectHeaders", + "assertions": [ + { + "description": "Correct traceId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.traceId", + "right": "currentTransaction.traceId" + } + } + }, + { + "description": "Correct spanId was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.spanId", + "right": "currentSegment.spanId" + } + } + }, + { + "description": "Correct sampled flag was injected", + "rule": { + "operator": "Equals", + "parameters": { + "left": "injected.sampled", + "right": "currentTransaction.sampled" + } + } + } + ] + } + ] + } + ] + } + ] + } + ], + + "agentOutput": { + "transactions": [ + { + "name": "Foo" + } + ], + "spans": [ + { + "name": "OTelSpan1", + "parentName": "Foo" + }, + { + "name": "segment1", + "parentName": "Foo" + }, + { + "name": "OTelSpan2", + "parentName": "Foo" + }, + { + "name": "segment2", + "parentName": "Foo" + } + ] + } + }, + + { + "testDescription": "Starting transaction tests", + + "operations": [ + { + "command": "DoWorkInSpan", + "parameters": { + "spanName": "Foo", + "spanKind": "Server" + } + }, + { + "command": "DoWorkInSpanWithRemoteParent", + "parameters": { + "spanName": "Bar", + "spanKind": "Server" + } + }, + { + "command": "DoWorkInTransaction", + "parameters": { + "transactionName": "Baz" + }, + "childOperations": [ + { + "command": "DoWorkInSpanWithRemoteParent", + "parameters": { + "spanName": "EdgeCase", + "spanKind": "Server" + } + } + ] + } + ], + + "agentOutput": { + "transactions": [ + { + "name": "Foo" + }, + { + "name": "Bar" + }, + { + "name": "Baz" + } + ], + "spans": [ + { + "name": "EdgeCase", + "parentName": "Baz" + }, + { + "name": "Baz", + "entryPoint": true + } + ] + } + }, + + { + "testDescription": "Inbound distributed tracing tests", + + "operations": [ + { + "command": "DoWorkInSpanWithInboundContext", + "parameters": { + "spanName": "Foo", + "spanKind": "Server", + "traceIdInHeader": "da8bc8cc6d062849b0efcf3c169afb5a", + "spanIdInHeader": "7d3efb1b173fecfa", + "sampledFlagInHeader": "0" + }, + "assertions": [ + { + "description": "Current span has expected traceId", + "rule": { + "operator": "Matches", + "parameters": { + "object": "currentOTelSpan.traceId", + "value": "da8bc8cc6d062849b0efcf3c169afb5a" + } + } + } + ] + } + ], + + "agentOutput": { + "transactions": [ + { + "name": "Foo" + } + ], + "spans": [] + } + } + +] diff --git a/tests/hybridagent_opentelemetry/test_traces_attributes.py b/tests/hybridagent_opentelemetry/test_attributes.py similarity index 64% rename from tests/hybridagent_opentelemetry/test_traces_attributes.py rename to tests/hybridagent_opentelemetry/test_attributes.py index b50a99e1b..f4b9890d8 100644 --- a/tests/hybridagent_opentelemetry/test_traces_attributes.py +++ b/tests/hybridagent_opentelemetry/test_attributes.py @@ -26,8 +26,8 @@ def test_trace_with_span_attributes(tracer): @validate_span_events( count=1, exact_intrinsics={ - "name": "Function/test_traces_attributes:test_trace_with_span_attributes.._test", - "transaction.name": "OtherTransaction/Function/test_traces_attributes:test_trace_with_span_attributes.._test", + "name": "Function/test_attributes:test_trace_with_span_attributes.._test", + "transaction.name": "OtherTransaction/Function/test_attributes:test_trace_with_span_attributes.._test", "sampled": True, }, ) @@ -50,50 +50,59 @@ def test_trace_with_otel_to_newrelic(tracer): This test adds custom attributes to the transaction and trace. * `add_custom_attribute` adds custom attributes to the transaction. * `add_custom_span_attribute` adds custom attributes to the trace. - NOTE: a transaction's custom attributes are added to the root - span's user attributes. + NOTE: + 1. Distinction between trace and span attributes is given for + whether they are added to the transaction or the span. A + transaction's custom attributes are added to the root span's + user attributes. + 2. Notation for attributes: + - NR trace attributes: "NR_trace_attribute_" + - NR span attributes: "NR_span_attribute_" + - OTel span attributes: "otel_span_attribute_" + Where is either `FT` or `BG`, for FunctionTrace + or BackgroundTask, respectively. """ @function_trace() def newrelic_function_trace(): - add_custom_attribute("NR_trace_attribute_from_function", "NR trace attribute") - add_custom_span_attribute("NR_span_attribute_from_function", "NR span attribute") + add_custom_attribute("NR_trace_attribute_FT", "NR trace attribute from FT") + add_custom_span_attribute("NR_span_attribute_FT", "NR span attribute from FT") otel_span = trace.get_current_span() - otel_span.set_attribute("otel_span_attribute_from_function", "OTel span attribute from FT") + otel_span.set_attribute("otel_span_attribute_FT", "OTel span attribute from FT") @validate_span_events( count=1, exact_intrinsics={ - "name": "Function/test_traces_attributes:test_trace_with_otel_to_newrelic..newrelic_background_task", - "transaction.name": "OtherTransaction/Function/test_traces_attributes:test_trace_with_otel_to_newrelic..newrelic_background_task", + "name": "Function/test_attributes:test_trace_with_otel_to_newrelic..newrelic_background_task", + "transaction.name": "OtherTransaction/Function/test_attributes:test_trace_with_otel_to_newrelic..newrelic_background_task", "sampled": True, }, - exact_users={"NR_trace_attribute_from_function": "NR trace attribute"}, + exact_users={"NR_trace_attribute_FT": "NR trace attribute from FT"}, ) @validate_span_events( count=1, exact_intrinsics={"name": "Function/foo", "sampled": True}, expected_intrinsics={"priority": None, "traceId": None, "guid": None}, exact_users={ - "nr_trace_attribute": "NR span attribute from BG", + "NR_span_attribute_BG": "NR span attribute from BG", "otel_span_attribute_BG": "OTel span attribute from BG", }, ) @validate_span_events( count=1, exact_intrinsics={ - "name": "Function/test_traces_attributes:test_trace_with_otel_to_newrelic..newrelic_function_trace", + "name": "Function/test_attributes:test_trace_with_otel_to_newrelic..newrelic_function_trace", "sampled": True, }, exact_users={ - "NR_span_attribute_from_function": "NR span attribute", - "otel_span_attribute_from_function": "OTel span attribute from FT", + "NR_span_attribute_FT": "NR span attribute from FT", + "otel_span_attribute_FT": "OTel span attribute from FT", }, ) @background_task() def newrelic_background_task(): with tracer.start_as_current_span("foo") as otel_span: - add_custom_span_attribute("nr_trace_attribute", "NR span attribute from BG") + add_custom_span_attribute("NR_span_attribute_BG", "NR span attribute from BG") otel_span.set_attribute("otel_span_attribute_BG", "OTel span attribute from BG") newrelic_function_trace() diff --git a/tests/hybridagent_opentelemetry/test_hybrid_cross_agent.py b/tests/hybridagent_opentelemetry/test_hybrid_cross_agent.py index 9a31d2bf6..dcd129062 100644 --- a/tests/hybridagent_opentelemetry/test_hybrid_cross_agent.py +++ b/tests/hybridagent_opentelemetry/test_hybrid_cross_agent.py @@ -12,17 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -from opentelemetry import trace as otel_api_trace -from testing_support.validators.validate_error_event_attributes import validate_error_event_attributes -from testing_support.validators.validate_span_events import validate_span_events -from testing_support.validators.validate_transaction_count import validate_transaction_count -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from opentelemetry import trace as otel_api_trace, propagate as otel_api_propagate from newrelic.api.application import application_instance +from newrelic.api.transaction import current_transaction +from newrelic.api.time_trace import current_trace from newrelic.api.background_task import BackgroundTask from newrelic.api.function_trace import FunctionTrace -from newrelic.api.time_trace import current_trace -from newrelic.api.transaction import current_transaction +from newrelic.api.external_trace import ExternalTrace + +from testing_support.validators.validate_transaction_count import validate_transaction_count +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_error_event_attributes import validate_error_event_attributes +from testing_support.fixtures import override_application_settings + +PROPAGATOR = otel_api_propagate.get_global_textmap() # Does not create segment without a transaction @@ -39,7 +44,18 @@ def test_does_not_create_segment_without_a_transaction(tracer): # Creates OpenTelemetry segment in a transaction @validate_transaction_metrics(name="Foo", background_task=True) @validate_span_events( - exact_intrinsics={"name": "Function/Bar", "category": "generic"}, expected_intrinsics=("parentId",) + exact_intrinsics={ + "name": "Function/Bar", + "category": "generic", + }, + expected_intrinsics=("parentId",) +) +@validate_span_events( + exact_intrinsics={ + "name": "Function/Foo", + "category": "generic", + "nr.entryPoint": True + }, ) @validate_span_events(exact_intrinsics={"name": "Function/Foo", "category": "generic", "nr.entryPoint": True}) def test_creates_opentelemetry_segment_in_a_transaction(tracer): @@ -49,7 +65,7 @@ def test_creates_opentelemetry_segment_in_a_transaction(tracer): with tracer.start_as_current_span(name="Bar", kind=otel_api_trace.SpanKind.INTERNAL): # OpenTelemetry API and New Relic API report the same traceId assert otel_api_trace.get_current_span().get_span_context().trace_id == int( - current_transaction()._trace_id, 16 + current_transaction().trace_id, 16 ) # OpenTelemetry API and New Relic API report the same spanId @@ -59,10 +75,25 @@ def test_creates_opentelemetry_segment_in_a_transaction(tracer): # Creates New Relic span as child of OpenTelemetry span @validate_transaction_metrics(name="Foo", background_task=True) @validate_span_events( - exact_intrinsics={"name": "Function/Baz", "category": "generic"}, expected_intrinsics=("parentId",) + exact_intrinsics={ + "name": "Function/Baz", + "category": "generic", + }, + expected_intrinsics=("parentId",) ) @validate_span_events( - exact_intrinsics={"name": "Function/Bar", "category": "generic"}, expected_intrinsics=("parentId",) + exact_intrinsics={ + "name": "Function/Bar", + "category": "generic", + }, + expected_intrinsics=("parentId",) +) +@validate_span_events( + exact_intrinsics={ + "name": "Function/Foo", + "category": "generic", + "nr.entryPoint": True + }, ) @validate_span_events(exact_intrinsics={"name": "Function/Foo", "category": "generic"}) def test_creates_new_relic_span_as_child_of_open_telemetry_span(tracer): @@ -110,3 +141,177 @@ def test_opentelemetry_api_can_record_errors(tracer): raise Exception("Test exception message") except Exception as e: otel_api_trace.get_current_span().record_exception(e) + + +# OpenTelemetry API and New Relic API can inject outbound trace context +@validate_transaction_metrics(name="Foo", background_task=True) +@validate_span_events( + exact_intrinsics={ + "name": "External/url1/OtelSpan1/GET", + }, + expected_intrinsics=("parentId",) +) +@validate_span_events( + exact_intrinsics={ + "name": "External/url2/segment1/GET", + }, + expected_intrinsics=("parentId",) +) +@validate_span_events( + exact_intrinsics={ + "name": "External/url3/OtelSpan2/GET", + }, + expected_intrinsics=("parentId",) +) +@validate_span_events( + exact_intrinsics={ + "name": "External/url4/segment2/GET", + }, + expected_intrinsics=("parentId",) +) +def test_opentelemetry_api_and_new_relic_api_can_inject_outbound_trace_context(tracer): + application = application_instance(activate=False) + + with BackgroundTask(application, name="Foo"): + transaction = current_transaction() + with tracer.start_as_current_span( + name="OtelSpan1", + kind=otel_api_trace.SpanKind.CLIENT, + attributes={"http.url": "http://url1", "http.method": "GET"} + ): + headers = {} + PROPAGATOR.inject(carrier=headers) + _, trace_id, span_id, sampled = headers["traceparent"].split("-") + + # Correct traceId was injected + assert transaction.trace_id == trace_id + + # Correct spanId was injected + assert current_trace().guid == span_id + + # Correct sampled flag was injected + assert transaction.sampled == (sampled == "01") + + # Reset the distributed trace state for the purposes of this test + transaction._distributed_trace_state = 0 + + with ExternalTrace(library="segment1", url="http://url2", method="GET"): + headers = {} + PROPAGATOR.inject(carrier=headers) + _, trace_id, span_id, sampled = headers["traceparent"].split("-") + + # Correct traceId was injected + assert current_transaction().trace_id == trace_id + + # Correct spanId was injected + assert current_trace().guid == span_id + + # Correct sampled flag was injected + assert current_transaction().sampled == (sampled == "01") + + # Reset the distributed trace state for the purposes of this test + transaction._distributed_trace_state = 0 + + with tracer.start_as_current_span( + name="OtelSpan2", + kind=otel_api_trace.SpanKind.CLIENT, + attributes={"http.url": "http://url3", "http.method": "GET"} + ): + headers = [] + transaction.insert_distributed_trace_headers(headers) + _, trace_id, span_id, sampled = headers[0][1].split("-") + + # Correct traceId was injected + assert transaction.trace_id == trace_id + + # Correct spanId was injected + assert current_trace().guid == span_id + + # Correct sampled flag was injected + assert transaction.sampled == (sampled == "01") + + # Reset the distributed trace state for the purposes of this test + transaction._distributed_trace_state = 0 + + with ExternalTrace(library="segment2", url="http://url4", method="GET"): + headers = [] + transaction.insert_distributed_trace_headers(headers) + _, trace_id, span_id, sampled = headers[0][1].split("-") + + # Correct traceId was injected + assert current_transaction().trace_id == trace_id + + # Correct spanId was injected + assert current_trace().guid == span_id + + # Correct sampled flag was injected + assert current_transaction().sampled == (sampled == "01") + + +# Starting transaction tests +@validate_transaction_metrics(name="Foo", index=-3) +@validate_transaction_metrics(name="Bar", index=-2) +@validate_transaction_metrics(name="Baz", background_task=True, index=-1) +@validate_span_events( + exact_intrinsics={ + "name": "Function/EdgeCase", + }, + expected_intrinsics=("parentId",) +) +@validate_span_events( + exact_intrinsics={ + "name": "Function/Baz", + "nr.entryPoint": True + }, +) +def test_starting_transaction_tests(tracer): + application = application_instance(activate=False) + + with tracer.start_as_current_span(name="Foo", kind=otel_api_trace.SpanKind.SERVER): + pass + + # Create remote span context and remote context + remote_span_context = otel_api_trace.SpanContext( + trace_id=0x1234567890abcdef1234567890abcdef, + span_id=0x1234567890abcdef, + is_remote=True, + trace_flags=otel_api_trace.TraceFlags.SAMPLED, + trace_state=otel_api_trace.TraceState() + ) + remote_context = otel_api_trace.set_span_in_context(otel_api_trace.NonRecordingSpan(remote_span_context)) + + with tracer.start_as_current_span(name="Bar", kind=otel_api_trace.SpanKind.SERVER, context=remote_context): + pass + + with BackgroundTask(application, name="Baz"): + with tracer.start_as_current_span(name="EdgeCase", kind=otel_api_trace.SpanKind.SERVER, context=remote_context): + pass + + +# Inbound distributed tracing tests +@validate_transaction_metrics(name="Foo") +@validate_span_events(count=0) +@override_application_settings( + { + "trusted_account_key": "1", + "account_id": "1", + } +) +def test_inbound_distributed_tracing_tests(tracer): + """ + This test intends to check for a scenario where an external call + span is made outside the context of an existing transaction. + By flagging that span as not sampled in OTel, it means that the OTel + api will reflect that our agent is also ignoring that span. In + this case, it will create the server transaction, but no spans. + """ + with tracer.start_as_current_span(name="Foo", kind=otel_api_trace.SpanKind.SERVER): + carrier = { + "traceparent": "00-da8bc8cc6d062849b0efcf3c169afb5a-7d3efb1b173fecfa-00", + "tracestate": "1@nr=0-0-1-12345678-7d3efb1b173fecfa-da8bc8cc6d062849-0-0.23456-1011121314151", + } + PROPAGATOR.extract(carrier=carrier) + + current_span = otel_api_trace.get_current_span() + + assert current_span.get_span_context().trace_id == 0xda8bc8cc6d062849b0efcf3c169afb5a diff --git a/tests/hybridagent_opentelemetry/test_settings.py b/tests/hybridagent_opentelemetry/test_settings.py index 6b50b020f..84c6bd789 100644 --- a/tests/hybridagent_opentelemetry/test_settings.py +++ b/tests/hybridagent_opentelemetry/test_settings.py @@ -19,6 +19,7 @@ from newrelic.api.transaction import current_transaction from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from testing_support.fixtures import override_application_settings @@ -26,14 +27,9 @@ "enabled", [True, False] ) -def test_distributed_tracing_enabled(tracer, enabled): +def test_opentelemetry_bridge_enabled(tracer, enabled): @override_application_settings({"otel_bridge.enabled": enabled}) - @validate_span_events( - count=1, - exact_intrinsics={ - "name": "Function/Foo" - } - ) + @validate_transaction_metrics(name="Foo", background_task=True) @validate_span_events( count=1 if enabled else 0, exact_intrinsics={ diff --git a/tox.ini b/tox.ini index 5017adceb..4187b6f4a 100644 --- a/tox.ini +++ b/tox.ini @@ -491,6 +491,7 @@ commands = framework_azurefunctions: {toxinidir}/.github/scripts/install_azure_functions_worker.sh + coverage run -m pytest -v [] allowlist_externals =