Expose AWS Progagator variables and update readme

This commit is contained in:
Nathaniel Ruiz Nowell
2020-11-09 12:55:20 -08:00
parent ea0988a26a
commit c0a3ea96d9
3 changed files with 101 additions and 96 deletions

View File

@ -41,7 +41,7 @@ Propagator:
:: ::
export OTEL_PYTHON_PROPAGATORS = aws_xray export OTEL_PROPAGATORS = aws_xray
References References

View File

@ -24,6 +24,27 @@ from opentelemetry.trace.propagation.textmap import (
TextMapPropagatorT, TextMapPropagatorT,
) )
TRACE_HEADER_KEY = "X-Amzn-Trace-Id"
KV_PAIR_DELIMITER = ";"
KEY_AND_VALUE_DELIMITER = "="
TRACE_ID_KEY = "Root"
TRACE_ID_LENGTH = 35
TRACE_ID_VERSION = "1"
TRACE_ID_DELIMITER = "-"
TRACE_ID_DELIMITER_INDEX_1 = 1
TRACE_ID_DELIMITER_INDEX_2 = 10
TRACE_ID_FIRST_PART_LENGTH = 8
PARENT_ID_KEY = "Parent"
PARENT_ID_LENGTH = 16
SAMPLED_FLAG_KEY = "Sampled"
SAMPLED_FLAG_LENGTH = 1
IS_SAMPLED = "1"
NOT_SAMPLED = "0"
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
@ -40,26 +61,6 @@ class AwsXRayFormat(TextMapPropagator):
""" """
# AWS # AWS
TRACE_HEADER_KEY = "X-Amzn-Trace-Id"
KV_PAIR_DELIMITER = ";"
KEY_AND_VALUE_DELIMITER = "="
TRACE_ID_KEY = "Root"
TRACE_ID_LENGTH = 35
TRACE_ID_VERSION = "1"
TRACE_ID_DELIMITER = "-"
TRACE_ID_DELIMITER_INDEX_1 = 1
TRACE_ID_DELIMITER_INDEX_2 = 10
TRACE_ID_FIRST_PART_LENGTH = 8
PARENT_ID_KEY = "Parent"
PARENT_ID_LENGTH = 16
SAMPLED_FLAG_KEY = "Sampled"
SAMPLED_FLAG_LENGTH = 1
IS_SAMPLED = "1"
NOT_SAMPLED = "0"
def extract( def extract(
self, self,
@ -67,8 +68,7 @@ class AwsXRayFormat(TextMapPropagator):
carrier: TextMapPropagatorT, carrier: TextMapPropagatorT,
context: typing.Optional[Context] = None, context: typing.Optional[Context] = None,
) -> Context: ) -> Context:
trace_header_list = getter(carrier, self.TRACE_HEADER_KEY) trace_header_list = getter.get(carrier, TRACE_HEADER_KEY)
trace_header_list = getter.get(carrier, self.TRACE_HEADER_KEY)
if not trace_header_list or len(trace_header_list) != 1: if not trace_header_list or len(trace_header_list) != 1:
return trace.set_span_in_context( return trace.set_span_in_context(
@ -83,9 +83,11 @@ class AwsXRayFormat(TextMapPropagator):
) )
try: try:
trace_id, span_id, sampled = self._extract_span_properties( (
trace_header trace_id,
) span_id,
sampled,
) = AwsXRayFormat._extract_span_properties(trace_header)
except AwsParseTraceHeaderError as err: except AwsParseTraceHeaderError as err:
_logger.debug(err.message) _logger.debug(err.message)
return trace.set_span_in_context( return trace.set_span_in_context(
@ -116,16 +118,15 @@ class AwsXRayFormat(TextMapPropagator):
trace.DefaultSpan(span_context), context=context trace.DefaultSpan(span_context), context=context
) )
def _extract_span_properties(self, trace_header): @staticmethod
def _extract_span_properties(trace_header):
trace_id = trace.INVALID_TRACE_ID trace_id = trace.INVALID_TRACE_ID
span_id = trace.INVALID_SPAN_ID span_id = trace.INVALID_SPAN_ID
sampled = False sampled = False
for kv_pair_str in trace_header.split(self.KV_PAIR_DELIMITER): for kv_pair_str in trace_header.split(KV_PAIR_DELIMITER):
try: try:
key_str, value_str = kv_pair_str.split( key_str, value_str = kv_pair_str.split(KEY_AND_VALUE_DELIMITER)
self.KEY_AND_VALUE_DELIMITER
)
key, value = key_str.strip(), value_str.strip() key, value = key_str.strip(), value_str.strip()
except ValueError as ex: except ValueError as ex:
raise AwsParseTraceHeaderError( raise AwsParseTraceHeaderError(
@ -134,32 +135,32 @@ class AwsXRayFormat(TextMapPropagator):
kv_pair_str, kv_pair_str,
) )
) from ex ) from ex
if key == self.TRACE_ID_KEY: if key == TRACE_ID_KEY:
if not self._validate_trace_id(value): if not AwsXRayFormat._validate_trace_id(value):
raise AwsParseTraceHeaderError( raise AwsParseTraceHeaderError(
( (
"Invalid TraceId in X-Ray trace header: '%s' with value '%s'. Returning INVALID span context.", "Invalid TraceId in X-Ray trace header: '%s' with value '%s'. Returning INVALID span context.",
self.TRACE_HEADER_KEY, TRACE_HEADER_KEY,
trace_header, trace_header,
) )
) )
try: try:
trace_id = self._parse_trace_id(value) trace_id = AwsXRayFormat._parse_trace_id(value)
except ValueError as ex: except ValueError as ex:
raise AwsParseTraceHeaderError( raise AwsParseTraceHeaderError(
( (
"Invalid TraceId in X-Ray trace header: '%s' with value '%s'. Returning INVALID span context.", "Invalid TraceId in X-Ray trace header: '%s' with value '%s'. Returning INVALID span context.",
self.TRACE_HEADER_KEY, TRACE_HEADER_KEY,
trace_header, trace_header,
) )
) from ex ) from ex
elif key == self.PARENT_ID_KEY: elif key == PARENT_ID_KEY:
if not self._validate_span_id(value): if not AwsXRayFormat._validate_span_id(value):
raise AwsParseTraceHeaderError( raise AwsParseTraceHeaderError(
( (
"Invalid ParentId in X-Ray trace header: '%s' with value '%s'. Returning INVALID span context.", "Invalid ParentId in X-Ray trace header: '%s' with value '%s'. Returning INVALID span context.",
self.TRACE_HEADER_KEY, TRACE_HEADER_KEY,
trace_header, trace_header,
) )
) )
@ -170,61 +171,63 @@ class AwsXRayFormat(TextMapPropagator):
raise AwsParseTraceHeaderError( raise AwsParseTraceHeaderError(
( (
"Invalid TraceId in X-Ray trace header: '%s' with value '%s'. Returning INVALID span context.", "Invalid TraceId in X-Ray trace header: '%s' with value '%s'. Returning INVALID span context.",
self.TRACE_HEADER_KEY, TRACE_HEADER_KEY,
trace_header, trace_header,
) )
) from ex ) from ex
elif key == self.SAMPLED_FLAG_KEY: elif key == SAMPLED_FLAG_KEY:
if not self._validate_sampled_flag(value): if not AwsXRayFormat._validate_sampled_flag(value):
raise AwsParseTraceHeaderError( raise AwsParseTraceHeaderError(
( (
"Invalid Sampling flag in X-Ray trace header: '%s' with value '%s'. Returning INVALID span context.", "Invalid Sampling flag in X-Ray trace header: '%s' with value '%s'. Returning INVALID span context.",
self.TRACE_HEADER_KEY, TRACE_HEADER_KEY,
trace_header, trace_header,
) )
) )
sampled = self._parse_sampled_flag(value) sampled = AwsXRayFormat._parse_sampled_flag(value)
return trace_id, span_id, sampled return trace_id, span_id, sampled
def _validate_trace_id(self, trace_id_str): @staticmethod
def _validate_trace_id(trace_id_str):
return ( return (
len(trace_id_str) == self.TRACE_ID_LENGTH len(trace_id_str) == TRACE_ID_LENGTH
and trace_id_str.startswith(self.TRACE_ID_VERSION) and trace_id_str.startswith(TRACE_ID_VERSION)
and trace_id_str[self.TRACE_ID_DELIMITER_INDEX_1] and trace_id_str[TRACE_ID_DELIMITER_INDEX_1] == TRACE_ID_DELIMITER
== self.TRACE_ID_DELIMITER and trace_id_str[TRACE_ID_DELIMITER_INDEX_2] == TRACE_ID_DELIMITER
and trace_id_str[self.TRACE_ID_DELIMITER_INDEX_2]
== self.TRACE_ID_DELIMITER
) )
def _parse_trace_id(self, trace_id_str): @staticmethod
def _parse_trace_id(trace_id_str):
timestamp_subset = trace_id_str[ timestamp_subset = trace_id_str[
self.TRACE_ID_DELIMITER_INDEX_1 TRACE_ID_DELIMITER_INDEX_1 + 1 : TRACE_ID_DELIMITER_INDEX_2
+ 1 : self.TRACE_ID_DELIMITER_INDEX_2
] ]
unique_id_subset = trace_id_str[ unique_id_subset = trace_id_str[
self.TRACE_ID_DELIMITER_INDEX_2 + 1 : self.TRACE_ID_LENGTH TRACE_ID_DELIMITER_INDEX_2 + 1 : TRACE_ID_LENGTH
] ]
return int(timestamp_subset + unique_id_subset, 16) return int(timestamp_subset + unique_id_subset, 16)
def _validate_span_id(self, span_id_str): @staticmethod
return len(span_id_str) == self.PARENT_ID_LENGTH def _validate_span_id(span_id_str):
return len(span_id_str) == PARENT_ID_LENGTH
@staticmethod @staticmethod
def _parse_span_id(span_id_str): def _parse_span_id(span_id_str):
return int(span_id_str, 16) return int(span_id_str, 16)
def _validate_sampled_flag(self, sampled_flag_str): @staticmethod
def _validate_sampled_flag(sampled_flag_str):
return len( return len(
sampled_flag_str sampled_flag_str
) == self.SAMPLED_FLAG_LENGTH and sampled_flag_str in ( ) == SAMPLED_FLAG_LENGTH and sampled_flag_str in (
self.IS_SAMPLED, IS_SAMPLED,
self.NOT_SAMPLED, NOT_SAMPLED,
) )
def _parse_sampled_flag(self, sampled_flag_str): @staticmethod
return sampled_flag_str[0] == self.IS_SAMPLED def _parse_sampled_flag(sampled_flag_str):
return sampled_flag_str[0] == IS_SAMPLED
def inject( def inject(
self, self,
@ -240,37 +243,37 @@ class AwsXRayFormat(TextMapPropagator):
otel_trace_id = "{:032x}".format(span_context.trace_id) otel_trace_id = "{:032x}".format(span_context.trace_id)
xray_trace_id = ( xray_trace_id = (
self.TRACE_ID_VERSION TRACE_ID_VERSION
+ self.TRACE_ID_DELIMITER + TRACE_ID_DELIMITER
+ otel_trace_id[: self.TRACE_ID_FIRST_PART_LENGTH] + otel_trace_id[:TRACE_ID_FIRST_PART_LENGTH]
+ self.TRACE_ID_DELIMITER + TRACE_ID_DELIMITER
+ otel_trace_id[self.TRACE_ID_FIRST_PART_LENGTH :] + otel_trace_id[TRACE_ID_FIRST_PART_LENGTH:]
) )
parent_id = "{:016x}".format(span_context.span_id) parent_id = "{:016x}".format(span_context.span_id)
sampling_flag = ( sampling_flag = (
self.IS_SAMPLED IS_SAMPLED
if span_context.trace_flags & trace.TraceFlags.SAMPLED if span_context.trace_flags & trace.TraceFlags.SAMPLED
else self.NOT_SAMPLED else NOT_SAMPLED
) )
# TODO: Add OT trace state to the X-Ray trace header # TODO: Add OT trace state to the X-Ray trace header
trace_header = ( trace_header = (
self.TRACE_ID_KEY TRACE_ID_KEY
+ self.KEY_AND_VALUE_DELIMITER + KEY_AND_VALUE_DELIMITER
+ xray_trace_id + xray_trace_id
+ self.KV_PAIR_DELIMITER + KV_PAIR_DELIMITER
+ self.PARENT_ID_KEY + PARENT_ID_KEY
+ self.KEY_AND_VALUE_DELIMITER + KEY_AND_VALUE_DELIMITER
+ parent_id + parent_id
+ self.KV_PAIR_DELIMITER + KV_PAIR_DELIMITER
+ self.SAMPLED_FLAG_KEY + SAMPLED_FLAG_KEY
+ self.KEY_AND_VALUE_DELIMITER + KEY_AND_VALUE_DELIMITER
+ sampling_flag + sampling_flag
) )
set_in_carrier( set_in_carrier(
carrier, self.TRACE_HEADER_KEY, trace_header, carrier, TRACE_HEADER_KEY, trace_header,
) )

View File

@ -18,6 +18,7 @@ from requests.structures import CaseInsensitiveDict
import opentelemetry.trace as trace_api import opentelemetry.trace as trace_api
from opentelemetry.sdk.extension.aws.trace.propagation.aws_xray_format import ( from opentelemetry.sdk.extension.aws.trace.propagation.aws_xray_format import (
TRACE_HEADER_KEY,
AwsXRayFormat, AwsXRayFormat,
) )
from opentelemetry.trace import ( from opentelemetry.trace import (
@ -29,6 +30,7 @@ from opentelemetry.trace import (
TraceState, TraceState,
set_span_in_context, set_span_in_context,
) )
from opentelemetry.trace.propagation.textmap import DictGetter
TRACE_ID_BASE16 = "8a3c60f7d188f8fa79d48a391a778fa6" TRACE_ID_BASE16 = "8a3c60f7d188f8fa79d48a391a778fa6"
@ -83,7 +85,7 @@ def build_test_span_context(
class AwsXRayPropagatorTest(unittest.TestCase): class AwsXRayPropagatorTest(unittest.TestCase):
carrier_setter = CaseInsensitiveDict.__setitem__ carrier_setter = CaseInsensitiveDict.__setitem__
carrier_getter = get_as_list carrier_getter = DictGetter()
XRAY_PROPAGATOR = AwsXRayFormat() XRAY_PROPAGATOR = AwsXRayFormat()
# Inject Tests # Inject Tests
@ -101,7 +103,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
expected_items = set( expected_items = set(
CaseInsensitiveDict( CaseInsensitiveDict(
{ {
AwsXRayFormat.TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0" TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0"
} }
).items() ).items()
) )
@ -123,7 +125,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
expected_items = set( expected_items = set(
CaseInsensitiveDict( CaseInsensitiveDict(
{ {
AwsXRayFormat.TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=1" TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=1"
} }
).items() ).items()
) )
@ -144,7 +146,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
expected_items = set( expected_items = set(
CaseInsensitiveDict( CaseInsensitiveDict(
{ {
AwsXRayFormat.TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0" TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0"
} }
).items() ).items()
) )
@ -168,7 +170,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
AwsXRayPropagatorTest.carrier_getter, AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict( CaseInsensitiveDict(
{ {
AwsXRayFormat.TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0" TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0"
} }
), ),
) )
@ -183,7 +185,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
AwsXRayPropagatorTest.carrier_getter, AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict( CaseInsensitiveDict(
{ {
AwsXRayFormat.TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=1" TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=1"
} }
), ),
) )
@ -200,7 +202,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
AwsXRayPropagatorTest.carrier_getter, AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict( CaseInsensitiveDict(
{ {
AwsXRayFormat.TRACE_HEADER_KEY: "Sampled=0;Parent=53995c3f42cd8ad8;Root=1-8a3c60f7-d188f8fa79d48a391a778fa6" TRACE_HEADER_KEY: "Sampled=0;Parent=53995c3f42cd8ad8;Root=1-8a3c60f7-d188f8fa79d48a391a778fa6"
} }
), ),
) )
@ -215,7 +217,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
AwsXRayPropagatorTest.carrier_getter, AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict( CaseInsensitiveDict(
{ {
AwsXRayFormat.TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0;Foo=Bar" TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0;Foo=Bar"
} }
), ),
) )
@ -230,7 +232,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
AwsXRayPropagatorTest.carrier_getter, AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict( CaseInsensitiveDict(
{ {
AwsXRayFormat.TRACE_HEADER_KEY: " Root = 1-8a3c60f7-d188f8fa79d48a391a778fa6 ; Parent = 53995c3f42cd8ad8 ; Sampled = 0 " TRACE_HEADER_KEY: " Root = 1-8a3c60f7-d188f8fa79d48a391a778fa6 ; Parent = 53995c3f42cd8ad8 ; Sampled = 0 "
} }
), ),
) )
@ -243,7 +245,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
def test_extract_invalid_xray_trace_header(self): def test_extract_invalid_xray_trace_header(self):
context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract( context_with_extracted = AwsXRayPropagatorTest.XRAY_PROPAGATOR.extract(
AwsXRayPropagatorTest.carrier_getter, AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict({AwsXRayFormat.TRACE_HEADER_KEY: ""}), CaseInsensitiveDict({TRACE_HEADER_KEY: ""}),
) )
self.assertEqual( self.assertEqual(
@ -256,7 +258,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
AwsXRayPropagatorTest.carrier_getter, AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict( CaseInsensitiveDict(
{ {
AwsXRayFormat.TRACE_HEADER_KEY: "Root=1-12345678-abcdefghijklmnopqrstuvwx;Parent=53995c3f42cd8ad8;Sampled=0" TRACE_HEADER_KEY: "Root=1-12345678-abcdefghijklmnopqrstuvwx;Parent=53995c3f42cd8ad8;Sampled=0"
} }
), ),
) )
@ -271,7 +273,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
AwsXRayPropagatorTest.carrier_getter, AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict( CaseInsensitiveDict(
{ {
AwsXRayFormat.TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa600;Parent=53995c3f42cd8ad8;Sampled=0=" TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa600;Parent=53995c3f42cd8ad8;Sampled=0="
} }
), ),
) )
@ -286,7 +288,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
AwsXRayPropagatorTest.carrier_getter, AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict( CaseInsensitiveDict(
{ {
AwsXRayFormat.TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=abcdefghijklmnop;Sampled=0" TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=abcdefghijklmnop;Sampled=0"
} }
), ),
) )
@ -301,7 +303,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
AwsXRayPropagatorTest.carrier_getter, AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict( CaseInsensitiveDict(
{ {
AwsXRayFormat.TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad800;Sampled=0" TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad800;Sampled=0"
} }
), ),
) )
@ -316,7 +318,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
AwsXRayPropagatorTest.carrier_getter, AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict( CaseInsensitiveDict(
{ {
AwsXRayFormat.TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=" TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled="
} }
), ),
) )
@ -331,7 +333,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
AwsXRayPropagatorTest.carrier_getter, AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict( CaseInsensitiveDict(
{ {
AwsXRayFormat.TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=011" TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=011"
} }
), ),
) )
@ -346,7 +348,7 @@ class AwsXRayPropagatorTest(unittest.TestCase):
AwsXRayPropagatorTest.carrier_getter, AwsXRayPropagatorTest.carrier_getter,
CaseInsensitiveDict( CaseInsensitiveDict(
{ {
AwsXRayFormat.TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=a" TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=a"
} }
), ),
) )