diff --git a/driver-core/build.gradle.kts b/driver-core/build.gradle.kts index 047b3a43a63..5efda24e3a3 100644 --- a/driver-core/build.gradle.kts +++ b/driver-core/build.gradle.kts @@ -57,9 +57,15 @@ dependencies { optionalImplementation(platform(libs.micrometer.observation.bom)) optionalImplementation(libs.micrometer.observation) + optionalImplementation(libs.micrometer.tracing) testImplementation(project(path = ":bson", configuration = "testArtifacts")) testImplementation(libs.reflections) + + // Tracing testing + testImplementation(platform(libs.micrometer.tracing.integration.test.bom)) + testImplementation(libs.micrometer.tracing.integration.test) { exclude(group = "org.junit.jupiter") } + testImplementation(libs.netty.tcnative.boringssl.static) listOf("linux-x86_64", "linux-aarch_64", "osx-x86_64", "osx-aarch_64", "windows-x86_64").forEach { arch -> testImplementation("${libs.netty.tcnative.boringssl.static.get()}::$arch") diff --git a/driver-core/src/main/com/mongodb/connection/ServerDescription.java b/driver-core/src/main/com/mongodb/connection/ServerDescription.java index 2d675bce217..fd82f1dcb16 100644 --- a/driver-core/src/main/com/mongodb/connection/ServerDescription.java +++ b/driver-core/src/main/com/mongodb/connection/ServerDescription.java @@ -65,7 +65,7 @@ public class ServerDescription { * The maximum supported driver wire version * @since 3.8 */ - public static final int MAX_DRIVER_WIRE_VERSION = 25; + public static final int MAX_DRIVER_WIRE_VERSION = 29; private static final int DEFAULT_MAX_DOCUMENT_SIZE = 0x1000000; // 16MB diff --git a/driver-core/src/main/com/mongodb/internal/connection/CommandMessage.java b/driver-core/src/main/com/mongodb/internal/connection/CommandMessage.java index 348349fd18c..074c0129cfc 100644 --- a/driver-core/src/main/com/mongodb/internal/connection/CommandMessage.java +++ b/driver-core/src/main/com/mongodb/internal/connection/CommandMessage.java @@ -25,6 +25,7 @@ import com.mongodb.internal.MongoNamespaceHelper; import com.mongodb.internal.TimeoutContext; import com.mongodb.internal.connection.MessageSequences.EmptyMessageSequences; +import com.mongodb.internal.observability.micrometer.Span; import com.mongodb.internal.session.SessionContext; import com.mongodb.lang.Nullable; import org.bson.BsonArray; @@ -34,6 +35,7 @@ import org.bson.BsonElement; import org.bson.BsonInt64; import org.bson.BsonString; +import org.bson.BsonValue; import org.bson.ByteBuf; import org.bson.FieldNameValidator; import org.bson.io.BsonOutput; @@ -64,6 +66,7 @@ import static com.mongodb.internal.connection.ByteBufBsonDocument.createList; import static com.mongodb.internal.connection.ByteBufBsonDocument.createOne; import static com.mongodb.internal.connection.ReadConcernHelper.getReadConcernDocument; +import static com.mongodb.internal.operation.ServerVersionHelper.NINE_DOT_ZERO_WIRE_VERSION; import static com.mongodb.internal.operation.ServerVersionHelper.UNKNOWN_WIRE_VERSION; /** @@ -80,6 +83,11 @@ public final class CommandMessage extends RequestMessage { * Specifies that the `OP_MSG` section payload is a sequence of BSON documents. */ private static final byte PAYLOAD_TYPE_1_DOCUMENT_SEQUENCE = 1; + /** + * Specifies that the `OP_MSG` section payload is a BSON telemetry document + * ({@code {otel: {traceparent: }}}). + */ + private static final byte PAYLOAD_TYPE_3_TELEMETRY = 3; private static final int UNINITIALIZED_POSITION = -1; @@ -101,6 +109,14 @@ public final class CommandMessage extends RequestMessage { private final ClusterConnectionMode clusterConnectionMode; private final ServerApi serverApi; + /** + * The command span to attach to the OP_MSG telemetry section during {@linkplain + * #encode(ByteBufferBsonOutput, OperationContext, Span) encoding}. Set for the duration of that overload's + * call and cleared afterward; {@code null} otherwise. + */ + @Nullable + private Span commandSpanForEncoding; + CommandMessage(final String database, final BsonDocument command, final FieldNameValidator commandFieldNameValidator, final ReadPreference readPreference, final MessageSettings settings, final ClusterConnectionMode clusterConnectionMode, @Nullable final ServerApi serverApi) { @@ -156,36 +172,42 @@ BsonDocument getCommandDocument(final ByteBufferBsonOutput bsonOutput) { byteBuf.position(firstDocumentPosition); ByteBufBsonDocument byteBufBsonDocument = createOne(byteBuf); - // If true, it means there is at least one `PAYLOAD_TYPE_1_DOCUMENT_SEQUENCE` section in the OP_MSG - if (byteBuf.hasRemaining()) { - BsonDocument commandBsonDocument = byteBufBsonDocument.toBaseBsonDocument(); - - // Each loop iteration processes one Document Sequence - // When there are no more bytes remaining, there are no more Document Sequences - while (byteBuf.hasRemaining()) { - // skip reading the payload type, we know it is `PAYLOAD_TYPE_1` - byteBuf.position(byteBuf.position() + 1); - int sequenceStart = byteBuf.position(); - int sequenceSizeInBytes = byteBuf.getInt(); - int sectionEnd = sequenceStart + sequenceSizeInBytes; - - String fieldName = getSequenceIdentifier(byteBuf); - // If this assertion fires, it means that the driver has started using document sequences for nested fields. If - // so, this method will need to change in order to append the value to the correct nested document. - assertFalse(fieldName.contains(".")); - - ByteBuf documentsByteBufSlice = byteBuf.duplicate().limit(sectionEnd); - try { - commandBsonDocument.append(fieldName, new BsonArray(createList(documentsByteBufSlice))); - } finally { - documentsByteBufSlice.release(); - } - byteBuf.position(sectionEnd); + // There may be more sections after the `PAYLOAD_TYPE_0_DOCUMENT` section: either one or more + // `PAYLOAD_TYPE_1_DOCUMENT_SEQUENCE` sections, and/or a trailing `PAYLOAD_TYPE_3_TELEMETRY` section. + // The command document is materialized lazily, only when a document sequence has to be folded into + // it; otherwise (no sections, or only a telemetry section) the cheap lazy document is returned. + BsonDocument commandBsonDocument = null; + + // Each loop iteration processes one Document Sequence + // When there are no more bytes remaining, there are no more Document Sequences + while (byteBuf.hasRemaining()) { + byte payloadType = byteBuf.get(); + // Document-sequence sections always precede any trailing non-sequence section (e.g. + // PAYLOAD_TYPE_3_TELEMETRY), and such sections carry no command-document fields, so stop here. + if (payloadType != PAYLOAD_TYPE_1_DOCUMENT_SEQUENCE) { + break; + } + if (commandBsonDocument == null) { + commandBsonDocument = byteBufBsonDocument.toBaseBsonDocument(); + } + int sequenceStart = byteBuf.position(); + int sequenceSizeInBytes = byteBuf.getInt(); + int sectionEnd = sequenceStart + sequenceSizeInBytes; + + String fieldName = getSequenceIdentifier(byteBuf); + // If this assertion fires, it means that the driver has started using document sequences for nested fields. If + // so, this method will need to change in order to append the value to the correct nested document. + assertFalse(fieldName.contains(".")); + + ByteBuf documentsByteBufSlice = byteBuf.duplicate().limit(sectionEnd); + try { + commandBsonDocument.append(fieldName, new BsonArray(createList(documentsByteBufSlice))); + } finally { + documentsByteBufSlice.release(); } - return commandBsonDocument; - } else { - return byteBufBsonDocument; + byteBuf.position(sectionEnd); } + return commandBsonDocument != null ? commandBsonDocument : byteBufBsonDocument; } finally { byteBuf.release(); } @@ -194,6 +216,24 @@ BsonDocument getCommandDocument(final ByteBufferBsonOutput bsonOutput) { } } + /** + * The command name (the first key of the command document). Available before {@code encode()} runs, unlike + * {@link #getCommandDocument(ByteBufferBsonOutput)}; identical to the encoded document's first + * key, since encoding only appends fields. + */ + public String getCommandName() { + return command.getFirstKey(); + } + + /** + * The cursor id if this is a {@code getMore} command, or {@code null} otherwise. + */ + @Nullable + public BsonInt64 getGetMoreCursorId() { + BsonValue value = command.get("getMore"); + return value instanceof BsonInt64 ? (BsonInt64) value : null; + } + /** * Get the field name from a buffer positioned at the start of the document sequence identifier of an OP_MSG Section of type * `PAYLOAD_TYPE_1_DOCUMENT_SEQUENCE`. @@ -235,6 +275,21 @@ protected void encodeMessageBody(final ByteBufferBsonOutput bsonOutput, final Op this.firstDocumentPosition = useOpMsg() ? writeOpMsg(bsonOutput, operationContext) : writeOpQuery(bsonOutput); } + /** + * Encodes the message, attaching the OP_MSG telemetry section with {@code commandSpan}'s trace context when + * the gating conditions hold (see {@link #writeTelemetryContextSection}). The 2-arg {@link + * #encode(ByteBufferBsonOutput, OperationContext)} never attaches the section. + */ + public void encode(final ByteBufferBsonOutput bsonOutput, final OperationContext operationContext, + @Nullable final Span commandSpan) { + this.commandSpanForEncoding = commandSpan; + try { + encode(bsonOutput, operationContext); + } finally { + this.commandSpanForEncoding = null; + } + } + @SuppressWarnings("try") private int writeOpMsg(final ByteBufferBsonOutput bsonOutput, final OperationContext operationContext) { int messageStartPosition = bsonOutput.getPosition() - MESSAGE_PROLOGUE_LENGTH; @@ -277,11 +332,35 @@ private int writeOpMsg(final ByteBufferBsonOutput bsonOutput, final OperationCon fail(sequences.toString()); } + writeTelemetryContextSection(bsonOutput); + // Write the flag bits bsonOutput.writeInt32(flagPosition, getOpMsgFlagBits()); return commandStartPosition; } + private void writeTelemetryContextSection(final ByteBufferBsonOutput bsonOutput) { + if (getSettings().getMaxWireVersion() < NINE_DOT_ZERO_WIRE_VERSION) { + return; + } + Span commandSpan = this.commandSpanForEncoding; + if (commandSpan == null) { + return; + } + String traceParent = commandSpan.context().traceParent(); + if (traceParent == null) { + return; + } + bsonOutput.writeByte(PAYLOAD_TYPE_3_TELEMETRY); + // {otel: {traceparent: }} + BsonBinaryWriter writer = new BsonBinaryWriter(bsonOutput); + writer.writeStartDocument(); + writer.writeStartDocument("otel"); + writer.writeString("traceparent", traceParent); + writer.writeEndDocument(); + writer.writeEndDocument(); + } + private int writeOpQuery(final ByteBufferBsonOutput bsonOutput) { bsonOutput.writeInt32(0); bsonOutput.writeCString(new MongoNamespace(getDatabase(), MongoNamespaceHelper.COMMAND_COLLECTION_NAME).getFullName()); diff --git a/driver-core/src/main/com/mongodb/internal/connection/InternalStreamConnection.java b/driver-core/src/main/com/mongodb/internal/connection/InternalStreamConnection.java index 0cad654a73a..b1baf434124 100644 --- a/driver-core/src/main/com/mongodb/internal/connection/InternalStreamConnection.java +++ b/driver-core/src/main/com/mongodb/internal/connection/InternalStreamConnection.java @@ -439,20 +439,19 @@ public boolean reauthenticationIsTriggered(@Nullable final Throwable t) { @Nullable private T sendAndReceiveInternal(final CommandMessage message, final Decoder decoder, final OperationContext operationContext) { - CommandEventSender commandEventSender; - Span tracingSpan; + CommandEventSender commandEventSender = new NoOpCommandEventSender(); + Span tracingSpan = null; try (ByteBufferBsonOutput bsonOutput = new ByteBufferBsonOutput(this)) { - message.encode(bsonOutput, operationContext); tracingSpan = operationContext .getTracingManager() .createTracingSpan(message, operationContext, - () -> message.getCommandDocument(bsonOutput), cmdName -> SECURITY_SENSITIVE_COMMANDS.contains(cmdName) || SECURITY_SENSITIVE_HELLO_COMMANDS.contains(cmdName), () -> getDescription().getServerAddress(), () -> getDescription().getConnectionId() ); + message.encode(bsonOutput, operationContext, tracingSpan); boolean isLoggingCommandNeeded = isLoggingCommandNeeded(); boolean isTracingCommandPayloadNeeded = tracingSpan != null && operationContext.getTracingManager().isCommandPayloadEnabled(); @@ -467,8 +466,6 @@ private T sendAndReceiveInternal(final CommandMessage message, final Decoder operationContext, message, commandDocument, COMMAND_PROTOCOL_LOGGER, loggerSettings); commandEventSender.sendStartedEvent(); - } else { - commandEventSender = new NoOpCommandEventSender(); } if (isTracingCommandPayloadNeeded) { tracingSpan.setQueryText(commandDocument); @@ -476,18 +473,15 @@ private T sendAndReceiveInternal(final CommandMessage message, final Decoder if (tracingSpan != null) { tracingSpan.openScope(); } - - try { - sendCommandMessage(message, bsonOutput, operationContext); - } catch (Exception e) { - if (tracingSpan != null) { - tracingSpan.error(e); - tracingSpan.closeScope(); - tracingSpan.end(); - } - commandEventSender.sendFailedEvent(e); - throw e; + sendCommandMessage(message, bsonOutput, operationContext); + } catch (Throwable t) { + if (tracingSpan != null) { + tracingSpan.error(t); + tracingSpan.closeScope(); + tracingSpan.end(); } + commandEventSender.sendFailedEvent(t); + throw t; } if (message.isResponseExpected()) { @@ -528,7 +522,7 @@ private void sendCommandMessage(final CommandMessage message, final ByteBufferBs final OperationContext operationContext) { Compressor localSendCompressor = sendCompressor; - if (localSendCompressor == null || SECURITY_SENSITIVE_COMMANDS.contains(message.getCommandDocument(bsonOutput).getFirstKey())) { + if (localSendCompressor == null || SECURITY_SENSITIVE_COMMANDS.contains(message.getCommandName())) { trySendMessage(message, bsonOutput, operationContext); } else { ByteBufferBsonOutput compressedBsonOutput; @@ -617,19 +611,18 @@ private void sendAndReceiveAsyncInternal(final CommandMessage message, final Span tracingSpan = null; try { - message.encode(bsonOutput, operationContext); - tracingSpan = operationContext .getTracingManager() .createTracingSpan(message, operationContext, - () -> message.getCommandDocument(bsonOutput), cmdName -> SECURITY_SENSITIVE_COMMANDS.contains(cmdName) || SECURITY_SENSITIVE_HELLO_COMMANDS.contains(cmdName), () -> getDescription().getServerAddress(), () -> getDescription().getConnectionId() ); + message.encode(bsonOutput, operationContext, tracingSpan); + CommandEventSender commandEventSender; boolean isLoggingCommandNeeded = isLoggingCommandNeeded(); boolean isTracingCommandPayloadNeeded = tracingSpan != null && operationContext.getTracingManager().isCommandPayloadEnabled(); @@ -670,7 +663,7 @@ private void sendAndReceiveAsyncInternal(final CommandMessage message, final commandEventSender.sendStartedEvent(); Compressor localSendCompressor = sendCompressor; - if (localSendCompressor == null || SECURITY_SENSITIVE_COMMANDS.contains(message.getCommandDocument(bsonOutput).getFirstKey())) { + if (localSendCompressor == null || SECURITY_SENSITIVE_COMMANDS.contains(message.getCommandName())) { sendCommandMessageAsync(message.getId(), decoder, operationContext, tracingCallback, bsonOutput, commandEventSender, message.isResponseExpected()); } else { diff --git a/driver-core/src/main/com/mongodb/internal/observability/micrometer/MicrometerTracer.java b/driver-core/src/main/com/mongodb/internal/observability/micrometer/MicrometerTracer.java index fa5c6223daa..c792e6771ed 100644 --- a/driver-core/src/main/com/mongodb/internal/observability/micrometer/MicrometerTracer.java +++ b/driver-core/src/main/com/mongodb/internal/observability/micrometer/MicrometerTracer.java @@ -47,11 +47,28 @@ * @since 5.7 */ public class MicrometerTracer implements Tracer { + /** + * {@code micrometer-tracing} is an optional dependency: consumers may have only {@code micrometer-observation} + * on the classpath (e.g. metrics/logging-only observability setups). Guard any use of {@code io.micrometer.tracing.*} + * types with this flag so that such setups never trigger a {@link NoClassDefFoundError} or {@link LinkageError}. + */ + static final boolean MICROMETER_TRACING_ON_CLASSPATH = isMicrometerTracingOnClasspath(); + private final ObservationRegistry observationRegistry; private final boolean allowCommandPayload; private final int textMaxLength; private final ObservationConvention convention; + private static boolean isMicrometerTracingOnClasspath() { + try { + Class.forName("io.micrometer.tracing.handler.TracingObservationHandler", + false, MicrometerTracer.class.getClassLoader()); + return true; + } catch (ClassNotFoundException | LinkageError e) { + return false; + } + } + /** * Constructs a new {@link MicrometerTracer} instance. * @@ -116,6 +133,68 @@ private static class MicrometerTraceContext implements TraceContext { MicrometerTraceContext(@Nullable final Observation observation) { this.observation = observation; } + + @Override + @Nullable + public String traceParent() { + if (observation == null) { + return null; + } + if (!MICROMETER_TRACING_ON_CLASSPATH) { + return null; + } + return MicrometerTracingSupport.traceParent(observation); + } + + /** + * The server ({@code validateW3CTraceparent}) rejects ids that are not exactly the expected + * length of lowercase hex, or that are all zeroes. Never emit a traceparent it would reject. + */ + @SuppressWarnings("BooleanMethodIsAlwaysInverted") + static boolean isValidNonZeroLowercaseHex(@Nullable final String value, final int expectedLength) { + if (value == null || value.length() != expectedLength) { + return false; + } + boolean nonZero = false; + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'))) { + return false; + } + if (c != '0') { + nonZero = true; + } + } + return nonZero; + } + } + + /** + * Isolates all hard references to {@code io.micrometer.tracing.*} types. This class must only be loaded/invoked + * after {@link #MICROMETER_TRACING_ON_CLASSPATH} has been confirmed {@code true}, so that JVMs which resolve + * classes referenced by bytecode eagerly (e.g. during verification) do not trigger a {@link NoClassDefFoundError} + * when {@code micrometer-tracing} is absent from the classpath. + */ + private static final class MicrometerTracingSupport { + @Nullable + static String traceParent(final Observation observation) { + io.micrometer.tracing.handler.TracingObservationHandler.TracingContext tracingContext = + observation.getContextView().get(io.micrometer.tracing.handler.TracingObservationHandler.TracingContext.class); + if (tracingContext == null || tracingContext.getSpan() == null) { + return null; + } + // Span.context() is non-null by contract (the package is @NullMarked); a no-op span yields a + // NOOP context whose empty ids fail the hex validation below. + io.micrometer.tracing.TraceContext ctx = tracingContext.getSpan().context(); + String traceId = ctx.traceId(); + String spanId = ctx.spanId(); + if (!MicrometerTraceContext.isValidNonZeroLowercaseHex(traceId, 32) + || !MicrometerTraceContext.isValidNonZeroLowercaseHex(spanId, 16)) { + return null; + } + Boolean sampled = ctx.sampled(); + return "00-" + traceId + "-" + spanId + (sampled != null && sampled ? "-01" : "-00"); + } } /** diff --git a/driver-core/src/main/com/mongodb/internal/observability/micrometer/Span.java b/driver-core/src/main/com/mongodb/internal/observability/micrometer/Span.java index d060c82b0d4..f0821f2375d 100644 --- a/driver-core/src/main/com/mongodb/internal/observability/micrometer/Span.java +++ b/driver-core/src/main/com/mongodb/internal/observability/micrometer/Span.java @@ -90,12 +90,14 @@ public MongodbObservationContext getMongodbObservationContext() { /** * Opens a scope for this span, making it the current observation on the thread. - * Must be paired with {@link #closeScope()} in a try-finally block. + * A successful {@code openScope()} must be followed by {@link #closeScope()}. */ void openScope(); /** * Closes the scope previously opened by {@link #openScope()}, restoring the previous observation. + * Calling this without a prior {@code openScope()}, or more than once, is a no-op — error-handling + * paths may therefore call it unconditionally. */ void closeScope(); diff --git a/driver-core/src/main/com/mongodb/internal/observability/micrometer/TraceContext.java b/driver-core/src/main/com/mongodb/internal/observability/micrometer/TraceContext.java index 5ca248db59d..c76d86e4d2e 100644 --- a/driver-core/src/main/com/mongodb/internal/observability/micrometer/TraceContext.java +++ b/driver-core/src/main/com/mongodb/internal/observability/micrometer/TraceContext.java @@ -16,8 +16,18 @@ package com.mongodb.internal.observability.micrometer; +import com.mongodb.lang.Nullable; + @SuppressWarnings("InterfaceIsType") public interface TraceContext { - TraceContext EMPTY = new TraceContext() { - }; + TraceContext EMPTY = () -> null; + + /** + * The 55-char W3C {@code traceparent} string for this context + * ({@code 00-<32 hex traceId>-<16 hex spanId>-}; flags {@code 01} sampled / {@code 00} unsampled), + * or {@code null} when there is no valid context (no-op span, missing/zero/malformed ids). + * Never includes tracestate. + */ + @Nullable + String traceParent(); } diff --git a/driver-core/src/main/com/mongodb/internal/observability/micrometer/TracingManager.java b/driver-core/src/main/com/mongodb/internal/observability/micrometer/TracingManager.java index d177f02fae7..6e25ad2779e 100644 --- a/driver-core/src/main/com/mongodb/internal/observability/micrometer/TracingManager.java +++ b/driver-core/src/main/com/mongodb/internal/observability/micrometer/TracingManager.java @@ -30,7 +30,7 @@ import com.mongodb.observability.micrometer.MongodbObservation; import com.mongodb.observability.micrometer.MongodbObservationContext; import io.micrometer.observation.ObservationRegistry; -import org.bson.BsonDocument; +import org.bson.BsonInt64; import java.util.function.Predicate; import java.util.function.Supplier; @@ -164,7 +164,6 @@ public boolean isCommandPayloadEnabled() { * * @param message the command message to trace * @param operationContext the operation context containing tracing and session information - * @param commandDocumentSupplier a supplier that provides the command document when needed * @param isSensitiveCommand a predicate that determines if a command is security-sensitive based on its name * @param serverAddressSupplier a supplier that provides the server address when needed * @param connectionIdSupplier a supplier that provides the connection ID when needed @@ -173,7 +172,6 @@ public boolean isCommandPayloadEnabled() { @Nullable public Span createTracingSpan(final CommandMessage message, final OperationContext operationContext, - final Supplier commandDocumentSupplier, final Predicate isSensitiveCommand, final Supplier serverAddressSupplier, final Supplier connectionIdSupplier @@ -182,8 +180,7 @@ public Span createTracingSpan(final CommandMessage message, if (!isEnabled()) { return null; } - BsonDocument command = commandDocumentSupplier.get(); - String commandName = command.getFirstKey(); + String commandName = message.getCommandName(); if (isSensitiveCommand.test(commandName)) { return null; } @@ -224,9 +221,9 @@ public Span createTracingSpan(final CommandMessage message, ConnectionId connectionId = connectionIdSupplier.get(); mongodbContext.setConnectionId(connectionId); - if (command.containsKey("getMore")) { - long cursorId = command.getInt64("getMore").longValue(); - mongodbContext.setCursorId(cursorId); + BsonInt64 getMoreCursorId = message.getGetMoreCursorId(); + if (getMoreCursorId != null) { + mongodbContext.setCursorId(getMoreCursorId.longValue()); } SessionContext sessionContext = operationContext.getSessionContext(); diff --git a/driver-core/src/main/com/mongodb/internal/operation/ServerVersionHelper.java b/driver-core/src/main/com/mongodb/internal/operation/ServerVersionHelper.java index 093d48e3781..482458c8615 100644 --- a/driver-core/src/main/com/mongodb/internal/operation/ServerVersionHelper.java +++ b/driver-core/src/main/com/mongodb/internal/operation/ServerVersionHelper.java @@ -31,7 +31,10 @@ public final class ServerVersionHelper { public static final int SIX_DOT_ZERO_WIRE_VERSION = 17; public static final int SEVEN_DOT_ZERO_WIRE_VERSION = 21; public static final int EIGHT_DOT_ZERO_WIRE_VERSION = 25; - public static final int LATEST_WIRE_VERSION = EIGHT_DOT_ZERO_WIRE_VERSION; + // Server 9.0 (WIRE_VERSION_90 = 29). Minimum wire version for the OP_MSG telemetry + // section (OTel trace-context propagation, DRIVERS-3454). + public static final int NINE_DOT_ZERO_WIRE_VERSION = 29; + public static final int LATEST_WIRE_VERSION = NINE_DOT_ZERO_WIRE_VERSION; public static boolean serverIsAtLeastVersionFourDotFour(final ConnectionDescription description) { return description.getMaxWireVersion() >= FOUR_DOT_FOUR_WIRE_VERSION; diff --git a/driver-core/src/test/unit/com/mongodb/internal/connection/CommandMessageOtelTraceContextTest.java b/driver-core/src/test/unit/com/mongodb/internal/connection/CommandMessageOtelTraceContextTest.java new file mode 100644 index 00000000000..762876bf253 --- /dev/null +++ b/driver-core/src/test/unit/com/mongodb/internal/connection/CommandMessageOtelTraceContextTest.java @@ -0,0 +1,293 @@ +/* + * Copyright 2008-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.mongodb.internal.connection; + +import com.mongodb.MongoNamespace; +import com.mongodb.ReadConcern; +import com.mongodb.ReadPreference; +import com.mongodb.connection.ClusterConnectionMode; +import com.mongodb.connection.ServerType; +import com.mongodb.internal.TimeoutContext; +import com.mongodb.internal.TimeoutSettings; +import com.mongodb.internal.bulk.InsertRequest; +import com.mongodb.internal.bulk.WriteRequestWithIndex; +import com.mongodb.internal.connection.MessageSequences.EmptyMessageSequences; +import com.mongodb.internal.observability.micrometer.Span; +import com.mongodb.internal.observability.micrometer.TraceContext; +import com.mongodb.internal.session.SessionContext; +import com.mongodb.internal.validator.NoOpFieldNameValidator; +import com.mongodb.lang.Nullable; +import org.bson.BsonBinaryReader; +import org.bson.BsonDocument; +import org.bson.BsonInt32; +import org.bson.BsonString; +import org.bson.ByteBuf; +import org.bson.ByteBufNIO; +import org.bson.codecs.BsonDocumentCodec; +import org.bson.codecs.DecoderContext; +import org.bson.codecs.EncoderContext; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.stream.Stream; + +import static com.mongodb.internal.mockito.MongoMockito.mock; +import static com.mongodb.internal.operation.ServerVersionHelper.EIGHT_DOT_ZERO_WIRE_VERSION; +import static com.mongodb.internal.operation.ServerVersionHelper.NINE_DOT_ZERO_WIRE_VERSION; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Named.named; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.Mockito.when; + +class CommandMessageOtelTraceContextTest { + + private static final String TRACEPARENT = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"; + private static final BsonDocument EXPECTED_TELEMETRY_DOCUMENT = + new BsonDocument("otel", new BsonDocument("traceparent", new BsonString(TRACEPARENT))); + private static final MongoNamespace NAMESPACE = new MongoNamespace("db.test"); + private static final BsonDocument COMMAND = new BsonDocument("find", new BsonString(NAMESPACE.getCollectionName())); + + /** + * Telemetry section is attached when supported and traced. + */ + @Test + void shouldWriteTelemetrySectionWhenWireVersionAtLeast29AndSpanActive() { + CommandMessage message = buildCommandMessage(NINE_DOT_ZERO_WIRE_VERSION, EmptyMessageSequences.INSTANCE); + Span span = spanWithTraceParent(TRACEPARENT); + OperationContext operationContext = buildOperationContext(); + + try (ByteBufferBsonOutput output = new ByteBufferBsonOutput(new SimpleBufferProvider())) { + message.encode(output, operationContext, span); + byte[] buffer = output.toByteArray(); + + BsonDocument telemetry = readTelemetrySectionDocument(buffer); + assertEquals(EXPECTED_TELEMETRY_DOCUMENT, telemetry); + } + } + + private static Stream telemetrySectionOmittedCases() { + return Stream.of( + arguments(named("wire version below 29", EIGHT_DOT_ZERO_WIRE_VERSION), + named("valid span", spanWithTraceParent(TRACEPARENT))), + arguments(named("wire version 29", NINE_DOT_ZERO_WIRE_VERSION), + named("no command span", null)), + arguments(named("wire version 29", NINE_DOT_ZERO_WIRE_VERSION), + named("span without traceparent", spanWithTraceParent(null)))); + } + + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("telemetrySectionOmittedCases") + void shouldNotWriteTelemetrySection(final int maxWireVersion, @Nullable final Span span) { + CommandMessage message = buildCommandMessage(maxWireVersion, EmptyMessageSequences.INSTANCE); + OperationContext operationContext = buildOperationContext(); + + try (ByteBufferBsonOutput output = new ByteBufferBsonOutput(new SimpleBufferProvider())) { + message.encode(output, operationContext, span); + byte[] buffer = output.toByteArray(); + + assertNull(findTelemetrySectionDocument(buffer)); + } + } + + @Test + void getCommandDocumentIgnoresTelemetrySection() { + // Regression guard: InternalStreamConnection calls getCommandDocument() on every send (logging/monitoring/ + // compression). The trailing kind-3 section must not corrupt command-document reconstruction. + CommandMessage message = buildCommandMessage(NINE_DOT_ZERO_WIRE_VERSION, EmptyMessageSequences.INSTANCE); + Span span = spanWithTraceParent(TRACEPARENT); + OperationContext operationContext = buildOperationContext(); + + try (ByteBufferBsonOutput output = new ByteBufferBsonOutput(new SimpleBufferProvider())) { + message.encode(output, operationContext, span); + BsonDocument commandDocument = message.getCommandDocument(output); + assertEquals("test", commandDocument.getString("find").getValue()); + assertEquals("db", commandDocument.getString("$db").getValue()); + // The reconstructed command is exactly the body section (find + $db); the trailing + // kind-3 section must not leak into it. + assertFalse(commandDocument.containsKey("otel")); + assertEquals(2, commandDocument.size()); + } + } + + @Test + void shouldWriteTelemetrySectionAfterDocumentSequences() { + SplittablePayload payload = new SplittablePayload(SplittablePayload.Type.INSERT, + Collections.singletonList(new WriteRequestWithIndex( + new InsertRequest(new BsonDocument("_id", new BsonString("1"))), 0)), + true, NoOpFieldNameValidator.INSTANCE); + CommandMessage message = buildCommandMessage(NINE_DOT_ZERO_WIRE_VERSION, payload); + Span span = spanWithTraceParent(TRACEPARENT); + OperationContext operationContext = buildOperationContext(); + + try (ByteBufferBsonOutput output = new ByteBufferBsonOutput(new SimpleBufferProvider())) { + message.encode(output, operationContext, span); + byte[] buffer = output.toByteArray(); + + // Sequence section(s) precede the kind-3 telemetry section; scanning left-to-right and taking + // the first kind-3 occurrence therefore validates ordering as well as content. + BsonDocument telemetry = readTelemetrySectionDocument(buffer); + assertEquals(EXPECTED_TELEMETRY_DOCUMENT, telemetry); + + BsonDocument commandDocument = message.getCommandDocument(output); + assertEquals("test", commandDocument.getString("find").getValue()); + } + } + + /** + * Same guarantee as {@link #shouldWriteTelemetrySectionAfterDocumentSequences()}, but for the + * {@link DualMessageSequences} branch (the {@code clientBulkWrite} path). + */ + @Test + void shouldWriteTelemetrySectionAfterDualMessageSequences() { + DualMessageSequences sequences = new DualMessageSequences( + "ops", NoOpFieldNameValidator.INSTANCE, "nsInfo", NoOpFieldNameValidator.INSTANCE) { + @Override + public EncodeDocumentsResult encodeDocuments(final WritersProviderAndLimitsChecker writersProviderAndLimitsChecker) { + writersProviderAndLimitsChecker.tryWrite((firstWriter, secondWriter) -> { + new BsonDocumentCodec().encode(firstWriter, + new BsonDocument("insert", new BsonInt32(0)), + EncoderContext.builder().build()); + new BsonDocumentCodec().encode(secondWriter, + new BsonDocument("ns", new BsonString(NAMESPACE.getFullName())), + EncoderContext.builder().build()); + return 1; + }); + return new EncodeDocumentsResult(true, Collections.emptyList()); + } + }; + CommandMessage message = buildCommandMessage(NINE_DOT_ZERO_WIRE_VERSION, sequences); + Span span = spanWithTraceParent(TRACEPARENT); + OperationContext operationContext = buildOperationContext(); + + try (ByteBufferBsonOutput output = new ByteBufferBsonOutput(new SimpleBufferProvider())) { + message.encode(output, operationContext, span); + byte[] buffer = output.toByteArray(); + + BsonDocument telemetry = readTelemetrySectionDocument(buffer); + assertEquals(EXPECTED_TELEMETRY_DOCUMENT, telemetry); + + BsonDocument commandDocument = message.getCommandDocument(output); + assertEquals("test", commandDocument.getString("find").getValue()); + assertFalse(commandDocument.containsKey("otel")); + } + } + + /** + * The 2-arg {@code encode} inherited from {@code RequestMessage} never attaches the telemetry section, + * even when the wire version supports it. This guards monitoring/compression/other callers that still use + * the 2-arg overload. + */ + @Test + void shouldNotWriteTelemetrySectionViaTwoArgEncode() { + CommandMessage message = buildCommandMessage(NINE_DOT_ZERO_WIRE_VERSION, EmptyMessageSequences.INSTANCE); + OperationContext operationContext = buildOperationContext(); + + try (ByteBufferBsonOutput output = new ByteBufferBsonOutput(new SimpleBufferProvider())) { + message.encode(output, operationContext); + byte[] buffer = output.toByteArray(); + + assertNull(findTelemetrySectionDocument(buffer)); + } + } + + // --- helpers --- + + private static Span spanWithTraceParent(final String traceParent) { + TraceContext traceContext = () -> traceParent; + return mock(Span.class, mock -> when(mock.context()).thenReturn(traceContext)); + } + + private static CommandMessage buildCommandMessage(final int maxWireVersion, final MessageSequences sequences) { + return new CommandMessage( + NAMESPACE.getDatabaseName(), + COMMAND, + NoOpFieldNameValidator.INSTANCE, + ReadPreference.primary(), + MessageSettings.builder() + .maxWireVersion(maxWireVersion) + .serverType(ServerType.REPLICA_SET_PRIMARY) + .sessionSupported(true) + .build(), + true, + sequences, + ClusterConnectionMode.MULTIPLE, + null); + } + + private static OperationContext buildOperationContext() { + SessionContext sessionContext = mock(SessionContext.class, mock -> { + when(mock.getClusterTime()).thenReturn(null); + when(mock.hasSession()).thenReturn(false); + when(mock.getReadConcern()).thenReturn(ReadConcern.DEFAULT); + when(mock.notifyMessageSent()).thenReturn(true); + when(mock.hasActiveTransaction()).thenReturn(false); + when(mock.isSnapshot()).thenReturn(false); + }); + TimeoutContext timeoutContext = new TimeoutContext(TimeoutSettings.DEFAULT); + return mock(OperationContext.class, mock -> { + when(mock.getSessionContext()).thenReturn(sessionContext); + when(mock.getTimeoutContext()).thenReturn(timeoutContext); + }); + } + + /** + * Walks the OP_MSG sections skip the type-0 body document, then for each subsequent section read the kind byte; + * for kind 1 (document sequence) skip past the {@code int32} section size, and for kind 3 (telemetry) decode and + * return the BSON document payload. Returns {@code null} if no kind-3 section is found. + */ + private static BsonDocument findTelemetrySectionDocument(final byte[] buffer) { + ByteBuf byteBuf = new ByteBufNIO(ByteBuffer.wrap(buffer)); + // MsgHeader (16 bytes) + flagBits (4 bytes) + payload type byte (1 byte) for the body section. + int position = 16 + 4 + 1; + byteBuf.position(position); + int bodyLength = byteBuf.getInt(byteBuf.position()); + byteBuf.position(byteBuf.position() + bodyLength); + + while (byteBuf.hasRemaining()) { + byte kind = byteBuf.get(); + if (kind == 1) { + int sectionStart = byteBuf.position(); + int sectionSize = byteBuf.getInt(); + byteBuf.position(sectionStart + sectionSize); + } else if (kind == 3) { + BsonBinaryReader reader = new BsonBinaryReader(byteBuf.asNIO()); + try { + return new BsonDocumentCodec().decode(reader, DecoderContext.builder().build()); + } finally { + reader.close(); + } + } else { + throw new AssertionError("Unexpected section kind byte: " + kind); + } + } + return null; + } + + private static BsonDocument readTelemetrySectionDocument(final byte[] buffer) { + BsonDocument telemetry = findTelemetrySectionDocument(buffer); + if (telemetry == null) { + throw new AssertionError("Expected a kind-3 telemetry section but none was found"); + } + return telemetry; + } +} diff --git a/driver-core/src/test/unit/com/mongodb/internal/observability/micrometer/MicrometerTraceParentTest.java b/driver-core/src/test/unit/com/mongodb/internal/observability/micrometer/MicrometerTraceParentTest.java new file mode 100644 index 00000000000..8cb3a670d1e --- /dev/null +++ b/driver-core/src/test/unit/com/mongodb/internal/observability/micrometer/MicrometerTraceParentTest.java @@ -0,0 +1,176 @@ +/* + * Copyright 2008-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.mongodb.internal.observability.micrometer; + +import io.micrometer.observation.ObservationRegistry; +import io.micrometer.tracing.test.simple.SimpleTracer; +import io.micrometer.tracing.test.simple.SimpleTraceContext; +import io.micrometer.tracing.handler.DefaultTracingObservationHandler; +import com.mongodb.observability.micrometer.MongodbObservation; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class MicrometerTraceParentTest { + private static final String VALID_TRACE_ID = "0af7651916cd43dd8448eb211c80319c"; + private static final String VALID_SPAN_ID = "b7ad6b7169203331"; + + @Test + void shouldFormatSampledTraceParent() { + String traceParent = traceParentFor(VALID_TRACE_ID, VALID_SPAN_ID, true); + assertEquals("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", traceParent); + assertEquals(55, traceParent.length()); + } + + @Test + void shouldFormatUnsampledTraceParentWithZeroFlags() { + assertEquals("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-00", + traceParentFor(VALID_TRACE_ID, VALID_SPAN_ID, false)); + assertEquals("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-00", + traceParentFor(VALID_TRACE_ID, VALID_SPAN_ID, null)); + } + + @Test + void shouldReturnNullForInvalidIds() { + assertNull(traceParentFor("00000000000000000000000000000000", VALID_SPAN_ID, true)); + assertNull(traceParentFor(VALID_TRACE_ID, "0000000000000000", true)); + assertNull(traceParentFor("abc", VALID_SPAN_ID, true)); + assertNull(traceParentFor(VALID_TRACE_ID, "abc", true)); + assertNull(traceParentFor("0AF7651916CD43DD8448EB211C80319C", VALID_SPAN_ID, true)); + assertNull(traceParentFor(null, VALID_SPAN_ID, true)); + assertNull(traceParentFor(VALID_TRACE_ID, null, true)); + } + + /** + * {@code micrometer-tracing} is an optional dependency. Consumers who only have {@code micrometer-observation} + * on the classpath (metrics/logging-only observability setups) must not see a {@link NoClassDefFoundError} or + * any other {@link LinkageError} the first time {@code traceParent()} is called. + */ + @Test + void shouldExposeMicrometerTracingClasspathGuard() { + // Sanity check on the current test classpath (micrometer-tracing IS present here). + assertTrue(MicrometerTracer.MICROMETER_TRACING_ON_CLASSPATH); + } + + /** + * Simulates a consumer that has only {@code micrometer-observation} on the classpath (no + * {@code micrometer-tracing}), by loading {@link MicrometerTracer} (and its dependency classes) through an + * isolated classloader that refuses to resolve {@code io.micrometer.tracing.*}. Asserts that {@code + * traceParent()} returns {@code null} rather than throwing {@link NoClassDefFoundError}. + */ + @Test + void shouldReturnNullWhenMicrometerTracingIsAbsentFromClasspath() throws Exception { + List urls = new ArrayList<>(); + String[] classpathEntries = System.getProperty("java.class.path").split(System.getProperty("path.separator")); + for (String entry : classpathEntries) { + if (entry.contains("micrometer-tracing")) { + continue; + } + urls.add(new java.io.File(entry).toURI().toURL()); + } + + try (URLClassLoader isolatedLoader = new URLClassLoader(urls.toArray(new URL[0]), null) { + @Override + protected Class loadClass(final String name, final boolean resolve) throws ClassNotFoundException { + if (name.startsWith("io.micrometer.tracing.")) { + throw new ClassNotFoundException(name); + } + if (name.startsWith("java.") || name.startsWith("javax.") || name.startsWith("jdk.")) { + return Class.forName(name, resolve, null); + } + return super.loadClass(name, resolve); + } + }) { + Class tracerClass = Class.forName( + "com.mongodb.internal.observability.micrometer.MicrometerTracer", true, isolatedLoader); + + Field guard = tracerClass.getDeclaredField("MICROMETER_TRACING_ON_CLASSPATH"); + guard.setAccessible(true); + assertFalse((Boolean) guard.get(null), + "guard should detect that io.micrometer.tracing is not resolvable in the isolated loader"); + + Class registryClass = Class.forName("io.micrometer.observation.ObservationRegistry", true, isolatedLoader); + Object registry = registryClass.getMethod("create").invoke(null); + + Constructor tracerCtor = tracerClass.getDeclaredConstructor( + registryClass, boolean.class, int.class, + Class.forName("io.micrometer.observation.ObservationConvention", true, isolatedLoader)); + Object micrometerTracer = tracerCtor.newInstance(registry, false, 1000, null); + + Class observationTypeClass = Class.forName( + "com.mongodb.observability.micrometer.MongodbObservation", true, isolatedLoader); + Object observationType = observationTypeClass.getField("MONGODB_COMMAND").get(null); + + Class tracerInterface = Class.forName( + "com.mongodb.internal.observability.micrometer.Tracer", true, isolatedLoader); + Method nextSpan = tracerInterface.getMethod("nextSpan", observationTypeClass, String.class, + Class.forName("com.mongodb.internal.observability.micrometer.TraceContext", true, isolatedLoader), + Class.forName("com.mongodb.MongoNamespace", true, isolatedLoader)); + Object span = nextSpan.invoke(micrometerTracer, observationType, "find", null, null); + + Class spanInterface = Class.forName( + "com.mongodb.internal.observability.micrometer.Span", true, isolatedLoader); + spanInterface.getMethod("openScope").invoke(span); + + Object traceContext = assertDoesNotThrow(() -> spanInterface.getMethod("context").invoke(span), + "obtaining the trace context must not throw when micrometer-tracing is absent"); + + Class traceContextInterface = Class.forName( + "com.mongodb.internal.observability.micrometer.TraceContext", true, isolatedLoader); + Object traceParent = assertDoesNotThrow( + () -> traceContextInterface.getMethod("traceParent").invoke(traceContext), + "traceParent() must not throw NoClassDefFoundError when micrometer-tracing is absent"); + + assertNull(traceParent); + + spanInterface.getMethod("closeScope").invoke(span); + spanInterface.getMethod("end").invoke(span); + } + } + + private static String traceParentFor(final String traceId, final String spanId, final Boolean sampled) { + ObservationRegistry registry = ObservationRegistry.create(); + SimpleTracer tracer = new SimpleTracer(); + registry.observationConfig().observationHandler(new DefaultTracingObservationHandler(tracer)); + + MicrometerTracer micrometerTracer = new MicrometerTracer(registry, false, 1000, null); + Span span = micrometerTracer.nextSpan(MongodbObservation.MONGODB_COMMAND, "find", null, null); + span.openScope(); + try { + SimpleTraceContext context = (SimpleTraceContext) tracer.lastSpan().context(); + context.setTraceId(traceId); + context.setSpanId(spanId); + context.setSampled(sampled); + return span.context().traceParent(); + } finally { + span.closeScope(); + span.end(); + } + } +} diff --git a/driver-core/src/test/unit/com/mongodb/internal/operation/OperationUnitSpecification.groovy b/driver-core/src/test/unit/com/mongodb/internal/operation/OperationUnitSpecification.groovy index ec5cb74156f..fb5fa791eda 100644 --- a/driver-core/src/test/unit/com/mongodb/internal/operation/OperationUnitSpecification.groovy +++ b/driver-core/src/test/unit/com/mongodb/internal/operation/OperationUnitSpecification.groovy @@ -65,7 +65,8 @@ class OperationUnitSpecification extends Specification { [6, 2]: 19, [6, 3]: 20, [7, 0]: 21, - [9, 0]: 25, + [8, 0]: 25, + [9, 0]: 29, ] static Integer getMaxWireVersionForServerVersion(List serverVersion) { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 7686cc15c41..848ae670815 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -186,8 +186,9 @@ project-reactor-test = { module = "io.projectreactor:reactor-test" } reactive-streams-tck = { module = " org.reactivestreams:reactive-streams-tck", version.ref = "reactive-streams" } reflections = { module = "org.reflections:reflections", version.ref = "reflections" } -micrometer-tracing-integration-test-bom = { module = " io.micrometer:micrometer-tracing-bom", version.ref = "micrometer-tracing" } -micrometer-tracing-integration-test = { module = " io.micrometer:micrometer-tracing-integration-test" } +micrometer-tracing = { module = "io.micrometer:micrometer-tracing", version.ref = "micrometer-tracing" } +micrometer-tracing-integration-test-bom = { module = "io.micrometer:micrometer-tracing-bom", version.ref = "micrometer-tracing" } +micrometer-tracing-integration-test = { module = "io.micrometer:micrometer-tracing-integration-test" } [bundles] aws-java-sdk-v1 = ["aws-java-sdk-v1-core", "aws-java-sdk-v1-sts"]