Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,26 @@ public static ValidationResponse asInvalid(String message) {
*/
ValidationResponse validate(Map<String, Object> schema, Object structuredContent);

/**
* Validates the structured content against the provided JSON schema, describing the
* validated data in any resulting error message.
* <p>
* Use this overload when the caller knows which data is being validated (for example
* tool input arguments versus tool output) so that error messages are not misleading
* to consumers or LLMs.
* @param schema The JSON schema to validate against.
* @param structuredContent The structured content to validate.
* @param dataDescription A short, human-readable description of the data being
* validated and the schema it is checked against, for example
* {@code "input arguments do not match tool inputSchema"}. Included verbatim in
* validation error messages.
* @return A ValidationResponse indicating whether the validation was successful or
* not.
*/
default ValidationResponse validate(Map<String, Object> schema, Object structuredContent, String dataDescription) {
return validate(schema, structuredContent);
}

/**
* Validates that the given schema document itself conforms to JSON Schema 2020-12
* (SEP-1613). Schemas that declare an explicit non-2020-12 {@code $schema} dialect
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public static CallToolResult validate(McpSchema.Tool tool, Map<String, Object> a
return null;
}
Map<String, Object> args = arguments != null ? arguments : Map.of();
var validation = validator.validate(tool.inputSchema(), args);
var validation = validator.validate(tool.inputSchema(), args, "input arguments do not match tool inputSchema");
if (!validation.valid()) {
logger.warn("Tool '{}' input validation failed: {}", tool.name(), validation.errorMessage());
return CallToolResult.builder()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.*;

/**
Expand All @@ -39,17 +40,17 @@ void validate_whenDisabled_returnsNull() {
CallToolResult result = ToolInputValidator.validate(toolWithSchema, Map.of("name", "test"), false, validator);

assertThat(result).isNull();
verify(validator, never()).validate(any(), any());
verify(validator, never()).validate(any(), any(), any());
}

@Test
void validate_whenNoSchema_returnsNull() {
when(validator.validate(any(), any())).thenReturn(ValidationResponse.asValid(null));
when(validator.validate(any(), any(), any())).thenReturn(ValidationResponse.asValid(null));

CallToolResult result = ToolInputValidator.validate(toolWithoutSchema, Map.of("name", "test"), true, validator);

assertThat(result).isNull();
verify(validator).validate(any(), any());
verify(validator).validate(any(), any(), any());
}

@Test
Expand All @@ -61,7 +62,7 @@ void validate_whenNoValidator_returnsNull() {

@Test
void validate_withValidInput_returnsNull() {
when(validator.validate(any(), any())).thenReturn(ValidationResponse.asValid(null));
when(validator.validate(any(), any(), any())).thenReturn(ValidationResponse.asValid(null));

CallToolResult result = ToolInputValidator.validate(toolWithSchema, Map.of("name", "test"), true, validator);

Expand All @@ -70,24 +71,36 @@ void validate_withValidInput_returnsNull() {

@Test
void validate_withInvalidInput_returnsErrorResult() {
when(validator.validate(any(), any())).thenReturn(ValidationResponse.asInvalid("missing required: 'name'"));
when(validator.validate(any(), any(), any()))
.thenReturn(ValidationResponse.asInvalid("missing required: 'name'"));

CallToolResult result = ToolInputValidator.validate(toolWithSchema, Map.of(), true, validator);

assertThat(result).isNotNull();
assertThat(result.isError()).isTrue();
assertThat(((TextContent) result.content().get(0)).text()).contains("missing required: 'name'");
verify(validator).validate(any(), any());
verify(validator).validate(any(), any(), any());
}

@Test
void validate_passesInputDataDescriptionToValidator() {
when(validator.validate(any(), any(), any())).thenReturn(ValidationResponse.asValid(null));

ToolInputValidator.validate(toolWithSchema, Map.of("name", "test"), true, validator);

// The data description must reference input/inputSchema so error messages are not
// misleading to consumers/LLMs.
verify(validator).validate(any(), any(), eq("input arguments do not match tool inputSchema"));
}

@Test
void validate_withNullArguments_usesEmptyMap() {
when(validator.validate(any(), any())).thenReturn(ValidationResponse.asValid(null));
when(validator.validate(any(), any(), any())).thenReturn(ValidationResponse.asValid(null));

CallToolResult result = ToolInputValidator.validate(toolWithSchema, null, true, validator);

assertThat(result).isNull();
verify(validator).validate(any(), any());
verify(validator).validate(any(), any(), any());
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,12 @@ public class DefaultJsonSchemaValidator implements JsonSchemaValidator {

private static final Logger logger = LoggerFactory.getLogger(DefaultJsonSchemaValidator.class);

/**
* Default description used by the two-argument {@code validate} overload, which is
* primarily used for tool output validation.
*/
private static final String DEFAULT_DATA_DESCRIPTION = "structuredContent does not match tool outputSchema";

private final ObjectMapper objectMapper;

private final SchemaRegistry schemaFactory;
Expand All @@ -57,6 +63,11 @@ public DefaultJsonSchemaValidator(ObjectMapper objectMapper) {

@Override
public ValidationResponse validate(Map<String, Object> schema, Object structuredContent) {
return validate(schema, structuredContent, DEFAULT_DATA_DESCRIPTION);
}

@Override
public ValidationResponse validate(Map<String, Object> schema, Object structuredContent, String dataDescription) {

if (schema == null) {
throw new IllegalArgumentException("Schema must not be null");
Expand All @@ -76,8 +87,7 @@ public ValidationResponse validate(Map<String, Object> schema, Object structured
// Check if validation passed
if (!validationResult.isEmpty()) {
return ValidationResponse
.asInvalid("Validation failed: structuredContent does not match tool outputSchema. "
+ "Validation errors: " + validationResult);
.asInvalid("Validation failed: " + dataDescription + ". Validation errors: " + validationResult);
}

return ValidationResponse.asValid(jsonStructuredOutput.toString());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,40 @@ void testValidateWithInvalidTypeSchema() {
assertTrue(response.errorMessage().contains("structuredContent does not match tool outputSchema"));
}

@Test
void testValidateWithCustomDataDescription() {
String schemaJson = """
{
"type": "object",
"properties": {
"name": {"type": "string"},
"age": {"type": "integer"}
},
"required": ["name", "age"]
}
""";

String contentJson = """
{
"name": "John Doe"
}
""";

Map<String, Object> schema = toMap(schemaJson);
Map<String, Object> structuredContent = toMap(contentJson);

ValidationResponse response = validator.validate(schema, structuredContent,
"input arguments do not match tool inputSchema");

assertFalse(response.valid());
assertNotNull(response.errorMessage());
assertTrue(response.errorMessage().contains("Validation failed"));
assertTrue(response.errorMessage().contains("input arguments do not match tool inputSchema"));
assertFalse(response.errorMessage().contains("outputSchema"));
assertFalse(response.errorMessage().contains("structuredContent"));
assertTrue(response.errorMessage().contains("age"));
}

@Test
void testValidateWithMissingRequiredField() {
String schemaJson = """
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ public class DefaultJsonSchemaValidator implements JsonSchemaValidator {

private static final Logger logger = LoggerFactory.getLogger(DefaultJsonSchemaValidator.class);

/**
* Default description used by the two-argument {@code validate} overload, which is
* primarily used for tool output validation.
*/
private static final String DEFAULT_DATA_DESCRIPTION = "structuredContent does not match tool outputSchema";

private final JsonMapper jsonMapper;

private final SchemaRegistry schemaFactory;
Expand All @@ -56,6 +62,11 @@ public DefaultJsonSchemaValidator(JsonMapper jsonMapper) {

@Override
public ValidationResponse validate(Map<String, Object> schema, Object structuredContent) {
return validate(schema, structuredContent, DEFAULT_DATA_DESCRIPTION);
}

@Override
public ValidationResponse validate(Map<String, Object> schema, Object structuredContent, String dataDescription) {

if (schema == null) {
throw new IllegalArgumentException("Schema must not be null");
Expand All @@ -75,8 +86,7 @@ public ValidationResponse validate(Map<String, Object> schema, Object structured
// Check if validation passed
if (!validationResult.isEmpty()) {
return ValidationResponse
.asInvalid("Validation failed: structuredContent does not match tool outputSchema. "
+ "Validation errors: " + validationResult);
.asInvalid("Validation failed: " + dataDescription + ". Validation errors: " + validationResult);
}

return ValidationResponse.asValid(jsonStructuredOutput.toString());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,40 @@ void testValidateWithInvalidTypeSchema() {
assertTrue(response.errorMessage().contains("structuredContent does not match tool outputSchema"));
}

@Test
void testValidateWithCustomDataDescription() {
String schemaJson = """
{
"type": "object",
"properties": {
"name": {"type": "string"},
"age": {"type": "integer"}
},
"required": ["name", "age"]
}
""";

String contentJson = """
{
"name": "John Doe"
}
""";

Map<String, Object> schema = toMap(schemaJson);
Map<String, Object> structuredContent = toMap(contentJson);

ValidationResponse response = validator.validate(schema, structuredContent,
"input arguments do not match tool inputSchema");

assertFalse(response.valid());
assertNotNull(response.errorMessage());
assertTrue(response.errorMessage().contains("Validation failed"));
assertTrue(response.errorMessage().contains("input arguments do not match tool inputSchema"));
assertFalse(response.errorMessage().contains("outputSchema"));
assertFalse(response.errorMessage().contains("structuredContent"));
assertTrue(response.errorMessage().contains("age"));
}

@Test
void testValidateWithMissingRequiredField() {
String schemaJson = """
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,12 @@ void invalidInput_withDefaultValidation_shouldReturnToolError(String serverType,
assertThat(result.isError()).isTrue();
String errorMessage = ((TextContent) result.content().get(0)).text();
assertThat(errorMessage).containsIgnoringCase(expectedErrorSubstring);
// The message must make clear the failure refers to tool input, not output,
// otherwise it is misleading to consumers/LLMs.
assertThat(errorMessage).contains("Validation failed")
.contains("input arguments do not match tool inputSchema")
.doesNotContain("outputSchema")
.doesNotContain("structuredContent");
}
finally {
closeServer(server, serverType);
Expand Down