From 3c5ce2c0d78c8eb33b0b8d301d1f56bd3f965545 Mon Sep 17 00:00:00 2001 From: vanshikaagupta22 Date: Fri, 24 Apr 2026 08:42:37 +0000 Subject: [PATCH] Enable xml type support for oracle for dts-connectors only --- .../cdap/plugin/oracle/OracleConnector.java | 3 +- .../plugin/oracle/OracleConnectorConfig.java | 16 +++++++- .../cdap/plugin/oracle/OracleConstants.java | 1 + .../io/cdap/plugin/oracle/OracleSource.java | 9 +++-- .../oracle/OracleSourceSchemaReader.java | 10 ++++- .../plugin/oracle/OracleSchemaReaderTest.java | 39 ++++++++++++++++++- oracle-plugin/widgets/Oracle-batchsource.json | 19 +++++++++ oracle-plugin/widgets/Oracle-connector.json | 19 +++++++++ 8 files changed, 106 insertions(+), 10 deletions(-) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index 3ed85b009..469ca45f2 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -115,7 +115,8 @@ protected DBConnectorPath getDBConnectorPath(String path) { protected SchemaReader getSchemaReader(String sessionID) { return new OracleSourceSchemaReader(sessionID, config.getTreatAsOldTimestamp(), config.getTreatPrecisionlessNumAsDeci(), - config.getTreatTimestampLTZAsTimestamp()); + config.getTreatTimestampLTZAsTimestamp(), + config.getXmlTypeEnabled()); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index 79d14215b..cd9f2b375 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -42,14 +42,15 @@ public OracleConnectorConfig(String host, int port, String user, String password public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database) { this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null, null, - null, null); + null, null, null); } public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database, String role, Boolean useSSL, @Nullable Boolean treatAsOldTimestamp, @Nullable Boolean treatPrecisionlessNumAsDeci, - @Nullable Boolean treatTimestampLTZAsTimestamp) { + @Nullable Boolean treatTimestampLTZAsTimestamp, + @Nullable Boolean enableXmlType) { this.host = host; this.port = port; @@ -64,6 +65,7 @@ public OracleConnectorConfig(String host, int port, String user, String password this.treatAsOldTimestamp = treatAsOldTimestamp; this.treatPrecisionlessNumAsDeci = treatPrecisionlessNumAsDeci; this.treatTimestampLTZAsTimestamp = treatTimestampLTZAsTimestamp; + this.enableXmlType = enableXmlType; } @Override @@ -105,6 +107,12 @@ public String getConnectionString() { @Nullable public Boolean treatTimestampLTZAsTimestamp; + @Name(OracleConstants.ENABLE_XML_TYPE) + @Description("A hidden field to handle mapping of Oracle XML type to BQ String.") + @Nullable + public Boolean enableXmlType; + + @Override protected int getDefaultPort() { return 1521; @@ -139,6 +147,10 @@ public Boolean getTreatTimestampLTZAsTimestamp() { return Boolean.TRUE.equals(treatTimestampLTZAsTimestamp); } + public Boolean getXmlTypeEnabled() { + return Boolean.TRUE.equals(enableXmlType); + } + @Override public Properties getConnectionArgumentsProperties() { Properties prop = super.getConnectionArgumentsProperties(); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java index a3560e969..1a9508382 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java @@ -47,6 +47,7 @@ private OracleConstants() { public static final String TREAT_AS_OLD_TIMESTAMP = "treatAsOldTimestamp"; public static final String TREAT_PRECISIONLESSNUM_AS_DECI = "treatPrecisionlessNumAsDeci"; public static final String TREAT_TIMESTAMP_LTZ_AS_TIMESTAMP = "treatTimestampLTZAsTimestamp"; + public static final String ENABLE_XML_TYPE = "enableXmlType"; /** * Constructs the Oracle connection string based on the provided connection type, host, port, and database. diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 38e610855..24ef25d19 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -69,9 +69,10 @@ protected SchemaReader getSchemaReader() { boolean treatAsOldTimestamp = oracleSourceConfig.getConnection().getTreatAsOldTimestamp(); boolean treatPrecisionlessNumAsDeci = oracleSourceConfig.getConnection().getTreatPrecisionlessNumAsDeci(); boolean treatTimestampLTZAsTimestamp = oracleSourceConfig.getConnection().getTreatTimestampLTZAsTimestamp(); + boolean enableXmlType = oracleSourceConfig.getConnection().getXmlTypeEnabled(); return new OracleSourceSchemaReader(null, treatAsOldTimestamp, treatPrecisionlessNumAsDeci, - treatTimestampLTZAsTimestamp); + treatTimestampLTZAsTimestamp, enableXmlType); } @Override @@ -139,10 +140,12 @@ public OracleSourceConfig(String host, int port, String user, String password, S int defaultBatchValue, int defaultRowPrefetch, String importQuery, Integer numSplits, int fetchSize, String boundingQuery, String splitBy, Boolean useSSL, Boolean treatAsOldTimestamp, - Boolean treatPrecisionlessNumAsDeci, Boolean treatTimestampLTZAsTimestamp) { + Boolean treatPrecisionlessNumAsDeci, Boolean treatTimestampLTZAsTimestamp, + Boolean enableXmlType) { this.connection = new OracleConnectorConfig(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, role, useSSL, treatAsOldTimestamp, - treatPrecisionlessNumAsDeci, treatTimestampLTZAsTimestamp); + treatPrecisionlessNumAsDeci, treatTimestampLTZAsTimestamp, + enableXmlType); this.defaultBatchValue = defaultBatchValue; this.defaultRowPrefetch = defaultRowPrefetch; this.fetchSize = fetchSize; diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java index 208b70410..ec7a9d8ce 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java @@ -61,6 +61,7 @@ public class OracleSourceSchemaReader extends CommonSchemaReader { BINARY_DOUBLE, BFILE, LONG, + Types.SQLXML, LONG_RAW, Types.NUMERIC, Types.DECIMAL @@ -70,16 +71,19 @@ public class OracleSourceSchemaReader extends CommonSchemaReader { private final Boolean isTimestampOldBehavior; private final Boolean isPrecisionlessNumAsDecimal; private final Boolean isTimestampLtzFieldTimestamp; + private final Boolean isXmlTypeEnabled; public OracleSourceSchemaReader() { - this(null, false, false, false); + this(null, false, false, false, false); } public OracleSourceSchemaReader(@Nullable String sessionID, boolean isTimestampOldBehavior, - boolean isPrecisionlessNumAsDecimal, boolean isTimestampLtzFieldTimestamp) { + boolean isPrecisionlessNumAsDecimal, boolean isTimestampLtzFieldTimestamp, + boolean isXmlTypeEnabled) { this.sessionID = sessionID; this.isTimestampOldBehavior = isTimestampOldBehavior; this.isPrecisionlessNumAsDecimal = isPrecisionlessNumAsDecimal; this.isTimestampLtzFieldTimestamp = isTimestampLtzFieldTimestamp; + this.isXmlTypeEnabled = isXmlTypeEnabled; } @Override @@ -104,6 +108,8 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti case INTERVAL_YM: case LONG: return Schema.of(Schema.Type.STRING); + case Types.SQLXML: + return isXmlTypeEnabled ? Schema.of(Schema.Type.STRING) : super.getSchema(metadata, index); case Types.NUMERIC: case Types.DECIMAL: // FLOAT and REAL are returned as java.sql.Types.NUMERIC but with value that is a java.lang.Double diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java index 1ff77c533..586ca1141 100644 --- a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java @@ -18,6 +18,7 @@ import com.google.common.collect.Lists; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.exception.ProgramFailureException; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -27,13 +28,14 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.Types; import java.util.List; public class OracleSchemaReaderTest { @Test public void getSchema_timestampLTZFieldTrue_returnTimestamp() throws SQLException { - OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, true); + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, true, false); ResultSet resultSet = Mockito.mock(ResultSet.class); ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); @@ -64,7 +66,7 @@ public void getSchema_timestampLTZFieldTrue_returnTimestamp() throws SQLExceptio @Test public void getSchema_timestampLTZFieldFalse_returnDatetime() throws SQLException { - OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, false); + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, false, false); ResultSet resultSet = Mockito.mock(ResultSet.class); ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); @@ -91,4 +93,37 @@ public void getSchema_timestampLTZFieldFalse_returnDatetime() throws SQLExceptio Assert.assertEquals(expectedSchemaFields.get(1).getName(), actualSchemaFields.get(1).getName()); Assert.assertEquals(expectedSchemaFields.get(1).getSchema(), actualSchemaFields.get(1).getSchema()); } + + @Test + public void getSchema_xmlField_returnString() throws SQLException { + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, false, true); + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + Mockito.when(metadata.getColumnCount()).thenReturn(1); + Mockito.when(metadata.getColumnType(1)).thenReturn(Types.SQLXML); + Mockito.when(metadata.getColumnName(1)).thenReturn("xmlData"); + + List actualSchemaFields = schemaReader.getSchemaFields(resultSet); + + List expectedSchemaFields = Lists.newArrayList(); + expectedSchemaFields.add(Schema.Field.of("xmlData", Schema.of(Schema.Type.STRING))); + Assert.assertEquals(expectedSchemaFields.get(0).getName(), actualSchemaFields.get(0).getName()); + Assert.assertEquals(expectedSchemaFields.get(0).getSchema(), actualSchemaFields.get(0).getSchema()); + } + + @Test + public void getSchema_xmlFieldDisabled_throwsProgramFailureException() throws SQLException { + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, + false, false, false, false); + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + Mockito.when(metadata.getColumnCount()).thenReturn(1); + Mockito.when(metadata.getColumnType(1)).thenReturn(Types.SQLXML); + Mockito.when(metadata.getColumnName(1)).thenReturn("xmlData"); + + Assert.assertThrows(ProgramFailureException.class, () -> schemaReader.getSchemaFields(resultSet)); + + } } diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index 37adc011d..f5565d78e 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -177,6 +177,25 @@ ] } }, + { + "widget-type": "hidden", + "label": "Enable Xml Type", + "name": "enableXmlType", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, { "name": "connectionType", "label": "Connection Type", diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 413aa4b7a..4aaddc358 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -186,6 +186,25 @@ } ] } + }, + { + "widget-type": "hidden", + "label": "Enable Xml Type", + "name": "enableXmlType", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } } ] },