Skip to content

Commit 00e4f7b

Browse files
MrRahulSharmaMrRahulSharma
authored
Converting precision less decimal/numbers into String type. (#323)
1 parent 5308017 commit 00e4f7b

File tree

4 files changed

+80
-22
lines changed

4 files changed

+80
-22
lines changed

‎oracle-plugin/docs/Oracle-batchsink.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,8 @@ Data Types Mapping
6565
| VARCHAR2 | string | |
6666
| NVARCHAR2 | string | |
6767
| VARCHAR | string | |
68-
| NUMBER | decimal | |
68+
| NUMBER | string | For NUMBER types defined without a precision and scale |
69+
| NUMBER | decimal | For NUMBER types with a defined precision and scale |
6970
| FLOAT | double | |
7071
| LONG | string | |
7172
| DATE | timestamp | |

‎oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -214,12 +214,17 @@ private void handleOracleSpecificType(ResultSet resultSet, StructuredRecord.Buil
214214
if (Double.class.getTypeName().equals(resultSet.getMetaData().getColumnClassName(columnIndex))) {
215215
recordBuilder.set(field.getName(), resultSet.getDouble(columnIndex));
216216
} else {
217-
// It's required to pass 'scale' parameter since in the case of Oracle, scale of 'BigDecimal' depends on the
218-
// scale set in the logical schema. For example for value '77.12' if the scale set in the logical schema is
219-
// set to 4 then the number will change to '77.1200'. Also if the value is '22.1274' and the logical schema
220-
// scale is set to 2 then the decimal value used will be '22.13' after rounding.
221-
BigDecimal decimal = resultSet.getBigDecimal(columnIndex, getScale(field.getSchema()));
222-
recordBuilder.setDecimal(field.getName(), decimal);
217+
if (precision == 0) {
218+
// In case of precision less decimal convert the field to String type
219+
recordBuilder.set(field.getName(), resultSet.getString(columnIndex));
220+
} else {
221+
// It's required to pass 'scale' parameter since in the case of Oracle, scale of 'BigDecimal' depends on the
222+
// scale set in the logical schema. For example for value '77.12' if the scale set in the logical schema is
223+
// set to 4 then the number will change to '77.1200'. Also if the value is '22.1274' and the logical schema
224+
// scale is set to 2 then the decimal value used will be '22.13' after rounding.
225+
BigDecimal decimal = resultSet.getBigDecimal(columnIndex, getScale(field.getSchema()));
226+
recordBuilder.setDecimal(field.getName(), decimal);
227+
}
223228
}
224229
}
225230
}

‎oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -105,13 +105,12 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti
105105
// For a Number type without specified precision and scale, precision will be 0 and scale will be -127
106106
if (precision == 0) {
107107
// reference : https://docs.oracle.com/cd/B28359_01/server.111/b28318/datatype.htm#CNCPT1832
108-
precision = 38;
109-
scale = 0;
110-
LOG.warn(String.format("%s type with undefined precision and scale is detected, "
111-
+ "there may be a precision loss while running the pipeline. "
112-
+ "Please define an output precision and scale for field '%s' to avoid precision loss.",
108+
LOG.warn(String.format("Field '%s' is a %s type without precision and scale, "
109+
+ "converting into STRING type to avoid any precision loss.",
110+
metadata.getColumnName(index),
113111
metadata.getColumnTypeName(index),
114112
metadata.getColumnName(index)));
113+
return Schema.of(Schema.Type.STRING);
115114
}
116115
return Schema.decimalOf(precision, scale);
117116
}

‎oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSourceDBRecordUnitTest.java‎

Lines changed: 63 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,11 @@ public class OracleSourceDBRecordUnitTest {
4848

4949
/**
5050
* Validate the precision less Numbers handling against following use cases.
51-
* 1. Ensure that for Number(0,-127) non nullable type a Number(38,0) is returned by default.
52-
* 2. Ensure that for Number(0,-127) non nullable type a Number(38,4) is returned,
51+
* 1. Ensure that for Number(0,-127) non nullable type a String is returned by default.
52+
* 2. Ensure that for Number(0,-127) non nullable type a String is returned,
5353
* if schema defined this as Number(38,4).
54-
* 3. Ensure that for Number(0,-127) nullable type a Number(38,0) is returned by default.
55-
* 4. Ensure that for Number(0,-127) nullable type a Number(38,4) is returned,
54+
* 3. Ensure that for Number(0,-127) nullable type a String type is returned by default.
55+
* 4. Ensure that for Number(0,-127) nullable type a String is returned,
5656
* if schema defined this as Number(38,4).
5757
* @throws Exception
5858
*/
@@ -72,10 +72,10 @@ public void validatePrecisionLessDecimalParsing() throws Exception {
7272
);
7373

7474
when(resultSet.getMetaData()).thenReturn(resultSetMetaData);
75-
when(resultSet.getBigDecimal(eq(1), eq(0))).thenReturn(new BigDecimal("123"));
76-
when(resultSet.getBigDecimal(eq(2), eq(4))).thenReturn(new BigDecimal("123.4568"));
77-
when(resultSet.getBigDecimal(eq(3), eq(0))).thenReturn(new BigDecimal("123"));
78-
when(resultSet.getBigDecimal(eq(4), eq(4))).thenReturn(new BigDecimal("123.4568"));
75+
when(resultSet.getString(eq(1))).thenReturn("123");
76+
when(resultSet.getString(eq(2))).thenReturn("123.4568");
77+
when(resultSet.getString(eq(3))).thenReturn("123");
78+
when(resultSet.getString(eq(4))).thenReturn("123.4568");
7979

8080
StructuredRecord.Builder builder = StructuredRecord.builder(schema);
8181
OracleSourceDBRecord dbRecord = new OracleSourceDBRecord(null, null);
@@ -85,9 +85,62 @@ public void validatePrecisionLessDecimalParsing() throws Exception {
8585
dbRecord.handleField(resultSet, builder, field4, 4, Types.NUMERIC, 0, -127);
8686

8787
StructuredRecord record = builder.build();
88+
Assert.assertTrue(record.get("ID1") instanceof String);
89+
Assert.assertEquals(record.get("ID1"), "123");
90+
Assert.assertTrue(record.get("ID2") instanceof String);
91+
Assert.assertEquals(record.get("ID2"), "123.4568");
92+
Assert.assertTrue(record.get("ID3") instanceof String);
93+
Assert.assertEquals(record.get("ID3"), "123");
94+
Assert.assertTrue(record.get("ID4") instanceof String);
95+
Assert.assertEquals(record.get("ID4"), "123.4568");
96+
}
97+
98+
/**
99+
* Validate the default precision Numbers handling against following use cases.
100+
* 1. Ensure that for Number(38, 0) non nullable type a Number(38,0) is returned.
101+
* 2. Ensure that for Number(38, 4) non nullable type a Number(38,6) is returned,
102+
* if schema defined this as Number(38,6).
103+
* 3. Ensure that for Number(38, 0) nullable type a Number(38,0) is returned by default.
104+
* 4. Ensure that for Number(38, 4) nullable type a Number(38,6) is returned,
105+
* if schema defined this as Number(38,6).
106+
* @throws Exception
107+
*/
108+
@Test
109+
public void validateDefaultDecimalParsing() throws Exception {
110+
Schema.Field field1 = Schema.Field.of("ID1", Schema.decimalOf(DEFAULT_PRECISION));
111+
Schema.Field field2 = Schema.Field.of("ID2", Schema.decimalOf(DEFAULT_PRECISION, 6));
112+
Schema.Field field3 = Schema.Field.of("ID3", Schema.nullableOf(Schema.decimalOf(DEFAULT_PRECISION)));
113+
Schema.Field field4 = Schema.Field.of("ID4", Schema.nullableOf(Schema.decimalOf(DEFAULT_PRECISION, 6)));
114+
115+
Schema schema = Schema.recordOf(
116+
"dbRecord",
117+
field1,
118+
field2,
119+
field3,
120+
field4
121+
);
122+
123+
when(resultSet.getMetaData()).thenReturn(resultSetMetaData);
124+
when(resultSet.getBigDecimal(eq(1), eq(0))).thenReturn(new BigDecimal("123"));
125+
when(resultSet.getBigDecimal(eq(2), eq(6))).thenReturn(new BigDecimal("123.456789"));
126+
when(resultSet.getBigDecimal(eq(3), eq(0))).thenReturn(new BigDecimal("123"));
127+
when(resultSet.getBigDecimal(eq(4), eq(6))).thenReturn(new BigDecimal("123.456789"));
128+
129+
StructuredRecord.Builder builder = StructuredRecord.builder(schema);
130+
OracleSourceDBRecord dbRecord = new OracleSourceDBRecord(null, null);
131+
dbRecord.handleField(resultSet, builder, field1, 1, Types.NUMERIC, DEFAULT_PRECISION, 0);
132+
dbRecord.handleField(resultSet, builder, field2, 2, Types.NUMERIC, DEFAULT_PRECISION, 4);
133+
dbRecord.handleField(resultSet, builder, field3, 3, Types.NUMERIC, DEFAULT_PRECISION, 0);
134+
dbRecord.handleField(resultSet, builder, field4, 4, Types.NUMERIC, DEFAULT_PRECISION, 4);
135+
136+
StructuredRecord record = builder.build();
137+
Assert.assertTrue(record.getDecimal("ID1") instanceof BigDecimal);
88138
Assert.assertEquals(record.getDecimal("ID1").toPlainString(), "123");
89-
Assert.assertEquals(record.getDecimal("ID2").toPlainString(), "123.4568");
139+
Assert.assertTrue(record.getDecimal("ID2") instanceof BigDecimal);
140+
Assert.assertEquals(record.getDecimal("ID2").toPlainString(), "123.456789");
141+
Assert.assertTrue(record.getDecimal("ID3") instanceof BigDecimal);
90142
Assert.assertEquals(record.getDecimal("ID3").toPlainString(), "123");
91-
Assert.assertEquals(record.getDecimal("ID4").toPlainString(), "123.4568");
143+
Assert.assertTrue(record.getDecimal("ID4") instanceof BigDecimal);
144+
Assert.assertEquals(record.getDecimal("ID4").toPlainString(), "123.456789");
92145
}
93146
}

0 commit comments

Comments
 (0)