Skip to content

Commit ab821ba

Browse files
snazysnazy
committed
Support Iceberg's timestamp with nano-precision data type
NOTE On hold, because Iceberg code as of 1.7.1 does not yet fully support that new data type, causing test failures. Fixes #10236
1 parent e129315 commit ab821ba

File tree

9 files changed

+179
-40
lines changed

9 files changed

+179
-40
lines changed

‎catalog/format/iceberg-fixturegen/src/main/java/org/projectnessie/catalog/formats/iceberg/fixtures/IcebergFixtures.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,10 @@
3939
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.stringType;
4040
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.structType;
4141
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timeType;
42+
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampNanosType;
43+
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampNanosTzType;
4244
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampType;
43-
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestamptzType;
45+
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampTzType;
4446
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.uuidType;
4547

4648
import java.util.Iterator;
@@ -98,7 +100,9 @@ public static Stream<IcebergType> icebergTypes(IntSupplier idSupplier) {
98100
decimalType(10, 3),
99101
fixedType(42),
100102
timestampType(),
101-
timestamptzType());
103+
timestampTzType(),
104+
timestampNanosType(),
105+
timestampNanosTzType());
102106
}
103107

104108
public static IcebergSchema icebergSchemaAllTypes() {

‎catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/nessie/NessieModelIceberg.java‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@
3636
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.TYPE_MAP;
3737
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.TYPE_STRUCT;
3838
import static org.projectnessie.catalog.model.id.NessieId.transientNessieId;
39-
import static org.projectnessie.catalog.model.schema.types.NessieType.DEFAULT_TIME_PRECISION;
39+
import static org.projectnessie.catalog.model.schema.types.NessieType.MICROS_TIME_PRECISION;
40+
import static org.projectnessie.catalog.model.schema.types.NessieType.NANOS_TIME_PRECISION;
4041
import static org.projectnessie.catalog.model.snapshot.NessieViewRepresentation.NessieViewSQLRepresentation.nessieViewSQLRepresentation;
4142
import static org.projectnessie.catalog.model.snapshot.TableFormat.ICEBERG;
4243
import static org.projectnessie.catalog.model.statistics.NessiePartitionStatisticsFile.partitionStatisticsFile;
@@ -295,18 +296,23 @@ public static IcebergType nessieTypeToIcebergType(NessieTypeSpec type) {
295296
return IcebergType.dateType();
296297
case TIME:
297298
NessieTimeTypeSpec time = (NessieTimeTypeSpec) type;
298-
if (time.precision() != DEFAULT_TIME_PRECISION || time.withTimeZone()) {
299+
if (time.precision() != MICROS_TIME_PRECISION || time.withTimeZone()) {
299300
throw new IllegalArgumentException("Data type not supported in Iceberg: " + type);
300301
}
301302
return IcebergType.timeType();
302303
case TIMESTAMP:
303304
NessieTimestampTypeSpec timestamp = (NessieTimestampTypeSpec) type;
304-
if (timestamp.precision() != DEFAULT_TIME_PRECISION) {
305-
throw new IllegalArgumentException("Data type not supported in Iceberg: " + type);
305+
switch (timestamp.precision()) {
306+
case MICROS_TIME_PRECISION:
307+
return timestamp.withTimeZone()
308+
? IcebergType.timestampTzType()
309+
: IcebergType.timestampType();
310+
case NANOS_TIME_PRECISION:
311+
return timestamp.withTimeZone()
312+
? IcebergType.timestampNanosTzType()
313+
: IcebergType.timestampNanosType();
306314
}
307-
return timestamp.withTimeZone()
308-
? IcebergType.timestamptzType()
309-
: IcebergType.timestampType();
315+
throw new IllegalArgumentException("Data type not supported in Iceberg: " + type);
310316
case BINARY:
311317
return IcebergType.binaryType();
312318
case DECIMAL:
@@ -357,9 +363,13 @@ static NessieTypeSpec icebergTypeToNessieType(
357363
IcebergType type, Map<Integer, NessieField> icebergFields) {
358364
switch (type.type()) {
359365
case IcebergType.TYPE_TIMESTAMP_TZ:
360-
return NessieType.timestampType(true);
366+
return NessieType.timestampType(MICROS_TIME_PRECISION, true);
361367
case IcebergType.TYPE_TIMESTAMP:
362-
return NessieType.timestampType(false);
368+
return NessieType.timestampType(MICROS_TIME_PRECISION, false);
369+
case IcebergType.TYPE_TIMESTAMP_NS_TZ:
370+
return NessieType.timestampType(NANOS_TIME_PRECISION, true);
371+
case IcebergType.TYPE_TIMESTAMP_NS:
372+
return NessieType.timestampType(NANOS_TIME_PRECISION, false);
363373
case IcebergType.TYPE_BOOLEAN:
364374
return NessieType.booleanType();
365375
case IcebergType.TYPE_UUID:

‎catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergTimestampNanosType.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Copyright (C) 2023 Dremio
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.projectnessie.catalog.formats.iceberg.types;
17+
18+
import org.apache.avro.LogicalTypes;
19+
import org.apache.avro.Schema;
20+
21+
/** Iceberg timestamp, nanosecond precision. */
22+
public final class IcebergTimestampNanosType extends IcebergPrimitiveType {
23+
private static final Schema TIMESTAMP_NS_SCHEMA =
24+
LogicalTypes.timestampNanos().addToSchema(Schema.create(Schema.Type.LONG));
25+
private static final Schema TIMESTAMPTZ_NS_SCHEMA =
26+
LogicalTypes.timestampNanos().addToSchema(Schema.create(Schema.Type.LONG));
27+
public static final String ADJUST_TO_UTC_PROP = "adjust-to-utc";
28+
29+
static {
30+
TIMESTAMP_NS_SCHEMA.addProp(ADJUST_TO_UTC_PROP, false);
31+
TIMESTAMPTZ_NS_SCHEMA.addProp(ADJUST_TO_UTC_PROP, true);
32+
}
33+
34+
private final boolean adjustToUTC;
35+
36+
IcebergTimestampNanosType(boolean adjustToUTC) {
37+
this.adjustToUTC = adjustToUTC;
38+
}
39+
40+
public boolean adjustToUTC() {
41+
return adjustToUTC;
42+
}
43+
44+
@Override
45+
public String type() {
46+
return adjustToUTC() ? TYPE_TIMESTAMP_NS_TZ : TYPE_TIMESTAMP_NS;
47+
}
48+
49+
@Override
50+
public Schema avroSchema(int fieldId) {
51+
return adjustToUTC() ? TIMESTAMPTZ_NS_SCHEMA : TIMESTAMP_NS_SCHEMA;
52+
}
53+
54+
@Override
55+
public byte[] serializeSingleValue(Object value) {
56+
return IcebergLongType.serializeLong((Long) value);
57+
}
58+
59+
@Override
60+
public Object deserializeSingleValue(byte[] value) {
61+
return IcebergLongType.deserializeLong(value);
62+
}
63+
64+
@Override
65+
public int hashCode() {
66+
return type().hashCode() ^ (adjustToUTC ? 1 : 0);
67+
}
68+
69+
@Override
70+
public boolean equals(Object obj) {
71+
if (!(obj instanceof IcebergTimestampNanosType)) {
72+
return false;
73+
}
74+
if (obj == this) {
75+
return true;
76+
}
77+
IcebergTimestampNanosType o = (IcebergTimestampNanosType) obj;
78+
return o.adjustToUTC == adjustToUTC;
79+
}
80+
}

‎catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergTimestampType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.apache.avro.LogicalTypes;
1919
import org.apache.avro.Schema;
2020

21+
/** Iceberg timestamp, microsecond precision. */
2122
public final class IcebergTimestampType extends IcebergPrimitiveType {
2223
private static final Schema TIMESTAMP_SCHEMA =
2324
LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG));

‎catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergType.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ public interface IcebergType {
4343
String TYPE_BINARY = "binary";
4444
String TYPE_TIMESTAMP = "timestamp";
4545
String TYPE_TIMESTAMP_TZ = "timestamptz";
46+
String TYPE_TIMESTAMP_NS = "timestamp_ns";
47+
String TYPE_TIMESTAMP_NS_TZ = "timestamptz_ns";
4648
String TYPE_FIXED = "fixed";
4749
String TYPE_DECIMAL = "decimal";
4850
String TYPE_STRUCT = "struct";
@@ -89,14 +91,22 @@ static IcebergBinaryType binaryType() {
8991
return IcebergTypes.BINARY;
9092
}
9193

92-
static IcebergTimestampType timestamptzType() {
94+
static IcebergTimestampType timestampTzType() {
9395
return IcebergTypes.TIMESTAMPTZ;
9496
}
9597

9698
static IcebergTimestampType timestampType() {
9799
return IcebergTypes.TIMESTAMP;
98100
}
99101

102+
static IcebergTimestampNanosType timestampNanosTzType() {
103+
return IcebergTypes.TIMESTAMPTZ_NS;
104+
}
105+
106+
static IcebergTimestampNanosType timestampNanosType() {
107+
return IcebergTypes.TIMESTAMP_NS;
108+
}
109+
100110
static IcebergFixedType fixedType(int length) {
101111
return ImmutableIcebergFixedType.of(length);
102112
}

‎catalog/format/iceberg/src/main/java/org/projectnessie/catalog/formats/iceberg/types/IcebergTypes.java‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -45,35 +45,41 @@ final class IcebergTypes {
4545
static final IcebergBinaryType BINARY = new IcebergBinaryType();
4646
static final IcebergTimestampType TIMESTAMPTZ = new IcebergTimestampType(true);
4747
static final IcebergTimestampType TIMESTAMP = new IcebergTimestampType(false);
48+
static final IcebergTimestampNanosType TIMESTAMPTZ_NS = new IcebergTimestampNanosType(true);
49+
static final IcebergTimestampNanosType TIMESTAMP_NS = new IcebergTimestampNanosType(false);
4850

4951
private IcebergTypes() {}
5052

5153
static IcebergPrimitiveType primitiveFromString(String primitiveType) {
5254
switch (primitiveType) {
53-
case IcebergBooleanType.TYPE_BOOLEAN:
55+
case IcebergType.TYPE_BOOLEAN:
5456
return IcebergType.booleanType();
55-
case IcebergUuidType.TYPE_UUID:
57+
case IcebergType.TYPE_UUID:
5658
return IcebergType.uuidType();
57-
case IcebergIntegerType.TYPE_INT:
59+
case IcebergType.TYPE_INT:
5860
return IcebergType.integerType();
59-
case IcebergLongType.TYPE_LONG:
61+
case IcebergType.TYPE_LONG:
6062
return IcebergType.longType();
61-
case IcebergFloatType.TYPE_FLOAT:
63+
case IcebergType.TYPE_FLOAT:
6264
return IcebergType.floatType();
63-
case IcebergDoubleType.TYPE_DOUBLE:
65+
case IcebergType.TYPE_DOUBLE:
6466
return IcebergType.doubleType();
65-
case IcebergDateType.TYPE_DATE:
67+
case IcebergType.TYPE_DATE:
6668
return IcebergType.dateType();
67-
case IcebergTimeType.TYPE_TIME:
69+
case IcebergType.TYPE_TIME:
6870
return IcebergType.timeType();
69-
case IcebergStringType.TYPE_STRING:
71+
case IcebergType.TYPE_STRING:
7072
return IcebergType.stringType();
71-
case IcebergBinaryType.TYPE_BINARY:
73+
case IcebergType.TYPE_BINARY:
7274
return IcebergType.binaryType();
73-
case IcebergTimestampType.TYPE_TIMESTAMP_TZ:
74-
return IcebergType.timestamptzType();
75-
case IcebergTimestampType.TYPE_TIMESTAMP:
75+
case IcebergType.TYPE_TIMESTAMP_TZ:
76+
return IcebergType.timestampTzType();
77+
case IcebergType.TYPE_TIMESTAMP:
7678
return IcebergType.timestampType();
79+
case IcebergType.TYPE_TIMESTAMP_NS_TZ:
80+
return IcebergType.timestampNanosTzType();
81+
case IcebergType.TYPE_TIMESTAMP_NS:
82+
return IcebergType.timestampNanosType();
7783
default:
7884
Matcher m = DECIMAL_PATTERN.matcher(primitiveType);
7985
if (m.matches()) {

‎catalog/format/iceberg/src/test/java/org/projectnessie/catalog/formats/iceberg/nessie/TestNessieModelIceberg.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.mapType;
3939
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.stringType;
4040
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.structType;
41-
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestamptzType;
41+
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampTzType;
4242
import static org.projectnessie.catalog.model.id.NessieIdHasher.nessieIdHasher;
4343

4444
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -367,7 +367,7 @@ static Stream<Arguments> icebergNested() {
367367
nestedField(102, "topic", false, stringType(), null),
368368
nestedField(103, "partition", false, integerType(), null),
369369
nestedField(104, "offset", false, longType(), null),
370-
nestedField(105, "timestamp", false, timestamptzType(), null),
370+
nestedField(105, "timestamp", false, timestampTzType(), null),
371371
nestedField(106, "timestampType", false, integerType(), null),
372372
nestedField(
373373
107,
@@ -433,7 +433,7 @@ static Stream<Arguments> icebergNested() {
433433
nestedField(3, "topic", false, stringType(), null),
434434
nestedField(4, "partition", false, integerType(), null),
435435
nestedField(5, "offset", false, longType(), null),
436-
nestedField(6, "timestamp", false, timestamptzType(), null),
436+
nestedField(6, "timestamp", false, timestampTzType(), null),
437437
nestedField(7, "timestampType", false, integerType(), null),
438438
nestedField(
439439
8,

‎catalog/format/iceberg/src/test/java/org/projectnessie/catalog/formats/iceberg/types/TestIcebergTypes.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,10 @@
3333
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.stringType;
3434
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.structType;
3535
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timeType;
36+
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampNanosType;
37+
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampNanosTzType;
3638
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampType;
37-
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestamptzType;
39+
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.timestampTzType;
3840
import static org.projectnessie.catalog.formats.iceberg.types.IcebergType.uuidType;
3941

4042
import java.util.stream.Stream;
@@ -75,7 +77,9 @@ static Stream<Arguments> types() {
7577
arguments(dateType(), "\"date\""),
7678
arguments(timeType(), "\"time\""),
7779
arguments(timestampType(), "\"timestamp\""),
78-
arguments(timestamptzType(), "\"timestamptz\""),
80+
arguments(timestampTzType(), "\"timestamptz\""),
81+
arguments(timestampNanosType(), "\"timestamp_ns\""),
82+
arguments(timestampNanosTzType(), "\"timestamptz_ns\""),
7983
arguments(uuidType(), "\"uuid\""),
8084
arguments(fixedType(42), "\"fixed[42]\""),
8185
arguments(decimalType(33, 11), "\"decimal(33, 11)\""),
@@ -153,6 +157,8 @@ static Stream<Arguments> icebergTypes() {
153157
arguments(Types.DecimalType.of(10, 3), decimalType(10, 3)),
154158
arguments(Types.FixedType.ofLength(42), fixedType(42)),
155159
arguments(Types.TimestampType.withoutZone(), timestampType()),
156-
arguments(Types.TimestampType.withZone(), timestamptzType()));
160+
arguments(Types.TimestampType.withZone(), timestampTzType()),
161+
arguments(Types.TimestampNanoType.withoutZone(), timestampNanosType()),
162+
arguments(Types.TimestampNanoType.withZone(), timestampNanosTzType()));
157163
}
158164
}

0 commit comments

Comments
 (0)