Skip to content

Commit ced7392

Browse files
authored
fix: fallback to Spark scan if encryption is enabled (native_datafusion/native_iceberg_compat) (#1785)
* fix: fallback to Spark scan if encryption is enabled (native_datafusion/native_iceberg_compat)
1 parent 7e64857 commit ced7392

File tree

2 files changed

+12
-4
lines changed

2 files changed

+12
-4
lines changed

spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,16 @@ case class CometScanRule(session: SparkSession) extends Rule[SparkPlan] {
132132
return withInfos(scanExec, fallbackReasons.toSet)
133133
}
134134

135+
val encryptionEnabled: Boolean =
136+
conf.getConfString("parquet.crypto.factory.class", "").nonEmpty &&
137+
conf.getConfString("parquet.encryption.kms.client.class", "").nonEmpty
138+
139+
if (scanImpl != CometConf.SCAN_NATIVE_COMET && encryptionEnabled) {
140+
fallbackReasons +=
141+
"Full native scan disabled because encryption is not supported"
142+
return withInfos(scanExec, fallbackReasons.toSet)
143+
}
144+
135145
val typeChecker = CometScanTypeChecker(scanImpl)
136146
val schemaSupported =
137147
typeChecker.isSchemaSupported(scanExec.requiredSchema, fallbackReasons)

spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,7 @@ class ParquetEncryptionITCase extends CometTestBase with SQLTestUtils {
4949
private val key2 = encoder.encodeToString("1234567890123451".getBytes(StandardCharsets.UTF_8))
5050

5151
test("SPARK-34990: Write and read an encrypted parquet") {
52-
// https://github.com/apache/datafusion-comet/issues/1488
53-
assume(CometConf.COMET_NATIVE_SCAN_IMPL.get() != CometConf.SCAN_NATIVE_ICEBERG_COMPAT)
52+
assume(CometConf.COMET_NATIVE_SCAN_IMPL.get() == CometConf.SCAN_NATIVE_COMET)
5453

5554
import testImplicits._
5655

@@ -93,8 +92,7 @@ class ParquetEncryptionITCase extends CometTestBase with SQLTestUtils {
9392
}
9493

9594
test("SPARK-37117: Can't read files in Parquet encryption external key material mode") {
96-
// https://github.com/apache/datafusion-comet/issues/1488
97-
assume(CometConf.COMET_NATIVE_SCAN_IMPL.get() != CometConf.SCAN_NATIVE_ICEBERG_COMPAT)
95+
assume(CometConf.COMET_NATIVE_SCAN_IMPL.get() == CometConf.SCAN_NATIVE_COMET)
9896

9997
import testImplicits._
10098

0 commit comments

Comments
 (0)