[SPARK-51318][TESTS] Remove jar files from Apache Spark repository and disable affected tests

vrozov · vrozov · commit 19b971625bdf · 2025-03-26T19:18:52.000-07:00
diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/application/ReplE2ESuite.scala
@@ -169,8 +169,9 @@ class ReplE2ESuite extends ConnectFunSuite with RemoteSparkSession with BeforeAn
     assertContains("Array[java.lang.Long] = Array(0L, 2L, 4L, 6L, 8L)", output)
   }
 
-  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
-    "tests: Client-side JAR") {
+  ignore(
+    "SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+      "tests: Client-side JAR") {
     // scalastyle:off classforname line.size.limit
     val sparkHome = IntegrationTestUtils.sparkHome
     val testJar = Paths
diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/UDFClassLoadingE2ESuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/UDFClassLoadingE2ESuite.scala
@@ -28,7 +28,6 @@ import org.apache.spark.connect.proto
 import org.apache.spark.sql.connect.common.ProtoDataTypes
 import org.apache.spark.sql.connect.test.{ConnectFunSuite, RemoteSparkSession}
 
-
 @Ignore
 class UDFClassLoadingE2ESuite extends ConnectFunSuite with RemoteSparkSession {
 
diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ArtifactSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/ArtifactSuite.scala
@@ -143,8 +143,9 @@ class ArtifactSuite extends ConnectFunSuite with BeforeAndAfterEach {
 
   singleChunkArtifactTest("smallClassFile.class")
 
-  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
-    "tests") {
+  ignore(
+    "SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+      "tests") {
     singleChunkArtifactTest("smallJar.jar")
   }
 
@@ -180,8 +181,9 @@ class ArtifactSuite extends ConnectFunSuite with BeforeAndAfterEach {
     }
   }
 
-  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
-    "tests: Chunked Artifact - junitLargeJar.jar") {
+  ignore(
+    "SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+      "tests: Chunked Artifact - junitLargeJar.jar") {
     val artifactPath = artifactFilePath.resolve("junitLargeJar.jar")
     artifactManager.addArtifact(artifactPath.toString)
     // Expected chunks = roundUp( file_size / chunk_size) = 12
@@ -200,8 +202,9 @@ class ArtifactSuite extends ConnectFunSuite with BeforeAndAfterEach {
     checkChunksDataAndCrc(artifactPath, dataChunks)
   }
 
-  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
-    "tests: Batched SingleChunkArtifacts") {
+  ignore(
+    "SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+      "tests: Batched SingleChunkArtifacts") {
     val file1 = artifactFilePath.resolve("smallClassFile.class").toUri
     val file2 = artifactFilePath.resolve("smallJar.jar").toUri
     artifactManager.addArtifacts(Seq(file1, file2))
@@ -223,8 +226,9 @@ class ArtifactSuite extends ConnectFunSuite with BeforeAndAfterEach {
     assertFileDataEquality(artifacts.get(1).getData, Paths.get(file2))
   }
 
-  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
-    "tests: Mix of SingleChunkArtifact and chunked artifact") {
+  ignore(
+    "SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+      "tests: Mix of SingleChunkArtifact and chunked artifact") {
     val file1 = artifactFilePath.resolve("smallClassFile.class").toUri
     val file2 = artifactFilePath.resolve("junitLargeJar.jar").toUri
     val file3 = artifactFilePath.resolve("smallClassFileDup.class").toUri
diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/AddArtifactsHandlerSuite.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/AddArtifactsHandlerSuite.scala
@@ -211,8 +211,9 @@ class AddArtifactsHandlerSuite extends SharedSparkSession with ResourceHelper {
     }
   }
 
-  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
-    "tests: Multi chunk artifact") {
+  ignore(
+    "SPARK-51318: Remove `jar` files from Apache Spark repository and disable " +
+      "affected tests: Multi chunk artifact") {
     val promise = Promise[AddArtifactsResponse]()
     val handler = new TestAddArtifactsHandler(new DummyStreamObserver(promise))
     try {
@@ -236,8 +237,9 @@ class AddArtifactsHandlerSuite extends SharedSparkSession with ResourceHelper {
     }
   }
 
-  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
-    "tests: Mix of single-chunk and chunked artifacts") {
+  ignore(
+    "SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+      "tests: Mix of single-chunk and chunked artifacts") {
     val promise = Promise[AddArtifactsResponse]()
     val handler = new TestAddArtifactsHandler(new DummyStreamObserver(promise))
     try {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerSuite.scala
@@ -394,19 +394,20 @@ class ArtifactManagerSuite extends SharedSparkSession {
       val testBytes = "test".getBytes(StandardCharsets.UTF_8)
       Files.write(randomFilePath, testBytes)
 
+      val artifacts = 2
+
       // Register multiple kinds of artifacts
       artifactManager.addArtifact( // Class
         Paths.get("classes/Hello.class"), path.resolve("Hello.class"), None)
       artifactManager.addArtifact( // Python
         Paths.get("pyfiles/abc.zip"), randomFilePath, None, deleteStagedFile = false)
-      ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
-        "tests") {
-        artifactManager.addArtifact( // JAR
-          Paths.get("jars/udf_noA.jar"), path.resolve("udf_noA.jar"), None)
-      }
+      // "SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected tests"
+      // artifactManager.addArtifact( // JAR
+      //    Paths.get("jars/udf_noA.jar"), path.resolve("udf_noA.jar"), None)
       artifactManager.addArtifact( // Cached
         Paths.get("cache/test"), randomFilePath, None)
-      assert(FileUtils.listFiles(artifactManager.artifactPath.toFile, null, true).size() === 3)
+      assert(FileUtils.listFiles(artifactManager.artifactPath.toFile, null, true).size() ===
+        artifacts)
 
       // Clone the artifact manager
       val newSession = spark.cloneSession()
@@ -426,7 +427,7 @@ class ArtifactManagerSuite extends SharedSparkSession {
       }
 
       val allFiles = FileUtils.listFiles(newArtifactManager.artifactPath.toFile, null, true)
-      assert(allFiles.size() === 3)
+      assert(allFiles.size() === artifacts)
       allFiles.forEach { file =>
         assert(!file.getCanonicalPath.contains(spark.sessionUUID))
         assert(file.getCanonicalPath.contains(newSession.sessionUUID))
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
@@ -156,7 +156,8 @@ class HiveSparkSubmitSuite
     runSparkSubmit(args)
   }
 
-  test("SPARK-8489: MissingRequirementError during reflection") {
+  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+    "tests: SPARK-8489: MissingRequirementError during reflection") {
     // This test uses a pre-built jar to test SPARK-8489. In a nutshell, this test creates
     // a HiveContext and uses it to create a data frame from an RDD using reflection.
     // Before the fix in SPARK-8470, this results in a MissingRequirementError because
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUDFDynamicLoadSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUDFDynamicLoadSuite.scala
@@ -149,7 +149,8 @@ class HiveUDFDynamicLoadSuite extends QueryTest with SQLTestUtils with TestHiveS
     val jarPath = "src/test/noclasspath/hive-test-udfs.jar"
     val jarUrl = s"file://${System.getProperty("user.dir")}/$jarPath"
 
-    test("Spark should be able to run Hive UDF using jar regardless of " +
+    ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+      "tests: Spark should be able to run Hive UDF using jar regardless of " +
       s"current thread context classloader (${udfInfo.identifier}") {
       Utils.withContextClassLoader(Utils.getSparkClassLoader) {
         withUserDefinedFunction(udfInfo.funcName -> false) {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -3240,7 +3240,8 @@ class HiveDDLSuite
     }
   }
 
-  test("SPARK-34261: Avoid side effect if create exists temporary function") {
+  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+    "tests: SPARK-34261: Avoid side effect if create exists temporary function") {
     withUserDefinedFunction("f1" -> true) {
       sql("CREATE TEMPORARY FUNCTION f1 AS 'org.apache.hadoop.hive.ql.udf.UDFUUID'")
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -835,7 +835,8 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
     }
   }
 
-  test("ADD JAR command 2") {
+  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+    "tests: ADD JAR command 2") {
     // this is a test case from mapjoin_addjar.q
     val testJar = HiveTestJars.getHiveHcatalogCoreJar().toURI
     val testData = TestHive.getHiveFile("data/files/sample.json").toURI
@@ -868,7 +869,8 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
     }
   }
 
-  test("CREATE TEMPORARY FUNCTION") {
+  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+    "tests: CREATE TEMPORARY FUNCTION") {
     val funcJar = TestHive.getHiveFile("TestUDTF.jar")
     val jarURL = funcJar.toURI.toURL
     sql(s"ADD JAR $jarURL")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -605,7 +605,8 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils {
     }
   }
 
-  test("UDTF") {
+  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+    "tests: UDTF") {
     withUserDefinedFunction("udtf_count2" -> true) {
       sql(s"ADD JAR ${hiveContext.getHiveFile("TestUDTF.jar").getCanonicalPath}")
       // The function source code can be found at:
@@ -626,7 +627,8 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils {
     }
   }
 
-  test("permanent UDTF") {
+  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+    "tests: permanent UDTF") {
     withUserDefinedFunction("udtf_count_temp" -> false) {
       sql(
         s"""
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -258,7 +258,8 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
         "when `expr1` = `expr4`, return `expr5`; else return `expr6`")
   }
 
-  test("describe functions - user defined functions") {
+  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+    "tests: describe functions - user defined functions") {
     withUserDefinedFunction("udtf_count" -> false) {
       sql(
         s"""
@@ -283,7 +284,8 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
     }
   }
 
-  test("describe functions - temporary user defined functions") {
+  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+    "tests: describe functions - temporary user defined functions") {
     withUserDefinedFunction("udtf_count_temp" -> true) {
       sql(
         s"""
@@ -2484,7 +2486,8 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
     }
   }
 
-  test("SPARK-32668: HiveGenericUDTF initialize UDTF should use StructObjectInspector method") {
+  ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
+    "tests: SPARK-32668: HiveGenericUDTF initialize UDTF should use StructObjectInspector method") {
     withUserDefinedFunction("udtf_stack1" -> true, "udtf_stack2" -> true) {
       sql(
         s"""

Original file line number	Diff line number	Diff line change
`@@ -211,8 +211,9 @@ class AddArtifactsHandlerSuite extends SharedSparkSession with ResourceHelper {`
`211`	`211`	`}`
`212`	`212`	`}`
`213`	`213`
`214`		- ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
`215`		`- "tests: Multi chunk artifact") {`
	`214`	`+ ignore(`
	`215`	+ "SPARK-51318: Remove `jar` files from Apache Spark repository and disable " +
	`216`	`+ "affected tests: Multi chunk artifact") {`
`216`	`217`	`val promise = Promise[AddArtifactsResponse]()`
`217`	`218`	`val handler = new TestAddArtifactsHandler(new DummyStreamObserver(promise))`
`218`	`219`	`try {`
`@@ -236,8 +237,9 @@ class AddArtifactsHandlerSuite extends SharedSparkSession with ResourceHelper {`
`236`	`237`	`}`
`237`	`238`	`}`
`238`	`239`
`239`		- ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
`240`		`- "tests: Mix of single-chunk and chunked artifacts") {`
	`240`	`+ ignore(`
	`241`	+ "SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
	`242`	`+ "tests: Mix of single-chunk and chunked artifacts") {`
`241`	`243`	`val promise = Promise[AddArtifactsResponse]()`
`242`	`244`	`val handler = new TestAddArtifactsHandler(new DummyStreamObserver(promise))`
`243`	`245`	`try {`
Original file line number	Diff line number	Diff line change
`@@ -156,7 +156,8 @@ class HiveSparkSubmitSuite`
`156`	`156`	`runSparkSubmit(args)`
`157`	`157`	`}`
`158`	`158`
`159`		`- test("SPARK-8489: MissingRequirementError during reflection") {`
	`159`	+ ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
	`160`	`+ "tests: SPARK-8489: MissingRequirementError during reflection") {`
`160`	`161`	`// This test uses a pre-built jar to test SPARK-8489. In a nutshell, this test creates`
`161`	`162`	`// a HiveContext and uses it to create a data frame from an RDD using reflection.`
`162`	`163`	`// Before the fix in SPARK-8470, this results in a MissingRequirementError because`
Original file line number	Diff line number	Diff line change
`@@ -3240,7 +3240,8 @@ class HiveDDLSuite`
`3240`	`3240`	`}`
`3241`	`3241`	`}`
`3242`	`3242`
`3243`		`- test("SPARK-34261: Avoid side effect if create exists temporary function") {`
	`3243`	+ ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
	`3244`	`+ "tests: SPARK-34261: Avoid side effect if create exists temporary function") {`
`3244`	`3245`	`withUserDefinedFunction("f1" -> true) {`
`3245`	`3246`	`sql("CREATE TEMPORARY FUNCTION f1 AS 'org.apache.hadoop.hive.ql.udf.UDFUUID'")`
`3246`	`3247`
Original file line number	Diff line number	Diff line change
`@@ -835,7 +835,8 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd`
`835`	`835`	`}`
`836`	`836`	`}`
`837`	`837`
`838`		`- test("ADD JAR command 2") {`
	`838`	+ ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
	`839`	`+ "tests: ADD JAR command 2") {`
`839`	`840`	`// this is a test case from mapjoin_addjar.q`
`840`	`841`	`val testJar = HiveTestJars.getHiveHcatalogCoreJar().toURI`
`841`	`842`	`val testData = TestHive.getHiveFile("data/files/sample.json").toURI`
`@@ -868,7 +869,8 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd`
`868`	`869`	`}`
`869`	`870`	`}`
`870`	`871`
`871`		`- test("CREATE TEMPORARY FUNCTION") {`
	`872`	+ ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
	`873`	`+ "tests: CREATE TEMPORARY FUNCTION") {`
`872`	`874`	`val funcJar = TestHive.getHiveFile("TestUDTF.jar")`
`873`	`875`	`val jarURL = funcJar.toURI.toURL`
`874`	`876`	`sql(s"ADD JAR $jarURL")`
Original file line number	Diff line number	Diff line change
`@@ -605,7 +605,8 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils {`
`605`	`605`	`}`
`606`	`606`	`}`
`607`	`607`
`608`		`- test("UDTF") {`
	`608`	+ ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
	`609`	`+ "tests: UDTF") {`
`609`	`610`	`withUserDefinedFunction("udtf_count2" -> true) {`
`610`	`611`	`sql(s"ADD JAR ${hiveContext.getHiveFile("TestUDTF.jar").getCanonicalPath}")`
`611`	`612`	`// The function source code can be found at:`
`@@ -626,7 +627,8 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils {`
`626`	`627`	`}`
`627`	`628`	`}`
`628`	`629`
`629`		`- test("permanent UDTF") {`
	`630`	+ ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
	`631`	`+ "tests: permanent UDTF") {`
`630`	`632`	`withUserDefinedFunction("udtf_count_temp" -> false) {`
`631`	`633`	`sql(`
`632`	`634`	`s"""`
Original file line number	Diff line number	Diff line change
`@@ -258,7 +258,8 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi`
`258`	`258`	"when `expr1` = `expr4`, return `expr5`; else return `expr6`")
`259`	`259`	`}`
`260`	`260`
`261`		`- test("describe functions - user defined functions") {`
	`261`	+ ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
	`262`	`+ "tests: describe functions - user defined functions") {`
`262`	`263`	`withUserDefinedFunction("udtf_count" -> false) {`
`263`	`264`	`sql(`
`264`	`265`	`s"""`
`@@ -283,7 +284,8 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi`
`283`	`284`	`}`
`284`	`285`	`}`
`285`	`286`
`286`		`- test("describe functions - temporary user defined functions") {`
	`287`	+ ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
	`288`	`+ "tests: describe functions - temporary user defined functions") {`
`287`	`289`	`withUserDefinedFunction("udtf_count_temp" -> true) {`
`288`	`290`	`sql(`
`289`	`291`	`s"""`
`@@ -2484,7 +2486,8 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi`
`2484`	`2486`	`}`
`2485`	`2487`	`}`
`2486`	`2488`
`2487`		`- test("SPARK-32668: HiveGenericUDTF initialize UDTF should use StructObjectInspector method") {`
	`2489`	+ ignore("SPARK-51318: Remove `jar` files from Apache Spark repository and disable affected " +
	`2490`	`+ "tests: SPARK-32668: HiveGenericUDTF initialize UDTF should use StructObjectInspector method") {`
`2488`	`2491`	`withUserDefinedFunction("udtf_stack1" -> true, "udtf_stack2" -> true) {`
`2489`	`2492`	`sql(`
`2490`	`2493`	`s"""`