Skip to content

chore: Make CI rules more consistent and reduce PR coverage for Spark 3.4 #1784

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
+79 −29
Draft

chore: Make CI rules more consistent and reduce PR coverage for Spark 3.4 #1784

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 79 additions & 29 deletions .github/workflows/pr_build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,13 @@ jobs:
test-target: [java]
spark-version: ['3.4']
scala-version: ['2.12', '2.13']
is_push_event:
- ${{ github.event_name == 'push' }}
exclude: # exclude java 11 and for pull_request event
- java_version: 11
is_push_event: false
- scala-version: '2.12'
is_push_event: false
fail-fast: false
name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
runs-on: ${{ matrix.os }}
Expand All @@ -207,8 +214,48 @@ jobs:
os: [macos-13]
java_version: [11, 17]
test-target: [rust, java]
spark-version: ['3.4', '3.5']
spark-version: ['3.5']
scala-version: ['2.12', '2.13']
is_push_event:
- ${{ github.event_name == 'push' }}
exclude: # exclude java 11 for pull_request event
- java_version: 11
is_push_event: false
fail-fast: false
if: github.event_name == 'push'
name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- name: Setup Rust & Java toolchain
uses: ./.github/actions/setup-macos-builder
with:
rust-version: ${{env.RUST_VERSION}}
jdk-version: ${{ matrix.java_version }}
- if: matrix.test-target == 'rust'
name: Rust test steps
uses: ./.github/actions/rust-test
- if: matrix.test-target == 'java'
name: Java test steps
uses: ./.github/actions/java-test
with:
maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}

macos-test-old-spark:
strategy:
matrix:
os: [macos-13]
java_version: [11, 17]
test-target: [rust, java]
spark-version: ['3.4']
scala-version: ['2.12', '2.13']
is_push_event:
- ${{ github.event_name == 'push' }}
exclude: # exclude java 11 and scala 2.12 for pull_request event
- java_version: 11
is_push_event: false
- scala-version: '2.12'
is_push_event: false
fail-fast: false
if: github.event_name == 'push'
name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
Expand All @@ -234,7 +281,7 @@ jobs:
matrix:
java_version: [11, 17]
test-target: [rust, java]
spark-version: ['3.4', '3.5']
spark-version: [''3.5']
scala-version: ['2.12', '2.13']
is_push_event:
- ${{ github.event_name == 'push' }}
Expand Down Expand Up @@ -262,6 +309,36 @@ jobs:
with:
maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}

macos-aarch64-test-with-old-spark:
strategy:
matrix:
java_version: [11, 17]
test-target: [java]
spark-version: ['3.4']
scala-version: ['2.12', '2.13']
exclude: # exclude java 11 and scala 2.12 for pull_request event
- java_version: 11
is_push_event: false
- scala-version: '2.12'
is_push_event: false
fail-fast: false
name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
runs-on: macos-14
steps:
- uses: actions/checkout@v4
- name: Setup Rust & Java toolchain
uses: ./.github/actions/setup-macos-builder
with:
rust-version: ${{env.RUST_VERSION}}
jdk-version: ${{ matrix.java_version }}
jdk-architecture: aarch64
protoc-architecture: aarch_64
- if: matrix.test-target == 'java'
name: Java test steps
uses: ./.github/actions/java-test
with:
maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}

macos-test-with-spark4_0:
strategy:
matrix:
Expand Down Expand Up @@ -315,30 +392,3 @@ jobs:
maven_opts: -Pspark-${{ matrix.spark-version }}
upload-test-reports: true

macos-aarch64-test-with-old-spark:
strategy:
matrix:
java_version: [17]
test-target: [java]
spark-version: ['3.4']
scala-version: ['2.12', '2.13']
exclude:
- java_version: 8
fail-fast: false
name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
runs-on: macos-14
steps:
- uses: actions/checkout@v4
- name: Setup Rust & Java toolchain
uses: ./.github/actions/setup-macos-builder
with:
rust-version: ${{env.RUST_VERSION}}
jdk-version: ${{ matrix.java_version }}
jdk-architecture: aarch64
protoc-architecture: aarch_64
- if: matrix.test-target == 'java'
name: Java test steps
uses: ./.github/actions/java-test
with:
maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}

Loading