Skip to content

Commit

Permalink
Add Spark 4.0 support
Browse files Browse the repository at this point in the history
  • Loading branch information
dolfinus committed Dec 4, 2024
1 parent 2296982 commit 068fe01
Show file tree
Hide file tree
Showing 44 changed files with 409 additions and 186 deletions.
19 changes: 14 additions & 5 deletions .github/workflows/data/clickhouse/matrix.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
min: &min
2x: &2x
# Clickhouse version with proper DateTime > DateTime64 comparison
clickhouse-image: yandex/clickhouse-server
clickhouse-version: '21.1'
Expand All @@ -8,7 +8,7 @@ min: &min
java-version: 8
os: ubuntu-latest

max: &max
3x: &3x
clickhouse-image: clickhouse/clickhouse-server
clickhouse-version: 24.8.2.3-alpine
spark-version: 3.5.3
Expand All @@ -17,6 +17,15 @@ max: &max
java-version: 20
os: ubuntu-latest

4x: &4x
clickhouse-image: clickhouse/clickhouse-server
clickhouse-version: 24.8.2.3-alpine
spark-version: 4.0.0
pydantic-version: 2
python-version: '3.12'
java-version: 22
os: ubuntu-latest

latest: &latest
clickhouse-image: clickhouse/clickhouse-server
clickhouse-version: latest-alpine
Expand All @@ -27,6 +36,6 @@ latest: &latest
os: ubuntu-latest

matrix:
small: [*max]
full: [*min, *max]
nightly: [*min, *max, *latest]
small: [*3x]
full: [*2x, *3x, *4x]
nightly: [*2x, *3x, *latest]
17 changes: 12 additions & 5 deletions .github/workflows/data/core/matrix.yml
Original file line number Diff line number Diff line change
@@ -1,17 +1,24 @@
min: &min
2x: &2x
spark-version: 2.3.1
pydantic-version: 1
python-version: '3.7'
java-version: 8
os: ubuntu-latest

max: &max
3x: &3x
spark-version: 3.5.3
pydantic-version: 2
python-version: '3.12'
java-version: 20
os: ubuntu-latest

4x: &4x
spark-version: 4.0.0
pydantic-version: 2
python-version: '3.12'
java-version: 22
os: ubuntu-latest

latest: &latest
spark-version: latest
pydantic-version: latest
Expand All @@ -20,6 +27,6 @@ latest: &latest
os: ubuntu-latest

matrix:
small: [*max]
full: [*min, *max]
nightly: [*min, *max, *latest]
small: [*3x]
full: [*2x, *3x, *4x]
nightly: [*2x, *3x, *latest]
10 changes: 5 additions & 5 deletions .github/workflows/data/greenplum/matrix.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
min: &min
23: &23
greenplum-version: 6.23.1
package-version: 2.2.0
# Spark 2.3.0 does not support passing ivysettings.xml
Expand All @@ -8,7 +8,7 @@ min: &min
java-version: 8
os: ubuntu-latest

max: &max
32: &32
greenplum-version: 7.0.0
package-version: 2.3.1
# Greenplum connector does not support Spark 3.3+
Expand All @@ -29,6 +29,6 @@ latest: &latest
os: ubuntu-latest

matrix:
small: [*max]
full: [*min, *max]
nightly: [*min, *max, *latest]
small: [*32]
full: [*23, *32]
nightly: [*23, *latest]
18 changes: 13 additions & 5 deletions .github/workflows/data/hdfs/matrix.yml
Original file line number Diff line number Diff line change
@@ -1,19 +1,27 @@
min: &min
2x: &2x
hadoop-version: hadoop2-hdfs
spark-version: 2.3.1
pydantic-version: 1
python-version: '3.7'
java-version: 8
os: ubuntu-latest

max: &max
3x: &3x
hadoop-version: hadoop3-hdfs
spark-version: 3.5.3
pydantic-version: 2
python-version: '3.12'
java-version: 20
os: ubuntu-latest

4x: &4x
hadoop-version: hadoop3-hdfs
spark-version: 4.0.0
pydantic-version: 2
python-version: '3.12'
java-version: 22
os: ubuntu-latest

latest: &latest
hadoop-version: hadoop3-hdfs
spark-version: latest
Expand All @@ -23,6 +31,6 @@ latest: &latest
os: ubuntu-latest

matrix:
small: [*max]
full: [*min, *max]
nightly: [*min, *max, *latest]
small: [*3x]
full: [*2x, *3x, *4x]
nightly: [*2x, *3x, *latest]
17 changes: 12 additions & 5 deletions .github/workflows/data/hive/matrix.yml
Original file line number Diff line number Diff line change
@@ -1,17 +1,24 @@
min: &min
2x: &2x
spark-version: 2.3.1
pydantic-version: 1
python-version: '3.7'
java-version: 8
os: ubuntu-latest

max: &max
3x: &3x
spark-version: 3.5.3
pydantic-version: 2
python-version: '3.12'
java-version: 20
os: ubuntu-latest

4x: &4x
spark-version: 4.0.0
pydantic-version: 2
python-version: '3.12'
java-version: 22
os: ubuntu-latest

latest: &latest
spark-version: latest
pydantic-version: latest
Expand All @@ -20,6 +27,6 @@ latest: &latest
os: ubuntu-latest

matrix:
small: [*max]
full: [*min, *max]
nightly: [*min, *latest]
small: [*3x]
full: [*2x, *3x, *4x]
nightly: [*2x, *3x, *latest]
20 changes: 14 additions & 6 deletions .github/workflows/data/kafka/matrix.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
min: &min
# Headers are supported only since 2.x.
2x: &2x
# Headers are supported only since Kafka 2x.
# Images before 3.2.3 are not creating kafka_jaas.conf properly, and failing to start
# https://github.com/bitnami/containers/blob/9db9064668365cac89bff58259f63eb78bb97e79/bitnami/kafka/README.md?plain=1#L933
kafka-version: 3.2.3
Expand All @@ -9,14 +9,22 @@ min: &min
java-version: 8
os: ubuntu-latest

max: &max
3x: &3x
kafka-version: 3.7.1
pydantic-version: 2
spark-version: 3.5.3
python-version: '3.12'
java-version: 20
os: ubuntu-latest

4x: &4x
kafka-version: 3.7.1
pydantic-version: 2
spark-version: 4.0.0
python-version: '3.12'
java-version: 22
os: ubuntu-latest

latest: &latest
kafka-version: latest
pydantic-version: latest
Expand All @@ -26,6 +34,6 @@ latest: &latest
os: ubuntu-latest

matrix:
small: [*max]
full: [*min, *max]
nightly: [*min, *max, *latest]
small: [*3x]
full: [*2x, *3x, *4x]
nightly: [*2x, *3x, *latest]
32 changes: 16 additions & 16 deletions .github/workflows/data/local-fs/matrix.yml
Original file line number Diff line number Diff line change
@@ -1,32 +1,41 @@
min: &min
23: &23
spark-version: 2.3.1
pydantic-version: 1
python-version: '3.7'
java-version: 8
os: ubuntu-latest

min_avro: &min_avro
24: &24
# Avro supported only since Spark 2.4
spark-version: 2.4.8
pydantic-version: 1
python-version: '3.7'
java-version: 8
os: ubuntu-latest

min_excel: &min_excel
32: &32
# Excel supported only since Spark 3.2
spark-version: 3.2.4
pydantic-version: 1
python-version: '3.7'
java-version: 8
os: ubuntu-latest

max: &max
35: &35
# Excel pagkage currently supports Spark 3.5.1 max
spark-version: 3.5.1
pydantic-version: 2
python-version: '3.12'
java-version: 20
os: ubuntu-latest

4x: &4x
spark-version: 4.0.0
pydantic-version: 2
python-version: '3.12'
java-version: 22
os: ubuntu-latest

latest: &latest
spark-version: latest
pydantic-version: latest
Expand All @@ -35,15 +44,6 @@ latest: &latest
os: ubuntu-latest

matrix:
small:
- <<: *max
full:
- <<: *min
- <<: *min_avro
- <<: *min_excel
- <<: *max
nightly:
- <<: *min
- <<: *min_avro
- <<: *min_excel
- <<: *latest
small: [*35]
full: [*23, *24, *32, *35, *4x]
nightly: [*23, *24, *32, *35, *latest]
20 changes: 14 additions & 6 deletions .github/workflows/data/mongodb/matrix.yml
Original file line number Diff line number Diff line change
@@ -1,20 +1,28 @@
min: &min
32: &32
mongodb-version: 4.0.0
# MongoDB connector does not support Spark 2.x
# MongoDB connector does not support Spark 2x
spark-version: 3.2.4
pydantic-version: 1
python-version: '3.7'
java-version: 8
os: ubuntu-latest

max: &max
35: &35
mongodb-version: 7.0.14
spark-version: 3.5.3
pydantic-version: 2
python-version: '3.12'
java-version: 20
os: ubuntu-latest

4x: &4x
mongodb-version: 7.0.12
spark-version: 4.0.0
pydantic-version: 2
python-version: '3.12'
java-version: 20
os: ubuntu-latest

latest: &latest
mongodb-version: latest
spark-version: latest
Expand All @@ -24,6 +32,6 @@ latest: &latest
os: ubuntu-latest

matrix:
small: [*max]
full: [*min, *max]
nightly: [*min, *latest]
small: [*35]
full: [*32, *35, *4x]
nightly: [*32, *35, *latest]
18 changes: 13 additions & 5 deletions .github/workflows/data/mssql/matrix.yml
Original file line number Diff line number Diff line change
@@ -1,19 +1,27 @@
min: &min
2x: &2x
mssql-version: 2017-latest
spark-version: 2.3.1
pydantic-version: 1
python-version: '3.7'
java-version: 8
os: ubuntu-20.04

max: &max
3x: &3x
mssql-version: 2022-latest
spark-version: 3.5.3
pydantic-version: 2
python-version: '3.12'
java-version: 20
os: ubuntu-latest

4x: &4x
mssql-version: 2022-latest
spark-version: 4.0.0
pydantic-version: 2
python-version: '3.12'
java-version: 22
os: ubuntu-latest

latest: &latest
mssql-version: latest
spark-version: latest
Expand All @@ -23,6 +31,6 @@ latest: &latest
os: ubuntu-latest

matrix:
small: [*max]
full: [*min, *max]
nightly: [*min, *latest]
small: [*3x]
full: [*2x, *3x, *4x]
nightly: [*2x, *3x, *latest]
Loading

0 comments on commit 068fe01

Please sign in to comment.