Skip to content

Commit d77e807

Browse files
dennygleedennyglee
authored
Merge branch 'master' into dvrowindex
2 parents f0c5676 + 751b5d4 commit d77e807

File tree

2,475 files changed

+95125
-346897
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

2,475 files changed

+95125
-346897
lines changed

‎.github/workflows/build.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: "Delta Build"
2+
on: [push, pull_request]
3+
jobs:
4+
test:
5+
name: "Build Test"
6+
runs-on: ubuntu-24.04
7+
steps:
8+
- uses: actions/checkout@v3
9+
10+
- name: Install Java 17
11+
uses: actions/setup-java@v3
12+
with:
13+
distribution: "zulu"
14+
java-version: "17"
15+
16+
- name: Set up Python
17+
uses: actions/setup-python@v4
18+
with:
19+
python-version: '3.9'
20+
21+
- name: Cache Scala, SBT
22+
uses: actions/cache@v3
23+
with:
24+
path: |
25+
~/.sbt
26+
~/.ivy2
27+
~/.cache/coursier
28+
key: delta-sbt-cache-cross-spark
29+
30+
- name: Run cross-Spark build test
31+
run: python project/tests/test_cross_spark_publish.py

‎.github/workflows/connectors_test.yaml‎

Lines changed: 0 additions & 45 deletions
This file was deleted.

‎.github/workflows/iceberg_test.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ jobs:
77
strategy:
88
matrix:
99
# These Scala versions must match those in the build.sbt
10-
scala: [2.12.18, 2.13.13]
10+
scala: [2.13.16]
1111
env:
1212
SCALA_VERSION: ${{ matrix.scala }}
1313
steps:
@@ -25,7 +25,7 @@ jobs:
2525
uses: actions/setup-java@v3
2626
with:
2727
distribution: "zulu"
28-
java-version: "8"
28+
java-version: "17"
2929
- name: Cache Scala, SBT
3030
uses: actions/cache@v3
3131
with:

‎.github/workflows/kernel_docs.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ jobs:
3131
uses: actions/setup-java@v3
3232
with:
3333
distribution: "zulu"
34-
java-version: "8"
34+
java-version: "11"
3535
- name: Generate docs
3636
run: |
3737
build/sbt kernelGroup/unidoc

‎.github/workflows/kernel_test.yaml‎

Lines changed: 65 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,80 @@
11
name: "Delta Kernel"
2+
23
on: [push, pull_request]
4+
5+
# Cancel previous runs when new commits are pushed
6+
concurrency:
7+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
8+
cancel-in-progress: true
9+
10+
env:
11+
# Point SBT to our cache directories for consistency
12+
SBT_OPTS: "-Dsbt.coursier.home-dir=/home/runner/.cache/coursier -Dsbt.ivy.home=/home/runner/.ivy2"
13+
314
jobs:
415
test:
5-
name: "DK"
16+
name: "DK: Shard ${{ matrix.shard }}"
617
runs-on: ubuntu-24.04
18+
strategy:
19+
fail-fast: false # Allow all shards to run even if one fails
20+
matrix:
21+
shard: [0, 1, 2, 3]
722
env:
8-
SCALA_VERSION: 2.12.18
23+
SCALA_VERSION: 2.13.16
24+
NUM_SHARDS: 4
25+
DISABLE_UNIDOC: true # Another unidoc workflow will test unidoc.
26+
TEST_PARALLELISM_COUNT: 4
27+
steps:
28+
- name: Show runner specs
29+
run: |
30+
echo "=== GitHub Runner Specs ==="
31+
echo "CPU cores: $(nproc)"
32+
echo "CPU info: $(lscpu | grep 'Model name' | cut -d':' -f2 | xargs)"
33+
echo "Total RAM: $(free -h | grep '^Mem:' | awk '{print $2}')"
34+
echo "Available RAM: $(free -h | grep '^Mem:' | awk '{print $7}')"
35+
echo "Disk space: $(df -h / | tail -1 | awk '{print $2 " total, " $4 " available"}')"
36+
echo "Runner OS: ${{ runner.os }}"
37+
echo "Runner arch: ${{ runner.arch }}"
38+
- name: Checkout code
39+
uses: actions/checkout@v4
40+
# Run unit tests with JDK 17. These unit tests depend on Spark, and Spark 4.0+ is JDK 17.
41+
- name: install java
42+
uses: actions/setup-java@v4
43+
with:
44+
distribution: "zulu"
45+
java-version: "17"
46+
- name: Cache SBT and dependencies
47+
id: cache-sbt
48+
uses: actions/cache@v4
49+
with:
50+
path: |
51+
~/.sbt
52+
~/.ivy2/cache
53+
~/.coursier/cache
54+
~/.cache/coursier
55+
key: sbt-kernel-${{ runner.os }}-scala${{ env.SCALA_VERSION }}
56+
- name: Check cache status
57+
run: |
58+
if [ "${{ steps.cache-sbt.outputs.cache-hit }}" == "true" ]; then
59+
echo "✅ Cache HIT - using cached dependencies"
60+
else
61+
echo "❌ Cache MISS - will download dependencies"
62+
fi
63+
- name: Run unit tests
64+
run: |
65+
python run-tests.py --group kernel --coverage --shard ${{ matrix.shard }}
66+
67+
integration-test:
68+
name: "DK: Integration"
69+
runs-on: ubuntu-24.04
970
steps:
1071
- uses: actions/checkout@v3
72+
# Run integration tests with JDK 11, as they have no Spark dependency
1173
- name: install java
1274
uses: actions/setup-java@v3
1375
with:
1476
distribution: "zulu"
15-
java-version: "8"
16-
- name: Run tests
17-
run: |
18-
python run-tests.py --group kernel --coverage
77+
java-version: "11"
1978
- name: Run integration tests
2079
run: |
2180
cd kernel/examples && python run-kernel-examples.py --use-local

‎.github/workflows/kernel_unitycatalog_test.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: "Kernel Unity Catalog"
2+
on: [push, pull_request]
3+
jobs:
4+
test:
5+
name: "Kernel Unity Catalog Tests"
6+
runs-on: ubuntu-24.04
7+
env:
8+
SCALA_VERSION: 2.13.16
9+
steps:
10+
- uses: actions/checkout@v3
11+
- uses: technote-space/get-diff-action@v4
12+
id: git-diff
13+
with:
14+
PATTERNS: |
15+
build.sbt
16+
version.sbt
17+
unity/**
18+
kernel/**
19+
storage/**
20+
.github/workflows/unity_test.yaml
21+
- name: install java
22+
uses: actions/setup-java@v3
23+
with:
24+
distribution: "zulu"
25+
java-version: "17"
26+
if: steps.git-diff.outputs.diff
27+
- name: Run Unity tests with coverage
28+
run: |
29+
./build/sbt "++ ${{ env.SCALA_VERSION }}" clean coverage kernelUnityCatalog/test coverageAggregate coverageOff -v
30+
if: steps.git-diff.outputs.diff

‎.github/workflows/publish_docs.yaml‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
name: Publish Docs
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
paths:
8+
- docs/**
9+
release:
10+
types:
11+
- published
12+
workflow_dispatch:
13+
14+
jobs:
15+
build_api_docs:
16+
name: Build API docs (${{ matrix.version.name }})
17+
runs-on: ubuntu-latest
18+
19+
strategy:
20+
matrix:
21+
version:
22+
- name: latest
23+
ref: v4.0.0
24+
java: 17
25+
out_dir: docs/apis/_site/api
26+
- name: 3.3.2
27+
ref: v3.3.2
28+
java: 8
29+
out_dir: docs/apis/_site/api
30+
31+
steps:
32+
- uses: actions/checkout@v4
33+
with:
34+
repository: delta-io/delta
35+
ref: ${{ matrix.version.ref }}
36+
37+
- uses: actions/setup-java@v3
38+
with:
39+
distribution: "zulu"
40+
java-version: ${{ matrix.version.java }}
41+
42+
- name: Setup python environment
43+
uses: conda-incubator/setup-miniconda@v3
44+
with:
45+
activate-environment: delta_docs
46+
environment-file: docs/environment.yml
47+
48+
- name: Fix generate_api_docs script
49+
if: contains(matrix.version.ref, 'v4')
50+
run: |
51+
sed -i 's|scala-2\.13|scala-2.13|g' docs/apis/generate_api_docs.py
52+
sed -i '/standalone_javadoc_gen_dir,/d' docs/apis/generate_api_docs.py
53+
sed -i '/flink_javadoc_gen_dir,/d' docs/apis/generate_api_docs.py
54+
55+
- name: Generate API docs
56+
shell: bash -el {0}
57+
run: python3 docs/generate_docs.py --api-docs
58+
env:
59+
_DELTA_LAKE_RELEASE_VERSION_: ${{ matrix.version.name }}
60+
61+
- name: Move doc contents up one level
62+
run: |
63+
find docs/apis/_site/api -type d \( -name "unidoc" -o -name "javaunidoc" -o -name "html" \) | while read dir; do
64+
echo "Processing $dir"
65+
parent="$(dirname "$dir")"
66+
# Move all files (including hidden ones) using find
67+
find "$dir" -maxdepth 1 -type f -exec mv {} "$parent"/ \;
68+
# Move all subdirectories
69+
find "$dir" -maxdepth 1 -type d ! -path "$dir" -exec mv {} "$parent"/ \;
70+
rmdir "$dir"
71+
done
72+
73+
- name: Upload artifact
74+
uses: actions/upload-artifact@v4
75+
with:
76+
name: ${{ github.run_id }}-apidocs-${{ matrix.version.name }}
77+
path: ${{ matrix.version.out_dir }}
78+
79+
build_site:
80+
name: Build site
81+
needs: build_api_docs
82+
runs-on: ubuntu-latest
83+
84+
steps:
85+
- uses: actions/checkout@v4
86+
87+
- name: Install Node.js
88+
uses: actions/setup-node@v4
89+
with:
90+
node-version-file: docs/.nvmrc
91+
92+
- uses: pnpm/action-setup@v4
93+
name: Install pnpm
94+
with:
95+
package_json_file: docs/package.json
96+
97+
- name: Install Node.js dependencies
98+
run: pnpm install
99+
working-directory: docs
100+
101+
- name: Download API docs artifacts
102+
uses: actions/download-artifact@v4
103+
with:
104+
pattern: ${{ github.run_id }}-apidocs-*
105+
path: docs/public/api
106+
107+
- name: Rename API docs artifact folders
108+
run: |
109+
for d in docs/public/api/${{ github.run_id }}-apidocs-*; do
110+
[ -d "$d" ] || continue
111+
new_name="$(echo "$d" | sed "s|docs/public/api/${{ github.run_id }}-apidocs-||")"
112+
mv "$d" "docs/public/api/$new_name"
113+
done
114+
115+
- name: Generate docs site
116+
run: python3 docs/generate_docs.py
117+
118+
- name: Install Netlify CLI
119+
run: pnpm i -g netlify-cli
120+
121+
- name: Publish site to Netlify
122+
run: netlify deploy --dir=docs/dist --prod
123+
env:
124+
NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
125+
NETLIFY_SITE_ID: ${{ vars.NETLIFY_SITE_ID }}

‎.github/workflows/spark_examples_test.yaml‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ jobs:
77
strategy:
88
matrix:
99
# These Scala versions must match those in the build.sbt
10-
scala: [2.12.18, 2.13.13]
10+
scala: [2.13.16]
1111
env:
1212
SCALA_VERSION: ${{ matrix.scala }}
1313
steps:
@@ -24,7 +24,7 @@ jobs:
2424
uses: actions/setup-java@v3
2525
with:
2626
distribution: "zulu"
27-
java-version: "8"
27+
java-version: "17"
2828
- name: Cache Scala, SBT
2929
uses: actions/cache@v3
3030
with:
@@ -44,13 +44,10 @@ jobs:
4444
sudo apt install libedit-dev
4545
if: steps.git-diff.outputs.diff
4646
- name: Run Delta Spark Local Publishing and Examples Compilation
47-
# examples/scala/build.sbt will compile against the local Delta relase version (e.g. 3.2.0-SNAPSHOT).
47+
# examples/scala/build.sbt will compile against the local Delta release version (e.g. 3.2.0-SNAPSHOT).
4848
# Thus, we need to publishM2 first so those jars are locally accessible.
49-
# We publish storage explicitly so that it is available for the Scala 2.13 build. As a java project
50-
# it is typically only released when publishing for Scala 2.12.
5149
run: |
5250
build/sbt clean
53-
build/sbt storage/publishM2
54-
build/sbt "++ $SCALA_VERSION publishM2"
51+
build/sbt publishM2
5552
cd examples/scala && build/sbt "++ $SCALA_VERSION compile"
5653
if: steps.git-diff.outputs.diff

0 commit comments

Comments
 (0)