Skip to content

Commit a519272

Browse files
sayakpaulsayakpaul
authored
[ci] revisit the installations in CI. (huggingface#12450)
* revisit the installations in CI. * up * up * up * empty * up * up * up
1 parent 345864e commit a519272

File tree

20 files changed

+148
-204
lines changed

20 files changed

+148
-204
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,8 @@ jobs:
3838
run: |
3939
apt update
4040
apt install -y libpq-dev postgresql-client
41-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
42-
python -m uv pip install -e [quality,test]
43-
python -m uv pip install -r benchmarks/requirements.txt
41+
uv pip install -e ".[quality]"
42+
uv pip install -r benchmarks/requirements.txt
4443
- name: Environment
4544
run: |
4645
python utils/print_env.py

‎.github/workflows/mirror_community_pipeline.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ jobs:
7474
python-version: "3.10"
7575
- name: Install dependencies
7676
run: |
77-
python -m pip install --upgrade pip
77+
pip install --upgrade pip
7878
pip install --upgrade huggingface_hub
7979
8080
# Check secret is set

‎.github/workflows/nightly_tests.yml‎

Lines changed: 42 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -71,10 +71,9 @@ jobs:
7171
run: nvidia-smi
7272
- name: Install dependencies
7373
run: |
74-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
75-
python -m uv pip install -e [quality,test]
76-
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
77-
python -m uv pip install pytest-reportlog
74+
uv pip install -e ".[quality]"
75+
uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
76+
uv pip install pytest-reportlog
7877
- name: Environment
7978
run: |
8079
python utils/print_env.py
@@ -84,7 +83,7 @@ jobs:
8483
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
8584
CUBLAS_WORKSPACE_CONFIG: :16:8
8685
run: |
87-
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
86+
pytest -n 1 --max-worker-restart=0 --dist=loadfile \
8887
-s -v -k "not Flax and not Onnx" \
8988
--make-reports=tests_pipeline_${{ matrix.module }}_cuda \
9089
--report-log=tests_pipeline_${{ matrix.module }}_cuda.log \
@@ -124,11 +123,10 @@ jobs:
124123

125124
- name: Install dependencies
126125
run: |
127-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
128-
python -m uv pip install -e [quality,test]
129-
python -m uv pip install peft@git+https://github.com/huggingface/peft.git
130-
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
131-
python -m uv pip install pytest-reportlog
126+
uv pip install -e ".[quality]"
127+
uv pip install peft@git+https://github.com/huggingface/peft.git
128+
uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
129+
uv pip install pytest-reportlog
132130
- name: Environment
133131
run: python utils/print_env.py
134132

@@ -139,7 +137,7 @@ jobs:
139137
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
140138
CUBLAS_WORKSPACE_CONFIG: :16:8
141139
run: |
142-
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
140+
pytest -n 1 --max-worker-restart=0 --dist=loadfile \
143141
-s -v -k "not Flax and not Onnx" \
144142
--make-reports=tests_torch_${{ matrix.module }}_cuda \
145143
--report-log=tests_torch_${{ matrix.module }}_cuda.log \
@@ -152,7 +150,7 @@ jobs:
152150
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
153151
CUBLAS_WORKSPACE_CONFIG: :16:8
154152
run: |
155-
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
153+
pytest -n 1 --max-worker-restart=0 --dist=loadfile \
156154
-s -v --make-reports=examples_torch_cuda \
157155
--report-log=examples_torch_cuda.log \
158156
examples/
@@ -191,8 +189,7 @@ jobs:
191189
nvidia-smi
192190
- name: Install dependencies
193191
run: |
194-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
195-
python -m uv pip install -e [quality,test,training]
192+
uv pip install -e ".[quality,training]"
196193
- name: Environment
197194
run: |
198195
python utils/print_env.py
@@ -201,7 +198,7 @@ jobs:
201198
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
202199
RUN_COMPILE: yes
203200
run: |
204-
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/
201+
pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v -k "compile" --make-reports=tests_torch_compile_cuda tests/
205202
- name: Failure short reports
206203
if: ${{ failure() }}
207204
run: cat reports/tests_torch_compile_cuda_failures_short.txt
@@ -232,11 +229,10 @@ jobs:
232229
run: nvidia-smi
233230
- name: Install dependencies
234231
run: |
235-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
236-
python -m uv pip install -e [quality,test]
237-
python -m uv pip install peft@git+https://github.com/huggingface/peft.git
238-
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
239-
python -m uv pip install pytest-reportlog
232+
uv pip install -e ".[quality]"
233+
uv pip install peft@git+https://github.com/huggingface/peft.git
234+
uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
235+
uv pip install pytest-reportlog
240236
- name: Environment
241237
run: |
242238
python utils/print_env.py
@@ -247,7 +243,7 @@ jobs:
247243
CUBLAS_WORKSPACE_CONFIG: :16:8
248244
BIG_GPU_MEMORY: 40
249245
run: |
250-
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
246+
pytest -n 1 --max-worker-restart=0 --dist=loadfile \
251247
-m "big_accelerator" \
252248
--make-reports=tests_big_gpu_torch_cuda \
253249
--report-log=tests_big_gpu_torch_cuda.log \
@@ -282,10 +278,9 @@ jobs:
282278

283279
- name: Install dependencies
284280
run: |
285-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
286-
python -m uv pip install -e [quality,test]
287-
python -m uv pip install peft@git+https://github.com/huggingface/peft.git
288-
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
281+
uv pip install -e ".[quality]"
282+
uv pip install peft@git+https://github.com/huggingface/peft.git
283+
uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
289284
290285
- name: Environment
291286
run: |
@@ -297,7 +292,7 @@ jobs:
297292
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
298293
CUBLAS_WORKSPACE_CONFIG: :16:8
299294
run: |
300-
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
295+
pytest -n 1 --max-worker-restart=0 --dist=loadfile \
301296
-s -v -k "not Flax and not Onnx" \
302297
--make-reports=tests_torch_minimum_version_cuda \
303298
tests/models/test_modeling_common.py \
@@ -357,13 +352,12 @@ jobs:
357352
run: nvidia-smi
358353
- name: Install dependencies
359354
run: |
360-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
361-
python -m uv pip install -e [quality,test]
362-
python -m uv pip install -U ${{ matrix.config.backend }}
355+
uv pip install -e ".[quality]"
356+
uv pip install -U ${{ matrix.config.backend }}
363357
if [ "${{ join(matrix.config.additional_deps, ' ') }}" != "" ]; then
364-
python -m uv pip install ${{ join(matrix.config.additional_deps, ' ') }}
358+
uv pip install ${{ join(matrix.config.additional_deps, ' ') }}
365359
fi
366-
python -m uv pip install pytest-reportlog
360+
uv pip install pytest-reportlog
367361
- name: Environment
368362
run: |
369363
python utils/print_env.py
@@ -374,7 +368,7 @@ jobs:
374368
CUBLAS_WORKSPACE_CONFIG: :16:8
375369
BIG_GPU_MEMORY: 40
376370
run: |
377-
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
371+
pytest -n 1 --max-worker-restart=0 --dist=loadfile \
378372
--make-reports=tests_${{ matrix.config.backend }}_torch_cuda \
379373
--report-log=tests_${{ matrix.config.backend }}_torch_cuda.log \
380374
tests/quantization/${{ matrix.config.test_location }}
@@ -409,10 +403,9 @@ jobs:
409403
run: nvidia-smi
410404
- name: Install dependencies
411405
run: |
412-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
413-
python -m uv pip install -e [quality,test]
414-
python -m uv pip install -U bitsandbytes optimum_quanto
415-
python -m uv pip install pytest-reportlog
406+
uv pip install -e ".[quality]"
407+
uv pip install -U bitsandbytes optimum_quanto
408+
uv pip install pytest-reportlog
416409
- name: Environment
417410
run: |
418411
python utils/print_env.py
@@ -423,7 +416,7 @@ jobs:
423416
CUBLAS_WORKSPACE_CONFIG: :16:8
424417
BIG_GPU_MEMORY: 40
425418
run: |
426-
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
419+
pytest -n 1 --max-worker-restart=0 --dist=loadfile \
427420
--make-reports=tests_pipeline_level_quant_torch_cuda \
428421
--report-log=tests_pipeline_level_quant_torch_cuda.log \
429422
tests/quantization/test_pipeline_level_quantization.py
@@ -523,11 +516,11 @@ jobs:
523516
# - name: Install dependencies
524517
# shell: arch -arch arm64 bash {0}
525518
# run: |
526-
# ${CONDA_RUN} python -m pip install --upgrade pip uv
527-
# ${CONDA_RUN} python -m uv pip install -e [quality,test]
528-
# ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
529-
# ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate
530-
# ${CONDA_RUN} python -m uv pip install pytest-reportlog
519+
# ${CONDA_RUN} pip install --upgrade pip uv
520+
# ${CONDA_RUN} uv pip install -e ".[quality]"
521+
# ${CONDA_RUN} uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
522+
# ${CONDA_RUN} uv pip install accelerate@git+https://github.com/huggingface/accelerate
523+
# ${CONDA_RUN} uv pip install pytest-reportlog
531524
# - name: Environment
532525
# shell: arch -arch arm64 bash {0}
533526
# run: |
@@ -538,7 +531,7 @@ jobs:
538531
# HF_HOME: /System/Volumes/Data/mnt/cache
539532
# HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
540533
# run: |
541-
# ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps \
534+
# ${CONDA_RUN} pytest -n 1 -s -v --make-reports=tests_torch_mps \
542535
# --report-log=tests_torch_mps.log \
543536
# tests/
544537
# - name: Failure short reports
@@ -579,11 +572,11 @@ jobs:
579572
# - name: Install dependencies
580573
# shell: arch -arch arm64 bash {0}
581574
# run: |
582-
# ${CONDA_RUN} python -m pip install --upgrade pip uv
583-
# ${CONDA_RUN} python -m uv pip install -e [quality,test]
584-
# ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
585-
# ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate
586-
# ${CONDA_RUN} python -m uv pip install pytest-reportlog
575+
# ${CONDA_RUN} pip install --upgrade pip uv
576+
# ${CONDA_RUN} uv pip install -e ".[quality]"
577+
# ${CONDA_RUN} uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
578+
# ${CONDA_RUN} uv pip install accelerate@git+https://github.com/huggingface/accelerate
579+
# ${CONDA_RUN} uv pip install pytest-reportlog
587580
# - name: Environment
588581
# shell: arch -arch arm64 bash {0}
589582
# run: |
@@ -594,7 +587,7 @@ jobs:
594587
# HF_HOME: /System/Volumes/Data/mnt/cache
595588
# HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
596589
# run: |
597-
# ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps \
590+
# ${CONDA_RUN} pytest -n 1 -s -v --make-reports=tests_torch_mps \
598591
# --report-log=tests_torch_mps.log \
599592
# tests/
600593
# - name: Failure short reports

‎.github/workflows/pr_dependency_test.yml‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,8 @@ jobs:
2525
python-version: "3.8"
2626
- name: Install dependencies
2727
run: |
28-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
29-
python -m pip install --upgrade pip uv
30-
python -m uv pip install -e .
31-
python -m uv pip install pytest
28+
pip install -e .
29+
pip install pytest
3230
- name: Check for soft dependencies
3331
run: |
34-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
35-
pytest tests/others/test_dependencies.py
32+
pytest tests/others/test_dependencies.py

‎.github/workflows/pr_modular_tests.yml‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ jobs:
4242
python-version: "3.10"
4343
- name: Install dependencies
4444
run: |
45-
python -m pip install --upgrade pip
45+
pip install --upgrade pip
4646
pip install .[quality]
4747
- name: Check quality
4848
run: make quality
@@ -62,7 +62,7 @@ jobs:
6262
python-version: "3.10"
6363
- name: Install dependencies
6464
run: |
65-
python -m pip install --upgrade pip
65+
pip install --upgrade pip
6666
pip install .[quality]
6767
- name: Check repo consistency
6868
run: |
@@ -108,21 +108,18 @@ jobs:
108108

109109
- name: Install dependencies
110110
run: |
111-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
112-
python -m uv pip install -e [quality,test]
113-
pip uninstall transformers -y && pip uninstall huggingface_hub -y && python -m uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
114-
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
111+
uv pip install -e ".[quality]"
112+
uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git
113+
uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
115114
116115
- name: Environment
117116
run: |
118-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
119117
python utils/print_env.py
120118
121119
- name: Run fast PyTorch Pipeline CPU tests
122120
if: ${{ matrix.config.framework == 'pytorch_pipelines' }}
123121
run: |
124-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
125-
python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile \
122+
pytest -n 8 --max-worker-restart=0 --dist=loadfile \
126123
-s -v -k "not Flax and not Onnx" \
127124
--make-reports=tests_${{ matrix.config.report }} \
128125
tests/modular_pipelines

‎.github/workflows/pr_test_fetcher.yml‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,7 @@ jobs:
3333
fetch-depth: 0
3434
- name: Install dependencies
3535
run: |
36-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
37-
python -m uv pip install -e [quality,test]
36+
uv pip install -e ".[quality]"
3837
- name: Environment
3938
run: |
4039
python utils/print_env.py
@@ -90,19 +89,16 @@ jobs:
9089

9190
- name: Install dependencies
9291
run: |
93-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
94-
python -m pip install -e [quality,test]
95-
python -m pip install accelerate
92+
uv pip install -e ".[quality]"
93+
uv pip install accelerate
9694
9795
- name: Environment
9896
run: |
99-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
10097
python utils/print_env.py
10198
10299
- name: Run all selected tests on CPU
103100
run: |
104-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
105-
python -m pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.modules }}_tests_cpu ${{ fromJson(needs.setup_pr_tests.outputs.test_map)[matrix.modules] }}
101+
pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.modules }}_tests_cpu ${{ fromJson(needs.setup_pr_tests.outputs.test_map)[matrix.modules] }}
106102
107103
- name: Failure short reports
108104
if: ${{ failure() }}
@@ -148,19 +144,16 @@ jobs:
148144

149145
- name: Install dependencies
150146
run: |
151-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
152-
python -m pip install -e [quality,test]
147+
pip install -e [quality]
153148
154149
- name: Environment
155150
run: |
156-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
157151
python utils/print_env.py
158152
159153
- name: Run Hub tests for models, schedulers, and pipelines on a staging env
160154
if: ${{ matrix.config.framework == 'hub_tests_pytorch' }}
161155
run: |
162-
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
163-
HUGGINGFACE_CO_STAGING=true python -m pytest \
156+
HUGGINGFACE_CO_STAGING=true pytest \
164157
-m "is_staging_test" \
165158
--make-reports=tests_${{ matrix.config.report }} \
166159
tests

0 commit comments

Comments
 (0)