Skip to content

Commit d6857ed

Browse files
PaulZhang12PaulZhang12
committed
remove upgrade pip
1 parent af577e7 commit d6857ed

File tree

1 file changed

+176
-16
lines changed

1 file changed

+176
-16
lines changed

‎.github/workflows/release_build.yml

Lines changed: 176 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -11,49 +11,209 @@ on:
1111
workflow_dispatch:
1212

1313
jobs:
14-
build_release:
14+
15+
# build on cpu hosts and upload to GHA
16+
build_on_cpu:
1517
runs-on: ${{ matrix.os }}
1618
strategy:
1719
matrix:
18-
os: [linux.4xlarge.nvidia.gpu]
19-
python-version: [3.9, "3.10", "3.11", "3.12"]
20-
cuda-tag: ["cu124"]
20+
include:
21+
- os: linux.2xlarge
22+
python-version: 3.8
23+
python-tag: "py38"
24+
cuda-tag: "cu121"
25+
- os: linux.2xlarge
26+
python-version: 3.9
27+
python-tag: "py39"
28+
cuda-tag: "cu121"
29+
- os: linux.2xlarge
30+
python-version: '3.10'
31+
python-tag: "py310"
32+
cuda-tag: "cu121"
33+
- os: linux.2xlarge
34+
python-version: '3.11'
35+
python-tag: "py311"
36+
cuda-tag: "cu121"
37+
- os: linux.2xlarge
38+
python-version: '3.12'
39+
python-tag: "py312"
40+
cuda-tag: "cu121"
2141
steps:
22-
- name: Set up Environment
23-
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
24-
- name: Push to PyPI
25-
env:
26-
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
27-
OFFICIAL_RELEASE: 1
42+
# Checkout the repository to the GitHub Actions runner
43+
- name: Check ldd --version
44+
run: ldd --version
45+
- name: Checkout
46+
uses: actions/checkout@v4
47+
- name: Update pip
48+
run: |
49+
sudo yum update -y
50+
sudo yum -y install git python3-pip
51+
- name: Setup conda
52+
run: |
53+
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
54+
bash ~/miniconda.sh -b -p $HOME/miniconda -u
55+
- name: setup Path
2856
run: |
29-
ldd --version
30-
conda create -y --name build_binary python=${{ matrix.python-version }}
57+
echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH
58+
echo "CONDA=/home/ec2-user/miniconda" >> $GITHUB_PATH
59+
- name: create conda env
60+
run: |
61+
conda create --name build_binary python=${{ matrix.python-version }}
3162
conda info
63+
- name: check python version no Conda
64+
run: |
3265
python --version
66+
- name: check python version
67+
run: |
3368
conda run -n build_binary python --version
34-
conda run -n build_binary pip install torch
69+
- name: Install C/C++ compilers
70+
run: |
71+
sudo yum install -y gcc gcc-c++
72+
- name: Install PyTorch and CUDA
73+
shell: bash
74+
run: |
75+
conda run -n build_binary pip install torch --index-url https://download.pytorch.org/whl/test/cu121
76+
- name: Install fbgemm
77+
shell: bash
78+
run: |
3579
conda run -n build_binary pip install numpy
36-
conda run -n build_binary pip install fbgemm-gpu
80+
conda run -n build_binary pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/test/cu121
81+
- name: Install Dependencies
82+
shell: bash
83+
run: |
3784
conda run -n build_binary python -m pip install -r requirements.txt
85+
- name: Test Installation of dependencies
86+
run: |
3887
conda run -n build_binary python -c "import torch.distributed"
3988
echo "torch.distributed succeeded"
4089
conda run -n build_binary python -c "import skbuild"
4190
echo "skbuild succeeded"
4291
conda run -n build_binary python -c "import numpy"
4392
echo "numpy succeeded"
93+
# for the conda run with quotes, we have to use "\" and double quotes
94+
# here is the issue: https://github.com/conda/conda/issues/10972
95+
- name: Build TorchRec
96+
env:
97+
OFFICIAL_RELEASE: 1
98+
run: |
4499
rm -r dist || true
45100
conda run -n build_binary \
46101
python setup.py bdist_wheel \
47102
--python-tag=${{ matrix.python-tag }}
103+
- name: Upload wheel as GHA artifact
104+
uses: actions/upload-artifact@v4
105+
with:
106+
name: torchrec_${{ matrix.python-version }}_${{ matrix.cuda-tag }}.whl
107+
path: dist/torchrec-*.whl
108+
109+
# download from GHA, sanity check on gpu and push to pypi
110+
sanity_check_on_gpu_and_push:
111+
runs-on: ${{ matrix.os }}
112+
strategy:
113+
matrix:
114+
os: [linux.4xlarge.nvidia.gpu]
115+
python-version: [3.8, 3.9, "3.10", "3.11", "3.12"]
116+
cuda-tag: ["cu121"]
117+
needs: build_on_cpu
118+
# the glibc version should match the version of the one we used to build the binary
119+
# for this case, it's 2.26
120+
steps:
121+
- name: Check ldd --version
122+
run: ldd --version
123+
- name: check cpu info
124+
shell: bash
125+
run: |
126+
cat /proc/cpuinfo
127+
- name: check distribution info
128+
shell: bash
129+
run: |
130+
cat /proc/version
131+
- name: Display EC2 information
132+
shell: bash
133+
run: |
134+
set -euo pipefail
135+
function get_ec2_metadata() {
136+
# Pulled from instance metadata endpoint for EC2
137+
# see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
138+
category=$1
139+
curl -fsSL "http://169.254.169.254/latest/meta-data/${category}"
140+
}
141+
echo "ami-id: $(get_ec2_metadata ami-id)"
142+
echo "instance-id: $(get_ec2_metadata instance-id)"
143+
echo "instance-type: $(get_ec2_metadata instance-type)"
144+
- name: check gpu info
145+
shell: bash
146+
run: |
147+
sudo yum install lshw -y
148+
sudo lshw -C display
149+
# Checkout the repository to the GitHub Actions runner
150+
- name: Checkout
151+
uses: actions/checkout@v4
152+
- name: Update pip
153+
run: |
154+
sudo yum update -y
155+
sudo yum -y install git python3-pip
156+
- name: Setup conda
157+
run: |
158+
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
159+
bash ~/miniconda.sh -b -p $HOME/miniconda
160+
- name: setup Path
161+
run: |
162+
echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH
163+
echo "CONDA=/home/ec2-user/miniconda" >> $GITHUB_PATH
164+
- name: create conda env
165+
run: |
166+
conda create --name build_binary python=${{ matrix.python-version }}
167+
conda info
168+
- name: check python version no Conda
169+
run: |
170+
python --version
171+
- name: check python version
172+
run: |
173+
conda run -n build_binary python --version
174+
- name: Install C/C++ compilers
175+
run: |
176+
sudo yum install -y gcc gcc-c++
177+
- name: Install PyTorch and CUDA
178+
shell: bash
179+
run: |
180+
conda run -n build_binary pip install torch --index-url https://download.pytorch.org/whl/test/cu121
181+
# download wheel from GHA
182+
- name: Install fbgemm
183+
shell: bash
184+
run: |
185+
conda run -n build_binary pip install numpy
186+
conda run -n build_binary pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/test/cu121
187+
- name: Install torchmetrics
188+
shell: bash
189+
run: |
190+
conda run -n build_binary pip install torchmetrics==1.0.3
191+
- name: Download wheel
192+
uses: actions/download-artifact@v4
193+
with:
194+
name: torchrec_${{ matrix.python-version }}_${{ matrix.cuda-tag }}.whl
195+
- name: Display structure of downloaded files
196+
run: ls -R
197+
- name: Install TorchRec
198+
run: |
199+
rm -r dist || true
200+
conda run -n build_binary python -m pip install *.whl
201+
- name: Test fbgemm_gpu and torchrec installation
202+
shell: bash
203+
run: |
48204
conda run -n build_binary \
49205
python -c "import fbgemm_gpu"
50206
conda run -n build_binary \
51207
python -c "import torchrec"
208+
# Push to Pypi
209+
- name: Push TorchRec Binary to PYPI
210+
env:
211+
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
212+
run: |
52213
conda run -n build_binary python -m pip install twine
53214
conda run -n build_binary \
54215
python -m twine upload \
55216
--username __token__ \
56217
--password "$PYPI_TOKEN" \
57218
--skip-existing \
58-
dist/torchrec-*.whl
59-
219+
torchrec-*.whl

0 commit comments

Comments
 (0)