Skip to content

Commit 6302a26

Browse files
PaulZhang12PaulZhang12
committed
Migrate release_build to use pytorch test-infra linux_job.yml
1 parent b23fd2c commit 6302a26

File tree

1 file changed

+15
-165
lines changed

1 file changed

+15
-165
lines changed

‎.github/workflows/release_build.yml

Lines changed: 15 additions & 165 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
1+
git# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
22
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
33

44
name: Push Binary Release
@@ -11,211 +11,61 @@ on:
1111
workflow_dispatch:
1212

1313
jobs:
14-
15-
# build on cpu hosts and upload to GHA
16-
build_on_cpu:
14+
build_release:
1715
runs-on: ${{ matrix.os }}
1816
strategy:
1917
matrix:
2018
include:
21-
- os: linux.2xlarge
22-
python-version: 3.8
23-
python-tag: "py38"
24-
cuda-tag: "cu121"
25-
- os: linux.2xlarge
19+
- os: linux.g5.12xlarge.nvidia.gpu
2620
python-version: 3.9
2721
python-tag: "py39"
2822
cuda-tag: "cu121"
29-
- os: linux.2xlarge
23+
- os: linux.g5.12xlarge.nvidia.gpu
3024
python-version: '3.10'
3125
python-tag: "py310"
3226
cuda-tag: "cu121"
33-
- os: linux.2xlarge
27+
- os: linux.g5.12xlarge.nvidia.gpu
3428
python-version: '3.11'
3529
python-tag: "py311"
3630
cuda-tag: "cu121"
37-
- os: linux.2xlarge
31+
- os: linux.g5.12xlarge.nvidia.gpu
3832
python-version: '3.12'
3933
python-tag: "py312"
4034
cuda-tag: "cu121"
41-
steps:
42-
# Checkout the repository to the GitHub Actions runner
43-
- name: Check ldd --version
44-
run: ldd --version
45-
- name: Checkout
46-
uses: actions/checkout@v4
47-
- name: Update pip
48-
run: |
49-
sudo yum update -y
50-
sudo yum -y install git python3-pip
51-
sudo pip3 install --upgrade pip
52-
- name: Setup conda
53-
run: |
54-
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
55-
bash ~/miniconda.sh -b -p $HOME/miniconda -u
56-
- name: setup Path
57-
run: |
58-
echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH
59-
echo "CONDA=/home/ec2-user/miniconda" >> $GITHUB_PATH
60-
- name: create conda env
61-
run: |
62-
conda create --name build_binary python=${{ matrix.python-version }}
35+
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
36+
with:
37+
runner: ${{ matrix.os }}
38+
timeout: 30
39+
script: |
40+
ldd --version
41+
conda create -y --name build_binary python=${{ matrix.python-version }}
6342
conda info
64-
- name: check python version no Conda
65-
run: |
6643
python --version
67-
- name: check python version
68-
run: |
6944
conda run -n build_binary python --version
70-
- name: Install C/C++ compilers
71-
run: |
72-
sudo yum install -y gcc gcc-c++
73-
- name: Install PyTorch and CUDA
74-
shell: bash
75-
run: |
7645
conda run -n build_binary pip install torch --index-url https://download.pytorch.org/whl/test/cu121
77-
- name: Install fbgemm
78-
shell: bash
79-
run: |
8046
conda run -n build_binary pip install numpy
8147
conda run -n build_binary pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/test/cu121
82-
- name: Install Dependencies
83-
shell: bash
84-
run: |
8548
conda run -n build_binary python -m pip install -r requirements.txt
86-
- name: Test Installation of dependencies
87-
run: |
8849
conda run -n build_binary python -c "import torch.distributed"
8950
echo "torch.distributed succeeded"
9051
conda run -n build_binary python -c "import skbuild"
9152
echo "skbuild succeeded"
9253
conda run -n build_binary python -c "import numpy"
9354
echo "numpy succeeded"
94-
# for the conda run with quotes, we have to use "\" and double quotes
95-
# here is the issue: https://github.com/conda/conda/issues/10972
96-
- name: Build TorchRec
97-
env:
98-
OFFICIAL_RELEASE: 1
99-
run: |
10055
rm -r dist || true
10156
conda run -n build_binary \
10257
python setup.py bdist_wheel \
10358
--python-tag=${{ matrix.python-tag }}
104-
- name: Upload wheel as GHA artifact
105-
uses: actions/upload-artifact@v4
106-
with:
107-
name: torchrec_${{ matrix.python-version }}_${{ matrix.cuda-tag }}.whl
108-
path: dist/torchrec-*.whl
109-
110-
# download from GHA, sanity check on gpu and push to pypi
111-
sanity_check_on_gpu_and_push:
112-
runs-on: ${{ matrix.os }}
113-
strategy:
114-
matrix:
115-
os: [linux.4xlarge.nvidia.gpu]
116-
python-version: [3.8, 3.9, "3.10", "3.11", "3.12"]
117-
cuda-tag: ["cu121"]
118-
needs: build_on_cpu
119-
# the glibc version should match the version of the one we used to build the binary
120-
# for this case, it's 2.26
121-
steps:
122-
- name: Check ldd --version
123-
run: ldd --version
124-
- name: check cpu info
125-
shell: bash
126-
run: |
127-
cat /proc/cpuinfo
128-
- name: check distribution info
129-
shell: bash
130-
run: |
131-
cat /proc/version
132-
- name: Display EC2 information
133-
shell: bash
134-
run: |
135-
set -euo pipefail
136-
function get_ec2_metadata() {
137-
# Pulled from instance metadata endpoint for EC2
138-
# see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
139-
category=$1
140-
curl -fsSL "http://169.254.169.254/latest/meta-data/${category}"
141-
}
142-
echo "ami-id: $(get_ec2_metadata ami-id)"
143-
echo "instance-id: $(get_ec2_metadata instance-id)"
144-
echo "instance-type: $(get_ec2_metadata instance-type)"
145-
- name: check gpu info
146-
shell: bash
147-
run: |
148-
sudo yum install lshw -y
149-
sudo lshw -C display
150-
# Checkout the repository to the GitHub Actions runner
151-
- name: Checkout
152-
uses: actions/checkout@v4
153-
- name: Update pip
154-
run: |
155-
sudo yum update -y
156-
sudo yum -y install git python3-pip
157-
sudo pip3 install --upgrade pip
158-
- name: Setup conda
159-
run: |
160-
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
161-
bash ~/miniconda.sh -b -p $HOME/miniconda
162-
- name: setup Path
163-
run: |
164-
echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH
165-
echo "CONDA=/home/ec2-user/miniconda" >> $GITHUB_PATH
166-
- name: create conda env
167-
run: |
168-
conda create --name build_binary python=${{ matrix.python-version }}
169-
conda info
170-
- name: check python version no Conda
171-
run: |
172-
python --version
173-
- name: check python version
174-
run: |
175-
conda run -n build_binary python --version
176-
- name: Install C/C++ compilers
177-
run: |
178-
sudo yum install -y gcc gcc-c++
179-
- name: Install PyTorch and CUDA
180-
shell: bash
181-
run: |
182-
conda run -n build_binary pip install torch --index-url https://download.pytorch.org/whl/test/cu121
183-
# download wheel from GHA
184-
- name: Install fbgemm
185-
shell: bash
186-
run: |
187-
conda run -n build_binary pip install numpy
188-
conda run -n build_binary pip install fbgemm-gpu --index-url https://download.pytorch.org/whl/test/cu121
189-
- name: Install torchmetrics
190-
shell: bash
191-
run: |
192-
conda run -n build_binary pip install torchmetrics==1.0.3
193-
- name: Download wheel
194-
uses: actions/download-artifact@v2
195-
with:
196-
name: torchrec_${{ matrix.python-version }}_${{ matrix.cuda-tag }}.whl
197-
- name: Display structure of downloaded files
198-
run: ls -R
199-
- name: Install TorchRec
200-
run: |
201-
rm -r dist || true
202-
conda run -n build_binary python -m pip install *.whl
203-
- name: Test fbgemm_gpu and torchrec installation
204-
shell: bash
205-
run: |
20659
conda run -n build_binary \
20760
python -c "import fbgemm_gpu"
20861
conda run -n build_binary \
20962
python -c "import torchrec"
210-
# Push to Pypi
211-
- name: Push TorchRec Binary to PYPI
212-
env:
213-
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
214-
run: |
21563
conda run -n build_binary python -m pip install twine
21664
conda run -n build_binary \
21765
python -m twine upload \
21866
--username __token__ \
21967
--password "$PYPI_TOKEN" \
22068
--skip-existing \
22169
torchrec-*.whl
70+
71+

0 commit comments

Comments
 (0)