Skip to content

Commit 90815ff

Browse files
echarlaixecharlaix
authored
Combine documentation from all optimum subpackages (#2353)
* Combine documentation from all optimum subpackages * style * add links * update workflow * fix habana section title * move habana section to make it closer to intel * add redirects
1 parent 86024c3 commit 90815ff

File tree

5 files changed

+66
-172
lines changed

5 files changed

+66
-172
lines changed

‎.github/workflows/build_main_documentation.yml‎

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -32,16 +32,6 @@ jobs:
3232
repository: "huggingface/optimum"
3333
path: optimum
3434

35-
- uses: actions/checkout@v2
36-
with:
37-
repository: "huggingface/optimum-habana"
38-
path: optimum-habana
39-
40-
- uses: actions/checkout@v2
41-
with:
42-
repository: "huggingface/optimum-intel"
43-
path: optimum-intel
44-
4535
- uses: actions/checkout@v2
4636
with:
4737
repository: "huggingface/optimum-amd"
@@ -66,22 +56,6 @@ jobs:
6656
pip install --upgrade pip uv
6757
uv pip install git+https://github.com/huggingface/doc-builder
6858
69-
- name: Make Intel documentation
70-
run: |
71-
cd optimum-intel
72-
uv pip install . nncf openvino neural-compressor[pt]>3.4 diffusers accelerate
73-
make doc BUILD_DIR=intel-doc-build VERSION=${{ env.VERSION }}
74-
sudo mv intel-doc-build ../optimum
75-
cd ..
76-
77-
- name: Make Habana documentation
78-
run: |
79-
sudo docker system prune -a -f
80-
cd optimum-habana
81-
make doc BUILD_DIR=habana-doc-build VERSION=${{ env.VERSION }}
82-
sudo mv habana-doc-build ../optimum
83-
cd ..
84-
8559
- name: Make AMD documentation
8660
run: |
8761
sudo docker system prune -a -f
@@ -100,7 +74,7 @@ jobs:
10074
- name: Combine subpackage documentation
10175
run: |
10276
cd optimum
103-
sudo python docs/combine_docs.py --subpackages nvidia amd intel neuron tpu habana furiosa executorch onnx --version ${{ env.VERSION }}
77+
sudo python docs/combine_docs.py --subpackages nvidia amd intel habana neuron tpu furiosa executorch onnx --version ${{ env.VERSION }}
10478
cd ..
10579
10680
- name: Push to repositories

‎.github/workflows/build_pr_documentation.yml‎

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -43,16 +43,6 @@ jobs:
4343
repository: "huggingface/optimum"
4444
path: optimum
4545

46-
- uses: actions/checkout@v2
47-
with:
48-
repository: "huggingface/optimum-habana"
49-
path: optimum-habana
50-
51-
- uses: actions/checkout@v2
52-
with:
53-
repository: "huggingface/optimum-intel"
54-
path: optimum-intel
55-
5646
- uses: actions/checkout@v2
5747
with:
5848
repository: "huggingface/optimum-amd"
@@ -63,22 +53,6 @@ jobs:
6353
pip install --upgrade pip uv
6454
uv pip install git+https://github.com/huggingface/doc-builder
6555
66-
- name: Make Intel documentation
67-
run: |
68-
cd optimum-intel
69-
uv pip install . nncf openvino neural-compressor[pt]>3.4 diffusers accelerate
70-
make doc BUILD_DIR=intel-doc-build VERSION=pr_$PR_NUMBER
71-
sudo mv intel-doc-build ../optimum
72-
cd ..
73-
74-
- name: Make Habana documentation
75-
run: |
76-
sudo docker system prune -a -f
77-
cd optimum-habana
78-
make doc BUILD_DIR=habana-doc-build VERSION=pr_$PR_NUMBER
79-
sudo mv habana-doc-build ../optimum
80-
cd ..
81-
8256
- name: Make AMD documentation
8357
run: |
8458
sudo docker system prune -a -f
@@ -97,7 +71,7 @@ jobs:
9771
- name: Combine subpackage documentation
9872
run: |
9973
cd optimum
100-
sudo python docs/combine_docs.py --subpackages nvidia amd intel neuron tpu habana furiosa executorch onnx --version pr_$PR_NUMBER
74+
sudo python docs/combine_docs.py --subpackages nvidia amd intel habana neuron tpu furiosa executorch onnx --version pr_$PR_NUMBER
10175
sudo mv optimum-doc-build ../
10276
cd ..
10377

‎docs/combine_docs.py‎

100755100644
Lines changed: 24 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -83,125 +83,43 @@ def rename_copy_subpackage_html_paths(subpackage: str, subpackage_path: Path, op
8383
shutil.copyfile(html_path, new_path_in_optimum)
8484

8585

86-
def add_neuron_doc(base_toc: List):
86+
# for docs created separately that needs to be combined
87+
# TODO: add AMD to EXTERNAL_DOCUMENTATION once added in optimum-amd
88+
EXTERNAL_DOCUMENTATION = {
89+
"executorch": "ExecuTorch",
90+
"habana": "Habana",
91+
"intel": "Intel",
92+
"neuron": "Neuron",
93+
"onnx": "ONNX",
94+
"tpu": "TPU",
95+
}
96+
# needed for cases where the section title differs from the subpackage name
97+
TITLE_SECTION = {"Habana": "For Intel Gaudi", "Neuron": "AWS Trainium/Inferentia", "TPU": "Google TPUs"}
98+
99+
100+
def add_subpackage_doc(base_toc: List, subpackage: str):
87101
"""
88-
Extends the table of content with a section about Optimum Neuron.
89-
90-
Args:
91-
base_toc (List): table of content for the doc of Optimum.
92-
"""
93-
# Update optimum table of contents
94-
base_toc.insert(
95-
SUBPACKAGE_TOC_INSERT_INDEX,
96-
{
97-
"sections": [
98-
{
99-
# Ideally this should directly point at https://huggingface.co/docs/optimum-neuron/index
100-
# Current hacky solution is to have a redirection in _redirects.yml
101-
"local": "docs/optimum-neuron/index",
102-
"title": "🤗 Optimum Neuron",
103-
}
104-
],
105-
"title": "AWS Trainium/Inferentia",
106-
"isExpanded": False,
107-
},
108-
)
109-
110-
111-
def add_tpu_doc(base_toc: List):
112-
"""
113-
Extends the table of content with a section about Optimum TPU.
114-
115-
Args:
116-
base_toc (List): table of content for the doc of Optimum.
117-
"""
118-
# Update optimum table of contents
119-
base_toc.insert(
120-
SUBPACKAGE_TOC_INSERT_INDEX,
121-
{
122-
"sections": [
123-
{
124-
# Ideally this should directly point at https://huggingface.co/docs/optimum-tpu/index
125-
# Current hacky solution is to have a redirection in _redirects.yml
126-
"local": "docs/optimum-tpu/index",
127-
"title": "🤗 Optimum-TPU",
128-
}
129-
],
130-
"title": "Google TPUs",
131-
"isExpanded": False,
132-
},
133-
)
134-
135-
136-
def add_executorch_doc(base_toc: List):
137-
"""
138-
Extends the table of content with a section about Optimum ExecuTorch.
139-
140-
Args:
141-
base_toc (List): table of content for the doc of Optimum.
142-
"""
143-
# Update optimum table of contents
144-
base_toc.insert(
145-
SUBPACKAGE_TOC_INSERT_INDEX,
146-
{
147-
"sections": [
148-
{
149-
# Ideally this should directly point at https://huggingface.co/docs/optimum-executorch/index
150-
# Current hacky solution is to have a redirection in _redirects.yml
151-
"local": "docs/optimum-executorch/index",
152-
"title": "🤗 Optimum ExecuTorch",
153-
}
154-
],
155-
"title": "ExecuTorch",
156-
"isExpanded": False,
157-
},
158-
)
159-
160-
161-
def add_furiosa_doc(base_toc: List):
162-
"""
163-
Extends the table of content with a section about Optimum Furiosa.
102+
Extends the table of content with a section generated by the original optimum-xxx.
164103
165104
Args:
166105
base_toc (List): table of content for the doc of Optimum.
106+
subpackage (str): subpackage name
167107
"""
168-
# Update optimum table of contents
169-
base_toc.insert(
170-
SUBPACKAGE_TOC_INSERT_INDEX,
171-
{
172-
"sections": [
173-
{
174-
# Ideally this should directly point at https://huggingface.co/docs/optimum-furiosa/index
175-
# Current hacky solution is to have a redirection in _redirects.yml
176-
"local": "docs/optimum-furiosa/index",
177-
"title": "🤗 Optimum Furiosa",
178-
}
179-
],
180-
"title": "Furiosa",
181-
"isExpanded": False,
182-
},
183-
)
184108

185-
186-
def add_onnx_doc(base_toc: List):
187-
"""
188-
Extends the table of content with a section about Optimum ONNX.
189-
Args:
190-
base_toc (List): table of content for the doc of Optimum.
191-
"""
109+
section_title = EXTERNAL_DOCUMENTATION[subpackage]
192110
# Update optimum table of contents
193111
base_toc.insert(
194112
SUBPACKAGE_TOC_INSERT_INDEX,
195113
{
196114
"sections": [
197115
{
198-
# Ideally this should directly point at https://huggingface.co/docs/optimum-onnx/index
116+
# Ideally this should directly point at https://huggingface.co/docs/optimum-xxx/index
199117
# Current hacky solution is to have a redirection in _redirects.yml
200-
"local": "docs/optimum-onnx/index",
201-
"title": "🤗 Optimum ONNX",
118+
"local": f"docs/optimum-{subpackage}/index",
119+
"title": f"🤗 Optimum {section_title}",
202120
}
203121
],
204-
"title": "ONNX",
122+
"title": TITLE_SECTION.get(section_title, section_title),
205123
"isExpanded": False,
206124
},
207125
)
@@ -214,24 +132,14 @@ def main():
214132
base_toc_path = next(optimum_path.rglob("_toctree.yml"))
215133
with open(base_toc_path, "r") as f:
216134
base_toc = yaml.safe_load(f)
217-
218135
# Copy and rename all files from subpackages' docs to Optimum doc
219136
for subpackage in args.subpackages[::-1]:
220-
if subpackage == "neuron":
221-
# Neuron has its own doc so it is managed differently
222-
add_neuron_doc(base_toc)
223-
elif subpackage == "tpu":
224-
# Optimum TPU has its own doc so it is managed differently
225-
add_tpu_doc(base_toc)
137+
if subpackage in EXTERNAL_DOCUMENTATION:
138+
add_subpackage_doc(base_toc, subpackage)
226139
elif subpackage == "nvidia":
227140
# At the moment, Optimum Nvidia's doc is the README of the GitHub repo
228141
# It is linked to in optimum/docs/source/nvidia_overview.mdx
229142
continue
230-
elif subpackage == "executorch":
231-
# Optimum ExecuTorch has its own doc so it is managed differently
232-
add_executorch_doc(base_toc)
233-
elif subpackage == "onnx":
234-
add_onnx_doc(base_toc)
235143
elif subpackage == "furiosa":
236144
# TODO: add furiosa doc when available
237145
# add_furiosa_doc(base_toc)

‎docs/source/_redirects.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,28 @@ habana_trainer: habana/package_reference/trainer
1616
habana_gaudi_config: habana/package_reference/gaudi_config
1717
habana/usage_guides/stable_diffusion: habana/tutorials/stable_diffusion
1818
habana/tutorials/pretraining: habana/usage_guides/pretraining
19+
docs/optimum-habana/index: /docs/optimum-habana/index
20+
habana/index: /docs/optimum-habana/index
21+
habana/installation: /docs/optimum-habana/installation
22+
habana/quickstart: /docs/optimum-habana/quickstart
23+
habana/tutorials/overview: /docs/optimum-habana/tutorials/overview
24+
habana/tutorials/single_hpu: /docs/optimum-habana/tutorials/single_hpu
25+
habana/tutorials/distributed: /docs/optimum-habana/tutorials/distributed
26+
habana/tutorials/inference: /docs/optimum-habana/tutorials/inference
27+
habana/tutorials/stable_diffusion: /docs/optimum-habana/tutorials/stable_diffusion
28+
habana/tutorials/tgi: /docs/optimum-habana/tutorials/tgi
29+
habana/usage_guides/overview: /docs/optimum-habana/usage_guides/overview
30+
habana/usage_guides/script_adaptation: /docs/optimum-habana/usage_guides/script_adaptation
31+
habana/usage_guides/pretraining: /docs/optimum-habana/usage_guides/pretraining
32+
habana/usage_guides/accelerate_training: /docs/optimum-habana/usage_guides/accelerate_training
33+
habana/usage_guides/accelerate_inference: /docs/optimum-habana/usage_guides/accelerate_inference
34+
habana/usage_guides/deepspeed: /docs/optimum-habana/usage_guides/deepspeed
35+
habana/usage_guides/multi_node_training: /docs/optimum-habana/usage_guides/multi_node_training
36+
habana/usage_guides/quantization: /docs/optimum-habana/usage_guides/quantization
37+
habana/usage_guides/safe_softmax: /docs/optimum-habana/usage_guides/safe_softmax
38+
habana/package_reference/trainer: /docs/optimum-habana/package_reference/trainer
39+
habana/package_reference/gaudi_config: /docs/optimum-habana/package_reference/gaudi_config
40+
habana/package_reference/distributed_runner: /docs/optimum-habana/package_reference/distributed_runner
1941

2042
# Optimum Intel
2143
intel_index: intel/index
@@ -31,6 +53,22 @@ intel/reference_ov: intel/openvino/reference
3153
intel/optimization_inc: intel/neural_compressor/optimization
3254
intel/distributed_training: intel/neural_compressor/distributed_training
3355
intel/reference_inc: intel/neural_compressor/reference
56+
docs/optimum-intel/index: /docs/optimum-intel/index
57+
intel/index: /docs/optimum-intel/index
58+
intel/installation: /docs/optimum-intel/installation
59+
intel/neural_compressor/optimization: /docs/optimum-intel/neural_compressor/optimization
60+
intel/neural_compressor/distributed_training: /docs/optimum-intel/neural_compressor/distributed_training
61+
intel/neural_compressor/reference: /docs/optimum-intel/neural_compressor/reference
62+
intel/openvino/export: /docs/optimum-intel/openvino/export
63+
intel/openvino/inference: /docs/optimum-intel/openvino/inference
64+
intel/openvino/optimization: /docs/optimum-intel/openvino/optimization
65+
intel/openvino/models: /docs/optimum-intel/openvino/models
66+
intel/openvino/reference: /docs/optimum-intel/openvino/reference
67+
intel/openvino/tutorials/notebooks: /docs/optimum-intel/openvino/tutorials/notebooks
68+
intel/openvino/tutorials/diffusers: /docs/optimum-intel/openvino/tutorials/diffusers
69+
intel/ipex/inference: /docs/optimum-intel/ipex/inference
70+
intel/ipex/models: /docs/optimum-intel/ipex/models
71+
intel/ipex/tutorials/notebooks: /docs/optimum-intel/ipex/tutorials/notebooks
3472

3573
# Optimum Neuron
3674
docs/optimum-neuron/index: /docs/optimum-neuron/index

‎docs/source/index.mdx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ The packages below enable you to get the best of the 🤗 Hugging Face ecosystem
3434
><div class="w-full text-center bg-gradient-to-br from-red-600 to-red-600 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">AMD</div>
3535
<p class="text-gray-700">Enable performance optimizations for <span class="underline" onclick="event.preventDefault(); window.open('https://www.amd.com/en/graphics/instinct-server-accelerators', '_blank');">AMD Instinct GPUs</span> and <span class="underline" onclick="event.preventDefault(); window.open('https://ryzenai.docs.amd.com/en/latest/index.html', '_blank');">AMD Ryzen AI NPUs</span></p>
3636
</a>
37-
<a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./intel/index"
37+
<a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="https://huggingface.co/docs/optimum-intel/index"
3838
><div class="w-full text-center bg-gradient-to-br from-blue-400 to-blue-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Intel</div>
3939
<p class="text-gray-700">Optimize your model to speedup inference with <span class="underline" onclick="event.preventDefault(); window.open('https://docs.openvino.ai/latest/index.html', '_blank');">OpenVINO</span> , <span class="underline" onclick="event.preventDefault(); window.open('https://www.intel.com/content/www/us/en/developer/tools/oneapi/neural-compressor.html', '_blank');">Neural Compressor</span> and <span class="underline" onclick="event.preventDefault(); window.open('https://intel.github.io/intel-extension-for-pytorch/index.html', '_blank');">IPEX</span></p>
4040
</a>
@@ -46,7 +46,7 @@ The packages below enable you to get the best of the 🤗 Hugging Face ecosystem
4646
><div class="w-full text-center bg-gradient-to-br from-blue-500 to-blue-600 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Google TPUs</div>
4747
<p class="text-gray-700">Accelerate your training and inference workflows with <span class="underline" onclick="event.preventDefault(); window.open('https://cloud.google.com/tpu', '_blank');">Google TPUs</span></p>
4848
</a>
49-
<a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./habana/index"
49+
<a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="https://huggingface.co/docs/optimum-habana/index"
5050
><div class="w-full text-center bg-gradient-to-br from-indigo-400 to-indigo-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Habana</div>
5151
<p class="text-gray-700">Maximize training throughput and efficiency with <span class="underline" onclick="event.preventDefault(); window.open('https://docs.habana.ai/en/latest/Gaudi_Overview/Gaudi_Architecture.html', '_blank');">Habana's Gaudi processor</span></p>
5252
</a>

0 commit comments

Comments
 (0)