Skip to content

Commit 7c5a729

Browse files
authored
🌱 Restart OCI download if too slow. Timeout after ten minutes. (#1619)
1 parent 7a44600 commit 7c5a729

File tree

2 files changed

+138
-104
lines changed

2 files changed

+138
-104
lines changed
Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
#!/bin/bash
2+
3+
# Copyright 2023 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# This scripts gets copied from the controller into the rescue system
18+
# of the bare-metal machine.
19+
20+
# Bash Strict Mode: https://github.com/guettli/bash-strict-mode
21+
trap 'echo -e "\n🤷 🚨 🔥 Warning: A command has failed. Exiting the script. Line was ($0:$LINENO): $(sed -n "${LINENO}p" "$0" 2>/dev/null || true) 🔥 🚨 🤷 "; exit 3' ERR
22+
set -Eeuo pipefail
23+
24+
image="${1:-}"
25+
outfile="${2:-}"
26+
27+
function usage {
28+
echo "$0 image outfile"
29+
echo " Download a machine image from a container registry"
30+
echo " image: for example ghcr.io/foo/bar/my-machine-image:v9"
31+
echo " outfile: Created file. Usually with file extensions '.tgz'"
32+
echo " If the oci registry needs a token, then the script uses OCI_REGISTRY_AUTH_TOKEN (if set)"
33+
echo " Example 1: of OCI_REGISTRY_AUTH_TOKEN: mygithubuser:mypassword"
34+
echo " Example 2: of OCI_REGISTRY_AUTH_TOKEN: ghp_SN51...."
35+
echo
36+
}
37+
if [ -z "$outfile" ]; then
38+
usage
39+
exit 1
40+
fi
41+
42+
OCI_REGISTRY_AUTH_TOKEN="${OCI_REGISTRY_AUTH_TOKEN:-}" # github:$GITHUB_TOKEN
43+
44+
# Extract registry
45+
registry="${image%%/*}"
46+
47+
# Extract scope and tag
48+
remainder="${image#*/}"
49+
scope="${remainder%:*}"
50+
tag="${remainder##*:}"
51+
52+
if [[ -z "$registry" || -z "$scope" || -z "$tag" ]]; then
53+
echo "failed to parse registry, scope and tag from image"
54+
echo "image=$image"
55+
echo "registry=$registry"
56+
echo "scope=$scope"
57+
echo "tag=$tag"
58+
exit 1
59+
fi
60+
61+
function get_token {
62+
echo "download with token (OCI_REGISTRY_AUTH_TOKEN set)"
63+
if [[ "$OCI_REGISTRY_AUTH_TOKEN" != *:* ]]; then
64+
echo "Using OCI_REGISTRY_AUTH_TOKEN directly (no colon in token)"
65+
token=$(echo "$OCI_REGISTRY_AUTH_TOKEN" | base64)
66+
return
67+
fi
68+
echo "OCI_REGISTRY_AUTH_TOKEN contains colon. Doing login first"
69+
token=$(curl -fsSL -u "$OCI_REGISTRY_AUTH_TOKEN" "https://${registry}/token?scope=repository:$scope:pull" | jq -r '.token')
70+
if [ -z "$token" ] || [ "$token" == null ]; then
71+
echo "Failed to get token for container registry"
72+
exit 1
73+
fi
74+
echo "Login to $registry was successful"
75+
}
76+
77+
AUTH_ARGS=()
78+
if [ -z "$OCI_REGISTRY_AUTH_TOKEN" ]; then
79+
echo "OCI_REGISTRY_AUTH_TOKEN is not set. Using no auth"
80+
else
81+
token=""
82+
get_token
83+
if [ -z "$token" ]; then
84+
echo "failed to get token"
85+
exit 1
86+
fi
87+
AUTH_ARGS+=("--header")
88+
AUTH_ARGS+=("Authorization: Bearer $token")
89+
fi
90+
manifest=$(curl -sSL "${AUTH_ARGS[@]}" \
91+
-H "Accept: application/vnd.oci.image.manifest.v1+json" \
92+
"https://${registry}/v2/${scope}/manifests/${tag}")
93+
94+
if [ -z "$manifest" ] || [ "$manifest" == null ]; then
95+
echo "Failed to get manifest from container registry for image $image"
96+
exit 1
97+
fi
98+
digest=$(echo "$manifest" | jq -r '.layers[0].digest')
99+
100+
if [ -z "$digest" ] || [ "$digest" == null ]; then
101+
echo "Failed to get digest from container registry. Manifest: $manifest"
102+
exit 1
103+
fi
104+
105+
expected_hash=$(echo "$manifest" | jq -r '.layers[0].digest' | cut -d':' -f2)
106+
if [ -z "$expected_hash" ]; then
107+
echo "Could not get hash from manifest. Manifest: $manifest"
108+
exit 1
109+
fi
110+
111+
echo "Start download of $image"
112+
# with speed 5111000 bytes/sec (5MB/sec) and a 2 GByte image,
113+
# it takes about 6 minutes to download the image.
114+
# max-time 600 --> 10 minutes
115+
# Usually the download is much fast: 40 MB/sec, which takes about 50 seconds.
116+
curl -fsSL "${AUTH_ARGS[@]}" \
117+
--retry 5 --retry-delay 2 --retry-connrefused \
118+
--speed-limit 5111000 --speed-time 10 --max-time 600 \
119+
--continue-at - \
120+
--write-out "Downloaded %{size_download} bytes in %{time_total} seconds\n" \
121+
-o"$outfile" "https://${registry}/v2/${scope}/blobs/$digest"
122+
123+
hash=$(sha256sum "$outfile" | awk '{print $1}')
124+
if [ -z "$hash" ]; then
125+
echo "Failed to calculate hash of downloaded file $outfile"
126+
exit 1
127+
fi
128+
129+
if [ "$hash" != "$expected_hash" ]; then
130+
echo "Hash of downloaded file $outfile does not match expected hash"
131+
echo "Expected: $expected_hash"
132+
echo "Got: $hash"
133+
exit 1
134+
fi
135+
136+
echo "Hash of downloaded file $outfile matches expected hash: $hash"

pkg/services/baremetal/client/ssh/ssh_client.go

Lines changed: 2 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -53,110 +53,8 @@ var checkDiskShellScript string
5353
//go:embed nic-info.sh
5454
var nicInfoShellScript string
5555

56-
var downloadFromOciShellScript = `#!/bin/bash
57-
58-
# Copyright 2023 The Kubernetes Authors.
59-
#
60-
# Licensed under the Apache License, Version 2.0 (the "License");
61-
# you may not use this file except in compliance with the License.
62-
# You may obtain a copy of the License at
63-
#
64-
# http://www.apache.org/licenses/LICENSE-2.0
65-
#
66-
# Unless required by applicable law or agreed to in writing, software
67-
# distributed under the License is distributed on an "AS IS" BASIS,
68-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
69-
# See the License for the specific language governing permissions and
70-
# limitations under the License.
71-
72-
# This scripts gets copied from the controller into the rescue system
73-
# of the bare-metal machine.
74-
75-
set -euo pipefail
76-
77-
image="${1:-}"
78-
outfile="${2:-}"
79-
80-
function usage {
81-
echo "$0 image outfile."
82-
echo " Download a machine image from a container registry"
83-
echo " image: for example ghcr.io/foo/bar/my-machine-image:v9"
84-
echo " outfile: Created file. Usually with file extensions '.tgz'"
85-
echo " If the oci registry needs a token, then the script uses OCI_REGISTRY_AUTH_TOKEN (if set)"
86-
echo " Example 1: of OCI_REGISTRY_AUTH_TOKEN: mygithubuser:mypassword"
87-
echo " Example 2: of OCI_REGISTRY_AUTH_TOKEN: ghp_SN51...."
88-
echo
89-
}
90-
if [ -z "$outfile" ]; then
91-
usage
92-
exit 1
93-
fi
94-
OCI_REGISTRY_AUTH_TOKEN="${OCI_REGISTRY_AUTH_TOKEN:-}" # github:$GITHUB_TOKEN
95-
96-
# Extract registry
97-
registry="${image%%/*}"
98-
99-
# Extract scope and tag
100-
remainder="${image#*/}"
101-
scope="${remainder%:*}"
102-
tag="${remainder##*:}"
103-
104-
if [[ -z "$registry" || -z "$scope" || -z "$tag" ]]; then
105-
echo "failed to parse registry, scope and tag from image"
106-
echo "image=$image"
107-
echo "registry=$registry"
108-
echo "scope=$scope"
109-
echo "tag=$tag"
110-
exit 1
111-
fi
112-
113-
function download_with_token {
114-
echo "download with token (OCI_REGISTRY_AUTH_TOKEN set)"
115-
if [[ "$OCI_REGISTRY_AUTH_TOKEN" != *:* ]]; then
116-
echo "Using OCI_REGISTRY_AUTH_TOKEN directly (no colon in token)"
117-
token=$(echo "$OCI_REGISTRY_AUTH_TOKEN" | base64)
118-
else
119-
echo "OCI_REGISTRY_AUTH_TOKEN contains colon. Doing login first"
120-
token=$(curl -fsSL -u "$OCI_REGISTRY_AUTH_TOKEN" "https://${registry}/token?scope=repository:$scope:pull" | jq -r '.token')
121-
if [ -z "$token" ]; then
122-
echo "Failed to get token for container registry"
123-
exit 1
124-
fi
125-
echo "Login to $registry was successful"
126-
fi
127-
digest=$(curl -sSL -H "Authorization: Bearer $token" -H "Accept: application/vnd.oci.image.manifest.v1+json" \
128-
"https://${registry}/v2/${scope}/manifests/${tag}" | jq -r '.layers[0].digest')
129-
130-
if [ -z "$digest" ]; then
131-
echo "Failed to get digest from container registry"
132-
exit 1
133-
fi
134-
135-
echo "Start download of $image"
136-
curl -fsSL -H "Authorization: Bearer $token" \
137-
"https://${registry}/v2/${scope}/blobs/$digest" >"$outfile"
138-
}
139-
140-
function download_without_token {
141-
echo "download without token (OCI_REGISTRY_AUTH_TOKEN empty)"
142-
digest=$(curl -sSL -H "Accept: application/vnd.oci.image.manifest.v1+json" \
143-
"https://${registry}/v2/${scope}/manifests/${tag}" | jq -r '.layers[0].digest')
144-
145-
if [ -z "$digest" ]; then
146-
echo "Failed to get digest from container registry"
147-
exit 1
148-
fi
149-
150-
echo "Start download of $image"
151-
curl -fsSL "https://${registry}/v2/${scope}/blobs/$digest" >"$outfile"
152-
}
153-
154-
if [ -z "$OCI_REGISTRY_AUTH_TOKEN" ]; then
155-
download_without_token
156-
else
157-
download_with_token
158-
fi
159-
`
56+
//go:embed download-from-oci.sh
57+
var downloadFromOciShellScript string
16058

16159
var (
16260
// ErrCommandExitedWithoutExitSignal means the ssh command exited unplanned.

0 commit comments

Comments
 (0)