Skip to content

Commit ca4b8c4

Browse files
sbueringersbueringer
committed
Use vSphere projects from Boskos
Signed-off-by: Stefan Büringer [email protected]
1 parent 8b7b506 commit ca4b8c4

File tree

1 file changed

+89
-15
lines changed

1 file changed

+89
-15
lines changed

‎images/capi/scripts/ci-ova.sh

Lines changed: 89 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,22 @@ cd "${CAPI_ROOT}" || exit 1
2424
export ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
2525
TARGETS=("ubuntu-2004" "ubuntu-2204" "photon-3" "photon-4" "photon-5" "rockylinux-8" "flatcar")
2626

27+
export BOSKOS_RESOURCE_OWNER=image-builder
28+
if [[ "${JOB_NAME}" != "" ]]; then
29+
export BOSKOS_RESOURCE_OWNER="${JOB_NAME}/${BUILD_ID}"
30+
fi
31+
export BOSKOS_RESOURCE_TYPE=vsphere-project-image-builder
32+
2733
on_exit() {
2834
#Cleanup VMs
2935
cleanup_build_vm
3036

37+
# Stop boskos heartbeat
38+
[[ -z ${HEART_BEAT_PID:-} ]] || kill -9 "${HEART_BEAT_PID}"
39+
40+
# If Boskos is being used then release the vsphere project.
41+
[ -z "${BOSKOS_HOST:-}" ] || docker run gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest release --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" --vsphere-username="${VSPHERE_USERNAME}" --vsphere-password="${VSPHERE_PASSWORD}" --vsphere-server="${VSPHERE_SERVER}" --vsphere-tls-thumbprint="${VSPHERE_TLS_THUMBPRINT}" --vsphere-folder="${BOSKOS_RESOURCE_FOLDER}" --vsphere-resource-pool="${BOSKOS_RESOURCE_POOL}"
42+
3143
# kill the VPN
3244
docker kill vpn
3345
}
@@ -42,24 +54,94 @@ cleanup_build_vm() {
4254
for target in ${TARGETS[@]};
4355
do
4456
# Adding || true to both commands so it does not exit after not being able to cleanup one target.
45-
govc vm.power -off -force -wait /${GOVC_DATACENTER}/vm/${FOLDER}/capv-ci-${target}-${TIMESTAMP} || true
46-
govc object.destroy /${GOVC_DATACENTER}/vm/${FOLDER}/capv-ci-${target}-${TIMESTAMP} || true
57+
govc vm.power -off -force -wait /${GOVC_DATACENTER}/vm/${VSPHERE_FOLDER}/capv-ci-${target}-${TIMESTAMP} || true
58+
govc object.destroy /${GOVC_DATACENTER}/vm/${VSPHERE_FOLDER}/capv-ci-${target}-${TIMESTAMP} || true
4759
done
4860

4961
}
5062

5163
trap on_exit EXIT
5264

65+
# For Boskos
66+
export VSPHERE_SERVER="${GOVC_URL:-}"
67+
export VSPHERE_USERNAME="${GOVC_USERNAME:-}"
68+
export VSPHERE_PASSWORD="${GOVC_PASSWORD:-}"
69+
5370
export PATH=${PWD}/.local/bin:$PATH
5471
export PATH=${PYTHON_BIN_DIR:-"/root/.local/bin"}:$PATH
5572
export GC_KIND="false"
5673
export TIMESTAMP="$(date -u '+%Y%m%dT%H%M%S')"
5774
export GOVC_DATACENTER="SDDC-Datacenter"
75+
export GOVC_CLUSTER="Cluster-1"
5876
export GOVC_INSECURE=true
59-
export FOLDER="Workloads/image-builder"
77+
78+
# FIXME(sbueringer) before merge this should be moved to the ProwJob config in test-infra (so we can move jobs incrementally to the new environment / boskos)
79+
export BOSKOS_HOST=http://192.168.6.138:32222
80+
81+
# Run the vpn client in container
82+
docker run --rm -d --name vpn -v "${HOME}/.openvpn/:${HOME}/.openvpn/" \
83+
-w "${HOME}/.openvpn/" --cap-add=NET_ADMIN --net=host --device=/dev/net/tun \
84+
gcr.io/k8s-staging-capi-vsphere/extra/openvpn:latest
85+
86+
# Tail the vpn logs
87+
docker logs vpn
88+
89+
# Wait until the VPN connection is active.
90+
function wait_for_vpn_up() {
91+
local n=0
92+
until [ $n -ge 30 ]; do
93+
curl "https://${VSPHERE_SERVER}" --connect-timeout 2 -k -v && RET=$? || RET=$?
94+
if [[ "$RET" -eq 0 ]]; then
95+
break
96+
fi
97+
n=$((n + 1))
98+
sleep 1
99+
done
100+
return "$RET"
101+
}
102+
wait_for_vpn_up
103+
104+
# If BOSKOS_HOST is set then acquire a vsphere-project from Boskos.
105+
if [ -n "${BOSKOS_HOST:-}" ]; then
106+
# Check out the account from Boskos and store the produced environment
107+
# variables in a temporary file.
108+
account_env_var_file="$(mktemp)"
109+
docker run gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest acquire --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-type="${BOSKOS_RESOURCE_TYPE}" 1>"${account_env_var_file}"
110+
checkout_account_status="${?}"
111+
112+
# If the checkout process was a success then load the account's
113+
# environment variables into this process.
114+
# shellcheck disable=SC1090
115+
[ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}"
116+
export BOSKOS_RESOURCE_NAME=${BOSKOS_RESOURCE_NAME}
117+
# Drop absolute prefix because packer needs the relative path.
118+
export VSPHERE_FOLDER="$(echo "${BOSKOS_RESOURCE_FOLDER}" | sed "s@/${GOVC_DATACENTER}/vm/@@")"
119+
export VSPHERE_RESOURCE_POOL="$(echo "${BOSKOS_RESOURCE_POOL}" | sed "s@/${GOVC_DATACENTER}/host/${GOVC_CLUSTER}/Resources/@@")"
120+
121+
# Always remove the account environment variable file. It contains
122+
# sensitive information.
123+
rm -f "${account_env_var_file}"
124+
125+
if [ ! "${checkout_account_status}" = "0" ]; then
126+
echo "error getting vsphere project from Boskos" 1>&2
127+
exit "${checkout_account_status}"
128+
fi
129+
130+
# Run the heartbeat to tell boskos periodically that we are still
131+
# using the checked out account.
132+
docker run gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest heartbeat --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" >>"${ARTIFACTS}/boskos-heartbeat.log" 2>&1 &
133+
HEART_BEAT_PID=$!
134+
else
135+
echo "error getting vsphere project from Boskos, BOSKOS_HOST not set" 1>&2
136+
exit 1
137+
fi
60138

61139
echo "Running build with timestamp ${TIMESTAMP}"
62140

141+
echo "Using user: ${GOVC_USERNAME}"
142+
echo "Using relative folder: ${VSPHERE_FOLDER}"
143+
echo "Using relative resource pool: ${VSPHERE_RESOURCE_POOL}"
144+
63145
cat << EOF > packer/ova/vsphere.json
64146
{
65147
"vcenter_server":"${GOVC_URL}",
@@ -68,10 +150,10 @@ cat << EOF > packer/ova/vsphere.json
68150
"password":"${GOVC_PASSWORD}",
69151
"datastore":"WorkloadDatastore",
70152
"datacenter":"${GOVC_DATACENTER}",
71-
"resource_pool": "Compute-ResourcePool/image-builder",
72-
"cluster": "Cluster-1",
73-
"network": "sddc-cgw-network-8",
74-
"folder": "${FOLDER}"
153+
"resource_pool": "${VSPHERE_RESOURCE_POOL}",
154+
"cluster": "${GOVC_CLUSTER}",
155+
"network": "sddc-cgw-network-10",
156+
"folder": "${VSPHERE_FOLDER}"
75157
}
76158
EOF
77159

@@ -81,14 +163,6 @@ cat packer/ova/packer-node.json | jq 'del(.builders[] | select( .name == "vsphe
81163
cat packer/ova/packer-node.json | jq 'del(.builders[] | select( .name == "vsphere-clone" ).export)' > packer/ova/packer-node.json.tmp && mv packer/ova/packer-node.json.tmp packer/ova/packer-node.json
82164
cat packer/ova/packer-node.json | jq 'del(."post-processors"[])' > packer/ova/packer-node.json.tmp && mv packer/ova/packer-node.json.tmp packer/ova/packer-node.json
83165

84-
# Run the vpn client in container
85-
docker run --rm -d --name vpn -v "${HOME}/.openvpn/:${HOME}/.openvpn/" \
86-
-w "${HOME}/.openvpn/" --cap-add=NET_ADMIN --net=host --device=/dev/net/tun \
87-
gcr.io/k8s-staging-capi-vsphere/extra/openvpn:latest
88-
89-
# Tail the vpn logs
90-
docker logs vpn
91-
92166
# install deps and build all images
93167
make deps-ova
94168

0 commit comments

Comments
 (0)