Skip to content

Commit 5cf2e1b

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #7550 from ameukam/boskos-projects-enable-ar
GCP: Enable Artifact Registry for the Boskos projects
2 parents f49b1ef + 16e0bbc commit 5cf2e1b

File tree

1 file changed

+156
-155
lines changed

1 file changed

+156
-155
lines changed

‎infra/gcp/bash/prow/ensure-e2e-projects.sh‎

Lines changed: 156 additions & 155 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,11 @@ SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
2525
. "${SCRIPT_DIR}/../lib.sh"
2626

2727
function usage() {
28-
echo "usage: $0 [repo...]" > /dev/stderr
29-
echo "example:" > /dev/stderr
30-
echo " $0 # do all projects" > /dev/stderr
31-
echo " $0 k8s-infra-node-e2e-project # just do one" > /dev/stderr
32-
echo > /dev/stderr
28+
echo "usage: $0 [repo...]" >/dev/stderr
29+
echo "example:" >/dev/stderr
30+
echo " $0 # do all projects" >/dev/stderr
31+
echo " $0 k8s-infra-node-e2e-project # just do one" >/dev/stderr
32+
echo >/dev/stderr
3333
}
3434

3535
## projects hosting prow build clusters managed by sig-k8s-infra
@@ -47,170 +47,171 @@ mapfile -t E2E_PROJECTS < <(k8s_infra_projects "e2e")
4747
readonly E2E_PROJECTS
4848

4949
function ensure_e2e_project() {
50-
if [ $# != 1 ] || [ -z "$1" ]; then
51-
echo "${FUNCNAME[0]}(project) requires 1 argument" >&2
52-
return 1
53-
fi
54-
local prj="${1}"
55-
56-
ensure_project "${prj}"
57-
58-
local project_number
59-
project_number=$(gcloud projects describe "${prj}" --format='value(projectNumber)')
60-
61-
color 6 "Ensure stale role bindings have been removed from e2e project: ${prj}"
62-
(
63-
echo "no stale bindings slated for removal"
64-
) 2>&1 | indent
65-
66-
color 6 "Ensuring only APIs necessary for kubernetes e2e jobs to use e2e project: ${prj}"
67-
ensure_only_services "${prj}" \
68-
cloudkms.googleapis.com \
69-
compute.googleapis.com \
70-
container.googleapis.com \
71-
containerregistry.googleapis.com \
72-
file.googleapis.com \
73-
logging.googleapis.com \
74-
monitoring.googleapis.com \
75-
storage-component.googleapis.com
76-
77-
# TODO: this is what prow.k8s.io uses today, but seems overprivileged, we
78-
# could consider using a more limited custom IAM role instead
79-
color 6 "Empower prow-build service account to edit e2e project: ${prj}"
80-
ensure_project_role_binding "${prj}" \
81-
"serviceAccount:${PROW_BUILD_SVCACCT}" \
82-
"roles/editor"
83-
84-
# TODO: Remove this binding and clean up permissions in projects
85-
# This permission is superseded by roles/cloudkms.admin below
86-
# Ensure GCP CSI driver tests can manage KMS keys
87-
ensure_project_role_binding "${prj}" \
88-
"serviceAccount:${PROW_BUILD_SVCACCT}" \
89-
"roles/cloudkms.cryptoKeyEncrypterDecrypter"
90-
91-
# Ensure GCP Default Compute Service Account can administer KMS keys
92-
ensure_project_role_binding "${prj}" \
93-
"serviceAccount:${PROW_BUILD_SVCACCT}" \
94-
"roles/cloudkms.admin"
95-
96-
# TODO: Remove this binding and clean up permissions in projects
97-
# Ensure GCP Default Compute Service Account can manage KMS keys
98-
ensure_project_role_binding "${prj}" \
99-
"serviceAccount:${project_number}[email protected]" \
100-
"roles/cloudkms.cryptoKeyEncrypterDecrypter"
101-
102-
# Ensure GCP Default Compute Engine Service Agent Account can manage KMS
103-
# keys
104-
ensure_project_role_binding "${prj}" \
105-
"serviceAccount:service-${project_number}@compute-system.iam.gserviceaccount.com" \
106-
"roles/cloudkms.cryptoKeyEncrypterDecrypter"
107-
108-
# TODO: Remove this binding and clean up permissions in projects
109-
# Ensure GCP CSI driver tests can use prow-build service account to
110-
# act as all other service accounts (eg: Compute Engine default service account)
111-
ensure_project_role_binding "${prj}" \
112-
"serviceAccount:${PROW_BUILD_SVCACCT}" \
113-
"roles/iam.serviceAccountUser"
114-
115-
# TODO: this is what prow.k8s.io uses today, but seems overprivileged, we
116-
# could consider using a more limited custom IAM role instead
117-
color 6 "Empower boskos-janitor service account to clean e2e project: ${prj}"
118-
ensure_project_role_binding "${prj}" \
119-
"serviceAccount:${BOSKOS_JANITOR_SVCACCT}" \
120-
"roles/editor"
121-
122-
color 6 "Empower [email protected] to admin e2e project: ${prj}"
50+
if [ $# != 1 ] || [ -z "$1" ]; then
51+
echo "${FUNCNAME[0]}(project) requires 1 argument" >&2
52+
return 1
53+
fi
54+
local prj="${1}"
55+
56+
ensure_project "${prj}"
57+
58+
local project_number
59+
project_number=$(gcloud projects describe "${prj}" --format='value(projectNumber)')
60+
61+
color 6 "Ensure stale role bindings have been removed from e2e project: ${prj}"
62+
(
63+
echo "no stale bindings slated for removal"
64+
) 2>&1 | indent
65+
66+
color 6 "Ensuring only APIs necessary for kubernetes e2e jobs to use e2e project: ${prj}"
67+
ensure_only_services "${prj}" \
68+
artifactregistry.googleapis.com \
69+
cloudkms.googleapis.com \
70+
compute.googleapis.com \
71+
container.googleapis.com \
72+
containerregistry.googleapis.com \
73+
file.googleapis.com \
74+
logging.googleapis.com \
75+
monitoring.googleapis.com \
76+
storage-component.googleapis.com
77+
78+
# TODO: this is what prow.k8s.io uses today, but seems overprivileged, we
79+
# could consider using a more limited custom IAM role instead
80+
color 6 "Empower prow-build service account to edit e2e project: ${prj}"
81+
ensure_project_role_binding "${prj}" \
82+
"serviceAccount:${PROW_BUILD_SVCACCT}" \
83+
"roles/editor"
84+
85+
# TODO: Remove this binding and clean up permissions in projects
86+
# This permission is superseded by roles/cloudkms.admin below
87+
# Ensure GCP CSI driver tests can manage KMS keys
88+
ensure_project_role_binding "${prj}" \
89+
"serviceAccount:${PROW_BUILD_SVCACCT}" \
90+
"roles/cloudkms.cryptoKeyEncrypterDecrypter"
91+
92+
# Ensure GCP Default Compute Service Account can administer KMS keys
93+
ensure_project_role_binding "${prj}" \
94+
"serviceAccount:${PROW_BUILD_SVCACCT}" \
95+
"roles/cloudkms.admin"
96+
97+
# TODO: Remove this binding and clean up permissions in projects
98+
# Ensure GCP Default Compute Service Account can manage KMS keys
99+
ensure_project_role_binding "${prj}" \
100+
"serviceAccount:${project_number}[email protected]" \
101+
"roles/cloudkms.cryptoKeyEncrypterDecrypter"
102+
103+
# Ensure GCP Default Compute Engine Service Agent Account can manage KMS
104+
# keys
105+
ensure_project_role_binding "${prj}" \
106+
"serviceAccount:service-${project_number}@compute-system.iam.gserviceaccount.com" \
107+
"roles/cloudkms.cryptoKeyEncrypterDecrypter"
108+
109+
# TODO: Remove this binding and clean up permissions in projects
110+
# Ensure GCP CSI driver tests can use prow-build service account to
111+
# act as all other service accounts (eg: Compute Engine default service account)
112+
ensure_project_role_binding "${prj}" \
113+
"serviceAccount:${PROW_BUILD_SVCACCT}" \
114+
"roles/iam.serviceAccountUser"
115+
116+
# TODO: this is what prow.k8s.io uses today, but seems overprivileged, we
117+
# could consider using a more limited custom IAM role instead
118+
color 6 "Empower boskos-janitor service account to clean e2e project: ${prj}"
119+
ensure_project_role_binding "${prj}" \
120+
"serviceAccount:${BOSKOS_JANITOR_SVCACCT}" \
121+
"roles/editor"
122+
123+
color 6 "Empower [email protected] to admin e2e project: ${prj}"
124+
ensure_project_role_binding "${prj}" \
125+
"group:[email protected]" \
126+
"roles/owner"
127+
128+
# NB: prow.viewer role is defined in ensure-organization.sh, that needs to have been run first
129+
color 6 "Empower [email protected] to view specific resources in e2e project: ${prj}"
130+
ensure_project_role_binding "${prj}" \
131+
"group:[email protected]" \
132+
"$(custom_org_role_name "prow.viewer")"
133+
134+
if [[ "${prj}" =~ k8s-infra-e2e.*scale ]]; then
135+
color 6 "Empower [email protected] to admin e2e project: ${prj}"
123136
ensure_project_role_binding "${prj}" \
124-
"group:k8s-infra-prow[email protected]" \
137+
"group:k8s-infra-sig-scalability[email protected]" \
125138
"roles/owner"
139+
fi
140+
141+
color 6 "Ensure prow-build prowjobs are able to ssh to instances in e2e project: ${prj}"
142+
prow_build_ssh_pubkey="prow:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCmYxHh/wwcV0P1aChuFLpl28w6DFyc7G5Xrw1F8wH1Re9AdxyemM2bTZ/PhsP3u9VDnNbyOw3UN00VFdumkFLjLf1WQ7Q6rZDlPjlw7urBIvAMqUecY6ae1znqsZ0dMBxOuPXHznlnjLjM5b7O7q5WsQMCA9Szbmz6DsuSyCuX0It2osBTN+8P/Fa6BNh3W8AF60M7L8/aUzLfbXVS2LIQKAHHD8CWqvXhLPuTJ03iSwFvgtAK1/J2XJwUP+OzAFrxj6A9LW5ZZgk3R3kRKr0xT/L7hga41rB1qy8Uz+Xr/PTVMNGW+nmU4bPgFchCK0JBK7B12ZcdVVFUEdpaAiKZ prow"
143+
k8s_prow_builds_ssh_pubkey="prow:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+/ZdafYYrJknk08g98sYS1Nr+aVdAnhHpQyXBx7EAT9pazCGaoiYnXgC82FAfTVMqdsqnIiP+7FgQTFLNYvBt8KsBd9qCkuMh/Q1QYVh4kfjjuGUrjfo020pxGSvp+67kbxm6lubaio9AgJ9XXE+SP1AYbyKTvXEzk5Tu7gGnRt3OrjVB+9eqTnVJOjS/BAOTJV5DWQ7xMubHlT9NmQ/S2hotMoiJJybYGUalOfcf8ZkyspU2oR+x13DCfjvFdzF4U0fb/uvTJZeu22w887M5y0YQulFY2LIeoAUE4XwoOv0nxzwbtZpqPHwtfLgq3G906KHW5e6slXu8kGda656n prow"
144+
ssh_keys_expected=(
145+
"${k8s_prow_builds_ssh_pubkey}"
146+
"${prow_build_ssh_pubkey}"
147+
# TODO(amwat,spiffxp): something is adding an extra prow: prefix, it is
148+
# unclear where in prow->kubetest2->cluster/log-dump.sh->`gcloud ssh`
149+
# this is happening
150+
"prow:${k8s_prow_builds_ssh_pubkey}"
151+
"prow:${prow_build_ssh_pubkey}"
152+
)
153+
154+
# append to project-wide ssh-keys metadata if not present
155+
ssh_keys_before="${TMPDIR}/ssh-keys.before.txt"
156+
ssh_keys_after="${TMPDIR}/ssh-keys.after.txt"
157+
gcloud compute project-info describe --project="${prj}" \
158+
--format='value(commonInstanceMetadata.items.filter(key:ssh-keys).extract(value).flatten())' |
159+
sed -e '/^$/d' >"${ssh_keys_before}"
160+
161+
cp "${ssh_keys_before}" "${ssh_keys_after}"
162+
163+
if [ "${K8S_INFRA_ENSURE_E2E_PROJECTS_RESETS_SSH_KEYS:-"false"}" == "true" ]; then
164+
printf '%s\n' "${ssh_keys_expected[@]}" >"${ssh_keys_after}"
165+
else
166+
for ssh_key in "${ssh_keys_expected[@]}"; do
167+
if ! grep -q "${ssh_key}" "${ssh_keys_before}"; then
168+
echo "${ssh_key}" >>"${ssh_keys_after}"
169+
fi
170+
done
171+
fi
126172

127-
# NB: prow.viewer role is defined in ensure-organization.sh, that needs to have been run first
128-
color 6 "Empower [email protected] to view specific resources in e2e project: ${prj}"
129-
ensure_project_role_binding "${prj}" \
130-
"group:[email protected]" \
131-
"$(custom_org_role_name "prow.viewer")"
132-
133-
if [[ "${prj}" =~ k8s-infra-e2e.*scale ]]; then
134-
color 6 "Empower [email protected] to admin e2e project: ${prj}"
135-
ensure_project_role_binding "${prj}" \
136-
"group:[email protected]" \
137-
"roles/owner"
138-
fi
139-
140-
color 6 "Ensure prow-build prowjobs are able to ssh to instances in e2e project: ${prj}"
141-
prow_build_ssh_pubkey="prow:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCmYxHh/wwcV0P1aChuFLpl28w6DFyc7G5Xrw1F8wH1Re9AdxyemM2bTZ/PhsP3u9VDnNbyOw3UN00VFdumkFLjLf1WQ7Q6rZDlPjlw7urBIvAMqUecY6ae1znqsZ0dMBxOuPXHznlnjLjM5b7O7q5WsQMCA9Szbmz6DsuSyCuX0It2osBTN+8P/Fa6BNh3W8AF60M7L8/aUzLfbXVS2LIQKAHHD8CWqvXhLPuTJ03iSwFvgtAK1/J2XJwUP+OzAFrxj6A9LW5ZZgk3R3kRKr0xT/L7hga41rB1qy8Uz+Xr/PTVMNGW+nmU4bPgFchCK0JBK7B12ZcdVVFUEdpaAiKZ prow"
142-
k8s_prow_builds_ssh_pubkey="prow:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+/ZdafYYrJknk08g98sYS1Nr+aVdAnhHpQyXBx7EAT9pazCGaoiYnXgC82FAfTVMqdsqnIiP+7FgQTFLNYvBt8KsBd9qCkuMh/Q1QYVh4kfjjuGUrjfo020pxGSvp+67kbxm6lubaio9AgJ9XXE+SP1AYbyKTvXEzk5Tu7gGnRt3OrjVB+9eqTnVJOjS/BAOTJV5DWQ7xMubHlT9NmQ/S2hotMoiJJybYGUalOfcf8ZkyspU2oR+x13DCfjvFdzF4U0fb/uvTJZeu22w887M5y0YQulFY2LIeoAUE4XwoOv0nxzwbtZpqPHwtfLgq3G906KHW5e6slXu8kGda656n prow"
143-
ssh_keys_expected=(
144-
"${k8s_prow_builds_ssh_pubkey}"
145-
"${prow_build_ssh_pubkey}"
146-
# TODO(amwat,spiffxp): something is adding an extra prow: prefix, it is
147-
# unclear where in prow->kubetest2->cluster/log-dump.sh->`gcloud ssh`
148-
# this is happening
149-
"prow:${k8s_prow_builds_ssh_pubkey}"
150-
"prow:${prow_build_ssh_pubkey}"
151-
)
152-
153-
# append to project-wide ssh-keys metadata if not present
154-
ssh_keys_before="${TMPDIR}/ssh-keys.before.txt"
155-
ssh_keys_after="${TMPDIR}/ssh-keys.after.txt"
156-
gcloud compute project-info describe --project="${prj}" \
157-
--format='value(commonInstanceMetadata.items.filter(key:ssh-keys).extract(value).flatten())' \
158-
| sed -e '/^$/d' > "${ssh_keys_before}"
159-
160-
cp "${ssh_keys_before}" "${ssh_keys_after}"
161-
162-
if [ "${K8S_INFRA_ENSURE_E2E_PROJECTS_RESETS_SSH_KEYS:-"false"}" == "true" ]; then
163-
printf '%s\n' "${ssh_keys_expected[@]}" > "${ssh_keys_after}"
164-
else
165-
for ssh_key in "${ssh_keys_expected[@]}"; do
166-
if ! grep -q "${ssh_key}" "${ssh_keys_before}"; then
167-
echo "${ssh_key}" >> "${ssh_keys_after}"
168-
fi
169-
done
170-
fi
171-
172-
if ! diff "${ssh_keys_before}" "${ssh_keys_after}" >/dev/null; then
173-
gcloud compute project-info add-metadata --project="${prj}" \
174-
--metadata-from-file ssh-keys="${ssh_keys_after}"
175-
diff_colorized "${ssh_keys_before}" "${ssh_keys_after}"
176-
fi
173+
if ! diff "${ssh_keys_before}" "${ssh_keys_after}" >/dev/null; then
174+
gcloud compute project-info add-metadata --project="${prj}" \
175+
--metadata-from-file ssh-keys="${ssh_keys_after}"
176+
diff_colorized "${ssh_keys_before}" "${ssh_keys_after}"
177+
fi
177178
}
178179

179180
# Disable OS Login at the project level
180181
# $1 The GCP Project
181182
function disable_project_oslogin() {
182-
if [ $# != 1 ] || [ -z "$1" ]; then
183-
echo "${FUNCNAME[0]}(project) requires 1 argument" >&2
184-
return 1
185-
fi
186-
187-
local prj="${1}"
188-
189-
enabled=$(gcloud compute project-info describe --project="${prj}" \
190-
--format='value(commonInstanceMetadata.items[enable-oslogin])')
191-
if [ "${enabled}" == "TRUE" ]; then
192-
gcloud compute project-info --project="${prj}" remove-metadata --keys "enable-oslogin"
193-
fi
183+
if [ $# != 1 ] || [ -z "$1" ]; then
184+
echo "${FUNCNAME[0]}(project) requires 1 argument" >&2
185+
return 1
186+
fi
187+
188+
local prj="${1}"
189+
190+
enabled=$(gcloud compute project-info describe --project="${prj}" \
191+
--format='value(commonInstanceMetadata.items[enable-oslogin])')
192+
if [ "${enabled}" == "TRUE" ]; then
193+
gcloud compute project-info --project="${prj}" remove-metadata --keys "enable-oslogin"
194+
fi
194195
}
195196

196197
function ensure_e2e_projects() {
197-
# default to all staging projects
198-
if [ $# = 0 ]; then
199-
set -- "${E2E_PROJECTS[@]}"
198+
# default to all staging projects
199+
if [ $# = 0 ]; then
200+
set -- "${E2E_PROJECTS[@]}"
201+
fi
202+
203+
for project in "${@}"; do
204+
if ! (printf '%s\n' "${E2E_PROJECTS[@]}" | grep -q "^${project}$"); then
205+
color 2 "Skipping unrecognized e2e project name: ${project}"
206+
continue
200207
fi
201208

202-
for project in "${@}"; do
203-
if ! (printf '%s\n' "${E2E_PROJECTS[@]}" | grep -q "^${project}$"); then
204-
color 2 "Skipping unrecognized e2e project name: ${project}"
205-
continue
206-
fi
207-
208-
color 3 "Configuring e2e project: ${project}"
209-
ensure_e2e_project "${project}" 2>&1 | indent
209+
color 3 "Configuring e2e project: ${project}"
210+
ensure_e2e_project "${project}" 2>&1 | indent
210211

211-
# color 3 "Ensuring OS Login is disabled for $project"
212-
# disable_project_oslogin "${project}" 2>&1 | indent
213-
done
212+
# color 3 "Ensuring OS Login is disabled for $project"
213+
# disable_project_oslogin "${project}" 2>&1 | indent
214+
done
214215
}
215216

216217
#

0 commit comments

Comments
 (0)