Merged PR 32: Support IcM ticket creation for OFR nodes

Support IcM ticket creation for OFR ndoes.

# Documents for testing new features

### Description:
Add node-recycler service to submit IcM ticket to UA nodes and recycle nodes.

### Test Actions:

1. Build the image for the node-recycler service.
2. Write the configuration file for the node-recycler under alert-manager:
```yaml
node-recycler:
  configured: False
  kusto:
    cluster: luciatrainingplatform
    datebase: ""
  icm:
    host: prod.microsofticm.com
    crt_name: cert.pem
    crt_path: /icm/cert.pem
    key_name: key.pem
    key_path: /icm/key.pem
    connector_id: ""
    routing_id: hpclucia://AzureHPC/Lucia/TrainingPlatform
  vmss:
    client_id: ""
    vmss_ids: "/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachineScaleSets/{vmss}"
    validation_image: bench.azurecr.io/superbench:0.11-rocm6.3
```
3. Prepare the certificate files for IcM connector.
4. Deploy the service to the cluster.
5. Mark some nodes as `triaged_hardware` in the node status table and add corresponding action record in the node action table.
6. Check whether the service can submit corresponding IcM tickets and deallocate/start the node.

### Expected Results:

1. Build the image successfully.
2. The service is deployed successfully.
3. The corresponding node should have the following status and action added to the node status table and node action table:
* status: `ua`, action: `triaged_hardware-ua`
* status: `deallocated_ua`, action: `ua-deallocated_ua`
* status: `allocated_ua`, action: `deallocated_ua-allocated-ua`
* status: `validating`, action: `allocated_ua-validating`
4. There exist one superbench job with the name `superbench_uaback_auto_xxxxxxxx` for corresponding node in the cluster.

Author: abuccts

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "src/alert-manager/src/node-recycler/python-icm"]
+	path = src/alert-manager/src/node-recycler/python-icm
+	url = https://dev.azure.com/msblox/python-icm/_git/python-icm

src/alert-manager/.gitignore

@@ -1,2 +1,4 @@
 # Dependency directories
 node_modules/
+
+deploy/icm-certs/

src/alert-manager/build/build-pre.sh

@@ -5,6 +5,7 @@
 
 pushd $(dirname "$0") > /dev/null
 
+cp -arfT "../../kusto/sdk" "../src/node-recycler"
 cp -arfT "../../database-controller/sdk" "../src/job-status-change-notification/openpaidbsdk"
 
 popd > /dev/null

src/alert-manager/build/node-recycler.common.dockerfile

@@ -0,0 +1,11 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+FROM mcr.microsoft.com/azurelinux/base/python:3.12
+
+WORKDIR /usr/src/app
+COPY ./src/node-recycler .
+
+RUN pip3 install -r requirements.txt
+
+ENTRYPOINT ["python3", "recycler.py"]

src/alert-manager/config/alert-manager.yaml

@@ -23,6 +23,23 @@ job-status-change-notification:
   log-level: 'info'
   db-poller-max-db-connection: 10
   db-poller-interval-second: 600
+node-recycler:
+  configured: False
+  kusto:
+    cluster: luciatrainingplatform
+    datebase: ""
+  icm:
+    host: prod.microsofticm.com
+    crt_name: cert.pem
+    crt_path: /icm/cert.pem
+    key_name: key.pem
+    key_path: /icm/key.pem
+    connector_id: ""
+    routing_id: hpclucia://AzureHPC/Lucia/TrainingPlatform
+  vmss:
+    client_id: ""
+    vmss_ids: ""
+    validation_image: bench.azurecr.io/superbench:0.11-rocm6.3
 use-pylon: False
 repeat-interval: '24h'
 cert-expiration-checker:

src/alert-manager/deploy/alert-manager-deployment.yaml.template

@@ -193,6 +193,68 @@ spec:
         - name: config-volume
           mountPath: /app/config
 {% endif %}
+
+{% if cluster_cfg["alert-manager"]["node-recycler"]["configured"] %}
+      - name: node-recycler
+        image: {{ cluster_cfg['cluster']['docker-registry']['prefix'] }}node-recycler:{{ cluster_cfg['cluster']['docker-registry']['tag'] }}
+        imagePullPolicy: Always
+        env:
+        - name: CLUSTER_ID
+          value: {{ cluster_cfg["cluster"]["common"]["cluster-id"] }}
+        - name: ENVIRONMENT
+          value: {{ cluster_cfg["alert-manager"]["node-recycler"]["environment"] | default("prod") }}
+        - name: KUSTO_USER_ASSIGNED_CLIENT_ID
+          value: {{ cluster_cfg["alert-manager"]["kusto-user-assigned-client-id"] }}
+{% if "kusto" in cluster_cfg["alert-manager"]["node-recycler"] %}
+        - name: LTP_KUSTO_CLUSTER
+          value: {{ cluster_cfg["alert-manager"]["node-recycler"]["kusto"]["cluster"] }}
+        - name: LTP_KUSTO_DATABASE_NAME
+          value: {{ cluster_cfg["alert-manager"]["node-recycler"]["kusto"]["database"] }}
+        - name: KUSTO_NODE_STATUS_TABLE_NAME
+          value: {{ cluster_cfg["alert-manager"]["node-recycler"]["kusto"]["node-status-table"] | default("NodeStatusRecord") }}
+        - name: KUSTO_NODE_ACTION_TABLE_NAME
+          value: {{ cluster_cfg["alert-manager"]["node-recycler"]["kusto"]["node-action-table"] | default("NodeActionRecord") }}
+{% endif %}
+{% if "icm" in cluster_cfg["alert-manager"]["node-recycler"] %}
+        - name: ICM_HOST
+          value: {{ cluster_cfg["alert-manager"]["node-recycler"]["icm"]["host"] }}
+        - name: ICM_CERT_PATH
+          value: /etc/icm/certs/{{ cluster_cfg["alert-manager"]["node-recycler"]["icm"]["crt_name"] | default("cert.pem") }}
+        - name: ICM_KEY_PATH
+          value: /etc/icm/certs/{{ cluster_cfg["alert-manager"]["node-recycler"]["icm"]["key_name"] | default("key.pem") }}
+        - name: ICM_CONNECTOR_ID
+          value: {{ cluster_cfg["alert-manager"]["node-recycler"]["icm"]["connector_id"] }}
+        - name: ICM_ROUTING_ID
+          value: {{ cluster_cfg["alert-manager"]["node-recycler"]["icm"]["routing_id"] }}
+{% endif %}
+{% if "vmss" in cluster_cfg["alert-manager"]["node-recycler"] %}
+        - name: AZURE_CLIENT_ID
+          value: {{ cluster_cfg["alert-manager"]["node-recycler"]["vmss"]["client_id"] }}
+        - name: LTP_VMSS_IDS
+          value: {{ cluster_cfg["alert-manager"]["node-recycler"]["vmss"]["vmss_ids"] }}
+        - name: LTP_VALIDATION_IMAGE
+          value: {{ cluster_cfg["alert-manager"]["node-recycler"]["vmss"]["validation_image"] }}
+{% endif %}
+        - name: REST_SERVER_URI
+          value: {{ cluster_cfg["rest-server"]["uri"] }}
+        - name: REST_SERVER_TOKEN
+          value: {{ cluster_cfg["alert-manager"]["pai-bearer-token"] }}
+        resources:
+          requests:
+            memory: "256Mi"
+            cpu: "100m"
+          limits:
+            memory: "512Mi"
+            cpu: "200m"
+        volumeMounts:
+        - name: config-volume
+          mountPath: /app/config
+{% if "icm" in cluster_cfg["alert-manager"]["node-recycler"] %}
+        - name: icm-certs
+          mountPath: /etc/icm/certs
+{% endif %}
+{% endif %}
+
       imagePullSecrets:
       - name: {{ cluster_cfg["cluster"]["docker-registry"]["secret-name"] }}
       volumes:
@@ -212,6 +274,13 @@ spec:
             path: {{ template }}/subject.ejs
           {% endfor -%}
 {% endif %}
+{% endif %}
+{% if cluster_cfg["alert-manager"]["node-recycler"]["configured"] %}
+{% if "icm" in cluster_cfg["alert-manager"]["node-recycler"] %}
+      - name: icm-certs
+        configMap:
+          name: icm-certs
+{% endif %}
 {% endif %}
       - name: alertmanager
         emptyDir: {}

src/alert-manager/deploy/start.sh.template

@@ -31,6 +31,14 @@ kubectl create configmap alert-templates \
 {% endif -%}
 {% endif -%}
 
+# create configmap for node-recycler
+{% if "icm" in cluster_cfg["alert-manager"]["node-recycler"] %}
+mkdir -p icm-certs/
+cp {{ cluster_cfg["alert-manager"]["node-recycler"]["icm"]["crt_path"] }} icm-certs/
+cp {{ cluster_cfg["alert-manager"]["node-recycler"]["icm"]["key_path"] }} icm-certs/
+kubectl create configmap icm-certs --from-file=icm-certs/ --dry-run=client -o yaml | kubectl apply --overwrite=true -f - || exit $?
+{% endif %}
+
 kubectl apply --overwrite=true -f rbac.yaml || exit $?
 kubectl apply --overwrite=true -f alert-manager-configmap.yaml || exit $?
 kubectl apply --overwrite=true -f alert-manager-deployment.yaml || exit $?