Merge pull request #20 from oracle-devrel/develop

v10 Commit Apprved

Author: xs2suruchi

README.md

@@ -9,15 +9,24 @@ The CD3 Automation toolkit has been developed to help in automating the OCI reso
 It reads input data in the form of CD3 Excel sheets and generates the terraform files instead of handling the task through the OCI console manually. The toolkit also reverse engineers the components in OCI back to the Excel sheet and Terraform configuration. This generated Terraform code can be used by the OCI Resource Manager or leveraged by the organisations’ CI/CD processes.
 <br><br>
 <img width="748" alt="Screenshot 2022-12-30 at 11 57 41 AM" src="https://user-images.githubusercontent.com/111430850/210614513-5d2e97a6-3c1e-4a2b-a793-3a1b6410c856.png">
-<br><br>
-
-`To ease the execution of toolkit, we have provided the steps to build an image which encloses the code base and its package dependencies. Follow the steps provided below to clone the repo and build the image.`
-<br><br>
+<br>
 
 #### OCI Services Currently Supported by CD3
-<img width="748" alt="Screenshot 2022-12-30 at 1 05 59 PM" src="https://user-images.githubusercontent.com/103475219/210046038-91acacfd-5d65-4bc3-a1a8-90d425d2e3d3.png">
 
+| OCI Services | Details |
+| --------- | ----------- |
+| IAM | Compartments, Groups, Dynamic Groups, Policies |
+| Network | VCNs, Subnets, DRGs, IGWs, NGWs, LPGs, Route Tables, DRG Route, Tables, Security Lists, Network Security Groups, Application Load Balancer, Network Load Balancers |
+| Governance | Tags (Namespaces, Tag Keys, Defined Tags, Default Tags, Cost Tracking) |
+| Compute | Instances – VM, BM, Dedicated VM Hosts |
+| Storage | FSS, Block and Boot Volumes, Backup Policies |
+| Database | Exa Infra, ExaCS, DB Systems VM and BM, ATP, ADW |
+| Management Services | Events, Notifications, Alarms, Service Connector Hub (SCH) |
+| Developer Services | Resource Manager, Oracle Kubernetes Engine (OKE) |
+| CIS Landing Zone Compliance | Download CIS Report Script, Execute CIS Report Script, VCN Flow Logs, Cloud Guard, Object Storage, Key Vault, Budget |
 
+`To ease the execution of toolkit, we have provided the steps to build an image which encloses the code base and its package dependencies. Follow the steps provided below to clone the repo and build the image.`
+<br>
 
 ## Getting Started
 

cd3_automation_toolkit/Compute/export_dedicatedvmhosts_nonGreenField.py

@@ -123,9 +123,6 @@ def export_dedicatedvmhosts(inputfile, _outdir, _config, network_compartments=[]
                 dedicatedvmhost=compute_client.get_dedicated_vm_host(dedicatedvmhost.id).data
                 print_dedicatedvmhosts(region, dedicatedvmhost,values_for_column, ntk_compartment_name)
 
-        if "linux" in sys.platform:
-            os.chmod(script_file, 0o755)
-
     commonTools.write_to_cd3(values_for_column, cd3file, "DedicatedVMHosts")
 
     print("Dedicated VM Hosts exported to CD3\n")

cd3_automation_toolkit/Compute/export_instances_nonGreenField.py

@@ -93,8 +93,7 @@ def __get_instances_info(compartment_name, compartment_id, reg_name, config,disp
     bc = oci.core.BlockstorageClient(config, retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY)
     instance_info = oci.pagination.list_call_get_all_results(compute.list_instances, compartment_id=compartment_id)
     # print(instance_info.data)
-    vnic_defined_tags=""
-    vnic_freeform_tags=""
+
     for ins in instance_info.data:
         ins_details = compute.get_instance(instance_id=ins.id)
         # print(ins_details.data)
@@ -181,7 +180,8 @@ def __get_instances_info(compartment_name, compartment_id, reg_name, config,disp
                 vnic_info = network.get_vnic(vnic_id=vnic_id).data
 
                 # print(vnic_info)
-
+                vnic_defined_tags = ""
+                vnic_freeform_tags = ""
                 for namespace,tagkey in vnic_info.defined_tags.items():
                     for tag,value in tagkey.items():
                         vnic_defined_tags=vnic_defined_tags+";"+namespace+"."+tag+"="+value
@@ -372,8 +372,6 @@ def export_instances(inputfile, outdir, config, network_compartments=[], display
         script_file = f'{outdir}/{reg}/tf_import_commands_instances_nonGF.sh'
         with open(script_file, 'a') as importCommands[reg]:
             importCommands[reg].write('\n\nterraform plan\n')
-        if "linux" in sys.platform:
-            os.chmod(script_file, 0o755)
 
     commonTools.write_to_cd3(values_for_column_instances, cd3file, "Instances")
     print("{0} Instance Details exported into CD3.\n".format(len(values_for_column_instances["Region"])))

cd3_automation_toolkit/Compute/templates/dedicatedvmhosts-template

@@ -6,7 +6,7 @@
 # Dedicated VM Hosts - tfvars
 # Allowed Values:
 # compartment_id and network_compartment_id can be the ocid or the name of the compartment hierarchy delimited by double hiphens "--"
-# Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "Network-root-cpt--Network" where "Network-root-cpt" is the parent of "Network" compartment
+# Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "AppDev--Prod" where "AppDev" is the parent of "Prod" compartment
 # Sample import command for Dedicated VM Hosts:
 # terraform import "module.dedicated-hosts[\"<<dedicated_hosts terraform variable name>>\"].oci_core_dedicated_vm_host.dedicated_vm_host" <<dedicated vm host ocid>>
 ############################

cd3_automation_toolkit/Compute/templates/instances-template

@@ -8,10 +8,10 @@
 # vcn_name must be the name of the VCN as in OCI
 # subnet_id can be the ocid of the subnet or the name as in OCI
 # compartment_id and network_compartment_id can be the ocid or the name of the compartment hierarchy delimited by double hiphens "--"
-# Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "Network-root-cpt--Network" where "Network-root-cpt" is the parent of "Network" compartment
+# Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "AppDev--Prod" where "AppDev" is the parent of "Prod" compartment
 # Sample import command for Instance and volume backup policy:
-# terraform import "module.instances[\"<<instances terraform variable name>>\"].oci_core_instance.instance" <<instance ocid>>
-# terraform import "module.instances[\"<<instances terraform variable name>>\"].oci_core_volume_backup_policy_assignment.volume_backup_policy_assignment[0]" <<volume backup policy ocid>>
+# terraform import "module.instances[\"<<instance terraform variable name>>\"].oci_core_instance.instance" <<instance ocid>>
+# terraform import "module.instances[\"<<instance terraform variable name>>\"].oci_core_volume_backup_policy_assignment.volume_backup_policy_assignment[0]" <<volume backup policy ocid>>
 ############################
 
 instances = {

cd3_automation_toolkit/Database/create_terraform_dbsystems_vm_bm.py

@@ -44,6 +44,7 @@ def create_terraform_dbsystems_vm_bm(inputfile, outdir, prefix, config=DEFAULT_L
     oname = {}
     tfStr = {}
     ADS = ["AD1", "AD2", "AD3"]
+    subnets = parseSubnets(filename)
 
     # Load the template file
     file_loader = FileSystemLoader(f'{Path(__file__).parent}/templates')
@@ -157,10 +158,28 @@ def create_terraform_dbsystems_vm_bm(inputfile, outdir, prefix, config=DEFAULT_L
                 tempdict = {'display_tf_name': display_tf_name}
 
             if columnname == 'Subnet Name':
-                columnvalue = commonTools.check_tf_variable(columnvalue)
-
-            if columnname == 'Backup Subnet Name':
-                columnvalue = commonTools.check_tf_variable(columnvalue)
+                subnet_tf_name = columnvalue.strip()
+                if ("ocid1.subnet.oc1" in subnet_tf_name):
+                    network_compartment_id = ""
+                    vcn_name = ""
+                    subnet_id = subnet_tf_name
+                else:
+                    try:
+                        key = region, subnet_tf_name
+                        network_compartment_id = subnets.vcn_subnet_map[key][0]
+                        vcn_name = subnets.vcn_subnet_map[key][1]
+                        subnet_id = subnets.vcn_subnet_map[key][2]
+                    except Exception as e:
+                        print("Invalid Subnet Name specified for row " + str(
+                            i + 3) + ". It Doesnt exist in Subnets sheet. Exiting!!!")
+                        exit()
+
+                tempdict = {'network_compartment_id': commonTools.check_tf_variable(network_compartment_id),
+                            'vcn_name': vcn_name,
+                            'subnet_id': subnet_id}
+
+            # if columnname == 'Backup Subnet Name':
+            #     columnvalue = commonTools.check_tf_variable(columnvalue)
 
 
             if columnname == 'Availability Domain(AD1|AD2|AD3)':

cd3_automation_toolkit/Database/create_terraform_exa_vmclusters.py

@@ -61,6 +61,7 @@ def create_terraform_exa_vmclusters(inputfile, outdir, prefix, config=DEFAULT_LO
 
     # List of the column headers
     dfcolumns = df.columns.values.tolist()
+    subnets = parseSubnets(filename)
 
     # Initialise empty TF string for each region
     for reg in ct.all_regions:
@@ -143,10 +144,44 @@ def create_terraform_exa_vmclusters(inputfile, outdir, prefix, config=DEFAULT_LO
                 tempdict = {'exadata_infra_display_tf_name': display_tf_name}
 
             if columnname == 'Client Subnet Name':
-                columnvalue = commonTools.check_tf_variable(columnvalue)
+                subnet_tf_name = columnvalue.strip()
+                if ("ocid1.subnet.oc1" in subnet_tf_name):
+                    network_compartment_id = ""
+                    vcn_name = ""
+                    subnet_id = subnet_tf_name
+                else:
+                    try:
+                        key = region, subnet_tf_name
+                        network_compartment_id = subnets.vcn_subnet_map[key][0]
+                        vcn_name = subnets.vcn_subnet_map[key][1]
+                        subnet_id = subnets.vcn_subnet_map[key][2]
+                    except Exception as e:
+                        print("Invalid Subnet Name specified for row " + str(
+                            i + 3) + ". It Doesnt exist in Subnets sheet. Exiting!!!")
+                        exit()
+
+                tempdict = {'network_compartment_id': commonTools.check_tf_variable(network_compartment_id),
+                            'vcn_name': vcn_name,
+                            'client_subnet_name': subnet_id}
 
             if columnname == 'Backup Subnet Name':
-                columnvalue = commonTools.check_tf_variable(columnvalue)
+                subnet_tf_name = columnvalue.strip()
+                if ("ocid1.subnet.oc1" in subnet_tf_name):
+                    network_compartment_id = ""
+                    vcn_name = ""
+                    subnet_id = subnet_tf_name
+                else:
+                    try:
+                        key = region, subnet_tf_name
+                        network_compartment_id = subnets.vcn_subnet_map[key][0]
+                        vcn_name = subnets.vcn_subnet_map[key][1]
+                        subnet_id = subnets.vcn_subnet_map[key][2]
+                    except Exception as e:
+                        print("Invalid Subnet Name specified for row " + str(
+                            i + 3) + ". It Doesnt exist in Subnets sheet. Exiting!!!")
+                        exit()
+
+                tempdict = {'backup_subnet_name': subnet_id}
 
             if columnname == 'NSGs':
                 if columnvalue != '':

cd3_automation_toolkit/Database/export_dbsystems_vm_bm_nonGreenField.py

@@ -222,8 +222,6 @@ def export_dbsystems_vm_bm(inputfile, _outdir, _config, network_compartments=[])
 
         with open(script_file, 'a') as importCommands[reg]:
             importCommands[reg].write('\n\nterraform plan\n')
-        if "linux" in sys.platform:
-            os.chmod(script_file, 0o755)
 
     commonTools.write_to_cd3(values_for_column, cd3file, sheetName)
     print("Virtual Machine and Bare Metal DB Systems exported to CD3\n")

cd3_automation_toolkit/Database/export_exa_infra_nonGreenField.py

@@ -126,9 +126,6 @@ def export_exa_infra(inputfile, _outdir, _config, network_compartments=[]):
             for exa_infra in exa_infras.data:
                 print_exa_infra(region, exa_infra,values_for_column, ntk_compartment_name)
 
-        if "linux" in sys.platform:
-            os.chmod(script_file, 0o755)
-
     commonTools.write_to_cd3(values_for_column, cd3file, sheetName)
 
     print("Exadata Infra exported to CD3\n")

cd3_automation_toolkit/Database/export_exa_vmclusters_nonGreenField.py

@@ -201,8 +201,6 @@ def export_exa_vmclusters(inputfile, _outdir, _config, network_compartments=[]):
 
         with open(script_file, 'a') as importCommands[reg]:
             importCommands[reg].write('\n\nterraform plan\n')
-        if "linux" in sys.platform:
-            os.chmod(script_file, 0o755)
 
     commonTools.write_to_cd3(values_for_column, cd3file, sheetName)
 

cd3_automation_toolkit/Database/templates/adb-template

@@ -6,7 +6,7 @@
 # ADB - tfvars
 # Allowed Values:
 # compartment_id and network_compartment_id can be the ocid or the name of the compartment hierarchy delimited by double hiphens "--"
-# Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "Network-root-cpt--Network" where "Network-root-cpt" is the parent of "Network" compartment
+# Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "Database--Prod" where "Database" is the parent of "Prod" compartment
 ############################
 
 adb = {

cd3_automation_toolkit/Database/templates/dbsystems-vm-bm-template

@@ -7,7 +7,7 @@
 # Allowed Values:
 # subnet_id and nsg_ids can be ocids or the key of subnets (map) and nsgs (map)
 # compartment_id and network_compartment_id can be the ocid or the name of the compartment hierarchy delimited by double hiphens "--"
-# Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "Network-root-cpt--Network" where "Network-root-cpt" is the parent of "Network" compartment
+# Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "Database--Prod" where "Database" is the parent of "Prod" compartment
 # Sample import command for DBSystem-VM-BM:
 # terraform import "module.dbsystems-vm-bm[\"<<dbsystems_vm_bm terraform variable name>>\"].oci_database_db_system.database_db_system" <<dbsystem ocid>>
 ############################
@@ -18,7 +18,9 @@ dbsystems_vm_bm = {
             availability_domain  = {{ availability_domain }}
             compartment_id       = "{{ compartment_name }}"
             shape                = "{{ shape }}"
-            subnet_id            = "{{ subnet_name }}"
+            network_compartment_id   = "{{ network_compartment_id }}"
+            vcn_name             = "{{ vcn_name }}"
+            subnet_id            = "{{ subnet_id }}"
             hostname             = "{{ hostname_prefix }}"
 
             {% if cpu_core_count %}

cd3_automation_toolkit/Database/templates/exa-infra-template

@@ -6,7 +6,7 @@
 # ExaInfra - tfvars
 # Allowed Values:
 # compartment_id and network_compartment_id can be the ocid or the name of the compartment hierarchy delimited by double hiphens "--"
-# Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "Network-root-cpt--Network" where "Network-root-cpt" is the parent of "Network" compartment
+# Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "Database--Prod" where "Database" is the parent of "Prod" compartment
 # Sample import command for ExaInfra:
 # terraform import "module.exa-infra[\"<<exa_infra terraform variable name>>\"].oci_database_cloud_exadata_infrastructure.exa_infra" <<exainfra ocid>>
 ############################

cd3_automation_toolkit/Database/templates/exa-vmclusters-template

@@ -7,7 +7,7 @@
 # Allowed Values:
 # cluster_subnet_id, backup_subnet_id and exadata_infrastructure_id can be ocids or the key of subnets (map) and exa_infra (map) respectively
 # compartment_id and network_compartment_id can be the ocid or the name of the compartment hierarchy delimited by double hiphens "--"
-# Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "Network-root-cpt--Network" where "Network-root-cpt" is the parent of "Network" compartment
+# Example : compartment_id = "ocid1.compartment.oc1..aaaaaaaahwwiefb56epvdlzfic6ah6jy3xf3c" or compartment_id = "Database--Prod" where "Database" is the parent of "Prod" compartment
 # Sample import command for ExaVMClusters:
 # terraform import "module.exa-vmclusters[\"<<exa_vmclusters terraform variable name>>\"].oci_database_cloud_vm_cluster.exa_vmcluster" <<exavmcluster ocid>>
 ############################
@@ -18,6 +18,8 @@ exa_vmclusters = {
          {{ vm_cluster_display_tf_name }} = {
             compartment_id           = "{{ compartment_name }}"
             exadata_infrastructure_id= "{{ exadata_infra_display_tf_name }}"
+            network_compartment_id   = "{{ network_compartment_id }}"
+            vcn_name                 = "{{ vcn_name }}"
             cluster_subnet_id        = "{{ client_subnet_name }}"
             gi_version               = "{{ oracle_grid_infrastructure_version }}"
             hostname                 = "{{ hostname_prefix }}"

cd3_automation_toolkit/DeveloperServices/OKE/__init__.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python3
+
+from .create_terraform_oke import create_terraform_oke
+from .export_oke_nonGreenField import export_oke