Merge pull request #38 from oracle-devrel/develop

Develop to main for v10.1

Author: xs2suruchi

README.md

@@ -18,15 +18,16 @@
 8. [Creating independent tfstate files for each resource](/cd3_automation_toolkit/documentation/user_guide/RestructuringOutDirectory.md)
 9. [Expected Behaviour Of Automation Toolkit](/cd3_automation_toolkit/documentation/user_guide/KnownBehaviour.md)
 10. [FAQs](/cd3_automation_toolkit/documentation/user_guide/FAQ.md)
-11. Learn More...
+11. [Steps to Upgrade Your Toolkit](/cd3_automation_toolkit/documentation/user_guide/Upgrade_Toolkit.md)
+12. Learn More...
     - CD3 Excel Information
       - [Excel Templates](/cd3_automation_toolkit/documentation/user_guide/RunningAutomationToolkit.md#excel-sheet-templates)
       - [Toolkit Supported OCI Services](/cd3_automation_toolkit/documentation/user_guide/learn_more/CD3ExcelTabs.md)
     - [CD3 Validator Features](/cd3_automation_toolkit/documentation/user_guide/learn_more/SupportForCD3Validator.md)
     - [Additional CIS Compliance Features](/cd3_automation_toolkit/documentation/user_guide/learn_more/CISFeatures.md)
     - [OCI Resource Manager Upload](/cd3_automation_toolkit/documentation/user_guide/learn_more/ResourceManagerUpload.md)
     - [Support for Additional Attributes](/cd3_automation_toolkit/documentation/user_guide/learn_more/SupportforAdditionalAttributes.md)
-12. [Automation Toolkit Learning Videos](/cd3_automation_toolkit/documentation/user_guide/LearningVideos.md)
+13. [Automation Toolkit Learning Videos](/cd3_automation_toolkit/documentation/user_guide/LearningVideos.md)
 
 ## Introduction
 CD3 stands for <b>C</b>loud <b>D</b>eployment <b>D</b>esign <b>D</b>eliverable.
@@ -39,7 +40,7 @@ It reads input data in the form of CD3 Excel sheet and generates the terraform f
 </kbd>
 <br>
 
-#### OCI Services Currently Supported by CD3
+#### OCI Services Currently Supported by Automation Toolkit
 
 | OCI Services | Details |
 | --------- | ----------- |
@@ -62,7 +63,7 @@ It reads input data in the form of CD3 Excel sheet and generates the terraform f
 Appropriate IAM policies must be in place for each of the resources that the user may try to create.
 Minimum requirement for the user to get started is to have the ability to read to the tenancy.
 
-[Click here](/cd3_automation_toolkit/documentation/user_guide/Launch_Docker_container.md) to get started and manage your OCI Infra!!! 
+[Click here](/cd3_automation_toolkit/documentation/user_guide/Launch_Docker_container.md) to get started and manage your OCI Infra!
 
 ## Contributing
 This project is open source.  Please submit your contributions by raising an <b>Issue</b> or through <b>Discussion topic</b> in this repository. Currently, we do not accept any pull requests. Oracle appreciates any contributions that are made by the open source community.

cd3_automation_toolkit/Compute/create_terraform_dedicatedhosts.py

@@ -28,13 +28,14 @@ def parse_args():
     parser = argparse.ArgumentParser(description='Create Dedicated VM Hosts terraform file')
     parser.add_argument('inputfile', help='Full Path of input CD3 excel file')
     parser.add_argument('outdir', help='Output directory for creation of TF files')
+    parser.add_argument("service_dir",help="subdirectory under region directory in case of separate out directory structure")
     parser.add_argument('prefix', help='TF files prefix')
     parser.add_argument('--config', default=DEFAULT_LOCATION, help='Config file name')
     return parser.parse_args()
 
 
 # If input is CD3 excel file
-def create_terraform_dedicatedhosts(inputfile, outdir, prefix, config):
+def create_terraform_dedicatedhosts(inputfile, outdir, service_dir,prefix, config):
     # Load the template file
     file_loader = FileSystemLoader(f'{Path(__file__).parent}/templates')
     env = Environment(loader=file_loader, keep_trailing_newline=True, trim_blocks=True, lstrip_blocks=True)
@@ -62,7 +63,7 @@ def create_terraform_dedicatedhosts(inputfile, outdir, prefix, config):
     # Take backup of files
     for eachregion in ct.all_regions:
         resource = sheetName.lower()
-        srcdir = outdir + "/" + eachregion + "/"
+        srcdir = outdir + "/" + eachregion + "/" + service_dir + "/"
         commonTools.backup_file(srcdir, resource, auto_tfvars_filename)
         tfStr[eachregion] = ''
 
@@ -133,7 +134,7 @@ def create_terraform_dedicatedhosts(inputfile, outdir, prefix, config):
 
     # Write to output
     for reg in ct.all_regions:
-        reg_out_dir = outdir + "/" + reg
+        reg_out_dir = outdir + "/" + reg + "/" + service_dir
         if not os.path.exists(reg_out_dir):
             os.makedirs(reg_out_dir)
 
@@ -158,4 +159,4 @@ def create_terraform_dedicatedhosts(inputfile, outdir, prefix, config):
 if __name__ == '__main__':
     args = parse_args()
     # Execution of the code begins here
-    create_terraform_dedicatedhosts(args.inputfile, args.outdir, args.prefix, args.config)
+    create_terraform_dedicatedhosts(args.inputfile, args.outdir, args.service_dir,args.prefix, args.config)

cd3_automation_toolkit/Compute/create_terraform_instances.py

@@ -24,13 +24,14 @@ def parse_args():
     parser = argparse.ArgumentParser(description='Creates Instances TF file')
     parser.add_argument('inputfile', help='Full Path of input CD3 excel file')
     parser.add_argument('outdir', help='Output directory for creation of TF files')
+    parser.add_argument("service_dir",help="subdirectory under region directory in case of separate out directory structure")
     parser.add_argument('prefix', help='TF files prefix')
     parser.add_argument('--config', default=DEFAULT_LOCATION, help='Config file name')
     return parser.parse_args()
 
 
 # If input is CD3 excel file
-def create_terraform_instances(inputfile, outdir, prefix, config):
+def create_terraform_instances(inputfile, outdir, service_dir,prefix, config):
     boot_policy_tfStr = {}
     tfStr = {}
     ADS = ["AD1", "AD2", "AD3"]
@@ -59,7 +60,7 @@ def create_terraform_instances(inputfile, outdir, prefix, config):
     # Take backup of files
     for eachregion in ct.all_regions:
         resource = sheetName.lower()
-        srcdir = outdir + "/" + eachregion + "/"
+        srcdir = outdir + "/" + eachregion+ "/" + service_dir + "/"
         commonTools.backup_file(srcdir, resource, auto_tfvars_filename)
         tfStr[eachregion] = ''
         boot_policy_tfStr[eachregion] = ''
@@ -106,7 +107,7 @@ def create_terraform_instances(inputfile, outdir, prefix, config):
             df.loc[i, 'Availability Domain(AD1|AD2|AD3)']).lower() == 'nan' or str(
             df.loc[i, 'Subnet Name']).lower() == 'nan' or str(df.loc[i, 'Source Details']).lower() == 'nan'):
             print(
-                "\nColumn Region, Shape, Compartment Name, Availability Domain, Display Name, Pub Address, Source Details and Subnet Name cannot be left empty in Instances sheet of CD3..exiting...")
+                "\nOne/All of the Column/Columns from Region, Shape, Compartment Name, Availability Domain, Display Name, Pub Address, Source Details and Subnet Name is empty in Instances sheet of CD3..exiting...Please check.")
             exit(1)
 
         for columnname in dfcolumns:
@@ -214,7 +215,7 @@ def create_terraform_instances(inputfile, outdir, prefix, config):
     # Write TF string to the file in respective region directory
     for reg in ct.all_regions:
 
-        reg_out_dir = outdir + "/" + reg
+        reg_out_dir = outdir + "/" + reg + "/" + service_dir
         if not os.path.exists(reg_out_dir):
             os.makedirs(reg_out_dir)
 

cd3_automation_toolkit/Compute/export_dedicatedvmhosts_nonGreenField.py

@@ -9,7 +9,6 @@
 #
 
 import argparse
-import sys
 import oci
 import os
 
@@ -51,12 +50,15 @@ def parse_args():
     parser = argparse.ArgumentParser(description="Export Block Volumes on OCI to CD3")
     parser.add_argument("inputfile", help="path of CD3 excel file to export Block Volume objects to")
     parser.add_argument("outdir", help="path to out directory containing script for TF import commands")
+    parser.add_argument("service_dir",help="subdirectory under region directory in case of separate out directory structure")
     parser.add_argument("--config", default=DEFAULT_LOCATION, help="Config file name")
-    parser.add_argument("--network-compartments", nargs='*', required=False, help="comma seperated Compartments for which to export Block Volume Objects")
+    parser.add_argument("--export-compartments", nargs='*', required=False, help="comma seperated Compartments for which to export Block Volume Objects")
+    parser.add_argument("--export-regions", nargs='*', help="comma seperated Regions for which to export Networking Objects",
+                        required=False)
     return parser.parse_args()
 
 
-def export_dedicatedvmhosts(inputfile, _outdir, _config, network_compartments=[]):
+def export_dedicatedvmhosts(inputfile, _outdir, service_dir, _config, ct, export_compartments=[], export_regions=[]):
     global tf_import_cmd
     global sheet_dict
     global importCommands
@@ -78,30 +80,27 @@ def export_dedicatedvmhosts(inputfile, _outdir, _config, network_compartments=[]
     config = oci.config.from_file(file_location=configFileName)
 
     sheetName="DedicatedVMHosts"
-    ct = commonTools()
-    ct.get_subscribedregions(configFileName)
-    ct.get_network_compartment_ids(config['tenancy'],"root",configFileName)
+    if ct==None:
+        ct = commonTools()
+        ct.get_subscribedregions(configFileName)
+        ct.get_network_compartment_ids(config['tenancy'],"root",configFileName)
 
     # Read CD3
     df, values_for_column= commonTools.read_cd3(cd3file,sheetName)
 
     # Get dict for columns from Excel_Columns
     sheet_dict=ct.sheet_dict[sheetName]
 
-    # Check Compartments
-    global comp_list_fetch
-    comp_list_fetch = commonTools.get_comp_list_for_export(network_compartments, ct.ntk_compartment_ids)
-
     print("\nCD3 excel file should not be opened during export process!!!")
     print("Tabs- DedicatedVMHosts  will be overwritten during export process!!!\n")
 
     # Create backups
     resource = 'tf_import_' + sheetName.lower()
     file_name = 'tf_import_commands_' + sheetName.lower() + '_nonGF.sh'
-    for reg in ct.all_regions:
-        script_file = f'{outdir}/{reg}/'+file_name
+    for reg in export_regions:
+        script_file = f'{outdir}/{reg}/{service_dir}/'+file_name
         if (os.path.exists(script_file)):
-            commonTools.backup_file(outdir + "/" + reg, resource, file_name)
+            commonTools.backup_file(outdir + "/" + reg+"/"+service_dir, resource, file_name)
         importCommands[reg] = open(script_file, "w")
         importCommands[reg].write("#!/bin/bash")
         importCommands[reg].write("\n")
@@ -110,14 +109,14 @@ def export_dedicatedvmhosts(inputfile, _outdir, _config, network_compartments=[]
     # Fetch Block Volume Details
     print("\nFetching details of Dedicated VM Hosts...")
 
-    for reg in ct.all_regions:
+    for reg in export_regions:
         importCommands[reg].write("\n\n######### Writing import for Dedicated VM Hosts #########\n\n")
         config.__setitem__("region", ct.region_dict[reg])
         region = reg.capitalize()
 
         compute_client = oci.core.ComputeClient(config,retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY)
 
-        for ntk_compartment_name in comp_list_fetch:
+        for ntk_compartment_name in export_compartments:
             dedicatedvmhosts = oci.pagination.list_call_get_all_results(compute_client.list_dedicated_vm_hosts,compartment_id=ct.ntk_compartment_ids[ntk_compartment_name], lifecycle_state="ACTIVE")
             for dedicatedvmhost in dedicatedvmhosts.data:
                 dedicatedvmhost=compute_client.get_dedicated_vm_host(dedicatedvmhost.id).data
@@ -131,4 +130,4 @@ def export_dedicatedvmhosts(inputfile, _outdir, _config, network_compartments=[]
 if __name__ == '__main__':
     args = parse_args()
     # Execution of the code begins here
-    export_dedicatedvmhosts(args.inputfile, args.outdir, args.config, args.network_compartments)
+    export_dedicatedvmhosts(args.inputfile, args.outdir, args.service_dir,args.config, args.export_compartments, args.export_regions)