Merge pull request #87 from oracle-devrel/develop

Automation Toolkit Release v2024.1.0

Author: ttscoff

Dockerfile

@@ -10,25 +10,62 @@ RUN yum install sudo -y && groupadd --gid $USER_GID $USERNAME \
 && useradd --uid $USER_UID --gid $USER_GID -d /$USERNAME -m $USERNAME \
 && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
 && chmod 0440 /etc/sudoers.d/$USERNAME \
-&& mkdir /cd3user/tenancies/ && sudo chown -R $USERNAME:$USERNAME /cd3user/tenancies/
+&& mkdir -p /cd3user/tenancies && sudo chown -R $USERNAME:$USERNAME /cd3user/tenancies/ \
+&& yum install -y vim && echo 'alias vi="vim"' >> /etc/bashrc
 
 
 USER $USERNAME
 WORKDIR /cd3user/oci_tools/
-COPY . .
+COPY cd3_automation_toolkit cd3_automation_toolkit/
+
+WORKDIR /cd3user/
 
 RUN sudo yum install -y oracle-softwarecollection-release-el7 \
 && sudo chown -R $USERNAME:$USERNAME /cd3user/
 
-
-WORKDIR /cd3user/
-
 RUN sudo sed -i -e 's/\r$//' /cd3user/oci_tools/cd3_automation_toolkit/shell_script.sh \
 && bash /cd3user/oci_tools/cd3_automation_toolkit/shell_script.sh \
-&& sudo chown -R cd3user:cd3user /cd3user/ && sudo yum clean all && sudo rm -rf /var/cache/yum /root/ocswork \
-&& sudo rm -rf /cd3user/oci_tools/Dockerfile && sudo rm -rf /cd3user/oci_tools/Readme.md \
-&& sudo rm -rf /cd3user/oci_tools/.git && sudo rm -rf /cd3user/oci_tools/.gitignore && sudo chmod -R 740 /cd3user/
+&& sudo chown -R cd3user:cd3user /cd3user/ && sudo yum clean all && sudo rm -rf /var/cache/yum \
+&& sudo chmod -R 740 /cd3user/
+
+
+##################################### START INSTALLING JENKINS ###################################
+ARG JENKINS_VERSION=2.401.1
+ARG JENKINS_SHA=600b73eabf797852e39919541b84f7686ff601b97c77b44eb00843eb91c7dd6c
+ARG JENKINS_PLUGIN_MANAGER_VERSION=2.12.13
+ARG PLUGIN_CLI_URL=https://github.com/jenkinsci/plugin-installation-manager-tool/releases/download/${JENKINS_PLUGIN_MANAGER_VERSION}/jenkins-plugin-manager-${JENKINS_PLUGIN_MANAGER_VERSION}.jar
+
+ARG JENKINS_HOME=/cd3user/tenancies/jenkins_home
+ARG JENKINS_INSTALL=/usr/share/jenkins
+ARG REF=/usr/share/jenkins/ref
+
+RUN sudo yum remove java-1.8.0-openjdk-1.8.0.345.b01-1.el7_9.x86_64 \
+&& sudo yum install -y java-11-openjdk  \
+&& sudo yum install -y java-11-openjdk-devel \
+&& sudo yum install unzip -y \
+&& sudo yum install git -y \
+&& sudo mkdir -p ${REF}/init.groovy.d \
+&& sudo chown -R cd3user:cd3user ${JENKINS_INSTALL} \
+&& sudo curl -fsSL http://updates.jenkins-ci.org/download/war/${JENKINS_VERSION}/jenkins.war -o ${JENKINS_INSTALL}/jenkins.war \
+&& echo "${JENKINS_SHA}  ${JENKINS_INSTALL}/jenkins.war" | sha256sum -c - \
+&& sudo curl -fsSL ${PLUGIN_CLI_URL} -o ${JENKINS_INSTALL}/jenkins-plugin-manager.jar
 
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-11.0.17.0.8-2.el8_6.x86_64
+ENV JENKINS_HOME ${JENKINS_HOME}
+ENV JENKINS_INSTALL ${JENKINS_INSTALL}
+ENV REF ${REF}
+ENV JENKINS_UC https://updates.jenkins.io
+ENV JENKINS_UC_EXPERIMENTAL=https://updates.jenkins.io/experimental
+ENV JENKINS_INCREMENTALS_REPO_MIRROR=https://repo.jenkins-ci.org/incrementals
+ENV JAVA_OPTS="-Djenkins.install.runSetupWizard=false"
+ENV COPY_REFERENCE_FILE_LOG ${JENKINS_HOME}/copy_reference_file.log
+ENV CASC_JENKINS_CONFIG ${JENKINS_HOME}/jcasc.yaml
 
+COPY --chown=cd3user:cd3user jenkins_install ${JENKINS_INSTALL}/
+COPY --chown=cd3user:cd3user jenkins_install/init/*.groovy ${REF}/init.groovy.d/
+COPY --chown=cd3user:cd3user jenkins_install/plugins.txt ${REF}/plugins.txt
 
 
+RUN sudo java -jar ${JENKINS_INSTALL}/jenkins-plugin-manager.jar --war ${JENKINS_INSTALL}/jenkins.war --verbose -f ${REF}/plugins.txt \
+&& sudo chown -R cd3user:cd3user ${JENKINS_INSTALL} \
+&& sudo chmod +x ${JENKINS_INSTALL}/jenkins.sh

README.md

@@ -1,86 +1,88 @@
 # CD3 Automation Toolkit
 
 [![License: UPL](https://img.shields.io/badge/license-UPL-green)](https://img.shields.io/badge/license-UPL-green) [![Quality gate](https://sonarcloud.io/api/project_badges/quality_gate?project=oracle-devrel_cd3-automation-toolkit)](https://sonarcloud.io/dashboard?id=oracle-devrel_cd3-automation-toolkit)
-
-
-
 <ul>
-
   <li> <a href= "#introduction">Introduction</a></li>
-
-  <li> <a href = "https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v12.1">What's new in this release</a></li>
+  <li> <a href = "https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2024.1.0">What's new in this release</a></li>
   <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/CD3ExcelTabs.md">Toolkit Supported OCI Services</a></li>
-  <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/RunningAutomationToolkit.md#excel-sheet-templates">Excel Templates</a></li>
-  
+  <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/ExcelTemplates.md">Excel Templates</a></li>  
 </ul>
 
-### New Users
 
+### New Users
 <ul>
-  
   <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/prerequisites.md">Prerequisites</a></li>
-  <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/Launch_Docker_container.md">Launch Docker container</a></li>
+  <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/Launch_Docker_container.md">Launch the container</a></li>
   <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/Connect_container_to_OCI_Tenancy.md">Connect container to OCI Tenancy</a></li>
-  <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/RunningAutomationToolkit.md">Getting Started with Automation Toolkit</a></li>
-
+</ul>
 
-  <li> <a href="/cd3_automation_toolkit/documentation/user_guide/Workflows.md">Using the Automation Toolkit</a>
-  <ul>
-  <li> <a href="/cd3_automation_toolkit/documentation/user_guide/GreenField.md">Green Field Tenancies</a>
+#### Using the Automation Toolkit via CLI
+<ul>
+  <li> <a href="/cd3_automation_toolkit/documentation/user_guide/Workflows.md">Before you Begin</a></li>
+  <li> <a href="/cd3_automation_toolkit/documentation/user_guide/GreenField.md">Create resources in OCI (Greenfield Workflow)</a>
     <ul>
       <li> <a href="/cd3_automation_toolkit/documentation/user_guide/learn_more/OPAForCompliance.md"</a> Enforcing OPA (Open Policy Agent) policies for Terraform </li>
-      <li> <a href="/cd3_automation_toolkit/documentation/user_guide/NetworkingScenariosGF.md"</a><b> Must Read :</b> Managing Network for Greenfield Tenancies</li>
-      <li> <a href="/cd3_automation_toolkit/documentation/user_guide/ComputeGF.md"</a><b> Must Read :</b> Managing Compute Instances for Greefield Tenancies</li>
-    </ul>
+      <li> <a href="/cd3_automation_toolkit/documentation/user_guide/NetworkingScenariosGF.md"</a><b> Must Read :</b> Managing Network for Greenfield Workflow</li>
+      <li> <a href="/cd3_automation_toolkit/documentation/user_guide/ComputeGF.md"</a><b> Must Read :</b> Managing Compute Instances for Greefield Workflow</li>
     </ul>
   </li>
-  <ul>
-  <li> <a href="/cd3_automation_toolkit/documentation/user_guide/NonGreenField.md">Non Greenfield Tenancies</a>
+  <li> <a href="/cd3_automation_toolkit/documentation/user_guide/NonGreenField.md">Export Resources from OCI (Non-Greenfield Workflow)</a>
     <ul>
-      <li><a href ="/cd3_automation_toolkit/documentation/user_guide/NetworkingScenariosNGF.md"</a><b> Must Read : </b> Managing Network for Non-Greenfield Tenancies</li>
-       <li><a href = "/cd3_automation_toolkit/documentation/user_guide/ComputeNGF.md"</a><b>  Must Read : </b> Managing Compute Instances for Non-Greenfield Tenancies </li> 
+      <li><a href ="/cd3_automation_toolkit/documentation/user_guide/NetworkingScenariosNGF.md"</a><b> Must Read : </b> Managing Network for Non-Greenfield Workflow</a></li>
+       <li><a href = "/cd3_automation_toolkit/documentation/user_guide/ComputeNGF.md"</a><b>  Must Read : </b> Managing Compute Instances for Non-Greenfield Workflow </a></li> 
     </ul>
-  </ul>
-  
-  
+</ul>
+        
+#### Using the Automation Toolkit via Jenkins
+<ul>
+  <li> <a href="/cd3_automation_toolkit/documentation/user_guide/Workflows-jenkins.md"</a>Before you Begin</a></li>
+  <li> <a href="/cd3_automation_toolkit/documentation/user_guide/Intro-Jenkins.md"</a>Introduction to Jenkins with the toolkit</a></li>
+  <li> <a href="/cd3_automation_toolkit/documentation/user_guide/GreenField-Jenkins.md"</a>Create resources in OCI via Jenkins(Greenfield Workflow)</li>
+    <ul>
+      <li> <a href="/cd3_automation_toolkit/documentation/user_guide/NetworkingScenariosGF-Jenkins.md"</a><b> Must Read :</b> Managing Network for Greenfield Workflow</li>
+      <li> <a href="/cd3_automation_toolkit/documentation/user_guide/GF-Jenkins.md"</a><b>  Must Read : </b> Provisioning of Resources - Instances/OKE/SDDC/Database </li>
+      <li> <a href="/cd3_automation_toolkit/documentation/user_guide/multiple_options_GF-Jenkins.md"</a><b>  Must Read : </b> Provisioning of multiple services together </li>
+    </ul>
+      <li> <a href="/cd3_automation_toolkit/documentation/user_guide/NonGreenField-Jenkins.md"</a>Export Resources from OCI via Jenkins(Non-Greenfield Workflow) </li>     
+  <li> <a href="/cd3_automation_toolkit/documentation/user_guide/cli_jenkins.md">Switch between CLI and Jenkins</a></li>
+  <li> <a href="/cd3_automation_toolkit/documentation/user_guide/remote_state.md">Remote Management of Terraform State File</a></li>
  </ul>
 
  ### Existing Users
-
-<ul>
- 
+<ul> 
   <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/Upgrade_Toolkit.md">Steps to Upgrade Your Toolkit</a></li>
- 
+  <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/Jobs_Migration.md">Migrating Jenkins Jobs</a></li>
 </ul> 
 
-### Learn More...
+### Videos
+<ul>
+  <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/LearningVideos.md">Automation Toolkit Learning Videos</a></li>
+</ul>
 
+### Known Behaviour
 <ul>
+  <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/KnownBehaviour.md">Expected Behaviour Of Automation Toolkit</a></li>
+   <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/FAQ.md">FAQs</a></li> 
+</ul>
 
+### Learn More...
+<ul>
    <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/RestructuringOutDirectory.md">Grouping of generated Terraform files</a></li>
    <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/ResourceManagerUpload.md">OCI Resource Manager Upload</a></li>
-   <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/OPAForCompliance.md">OPA For Compliance with Terraform</a></li>
    <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/CISFeatures.md">Additional CIS Compliance Features</a></li>
-   <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/SupportForCD3Validator.md">CD3 Validator Features</a></li>
- 
-   
+   <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/SupportForCD3Validator.md">CD3 Validator Features</a></li>   
    <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/SupportforAdditionalAttributes.md">Support for Additional Attributes</a></li>
-   <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/LearningVideos.md">Automation Toolkit Learning Videos</a></li>
-   <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/KnownBehaviour.md">Expected Behaviour Of Automation Toolkit</a></li>
-   <li> <a href = "/cd3_automation_toolkit/documentation/user_guide/FAQ.md">FAQs</a></li>
- 
 </ul>
 
-
-
 ## Introduction
 CD3 stands for <b>C</b>loud <b>D</b>eployment <b>D</b>esign <b>D</b>eliverable.
 The CD3 Automation toolkit has been developed to help in automating the OCI resource object management. 
 <br><br>
 It reads input data in the form of CD3 Excel sheet and generates Terraform files which can be used to provision the resources in OCI instead of handling the task through the OCI console manually. The toolkit also reverse engineers the components in OCI back to the Excel sheet and Terraform configuration. The toolkit can be used throughout the lifecycle of tenancy to continuously create or modify existing resources. The generated Terraform code can be used by the OCI Resource Manager or can be integrated into organization's existing devops CI/CD ecosystem.
 <br><br>
 <kbd>
-<img width="748" alt="Screenshot 2022-12-30 at 11 57 41 AM" src="https://user-images.githubusercontent.com/111430850/210614513-5d2e97a6-3c1e-4a2b-a793-3a1b6410c856.png">
+<img width="923" alt="Screenshot 2024-02-02 at 8 23 51 PM" src="https://github.com/oracle-devrel/cd3-automation-toolkit/assets/103508105/c2d06575-1bcb-4cb3-9c49-682bf19f3ac9">
+
 </kbd>
 <br>
 
@@ -98,11 +100,10 @@ It reads input data in the form of CD3 Excel sheet and generates Terraform files
 | [Management Services](/cd3_automation_toolkit/documentation/user_guide/learn_more/CD3ExcelTabs.md#management-services) | Events, Notifications, Alarms, Service Connector Hub (SCH) |
 | [Developer Services](/cd3_automation_toolkit/documentation/user_guide/learn_more/CD3ExcelTabs.md#developer-services) | Resource Manager, Oracle Kubernetes Engine (OKE) |
 | [Logging Services](/cd3_automation_toolkit/documentation/user_guide/learn_more/CD3ExcelTabs.md#logging-Services) | VCN Flow Logs, LBaaS access and error Logs, OSS Buckets write Logs |
-| [SDDCs ](/cd3_automation_toolkit/documentation/user_guide/learn_more/CD3ExcelTabs.md#sddcs-tab) | Oracle Cloud VMWare Solutions |
+| [SDDCs ](/cd3_automation_toolkit/documentation/user_guide/learn_more/CD3ExcelTabs.md#sddcs-tab) | Oracle Cloud VMWare Solutions (Single Cluster is supported as of now. Multi-cluster support will be included in the upcoming release) |
 | [CIS Landing Zone Compliance](/cd3_automation_toolkit/documentation/user_guide/learn_more/CISFeatures.md#additional-cis-compliance-features) | Download and Execute CIS Compliance Check Script, Cloud Guard, Key Vault, Budget |
 [Policy Enforcement](/cd3_automation_toolkit/documentation/user_guide/learn_more/OPAForCompliance.md) | OPA - Open Policy Agent |
 
-
 [Click here](/cd3_automation_toolkit/documentation/user_guide/prerequisites.md) to get started and manage your OCI Infra!
 
 ## Contributing

cd3_automation_toolkit/Compute/create_terraform_dedicatedhosts.py

@@ -25,19 +25,16 @@
 
 # If input is CD3 excel file
 # Execution of the code begins here
-def create_terraform_dedicatedhosts(inputfile, outdir, service_dir,prefix, config):
+def create_terraform_dedicatedhosts(inputfile, outdir, service_dir,prefix, ct):
     # Load the template file
     file_loader = FileSystemLoader(f'{Path(__file__).parent}/templates')
     env = Environment(loader=file_loader, keep_trailing_newline=True, trim_blocks=True, lstrip_blocks=True)
     template = env.get_template('dedicatedvmhosts-template')
 
     filename = inputfile
-    configFileName = config
 
     sheetName = "DedicatedVMHosts"
     auto_tfvars_filename = prefix + '_' + sheetName.lower() + '.auto.tfvars'
-    ct = commonTools()
-    ct.get_subscribedregions(configFileName)
 
     outfile = {}
     oname = {}

cd3_automation_toolkit/Compute/create_terraform_instances.py

@@ -22,18 +22,15 @@
 
 # If input is CD3 excel file
 # Execution of the code begins here
-def create_terraform_instances(inputfile, outdir, service_dir, prefix, config):
+def create_terraform_instances(inputfile, outdir, service_dir, prefix, ct):
     boot_policy_tfStr = {}
     tfStr = {}
     ADS = ["AD1", "AD2", "AD3"]
 
     filename = inputfile
-    configFileName = config
 
     sheetName = "Instances"
     auto_tfvars_filename = prefix + '_' + sheetName.lower() + '.auto.tfvars'
-    ct = commonTools()
-    ct.get_subscribedregions(configFileName)
 
     # Load the template file
     file_loader = FileSystemLoader(f'{Path(__file__).parent}/templates')
@@ -153,7 +150,7 @@ def create_terraform_instances(inputfile, outdir, service_dir, prefix, config):
                     except Exception as e:
                         print("Invalid Subnet Name specified for row " + str(
                             i + 3) + ". It Doesnt exist in Subnets sheet. Exiting!!!")
-                        exit()
+                        exit(1)
 
                 tempdict = {'network_compartment_id': commonTools.check_tf_variable(network_compartment_id),
                             'vcn_name': vcn_name,

cd3_automation_toolkit/Compute/export_dedicatedvmhosts_nonGreenField.py

@@ -44,14 +44,12 @@ def print_dedicatedvmhosts(region, dedicatedvmhost, values_for_column, ntk_compa
             values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column)
 
 # Execution of the code begins here
-def export_dedicatedvmhosts(inputfile, _outdir, service_dir, _config, ct, export_compartments=[], export_regions=[]):
+def export_dedicatedvmhosts(inputfile, outdir, service_dir, config, signer, ct, export_compartments=[], export_regions=[]):
     global tf_import_cmd
     global sheet_dict
     global importCommands
-    global config
     global cd3file
     global reg
-    global outdir
     global values_for_column
 
 
@@ -60,16 +58,7 @@ def export_dedicatedvmhosts(inputfile, _outdir, service_dir, _config, ct, export
         print("\nAcceptable cd3 format: .xlsx")
         exit()
 
-
-    outdir = _outdir
-    configFileName = _config
-    config = oci.config.from_file(file_location=configFileName)
-
     sheetName="DedicatedVMHosts"
-    if ct==None:
-        ct = commonTools()
-        ct.get_subscribedregions(configFileName)
-        ct.get_network_compartment_ids(config['tenancy'],"root",configFileName)
 
     # Read CD3
     df, values_for_column= commonTools.read_cd3(cd3file,sheetName)
@@ -100,14 +89,20 @@ def export_dedicatedvmhosts(inputfile, _outdir, service_dir, _config, ct, export
         config.__setitem__("region", ct.region_dict[reg])
         region = reg.capitalize()
 
-        compute_client = oci.core.ComputeClient(config,retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY)
+        compute_client = oci.core.ComputeClient(config=config,retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY, signer=signer)
 
         for ntk_compartment_name in export_compartments:
             dedicatedvmhosts = oci.pagination.list_call_get_all_results(compute_client.list_dedicated_vm_hosts,compartment_id=ct.ntk_compartment_ids[ntk_compartment_name], lifecycle_state="ACTIVE")
             for dedicatedvmhost in dedicatedvmhosts.data:
                 dedicatedvmhost=compute_client.get_dedicated_vm_host(dedicatedvmhost.id).data
                 print_dedicatedvmhosts(region, dedicatedvmhost,values_for_column, ntk_compartment_name)
 
+    # write data into file
+    for reg in export_regions:
+        script_file = f'{outdir}/{reg}/{service_dir}/'+file_name
+        with open(script_file, 'a') as importCommands[reg]:
+            importCommands[reg].write('\n\nterraform plan\n')
+
     commonTools.write_to_cd3(values_for_column, cd3file, "DedicatedVMHosts")
 
     print("Dedicated VM Hosts exported to CD3\n")