Skip to content

Commit 7db16a3

Browse files
authored
Merge pull request #87 from oracle-devrel/develop
Automation Toolkit Release v2024.1.0
2 parents 1263f29 + fba8903 commit 7db16a3

File tree

240 files changed

+12426
-2061
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

240 files changed

+12426
-2061
lines changed

Dockerfile

Lines changed: 45 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,25 +10,62 @@ RUN yum install sudo -y && groupadd --gid $USER_GID $USERNAME \
1010
&& useradd --uid $USER_UID --gid $USER_GID -d /$USERNAME -m $USERNAME \
1111
&& echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
1212
&& chmod 0440 /etc/sudoers.d/$USERNAME \
13-
&& mkdir /cd3user/tenancies/ && sudo chown -R $USERNAME:$USERNAME /cd3user/tenancies/
13+
&& mkdir -p /cd3user/tenancies && sudo chown -R $USERNAME:$USERNAME /cd3user/tenancies/ \
14+
&& yum install -y vim && echo 'alias vi="vim"' >> /etc/bashrc
1415

1516

1617
USER $USERNAME
1718
WORKDIR /cd3user/oci_tools/
18-
COPY . .
19+
COPY cd3_automation_toolkit cd3_automation_toolkit/
20+
21+
WORKDIR /cd3user/
1922

2023
RUN sudo yum install -y oracle-softwarecollection-release-el7 \
2124
&& sudo chown -R $USERNAME:$USERNAME /cd3user/
2225

23-
24-
WORKDIR /cd3user/
25-
2626
RUN sudo sed -i -e 's/\r$//' /cd3user/oci_tools/cd3_automation_toolkit/shell_script.sh \
2727
&& bash /cd3user/oci_tools/cd3_automation_toolkit/shell_script.sh \
28-
&& sudo chown -R cd3user:cd3user /cd3user/ && sudo yum clean all && sudo rm -rf /var/cache/yum /root/ocswork \
29-
&& sudo rm -rf /cd3user/oci_tools/Dockerfile && sudo rm -rf /cd3user/oci_tools/Readme.md \
30-
&& sudo rm -rf /cd3user/oci_tools/.git && sudo rm -rf /cd3user/oci_tools/.gitignore && sudo chmod -R 740 /cd3user/
28+
&& sudo chown -R cd3user:cd3user /cd3user/ && sudo yum clean all && sudo rm -rf /var/cache/yum \
29+
&& sudo chmod -R 740 /cd3user/
30+
31+
32+
##################################### START INSTALLING JENKINS ###################################
33+
ARG JENKINS_VERSION=2.401.1
34+
ARG JENKINS_SHA=600b73eabf797852e39919541b84f7686ff601b97c77b44eb00843eb91c7dd6c
35+
ARG JENKINS_PLUGIN_MANAGER_VERSION=2.12.13
36+
ARG PLUGIN_CLI_URL=https://github.com/jenkinsci/plugin-installation-manager-tool/releases/download/${JENKINS_PLUGIN_MANAGER_VERSION}/jenkins-plugin-manager-${JENKINS_PLUGIN_MANAGER_VERSION}.jar
37+
38+
ARG JENKINS_HOME=/cd3user/tenancies/jenkins_home
39+
ARG JENKINS_INSTALL=/usr/share/jenkins
40+
ARG REF=/usr/share/jenkins/ref
41+
42+
RUN sudo yum remove java-1.8.0-openjdk-1.8.0.345.b01-1.el7_9.x86_64 \
43+
&& sudo yum install -y java-11-openjdk \
44+
&& sudo yum install -y java-11-openjdk-devel \
45+
&& sudo yum install unzip -y \
46+
&& sudo yum install git -y \
47+
&& sudo mkdir -p ${REF}/init.groovy.d \
48+
&& sudo chown -R cd3user:cd3user ${JENKINS_INSTALL} \
49+
&& sudo curl -fsSL http://updates.jenkins-ci.org/download/war/${JENKINS_VERSION}/jenkins.war -o ${JENKINS_INSTALL}/jenkins.war \
50+
&& echo "${JENKINS_SHA} ${JENKINS_INSTALL}/jenkins.war" | sha256sum -c - \
51+
&& sudo curl -fsSL ${PLUGIN_CLI_URL} -o ${JENKINS_INSTALL}/jenkins-plugin-manager.jar
3152

53+
ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-11.0.17.0.8-2.el8_6.x86_64
54+
ENV JENKINS_HOME ${JENKINS_HOME}
55+
ENV JENKINS_INSTALL ${JENKINS_INSTALL}
56+
ENV REF ${REF}
57+
ENV JENKINS_UC https://updates.jenkins.io
58+
ENV JENKINS_UC_EXPERIMENTAL=https://updates.jenkins.io/experimental
59+
ENV JENKINS_INCREMENTALS_REPO_MIRROR=https://repo.jenkins-ci.org/incrementals
60+
ENV JAVA_OPTS="-Djenkins.install.runSetupWizard=false"
61+
ENV COPY_REFERENCE_FILE_LOG ${JENKINS_HOME}/copy_reference_file.log
62+
ENV CASC_JENKINS_CONFIG ${JENKINS_HOME}/jcasc.yaml
3263

64+
COPY --chown=cd3user:cd3user jenkins_install ${JENKINS_INSTALL}/
65+
COPY --chown=cd3user:cd3user jenkins_install/init/*.groovy ${REF}/init.groovy.d/
66+
COPY --chown=cd3user:cd3user jenkins_install/plugins.txt ${REF}/plugins.txt
3367

3468

69+
RUN sudo java -jar ${JENKINS_INSTALL}/jenkins-plugin-manager.jar --war ${JENKINS_INSTALL}/jenkins.war --verbose -f ${REF}/plugins.txt \
70+
&& sudo chown -R cd3user:cd3user ${JENKINS_INSTALL} \
71+
&& sudo chmod +x ${JENKINS_INSTALL}/jenkins.sh

README.md

Lines changed: 45 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,86 +1,88 @@
11
# CD3 Automation Toolkit
22

33
[![License: UPL](https://img.shields.io/badge/license-UPL-green)](https://img.shields.io/badge/license-UPL-green) [![Quality gate](https://sonarcloud.io/api/project_badges/quality_gate?project=oracle-devrel_cd3-automation-toolkit)](https://sonarcloud.io/dashboard?id=oracle-devrel_cd3-automation-toolkit)
4-
5-
6-
74
<ul>
8-
95
<li> <a href= "#introduction">Introduction</a></li>
10-
11-
<li> <a href = "https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v12.1">What's new in this release</a></li>
6+
<li> <a href = "https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2024.1.0">What's new in this release</a></li>
127
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/CD3ExcelTabs.md">Toolkit Supported OCI Services</a></li>
13-
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/RunningAutomationToolkit.md#excel-sheet-templates">Excel Templates</a></li>
14-
8+
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/ExcelTemplates.md">Excel Templates</a></li>
159
</ul>
1610

17-
### New Users
1811

12+
### New Users
1913
<ul>
20-
2114
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/prerequisites.md">Prerequisites</a></li>
22-
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/Launch_Docker_container.md">Launch Docker container</a></li>
15+
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/Launch_Docker_container.md">Launch the container</a></li>
2316
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/Connect_container_to_OCI_Tenancy.md">Connect container to OCI Tenancy</a></li>
24-
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/RunningAutomationToolkit.md">Getting Started with Automation Toolkit</a></li>
25-
17+
</ul>
2618

27-
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/Workflows.md">Using the Automation Toolkit</a>
28-
<ul>
29-
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/GreenField.md">Green Field Tenancies</a>
19+
#### Using the Automation Toolkit via CLI
20+
<ul>
21+
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/Workflows.md">Before you Begin</a></li>
22+
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/GreenField.md">Create resources in OCI (Greenfield Workflow)</a>
3023
<ul>
3124
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/learn_more/OPAForCompliance.md"</a> Enforcing OPA (Open Policy Agent) policies for Terraform </li>
32-
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/NetworkingScenariosGF.md"</a><b> Must Read :</b> Managing Network for Greenfield Tenancies</li>
33-
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/ComputeGF.md"</a><b> Must Read :</b> Managing Compute Instances for Greefield Tenancies</li>
34-
</ul>
25+
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/NetworkingScenariosGF.md"</a><b> Must Read :</b> Managing Network for Greenfield Workflow</li>
26+
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/ComputeGF.md"</a><b> Must Read :</b> Managing Compute Instances for Greefield Workflow</li>
3527
</ul>
3628
</li>
37-
<ul>
38-
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/NonGreenField.md">Non Greenfield Tenancies</a>
29+
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/NonGreenField.md">Export Resources from OCI (Non-Greenfield Workflow)</a>
3930
<ul>
40-
<li><a href ="/cd3_automation_toolkit/documentation/user_guide/NetworkingScenariosNGF.md"</a><b> Must Read : </b> Managing Network for Non-Greenfield Tenancies</li>
41-
<li><a href = "/cd3_automation_toolkit/documentation/user_guide/ComputeNGF.md"</a><b> Must Read : </b> Managing Compute Instances for Non-Greenfield Tenancies </li>
31+
<li><a href ="/cd3_automation_toolkit/documentation/user_guide/NetworkingScenariosNGF.md"</a><b> Must Read : </b> Managing Network for Non-Greenfield Workflow</a></li>
32+
<li><a href = "/cd3_automation_toolkit/documentation/user_guide/ComputeNGF.md"</a><b> Must Read : </b> Managing Compute Instances for Non-Greenfield Workflow </a></li>
4233
</ul>
43-
</ul>
44-
45-
34+
</ul>
35+
36+
#### Using the Automation Toolkit via Jenkins
37+
<ul>
38+
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/Workflows-jenkins.md"</a>Before you Begin</a></li>
39+
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/Intro-Jenkins.md"</a>Introduction to Jenkins with the toolkit</a></li>
40+
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/GreenField-Jenkins.md"</a>Create resources in OCI via Jenkins(Greenfield Workflow)</li>
41+
<ul>
42+
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/NetworkingScenariosGF-Jenkins.md"</a><b> Must Read :</b> Managing Network for Greenfield Workflow</li>
43+
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/GF-Jenkins.md"</a><b> Must Read : </b> Provisioning of Resources - Instances/OKE/SDDC/Database </li>
44+
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/multiple_options_GF-Jenkins.md"</a><b> Must Read : </b> Provisioning of multiple services together </li>
45+
</ul>
46+
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/NonGreenField-Jenkins.md"</a>Export Resources from OCI via Jenkins(Non-Greenfield Workflow) </li>
47+
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/cli_jenkins.md">Switch between CLI and Jenkins</a></li>
48+
<li> <a href="/cd3_automation_toolkit/documentation/user_guide/remote_state.md">Remote Management of Terraform State File</a></li>
4649
</ul>
4750

4851
### Existing Users
49-
50-
<ul>
51-
52+
<ul>
5253
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/Upgrade_Toolkit.md">Steps to Upgrade Your Toolkit</a></li>
53-
54+
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/Jobs_Migration.md">Migrating Jenkins Jobs</a></li>
5455
</ul>
5556

56-
### Learn More...
57+
### Videos
58+
<ul>
59+
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/LearningVideos.md">Automation Toolkit Learning Videos</a></li>
60+
</ul>
5761

62+
### Known Behaviour
5863
<ul>
64+
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/KnownBehaviour.md">Expected Behaviour Of Automation Toolkit</a></li>
65+
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/FAQ.md">FAQs</a></li>
66+
</ul>
5967

68+
### Learn More...
69+
<ul>
6070
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/RestructuringOutDirectory.md">Grouping of generated Terraform files</a></li>
6171
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/ResourceManagerUpload.md">OCI Resource Manager Upload</a></li>
62-
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/OPAForCompliance.md">OPA For Compliance with Terraform</a></li>
6372
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/CISFeatures.md">Additional CIS Compliance Features</a></li>
64-
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/SupportForCD3Validator.md">CD3 Validator Features</a></li>
65-
66-
73+
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/SupportForCD3Validator.md">CD3 Validator Features</a></li>
6774
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/learn_more/SupportforAdditionalAttributes.md">Support for Additional Attributes</a></li>
68-
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/LearningVideos.md">Automation Toolkit Learning Videos</a></li>
69-
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/KnownBehaviour.md">Expected Behaviour Of Automation Toolkit</a></li>
70-
<li> <a href = "/cd3_automation_toolkit/documentation/user_guide/FAQ.md">FAQs</a></li>
71-
7275
</ul>
7376

74-
75-
7677
## Introduction
7778
CD3 stands for <b>C</b>loud <b>D</b>eployment <b>D</b>esign <b>D</b>eliverable.
7879
The CD3 Automation toolkit has been developed to help in automating the OCI resource object management.
7980
<br><br>
8081
It reads input data in the form of CD3 Excel sheet and generates Terraform files which can be used to provision the resources in OCI instead of handling the task through the OCI console manually. The toolkit also reverse engineers the components in OCI back to the Excel sheet and Terraform configuration. The toolkit can be used throughout the lifecycle of tenancy to continuously create or modify existing resources. The generated Terraform code can be used by the OCI Resource Manager or can be integrated into organization's existing devops CI/CD ecosystem.
8182
<br><br>
8283
<kbd>
83-
<img width="748" alt="Screenshot 2022-12-30 at 11 57 41 AM" src="https://user-images.githubusercontent.com/111430850/210614513-5d2e97a6-3c1e-4a2b-a793-3a1b6410c856.png">
84+
<img width="923" alt="Screenshot 2024-02-02 at 8 23 51 PM" src="https://github.com/oracle-devrel/cd3-automation-toolkit/assets/103508105/c2d06575-1bcb-4cb3-9c49-682bf19f3ac9">
85+
8486
</kbd>
8587
<br>
8688

@@ -98,11 +100,10 @@ It reads input data in the form of CD3 Excel sheet and generates Terraform files
98100
| [Management Services](/cd3_automation_toolkit/documentation/user_guide/learn_more/CD3ExcelTabs.md#management-services) | Events, Notifications, Alarms, Service Connector Hub (SCH) |
99101
| [Developer Services](/cd3_automation_toolkit/documentation/user_guide/learn_more/CD3ExcelTabs.md#developer-services) | Resource Manager, Oracle Kubernetes Engine (OKE) |
100102
| [Logging Services](/cd3_automation_toolkit/documentation/user_guide/learn_more/CD3ExcelTabs.md#logging-Services) | VCN Flow Logs, LBaaS access and error Logs, OSS Buckets write Logs |
101-
| [SDDCs ](/cd3_automation_toolkit/documentation/user_guide/learn_more/CD3ExcelTabs.md#sddcs-tab) | Oracle Cloud VMWare Solutions |
103+
| [SDDCs ](/cd3_automation_toolkit/documentation/user_guide/learn_more/CD3ExcelTabs.md#sddcs-tab) | Oracle Cloud VMWare Solutions (Single Cluster is supported as of now. Multi-cluster support will be included in the upcoming release) |
102104
| [CIS Landing Zone Compliance](/cd3_automation_toolkit/documentation/user_guide/learn_more/CISFeatures.md#additional-cis-compliance-features) | Download and Execute CIS Compliance Check Script, Cloud Guard, Key Vault, Budget |
103105
[Policy Enforcement](/cd3_automation_toolkit/documentation/user_guide/learn_more/OPAForCompliance.md) | OPA - Open Policy Agent |
104106

105-
106107
[Click here](/cd3_automation_toolkit/documentation/user_guide/prerequisites.md) to get started and manage your OCI Infra!
107108

108109
## Contributing

cd3_automation_toolkit/Compute/create_terraform_dedicatedhosts.py

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,19 +25,16 @@
2525

2626
# If input is CD3 excel file
2727
# Execution of the code begins here
28-
def create_terraform_dedicatedhosts(inputfile, outdir, service_dir,prefix, config):
28+
def create_terraform_dedicatedhosts(inputfile, outdir, service_dir,prefix, ct):
2929
# Load the template file
3030
file_loader = FileSystemLoader(f'{Path(__file__).parent}/templates')
3131
env = Environment(loader=file_loader, keep_trailing_newline=True, trim_blocks=True, lstrip_blocks=True)
3232
template = env.get_template('dedicatedvmhosts-template')
3333

3434
filename = inputfile
35-
configFileName = config
3635

3736
sheetName = "DedicatedVMHosts"
3837
auto_tfvars_filename = prefix + '_' + sheetName.lower() + '.auto.tfvars'
39-
ct = commonTools()
40-
ct.get_subscribedregions(configFileName)
4138

4239
outfile = {}
4340
oname = {}

cd3_automation_toolkit/Compute/create_terraform_instances.py

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,18 +22,15 @@
2222

2323
# If input is CD3 excel file
2424
# Execution of the code begins here
25-
def create_terraform_instances(inputfile, outdir, service_dir, prefix, config):
25+
def create_terraform_instances(inputfile, outdir, service_dir, prefix, ct):
2626
boot_policy_tfStr = {}
2727
tfStr = {}
2828
ADS = ["AD1", "AD2", "AD3"]
2929

3030
filename = inputfile
31-
configFileName = config
3231

3332
sheetName = "Instances"
3433
auto_tfvars_filename = prefix + '_' + sheetName.lower() + '.auto.tfvars'
35-
ct = commonTools()
36-
ct.get_subscribedregions(configFileName)
3734

3835
# Load the template file
3936
file_loader = FileSystemLoader(f'{Path(__file__).parent}/templates')
@@ -153,7 +150,7 @@ def create_terraform_instances(inputfile, outdir, service_dir, prefix, config):
153150
except Exception as e:
154151
print("Invalid Subnet Name specified for row " + str(
155152
i + 3) + ". It Doesnt exist in Subnets sheet. Exiting!!!")
156-
exit()
153+
exit(1)
157154

158155
tempdict = {'network_compartment_id': commonTools.check_tf_variable(network_compartment_id),
159156
'vcn_name': vcn_name,

cd3_automation_toolkit/Compute/export_dedicatedvmhosts_nonGreenField.py

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,12 @@ def print_dedicatedvmhosts(region, dedicatedvmhost, values_for_column, ntk_compa
4444
values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column)
4545

4646
# Execution of the code begins here
47-
def export_dedicatedvmhosts(inputfile, _outdir, service_dir, _config, ct, export_compartments=[], export_regions=[]):
47+
def export_dedicatedvmhosts(inputfile, outdir, service_dir, config, signer, ct, export_compartments=[], export_regions=[]):
4848
global tf_import_cmd
4949
global sheet_dict
5050
global importCommands
51-
global config
5251
global cd3file
5352
global reg
54-
global outdir
5553
global values_for_column
5654

5755

@@ -60,16 +58,7 @@ def export_dedicatedvmhosts(inputfile, _outdir, service_dir, _config, ct, export
6058
print("\nAcceptable cd3 format: .xlsx")
6159
exit()
6260

63-
64-
outdir = _outdir
65-
configFileName = _config
66-
config = oci.config.from_file(file_location=configFileName)
67-
6861
sheetName="DedicatedVMHosts"
69-
if ct==None:
70-
ct = commonTools()
71-
ct.get_subscribedregions(configFileName)
72-
ct.get_network_compartment_ids(config['tenancy'],"root",configFileName)
7362

7463
# Read CD3
7564
df, values_for_column= commonTools.read_cd3(cd3file,sheetName)
@@ -100,14 +89,20 @@ def export_dedicatedvmhosts(inputfile, _outdir, service_dir, _config, ct, export
10089
config.__setitem__("region", ct.region_dict[reg])
10190
region = reg.capitalize()
10291

103-
compute_client = oci.core.ComputeClient(config,retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY)
92+
compute_client = oci.core.ComputeClient(config=config,retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY, signer=signer)
10493

10594
for ntk_compartment_name in export_compartments:
10695
dedicatedvmhosts = oci.pagination.list_call_get_all_results(compute_client.list_dedicated_vm_hosts,compartment_id=ct.ntk_compartment_ids[ntk_compartment_name], lifecycle_state="ACTIVE")
10796
for dedicatedvmhost in dedicatedvmhosts.data:
10897
dedicatedvmhost=compute_client.get_dedicated_vm_host(dedicatedvmhost.id).data
10998
print_dedicatedvmhosts(region, dedicatedvmhost,values_for_column, ntk_compartment_name)
11099

100+
# write data into file
101+
for reg in export_regions:
102+
script_file = f'{outdir}/{reg}/{service_dir}/'+file_name
103+
with open(script_file, 'a') as importCommands[reg]:
104+
importCommands[reg].write('\n\nterraform plan\n')
105+
111106
commonTools.write_to_cd3(values_for_column, cd3file, "DedicatedVMHosts")
112107

113108
print("Dedicated VM Hosts exported to CD3\n")

0 commit comments

Comments
 (0)