Merge pull request HarshCasper#413 from sameersrivastava13/master

solved HarshCasper#392 added : Loading data from Amazon S3 to Redshift

Author: HarshCasper

Python/Aws/s3ToRedshift.md

@@ -0,0 +1,10 @@
+# Load data from S3 to Redshift
+
+The code example executes the following steps:
+
+1. import modules that are bundled by AWS Glue by default.
+2. Define some configuration parameters (e.g., the Redshift hostname RedshiftHOST).
+3. Read the S3 bucket and object from the arguments (see getResolvedOptions) handed over when starting the job.
+4. Establish a connection to Redshift: connect(...).
+5. Increase the statement timeout (see statement_timeout) to one hour.
+6. Execute the COPY query to tell Redshift to the object from S3.

Python/Aws/s3ToRedshift.py

@@ -0,0 +1,38 @@
+from pgdb import connect
+import os
+import sys
+from awsglue.utils import getResolvedOptions
+
+# CONFIGURATION
+redshiftHost = "xyz.redshift.amazonaws.com"
+redshiftPort = "5439"
+redshiftDatabase = "mydatabase"
+redshiftUser = "myadmin"
+redshiftPassword = "XYZ"
+redshiftSchema = "myschema"
+redshifttable = "mytable"
+redshiftColumns = "timestamp,value_a,value_b,value_c"
+DELIMITER = "\t"
+DATEFORMAT = "YYYY-MM-DD"
+
+# ARGUMENTS
+args = getResolvedOptions(sys.argv, ["s3-bucket", "s3-object"])
+s3Bucket = args["s3_bucket"]
+s3Object = args["s3_object"]
+
+con = connect(
+    host=redshiftHost + ":" + redshiftPort,
+    database=redshiftDatabase,
+    user=redshiftUser,
+    password=redshiftPassword,
+)
+cursor = con.cursor()
+
+cursor.execute("set statement_timeout = 360000")
+
+copyQuery = f"COPY {redshiftSchema}.{redshifttable}({redshiftColumns}) from 's3://{s3Bucket}/{s3Object}' iam_role 'arn:aws:iam::111111111111:role/LoadFromS3ToRedshiftJob' delimiter {DELIMITER} DATEFORMAT AS {DATEFORMAT} ROUNDEC TRUNCATECOLUMNS ESCAPE MAXERROR AS 500;"
+
+cursor.execute(copyQuery)
+con.commit()
+cursor.close()
+con.close()