refactor: Add support for plain SOQL query

feat: Include deleted records

Author: Asafffff

CHANGELOG.md

@@ -1,3 +1,10 @@
+## 4.0.0
+  - refactor: Change behavior to support SOQL queries
+  - feat: Include deleted records
+
+## 3.4.0
+  - feat: Add support for plain SOQL query
+
 ## 3.3.0
   - feat: Added `timeout` configuration to control RESTForce timeout settings (defaults to `60`)
   - feat: Added support for reference fields (`parent__r.child`)

lib/logstash/inputs/salesforce.rb

@@ -33,7 +33,7 @@
 #     username => '[email protected]'
 #     password => 'super-secret'
 #     security_token => 'SECURITY TOKEN FOR THIS USER'
-#     sfdc_object_name => 'Opportunity'
+#     sfdc_soql_query => 'SELECT Id FROM Account'
 #   }
 # }
 #
@@ -58,6 +58,8 @@ class LogStash::Inputs::Salesforce < LogStash::Inputs::Base
   # Set this to true to connect to a sandbox sfdc instance
   # logging in through test.salesforce.com
   config :use_test_sandbox, :validate => :boolean, :default => false
+  # Include deleted records
+  config :include_deleted, :validate => :boolean, :default => false
   # Set this to the instance url of the sfdc instance you want
   # to connect to already during login. If you have configured
   # a MyDomain in your sfdc instance you would provide
@@ -85,29 +87,25 @@ class LogStash::Inputs::Salesforce < LogStash::Inputs::Base
   # generting a security token, see:
   # https://help.salesforce.com/apex/HTViewHelpDoc?id=user_security_token.htm
   config :security_token, :validate => :string, :required => true
-  # The name of the salesforce object you are creating or updating
-  config :sfdc_object_name, :validate => :string, :required => true
-  # These are the field names to return in the Salesforce query
-  # If this is empty, all fields are returned.
-  config :sfdc_fields, :validate => :array, :default => []
-  # These options will be added to the WHERE clause in the
-  # SOQL statement. Additional fields can be filtered on by
-  # adding field1 = value1 AND field2 = value2 AND...
-  config :sfdc_filters, :validate => :string, :default => ""
+  # Plain SOQL query
+  config :sfdc_soql_query, :validate => :string, :required => true
   # Setting this to true will convert SFDC's NamedFields__c to named_fields__c
   config :to_underscores, :validate => :boolean, :default => false
 
   public
   def register
     require 'restforce'
-    obj_desc = client.describe(@sfdc_object_name)
-    @sfdc_field_types = get_field_types(obj_desc)
-    @sfdc_fields = get_all_fields if @sfdc_fields.empty?
+    @sfdc_fields = get_sfdc_fields
   end # def register
 
   public
   def run(queue)
-    results = client.query(get_query())
+    if @include_deleted
+      results = client.query_all(@sfdc_soql_query)
+    else
+      results = client.query(@sfdc_soql_query)
+    end
+
     @logger.debug("Query results:", :results => results)
     if results && results.first
       results.each do |result|
@@ -162,29 +160,12 @@ def client_options
   end
 
   private
-  def get_query()
-    query = ["SELECT",@sfdc_fields.join(','),
-             "FROM",@sfdc_object_name]
-    query << ["WHERE",@sfdc_filters] unless @sfdc_filters.empty?
-    query << "ORDER BY LastModifiedDate DESC" if @sfdc_fields.include?('LastModifiedDate')
-    query_str = query.flatten.join(" ")
-    @logger.debug? && @logger.debug("SFDC Query", :query => query_str)
-    return query_str
-  end
-
-  private
-  def get_field_types(obj_desc)
-    field_types = {}
-    obj_desc.fields.each do |f|
-      field_types[f.name] = f.type
-    end
-    @logger.debug? && @logger.debug("Field types", :field_types => field_types.to_s)
-    return field_types
-  end
-
-  private
-  def get_all_fields
-    return @sfdc_field_types.keys
+  def get_sfdc_fields
+    extracted_fields = sfdc_soql_query.gsub(/SELECT (.+) FROM.+/i, '\1')
+    extracted_fields_as_array = extracted_fields.split(',').collect { |item| item.strip }
+    @logger.debug("Extracted fields: ", :extracted_fields_as_array => extracted_fields_as_array.to_s)
+    
+    return extracted_fields_as_array;
   end
 
   private

logstash-input-salesforce.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-salesforce'
-  s.version         = '3.3.0'
+  s.version         = '4.0.0'
   s.licenses = ['Apache-2.0']
   s.summary = "Creates events based on a Salesforce SOQL query"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"