Feat: always check ES license (#1005)

Author: kares

.travis.yml

@@ -9,4 +9,3 @@ env:
   - DISTRIBUTION=default INTEGRATION=true ELASTIC_STACK_VERSION=7.x SNAPSHOT=true
   - DISTRIBUTION=default INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true
   - DISTRIBUTION=default SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x
-  - DISTRIBUTION=oss INTEGRATION=true ELASTIC_STACK_VERSION=7.10.2 # since 7.11 no OSS artifact

CHANGELOG.md

@@ -1,6 +1,9 @@
-## 10.9.0
+## 11.0.0
  - Feat: Data stream support [#988](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/988)
  - Refactor: reviewed logging format + restored ES (initial) setup error logging
+ - Feat: always check ES license [#1005](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1005)
+
+   Since Elasticsearch no longer provides an OSS artifact the plugin will no longer skip the license check on OSS Logstash. 
 
 ## 10.8.6
  - Fixed an issue where a single over-size event being rejected by Elasticsearch would cause the entire entire batch to be retried indefinitely. The oversize event will still be retried on its own and logging has been improved to include payload sizes in this situation [#972](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/972)

lib/logstash/outputs/elasticsearch/license_checker.rb

@@ -7,41 +7,38 @@ def initialize(logger)
 
     # Figure out if the provided license is appropriate or not
     # The appropriate_license? methods is the method called from LogStash::Outputs::ElasticSearch::HttpClient::Pool#healthcheck!
+    # @param pool
     # @param url [LogStash::Util::SafeURI] ES node URL
-    # @param license [Hash] ES node deserialized licence document
     # @return [Boolean] true if provided license is deemed appropriate
     def appropriate_license?(pool, url)
-      return true if oss?
-
       license = pool.get_license(url)
-      if valid_es_license?(license)
+      case license_status(license)
+      when 'active'
         true
-      else
-        # As this version is to be shipped with Logstash 7.x we won't mark the connection as unlicensed
-        #
-        #  @logger.error("Cannot connect to the Elasticsearch cluster configured in the Elasticsearch output. Logstash requires the default distribution of Elasticsearch. Please update to the default distribution of Elasticsearch for full access to all free features, or switch to the OSS distribution of Logstash.", :url => url.sanitized.to_s)
-        #  meta[:state] = :unlicensed
-        #
-        # Instead we'll log a deprecation warning and mark it as alive:
-        #
-        log_license_deprecation_warn(url)
+      when nil
+        warn_no_license(url)
+        false
+      else # 'invalid', 'expired'
+        warn_invalid_license(url, license)
         true
       end
     end
 
-    # Note that oss? could be private but is used by the Pool specs
-    def oss?
-      LogStash::OSS
+    def license_status(license)
+      license.fetch("license", {}).fetch("status", nil)
     end
 
-    # Note that valid_es_license? could be private but is used by the Pool specs
-    def valid_es_license?(license)
-      license.fetch("license", {}).fetch("status", nil) == "active"
+    private
+
+    def warn_no_license(url)
+      @logger.error("Connecting to an OSS distribution of Elasticsearch is no longer supported, " +
+                    "please upgrade to the default distribution of Elasticsearch", url: url.sanitized.to_s)
     end
 
-    # Note that log_license_deprecation_warn could be private but is used by the Pool specs
-    def log_license_deprecation_warn(url)
-      @logger.warn("DEPRECATION WARNING: Connecting to an OSS distribution of Elasticsearch using the default distribution of Logstash will stop working in Logstash 8.0.0. Please upgrade to the default distribution of Elasticsearch, or use the OSS distribution of Logstash", :url => url.sanitized.to_s)
+    def warn_invalid_license(url, license)
+      @logger.warn("WARNING: Current Elasticsearch license is not active, " +
+                   "please check Elasticsearch's licensing information", url: url.sanitized.to_s, license: license)
     end
+
   end
 end; end; end

logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '10.9.0'
+  s.version         = '11.0.0'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"

spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -9,7 +9,7 @@
   let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
   let(:es_node_versions) { [ "0.0.0" ] }
   let(:oss) { true }
-  let(:valid_license) { true }
+  let(:license_status) { 'active' }
 
   subject { described_class.new(logger, adapter, initial_urls, options) }
 
@@ -26,8 +26,7 @@
     allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
 
     allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_node_versions)
-    allow(subject.license_checker).to receive(:oss?).and_return(oss)
-    allow(subject.license_checker).to receive(:valid_es_license?).and_return(valid_license)
+    allow(subject.license_checker).to receive(:license_status).and_return(license_status)
   end
 
   after do
@@ -283,29 +282,44 @@
       let(:oss) { false }
 
       context "if ES doesn't return a valid license" do
-        let(:valid_license) { false }
+        let(:license_status) { nil }
 
-        it "marks the url as active" do
+        it "marks the url as dead" do
           subject.update_initial_urls
-          expect(subject.alive_urls_count).to eq(1)
+          expect(subject.alive_urls_count).to eq(0)
         end
 
         it "logs a warning" do
-          expect(subject.license_checker).to receive(:log_license_deprecation_warn).once
+          expect(subject.license_checker).to receive(:warn_no_license).once.and_call_original
           subject.update_initial_urls
         end
       end
 
       context "if ES returns a valid license" do
-        let(:valid_license) { true }
+        let(:license_status) { 'active' }
 
         it "marks the url as active" do
           subject.update_initial_urls
           expect(subject.alive_urls_count).to eq(1)
         end
 
         it "does not log a warning" do
-          expect(subject.license_checker).to_not receive(:log_license_deprecation_warn)
+          expect(subject.license_checker).to_not receive(:warn_no_license)
+          expect(subject.license_checker).to_not receive(:warn_invalid_license)
+          subject.update_initial_urls
+        end
+      end
+
+      context "if ES returns an invalid license" do
+        let(:license_status) { 'invalid' }
+
+        it "marks the url as active" do
+          subject.update_initial_urls
+          expect(subject.alive_urls_count).to eq(1)
+        end
+
+        it "logs a warning" do
+          expect(subject.license_checker).to receive(:warn_invalid_license).and_call_original
           subject.update_initial_urls
         end
       end

spec/unit/outputs/license_check_spec.rb

@@ -14,22 +14,6 @@
     end
   end
 
-  context "LicenseChecker API required by Pool specs" do
-    subject { described_class }
-
-    it "defines the oss? method" do
-      expect(subject.instance_methods).to include(:oss?)
-    end
-
-    it "defines the valid_es_license? method" do
-      expect(subject.instance_methods).to include(:valid_es_license?)
-    end
-
-    it "defines the log_license_deprecation_warn method" do
-      expect(subject.instance_methods).to include(:log_license_deprecation_warn)
-    end
-  end
-
   context "Pool class API required by the LicenseChecker" do
     subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool }