[APMAPI-1068] Enable Service Discovery (#4649)

Author: vpellan

.github/forced-tests-list.json

@@ -1,5 +1,6 @@
 {
     "PARAMETRIC": [
-        "tests/parametric/test_config_consistency.py::Test_Stable_Config_Default"
+        "tests/parametric/test_config_consistency.py::Test_Stable_Config_Default",
+        "tests/parametric/test_process_discovery.py"
     ]
 }

Matrixfile

@@ -13,6 +13,9 @@
   'error_tracking' => {
     '' => '❌ 2.5 / ❌ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby',
   },
+  'process_discovery' => {
+    '' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby'
+  },
   'appsec:main' => {
     '' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
   },

Rakefile

@@ -70,13 +70,13 @@ namespace :spec do
              :graphql, :graphql_unified_trace_patcher, :graphql_trace_patcher, :graphql_tracing_patcher,
              :rails, :railsredis, :railsredis_activesupport, :railsactivejob,
              :elasticsearch, :http, :redis, :sidekiq, :sinatra, :hanami, :hanami_autoinstrument,
-             :profiling, :crashtracking, :error_tracking]
+             :profiling, :crashtracking, :error_tracking, :process_discovery]
 
   desc '' # "Explicitly hiding from `rake -T`"
   RSpec::Core::RakeTask.new(:main) do |t, args|
     t.pattern = 'spec/**/*_spec.rb'
     t.exclude_pattern = 'spec/**/{appsec/integration,contrib,benchmark,redis,auto_instrument,opentelemetry,profiling,crashtracking,error_tracking}/**/*_spec.rb,'\
-                        ' spec/**/{auto_instrument,opentelemetry}_spec.rb, spec/datadog/gem_packaging_spec.rb'
+                        ' spec/**/{auto_instrument,opentelemetry,process_discovery}_spec.rb, spec/datadog/gem_packaging_spec.rb'
     t.rspec_opts = args.to_a.join(' ')
   end
 
@@ -205,6 +205,15 @@ namespace :spec do
   end
   # rubocop:enable Style/MultilineBlockChain
 
+  # rubocop:disable Style/MultilineBlockChain
+  RSpec::Core::RakeTask.new(:process_discovery) do |t, args|
+    t.pattern = 'spec/datadog/core/process_discovery_spec.rb'
+    t.rspec_opts = args.to_a.join(' ')
+  end.tap do |t|
+    Rake::Task[t.name].enhance(["compile:libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"])
+  end
+  # rubocop:enable Style/MultilineBlockChain
+
   desc '' # "Explicitly hiding from `rake -T`"
   RSpec::Core::RakeTask.new(:error_tracking) do |t, args|
     t.pattern = 'spec/datadog/error_tracking/**/*_spec.rb'

ext/libdatadog_api/process_discovery.c

@@ -1,7 +1,7 @@
 #include <errno.h>
 #include <stdlib.h>
 #include <ruby.h>
-#include <datadog/common.h>
+#include <datadog/library-config.h>
 
 #include "datadog_ruby_common.h"
 
@@ -36,8 +36,7 @@ void process_discovery_init(VALUE core_module) {
   rb_define_singleton_method(process_discovery_class, "_native_close_tracer_memfd", _native_close_tracer_memfd, 2);
 }
 
-// TODO: Remove DDTRACE_UNUSED and rename _self to self once we have updated libdatadog to 17.1
-static VALUE _native_store_tracer_metadata(int argc, VALUE *argv, DDTRACE_UNUSED VALUE _self) {
+static VALUE _native_store_tracer_metadata(int argc, VALUE *argv, VALUE self) {
   VALUE logger;
   VALUE options;
   rb_scan_args(argc, argv, "1:", &logger, &options);
@@ -61,7 +60,6 @@ static VALUE _native_store_tracer_metadata(int argc, VALUE *argv, DDTRACE_UNUSED
   ENFORCE_TYPE(service_env, T_STRING);
   ENFORCE_TYPE(service_version, T_STRING);
 
-  /*
   ddog_Result_TracerMemfdHandle result = ddog_store_tracer_metadata(
     (uint8_t) NUM2UINT(schema_version),
     char_slice_from_ruby_string(runtime_id),
@@ -86,9 +84,6 @@ static VALUE _native_store_tracer_metadata(int argc, VALUE *argv, DDTRACE_UNUSED
   VALUE tracer_memfd_class = rb_const_get(self, rb_intern("TracerMemfd"));
   VALUE tracer_memfd = TypedData_Wrap_Struct(tracer_memfd_class, &tracer_memfd_type, fd);
   return tracer_memfd;
-  */
-
-  rb_raise(rb_eNotImpError, "TODO: Not in use yet, waiting for libdatadog 17.1");
 }
 
 static VALUE _native_to_rb_int(DDTRACE_UNUSED VALUE _self, VALUE tracer_memfd) {

lib/datadog/core/configuration/components.rb

@@ -126,8 +126,7 @@ def initialize(settings)
           @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, @logger, telemetry: telemetry)
           @error_tracking = Datadog::ErrorTracking::Component.build(settings, @tracer, @logger)
           @environment_logger_extra[:dynamic_instrumentation_enabled] = !!@dynamic_instrumentation
-          # TODO: Re-enable this once we have updated libdatadog to 17.1
-          # @process_discovery_fd = Core::ProcessDiscovery.get_and_store_metadata(settings, @logger)
+          @process_discovery_fd = Core::ProcessDiscovery.get_and_store_metadata(settings, @logger)
 
           self.class.configure_tracing(settings)
         end
@@ -211,8 +210,7 @@ def shutdown!(replacement = nil)
           telemetry.emit_closing! unless replacement&.telemetry&.enabled
           telemetry.shutdown!
 
-          # TODO: Re-enable this once we have updated libdatadog to 17.1
-          # Core::ProcessDiscovery._native_close_tracer_memfd(@process_discovery_fd, @logger) if @process_discovery_fd
+          Core::ProcessDiscovery._native_close_tracer_memfd(@process_discovery_fd, @logger) if @process_discovery_fd
         end
 
         # Returns the current state of various components.

spec/datadog/core/configuration/components_spec.rb

@@ -28,7 +28,12 @@
 RSpec.describe Datadog::Core::Configuration::Components do
   subject(:components) { described_class.new(settings) }
 
-  let(:logger) { instance_double(Datadog::Core::Logger) }
+  # Using a double does not give access to superclass methods like debug
+  let(:logger) do
+    instance_double(Datadog::Core::Logger).tap do |logger|
+      allow(logger).to receive(:debug)
+    end
+  end
   let(:settings) { Datadog::Core::Configuration::Settings.new }
   let(:agent_settings) { Datadog::Core::Configuration::AgentSettingsResolver.call(settings, logger: nil) }
   let(:agent_info) { Datadog::Core::Environment::AgentInfo.new(agent_settings, logger: logger) }

spec/datadog/core/process_discovery_spec.rb

@@ -5,8 +5,7 @@
 require 'spec_helper'
 require 'datadog/core/process_discovery'
 
-# TODO: Re-enable this once we have updated libdatadog to 17.1
-RSpec.xdescribe Datadog::Core::ProcessDiscovery do
+RSpec.describe Datadog::Core::ProcessDiscovery do
   describe '.get_and_store_metadata' do
     context 'when libdatadog API is not available' do
       it 'returns nil' do