Revert "Use Erlang.mk's native Elixir support for CLI"

This reverts commit 398a845.

Author: dcorbacho

Makefile

@@ -31,6 +31,10 @@ DISABLE_DISTCLEAN = 1
 
 XREF_SCOPE = app deps
 
+# We add all the applications that are in non-standard paths
+# so they are included in the analyses as well.
+XREF_EXTRA_APP_DIRS = $(filter-out deps/rabbitmq_cli/_build/dev/lib/rabbit_common/,$(wildcard deps/rabbitmq_cli/_build/dev/lib/*/)) deps/rabbitmq_prelaunch/
+
 # For Elixir protocols the right fix is to include the consolidated/
 # folders in addition to ebin/. However this creates conflicts because
 # some modules are duplicated. So instead we ignore warnings from
@@ -44,12 +48,64 @@ XREF_IGNORE = [ \
 # Include Elixir libraries in the Xref checks.
 xref: ERL_LIBS := $(ERL_LIBS):$(CURDIR)/apps:$(CURDIR)/deps:$(dir $(shell elixir --eval ":io.format '~s~n', [:code.lib_dir :elixir ]"))
 
+ifneq ($(wildcard deps/.hex/cache.erl),)
+deps:: restore-hex-cache-ets-file
+endif
+
 include rabbitmq-components.mk
 include erlang.mk
 include mk/github-actions.mk
 include mk/bazel.mk
 include mk/topic-branches.mk
 
+# --------------------------------------------------------------------
+# Mix Hex cache management.
+# --------------------------------------------------------------------
+
+# We restore the initial Hex cache.ets file from an Erlang term created
+# at the time the source archive was prepared.
+#
+# See the `$(SOURCE_DIST)` recipe for the reason behind this step.
+
+restore-hex-cache-ets-file: deps/.hex/cache.ets
+
+deps/.hex/cache.ets: deps/.hex/cache.erl
+	$(gen_verbose) $(call erlang,$(call restore_hex_cache_from_erl_term,$<,$@))
+
+define restore_hex_cache_from_erl_term
+  In = "$(1)",
+  Out = "$(2)",
+  {ok, [Props, Entries]} = file:consult(In),
+  Name = proplists:get_value(name, Props),
+  Type = proplists:get_value(type, Props),
+  Access = proplists:get_value(protection, Props),
+  NamedTable = proplists:get_bool(named_table, Props),
+  Keypos = proplists:get_value(keypos, Props),
+  Heir = proplists:get_value(heir, Props),
+  ReadConc = proplists:get_bool(read_concurrency, Props),
+  WriteConc = proplists:get_bool(write_concurrency, Props),
+  Compressed = proplists:get_bool(compressed, Props),
+  Options0 = [
+    Type,
+    Access,
+    {keypos, Keypos},
+    {heir, Heir},
+    {read_concurrency, ReadConc},
+    {write_concurrency, WriteConc}],
+  Options1 = case NamedTable of
+    true  -> [named_table | Options0];
+    false -> Options0
+  end,
+  Options2 = case Compressed of
+    true  -> [compressed | Options0];
+    false -> Options0
+  end,
+  Tab = ets:new(Name, Options2),
+  [true = ets:insert(Tab, Entry) || Entry <- Entries],
+  ok = ets:tab2file(Tab, Out),
+  init:stop().
+endef
+
 # --------------------------------------------------------------------
 # Distribution - common variables and generic functions.
 # --------------------------------------------------------------------
@@ -192,6 +248,14 @@ $(1): $(ERLANG_MK_RECURSIVE_DEPS_LIST)
 		sed -E -i.bak "s|^[[:blank:]]*include[[:blank:]]+\.\./.*erlang.mk$$$$|include ../../erlang.mk|" \
 		 $$@/deps/$$$$(basename $$$$dep)/Makefile && \
 		rm $$@/deps/$$$$(basename $$$$dep)/Makefile.bak; \
+		mix_exs=$$@/deps/$$$$(basename $$$$dep)/mix.exs; \
+		if test -f $$$$mix_exs; then \
+			(cd $$$$(dirname "$$$$mix_exs") && \
+			 (test -d $$@/deps/.hex || env DEPS_DIR=$$@/deps MIX_HOME=$$@/deps/.mix HEX_HOME=$$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix local.hex --force) && \
+			 env DEPS_DIR=$$@/deps MIX_HOME=$$@/deps/.mix HEX_HOME=$$@/deps/.hex MIX_ENV=prod FILL_HEX_CACHE=yes mix deps.get --only prod && \
+			 cp $(CURDIR)/mk/rabbitmq-mix.mk . && \
+			 rm -rf _build deps); \
+		fi; \
 		if test -f "$$$$dep/license_info"; then \
 			cp "$$$$dep/license_info" "$$@/deps/licensing/license_info_$$$$(basename $$$$dep)"; \
 			cat "$$$$dep/license_info" >> $$@/LICENSE; \
@@ -216,6 +280,7 @@ $(1): $(ERLANG_MK_RECURSIVE_DEPS_LIST)
 	done
 	$${verbose} echo "PLUGINS := $(PLUGINS)" > $$@/plugins.mk
 	$${verbose} sort -r < "$$@.git-times.txt" | head -n 1 > "$$@.git-time.txt"
+	$${verbose} $$(call erlang,$$(call dump_hex_cache_to_erl_term,$$(call core_native_path,$$@),$$(call core_native_path,$$@.git-time.txt)))
 	$${verbose} find $$@ -print0 | xargs -0 touch -t "$$$$(cat $$@.git-time.txt)"
 	$${verbose} rm "$$@.git-times.txt" "$$@.git-time.txt"
 
@@ -257,6 +322,47 @@ clean-$(1):
 clean:: clean-$(1)
 endef
 
+# Mix Hex component requires a cache file, otherwise it refuses to build
+# offline... That cache is an ETS table with all the applications we
+# depend on, plus some versioning informations and checksums. There
+# are two problems with that: the table contains a date (`last_update`
+# field) and `ets:tab2file()` produces a different file each time it's
+# called.
+#
+# To make our source archive reproducible, we fix the time of the
+# `last_update` field to the last Git commit and dump the content of the
+# table as an Erlang term to a text file.
+#
+# The ETS file must be recreated before compiling RabbitMQ. See the
+# `restore-hex-cache-ets-file` Make target.
+define dump_hex_cache_to_erl_term
+  In = "$(1)/deps/.hex/cache.ets",
+  Out = "$(1)/deps/.hex/cache.erl",
+  {ok, DateStr} = file:read_file("$(2)"),
+  {match, Date} = re:run(DateStr,
+    "^([0-9]{4})([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{2})\.([0-9]{2})",
+    [{capture, all_but_first, list}]),
+  [Year, Month, Day, Hour, Min, Sec] = [erlang:list_to_integer(V) || V <- Date],
+  {ok, Tab} = ets:file2tab(In),
+  true = ets:insert(Tab, {last_update, {{Year, Month, Day}, {Hour, Min, Sec}}}),
+  Props = [
+    Prop
+    || {Key, _} = Prop <- ets:info(Tab),
+    Key =:= name orelse
+    Key =:= type orelse
+    Key =:= protection orelse
+    Key =:= named_table orelse
+    Key =:= keypos orelse
+    Key =:= heir orelse
+    Key =:= read_concurrency orelse
+    Key =:= write_concurrency orelse
+    Key =:= compressed],
+  Entries = ets:tab2list(Tab),
+  ok = file:write_file(Out, io_lib:format("~w.~n~w.~n", [Props, Entries])),
+  ok = file:delete(In),
+  init:stop().
+endef
+
 # --------------------------------------------------------------------
 # Distribution - public targets
 # --------------------------------------------------------------------

deps/amqp10_client/Makefile

@@ -1,5 +1,6 @@
 PROJECT = amqp10_client
-PROJECT_DESCRIPTION = AMQP 1.0 client
+PROJECT_DESCRIPTION = AMQP 1.0 client from the RabbitMQ Project
+PROJECT_MOD = amqp10_client_app
 
 define PROJECT_APP_EXTRA_KEYS
 %% Hex.pm package informations.

deps/oauth2_client/Makefile

@@ -1,5 +1,6 @@
 PROJECT = oauth2_client
 PROJECT_DESCRIPTION = OAuth2 client from the RabbitMQ Project
+PROJECT_MOD = oauth2_client_app
 
 BUILD_DEPS = rabbit
 DEPS = rabbit_common jose
@@ -17,8 +18,5 @@ DEP_PLUGINS = rabbit_common/mk/rabbitmq-macros.mk \
 	      rabbit_common/mk/rabbitmq-test.mk \
 	      rabbit_common/mk/rabbitmq-tools.mk
 
-# Required to properly autopatch jose.
-ELIXIR = system
-
 include rabbitmq-components.mk
 include erlang.mk

deps/rabbit/src/rabbit_variable_queue.erl

@@ -2218,7 +2218,6 @@ determine_persist_to(Version,
     %% via the direct client), we make a guess based on the number of
     %% headers.
 
-    %% @todo We can probably simplify this.
     {MetaSize, _BodySize} = mc:size(Msg),
      case BodySize >= IndexMaxSize of
          true  -> msg_store;

deps/rabbit_common/mk/rabbitmq-build.mk

@@ -15,8 +15,7 @@ TEST_ERLC_OPTS += +nowarn_export_all
 ifneq ($(filter-out rabbit_common amqp_client,$(PROJECT)),)
 # Add the CLI ebin directory to the code path for the compiler: plugin
 # CLI extensions may access behaviour modules defined in this directory.
-
-RMQ_ERLC_OPTS += -pa $(DEPS_DIR)/rabbitmq_cli/ebin
+RMQ_ERLC_OPTS += -pa $(DEPS_DIR)/rabbitmq_cli/_build/dev/lib/rabbitmqctl/ebin
 endif
 
 RMQ_ERLC_OPTS += +deterministic

deps/rabbit_common/mk/rabbitmq-dist.mk

@@ -1,8 +1,9 @@
 .PHONY: dist test-dist do-dist cli-scripts cli-escripts clean-dist
 
-DIST_DIR ?= $(CURDIR)/plugins
-CLI_SCRIPTS_DIR ?= $(CURDIR)/sbin
-CLI_ESCRIPTS_DIR ?= $(CURDIR)/escript
+DIST_DIR = plugins
+CLI_SCRIPTS_DIR = sbin
+CLI_ESCRIPTS_DIR = escript
+MIX = echo y | mix
 
 # Set $(DIST_AS_EZS) to a non-empty value to enable the packaging of
 # plugins as .ez archives.
@@ -80,13 +81,17 @@ endef
 
 # Real entry point: it tests the existence of an .app file to determine
 # if it is an Erlang application (and therefore if it should be provided
-# as an .ez plugin archive) and calls do_ez_target_erlangmk.
+# as an .ez plugin archive) and calls do_ez_target_erlangmk. If instead
+# it finds a Mix configuration file, it is skipped, as the only elixir
+# applications in the directory are used by rabbitmq_cli and compiled
+# with it.
 #
 #   $(call ez_target,path_to_app)
 
 define ez_target
 dist_$(1)_appdir  = $(2)
 dist_$(1)_appfile = $$(dist_$(1)_appdir)/ebin/$(1).app
+dist_$(1)_mixfile = $$(dist_$(1)_appdir)/mix.exs
 
 $$(if $$(shell test -f $$(dist_$(1)_appfile) && echo OK), \
   $$(eval $$(call do_ez_target_erlangmk,$(1),$$(call get_app_version,$$(dist_$(1)_appfile)),$$(dist_$(1)_appdir))))
@@ -112,8 +117,9 @@ endif
 endif
 
 # The actual recipe to create the .ez plugin archive. Some variables
-# are defined in the do_ez_target_erlangmk macro
-# above. All .ez archives are also listed in this macro.
+# are defined in the do_ez_target_erlangmk and do_ez_target_mix macros
+# above. All .ez archives are also listed in this do_ez_target_erlangmk
+# and do_ez_target_mix macros.
 
 RSYNC ?= rsync
 RSYNC_V_0 =
@@ -198,7 +204,7 @@ test-dist:: $(ERLANG_MK_RECURSIVE_TEST_DEPS_LIST) test-build
 		$(MAYBE_APPS_LIST)"; \
 	fi
 
-DIST_EZS = $(ERLANGMK_DIST_EZS)
+DIST_EZS = $(ERLANGMK_DIST_EZS) $(MIX_DIST_EZS)
 
 do-dist:: $(DIST_EZS)
 	$(verbose) unwanted='$(filter-out $(DIST_EZS) $(EXTRA_DIST_EZS), \
@@ -216,21 +222,90 @@ endif
 install-cli: install-cli-scripts install-cli-escripts
 	@:
 
-install-cli-scripts: | $(CLI_SCRIPTS_DIR)
+ifeq ($(PROJECT),rabbit)
+install-cli-scripts:
+	$(gen_verbose) \
+	if command -v flock >/dev/null; then \
+		flock $(CLI_SCRIPTS_LOCK) \
+		sh -c 'mkdir -p "$(CLI_SCRIPTS_DIR)" && \
+		for file in scripts/*; do \
+			cmp -s "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")" || \
+			cp -a "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")"; \
+		done'; \
+	elif command -v lockf >/dev/null; then \
+		lockf $(CLI_SCRIPTS_LOCK) \
+		sh -c 'mkdir -p "$(CLI_SCRIPTS_DIR)" && \
+		for file in scripts/*; do \
+			cmp -s "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")" || \
+			cp -a "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")"; \
+		done'; \
+	else \
+		mkdir -p "$(CLI_SCRIPTS_DIR)" && \
+		for file in scripts/*; do \
+			cmp -s "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")" || \
+			cp -a "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")"; \
+		done; \
+	fi
+else
+
+install-cli-scripts:
 	$(gen_verbose) \
 	set -e; \
-	test -d "$(DEPS_DIR)/rabbit/scripts"; \
-	$(call maybe_flock,$(CLI_SCRIPTS_LOCK), \
-		cp -a $(DEPS_DIR)/rabbit/scripts/* $(CLI_SCRIPTS_DIR)/)
+	if test -d "$(DEPS_DIR)/rabbit/scripts"; then \
+		rabbit_scripts_dir='$(DEPS_DIR)/rabbit/scripts'; \
+	elif test -d "$(DEPS_DIR)/../scripts"; then \
+		rabbit_scripts_dir='$(DEPS_DIR)/../scripts'; \
+	else \
+		echo 'rabbit/scripts directory not found' 1>&2; \
+		exit 1; \
+	fi; \
+	test -d "$$rabbit_scripts_dir"; \
+	if command -v flock >/dev/null; then \
+		flock $(CLI_SCRIPTS_LOCK) \
+		sh -e -c 'mkdir -p "$(CLI_SCRIPTS_DIR)" && \
+		for file in "'$$rabbit_scripts_dir'"/*; do \
+			test -f "$$file"; \
+			cmp -s "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")" || \
+			cp -a "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")"; \
+		done'; \
+	elif command -v lockf >/dev/null; then \
+		lockf $(CLI_SCRIPTS_LOCK) \
+		sh -e -c 'mkdir -p "$(CLI_SCRIPTS_DIR)" && \
+		for file in "'$$rabbit_scripts_dir'"/*; do \
+			test -f "$$file"; \
+			cmp -s "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")" || \
+			cp -a "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")"; \
+		done'; \
+	else \
+		mkdir -p "$(CLI_SCRIPTS_DIR)" && \
+		for file in "$$rabbit_scripts_dir"/*; do \
+			test -f "$$file"; \
+			cmp -s "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")" || \
+			cp -a "$$file" "$(CLI_SCRIPTS_DIR)/$$(basename "$$file")"; \
+		done; \
+	fi
+endif
 
-install-cli-escripts: | $(CLI_ESCRIPTS_DIR)
-	$(gen_verbose) $(call maybe_flock,$(CLI_ESCRIPTS_LOCK), \
+install-cli-escripts:
+	$(gen_verbose) \
+	if command -v flock >/dev/null; then \
+		flock $(CLI_ESCRIPTS_LOCK) \
+		sh -c 'mkdir -p "$(CLI_ESCRIPTS_DIR)" && \
 		$(MAKE) -C "$(DEPS_DIR)/rabbitmq_cli" install \
 			PREFIX="$(abspath $(CLI_ESCRIPTS_DIR))" \
-			DESTDIR= IS_DEP=1)
-
-$(CLI_SCRIPTS_DIR) $(CLI_ESCRIPTS_DIR):
-	$(verbose) mkdir -p $@
+			DESTDIR='; \
+	elif command -v lockf >/dev/null; then \
+		lockf $(CLI_ESCRIPTS_LOCK) \
+		sh -c 'mkdir -p "$(CLI_ESCRIPTS_DIR)" && \
+		$(MAKE) -C "$(DEPS_DIR)/rabbitmq_cli" install \
+			PREFIX="$(abspath $(CLI_ESCRIPTS_DIR))" \
+			DESTDIR='; \
+	else \
+		mkdir -p "$(CLI_ESCRIPTS_DIR)" && \
+		$(MAKE) -C "$(DEPS_DIR)/rabbitmq_cli" install \
+			PREFIX="$(abspath $(CLI_ESCRIPTS_DIR))" \
+			DESTDIR= ; \
+	fi
 
 clean-dist::
 	$(gen_verbose) rm -rf \