diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index d5d03bad0ae9..fdb8fe07e2e2 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -73,6 +73,7 @@ lua { {% if status then %} lua_shared_dict status-report {* meta.lua_shared_dict["status-report"] *}; {% end %} + lua_shared_dict nacos 10m; } {% if enabled_stream_plugins["prometheus"] and not enable_http then %} diff --git a/apisix/discovery/nacos/factory.lua b/apisix/discovery/nacos/factory.lua new file mode 100644 index 000000000000..7b21b0f54940 --- /dev/null +++ b/apisix/discovery/nacos/factory.lua @@ -0,0 +1,303 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local require = require +local core = require("apisix.core") +local http = require('resty.http') +local ngx = ngx +local ngx_re = require('ngx.re') +local utils = require("apisix.discovery.nacos.utils") +local string = string +local string_sub = string.sub +local str_find = core.string.find +local ngx_timer_at = ngx.timer.at +local math_random = math.random +local ipairs = ipairs +local pairs = pairs +local shdict_name = "nacos" + +local nacos_dict = ngx.shared[shdict_name] +local NACOS_LOGIN_PATH = "/auth/login" +local NACOS_INSTANCE_PATH = "/ns/instance/list" +local _M = {} + + +local function _request(method, uri, params, headers, body, options) + local url = uri + if params ~= nil and params ~= {} then + url = uri .. "?" .. ngx.encode_args(params) + end + local httpc = http.new() + local timeout = options and options.timeout or {} + local connect_timeout = timeout.connect and timeout.connect * 1000 or 2000 + local read_timeout = timeout.read and timeout.read * 1000 or 2000 + local write_timeout = timeout.write and timeout.write * 1000 or 5000 + + httpc:set_timeouts(connect_timeout, read_timeout, write_timeout ) + local res, err = httpc:request_uri(url, { + method = method, + headers = headers, + body = body, + ssl_verify = true, + }) + + if not res then + return nil, err + end + + if not res.body or res.status ~= 200 then + return nil, 'status = ' .. res.status + end + + core.log.info("request to nacos, uri: ", url, "response: ", res.body) + + local data, err = core.json.decode(res.body) + if not data then + return nil, err + end + + return data +end + + +local function get_base_uri(hosts) + local host = hosts + -- TODO Add health check to get healthy nodes. + local url = host[math_random(#host)] + local auth_idx = core.string.rfind_char(url, '@') + local username, password + if auth_idx then + local protocol_idx = str_find(url, '://') + local protocol = string_sub(url, 1, protocol_idx + 2) + local user_and_password = string_sub(url, protocol_idx + 3, auth_idx - 1) + local arr = ngx_re.split(user_and_password, ':') + if #arr == 2 then + username = arr[1] + password = arr[2] + end + local other = string_sub(url, auth_idx + 1) + url = protocol .. other + end + return url, username, password +end + +local function request_login(self, host, username, password) + local params = { + username = username, + password = password, + } + -- backward compat: NACOS_LOGIN_PATH starts with "/" so + -- we need to remove the last "/" from prefix + if string_sub(self.config.prefix, -1) == "/" then + self.config.prefix = string_sub(self.config.prefix, 1, -2) + end + local uri = host .. self.config.prefix .. NACOS_LOGIN_PATH + + local headers = { + ["Content-Type"] ="application/x-www-form-urlencoded" + } + + local resp, err = _request("POST", uri, params, headers, nil, {timeout=self.config.timeout}) + if not resp then + core.log.error("failed to fetch token from nacos, uri: ", uri, " err: ", err) + return "" + end + return resp.accessToken +end + + +local function request_instance_list(self, params, host) + -- backward compat: NACOS_INSTANCE_PATH starts with "/" so + -- we need to remove the last "/" from prefix + if string_sub(self.config.prefix, -1) == "/" then + self.config.prefix = string_sub(self.config.prefix, 1, -2) + end + local uri = host .. self.config.prefix .. NACOS_INSTANCE_PATH + + local resp, err = _request("GET", uri, params) + if not resp then + core.log.error("failed to fetch instances list from nacos, uri: ", uri, " err: ", err) + return {} + end + return resp.hosts or {} +end + +local function is_grpc(scheme) + return scheme == "grpc" or scheme == "grpcs" +end + + +local function fetch_instances(self, serv) + local config = self.config + + local params = { + namespaceId = serv.namespace_id or "", + groupName = serv.group_name or "DEFAULT_GROUP", + serviceName = serv.name, + healthyOnly = "true" + } + + local auth = config.auth or {} + -- for backward compat: + -- In older method, we passed username and password inside of host + -- In new method its passed separately + local username, password, host + if config.old_conf then + -- extract username and password from host + host, username, password = get_base_uri(config.hosts) + else + host = config.hosts[math_random(#config.hosts)] + if (auth.username and auth.username ~= "") and (auth.password and auth.password ~= "") then + username = auth.username + password = auth.password + end + end + if username and username ~= "" and password and password ~= "" then + local token = request_login(self, host, username, password) + params["accessToken"] = token + end + + if auth.token and auth.token ~= "" then + params["accessToken"] = auth.token + end + + if (auth.access_key and auth.access_key ~= "") and + (auth.secret_key and auth.secret_key ~= "") then + local ak, data, signature = utils.generate_signature(serv.group_name, + serv.name, auth.access_key, auth.secret_key) + params["ak"] = ak + params["data"] = data + params["signature"] = signature + end + + local instances = request_instance_list(self, params, host) + local nodes = {} + for _, instance in ipairs(instances) do + local node = { + host = instance.ip, + port = instance.port, + weight = instance.weight or self.config.default_weight, + metadata = instance.metadata, + } + -- docs: https://github.com/yidongnan/grpc-spring-boot-starter/pull/496 + if is_grpc(instance.scheme) and instance.metadata and instance.metadata.gRPC_port then + node.port = host.metadata.gRPC_port + end + core.table.insert(nodes, node) + end + return nodes +end + +local curr_services_in_use +local function fetch_full_registry(self) + return function (premature) + if premature then + return + end + + local config = self.config + local services_in_use = utils.get_nacos_services(config.id) + local service_names = {} + for _, serv in ipairs(services_in_use) do + if self.stop_flag then + core.log.error("nacos client is exited, id: ", config.id) + return + end + local key = utils.generate_key(serv.id, serv.namespace_id, + serv.group_name, serv.name) + service_names[key] = true + local nodes = self:fetch_instances(serv) + if #nodes > 0 then + local content = core.json.encode(nodes) + local key = key + nacos_dict:set(key, content) + end + end + -- delete unused service names + if not curr_services_in_use then + curr_services_in_use = {} + end + if not curr_services_in_use[config.id] then + curr_services_in_use[config.id] = {} + end + for name in pairs(service_names) do + curr_services_in_use[config.id][name] = true + end + for name in pairs(curr_services_in_use[config.id]) do + if not service_names[name] then + nacos_dict:delete(name) + curr_services_in_use[config.id][name] = nil + end + end + ngx_timer_at(self.config.fetch_interval, self:fetch_full_registry()) + end +end + + +local function stop(self) + self.stop_flag = true + + if self.checker then + self.checker:clear() + end +end + + +local function start(self) + ngx_timer_at(0, self:fetch_full_registry()) +end + + +function _M.new(config) + local version = ngx.md5(core.json.encode(config, true)) + + local client = { + id = config.id, + version = version, + config = config, + stop_flag = false, + + start = start, + stop = stop, + fetch_instances = fetch_instances, + fetch_full_registry = fetch_full_registry, + } + + if config.check then + local health_check = require("resty.healthcheck") + local checker = health_check.new({ + name = config.id, + shm_name = "nacos", + checks = config.check + }) + + local ok, err = checker:add_target(config.check.active.host, + config.check.active.port, nil, false) + if not ok then + core.log.error("failed to add health check target", + core.json.encode(config), " err: ", err) + else + core.log.info("success to add health checker, id ", config.id, + " host ", config.check.active.host, " port ", config.check.active.port) + client.checker = checker + end + end + + return client +end + +return _M diff --git a/apisix/discovery/nacos/init.lua b/apisix/discovery/nacos/init.lua index 86e9d4125a83..cc0d5427d829 100644 --- a/apisix/discovery/nacos/init.lua +++ b/apisix/discovery/nacos/init.lua @@ -15,410 +15,147 @@ -- limitations under the License. -- -local require = require -local local_conf = require('apisix.core.config_local').local_conf() -local http = require('resty.http') -local core = require('apisix.core') -local ipairs = ipairs -local type = type -local math = math -local math_random = math.random -local ngx = ngx -local ngx_re = require('ngx.re') -local ngx_timer_at = ngx.timer.at -local ngx_timer_every = ngx.timer.every -local string = string -local string_sub = string.sub -local str_byte = string.byte -local str_find = core.string.find -local log = core.log +local core = require("apisix.core") +local nacos_factory = require("apisix.discovery.nacos.factory") +local utils = require("apisix.discovery.nacos.utils") +local process = require("ngx.process") +local ipairs = ipairs +local require = require +local table = require("apisix.core.table") +local pcall = pcall +local ngx = ngx + +local shdict_name = "nacos" + +local nacos_dict = ngx.shared[shdict_name] +local OLD_CONFIG_ID = utils.old_config_id +local _M = {} -local default_weight -local applications -local auth_path = 'auth/login' -local instance_list_path = 'ns/instance/list?healthyOnly=true&serviceName=' -local default_namespace_id = "public" -local default_group_name = "DEFAULT_GROUP" -local access_key -local secret_key - -local events -local events_list - - -local _M = {} - -local function discovery_nacos_callback(data, event, source, pid) - applications = data - log.notice("update local variable application, event is: ", event, - "source: ", source, "server pid:", pid, - ", application: ", core.json.encode(applications, true)) -end - - -local function request(request_uri, path, body, method, basic_auth) - local url = request_uri .. path - log.info('request url:', url) - local headers = {} - headers['Accept'] = 'application/json' - - if basic_auth then - headers['Authorization'] = basic_auth - end +function _M.nodes(service_name, discovery_args) + local ns_id = discovery_args and discovery_args.namespace_id or utils.default_namespace_id + local group_name = discovery_args and discovery_args.group_name or utils.default_group_name + local id = discovery_args and discovery_args.id or OLD_CONFIG_ID + local key = utils.generate_key(id, + ns_id, + group_name, + service_name) + local value = nacos_dict:get(key) + local nodes = {} + if not value then + -- maximum waiting time: 5 seconds + local waiting_time = 5 + local step = 0.1 + local logged = false + while not value and waiting_time > 0 do + if not logged then + logged = true + end - if body and 'table' == type(body) then - local err - body, err = core.json.encode(body) - if not body then - return nil, 'invalid body : ' .. err + ngx.sleep(step) + waiting_time = waiting_time - step + value = nacos_dict:get(key) end - headers['Content-Type'] = 'application/json' - end - - local httpc = http.new() - local timeout = local_conf.discovery.nacos.timeout - local connect_timeout = timeout.connect - local send_timeout = timeout.send - local read_timeout = timeout.read - log.info('connect_timeout:', connect_timeout, ', send_timeout:', send_timeout, - ', read_timeout:', read_timeout) - httpc:set_timeouts(connect_timeout, send_timeout, read_timeout) - local res, err = httpc:request_uri(url, { - method = method, - headers = headers, - body = body, - ssl_verify = true, - }) - if not res then - return nil, err - end - - if not res.body or res.status ~= 200 then - return nil, 'status = ' .. res.status - end - - local json_str = res.body - local data, err = core.json.decode(json_str) - if not data then - return nil, err end - return data -end - - -local function get_url(request_uri, path) - return request(request_uri, path, nil, 'GET', nil) -end - - -local function post_url(request_uri, path, body) - return request(request_uri, path, body, 'POST', nil) -end - - -local function get_token_param(base_uri, username, password) - if not username or not password then - return '' + if not value then + core.log.error("nacos service not found: ", service_name) + return nodes end - local args = { username = username, password = password} - local data, err = post_url(base_uri, auth_path .. '?' .. ngx.encode_args(args), nil) - if err then - log.error('nacos login fail:', username, ' ', password, ' desc:', err) - return nil, err - end - return '&accessToken=' .. data.accessToken -end + nodes = core.json.decode(value) - -local function get_namespace_param(namespace_id) - local param = '' - if namespace_id then - local args = {namespaceId = namespace_id} - param = '&' .. ngx.encode_args(args) - end - return param -end - - -local function get_group_name_param(group_name) - local param = '' - if group_name then - local args = {groupName = group_name} - param = '&' .. ngx.encode_args(args) - end - return param -end - - -local function get_signed_param(group_name, service_name) - local param = '' - if access_key ~= '' and secret_key ~= '' then - local str_to_sign = ngx.now() * 1000 .. '@@' .. group_name .. '@@' .. service_name - local args = { - ak = access_key, - data = str_to_sign, - signature = ngx.encode_base64(ngx.hmac_sha1(secret_key, str_to_sign)) - } - param = '&' .. ngx.encode_args(args) - end - return param -end - - -local function get_base_uri() - local host = local_conf.discovery.nacos.host - -- TODO Add health check to get healthy nodes. - local url = host[math_random(#host)] - local auth_idx = core.string.rfind_char(url, '@') - local username, password - if auth_idx then - local protocol_idx = str_find(url, '://') - local protocol = string_sub(url, 1, protocol_idx + 2) - local user_and_password = string_sub(url, protocol_idx + 3, auth_idx - 1) - local arr = ngx_re.split(user_and_password, ':') - if #arr == 2 then - username = arr[1] - password = arr[2] + local res = {} + for _, node in ipairs(nodes) do + if discovery_args then + if utils.match_metdata(node.metadata, discovery_args.metadata) then + core.table.insert(res, node) + end + else + core.table.insert(res, node) end - local other = string_sub(url, auth_idx + 1) - url = protocol .. other - end - - if local_conf.discovery.nacos.prefix then - url = url .. local_conf.discovery.nacos.prefix end - if str_byte(url, #url) ~= str_byte('/') then - url = url .. '/' - end - - return url, username, password + core.log.info("nacos service_name: ", service_name, " nodes: ", core.json.encode(res)) + return res end - -local function de_duplication(services, namespace_id, group_name, service_name, scheme) - for _, service in ipairs(services) do - if service.namespace_id == namespace_id and service.group_name == group_name - and service.service_name == service_name and service.scheme == scheme then - return true - end - end - return false +local function generate_new_config_from_old(discovery_conf) + local config = { + id = OLD_CONFIG_ID, + hosts = discovery_conf.host, + prefix = discovery_conf.prefix, + fetch_interval = discovery_conf.fetch_interval, + auth = { + access_key = discovery_conf.access_key, + secret_key = discovery_conf.secret_key, + }, + default_weight = discovery_conf.weight, + timeout = discovery_conf.timeout, + old_conf = true + } + return {config} end +function _M.init_worker() + local local_conf = require("apisix.core.config_local").local_conf(true) + local discovery_conf = local_conf.discovery and local_conf.discovery.nacos or {} -local function iter_and_add_service(services, values) - if not values then + if process.type() ~= "privileged agent" then return end - for _, value in core.config_util.iterate_values(values) do - local conf = value.value - if not conf then - goto CONTINUE - end - - local up - if conf.upstream then - up = conf.upstream - else - up = conf - end - - local namespace_id = (up.discovery_args and up.discovery_args.namespace_id) - or default_namespace_id - - local group_name = (up.discovery_args and up.discovery_args.group_name) - or default_group_name - - local dup = de_duplication(services, namespace_id, group_name, - up.service_name, up.scheme) - if dup then - goto CONTINUE - end - - if up.discovery_type == 'nacos' then - core.table.insert(services, { - service_name = up.service_name, - namespace_id = namespace_id, - group_name = group_name, - scheme = up.scheme, - }) - end - ::CONTINUE:: + -- support old way + if discovery_conf.host then + discovery_conf = generate_new_config_from_old(discovery_conf) end -end - - -local function get_nacos_services() - local services = {} - - -- here we use lazy load to work around circle dependency - local get_upstreams = require('apisix.upstream').upstreams - local get_routes = require('apisix.router').http_routes - local get_stream_routes = require('apisix.router').stream_routes - local get_services = require('apisix.http.service').services - local values = get_upstreams() - iter_and_add_service(services, values) - values = get_routes() - iter_and_add_service(services, values) - values = get_services() - iter_and_add_service(services, values) - values = get_stream_routes() - iter_and_add_service(services, values) - return services -end - -local function is_grpc(scheme) - if scheme == 'grpc' or scheme == 'grpcs' then - return true + for _, val in ipairs(discovery_conf) do + local new_client = nacos_factory.new(val) + new_client:start() end - - return false end -local function fetch_full_registry(premature) - if premature then - return - end +-- Now we use control plane to list the services +function _M.list_all_services() + return {} +end - local up_apps = {} - local base_uri, username, password = get_base_uri() - local token_param, err = get_token_param(base_uri, username, password) - if err then - log.error('get_token_param error:', err) - if not applications then - applications = up_apps - end - return - end - local infos = get_nacos_services() - if #infos == 0 then - applications = up_apps - return - end +local cjson = require "cjson" - for _, service_info in ipairs(infos) do - local data, err - local namespace_id = service_info.namespace_id - local group_name = service_info.group_name - local scheme = service_info.scheme or '' - local namespace_param = get_namespace_param(service_info.namespace_id) - local group_name_param = get_group_name_param(service_info.group_name) - local signature_param = get_signed_param(service_info.group_name, service_info.service_name) - local query_path = instance_list_path .. service_info.service_name - .. token_param .. namespace_param .. group_name_param - .. signature_param - data, err = get_url(base_uri, query_path) - if err then - log.error('get_url:', query_path, ' err:', err) - goto CONTINUE - end - - if not up_apps[namespace_id] then - up_apps[namespace_id] = {} - end +function _M.dump_data() + local applications = {} + local keys = nacos_dict:get_keys() or {} - if not up_apps[namespace_id][group_name] then - up_apps[namespace_id][group_name] = {} + for _, key in ipairs(keys) do + local parts = {} + for part in key:gmatch("[^/]+") do + table.insert(parts, part) end - for _, host in ipairs(data.hosts) do - local nodes = up_apps[namespace_id] - [group_name][service_info.service_name] - if not nodes then - nodes = {} - up_apps[namespace_id] - [group_name][service_info.service_name] = nodes - end - - local node = { - host = host.ip, - port = host.port, - weight = host.weight or default_weight, - } - - -- docs: https://github.com/yidongnan/grpc-spring-boot-starter/pull/496 - if is_grpc(scheme) and host.metadata and host.metadata.gRPC_port then - node.port = host.metadata.gRPC_port + if #parts == 4 then + local id, namespace_id, + group_name, service_name = parts[1], parts[2], parts[3], parts[4] + local data_str = nacos_dict:get(key) + + if data_str and data_str ~= "" then + -- Decode JSON string to Lua table + local success, data = pcall(cjson.decode, data_str) + if success then + applications[id] = applications[id] or {} + applications[id][namespace_id] = applications[id][namespace_id] or {} + applications[id][namespace_id][group_name] = applications[id] + [namespace_id][group_name] or {} + applications[id][namespace_id][group_name][service_name] = data + else + ngx.log(ngx.ERR, "failed to decode data for key ", key, ": ", data) + end end - - core.table.insert(nodes, node) - end - - ::CONTINUE:: - end - local new_apps_md5sum = ngx.md5(core.json.encode(up_apps)) - local old_apps_md5sum = ngx.md5(core.json.encode(applications)) - if new_apps_md5sum == old_apps_md5sum then - return - end - applications = up_apps - local ok, err = events:post(events_list._source, events_list.updating, - applications) - if not ok then - log.error("post_event failure with ", events_list._source, - ", update application error: ", err) - end -end - - -function _M.nodes(service_name, discovery_args) - local namespace_id = discovery_args and - discovery_args.namespace_id or default_namespace_id - local group_name = discovery_args - and discovery_args.group_name or default_group_name - - local logged = false - -- maximum waiting time: 5 seconds - local waiting_time = 5 - local step = 0.1 - while not applications and waiting_time > 0 do - if not logged then - log.warn('wait init') - logged = true end - ngx.sleep(step) - waiting_time = waiting_time - step end - if not applications or not applications[namespace_id] - or not applications[namespace_id][group_name] - then - return nil - end - return applications[namespace_id][group_name][service_name] + return { + services = applications + } end - -function _M.init_worker() - events = require("apisix.events") - events_list = events:event_list("discovery_nacos_update_application", - "updating") - - if 0 ~= ngx.worker.id() then - events:register(discovery_nacos_callback, events_list._source, - events_list.updating) - return - end - - default_weight = local_conf.discovery.nacos.weight - log.info('default_weight:', default_weight) - local fetch_interval = local_conf.discovery.nacos.fetch_interval - log.info('fetch_interval:', fetch_interval) - access_key = local_conf.discovery.nacos.access_key - secret_key = local_conf.discovery.nacos.secret_key - ngx_timer_at(0, fetch_full_registry) - ngx_timer_every(fetch_interval, fetch_full_registry) -end - - -function _M.dump_data() - return {config = local_conf.discovery.nacos, services = applications or {}} -end - - return _M diff --git a/apisix/discovery/nacos/schema.lua b/apisix/discovery/nacos/schema.lua index 294048736e58..d63d3b3fdaf6 100644 --- a/apisix/discovery/nacos/schema.lua +++ b/apisix/discovery/nacos/schema.lua @@ -19,41 +19,106 @@ local prefix_pattern = [[^[\/a-zA-Z0-9-_.]+$]] return { - type = 'object', - properties = { - host = { - type = 'array', - minItems = 1, - items = { - type = 'string', - pattern = host_pattern, - minLength = 2, - maxLength = 100, - }, - }, - fetch_interval = {type = 'integer', minimum = 1, default = 30}, - prefix = { - type = 'string', - pattern = prefix_pattern, - maxLength = 100, - default = '/nacos/v1/' - }, - weight = {type = 'integer', minimum = 1, default = 100}, - timeout = { + oneOf = { + -- Legacy object format + { type = 'object', properties = { - connect = {type = 'integer', minimum = 1, default = 2000}, - send = {type = 'integer', minimum = 1, default = 2000}, - read = {type = 'integer', minimum = 1, default = 5000}, + host = { + type = 'array', + minItems = 1, + items = { + type = 'string', + pattern = host_pattern, + minLength = 2, + maxLength = 100, + }, + }, + fetch_interval = {type = 'integer', minimum = 1, default = 30}, + prefix = { + type = 'string', + pattern = prefix_pattern, + maxLength = 100, + default = '/nacos/v1/' + }, + weight = {type = 'integer', minimum = 1, default = 100}, + timeout = { + type = 'object', + properties = { + connect = {type = 'integer', minimum = 1, default = 2000}, + send = {type = 'integer', minimum = 1, default = 2000}, + read = {type = 'integer', minimum = 1, default = 5000}, + }, + default = { + connect = 2000, + send = 2000, + read = 5000, + } + }, + access_key = {type = 'string', default = ''}, + secret_key = {type = 'string', default = ''}, }, - default = { - connect = 2000, - send = 2000, - read = 5000, - } + required = {'host'}, + additionalProperties = false }, - access_key = {type = 'string', default = ''}, - secret_key = {type = 'string', default = ''}, - }, - required = {'host'} + -- New array format + { + type = 'array', + minItems = 1, + items = { + type = 'object', + properties = { + id = { + type = 'string', + minLength = 1, + maxLength = 50, + pattern = '^[a-zA-Z0-9_-]+$' + }, + hosts = { + type = 'array', + minItems = 1, + items = { + type = 'string', + pattern = host_pattern, + minLength = 2, + maxLength = 100, + }, + }, + prefix = { + type = 'string', + pattern = prefix_pattern, + maxLength = 100, + default = '/nacos/v1/' + }, + fetch_interval = {type = 'integer', minimum = 1, default = 30}, + default_weight = {type = 'integer', minimum = 1, default = 100}, + timeout = { + type = 'object', + properties = { + connect = {type = 'integer', minimum = 1, default = 2000}, + send = {type = 'integer', minimum = 1, default = 2000}, + read = {type = 'integer', minimum = 1, default = 5000}, + }, + default = { + connect = 2000, + send = 2000, + read = 5000, + } + }, + auth = { + type = 'object', + properties = { + username = {type = 'string', default = ''}, + password = {type = 'string', default = ''}, + access_key = {type = 'string', default = ''}, + secret_key = {type = 'string', default = ''}, + }, + default = {} + } + }, + required = {'id', 'hosts'}, + additionalProperties = false + } + } + } } diff --git a/apisix/discovery/nacos/utils.lua b/apisix/discovery/nacos/utils.lua new file mode 100644 index 000000000000..d9d8ded2feaf --- /dev/null +++ b/apisix/discovery/nacos/utils.lua @@ -0,0 +1,196 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local core = require("apisix.core") +local ngx = ngx +local ipairs = ipairs +local require = require +local pairs = pairs +local str_format = string.format +local OLD_CONFIG_ID = "0" +local default_namespace_id = "public" +local default_group_name = "DEFAULT_GROUP" +local _M = { + old_config_id = OLD_CONFIG_ID, + default_namespace_id = default_namespace_id, + default_group_name = default_group_name, +} + +local function parse_service_name(service_name) + -- support old data + -- NOTE: If old service_name contains "/", it will be parsed as + -- registry_id/namespace_id/group_name/service_name + -- and termed invalid + if not service_name:find("/") then + return OLD_CONFIG_ID, "", "", service_name + end + local pattern = "^(.*)/(.*)/(.*)/(.*)$" -- registry_id/namespace_id/group_name/service_name + local match = ngx.re.match(service_name, pattern, "jo") + if not match then + core.log.error("get unexpected upstream service_name: ", service_name) + return "" + end + + return match[1], match[2], match[3], match[4] +end + +_M.parse_service_name = parse_service_name + +local function de_duplication(services, namespace_id, group_name, service_name, scheme) + for _, service in ipairs(services) do + if service.namespace_id == namespace_id and service.group_name == group_name + and service.service_name == service_name and service.scheme == scheme then + return true + end + end + return false +end + +local function iter_and_add_service(services, hash, id, values) + if not values then + return + end + + for _, value in core.config_util.iterate_values(values) do + local conf = value.value + if not conf then + goto CONTINUE + end + + local upstream + if conf.upstream then + upstream = conf.upstream + else + upstream = conf + end + + if upstream.discovery_type ~= "nacos" then + goto CONTINUE + end + + if hash[upstream.service_name] then + goto CONTINUE + end + + local service_registry_id, namespace_id, + group_name, name = parse_service_name(upstream.service_name) + if service_registry_id ~= id then + goto CONTINUE + end + + if not namespace_id or namespace_id == "" then + namespace_id = upstream.discovery_args and upstream.discovery_args.namespace_id + or default_namespace_id + end + if not group_name or group_name == "" then + group_name = upstream.discovery_args and upstream.discovery_args.group_name + or default_group_name + end + local dup = de_duplication(services, namespace_id, group_name, + upstream.service_name, upstream.scheme) + if dup then + goto CONTINUE + end + core.table.insert(services, { + name = name, + namespace_id = namespace_id, + group_name = group_name, + service_name = upstream.service_name, + scheme = upstream.scheme, + id = id, + }) + + ::CONTINUE:: + end +end + +function _M.generate_key(id, ns_id, group_name, service_name) + -- new data expects service_name to be in the format and + -- will use that as key directly + if service_name:find("/") then + return service_name + end + return str_format("%s/%s/%s/%s", id, ns_id, group_name, service_name) +end + +function _M.get_nacos_services(service_registry_id) + local services = {} + local services_hash = {} + + -- here we use lazy load to work around circle dependency + local get_upstreams = require('apisix.upstream').upstreams + local get_routes = require('apisix.router').http_routes + local get_stream_routes = require('apisix.router').stream_routes + local get_services = require('apisix.http.service').services + local values = get_upstreams() + iter_and_add_service(services, services_hash, service_registry_id, values) + values = get_routes() + iter_and_add_service(services, services_hash, service_registry_id, values) + values = get_services() + iter_and_add_service(services, services_hash, service_registry_id, values) + values = get_stream_routes() + iter_and_add_service(services, services_hash, service_registry_id, values) + return services +end + + +function _M.generate_signature(group_name, service_name, access_key, secret_key) + local str_to_sign = ngx.now() * 1000 .. '@@' .. group_name .. '@@' .. service_name + return access_key, str_to_sign, ngx.encode_base64(ngx.hmac_sha1(secret_key, str_to_sign)) +end + + +function _M.generate_request_params(params) + if params == nil then + return "" + end + + local args = "" + local first = false + for k, v in pairs(params) do + if not first then + args = str_format("%s&%s=%s", args, k, v) + else + first = true + args = str_format("%s=%s", k, v) + end + end + + return args +end + + +function _M.match_metdata(node_metadata, upstream_metadata) + if upstream_metadata == nil then + return true + end + + if not node_metadata then + node_metadata = {} + end + + for k, v in pairs(upstream_metadata) do + if not node_metadata[k] or node_metadata[k] ~= v then + return false + end + end + + return true +end + + +return _M diff --git a/docs/en/latest/discovery/nacos.md b/docs/en/latest/discovery/nacos.md index 5ebbcee46b49..1f47185bd801 100644 --- a/docs/en/latest/discovery/nacos.md +++ b/docs/en/latest/discovery/nacos.md @@ -29,8 +29,16 @@ The performance of this module needs to be improved: ### Configuration for Nacos +```yaml title="conf/config.yaml" +Nacos can be configured in two ways: single instance or multi-instance. +``` + Add following configuration in `conf/config.yaml` : +### Single Instance method + +The below is a single nacos instance method that is also supported. + ```yaml discovery: nacos: @@ -56,11 +64,44 @@ discovery: - "http://192.168.33.1:8848" ``` +#### Multi Instance method + +```yaml +discovery: + nacos: + - id: "nacos-cluster-1" # Unique ID for this Nacos cluster (required) + hosts: # List of Nacos hosts (required) + - "http://${host1}:${port1}" + prefix: "/nacos/v1/" # API prefix + fetch_interval: 30 # default 30 sec + default_weight: 100 # default 100 + auth: # Authentication credentials + username: username + password: password + timeout: + connect: 2000 # default 2000 ms + send: 2000 # default 2000 ms + read: 5000 # default 5000 ms + - id: "nacos-cluster-2" # Unique ID for this Nacos cluster (required) + hosts: # List of Nacos hosts (required) + - "http://${host2}:${port2}" + prefix: "/nacos/v1/" # API prefix + fetch_interval: 30 # default 30 sec + default_weight: 100 # default 100 + auth: # Authentication credentials + username: username2 + password: password2 + timeout: + connect: 2000 # default 2000 ms + send: 2000 # default 2000 ms + read: 5000 # default 5000 ms +``` + ### Upstream setting #### L7 -Here is an example of routing a request with an URI of "/nacos/*" to a service which named "http://192.168.33.1:8848/nacos/v1/ns/instance/list?serviceName=APISIX-NACOS" and use nacos discovery client in the registry: +The following is an example of routing requests matching the URI `/nacos/*` to a service at `http://192.168.33.1:8848/nacos/v1/ns/instance/list?serviceName=APISIX-NACOS`, configured with the ID `nacos-cluster-1` and using the Nacos discovery client in the registry.” :::note You can fetch the `admin_key` from `config.yaml` and save to an environment variable with the following command: @@ -79,6 +120,9 @@ $ curl http://127.0.0.1:9180/apisix/admin/routes/1 -H "X-API-KEY: $admin_key" -X "service_name": "APISIX-NACOS", "type": "roundrobin", "discovery_type": "nacos" + "discovery_args":{ + "id": "nacos-cluster-1" + } } }' ``` @@ -100,7 +144,10 @@ The formatted response as below: "scheme": "http", "service_name": "APISIX-NACOS", "type": "roundrobin", - "discovery_type": "nacos" + "discovery_type": "nacos", + "discovery_args":{ + "id": "nacos-cluster-1" + } }, "priority": 0, "uri": "\/nacos\/*" @@ -130,6 +177,7 @@ $ curl http://127.0.0.1:9180/apisix/admin/stream_routes/1 -H "X-API-KEY: $admin_ | Name | Type | Requirement | Default | Valid | Description | | ------------ | ------ | ----------- | ------- | ----- | ------------------------------------------------------------ | +| id | string | optional | "0" | | (Used in multi instance mode)This parameter is used to specify the unique ID of a nacos instance. | | namespace_id | string | optional | public | | This parameter is used to specify the namespace of the corresponding service | | group_name | string | optional | DEFAULT_GROUP | | This parameter is used to specify the group of the corresponding service | diff --git a/t/APISIX.pm b/t/APISIX.pm index 69f4b6fb18ac..035a661f921c 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -277,6 +277,7 @@ lua { lua_shared_dict prometheus-metrics 15m; lua_shared_dict standalone-config 10m; lua_shared_dict status-report 1m; + lua_shared_dict nacos 10m; } _EOC_ } diff --git a/t/cli/test_validate_config.sh b/t/cli/test_validate_config.sh index 0f8a09a43c2d..598d12b74189 100755 --- a/t/cli/test_validate_config.sh +++ b/t/cli/test_validate_config.sh @@ -28,7 +28,9 @@ discovery: ' > conf/config.yaml out=$(make init 2>&1 || true) -if ! echo "$out" | grep 'property "host" validation failed: wrong type: expected array, got string'; then + +if ! echo "$out" | grep 'invalid discovery nacos configuration: value should match only one schema, but matches none'; then +cat logs/error.log echo "failed: should check discovery schema during init" exit 1 fi diff --git a/t/discovery/nacos-multi-cluster.t b/t/discovery/nacos-multi-cluster.t new file mode 100644 index 000000000000..29b5d580496e --- /dev/null +++ b/t/discovery/nacos-multi-cluster.t @@ -0,0 +1,959 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('info'); +worker_connections(256); +no_root_location(); +no_shuffle(); +workers(4); + +our $yaml_config = <<_EOC_; +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + nacos: + - id: "0" + hosts: + - "http://127.0.0.1:8858" + prefix: "/nacos/v1/" + fetch_interval: 1 + default_weight: 1 + timeout: + connect: 2000 + send: 2000 + read: 5000 + +_EOC_ + +our $yaml_auth_config = <<_EOC_; +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + nacos: + - id: "0" + hosts: + - "http://127.0.0.1:8858" + prefix: "/nacos/v1/" + fetch_interval: 1 + default_weight: 1 + auth: + username: nacos + password: nacos + timeout: + connect: 2000 + send: 2000 + read: 5000 + +_EOC_ + +run_tests(); + +__DATA__ + +=== TEST 1: get APISIX-NACOS info from NACOS - no auth +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: APISIX-NACOS + discovery_type: nacos + type: roundrobin +#END +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello", +] +--- response_body_like eval +[ + qr/server [1-2]/, + qr/server [1-2]/, +] +--- no_error_log +[error, error] + + + +=== TEST 2: error service_name name - no auth +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: APISIX-NACOS-DEMO + discovery_type: nacos + type: roundrobin + +#END +--- request +GET /hello +--- error_code: 503 +--- error_log +no valid upstream node +--- timeout: 10 + + + +=== TEST 3: get APISIX-NACOS info from NACOS - auth +--- yaml_config eval: $::yaml_auth_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: APISIX-NACOS + discovery_type: nacos + type: roundrobin + +#END +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello", +] +--- response_body_like eval +[ + qr/server [1-2]/, + qr/server [1-2]/, +] +--- no_error_log +[error, error] +--- timeout: 10 + + + +=== TEST 4: error service_name name - auth +--- yaml_config eval: $::yaml_auth_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: APISIX-NACOS-DEMO + discovery_type: nacos + type: roundrobin + +#END +--- request +GET /hello +--- error_code: 503 +--- error_log +no valid upstream node +--- timeout: 10 + + + +=== TEST 5: get APISIX-NACOS info from NACOS - configured in services +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + service_id: 1 +services: + - + id: 1 + upstream: + service_name: APISIX-NACOS + discovery_type: nacos + type: roundrobin +#END +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello", +] +--- response_body_like eval +[ + qr/server [1-2]/, + qr/server [1-2]/, +] +--- timeout: 10 + + + +=== TEST 6: get APISIX-NACOS info from NACOS - configured in upstreams + etcd +--- extra_yaml_config +discovery: + nacos: + host: + - "http://127.0.0.1:8858" + fetch_interval: 1 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/upstreams/1', + ngx.HTTP_PUT, + [[{ + "service_name": "APISIX-NACOS", + "discovery_type": "nacos", + "type": "roundrobin" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/hello", + "upstream_id": 1 + }]] + ) + + if code >= 300 then + ngx.status = code + end + + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- timeout: 10 + + + +=== TEST 7: hit +--- extra_yaml_config +discovery: + nacos: + host: + - "http://127.0.0.1:8858" + fetch_interval: 1 +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello", +] +--- response_body_like eval +[ + qr/server [1-2]/, + qr/server [1-2]/, +] +--- timeout: 10 + + + +=== TEST 8: get APISIX-NACOS info from NACOS - no auth with namespace +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: APISIX-NACOS + discovery_type: nacos + type: roundrobin + discovery_args: + namespace_id: test_ns +#END +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello", +] +--- response_body_like eval +[ + qr/server [1-2]/, + qr/server [1-2]/, +] +--- timeout: 10 + + + +=== TEST 9: error namespace_id - no auth +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: APISIX-NACOS-DEMO + discovery_type: nacos + type: roundrobin + discovery_args: + namespace_id: err_ns +#END +--- request +GET /hello +--- error_code: 503 +--- error_log +no valid upstream node +--- timeout: 10 + + + +=== TEST 10: get APISIX-NACOS info from NACOS - configured in services with namespace +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + service_id: 1 +services: + - + id: 1 + upstream: + service_name: APISIX-NACOS + discovery_type: nacos + type: roundrobin + discovery_args: + namespace_id: test_ns +#END +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello", +] +--- response_body_like eval +[ + qr/server [1-2]/, + qr/server [1-2]/, +] +--- timeout: 10 + + + +=== TEST 11: get APISIX-NACOS info from NACOS - configured in upstreams + etcd with namespace +--- extra_yaml_config +discovery: + nacos: + host: + - "http://127.0.0.1:8858" + fetch_interval: 1 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/upstreams/1', + ngx.HTTP_PUT, + [[{ + "service_name": "APISIX-NACOS", + "discovery_type": "nacos", + "type": "roundrobin", + "discovery_args": { + "namespace_id": "test_ns" + } + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/hello", + "upstream_id": 1 + }]] + ) + + if code >= 300 then + ngx.status = code + end + + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- timeout: 10 + + + +=== TEST 12: hit with namespace +--- extra_yaml_config +discovery: + nacos: + host: + - "http://127.0.0.1:8858" + fetch_interval: 1 +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello", +] +--- response_body_like eval +[ + qr/server [1-2]/, + qr/server [1-2]/, +] +--- timeout: 10 + + + +=== TEST 13: get APISIX-NACOS info from NACOS - no auth with group_name +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: APISIX-NACOS + discovery_type: nacos + type: roundrobin + discovery_args: + group_name: test_group +#END +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello", +] +--- response_body_like eval +[ + qr/server [1-2]/, + qr/server [1-2]/, +] +--- timeout: 10 + + + +=== TEST 14: error group_name - no auth +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: APISIX-NACOS-DEMO + discovery_type: nacos + type: roundrobin + discovery_args: + group_name: err_group_name +#END +--- request +GET /hello +--- error_code: 503 +--- error_log +no valid upstream node +--- timeout: 10 + + + +=== TEST 15: get APISIX-NACOS info from NACOS - configured in services with group_name +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + service_id: 1 +services: + - + id: 1 + upstream: + service_name: APISIX-NACOS + discovery_type: nacos + type: roundrobin + discovery_args: + group_name: test_group +#END +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello", +] +--- response_body_like eval +[ + qr/server [1-2]/, + qr/server [1-2]/, +] +--- timeout: 10 + + + +=== TEST 16: get APISIX-NACOS info from NACOS - configured in upstreams + etcd with group_name +--- extra_yaml_config +discovery: + nacos: + host: + - "http://127.0.0.1:8858" + fetch_interval: 1 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/upstreams/1', + ngx.HTTP_PUT, + [[{ + "service_name": "APISIX-NACOS", + "discovery_type": "nacos", + "type": "roundrobin", + "discovery_args": { + "group_name": "test_group" + } + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/hello", + "upstream_id": 1 + }]] + ) + + if code >= 300 then + ngx.status = code + end + + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- timeout: 10 + + + +=== TEST 17: hit with group_name +--- extra_yaml_config +discovery: + nacos: + host: + - "http://127.0.0.1:8858" + fetch_interval: 1 +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello", +] +--- response_body_like eval +[ + qr/server [1-2]/, + qr/server [1-2]/, +] +--- timeout: 10 + + + +=== TEST 18: get APISIX-NACOS info from NACOS - no auth with namespace_id and group_name +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: APISIX-NACOS + discovery_type: nacos + type: roundrobin + discovery_args: + namespace_id: test_ns + group_name: test_group +#END +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello", +] +--- response_body_like eval +[ + qr/server [1-2]/, + qr/server [1-2]/, +] +--- timeout: 10 + + + +=== TEST 19: error group_name and correct namespace_id - no auth +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: APISIX-NACOS-DEMO + discovery_type: nacos + type: roundrobin + discovery_args: + namespace_id: test_ns + group_name: err_group_name +#END +--- request +GET /hello +--- error_code: 503 +--- error_log +no valid upstream node +--- timeout: 10 + + + +=== TEST 20: error namespace_id and correct group_name - no auth +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: APISIX-NACOS-DEMO + discovery_type: nacos + type: roundrobin + discovery_args: + namespace_id: err_ns + group_name: test_group +#END +--- request +GET /hello +--- error_code: 503 +--- error_log +no valid upstream node +--- timeout: 10 + + + +=== TEST 21: error namespace_id and error group_name - no auth +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: APISIX-NACOS-DEMO + discovery_type: nacos + type: roundrobin + discovery_args: + namespace_id: err_ns + group_name: err_group_name +#END +--- request +GET /hello +--- error_code: 503 +--- error_log +no valid upstream node +--- timeout: 10 + + + +=== TEST 22: get APISIX-NACOS info from NACOS - configured in services with namespace_id and group_name +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + service_id: 1 +services: + - + id: 1 + upstream: + service_name: APISIX-NACOS + discovery_type: nacos + type: roundrobin + discovery_args: + namespace_id: test_ns + group_name: test_group +#END +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello", +] +--- response_body_like eval +[ + qr/server [1-2]/, + qr/server [1-2]/, +] +--- timeout: 10 + + + +=== TEST 23: get APISIX-NACOS info from NACOS - configured in upstreams + etcd with namespace_id and group_name +--- extra_yaml_config +discovery: + nacos: + host: + - "http://127.0.0.1:8858" + fetch_interval: 1 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/upstreams/1', + ngx.HTTP_PUT, + [[{ + "service_name": "APISIX-NACOS", + "discovery_type": "nacos", + "type": "roundrobin", + "discovery_args": { + "namespace_id": "test_ns", + "group_name": "test_group" + } + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/hello", + "upstream_id": 1 + }]] + ) + + if code >= 300 then + ngx.status = code + end + + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- timeout: 10 + + + +=== TEST 24: hit with namespace_id and group_name +--- extra_yaml_config +discovery: + nacos: + host: + - "http://127.0.0.1:8858" + fetch_interval: 1 +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello", +] +--- response_body_like eval +[ + qr/server [1-2]/, + qr/server [1-2]/, +] +--- timeout: 10 + + + +=== TEST 25: same namespace_id and service_name, different group_name +--- extra_yaml_config +discovery: + nacos: + host: + - "http://127.0.0.1:8858" + fetch_interval: 1 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + + -- use nacos-service5 + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/hello", + "upstream": { + "service_name": "APISIX-NACOS", + "discovery_type": "nacos", + "type": "roundrobin", + "discovery_args": { + "namespace_id": "test_ns", + "group_name": "test_group" + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + + -- use nacos-service6 + local code, body = t('/apisix/admin/routes/2', + ngx.HTTP_PUT, + [[{ + "uri": "/hello1", + "upstream": { + "service_name": "APISIX-NACOS", + "discovery_type": "nacos", + "type": "roundrobin", + "discovery_args": { + "namespace_id": "test_ns", + "group_name": "test_group2" + } + }, + "plugins": { + "proxy-rewrite": { + "uri": "/hello" + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + + ngx.sleep(1.5) + + local http = require "resty.http" + local httpc = http.new() + local uri1 = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local res, err = httpc:request_uri(uri1, { method = "GET"}) + if err then + ngx.log(ngx.ERR, err) + ngx.status = res.status + return + end + ngx.say(res.body) + + local uri2 = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello1" + res, err = httpc:request_uri(uri2, { method = "GET"}) + if err then + ngx.log(ngx.ERR, err) + ngx.status = res.status + return + end + ngx.say(res.body) + } + } +--- request +GET /t +--- response_body +server 1 +server 3 +--- timeout: 10 + + + +=== TEST 26: same group_name and service_name, different namespace_id +--- extra_yaml_config +discovery: + nacos: + host: + - "http://127.0.0.1:8858" + fetch_interval: 1 +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + + -- use nacos-service5 + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/hello", + "upstream": { + "service_name": "APISIX-NACOS", + "discovery_type": "nacos", + "type": "roundrobin", + "discovery_args": { + "namespace_id": "test_ns", + "group_name": "test_group" + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + + -- use nacos-service7 + local code, body = t('/apisix/admin/routes/2', + ngx.HTTP_PUT, + [[{ + "uri": "/hello1", + "upstream": { + "service_name": "APISIX-NACOS", + "discovery_type": "nacos", + "type": "roundrobin", + "discovery_args": { + "namespace_id": "test_ns2", + "group_name": "test_group" + } + }, + "plugins": { + "proxy-rewrite": { + "uri": "/hello" + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + + ngx.sleep(1.5) + + local http = require "resty.http" + local httpc = http.new() + local uri1 = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local res, err = httpc:request_uri(uri1, { method = "GET"}) + if err then + ngx.log(ngx.ERR, err) + ngx.status = res.status + return + end + ngx.say(res.body) + + local uri2 = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello1" + res, err = httpc:request_uri(uri2, { method = "GET"}) + if err then + ngx.log(ngx.ERR, err) + ngx.status = res.status + return + end + ngx.say(res.body) + } + } +--- request +GET /t +--- response_body +server 1 +server 4 +--- timeout: 10 diff --git a/t/discovery/nacos-multi-cluster2.t b/t/discovery/nacos-multi-cluster2.t new file mode 100644 index 000000000000..d38f53efe3cd --- /dev/null +++ b/t/discovery/nacos-multi-cluster2.t @@ -0,0 +1,133 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('info'); +worker_connections(256); +no_root_location(); +no_shuffle(); +workers(4); + +our $yaml_config = <<_EOC_; +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + nacos: + - id: "svc1" + hosts: + - "http://127.0.0.1:8858" + prefix: "/nacos/v1/" + fetch_interval: 1 + default_weight: 1 + timeout: + connect: 2000 + send: 2000 + read: 5000 + - id: "svc2" + hosts: + - "http://127.0.0.1:8858" + prefix: "/nacos/v1/" + fetch_interval: 1 + default_weight: 1 + timeout: + connect: 2000 + send: 2000 + read: 5000 + +_EOC_ + +our $yaml_auth_config = <<_EOC_; +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + nacos: + - id: "svc1" + hosts: + - "http://127.0.0.1:8858" + prefix: "/nacos/v1/" + fetch_interval: 1 + default_weight: 1 + auth: + username: nacos + password: nacos + timeout: + connect: 2000 + send: 2000 + read: 5000 + - id: "svc2" + hosts: + - "http://127.0.0.1:8858" + prefix: "/nacos/v1/" + fetch_interval: 1 + default_weight: 1 + auth: + username: nacos + password: nacos + timeout: + connect: 2000 + send: 2000 + read: 5000 + +_EOC_ + +run_tests(); + +__DATA__ + +=== TEST 1: get APISIX-NACOS info from NACOS - no auth +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: svc1/public/DEFAULT_GROUP/APISIX-NACOS + discovery_type: nacos + type: roundrobin + discovery_args: + id: svc1 + - + uri: /hello2 + upstream: + service_name: svc2/public/DEFAULT_GROUP/APISIX-NACOS + discovery_type: nacos + type: roundrobin + discovery_args: + id: svc2 +#END +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello", +] +--- response_body_like eval +[ + qr/server [1-2]/, + qr/server [1-2]/, +] +--- no_error_log +[error, error] +--- timeout: 10 diff --git a/t/discovery/nacos.t b/t/discovery/nacos.t index 9af1ee14a814..77f694530a78 100644 --- a/t/discovery/nacos.t +++ b/t/discovery/nacos.t @@ -65,6 +65,12 @@ discovery: _EOC_ +add_block_preprocessor(sub { + my ($block) = @_; + $block->set_value("timeout", "10"); + +}); + run_tests(); __DATA__ diff --git a/t/discovery/nacos2.t b/t/discovery/nacos2.t index e7dc8f97318a..0278edc76b47 100644 --- a/t/discovery/nacos2.t +++ b/t/discovery/nacos2.t @@ -24,6 +24,7 @@ add_block_preprocessor(sub { if (!$block->request) { $block->set_value("request", "GET /t"); } + $block->set_value("timeout", "10"); }); run_tests(); @@ -83,7 +84,7 @@ GET /hello --- response_body_like eval qr/server [1-2]/ --- error_log -err:status = 502 +err: status = 502 @@ -308,7 +309,7 @@ discovery: local body = json_decode(res.body) local services = body.services - local service = services["public"]["DEFAULT_GROUP"]["APISIX-NACOS"] + local service = services["0"]["public"]["DEFAULT_GROUP"]["APISIX-NACOS"] local number = table.getn(service) ngx.say(number) } diff --git a/t/discovery/nacos3.t b/t/discovery/nacos3.t index c71259386c7e..41050c7ea0df 100644 --- a/t/discovery/nacos3.t +++ b/t/discovery/nacos3.t @@ -26,6 +26,7 @@ add_block_preprocessor(sub { if (!$block->request) { $block->set_value("request", "GET /t"); } + $block->set_value("timeout", "10"); }); our $yaml_config = <<_EOC_; diff --git a/t/discovery/stream/nacos.t b/t/discovery/stream/nacos.t index 1a1053cfdffd..bb8106fb06db 100644 --- a/t/discovery/stream/nacos.t +++ b/t/discovery/stream/nacos.t @@ -50,6 +50,7 @@ add_block_preprocessor(sub { if (!$block->stream_request) { $block->set_value("stream_request", "GET /hello HTTP/1.1\r\nHost: 127.0.0.1:1985\r\nConnection: close\r\n\r\n"); } + $block->set_value("timeout", "10"); });