Skip to content

fix(healthcheck): use immediate clear() to ensure k8s endpoint change… #13029

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tyq010101 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix(healthcheck): use immediate clear() to ensure k8s endpoint change… #13029

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions apisix/healthcheck_manager.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ local _M = {}
local working_pool = {} -- resource_path -> {version = ver, checker = checker}
local waiting_pool = {} -- resource_path -> resource_ver

local DELAYED_CLEAR_TIMEOUT = 10
local healthcheck_shdict_name = "upstream-healthcheck"


Expand Down Expand Up @@ -192,7 +191,14 @@ local function timer_create_checker()
-- if a checker exists then delete it before creating a new one
local existing_checker = working_pool[resource_path]
if existing_checker then
existing_checker.checker:delayed_clear(DELAYED_CLEAR_TIMEOUT)
-- use immediate clear() instead of delayed_clear() to ensure
-- k8s endpoint changes take effect immediately, avoiding stale IP addresses
local clear_ok, clear_err = pcall(function()
existing_checker.checker:clear()
end)
if not clear_ok then
core.log.warn("failed to clear checker: ", clear_err)
end
existing_checker.checker:stop()
core.log.info("releasing existing checker: ", tostring(existing_checker.checker),
" for resource: ", resource_path, " and version: ",
Expand Down Expand Up @@ -251,7 +257,14 @@ local function timer_working_pool_check()
if need_destroy then
working_pool[resource_path] = nil
item.checker.dead = true
item.checker:delayed_clear(DELAYED_CLEAR_TIMEOUT)
-- use immediate clear() instead of delayed_clear() to ensure
-- k8s endpoint changes take effect immediately, avoiding stale IP addresses
local clear_ok, clear_err = pcall(function()
item.checker:clear()
end)
if not clear_ok then
core.log.warn("failed to clear checker: ", clear_err)
end
item.checker:stop()
core.log.info("try to release checker: ", tostring(item.checker), " for resource: ",
resource_path, " and version : ", item.version)
Expand Down
7 changes: 6 additions & 1 deletion t/node/healthcheck-leak-bugfix.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,13 @@ __DATA__
local new = healthcheck.new
healthcheck.new = function(...)
local obj = new(...)
local clear = obj.delayed_clear
local delayed_clear = obj.delayed_clear
obj.delayed_clear = function(...)
ngx.log(ngx.WARN, "clear checker")
return delayed_clear(...)
end
local clear = obj.clear
obj.clear = function(...)
ngx.log(ngx.WARN, "clear checker")
return clear(...)
end
Expand Down