diff --git a/apisix/core/response.lua b/apisix/core/response.lua index baee97749598..d6199bbb2b28 100644 --- a/apisix/core/response.lua +++ b/apisix/core/response.lua @@ -19,6 +19,7 @@ -- -- @module core.response +local tracer = require("apisix.utils.tracer") local encode_json = require("cjson.safe").encode local ngx = ngx local arg = ngx.arg @@ -62,6 +63,7 @@ function resp_exit(code, ...) ngx.status = code end + local message for i = 1, select('#', ...) do local v = select(i, ...) if type(v) == "table" then @@ -73,6 +75,7 @@ function resp_exit(code, ...) t[idx] = body idx = idx + 1 t[idx] = "\n" + message = body end elseif v ~= nil then @@ -86,6 +89,9 @@ function resp_exit(code, ...) end if code then + if code >= 400 then + tracer.finish_all_spans(tracer.status.ERROR, message or ("response code " .. code)) + end return ngx_exit(code) end end diff --git a/apisix/init.lua b/apisix/init.lua index 430572e27e48..cffffc991761 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -47,6 +47,7 @@ local debug = require("apisix.debug") local pubsub_kafka = require("apisix.pubsub.kafka") local resource = require("apisix.resource") local trusted_addresses_util = require("apisix.utils.trusted-addresses") +local tracer = require("apisix.utils.tracer") local ngx = ngx local get_method = ngx.req.get_method local ngx_exit = ngx.exit @@ -203,6 +204,8 @@ function _M.ssl_client_hello_phase() local api_ctx = core.tablepool.fetch("api_ctx", 0, 32) ngx_ctx.api_ctx = api_ctx + local span = tracer.new_span("ssl_client_hello_phase", tracer.kind.server) + local ok, err = router.router_ssl.match_and_set(api_ctx, true, sni) ngx_ctx.matched_ssl = api_ctx.matched_ssl @@ -215,18 +218,23 @@ function _M.ssl_client_hello_phase() core.log.error("failed to fetch ssl config: ", err) end core.log.error("failed to match any SSL certificate by SNI: ", sni) + span:set_status(tracer.status.ERROR, "failed match SNI") + tracer.finish_current_span() ngx_exit(-1) end ok, err = apisix_ssl.set_protocols_by_clienthello(ngx_ctx.matched_ssl.value.ssl_protocols) if not ok then core.log.error("failed to set ssl protocols: ", err) + span:set_status(tracer.status.ERROR, "failed set protocols") + tracer.finish_current_span() ngx_exit(-1) end -- in stream subsystem, ngx.ssl.server_name() return hostname of ssl session in preread phase, -- so that we can't get real SNI without recording it in ngx.ctx during client_hello phase ngx.ctx.client_hello_sni = sni + tracer.finish_current_span() end @@ -666,6 +674,7 @@ end function _M.http_access_phase() + tracer.new_span("apisix.phase.access", tracer.kind.server) -- from HTTP/3 to HTTP/1.1 we need to convert :authority pesudo-header -- to Host header, so we set upstream_host variable here. if ngx.req.http_version() == 3 then @@ -716,19 +725,26 @@ function _M.http_access_phase() handle_x_forwarded_headers(api_ctx) + local router_match_span = tracer.new_span("http_router_match", tracer.kind.internal) router.router_http.match(api_ctx) local route = api_ctx.matched_route if not route then + tracer.new_span("run_global_rules", tracer.kind.internal) -- run global rule when there is no matching route local global_rules = apisix_global_rules.global_rules() plugin.run_global_rules(api_ctx, global_rules, nil) + tracer.finish_current_span() core.log.info("not find any matched route") + router_match_span:set_status(tracer.status.ERROR, "no matched route") + tracer.finish_current_span() return core.response.exit(404, {error_msg = "404 Route Not Found"}) end + tracer.finish_current_span() + core.log.info("matched route: ", core.json.delay_encode(api_ctx.matched_route, true)) @@ -784,7 +800,6 @@ function _M.http_access_phase() else local plugins = plugin.filter(api_ctx, route) api_ctx.plugins = plugins - plugin.run_plugin("rewrite", plugins, api_ctx) if api_ctx.consumer then local changed @@ -904,8 +919,10 @@ end function _M.http_body_filter_phase() + tracer.new_span("apisix.phase.body_filter", tracer.kind.server) common_phase("body_filter") common_phase("delayed_body_filter") + tracer.finish_current_span() end diff --git a/apisix/plugin.lua b/apisix/plugin.lua index 789eb528d546..be2e12cd526e 100644 --- a/apisix/plugin.lua +++ b/apisix/plugin.lua @@ -38,6 +38,7 @@ local tostring = tostring local error = error local getmetatable = getmetatable local setmetatable = setmetatable +local tracer = require("apisix.utils.tracer") -- make linter happy to avoid error: getting the Lua global "load" -- luacheck: globals load, ignore lua_load local lua_load = load @@ -1221,7 +1222,7 @@ function _M.run_plugin(phase, plugins, api_ctx) end return api_ctx, plugin_run end - + tracer.new_span("apisix.plugins.phase." .. phase) for i = 1, #plugins, 2 do local phase_func = plugins[i][phase] local conf = plugins[i + 1] @@ -1229,11 +1230,13 @@ function _M.run_plugin(phase, plugins, api_ctx) plugin_run = true run_meta_pre_function(conf, api_ctx, plugins[i]["name"]) api_ctx._plugin_name = plugins[i]["name"] + tracer.new_span("apisix.phase." .. phase .. "." .. api_ctx._plugin_name) phase_func(conf, api_ctx) + tracer.finish_current_span() api_ctx._plugin_name = nil end end - + tracer.finish_current_span() return api_ctx, plugin_run end diff --git a/apisix/plugins/opentelemetry.lua b/apisix/plugins/opentelemetry.lua index d98ac44ae69d..0931ac0ac97e 100644 --- a/apisix/plugins/opentelemetry.lua +++ b/apisix/plugins/opentelemetry.lua @@ -376,6 +376,10 @@ function _M.rewrite(conf, api_ctx) ngx_var.opentelemetry_span_id = span_context.span_id end + if not ctx:span():is_recording() then + ngx.ctx._apisix_skip_tracing = true + end + api_ctx.otel_context_token = ctx:attach() -- inject trace context into the headers of upstream HTTP request @@ -383,42 +387,74 @@ function _M.rewrite(conf, api_ctx) end -function _M.delayed_body_filter(conf, api_ctx) - if api_ctx.otel_context_token and ngx.arg[2] then - local ctx = context:current() - ctx:detach(api_ctx.otel_context_token) - api_ctx.otel_context_token = nil +local function create_child_span(tracer, parent_span_ctx, span) + local new_span_ctx, new_span = tracer:start(parent_span_ctx, span.name, + { + kind = span.kind, + attributes = span.attributes, + }) + new_span.start_time = span.start_time - -- get span from current context - local span = ctx:span() - local upstream_status = core.response.get_upstream_status(api_ctx) - if upstream_status and upstream_status >= 500 then - span:set_status(span_status.ERROR, - "upstream response status: " .. upstream_status) - end + for _, child in ipairs(span.children or {}) do + create_child_span(tracer, new_span_ctx, child) + end + if span.status then + new_span:set_status(span.status.code, span.status.message) + end + new_span:finish(span.end_time) +end - span:set_attributes(attr.int("http.status_code", upstream_status)) - span:finish() +local function inject_core_spans(root_span_ctx, api_ctx, conf) + local metadata = plugin.plugin_metadata(plugin_name) + local plugin_info = metadata.value + if root_span_ctx.span and not root_span_ctx:span():is_recording() then + return + end + local inject_conf = { + sampler = { + name = "always_on", + options = conf.sampler.options + }, + additional_attributes = conf.additional_attributes, + additional_header_prefix_attributes = conf.additional_header_prefix_attributes + } + local tracer, err = core.lrucache.plugin_ctx(lrucache, api_ctx, nil, + create_tracer_obj, inject_conf, plugin_info) + if not tracer then + core.log.error("failed to fetch tracer object: ", err) + return + end + for _, sp in ipairs(ngx.ctx._apisix_spans or {}) do + if root_span_ctx.span_context then + create_child_span(tracer, root_span_ctx, sp) + end end end --- body_filter maybe not called because of empty http body response --- so we need to check if the span has finished in log phase function _M.log(conf, api_ctx) if api_ctx.otel_context_token then -- ctx:detach() is not necessary, because of ctx is stored in ngx.ctx local upstream_status = core.response.get_upstream_status(api_ctx) -- get span from current context - local span = context:current():span() + local ctx = context:current() + local span = ctx:span() if upstream_status and upstream_status >= 500 then span:set_status(span_status.ERROR, "upstream response status: " .. upstream_status) end + inject_core_spans(ctx, api_ctx, conf) + span:set_attributes(attr.int("http.status_code", upstream_status)) span:finish() + if ngx.ctx._apisix_spans then + for _, sp in ipairs(ngx.ctx._apisix_spans) do + sp:release() + end + ngx.ctx._apisix_spans = nil + end end end diff --git a/apisix/secret.lua b/apisix/secret.lua index b8d7b19a522c..1db6e7246d82 100644 --- a/apisix/secret.lua +++ b/apisix/secret.lua @@ -18,6 +18,7 @@ local require = require local core = require("apisix.core") local string = require("apisix.core.string") +local tracer = require("apisix.utils.tracer") local local_conf = require("apisix.core.config_local").local_conf() @@ -148,6 +149,7 @@ local function fetch_by_uri(secret_uri) return nil, "no secret conf, secret_uri: " .. secret_uri end + local span = tracer.new_span("fetch_secret", tracer.kind.client) local ok, sm = pcall(require, "apisix.secret." .. opts.manager) if not ok then return nil, "no secret manager: " .. opts.manager @@ -155,9 +157,12 @@ local function fetch_by_uri(secret_uri) local value, err = sm.get(conf, opts.key) if err then + span:set_status(tracer.status.ERROR, err) + tracer.finish_current_span() return nil, err end + tracer.finish_current_span() return value end diff --git a/apisix/ssl/router/radixtree_sni.lua b/apisix/ssl/router/radixtree_sni.lua index ae7e5b265bf9..027cbb19a303 100644 --- a/apisix/ssl/router/radixtree_sni.lua +++ b/apisix/ssl/router/radixtree_sni.lua @@ -21,6 +21,7 @@ local apisix_ssl = require("apisix.ssl") local secret = require("apisix.secret") local ngx_ssl = require("ngx.ssl") local config_util = require("apisix.core.config_util") +local tracer = require("apisix.utils.tracer") local ngx = ngx local ipairs = ipairs local type = type @@ -149,11 +150,15 @@ function _M.match_and_set(api_ctx, match_only, alt_sni) local err if not radixtree_router or radixtree_router_ver ~= ssl_certificates.conf_version then + local span = tracer.new_span("create_router", tracer.kind.internal) radixtree_router, err = create_router(ssl_certificates.values) if not radixtree_router then + span:set_status(tracer.status.ERROR, "failed create router") + tracer.finish_current_span() return false, "failed to create radixtree router: " .. err end radixtree_router_ver = ssl_certificates.conf_version + tracer.finish_current_span() end local sni = alt_sni @@ -170,6 +175,7 @@ function _M.match_and_set(api_ctx, match_only, alt_sni) core.log.debug("sni: ", sni) local sni_rev = sni:reverse() + local span = tracer.new_span("sni_radixtree_match", tracer.kind.internal) local ok = radixtree_router:dispatch(sni_rev, nil, api_ctx) if not ok then if not alt_sni then @@ -177,8 +183,11 @@ function _M.match_and_set(api_ctx, match_only, alt_sni) -- with it sometimes core.log.error("failed to find any SSL certificate by SNI: ", sni) end + span:set_status(tracer.status.ERROR, "failed match SNI") + tracer.finish_current_span() return false end + tracer.finish_current_span() if type(api_ctx.matched_sni) == "table" then diff --git a/apisix/utils/span.lua b/apisix/utils/span.lua new file mode 100644 index 000000000000..126c25116473 --- /dev/null +++ b/apisix/utils/span.lua @@ -0,0 +1,87 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local tablepool = require("tablepool") +local util = require("opentelemetry.util") +local span_status = require("opentelemetry.trace.span_status") +local setmetatable = setmetatable +local table = table +local select = select +local pool_name = "opentelemetry_span" + +local _M = {} + + +local mt = { + __index = _M +} + + +function _M.new(name, kind) + local self = tablepool.fetch(pool_name, 0, 8) + self.name = name + self.start_time = util.time_nano() + self.end_time = 0 + self.kind = kind + self.attributes = self.attributes or {} + self.children = self.children or {} + self.status = nil + return setmetatable(self, mt) +end + + +function _M.append_child(self, span) + table.insert(self.children, span) +end + + +function _M.set_status(self, code, message) + code = span_status.validate(code) + local status = self.status + if not status then + status = { + code = code, + message = "" + } + self.status = status + else + status.code = code + end + + if code == span_status.ERROR then + status.message = message + end +end + + +function _M.set_attributes(self, ...) + local count = select('#', ...) + for i = 1, count do + local attr = select(i, ...) + table.insert(self.attributes, attr) + end +end + + +function _M.finish(self) + self.end_time = util.time_nano() +end + +function _M.release(self) + tablepool.release(pool_name, self) +end + +return _M diff --git a/apisix/utils/stack.lua b/apisix/utils/stack.lua new file mode 100644 index 000000000000..ad639d55337b --- /dev/null +++ b/apisix/utils/stack.lua @@ -0,0 +1,75 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local _M = {} +local mt = { __index = _M } +local setmetatable = setmetatable +local table = require("apisix.core.table") + +function _M.new() + local self = { + _data = {}, + _n = 0, + } + return setmetatable(self, mt) +end + + +function _M.push(self, value) + self._n = self._n + 1 + self._data[self._n] = value +end + + +function _M.pop(self) + if self._n == 0 then + return nil + end + + local value = self._data[self._n] + self._data[self._n] = nil + self._n = self._n - 1 + return value +end + + +function _M.peek(self) + if self._n == 0 then + return nil + end + + return self._data[self._n] +end + + +function _M.is_empty(self) + return self._n == 0 +end + + +function _M.size(self) + return self._n +end + + +function _M.clear(self) + table.clear(self._data) + self._n = 0 +end + + +return _M + diff --git a/apisix/utils/tracer.lua b/apisix/utils/tracer.lua new file mode 100644 index 000000000000..86e3ae0cec05 --- /dev/null +++ b/apisix/utils/tracer.lua @@ -0,0 +1,80 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local ngx = ngx +local stack = require("apisix.utils.stack") +local span = require("apisix.utils.span") +local span_kind = require("opentelemetry.trace.span_kind") +local span_status = require("opentelemetry.trace.span_status") +local table = table +local pairs = pairs + +local _M = { + kind = span_kind, + status = span_status, +} + + +function _M.new_span(name, kind) + local ctx = ngx.ctx + if not ctx._apisix_spans then + ctx._apisix_spans = {} + end + if not ctx._apisix_span_stack then + ctx._apisix_span_stack = stack.new() + end + local sp = span.new(name, kind) + if ctx._apisix_skip_tracing then + return sp + end + if ctx._apisix_span_stack:is_empty() then + table.insert(ctx._apisix_spans, sp) + else + local parent_span = ctx._apisix_span_stack:peek() + parent_span:append_child(sp) + end + ctx._apisix_span_stack:push(sp) + return sp +end + + +function _M.finish_current_span(code, message) + if not ngx.ctx._apisix_span_stack then + return + end + local sp = ngx.ctx._apisix_span_stack:pop() + if code then + sp:set_status(code, message) + end + sp:finish() +end + +function _M.finish_all_spans(code, message) + local apisix_spans = ngx.ctx._apisix_spans + if not apisix_spans then + return + end + + for _, sp in pairs(apisix_spans) do + if code then + sp:set_status(code, message) + end + sp:finish() + end +end + + +return _M diff --git a/apisix/utils/upstream.lua b/apisix/utils/upstream.lua index 49e8a18a1075..a55eaae63e49 100644 --- a/apisix/utils/upstream.lua +++ b/apisix/utils/upstream.lua @@ -20,7 +20,7 @@ local ngx_now = ngx.now local ipairs = ipairs local type = type local tostring = tostring - +local tracer = require("apisix.utils.tracer") local _M = {} @@ -80,6 +80,7 @@ _M.compare_upstream_node = compare_upstream_node local function parse_domain_for_nodes(nodes) + tracer.new_span("resolve_dns", tracer.kind.internal) local new_nodes = core.table.new(#nodes, 0) for _, node in ipairs(nodes) do local host = node.host @@ -100,6 +101,7 @@ local function parse_domain_for_nodes(nodes) core.table.insert(new_nodes, node) end end + tracer.finish_current_span() return new_nodes end _M.parse_domain_for_nodes = parse_domain_for_nodes diff --git a/docs/en/latest/plugins/opentelemetry.md b/docs/en/latest/plugins/opentelemetry.md index 061c26212dd5..6891a377e273 100644 --- a/docs/en/latest/plugins/opentelemetry.md +++ b/docs/en/latest/plugins/opentelemetry.md @@ -153,37 +153,152 @@ In OpenTelemetry collector's log, you should see information similar to the foll ```text 2024-02-18T17:14:03.825Z info ResourceSpans #0 -Resource SchemaURL: -Resource attributes: - -> telemetry.sdk.language: Str(lua) - -> telemetry.sdk.name: Str(opentelemetry-lua) - -> telemetry.sdk.version: Str(0.1.1) - -> hostname: Str(e34673e24631) - -> service.name: Str(APISIX) ScopeSpans #0 ScopeSpans SchemaURL: InstrumentationScope opentelemetry-lua Span #0 - Trace ID : fbd0a38d4ea4a128ff1a688197bc58b0 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 905f850f13e32bfb + ID : 5a3835b61110d942 + Name : http_router_match + Kind : Internal + Start time : 2025-10-24 06:58:04.430430976 +0000 UTC + End time : 2025-10-24 06:58:04.431542016 +0000 UTC + Status code : Unset + Status message : +Span #1 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 905f850f13e32bfb + ID : 4ab25e2b92f394e1 + Name : resolve_dns + Kind : Internal + Start time : 2025-10-24 06:58:04.432521984 +0000 UTC + End time : 2025-10-24 06:58:04.44903296 +0000 UTC + Status code : Unset + Status message : +Span #2 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 905f850f13e32bfb + ID : 3620c0f05dd2be4f + Name : apisix.phase.header_filter + Kind : Internal + Start time : 2025-10-24 06:58:06.960481024 +0000 UTC + End time : 2025-10-24 06:58:06.960510976 +0000 UTC + Status code : Unset + Status message : +Span #3 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 4c5f3476f62a7e8a + ID : a9bfad7bb6986e41 + Name : apisix.phase.body_filter + Kind : Internal + Start time : 2025-10-24 06:58:06.960579072 +0000 UTC + End time : 2025-10-24 06:58:06.96059008 +0000 UTC + Status code : Unset + Status message : +Span #4 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : b2994675df6baa83 + ID : 26705f9c47584a5b + Name : apisix.phase.delayed_body_filter.opentelemetry + Kind : Internal + Start time : 2025-10-24 06:58:06.960613888 +0000 UTC + End time : 2025-10-24 06:58:06.960687104 +0000 UTC + Status code : Unset + Status message : +Span #5 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 4c5f3476f62a7e8a + ID : b2994675df6baa83 + Name : apisix.phase.delayed_body_filter + Kind : Internal + Start time : 2025-10-24 06:58:06.96059904 +0000 UTC + End time : 2025-10-24 06:58:06.960692992 +0000 UTC + Status code : Unset + Status message : +Span #6 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 905f850f13e32bfb + ID : 4c5f3476f62a7e8a + Name : apisix.phase.body_filter + Kind : Server + Start time : 2025-10-24 06:58:06.96056704 +0000 UTC + End time : 2025-10-24 06:58:06.960698112 +0000 UTC + Status code : Unset + Status message : +Span #7 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 2024d73d32cbd81b + ID : 223c64fb691a24e8 + Name : apisix.phase.body_filter + Kind : Internal + Start time : 2025-10-24 06:58:06.961624064 +0000 UTC + End time : 2025-10-24 06:58:06.961635072 +0000 UTC + Status code : Unset + Status message : +Span #8 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : fd193dd24c618f60 + ID : 8729ad6e0d94a23b + Name : apisix.phase.delayed_body_filter.opentelemetry + Kind : Internal + Start time : 2025-10-24 06:58:06.961648896 +0000 UTC + End time : 1970-01-01 00:00:00 +0000 UTC + Status code : Unset + Status message : +Span #9 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 2024d73d32cbd81b + ID : fd193dd24c618f60 + Name : apisix.phase.delayed_body_filter + Kind : Internal + Start time : 2025-10-24 06:58:06.961641984 +0000 UTC + End time : 1970-01-01 00:00:00 +0000 UTC + Status code : Unset + Status message : +Span #10 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 905f850f13e32bfb + ID : 2024d73d32cbd81b + Name : apisix.phase.body_filter + Kind : Server + Start time : 2025-10-24 06:58:06.960980992 +0000 UTC + End time : 1970-01-01 00:00:00 +0000 UTC + Status code : Unset + Status message : +Span #11 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : cfb0b4603dc2e385 + ID : 905f850f13e32bfb + Name : apisix.phase.access + Kind : Server + Start time : 2025-10-24 06:58:04.427932928 +0000 UTC + End time : 1970-01-01 00:00:00 +0000 UTC + Status code : Unset + Status message : +Span #12 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a Parent ID : - ID : af3dc7642104748a - Name : GET /anything + ID : cfb0b4603dc2e385 + Name : GET /headers Kind : Server - Start time : 2024-02-18 17:14:03.763244032 +0000 UTC - End time : 2024-02-18 17:14:03.920229888 +0000 UTC + Start time : 2025-10-24 06:58:04.432427008 +0000 UTC + End time : 2025-10-24 06:58:06.962299904 +0000 UTC Status code : Unset Status message : Attributes: -> net.host.name: Str(127.0.0.1) -> http.method: Str(GET) -> http.scheme: Str(http) - -> http.target: Str(/anything) - -> http.user_agent: Str(curl/7.64.1) + -> http.target: Str(/headers) + -> http.user_agent: Str(curl/8.16.0) -> apisix.route_id: Str(otel-tracing-route) -> apisix.route_name: Empty() - -> http.route: Str(/anything) + -> http.route: Str(/headers) -> http.status_code: Int(200) -{"kind": "exporter", "data_type": "traces", "name": "debug"} +{"resource": {"service.instance.id": "5006c483-d64c-4d1d-87ac-edb037ba3669", "service.name": "otelcol-contrib", "service.version": "0.138.0"}, "otelcol.component.id": "debug", "otelcol.component.kind": "exporter", "otelcol.signal": "traces"} +2025-10-24T06:58:13.893Z info Metrics {"resource": {"service.instance.id": "5006c483-d64c-4d1d-87ac-edb037ba3669", "service.name": "otelcol-contrib", "service.version": "0.138.0"}, "otelcol.component.id": "debug", "otelcol.component.kind": "exporter", "otelcol.signal": "metrics", "resource metrics": 1, "metrics": 25, "data points": 26} +2025-10-24T06:58:13.893Z info ResourceMetrics #0 ``` To visualize these traces, you can export your telemetry to backend Services, such as Zipkin and Prometheus. See [exporters](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter) for more details. diff --git a/docs/zh/latest/plugins/opentelemetry.md b/docs/zh/latest/plugins/opentelemetry.md index f22d90c932b3..18f0566bd03d 100644 --- a/docs/zh/latest/plugins/opentelemetry.md +++ b/docs/zh/latest/plugins/opentelemetry.md @@ -152,37 +152,152 @@ curl "http://127.0.0.1:9080/anything" ```text 2024-02-18T17:14:03.825Z info ResourceSpans #0 -Resource SchemaURL: -Resource attributes: - -> telemetry.sdk.language: Str(lua) - -> telemetry.sdk.name: Str(opentelemetry-lua) - -> telemetry.sdk.version: Str(0.1.1) - -> hostname: Str(e34673e24631) - -> service.name: Str(APISIX) ScopeSpans #0 ScopeSpans SchemaURL: InstrumentationScope opentelemetry-lua Span #0 - Trace ID : fbd0a38d4ea4a128ff1a688197bc58b0 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 905f850f13e32bfb + ID : 5a3835b61110d942 + Name : http_router_match + Kind : Internal + Start time : 2025-10-24 06:58:04.430430976 +0000 UTC + End time : 2025-10-24 06:58:04.431542016 +0000 UTC + Status code : Unset + Status message : +Span #1 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 905f850f13e32bfb + ID : 4ab25e2b92f394e1 + Name : resolve_dns + Kind : Internal + Start time : 2025-10-24 06:58:04.432521984 +0000 UTC + End time : 2025-10-24 06:58:04.44903296 +0000 UTC + Status code : Unset + Status message : +Span #2 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 905f850f13e32bfb + ID : 3620c0f05dd2be4f + Name : apisix.phase.header_filter + Kind : Internal + Start time : 2025-10-24 06:58:06.960481024 +0000 UTC + End time : 2025-10-24 06:58:06.960510976 +0000 UTC + Status code : Unset + Status message : +Span #3 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 4c5f3476f62a7e8a + ID : a9bfad7bb6986e41 + Name : apisix.phase.body_filter + Kind : Internal + Start time : 2025-10-24 06:58:06.960579072 +0000 UTC + End time : 2025-10-24 06:58:06.96059008 +0000 UTC + Status code : Unset + Status message : +Span #4 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : b2994675df6baa83 + ID : 26705f9c47584a5b + Name : apisix.phase.delayed_body_filter.opentelemetry + Kind : Internal + Start time : 2025-10-24 06:58:06.960613888 +0000 UTC + End time : 2025-10-24 06:58:06.960687104 +0000 UTC + Status code : Unset + Status message : +Span #5 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 4c5f3476f62a7e8a + ID : b2994675df6baa83 + Name : apisix.phase.delayed_body_filter + Kind : Internal + Start time : 2025-10-24 06:58:06.96059904 +0000 UTC + End time : 2025-10-24 06:58:06.960692992 +0000 UTC + Status code : Unset + Status message : +Span #6 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 905f850f13e32bfb + ID : 4c5f3476f62a7e8a + Name : apisix.phase.body_filter + Kind : Server + Start time : 2025-10-24 06:58:06.96056704 +0000 UTC + End time : 2025-10-24 06:58:06.960698112 +0000 UTC + Status code : Unset + Status message : +Span #7 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 2024d73d32cbd81b + ID : 223c64fb691a24e8 + Name : apisix.phase.body_filter + Kind : Internal + Start time : 2025-10-24 06:58:06.961624064 +0000 UTC + End time : 2025-10-24 06:58:06.961635072 +0000 UTC + Status code : Unset + Status message : +Span #8 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : fd193dd24c618f60 + ID : 8729ad6e0d94a23b + Name : apisix.phase.delayed_body_filter.opentelemetry + Kind : Internal + Start time : 2025-10-24 06:58:06.961648896 +0000 UTC + End time : 1970-01-01 00:00:00 +0000 UTC + Status code : Unset + Status message : +Span #9 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 2024d73d32cbd81b + ID : fd193dd24c618f60 + Name : apisix.phase.delayed_body_filter + Kind : Internal + Start time : 2025-10-24 06:58:06.961641984 +0000 UTC + End time : 1970-01-01 00:00:00 +0000 UTC + Status code : Unset + Status message : +Span #10 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : 905f850f13e32bfb + ID : 2024d73d32cbd81b + Name : apisix.phase.body_filter + Kind : Server + Start time : 2025-10-24 06:58:06.960980992 +0000 UTC + End time : 1970-01-01 00:00:00 +0000 UTC + Status code : Unset + Status message : +Span #11 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a + Parent ID : cfb0b4603dc2e385 + ID : 905f850f13e32bfb + Name : apisix.phase.access + Kind : Server + Start time : 2025-10-24 06:58:04.427932928 +0000 UTC + End time : 1970-01-01 00:00:00 +0000 UTC + Status code : Unset + Status message : +Span #12 + Trace ID : 95a1644afaaf65e1f0193b1f193b990a Parent ID : - ID : af3dc7642104748a - Name : GET /anything + ID : cfb0b4603dc2e385 + Name : GET /headers Kind : Server - Start time : 2024-02-18 17:14:03.763244032 +0000 UTC - End time : 2024-02-18 17:14:03.920229888 +0000 UTC + Start time : 2025-10-24 06:58:04.432427008 +0000 UTC + End time : 2025-10-24 06:58:06.962299904 +0000 UTC Status code : Unset Status message : Attributes: - -> net.host.name: Str(127.0.0.1) - -> http.method: Str(GET) - -> http.scheme: Str(http) - -> http.target: Str(/anything) - -> http.user_agent: Str(curl/7.64.1) - -> apisix.route_id: Str(otel-tracing-route) - -> apisix.route_name: Empty() - -> http.route: Str(/anything) - -> http.status_code: Int(200) -{"kind": "exporter", "data_type": "traces", "name": "debug"} + -> net.host.name: Str(127.0.0.1) + -> http.method: Str(GET) + -> http.scheme: Str(http) + -> http.target: Str(/headers) + -> http.user_agent: Str(curl/8.16.0) + -> apisix.route_id: Str(otel-tracing-route) + -> apisix.route_name: Empty() + -> http.route: Str(/headers) + -> http.status_code: Int(200) +{"resource": {"service.instance.id": "5006c483-d64c-4d1d-87ac-edb037ba3669", "service.name": "otelcol-contrib", "service.version": "0.138.0"}, "otelcol.component.id": "debug", "otelcol.component.kind": "exporter", "otelcol.signal": "traces"} +2025-10-24T06:58:13.893Z info Metrics {"resource": {"service.instance.id": "5006c483-d64c-4d1d-87ac-edb037ba3669", "service.name": "otelcol-contrib", "service.version": "0.138.0"}, "otelcol.component.id": "debug", "otelcol.component.kind": "exporter", "otelcol.signal": "metrics", "resource metrics": 1, "metrics": 25, "data points": 26} +2025-10-24T06:58:13.893Z info ResourceMetrics #0 ``` 要可视化这些追踪,你可以将 traces 导出到后端服务,例如 Zipkin 和 Prometheus。有关更多详细信息,请参阅[exporters](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter)。 diff --git a/t/plugin/opentelemetry3.t b/t/plugin/opentelemetry3.t index ff7ea14e56e5..228f0f6d6763 100644 --- a/t/plugin/opentelemetry3.t +++ b/t/plugin/opentelemetry3.t @@ -161,6 +161,19 @@ hello world qr/opentelemetry export span/ --- grep_error_log_out opentelemetry export span +opentelemetry export span +opentelemetry export span +opentelemetry export span +opentelemetry export span +opentelemetry export span +opentelemetry export span +opentelemetry export span +opentelemetry export span +opentelemetry export span +opentelemetry export span +opentelemetry export span +opentelemetry export span +opentelemetry export span --- error_log eval qr/request log: \{.*"opentelemetry_context_traceparent":"00-\w{32}-\w{16}-01".*\}/ diff --git a/t/plugin/opentelemetry6.t b/t/plugin/opentelemetry6.t new file mode 100644 index 000000000000..0b01f623a0f9 --- /dev/null +++ b/t/plugin/opentelemetry6.t @@ -0,0 +1,231 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +BEGIN { + sub set_env_from_file { + my ($env_name, $file_path) = @_; + + open my $fh, '<', $file_path or die $!; + my $content = do { local $/; <$fh> }; + close $fh; + + $ENV{$env_name} = $content; + } + # set env + set_env_from_file('TEST_CERT', 't/certs/apisix.crt'); + set_env_from_file('TEST_KEY', 't/certs/apisix.key'); + set_env_from_file('TEST2_CERT', 't/certs/test2.crt'); + set_env_from_file('TEST2_KEY', 't/certs/test2.key'); +} +use t::APISIX 'no_plan'; +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->extra_yaml_config) { + my $extra_yaml_config = <<_EOC_; +plugins: + - opentelemetry +_EOC_ + $block->set_value("extra_yaml_config", $extra_yaml_config); + } + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } + + if (!defined $block->response_body) { + $block->set_value("response_body", "passed\n"); + } + $block; +}); +repeat_each(1); +no_long_string(); +no_root_location(); +log_level("debug"); + +run_tests; + +__DATA__ + +=== TEST 1: empty file +--- exec +echo '' > ci/pod/otelcol-contrib/data-otlp.json +--- response_body eval +qr// + + + +=== TEST 2: add plugin metadata +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/plugin_metadata/opentelemetry', + ngx.HTTP_PUT, + [[{ + "batch_span_processor": { + "max_export_batch_size": 1, + "inactive_timeout": 0.5 + }, + "collector": { + "address": "127.0.0.1:4318", + "request_timeout": 3, + "request_headers": { + "foo": "bar" + } + }, + "trace_id_source": "x-request-id" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 3: set route +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "opentelemetry": { + "sampler": { + "name": "always_on" + } + } + }, + "upstream": { + "nodes": { + "test1.com:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t + + + +=== TEST 4: set ssl with two certs and keys in env +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin") + + local data = { + snis = {"test.com"}, + key = "$env://TEST_KEY", + cert = "$env://TEST_CERT", + keys = {"$env://TEST2_KEY"}, + certs = {"$env://TEST2_CERT"} + } + + local code, body = t.test('/apisix/admin/ssls/1', + ngx.HTTP_PUT, + core.json.encode(data), + [[{ + "value": { + "snis": ["test.com"], + "key": "$env://TEST_KEY", + "cert": "$env://TEST_CERT", + "keys": ["$env://TEST2_KEY"], + "certs": ["$env://TEST2_CERT"] + }, + "key": "/apisix/ssls/1" + }]] + ) + + ngx.status = code + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 5: trigger SSL match with SNI +--- init_by_lua_block + require "resty.core" + apisix = require("apisix") + core = require("apisix.core") + apisix.http_init() + + local utils = require("apisix.core.utils") + utils.dns_parse = function (domain) -- mock: DNS parser + if domain == "test1.com" then + return {address = "127.0.0.2"} + end + + error("unknown domain: " .. domain) + end +--- exec +curl -k --resolve "test.com:1994:127.0.0.1" https://test.com:1994/opentracing +--- wait: 5 +--- response_body +opentracing + + + +=== TEST 6: check sni_radixtree_match span +--- exec +sed -i '$d' ci/pod/otelcol-contrib/data-otlp.json +tail -n 13 ci/pod/otelcol-contrib/data-otlp.json +--- response_body eval +qr/.*sni_radixtree_match.*/ + + + +=== TEST 7: check resolve_dns span +--- exec +tail -n 13 ci/pod/otelcol-contrib/data-otlp.json +--- response_body eval +qr/.*resolve_dns.*/ + + + +=== TEST 8: check apisix.phase.access span +--- exec +tail ci/pod/otelcol-contrib/data-otlp.json +--- response_body eval +qr/.*apisix.phase.access.*/ + + + +=== TEST 9: check apisix.plugins.phase.header_filter span +--- exec +tail -n 12 ci/pod/otelcol-contrib/data-otlp.json +--- response_body eval +qr/.*apisix.plugins.phase.header_filter.*/ diff --git a/t/router/radixtree-sni2.t b/t/router/radixtree-sni2.t index 79d23df8e232..bbc36a733d3d 100644 --- a/t/router/radixtree-sni2.t +++ b/t/router/radixtree-sni2.t @@ -611,6 +611,9 @@ ssl handshake: true qr/(fetch|release) table \w+/ --- grep_error_log_out fetch table api_ctx +fetch table opentelemetry_span +fetch table opentelemetry_span +fetch table opentelemetry_span release table api_ctx diff --git a/t/stream-node/sanity.t b/t/stream-node/sanity.t index 556c4bc9983a..c0fc0bd5b5b2 100644 --- a/t/stream-node/sanity.t +++ b/t/stream-node/sanity.t @@ -394,6 +394,7 @@ hello world qr/(fetch|release) (ctx var|table \w+)/ --- grep_error_log_out fetch table api_ctx +fetch table opentelemetry_span fetch ctx var fetch table ctx_var fetch table plugins diff --git a/t/stream-node/tls.t b/t/stream-node/tls.t index 13bdcba0f1f5..2c0003ef5823 100644 --- a/t/stream-node/tls.t +++ b/t/stream-node/tls.t @@ -126,8 +126,12 @@ hello world qr/(fetch|release) table \w+/ --- grep_error_log_out fetch table api_ctx +fetch table opentelemetry_span +fetch table opentelemetry_span +fetch table opentelemetry_span release table api_ctx fetch table api_ctx +fetch table opentelemetry_span fetch table ctx_var fetch table plugins release table ctx_var