diff --git a/apisix/plugins/clickhouse-logger.lua b/apisix/plugins/clickhouse-logger.lua index b3e791f079b1..ada2133fe942 100644 --- a/apisix/plugins/clickhouse-logger.lua +++ b/apisix/plugins/clickhouse-logger.lua @@ -14,7 +14,6 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- - local bp_manager_mod = require("apisix.utils.batch-processor-manager") local log_util = require("apisix.utils.log-util") local plugin = require("apisix.plugin") @@ -57,7 +56,9 @@ local schema = { items = { type = "array" } - } + }, + max_req_body_bytes = {type = "integer", minimum = 1, default = 524288}, + max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288}, }, oneOf = { {required = {"endpoint_addr", "user", "password", "database", "logtable"}}, @@ -174,6 +175,11 @@ local function send_http_data(conf, log_message) end +function _M.access(conf, ctx) + log_util.check_and_read_req_body(conf, ctx) +end + + function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end diff --git a/apisix/plugins/elasticsearch-logger.lua b/apisix/plugins/elasticsearch-logger.lua index a3114e98eab2..3566da2c636d 100644 --- a/apisix/plugins/elasticsearch-logger.lua +++ b/apisix/plugins/elasticsearch-logger.lua @@ -14,7 +14,6 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- - local core = require("apisix.core") local http = require("resty.http") local log_util = require("apisix.utils.log-util") @@ -104,6 +103,8 @@ local schema = { type = "array" } }, + max_req_body_bytes = { type = "integer", minimum = 1, default = 524288 }, + max_resp_body_bytes = { type = "integer", minimum = 1, default = 524288 }, }, encrypt_fields = {"auth.password"}, oneOf = { @@ -214,6 +215,7 @@ local function get_logger_entry(conf, ctx) core.json.encode(entry) .. "\n" end + local function fetch_and_update_es_version(conf) if conf._version then return @@ -290,12 +292,16 @@ function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end -function _M.access(conf) + +function _M.access(conf, ctx) -- fetch_and_update_es_version will call ES server only the first time -- so this should not amount to considerable overhead fetch_and_update_es_version(conf) + + log_util.check_and_read_req_body(conf, ctx) end + function _M.log(conf, ctx) local metadata = plugin.plugin_metadata(plugin_name) local max_pending_entries = metadata and metadata.value and diff --git a/apisix/plugins/file-logger.lua b/apisix/plugins/file-logger.lua index 20ecd18ad1fa..f8271c4f5791 100644 --- a/apisix/plugins/file-logger.lua +++ b/apisix/plugins/file-logger.lua @@ -14,6 +14,7 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- + local log_util = require("apisix.utils.log-util") local core = require("apisix.core") local plugin = require("apisix.plugin") @@ -49,6 +50,8 @@ local schema = { type = "array" } }, + max_req_body_bytes = {type = "integer", minimum = 1, default = 524288}, + max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288}, match = { type = "array", maxItems = 20, @@ -204,10 +207,17 @@ local function write_file_data(conf, log_message) end end + +function _M.access(conf, ctx) + log_util.check_and_read_req_body(conf, ctx) +end + + function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end + function _M.log(conf, ctx) local entry = log_util.get_log_entry(plugin_name, conf, ctx) if entry == nil then diff --git a/apisix/plugins/http-logger.lua b/apisix/plugins/http-logger.lua index d3259e5db53f..47d8364fcb7d 100644 --- a/apisix/plugins/http-logger.lua +++ b/apisix/plugins/http-logger.lua @@ -14,7 +14,6 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- - local bp_manager_mod = require("apisix.utils.batch-processor-manager") local plugin = require("apisix.plugin") local log_util = require("apisix.utils.log-util") @@ -51,6 +50,8 @@ local schema = { type = "array" } }, + max_req_body_bytes = {type = "integer", minimum = 1, default = 524288}, + max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288}, concat_method = {type = "string", default = "json", enum = {"json", "new_line"}}, ssl_verify = {type = "boolean", default = false}, @@ -168,6 +169,11 @@ local function send_http_data(conf, log_message) end +function _M.access(conf, ctx) + log_util.check_and_read_req_body(conf, ctx) +end + + function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end diff --git a/apisix/plugins/kafka-logger.lua b/apisix/plugins/kafka-logger.lua index ec4f694f61ec..a8719684eff7 100644 --- a/apisix/plugins/kafka-logger.lua +++ b/apisix/plugins/kafka-logger.lua @@ -14,7 +14,6 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- -local expr = require("resty.expr.v1") local core = require("apisix.core") local log_util = require("apisix.utils.log-util") local producer = require ("resty.kafka.producer") @@ -24,7 +23,7 @@ local plugin = require("apisix.plugin") local math = math local pairs = pairs local type = type -local req_read_body = ngx.req.read_body + local plugin_name = "kafka-logger" local batch_processor_manager = bp_manager_mod.new("kafka logger") @@ -221,28 +220,7 @@ end function _M.access(conf, ctx) - if conf.include_req_body then - local should_read_body = true - if conf.include_req_body_expr then - if not conf.request_expr then - local request_expr, err = expr.new(conf.include_req_body_expr) - if not request_expr then - core.log.error('generate request expr err ', err) - return - end - conf.request_expr = request_expr - end - - local result = conf.request_expr:eval(ctx.var) - - if not result then - should_read_body = false - end - end - if should_read_body then - req_read_body() - end - end + log_util.check_and_read_req_body(conf, ctx) end diff --git a/apisix/plugins/loggly.lua b/apisix/plugins/loggly.lua index 16dc9b4a6879..2b7d9210142d 100644 --- a/apisix/plugins/loggly.lua +++ b/apisix/plugins/loggly.lua @@ -77,6 +77,8 @@ local schema = { type = "array" } }, + max_req_body_bytes = {type = "integer", minimum = 1, default = 524288}, + max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288}, tags = { type = "array", minItems = 1, @@ -177,6 +179,11 @@ function _M.check_schema(conf, schema_type) end +function _M.access(conf, ctx) + log_util.check_and_read_req_body(conf, ctx) +end + + function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end diff --git a/apisix/plugins/loki-logger.lua b/apisix/plugins/loki-logger.lua index bb7c614f17f0..6aa1eb6b36c6 100644 --- a/apisix/plugins/loki-logger.lua +++ b/apisix/plugins/loki-logger.lua @@ -14,7 +14,6 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- - local bp_manager_mod = require("apisix.utils.batch-processor-manager") local log_util = require("apisix.utils.log-util") local core = require("apisix.core") @@ -106,6 +105,8 @@ local schema = { type = "array" } }, + max_req_body_bytes = {type = "integer", minimum = 1, default = 524288}, + max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288}, }, required = {"endpoint_addrs"} } @@ -193,6 +194,11 @@ local function send_http_data(conf, log) end +function _M.access(conf, ctx) + log_util.check_and_read_req_body(conf, ctx) +end + + function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end diff --git a/apisix/plugins/rocketmq-logger.lua b/apisix/plugins/rocketmq-logger.lua index 0d3a6f22392f..ea2f88094f10 100644 --- a/apisix/plugins/rocketmq-logger.lua +++ b/apisix/plugins/rocketmq-logger.lua @@ -68,6 +68,8 @@ local schema = { type = "array" } }, + max_req_body_bytes = {type = "integer", minimum = 1, default = 524288}, + max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288}, }, encrypt_fields = {"secret_key"}, required = {"nameserver_list", "topic"} @@ -138,6 +140,11 @@ local function send_rocketmq_data(conf, log_message, prod) end +function _M.access(conf, ctx) + log_util.check_and_read_req_body(conf, ctx) +end + + function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end diff --git a/apisix/plugins/skywalking-logger.lua b/apisix/plugins/skywalking-logger.lua index c7c8dcae8b14..b54445e3dd80 100644 --- a/apisix/plugins/skywalking-logger.lua +++ b/apisix/plugins/skywalking-logger.lua @@ -14,7 +14,6 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- - local bp_manager_mod = require("apisix.utils.batch-processor-manager") local plugin = require("apisix.plugin") local log_util = require("apisix.utils.log-util") @@ -55,6 +54,8 @@ local schema = { type = "array" } }, + max_req_body_bytes = {type = "integer", minimum = 1, default = 524288}, + max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288}, }, required = {"endpoint_addr"}, } @@ -139,6 +140,11 @@ local function send_http_data(conf, log_message) end +function _M.access(conf, ctx) + log_util.check_and_read_req_body(conf, ctx) +end + + function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end diff --git a/apisix/plugins/sls-logger.lua b/apisix/plugins/sls-logger.lua index 819f0841bfb2..a84b6680c2e4 100644 --- a/apisix/plugins/sls-logger.lua +++ b/apisix/plugins/sls-logger.lua @@ -48,6 +48,8 @@ local schema = { type = "array" } }, + max_req_body_bytes = {type = "integer", minimum = 1, default = 524288}, + max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288}, timeout = {type = "integer", minimum = 1, default= 5000}, log_format = {type = "object"}, host = {type = "string"}, @@ -158,6 +160,11 @@ local function handle_log(entries) end +function _M.access(conf, ctx) + log_util.check_and_read_req_body(conf, ctx) +end + + function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end diff --git a/apisix/plugins/syslog.lua b/apisix/plugins/syslog.lua index 1f35395305f4..9420d5342d0f 100644 --- a/apisix/plugins/syslog.lua +++ b/apisix/plugins/syslog.lua @@ -14,7 +14,6 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- - local core = require("apisix.core") local log_util = require("apisix.utils.log-util") local bp_manager_mod = require("apisix.utils.batch-processor-manager") @@ -50,6 +49,8 @@ local schema = { type = "array" } }, + max_req_body_bytes = {type = "integer", minimum = 1, default = 524288}, + max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288}, }, required = {"host", "port"} } @@ -85,6 +86,11 @@ function _M.check_schema(conf, schema_type) end +function _M.access(conf, ctx) + log_util.check_and_read_req_body(conf, ctx) +end + + function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end diff --git a/apisix/plugins/tcp-logger.lua b/apisix/plugins/tcp-logger.lua index 9b3df668ea64..ef78371d3c55 100644 --- a/apisix/plugins/tcp-logger.lua +++ b/apisix/plugins/tcp-logger.lua @@ -50,6 +50,8 @@ local schema = { type = "array" } }, + max_req_body_bytes = {type = "integer", minimum = 1, default = 524288}, + max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288}, }, required = {"host", "port"} } @@ -132,6 +134,11 @@ local function send_tcp_data(conf, log_message) end +function _M.access(conf, ctx) + log_util.check_and_read_req_body(conf, ctx) +end + + function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end diff --git a/apisix/plugins/tencent-cloud-cls.lua b/apisix/plugins/tencent-cloud-cls.lua index f8dac160ed87..8fdc5bb769d5 100644 --- a/apisix/plugins/tencent-cloud-cls.lua +++ b/apisix/plugins/tencent-cloud-cls.lua @@ -14,7 +14,6 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- - local core = require("apisix.core") local plugin = require("apisix.plugin") local log_util = require("apisix.utils.log-util") @@ -23,7 +22,6 @@ local cls_sdk = require("apisix.plugins.tencent-cloud-cls.cls-sdk") local math = math local pairs = pairs - local plugin_name = "tencent-cloud-cls" local batch_processor_manager = bp_manager_mod.new(plugin_name) local schema = { @@ -55,6 +53,8 @@ local schema = { type = "array" } }, + max_req_body_bytes = { type = "integer", minimum = 1, default = 524288 }, + max_resp_body_bytes = { type = "integer", minimum = 1, default = 524288 }, global_tag = { type = "object" }, log_format = {type = "object"}, }, @@ -105,8 +105,11 @@ function _M.access(conf, ctx) if conf.sample_ratio == 1 or math.random() < conf.sample_ratio then core.log.debug("cls sampled") ctx.cls_sample = true + else return end + + log_util.check_and_read_req_body(conf, ctx) end diff --git a/apisix/plugins/udp-logger.lua b/apisix/plugins/udp-logger.lua index a2333b0b5b2e..2bd70e2bbaa3 100644 --- a/apisix/plugins/udp-logger.lua +++ b/apisix/plugins/udp-logger.lua @@ -48,6 +48,8 @@ local schema = { type = "array" } }, + max_req_body_bytes = {type = "integer", minimum = 1, default = 524288}, + max_resp_body_bytes = {type = "integer", minimum = 1, default = 524288}, }, required = {"host", "port"} } @@ -117,6 +119,11 @@ local function send_udp_data(conf, log_message) end +function _M.access(conf, ctx) + log_util.check_and_read_req_body(conf, ctx) +end + + function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end diff --git a/apisix/utils/log-util.lua b/apisix/utils/log-util.lua index 828038b78f15..f1fea458db6c 100644 --- a/apisix/utils/log-util.lua +++ b/apisix/utils/log-util.lua @@ -35,6 +35,7 @@ local MAX_REQ_BODY = 524288 -- 512 KiB local MAX_RESP_BODY = 524288 -- 512 KiB local MAX_LOG_FORMAT_DEPTH = 5 local io = io +local req_read_body = ngx.req.read_body local lru_log_format = core.lrucache.new({ ttl = 300, count = 512 @@ -433,4 +434,30 @@ function _M.get_rfc3339_zulu_timestamp(timestamp) end +function _M.check_and_read_req_body(conf, ctx) + if conf.include_req_body then + local should_read_body = true + if conf.include_req_body_expr then + if not conf.request_expr then + local request_expr, err = expr.new(conf.include_req_body_expr) + if not request_expr then + core.log.error('generate request expr err ', err) + return + end + conf.request_expr = request_expr + end + + local result = conf.request_expr:eval(ctx.var) + + if not result then + should_read_body = false + end + end + if should_read_body then + req_read_body() + end + end +end + + return _M diff --git a/docs/en/latest/plugins/clickhouse-logger.md b/docs/en/latest/plugins/clickhouse-logger.md index 64d8caa79e1d..f8cf4cc09d41 100644 --- a/docs/en/latest/plugins/clickhouse-logger.md +++ b/docs/en/latest/plugins/clickhouse-logger.md @@ -47,8 +47,10 @@ The `clickhouse-logger` Plugin is used to push logs to [ClickHouse](https://clic | log_format | object | False | | | Log format declared as key-value pairs in JSON. Values support strings and nested objects (up to five levels deep; deeper fields are truncated). Within strings, [APISIX](../apisix-variable.md) or [NGINX](http://nginx.org/en/docs/varindex.html) variables can be referenced by prefixing with `$`. | | include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. | | include_req_body_expr | array | False | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_req_body_bytes | integer | False | 524288 | >=1 | Request bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | include_resp_body | boolean | False | false | [false, true] | When set to `true` includes the response body in the log. | | include_resp_body_expr | array | False | | | Filter for when the `include_resp_body` attribute is set to `true`. Response body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_resp_body_bytes | integer | False | 524288 | >=1 | Response bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | NOTE: `encrypt_fields = {"password"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields). diff --git a/docs/en/latest/plugins/elasticsearch-logger.md b/docs/en/latest/plugins/elasticsearch-logger.md index ff75377830bc..d555cd122205 100644 --- a/docs/en/latest/plugins/elasticsearch-logger.md +++ b/docs/en/latest/plugins/elasticsearch-logger.md @@ -51,8 +51,10 @@ The `elasticsearch-logger` Plugin pushes request and response logs in batches to | timeout | integer | False | 10 | Elasticsearch send data timeout in seconds. | | include_req_body | boolean | False | false | If true, include the request body in the log. Note that if the request body is too big to be kept in the memory, it can not be logged due to NGINX's limitations. | | include_req_body_expr | array[array] | False | | An array of one or more conditions in the form of [lua-resty-expr](https://github.com/api7/lua-resty-expr). Used when the `include_req_body` is true. Request body would only be logged when the expressions configured here evaluate to true. | +| max_req_body_bytes | integer | False | 524288 | Request bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | include_resp_body | boolean | False | false | If true, include the response body in the log. | | include_resp_body_expr | array[array] | False | | An array of one or more conditions in the form of [lua-resty-expr](https://github.com/api7/lua-resty-expr). Used when the `include_resp_body` is true. Response body would only be logged when the expressions configured here evaluate to true. | +| max_resp_body_bytes | integer | False | 524288 | Response bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | NOTE: `encrypt_fields = {"auth.password"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields). diff --git a/docs/en/latest/plugins/file-logger.md b/docs/en/latest/plugins/file-logger.md index 45d498602ecf..183ee55b6268 100644 --- a/docs/en/latest/plugins/file-logger.md +++ b/docs/en/latest/plugins/file-logger.md @@ -49,8 +49,10 @@ The `file-logger` Plugin is used to push log streams to a specific location. | log_format | object | False | Log format declared as key-value pairs in JSON. Values support strings and nested objects (up to five levels deep; deeper fields are truncated). Within strings, [APISIX](../apisix-variable.md) or [NGINX](http://nginx.org/en/docs/varindex.html) variables can be referenced by prefixing with `$`. | | include_req_body | boolean | False | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. | | include_req_body_expr | array | False | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_req_body_bytes | integer | False | Request bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | include_resp_body | boolean | False | When set to `true` includes the response body in the log file. | | include_resp_body_expr | array | False | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response into file if the expression evaluates to `true`. | +| max_resp_body_bytes | integer | False | Response bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | match | array[array] | False | Logs will be recorded when the rule matching is successful if the option is set. See [lua-resty-expr](https://github.com/api7/lua-resty-expr#operator-list) for a list of available expressions. | ### Example of default log format diff --git a/docs/en/latest/plugins/http-logger.md b/docs/en/latest/plugins/http-logger.md index 0a8dd7dae4e0..5f93302bdf8b 100644 --- a/docs/en/latest/plugins/http-logger.md +++ b/docs/en/latest/plugins/http-logger.md @@ -43,8 +43,10 @@ This will allow the ability to send log data requests as JSON objects to monitor | log_format | object | False | | | Log format declared as key-value pairs in JSON. Values support strings and nested objects (up to five levels deep; deeper fields are truncated). Within strings, [APISIX](../apisix-variable.md) or [NGINX](http://nginx.org/en/docs/varindex.html) variables can be referenced by prefixing with `$`. | | include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. | | include_req_body_expr | array | False | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_req_body_bytes | integer | False | 524288 | >=1 | Request bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | include_resp_body | boolean | False | false | [false, true] | When set to `true` includes the response body in the log. | | include_resp_body_expr | array | False | | | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`. | +| max_resp_body_bytes | integer | False | 524288 | >=1 | Response bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | concat_method | string | False | "json" | ["json", "new_line"] | Sets how to concatenate logs. When set to `json`, uses `json.encode` for all pending logs and when set to `new_line`, also uses `json.encode` but uses the newline (`\n`) to concatenate lines. | | ssl_verify | boolean | False | false | [false, true] | When set to `true` verifies the SSL certificate. | diff --git a/docs/en/latest/plugins/loggly.md b/docs/en/latest/plugins/loggly.md index 3899dca37c85..4d6ccb737afc 100644 --- a/docs/en/latest/plugins/loggly.md +++ b/docs/en/latest/plugins/loggly.md @@ -46,8 +46,10 @@ When the maximum batch size is exceeded, the data in the queue is pushed to Logg | log_format | object | False | {"host": "$host", "@timestamp": "$time_iso8601", "client_ip": "$remote_addr"} | Log format declared as key-value pairs in JSON. Values support strings and nested objects (up to five levels deep; deeper fields are truncated). Within strings, [APISIX](../apisix-variable.md) or [NGINX](http://nginx.org/en/docs/varindex.html) variables can be referenced by prefixing with `$`. | | include_req_body | boolean | False | false | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. | | include_req_body_expr | array | False | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_req_body_bytes | integer | False | 524288 | Request bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | include_resp_body | boolean | False | false | When set to `true` includes the response body in the log. | | include_resp_body_expr | array | False | | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`. | +| max_resp_body_bytes | integer | False | 524288 | Response bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration. diff --git a/docs/en/latest/plugins/loki-logger.md b/docs/en/latest/plugins/loki-logger.md index cb35f9c915f0..0f809ba0ecea 100644 --- a/docs/en/latest/plugins/loki-logger.md +++ b/docs/en/latest/plugins/loki-logger.md @@ -56,8 +56,10 @@ When enabled, the Plugin will serialize the request context information to [JSON | name | string | False | loki-logger | | Unique identifier of the Plugin for the batch processor. If you use [Prometheus](./prometheus.md) to monitor APISIX metrics, the name is exported in `apisix_batch_process_entries`. | | include_req_body | boolean | False | false | | If true, include the request body in the log. Note that if the request body is too big to be kept in the memory, it can not be logged due to NGINX's limitations. | | include_req_body_expr | array[array] | False | | | An array of one or more conditions in the form of [lua-resty-expr](https://github.com/api7/lua-resty-expr). Used when the `include_req_body` is true. Request body would only be logged when the expressions configured here evaluate to true. | +| max_req_body_bytes | integer | False | 524288 | >=1 | Request bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | include_resp_body | boolean | False | false | | If true, include the response body in the log. | | include_resp_body_expr | array[array] | False | | | An array of one or more conditions in the form of [lua-resty-expr](https://github.com/api7/lua-resty-expr). Used when the `include_resp_body` is true. Response body would only be logged when the expressions configured here evaluate to true. | +| max_resp_body_bytes | integer | False | 524288 | >=1 | Response bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration. diff --git a/docs/en/latest/plugins/rocketmq-logger.md b/docs/en/latest/plugins/rocketmq-logger.md index 84af0040ebbd..7da26a780a64 100644 --- a/docs/en/latest/plugins/rocketmq-logger.md +++ b/docs/en/latest/plugins/rocketmq-logger.md @@ -49,8 +49,10 @@ It might take some time to receive the log data. It will be automatically sent a | meta_format | enum | False | "default" | ["default","origin"] | Format to collect the request information. Setting to `default` collects the information in JSON format and `origin` collects the information with the original HTTP request. See [examples](#meta_format-example) below. | | include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. | | include_req_body_expr | array | False | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_req_body_bytes | integer | False | 524288 | >=1 | Request bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | include_resp_body | boolean | False | false | [false, true] | When set to `true` includes the response body in the log. | | include_resp_body_expr | array | False | | | Filter for when the `include_resp_body` attribute is set to `true`. Response body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_resp_body_bytes | integer | False | 524288 | >=1 | Response bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | NOTE: `encrypt_fields = {"secret_key"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields). diff --git a/docs/en/latest/plugins/skywalking-logger.md b/docs/en/latest/plugins/skywalking-logger.md index 9b2662bc2ac0..f1e6aad543b1 100644 --- a/docs/en/latest/plugins/skywalking-logger.md +++ b/docs/en/latest/plugins/skywalking-logger.md @@ -50,8 +50,10 @@ If there is an existing tracing context, it sets up the trace-log correlation au | name | string | False | "skywalking logger" | | Unique identifier to identify the logger. If you use Prometheus to monitor APISIX metrics, the name is exported in `apisix_batch_process_entries`. | | include_req_body | boolean | False | false | If true, include the request body in the log. Note that if the request body is too big to be kept in the memory, it can not be logged due to NGINX's limitations. | | include_req_body_expr | array[array] | False | | An array of one or more conditions in the form of [lua-resty-expr](https://github.com/api7/lua-resty-expr). Used when the `include_req_body` is true. Request body would only be logged when the expressions configured here evaluate to true. | +| max_req_body_bytes | integer | False | 524288 | >=1 | Request bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | include_resp_body | boolean | False | false | If true, include the response body in the log. | | include_resp_body_expr | array[array] | False | | An array of one or more conditions in the form of [lua-resty-expr](https://github.com/api7/lua-resty-expr). Used when the `include_resp_body` is true. Response body would only be logged when the expressions configured here evaluate to true. | +| max_resp_body_bytes | integer | False | 524288 | >=1 | Response bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration. diff --git a/docs/en/latest/plugins/sls-logger.md b/docs/en/latest/plugins/sls-logger.md index d2c5b51f98d9..ab49ca329151 100644 --- a/docs/en/latest/plugins/sls-logger.md +++ b/docs/en/latest/plugins/sls-logger.md @@ -47,8 +47,10 @@ It might take some time to receive the log data. It will be automatically sent a | access_key_secret | True | AccessKey Secret in Alibaba Cloud. See [Authorization](https://www.alibabacloud.com/help/en/log-service/latest/create-a-ram-user-and-authorize-the-ram-user-to-access-log-service) for more details. | | include_req_body | True | When set to `true`, includes the request body in the log. | | include_req_body_expr | No | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_req_body_bytes | False | Request bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | include_resp_body | No | When set to `true` includes the response body in the log. | | include_resp_body_expr | No | Filter for when the `include_resp_body` attribute is set to `true`. Response body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_resp_body_bytes | False | Response bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | name | False | Unique identifier for the batch processor. If you use Prometheus to monitor APISIX metrics, the name is exported in `apisix_batch_process_entries`. | NOTE: `encrypt_fields = {"access_key_secret"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields). diff --git a/docs/en/latest/plugins/syslog.md b/docs/en/latest/plugins/syslog.md index 7acb7f3b1a31..9966dac76ece 100644 --- a/docs/en/latest/plugins/syslog.md +++ b/docs/en/latest/plugins/syslog.md @@ -48,8 +48,10 @@ Logs can be set as JSON objects. | log_format | object | False | | | Log format declared as key-value pairs in JSON. Values support strings and nested objects (up to five levels deep; deeper fields are truncated). Within strings, [APISIX](../apisix-variable.md) or [NGINX](http://nginx.org/en/docs/varindex.html) variables can be referenced by prefixing with `$`. | | include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. | | include_req_body_expr | array | False | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_req_body_bytes | integer | False | 524288 | >=1 | Request bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | include_resp_body | boolean | False | false | [false, true] | When set to `true` includes the response body in the log. | | include_resp_body_expr | array | False | | | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`. | +| max_resp_body_bytes | integer | False | 524288 | >=1 | Response bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration. diff --git a/docs/en/latest/plugins/tcp-logger.md b/docs/en/latest/plugins/tcp-logger.md index cd2c5c59fd1c..582610994a88 100644 --- a/docs/en/latest/plugins/tcp-logger.md +++ b/docs/en/latest/plugins/tcp-logger.md @@ -48,8 +48,10 @@ This plugin also allows to push logs as a batch to your external TCP server. It | tls_options | string | False | | | TLS options. | | include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. | | include_req_body_expr | array | No | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_req_body_bytes | integer | False | 524288 | >=1 | Request bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | include_resp_body | boolean | No | false | [false, true] | When set to `true` includes the response body in the log. | | include_resp_body_expr | array | No | | | Filter for when the `include_resp_body` attribute is set to `true`. Response body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_resp_body_bytes | integer | False | 524288 | >=1 | Response bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration. diff --git a/docs/en/latest/plugins/tencent-cloud-cls.md b/docs/en/latest/plugins/tencent-cloud-cls.md index e9d633ef19d7..a66266ad26ae 100644 --- a/docs/en/latest/plugins/tencent-cloud-cls.md +++ b/docs/en/latest/plugins/tencent-cloud-cls.md @@ -43,8 +43,10 @@ The `tencent-cloud-cls` Plugin uses [TencentCloud CLS](https://cloud.tencent.com | sample_ratio | number | No | 1 | [0.00001, 1] | How often to sample the requests. Setting to `1` will sample all requests. | | include_req_body | boolean | No | false | [false, true] | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to NGINX's limitations. | | include_req_body_expr | array | No | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_req_body_bytes | integer | False | 524288 | >=1 | Request bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | include_resp_body | boolean | No | false | [false, true] | When set to `true` includes the response body in the log. | | include_resp_body_expr | array | No | | | Filter for when the `include_resp_body` attribute is set to `true`. Response body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_resp_body_bytes | integer | False | 524288 | >=1 | Response bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | global_tag | object | No | | | kv pairs in JSON,send with each log. | | log_format | object | No | | | Log format declared as key-value pairs in JSON. Values support strings and nested objects (up to five levels deep; deeper fields are truncated). Within strings, [APISIX](../apisix-variable.md) or [NGINX](http://nginx.org/en/docs/varindex.html) variables can be referenced by prefixing with `$`. | diff --git a/docs/en/latest/plugins/udp-logger.md b/docs/en/latest/plugins/udp-logger.md index 6d1511bb545b..1d81c11f784f 100644 --- a/docs/en/latest/plugins/udp-logger.md +++ b/docs/en/latest/plugins/udp-logger.md @@ -46,8 +46,10 @@ This plugin also allows to push logs as a batch to your external UDP server. It | name | string | False | "udp logger" | | Unique identifier for the batch processor. If you use Prometheus to monitor APISIX metrics, the name is exported in `apisix_batch_process_entries`. processor. | | include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. | | include_req_body_expr | array | No | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_req_body_bytes | integer | False | 524288 | >=1 | Request bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | | include_resp_body | boolean | No | false | [false, true] | When set to `true` includes the response body in the log. | | include_resp_body_expr | array | No | | | Filter for when the `include_resp_body` attribute is set to `true`. Response body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| max_resp_body_bytes | integer | False | 524288 | >=1 | Response bodies within this size will be logged, if the size exceeds the configured value it will be truncated before logging. | This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration. diff --git a/t/plugin/http-logger-large-body.t b/t/plugin/http-logger-large-body.t new file mode 100644 index 000000000000..626d9d18ede7 --- /dev/null +++ b/t/plugin/http-logger-large-body.t @@ -0,0 +1,919 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } + + my $http_config = $block->http_config // <<_EOC_; + # fake server, only for test + server { + listen 1970; + location /large_resp { + content_by_lua_block { + local large_body = { + "h", "e", "l", "l", "o" + } + + local size_in_bytes = 1024 * 1024 -- 1mb + for i = 1, size_in_bytes do + large_body[i+5] = "l" + end + large_body = table.concat(large_body, "") + + ngx.say(large_body) + } + } + } +_EOC_ + + $block->set_value("http_config", $http_config); +}); + +run_tests; + +__DATA__ + +=== TEST 1: max_body_bytes is not an integer +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.http-logger") + local ok, err = plugin.check_schema({ + uri = "http://127.0.0.1:1980/hello", + timeout = 1, + batch_max_size = 1, + max_req_body_bytes = "10", + include_req_body = true + }) + if not ok then + ngx.say(err) + end + ngx.say("done") + } + } +--- response_body +property "max_req_body_bytes" validation failed: wrong type: expected integer, got string +done + + + +=== TEST 2: max_resp_body_bytes is not an integer +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.http-logger") + local ok, err = plugin.check_schema({ + uri = "http://127.0.0.1:1980/hello", + timeout = 1, + batch_max_size = 1, + max_resp_body_bytes = "10", + include_resp_body = true + }) + if not ok then + ngx.say(err) + end + ngx.say("done") + } + } +--- response_body +property "max_resp_body_bytes" validation failed: wrong type: expected integer, got string +done + + + +=== TEST 3: set route(include_req_body = true, concat_method = json) +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1982/hello", + "timeout": 1, + "batch_max_size": 1, + "max_req_body_bytes": 5, + "include_req_body": true, + "concat_method": "json" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 4: hit route(include_req_body = true, concat_method = json) +--- request +POST /hello?ab=cd +abcdef +--- response_body +hello world +--- error_log_like eval +qr/"body":"abcde"/ +--- wait: 2 + + + +=== TEST 5: set route(include_resp_body = true, concat_method = json) +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1982/log", + "timeout": 1, + "max_resp_body_bytes": 5, + "include_resp_body": true, + "batch_max_size": 1, + "concat_method": "json" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 6: hit route(include_resp_body = true, concat_method = json) +--- request +POST /hello?name=qwerty +abcdef +--- response_body +hello world +--- error_log eval +qr/request log:.*"response":\{"body":"hello"/ +--- wait: 2 + + + +=== TEST 7: set route(include_resp_body = true, include_req_body = true, concat_method = json) +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1982/log", + "timeout": 1, + "include_req_body": true, + "max_req_body_bytes": 5, + "include_resp_body": true, + "max_resp_body_bytes": 5, + "batch_max_size": 1, + "concat_method": "json" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 8: hit route(include_resp_body = true, include_req_body = true, concat_method = json) +--- request +POST /hello?name=qwerty +abcdef +--- response_body +hello world +--- error_log eval +qr/request log:.*"response":\{"body":"hello"/ +--- error_log_like +qr/"body":"abcde"/ +--- wait: 2 + + + +=== TEST 9: set route(include_resp_body = false, include_req_body = false) +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1982/log", + "timeout": 1, + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 10: hit route(include_resp_body = false, include_req_body = false) +--- request +POST /hello?name=qwerty +abcdef +--- response_body +hello world +--- no_error_log eval +qr/request log:.*"response":\{"body":.*/ +--- wait: 2 + + + +=== TEST 11: set route(large_body, include_resp_body = true, include_req_body = true) +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1982/log", + "timeout": 1, + "include_req_body": true, + "max_req_body_bytes": 256, + "include_resp_body": true, + "max_resp_body_bytes": 256, + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/echo" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 12: hit route(large_body, include_resp_body = true, include_req_body = true) +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin") + local http = require("resty.http") + + local large_body = { + "h", "e", "l", "l", "o" + } + + local size_in_bytes = 10 * 1024 -- 10kb + for i = 1, size_in_bytes do + large_body[i+5] = "l" + end + large_body = table.concat(large_body, "") + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/echo" + + local httpc = http.new() + local res, err = httpc:request_uri(uri, + { + method = "POST", + body = large_body, + } + ) + ngx.say(res.body) + } + } +--- request +GET /t +--- error_log eval +qr/request log:.*"response":\{"body":"hello(l{251})"/ +--- response_body eval +qr/hello.*/ + + + +=== TEST 13: set route(large_body, include_resp_body = true) +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1982/log", + "timeout": 1, + "include_resp_body": true, + "max_resp_body_bytes": 256, + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/echo" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 14: hit route(large_body, include_resp_body = true) +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin") + local http = require("resty.http") + + local large_body = { + "h", "e", "l", "l", "o" + } + + local size_in_bytes = 10 * 1024 -- 10kb + for i = 1, size_in_bytes do + large_body[i+5] = "l" + end + large_body = table.concat(large_body, "") + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/echo" + + local httpc = http.new() + local res, err = httpc:request_uri(uri, + { + method = "POST", + body = large_body, + } + ) + ngx.say(res.body) + } + } +--- request +GET /t +--- error_log eval +qr/request log:.*"response":\{"body":"hello(l{251})"/ +--- response_body eval +qr/hello.*/ + + + +=== TEST 15: set route(large_body, include_req_body = true) +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1982/log", + "timeout": 1, + "include_req_body": true, + "max_req_body_bytes": 256, + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/echo" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 16: hit route(large_body, include_req_body = true) +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin") + local http = require("resty.http") + + local large_body = { + "h", "e", "l", "l", "o" + } + + local size_in_bytes = 10 * 1024 -- 10kb + for i = 1, size_in_bytes do + large_body[i+5] = "l" + end + large_body = table.concat(large_body, "") + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/echo" + + local httpc = http.new() + local res, err = httpc:request_uri(uri, + { + method = "POST", + body = large_body, + } + ) + ngx.say(res.body) + } + } +--- request +GET /t +--- error_log eval +qr/request log:.*"body":"hello(l{251})"/ +--- response_body eval +qr/hello.*/ + + + +=== TEST 17: set route(large_body, include_resp_body = true) +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1982/log", + "timeout": 1, + "include_resp_body": true, + "max_resp_body_bytes": 256, + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1970": 1 + }, + "type": "roundrobin" + }, + "uri": "/large_resp" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 18: truncate upstream response body 1m to 256 bytes +--- request +GET /large_resp +--- error_log eval +qr/request log:.*"response":\{"body":"hello(l{251})"/ +--- response_body eval +qr/hello.*/ + + + +=== TEST 19: set route(large_body, include_req_body = true) +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1982/log", + "timeout": 1, + "include_req_body": true, + "max_req_body_bytes": 256, + "batch_max_size": 1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 20: truncate upstream request body 100kb to 256 bytes +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin") + local http = require("resty.http") + + local large_body = { + "h", "e", "l", "l", "o" + } + + local size_in_bytes = 100 * 1024 -- 100kb + for i = 1, size_in_bytes do + large_body[i+5] = "l" + end + large_body = table.concat(large_body, "") + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + + local httpc = http.new() + local res, err = httpc:request_uri(uri, + { + method = "POST", + body = large_body, + } + ) + + if err then + ngx.say(err) + end + + ngx.say(res.body) + } + } +--- request +GET /t +--- response_body_like +hello world +--- error_log eval +qr/request log:.*"body":"hello(l{251})"/ + + + +=== TEST 21: set route(include_req_body = true) +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1982/log", + "timeout": 1, + "batch_max_size": 1, + "max_req_body_bytes": 5, + "include_req_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 22: empty request body +--- request +GET /hello?ab=cd +--- response_body +hello world +--- no_error_log eval +qr/"body":/ +--- wait: 2 + + + +=== TEST 23: add plugin metadata +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/plugin_metadata/http-logger', + ngx.HTTP_PUT, + [[{ + "log_format": { + "request_body": "$request_body" + } + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 24: set route with plugin metadata +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1982/log", + "timeout": 1, + "batch_max_size": 1, + "max_req_body_bytes": 5 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 25: hit route with custom log_format +--- request +POST /hello?ab=cd +abcdef +--- response_body +hello world +--- error_log_like eval +qr/"request_body": "abcde"/ +--- wait: 2 + + + +=== TEST 26: set route(include_req_body = true, concat_method = new_line) +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1982/log", + "timeout": 1, + "batch_max_size": 2, + "max_req_body_bytes": 5, + "include_req_body": true, + "concat_method": "new_line" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 27: hit route(concat_method = new_line, batch_max_size = 2) +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + + for i = 1, 2 do + local httpc = http.new() + local res, err = httpc:request_uri(uri, + { + method = "POST", + body = "test_body" .. i, + } + ) + if err then + ngx.say(err) + end + end + ngx.say("done") + } + } +--- request +GET /t +--- response_body +done +--- error_log_like eval +qr/request log:.*"body":"test_"\}\\n.*"body":"test_"/ +--- wait: 2 + + + +=== TEST 28: set route(include_resp_body = true, concat_method = new_line) +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1982/log", + "timeout": 1, + "batch_max_size": 2, + "max_resp_body_bytes": 6, + "include_resp_body": true, + "concat_method": "new_line" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 29: hit route(concat_method = new_line, include_resp_body = true) +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + + for i = 1, 2 do + local httpc = http.new() + local res, err = httpc:request_uri(uri, {method = "GET"}) + if err then + ngx.say(err) + end + end + ngx.say("done") + } + } +--- request +GET /t +--- response_body +done +--- error_log_like eval +qr/request log:.*"body":"hello \\n"\}\\n.*"body":"hello \\n"/ +--- wait: 2