From 0235752a14732a36a023df4d3022aea388eb7968 Mon Sep 17 00:00:00 2001 From: Stefan Schueffler Date: Tue, 2 Jul 2024 19:36:45 +0200 Subject: [PATCH] Add support for configurable dequeue size When sending messages to remote MTAs, the messages get dequeued in batches from the local queue. As the batch-key is the given remote MX server, those messages will be delivered to this remote MTA in one SMTP session. Although this is good for performance (to reuse the same SMTP session for many mails), many of the real-world MTAs do not like sending too much mails at once in one single session. Example error messages are similar to "421 too many messages in this connection" Therefore, we make the limit adjustable (with the default value of 100 to be backwards compatible). From our experiences with the last 5 million emails sent, having a batch size of 10 works almost ever, and 50 seems to be the upper "real world" limit before hitting those rate limits by the remote MTAs. --- app/lib/message_dequeuer/initial_processor.rb | 2 +- doc/config/yaml.yml | 2 ++ lib/postal/config_schema.rb | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/lib/message_dequeuer/initial_processor.rb b/app/lib/message_dequeuer/initial_processor.rb index 37ad9aa49..a49495b69 100644 --- a/app/lib/message_dequeuer/initial_processor.rb +++ b/app/lib/message_dequeuer/initial_processor.rb @@ -57,7 +57,7 @@ def check_message_is_ready def find_other_messages_for_batch return unless Postal::Config.postal.batch_queued_messages? - @other_messages = @queued_message.batchable_messages(100) + @other_messages = @queued_message.batchable_messages(Postal::Config.postal.batch_queued_messages_limit) log "found #{@other_messages.size} associated messages to process at the same time", batch_key: @queued_message.batch_key rescue StandardError @queued_message.unlock diff --git a/doc/config/yaml.yml b/doc/config/yaml.yml index 1035ec99f..cbf83d968 100644 --- a/doc/config/yaml.yml +++ b/doc/config/yaml.yml @@ -33,6 +33,8 @@ postal: queued_message_lock_stale_days: 1 # When enabled queued messages will be de-queued in batches based on their destination batch_queued_messages: true + # When de-queuing in batches, use this limit for the batch size + batch_queued_messages_limit: 100 web_server: # The default port the web server should listen on unless overriden by the PORT environment variable diff --git a/lib/postal/config_schema.rb b/lib/postal/config_schema.rb index af3c7330e..8553aafdf 100644 --- a/lib/postal/config_schema.rb +++ b/lib/postal/config_schema.rb @@ -101,6 +101,11 @@ module Postal description "When enabled queued messages will be de-queued in batches based on their destination" default true end + + integer :batch_queued_messages_limit do + description "When de-queuing in batches, use this limit for the batch size" + default 100 + end end group :web_server do