diff --git a/docker/README.md b/docker/README.md index 2727dad..9858ef3 100644 --- a/docker/README.md +++ b/docker/README.md @@ -48,3 +48,21 @@ Enable linger by `loginctl enable-linger ` for your user. Start the service with `systemctl --user start servo_ci`. You will find logs in `journalctl`. You can enable it to start at boot by running `systemctl --user enable servo_ci`. +### Setup email notifications on monitor failure + +In the rare case that the runner fails and intervention is required, it can be very useful to receive +email notifications. The guide below uses `ssmtp` so that we can send authenticated emails. Using just +`mail` can work to, but increases the risk of mails being filtered out by a spam filter, depending on +the whole setup. + +- Install `ssmtp`: `apt-get install ssmtp mailutils` +- Configure `/etc/ssmtp/ssmtp.conf` so that it can send emails via the CI email account. +- Copy `docker_jit_monitor/etc/servo_ci_runner_send_mail.sh` to `$CI_USER/.local/bin/servo_ci_runner_send_mail.sh` +- Copy `docker_jit_monitor/etc/notify-email@.service` to `~/.config/systemd/user` +- Test the email configuration by running `systemctl start --user notify-email@servo_ci.service` +- Edit the `Unit` section of `servo_ci.service to contain: + ``` + [Unit] + Description=Run the servo_ci runner + OnFailure=notify-email@%i.service + ``` diff --git a/docker/docker_jit_monitor/etc/notify-email@.service b/docker/docker_jit_monitor/etc/notify-email@.service new file mode 100644 index 0000000..aeaae76 --- /dev/null +++ b/docker/docker_jit_monitor/etc/notify-email@.service @@ -0,0 +1,7 @@ +[Unit] +Description=Send email + +[Service] +Type=oneshot +ExecStart=/usr/bin/bash -c '/home/servo_ci/.local/bin/servo_ci_runner_send_mail.sh "%i"' + diff --git a/docker/docker_jit_monitor/etc/servo_ci_runner_send_mail.sh b/docker/docker_jit_monitor/etc/servo_ci_runner_send_mail.sh new file mode 100644 index 0000000..32658f6 --- /dev/null +++ b/docker/docker_jit_monitor/etc/servo_ci_runner_send_mail.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# A small script to send an email when a service fails. +# Usage: servo_ci_runner_send_mail.sh +# This script is intended to be used with systemd's OnFailure directive. +# Before using this script, ensure that you have configured your email settings correctly in /etc/ssmtp/ssmtp.conf. +# Make sure to replace XXXinsert your email address hereXXX with your actual email address in the script. +# You may setup multiple receivers. + +set -eu +service_name="$1" +host_name=$(hostname) +{ + echo "To: XXXinsert your email address hereXXX" + echo "From: DRC servo CI " + echo "Subject: Service ${service_name} failed on Host ${host_name}" + echo 'Content-Type: text/plain; charset="UTF-8"' + echo + echo "Service ${service_name} failed on Host ${host_name}!" + echo "Systemctl status:" + echo "\`\`\`" + echo "$(systemctl --user status ${service_name})" + echo "\`\`\`" + echo "Service logs of last 5 minutes:" + echo "\`\`\`" + echo "$(journalctl --user -u servo_ci --since '5 minutes ago')" + echo "\`\`\`" + echo "Disk usage:" + echo "\`\`\`" + echo "$(df -h)" + echo "\`\`\`" +} | tee /tmp/servo_ci_mail_dump.txt | /usr/bin/env ssmtp XXXinsert your email address hereXXX