From bd73182111a6f0c4a457fbe97e709dceacd02a62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82a=C5=BCej=20Sowa?= Date: Thu, 9 Oct 2025 16:57:41 +0200 Subject: [PATCH 01/33] Reorganize files --- CMakeLists.txt | 15 +- include/web_video_server/image_streamer.hpp | 36 --- .../{ => streamers}/h264_streamer.hpp | 3 +- .../streamers/image_transport_streamer.hpp | 74 ++++++ .../{ => streamers}/jpeg_streamers.hpp | 3 +- .../{ => streamers}/libav_streamer.hpp | 7 +- .../{ => streamers}/png_streamers.hpp | 7 +- .../ros_compressed_streamer.hpp | 7 +- .../{ => streamers}/vp8_streamer.hpp | 7 +- .../{ => streamers}/vp9_streamer.hpp | 7 +- include/web_video_server/web_video_server.hpp | 7 +- src/image_streamer.cpp | 182 --------------- src/multipart_stream.cpp | 1 + src/{ => streamers}/h264_streamer.cpp | 2 +- src/streamers/image_transport_streamer.cpp | 218 ++++++++++++++++++ src/{ => streamers}/jpeg_streamers.cpp | 3 +- src/{ => streamers}/libav_streamer.cpp | 3 +- src/{ => streamers}/png_streamers.cpp | 3 +- .../ros_compressed_streamer.cpp | 2 +- src/{ => streamers}/vp8_streamer.cpp | 2 +- src/{ => streamers}/vp9_streamer.cpp | 2 +- src/web_video_server.cpp | 17 +- 22 files changed, 348 insertions(+), 260 deletions(-) rename include/web_video_server/{ => streamers}/h264_streamer.hpp (97%) create mode 100644 include/web_video_server/streamers/image_transport_streamer.hpp rename include/web_video_server/{ => streamers}/jpeg_streamers.hpp (97%) rename include/web_video_server/{ => streamers}/libav_streamer.hpp (98%) rename include/web_video_server/{ => streamers}/png_streamers.hpp (97%) rename include/web_video_server/{ => streamers}/ros_compressed_streamer.hpp (97%) rename include/web_video_server/{ => streamers}/vp8_streamer.hpp (97%) rename include/web_video_server/{ => streamers}/vp9_streamer.hpp (97%) rename src/{ => streamers}/h264_streamer.cpp (98%) create mode 100644 src/streamers/image_transport_streamer.cpp rename src/{ => streamers}/jpeg_streamers.cpp (98%) rename src/{ => streamers}/libav_streamer.cpp (99%) rename src/{ => streamers}/png_streamers.cpp (98%) rename src/{ => streamers}/ros_compressed_streamer.cpp (98%) rename src/{ => streamers}/vp8_streamer.cpp (98%) rename src/{ => streamers}/vp9_streamer.cpp (98%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 766b4a2..bf06404 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,15 +51,16 @@ include_directories(include add_library(${PROJECT_NAME} SHARED src/web_video_server.cpp src/image_streamer.cpp - src/libav_streamer.cpp - src/vp8_streamer.cpp - src/h264_streamer.cpp - src/vp9_streamer.cpp src/multipart_stream.cpp - src/ros_compressed_streamer.cpp - src/jpeg_streamers.cpp - src/png_streamers.cpp src/utils.cpp + src/streamers/image_transport_streamer.cpp + src/streamers/libav_streamer.cpp + src/streamers/h264_streamer.cpp + src/streamers/vp8_streamer.cpp + src/streamers/vp9_streamer.cpp + src/streamers/ros_compressed_streamer.cpp + src/streamers/jpeg_streamers.cpp + src/streamers/png_streamers.cpp ) ## Specify libraries to link a library or executable target against diff --git a/include/web_video_server/image_streamer.hpp b/include/web_video_server/image_streamer.hpp index 0e001c5..2a78521 100644 --- a/include/web_video_server/image_streamer.hpp +++ b/include/web_video_server/image_streamer.hpp @@ -81,42 +81,6 @@ class ImageStreamer std::string topic_; }; - -class ImageTransportImageStreamer : public ImageStreamer -{ -public: - ImageTransportImageStreamer( - const async_web_server_cpp::HttpRequest & request, - async_web_server_cpp::HttpConnectionPtr connection, - rclcpp::Node::SharedPtr node); - virtual ~ImageTransportImageStreamer(); - - virtual void start(); - -protected: - virtual cv::Mat decodeImage(const sensor_msgs::msg::Image::ConstSharedPtr & msg); - virtual void sendImage(const cv::Mat &, const std::chrono::steady_clock::time_point & time) = 0; - virtual void restreamFrame(std::chrono::duration max_age); - virtual void initialize(const cv::Mat &); - - image_transport::Subscriber image_sub_; - int output_width_; - int output_height_; - bool invert_; - std::string default_transport_; - std::string qos_profile_name_; - - std::chrono::steady_clock::time_point last_frame_; - cv::Mat output_size_image; - std::mutex send_mutex_; - -private: - image_transport::ImageTransport it_; - bool initialized_; - - void imageCallback(const sensor_msgs::msg::Image::ConstSharedPtr & msg); -}; - class ImageStreamerType { public: diff --git a/include/web_video_server/h264_streamer.hpp b/include/web_video_server/streamers/h264_streamer.hpp similarity index 97% rename from include/web_video_server/h264_streamer.hpp rename to include/web_video_server/streamers/h264_streamer.hpp index 3df01b4..ae120a1 100644 --- a/include/web_video_server/h264_streamer.hpp +++ b/include/web_video_server/streamers/h264_streamer.hpp @@ -33,10 +33,11 @@ #include #include "image_transport/image_transport.hpp" -#include "web_video_server/libav_streamer.hpp" #include "async_web_server_cpp/http_request.hpp" #include "async_web_server_cpp/http_connection.hpp" +#include "web_video_server/streamers/libav_streamer.hpp" + namespace web_video_server { diff --git a/include/web_video_server/streamers/image_transport_streamer.hpp b/include/web_video_server/streamers/image_transport_streamer.hpp new file mode 100644 index 0000000..042bbbc --- /dev/null +++ b/include/web_video_server/streamers/image_transport_streamer.hpp @@ -0,0 +1,74 @@ +// Copyright (c) 2024-2025, The Robot Web Tools Contributors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include + +#include "web_video_server/image_streamer.hpp" + +namespace web_video_server +{ + +class ImageTransportImageStreamer : public ImageStreamer +{ +public: + ImageTransportImageStreamer( + const async_web_server_cpp::HttpRequest & request, + async_web_server_cpp::HttpConnectionPtr connection, + rclcpp::Node::SharedPtr node); + virtual ~ImageTransportImageStreamer(); + + virtual void start(); + +protected: + virtual cv::Mat decodeImage(const sensor_msgs::msg::Image::ConstSharedPtr & msg); + virtual void sendImage(const cv::Mat &, const std::chrono::steady_clock::time_point & time) = 0; + virtual void restreamFrame(std::chrono::duration max_age); + virtual void initialize(const cv::Mat &); + + image_transport::Subscriber image_sub_; + int output_width_; + int output_height_; + bool invert_; + std::string default_transport_; + std::string qos_profile_name_; + + std::chrono::steady_clock::time_point last_frame_; + cv::Mat output_size_image; + std::mutex send_mutex_; + +private: + image_transport::ImageTransport it_; + bool initialized_; + + void imageCallback(const sensor_msgs::msg::Image::ConstSharedPtr & msg); +}; + +} // namespace web_video_server diff --git a/include/web_video_server/jpeg_streamers.hpp b/include/web_video_server/streamers/jpeg_streamers.hpp similarity index 97% rename from include/web_video_server/jpeg_streamers.hpp rename to include/web_video_server/streamers/jpeg_streamers.hpp index 364357f..c63c9ce 100644 --- a/include/web_video_server/jpeg_streamers.hpp +++ b/include/web_video_server/streamers/jpeg_streamers.hpp @@ -34,10 +34,11 @@ #include #include "image_transport/image_transport.hpp" -#include "web_video_server/image_streamer.hpp" #include "async_web_server_cpp/http_request.hpp" #include "async_web_server_cpp/http_connection.hpp" + #include "web_video_server/multipart_stream.hpp" +#include "web_video_server/streamers/image_transport_streamer.hpp" namespace web_video_server { diff --git a/include/web_video_server/libav_streamer.hpp b/include/web_video_server/streamers/libav_streamer.hpp similarity index 98% rename from include/web_video_server/libav_streamer.hpp rename to include/web_video_server/streamers/libav_streamer.hpp index 12914bb..90d75ed 100644 --- a/include/web_video_server/libav_streamer.hpp +++ b/include/web_video_server/streamers/libav_streamer.hpp @@ -46,10 +46,11 @@ extern "C" #include #include -#include "image_transport/image_transport.hpp" -#include "web_video_server/image_streamer.hpp" -#include "async_web_server_cpp/http_request.hpp" #include "async_web_server_cpp/http_connection.hpp" +#include "async_web_server_cpp/http_request.hpp" +#include "image_transport/image_transport.hpp" + +#include "web_video_server/streamers/image_transport_streamer.hpp" namespace web_video_server { diff --git a/include/web_video_server/png_streamers.hpp b/include/web_video_server/streamers/png_streamers.hpp similarity index 97% rename from include/web_video_server/png_streamers.hpp rename to include/web_video_server/streamers/png_streamers.hpp index c261fe6..b68533b 100644 --- a/include/web_video_server/png_streamers.hpp +++ b/include/web_video_server/streamers/png_streamers.hpp @@ -32,11 +32,12 @@ #include #include -#include "image_transport/image_transport.hpp" -#include "web_video_server/image_streamer.hpp" -#include "async_web_server_cpp/http_request.hpp" #include "async_web_server_cpp/http_connection.hpp" +#include "async_web_server_cpp/http_request.hpp" +#include "image_transport/image_transport.hpp" + #include "web_video_server/multipart_stream.hpp" +#include "web_video_server/streamers/image_transport_streamer.hpp" namespace web_video_server { diff --git a/include/web_video_server/ros_compressed_streamer.hpp b/include/web_video_server/streamers/ros_compressed_streamer.hpp similarity index 97% rename from include/web_video_server/ros_compressed_streamer.hpp rename to include/web_video_server/streamers/ros_compressed_streamer.hpp index 9fc51bd..0d458d1 100644 --- a/include/web_video_server/ros_compressed_streamer.hpp +++ b/include/web_video_server/streamers/ros_compressed_streamer.hpp @@ -33,11 +33,12 @@ #include #include -#include "sensor_msgs/msg/compressed_image.hpp" -#include "web_video_server/image_streamer.hpp" -#include "async_web_server_cpp/http_request.hpp" #include "async_web_server_cpp/http_connection.hpp" +#include "async_web_server_cpp/http_request.hpp" +#include "sensor_msgs/msg/compressed_image.hpp" + #include "web_video_server/multipart_stream.hpp" +#include "web_video_server/streamers/image_transport_streamer.hpp" namespace web_video_server { diff --git a/include/web_video_server/vp8_streamer.hpp b/include/web_video_server/streamers/vp8_streamer.hpp similarity index 97% rename from include/web_video_server/vp8_streamer.hpp rename to include/web_video_server/streamers/vp8_streamer.hpp index ad6fd2f..c92585f 100644 --- a/include/web_video_server/vp8_streamer.hpp +++ b/include/web_video_server/streamers/vp8_streamer.hpp @@ -33,10 +33,11 @@ #include #include -#include "image_transport/image_transport.hpp" -#include "web_video_server/libav_streamer.hpp" -#include "async_web_server_cpp/http_request.hpp" #include "async_web_server_cpp/http_connection.hpp" +#include "async_web_server_cpp/http_request.hpp" +#include "image_transport/image_transport.hpp" + +#include "web_video_server/streamers/libav_streamer.hpp" namespace web_video_server { diff --git a/include/web_video_server/vp9_streamer.hpp b/include/web_video_server/streamers/vp9_streamer.hpp similarity index 97% rename from include/web_video_server/vp9_streamer.hpp rename to include/web_video_server/streamers/vp9_streamer.hpp index 5491db7..0cc7d90 100644 --- a/include/web_video_server/vp9_streamer.hpp +++ b/include/web_video_server/streamers/vp9_streamer.hpp @@ -31,10 +31,11 @@ #include -#include "image_transport/image_transport.hpp" -#include "web_video_server/libav_streamer.hpp" -#include "async_web_server_cpp/http_request.hpp" #include "async_web_server_cpp/http_connection.hpp" +#include "async_web_server_cpp/http_request.hpp" +#include "image_transport/image_transport.hpp" + +#include "web_video_server/streamers/libav_streamer.hpp" namespace web_video_server { diff --git a/include/web_video_server/web_video_server.hpp b/include/web_video_server/web_video_server.hpp index 6e9bd95..27d650b 100644 --- a/include/web_video_server/web_video_server.hpp +++ b/include/web_video_server/web_video_server.hpp @@ -41,11 +41,12 @@ #include "cv_bridge/cv_bridge.hpp" #endif +#include "async_web_server_cpp/http_connection.hpp" +#include "async_web_server_cpp/http_request.hpp" +#include "async_web_server_cpp/http_server.hpp" #include "rclcpp/rclcpp.hpp" + #include "web_video_server/image_streamer.hpp" -#include "async_web_server_cpp/http_server.hpp" -#include "async_web_server_cpp/http_request.hpp" -#include "async_web_server_cpp/http_connection.hpp" namespace web_video_server { diff --git a/src/image_streamer.cpp b/src/image_streamer.cpp index 564e1dc..dc95867 100644 --- a/src/image_streamer.cpp +++ b/src/image_streamer.cpp @@ -30,12 +30,6 @@ #include "web_video_server/image_streamer.hpp" -#ifdef CV_BRIDGE_USES_OLD_HEADERS -#include -#else -#include -#endif - #include namespace web_video_server @@ -53,180 +47,4 @@ ImageStreamer::~ImageStreamer() { } -ImageTransportImageStreamer::ImageTransportImageStreamer( - const async_web_server_cpp::HttpRequest & request, - async_web_server_cpp::HttpConnectionPtr connection, rclcpp::Node::SharedPtr node) -: ImageStreamer(request, connection, node), it_(node), initialized_(false) -{ - output_width_ = request.get_query_param_value_or_default("width", -1); - output_height_ = request.get_query_param_value_or_default("height", -1); - invert_ = request.has_query_param("invert"); - default_transport_ = request.get_query_param_value_or_default("default_transport", "raw"); - qos_profile_name_ = request.get_query_param_value_or_default("qos_profile", "default"); -} - -ImageTransportImageStreamer::~ImageTransportImageStreamer() -{ -} - -void ImageTransportImageStreamer::start() -{ - image_transport::TransportHints hints(node_.get(), default_transport_); - auto tnat = node_->get_topic_names_and_types(); - inactive_ = true; - for (auto topic_and_types : tnat) { - if (topic_and_types.second.size() > 1) { - // skip over topics with more than one type - continue; - } - auto & topic_name = topic_and_types.first; - if (topic_name == topic_ || (topic_name.find("/") == 0 && topic_name.substr(1) == topic_)) { - inactive_ = false; - break; - } - } - - // Get QoS profile from query parameter - RCLCPP_INFO( - node_->get_logger(), "Streaming topic %s with QoS profile %s", topic_.c_str(), - qos_profile_name_.c_str()); - auto qos_profile = get_qos_profile_from_name(qos_profile_name_); - if (!qos_profile) { - qos_profile = rmw_qos_profile_default; - RCLCPP_ERROR( - node_->get_logger(), - "Invalid QoS profile %s specified. Using default profile.", - qos_profile_name_.c_str()); - } - - // Create subscriber - image_sub_ = image_transport::create_subscription( - node_.get(), topic_, - std::bind(&ImageTransportImageStreamer::imageCallback, this, std::placeholders::_1), - default_transport_, qos_profile.value()); -} - -void ImageTransportImageStreamer::initialize(const cv::Mat &) -{ -} - -void ImageTransportImageStreamer::restreamFrame(std::chrono::duration max_age) -{ - if (inactive_ || !initialized_) { - return; - } - try { - if (last_frame_ + max_age < std::chrono::steady_clock::now()) { - std::scoped_lock lock(send_mutex_); - // don't update last_frame, it may remain an old value. - sendImage(output_size_image, std::chrono::steady_clock::now()); - } - } catch (boost::system::system_error & e) { - // happens when client disconnects - RCLCPP_DEBUG(node_->get_logger(), "system_error exception: %s", e.what()); - inactive_ = true; - return; - } catch (std::exception & e) { - auto & clk = *node_->get_clock(); - RCLCPP_ERROR_THROTTLE(node_->get_logger(), clk, 40, "exception: %s", e.what()); - inactive_ = true; - return; - } catch (...) { - auto & clk = *node_->get_clock(); - RCLCPP_ERROR_THROTTLE(node_->get_logger(), clk, 40, "exception"); - inactive_ = true; - return; - } -} - -cv::Mat ImageTransportImageStreamer::decodeImage( - const sensor_msgs::msg::Image::ConstSharedPtr & msg) -{ - if (msg->encoding.find("F") != std::string::npos) { - // scale floating point images - cv::Mat float_image_bridge = cv_bridge::toCvCopy(msg, msg->encoding)->image; - cv::Mat_ float_image = float_image_bridge; - double max_val; - cv::minMaxIdx(float_image, 0, &max_val); - - if (max_val > 0) { - float_image *= (255 / max_val); - } - return float_image; - } else { - // Convert to OpenCV native BGR color - return cv_bridge::toCvCopy(msg, "bgr8")->image; - } -} - -void ImageTransportImageStreamer::imageCallback(const sensor_msgs::msg::Image::ConstSharedPtr & msg) -{ - if (inactive_) { - return; - } - - cv::Mat img; - try { - img = decodeImage(msg); - int input_width = img.cols; - int input_height = img.rows; - - if (output_width_ == -1) { - output_width_ = input_width; - } - if (output_height_ == -1) { - output_height_ = input_height; - } - - if (invert_) { - // Rotate 180 degrees - cv::flip(img, img, false); - cv::flip(img, img, true); - } - - std::scoped_lock lock(send_mutex_); // protects output_size_image - if (output_width_ != input_width || output_height_ != input_height) { - cv::Mat img_resized; - cv::Size new_size(output_width_, output_height_); - cv::resize(img, img_resized, new_size); - output_size_image = img_resized; - } else { - output_size_image = img; - } - - if (!initialized_) { - initialize(output_size_image); - initialized_ = true; - } - - last_frame_ = std::chrono::steady_clock::now(); - sendImage(output_size_image, last_frame_); - } catch (cv_bridge::Exception & e) { - auto & clk = *node_->get_clock(); - RCLCPP_ERROR_THROTTLE(node_->get_logger(), clk, 40, "cv_bridge exception: %s", e.what()); - inactive_ = true; - return; - } catch (cv::Exception & e) { - auto & clk = *node_->get_clock(); - RCLCPP_ERROR_THROTTLE(node_->get_logger(), clk, 40, "cv_bridge exception: %s", e.what()); - inactive_ = true; - return; - } catch (boost::system::system_error & e) { - // happens when client disconnects - RCLCPP_DEBUG(node_->get_logger(), "system_error exception: %s", e.what()); - inactive_ = true; - return; - } catch (std::exception & e) { - auto & clk = *node_->get_clock(); - RCLCPP_ERROR_THROTTLE(node_->get_logger(), clk, 40, "exception: %s", e.what()); - inactive_ = true; - return; - } catch (...) { - auto & clk = *node_->get_clock(); - RCLCPP_ERROR_THROTTLE(node_->get_logger(), clk, 40, "exception"); - inactive_ = true; - return; - } -} - } // namespace web_video_server diff --git a/src/multipart_stream.cpp b/src/multipart_stream.cpp index 0429272..3c4af01 100644 --- a/src/multipart_stream.cpp +++ b/src/multipart_stream.cpp @@ -29,6 +29,7 @@ // POSSIBILITY OF SUCH DAMAGE. #include "web_video_server/multipart_stream.hpp" + #include "async_web_server_cpp/http_reply.hpp" namespace web_video_server diff --git a/src/h264_streamer.cpp b/src/streamers/h264_streamer.cpp similarity index 98% rename from src/h264_streamer.cpp rename to src/streamers/h264_streamer.cpp index 05e487f..ab6326b 100644 --- a/src/h264_streamer.cpp +++ b/src/streamers/h264_streamer.cpp @@ -27,7 +27,7 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. -#include "web_video_server/h264_streamer.hpp" +#include "web_video_server/streamers/h264_streamer.hpp" namespace web_video_server { diff --git a/src/streamers/image_transport_streamer.cpp b/src/streamers/image_transport_streamer.cpp new file mode 100644 index 0000000..f17e8d1 --- /dev/null +++ b/src/streamers/image_transport_streamer.cpp @@ -0,0 +1,218 @@ +// Copyright (c) 2014, Worcester Polytechnic Institute +// Copyright (c) 2024-2025, The Robot Web Tools Contributors +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. + +#include "web_video_server/streamers/image_transport_streamer.hpp" + +#ifdef CV_BRIDGE_USES_OLD_HEADERS +#include "cv_bridge/cv_bridge.h" +#else +#include "cv_bridge/cv_bridge.hpp" +#endif + +namespace web_video_server +{ + +ImageTransportImageStreamer::ImageTransportImageStreamer( + const async_web_server_cpp::HttpRequest & request, + async_web_server_cpp::HttpConnectionPtr connection, rclcpp::Node::SharedPtr node) +: ImageStreamer(request, connection, node), it_(node), initialized_(false) +{ + output_width_ = request.get_query_param_value_or_default("width", -1); + output_height_ = request.get_query_param_value_or_default("height", -1); + invert_ = request.has_query_param("invert"); + default_transport_ = request.get_query_param_value_or_default("default_transport", "raw"); + qos_profile_name_ = request.get_query_param_value_or_default("qos_profile", "default"); +} + +ImageTransportImageStreamer::~ImageTransportImageStreamer() +{ +} + +void ImageTransportImageStreamer::start() +{ + image_transport::TransportHints hints(node_.get(), default_transport_); + auto tnat = node_->get_topic_names_and_types(); + inactive_ = true; + for (auto topic_and_types : tnat) { + if (topic_and_types.second.size() > 1) { + // skip over topics with more than one type + continue; + } + auto & topic_name = topic_and_types.first; + if (topic_name == topic_ || (topic_name.find("/") == 0 && topic_name.substr(1) == topic_)) { + inactive_ = false; + break; + } + } + + // Get QoS profile from query parameter + RCLCPP_INFO( + node_->get_logger(), "Streaming topic %s with QoS profile %s", topic_.c_str(), + qos_profile_name_.c_str()); + auto qos_profile = get_qos_profile_from_name(qos_profile_name_); + if (!qos_profile) { + qos_profile = rmw_qos_profile_default; + RCLCPP_ERROR( + node_->get_logger(), + "Invalid QoS profile %s specified. Using default profile.", + qos_profile_name_.c_str()); + } + + // Create subscriber + image_sub_ = image_transport::create_subscription( + node_.get(), topic_, + std::bind(&ImageTransportImageStreamer::imageCallback, this, std::placeholders::_1), + default_transport_, qos_profile.value()); +} + +void ImageTransportImageStreamer::initialize(const cv::Mat &) +{ +} + +void ImageTransportImageStreamer::restreamFrame(std::chrono::duration max_age) +{ + if (inactive_ || !initialized_) { + return; + } + try { + if (last_frame_ + max_age < std::chrono::steady_clock::now()) { + std::scoped_lock lock(send_mutex_); + // don't update last_frame, it may remain an old value. + sendImage(output_size_image, std::chrono::steady_clock::now()); + } + } catch (boost::system::system_error & e) { + // happens when client disconnects + RCLCPP_DEBUG(node_->get_logger(), "system_error exception: %s", e.what()); + inactive_ = true; + return; + } catch (std::exception & e) { + auto & clk = *node_->get_clock(); + RCLCPP_ERROR_THROTTLE(node_->get_logger(), clk, 40, "exception: %s", e.what()); + inactive_ = true; + return; + } catch (...) { + auto & clk = *node_->get_clock(); + RCLCPP_ERROR_THROTTLE(node_->get_logger(), clk, 40, "exception"); + inactive_ = true; + return; + } +} + +void ImageTransportImageStreamer::imageCallback(const sensor_msgs::msg::Image::ConstSharedPtr & msg) +{ + if (inactive_) { + return; + } + + cv::Mat img; + try { + img = decodeImage(msg); + int input_width = img.cols; + int input_height = img.rows; + + if (output_width_ == -1) { + output_width_ = input_width; + } + if (output_height_ == -1) { + output_height_ = input_height; + } + + if (invert_) { + // Rotate 180 degrees + cv::flip(img, img, false); + cv::flip(img, img, true); + } + + std::scoped_lock lock(send_mutex_); // protects output_size_image + if (output_width_ != input_width || output_height_ != input_height) { + cv::Mat img_resized; + cv::Size new_size(output_width_, output_height_); + cv::resize(img, img_resized, new_size); + output_size_image = img_resized; + } else { + output_size_image = img; + } + + if (!initialized_) { + initialize(output_size_image); + initialized_ = true; + } + + last_frame_ = std::chrono::steady_clock::now(); + sendImage(output_size_image, last_frame_); + } catch (cv_bridge::Exception & e) { + auto & clk = *node_->get_clock(); + RCLCPP_ERROR_THROTTLE(node_->get_logger(), clk, 40, "cv_bridge exception: %s", e.what()); + inactive_ = true; + return; + } catch (cv::Exception & e) { + auto & clk = *node_->get_clock(); + RCLCPP_ERROR_THROTTLE(node_->get_logger(), clk, 40, "cv_bridge exception: %s", e.what()); + inactive_ = true; + return; + } catch (boost::system::system_error & e) { + // happens when client disconnects + RCLCPP_DEBUG(node_->get_logger(), "system_error exception: %s", e.what()); + inactive_ = true; + return; + } catch (std::exception & e) { + auto & clk = *node_->get_clock(); + RCLCPP_ERROR_THROTTLE(node_->get_logger(), clk, 40, "exception: %s", e.what()); + inactive_ = true; + return; + } catch (...) { + auto & clk = *node_->get_clock(); + RCLCPP_ERROR_THROTTLE(node_->get_logger(), clk, 40, "exception"); + inactive_ = true; + return; + } +} + +cv::Mat ImageTransportImageStreamer::decodeImage( + const sensor_msgs::msg::Image::ConstSharedPtr & msg) +{ + if (msg->encoding.find("F") != std::string::npos) { + // scale floating point images + cv::Mat float_image_bridge = cv_bridge::toCvCopy(msg, msg->encoding)->image; + cv::Mat_ float_image = float_image_bridge; + double max_val; + cv::minMaxIdx(float_image, 0, &max_val); + + if (max_val > 0) { + float_image *= (255 / max_val); + } + return float_image; + } else { + // Convert to OpenCV native BGR color + return cv_bridge::toCvCopy(msg, "bgr8")->image; + } +} + +} // namespace web_video_server diff --git a/src/jpeg_streamers.cpp b/src/streamers/jpeg_streamers.cpp similarity index 98% rename from src/jpeg_streamers.cpp rename to src/streamers/jpeg_streamers.cpp index bfc6c62..3e0afa5 100644 --- a/src/jpeg_streamers.cpp +++ b/src/streamers/jpeg_streamers.cpp @@ -28,7 +28,8 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. -#include "web_video_server/jpeg_streamers.hpp" +#include "web_video_server/streamers/jpeg_streamers.hpp" + #include "async_web_server_cpp/http_reply.hpp" namespace web_video_server diff --git a/src/libav_streamer.cpp b/src/streamers/libav_streamer.cpp similarity index 99% rename from src/libav_streamer.cpp rename to src/streamers/libav_streamer.cpp index 327a919..a29408c 100644 --- a/src/libav_streamer.cpp +++ b/src/streamers/libav_streamer.cpp @@ -28,7 +28,8 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. -#include "web_video_server/libav_streamer.hpp" +#include "web_video_server/streamers/libav_streamer.hpp" + #include "async_web_server_cpp/http_reply.hpp" // https://stackoverflow.com/questions/46884682/error-in-building-opencv-with-ffmpeg diff --git a/src/png_streamers.cpp b/src/streamers/png_streamers.cpp similarity index 98% rename from src/png_streamers.cpp rename to src/streamers/png_streamers.cpp index c3c9327..d0c8db8 100644 --- a/src/png_streamers.cpp +++ b/src/streamers/png_streamers.cpp @@ -27,7 +27,8 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. -#include "web_video_server/png_streamers.hpp" +#include "web_video_server/streamers/png_streamers.hpp" + #include "async_web_server_cpp/http_reply.hpp" #ifdef CV_BRIDGE_USES_OLD_HEADERS diff --git a/src/ros_compressed_streamer.cpp b/src/streamers/ros_compressed_streamer.cpp similarity index 98% rename from src/ros_compressed_streamer.cpp rename to src/streamers/ros_compressed_streamer.cpp index 76c965d..51347b0 100644 --- a/src/ros_compressed_streamer.cpp +++ b/src/streamers/ros_compressed_streamer.cpp @@ -28,7 +28,7 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. -#include "web_video_server/ros_compressed_streamer.hpp" +#include "web_video_server/streamers/ros_compressed_streamer.hpp" namespace web_video_server { diff --git a/src/vp8_streamer.cpp b/src/streamers/vp8_streamer.cpp similarity index 98% rename from src/vp8_streamer.cpp rename to src/streamers/vp8_streamer.cpp index c4c1d0c..260e88d 100644 --- a/src/vp8_streamer.cpp +++ b/src/streamers/vp8_streamer.cpp @@ -28,7 +28,7 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. -#include "web_video_server/vp8_streamer.hpp" +#include "web_video_server/streamers/vp8_streamer.hpp" namespace web_video_server { diff --git a/src/vp9_streamer.cpp b/src/streamers/vp9_streamer.cpp similarity index 98% rename from src/vp9_streamer.cpp rename to src/streamers/vp9_streamer.cpp index 65511a7..765dcef 100644 --- a/src/vp9_streamer.cpp +++ b/src/streamers/vp9_streamer.cpp @@ -27,7 +27,7 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. -#include "web_video_server/vp9_streamer.hpp" +#include "web_video_server/streamers/vp9_streamer.hpp" namespace web_video_server { diff --git a/src/web_video_server.cpp b/src/web_video_server.cpp index 24db2fb..aa9f226 100644 --- a/src/web_video_server.cpp +++ b/src/web_video_server.cpp @@ -36,16 +36,17 @@ #include #include +#include "async_web_server_cpp/http_reply.hpp" #include "rclcpp/rclcpp.hpp" - #include "sensor_msgs/image_encodings.hpp" -#include "web_video_server/ros_compressed_streamer.hpp" -#include "web_video_server/jpeg_streamers.hpp" -#include "web_video_server/png_streamers.hpp" -#include "web_video_server/vp8_streamer.hpp" -#include "web_video_server/h264_streamer.hpp" -#include "web_video_server/vp9_streamer.hpp" -#include "async_web_server_cpp/http_reply.hpp" + +#include "web_video_server/streamers/ros_compressed_streamer.hpp" +#include "web_video_server/streamers/image_transport_streamer.hpp" +#include "web_video_server/streamers/jpeg_streamers.hpp" +#include "web_video_server/streamers/png_streamers.hpp" +#include "web_video_server/streamers/vp8_streamer.hpp" +#include "web_video_server/streamers/h264_streamer.hpp" +#include "web_video_server/streamers/vp9_streamer.hpp" using namespace std::chrono_literals; using namespace boost::placeholders; // NOLINT From bf4af8e5ad9ba6f9f334e44694c824d44c9700a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82a=C5=BCej=20Sowa?= Date: Thu, 9 Oct 2025 17:54:34 +0200 Subject: [PATCH 02/33] Clean up include directives by including only what's used --- include/web_video_server/image_streamer.hpp | 10 ++--- include/web_video_server/multipart_stream.hpp | 5 ++- .../streamers/h264_streamer.hpp | 3 +- .../streamers/image_transport_streamer.hpp | 11 +++++ .../streamers/jpeg_streamers.hpp | 6 ++- .../streamers/libav_streamer.hpp | 15 ++++--- .../streamers/png_streamers.hpp | 9 +++- .../streamers/ros_compressed_streamer.hpp | 6 ++- .../streamers/vp8_streamer.hpp | 3 +- .../streamers/vp9_streamer.hpp | 3 +- include/web_video_server/utils.hpp | 3 +- include/web_video_server/web_video_server.hpp | 13 +++--- src/image_streamer.cpp | 5 ++- src/multipart_stream.cpp | 11 +++++ src/streamers/h264_streamer.cpp | 18 ++++++++ src/streamers/image_transport_streamer.cpp | 23 +++++++++++ src/streamers/jpeg_streamers.cpp | 23 ++++++++++- src/streamers/libav_streamer.cpp | 41 ++++++++++++++++++- src/streamers/png_streamers.cpp | 25 ++++++++++- src/streamers/ros_compressed_streamer.cpp | 21 ++++++++++ src/streamers/vp8_streamer.cpp | 17 ++++++++ src/streamers/vp9_streamer.cpp | 16 ++++++++ src/utils.cpp | 6 +++ src/web_video_server.cpp | 21 ++++++++-- src/web_video_server_node.cpp | 3 ++ 25 files changed, 277 insertions(+), 40 deletions(-) diff --git a/include/web_video_server/image_streamer.hpp b/include/web_video_server/image_streamer.hpp index 2a78521..ec6516d 100644 --- a/include/web_video_server/image_streamer.hpp +++ b/include/web_video_server/image_streamer.hpp @@ -34,14 +34,10 @@ #include #include -#include - -#include "rclcpp/rclcpp.hpp" -#include "image_transport/image_transport.hpp" -#include "image_transport/transport_hints.hpp" -#include "web_video_server/utils.hpp" -#include "async_web_server_cpp/http_server.hpp" +#include "async_web_server_cpp/http_connection.hpp" #include "async_web_server_cpp/http_request.hpp" +#include "image_transport/subscriber.hpp" +#include "rclcpp/node.hpp" namespace web_video_server { diff --git a/include/web_video_server/multipart_stream.hpp b/include/web_video_server/multipart_stream.hpp index fae1352..813e3ae 100644 --- a/include/web_video_server/multipart_stream.hpp +++ b/include/web_video_server/multipart_stream.hpp @@ -30,12 +30,15 @@ #pragma once +#include +#include #include #include #include #include -#include "rclcpp/rclcpp.hpp" +#include + #include "async_web_server_cpp/http_connection.hpp" namespace web_video_server diff --git a/include/web_video_server/streamers/h264_streamer.hpp b/include/web_video_server/streamers/h264_streamer.hpp index ae120a1..a625750 100644 --- a/include/web_video_server/streamers/h264_streamer.hpp +++ b/include/web_video_server/streamers/h264_streamer.hpp @@ -32,10 +32,11 @@ #include #include -#include "image_transport/image_transport.hpp" #include "async_web_server_cpp/http_request.hpp" #include "async_web_server_cpp/http_connection.hpp" +#include "rclcpp/node.hpp" +#include "web_video_server/image_streamer.hpp" #include "web_video_server/streamers/libav_streamer.hpp" namespace web_video_server diff --git a/include/web_video_server/streamers/image_transport_streamer.hpp b/include/web_video_server/streamers/image_transport_streamer.hpp index 042bbbc..bb01c9b 100644 --- a/include/web_video_server/streamers/image_transport_streamer.hpp +++ b/include/web_video_server/streamers/image_transport_streamer.hpp @@ -29,8 +29,19 @@ #pragma once +#include +#include #include +#include + +#include "async_web_server_cpp/http_connection.hpp" +#include "async_web_server_cpp/http_request.hpp" +#include "image_transport/image_transport.hpp" +#include "image_transport/subscriber.hpp" +#include "rclcpp/node.hpp" +#include "sensor_msgs/msg/image.hpp" + #include "web_video_server/image_streamer.hpp" namespace web_video_server diff --git a/include/web_video_server/streamers/jpeg_streamers.hpp b/include/web_video_server/streamers/jpeg_streamers.hpp index c63c9ce..cf7ee76 100644 --- a/include/web_video_server/streamers/jpeg_streamers.hpp +++ b/include/web_video_server/streamers/jpeg_streamers.hpp @@ -30,13 +30,17 @@ #pragma once +#include #include #include -#include "image_transport/image_transport.hpp" +#include + #include "async_web_server_cpp/http_request.hpp" #include "async_web_server_cpp/http_connection.hpp" +#include "rclcpp/node.hpp" +#include "web_video_server/image_streamer.hpp" #include "web_video_server/multipart_stream.hpp" #include "web_video_server/streamers/image_transport_streamer.hpp" diff --git a/include/web_video_server/streamers/libav_streamer.hpp b/include/web_video_server/streamers/libav_streamer.hpp index 90d75ed..966571f 100644 --- a/include/web_video_server/streamers/libav_streamer.hpp +++ b/include/web_video_server/streamers/libav_streamer.hpp @@ -33,23 +33,26 @@ extern "C" { #include +#include #include -#include -#include +#include +#include #include -#include -#include -#include } #include +#include #include +#include #include +#include + #include "async_web_server_cpp/http_connection.hpp" #include "async_web_server_cpp/http_request.hpp" -#include "image_transport/image_transport.hpp" +#include "rclcpp/node.hpp" +#include "web_video_server/image_streamer.hpp" #include "web_video_server/streamers/image_transport_streamer.hpp" namespace web_video_server diff --git a/include/web_video_server/streamers/png_streamers.hpp b/include/web_video_server/streamers/png_streamers.hpp index b68533b..7438d44 100644 --- a/include/web_video_server/streamers/png_streamers.hpp +++ b/include/web_video_server/streamers/png_streamers.hpp @@ -29,13 +29,18 @@ #pragma once +#include #include #include -#include "async_web_server_cpp/http_connection.hpp" +#include + #include "async_web_server_cpp/http_request.hpp" -#include "image_transport/image_transport.hpp" +#include "async_web_server_cpp/http_connection.hpp" +#include "rclcpp/node.hpp" +#include "sensor_msgs/msg/image.hpp" +#include "web_video_server/image_streamer.hpp" #include "web_video_server/multipart_stream.hpp" #include "web_video_server/streamers/image_transport_streamer.hpp" diff --git a/include/web_video_server/streamers/ros_compressed_streamer.hpp b/include/web_video_server/streamers/ros_compressed_streamer.hpp index 0d458d1..7ce7ba5 100644 --- a/include/web_video_server/streamers/ros_compressed_streamer.hpp +++ b/include/web_video_server/streamers/ros_compressed_streamer.hpp @@ -30,15 +30,19 @@ #pragma once +#include #include +#include #include #include "async_web_server_cpp/http_connection.hpp" #include "async_web_server_cpp/http_request.hpp" +#include "rclcpp/node.hpp" +#include "rclcpp/subscription.hpp" #include "sensor_msgs/msg/compressed_image.hpp" +#include "web_video_server/image_streamer.hpp" #include "web_video_server/multipart_stream.hpp" -#include "web_video_server/streamers/image_transport_streamer.hpp" namespace web_video_server { diff --git a/include/web_video_server/streamers/vp8_streamer.hpp b/include/web_video_server/streamers/vp8_streamer.hpp index c92585f..8e148c3 100644 --- a/include/web_video_server/streamers/vp8_streamer.hpp +++ b/include/web_video_server/streamers/vp8_streamer.hpp @@ -35,8 +35,9 @@ #include "async_web_server_cpp/http_connection.hpp" #include "async_web_server_cpp/http_request.hpp" -#include "image_transport/image_transport.hpp" +#include "rclcpp/node.hpp" +#include "web_video_server/image_streamer.hpp" #include "web_video_server/streamers/libav_streamer.hpp" namespace web_video_server diff --git a/include/web_video_server/streamers/vp9_streamer.hpp b/include/web_video_server/streamers/vp9_streamer.hpp index 0cc7d90..348e1b0 100644 --- a/include/web_video_server/streamers/vp9_streamer.hpp +++ b/include/web_video_server/streamers/vp9_streamer.hpp @@ -33,8 +33,9 @@ #include "async_web_server_cpp/http_connection.hpp" #include "async_web_server_cpp/http_request.hpp" -#include "image_transport/image_transport.hpp" +#include "rclcpp/node.hpp" +#include "web_video_server/image_streamer.hpp" #include "web_video_server/streamers/libav_streamer.hpp" namespace web_video_server diff --git a/include/web_video_server/utils.hpp b/include/web_video_server/utils.hpp index 13fe6d4..6176e74 100644 --- a/include/web_video_server/utils.hpp +++ b/include/web_video_server/utils.hpp @@ -31,7 +31,8 @@ #include #include -#include "rmw/qos_profiles.h" + +#include "rmw/types.h" namespace web_video_server { diff --git a/include/web_video_server/web_video_server.hpp b/include/web_video_server/web_video_server.hpp index 27d650b..468f12c 100644 --- a/include/web_video_server/web_video_server.hpp +++ b/include/web_video_server/web_video_server.hpp @@ -30,21 +30,20 @@ #pragma once +#include #include #include +#include #include #include -#ifdef CV_BRIDGE_USES_OLD_HEADERS -#include "cv_bridge/cv_bridge.h" -#else -#include "cv_bridge/cv_bridge.hpp" -#endif - #include "async_web_server_cpp/http_connection.hpp" #include "async_web_server_cpp/http_request.hpp" +#include "async_web_server_cpp/http_request_handler.hpp" #include "async_web_server_cpp/http_server.hpp" -#include "rclcpp/rclcpp.hpp" +#include "rclcpp/node.hpp" +#include "rclcpp/node_options.hpp" +#include "rclcpp/timer.hpp" #include "web_video_server/image_streamer.hpp" diff --git a/src/image_streamer.cpp b/src/image_streamer.cpp index dc95867..7a3619d 100644 --- a/src/image_streamer.cpp +++ b/src/image_streamer.cpp @@ -30,7 +30,10 @@ #include "web_video_server/image_streamer.hpp" -#include +#include "rclcpp/node.hpp" + +#include "async_web_server_cpp/http_connection.hpp" +#include "async_web_server_cpp/http_request.hpp" namespace web_video_server { diff --git a/src/multipart_stream.cpp b/src/multipart_stream.cpp index 3c4af01..3d9b33f 100644 --- a/src/multipart_stream.cpp +++ b/src/multipart_stream.cpp @@ -30,6 +30,17 @@ #include "web_video_server/multipart_stream.hpp" +#include +#include +#include +#include +#include +#include + +#include + +#include "async_web_server_cpp/http_connection.hpp" +#include "async_web_server_cpp/http_header.hpp" #include "async_web_server_cpp/http_reply.hpp" namespace web_video_server diff --git a/src/streamers/h264_streamer.cpp b/src/streamers/h264_streamer.cpp index ab6326b..b190d38 100644 --- a/src/streamers/h264_streamer.cpp +++ b/src/streamers/h264_streamer.cpp @@ -29,6 +29,24 @@ #include "web_video_server/streamers/h264_streamer.hpp" +extern "C" +{ +#include +#include +#include +#include +} + +#include +#include + +#include "async_web_server_cpp/http_connection.hpp" +#include "async_web_server_cpp/http_request.hpp" +#include "rclcpp/node.hpp" + +#include "web_video_server/image_streamer.hpp" +#include "web_video_server/streamers/libav_streamer.hpp" + namespace web_video_server { diff --git a/src/streamers/image_transport_streamer.cpp b/src/streamers/image_transport_streamer.cpp index f17e8d1..f7ab8e3 100644 --- a/src/streamers/image_transport_streamer.cpp +++ b/src/streamers/image_transport_streamer.cpp @@ -30,12 +30,35 @@ #include "web_video_server/streamers/image_transport_streamer.hpp" +#include +#include +#include +#include + +#include +#include +#include +#include +#include + #ifdef CV_BRIDGE_USES_OLD_HEADERS #include "cv_bridge/cv_bridge.h" #else #include "cv_bridge/cv_bridge.hpp" #endif +#include "async_web_server_cpp/http_connection.hpp" +#include "async_web_server_cpp/http_request.hpp" +#include "image_transport/image_transport.hpp" +#include "image_transport/transport_hints.hpp" +#include "rclcpp/node.hpp" +#include "rclcpp/logging.hpp" +#include "rmw/qos_profiles.h" +#include "sensor_msgs/msg/image.hpp" + +#include "web_video_server/image_streamer.hpp" +#include "web_video_server/utils.hpp" + namespace web_video_server { diff --git a/src/streamers/jpeg_streamers.cpp b/src/streamers/jpeg_streamers.cpp index 3e0afa5..d058943 100644 --- a/src/streamers/jpeg_streamers.cpp +++ b/src/streamers/jpeg_streamers.cpp @@ -30,7 +30,26 @@ #include "web_video_server/streamers/jpeg_streamers.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "async_web_server_cpp/http_connection.hpp" #include "async_web_server_cpp/http_reply.hpp" +#include "async_web_server_cpp/http_request.hpp" +#include "rclcpp/node.hpp" + +#include "web_video_server/image_streamer.hpp" +#include "web_video_server/streamers/image_transport_streamer.hpp" namespace web_video_server { @@ -59,7 +78,7 @@ void MjpegStreamer::sendImage( encode_params.push_back(cv::IMWRITE_JPEG_QUALITY); encode_params.push_back(quality_); - std::vector encoded_buffer; + std::vector encoded_buffer; cv::imencode(".jpeg", img, encoded_buffer, encode_params); stream_.sendPartAndClear(time, "image/jpeg", encoded_buffer); @@ -105,7 +124,7 @@ void JpegSnapshotStreamer::sendImage( encode_params.push_back(cv::IMWRITE_JPEG_QUALITY); encode_params.push_back(quality_); - std::vector encoded_buffer; + std::vector encoded_buffer; cv::imencode(".jpeg", img, encoded_buffer, encode_params); char stamp[20]; diff --git a/src/streamers/libav_streamer.cpp b/src/streamers/libav_streamer.cpp index a29408c..dcbb0c9 100644 --- a/src/streamers/libav_streamer.cpp +++ b/src/streamers/libav_streamer.cpp @@ -30,7 +30,44 @@ #include "web_video_server/streamers/libav_streamer.hpp" +extern "C" +{ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +} + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "async_web_server_cpp/http_connection.hpp" #include "async_web_server_cpp/http_reply.hpp" +#include "async_web_server_cpp/http_request.hpp" +#include "rclcpp/node.hpp" +#include "rclcpp/logging.hpp" + +#include "web_video_server/image_streamer.hpp" +#include "web_video_server/streamers/image_transport_streamer.hpp" // https://stackoverflow.com/questions/46884682/error-in-building-opencv-with-ffmpeg #define AV_CODEC_FLAG_GLOBAL_HEADER (1 << 22) @@ -249,7 +286,7 @@ void LibavStreamer::sendImage( } - int ret = sws_scale( + sws_scale( sws_context_, (const uint8_t * const *)raw_frame->data, raw_frame->linesize, 0, output_height_, frame_->data, frame_->linesize); @@ -259,7 +296,7 @@ void LibavStreamer::sendImage( // Encode the frame AVPacket * pkt = av_packet_alloc(); - ret = avcodec_send_frame(codec_context_, frame_); + int ret = avcodec_send_frame(codec_context_, frame_); if (ret == AVERROR_EOF) { RCLCPP_DEBUG_STREAM(node_->get_logger(), "avcodec_send_frame() encoder flushed\n"); } else if (ret == AVERROR(EAGAIN)) { diff --git a/src/streamers/png_streamers.cpp b/src/streamers/png_streamers.cpp index d0c8db8..95a1d9b 100644 --- a/src/streamers/png_streamers.cpp +++ b/src/streamers/png_streamers.cpp @@ -29,7 +29,28 @@ #include "web_video_server/streamers/png_streamers.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "async_web_server_cpp/http_connection.hpp" #include "async_web_server_cpp/http_reply.hpp" +#include "async_web_server_cpp/http_request.hpp" +#include "rclcpp/node.hpp" +#include "sensor_msgs/image_encodings.hpp" +#include "sensor_msgs/msg/image.hpp" + +#include "web_video_server/image_streamer.hpp" +#include "web_video_server/streamers/image_transport_streamer.hpp" #ifdef CV_BRIDGE_USES_OLD_HEADERS #include "cv_bridge/cv_bridge.h" @@ -72,7 +93,7 @@ void PngStreamer::sendImage(const cv::Mat & img, const std::chrono::steady_clock encode_params.push_back(cv::IMWRITE_PNG_COMPRESSION); encode_params.push_back(quality_); - std::vector encoded_buffer; + std::vector encoded_buffer; cv::imencode(".png", img, encoded_buffer, encode_params); stream_.sendPartAndClear(time, "image/png", encoded_buffer); @@ -129,7 +150,7 @@ void PngSnapshotStreamer::sendImage( encode_params.push_back(cv::IMWRITE_PNG_COMPRESSION); encode_params.push_back(quality_); - std::vector encoded_buffer; + std::vector encoded_buffer; cv::imencode(".png", img, encoded_buffer, encode_params); char stamp[20]; diff --git a/src/streamers/ros_compressed_streamer.cpp b/src/streamers/ros_compressed_streamer.cpp index 51347b0..a4f4111 100644 --- a/src/streamers/ros_compressed_streamer.cpp +++ b/src/streamers/ros_compressed_streamer.cpp @@ -30,6 +30,27 @@ #include "web_video_server/streamers/ros_compressed_streamer.hpp" +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "async_web_server_cpp/http_connection.hpp" +#include "async_web_server_cpp/http_request.hpp" +#include "rclcpp/logging.hpp" +#include "rclcpp/node.hpp" +#include "rmw/qos_profiles.h" +#include "sensor_msgs/msg/compressed_image.hpp" + +#include "web_video_server/image_streamer.hpp" +#include "web_video_server/utils.hpp" + namespace web_video_server { diff --git a/src/streamers/vp8_streamer.cpp b/src/streamers/vp8_streamer.cpp index 260e88d..df721f7 100644 --- a/src/streamers/vp8_streamer.cpp +++ b/src/streamers/vp8_streamer.cpp @@ -30,6 +30,23 @@ #include "web_video_server/streamers/vp8_streamer.hpp" +extern "C" +{ +#include +#include +} + +#include +#include +#include + +#include "async_web_server_cpp/http_connection.hpp" +#include "async_web_server_cpp/http_request.hpp" +#include "rclcpp/node.hpp" + +#include "web_video_server/image_streamer.hpp" +#include "web_video_server/streamers/libav_streamer.hpp" + namespace web_video_server { diff --git a/src/streamers/vp9_streamer.cpp b/src/streamers/vp9_streamer.cpp index 765dcef..32f3f63 100644 --- a/src/streamers/vp9_streamer.cpp +++ b/src/streamers/vp9_streamer.cpp @@ -29,6 +29,22 @@ #include "web_video_server/streamers/vp9_streamer.hpp" +extern "C" +{ +#include +#include +} + +#include +#include + +#include "async_web_server_cpp/http_connection.hpp" +#include "async_web_server_cpp/http_request.hpp" +#include "rclcpp/node.hpp" + +#include "web_video_server/image_streamer.hpp" +#include "web_video_server/streamers/libav_streamer.hpp" + namespace web_video_server { diff --git a/src/utils.cpp b/src/utils.cpp index f848e51..d0fe254 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -29,6 +29,12 @@ #include "web_video_server/utils.hpp" +#include +#include + +#include "rmw/qos_profiles.h" +#include "rmw/types.h" + namespace web_video_server { diff --git a/src/web_video_server.cpp b/src/web_video_server.cpp index aa9f226..3e9f586 100644 --- a/src/web_video_server.cpp +++ b/src/web_video_server.cpp @@ -30,18 +30,31 @@ #include "web_video_server/web_video_server.hpp" +#include #include +#include +#include +#include +#include +#include +#include #include #include -#include +#include +#include +#include +#include "async_web_server_cpp/http_connection.hpp" +#include "async_web_server_cpp/http_request.hpp" #include "async_web_server_cpp/http_reply.hpp" -#include "rclcpp/rclcpp.hpp" -#include "sensor_msgs/image_encodings.hpp" +#include "async_web_server_cpp/http_server.hpp" +#include "rclcpp/node.hpp" +#include "rclcpp/node_options.hpp" +#include "rclcpp/logging.hpp" +#include "web_video_server/image_streamer.hpp" #include "web_video_server/streamers/ros_compressed_streamer.hpp" -#include "web_video_server/streamers/image_transport_streamer.hpp" #include "web_video_server/streamers/jpeg_streamers.hpp" #include "web_video_server/streamers/png_streamers.hpp" #include "web_video_server/streamers/vp8_streamer.hpp" diff --git a/src/web_video_server_node.cpp b/src/web_video_server_node.cpp index 83d1c49..473905e 100644 --- a/src/web_video_server_node.cpp +++ b/src/web_video_server_node.cpp @@ -27,6 +27,9 @@ // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. + +#include + #include "rclcpp/rclcpp.hpp" #include "web_video_server/web_video_server.hpp" From 26c13221ad70e40d27147b13891f0fda97f2f35f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82a=C5=BCej=20Sowa?= Date: Thu, 9 Oct 2025 18:32:19 +0200 Subject: [PATCH 03/33] Clean up package.xml depend tags --- package.xml | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/package.xml b/package.xml index 32596b9..e0c6ecc 100644 --- a/package.xml +++ b/package.xml @@ -18,23 +18,19 @@ ament_cmake_ros pkg-config - rclcpp + + async_web_server_cpp + boost + cv_bridge + ffmpeg + image_transport + rclcpp + rmw + sensor_msgs + + rclcpp_components - cv_bridge - image_transport - async_web_server_cpp - ffmpeg - sensor_msgs - boost - - rclcpp rclcpp_components - cv_bridge - image_transport - async_web_server_cpp - ffmpeg - sensor_msgs - boost ament_lint_auto ament_cmake_copyright From 17ea6f6953274fff578013a1b3d5a37aa1164819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82a=C5=BCej=20Sowa?= Date: Fri, 10 Oct 2025 03:13:15 +0000 Subject: [PATCH 04/33] Load streamer plugins using pluginlib --- CMakeLists.txt | 73 ++++++++++++++----- ...e_streamer.hpp => base_image_streamer.hpp} | 15 ++-- ....hpp => base_image_transport_streamer.hpp} | 8 +- ...v_streamer.hpp => base_libav_streamer.hpp} | 30 ++------ .../streamers/h264_streamer.hpp | 16 ++-- .../streamers/jpeg_streamers.hpp | 20 ++--- .../streamers/png_streamers.hpp | 20 ++--- .../streamers/ros_compressed_streamer.hpp | 16 ++-- .../streamers/vp8_streamer.hpp | 16 ++-- .../streamers/vp9_streamer.hpp | 16 ++-- include/web_video_server/web_video_server.hpp | 8 +- package.xml | 3 + plugins.xml | 33 +++++++++ ...e_streamer.cpp => base_image_streamer.cpp} | 16 +++- ....cpp => base_image_transport_streamer.cpp} | 22 +++--- ...v_streamer.cpp => base_libav_streamer.cpp} | 40 +++------- src/streamers/h264_streamer.cpp | 23 +++--- src/streamers/jpeg_streamers.cpp | 27 +++---- src/streamers/png_streamers.cpp | 33 ++++----- src/streamers/ros_compressed_streamer.cpp | 26 +++---- src/streamers/vp8_streamer.cpp | 23 +++--- src/streamers/vp9_streamer.cpp | 23 +++--- src/web_video_server.cpp | 62 ++++++++-------- 23 files changed, 309 insertions(+), 260 deletions(-) rename include/web_video_server/{image_streamer.hpp => base_image_streamer.hpp} (92%) rename include/web_video_server/{streamers/image_transport_streamer.hpp => base_image_transport_streamer.hpp} (94%) rename include/web_video_server/{streamers/libav_streamer.hpp => base_libav_streamer.hpp} (79%) create mode 100644 plugins.xml rename src/{image_streamer.cpp => base_image_streamer.cpp} (85%) rename src/{streamers/image_transport_streamer.cpp => base_image_transport_streamer.cpp} (90%) rename src/{streamers/libav_streamer.cpp => base_libav_streamer.cpp} (91%) diff --git a/CMakeLists.txt b/CMakeLists.txt index bf06404..c8904b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,7 @@ find_package(ament_cmake_ros REQUIRED) find_package(async_web_server_cpp REQUIRED) find_package(cv_bridge REQUIRED) find_package(image_transport REQUIRED) +find_package(pluginlib REQUIRED) find_package(rclcpp REQUIRED) find_package(rclcpp_components REQUIRED) find_package(sensor_msgs REQUIRED) @@ -39,37 +40,67 @@ if(${cv_bridge_VERSION} VERSION_LESS "3.3.0") add_compile_definitions(CV_BRIDGE_USES_OLD_HEADERS) endif() -## Specify additional locations of header files -include_directories(include - ${avcodec_INCLUDE_DIRS} - ${avformat_INCLUDE_DIRS} - ${avutil_INCLUDE_DIRS} - ${swscale_INCLUDE_DIRS} -) - ## Declare a cpp library add_library(${PROJECT_NAME} SHARED src/web_video_server.cpp - src/image_streamer.cpp + src/base_image_streamer.cpp + src/base_image_transport_streamer.cpp + src/base_libav_streamer.cpp src/multipart_stream.cpp src/utils.cpp - src/streamers/image_transport_streamer.cpp - src/streamers/libav_streamer.cpp +) + +target_include_directories(${PROJECT_NAME} + PUBLIC + "$" + "$" + ${avcodec_INCLUDE_DIRS} + ${avformat_INCLUDE_DIRS} + ${avutil_INCLUDE_DIRS} + ${swscale_INCLUDE_DIRS} +) + +## Specify libraries to link a library or executable target against +target_link_libraries(${PROJECT_NAME} + PUBLIC + async_web_server_cpp::async_web_server_cpp + image_transport::image_transport + pluginlib::pluginlib + rclcpp::rclcpp + ${sensor_msgs_TARGETS} + Boost::boost + ${OpenCV_LIBS} + ${avcodec_LIBRARIES} + ${avformat_LIBRARIES} + ${avutil_LIBRARIES} + ${swscale_LIBRARIES} + PRIVATE + cv_bridge::cv_bridge + rclcpp_components::component +) + +add_library(${PROJECT_NAME}_streamers SHARED src/streamers/h264_streamer.cpp - src/streamers/vp8_streamer.cpp - src/streamers/vp9_streamer.cpp - src/streamers/ros_compressed_streamer.cpp src/streamers/jpeg_streamers.cpp src/streamers/png_streamers.cpp + src/streamers/ros_compressed_streamer.cpp + src/streamers/vp8_streamer.cpp + src/streamers/vp9_streamer.cpp ) -## Specify libraries to link a library or executable target against -target_link_libraries(${PROJECT_NAME} +target_include_directories(${PROJECT_NAME}_streamers + PUBLIC + "$" + "$" +) + +target_link_libraries(${PROJECT_NAME}_streamers + ${PROJECT_NAME} async_web_server_cpp::async_web_server_cpp cv_bridge::cv_bridge image_transport::image_transport + pluginlib::pluginlib rclcpp::rclcpp - rclcpp_components::component ${sensor_msgs_TARGETS} Boost::boost Boost::system @@ -91,11 +122,12 @@ target_link_libraries(${PROJECT_NAME}_node rclcpp_components_register_nodes(${PROJECT_NAME} "web_video_server::WebVideoServer") +pluginlib_export_plugin_description_file(web_video_server plugins.xml) + ############# ## Install ## ############# -## Mark executables and/or libraries for installation install( DIRECTORY include/ DESTINATION include/${PROJECT_NAME} @@ -116,6 +148,11 @@ install( DESTINATION lib/${PROJECT_NAME} ) +install( + TARGETS ${PROJECT_NAME}_streamers + LIBRARY DESTINATION lib +) + ########### ## Tests ## ########### diff --git a/include/web_video_server/image_streamer.hpp b/include/web_video_server/base_image_streamer.hpp similarity index 92% rename from include/web_video_server/image_streamer.hpp rename to include/web_video_server/base_image_streamer.hpp index ec6516d..8a4946e 100644 --- a/include/web_video_server/image_streamer.hpp +++ b/include/web_video_server/base_image_streamer.hpp @@ -42,16 +42,16 @@ namespace web_video_server { -class ImageStreamer +class BaseImageStreamer { public: - ImageStreamer( + BaseImageStreamer( const async_web_server_cpp::HttpRequest & request, async_web_server_cpp::HttpConnectionPtr connection, rclcpp::Node::SharedPtr node); virtual void start() = 0; - virtual ~ImageStreamer(); + virtual ~BaseImageStreamer(); bool isInactive() { @@ -73,19 +73,20 @@ class ImageStreamer async_web_server_cpp::HttpRequest request_; rclcpp::Node::SharedPtr node_; bool inactive_; - image_transport::Subscriber image_sub_; std::string topic_; }; -class ImageStreamerType +class BaseImageStreamerFactory { public: - virtual std::shared_ptr create_streamer( + virtual std::string get_type() = 0; + + virtual std::shared_ptr create_streamer( const async_web_server_cpp::HttpRequest & request, async_web_server_cpp::HttpConnectionPtr connection, rclcpp::Node::SharedPtr node) = 0; - virtual std::string create_viewer(const async_web_server_cpp::HttpRequest & request) = 0; + virtual std::string create_viewer(const async_web_server_cpp::HttpRequest & request); }; } // namespace web_video_server diff --git a/include/web_video_server/streamers/image_transport_streamer.hpp b/include/web_video_server/base_image_transport_streamer.hpp similarity index 94% rename from include/web_video_server/streamers/image_transport_streamer.hpp rename to include/web_video_server/base_image_transport_streamer.hpp index bb01c9b..3b667eb 100644 --- a/include/web_video_server/streamers/image_transport_streamer.hpp +++ b/include/web_video_server/base_image_transport_streamer.hpp @@ -42,19 +42,19 @@ #include "rclcpp/node.hpp" #include "sensor_msgs/msg/image.hpp" -#include "web_video_server/image_streamer.hpp" +#include "web_video_server/base_image_streamer.hpp" namespace web_video_server { -class ImageTransportImageStreamer : public ImageStreamer +class BaseImageTransportStreamer : public BaseImageStreamer { public: - ImageTransportImageStreamer( + BaseImageTransportStreamer( const async_web_server_cpp::HttpRequest & request, async_web_server_cpp::HttpConnectionPtr connection, rclcpp::Node::SharedPtr node); - virtual ~ImageTransportImageStreamer(); + virtual ~BaseImageTransportStreamer(); virtual void start(); diff --git a/include/web_video_server/streamers/libav_streamer.hpp b/include/web_video_server/base_libav_streamer.hpp similarity index 79% rename from include/web_video_server/streamers/libav_streamer.hpp rename to include/web_video_server/base_libav_streamer.hpp index 966571f..2bcb05e 100644 --- a/include/web_video_server/streamers/libav_streamer.hpp +++ b/include/web_video_server/base_libav_streamer.hpp @@ -52,25 +52,25 @@ extern "C" #include "async_web_server_cpp/http_request.hpp" #include "rclcpp/node.hpp" -#include "web_video_server/image_streamer.hpp" -#include "web_video_server/streamers/image_transport_streamer.hpp" +#include "web_video_server/base_image_streamer.hpp" +#include "web_video_server/base_image_transport_streamer.hpp" namespace web_video_server { -class LibavStreamer : public ImageTransportImageStreamer +class BaseLibavStreamer : public BaseImageTransportStreamer { public: - LibavStreamer( + BaseLibavStreamer( const async_web_server_cpp::HttpRequest & request, async_web_server_cpp::HttpConnectionPtr connection, rclcpp::Node::SharedPtr node, const std::string & format_name, const std::string & codec_name, const std::string & content_type); - ~LibavStreamer(); + ~BaseLibavStreamer(); protected: - virtual void initializeEncoder(); + virtual void initializeEncoder() = 0; virtual void sendImage(const cv::Mat &, const std::chrono::steady_clock::time_point & time); virtual void initialize(const cv::Mat &); AVFormatContext * format_context_; @@ -98,24 +98,10 @@ class LibavStreamer : public ImageTransportImageStreamer uint8_t * io_buffer_; // custom IO buffer }; -class LibavStreamerType : public ImageStreamerType +class BaseLibavStreamerFactory : public BaseImageStreamerFactory { public: - LibavStreamerType( - const std::string & format_name, const std::string & codec_name, - const std::string & content_type); - - std::shared_ptr create_streamer( - const async_web_server_cpp::HttpRequest & request, - async_web_server_cpp::HttpConnectionPtr connection, - rclcpp::Node::SharedPtr node); - - std::string create_viewer(const async_web_server_cpp::HttpRequest & request); - -private: - const std::string format_name_; - const std::string codec_name_; - const std::string content_type_; + virtual std::string create_viewer(const async_web_server_cpp::HttpRequest & request); }; } // namespace web_video_server diff --git a/include/web_video_server/streamers/h264_streamer.hpp b/include/web_video_server/streamers/h264_streamer.hpp index a625750..b9a8ab3 100644 --- a/include/web_video_server/streamers/h264_streamer.hpp +++ b/include/web_video_server/streamers/h264_streamer.hpp @@ -36,13 +36,13 @@ #include "async_web_server_cpp/http_connection.hpp" #include "rclcpp/node.hpp" -#include "web_video_server/image_streamer.hpp" -#include "web_video_server/streamers/libav_streamer.hpp" +#include "web_video_server/base_image_streamer.hpp" +#include "web_video_server/base_libav_streamer.hpp" -namespace web_video_server +namespace web_video_server_streamers { -class H264Streamer : public LibavStreamer +class H264Streamer : public web_video_server::BaseLibavStreamer { public: H264Streamer( @@ -56,14 +56,14 @@ class H264Streamer : public LibavStreamer std::string preset_; }; -class H264StreamerType : public LibavStreamerType +class H264StreamerFactory : public web_video_server::BaseLibavStreamerFactory { public: - H264StreamerType(); - std::shared_ptr create_streamer( + std::string get_type() {return "h264";} + std::shared_ptr create_streamer( const async_web_server_cpp::HttpRequest & request, async_web_server_cpp::HttpConnectionPtr connection, rclcpp::Node::SharedPtr node); }; -} // namespace web_video_server +} // namespace web_video_server_streamers diff --git a/include/web_video_server/streamers/jpeg_streamers.hpp b/include/web_video_server/streamers/jpeg_streamers.hpp index cf7ee76..a53123c 100644 --- a/include/web_video_server/streamers/jpeg_streamers.hpp +++ b/include/web_video_server/streamers/jpeg_streamers.hpp @@ -40,14 +40,14 @@ #include "async_web_server_cpp/http_connection.hpp" #include "rclcpp/node.hpp" -#include "web_video_server/image_streamer.hpp" +#include "web_video_server/base_image_streamer.hpp" +#include "web_video_server/base_image_transport_streamer.hpp" #include "web_video_server/multipart_stream.hpp" -#include "web_video_server/streamers/image_transport_streamer.hpp" -namespace web_video_server +namespace web_video_server_streamers { -class MjpegStreamer : public ImageTransportImageStreamer +class MjpegStreamer : public web_video_server::BaseImageTransportStreamer { public: MjpegStreamer( @@ -60,21 +60,21 @@ class MjpegStreamer : public ImageTransportImageStreamer virtual void sendImage(const cv::Mat &, const std::chrono::steady_clock::time_point & time); private: - MultipartStream stream_; + web_video_server::MultipartStream stream_; int quality_; }; -class MjpegStreamerType : public ImageStreamerType +class MjpegStreamerFactory : public web_video_server::BaseImageStreamerFactory { public: - std::shared_ptr create_streamer( + std::string get_type() {return "mjpeg";} + std::shared_ptr create_streamer( const async_web_server_cpp::HttpRequest & request, async_web_server_cpp::HttpConnectionPtr connection, rclcpp::Node::SharedPtr node); - std::string create_viewer(const async_web_server_cpp::HttpRequest & request); }; -class JpegSnapshotStreamer : public ImageTransportImageStreamer +class JpegSnapshotStreamer : public web_video_server::BaseImageTransportStreamer { public: JpegSnapshotStreamer( @@ -89,4 +89,4 @@ class JpegSnapshotStreamer : public ImageTransportImageStreamer int quality_; }; -} // namespace web_video_server +} // namespace web_video_server_streamers diff --git a/include/web_video_server/streamers/png_streamers.hpp b/include/web_video_server/streamers/png_streamers.hpp index 7438d44..b002282 100644 --- a/include/web_video_server/streamers/png_streamers.hpp +++ b/include/web_video_server/streamers/png_streamers.hpp @@ -40,14 +40,14 @@ #include "rclcpp/node.hpp" #include "sensor_msgs/msg/image.hpp" -#include "web_video_server/image_streamer.hpp" +#include "web_video_server/base_image_streamer.hpp" +#include "web_video_server/base_image_transport_streamer.hpp" #include "web_video_server/multipart_stream.hpp" -#include "web_video_server/streamers/image_transport_streamer.hpp" -namespace web_video_server +namespace web_video_server_streamers { -class PngStreamer : public ImageTransportImageStreamer +class PngStreamer : public web_video_server::BaseImageTransportStreamer { public: PngStreamer( @@ -61,21 +61,21 @@ class PngStreamer : public ImageTransportImageStreamer virtual cv::Mat decodeImage(const sensor_msgs::msg::Image::ConstSharedPtr & msg); private: - MultipartStream stream_; + web_video_server::MultipartStream stream_; int quality_; }; -class PngStreamerType : public ImageStreamerType +class PngStreamerFactory : public web_video_server::BaseImageStreamerFactory { public: - std::shared_ptr create_streamer( + std::string get_type() {return "png";} + std::shared_ptr create_streamer( const async_web_server_cpp::HttpRequest & request, async_web_server_cpp::HttpConnectionPtr connection, rclcpp::Node::SharedPtr node); - std::string create_viewer(const async_web_server_cpp::HttpRequest & request); }; -class PngSnapshotStreamer : public ImageTransportImageStreamer +class PngSnapshotStreamer : public web_video_server::BaseImageTransportStreamer { public: PngSnapshotStreamer( @@ -91,4 +91,4 @@ class PngSnapshotStreamer : public ImageTransportImageStreamer int quality_; }; -} // namespace web_video_server +} // namespace web_video_server_streamers diff --git a/include/web_video_server/streamers/ros_compressed_streamer.hpp b/include/web_video_server/streamers/ros_compressed_streamer.hpp index 7ce7ba5..c8429cf 100644 --- a/include/web_video_server/streamers/ros_compressed_streamer.hpp +++ b/include/web_video_server/streamers/ros_compressed_streamer.hpp @@ -41,13 +41,13 @@ #include "rclcpp/subscription.hpp" #include "sensor_msgs/msg/compressed_image.hpp" -#include "web_video_server/image_streamer.hpp" +#include "web_video_server/base_image_streamer.hpp" #include "web_video_server/multipart_stream.hpp" -namespace web_video_server +namespace web_video_server_streamers { -class RosCompressedStreamer : public ImageStreamer +class RosCompressedStreamer : public web_video_server::BaseImageStreamer { public: RosCompressedStreamer( @@ -65,7 +65,7 @@ class RosCompressedStreamer : public ImageStreamer private: void imageCallback(const sensor_msgs::msg::CompressedImage::ConstSharedPtr msg); - MultipartStream stream_; + web_video_server::MultipartStream stream_; rclcpp::Subscription::SharedPtr image_sub_; std::chrono::steady_clock::time_point last_frame_; sensor_msgs::msg::CompressedImage::ConstSharedPtr last_msg; @@ -73,14 +73,14 @@ class RosCompressedStreamer : public ImageStreamer std::string qos_profile_name_; }; -class RosCompressedStreamerType : public ImageStreamerType +class RosCompressedStreamerFactory : public web_video_server::BaseImageStreamerFactory { public: - std::shared_ptr create_streamer( + std::string get_type() {return "ros_compressed";} + std::shared_ptr create_streamer( const async_web_server_cpp::HttpRequest & request, async_web_server_cpp::HttpConnectionPtr connection, rclcpp::Node::SharedPtr node); - std::string create_viewer(const async_web_server_cpp::HttpRequest & request); }; -} // namespace web_video_server +} // namespace web_video_server_streamers diff --git a/include/web_video_server/streamers/vp8_streamer.hpp b/include/web_video_server/streamers/vp8_streamer.hpp index 8e148c3..e689c0a 100644 --- a/include/web_video_server/streamers/vp8_streamer.hpp +++ b/include/web_video_server/streamers/vp8_streamer.hpp @@ -37,13 +37,13 @@ #include "async_web_server_cpp/http_request.hpp" #include "rclcpp/node.hpp" -#include "web_video_server/image_streamer.hpp" -#include "web_video_server/streamers/libav_streamer.hpp" +#include "web_video_server/base_image_streamer.hpp" +#include "web_video_server/base_libav_streamer.hpp" -namespace web_video_server +namespace web_video_server_streamers { -class Vp8Streamer : public LibavStreamer +class Vp8Streamer : public web_video_server::BaseLibavStreamer { public: Vp8Streamer( @@ -59,14 +59,14 @@ class Vp8Streamer : public LibavStreamer std::string quality_; }; -class Vp8StreamerType : public LibavStreamerType +class Vp8StreamerFactory : public web_video_server::BaseLibavStreamerFactory { public: - Vp8StreamerType(); - std::shared_ptr create_streamer( + std::string get_type() {return "vp8";} + std::shared_ptr create_streamer( const async_web_server_cpp::HttpRequest & request, async_web_server_cpp::HttpConnectionPtr connection, rclcpp::Node::SharedPtr node); }; -} // namespace web_video_server +} // namespace web_video_server_streamers diff --git a/include/web_video_server/streamers/vp9_streamer.hpp b/include/web_video_server/streamers/vp9_streamer.hpp index 348e1b0..7341901 100644 --- a/include/web_video_server/streamers/vp9_streamer.hpp +++ b/include/web_video_server/streamers/vp9_streamer.hpp @@ -35,13 +35,13 @@ #include "async_web_server_cpp/http_request.hpp" #include "rclcpp/node.hpp" -#include "web_video_server/image_streamer.hpp" -#include "web_video_server/streamers/libav_streamer.hpp" +#include "web_video_server/base_image_streamer.hpp" +#include "web_video_server/base_libav_streamer.hpp" -namespace web_video_server +namespace web_video_server_streamers { -class Vp9Streamer : public LibavStreamer +class Vp9Streamer : public web_video_server::BaseLibavStreamer { public: Vp9Streamer( @@ -54,14 +54,14 @@ class Vp9Streamer : public LibavStreamer virtual void initializeEncoder(); }; -class Vp9StreamerType : public LibavStreamerType +class Vp9StreamerFactory : public web_video_server::BaseLibavStreamerFactory { public: - Vp9StreamerType(); - std::shared_ptr create_streamer( + std::string get_type() {return "vp9";} + std::shared_ptr create_streamer( const async_web_server_cpp::HttpRequest & request, async_web_server_cpp::HttpConnectionPtr connection, rclcpp::Node::SharedPtr node); }; -} // namespace web_video_server +} // namespace web_video_server_streamers diff --git a/include/web_video_server/web_video_server.hpp b/include/web_video_server/web_video_server.hpp index 468f12c..62ef9d5 100644 --- a/include/web_video_server/web_video_server.hpp +++ b/include/web_video_server/web_video_server.hpp @@ -41,11 +41,12 @@ #include "async_web_server_cpp/http_request.hpp" #include "async_web_server_cpp/http_request_handler.hpp" #include "async_web_server_cpp/http_server.hpp" +#include "pluginlib/class_loader.hpp" #include "rclcpp/node.hpp" #include "rclcpp/node_options.hpp" #include "rclcpp/timer.hpp" -#include "web_video_server/image_streamer.hpp" +#include "web_video_server/base_image_streamer.hpp" namespace web_video_server { @@ -110,8 +111,9 @@ class WebVideoServer : public rclcpp::Node std::shared_ptr server_; async_web_server_cpp::HttpRequestHandlerGroup handler_group_; - std::vector> image_subscribers_; - std::map> stream_types_; + std::vector> streamers_; + pluginlib::ClassLoader streamer_factory_loader_; + std::map> streamer_factories_; std::mutex subscriber_mutex_; }; diff --git a/package.xml b/package.xml index e0c6ecc..2bc3495 100644 --- a/package.xml +++ b/package.xml @@ -24,11 +24,14 @@ cv_bridge ffmpeg image_transport + libopencv-dev rclcpp rmw sensor_msgs + pluginlib + pluginlib rclcpp_components rclcpp_components diff --git a/plugins.xml b/plugins.xml new file mode 100644 index 0000000..b9d42fa --- /dev/null +++ b/plugins.xml @@ -0,0 +1,33 @@ + + + Streams images as multipart/x-mixed-replace (MJPEG). + + + Streams images as multipart/x-mixed-replace (PNG). + + + Streams images published using compressed_image_transport as + multipart/x-mixed-replace. + + + Streams images as H.264 encoded video in an MP4 container. + + + Streams images as VP8 encoded video in a WebM container. + + + Streams images as VP9 encoded video in a WebM container. + + \ No newline at end of file diff --git a/src/image_streamer.cpp b/src/base_image_streamer.cpp similarity index 85% rename from src/image_streamer.cpp rename to src/base_image_streamer.cpp index 7a3619d..19a114a 100644 --- a/src/image_streamer.cpp +++ b/src/base_image_streamer.cpp @@ -28,7 +28,7 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. -#include "web_video_server/image_streamer.hpp" +#include "web_video_server/base_image_streamer.hpp" #include "rclcpp/node.hpp" @@ -38,7 +38,7 @@ namespace web_video_server { -ImageStreamer::ImageStreamer( +BaseImageStreamer::BaseImageStreamer( const async_web_server_cpp::HttpRequest & request, async_web_server_cpp::HttpConnectionPtr connection, rclcpp::Node::SharedPtr node) : connection_(connection), request_(request), node_(node), inactive_(false) @@ -46,8 +46,18 @@ ImageStreamer::ImageStreamer( topic_ = request.get_query_param_value_or_default("topic", ""); } -ImageStreamer::~ImageStreamer() +BaseImageStreamer::~BaseImageStreamer() { } +std::string BaseImageStreamerFactory::create_viewer( + const async_web_server_cpp::HttpRequest & request) +{ + std::stringstream ss; + ss << ""; + return ss.str(); +} + } // namespace web_video_server diff --git a/src/streamers/image_transport_streamer.cpp b/src/base_image_transport_streamer.cpp similarity index 90% rename from src/streamers/image_transport_streamer.cpp rename to src/base_image_transport_streamer.cpp index f7ab8e3..4148a0d 100644 --- a/src/streamers/image_transport_streamer.cpp +++ b/src/base_image_transport_streamer.cpp @@ -28,7 +28,7 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. -#include "web_video_server/streamers/image_transport_streamer.hpp" +#include "web_video_server/base_image_transport_streamer.hpp" #include #include @@ -56,16 +56,16 @@ #include "rmw/qos_profiles.h" #include "sensor_msgs/msg/image.hpp" -#include "web_video_server/image_streamer.hpp" +#include "web_video_server/base_image_streamer.hpp" #include "web_video_server/utils.hpp" namespace web_video_server { -ImageTransportImageStreamer::ImageTransportImageStreamer( +BaseImageTransportStreamer::BaseImageTransportStreamer( const async_web_server_cpp::HttpRequest & request, async_web_server_cpp::HttpConnectionPtr connection, rclcpp::Node::SharedPtr node) -: ImageStreamer(request, connection, node), it_(node), initialized_(false) +: BaseImageStreamer(request, connection, node), it_(node), initialized_(false) { output_width_ = request.get_query_param_value_or_default("width", -1); output_height_ = request.get_query_param_value_or_default("height", -1); @@ -74,11 +74,11 @@ ImageTransportImageStreamer::ImageTransportImageStreamer( qos_profile_name_ = request.get_query_param_value_or_default("qos_profile", "default"); } -ImageTransportImageStreamer::~ImageTransportImageStreamer() +BaseImageTransportStreamer::~BaseImageTransportStreamer() { } -void ImageTransportImageStreamer::start() +void BaseImageTransportStreamer::start() { image_transport::TransportHints hints(node_.get(), default_transport_); auto tnat = node_->get_topic_names_and_types(); @@ -111,15 +111,15 @@ void ImageTransportImageStreamer::start() // Create subscriber image_sub_ = image_transport::create_subscription( node_.get(), topic_, - std::bind(&ImageTransportImageStreamer::imageCallback, this, std::placeholders::_1), + std::bind(&BaseImageTransportStreamer::imageCallback, this, std::placeholders::_1), default_transport_, qos_profile.value()); } -void ImageTransportImageStreamer::initialize(const cv::Mat &) +void BaseImageTransportStreamer::initialize(const cv::Mat &) { } -void ImageTransportImageStreamer::restreamFrame(std::chrono::duration max_age) +void BaseImageTransportStreamer::restreamFrame(std::chrono::duration max_age) { if (inactive_ || !initialized_) { return; @@ -148,7 +148,7 @@ void ImageTransportImageStreamer::restreamFrame(std::chrono::duration ma } } -void ImageTransportImageStreamer::imageCallback(const sensor_msgs::msg::Image::ConstSharedPtr & msg) +void BaseImageTransportStreamer::imageCallback(const sensor_msgs::msg::Image::ConstSharedPtr & msg) { if (inactive_) { return; @@ -218,7 +218,7 @@ void ImageTransportImageStreamer::imageCallback(const sensor_msgs::msg::Image::C } } -cv::Mat ImageTransportImageStreamer::decodeImage( +cv::Mat BaseImageTransportStreamer::decodeImage( const sensor_msgs::msg::Image::ConstSharedPtr & msg) { if (msg->encoding.find("F") != std::string::npos) { diff --git a/src/streamers/libav_streamer.cpp b/src/base_libav_streamer.cpp similarity index 91% rename from src/streamers/libav_streamer.cpp rename to src/base_libav_streamer.cpp index dcbb0c9..5b8efcc 100644 --- a/src/streamers/libav_streamer.cpp +++ b/src/base_libav_streamer.cpp @@ -28,7 +28,7 @@ // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. -#include "web_video_server/streamers/libav_streamer.hpp" +#include "web_video_server/base_libav_streamer.hpp" extern "C" { @@ -66,8 +66,8 @@ extern "C" #include "rclcpp/node.hpp" #include "rclcpp/logging.hpp" -#include "web_video_server/image_streamer.hpp" -#include "web_video_server/streamers/image_transport_streamer.hpp" +#include "web_video_server/base_image_streamer.hpp" +#include "web_video_server/base_image_transport_streamer.hpp" // https://stackoverflow.com/questions/46884682/error-in-building-opencv-with-ffmpeg #define AV_CODEC_FLAG_GLOBAL_HEADER (1 << 22) @@ -76,12 +76,12 @@ extern "C" namespace web_video_server { -LibavStreamer::LibavStreamer( +BaseLibavStreamer::BaseLibavStreamer( const async_web_server_cpp::HttpRequest & request, async_web_server_cpp::HttpConnectionPtr connection, rclcpp::Node::SharedPtr node, const std::string & format_name, const std::string & codec_name, const std::string & content_type) -: ImageTransportImageStreamer(request, connection, node), format_context_(0), codec_(0), +: BaseImageTransportStreamer(request, connection, node), format_context_(0), codec_(0), codec_context_(0), video_stream_(0), opt_(0), frame_(0), sws_context_(0), first_image_received_(false), first_image_time_(), format_name_(format_name), codec_name_(codec_name), content_type_(content_type), io_buffer_(0) @@ -92,7 +92,7 @@ LibavStreamer::LibavStreamer( gop_ = request.get_query_param_value_or_default("gop", 25); } -LibavStreamer::~LibavStreamer() +BaseLibavStreamer::~BaseLibavStreamer() { if (codec_context_) { avcodec_free_context(&codec_context_); @@ -129,7 +129,7 @@ static int dispatch_output_packet(void * opaque, const uint8_t * buffer, int buf return 0; } -void LibavStreamer::initialize(const cv::Mat & /* img */) +void BaseLibavStreamer::initialize(const cv::Mat & /* img */) { // Load format format_context_ = avformat_alloc_context(); @@ -253,11 +253,7 @@ void LibavStreamer::initialize(const cv::Mat & /* img */) } } -void LibavStreamer::initializeEncoder() -{ -} - -void LibavStreamer::sendImage( +void BaseLibavStreamer::sendImage( const cv::Mat & img, const std::chrono::steady_clock::time_point & time) { @@ -339,24 +335,8 @@ void LibavStreamer::sendImage( av_packet_unref(pkt); } -LibavStreamerType::LibavStreamerType( - const std::string & format_name, const std::string & codec_name, - const std::string & content_type) -: format_name_(format_name), codec_name_(codec_name), content_type_(content_type) -{ -} - -std::shared_ptr LibavStreamerType::create_streamer( - const async_web_server_cpp::HttpRequest & request, - async_web_server_cpp::HttpConnectionPtr connection, - rclcpp::Node::SharedPtr node) -{ - return std::make_shared( - request, connection, node, format_name_, codec_name_, - content_type_); -} - -std::string LibavStreamerType::create_viewer(const async_web_server_cpp::HttpRequest & request) +std::string BaseLibavStreamerFactory::create_viewer( + const async_web_server_cpp::HttpRequest & request) { std::stringstream ss; ss << "