From b19e1ee6743e60bbed55abcb435fa64b86270348 Mon Sep 17 00:00:00 2001 From: "Alejandro G. Castro" Date: Wed, 10 Aug 2016 12:43:43 +0200 Subject: [PATCH 01/26] OwrTransportAgent: Add bundle policy support to the transport agent. --- owr/owr_types.c | 18 ++++++++++++++++++ owr/owr_types.h | 9 +++++++++ tests/test_client.c | 2 +- tests/test_data_channel.c | 4 ++-- tests/test_send_receive.c | 4 ++-- transport/owr_transport_agent.c | 18 ++++++++++++++++-- transport/owr_transport_agent.h | 3 ++- 7 files changed, 50 insertions(+), 8 deletions(-) diff --git a/owr/owr_types.c b/owr/owr_types.c index 082291bb..65212d1b 100644 --- a/owr/owr_types.c +++ b/owr/owr_types.c @@ -107,3 +107,21 @@ if (g_once_init_enter((gsize *)&id)) { return id; } + +GType owr_bundle_policy_type_get_type(void) +{ + static const GEnumValue types[] = { + {OWR_BUNDLE_POLICY_TYPE_BALANCED, "Pick two tracks to send, audio and video", "balanced"}, + {OWR_BUNDLE_POLICY_TYPE_MAX_COMPAT, "Separate each track into its own connection", "max_compat"}, + {OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE, "Pick one media track to negotiate and send just that one", "max_bundle"}, + {0, NULL, NULL} + }; +static volatile GType id = 0; + +if (g_once_init_enter((gsize *)&id)) { + GType _id = g_enum_register_static("OwrBundlePolicyTypes", types); + g_once_init_leave((gsize *)&id, _id); +} + +return id; +} diff --git a/owr/owr_types.h b/owr/owr_types.h index 762ca06c..8e61df4e 100644 --- a/owr/owr_types.h +++ b/owr/owr_types.h @@ -62,6 +62,12 @@ typedef enum _OwrAdaptationType { OWR_ADAPTATION_TYPE_SCREAM } OwrAdaptationType; +typedef enum _OwrBundlePolicyType { + OWR_BUNDLE_POLICY_TYPE_BALANCED, + OWR_BUNDLE_POLICY_TYPE_MAX_COMPAT, + OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE +} OwrBundlePolicyType; + #define OWR_TYPE_CODEC_TYPE (owr_codec_type_get_type()) GType owr_codec_type_get_type(void); @@ -74,6 +80,9 @@ GType owr_media_type_get_type(void); #define OWR_TYPE_ADAPTATION_TYPE (owr_adaptation_type_get_type()) GType owr_adaptation_type_get_type(void); +#define OWR_TYPE_BUNDLE_POLICY_TYPE (owr_bundle_policy_type_get_type()) +GType owr_bundle_policy_type_get_type(void); + G_END_DECLS diff --git a/tests/test_client.c b/tests/test_client.c index 520f647e..5db71f89 100644 --- a/tests/test_client.c +++ b/tests/test_client.c @@ -771,7 +771,7 @@ static void send_eventsource_request(const gchar *url) static void got_local_sources(GList *sources, gchar *url) { local_sources = g_list_copy(sources); - transport_agent = owr_transport_agent_new(FALSE); + transport_agent = owr_transport_agent_new(FALSE, OWR_BUNDLE_POLICY_TYPE_BALANCED); owr_transport_agent_add_helper_server(transport_agent, OWR_HELPER_SERVER_TYPE_STUN, "stun.services.mozilla.com", 3478, NULL, NULL); if (url) { diff --git a/tests/test_data_channel.c b/tests/test_data_channel.c index 2656dff3..d9cd4bb0 100644 --- a/tests/test_data_channel.c +++ b/tests/test_data_channel.c @@ -223,14 +223,14 @@ static gboolean setup_transport_agents() { g_print("Setting up transport agents\n"); // LEFT - left_transport_agent = owr_transport_agent_new(FALSE); + left_transport_agent = owr_transport_agent_new(FALSE, OWR_BUNDLE_POLICY_TYPE_BALANCED); g_assert(OWR_IS_TRANSPORT_AGENT(left_transport_agent)); owr_transport_agent_set_local_port_range(left_transport_agent, 5000, 5999); owr_transport_agent_add_local_address(left_transport_agent, "127.0.0.1"); // RIGHT - right_transport_agent = owr_transport_agent_new(TRUE); + right_transport_agent = owr_transport_agent_new(TRUE, OWR_BUNDLE_POLICY_TYPE_BALANCED); g_assert(OWR_IS_TRANSPORT_AGENT(right_transport_agent)); owr_transport_agent_set_local_port_range(right_transport_agent, 5000, 5999); diff --git a/tests/test_send_receive.c b/tests/test_send_receive.c index 35fb0c77..290f0e55 100644 --- a/tests/test_send_receive.c +++ b/tests/test_send_receive.c @@ -412,7 +412,7 @@ int main(int argc, char **argv) owr_bus_add_message_origin(bus, OWR_MESSAGE_ORIGIN(owr_window_registry_get())); - recv_transport_agent = owr_transport_agent_new(FALSE); + recv_transport_agent = owr_transport_agent_new(FALSE, OWR_BUNDLE_POLICY_TYPE_BALANCED); g_assert(OWR_IS_TRANSPORT_AGENT(recv_transport_agent)); owr_bus_add_message_origin(bus, OWR_MESSAGE_ORIGIN(recv_transport_agent)); @@ -423,7 +423,7 @@ int main(int argc, char **argv) owr_transport_agent_add_local_address(recv_transport_agent, local_addr); // SEND - send_transport_agent = owr_transport_agent_new(TRUE); + send_transport_agent = owr_transport_agent_new(TRUE, OWR_BUNDLE_POLICY_TYPE_BALANCED); g_assert(OWR_IS_TRANSPORT_AGENT(send_transport_agent)); owr_bus_add_message_origin(bus, OWR_MESSAGE_ORIGIN(send_transport_agent)); diff --git a/transport/owr_transport_agent.c b/transport/owr_transport_agent.c index cffa2242..a586f1ed 100644 --- a/transport/owr_transport_agent.c +++ b/transport/owr_transport_agent.c @@ -79,11 +79,13 @@ GST_DEBUG_CATEGORY_EXTERN(_owrsession_debug); #define GST_CAT_DEFAULT _owrtransportagent_debug #define DEFAULT_ICE_CONTROLLING_MODE TRUE +#define DEFAULT_BUNDLE_POLICY OWR_BUNDLE_POLICY_TYPE_BALANCED #define GST_RTCP_RTPFB_TYPE_SCREAM 18 enum { PROP_0, PROP_ICE_CONTROLLING_MODE, + PROP_BUNDLE_POLICY, N_PROPERTIES }; @@ -135,6 +137,7 @@ typedef struct { struct _OwrTransportAgentPrivate { NiceAgent *nice_agent; gboolean ice_controlling_mode; + gboolean bundle_policy; GMutex sessions_lock; GHashTable *sessions; @@ -342,6 +345,10 @@ static void owr_transport_agent_class_init(OwrTransportAgentClass *klass) "Ice controlling mode", "Whether the ice agent is in controlling mode", DEFAULT_ICE_CONTROLLING_MODE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + obj_properties[PROP_BUNDLE_POLICY] = g_param_spec_enum("bundle-policy", "bundle-policy" + "Bundle policy of the data streams", "What kind of bundle policy we will use for the streams", + OWR_TYPE_BUNDLE_POLICY_TYPE, DEFAULT_BUNDLE_POLICY, G_PARAM_CONSTRUCT_ONLY | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + gobject_class->set_property = owr_transport_agent_set_property; gobject_class->get_property = owr_transport_agent_get_property; gobject_class->finalize = owr_transport_agent_finalize; @@ -434,6 +441,7 @@ static void owr_transport_agent_init(OwrTransportAgent *transport_agent) transport_agent->priv = priv = OWR_TRANSPORT_AGENT_GET_PRIVATE(transport_agent); priv->ice_controlling_mode = DEFAULT_ICE_CONTROLLING_MODE; + priv->bundle_policy = DEFAULT_BUNDLE_POLICY; priv->agent_id = next_transport_agent_id++; priv->nice_agent = NULL; @@ -527,6 +535,9 @@ static void owr_transport_agent_set_property(GObject *object, guint property_id, case PROP_ICE_CONTROLLING_MODE: priv->ice_controlling_mode = g_value_get_boolean(value); break; + case PROP_BUNDLE_POLICY: + priv->bundle_policy = g_value_get_enum(value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -548,16 +559,19 @@ static void owr_transport_agent_get_property(GObject *object, guint property_id, case PROP_ICE_CONTROLLING_MODE: g_value_set_boolean(value, priv->ice_controlling_mode); break; + case PROP_BUNDLE_POLICY: + g_value_set_enum(value, priv->bundle_policy); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; } } -OwrTransportAgent * owr_transport_agent_new(gboolean ice_controlling_mode) +OwrTransportAgent * owr_transport_agent_new(gboolean ice_controlling_mode, OwrBundlePolicyType bundle_policy_type) { return g_object_new(OWR_TYPE_TRANSPORT_AGENT, "ice-controlling_mode", ice_controlling_mode, - NULL); + "bundle-policy", bundle_policy_type, NULL); } /** diff --git a/transport/owr_transport_agent.h b/transport/owr_transport_agent.h index 950e4b4b..3d23afef 100644 --- a/transport/owr_transport_agent.h +++ b/transport/owr_transport_agent.h @@ -33,6 +33,7 @@ #define __OWR_TRANSPORT_AGENT_H__ #include "owr_session.h" +#include "owr_types.h" #include @@ -69,7 +70,7 @@ struct _OwrTransportAgentClass { GType owr_transport_agent_get_type(void) G_GNUC_CONST; -OwrTransportAgent * owr_transport_agent_new(gboolean ice_controlling_mode); +OwrTransportAgent * owr_transport_agent_new(gboolean ice_controlling_mode, OwrBundlePolicyType bundle_policy_type); void owr_transport_agent_add_helper_server(OwrTransportAgent *transport_agent, OwrHelperServerType type, const gchar *address, guint port, const gchar *username, const gchar *password); void owr_transport_agent_add_local_address(OwrTransportAgent *transport_agent, const gchar *local_address); From a5c3d989a0a5b82591d0e85d0cedb3ba4294ba4c Mon Sep 17 00:00:00 2001 From: "Alejandro G. Castro" Date: Fri, 7 Oct 2016 14:48:24 +0200 Subject: [PATCH 02/26] Add receive ssrcs attributes to the media session. This is required for the proper RTX implementation. --- transport/owr_media_session.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/transport/owr_media_session.c b/transport/owr_media_session.c index adb2bc2a..e024169b 100644 --- a/transport/owr_media_session.c +++ b/transport/owr_media_session.c @@ -66,6 +66,8 @@ struct _OwrMediaSessionPrivate { gchar *incoming_srtp_key; gchar *outgoing_srtp_key; guint send_ssrc; + guint receive_ssrc; + guint receive_rtx_ssrc; gchar *cname; GRWLock rw_lock; OwrPayload *send_payload; @@ -94,6 +96,8 @@ enum { PROP_INCOMING_SRTP_KEY, PROP_OUTGOING_SRTP_KEY, PROP_SEND_SSRC, + PROP_RECEIVE_SSRC, + PROP_RECEIVE_RTX_SSRC, PROP_CNAME, PROP_JITTER_BUFFER_LATENCY, @@ -138,6 +142,14 @@ static void owr_media_session_set_property(GObject *object, guint property_id, c priv->send_ssrc = g_value_get_uint(value); break; + case PROP_RECEIVE_SSRC: + priv->receive_ssrc = g_value_get_uint(value); + break; + + case PROP_RECEIVE_RTX_SSRC: + priv->receive_rtx_ssrc = g_value_get_uint(value); + break; + case PROP_CNAME: priv->cname = g_value_dup_string(value); break; @@ -173,6 +185,14 @@ static void owr_media_session_get_property(GObject *object, guint property_id, G g_value_set_uint(value, priv->send_ssrc); break; + case PROP_RECEIVE_SSRC: + g_value_set_uint(value, priv->receive_ssrc); + break; + + case PROP_RECEIVE_RTX_SSRC: + g_value_set_uint(value, priv->receive_rtx_ssrc); + break; + case PROP_CNAME: g_value_set_string(value, priv->cname); break; @@ -294,6 +314,14 @@ static void owr_media_session_class_init(OwrMediaSessionClass *klass) "The ssrc (to be) used for the outgoing RTP media stream", 0, G_MAXUINT, 0, G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + obj_properties[PROP_RECEIVE_SSRC] = g_param_spec_uint("receive-ssrc", "Receive ssrc", + "The ssrc (to be) used for the incoming RTP media stream", + 0, G_MAXUINT, 0, G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + + obj_properties[PROP_RECEIVE_RTX_SSRC] = g_param_spec_uint("receive-rtx-ssrc", "Receive RTX ssrc", + "The ssrc (to be) used for the incoming RTP RTX media stream", + 0, G_MAXUINT, 0, G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + obj_properties[PROP_CNAME] = g_param_spec_string("cname", "CNAME", "The canonical name identifying this endpoint", NULL, G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); @@ -317,6 +345,8 @@ static void owr_media_session_init(OwrMediaSession *media_session) priv->incoming_srtp_key = NULL; priv->outgoing_srtp_key = NULL; priv->send_ssrc = 0; + priv->receive_ssrc = 0; + priv->receive_rtx_ssrc = 0; priv->cname = NULL; priv->send_payload = NULL; priv->send_source = NULL; From b70779341080149a3c0867146ce058af02ac9b88 Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Wed, 19 Oct 2016 16:11:13 +0200 Subject: [PATCH 03/26] media_source: fix remote source rendering The previous version of the source bin had the capsfilter after the glcolorconvert element, triggering a caps negotiation issue. And without videoscale element the capsfilter is not very useful anyway. So now the video resize is performed before uploading to the GPU. --- owr/owr_media_source.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/owr/owr_media_source.c b/owr/owr_media_source.c index 1580c466..d7a59d95 100644 --- a/owr/owr_media_source.c +++ b/owr/owr_media_source.c @@ -328,16 +328,18 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media gst_structure_remove_field(s, "framerate"); gst_bin_add(GST_BIN(source_bin), videorate); } - g_object_set(capsfilter, "caps", caps, NULL); + g_object_set(capsfilter, "caps", caps, NULL); features = gst_caps_get_features(caps, 0); if (gst_caps_features_contains(features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) { GstElement *glupload; CREATE_ELEMENT_WITH_ID(glupload, "glupload", "source-glupload", source_id); + CREATE_ELEMENT_WITH_ID(videoscale, "gleffects_identity", "source-glcolorscale", source_id); CREATE_ELEMENT_WITH_ID(videoconvert, "glcolorconvert", "source-glcolorconvert", source_id); + gst_bin_add_many(GST_BIN(source_bin), - queue_pre, glupload, videoconvert, capsfilter, queue_post, NULL); + queue_pre, glupload, videoconvert, videoscale, queue_post, NULL); if (videorate) { LINK_ELEMENTS(queue_pre, videorate); @@ -346,6 +348,8 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media LINK_ELEMENTS(queue_pre, glupload); } LINK_ELEMENTS(glupload, videoconvert); + LINK_ELEMENTS(videoconvert, videoscale); + LINK_ELEMENTS(videoscale, queue_post); } else { GstElement *gldownload; @@ -362,9 +366,9 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media } LINK_ELEMENTS(gldownload, videoscale); LINK_ELEMENTS(videoscale, videoconvert); + LINK_ELEMENTS(videoconvert, capsfilter); + LINK_ELEMENTS(capsfilter, queue_post); } - LINK_ELEMENTS(videoconvert, capsfilter); - LINK_ELEMENTS(capsfilter, queue_post); break; } @@ -420,6 +424,7 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media gst_object_unref(source_pipeline); gst_object_unref(tee); + GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(source_bin), GST_DEBUG_GRAPH_SHOW_ALL, GST_OBJECT_NAME(GST_OBJECT(source_bin))); return source_bin; } From 71fa5ac6d46d746313a26a03d511eab07772fadc Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Wed, 19 Oct 2016 16:47:08 +0200 Subject: [PATCH 04/26] transport_agent: new API to explicitely start the pipeline The owr_transport_agent_add_session function is now limited to storing the session in an internal list. The list is processed by a new owr_transport_agent_start function which will perform the necessary operations in one go to prepare the GStreamer pipeline and start playback. This change of behavior is needed for the RTP bundling support because the agent needs to be aware of all the sessions it will manage as soon as rtpbin starts prerolling. --- bridge/worker/peerhandler.js | 1 + libopenwebrtc.exp | 1 + tests/test_client.c | 1 + tests/test_client.py | 2 +- tests/test_data_channel.c | 4 ++- tests/test_send_receive.c | 2 ++ transport/owr_transport_agent.c | 60 +++++++++++++++++++++++++++------ transport/owr_transport_agent.h | 2 ++ 8 files changed, 60 insertions(+), 13 deletions(-) diff --git a/bridge/worker/peerhandler.js b/bridge/worker/peerhandler.js index 7a0d6b38..34fa53f2 100644 --- a/bridge/worker/peerhandler.js +++ b/bridge/worker/peerhandler.js @@ -62,6 +62,7 @@ function PeerHandler(configuration, keyCert, client, jsonRpc) { transportAgent.add_session(sessions[i]); } + transportAgent.start(); numberOfReceivePreparedSessions = sessions.length; function prepareSession(session, mdesc) { diff --git a/libopenwebrtc.exp b/libopenwebrtc.exp index f6a3c72f..627ca640 100644 --- a/libopenwebrtc.exp +++ b/libopenwebrtc.exp @@ -69,6 +69,7 @@ owr_transport_agent_get_dot_data owr_transport_agent_get_type owr_transport_agent_new owr_transport_agent_set_local_port_range +owr_transport_agent_start owr_transport_type_get_type owr_uri_source_agent_get_dot_data owr_uri_source_agent_get_type diff --git a/tests/test_client.c b/tests/test_client.c index 5db71f89..19f6fe2e 100644 --- a/tests/test_client.c +++ b/tests/test_client.c @@ -610,6 +610,7 @@ static void handle_offer(JsonReader *reader) g_object_set_data(G_OBJECT(transport_agent), "media-sessions", media_sessions); owr_transport_agent_add_session(transport_agent, OWR_SESSION(media_session)); } + owr_transport_agent_start(transport_agent); json_reader_end_member(reader); } diff --git a/tests/test_client.py b/tests/test_client.py index 759ae8a2..664661fb 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -289,7 +289,7 @@ def handle_offer(message): session.set_send_source(source) ALL_SESSIONS.append((session, session_data)) TRANSPORT_AGENT.add_session(session) - + TRANSPORT_AGENT.start() def handle_remote_candidate(message): print("Handling remote candidate") diff --git a/tests/test_data_channel.c b/tests/test_data_channel.c index d9cd4bb0..3d179bf3 100644 --- a/tests/test_data_channel.c +++ b/tests/test_data_channel.c @@ -247,7 +247,9 @@ static gboolean setup_transport_agents() owr_transport_agent_add_session(left_transport_agent, OWR_SESSION(left_session)); owr_transport_agent_add_session(right_transport_agent, OWR_SESSION(right_session)); - + owr_transport_agent_start(left_transport_agent); + owr_transport_agent_start(right_transport_agent); + if (wait_for_dtls) { gboolean peer_certificate_received; GAsyncQueue *msg_queue = g_async_queue_new(); diff --git a/tests/test_send_receive.c b/tests/test_send_receive.c index 290f0e55..e14bd2cd 100644 --- a/tests/test_send_receive.c +++ b/tests/test_send_receive.c @@ -281,6 +281,8 @@ static void got_sources(GList *sources, gpointer user_data) sources = sources->next; } + owr_transport_agent_start(recv_transport_agent); + owr_transport_agent_start(send_transport_agent); } void on_new_source(gpointer *unused, OwrMediaSource *source) diff --git a/transport/owr_transport_agent.c b/transport/owr_transport_agent.c index a586f1ed..99154443 100644 --- a/transport/owr_transport_agent.c +++ b/transport/owr_transport_agent.c @@ -166,6 +166,8 @@ struct _OwrTransportAgentPrivate { GRWLock data_channels_rw_mutex; gboolean data_session_added, data_session_established; OwrMessageOriginBusSet *message_origin_bus_set; + + GSList* unstarted_sessions; }; typedef struct { @@ -295,6 +297,11 @@ static void owr_transport_agent_finalize(GObject *object) gst_element_set_state(priv->pipeline, GST_STATE_NULL); gst_object_unref(priv->pipeline); + if (priv->unstarted_sessions) { + g_slist_free_full(priv->unstarted_sessions, g_object_unref); + priv->unstarted_sessions = NULL; + } + sessions_list = g_hash_table_get_values(priv->sessions); for (item = sessions_list; item; item = item->next) { session = item->data; @@ -645,19 +652,19 @@ void owr_transport_agent_set_local_port_range(OwrTransportAgent *transport_agent */ void owr_transport_agent_add_session(OwrTransportAgent *agent, OwrSession *session) { - GHashTable *args; - g_return_if_fail(agent); g_return_if_fail(OWR_IS_MEDIA_SESSION(session) || OWR_IS_DATA_SESSION(session)); - args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(agent)); - g_hash_table_insert(args, "transport_agent", agent); - g_hash_table_insert(args, "session", session); - - g_object_ref(agent); - - _owr_schedule_with_hash_table((GSourceFunc)add_session, args); + if (GST_STATE(agent->priv->pipeline) >= GST_STATE_READY) { + GHashTable *args; + args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(agent)); + g_hash_table_insert(args, "transport_agent", agent); + g_hash_table_insert(args, "session", g_object_ref(session)); + g_object_ref(agent); + _owr_schedule_with_hash_table((GSourceFunc) add_session, args); + } else + agent->priv->unstarted_sessions = g_slist_append(agent->priv->unstarted_sessions, g_object_ref(session)); } @@ -1136,8 +1143,6 @@ static gboolean add_session(GHashTable *args) if (_owr_session_get_forced_remote_candidates(session)) on_new_remote_candidate(transport_agent, TRUE, session); - state_change_status = gst_element_set_state(transport_agent->priv->pipeline, GST_STATE_PLAYING); - g_warn_if_fail(state_change_status != GST_STATE_CHANGE_FAILURE); end: g_object_unref(session); @@ -4183,6 +4188,39 @@ gchar * owr_transport_agent_get_dot_data(OwrTransportAgent *transport_agent) #endif } +static gboolean dump_bin(gpointer data) +{ + GstPipeline* pipeline = GST_PIPELINE(data); + + GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline), GST_DEBUG_GRAPH_SHOW_ALL, GST_OBJECT_NAME(GST_OBJECT(pipeline))); + gst_object_unref(pipeline); + return G_SOURCE_REMOVE; +} + +void owr_transport_agent_start(OwrTransportAgent *agent) +{ + if (!agent->priv->unstarted_sessions) + return; + for (GSList *walk = agent->priv->unstarted_sessions; walk; walk = g_slist_next(walk)) { + GHashTable *args; + args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(agent)); + g_hash_table_insert(args, "transport_agent", agent); + g_hash_table_insert(args, "session", g_object_ref(OWR_SESSION(walk->data))); + + g_object_ref(agent); + + add_session(args); + g_hash_table_unref(args); + } + g_slist_free_full(agent->priv->unstarted_sessions, g_object_unref); + agent->priv->unstarted_sessions = NULL; + + GstStateChangeReturn state_change_status; + state_change_status = gst_element_set_state(agent->priv->pipeline, GST_STATE_PLAYING); + g_warn_if_fail(state_change_status != GST_STATE_CHANGE_FAILURE); + + g_timeout_add_seconds(5, dump_bin, gst_object_ref(agent->priv->pipeline)); +} static void on_feedback_rtcp(GObject *session, guint type, guint fbtype, guint sender_ssrc, guint media_ssrc, GstBuffer *fci, OwrTransportAgent *transport_agent) diff --git a/transport/owr_transport_agent.h b/transport/owr_transport_agent.h index 3d23afef..7e2d6541 100644 --- a/transport/owr_transport_agent.h +++ b/transport/owr_transport_agent.h @@ -78,6 +78,8 @@ void owr_transport_agent_set_local_port_range(OwrTransportAgent *transport_agent void owr_transport_agent_add_session(OwrTransportAgent *agent, OwrSession *session); gchar * owr_transport_agent_get_dot_data(OwrTransportAgent *transport_agent); +void owr_transport_agent_start(OwrTransportAgent *agent); + G_END_DECLS #endif /* __OWR_TRANSPORT_AGENT_H__ */ From 7a10100f47a8f57be9a909a3010c0af9f45a753d Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Mon, 14 Nov 2016 12:18:35 +0100 Subject: [PATCH 05/26] transport_agent: RTP bundling support We now differentiate session IDs and stream IDs. The streams represent what goes from/to the Nice GStreamer elements. When SDP bundle have been negotiated with a peer, the transport agent will now create a single Nice src and sink elements. If RTCP muxing is enabled, the RTCP packets will flow through these elements. Otherwise, new dedicated Nice src and sink elements are created. --- transport/owr_remote_media_source.c | 6 +- transport/owr_remote_media_source_private.h | 2 +- transport/owr_session.c | 12 + transport/owr_session_private.h | 3 + transport/owr_transport_agent.c | 1195 ++++++++++++------- 5 files changed, 780 insertions(+), 438 deletions(-) diff --git a/transport/owr_remote_media_source.c b/transport/owr_remote_media_source.c index d2f62f15..8ed45b20 100644 --- a/transport/owr_remote_media_source.c +++ b/transport/owr_remote_media_source.c @@ -87,7 +87,7 @@ static void on_caps(GstElement *source, GParamSpec *pspec, OwrMediaSource *media GST_ERROR("Failed to link " #a " -> " #b); OwrMediaSource *_owr_remote_media_source_new(OwrMediaType media_type, - guint stream_id, OwrCodecType codec_type, gpointer transport_bin_ptr) + guint session_id, guint stream_id, OwrCodecType codec_type, gpointer transport_bin_ptr) { GstElement *transport_bin = GST_ELEMENT(transport_bin_ptr); OwrRemoteMediaSource *source; @@ -119,10 +119,10 @@ OwrMediaSource *_owr_remote_media_source_new(OwrMediaType media_type, /* create source tee and everything */ if (media_type == OWR_MEDIA_TYPE_VIDEO) { bin_name = g_strdup_printf("video-src-%u-%u", codec_type, stream_id); - pad_name = g_strdup_printf("video_src_%u_%u", codec_type, stream_id); + pad_name = g_strdup_printf("video_src_%u_%u_%u", codec_type, session_id, stream_id); } else if (media_type == OWR_MEDIA_TYPE_AUDIO) { bin_name = g_strdup_printf("audio-src-%u-%u", codec_type, stream_id); - pad_name = g_strdup_printf("audio_raw_src_%u", stream_id); + pad_name = g_strdup_printf("audio_raw_src_%u_%u", session_id, stream_id); } else g_assert_not_reached(); diff --git a/transport/owr_remote_media_source_private.h b/transport/owr_remote_media_source_private.h index 1729d3b4..3cb7b84d 100644 --- a/transport/owr_remote_media_source_private.h +++ b/transport/owr_remote_media_source_private.h @@ -40,7 +40,7 @@ G_BEGIN_DECLS OwrMediaSource *_owr_remote_media_source_new(OwrMediaType media_type, - guint stream_id, OwrCodecType codec_type, gpointer transport_bin); + guint session_id, guint stream_id, OwrCodecType codec_type, gpointer transport_bin); G_END_DECLS diff --git a/transport/owr_session.c b/transport/owr_session.c index 6d7a3a13..d0109ac8 100644 --- a/transport/owr_session.c +++ b/transport/owr_session.c @@ -66,6 +66,7 @@ G_DEFINE_TYPE_WITH_CODE(OwrSession, owr_session, G_TYPE_OBJECT, G_IMPLEMENT_INTERFACE(OWR_TYPE_MESSAGE_ORIGIN, owr_message_origin_interface_init)) struct _OwrSessionPrivate { + guint stream_id; gboolean dtls_client_mode; gchar *dtls_certificate; gchar *dtls_key; @@ -332,6 +333,7 @@ static void owr_session_init(OwrSession *session) OwrSessionPrivate *priv; session->priv = priv = OWR_SESSION_GET_PRIVATE(session); + priv->stream_id = 0; priv->dtls_client_mode = DEFAULT_DTLS_CLIENT_MODE; priv->dtls_certificate = g_strdup(DEFAULT_DTLS_CERTIFICATE); priv->dtls_key = g_strdup(DEFAULT_DTLS_KEY); @@ -776,3 +778,13 @@ void _owr_session_emit_ice_state_changed(OwrSession *session, guint session_id, session->priv->ice_state = new_state; g_object_notify_by_pspec(G_OBJECT(session), obj_properties[PROP_ICE_STATE]); } + +void _owr_session_set_stream_id(OwrSession *session, guint stream_id) +{ + session->priv->stream_id = stream_id; +} + +guint _owr_session_get_stream_id(OwrSession *session) +{ + return session->priv->stream_id; +} diff --git a/transport/owr_session_private.h b/transport/owr_session_private.h index b6adaba8..bceaee20 100644 --- a/transport/owr_session_private.h +++ b/transport/owr_session_private.h @@ -50,6 +50,9 @@ void _owr_session_set_dtls_peer_certificate(OwrSession *, const gchar *certifica void _owr_session_emit_ice_state_changed(OwrSession *session, guint session_id, OwrComponentType component_type, OwrIceState state); +void _owr_session_set_stream_id(OwrSession *session, guint session_id); +guint _owr_session_get_stream_id(OwrSession *session); + G_END_DECLS #endif /* __GTK_DOC_IGNORE__ */ diff --git a/transport/owr_transport_agent.c b/transport/owr_transport_agent.c index 99154443..5cde8824 100644 --- a/transport/owr_transport_agent.c +++ b/transport/owr_transport_agent.c @@ -102,6 +102,7 @@ G_DEFINE_TYPE_WITH_CODE(OwrTransportAgent, owr_transport_agent, G_TYPE_OBJECT, typedef struct { OwrDataChannelState state; GstElement *data_sink, *data_src; + guint stream_id; guint session_id; gboolean ordered; @@ -136,6 +137,7 @@ typedef struct { struct _OwrTransportAgentPrivate { NiceAgent *nice_agent; + guint next_session_id; gboolean ice_controlling_mode; gboolean bundle_policy; @@ -178,6 +180,7 @@ typedef struct { typedef struct { OwrTransportAgent *transport_agent; guint session_id; + guint stream_id; gint rtx_pt; gboolean adapt; @@ -190,7 +193,8 @@ typedef struct { guint32 last_feedback_wallclock; } ScreamRx; -#define GEN_HASH_KEY(seq, ssrc) (seq ^ ssrc) +#define AGENT_SESSIONS_LOCK(agent) g_mutex_lock(&agent->priv->sessions_lock); +#define AGENT_SESSIONS_UNLOCK(agent) g_mutex_unlock(&agent->priv->sessions_lock); static void owr_transport_agent_set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec); @@ -201,14 +205,16 @@ static void owr_transport_agent_get_property(GObject *object, guint property_id, static void add_helper_server_info(GResolver *resolver, GAsyncResult *result, GHashTable *info); static void update_helper_servers(OwrTransportAgent *transport_agent, guint stream_id); static gboolean add_session(GHashTable *args); -static guint get_stream_id(OwrTransportAgent *transport_agent, OwrSession *session); -static OwrSession * get_session(OwrTransportAgent *transport_agent, guint stream_id); -static void prepare_transport_bin_send_elements(OwrTransportAgent *transport_agent, guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info); -static void prepare_transport_bin_receive_elements(OwrTransportAgent *transport_agent, guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info); -static void prepare_transport_bin_data_receive_elements(OwrTransportAgent *transport_agent, - guint stream_id); -static void prepare_transport_bin_data_send_elements(OwrTransportAgent *transport_agent, - guint stream_id); +static guint get_session_id_unlocked(OwrTransportAgent *transport_agent, OwrSession *session); +static guint get_session_id(OwrTransportAgent *transport_agent, OwrSession *session); +static OwrSession * get_session_unlocked(OwrTransportAgent *transport_agent, guint session_id); +static OwrSession * get_session(OwrTransportAgent *transport_agent, guint session_id); +static OwrSession * get_session_from_stream_id(OwrTransportAgent *transport_agent, guint stream_id); +static GSList * get_sessions_from_stream_id(OwrTransportAgent *transport_agent, guint stream_id); +static void prepare_transport_bin_send_elements(OwrTransportAgent *transport_agent, guint session_id, guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info); +static void prepare_transport_bin_receive_elements(OwrTransportAgent *transport_agent, guint session_id, guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info); +static void prepare_transport_bin_data_receive_elements(OwrTransportAgent *transport_agent, guint session_id, guint stream_id); +static void prepare_transport_bin_data_send_elements(OwrTransportAgent *transport_agent, guint session_id, guint stream_id); static void set_send_ssrc_and_cname(OwrTransportAgent *agent, OwrMediaSession *media_session); static void on_new_candidate(NiceAgent *nice_agent, NiceCandidate *nice_candidate, OwrTransportAgent *transport_agent); static void on_candidate_gathering_done(NiceAgent *nice_agent, guint stream_id, OwrTransportAgent *transport_agent); @@ -219,11 +225,11 @@ static void on_local_candidate_change(OwrTransportAgent *transport_agent, OwrCan static void on_transport_bin_pad_added(GstElement *transport_bin, GstPad *new_pad, OwrTransportAgent *transport_agent); static void on_rtpbin_pad_added(GstElement *rtpbin, GstPad *new_pad, OwrTransportAgent *agent); -static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, OwrPayload *payload, OwrTransportAgent *transport_agent); -static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, OwrPayload *payload, OwrTransportAgent *transport_agent); -static GstCaps * on_rtpbin_request_pt_map(GstElement *rtpbin, guint session_id, guint pt, OwrTransportAgent *agent); -static GstElement * on_rtpbin_request_aux_sender(GstElement *rtpbin, guint session_id, OwrTransportAgent *transport_agent); -static GstElement * on_rtpbin_request_aux_receiver(GstElement *rtpbin, guint session_id, OwrTransportAgent *transport_agent); +static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, guint32 stream_id, OwrPayload *payload, OwrTransportAgent *transport_agent); +static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, guint32 stream_id, OwrPayload *payload, OwrTransportAgent *transport_agent); +static GstCaps * on_rtpbin_request_pt_map(GstElement *rtpbin, guint stream_id, guint pt, OwrTransportAgent *agent); +static GstElement * on_rtpbin_request_aux_sender(GstElement *rtpbin, guint stream_id, OwrTransportAgent *transport_agent); +static GstElement * on_rtpbin_request_aux_receiver(GstElement *rtpbin, guint stream_id, OwrTransportAgent *transport_agent); static void on_dtls_enc_key_set(GstElement *dtls_srtp_enc, AgentAndSessionIdPair *data); static void on_new_selected_pair(NiceAgent *nice_agent, guint stream_id, guint component_id, @@ -235,8 +241,9 @@ static void on_receiving_rtcp(GObject *session, GstBuffer *buffer, OwrTransportA static void on_feedback_rtcp(GObject *session, guint type, guint fbtype, guint sender_ssrc, guint media_ssrc, GstBuffer *fci, OwrTransportAgent *transport_agent); static GstPadProbeReturn probe_save_ts(GstPad *srcpad, GstPadProbeInfo *info, void *user_data); static GstPadProbeReturn probe_rtp_info(GstPad *srcpad, GstPadProbeInfo *info, ScreamRx *scream_rx); -static void on_ssrc_active(GstElement *rtpbin, guint session_id, guint ssrc, OwrTransportAgent *transport_agent); -static void on_new_jitterbuffer(GstElement *rtpbin, GstElement *jitterbuffer, guint session_id, guint ssrc, OwrTransportAgent *transport_agent); +static void on_ssrc_active(GstElement *rtpbin, guint stream_id, guint ssrc, OwrTransportAgent *transport_agent); +static guint on_bundled_ssrc(GstElement *rtpbin, guint ssrc, OwrTransportAgent *transport_agent); +static void on_new_jitterbuffer(GstElement *rtpbin, GstElement *jitterbuffer, guint stream_id, guint ssrc, OwrTransportAgent *transport_agent); static void prepare_rtcp_stats(OwrMediaSession *media_session, GObject *rtp_source); static void data_channel_free(DataChannel *data_channel); @@ -251,7 +258,7 @@ static void on_sctp_association_established(GstElement *sctpenc, gboolean establ static GstFlowReturn new_data_callback(GstAppSink *appsink, OwrTransportAgent *transport_agent); static gboolean create_datachannel_appsrc(OwrTransportAgent *transport_agent, OwrDataChannel *data_channel); -static gboolean is_valid_sctp_stream_id(OwrTransportAgent *transport_agent, guint32 session_id, +static gboolean is_valid_sctp_session_id(OwrTransportAgent *transport_agent, guint32 session_id, guint16 sctp_stream_id, gboolean remotly_initiated); static guint8 * create_datachannel_open_request(OwrDataChannelChannelType channel_type, guint32 reliability_param, guint16 priority, const gchar *label, guint16 label_len, @@ -451,6 +458,7 @@ static void owr_transport_agent_init(OwrTransportAgent *transport_agent) priv->bundle_policy = DEFAULT_BUNDLE_POLICY; priv->agent_id = next_transport_agent_id++; priv->nice_agent = NULL; + priv->next_session_id = 1; priv->sessions = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, g_object_unref); priv->pending_sessions = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, g_free); @@ -544,6 +552,8 @@ static void owr_transport_agent_set_property(GObject *object, guint property_id, break; case PROP_BUNDLE_POLICY: priv->bundle_policy = g_value_get_enum(value); + if (priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) + g_signal_connect(priv->rtpbin, "on-bundled-ssrc", G_CALLBACK(on_bundled_ssrc), transport_agent); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); @@ -674,12 +684,15 @@ static void add_helper_server_info(GResolver *resolver, GAsyncResult *result, GH { OwrTransportAgent *transport_agent; OwrTransportAgentPrivate *priv; - GList *stream_ids, *item, *address_list; + GList *address_list; GHashTable *stats_table; OwrMessageOrigin *message_origin; GValue *value; - guint stream_id; GError *error = NULL; + GHashTableIter iter; + gpointer key, session; + GList *stream_id_list = NULL; + GList *list_iter; transport_agent = OWR_TRANSPORT_AGENT(g_hash_table_lookup(info, "transport_agent")); g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); @@ -718,17 +731,22 @@ static void add_helper_server_info(GResolver *resolver, GAsyncResult *result, GH priv->helper_server_infos = g_list_append(priv->helper_server_infos, info); } - g_mutex_lock(&priv->sessions_lock); - stream_ids = g_hash_table_get_keys(priv->sessions); - g_mutex_unlock(&priv->sessions_lock); - for (item = stream_ids; item; item = item->next) { - stream_id = GPOINTER_TO_UINT(item->data); + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_iter_init(&iter, priv->sessions); + while (g_hash_table_iter_next (&iter, &key, &session)) { + stream_id_list = g_list_prepend(stream_id_list, + GUINT_TO_POINTER(_owr_session_get_stream_id(OWR_SESSION(session)))); + } + AGENT_SESSIONS_UNLOCK(transport_agent); + + for (list_iter = stream_id_list; list_iter != NULL; list_iter = list_iter->next) { if (info) - update_helper_servers(transport_agent, stream_id); + update_helper_servers(transport_agent, GPOINTER_TO_UINT(list_iter->data)); if (!priv->deferred_helper_server_adds) - nice_agent_gather_candidates(priv->nice_agent, stream_id); + nice_agent_gather_candidates(priv->nice_agent, GPOINTER_TO_UINT(list_iter->data)); } - g_list_free(stream_ids); + + g_list_free(stream_id_list); g_object_unref(transport_agent); @@ -850,8 +868,7 @@ static void handle_new_send_source(OwrTransportAgent *transport_agent, g_assert(srcpad); transport_bin = transport_agent->priv->transport_bin; - stream_id = get_stream_id(transport_agent, OWR_SESSION(media_session)); - g_return_if_fail(stream_id); + stream_id = _owr_session_get_stream_id(OWR_SESSION(media_session)); gst_bin_add(GST_BIN(transport_agent->priv->pipeline), src); if (!link_source_to_transport_bin(srcpad, transport_agent->priv->pipeline, transport_bin, media_type, codec_type, stream_id)) { @@ -864,25 +881,13 @@ static void handle_new_send_source(OwrTransportAgent *transport_agent, gst_element_sync_state_with_parent(src); } -static void maybe_handle_new_send_source_with_payload(OwrTransportAgent *transport_agent, - OwrMediaSession *media_session) +static void maybe_handle_new_send_source_with_payload_for_session(OwrTransportAgent *transport_agent, + OwrMediaSession *media_session, gboolean pending) { OwrPayload *payload = NULL; OwrMediaSource *media_source = NULL; GHashTable *event_data; GValue *value; - guint stream_id; - gboolean pending; - - stream_id = get_stream_id(transport_agent, OWR_SESSION(media_session)); - g_return_if_fail(stream_id); - - g_mutex_lock(&transport_agent->priv->sessions_lock); - pending = g_hash_table_lookup_extended(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(stream_id), NULL, NULL); - g_mutex_unlock(&transport_agent->priv->sessions_lock); - - g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); - g_return_if_fail(OWR_IS_MEDIA_SESSION(media_session)); if (!pending && (payload = _owr_media_session_get_send_payload(media_session)) && @@ -906,10 +911,46 @@ static void maybe_handle_new_send_source_with_payload(OwrTransportAgent *transpo g_object_unref(media_source); } +static gboolean process_pending_bundled_session(gpointer key, gpointer value, gpointer user_data) +{ + guint session_id = GPOINTER_TO_UINT(key); + OwrSession* session; + OwrTransportAgent* transport_agent = (OwrTransportAgent*) user_data; + OWR_UNUSED(value); + + session = get_session_unlocked(transport_agent, session_id); + maybe_handle_new_send_source_with_payload_for_session(transport_agent, OWR_MEDIA_SESSION(session), FALSE); + g_object_unref(session); + return TRUE; +} + +static void maybe_handle_new_send_source_with_payload(OwrTransportAgent *transport_agent, + OwrMediaSession *media_session, guint process_bundled_sessions) +{ + g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); + g_return_if_fail(OWR_IS_MEDIA_SESSION(media_session)); + + if (process_bundled_sessions) { + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_foreach_remove(transport_agent->priv->pending_sessions, (GHRFunc) process_pending_bundled_session, transport_agent); + AGENT_SESSIONS_UNLOCK(transport_agent); + } else { + gboolean pending; + guint stream_id; + + stream_id = _owr_session_get_stream_id(OWR_SESSION(media_session)); + AGENT_SESSIONS_LOCK(transport_agent); + pending = g_hash_table_lookup_extended(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(stream_id), NULL, NULL); + AGENT_SESSIONS_UNLOCK(transport_agent); + + maybe_handle_new_send_source_with_payload_for_session(transport_agent, media_session, pending); + } +} + static void remove_existing_send_source_and_payload(OwrTransportAgent *transport_agent, OwrMediaSource *media_source, OwrMediaSession *media_session) { - guint stream_id; + guint session_id, stream_id; gchar *pad_name = NULL, *bin_name; GstPad *bin_src_pad, *sinkpad; GstElement *send_input_bin, *source_bin; @@ -925,7 +966,8 @@ static void remove_existing_send_source_and_payload(OwrTransportAgent *transport /* Setting a new, different source but have one already */ - stream_id = get_stream_id(transport_agent, OWR_SESSION(media_session)); + stream_id = _owr_session_get_stream_id(OWR_SESSION(media_session)); + session_id = get_session_id(transport_agent, OWR_SESSION(media_session)); /* Unlink the source bin */ g_object_get(media_source, "media-type", &media_type, NULL); @@ -951,7 +993,7 @@ static void remove_existing_send_source_and_payload(OwrTransportAgent *transport gst_object_unref(source_bin); /* Now the payload bin */ - bin_name = g_strdup_printf("send-input-bin-%u", stream_id); + bin_name = g_strdup_printf("send-input-bin-%u-%u", session_id, stream_id); send_input_bin = gst_bin_get_by_name(GST_BIN(transport_agent->priv->transport_bin), bin_name); g_assert(send_input_bin); g_free(bin_name); @@ -985,7 +1027,7 @@ static void on_new_send_payload(OwrTransportAgent *transport_agent, } if (new_payload && old_payload != new_payload) - maybe_handle_new_send_source_with_payload(transport_agent, media_session); + maybe_handle_new_send_source_with_payload(transport_agent, media_session, 0); } static void on_new_send_source(OwrTransportAgent *transport_agent, @@ -1002,7 +1044,7 @@ static void on_new_send_source(OwrTransportAgent *transport_agent, remove_existing_send_source_and_payload(transport_agent, old_media_source, media_session); if (new_media_source && old_media_source != new_media_source) - maybe_handle_new_send_source_with_payload(transport_agent, media_session); + maybe_handle_new_send_source_with_payload(transport_agent, media_session, 0); } static gboolean add_session(GHashTable *args) @@ -1011,11 +1053,12 @@ static gboolean add_session(GHashTable *args) OwrTransportAgentPrivate *priv; OwrSession *session; guint stream_id; + guint session_id; gboolean rtcp_mux = TRUE; GObject *rtp_session; - GstStateChangeReturn state_change_status; PendingSessionInfo *pending_session_info; guint port; + guint number_sessions; g_return_val_if_fail(args, FALSE); @@ -1026,34 +1069,50 @@ static gboolean add_session(GHashTable *args) g_return_val_if_fail(session, FALSE); priv = transport_agent->priv; - g_mutex_lock(&priv->sessions_lock); + AGENT_SESSIONS_LOCK(transport_agent); if (g_hash_table_find(priv->sessions, (GHRFunc)is_same_session, session)) { g_warning("An already existing media session was added to the transport agent. Action aborted."); g_mutex_unlock(&priv->sessions_lock); goto end; } - g_mutex_unlock(&priv->sessions_lock); + number_sessions = g_hash_table_size(priv->sessions); + AGENT_SESSIONS_UNLOCK(transport_agent); if (OWR_IS_MEDIA_SESSION(session)) g_object_get(OWR_MEDIA_SESSION(session), "rtcp-mux", &rtcp_mux, NULL); - stream_id = nice_agent_add_stream(priv->nice_agent, rtcp_mux ? 1 : 2); - if (!stream_id) { - g_warning("Failed to add media session."); - goto end; + if ((priv->bundle_policy != OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) || (number_sessions == 0)) { + stream_id = nice_agent_add_stream(priv->nice_agent, rtcp_mux ? 1 : 2); + if (!stream_id) { + g_warning("Failed to add media session."); + goto end; + } + _owr_session_set_stream_id(OWR_SESSION(session), stream_id); + } else { + GHashTableIter iter; + gpointer key, value; + + g_hash_table_iter_init(&iter, priv->sessions); + g_hash_table_iter_next(&iter, &key, &value); + stream_id = _owr_session_get_stream_id(OWR_SESSION(value)); + _owr_session_set_stream_id(OWR_SESSION(session), stream_id); } - g_mutex_lock(&priv->sessions_lock); - g_hash_table_insert(priv->sessions, GUINT_TO_POINTER(stream_id), session); + session_id = priv->next_session_id++; + + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_insert(priv->sessions, GUINT_TO_POINTER(session_id), session); g_object_ref(session); - g_mutex_unlock(&priv->sessions_lock); + AGENT_SESSIONS_UNLOCK(transport_agent); - update_helper_servers(transport_agent, stream_id); + if ((priv->bundle_policy != OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) || (number_sessions == 0)) { + update_helper_servers(transport_agent, stream_id); - _owr_session_set_on_remote_candidate(session, - g_cclosure_new_object_swap(G_CALLBACK(on_new_remote_candidate), G_OBJECT(transport_agent))); - _owr_session_set_on_local_candidate_change(session, - g_cclosure_new_object_swap(G_CALLBACK(on_local_candidate_change), G_OBJECT(transport_agent))); + _owr_session_set_on_remote_candidate(session, + g_cclosure_new_object_swap(G_CALLBACK(on_new_remote_candidate), G_OBJECT(transport_agent))); + _owr_session_set_on_local_candidate_change(session, + g_cclosure_new_object_swap(G_CALLBACK(on_local_candidate_change), G_OBJECT(transport_agent))); + } if (OWR_IS_MEDIA_SESSION(session)) { guint send_ssrc = 0; @@ -1064,13 +1123,16 @@ static gboolean add_session(GHashTable *args) _owr_media_session_set_on_send_payload(OWR_MEDIA_SESSION(session), g_cclosure_new_object_swap(G_CALLBACK(on_new_send_payload), G_OBJECT(transport_agent))); - pending_session_info = g_new0(PendingSessionInfo, 1); - prepare_transport_bin_receive_elements(transport_agent, stream_id, rtcp_mux, pending_session_info); - prepare_transport_bin_send_elements(transport_agent, stream_id, rtcp_mux, pending_session_info); - g_mutex_lock(&transport_agent->priv->sessions_lock); - g_hash_table_insert(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(stream_id), pending_session_info); - g_mutex_unlock(&transport_agent->priv->sessions_lock); + pending_session_info = g_new0(PendingSessionInfo, 1); + if (((priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) && (number_sessions == 0)) + || (priv->bundle_policy != OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE)) { + prepare_transport_bin_receive_elements(transport_agent, session_id, stream_id, rtcp_mux, pending_session_info); + } + prepare_transport_bin_send_elements(transport_agent, session_id, stream_id, rtcp_mux, pending_session_info); + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_insert(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(session_id), pending_session_info); + AGENT_SESSIONS_UNLOCK(transport_agent); g_object_get(session, "send-ssrc", &send_ssrc, "cname", &cname, NULL); if (!send_ssrc || !cname) @@ -1079,8 +1141,8 @@ static gboolean add_session(GHashTable *args) _owr_data_session_set_on_datachannel_added(OWR_DATA_SESSION(session), g_cclosure_new_object_swap(G_CALLBACK(on_new_datachannel), G_OBJECT(transport_agent))); - prepare_transport_bin_data_receive_elements(transport_agent, stream_id); - prepare_transport_bin_data_send_elements(transport_agent, stream_id); + prepare_transport_bin_data_receive_elements(transport_agent, session_id, stream_id); + prepare_transport_bin_data_send_elements(transport_agent, session_id, stream_id); } if (priv->local_max_port > 0) { @@ -1129,13 +1191,14 @@ static gboolean add_session(GHashTable *args) g_object_set(rtp_session, "bandwidth", (gdouble)700000, "rtcp-fraction", (gdouble)100000, "rtcp-min-interval", (guint64)200000000, NULL);*/ - g_object_set_data(rtp_session, "session_id", GUINT_TO_POINTER(stream_id)); + g_object_set_data(rtp_session, "stream-id", GUINT_TO_POINTER(stream_id)); + g_object_set_data(rtp_session, "session-id", GUINT_TO_POINTER(session_id)); + g_object_set_data(rtp_session, "session", session); g_signal_connect_after(rtp_session, "on-sending-rtcp", G_CALLBACK(on_sending_rtcp), transport_agent); g_signal_connect(rtp_session, "on-feedback-rtcp", G_CALLBACK(on_feedback_rtcp), transport_agent); g_signal_connect_after(rtp_session, "on-receiving-rtcp", G_CALLBACK(on_receiving_rtcp), NULL); g_object_unref(rtp_session); - - maybe_handle_new_send_source_with_payload(transport_agent, OWR_MEDIA_SESSION(session)); + maybe_handle_new_send_source_with_payload(transport_agent, OWR_MEDIA_SESSION(session), 0); } if (_owr_session_get_remote_candidates(session)) @@ -1151,7 +1214,7 @@ static gboolean add_session(GHashTable *args) return FALSE; } -static GstElement *add_nice_element(OwrTransportAgent *transport_agent, guint stream_id, +static GstElement *add_nice_element(OwrTransportAgent *transport_agent, guint session_id, guint stream_id, gboolean is_sink, gboolean is_rtcp, GstElement *bin) { GstElement *nice_element = NULL; @@ -1160,8 +1223,18 @@ static GstElement *add_nice_element(OwrTransportAgent *transport_agent, guint st g_return_val_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent), NULL); - element_name = g_strdup_printf("nice-%s-%s-%u", is_rtcp ? "rtcp" : "rtp", is_sink - ? "sink" : "src", stream_id); + if ((transport_agent->priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) && !is_rtcp) + element_name = g_strdup_printf("nice-%s-%s-%u", is_rtcp ? "rtcp" : "rtp", is_sink + ? "sink" : "src", stream_id); + else + element_name = g_strdup_printf("nice-%s-%s-%u-%u", is_rtcp ? "rtcp" : "rtp", is_sink + ? "sink" : "src", session_id, stream_id); + nice_element = gst_bin_get_by_name(GST_BIN(bin), element_name); + if (nice_element && (transport_agent->priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE)) { + gst_object_unref(nice_element); + return NULL; + } + nice_element = gst_element_factory_make(is_sink ? "nicesink" : "nicesrc", element_name); g_free(element_name); @@ -1249,7 +1322,7 @@ static void on_dtls_peer_certificate(GstElement *dtls_srtp_bin, GParamSpec *pspe g_free(certificate); } -static GstElement *add_dtls_srtp_bin(OwrTransportAgent *transport_agent, guint stream_id, +static GstElement *add_dtls_srtp_bin(OwrTransportAgent *transport_agent, guint session_id, guint stream_id, gboolean is_encoder, gboolean is_rtcp, GstElement *bin) { OwrTransportAgentPrivate *priv; @@ -1262,8 +1335,8 @@ static GstElement *add_dtls_srtp_bin(OwrTransportAgent *transport_agent, guint s g_return_val_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent), NULL); priv = transport_agent->priv; - element_name = g_strdup_printf("dtls_srtp_%s_%s_%u", is_rtcp ? "rtcp" : "rtp", - is_encoder ? "encoder" : "decoder", stream_id); + element_name = g_strdup_printf("dtls_srtp_%s_%s_%u_%u", is_rtcp ? "rtcp" : "rtp", + is_encoder ? "encoder" : "decoder", session_id, stream_id); dtls_srtp_bin = gst_element_factory_make(is_encoder ? "dtlssrtpenc" : "dtlssrtpdec", element_name); @@ -1272,7 +1345,7 @@ static GstElement *add_dtls_srtp_bin(OwrTransportAgent *transport_agent, guint s g_object_set(dtls_srtp_bin, "connection-id", connection_id, NULL); g_free(connection_id); - session = get_session(transport_agent, stream_id); + session = get_session(transport_agent, session_id); if (!is_encoder) { g_object_get(session, "dtls-certificate", &cert, NULL); @@ -1399,10 +1472,10 @@ static void on_bitrate_change(GstElement *scream_queue, guint bitrate, guint ssr _owr_schedule_with_hash_table((GSourceFunc)emit_bitrate_change, args); } -static void link_rtpbin_to_send_output_bin(OwrTransportAgent *transport_agent, guint stream_id, gboolean rtp, gboolean rtcp) +static void link_rtpbin_to_send_output_bin(OwrTransportAgent *transport_agent, guint session_id, guint stream_id, gboolean rtp, gboolean rtcp) { gchar *rtpbin_pad_name, *dtls_srtp_pad_name; - gchar *output_selector_name; + gchar *output_selector_name, *queue_name; gboolean linked_ok; GstPad *sink_pad, *src_pad; GstElement *output_selector; @@ -1416,7 +1489,8 @@ static void link_rtpbin_to_send_output_bin(OwrTransportAgent *transport_agent, g g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); send_bin_info = g_hash_table_lookup(transport_agent->priv->send_bins, GINT_TO_POINTER(stream_id)); - g_return_if_fail(send_bin_info); + if (!send_bin_info) + return; dtls_srtp_bin_rtp = send_bin_info->dtls_srtp_bin_rtp; dtls_srtp_bin_rtcp = send_bin_info->dtls_srtp_bin_rtcp; @@ -1425,9 +1499,11 @@ static void link_rtpbin_to_send_output_bin(OwrTransportAgent *transport_agent, g g_return_if_fail(GST_IS_ELEMENT(dtls_srtp_bin_rtp)); g_return_if_fail(!dtls_srtp_bin_rtcp || GST_IS_ELEMENT(dtls_srtp_bin_rtcp)); - media_session = OWR_MEDIA_SESSION(get_session(transport_agent, stream_id)); + media_session = OWR_MEDIA_SESSION(get_session_unlocked(transport_agent, session_id)); - scream_queue = gst_bin_get_by_name(GST_BIN(send_output_bin), "screamqueue"); + queue_name = g_strdup_printf("screamqueue-%u-%u", session_id, stream_id); + scream_queue = gst_bin_get_by_name(GST_BIN(send_output_bin), queue_name); + g_free(queue_name); g_assert(scream_queue); g_signal_connect_object(scream_queue, "on-bitrate-change", G_CALLBACK(on_bitrate_change), media_session, 0); @@ -1435,8 +1511,8 @@ static void link_rtpbin_to_send_output_bin(OwrTransportAgent *transport_agent, g /* RTP */ if (rtp) { - rtpbin_pad_name = g_strdup_printf("send_rtp_src_%u", stream_id); - dtls_srtp_pad_name = g_strdup_printf("rtp_sink_%u", stream_id); + rtpbin_pad_name = g_strdup_printf("send_rtp_src_%u", session_id); + dtls_srtp_pad_name = g_strdup_printf("rtp_sink_%u", session_id + stream_id); sink_pad = gst_element_get_static_pad(scream_queue, "sink"); g_assert(GST_IS_PAD(sink_pad)); @@ -1451,23 +1527,28 @@ static void link_rtpbin_to_send_output_bin(OwrTransportAgent *transport_agent, g g_free(dtls_srtp_pad_name); } + gst_object_unref(scream_queue); + /* RTCP */ - if (rtcp && !send_bin_info->linked_rtcp) { - output_selector_name = g_strdup_printf("rtcp_output_selector_%u", stream_id); + output_selector_name = g_strdup_printf("rtcp-output-selector-%u-%u", session_id, stream_id); + output_selector = gst_bin_get_by_name(GST_BIN(send_output_bin), output_selector_name); + if (output_selector) { + gst_object_unref(output_selector); + } else if (rtcp) { output_selector = gst_element_factory_make("output-selector", output_selector_name); - g_free(output_selector_name); g_object_set(output_selector, "pad-negotiation-mode", 0, NULL); gst_bin_add(GST_BIN(send_output_bin), output_selector); gst_element_sync_state_with_parent(output_selector); - rtpbin_pad_name = g_strdup_printf("send_rtcp_src_%u", stream_id); - dtls_srtp_pad_name = g_strdup_printf("rtcp_sink_%u", stream_id); + rtpbin_pad_name = g_strdup_printf("send_rtcp_src_%u", session_id); + dtls_srtp_pad_name = g_strdup_printf("rtcp_sink_%u", session_id); /* RTCP muxing */ - sink_pad = gst_element_get_request_pad(dtls_srtp_bin_rtp, dtls_srtp_pad_name); - g_assert(GST_IS_PAD(sink_pad)); src_pad = gst_element_get_request_pad(output_selector, "src_%u"); g_assert(GST_IS_PAD(src_pad)); + sink_pad = gst_element_get_request_pad(dtls_srtp_bin_rtp, dtls_srtp_pad_name); + g_assert(GST_IS_PAD(sink_pad)); + linked_ok = gst_pad_link(src_pad, sink_pad) == GST_PAD_LINK_OK; g_warn_if_fail(linked_ok); g_object_set(output_selector, "active-pad", src_pad, NULL); @@ -1505,73 +1586,126 @@ static void link_rtpbin_to_send_output_bin(OwrTransportAgent *transport_agent, g send_bin_info->linked_rtcp = TRUE; } + g_free(output_selector_name); } static void prepare_transport_bin_send_elements(OwrTransportAgent *transport_agent, - guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info) + guint session_id, guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info) { GstElement *nice_element, *dtls_srtp_bin_rtp, *dtls_srtp_bin_rtcp = NULL; GstElement *scream_queue = NULL; gboolean linked_ok, synced_ok; - GstElement *send_output_bin; + GstElement *send_output_bin = NULL; SendBinInfo *send_bin_info; - gchar *bin_name, *dtls_srtp_pad_name; + gchar *bin_name, *dtls_srtp_pad_name, *queue_name; OwrMediaSession *media_session; AgentAndSessionIdPair *agent_and_session_id_pair; g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); bin_name = g_strdup_printf("send-output-bin-%u", stream_id); - send_output_bin = gst_bin_new(bin_name); - g_free(bin_name); - - if (!gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), send_output_bin)) { - GST_ERROR("Failed to add send-output-bin-%u to parent bin", stream_id); - return; + if (transport_agent->priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) { + send_output_bin = gst_bin_get_by_name(GST_BIN(transport_agent->priv->transport_bin), bin_name); } - if (!gst_element_sync_state_with_parent(send_output_bin)) { - GST_ERROR("Failed to sync send-output-bin-%u state with parent bin", stream_id); - return; + if (!send_output_bin) { + send_output_bin = gst_bin_new(bin_name); + + if (!gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), send_output_bin)) { + GST_ERROR("Failed to add %s to parent bin", bin_name); + g_free(bin_name); + return; + } + + if (!gst_element_sync_state_with_parent(send_output_bin)) { + GST_ERROR("Failed to sync %s state with parent bin", bin_name); + g_free(bin_name); + return; + } } - media_session = OWR_MEDIA_SESSION(get_session(transport_agent, stream_id)); - scream_queue = gst_element_factory_make("screamqueue", "screamqueue"); + g_free(bin_name); + + media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); + queue_name = g_strdup_printf("screamqueue-%u-%u", session_id, stream_id); + scream_queue = gst_element_factory_make("screamqueue", queue_name); + g_free(queue_name); g_assert(scream_queue); g_object_set(scream_queue, "scream-controller-id", transport_agent->priv->agent_id, NULL); g_signal_connect(scream_queue, "on-payload-adaptation-request", (GCallback)on_payload_adaptation_request, media_session); gst_bin_add(GST_BIN(send_output_bin), scream_queue); - pending_session_info->nice_sink_rtp = nice_element = add_nice_element(transport_agent, stream_id, TRUE, FALSE, send_output_bin); - pending_session_info->dtls_enc_rtp = dtls_srtp_bin_rtp = add_dtls_srtp_bin(transport_agent, stream_id, TRUE, FALSE, send_output_bin); - linked_ok = gst_element_link(dtls_srtp_bin_rtp, nice_element); - g_warn_if_fail(linked_ok); + nice_element = add_nice_element(transport_agent, session_id, stream_id, TRUE, FALSE, send_output_bin); + // In MAX_BUNDLE case we need only one dtlssrtpenc and nicesink. + if (nice_element) { +#ifdef TEST_RTX + GstElement* identity = gst_element_factory_make("identity", NULL); + g_object_set(identity, "drop-probability", 0.01, NULL); + + gst_bin_add(GST_BIN(send_output_bin), identity); +#endif + pending_session_info->nice_sink_rtp = nice_element; + pending_session_info->dtls_enc_rtp = dtls_srtp_bin_rtp = add_dtls_srtp_bin(transport_agent, session_id, stream_id, TRUE, FALSE, send_output_bin); +#ifdef TEST_RTX + linked_ok = gst_element_link(identity, nice_element); + linked_ok &= gst_element_link(dtls_srtp_bin_rtp, identity); +#else + linked_ok = gst_element_link(dtls_srtp_bin_rtp, nice_element); +#endif + g_warn_if_fail(linked_ok); - agent_and_session_id_pair = g_new0(AgentAndSessionIdPair, 1); - agent_and_session_id_pair->transport_agent = transport_agent; - agent_and_session_id_pair->session_id = stream_id; - g_signal_connect_data(dtls_srtp_bin_rtp, "on-key-set", G_CALLBACK(on_dtls_enc_key_set), agent_and_session_id_pair, (GClosureNotify) g_free, 0); + agent_and_session_id_pair = g_new0(AgentAndSessionIdPair, 1); + agent_and_session_id_pair->transport_agent = transport_agent; + agent_and_session_id_pair->session_id = session_id; + g_signal_connect_data(dtls_srtp_bin_rtp, "on-key-set", G_CALLBACK(on_dtls_enc_key_set), agent_and_session_id_pair, (GClosureNotify) g_free, 0); - dtls_srtp_pad_name = g_strdup_printf("rtp_sink_%u", stream_id); + dtls_srtp_pad_name = g_strdup_printf("rtp_sink_%u", stream_id); + } else { + GSList *sessions = get_sessions_from_stream_id(transport_agent, stream_id); + guint previous_session_id = 0; + // Look for the session for which the global encoder was created. + for (GSList *walk = sessions; walk && previous_session_id == 0; walk = g_slist_next(walk)) { + OwrMediaSession *current = OWR_MEDIA_SESSION(walk->data); + guint current_session_id; + + if (!OWR_IS_MEDIA_SESSION(current)) + continue; + + current_session_id = get_session_id(transport_agent, OWR_SESSION(current)); + if (current_session_id != session_id) + previous_session_id = current_session_id; + } + g_slist_free_full(sessions, g_object_unref); + if (previous_session_id > 0) { + bin_name = g_strdup_printf("dtls_srtp_rtp_encoder_%u_%u", previous_session_id, stream_id); + dtls_srtp_bin_rtp = gst_bin_get_by_name(GST_BIN(send_output_bin), bin_name); + g_free(bin_name); + dtls_srtp_pad_name = g_strdup_printf("rtp_sink_%u", session_id); + } + } linked_ok = gst_element_link_pads(scream_queue, "src", dtls_srtp_bin_rtp, dtls_srtp_pad_name); - g_free(dtls_srtp_pad_name); g_warn_if_fail(linked_ok); + g_free(dtls_srtp_pad_name); + + if (nice_element) { + synced_ok = gst_element_sync_state_with_parent(nice_element); + g_warn_if_fail(synced_ok); + } - synced_ok = gst_element_sync_state_with_parent(nice_element); - g_warn_if_fail(synced_ok); synced_ok = gst_element_sync_state_with_parent(scream_queue); g_warn_if_fail(synced_ok); if (!rtcp_mux) { - pending_session_info->nice_sink_rtcp = nice_element = add_nice_element(transport_agent, stream_id, TRUE, TRUE, send_output_bin); - pending_session_info->dtls_enc_rtcp = dtls_srtp_bin_rtcp = add_dtls_srtp_bin(transport_agent, stream_id, TRUE, TRUE, send_output_bin); + nice_element = add_nice_element(transport_agent, session_id, stream_id, TRUE, TRUE, send_output_bin); + pending_session_info->nice_sink_rtcp = nice_element; + pending_session_info->dtls_enc_rtcp = dtls_srtp_bin_rtcp = add_dtls_srtp_bin(transport_agent, session_id, stream_id, TRUE, TRUE, send_output_bin); linked_ok = gst_element_link(dtls_srtp_bin_rtcp, nice_element); g_warn_if_fail(linked_ok); agent_and_session_id_pair = g_new0(AgentAndSessionIdPair, 1); agent_and_session_id_pair->transport_agent = transport_agent; - agent_and_session_id_pair->session_id = stream_id; + agent_and_session_id_pair->session_id = session_id; g_signal_connect_data(dtls_srtp_bin_rtp, "on-key-set", G_CALLBACK(on_dtls_enc_key_set), agent_and_session_id_pair, (GClosureNotify) g_free, 0); synced_ok = gst_element_sync_state_with_parent(nice_element); @@ -1587,7 +1721,7 @@ static void prepare_transport_bin_send_elements(OwrTransportAgent *transport_age g_hash_table_insert(transport_agent->priv->send_bins, GINT_TO_POINTER(stream_id), send_bin_info); } -static GstPadProbeReturn nice_src_pad_block(GstPad *pad, GstPadProbeInfo *info, AgentAndSessionIdPair *data) +static GstPadProbeReturn nice_src_pad_block(GstPad *pad, GstPadProbeInfo *info, gpointer data) { OWR_UNUSED(pad); OWR_UNUSED(info); @@ -1597,70 +1731,77 @@ static GstPadProbeReturn nice_src_pad_block(GstPad *pad, GstPadProbeInfo *info, } static void prepare_transport_bin_receive_elements(OwrTransportAgent *transport_agent, - guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info) + guint session_id, guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info) { GstElement *nice_element, *dtls_srtp_bin, *funnel; GstPad *rtp_src_pad, *rtcp_src_pad, *rtp_sink_pad; GstPad *nice_src_pad; - gchar *rtpbin_pad_name; gboolean linked_ok, synced_ok; GstElement *receive_input_bin; ScreamRx *scream_rx; + gchar *bin_name; + gchar *rtp_src_pad_name; + gchar *rtpbin_pad_name; #ifdef TEST_RTX GstElement *identity; #endif - gchar *bin_name; AgentAndSessionIdPair *agent_and_session_id_pair; g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); - bin_name = g_strdup_printf("receive-input-bin-%u", stream_id); + if (transport_agent->priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) { + bin_name = g_strdup_printf("receive-input-bin-%u", session_id); + } else { + bin_name = g_strdup_printf("receive-input-bin-%u", stream_id); + } + receive_input_bin = gst_bin_new(bin_name); - g_free(bin_name); if (!gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), receive_input_bin)) { - GST_ERROR("Failed to add receive-input-bin-%u to parent bin", stream_id); + GST_ERROR("Failed to add %s to parent bin", bin_name); + g_free(bin_name); return; } if (!gst_element_sync_state_with_parent(receive_input_bin)) { - GST_ERROR("Failed to sync receive-input-bin-%u state with parent bin", stream_id); + GST_ERROR("Failed to sync %s state with parent bin", bin_name); + g_free(bin_name); return; } + g_free(bin_name); - pending_session_info->nice_src_rtp = nice_element = add_nice_element(transport_agent, stream_id, FALSE, FALSE, receive_input_bin); - pending_session_info->dtls_dec_rtp = dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, stream_id, FALSE, FALSE, receive_input_bin); + pending_session_info->nice_src_rtp = nice_element = add_nice_element(transport_agent, session_id, stream_id, FALSE, FALSE, receive_input_bin); + pending_session_info->dtls_dec_rtp = dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, session_id, stream_id, FALSE, FALSE, receive_input_bin); - agent_and_session_id_pair = g_new0(AgentAndSessionIdPair, 1); - agent_and_session_id_pair->transport_agent = transport_agent; - agent_and_session_id_pair->session_id = stream_id; nice_src_pad = gst_element_get_static_pad(nice_element, "src"); - pending_session_info->nice_src_block_rtp = gst_pad_add_probe(nice_src_pad, GST_PAD_PROBE_TYPE_BLOCK | GST_PAD_PROBE_TYPE_BUFFER | GST_PAD_PROBE_TYPE_BUFFER_LIST, (GstPadProbeCallback)nice_src_pad_block, agent_and_session_id_pair, (GDestroyNotify) g_free); + pending_session_info->nice_src_block_rtp = gst_pad_add_probe(nice_src_pad, GST_PAD_PROBE_TYPE_BLOCK | GST_PAD_PROBE_TYPE_BUFFER | GST_PAD_PROBE_TYPE_BUFFER_LIST, (GstPadProbeCallback)nice_src_pad_block, NULL, NULL); gst_object_unref(nice_src_pad); synced_ok = gst_element_sync_state_with_parent(nice_element); g_warn_if_fail(synced_ok); - rtp_src_pad = gst_element_get_static_pad(dtls_srtp_bin, "rtp_src"); - ghost_pad_and_add_to_bin(rtp_src_pad, receive_input_bin, "rtp_src"); + rtp_src_pad_name = g_strdup("rtp_src"); + rtp_src_pad = gst_element_get_static_pad(dtls_srtp_bin, rtp_src_pad_name); + ghost_pad_and_add_to_bin(rtp_src_pad, receive_input_bin, rtp_src_pad_name); gst_object_unref(rtp_src_pad); rtpbin_pad_name = g_strdup_printf("recv_rtp_sink_%u", stream_id); #ifndef TEST_RTX - linked_ok = gst_element_link_pads(receive_input_bin, "rtp_src", transport_agent->priv->rtpbin, + linked_ok = gst_element_link_pads(receive_input_bin, rtp_src_pad_name, transport_agent->priv->rtpbin, rtpbin_pad_name); #else identity = gst_element_factory_make("identity", NULL); g_object_set(identity, "drop-probability", 0.01, NULL); gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), identity); - gst_element_link_pads(receive_input_bin, "rtp_src", identity, "sink"); + gst_element_link_pads(receive_input_bin, rtp_src_pad_name, identity, "sink"); linked_ok = gst_element_link_pads(identity, "src", transport_agent->priv->rtpbin, rtpbin_pad_name); gst_element_sync_state_with_parent(identity); #endif g_warn_if_fail(linked_ok); g_free(rtpbin_pad_name); + g_free(rtp_src_pad_name); if (!rtcp_mux) { funnel = gst_element_factory_make("funnel", NULL); @@ -1669,12 +1810,12 @@ static void prepare_transport_bin_receive_elements(OwrTransportAgent *transport_ linked_ok = gst_element_link_pads(dtls_srtp_bin, "rtcp_src", funnel, "sink_0"); g_warn_if_fail(linked_ok); - pending_session_info->nice_src_rtcp = nice_element = add_nice_element(transport_agent, stream_id, FALSE, TRUE, receive_input_bin); - pending_session_info->dtls_dec_rtcp = dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, stream_id, FALSE, TRUE, receive_input_bin); + pending_session_info->nice_src_rtcp = nice_element = add_nice_element(transport_agent, session_id, stream_id, FALSE, TRUE, receive_input_bin); + pending_session_info->dtls_dec_rtcp = dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, session_id, stream_id, FALSE, TRUE, receive_input_bin); agent_and_session_id_pair = g_new0(AgentAndSessionIdPair, 1); agent_and_session_id_pair->transport_agent = transport_agent; - agent_and_session_id_pair->session_id = stream_id; + agent_and_session_id_pair->session_id = session_id; nice_src_pad = gst_element_get_static_pad(nice_element, "src"); pending_session_info->nice_src_block_rtcp = gst_pad_add_probe(nice_src_pad, GST_PAD_PROBE_TYPE_BLOCK | GST_PAD_PROBE_TYPE_BUFFER | GST_PAD_PROBE_TYPE_BUFFER_LIST, (GstPadProbeCallback)nice_src_pad_block, agent_and_session_id_pair, (GDestroyNotify) g_free); gst_object_unref(nice_src_pad); @@ -1710,7 +1851,8 @@ static void prepare_transport_bin_receive_elements(OwrTransportAgent *transport_ scream_rx = g_new0(ScreamRx, 1); scream_rx->rtx_pt = -2; /* unknown */ scream_rx->transport_agent = transport_agent; - scream_rx->session_id = stream_id; + scream_rx->stream_id = stream_id; + scream_rx->session_id = session_id; scream_rx->adapt = TRUE; /* Always initiates to TRUE. Sets to TRUE or FALSE in probe_rtp_info */ gst_pad_add_probe(rtp_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, (GstPadProbeCallback)probe_rtp_info, scream_rx, g_free); @@ -1719,7 +1861,7 @@ static void prepare_transport_bin_receive_elements(OwrTransportAgent *transport_ static void prepare_transport_bin_data_receive_elements(OwrTransportAgent *transport_agent, - guint stream_id) + guint session_id, guint stream_id) { OwrTransportAgentPrivate *priv; GstElement *nice_element, *dtls_srtp_bin, *sctpdec; @@ -1748,12 +1890,12 @@ static void prepare_transport_bin_data_receive_elements(OwrTransportAgent *trans return; } - nice_element = add_nice_element(transport_agent, stream_id, FALSE, FALSE, receive_input_bin); - dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, stream_id, FALSE, FALSE, receive_input_bin); + nice_element = add_nice_element(transport_agent, session_id, stream_id, FALSE, FALSE, receive_input_bin); + dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, session_id, stream_id, FALSE, FALSE, receive_input_bin); sctpdec = _owr_data_session_create_decoder(data_session); g_return_if_fail(nice_element && dtls_srtp_bin && sctpdec); - g_object_set_data(G_OBJECT(sctpdec), "session-id", GUINT_TO_POINTER(stream_id)); + g_object_set_data(G_OBJECT(sctpdec), "stream_id", GUINT_TO_POINTER(stream_id)); gst_bin_add(GST_BIN(receive_input_bin), sctpdec); g_signal_connect(sctpdec, "pad-added", (GCallback)sctpdec_pad_added, transport_agent); g_signal_connect(sctpdec, "pad-removed", (GCallback)sctpdec_pad_removed, transport_agent); @@ -1769,7 +1911,7 @@ static void prepare_transport_bin_data_receive_elements(OwrTransportAgent *trans } static void prepare_transport_bin_data_send_elements(OwrTransportAgent *transport_agent, - guint stream_id) + guint session_id, guint stream_id) { OwrTransportAgentPrivate *priv; GstElement *nice_element, *dtls_srtp_bin, *sctpenc, *send_output_bin; @@ -1783,23 +1925,25 @@ static void prepare_transport_bin_data_send_elements(OwrTransportAgent *transpor name = g_strdup_printf("send-output-bin-%u", stream_id); send_output_bin = gst_bin_new(name); - g_free(name); if (!gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), send_output_bin)) { - GST_ERROR("Failed to add send-output-bin-%u to parent bin", stream_id); + GST_ERROR("Failed to add %s to parent bin", name); + g_free(name); return; } if (!gst_element_sync_state_with_parent(send_output_bin)) { - GST_ERROR("Failed to sync send-output-bin-%u to parent bin", stream_id); + GST_ERROR("Failed to sync %s to parent bin", name); + g_free(name); return; } + g_free(name); - nice_element = add_nice_element(transport_agent, stream_id, TRUE, FALSE, send_output_bin); - dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, stream_id, TRUE, FALSE, send_output_bin); + nice_element = add_nice_element(transport_agent, session_id, stream_id, TRUE, FALSE, send_output_bin); + dtls_srtp_bin = add_dtls_srtp_bin(transport_agent, session_id, stream_id, TRUE, FALSE, send_output_bin); sctpenc = _owr_data_session_create_encoder(data_session); g_warn_if_fail(sctpenc); - g_object_set_data(G_OBJECT(sctpenc), "session-id", GUINT_TO_POINTER(stream_id)); + g_object_set_data(G_OBJECT(sctpenc), "stream_id", GUINT_TO_POINTER(stream_id)); gst_bin_add(GST_BIN(send_output_bin), sctpenc); g_signal_connect(sctpenc, "sctp-association-established", G_CALLBACK(on_sctp_association_established), transport_agent); @@ -1824,7 +1968,7 @@ static void set_send_ssrc_and_cname(OwrTransportAgent *transport_agent, OwrMedia g_return_if_fail(transport_agent); g_return_if_fail(media_session); - stream_id = get_stream_id(transport_agent, OWR_SESSION(media_session)); + stream_id = _owr_session_get_stream_id(OWR_SESSION(media_session)); g_signal_emit_by_name(transport_agent->priv->rtpbin, "get-internal-session", stream_id, &session); g_warn_if_fail(session); g_object_get(session, "internal-ssrc", &send_ssrc, "sdes", &sdes, NULL); @@ -1838,11 +1982,11 @@ static gboolean emit_new_candidate(GHashTable *args) { OwrTransportAgent *transport_agent; OwrTransportAgentPrivate *priv; - OwrSession *session; NiceCandidate *nice_candidate; OwrCandidate *owr_candidate; gchar *ufrag = NULL, *password = NULL; gboolean got_credentials; + GSList *sessions; transport_agent = OWR_TRANSPORT_AGENT(g_hash_table_lookup(args, "transport_agent")); g_return_val_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent), FALSE); @@ -1850,33 +1994,36 @@ static gboolean emit_new_candidate(GHashTable *args) nice_candidate = (NiceCandidate *)g_hash_table_lookup(args, "nice_candidate"); g_return_val_if_fail(nice_candidate, FALSE); - session = get_session(transport_agent, nice_candidate->stream_id); - g_return_val_if_fail(OWR_IS_SESSION(session), FALSE); - if (!nice_candidate->username || !nice_candidate->password) { - got_credentials = nice_agent_get_local_credentials(priv->nice_agent, - nice_candidate->stream_id, &ufrag, &password); - g_warn_if_fail(got_credentials); + sessions = get_sessions_from_stream_id(transport_agent, nice_candidate->stream_id); + for (GSList *walk = sessions; walk; walk = g_slist_next(walk)) { + OwrSession *session = OWR_SESSION(walk->data); + if (!nice_candidate->username || !nice_candidate->password) { + got_credentials = nice_agent_get_local_credentials(priv->nice_agent, + nice_candidate->stream_id, &ufrag, &password); + g_warn_if_fail(got_credentials); + + if (!nice_candidate->username) + nice_candidate->username = ufrag; + else + g_free(ufrag); + + if (!nice_candidate->password) + nice_candidate->password = password; + else + g_free(password); + } - if (!nice_candidate->username) - nice_candidate->username = ufrag; - else - g_free(ufrag); + owr_candidate = _owr_candidate_new_from_nice_candidate(nice_candidate); + if (!owr_candidate) + continue; - if (!nice_candidate->password) - nice_candidate->password = password; - else - g_free(password); + g_signal_emit_by_name(session, "on-new-candidate", owr_candidate); + g_object_unref(owr_candidate); } - owr_candidate = _owr_candidate_new_from_nice_candidate(nice_candidate); - g_return_val_if_fail(owr_candidate, FALSE); + g_slist_free_full(sessions, g_object_unref); nice_candidate_free(nice_candidate); - - g_signal_emit_by_name(session, "on-new-candidate", owr_candidate); - g_object_unref(owr_candidate); - - g_object_unref(session); g_hash_table_destroy(args); g_object_unref(transport_agent); @@ -1908,56 +2055,55 @@ static gboolean emit_candidate_gathering_done(GHashTable *args) OwrCandidate *local_candidate, *remote_candidate; guint stream_id; int i; + GSList *sessions; transport_agent = g_hash_table_lookup(args, "transport-agent"); - session = g_hash_table_lookup(args, "session"); - g_signal_emit_by_name(session, "on-candidate-gathering-done", NULL); + stream_id = GPOINTER_TO_UINT(g_hash_table_lookup(args, "stream-id")); + + sessions = get_sessions_from_stream_id(transport_agent, stream_id); + for (GSList *walk = sessions; walk; walk = g_slist_next(walk)) { + session = OWR_SESSION(walk->data); - stream_id = get_stream_id(transport_agent, session); - g_return_val_if_fail(stream_id, FALSE); + g_signal_emit_by_name(session, "on-candidate-gathering-done", NULL); - for (i = 0; i < OWR_COMPONENT_MAX; i++) { - _owr_session_get_candidate_pair(session, i, &local_candidate, &remote_candidate); + for (i = 0; i < OWR_COMPONENT_MAX; i++) { + _owr_session_get_candidate_pair(session, i, &local_candidate, &remote_candidate); - if (local_candidate && remote_candidate) { - gchar *lfoundation = NULL, *rfoundation = NULL; + if (local_candidate && remote_candidate) { + gchar *lfoundation = NULL, *rfoundation = NULL; - g_object_get(local_candidate, "foundation", &lfoundation, NULL); - g_object_get(remote_candidate, "foundation", &rfoundation, NULL); + g_object_get(local_candidate, "foundation", &lfoundation, NULL); + g_object_get(remote_candidate, "foundation", &rfoundation, NULL); - if (lfoundation && rfoundation) { - GST_DEBUG_OBJECT(transport_agent, "Forcing pair %s:%s on stream %u", - lfoundation, rfoundation, stream_id); - nice_agent_set_selected_pair(transport_agent->priv->nice_agent, - stream_id, i, lfoundation, rfoundation); - } else - g_assert_not_reached(); + if (lfoundation && rfoundation) { + GST_DEBUG_OBJECT(transport_agent, "Forcing pair %s:%s on stream %u", + lfoundation, rfoundation, stream_id); + nice_agent_set_selected_pair(transport_agent->priv->nice_agent, + stream_id, i, lfoundation, rfoundation); + } else + g_assert_not_reached(); - g_free(lfoundation); - g_free(rfoundation); + g_free(lfoundation); + g_free(rfoundation); + } } } - + g_slist_free_full(sessions, g_object_unref); g_hash_table_destroy(args); - g_object_unref(session); return FALSE; } static void on_candidate_gathering_done(NiceAgent *nice_agent, guint stream_id, OwrTransportAgent *transport_agent) { - OwrSession *session; GHashTable *args; g_return_if_fail(nice_agent); g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); - session = get_session(transport_agent, stream_id); - g_return_if_fail(OWR_IS_SESSION(session)); - - args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(session)); + args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(transport_agent)); g_hash_table_insert(args, "transport-agent", transport_agent); - g_hash_table_insert(args, "session", session); + g_hash_table_insert(args, "stream-id", GUINT_TO_POINTER(stream_id)); _owr_schedule_with_hash_table((GSourceFunc)emit_candidate_gathering_done, args); @@ -1965,20 +2111,25 @@ static void on_candidate_gathering_done(NiceAgent *nice_agent, guint stream_id, static gboolean emit_ice_state_changed(GHashTable *args) { - OwrSession *session; - guint session_id; + OwrTransportAgent *transport_agent; + guint stream_id; OwrComponentType component_type; OwrIceState state; + GSList *sessions; - session = g_hash_table_lookup(args, "session"); - session_id = GPOINTER_TO_UINT(g_hash_table_lookup(args, "session-id")); + transport_agent = g_hash_table_lookup(args, "transport-agent"); + stream_id = GPOINTER_TO_UINT(g_hash_table_lookup(args, "stream-id")); component_type = GPOINTER_TO_UINT(g_hash_table_lookup(args, "component-type")); state = GPOINTER_TO_UINT(g_hash_table_lookup(args, "ice-state")); - _owr_session_emit_ice_state_changed(session, session_id, component_type, state); + sessions = get_sessions_from_stream_id(transport_agent, stream_id); + for (GSList *walk = sessions; walk; walk = g_slist_next(walk)) { + OwrSession *session = OWR_SESSION(walk->data); + _owr_session_emit_ice_state_changed(session, stream_id, component_type, state); + } + g_slist_free_full(sessions, g_object_unref); g_hash_table_destroy(args); - g_object_unref(session); return FALSE; } @@ -1986,18 +2137,14 @@ static gboolean emit_ice_state_changed(GHashTable *args) static void on_component_state_changed(NiceAgent *nice_agent, guint stream_id, guint component_id, OwrIceState state, OwrTransportAgent *transport_agent) { - OwrSession *session; GHashTable *args; g_return_if_fail(nice_agent); g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); - session = get_session(transport_agent, stream_id); - g_return_if_fail(OWR_IS_SESSION(session)); - - args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(session)); - g_hash_table_insert(args, "session", session); - g_hash_table_insert(args, "session-id", GUINT_TO_POINTER(stream_id)); + args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(transport_agent)); + g_hash_table_insert(args, "transport-agent", transport_agent); + g_hash_table_insert(args, "stream-id", GUINT_TO_POINTER(stream_id)); g_hash_table_insert(args, "component-type", GUINT_TO_POINTER(component_id)); g_hash_table_insert(args, "ice-state", GUINT_TO_POINTER(state)); @@ -2010,8 +2157,7 @@ static void on_new_selected_pair(NiceAgent *nice_agent, NiceCandidate *lcandidate, NiceCandidate *rcandidate, OwrTransportAgent *transport_agent) { - OwrSession *session; - PendingSessionInfo *pending_session_info; + GSList *sessions; OWR_UNUSED(nice_agent); OWR_UNUSED(lcandidate); @@ -2019,70 +2165,78 @@ static void on_new_selected_pair(NiceAgent *nice_agent, g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); - session = get_session(transport_agent, stream_id); - g_return_if_fail(OWR_IS_SESSION(session)); - - g_mutex_lock(&transport_agent->priv->sessions_lock); - pending_session_info = g_hash_table_lookup(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(stream_id)); - if (pending_session_info) { - if (component_id == NICE_COMPONENT_TYPE_RTP && pending_session_info->nice_src_block_rtp) { - GstPad *pad; - gboolean sync_ok, link_ok; - - link_ok = gst_element_link(pending_session_info->nice_src_rtp, pending_session_info->dtls_dec_rtp); - g_warn_if_fail(link_ok); - - gst_element_set_locked_state(pending_session_info->dtls_dec_rtp, FALSE); - sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_dec_rtp); - g_warn_if_fail(sync_ok); - gst_element_set_locked_state(pending_session_info->dtls_enc_rtp, FALSE); - sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_enc_rtp); - g_warn_if_fail(sync_ok); - - pad = gst_element_get_static_pad(pending_session_info->nice_src_rtp, "src"); - gst_pad_remove_probe(pad, pending_session_info->nice_src_block_rtp); - gst_object_unref(pad); - pending_session_info->nice_src_block_rtp = 0; - } + sessions = get_sessions_from_stream_id(transport_agent, stream_id); + for (GSList *walk = sessions; walk; walk = g_slist_next(walk)) { + OwrSession *session = OWR_SESSION(walk->data); + guint session_id = get_session_id(transport_agent, session); + PendingSessionInfo *pending_session_info; + + AGENT_SESSIONS_LOCK(transport_agent); + pending_session_info = g_hash_table_lookup(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(session_id)); + if (pending_session_info) { + if (component_id == NICE_COMPONENT_TYPE_RTP && pending_session_info->nice_src_block_rtp) { + GstPad *pad; + gboolean sync_ok, link_ok; + + link_ok = gst_element_link(pending_session_info->nice_src_rtp, pending_session_info->dtls_dec_rtp); + g_warn_if_fail(link_ok); + + gst_element_set_locked_state(pending_session_info->dtls_dec_rtp, FALSE); + sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_dec_rtp); + g_warn_if_fail(sync_ok); + gst_element_set_locked_state(pending_session_info->dtls_enc_rtp, FALSE); + sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_enc_rtp); + g_warn_if_fail(sync_ok); + + pad = gst_element_get_static_pad(pending_session_info->nice_src_rtp, "src"); + gst_pad_remove_probe(pad, pending_session_info->nice_src_block_rtp); + gst_object_unref(pad); + pending_session_info->nice_src_block_rtp = 0; + } - /* when doing standalone RTCP we unblock the RTCP bin and sync its state whenever the first component of - * the stream is connected. This is because the actual RTCP connection might be established - * much later (or even never). Also see link_rtpbin_to_send_output_bin. - */ - if (pending_session_info->nice_src_block_rtcp) { - GstPad *pad; - gboolean sync_ok, link_ok; - - link_ok = gst_element_link(pending_session_info->nice_src_rtcp, pending_session_info->dtls_dec_rtcp); - g_warn_if_fail(link_ok); - - gst_element_set_locked_state(pending_session_info->dtls_dec_rtcp, FALSE); - sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_dec_rtcp); - g_warn_if_fail(sync_ok); - gst_element_set_locked_state(pending_session_info->dtls_enc_rtcp, FALSE); - sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_enc_rtcp); - g_warn_if_fail(sync_ok); - - pad = gst_element_get_static_pad(pending_session_info->nice_src_rtcp, "src"); - gst_pad_remove_probe(pad, pending_session_info->nice_src_block_rtcp); - gst_object_unref(pad); - pending_session_info->nice_src_block_rtcp = 0; + /* when doing standalone RTCP we unblock the RTCP bin and sync its state whenever the first component of + * the stream is connected. This is because the actual RTCP connection might be established + * much later (or even never). Also see link_rtpbin_to_send_output_bin. + */ + if (pending_session_info->nice_src_block_rtcp) { + GstPad *pad; + gboolean sync_ok, link_ok; + + link_ok = gst_element_link(pending_session_info->nice_src_rtcp, pending_session_info->dtls_dec_rtcp); + g_warn_if_fail(link_ok); + + gst_element_set_locked_state(pending_session_info->dtls_dec_rtcp, FALSE); + sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_dec_rtcp); + g_warn_if_fail(sync_ok); + gst_element_set_locked_state(pending_session_info->dtls_enc_rtcp, FALSE); + sync_ok = gst_element_sync_state_with_parent(pending_session_info->dtls_enc_rtcp); + g_warn_if_fail(sync_ok); + + pad = gst_element_get_static_pad(pending_session_info->nice_src_rtcp, "src"); + gst_pad_remove_probe(pad, pending_session_info->nice_src_block_rtcp); + gst_object_unref(pad); + pending_session_info->nice_src_block_rtcp = 0; + } } + AGENT_SESSIONS_UNLOCK(transport_agent); } - g_mutex_unlock(&transport_agent->priv->sessions_lock); + g_slist_free_full(sessions, g_object_unref); + } static gboolean maybe_handle_new_send_source_with_payload_from_main_thread(GHashTable *args) { OwrTransportAgent *transport_agent; OwrMediaSession *session; + guint process_bundled_sessions; session = OWR_MEDIA_SESSION(g_hash_table_lookup(args, "session")); g_return_val_if_fail(OWR_IS_MEDIA_SESSION(session), FALSE); transport_agent = OWR_TRANSPORT_AGENT(g_hash_table_lookup(args, "transport_agent")); g_return_val_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent), FALSE); + process_bundled_sessions = GPOINTER_TO_UINT(g_hash_table_lookup(args, "process_bundled_sessions")); - maybe_handle_new_send_source_with_payload(transport_agent, session); + maybe_handle_new_send_source_with_payload(transport_agent, session, process_bundled_sessions); g_hash_table_destroy(args); g_object_unref(session); @@ -2096,67 +2250,155 @@ on_dtls_enc_key_set(GstElement *dtls_srtp_enc, AgentAndSessionIdPair *data) { OwrTransportAgent *transport_agent = data->transport_agent; OwrSession *session; - guint stream_id = data->session_id; + guint session_id = data->session_id; PendingSessionInfo *pending_session_info; - session = get_session(transport_agent, stream_id); + session = get_session(transport_agent, session_id); g_return_if_fail(session); /* Once we have the key, the DTLS handshake is done and we can start sending data here. Note * that we only wait for the DTLS handshake to be completed for the RTP component. */ - g_mutex_lock(&transport_agent->priv->sessions_lock); - pending_session_info = g_hash_table_lookup(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(stream_id)); + AGENT_SESSIONS_LOCK(transport_agent); + pending_session_info = g_hash_table_lookup(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(session_id)); /* FIXME: What to do about RTCP? It's not guaranteed to ever be enabled if * RTCP muxing is used but the usage wasn't known beforehand */ if (pending_session_info && dtls_srtp_enc == pending_session_info->dtls_enc_rtp) { GHashTable *args; + gboolean process_bundled_sessions = transport_agent->priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE; - g_hash_table_remove(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(stream_id)); + if (!process_bundled_sessions) + g_hash_table_remove(transport_agent->priv->pending_sessions, GUINT_TO_POINTER(session_id)); args = g_hash_table_new(g_str_hash, g_str_equal); g_hash_table_insert(args, "session", session); g_hash_table_insert(args, "transport_agent", g_object_ref(transport_agent)); + g_hash_table_insert(args, "process_bundled_sessions", GUINT_TO_POINTER(process_bundled_sessions)); _owr_schedule_with_hash_table((GSourceFunc)maybe_handle_new_send_source_with_payload_from_main_thread, args); } - g_mutex_unlock(&transport_agent->priv->sessions_lock); + AGENT_SESSIONS_UNLOCK(transport_agent); g_object_unref(session); } -static guint get_stream_id(OwrTransportAgent *transport_agent, OwrSession *session) +static guint get_session_id_unlocked(OwrTransportAgent *transport_agent, OwrSession *session) { GHashTableIter iter; OwrSession *s; - gpointer stream_id = GUINT_TO_POINTER(0); + gpointer session_id = GUINT_TO_POINTER(0); - g_mutex_lock(&transport_agent->priv->sessions_lock); g_hash_table_iter_init(&iter, transport_agent->priv->sessions); - while (g_hash_table_iter_next(&iter, &stream_id, (gpointer)&s)) { + while (g_hash_table_iter_next(&iter, &session_id, (gpointer)&s)) { if (s == session) { - g_mutex_unlock(&transport_agent->priv->sessions_lock); - return GPOINTER_TO_UINT(stream_id); + return GPOINTER_TO_UINT(session_id); } } - g_mutex_unlock(&transport_agent->priv->sessions_lock); - g_warn_if_reached(); return 0; } -static OwrSession * get_session(OwrTransportAgent *transport_agent, guint stream_id) +static guint get_session_id(OwrTransportAgent *transport_agent, OwrSession *session) +{ + guint session_id = 0; + + AGENT_SESSIONS_LOCK(transport_agent); + session_id = get_session_id_unlocked(transport_agent, session); + AGENT_SESSIONS_UNLOCK(transport_agent); + + return session_id; +} + +static OwrSession * get_session_unlocked(OwrTransportAgent *transport_agent, guint session_id) { OwrSession *s; - g_mutex_lock(&transport_agent->priv->sessions_lock); - s = OWR_SESSION(g_hash_table_lookup(transport_agent->priv->sessions, GUINT_TO_POINTER(stream_id))); + s = OWR_SESSION(g_hash_table_lookup(transport_agent->priv->sessions, GUINT_TO_POINTER(session_id))); if (s) g_object_ref(s); - g_mutex_unlock(&transport_agent->priv->sessions_lock); return s; } +static OwrSession * get_session(OwrTransportAgent *transport_agent, guint session_id) +{ + OwrSession *s; + + AGENT_SESSIONS_LOCK(transport_agent); + s = get_session_unlocked(transport_agent, session_id); + AGENT_SESSIONS_UNLOCK(transport_agent); + + return s; +} + +/* FIXME: for bundling it must return a GList of sessions. */ +static OwrSession * get_session_from_stream_id(OwrTransportAgent *transport_agent, guint stream_id) +{ + GHashTableIter iter; + OwrSession *session; + gpointer session_id = GUINT_TO_POINTER(0); + + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_iter_init(&iter, transport_agent->priv->sessions); + while (g_hash_table_iter_next(&iter, &session_id, (gpointer)&session)) { + if (_owr_session_get_stream_id(session) == stream_id) { + AGENT_SESSIONS_UNLOCK(transport_agent); + g_object_ref(session); + return session; + } + } + AGENT_SESSIONS_UNLOCK(transport_agent); + + g_warn_if_reached(); + return NULL; +} + +static GSList * get_sessions_from_stream_id(OwrTransportAgent *transport_agent, guint stream_id) +{ + GSList *sessions = NULL; + GHashTableIter iter; + OwrSession *session; + gpointer session_id = GUINT_TO_POINTER(0); + + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_iter_init(&iter, transport_agent->priv->sessions); + while (g_hash_table_iter_next(&iter, &session_id, (gpointer)&session)) { + if (_owr_session_get_stream_id(session) == stream_id) { + sessions = g_slist_prepend(sessions, g_object_ref(session)); + } + } + sessions = g_slist_reverse(sessions); + AGENT_SESSIONS_UNLOCK(transport_agent); + + return sessions; +} + +static OwrPayload * get_payload(OwrTransportAgent *transport_agent, guint pt, OwrMediaSession **media_session) +{ + GHashTableIter iter; + OwrSession *s; + gpointer session_id = GUINT_TO_POINTER(0); + OwrPayload *payload = NULL; + + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_iter_init(&iter, transport_agent->priv->sessions); + while (g_hash_table_iter_next(&iter, &session_id, (gpointer)&s)) { + if (!OWR_IS_MEDIA_SESSION(s)) + continue; + + payload = _owr_media_session_get_receive_payload(OWR_MEDIA_SESSION(s), pt); + if (payload) { + if (media_session) + *media_session = g_object_ref(OWR_MEDIA_SESSION(s)); + AGENT_SESSIONS_UNLOCK(transport_agent); + return payload; + } + } + AGENT_SESSIONS_UNLOCK(transport_agent); + + g_warn_if_reached(); + return NULL; +} + static void update_flip_method(OwrPayload *payload, GParamSpec *pspec, GstElement *flip) { guint rotation = 0; @@ -2198,6 +2440,7 @@ static void on_caps(GstElement *sink, GParamSpec *pspec, OwrSession *session) static void handle_new_send_payload(OwrTransportAgent *transport_agent, OwrMediaSession *media_session, OwrPayload * payload) { guint stream_id; + guint session_id; GstElement *send_input_bin = NULL; GstElement *encoder = NULL, *parser = NULL, *payloader = NULL, *rtp_capsfilter = NULL, *rtpbin = NULL; @@ -2215,25 +2458,26 @@ static void handle_new_send_payload(OwrTransportAgent *transport_agent, OwrMedia g_return_if_fail(media_session); g_assert(payload); - stream_id = get_stream_id(transport_agent, OWR_SESSION(media_session)); - g_return_if_fail(stream_id); + stream_id = _owr_session_get_stream_id(OWR_SESSION(media_session)); + session_id = get_session_id_unlocked(transport_agent, OWR_SESSION(media_session)); - name = g_strdup_printf("send-input-bin-%u", stream_id); + name = g_strdup_printf("send-input-bin-%u-%u", session_id, stream_id); send_input_bin = gst_bin_new(name); - g_free(name); gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), send_input_bin); if (!gst_element_sync_state_with_parent(send_input_bin)) { - GST_ERROR("Failed to sync send-input-bin-%u state with parent", stream_id); + GST_ERROR("Failed to sync %s state with parent", name); + g_free(name); return; } + g_free(name); rtpbin = transport_agent->priv->rtpbin; - name = g_strdup_printf("send_rtp_sink_%u", stream_id); + name = g_strdup_printf("send_rtp_sink_%u", session_id); rtp_sink_pad = gst_element_get_request_pad(rtpbin, name); g_free(name); - link_rtpbin_to_send_output_bin(transport_agent, stream_id, TRUE, TRUE); + link_rtpbin_to_send_output_bin(transport_agent, session_id, stream_id, TRUE, TRUE); g_object_get(payload, "media-type", &media_type, NULL); @@ -2389,8 +2633,7 @@ static void on_new_remote_candidate(OwrTransportAgent *transport_agent, gboolean g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); g_return_if_fail(OWR_IS_SESSION(session)); - stream_id = get_stream_id(transport_agent, session); - g_return_if_fail(stream_id); + stream_id = _owr_session_get_stream_id(session); item = forced ? _owr_session_get_forced_remote_candidates(session) : _owr_session_get_remote_candidates(session); @@ -2447,8 +2690,7 @@ static void on_local_candidate_change(OwrTransportAgent *transport_agent, OwrCan g_return_if_fail(OWR_IS_CANDIDATE(candidate)); g_return_if_fail(OWR_IS_SESSION(session)); - stream_id = get_stream_id(transport_agent, session); - g_return_if_fail(stream_id); + stream_id = _owr_session_get_stream_id(session); g_object_get(G_OBJECT(candidate), "ufrag", &ufrag, "password", &password, NULL); nice_agent_set_local_credentials(transport_agent->priv->nice_agent, stream_id, ufrag, password); @@ -2473,7 +2715,7 @@ static gboolean emit_on_incoming_source(GHashTable *args) } static void signal_incoming_source(OwrMediaType type, OwrTransportAgent *transport_agent, - guint stream_id, OwrCodecType codec_type) + guint session_id, guint stream_id, OwrCodecType codec_type) { OwrMediaSession *media_session; OwrMediaSource *source; @@ -2481,9 +2723,9 @@ static void signal_incoming_source(OwrMediaType type, OwrTransportAgent *transpo g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); - media_session = OWR_MEDIA_SESSION(get_session(transport_agent, stream_id)); + media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); - source = _owr_remote_media_source_new(type, stream_id, codec_type, + source = _owr_remote_media_source_new(type, session_id, stream_id, codec_type, transport_agent->priv->transport_bin); g_return_if_fail(source); @@ -2502,6 +2744,7 @@ static void on_transport_bin_pad_added(GstElement *transport_bin, GstPad *new_pa OwrMediaType media_type = OWR_MEDIA_TYPE_UNKNOWN; OwrCodecType codec_type = OWR_CODEC_TYPE_NONE; guint stream_id = 0; + guint session_id = 0; g_return_if_fail(GST_IS_BIN(transport_bin)); g_return_if_fail(GST_IS_PAD(new_pad)); @@ -2512,14 +2755,14 @@ static void on_transport_bin_pad_added(GstElement *transport_bin, GstPad *new_pa if (g_str_has_prefix(new_pad_name, "audio_raw_src")) { media_type = OWR_MEDIA_TYPE_AUDIO; codec_type = OWR_CODEC_TYPE_NONE; - sscanf(new_pad_name, "audio_raw_src_%u", &stream_id); + sscanf(new_pad_name, "audio_raw_src_%u_%u", &session_id, &stream_id); } else if (g_str_has_prefix(new_pad_name, "video_src_")) { media_type = OWR_MEDIA_TYPE_VIDEO; - sscanf(new_pad_name, "video_src_%u_%u", &codec_type, &stream_id); + sscanf(new_pad_name, "video_src_%u_%u_%u", &codec_type, &session_id, &stream_id); } if (media_type != OWR_MEDIA_TYPE_UNKNOWN && codec_type == OWR_CODEC_TYPE_NONE) - signal_incoming_source(media_type, transport_agent, stream_id, codec_type); + signal_incoming_source(media_type, transport_agent, session_id, stream_id, codec_type); g_free(new_pad_name); } @@ -2534,46 +2777,55 @@ static void on_rtpbin_pad_added(GstElement *rtpbin, GstPad *new_pad, OwrTranspor new_pad_name = gst_pad_get_name(new_pad); if (g_str_has_prefix(new_pad_name, "recv_rtp_src_")) { - guint32 session_id = 0, ssrc = 0, pt = 0; + guint32 stream_id = 0, ssrc = 0, pt = 0, session_id = 0; OwrMediaSession *media_session = NULL; OwrPayload *payload = NULL; OwrMediaType media_type; sscanf(new_pad_name, "recv_rtp_src_%u_%u_%u", &session_id, &ssrc, &pt); + g_free(new_pad_name); media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); + g_return_if_fail(OWR_IS_MEDIA_SESSION(media_session)); + + stream_id = _owr_session_get_stream_id(OWR_SESSION(media_session)); payload = _owr_media_session_get_receive_payload(media_session, pt); - g_return_if_fail(OWR_IS_MEDIA_SESSION(media_session)); g_return_if_fail(OWR_IS_PAYLOAD(payload)); + g_object_get(payload, "media-type", &media_type, NULL); g_object_set_data(G_OBJECT(media_session), "ssrc", GUINT_TO_POINTER(ssrc)); + session_id = get_session_id(transport_agent, OWR_SESSION(media_session)); + if (media_type == OWR_MEDIA_TYPE_VIDEO) - setup_video_receive_elements(new_pad, session_id, payload, transport_agent); + setup_video_receive_elements(new_pad, session_id, stream_id, payload, transport_agent); else - setup_audio_receive_elements(new_pad, session_id, payload, transport_agent); + setup_audio_receive_elements(new_pad, session_id, stream_id, payload, transport_agent); /* Hook up RTCP sending if it isn't already */ - link_rtpbin_to_send_output_bin(transport_agent, session_id, FALSE, TRUE); + link_rtpbin_to_send_output_bin(transport_agent, session_id, stream_id, FALSE, TRUE); - g_object_unref(media_session); + if (media_session) + g_object_unref(media_session); g_object_unref(payload); } else if (g_str_has_prefix(new_pad_name, "send_rtp_src")) { - guint32 session_id = 0; - sscanf(new_pad_name, "send_rtp_src_%u", &session_id); + guint32 stream_id = 0; + sscanf(new_pad_name, "send_rtp_src_%u", &stream_id); + g_free(new_pad_name); } - g_free(new_pad_name); } typedef struct { OwrSession *session; - guint session_id; + guint stream_id; } SessionData; -static void session_data_free(gpointer session_data) +static void session_data_free(gpointer user_data) { + SessionData *session_data = (SessionData *)user_data; + g_object_unref(session_data->session); g_slice_free(SessionData, session_data); } @@ -2585,14 +2837,14 @@ static GstPadProbeReturn check_for_keyframe(GstPad *pad, GstPadProbeInfo *info, if (!GST_BUFFER_FLAG_IS_SET(info->data, GST_BUFFER_FLAG_DELTA_UNIT)) { GST_CAT_INFO_OBJECT(_owrsession_debug, session_data->session, - "Session %u, Received keyframe for %u\n", session_data->session_id, + "Session %u, Received keyframe for %u", session_data->stream_id, GPOINTER_TO_UINT(g_object_get_data(G_OBJECT(session_data->session), "ssrc"))); } return GST_PAD_PROBE_OK; } -static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, OwrPayload *payload, OwrTransportAgent *transport_agent) +static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, guint32 stream_id, OwrPayload *payload, OwrTransportAgent *transport_agent) { GstPad *depay_sink_pad = NULL, *ghost_pad = NULL; gboolean sync_ok = TRUE; @@ -2605,23 +2857,23 @@ static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, Ow GstPad *pad; SessionData *session_data; - g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "receive-output-bin-%u", session_id); + g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "receive-output-bin-%u-%u", session_id, stream_id); receive_output_bin = gst_bin_new(name); gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), receive_output_bin); if (!gst_element_sync_state_with_parent(receive_output_bin)) { - GST_ERROR("Failed to sync receive-output-bin-%u state with parent", session_id); + GST_ERROR("Failed to sync %s state with parent", name); return; } rtpdepay = _owr_payload_create_payload_depacketizer(payload); - g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "videorepair1_%u", session_id); + g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "videorepair1_%u", stream_id); videorepair1 = gst_element_factory_make("videorepair", name); pad = gst_element_get_static_pad(videorepair1, "src"); session_data = g_slice_new(SessionData); session_data->session = get_session(transport_agent, session_id); - session_data->session_id = session_id; + session_data->stream_id = stream_id; gst_pad_add_probe(pad, GST_PAD_PROBE_TYPE_BUFFER, check_for_keyframe, session_data, session_data_free); gst_object_unref(pad); @@ -2654,13 +2906,13 @@ static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, Ow g_warn_if_fail(sync_ok); pad = gst_element_get_static_pad(decoder, "src"); - g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "video_src_%u_%u", OWR_CODEC_TYPE_NONE, - session_id); + g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "video_src_%u_%u_%u", OWR_CODEC_TYPE_NONE, + session_id, stream_id); add_pads_to_bin_and_transport_bin(pad, receive_output_bin, transport_agent->priv->transport_bin, name); gst_object_unref(pad); } -static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, OwrPayload *payload, OwrTransportAgent *transport_agent) +static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, guint32 stream_id, OwrPayload *payload, OwrTransportAgent *transport_agent) { GstElement *receive_output_bin; gchar *pad_name = NULL; @@ -2671,7 +2923,7 @@ static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, Ow gboolean link_ok = FALSE; gboolean sync_ok = TRUE; - pad_name = g_strdup_printf("receive-output-bin-%u", session_id); + pad_name = g_strdup_printf("receive-output-bin-%u-%u", session_id, stream_id); receive_output_bin = gst_bin_new(pad_name); g_free(pad_name); pad_name = NULL; @@ -2679,7 +2931,7 @@ static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, Ow gst_bin_add(GST_BIN(transport_agent->priv->transport_bin), receive_output_bin); sync_ok &= gst_element_sync_state_with_parent(receive_output_bin); - element_name = g_strdup_printf("recv-rtp-capsfilter-%u", session_id); + element_name = g_strdup_printf("recv-rtp-capsfilter-%u", stream_id); rtp_capsfilter = gst_element_factory_make("capsfilter", element_name); g_free(element_name); rtp_caps = _owr_payload_create_rtp_caps(payload); @@ -2706,7 +2958,7 @@ static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, Ow ghost_pad = ghost_pad_and_add_to_bin(rtp_caps_sink_pad, receive_output_bin, "sink"); gst_object_unref(rtp_caps_sink_pad); if (!GST_PAD_LINK_SUCCESSFUL(gst_pad_link(new_pad, ghost_pad))) { - GST_ERROR("Failed to link rtpbin with receive-output-bin-%u", session_id); + GST_ERROR("Failed to link rtpbin with receive-output-bin-%u", stream_id); return; } ghost_pad = NULL; @@ -2719,7 +2971,7 @@ static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, Ow g_warn_if_fail(sync_ok); pad = gst_element_get_static_pad(decoder, "src"); - pad_name = g_strdup_printf("audio_raw_src_%u", session_id); + pad_name = g_strdup_printf("audio_raw_src_%u_%u", session_id, stream_id); add_pads_to_bin_and_transport_bin(pad, receive_output_bin, transport_agent->priv->transport_bin, pad_name); gst_object_unref(pad); @@ -2736,16 +2988,22 @@ static GstCaps * on_rtpbin_request_pt_map(GstElement *rtpbin, guint session_id, g_return_val_if_fail(rtpbin, NULL); g_return_val_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent), NULL); - media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); - g_return_val_if_fail(OWR_IS_MEDIA_SESSION(media_session), NULL); + if (transport_agent->priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) { + payload = get_payload(transport_agent, pt, NULL); + } else { + media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); + g_return_val_if_fail(OWR_IS_MEDIA_SESSION(media_session), NULL); - payload = _owr_media_session_get_receive_payload(media_session, pt); + payload = _owr_media_session_get_receive_payload(media_session, pt); + } if (payload) { caps = _owr_payload_create_rtp_caps(payload); g_object_unref(payload); } - g_object_unref(media_session); + + if (media_session) + g_object_unref(media_session); return caps; } @@ -2758,6 +3016,7 @@ static GstElement * create_aux_bin(gchar *prefix, GstElement *rtx, guint session tmp = g_strdup_printf("%s_%u", prefix, session_id); bin = gst_bin_new(tmp); + GST_DEBUG("Retransmission auxiliary bin created: %s", tmp); g_free(tmp); gst_bin_add(GST_BIN(bin), rtx); @@ -2791,7 +3050,7 @@ static GstElement * on_rtpbin_request_aux_sender(G_GNUC_UNUSED GstElement *rtpbi guint pt, rtx_time; gchar *tmp; - media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); + media_session = OWR_MEDIA_SESSION(get_session_unlocked(transport_agent, session_id)); g_return_val_if_fail(media_session, NULL); payload = _owr_media_session_get_send_payload(media_session); @@ -2827,21 +3086,35 @@ static GstElement * on_rtpbin_request_aux_sender(G_GNUC_UNUSED GstElement *rtpbi return create_aux_bin("rtprtxsend", rtxsend, session_id); no_retransmission: + GST_DEBUG("Retransmission support disabled on sending side"); return NULL; } -static GstElement * on_rtpbin_request_aux_receiver(G_GNUC_UNUSED GstElement *rtpbin, G_GNUC_UNUSED guint session_id, OwrTransportAgent *transport_agent) +static GstElement * on_rtpbin_request_aux_receiver(G_GNUC_UNUSED GstElement *rtpbin, guint session_id, OwrTransportAgent *transport_agent) { OwrMediaSession *media_session; - GstElement *rtxrecv; + GstElement *rtxrecv = NULL; GstStructure *pt_map; - media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); + media_session = OWR_MEDIA_SESSION(get_session_unlocked(transport_agent, session_id)); g_return_val_if_fail(media_session, NULL); pt_map = _owr_media_session_get_receive_rtx_pt_map(media_session); g_object_unref(media_session); + if ((transport_agent->priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) && !pt_map) { + GST_DEBUG("no valid pt_map found, looking for one in the staged sessions"); + for (GSList* walk = transport_agent->priv->unstarted_sessions; walk; walk = g_slist_next(walk)) { + OwrSession* session = OWR_SESSION(walk->data); + if (!OWR_IS_MEDIA_SESSION(session)) + continue; + pt_map = _owr_media_session_get_receive_rtx_pt_map(OWR_MEDIA_SESSION(session)); + GST_DEBUG("pt_map found on staged session: %p", pt_map); + if (pt_map) + break; + } + } + if (!pt_map) goto no_retransmission; @@ -2855,13 +3128,14 @@ static GstElement * on_rtpbin_request_aux_receiver(G_GNUC_UNUSED GstElement *rtp return create_aux_bin("rtprtxrecv", rtxrecv, session_id); no_retransmission: + GST_DEBUG("Retransmission support disabled on receiving side"); return NULL; } -static void print_rtcp_type(GObject *session, guint session_id, +static void print_rtcp_type(GObject *session, guint stream_id, GstRTCPType packet_type) { - GST_CAT_DEBUG_OBJECT(_owrsession_debug, session, "Session %u, Received RTCP %s\n", session_id, + GST_CAT_DEBUG_OBJECT(_owrsession_debug, session, "Session %u, Received RTCP %s", stream_id, packet_type == GST_RTCP_TYPE_INVALID ? "Invalid type (INVALID)" : packet_type == GST_RTCP_TYPE_SR ? "Sender Report (SR)" : packet_type == GST_RTCP_TYPE_RR ? "Receiver Report (RR)" : @@ -2873,77 +3147,77 @@ static void print_rtcp_type(GObject *session, guint session_id, "unknown"); } -static void print_rtcp_feedback_type(GObject *session, guint session_id, +static void print_rtcp_feedback_type(GObject *session, guint stream_id, guint fbtype, guint media_ssrc, GstRTCPType packet_type, guint8 *fci, gboolean is_received) { if (fbtype == GST_RTCP_FB_TYPE_INVALID) { - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Invalid type\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Invalid type", + stream_id, is_received ? "Received" : "Sent", media_ssrc); } else if (packet_type == GST_RTCP_TYPE_RTPFB) { switch (fbtype) { case GST_RTCP_RTPFB_TYPE_NACK: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Generic NACK\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Generic NACK", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_RTPFB_TYPE_TMMBR: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporary Maximum Media Stream Bit Rate Request\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporary Maximum Media Stream Bit Rate Request", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_RTPFB_TYPE_TMMBN: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporary Maximum Media Stream Bit Rate Notification\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporary Maximum Media Stream Bit Rate Notification", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_RTPFB_TYPE_RTCP_SR_REQ: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Request an SR packet for early synchronization\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Request an SR packet for early synchronization", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_RTPFB_TYPE_SCREAM: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: SCReAM\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: SCReAM", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; default: - GST_CAT_WARNING_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Unknown feedback type %u\n", - session_id, is_received ? "Received" : "Sent", media_ssrc, fbtype); + GST_CAT_WARNING_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Unknown feedback type %u", + stream_id, is_received ? "Received" : "Sent", media_ssrc, fbtype); break; } } else if (packet_type == GST_RTCP_TYPE_PSFB) { switch (fbtype) { case GST_RTCP_PSFB_TYPE_PLI: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Picture Loss Indication\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Picture Loss Indication", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_PSFB_TYPE_SLI: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Slice Loss Indication\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Slice Loss Indication", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_PSFB_TYPE_RPSI: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Reference Picture Selection Indication\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Reference Picture Selection Indication", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_PSFB_TYPE_AFB: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Application layer Feedback\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Application layer Feedback", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_PSFB_TYPE_FIR: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Full Intra Request Command\n", - session_id, is_received ? "Received" : "Sent", fci ? GST_READ_UINT32_BE(fci) : 0); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Full Intra Request Command", + stream_id, is_received ? "Received" : "Sent", fci ? GST_READ_UINT32_BE(fci) : 0); break; case GST_RTCP_PSFB_TYPE_TSTR: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporal-Spatial Trade-off Request\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporal-Spatial Trade-off Request", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_PSFB_TYPE_TSTN: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporal-Spatial Trade-off Notification\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Temporal-Spatial Trade-off Notification", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; case GST_RTCP_PSFB_TYPE_VBCN: - GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Video Back Channel Message\n", - session_id, is_received ? "Received" : "Sent", media_ssrc); + GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Video Back Channel Message", + stream_id, is_received ? "Received" : "Sent", media_ssrc); break; default: - GST_CAT_WARNING_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Unknown feedback type %u\n", - session_id, is_received ? "Received" : "Sent", media_ssrc, fbtype); + GST_CAT_WARNING_OBJECT(_owrsession_debug, session, "Session %u, %s RTCP feedback for %u: Unknown feedback type %u", + stream_id, is_received ? "Received" : "Sent", media_ssrc, fbtype); break; } } @@ -2958,17 +3232,15 @@ static gboolean on_sending_rtcp(GObject *session, GstBuffer *buffer, gboolean ea GstRTCPType packet_type; gboolean has_packet, do_not_suppress = FALSE; OwrMediaSession *media_session; - OwrPayload *send_payload; - OwrMediaType media_type = -1; GValueArray *sources = NULL; GObject *source = NULL; - guint session_id = 0, rtcp_session_id = 0; + guint stream_id = 0, rtcp_stream_id = 0; GList *it, *next; GHashTable *rtcp_info; OWR_UNUSED(early); - session_id = GPOINTER_TO_UINT(g_object_get_data(session, "session_id")); + stream_id = GPOINTER_TO_UINT(g_object_get_data(session, "stream-id")); if (gst_rtcp_buffer_map(buffer, GST_MAP_READ | GST_MAP_WRITE, &rtcp_buffer)) { guint pt, fmt, ssrc, last_fb_wc, highest_seq, n_loss, n_ecn; @@ -2976,9 +3248,9 @@ static gboolean on_sending_rtcp(GObject *session, GstBuffer *buffer, gboolean ea has_packet = gst_rtcp_buffer_get_first_packet(&rtcp_buffer, &rtcp_packet); for (; has_packet; has_packet = gst_rtcp_packet_move_to_next(&rtcp_packet)) { packet_type = gst_rtcp_packet_get_type(&rtcp_packet); - print_rtcp_type(session, session_id, packet_type); + print_rtcp_type(session, stream_id, packet_type); if (packet_type == GST_RTCP_TYPE_PSFB || packet_type == GST_RTCP_TYPE_RTPFB) { - print_rtcp_feedback_type(session, session_id, gst_rtcp_packet_fb_get_type(&rtcp_packet), + print_rtcp_feedback_type(session, stream_id, gst_rtcp_packet_fb_get_type(&rtcp_packet), gst_rtcp_packet_fb_get_media_ssrc(&rtcp_packet), packet_type, gst_rtcp_packet_fb_get_fci(&rtcp_packet), FALSE); do_not_suppress = TRUE; @@ -2993,10 +3265,10 @@ static gboolean on_sending_rtcp(GObject *session, GstBuffer *buffer, gboolean ea pt = GPOINTER_TO_UINT(g_hash_table_lookup(rtcp_info, "pt")); ssrc = GPOINTER_TO_UINT(g_hash_table_lookup(rtcp_info, "ssrc")); - rtcp_session_id = GPOINTER_TO_UINT(g_hash_table_lookup(rtcp_info, "session-id")); + rtcp_stream_id = GPOINTER_TO_UINT(g_hash_table_lookup(rtcp_info, "stream_id")); if (pt == GST_RTCP_TYPE_RTPFB) { - if (session_id != rtcp_session_id) { + if (stream_id != rtcp_stream_id) { it = g_list_next(it); continue; } @@ -3051,19 +3323,13 @@ static gboolean on_sending_rtcp(GObject *session, GstBuffer *buffer, gboolean ea g_return_val_if_fail(OWR_IS_TRANSPORT_AGENT(agent), do_not_suppress); - media_session = OWR_MEDIA_SESSION(get_session(agent, session_id)); + media_session = OWR_MEDIA_SESSION(g_object_get_data(session, "session")); g_return_val_if_fail(OWR_IS_MEDIA_SESSION(media_session), do_not_suppress); - send_payload = _owr_media_session_get_send_payload(media_session); - if (send_payload) { - g_object_get(send_payload, "media-type", &media_type, NULL); - g_object_unref(send_payload); - } g_object_get(session, "sources", &sources, NULL); source = g_value_get_object(g_value_array_get_nth(sources, 0)); prepare_rtcp_stats(media_session, source); g_value_array_free(sources); - g_object_unref(media_session); return do_not_suppress; } @@ -3075,19 +3341,19 @@ static void on_receiving_rtcp(GObject *session, GstBuffer *buffer, GstRTCPPacket rtcp_packet; GstRTCPType packet_type; gboolean has_packet; - guint session_id = 0; + guint stream_id = 0; OWR_UNUSED(agent); - session_id = GPOINTER_TO_UINT(g_object_get_data(session, "session_id")); + stream_id = GPOINTER_TO_UINT(g_object_get_data(session, "stream-id")); if (gst_rtcp_buffer_map(buffer, GST_MAP_READ, &rtcp_buffer)) { has_packet = gst_rtcp_buffer_get_first_packet(&rtcp_buffer, &rtcp_packet); for (; has_packet; has_packet = gst_rtcp_packet_move_to_next(&rtcp_packet)) { packet_type = gst_rtcp_packet_get_type(&rtcp_packet); - print_rtcp_type(session, session_id, packet_type); + print_rtcp_type(session, stream_id, packet_type); if (packet_type == GST_RTCP_TYPE_PSFB || packet_type == GST_RTCP_TYPE_RTPFB) { - print_rtcp_feedback_type(session, session_id, gst_rtcp_packet_fb_get_type(&rtcp_packet), + print_rtcp_feedback_type(session, stream_id, gst_rtcp_packet_fb_get_type(&rtcp_packet), gst_rtcp_packet_fb_get_media_ssrc(&rtcp_packet), packet_type, gst_rtcp_packet_fb_get_fci(&rtcp_packet), TRUE); break; @@ -3151,6 +3417,7 @@ static void on_ssrc_active(GstElement *rtpbin, guint session_id, guint ssrc, GObject *rtp_session, *rtp_source; g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); + media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); g_return_if_fail(OWR_IS_MEDIA_SESSION(media_session)); @@ -3162,13 +3429,38 @@ static void on_ssrc_active(GstElement *rtpbin, guint session_id, guint ssrc, g_object_unref(media_session); } +static guint on_bundled_ssrc(GstElement *rtpbin, guint ssrc, OwrTransportAgent *transport_agent) +{ + GHashTableIter iter; + OwrSession *session; + gpointer session_id = GUINT_TO_POINTER(0); + guint found_session_id = 0; + + OWR_UNUSED(rtpbin); + + AGENT_SESSIONS_LOCK(transport_agent); + g_hash_table_iter_init(&iter, transport_agent->priv->sessions); + while (g_hash_table_iter_next(&iter, &session_id, (gpointer)&session)) { + guint receive_ssrc, receive_rtx_ssrc; + if (!OWR_IS_MEDIA_SESSION(session)) + continue; + g_object_get(OWR_MEDIA_SESSION(session), "receive-ssrc", &receive_ssrc, "receive-rtx-ssrc", &receive_rtx_ssrc, NULL); + if ((receive_rtx_ssrc == ssrc) || (receive_ssrc == ssrc)) { + found_session_id = GPOINTER_TO_UINT(session_id); + break; + } + } + AGENT_SESSIONS_UNLOCK(transport_agent); + + return found_session_id; +} + static void on_new_jitterbuffer(G_GNUC_UNUSED GstElement *rtpbin, GstElement *jitterbuffer, guint session_id, G_GNUC_UNUSED guint ssrc, OwrTransportAgent *transport_agent) { OwrMediaSession *media_session; g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); - g_return_if_fail(OWR_IS_MEDIA_SESSION(media_session)); if (_owr_media_session_want_receive_rtx(media_session)) g_object_set(jitterbuffer, "do-retransmission", TRUE, NULL); @@ -3205,6 +3497,7 @@ static gboolean create_datachannel(OwrTransportAgent *transport_agent, guint32 s gboolean ordered, negotiated; gint max_packet_life_time, max_packet_retransmits, sctp_stream_id; gchar *protocol, *label; + OwrSession *session; g_object_get(data_channel, "ordered", &ordered, "max-packet-life-time", &max_packet_life_time, "max-retransmits", &max_packet_retransmits, "protocol", &protocol, "negotiated", &negotiated, @@ -3217,7 +3510,7 @@ static gboolean create_datachannel(OwrTransportAgent *transport_agent, guint32 s goto end; } - if (!negotiated && !is_valid_sctp_stream_id(transport_agent, session_id, sctp_stream_id, FALSE)) { + if (!negotiated && !is_valid_sctp_session_id(transport_agent, session_id, sctp_stream_id, FALSE)) { g_warning("Invalid stream_id"); g_free(protocol); g_free(label); @@ -3232,11 +3525,14 @@ static gboolean create_datachannel(OwrTransportAgent *transport_agent, guint32 s goto end; } + session = get_session(transport_agent, session_id); + data_channel_info = g_new0(DataChannel, 1); data_channel_info->state = OWR_DATA_CHANNEL_STATE_CONNECTING; data_channel_info->id = sctp_stream_id; data_channel_info->label = label; data_channel_info->protocol = protocol; + data_channel_info->stream_id = _owr_session_get_stream_id(session); data_channel_info->session_id = session_id; data_channel_info->ctrl_bytes_sent = 0; data_channel_info->negotiated = negotiated; @@ -3287,16 +3583,18 @@ static void sctpdec_pad_added(GstElement *sctpdec, GstPad *sctpdec_srcpad, GstElement *data_sink, *receive_input_bin; GstPad *appsink_sinkpad; GstPadLinkReturn link_ret; - guint sctp_stream_id, session_id; + guint sctp_stream_id, stream_id; gchar *name; GstAppSinkCallbacks callbacks; DataChannel *data_channel_info; gboolean remotely_initiated = FALSE, valid_id; + OwrSession *session; + guint session_id; name = gst_pad_get_name(sctpdec_srcpad); sscanf(name, "src_%u", &sctp_stream_id); g_free(name); - session_id = GPOINTER_TO_UINT(g_object_get_data(G_OBJECT(sctpdec), "session-id")); + stream_id = GPOINTER_TO_UINT(g_object_get_data(G_OBJECT(sctpdec), "stream_id")); g_rw_lock_writer_lock(&priv->data_channels_rw_mutex); data_channel_info = (DataChannel *)g_hash_table_lookup(priv->data_channels, @@ -3305,7 +3603,7 @@ static void sctpdec_pad_added(GstElement *sctpdec, GstPad *sctpdec_srcpad, remotely_initiated = TRUE; data_channel_info = g_new0(DataChannel, 1); data_channel_info->state = OWR_DATA_CHANNEL_STATE_CONNECTING; - data_channel_info->session_id = session_id; + data_channel_info->stream_id = stream_id; data_channel_info->label = NULL; data_channel_info->protocol = NULL; data_channel_info->negotiated = FALSE; @@ -3315,7 +3613,10 @@ static void sctpdec_pad_added(GstElement *sctpdec, GstPad *sctpdec_srcpad, } g_rw_lock_writer_unlock(&priv->data_channels_rw_mutex); - valid_id = is_valid_sctp_stream_id(transport_agent, session_id, sctp_stream_id, + session = get_session_from_stream_id(transport_agent, stream_id); + session_id = get_session_id(transport_agent, session); + + valid_id = is_valid_sctp_session_id(transport_agent, session_id, sctp_stream_id, remotely_initiated); if (!data_channel_info->negotiated && !valid_id) { g_warning("Invalid stream_id"); @@ -3344,7 +3645,7 @@ static void sctpdec_pad_added(GstElement *sctpdec, GstPad *sctpdec_srcpad, data_channel_info->data_sink = data_sink; data_channel_info->id = sctp_stream_id; - name = g_strdup_printf("receive-input-bin-%u", session_id); + name = g_strdup_printf("receive-input-bin-%u", stream_id); receive_input_bin = gst_bin_get_by_name(GST_BIN(priv->transport_bin), name); g_free(name); @@ -3387,8 +3688,8 @@ static void sctpdec_pad_removed(GstElement *sctpdec, GstPad *sctpdec_srcpad, already_closing = TRUE; else { data_channel_info->state = OWR_DATA_CHANNEL_STATE_CLOSING; - data_session = OWR_DATA_SESSION( - get_session(transport_agent, data_channel_info->session_id)); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, + data_channel_info->stream_id)); g_assert(OWR_IS_DATA_SESSION(data_session)); data_channel = _owr_data_session_get_datachannel(data_session, data_channel_info->id); @@ -3440,15 +3741,15 @@ static void sctpdec_pad_removed(GstElement *sctpdec, GstPad *sctpdec_srcpad, static void on_sctp_association_established(GstElement *sctpenc, gboolean established, OwrTransportAgent *transport_agent) { - guint session_id; + guint stream_id; OwrDataSession *data_session; GList *data_channels, *it; transport_agent->priv->data_session_established = established; if (established) { g_log(G_LOG_DOMAIN, G_LOG_LEVEL_INFO, "An SCTP association has been established"); - session_id = GPOINTER_TO_UINT(g_object_get_data(G_OBJECT(sctpenc), "session-id")); - data_session = OWR_DATA_SESSION(get_session(transport_agent, session_id)); + stream_id = GPOINTER_TO_UINT(g_object_get_data(G_OBJECT(sctpenc), "stream_id")); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, stream_id)); g_assert(data_session); data_channels = _owr_data_session_get_datachannels(data_session); @@ -3556,10 +3857,11 @@ static gboolean create_datachannel_appsrc(OwrTransportAgent *transport_agent, g_rw_lock_reader_unlock(&priv->data_channels_rw_mutex); g_assert(data_channel_info); - data_session = OWR_DATA_SESSION(get_session(transport_agent, data_channel_info->session_id)); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, + data_channel_info->stream_id)); g_return_val_if_fail(data_session, FALSE); - name = g_strdup_printf("send-output-bin-%u", data_channel_info->session_id); + name = g_strdup_printf("send-output-bin-%u", data_channel_info->stream_id); send_output_bin = gst_bin_get_by_name(GST_BIN(priv->transport_bin), name); g_return_val_if_fail(send_output_bin, FALSE); g_free(name); @@ -3613,7 +3915,7 @@ static gboolean create_datachannel_appsrc(OwrTransportAgent *transport_agent, return result; } -static gboolean is_valid_sctp_stream_id(OwrTransportAgent *transport_agent, guint32 session_id, +static gboolean is_valid_sctp_session_id(OwrTransportAgent *transport_agent, guint32 session_id, guint16 sctp_stream_id, gboolean remotly_initiated) { gboolean is_client = FALSE; @@ -3731,12 +4033,13 @@ static void handle_data_channel_open_request(OwrTransportAgent *transport_agent, g_log(G_LOG_DOMAIN, G_LOG_LEVEL_INFO, "Received data channel open request for data channel (%u)" ":\nordered = %u\nmax_packets_life_time = %d\nmax_packet_retransmits = %d\nprotocols = %s" - "\nnegotiated=%u\nlabel= %s\n", data_channel_info->id, data_channel_info->ordered, + "\nnegotiated=%u\nlabel= %s", data_channel_info->id, data_channel_info->ordered, data_channel_info->max_packet_life_time, data_channel_info->max_packet_retransmits, data_channel_info->protocol, data_channel_info->negotiated, data_channel_info->label); - data_session = OWR_DATA_SESSION(get_session(transport_agent, data_channel_info->session_id)); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, + data_channel_info->stream_id)); g_object_ref(data_session); args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(data_session)); g_hash_table_insert(args, "session", data_session); @@ -3790,14 +4093,15 @@ static void handle_data_channel_ack(OwrTransportAgent *transport_agent, guint8 * OWR_UNUSED(data); OWR_UNUSED(size); - g_log(G_LOG_DOMAIN, G_LOG_LEVEL_INFO, "Received ACK for data channel %u\n", sctp_stream_id); + g_log(G_LOG_DOMAIN, G_LOG_LEVEL_INFO, "Received ACK for data channel %u", sctp_stream_id); g_rw_lock_reader_lock(&priv->data_channels_rw_mutex); data_channel_info = g_hash_table_lookup(priv->data_channels, GUINT_TO_POINTER(sctp_stream_id)); g_assert(data_channel_info); g_rw_lock_reader_unlock(&priv->data_channels_rw_mutex); g_rw_lock_writer_lock(&data_channel_info->rw_mutex); - data_session = OWR_DATA_SESSION(get_session(transport_agent, data_channel_info->session_id)); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, + data_channel_info->stream_id)); g_assert(OWR_IS_DATA_SESSION(data_session)); data_channel = _owr_data_session_get_datachannel(data_session, data_channel_info->id); g_assert(OWR_IS_DATA_CHANNEL(data_channel)); @@ -3830,7 +4134,8 @@ static void handle_data_channel_message(OwrTransportAgent *transport_agent, guin goto end; } - data_session = OWR_DATA_SESSION(get_session(transport_agent, data_channel_info->session_id)); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, + data_channel_info->stream_id)); owr_data_channel = _owr_data_session_get_datachannel(data_session, data_channel_info->id); g_assert(owr_data_channel); g_rw_lock_reader_unlock(&data_channel_info->rw_mutex); @@ -3896,7 +4201,7 @@ static void on_new_datachannel(OwrTransportAgent *transport_agent, OwrDataChanne remote_initiated = !!data_channel_info; if (!remote_initiated) { - guint session_id = get_stream_id(transport_agent, OWR_SESSION(data_session)); + guint session_id = get_session_id(transport_agent, OWR_SESSION(data_session)); if (!create_datachannel(transport_agent, session_id, data_channel)) { g_warning("Failed to create new datachannel"); goto end; @@ -3930,7 +4235,7 @@ static void on_new_datachannel(OwrTransportAgent *transport_agent, OwrDataChanne g_free(protocol); g_free(label); - g_log(G_LOG_DOMAIN, G_LOG_LEVEL_INFO, "New data channel (%u) added\n", id); + g_log(G_LOG_DOMAIN, G_LOG_LEVEL_INFO, "New data channel (%u) added", id); g_rw_lock_reader_unlock(&data_channel_info->rw_mutex); } @@ -4052,7 +4357,7 @@ static guint64 on_datachannel_request_bytes_sent(OwrTransportAgent *transport_ag DataChannel *data_channel_info; OwrDataSession *data_session; gchar *name; - guint ctrl_bytes_sent, session_id; + guint ctrl_bytes_sent, stream_id; g_object_get(data_channel, "id", &data_channel_id, NULL); g_rw_lock_reader_lock(&priv->data_channels_rw_mutex); @@ -4062,14 +4367,15 @@ static guint64 on_datachannel_request_bytes_sent(OwrTransportAgent *transport_ag g_rw_lock_reader_lock(&data_channel_info->rw_mutex); ctrl_bytes_sent = data_channel_info->ctrl_bytes_sent; - session_id = data_channel_info->session_id; + stream_id = data_channel_info->stream_id; g_rw_lock_reader_unlock(&data_channel_info->rw_mutex); - name = g_strdup_printf("send-output-bin-%u", session_id); + name = g_strdup_printf("send-output-bin-%u", stream_id); send_output_bin = gst_bin_get_by_name(GST_BIN(priv->transport_bin), name); g_free(name); - data_session = OWR_DATA_SESSION(get_session(transport_agent, session_id)); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, + stream_id)); name = _owr_data_session_get_encoder_name(data_session); sctpenc = gst_bin_get_by_name(GST_BIN(send_output_bin), name); g_free(name); @@ -4094,8 +4400,8 @@ static void maybe_close_data_channel(OwrTransportAgent *transport_agent, OwrDataSession *data_session; OwrDataChannel *data_channel; - data_session = OWR_DATA_SESSION(get_session(transport_agent, - data_channel_info->session_id)); + data_session = OWR_DATA_SESSION(get_session_from_stream_id(transport_agent, + data_channel_info->stream_id)); data_channel = _owr_data_session_get_datachannel(data_session, data_channel_info->id); _owr_data_channel_set_ready_state(data_channel, OWR_DATA_CHANNEL_READY_STATE_CLOSED); g_hash_table_steal(priv->data_channels, GUINT_TO_POINTER(data_channel_info->id)); @@ -4233,12 +4539,15 @@ static void on_feedback_rtcp(GObject *session, guint type, guint fbtype, guint s if (type == GST_RTCP_TYPE_RTPFB && fbtype == GST_RTCP_RTPFB_TYPE_SCREAM) { GstElement *send_output_bin, *scream_queue = NULL; GstMapInfo info = {NULL, 0, NULL, 0, 0, {0}, {0}}; /*GST_MAP_INFO_INIT;*/ - guint session_id = GPOINTER_TO_UINT(g_object_get_data(session, "session_id")); + guint stream_id = GPOINTER_TO_UINT(g_object_get_data(session, "stream-id")); + guint session_id = GPOINTER_TO_UINT(g_object_get_data(session, "session-id")); + gchar *queue_name = g_strdup_printf("screamqueue-%u-%u", session_id, stream_id); + gchar *name = g_strdup_printf("send-output-bin-%u", stream_id); - gchar *name = g_strdup_printf("send-output-bin-%u", session_id); send_output_bin = gst_bin_get_by_name(GST_BIN(transport_agent->priv->transport_bin), name); g_free(name); - scream_queue = gst_bin_get_by_name(GST_BIN(send_output_bin), "screamqueue"); + scream_queue = gst_bin_get_by_name(GST_BIN(send_output_bin), queue_name); + g_free(queue_name); gst_object_unref(send_output_bin); /* Read feedback from FCI */ @@ -4279,14 +4588,14 @@ static GstPadProbeReturn probe_save_ts(GstPad *srcpad, GstPadProbeInfo *info, vo static gint compare_rtcp_scream(GHashTable *a, GHashTable *b) { - guint session_id_a, session_id_b, ssrc_a, ssrc_b; + guint stream_id_a, stream_id_b, ssrc_a, ssrc_b; - session_id_a = GPOINTER_TO_UINT(g_hash_table_lookup(a, "session-id")); + stream_id_a = GPOINTER_TO_UINT(g_hash_table_lookup(a, "stream_id")); ssrc_a = GPOINTER_TO_UINT(g_hash_table_lookup(a, "ssrc")); - session_id_b = GPOINTER_TO_UINT(g_hash_table_lookup(b, "session-id")); + stream_id_b = GPOINTER_TO_UINT(g_hash_table_lookup(b, "stream_id")); ssrc_b = GPOINTER_TO_UINT(g_hash_table_lookup(b, "ssrc")); - if (session_id_a == session_id_b && ssrc_a == ssrc_b) + if (stream_id_a == stream_id_b && ssrc_a == ssrc_b) return 0; return -1; } @@ -4299,12 +4608,14 @@ static GstPadProbeReturn probe_rtp_info(GstPad *srcpad, GstPadProbeInfo *info, S guint64 arrival_time = GST_CLOCK_TIME_NONE; OwrTransportAgent *transport_agent = NULL; OwrTransportAgentPrivate *priv = NULL; + guint stream_id = 0; guint session_id = 0; guint8 pt = 0; gboolean rtp_mapped = FALSE; GObject *rtp_session = NULL; transport_agent = scream_rx->transport_agent; + stream_id = scream_rx->stream_id; session_id = scream_rx->session_id; g_assert(transport_agent); @@ -4322,21 +4633,37 @@ static GstPadProbeReturn probe_rtp_info(GstPad *srcpad, GstPadProbeInfo *info, S pt = gst_rtp_buffer_get_payload_type(&rtp_buf); } - g_signal_emit_by_name(priv->rtpbin, "get-internal-session", session_id, &rtp_session); + g_signal_emit_by_name(priv->rtpbin, "get-internal-session", stream_id, &rtp_session); if (G_UNLIKELY(scream_rx->rtx_pt == -2)) { - OwrMediaSession *media_session; - OwrPayload *rx_payload; + OwrMediaSession *media_session = NULL; + OwrPayload *rx_payload = NULL; OwrAdaptationType adapt_type; - media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); - rx_payload = _owr_media_session_get_receive_payload(media_session, pt); + + if (transport_agent->priv->bundle_policy != OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) { + media_session = OWR_MEDIA_SESSION(get_session(transport_agent, session_id)); + rx_payload = _owr_media_session_get_receive_payload(media_session, pt); + } else { + GSList *sessions = get_sessions_from_stream_id(transport_agent, stream_id); + for (GSList *walk = sessions; walk; walk = g_slist_next(walk)) { + media_session = OWR_MEDIA_SESSION(walk->data); + rx_payload = _owr_media_session_get_receive_payload(media_session, pt); + if (rx_payload) + break; + } + g_slist_free_full(sessions, g_object_unref); + } + + g_assert(media_session); + g_assert(rx_payload); + g_object_get(rx_payload, "rtx-payload-type", &scream_rx->rtx_pt, "adaptation", &adapt_type, NULL); scream_rx->adapt = (adapt_type == OWR_ADAPTATION_TYPE_SCREAM); g_object_unref(media_session); g_object_unref(rx_payload); - g_object_set(rtp_session, "rtcp-reduced-size", TRUE, NULL); + g_object_set(rtp_session, "rtcp-reduced-size", TRUE, NULL); } OWR_UNUSED(srcpad); @@ -4420,7 +4747,7 @@ static GstPadProbeReturn probe_rtp_info(GstPad *srcpad, GstPadProbeInfo *info, S GUINT_TO_POINTER(scream_rx->highest_seq)); g_hash_table_insert(rtcp_info, "n-loss", GUINT_TO_POINTER(scream_rx->n_loss)); g_hash_table_insert(rtcp_info, "n-ecn", GUINT_TO_POINTER(scream_rx->n_ecn)); - g_hash_table_insert(rtcp_info, "session-id", GUINT_TO_POINTER(session_id)); + g_hash_table_insert(rtcp_info, "stream_id", GUINT_TO_POINTER(stream_id)); GST_LOG_OBJECT(transport_agent, "queuing up scream feedback: %u, %u, %u, %u", scream_rx->highest_seq, scream_rx->n_loss, scream_rx->n_ecn, From 78e55742cc9ac85d6f8d7c9ae433b2e3708d8594 Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Wed, 9 Nov 2016 13:04:05 +0100 Subject: [PATCH 06/26] VP9 support --- owr/owr_types.c | 1 + owr/owr_types.h | 3 ++- owr/owr_utils.c | 2 ++ transport/owr_payload.c | 54 +++++++++++++++++++++++++++++++++++++---- 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/owr/owr_types.c b/owr/owr_types.c index 65212d1b..8e3d2ddd 100644 --- a/owr/owr_types.c +++ b/owr/owr_types.c @@ -43,6 +43,7 @@ GType owr_codec_type_get_type(void) {OWR_CODEC_TYPE_OPUS, "Opus", "opus"}, {OWR_CODEC_TYPE_H264, "H264", "h264"}, {OWR_CODEC_TYPE_VP8, "VP8", "vp8"}, + {OWR_CODEC_TYPE_VP9, "VP9", "vp9"}, {0, NULL, NULL} }; static volatile GType id = 0; diff --git a/owr/owr_types.h b/owr/owr_types.h index 8e61df4e..a9c8ea32 100644 --- a/owr/owr_types.h +++ b/owr/owr_types.h @@ -42,7 +42,8 @@ typedef enum _OwrCodecType { OWR_CODEC_TYPE_PCMA, OWR_CODEC_TYPE_OPUS, OWR_CODEC_TYPE_H264, - OWR_CODEC_TYPE_VP8 + OWR_CODEC_TYPE_VP8, + OWR_CODEC_TYPE_VP9 } OwrCodecType; typedef enum _OwrMediaType { diff --git a/owr/owr_utils.c b/owr/owr_utils.c index 6f41b736..c84eedf1 100644 --- a/owr/owr_utils.c +++ b/owr/owr_utils.c @@ -54,6 +54,8 @@ OwrCodecType _owr_caps_to_codec_type(GstCaps *caps) return OWR_CODEC_TYPE_H264; if (gst_structure_has_name(structure, "video/x-vp8")) return OWR_CODEC_TYPE_VP8; + if (gst_structure_has_name(structure, "video/x-vp9")) + return OWR_CODEC_TYPE_VP9; GST_ERROR("Unknown caps: %" GST_PTR_FORMAT, (gpointer)caps); return OWR_CODEC_TYPE_NONE; diff --git a/transport/owr_payload.c b/transport/owr_payload.c index 151c1721..1b2f1436 100644 --- a/transport/owr_payload.c +++ b/transport/owr_payload.c @@ -188,6 +188,8 @@ static GList *h264_decoders = NULL; static GList *h264_encoders = NULL; static GList *vp8_decoders = NULL; static GList *vp8_encoders = NULL; +static GList *vp9_decoders = NULL; +static GList *vp9_encoders = NULL; static gpointer owr_payload_detect_codecs(gpointer data) { @@ -214,6 +216,11 @@ static gpointer owr_payload_detect_codecs(gpointer data) vp8_encoders = gst_element_factory_list_filter(encoder_factories, caps, GST_PAD_SRC, FALSE); gst_caps_unref(caps); + caps = gst_caps_new_empty_simple("video/x-vp9"); + vp9_decoders = gst_element_factory_list_filter(decoder_factories, caps, GST_PAD_SINK, FALSE); + vp9_encoders = gst_element_factory_list_filter(encoder_factories, caps, GST_PAD_SRC, FALSE); + gst_caps_unref(caps); + gst_plugin_feature_list_free(decoder_factories); gst_plugin_feature_list_free(encoder_factories); @@ -221,6 +228,8 @@ static gpointer owr_payload_detect_codecs(gpointer data) h264_encoders = g_list_sort(h264_encoders, gst_plugin_feature_rank_compare_func); vp8_decoders = g_list_sort(vp8_decoders, gst_plugin_feature_rank_compare_func); vp8_encoders = g_list_sort(vp8_encoders, gst_plugin_feature_rank_compare_func); + vp9_decoders = g_list_sort(vp9_decoders, gst_plugin_feature_rank_compare_func); + vp9_encoders = g_list_sort(vp9_encoders, gst_plugin_feature_rank_compare_func); return NULL; } @@ -300,11 +309,11 @@ static void owr_payload_init(OwrPayload *payload) /* Private methods */ -static const gchar *OwrCodecTypeEncoderElementName[] = {"none", "mulawenc", "alawenc", "opusenc", "openh264enc", "vp8enc"}; -static const gchar *OwrCodecTypeDecoderElementName[] = {"none", "mulawdec", "alawdec", "opusdec", "openh264dec", "vp8dec"}; -static const gchar *OwrCodecTypeParserElementName[] = {"none", "none", "none", "none", "h264parse", "none"}; -static const gchar *OwrCodecTypePayElementName[] = {"none", "rtppcmupay", "rtppcmapay", "rtpopuspay", "rtph264pay", "rtpvp8pay"}; -static const gchar *OwrCodecTypeDepayElementName[] = {"none", "rtppcmudepay", "rtppcmadepay", "rtpopusdepay", "rtph264depay", "rtpvp8depay"}; +static const gchar *OwrCodecTypeEncoderElementName[] = {"none", "mulawenc", "alawenc", "opusenc", "openh264enc", "vp8enc", "vp9enc"}; +static const gchar *OwrCodecTypeDecoderElementName[] = {"none", "mulawdec", "alawdec", "opusdec", "openh264dec", "vp8dec", "vp9dec"}; +static const gchar *OwrCodecTypeParserElementName[] = {"none", "none", "none", "none", "h264parse", "none", "none"}; +static const gchar *OwrCodecTypePayElementName[] = {"none", "rtppcmupay", "rtppcmapay", "rtpopuspay", "rtph264pay", "rtpvp8pay", "rtpvp9pay"}; +static const gchar *OwrCodecTypeDepayElementName[] = {"none", "rtppcmudepay", "rtppcmadepay", "rtpopusdepay", "rtph264depay", "rtpvp8depay", "rtpvp9depay"}; static guint evaluate_bitrate_from_payload(OwrPayload *payload) { @@ -447,6 +456,30 @@ GstElement * _owr_payload_create_encoder(OwrPayload *payload) "keyframe-mode", 0, /* VPX_KF_DISABLED */ NULL); + g_object_bind_property(payload, "bitrate", encoder, "target-bitrate", G_BINDING_SYNC_CREATE); + g_object_set(payload, "bitrate", evaluate_bitrate_from_payload(payload), NULL); + break; + case OWR_CODEC_TYPE_VP9: + encoder = try_codecs(vp9_encoders, "encoder"); + g_return_val_if_fail(encoder, NULL); + /* values are inspired by webrtc.org values in vp9_impl.cc */ + g_object_set(encoder, + "end-usage", 1, /* VPX_CBR */ + "deadline", G_GINT64_CONSTANT(1), /* VPX_DL_REALTIME */ + "cpu-used", 3, + "min-quantizer", 2, + "max-quantizer", 52, + "buffer-initial-size", 500, + "buffer-optimal-size", 600, + "buffer-size", 1000, + "dropframe-threshold", 30, + "lag-in-frames", 0, + "timebase", 1, 90000, + "error-resilient", 1, + "resize-allowed", TRUE, + "keyframe-mode", 0, /* VPX_KF_DISABLED */ + NULL); + g_object_bind_property(payload, "bitrate", encoder, "target-bitrate", G_BINDING_SYNC_CREATE); g_object_set(payload, "bitrate", evaluate_bitrate_from_payload(payload), NULL); break; @@ -477,6 +510,10 @@ GstElement * _owr_payload_create_decoder(OwrPayload *payload) decoder = try_codecs(vp8_decoders, "decoder"); g_return_val_if_fail(decoder, NULL); break; + case OWR_CODEC_TYPE_VP9: + decoder = try_codecs(vp9_decoders, "decoder"); + g_return_val_if_fail(decoder, NULL); + break; default: element_name = g_strdup_printf("decoder_%s_%u", OwrCodecTypeDecoderElementName[payload->priv->codec_type], get_unique_id()); decoder = gst_element_factory_make(OwrCodecTypeDecoderElementName[payload->priv->codec_type], element_name); @@ -607,6 +644,10 @@ GstCaps * _owr_payload_create_rtp_caps(OwrPayload *payload) encoding_name = "VP8-DRAFT-IETF-01"; break; + case OWR_CODEC_TYPE_VP9: + encoding_name = "VP9-DRAFT-IETF-01"; + break; + default: g_return_val_if_reached(NULL); } @@ -718,6 +759,9 @@ GstCaps * _owr_payload_create_encoded_caps(OwrPayload *payload) case OWR_CODEC_TYPE_VP8: caps = gst_caps_new_empty_simple("video/x-vp8"); break; + case OWR_CODEC_TYPE_VP9: + caps = gst_caps_new_empty_simple("video/x-vp9"); + break; default: caps = gst_caps_new_any(); } From d3bf3c05044c0c35f31128f9d56dbf55f93064c9 Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Mon, 14 Nov 2016 17:48:11 +0100 Subject: [PATCH 07/26] OwrLocalMediaSource: add volume and mute properties Fixes #572 Based on a preliminary patch by Adrian Bauer --- configure.ac | 2 +- local/owr_local_media_source.c | 186 +++++++++++++++++++++++++++++++-- 2 files changed, 176 insertions(+), 12 deletions(-) diff --git a/configure.ac b/configure.ac index b509d3f9..70939e30 100644 --- a/configure.ac +++ b/configure.ac @@ -28,7 +28,7 @@ AC_SUBST(LIBOPENWEBRTC_CFLAGS) GST_REQUIRED=1.4 PKG_CHECK_MODULES(GLIB, [glib-2.0, gobject-2.0, gmodule-2.0, gthread-2.0]) -PKG_CHECK_MODULES(GSTREAMER, [gstreamer-1.0 >= $GST_REQUIRED gstreamer-rtp-1.0 >= $GST_REQUIRED gstreamer-video-1.0 >= $GST_REQUIRED gstreamer-app-1.0 >= $GST_REQUIRED gstreamer-gl-1.0 >= $GST_REQUIRED]) +PKG_CHECK_MODULES(GSTREAMER, [gstreamer-1.0 >= $GST_REQUIRED gstreamer-rtp-1.0 >= $GST_REQUIRED gstreamer-video-1.0 >= $GST_REQUIRED gstreamer-audio-1.0 >= $GST_REQUIRED gstreamer-app-1.0 >= $GST_REQUIRED gstreamer-gl-1.0 >= $GST_REQUIRED]) PKG_CHECK_MODULES(NICE, [nice >= 0.1.7.1]) PKG_CHECK_MODULES(GSTREAMER_SCTP, [gstreamer-sctp-1.0]) PKG_CHECK_MODULES(ORC, [orc-0.4]) diff --git a/local/owr_local_media_source.c b/local/owr_local_media_source.c index b417fab5..9fd54d42 100644 --- a/local/owr_local_media_source.c +++ b/local/owr_local_media_source.c @@ -45,6 +45,7 @@ #include "owr_utils.h" #include +#include #include #include @@ -78,6 +79,18 @@ static guint unique_bin_id = 0; #define OWR_LOCAL_MEDIA_SOURCE_GET_PRIVATE(obj) \ (G_TYPE_INSTANCE_GET_PRIVATE((obj), OWR_TYPE_LOCAL_MEDIA_SOURCE, OwrLocalMediaSourcePrivate)) +#define LINK_ELEMENTS(a, b) do { \ + if (!gst_element_link(a, b)) \ + GST_ERROR("Failed to link " #a " -> " #b); \ +} while (0) + +#define CREATE_ELEMENT(elem, factory, name) do { \ + elem = gst_element_factory_make(factory, name); \ + if (!elem) \ + GST_ERROR("Could not create " name " from factory " factory); \ + g_assert(elem); \ +} while (0) + static void owr_message_origin_interface_init(OwrMessageOriginInterface *interface); G_DEFINE_TYPE_WITH_CODE(OwrLocalMediaSource, owr_local_media_source, OWR_TYPE_MEDIA_SOURCE, @@ -86,6 +99,12 @@ G_DEFINE_TYPE_WITH_CODE(OwrLocalMediaSource, owr_local_media_source, OWR_TYPE_ME struct _OwrLocalMediaSourcePrivate { gint device_index; OwrMessageOriginBusSet *message_origin_bus_set; + + /* Volume control for audio sources */ + GstElement *source_volume; + /* Volume and mute are for before source_volume gets created */ + double volume; + gboolean mute; }; static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_source, GstCaps *caps); @@ -98,6 +117,8 @@ static void owr_local_media_source_get_property(GObject *object, guint property_ enum { PROP_0, PROP_DEVICE_INDEX, + PROP_VOLUME, + PROP_MUTE, N_PROPERTIES }; @@ -107,6 +128,8 @@ static void owr_local_media_source_finalize(GObject *object) owr_message_origin_bus_set_free(source->priv->message_origin_bus_set); source->priv->message_origin_bus_set = NULL; + + g_clear_object(&source->priv->source_volume); } static void owr_local_media_source_class_init(OwrLocalMediaSourceClass *klass) @@ -128,6 +151,17 @@ static void owr_local_media_source_class_init(OwrLocalMediaSourceClass *klass) "Index of the device to be used for this source (-1 => auto)", -1, G_MAXINT16, -1, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property(gobject_class, PROP_VOLUME, + g_param_spec_double("volume", "Volume", + "Volume factor (only applicable to audio sources)", + 0, 1, 0.8, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property(gobject_class, PROP_MUTE, + g_param_spec_boolean("mute", "Mute", + "Mute state (only applicable to audio sources)", + FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); } static gpointer owr_local_media_source_get_bus_set(OwrMessageOrigin *origin) @@ -143,21 +177,86 @@ static void owr_message_origin_interface_init(OwrMessageOriginInterface *interfa static void owr_local_media_source_init(OwrLocalMediaSource *source) { OwrLocalMediaSourcePrivate *priv; + source->priv = priv = OWR_LOCAL_MEDIA_SOURCE_GET_PRIVATE(source); priv->device_index = -1; priv->message_origin_bus_set = owr_message_origin_bus_set_new(); + priv->source_volume = NULL; + priv->volume = 0.8; + priv->mute = FALSE; } static void owr_local_media_source_set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec) { OwrLocalMediaSource *source = OWR_LOCAL_MEDIA_SOURCE(object); + OwrMediaType media_type = OWR_MEDIA_TYPE_UNKNOWN; switch (property_id) { case PROP_DEVICE_INDEX: source->priv->device_index = g_value_get_int(value); break; - + case PROP_VOLUME: { + gdouble volume_value = g_value_get_double(value); + + g_object_get(source, "media-type", &media_type, NULL); + + if (media_type == OWR_MEDIA_TYPE_AUDIO) { + /* Set this anyway in case the source gets shutdown and restarted */ + source->priv->volume = volume_value; + + GST_DEBUG_OBJECT(source, "setting volume to %f\n", volume_value); + if (source->priv->source_volume) + g_object_set(source->priv->source_volume, "volume", volume_value, NULL); + else { + GstElement* source_bin = _owr_media_source_get_source_bin(OWR_MEDIA_SOURCE(source)); + + if (source_bin) { + GstElement* source_element = gst_bin_get_by_name(GST_BIN_CAST(source_bin), "audio-source"); + if (source_element) { + if (GST_IS_STREAM_VOLUME(source_element)) + gst_stream_volume_set_volume(GST_STREAM_VOLUME(source_element), GST_STREAM_VOLUME_FORMAT_CUBIC, volume_value); + gst_object_unref(source_element); + } else + GST_WARNING_OBJECT(source, "The audio-source element was not found in source bin"); + gst_object_unref(source_bin); + } else + GST_WARNING_OBJECT(source, "No source bin set for the audio source"); + } + } else + GST_WARNING_OBJECT(source, "Tried to set volume on non-audio source"); + break; + } + case PROP_MUTE: { + gboolean mute_value = g_value_get_boolean(value); + g_object_get(source, "media-type", &media_type, NULL); + + if (media_type == OWR_MEDIA_TYPE_AUDIO) { + /* Set this anyway in case the source gets shutdown and restarted */ + source->priv->mute = mute_value; + + GST_DEBUG_OBJECT(source, "setting mute to %d\n", mute_value); + if (source->priv->source_volume) + g_object_set(source->priv->source_volume, "mute", mute_value, NULL); + else { + GstElement* source_bin = _owr_media_source_get_source_bin(OWR_MEDIA_SOURCE(source)); + + if (source_bin) { + GstElement* source_element = gst_bin_get_by_name(GST_BIN_CAST(source_bin), "audio-source"); + if (source_element) { + if (GST_IS_STREAM_VOLUME(source_element)) + gst_stream_volume_set_mute(GST_STREAM_VOLUME(source_element), mute_value); + gst_object_unref(source_element); + } else + GST_WARNING_OBJECT(source, "The audio-source element was not found in source bin"); + gst_object_unref(source_bin); + } else + GST_WARNING_OBJECT(source, "No source bin set for the audio source"); + } + } else + GST_WARNING_OBJECT(source, "Tried to set mute on non-audio source"); + break; + } default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -168,11 +267,72 @@ static void owr_local_media_source_get_property(GObject *object, guint property_ GValue *value, GParamSpec *pspec) { OwrLocalMediaSource *source = OWR_LOCAL_MEDIA_SOURCE(object); + OwrMediaType media_type = OWR_MEDIA_TYPE_UNKNOWN; switch (property_id) { case PROP_DEVICE_INDEX: g_value_set_int(value, source->priv->device_index); break; + case PROP_VOLUME: + g_object_get(source, "media-type", &media_type, NULL); + if (media_type == OWR_MEDIA_TYPE_AUDIO) { + if (source->priv->source_volume) + g_object_get_property(G_OBJECT(source->priv->source_volume), "volume", value); + else { + GstElement* source_bin = _owr_media_source_get_source_bin(OWR_MEDIA_SOURCE(source)); + + if (source_bin) { + GstElement* source_element = gst_bin_get_by_name(GST_BIN_CAST(source_bin), "audio-source"); + if (source_element) { + if (GST_IS_STREAM_VOLUME(source_element)) + g_value_set_double(value, gst_stream_volume_get_volume(GST_STREAM_VOLUME(source_element), GST_STREAM_VOLUME_FORMAT_CUBIC)); + else + g_value_set_double(value, source->priv->volume); + + gst_object_unref(source_element); + } else { + GST_WARNING_OBJECT(source, "The audio-source element was not found in source bin"); + g_value_set_double(value, source->priv->volume); + } + gst_object_unref(source_bin); + } else { + GST_WARNING_OBJECT(source, "No source bin set for the audio source"); + g_value_set_double(value, source->priv->volume); + } + } + } else + GST_WARNING_OBJECT(source, "Tried to get volume on non-audio source"); + break; + case PROP_MUTE: + g_object_get(source, "media-type", &media_type, NULL); + if (media_type == OWR_MEDIA_TYPE_AUDIO) { + if (source->priv->source_volume) + g_object_get_property(G_OBJECT(source->priv->source_volume), "mute", value); + else { + GstElement* source_bin = _owr_media_source_get_source_bin(OWR_MEDIA_SOURCE(source)); + + if (source_bin) { + GstElement* source_element = gst_bin_get_by_name(GST_BIN_CAST(source_bin), "audio-source"); + if (source_element) { + if (GST_IS_STREAM_VOLUME(source_element)) + g_value_set_boolean(value, gst_stream_volume_get_mute(GST_STREAM_VOLUME(source_element))); + else + g_value_set_boolean(value, source->priv->mute); + + gst_object_unref(source_element); + } else { + GST_WARNING_OBJECT(source, "The audio-source element was not found in source bin"); + g_value_set_boolean(value, source->priv->mute); + } + gst_object_unref(source_bin); + } else { + GST_WARNING_OBJECT(source, "No source bin set for the audio source"); + g_value_set_boolean(value, source->priv->mute); + } + } + } else + GST_WARNING_OBJECT(source, "Tried to get volume on non-audio source"); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); @@ -180,16 +340,6 @@ static void owr_local_media_source_get_property(GObject *object, guint property_ } } -#define LINK_ELEMENTS(a, b) \ - if (!gst_element_link(a, b)) \ - GST_ERROR("Failed to link " #a " -> " #b); - -#define CREATE_ELEMENT(elem, factory, name) \ - elem = gst_element_factory_make(factory, name); \ - if (!elem) \ - GST_ERROR("Could not create " name " from factory " factory); \ - g_assert(elem); - /* FIXME: Copy from owr/orw.c without any error handling whatsoever */ static gboolean bus_call(GstBus *bus, GstMessage *msg, gpointer user_data) { @@ -266,6 +416,7 @@ static gboolean bus_call(GstBus *bus, GstMessage *msg, gpointer user_data) static gboolean shutdown_media_source(GHashTable *args) { OwrMediaSource *media_source; + OwrLocalMediaSource *local_media_source; GstElement *source_pipeline, *source_tee; GHashTable *event_data; GValue *value; @@ -277,6 +428,10 @@ static gboolean shutdown_media_source(GHashTable *args) media_source = g_hash_table_lookup(args, "media_source"); g_assert(media_source); + local_media_source = OWR_LOCAL_MEDIA_SOURCE(media_source); + if (local_media_source->priv->source_volume) + gst_object_unref(local_media_source->priv->source_volume); + source_pipeline = _owr_media_source_get_source_bin(media_source); if (!source_pipeline) { g_object_unref(media_source); @@ -451,6 +606,8 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s local_source = OWR_LOCAL_MEDIA_SOURCE(media_source); priv = local_source->priv; + GST_DEBUG_OBJECT(media_source, "source requested"); + /* only create the source bin for this media source once */ if ((source_pipeline = _owr_media_source_get_source_bin(media_source))) GST_DEBUG_OBJECT(media_source, "Re-using existing source element/bin"); @@ -548,6 +705,13 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s goto done; } + if (!GST_IS_STREAM_VOLUME(source)) { + CREATE_ELEMENT(priv->source_volume, "volume", "audio-source-volume"); + g_object_set(priv->source_volume, "volume", priv->volume, "mute", priv->mute, NULL); + source_process = gst_object_ref(priv->source_volume); + gst_bin_add(GST_BIN(source_pipeline), source_process); + } + break; } case OWR_MEDIA_TYPE_VIDEO: From 75b53087b0afe65f2e6a7fc5e4527084a4d12f82 Mon Sep 17 00:00:00 2001 From: Arun Raghavan Date: Tue, 27 Jan 2015 16:25:02 +0530 Subject: [PATCH 08/26] local: Use echo cancellation on Linux if available --- local/owr_audio_renderer.c | 27 +++++++++++++++++++++++++++ local/owr_local_media_source.c | 23 +++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/local/owr_audio_renderer.c b/local/owr_audio_renderer.c index f80aef79..9157de7e 100644 --- a/local/owr_audio_renderer.c +++ b/local/owr_audio_renderer.c @@ -55,6 +55,7 @@ GST_DEBUG_CATEGORY_EXTERN(_owraudiorenderer_debug); #define AUDIO_SINK "openslessink" #elif __linux__ +#include #define AUDIO_SINK "pulsesink" @@ -119,6 +120,27 @@ OwrAudioRenderer *owr_audio_renderer_new(void) NULL); } +static void +setup_sink_for_aec(GstElement *sink) +{ +#if defined(__linux__) && !defined(__ANDROID__) + /* pulsesink */ + GstStructure *s; + + s = gst_structure_new("props", PA_PROP_FILTER_WANT, G_TYPE_STRING, "echo-cancel", NULL); + g_object_set(G_OBJECT(sink), "stream-properties", s, NULL); + gst_structure_free(s); + +#elif defined(__ANDROID__) + /* openslessink */ + +#elif defined(__APPLE__) && !TARGET_IPHONE_SIMULATOR + /* osxaudiosink */ + +#endif +} + + #define LINK_ELEMENTS(a, b) \ if (!gst_element_link(a, b)) \ GST_ERROR("Failed to link " #a " -> " #b); @@ -151,6 +173,11 @@ static GstElement *owr_audio_renderer_get_element(OwrMediaRenderer *renderer) sink = OWR_MEDIA_RENDERER_GET_CLASS(renderer)->get_sink(renderer); g_assert(sink); + g_object_set(sink, "buffer-time", SINK_BUFFER_TIME, + "latency-time", G_GINT64_CONSTANT(10000), NULL); + + setup_sink_for_aec(sink); + gst_bin_add_many(GST_BIN(renderer_bin), audioresample, audioconvert, capsfilter, volume, sink, NULL); diff --git a/local/owr_local_media_source.c b/local/owr_local_media_source.c index 9fd54d42..7fb1be76 100644 --- a/local/owr_local_media_source.c +++ b/local/owr_local_media_source.c @@ -66,6 +66,8 @@ GST_DEBUG_CATEGORY_EXTERN(_owrlocalmediasource_debug); #define VIDEO_SRC "androidvideosource" #elif defined(__linux__) +#include + #define AUDIO_SRC "pulsesrc" #define VIDEO_SRC "v4l2src" @@ -564,6 +566,26 @@ static void on_caps(GstElement *source, GParamSpec *pspec, OwrMediaSource *media } } +static void +setup_source_for_aec(GstElement *src) +{ +#if defined(__linux__) && !defined(__ANDROID__) + /* pulsesrc */ + GstStructure *s; + + s = gst_structure_new("props", PA_PROP_FILTER_WANT, G_TYPE_STRING, "echo-cancel", NULL); + g_object_set(G_OBJECT(src), "stream-properties", s, NULL); + gst_structure_free(s); + +#elif defined(__ANDROID__) + /* openslessrc */ + +#elif defined(__APPLE__) && !TARGET_IPHONE_SIMULATOR + /* osxaudiosrc */ + +#endif +} + /* * owr_local_media_source_get_pad * @@ -694,6 +716,7 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s #endif } #endif + setup_source_for_aec(source); break; case OWR_SOURCE_TYPE_TEST: CREATE_ELEMENT(source, "audiotestsrc", "audio-source"); From 7f3d23e034818893db198f4b56e41609abd8847b Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Wed, 16 Nov 2016 16:36:04 +0100 Subject: [PATCH 09/26] transport_agent: TURN TLS helper server support --- transport/owr_transport_agent.c | 6 ++++++ transport/owr_transport_agent.h | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/transport/owr_transport_agent.c b/transport/owr_transport_agent.c index 5cde8824..e757189b 100644 --- a/transport/owr_transport_agent.c +++ b/transport/owr_transport_agent.c @@ -800,6 +800,12 @@ static void update_helper_servers(OwrTransportAgent *transport_agent, guint stre nice_agent_set_relay_info(priv->nice_agent, stream_id, NICE_COMPONENT_TYPE_RTCP, address, port, username, password, NICE_RELAY_TYPE_TURN_TCP); break; + case OWR_HELPER_SERVER_TYPE_TURN_TLS: + nice_agent_set_relay_info(priv->nice_agent, stream_id, NICE_COMPONENT_TYPE_RTP, + address, port, username, password, NICE_RELAY_TYPE_TURN_TLS); + nice_agent_set_relay_info(priv->nice_agent, stream_id, NICE_COMPONENT_TYPE_RTCP, + address, port, username, password, NICE_RELAY_TYPE_TURN_TLS); + break; } } } diff --git a/transport/owr_transport_agent.h b/transport/owr_transport_agent.h index 7e2d6541..c149cea0 100644 --- a/transport/owr_transport_agent.h +++ b/transport/owr_transport_agent.h @@ -42,7 +42,8 @@ G_BEGIN_DECLS typedef enum _OwrHelperServerType { OWR_HELPER_SERVER_TYPE_STUN, OWR_HELPER_SERVER_TYPE_TURN_UDP, - OWR_HELPER_SERVER_TYPE_TURN_TCP + OWR_HELPER_SERVER_TYPE_TURN_TCP, + OWR_HELPER_SERVER_TYPE_TURN_TLS } OwrHelperServerType; #define OWR_TYPE_TRANSPORT_AGENT (owr_transport_agent_get_type()) From 242281580bf1eae788ee97cda7da292dfa972f6c Mon Sep 17 00:00:00 2001 From: Konstantin Tokarev Date: Thu, 30 Jun 2016 16:26:46 +0300 Subject: [PATCH 10/26] Don't require gtk-doc if autogen.sh is invoked with --disable-gtk-doc. --- autogen.sh | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/autogen.sh b/autogen.sh index 2c45ea9d..878cf054 100755 --- a/autogen.sh +++ b/autogen.sh @@ -6,27 +6,42 @@ srcdir=`dirname $0` (test -d $srcdir/m4) || mkdir $srcdir/m4 -pushd $srcdir > /dev/null -gtkdocize && \ -autoreconf --verbose --force --install --make || { - echo 'autogen.sh failed'; - exit 1; -} - -popd > /dev/null - -while test "x$@" != "x" ; do -optarg=`expr "x$@" : 'x[^=]*=\(.*\)'` -case "$@" in +for ag_option in $@ +do +case $ag_option in --noconfigure) NOCONFIGURE=defined AUTOGEN_EXT_OPT="$AUTOGEN_EXT_OPT --noconfigure" echo "+ configure run disabled" - shift + ;; + --disable-gtk-doc) + enable_gtk_doc=no + echo "+ gtk-doc disabled" ;; esac done +pushd $srcdir > /dev/null + +if test x$enable_gtk_doc = xno; then + if test -f gtk-doc.make; then :; else + echo "EXTRA_DIST = missing-gtk-doc" > gtk-doc.make + fi + echo "WARNING: You have disabled gtk-doc." + echo " As a result, you will not be able to generate the API" + echo " documentation and 'make dist' will not work." + echo +else + gtkdocize || exit $? +fi + +autoreconf --verbose --force --install --make || { + echo 'autogen.sh failed'; + exit 1; +} + +popd > /dev/null + for arg do CONFIGURE_EXT_OPT="$CONFIGURE_EXT_OPT $arg"; done if test ! -z "$CONFIGURE_EXT_OPT" then From 0acc6c5d848833e6aed4756b6e8d80a6905a6dd7 Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Tue, 28 Apr 2015 17:05:09 +0200 Subject: [PATCH 11/26] build: relax dependency on json-glib Most tests don't require json-glib so we can enable them and build test-client only json-glib was detected on the host. --- configure.ac | 6 ++++-- tests/Makefile.am | 42 +++++++++++++++++++++++------------------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/configure.ac b/configure.ac index 70939e30..7db54af9 100644 --- a/configure.ac +++ b/configure.ac @@ -82,7 +82,8 @@ if test "x$enable_debug" = xyes; then fi AM_CONDITIONAL(OWR_DEBUG, test x$enable_debug = xyes) -dnl build debug or not +dnl build tests or not +have_json_glib=no AC_MSG_CHECKING([whether to build tests or not]) AC_ARG_ENABLE(tests, AC_HELP_STRING( @@ -95,12 +96,13 @@ AC_HELP_STRING( esac],[enable_tests=yes]) AC_MSG_RESULT([$enable_tests]) if test "x$enable_tests" = xyes; then - PKG_CHECK_MODULES(JSON_GLIB, [json-glib-1.0]) + PKG_CHECK_MODULES(JSON_GLIB, [json-glib-1.0], [$have_json_glib=yes], [$have_json_glib=no]) PKG_CHECK_MODULES(LIBSOUP, [libsoup-2.4]) AC_DEFINE(OWR_TESTS, 1, [Define if building tests]) fi AM_CONDITIONAL(OWR_TESTS, test x$enable_tests = xyes) +AM_CONDITIONAL(HAVE_JSON_GLIB, test x$have_json_glib = xyes) dnl build static or not AC_MSG_CHECKING([whether to build static owr or not]) diff --git a/tests/Makefile.am b/tests/Makefile.am index 28ac557f..cd2cc590 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -20,10 +20,9 @@ bin_PROGRAMS = \ test-send-receive \ test-data-channel \ test-init \ - test-bus \ test-uri \ - test-client \ - test-crypto-utils + test-crypto-utils \ + test-bus if OWR_GST AM_CPPFLAGS += \ @@ -45,6 +44,27 @@ test_gst_io_LDADD = \ $(top_builddir)/gst/libopenwebrtc_gst.la endif +if HAVE_JSON_GLIB +bin_PROGRAMS += \ + test-client + +test_client_SOURCES = test_client.c + +test_client_CFLAGS = \ + $(AM_CFLAGS) \ + $(JSON_GLIB_CFLAGS) \ + $(LIBSOUP_CFLAGS) \ + -I$(top_srcdir)/local \ + -I$(top_srcdir)/transport \ + -I$(top_srcdir)/owr + +test_client_LDADD = \ + $(JSON_GLIB_LIBS) \ + $(LIBSOUP_LIBS) \ + $(GLIB_LIBS) \ + $(top_builddir)/owr/libopenwebrtc.la +endif + list_devices_SOURCES = list_devices.c list_devices_CFLAGS = \ @@ -113,22 +133,6 @@ test_bus_LDADD = \ $(GLIB_LIBS) \ $(top_builddir)/owr/libopenwebrtc.la -test_client_SOURCES = test_client.c - -test_client_CFLAGS = \ - $(AM_CFLAGS) \ - $(JSON_GLIB_CFLAGS) \ - $(LIBSOUP_CFLAGS) \ - -I$(top_srcdir)/local \ - -I$(top_srcdir)/transport \ - -I$(top_srcdir)/owr - -test_client_LDADD = \ - $(JSON_GLIB_LIBS) \ - $(LIBSOUP_LIBS) \ - $(GLIB_LIBS) \ - $(top_builddir)/owr/libopenwebrtc.la - test_uri_SOURCES = test_uri.c test_utils.c test_uri_CFLAGS = \ From 779ffc50c669e40c906c132e321722a88160877e Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Tue, 28 Apr 2015 17:41:19 +0200 Subject: [PATCH 12/26] rpi: rpicamsrc support --- configure.ac | 5 +++++ local/owr_device_list.c | 6 ++++++ local/owr_local_media_source.c | 7 +++++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 7db54af9..31eee080 100644 --- a/configure.ac +++ b/configure.ac @@ -186,6 +186,11 @@ esac AM_CONDITIONAL(TARGET_APPLE, test x$is_apple = xyes) dnl We substitute OWR_DEVICE_LIST_EXT_LIBS in the end +AC_CHECK_LIB([bcm_host], [bcm_host_init], +[AC_DEFINE(TARGET_RPI, 1, [Define if building for the Raspberry Pi])], +[AC_DEFINE(TARGET_RPI, 0, [Define if building for the Raspberry Pi])] +) + dnl generate java bindings or not AC_MSG_CHECKING([whether to generate java bindings or not]) AC_ARG_ENABLE(owr-java, diff --git a/local/owr_device_list.c b/local/owr_device_list.c index 81453815..9f8faa50 100644 --- a/local/owr_device_list.c +++ b/local/owr_device_list.c @@ -378,6 +378,12 @@ static gboolean enumerate_video_source_devices(GClosure *callback) GDir *dev_dir; const gchar *filename; +#if TARGET_RPI + source = _owr_local_media_source_new_cached(-1, "RPiCam", + OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE); + sources = g_list_prepend(sources, source); +#endif + dev_dir = g_dir_open("/dev", 0, &error); while ((filename = g_dir_read_name(dev_dir))) { diff --git a/local/owr_local_media_source.c b/local/owr_local_media_source.c index 7fb1be76..66195154 100644 --- a/local/owr_local_media_source.c +++ b/local/owr_local_media_source.c @@ -69,8 +69,11 @@ GST_DEBUG_CATEGORY_EXTERN(_owrlocalmediasource_debug); #include #define AUDIO_SRC "pulsesrc" +#if TARGET_RPI +#define VIDEO_SRC "rpicamsrc" +#else #define VIDEO_SRC "v4l2src" - +#endif /* TARGET_RPI */ #else #define AUDIO_SRC "audiotestsrc" #define VIDEO_SRC "videotestsrc" @@ -750,7 +753,7 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s g_object_set(source, "device-index", priv->device_index, NULL); #elif defined(__ANDROID__) g_object_set(source, "cam-index", priv->device_index, NULL); -#elif defined(__linux__) +#elif defined(__linux__) && !TARGET_RPI tmp = g_strdup_printf("/dev/video%d", priv->device_index); g_object_set(source, "device", tmp, NULL); g_free(tmp); From a85f7455ed5e6ff99271cdf11d93ed40a4933802 Mon Sep 17 00:00:00 2001 From: Xabier Rodriguez Calvar Date: Fri, 15 Jul 2016 17:37:15 +0200 Subject: [PATCH 13/26] owr: compressed sources support --- local/owr_device_list.c | 19 +- local/owr_local.c | 4 +- local/owr_local_media_source.c | 19 +- local/owr_local_media_source_private.h | 4 +- local/owr_media_renderer.c | 19 ++ local/owr_media_renderer.h | 1 + local/owr_media_renderer_private.h | 2 + local/owr_video_renderer.c | 218 +++++++++++++++------ owr/owr.c | 4 + owr/owr_media_source.c | 147 ++++++++------ owr/owr_media_source_private.h | 14 ++ owr/owr_utils.c | 253 ++++++++++++++++++++++++ owr/owr_utils.h | 14 ++ tests/test_self_view.c | 9 +- tests/test_send_receive.c | 2 +- transport/owr_payload.c | 186 ++---------------- transport/owr_payload_private.h | 3 +- transport/owr_remote_media_source.c | 2 + transport/owr_transport_agent.c | 259 +++++++++++-------------- 19 files changed, 732 insertions(+), 447 deletions(-) diff --git a/local/owr_device_list.c b/local/owr_device_list.c index 9f8faa50..3d421d28 100644 --- a/local/owr_device_list.c +++ b/local/owr_device_list.c @@ -262,7 +262,8 @@ static void source_info_iterator(pa_context *pa_context, const pa_source_info *i } source = _owr_local_media_source_new_cached(info->index, info->description, - OWR_MEDIA_TYPE_AUDIO, OWR_SOURCE_TYPE_CAPTURE); + OWR_MEDIA_TYPE_AUDIO, OWR_SOURCE_TYPE_CAPTURE, + OWR_MEDIA_SOURCE_SUPPORTS_NONE); context->list = g_list_prepend(context->list, source); } else { @@ -283,7 +284,8 @@ static gboolean enumerate_audio_source_devices(GClosure *callback) GList *sources = NULL; source = _owr_local_media_source_new_cached(-1, - "Default audio input", OWR_MEDIA_TYPE_AUDIO, OWR_SOURCE_TYPE_CAPTURE); + "Default audio input", OWR_MEDIA_TYPE_AUDIO, OWR_SOURCE_TYPE_CAPTURE, + OWR_MEDIA_SOURCE_SUPPORTS_NONE); sources = g_list_prepend(sources, source); _owr_utils_call_closure_with_list(callback, sources); g_list_free_full(sources, g_object_unref); @@ -358,7 +360,8 @@ static OwrLocalMediaSource *maybe_create_source_from_filename(const gchar *name) return NULL; source = _owr_local_media_source_new_cached(index, device_name, - OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE); + OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE, + OWR_MEDIA_SOURCE_SUPPORTS_NONE); g_debug("v4l: filename match: %s", device_name); @@ -380,7 +383,9 @@ static gboolean enumerate_video_source_devices(GClosure *callback) #if TARGET_RPI source = _owr_local_media_source_new_cached(-1, "RPiCam", - OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE); + OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE, + OWR_MEDIA_SOURCE_SUPPORTS_VIDEO_ORIENTATION | OWR_MEDIA_SOURCE_SUPPORTS_COLOR_BALANCE); + _owr_media_source_set_codec(OWR_MEDIA_SOURCE(source), OWR_CODEC_TYPE_H264); sources = g_list_prepend(sources, source); #endif @@ -687,11 +692,13 @@ static gboolean enumerate_video_source_devices(GClosure *callback) if (facing == CameraInfo.CAMERA_FACING_FRONT) { source = _owr_local_media_source_new_cached(i, "Front facing Camera", - OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE); + OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE, + OWR_MEDIA_SOURCE_SUPPORTS_NONE); sources = g_list_prepend(sources, source); } else if (facing == CameraInfo.CAMERA_FACING_BACK) { source = _owr_local_media_source_new_cached(i, "Back facing Camera", - OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE); + OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_CAPTURE, + OWR_MEDIA_SOURCE_SUPPORTS_NONE); sources = g_list_append(sources, source); } diff --git a/local/owr_local.c b/local/owr_local.c index 56b5bc97..353bbdd1 100644 --- a/local/owr_local.c +++ b/local/owr_local.c @@ -69,10 +69,10 @@ static GList *get_test_sources(OwrMediaType types) if (g_once_init_enter(&cached_sources)) { GList *sources = NULL; - source = _owr_local_media_source_new_cached(-1, "Audio test source", OWR_MEDIA_TYPE_AUDIO, OWR_SOURCE_TYPE_TEST); + source = _owr_local_media_source_new_cached(-1, "Audio test source", OWR_MEDIA_TYPE_AUDIO, OWR_SOURCE_TYPE_TEST, OWR_MEDIA_SOURCE_SUPPORTS_NONE); sources = g_list_append(sources, OWR_MEDIA_SOURCE(source)); - source = _owr_local_media_source_new_cached(-1, "Video test source", OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_TEST); + source = _owr_local_media_source_new_cached(-1, "Video test source", OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_TEST, OWR_MEDIA_SOURCE_SUPPORTS_NONE); sources = g_list_append(sources, OWR_MEDIA_SOURCE(source)); g_once_init_leave(&cached_sources, sources); diff --git a/local/owr_local_media_source.c b/local/owr_local_media_source.c index 66195154..d9023cac 100644 --- a/local/owr_local_media_source.c +++ b/local/owr_local_media_source.c @@ -566,7 +566,9 @@ static void on_caps(GstElement *source, GParamSpec *pspec, OwrMediaSource *media if (GST_IS_CAPS(caps)) { GST_INFO_OBJECT(source, "%s - configured with caps: %" GST_PTR_FORMAT, media_source_name, caps); + gst_caps_unref(caps); } + g_free(media_source_name); } static void @@ -748,17 +750,21 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s switch (source_type) { case OWR_SOURCE_TYPE_CAPTURE: CREATE_ELEMENT(source, VIDEO_SRC, "video-source"); +#if !defined(TARGET_RPI) || !TARGET_RPI if (priv->device_index > -1) { #if defined(__APPLE__) && !TARGET_IPHONE_SIMULATOR g_object_set(source, "device-index", priv->device_index, NULL); #elif defined(__ANDROID__) g_object_set(source, "cam-index", priv->device_index, NULL); -#elif defined(__linux__) && !TARGET_RPI +#elif defined(__linux__) tmp = g_strdup_printf("/dev/video%d", priv->device_index); g_object_set(source, "device", tmp, NULL); g_free(tmp); #endif } +#else + g_object_set(source, "preview", FALSE, "fullscreen", FALSE, "inline-headers", TRUE, NULL); +#endif break; case OWR_SOURCE_TYPE_TEST: { GstElement *src, *time; @@ -912,7 +918,8 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s } static OwrLocalMediaSource *_owr_local_media_source_new(gint device_index, const gchar *name, - OwrMediaType media_type, OwrSourceType source_type) + OwrMediaType media_type, OwrSourceType source_type, + OwrMediaSourceSupportedInterfaces interfaces) { OwrLocalMediaSource *source; @@ -923,12 +930,14 @@ static OwrLocalMediaSource *_owr_local_media_source_new(gint device_index, const NULL); _owr_media_source_set_type(OWR_MEDIA_SOURCE(source), source_type); + _owr_media_source_set_supported_interfaces(OWR_MEDIA_SOURCE(source), interfaces); return source; } OwrLocalMediaSource *_owr_local_media_source_new_cached(gint device_index, const gchar *name, - OwrMediaType media_type, OwrSourceType source_type) + OwrMediaType media_type, OwrSourceType source_type, + OwrMediaSourceSupportedInterfaces interfaces) { static OwrLocalMediaSource *test_sources[2] = { NULL, }; static GHashTable *sources[2] = { NULL, }; @@ -949,7 +958,7 @@ OwrLocalMediaSource *_owr_local_media_source_new_cached(gint device_index, const if (source_type == OWR_SOURCE_TYPE_TEST) { if (!test_sources[i]) - test_sources[i] = _owr_local_media_source_new(device_index, name, media_type, source_type); + test_sources[i] = _owr_local_media_source_new(device_index, name, media_type, source_type, interfaces); ret = test_sources[i]; @@ -969,7 +978,7 @@ OwrLocalMediaSource *_owr_local_media_source_new_cached(gint device_index, const } if (!ret) { - ret = _owr_local_media_source_new(device_index, name, media_type, source_type); + ret = _owr_local_media_source_new(device_index, name, media_type, source_type, interfaces); g_hash_table_insert(sources[i], GINT_TO_POINTER(device_index), ret); } diff --git a/local/owr_local_media_source_private.h b/local/owr_local_media_source_private.h index 34c36528..b0f1eba1 100644 --- a/local/owr_local_media_source_private.h +++ b/local/owr_local_media_source_private.h @@ -33,6 +33,7 @@ #define __OWR_LOCAL_MEDIA_SOURCE_PRIVATE_H__ #include "owr_local_media_source.h" +#include "owr_media_source_private.h" #include "owr_types.h" @@ -43,7 +44,8 @@ G_BEGIN_DECLS OwrLocalMediaSource *_owr_local_media_source_new_cached(gint device_index, - const gchar *name, OwrMediaType media_type, OwrSourceType source_type); + const gchar *name, OwrMediaType media_type, OwrSourceType source_type, + OwrMediaSourceSupportedInterfaces interfaces); void _owr_local_media_source_set_capture_device_index(OwrLocalMediaSource *source, guint index); G_END_DECLS diff --git a/local/owr_media_renderer.c b/local/owr_media_renderer.c index 5d688e8d..25853295 100644 --- a/local/owr_media_renderer.c +++ b/local/owr_media_renderer.c @@ -115,6 +115,10 @@ static void owr_media_renderer_finalize(GObject *object) G_OBJECT_CLASS(owr_media_renderer_parent_class)->finalize(object); } +static void owr_media_renderer_reconfigure_element_default(G_GNUC_UNUSED OwrMediaRenderer *renderer) +{ +} + static void owr_media_renderer_class_init(OwrMediaRendererClass *klass) { GObjectClass *gobject_class = G_OBJECT_CLASS(klass); @@ -135,6 +139,8 @@ static void owr_media_renderer_class_init(OwrMediaRendererClass *klass) gobject_class->finalize = owr_media_renderer_finalize; g_object_class_install_properties(gobject_class, N_PROPERTIES, obj_properties); + + klass->reconfigure_element = owr_media_renderer_reconfigure_element_default; } static gpointer owr_media_renderer_get_bus_set(OwrMessageOrigin *origin) @@ -305,6 +311,7 @@ static void on_caps(GstElement *sink, GParamSpec *pspec, OwrMediaRenderer *media media_renderer->priv->media_type == OWR_MEDIA_TYPE_AUDIO ? "Audio" : media_renderer->priv->media_type == OWR_MEDIA_TYPE_VIDEO ? "Video" : "Unknown", caps); + gst_caps_unref(caps); } } @@ -389,6 +396,7 @@ static gboolean set_source(GHashTable *args) priv->source = g_object_ref(source); + _owr_media_renderer_reconfigure_element(renderer); maybe_start_renderer(renderer); g_mutex_unlock(&priv->media_renderer_lock); @@ -469,6 +477,17 @@ void _owr_media_renderer_set_sink(OwrMediaRenderer *renderer, gpointer sink_ptr) g_mutex_unlock(&priv->media_renderer_lock); } +OwrMediaSource* _owr_media_renderer_get_source(OwrMediaRenderer *renderer) +{ + return renderer->priv->source; +} + +void _owr_media_renderer_reconfigure_element(OwrMediaRenderer *renderer) +{ + g_return_if_fail(OWR_IS_MEDIA_RENDERER(renderer)); + OWR_MEDIA_RENDERER_GET_CLASS(renderer)->reconfigure_element(renderer); +} + gchar * owr_media_renderer_get_dot_data(OwrMediaRenderer *renderer) { g_return_val_if_fail(OWR_IS_MEDIA_RENDERER(renderer), NULL); diff --git a/local/owr_media_renderer.h b/local/owr_media_renderer.h index bb29a214..69974c86 100644 --- a/local/owr_media_renderer.h +++ b/local/owr_media_renderer.h @@ -63,6 +63,7 @@ struct _OwrMediaRendererClass { /*< private >*/ void *(*get_caps)(OwrMediaRenderer *renderer); void *(*get_sink)(OwrMediaRenderer *renderer); + void (*reconfigure_element)(OwrMediaRenderer *renderer); }; GType owr_media_renderer_get_type(void) G_GNUC_CONST; diff --git a/local/owr_media_renderer_private.h b/local/owr_media_renderer_private.h index dc99e22a..045491c7 100644 --- a/local/owr_media_renderer_private.h +++ b/local/owr_media_renderer_private.h @@ -39,6 +39,8 @@ G_BEGIN_DECLS void _owr_media_renderer_set_sink(OwrMediaRenderer *renderer, gpointer sink); GstPipeline * _owr_media_renderer_get_pipeline(OwrMediaRenderer *renderer); +OwrMediaSource* _owr_media_renderer_get_source(OwrMediaRenderer *renderer); +void _owr_media_renderer_reconfigure_element(OwrMediaRenderer *renderer); G_END_DECLS diff --git a/local/owr_video_renderer.c b/local/owr_video_renderer.c index 8f1ab142..adbea545 100644 --- a/local/owr_video_renderer.c +++ b/local/owr_video_renderer.c @@ -35,12 +35,14 @@ #include "owr_video_renderer.h" #include "owr_media_renderer_private.h" +#include "owr_media_source_private.h" #include "owr_private.h" #include "owr_utils.h" #include "owr_video_renderer_private.h" #include "owr_window_registry.h" #include "owr_window_registry_private.h" +#include #include #include @@ -86,7 +88,9 @@ static void owr_video_renderer_get_property(GObject *object, guint property_id, GValue *value, GParamSpec *pspec); static void owr_video_renderer_constructed(GObject *object); -static GstElement *owr_video_renderer_get_element(OwrMediaRenderer *renderer, guintptr window_handle); +static GstElement *owr_video_renderer_get_element(OwrMediaRenderer *renderer); +static void owr_video_renderer_reconfigure_element(OwrMediaRenderer *renderer); +static GstElement *owr_video_renderer_get_element_with_window_handle(OwrMediaRenderer *renderer, guintptr window_handle); static GstCaps *owr_video_renderer_get_caps(OwrMediaRenderer *renderer); static GstElement *owr_video_renderer_get_sink(OwrMediaRenderer *renderer); @@ -99,6 +103,7 @@ struct _OwrVideoRendererPrivate { gchar *tag; GMutex closure_mutex; GClosure *request_context; + GstElement *renderer_bin; }; static void owr_video_renderer_finalize(GObject *object) @@ -119,6 +124,11 @@ static void owr_video_renderer_finalize(GObject *object) priv->request_context = NULL; } + if (priv->renderer_bin) { + gst_object_unref(priv->renderer_bin); + priv->renderer_bin = NULL; + } + G_OBJECT_CLASS(owr_video_renderer_parent_class)->finalize(object); } @@ -161,6 +171,7 @@ static void owr_video_renderer_class_init(OwrVideoRendererClass *klass) media_renderer_class->get_caps = (void *(*)(OwrMediaRenderer *))owr_video_renderer_get_caps; media_renderer_class->get_sink = (void *(*)(OwrMediaRenderer *))owr_video_renderer_get_sink; + media_renderer_class->reconfigure_element = (void (*)(OwrMediaRenderer *))owr_video_renderer_reconfigure_element; g_object_class_install_properties(gobject_class, N_PROPERTIES, obj_properties); } @@ -178,6 +189,7 @@ static void owr_video_renderer_init(OwrVideoRenderer *renderer) priv->mirror = DEFAULT_MIRROR; g_mutex_init(&priv->closure_mutex); priv->request_context = NULL; + priv->renderer_bin = NULL; } static void owr_video_renderer_set_property(GObject *object, guint property_id, @@ -269,31 +281,66 @@ OwrVideoRenderer *owr_video_renderer_new(const gchar *tag) if (!gst_element_link(a, b)) \ GST_ERROR("Failed to link " #a " -> " #b); -static void renderer_disabled(OwrMediaRenderer *renderer, GParamSpec *pspec, GstElement *balance) +static void renderer_disabled(OwrMediaRenderer *renderer, G_GNUC_UNUSED GParamSpec *pspec, GstElement *balance) { + // FIXME: We need to be able to disable rendering without a + // balance element. This is highly inneficient. gboolean disabled = FALSE; + GstColorBalance* color_balance = NULL; g_return_if_fail(OWR_IS_MEDIA_RENDERER(renderer)); - g_return_if_fail(G_IS_PARAM_SPEC(pspec) || !pspec); - g_return_if_fail(GST_IS_ELEMENT(balance)); + + if (GST_IS_COLOR_BALANCE(balance)) { + color_balance = GST_COLOR_BALANCE(gst_object_ref(balance)); + } else { + OwrMediaSource* media_source = _owr_media_renderer_get_source(renderer); + GstElement* src_bin = _owr_media_source_get_source_bin(media_source); + balance = gst_bin_get_by_interface(GST_BIN(src_bin), GST_TYPE_COLOR_BALANCE); + gst_object_unref(src_bin); + g_return_if_fail(GST_IS_COLOR_BALANCE(balance)); + color_balance = GST_COLOR_BALANCE(balance); + } g_object_get(renderer, "disabled", &disabled, NULL); - g_object_set(balance, "saturation", (gdouble)!disabled, "brightness", (gdouble)-disabled, NULL); + + const GList* controls = gst_color_balance_list_channels(color_balance); + gint index = 0; + for (const GList* item = controls; item != NULL; item = item->next, ++index) { + GstColorBalanceChannel* channel = item->data; + if (g_strcmp0(channel->label, "SATURATION") == 0 || g_strcmp0(channel->label, "BRIGHTNESS") == 0) { + gint new_value = disabled ? channel->min_value : ((channel->min_value + channel->max_value) / 2); + gst_color_balance_set_value(color_balance, channel, new_value); + } + } + + gst_object_unref(color_balance); } static void update_flip_method(OwrMediaRenderer *renderer, GParamSpec *pspec, GstElement *flip) { - guint rotation = 0; - gboolean mirror = FALSE; - gint flip_method; + g_assert(OWR_IS_MEDIA_RENDERER(renderer)); - g_return_if_fail(OWR_IS_MEDIA_RENDERER(renderer)); - g_return_if_fail(G_IS_PARAM_SPEC(pspec) || !pspec); - g_return_if_fail(GST_IS_ELEMENT(flip)); + if (!flip) { + OwrMediaSource* source = _owr_media_renderer_get_source(renderer); + + if (_owr_media_source_supports_interfaces(source, OWR_MEDIA_SOURCE_SUPPORTS_VIDEO_ORIENTATION)) { + GstElement* bin = _owr_media_source_get_source_bin(source); + + flip = gst_bin_get_by_name(GST_BIN(bin), "video-source"); + g_assert(flip); + + pspec = g_object_class_find_property(G_OBJECT_GET_CLASS(flip), "orientation"); + + // For simplicity, and considering that we already assume + // with the parameter that the object is alive, we can + // safely unref here. + gst_object_unref(flip); + gst_object_unref(bin); + } + } - g_object_get(renderer, "rotation", &rotation, "mirror", &mirror, NULL); - flip_method = _owr_rotation_and_mirror_to_video_flip_method(rotation, mirror); - g_object_set(flip, "method", flip_method, NULL); + if (flip) + _owr_update_flip_method(G_OBJECT(renderer), pspec, flip); } static void disable_last_sample_on_sink(const GValue *item, gpointer data) @@ -309,40 +356,87 @@ static void disable_last_sample_on_sink(const GValue *item, gpointer data) g_object_set(element, "enable-last-sample", FALSE, NULL); } -static GstElement *owr_video_renderer_get_element(OwrMediaRenderer *renderer, guintptr window_handle) +static GstElement *owr_video_renderer_get_element(OwrMediaRenderer *renderer) { OwrVideoRenderer *video_renderer; OwrVideoRendererPrivate *priv; - GstElement *renderer_bin; - GstElement *upload, *convert, *balance, *flip, *sink; - GstPad *ghostpad, *sinkpad; gchar *bin_name; - GValue value = G_VALUE_INIT; - g_assert(renderer); + g_assert(OWR_IS_VIDEO_RENDERER(renderer)); video_renderer = OWR_VIDEO_RENDERER(renderer); priv = video_renderer->priv; bin_name = g_strdup_printf("video-renderer-bin-%u", g_atomic_int_add(&unique_bin_id, 1)); - renderer_bin = gst_bin_new(bin_name); + priv->renderer_bin = gst_bin_new(bin_name); g_free(bin_name); + return GST_ELEMENT(gst_object_ref(priv->renderer_bin)); +} + +static void owr_video_renderer_reconfigure_element(OwrMediaRenderer *renderer) +{ + OwrVideoRenderer *video_renderer; + OwrVideoRendererPrivate *priv; + GstElement *parser; + GstElement *decoder; + GstElement *balance = NULL; + GstElement *upload, *sink, *flip = NULL; + GstPad *ghostpad, *sinkpad; + GValue value = G_VALUE_INIT; + OwrMediaSource *source; + OwrCodecType codec_type; + gboolean link_ok = TRUE; + GstElement *first = NULL; + + g_assert(OWR_IS_VIDEO_RENDERER(renderer)); + video_renderer = OWR_VIDEO_RENDERER(renderer); + priv = video_renderer->priv; + + source = _owr_media_renderer_get_source(renderer); + codec_type = _owr_media_source_get_codec(source); + + parser = _owr_create_parser(codec_type); + decoder = _owr_create_decoder(codec_type); + if (parser) + gst_bin_add(GST_BIN(priv->renderer_bin), parser); + if (decoder) + gst_bin_add(GST_BIN(priv->renderer_bin), decoder); + upload = gst_element_factory_make("glupload", "video-renderer-upload"); - convert = gst_element_factory_make("glcolorconvert", "video-renderer-convert"); + gst_bin_add(GST_BIN(priv->renderer_bin), upload); - balance = gst_element_factory_make("glcolorbalance", "video-renderer-balance"); - g_signal_connect_object(renderer, "notify::disabled", G_CALLBACK(renderer_disabled), - balance, 0); - renderer_disabled(renderer, NULL, balance); + if (!_owr_media_source_supports_interfaces(source, OWR_MEDIA_SOURCE_SUPPORTS_COLOR_BALANCE)) { + GstElement *convert = NULL; - flip = gst_element_factory_make("glvideoflip", "video-renderer-flip"); - if (!flip) { - g_warning("The glvideoflip GStreamer element isn't available. Video mirroring and rotation functionalities are thus disabled."); - } else { - g_signal_connect_object(renderer, "notify::rotation", G_CALLBACK(update_flip_method), flip, 0); - g_signal_connect_object(renderer, "notify::mirror", G_CALLBACK(update_flip_method), flip, 0); - update_flip_method(renderer, NULL, flip); + balance = gst_element_factory_make("glcolorbalance", "video-renderer-balance"); + + if (G_LIKELY(balance)) { + convert = gst_element_factory_make("glcolorconvert", "video-renderer-convert"); + + if (G_LIKELY(convert)) { + renderer_disabled(renderer, NULL, balance); + gst_bin_add_many(GST_BIN(priv->renderer_bin), convert, balance, NULL); + } else + g_object_unref(balance); + } + + if (!convert || !balance) + g_warning("cannot create convert or balance elements to disable rendering"); + } + g_signal_connect_object(renderer, "notify::disabled", G_CALLBACK(renderer_disabled), balance, 0); + + if (!_owr_media_source_supports_interfaces(source, OWR_MEDIA_SOURCE_SUPPORTS_VIDEO_ORIENTATION)) { + flip = gst_element_factory_make("glvideoflip", "video-renderer-flip"); + if (G_LIKELY(flip)) { + _owr_update_flip_method(G_OBJECT(renderer), NULL, flip); + gst_bin_add(GST_BIN(priv->renderer_bin), flip); + } else + g_warning("no suitable flipping element"); } + g_signal_connect_object(renderer, "notify::rotation", G_CALLBACK(update_flip_method), flip, 0); + g_signal_connect_object(renderer, "notify::mirror", G_CALLBACK(update_flip_method), flip, 0); + + g_object_unref(source); sink = OWR_MEDIA_RENDERER_GET_CLASS(renderer)->get_sink(renderer); g_assert(sink); @@ -352,7 +446,34 @@ static GstElement *owr_video_renderer_get_element(OwrMediaRenderer *renderer, gu disable_last_sample_on_sink(&value, NULL); g_value_unset(&value); - if (priv->tag) { + // FIXME: huhhhhh really? + g_object_set(sink, "sync", FALSE, NULL); + + gst_bin_add(GST_BIN(priv->renderer_bin), sink); + + _owr_bin_link_and_sync_elements(GST_BIN(priv->renderer_bin), &link_ok, NULL, &first, NULL); + g_warn_if_fail(link_ok); + sinkpad = gst_element_get_static_pad(first, "sink"); + + g_assert(sinkpad); + ghostpad = gst_ghost_pad_new("sink", sinkpad); + gst_pad_set_active(ghostpad, TRUE); + gst_element_add_pad(priv->renderer_bin, ghostpad); + gst_object_unref(sinkpad); +} + +static GstElement *owr_video_renderer_get_element_with_window_handle(OwrMediaRenderer *renderer, guintptr window_handle) +{ + GstElement *renderer_bin, *sink; + + g_assert(OWR_IS_VIDEO_RENDERER(renderer)); + + renderer_bin = owr_video_renderer_get_element(renderer); + owr_video_renderer_reconfigure_element(renderer); + + sink = OWR_MEDIA_RENDERER_GET_CLASS(renderer)->get_sink(renderer); + g_assert(sink); + if (OWR_VIDEO_RENDERER(renderer)->priv->tag) { GstElement *sink_element = GST_IS_BIN(sink) ? gst_bin_get_by_interface(GST_BIN(sink), GST_TYPE_VIDEO_OVERLAY) : sink; if (GST_IS_ELEMENT(sink_element) && GST_IS_VIDEO_OVERLAY(sink)) @@ -363,26 +484,6 @@ static GstElement *owr_video_renderer_get_element(OwrMediaRenderer *renderer, gu g_object_unref(sink_element); } - gst_bin_add_many(GST_BIN(renderer_bin), upload, convert, balance, sink, NULL); - - LINK_ELEMENTS(upload, convert); - LINK_ELEMENTS(convert, balance); - - if (flip) { - gst_bin_add(GST_BIN(renderer_bin), flip); - LINK_ELEMENTS(balance, flip); - LINK_ELEMENTS(flip, sink); - } else { - LINK_ELEMENTS(balance, sink); - } - - sinkpad = gst_element_get_static_pad(upload, "sink"); - g_assert(sinkpad); - ghostpad = gst_ghost_pad_new("sink", sinkpad); - gst_pad_set_active(ghostpad, TRUE); - gst_element_add_pad(renderer_bin, ghostpad); - gst_object_unref(sinkpad); - return renderer_bin; } @@ -452,7 +553,7 @@ static void owr_video_renderer_constructed(GObject *object) /* If we have no tag, just directly create the sink */ if (!priv->tag) - _owr_media_renderer_set_sink(OWR_MEDIA_RENDERER(video_renderer), owr_video_renderer_get_element(OWR_MEDIA_RENDERER(video_renderer), 0)); + _owr_media_renderer_set_sink(OWR_MEDIA_RENDERER(video_renderer), owr_video_renderer_get_element(OWR_MEDIA_RENDERER(video_renderer))); pipeline = _owr_media_renderer_get_pipeline(OWR_MEDIA_RENDERER(video_renderer)); g_assert(pipeline); @@ -469,6 +570,7 @@ static GstCaps *owr_video_renderer_get_caps(OwrMediaRenderer *renderer) GstCaps *caps; guint width = 0, height = 0; gdouble max_framerate = 0.0; + OwrMediaSource *source; g_object_get(OWR_VIDEO_RENDERER(renderer), "width", &width, @@ -476,7 +578,9 @@ static GstCaps *owr_video_renderer_get_caps(OwrMediaRenderer *renderer) "max-framerate", &max_framerate, NULL); - caps = gst_caps_new_empty_simple("video/x-raw"); + source = _owr_media_renderer_get_source(renderer); + caps = gst_caps_new_empty_simple(_owr_codec_type_to_caps_mime(_owr_media_source_get_media_type(source), + _owr_media_source_get_codec(source))); gst_caps_set_features(caps, 0, gst_caps_features_new_any()); if (width > 0) gst_caps_set_simple(caps, "width", G_TYPE_INT, width, NULL); @@ -511,6 +615,6 @@ void _owr_video_renderer_notify_tag_changed(OwrVideoRenderer *video_renderer, co _owr_media_renderer_set_sink(OWR_MEDIA_RENDERER(video_renderer), NULL); if (have_handle) { _owr_media_renderer_set_sink(OWR_MEDIA_RENDERER(video_renderer), - owr_video_renderer_get_element(OWR_MEDIA_RENDERER(video_renderer), new_handle)); + owr_video_renderer_get_element_with_window_handle(OWR_MEDIA_RENDERER(video_renderer), new_handle)); } } diff --git a/owr/owr.c b/owr/owr.c index eb705645..3f49c008 100644 --- a/owr/owr.c +++ b/owr/owr.c @@ -196,6 +196,8 @@ static void gst_log_android_handler(GstDebugCategory *category, */ void owr_init(GMainContext *main_context) { + static GOnce g_once = G_ONCE_INIT; + g_return_if_fail(!owr_initialized); #ifdef __ANDROID__ @@ -316,6 +318,8 @@ void owr_init(GMainContext *main_context) owr_main_context = g_main_context_ref_thread_default(); else g_main_context_ref(owr_main_context); + + g_once(&g_once, _owr_detect_codecs, NULL); } static gboolean owr_running_callback(GAsyncQueue *msg_queue) diff --git a/owr/owr_media_source.c b/owr/owr_media_source.c index d7a59d95..8efb3dc1 100644 --- a/owr/owr_media_source.c +++ b/owr/owr_media_source.c @@ -121,6 +121,8 @@ struct _OwrMediaSourcePrivate { GstElement *source_bin; /* Tee element from which we can tap the source for multiple consumers */ GstElement *source_tee; + + OwrMediaSourceSupportedInterfaces supported_interfaces; }; static void owr_media_source_set_property(GObject *object, guint property_id, @@ -197,6 +199,7 @@ static void owr_media_source_init(OwrMediaSource *source) priv->source_bin = NULL; priv->source_tee = NULL; + priv->supported_interfaces = OWR_MEDIA_SOURCE_SUPPORTS_NONE; g_mutex_init(&source->lock); } @@ -263,7 +266,7 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media { OwrMediaType media_type; GstElement *source_pipeline, *tee; - GstElement *source_bin, *source = NULL, *queue_pre, *queue_post; + GstElement *source_bin, *source = NULL, *queue_pre = NULL, *queue_post = NULL; GstElement *capsfilter; GstElement *sink, *sink_queue, *sink_bin; GstPad *bin_pad = NULL, *srcpad, *sinkpad; @@ -283,9 +286,6 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media source_bin = gst_bin_new(bin_name); g_free(bin_name); - CREATE_ELEMENT_WITH_ID(queue_pre, "queue", "source-queue", source_id); - CREATE_ELEMENT_WITH_ID(capsfilter, "capsfilter", "source-output-capsfilter", source_id); - CREATE_ELEMENT_WITH_ID(queue_post, "queue", "source-output-queue", source_id); CREATE_ELEMENT_WITH_ID(sink_queue, "queue", "sink-queue", source_id); @@ -295,8 +295,12 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media { GstElement *audioresample, *audioconvert; + CREATE_ELEMENT_WITH_ID(capsfilter, "capsfilter", "source-output-capsfilter", source_id); g_object_set(capsfilter, "caps", caps, NULL); + CREATE_ELEMENT_WITH_ID(queue_pre, "queue", "source-queue", source_id); + CREATE_ELEMENT_WITH_ID(queue_post, "queue", "source-output-queue", source_id); + CREATE_ELEMENT_WITH_ID(audioresample, "audioresample", "source-audio-resample", source_id); CREATE_ELEMENT_WITH_ID(audioconvert, "audioconvert", "source-audio-convert", source_id); @@ -311,65 +315,71 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media } case OWR_MEDIA_TYPE_VIDEO: { - GstElement *videorate = NULL, *videoscale = NULL, *videoconvert; - GstStructure *s; - GstCapsFeatures *features; - - s = gst_caps_get_structure(caps, 0); - if (gst_structure_has_field(s, "framerate")) { - gint fps_n = 0, fps_d = 0; + if (_owr_codec_type_is_raw(_owr_media_source_get_codec(media_source))) { + GstElement *videorate = NULL, *videoscale = NULL, *videoconvert; + GstStructure *s; + GstCapsFeatures *features; - gst_structure_get_fraction(s, "framerate", &fps_n, &fps_d); - g_assert(fps_d); + CREATE_ELEMENT_WITH_ID(queue_pre, "queue", "source-queue", source_id); + CREATE_ELEMENT_WITH_ID(queue_post, "queue", "source-output-queue", source_id); - CREATE_ELEMENT_WITH_ID(videorate, "videorate", "source-video-rate", source_id); - g_object_set(videorate, "drop-only", TRUE, "max-rate", fps_n / fps_d, NULL); + s = gst_caps_get_structure(caps, 0); + if (gst_structure_has_field(s, "framerate")) { + gint fps_n = 0, fps_d = 0; - gst_structure_remove_field(s, "framerate"); - gst_bin_add(GST_BIN(source_bin), videorate); - } - - g_object_set(capsfilter, "caps", caps, NULL); - features = gst_caps_get_features(caps, 0); - if (gst_caps_features_contains(features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) { - GstElement *glupload; + gst_structure_get_fraction(s, "framerate", &fps_n, &fps_d); + g_assert(fps_d); - CREATE_ELEMENT_WITH_ID(glupload, "glupload", "source-glupload", source_id); - CREATE_ELEMENT_WITH_ID(videoscale, "gleffects_identity", "source-glcolorscale", source_id); - CREATE_ELEMENT_WITH_ID(videoconvert, "glcolorconvert", "source-glcolorconvert", source_id); + CREATE_ELEMENT_WITH_ID(videorate, "videorate", "source-video-rate", source_id); + g_object_set(videorate, "drop-only", TRUE, "max-rate", fps_n / fps_d, NULL); - gst_bin_add_many(GST_BIN(source_bin), - queue_pre, glupload, videoconvert, videoscale, queue_post, NULL); - - if (videorate) { - LINK_ELEMENTS(queue_pre, videorate); - LINK_ELEMENTS(videorate, glupload); - } else { - LINK_ELEMENTS(queue_pre, glupload); + gst_structure_remove_field(s, "framerate"); + gst_bin_add(GST_BIN(source_bin), videorate); } - LINK_ELEMENTS(glupload, videoconvert); - LINK_ELEMENTS(videoconvert, videoscale); - LINK_ELEMENTS(videoscale, queue_post); - } else { - GstElement *gldownload; - - CREATE_ELEMENT_WITH_ID(gldownload, "gldownload", "source-gldownload", source_id); - CREATE_ELEMENT_WITH_ID(videoscale, "videoscale", "source-video-scale", source_id); - CREATE_ELEMENT_WITH_ID(videoconvert, VIDEO_CONVERT, "source-video-convert", source_id); - gst_bin_add_many(GST_BIN(source_bin), - queue_pre, gldownload, videoscale, videoconvert, capsfilter, queue_post, NULL); - if (videorate) { - LINK_ELEMENTS(queue_pre, videorate); - LINK_ELEMENTS(videorate, gldownload); + + features = gst_caps_get_features(caps, 0); + if (gst_caps_features_contains(features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) { + GstElement *glupload; + + CREATE_ELEMENT_WITH_ID(glupload, "glupload", "source-glupload", source_id); + CREATE_ELEMENT_WITH_ID(videoscale, "gleffects_identity", "source-glcolorscale", source_id); + CREATE_ELEMENT_WITH_ID(videoconvert, "glcolorconvert", "source-glcolorconvert", source_id); + + gst_bin_add_many(GST_BIN(source_bin), + queue_pre, glupload, videoconvert, videoscale, queue_post, NULL); + + if (videorate) { + LINK_ELEMENTS(queue_pre, videorate); + LINK_ELEMENTS(videorate, glupload); + } else { + LINK_ELEMENTS(queue_pre, glupload); + } + LINK_ELEMENTS(glupload, videoconvert); + LINK_ELEMENTS(videoconvert, videoscale); + LINK_ELEMENTS(videoscale, queue_post); } else { - LINK_ELEMENTS(queue_pre, gldownload); + GstElement *gldownload; + + CREATE_ELEMENT_WITH_ID(capsfilter, "capsfilter", "source-output-capsfilter", source_id); + g_object_set(capsfilter, "caps", caps, NULL); + + CREATE_ELEMENT_WITH_ID(gldownload, "gldownload", "source-gldownload", source_id); + CREATE_ELEMENT_WITH_ID(videoscale, "videoscale", "source-video-scale", source_id); + CREATE_ELEMENT_WITH_ID(videoconvert, VIDEO_CONVERT, "source-video-convert", source_id); + gst_bin_add_many(GST_BIN(source_bin), + queue_pre, gldownload, videoscale, videoconvert, capsfilter, queue_post, NULL); + if (videorate) { + LINK_ELEMENTS(queue_pre, videorate); + LINK_ELEMENTS(videorate, gldownload); + } else { + LINK_ELEMENTS(queue_pre, gldownload); + } + LINK_ELEMENTS(gldownload, videoscale); + LINK_ELEMENTS(videoscale, videoconvert); + LINK_ELEMENTS(videoconvert, capsfilter); + LINK_ELEMENTS(capsfilter, queue_post); } - LINK_ELEMENTS(gldownload, videoscale); - LINK_ELEMENTS(videoscale, videoconvert); - LINK_ELEMENTS(videoconvert, capsfilter); - LINK_ELEMENTS(capsfilter, queue_post); } - break; } case OWR_MEDIA_TYPE_UNKNOWN: @@ -408,7 +418,12 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media LINK_ELEMENTS(tee, sink_bin); /* Start up our new bin and link it all */ - srcpad = gst_element_get_static_pad(queue_post, "src"); + gst_bin_add(GST_BIN(source_bin), source); + if (queue_post) { + srcpad = gst_element_get_static_pad(queue_post, "src"); + } else { + srcpad = gst_element_get_static_pad(source, "src"); + } g_assert(srcpad); bin_pad = gst_ghost_pad_new("src", srcpad); @@ -416,8 +431,8 @@ static GstElement *owr_media_source_request_source_default(OwrMediaSource *media gst_pad_set_active(bin_pad, TRUE); gst_element_add_pad(source_bin, bin_pad); - gst_bin_add(GST_BIN(source_bin), source); - LINK_ELEMENTS(source, queue_pre); + if (queue_pre) + LINK_ELEMENTS(source, queue_pre); done: @@ -620,6 +635,12 @@ void _owr_media_source_set_type(OwrMediaSource *media_source, OwrSourceType type g_atomic_int_set(&media_source->priv->type, type); } +OwrSourceType _owr_media_source_get_media_type(OwrMediaSource *media_source) +{ + g_return_val_if_fail(OWR_IS_MEDIA_SOURCE(media_source), OWR_MEDIA_TYPE_UNKNOWN); + return media_source->priv->media_type; +} + OwrCodecType _owr_media_source_get_codec(OwrMediaSource *media_source) { g_return_val_if_fail(OWR_IS_MEDIA_SOURCE(media_source), OWR_CODEC_TYPE_NONE); @@ -646,3 +667,15 @@ gchar * owr_media_source_get_dot_data(OwrMediaSource *source) return g_strdup(""); #endif } + +void _owr_media_source_set_supported_interfaces(OwrMediaSource *source, OwrMediaSourceSupportedInterfaces interfaces) +{ + g_return_if_fail(OWR_IS_MEDIA_SOURCE(source)); + source->priv->supported_interfaces = interfaces; +} + +gboolean _owr_media_source_supports_interfaces(OwrMediaSource *source, OwrMediaSourceSupportedInterfaces interfaces) +{ + g_return_val_if_fail(OWR_IS_MEDIA_SOURCE(source), FALSE); + return source->priv->supported_interfaces & interfaces; +} diff --git a/owr/owr_media_source_private.h b/owr/owr_media_source_private.h index ca9bb4ae..8aeab425 100644 --- a/owr/owr_media_source_private.h +++ b/owr/owr_media_source_private.h @@ -43,6 +43,12 @@ G_BEGIN_DECLS +typedef enum { + OWR_MEDIA_SOURCE_SUPPORTS_NONE = 0, + OWR_MEDIA_SOURCE_SUPPORTS_VIDEO_ORIENTATION = (1 << 0), + OWR_MEDIA_SOURCE_SUPPORTS_COLOR_BALANCE = (1 << 1), +} OwrMediaSourceSupportedInterfaces; + GstElement *_owr_media_source_get_source_bin(OwrMediaSource *media_source); void _owr_media_source_set_source_bin(OwrMediaSource *media_source, GstElement *bin); @@ -52,11 +58,19 @@ void _owr_media_source_set_source_tee(OwrMediaSource *media_source, GstElement * GstElement *_owr_media_source_request_source(OwrMediaSource *media_source, GstCaps *caps); void _owr_media_source_release_source(OwrMediaSource *media_source, GstElement *source); +/* FIXME: At some point we should rename this function to + * set_media_type because get_type could eventually conflict with + * GObject required function if this function becomes public. */ void _owr_media_source_set_type(OwrMediaSource *source, OwrSourceType type); +OwrSourceType _owr_media_source_get_media_type(OwrMediaSource *source); void _owr_media_source_set_codec(OwrMediaSource *source, OwrCodecType codec_type); OwrCodecType _owr_media_source_get_codec(OwrMediaSource *source); +void _owr_media_source_set_supported_interfaces(OwrMediaSource *source, OwrMediaSourceSupportedInterfaces interfaces); + +gboolean _owr_media_source_supports_interfaces(OwrMediaSource *source, OwrMediaSourceSupportedInterfaces interfaces); + G_END_DECLS #endif /* __GTK_DOC_IGNORE__ */ diff --git a/owr/owr_utils.c b/owr/owr_utils.c index c84eedf1..e0228e8b 100644 --- a/owr/owr_utils.c +++ b/owr/owr_utils.c @@ -36,6 +36,25 @@ #include "owr_types.h" +/* To be extended once more codecs are supported */ +static GList *h264_decoders = NULL; +static GList *h264_encoders = NULL; +static GList *vp8_decoders = NULL; +static GList *vp8_encoders = NULL; +static GList *vp9_decoders = NULL; +static GList *vp9_encoders = NULL; + +static const gchar *OwrCodecTypeEncoderElementName[] = { NULL, "mulawenc", "alawenc", "opusenc", "openh264enc", "vp8enc", "vp9enc" }; +static const gchar *OwrCodecTypeDecoderElementName[] = { NULL, "mulawdec", "alawdec", "opusdec", "openh264dec", "vp8dec", "vp9dec" }; + +static const gchar *OwrCodecTypeParserElementName[] = { NULL, NULL, NULL, NULL, "h264parse", NULL, NULL }; + +guint _owr_get_unique_uint_id() +{ + static guint id = 0; + return g_atomic_int_add(&id, 1); +} + OwrCodecType _owr_caps_to_codec_type(GstCaps *caps) { GstStructure *structure; @@ -61,6 +80,227 @@ OwrCodecType _owr_caps_to_codec_type(GstCaps *caps) return OWR_CODEC_TYPE_NONE; } +const gchar* _owr_codec_type_to_caps_mime(OwrMediaType media_type, OwrCodecType codec_type) +{ + switch (codec_type) + { + case OWR_CODEC_TYPE_NONE: + switch (media_type) + { + case OWR_MEDIA_TYPE_AUDIO: + return "audio/x-raw"; + break; + case OWR_MEDIA_TYPE_VIDEO: + return "video/x-raw"; + break; + default: + g_return_val_if_reached("audio/x-raw"); + } + break; + case OWR_CODEC_TYPE_PCMU: + return "audio/x-mulaw"; + break; + case OWR_CODEC_TYPE_PCMA: + return "audio/x-alaw"; + break; + case OWR_CODEC_TYPE_OPUS: + return "audio/x-opus"; + break; + case OWR_CODEC_TYPE_H264: + return "video/x-h264"; + break; + case OWR_CODEC_TYPE_VP8: + return "video/x-vp8"; + break; + case OWR_CODEC_TYPE_VP9: + return "video/x-vp9"; + break; + default: + break; + } + g_return_val_if_reached("audio/x-raw"); +} + +gpointer _owr_detect_codecs(gpointer data) +{ + GList *decoder_factories; + GList *encoder_factories; + GstCaps *caps; + + OWR_UNUSED(data); + + decoder_factories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | + GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, + GST_RANK_MARGINAL); + encoder_factories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_ENCODER | + GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, + GST_RANK_MARGINAL); + + caps = gst_caps_new_empty_simple("video/x-h264"); + h264_decoders = gst_element_factory_list_filter(decoder_factories, caps, GST_PAD_SINK, FALSE); + h264_encoders = gst_element_factory_list_filter(encoder_factories, caps, GST_PAD_SRC, FALSE); + gst_caps_unref(caps); + + caps = gst_caps_new_empty_simple("video/x-vp8"); + vp8_decoders = gst_element_factory_list_filter(decoder_factories, caps, GST_PAD_SINK, FALSE); + vp8_encoders = gst_element_factory_list_filter(encoder_factories, caps, GST_PAD_SRC, FALSE); + gst_caps_unref(caps); + + caps = gst_caps_new_empty_simple("video/x-vp9"); + vp9_decoders = gst_element_factory_list_filter(decoder_factories, caps, GST_PAD_SINK, FALSE); + vp9_encoders = gst_element_factory_list_filter(encoder_factories, caps, GST_PAD_SRC, FALSE); + gst_caps_unref(caps); + + gst_plugin_feature_list_free(decoder_factories); + gst_plugin_feature_list_free(encoder_factories); + + h264_decoders = g_list_sort(h264_decoders, gst_plugin_feature_rank_compare_func); + h264_encoders = g_list_sort(h264_encoders, gst_plugin_feature_rank_compare_func); + vp8_decoders = g_list_sort(vp8_decoders, gst_plugin_feature_rank_compare_func); + vp8_encoders = g_list_sort(vp8_encoders, gst_plugin_feature_rank_compare_func); + vp9_decoders = g_list_sort(vp9_decoders, gst_plugin_feature_rank_compare_func); + vp9_encoders = g_list_sort(vp9_encoders, gst_plugin_feature_rank_compare_func); + + return NULL; +} + +const GList *_owr_get_detected_h264_encoders() +{ + return h264_encoders; +} + +const GList *_owr_get_detected_vp8_encoders() +{ + return vp8_encoders; +} + +const GList *_owr_get_detected_vp9_encoders() +{ + return vp9_encoders; +} + +GstElement *_owr_try_codecs(const GList *codecs, const gchar *name_prefix) +{ + GList *l; + gchar *element_name; + + for (l = (GList*) codecs; l; l = l->next) { + GstElementFactory *f = l->data; + GstElement *e; + + element_name = g_strdup_printf("%s_%s_%u", name_prefix, + gst_plugin_feature_get_name(GST_PLUGIN_FEATURE(f)), + _owr_get_unique_uint_id()); + + e = gst_element_factory_create(f, element_name); + g_free(element_name); + + if (!e) + continue; + + /* Try setting to READY. If this fails the codec does not work, for + * example because the hardware codec is currently busy + */ + if (gst_element_set_state(e, GST_STATE_READY) != GST_STATE_CHANGE_SUCCESS) { + gst_element_set_state(e, GST_STATE_NULL); + gst_object_unref(e); + continue; + } + + return e; + } + + return NULL; +} + +GstElement * _owr_create_decoder(OwrCodecType codec_type) +{ + GstElement * decoder = NULL; + gchar *element_name = NULL; + + switch (codec_type) { + case OWR_CODEC_TYPE_H264: + decoder = _owr_try_codecs(h264_decoders, "decoder"); + g_return_val_if_fail(decoder, NULL); + break; + case OWR_CODEC_TYPE_VP8: + decoder = _owr_try_codecs(vp8_decoders, "decoder"); + g_return_val_if_fail(decoder, NULL); + break; + case OWR_CODEC_TYPE_VP9: + decoder = _owr_try_codecs(vp9_decoders, "decoder"); + g_return_val_if_fail(decoder, NULL); + break; + default: + element_name = g_strdup_printf("decoder_%s_%u", OwrCodecTypeDecoderElementName[codec_type], _owr_get_unique_uint_id()); + decoder = gst_element_factory_make(OwrCodecTypeDecoderElementName[codec_type], element_name); + g_free(element_name); + g_return_val_if_fail(decoder, NULL); + break; + } + + return decoder; +} + +GstElement * _owr_create_parser(OwrCodecType codec_type) +{ + GstElement * parser = NULL; + gchar *element_name = NULL; + + if (!OwrCodecTypeParserElementName[codec_type]) + return NULL; + + element_name = g_strdup_printf("parser_%s_%u", OwrCodecTypeParserElementName[codec_type], _owr_get_unique_uint_id()); + parser = gst_element_factory_make(OwrCodecTypeParserElementName[codec_type], element_name); + g_free(element_name); + + switch (codec_type) { + case OWR_CODEC_TYPE_H264: + g_object_set(parser, "disable-passthrough", TRUE, NULL); + break; + default: + break; + } + return parser; +} + +const gchar* _owr_get_encoder_name(OwrCodecType codec_type) +{ + return OwrCodecTypeEncoderElementName[codec_type]; +} + +void _owr_bin_link_and_sync_elements(GstBin *bin, gboolean *out_link_ok, gboolean *out_sync_ok, GstElement **out_first, GstElement **out_last) +{ + GList *bin_elements, *current; + gboolean link_ok = TRUE, sync_ok = TRUE; + + g_assert(bin); + + bin_elements = g_list_last(bin->children); + g_assert(bin_elements); + for (current = bin_elements; current && current->prev && link_ok && sync_ok; current = g_list_previous(current)) { + if (out_link_ok) + link_ok &= gst_element_link(current->data, current->prev->data); + if (out_sync_ok) + sync_ok &= gst_element_sync_state_with_parent(current->data); + } + if (out_sync_ok && link_ok && sync_ok && current && !current->prev) + sync_ok &= gst_element_sync_state_with_parent(current->data); + + if (out_link_ok) + *out_link_ok = link_ok; + + if (out_sync_ok) + *out_sync_ok = sync_ok; + + if (link_ok && sync_ok) { + if (out_first) + *out_first = bin_elements->data; + if (out_last) + *out_last = current->data; + } +} + typedef struct { GClosure *callback; GList *list; @@ -222,6 +462,19 @@ int _owr_rotation_and_mirror_to_video_flip_method(guint rotation, gboolean mirro } } +void _owr_update_flip_method(GObject *source, GParamSpec *pspec, GstElement *flip) +{ + guint rotation = 0; + gboolean mirror = FALSE; + gint flip_method; + + g_assert(GST_IS_ELEMENT(flip)); + + g_object_get(source, "rotation", &rotation, "mirror", &mirror, NULL); + flip_method = _owr_rotation_and_mirror_to_video_flip_method(rotation, mirror); + g_object_set(flip, pspec ? pspec->name : "method", flip_method, NULL); +} + static void value_slice_free(gpointer value) { g_value_unset(value); diff --git a/owr/owr_utils.h b/owr/owr_utils.h index 85e2754e..ae1d9109 100644 --- a/owr/owr_utils.h +++ b/owr/owr_utils.h @@ -39,8 +39,21 @@ G_BEGIN_DECLS #define OWR_UNUSED(x) (void)x +#define _owr_codec_type_is_raw(codec_type) (codec_type == OWR_CODEC_TYPE_NONE) + void *_owr_require_symbols(void); +guint _owr_get_unique_uint_id(); OwrCodecType _owr_caps_to_codec_type(GstCaps *caps); +const gchar* _owr_codec_type_to_caps_mime(OwrMediaType media_type, OwrCodecType codec_type); +gpointer _owr_detect_codecs(gpointer data); +const GList *_owr_get_detected_h264_encoders(); +const GList *_owr_get_detected_vp8_encoders(); +const GList *_owr_get_detected_vp9_encoders(); +GstElement *_owr_try_codecs(const GList *codecs, const gchar *name_prefix); +GstElement *_owr_create_decoder(OwrCodecType codec_type); +GstElement *_owr_create_parser(OwrCodecType codec_type); +const gchar* _owr_get_encoder_name(OwrCodecType codec_type); +void _owr_bin_link_and_sync_elements(GstBin *bin, gboolean *out_link_ok, gboolean *out_sync_ok, GstElement **out_first, GstElement **out_last); void _owr_utils_call_closure_with_list(GClosure *callback, GList *list); GClosure *_owr_utils_list_closure_merger_new(GClosure *final_callback, GCopyFunc list_item_copy, @@ -59,6 +72,7 @@ gboolean _owr_gst_caps_foreach(const GstCaps *caps, OwrGstCapsForeachFunc func, void _owr_deep_notify(GObject *object, GstObject *orig, GParamSpec *pspec, gpointer user_data); +void _owr_update_flip_method(GObject *renderer, GParamSpec *pspec, GstElement *flip); int _owr_rotation_and_mirror_to_video_flip_method(guint rotation, gboolean mirror); GHashTable *_owr_value_table_new(); diff --git a/tests/test_self_view.c b/tests/test_self_view.c index b83b7747..8d53f4b7 100644 --- a/tests/test_self_view.c +++ b/tests/test_self_view.c @@ -49,7 +49,12 @@ static OwrMediaRenderer *audio_renderer = NULL, *video_renderer = NULL; gboolean dump_pipeline(gpointer user_data) { - g_print("Dumping pipelines\n"); + if (!g_getenv("OWR_DEBUG_DUMP_DOT_DIR")) { + g_print("Not dumping pipelines because OWR_DEBUG_DUMP_DOT_DIR environment variable is empty.\n"); + return FALSE; + } + + g_print("Dumping pipelines..."); if (audio_source) write_dot_file("test_self_view-audio_source", owr_media_source_get_dot_data(audio_source), FALSE); @@ -61,6 +66,8 @@ gboolean dump_pipeline(gpointer user_data) if (video_renderer) write_dot_file("test_self_view-video_renderer", owr_media_renderer_get_dot_data(video_renderer), FALSE); + g_print(" done.\n"); + return FALSE; } diff --git a/tests/test_send_receive.c b/tests/test_send_receive.c index e14bd2cd..f4bb44ea 100644 --- a/tests/test_send_receive.c +++ b/tests/test_send_receive.c @@ -236,7 +236,7 @@ static void got_sources(GList *sources, gpointer user_data) owr_bus_add_message_origin(bus, OWR_MESSAGE_ORIGIN(source)); - payload = owr_video_payload_new(OWR_CODEC_TYPE_VP8, 103, 90000, TRUE, FALSE); + payload = owr_video_payload_new(OWR_CODEC_TYPE_H264, 103, 90000, TRUE, FALSE); g_object_set(payload, "width", 640, "height", 480, "framerate", 30.0, NULL); g_object_set(payload, "rtx-payload-type", 123, NULL); if (adaptation) diff --git a/transport/owr_payload.c b/transport/owr_payload.c index 1b2f1436..d4559665 100644 --- a/transport/owr_payload.c +++ b/transport/owr_payload.c @@ -92,8 +92,6 @@ enum { static GParamSpec *obj_properties[N_PROPERTIES] = {NULL, }; -static guint get_unique_id(); - static void owr_payload_set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec) { @@ -183,58 +181,6 @@ static void owr_payload_get_property(GObject *object, guint property_id, GValue } } -/* To be extended once more codecs are supported */ -static GList *h264_decoders = NULL; -static GList *h264_encoders = NULL; -static GList *vp8_decoders = NULL; -static GList *vp8_encoders = NULL; -static GList *vp9_decoders = NULL; -static GList *vp9_encoders = NULL; - -static gpointer owr_payload_detect_codecs(gpointer data) -{ - GList *decoder_factories; - GList *encoder_factories; - GstCaps *caps; - - OWR_UNUSED(data); - - decoder_factories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | - GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, - GST_RANK_MARGINAL); - encoder_factories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_ENCODER | - GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, - GST_RANK_MARGINAL); - - caps = gst_caps_new_empty_simple("video/x-h264"); - h264_decoders = gst_element_factory_list_filter(decoder_factories, caps, GST_PAD_SINK, FALSE); - h264_encoders = gst_element_factory_list_filter(encoder_factories, caps, GST_PAD_SRC, FALSE); - gst_caps_unref(caps); - - caps = gst_caps_new_empty_simple("video/x-vp8"); - vp8_decoders = gst_element_factory_list_filter(decoder_factories, caps, GST_PAD_SINK, FALSE); - vp8_encoders = gst_element_factory_list_filter(encoder_factories, caps, GST_PAD_SRC, FALSE); - gst_caps_unref(caps); - - caps = gst_caps_new_empty_simple("video/x-vp9"); - vp9_decoders = gst_element_factory_list_filter(decoder_factories, caps, GST_PAD_SINK, FALSE); - vp9_encoders = gst_element_factory_list_filter(encoder_factories, caps, GST_PAD_SRC, FALSE); - gst_caps_unref(caps); - - gst_plugin_feature_list_free(decoder_factories); - gst_plugin_feature_list_free(encoder_factories); - - h264_decoders = g_list_sort(h264_decoders, gst_plugin_feature_rank_compare_func); - h264_encoders = g_list_sort(h264_encoders, gst_plugin_feature_rank_compare_func); - vp8_decoders = g_list_sort(vp8_decoders, gst_plugin_feature_rank_compare_func); - vp8_encoders = g_list_sort(vp8_encoders, gst_plugin_feature_rank_compare_func); - vp9_decoders = g_list_sort(vp9_decoders, gst_plugin_feature_rank_compare_func); - vp9_encoders = g_list_sort(vp9_encoders, gst_plugin_feature_rank_compare_func); - - return NULL; -} - - static void owr_payload_class_init(OwrPayloadClass *klass) { GObjectClass *gobject_class = G_OBJECT_CLASS(klass); @@ -293,10 +239,6 @@ static void owr_payload_class_init(OwrPayloadClass *klass) static void owr_payload_init(OwrPayload *payload) { - static GOnce g_once = G_ONCE_INIT; - - g_once(&g_once, owr_payload_detect_codecs, NULL); - payload->priv = OWR_PAYLOAD_GET_PRIVATE(payload); payload->priv->mtu = DEFAULT_MTU; payload->priv->bitrate = DEFAULT_BITRATE; @@ -309,11 +251,8 @@ static void owr_payload_init(OwrPayload *payload) /* Private methods */ -static const gchar *OwrCodecTypeEncoderElementName[] = {"none", "mulawenc", "alawenc", "opusenc", "openh264enc", "vp8enc", "vp9enc"}; -static const gchar *OwrCodecTypeDecoderElementName[] = {"none", "mulawdec", "alawdec", "opusdec", "openh264dec", "vp8dec", "vp9dec"}; -static const gchar *OwrCodecTypeParserElementName[] = {"none", "none", "none", "none", "h264parse", "none", "none"}; -static const gchar *OwrCodecTypePayElementName[] = {"none", "rtppcmupay", "rtppcmapay", "rtpopuspay", "rtph264pay", "rtpvp8pay", "rtpvp9pay"}; -static const gchar *OwrCodecTypeDepayElementName[] = {"none", "rtppcmudepay", "rtppcmadepay", "rtpopusdepay", "rtph264depay", "rtpvp8depay", "rtpvp9depay"}; +static const gchar *OwrCodecTypePayElementName[] = { NULL, "rtppcmupay", "rtppcmapay", "rtpopuspay", "rtph264pay", "rtpvp8pay", "rtpvp9pay" }; +static const gchar *OwrCodecTypeDepayElementName[] = { NULL, "rtppcmudepay", "rtppcmadepay", "rtpopusdepay", "rtph264depay", "rtpvp8depay", "rtpvp9depay" }; static guint evaluate_bitrate_from_payload(OwrPayload *payload) { @@ -336,40 +275,6 @@ static guint evaluate_bitrate_from_payload(OwrPayload *payload) return bitrate; } -static GstElement * try_codecs(GList *codecs, const gchar *name_prefix) -{ - GList *l; - gchar *element_name; - - for (l = codecs; l; l = l->next) { - GstElementFactory *f = l->data; - GstElement *e; - - element_name = g_strdup_printf("%s_%s_%u", name_prefix, - gst_plugin_feature_get_name(GST_PLUGIN_FEATURE(f)), - get_unique_id()); - - e = gst_element_factory_create(f, element_name); - g_free(element_name); - - if (!e) - continue; - - /* Try setting to READY. If this fails the codec does not work, for - * example because the hardware codec is currently busy - */ - if (gst_element_set_state(e, GST_STATE_READY) != GST_STATE_CHANGE_SUCCESS) { - gst_element_set_state(e, GST_STATE_NULL); - gst_object_unref(e); - continue; - } - - return e; - } - - return NULL; -} - static gboolean binding_transform_to_kbps(GBinding *binding, const GValue *from_value, GValue *to_value, gpointer user_data) { guint bitrate; @@ -395,7 +300,7 @@ GstElement * _owr_payload_create_encoder(OwrPayload *payload) switch (payload->priv->codec_type) { case OWR_CODEC_TYPE_H264: - encoder = try_codecs(h264_encoders, "encoder"); + encoder = _owr_try_codecs(_owr_get_detected_h264_encoders(), "encoder"); g_return_val_if_fail(encoder, NULL); factory = gst_element_get_factory(encoder); @@ -424,7 +329,7 @@ GstElement * _owr_payload_create_encoder(OwrPayload *payload) #endif "max-keyframe-interval", G_MAXINT, NULL); - } else { + } else if (strcmp(factory_name, "omxh264enc")) { /* Assume bits/s instead of kbit/s */ g_object_bind_property(payload, "bitrate", encoder, "bitrate", G_BINDING_SYNC_CREATE); } @@ -432,7 +337,7 @@ GstElement * _owr_payload_create_encoder(OwrPayload *payload) break; case OWR_CODEC_TYPE_VP8: - encoder = try_codecs(vp8_encoders, "encoder"); + encoder = _owr_try_codecs(_owr_get_detected_vp8_encoders(), "encoder"); g_return_val_if_fail(encoder, NULL); #if (defined(__APPLE__) && TARGET_OS_IPHONE && !TARGET_IPHONE_SIMULATOR) || defined(__ANDROID__) @@ -460,7 +365,7 @@ GstElement * _owr_payload_create_encoder(OwrPayload *payload) g_object_set(payload, "bitrate", evaluate_bitrate_from_payload(payload), NULL); break; case OWR_CODEC_TYPE_VP9: - encoder = try_codecs(vp9_encoders, "encoder"); + encoder = _owr_try_codecs(_owr_get_detected_vp9_encoders(), "encoder"); g_return_val_if_fail(encoder, NULL); /* values are inspired by webrtc.org values in vp9_impl.cc */ g_object_set(encoder, @@ -484,8 +389,8 @@ GstElement * _owr_payload_create_encoder(OwrPayload *payload) g_object_set(payload, "bitrate", evaluate_bitrate_from_payload(payload), NULL); break; default: - element_name = g_strdup_printf("encoder_%s_%u", OwrCodecTypeEncoderElementName[payload->priv->codec_type], get_unique_id()); - encoder = gst_element_factory_make(OwrCodecTypeEncoderElementName[payload->priv->codec_type], element_name); + element_name = g_strdup_printf("encoder_%s_%u", _owr_get_encoder_name(payload->priv->codec_type), _owr_get_unique_uint_id()); + encoder = gst_element_factory_make(_owr_get_encoder_name(payload->priv->codec_type), element_name); g_free(element_name); g_return_val_if_fail(encoder, NULL); break; @@ -494,61 +399,6 @@ GstElement * _owr_payload_create_encoder(OwrPayload *payload) return encoder; } -GstElement * _owr_payload_create_decoder(OwrPayload *payload) -{ - GstElement * decoder = NULL; - gchar *element_name = NULL; - - g_return_val_if_fail(payload, NULL); - - switch (payload->priv->codec_type) { - case OWR_CODEC_TYPE_H264: - decoder = try_codecs(h264_decoders, "decoder"); - g_return_val_if_fail(decoder, NULL); - break; - case OWR_CODEC_TYPE_VP8: - decoder = try_codecs(vp8_decoders, "decoder"); - g_return_val_if_fail(decoder, NULL); - break; - case OWR_CODEC_TYPE_VP9: - decoder = try_codecs(vp9_decoders, "decoder"); - g_return_val_if_fail(decoder, NULL); - break; - default: - element_name = g_strdup_printf("decoder_%s_%u", OwrCodecTypeDecoderElementName[payload->priv->codec_type], get_unique_id()); - decoder = gst_element_factory_make(OwrCodecTypeDecoderElementName[payload->priv->codec_type], element_name); - g_free(element_name); - g_return_val_if_fail(decoder, NULL); - break; - } - - return decoder; -} - -GstElement * _owr_payload_create_parser(OwrPayload *payload) -{ - GstElement * parser = NULL; - gchar *element_name = NULL; - - g_return_val_if_fail(payload, NULL); - - if (!g_strcmp0(OwrCodecTypeParserElementName[payload->priv->codec_type], "none")) - return NULL; - - element_name = g_strdup_printf("parser_%s_%u", OwrCodecTypeParserElementName[payload->priv->codec_type], get_unique_id()); - parser = gst_element_factory_make(OwrCodecTypeParserElementName[payload->priv->codec_type], element_name); - g_free(element_name); - - switch (payload->priv->codec_type) { - case OWR_CODEC_TYPE_H264: - g_object_set(parser, "disable-passthrough", TRUE, NULL); - break; - default: - break; - } - return parser; -} - GstElement * _owr_payload_create_payload_packetizer(OwrPayload *payload) { GstElement * pay = NULL; @@ -557,7 +407,7 @@ GstElement * _owr_payload_create_payload_packetizer(OwrPayload *payload) g_return_val_if_fail(payload, NULL); - element_name = g_strdup_printf("pay_%s_%u", OwrCodecTypePayElementName[payload->priv->codec_type], get_unique_id()); + element_name = g_strdup_printf("pay_%s_%u", OwrCodecTypePayElementName[payload->priv->codec_type], _owr_get_unique_uint_id()); pay = gst_element_factory_make(OwrCodecTypePayElementName[payload->priv->codec_type], element_name); g_free(element_name); @@ -593,7 +443,7 @@ GstElement * _owr_payload_create_payload_depacketizer(OwrPayload *payload) g_return_val_if_fail(payload, NULL); - element_name = g_strdup_printf("depay_%s_%u", OwrCodecTypeDepayElementName[payload->priv->codec_type], get_unique_id()); + element_name = g_strdup_printf("depay_%s_%u", OwrCodecTypeDepayElementName[payload->priv->codec_type], _owr_get_unique_uint_id()); depay = gst_element_factory_make(OwrCodecTypeDepayElementName[payload->priv->codec_type], element_name); g_free(element_name); @@ -609,6 +459,14 @@ OwrMediaType _owr_payload_get_media_type(OwrPayload *payload) return media_type; } +OwrCodecType _owr_payload_get_codec_type(OwrPayload *payload) +{ + OwrCodecType codec_type = OWR_CODEC_TYPE_NONE; + + g_object_get(payload, "codec-type", &codec_type, NULL); + return codec_type; +} + GstCaps * _owr_payload_create_rtp_caps(OwrPayload *payload) { @@ -721,7 +579,7 @@ GstCaps * _owr_payload_create_raw_caps(OwrPayload *payload) "framerate", &framerate, NULL); } - caps = gst_caps_new_empty_simple("video/x-raw"); + caps = gst_caps_new_empty_simple(_owr_codec_type_to_caps_mime(media_type, priv->codec_type)); #ifdef __APPLE__ if (priv->codec_type == OWR_CODEC_TYPE_H264) gst_caps_set_features(caps, 0, gst_caps_features_new_any()); @@ -772,9 +630,3 @@ GstCaps * _owr_payload_create_encoded_caps(OwrPayload *payload) /* local functions */ - -static guint get_unique_id() -{ - static guint id = 0; - return g_atomic_int_add(&id, 1); -} diff --git a/transport/owr_payload_private.h b/transport/owr_payload_private.h index 8f9ec579..b1deb25e 100644 --- a/transport/owr_payload_private.h +++ b/transport/owr_payload_private.h @@ -38,11 +38,10 @@ G_BEGIN_DECLS /*< private >*/ GstElement * _owr_payload_create_encoder(OwrPayload *payload); -GstElement * _owr_payload_create_decoder(OwrPayload *payload); -GstElement * _owr_payload_create_parser(OwrPayload *payload); GstElement * _owr_payload_create_payload_packetizer(OwrPayload *payload); GstElement * _owr_payload_create_payload_depacketizer(OwrPayload *payload); OwrMediaType _owr_payload_get_media_type(OwrPayload *payload); +OwrCodecType _owr_payload_get_codec_type(OwrPayload *payload); GstCaps * _owr_payload_create_rtp_caps(OwrPayload *payload); GstCaps * _owr_payload_create_raw_caps(OwrPayload *payload); GstCaps * _owr_payload_create_encoded_caps(OwrPayload *payload); diff --git a/transport/owr_remote_media_source.c b/transport/owr_remote_media_source.c index 8ed45b20..e913a1c1 100644 --- a/transport/owr_remote_media_source.c +++ b/transport/owr_remote_media_source.c @@ -79,7 +79,9 @@ static void on_caps(GstElement *source, GParamSpec *pspec, OwrMediaSource *media if (GST_IS_CAPS(caps)) { GST_INFO_OBJECT(source, "%s - configured with caps: %" GST_PTR_FORMAT, media_source_name, caps); + gst_caps_unref(caps); } + g_free(media_source_name); } #define LINK_ELEMENTS(a, b) \ diff --git a/transport/owr_transport_agent.c b/transport/owr_transport_agent.c index e757189b..8985c8eb 100644 --- a/transport/owr_transport_agent.c +++ b/transport/owr_transport_agent.c @@ -861,10 +861,7 @@ static void handle_new_send_source(OwrTransportAgent *transport_agent, return; } - /* FIXME - communicate what codec types are supported by the source - * and if one is reusable, use it, else raw? g_object_get(send_payload, "codec-type", &codec_type, NULL); - */ caps = _owr_payload_create_raw_caps(send_payload); src = _owr_media_source_request_source(send_source, caps); @@ -963,6 +960,8 @@ static void remove_existing_send_source_and_payload(OwrTransportAgent *transport OwrMediaType media_type = OWR_MEDIA_TYPE_UNKNOWN; GHashTable *event_data; GValue *value; + OwrPayload *send_payload; + OwrCodecType codec_type = OWR_CODEC_TYPE_NONE; g_assert(media_source); @@ -970,6 +969,12 @@ static void remove_existing_send_source_and_payload(OwrTransportAgent *transport value = _owr_value_table_add(event_data, "start_time", G_TYPE_INT64); g_value_set_int64(value, g_get_monotonic_time()); + send_payload = _owr_media_session_get_send_payload(media_session); + if (send_payload) { + g_object_get(send_payload, "codec-type", &codec_type, NULL); + g_object_unref(send_payload); + } + /* Setting a new, different source but have one already */ stream_id = _owr_session_get_stream_id(OWR_SESSION(media_session)); @@ -979,7 +984,7 @@ static void remove_existing_send_source_and_payload(OwrTransportAgent *transport g_object_get(media_source, "media-type", &media_type, NULL); g_warn_if_fail(media_type != OWR_MEDIA_TYPE_UNKNOWN); if (media_type == OWR_MEDIA_TYPE_VIDEO) - pad_name = g_strdup_printf("video_sink_%u_%u", OWR_CODEC_TYPE_NONE, stream_id); + pad_name = g_strdup_printf("video_sink_%u_%u", codec_type, stream_id); else pad_name = g_strdup_printf("audio_raw_sink_%u", stream_id); sinkpad = gst_element_get_static_pad(transport_agent->priv->transport_bin, pad_name); @@ -2405,21 +2410,6 @@ static OwrPayload * get_payload(OwrTransportAgent *transport_agent, guint pt, Ow return NULL; } -static void update_flip_method(OwrPayload *payload, GParamSpec *pspec, GstElement *flip) -{ - guint rotation = 0; - gboolean mirror = FALSE; - gint flip_method; - - g_return_if_fail(OWR_IS_VIDEO_PAYLOAD(payload)); - g_return_if_fail(G_IS_PARAM_SPEC(pspec) || !pspec); - g_return_if_fail(GST_IS_ELEMENT(flip)); - - g_object_get(payload, "rotation", &rotation, "mirror", &mirror, NULL); - flip_method = _owr_rotation_and_mirror_to_video_flip_method(rotation, mirror); - g_object_set(flip, "method", flip_method, NULL); -} - /* pad is transfer full */ static void add_pads_to_bin_and_transport_bin(GstPad *pad, GstElement *bin, GstElement *transport_bin, const gchar *pad_name) @@ -2439,8 +2429,10 @@ static void on_caps(GstElement *sink, GParamSpec *pspec, OwrSession *session) g_object_get(sink, "caps", &caps, NULL); - if (GST_IS_CAPS(caps)) + if (GST_IS_CAPS(caps)) { GST_CAT_INFO_OBJECT(_owrsession_debug, session, "Sending media configured with caps: %" GST_PTR_FORMAT, caps); + gst_caps_unref(caps); + } } static void handle_new_send_payload(OwrTransportAgent *transport_agent, OwrMediaSession *media_session, OwrPayload * payload) @@ -2455,10 +2447,12 @@ static void handle_new_send_payload(OwrTransportAgent *transport_agent, OwrMedia gboolean link_ok = TRUE, sync_ok = TRUE; GstPad *sink_pad = NULL, *rtp_sink_pad = NULL, *rtp_capsfilter_src_pad = NULL, *ghost_src_pad = NULL, *encoder_sink_pad; + OwrCodecType codec_type = OWR_CODEC_TYPE_NONE; OwrMediaType media_type; - GstPadLinkReturn link_res; guint send_ssrc = 0; gchar *cname = NULL; + OwrMediaSource *media_source = NULL; + GstElement *first = NULL; g_return_if_fail(transport_agent); g_return_if_fail(media_session); @@ -2485,7 +2479,7 @@ static void handle_new_send_payload(OwrTransportAgent *transport_agent, OwrMedia link_rtpbin_to_send_output_bin(transport_agent, session_id, stream_id, TRUE, TRUE); - g_object_get(payload, "media-type", &media_type, NULL); + g_object_get(payload, "media-type", &media_type, "codec-type", &codec_type, NULL); name = g_strdup_printf("send-rtp-capsfilter-%u", stream_id); rtp_capsfilter = gst_element_factory_make("capsfilter", name); @@ -2512,119 +2506,95 @@ static void handle_new_send_payload(OwrTransportAgent *transport_agent, OwrMedia g_object_set(rtp_capsfilter, "caps", rtp_caps, NULL); gst_caps_unref(rtp_caps); - gst_bin_add(GST_BIN(send_input_bin), rtp_capsfilter); - rtp_capsfilter_src_pad = gst_element_get_static_pad(rtp_capsfilter, "src"); - name = g_strdup_printf("src_%u", stream_id); - ghost_src_pad = ghost_pad_and_add_to_bin(rtp_capsfilter_src_pad, send_input_bin, name); - gst_object_unref(rtp_capsfilter_src_pad); - g_free(name); + media_source = _owr_media_session_get_send_source(media_session); - link_res = gst_pad_link(ghost_src_pad, rtp_sink_pad); - g_warn_if_fail(link_res == GST_PAD_LINK_OK); - gst_object_unref(rtp_sink_pad); - - sync_ok &= gst_element_sync_state_with_parent(rtp_capsfilter); - g_warn_if_fail(sync_ok); - - if (media_type == OWR_MEDIA_TYPE_VIDEO) { - GstElement *gldownload, *flip, *queue = NULL, *encoder_capsfilter; - - name = g_strdup_printf("send-input-video-gldownload-%u", stream_id); - gldownload = gst_element_factory_make("gldownload", name); - g_free(name); - - name = g_strdup_printf("send-input-video-flip-%u", stream_id); - flip = gst_element_factory_make("videoflip", name); - g_assert(flip); - g_free(name); - g_return_if_fail(OWR_IS_VIDEO_PAYLOAD(payload)); - g_signal_connect_object(payload, "notify::rotation", G_CALLBACK(update_flip_method), flip, 0); - g_signal_connect_object(payload, "notify::mirror", G_CALLBACK(update_flip_method), flip, 0); - update_flip_method(payload, NULL, flip); - - name = g_strdup_printf("send-input-video-queue-%u", stream_id); - queue = gst_element_factory_make("queue", name); - g_free(name); - g_object_set(queue, "max-size-buffers", 3, "max-size-bytes", 0, - "max-size-time", G_GUINT64_CONSTANT(0), NULL); + if (media_type == OWR_MEDIA_TYPE_VIDEO && OWR_IS_VIDEO_PAYLOAD(payload)) { + GstElement *gldownload; + GstElement *flip = NULL, *queue = NULL, *encoder_capsfilter = NULL; + if (_owr_codec_type_is_raw(_owr_payload_get_codec_type(payload))) { + name = g_strdup_printf("send-input-video-gldownload-%u", stream_id); + gldownload = gst_element_factory_make("gldownload", name); + g_free(name); + gst_bin_add(GST_BIN(send_input_bin), gldownload); + } + if (!_owr_media_source_supports_interfaces(media_source, OWR_MEDIA_SOURCE_SUPPORTS_VIDEO_ORIENTATION)) { + name = g_strdup_printf("send-input-video-flip-%u", stream_id); + flip = gst_element_factory_make("videoflip", name); + g_assert(flip); + g_free(name); + g_signal_connect_object(payload, "notify::rotation", G_CALLBACK(_owr_update_flip_method), flip, 0); + g_signal_connect_object(payload, "notify::mirror", G_CALLBACK(_owr_update_flip_method), flip, 0); + _owr_update_flip_method(G_OBJECT(payload), NULL, flip); + + name = g_strdup_printf("send-input-video-queue-%u", stream_id); + queue = gst_element_factory_make("queue", name); + g_free(name); + g_object_set(queue, "max-size-buffers", 3, "max-size-bytes", 0, + "max-size-time", G_GUINT64_CONSTANT(0), NULL); + + encoder = _owr_payload_create_encoder(payload); + parser = _owr_create_parser(_owr_payload_get_codec_type(payload)); + + g_warn_if_fail(encoder); + + encoder_sink_pad = gst_element_get_static_pad(encoder, "sink"); + g_signal_connect(encoder_sink_pad, "notify::caps", G_CALLBACK(on_caps), OWR_SESSION(media_session)); + gst_object_unref(encoder_sink_pad); + + name = g_strdup_printf("send-input-video-encoder-capsfilter-%u", stream_id); + encoder_capsfilter = gst_element_factory_make("capsfilter", name); + g_free(name); + caps = _owr_payload_create_encoded_caps(payload); + g_object_set(encoder_capsfilter, "caps", caps, NULL); + gst_caps_unref(caps); + + gst_bin_add_many(GST_BIN(send_input_bin), flip, queue, encoder, NULL); + + if (parser) + gst_bin_add(GST_BIN(send_input_bin), parser); + gst_bin_add(GST_BIN(send_input_bin), encoder_capsfilter); + } + } else { /* Audio */ encoder = _owr_payload_create_encoder(payload); - parser = _owr_payload_create_parser(payload); - payloader = _owr_payload_create_payload_packetizer(payload); - g_warn_if_fail(payloader && encoder); + parser = _owr_create_parser(_owr_payload_get_codec_type(payload)); encoder_sink_pad = gst_element_get_static_pad(encoder, "sink"); g_signal_connect(encoder_sink_pad, "notify::caps", G_CALLBACK(on_caps), OWR_SESSION(media_session)); gst_object_unref(encoder_sink_pad); - name = g_strdup_printf("send-input-video-encoder-capsfilter-%u", stream_id); - encoder_capsfilter = gst_element_factory_make("capsfilter", name); - g_free(name); - caps = _owr_payload_create_encoded_caps(payload); - g_object_set(encoder_capsfilter, "caps", caps, NULL); - gst_caps_unref(caps); - - gst_bin_add_many(GST_BIN(send_input_bin), gldownload, flip, queue, encoder, encoder_capsfilter, payloader, NULL); - if (parser) { + gst_bin_add(GST_BIN(send_input_bin), encoder); + if (parser) gst_bin_add(GST_BIN(send_input_bin), parser); - link_ok &= gst_element_link_many(gldownload, flip, queue, encoder, parser, encoder_capsfilter, payloader, NULL); - } else - link_ok &= gst_element_link_many(gldownload, flip, queue, encoder, encoder_capsfilter, payloader, NULL); - - link_ok &= gst_element_link_many(payloader, rtp_capsfilter, NULL); + } - g_warn_if_fail(link_ok); + payloader = _owr_payload_create_payload_packetizer(payload); + g_assert(payloader); + gst_bin_add_many(GST_BIN(send_input_bin), payloader, rtp_capsfilter, NULL); - sync_ok &= gst_element_sync_state_with_parent(rtp_capsfilter); - sync_ok &= gst_element_sync_state_with_parent(payloader); - if (parser) - sync_ok &= gst_element_sync_state_with_parent(parser); - sync_ok &= gst_element_sync_state_with_parent(encoder_capsfilter); - sync_ok &= gst_element_sync_state_with_parent(encoder); - sync_ok &= gst_element_sync_state_with_parent(queue); - sync_ok &= gst_element_sync_state_with_parent(flip); - sync_ok &= gst_element_sync_state_with_parent(gldownload); - - name = g_strdup_printf("video_sink_%u_%u", OWR_CODEC_TYPE_NONE, stream_id); - sink_pad = gst_element_get_static_pad(gldownload, "sink"); - add_pads_to_bin_and_transport_bin(sink_pad, send_input_bin, - transport_agent->priv->transport_bin, name); - gst_object_unref(sink_pad); - g_free(name); - } else { /* Audio */ - encoder = _owr_payload_create_encoder(payload); - parser = _owr_payload_create_parser(payload); - payloader = _owr_payload_create_payload_packetizer(payload); + _owr_bin_link_and_sync_elements(GST_BIN(send_input_bin), &link_ok, &sync_ok, &first, NULL); + g_warn_if_fail(link_ok && sync_ok); - encoder_sink_pad = gst_element_get_static_pad(encoder, "sink"); - g_signal_connect(encoder_sink_pad, "notify::caps", G_CALLBACK(on_caps), OWR_SESSION(media_session)); - gst_object_unref(encoder_sink_pad); + name = g_strdup_printf("%s_sink_%u_%u", media_type == OWR_MEDIA_TYPE_VIDEO ? "video" : "audio", + codec_type, stream_id); + sink_pad = gst_element_get_static_pad(first, "sink"); + add_pads_to_bin_and_transport_bin(sink_pad, send_input_bin, + transport_agent->priv->transport_bin, name); + gst_object_unref(sink_pad); + g_free(name); - gst_bin_add_many(GST_BIN(send_input_bin), encoder, payloader, NULL); - if (parser) { - gst_bin_add(GST_BIN(send_input_bin), parser); - link_ok &= gst_element_link_many(encoder, parser, payloader, NULL); - } else - link_ok &= gst_element_link_many(encoder, payloader, NULL); + rtp_capsfilter_src_pad = gst_element_get_static_pad(rtp_capsfilter, "src"); + name = g_strdup_printf("src_%u", stream_id); + ghost_src_pad = ghost_pad_and_add_to_bin(rtp_capsfilter_src_pad, send_input_bin, name); + gst_object_unref(rtp_capsfilter_src_pad); + g_free(name); + g_warn_if_fail(gst_pad_link(ghost_src_pad, rtp_sink_pad) == GST_PAD_LINK_OK); + gst_object_unref(rtp_sink_pad); - link_ok &= gst_element_link_many(payloader, rtp_capsfilter, NULL); - g_warn_if_fail(link_ok); + GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(send_input_bin), GST_DEBUG_GRAPH_SHOW_ALL, "send-input-bin"); - sync_ok &= gst_element_sync_state_with_parent(rtp_capsfilter); - sync_ok &= gst_element_sync_state_with_parent(payloader); - if (parser) - sync_ok &= gst_element_sync_state_with_parent(parser); - sync_ok &= gst_element_sync_state_with_parent(encoder); - g_warn_if_fail(sync_ok); - - name = g_strdup_printf("audio_raw_sink_%u", stream_id); - sink_pad = gst_element_get_static_pad(encoder, "sink"); - add_pads_to_bin_and_transport_bin(sink_pad, send_input_bin, - transport_agent->priv->transport_bin, name); - gst_object_unref(sink_pad); - g_free(name); - } + g_object_unref(media_source); } static void on_new_remote_candidate(OwrTransportAgent *transport_agent, gboolean forced, OwrSession *session) @@ -2767,7 +2737,7 @@ static void on_transport_bin_pad_added(GstElement *transport_bin, GstPad *new_pa sscanf(new_pad_name, "video_src_%u_%u_%u", &codec_type, &session_id, &stream_id); } - if (media_type != OWR_MEDIA_TYPE_UNKNOWN && codec_type == OWR_CODEC_TYPE_NONE) + if (media_type != OWR_MEDIA_TYPE_UNKNOWN) signal_incoming_source(media_type, transport_agent, session_id, stream_id, codec_type); g_free(new_pad_name); @@ -2852,16 +2822,17 @@ static GstPadProbeReturn check_for_keyframe(GstPad *pad, GstPadProbeInfo *info, static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, guint32 stream_id, OwrPayload *payload, OwrTransportAgent *transport_agent) { - GstPad *depay_sink_pad = NULL, *ghost_pad = NULL; + GstPad *sink_pad = NULL, *ghost_pad = NULL; gboolean sync_ok = TRUE; GstElement *receive_output_bin; - GstElement *rtpdepay, *videorepair1, *parser, *decoder; + GstElement *rtpdepay, *videorepair1, *parser; GstPadLinkReturn link_res; gboolean link_ok = TRUE; OwrCodecType codec_type; gchar name[100]; GstPad *pad; SessionData *session_data; + GstElement *first = NULL, *last = NULL; g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "receive-output-bin-%u-%u", session_id, stream_id); receive_output_bin = gst_bin_new(name); @@ -2873,8 +2844,16 @@ static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, gu } rtpdepay = _owr_payload_create_payload_depacketizer(payload); + gst_bin_add(GST_BIN(receive_output_bin), rtpdepay); + + codec_type = _owr_payload_get_codec_type(payload); + parser = _owr_create_parser(codec_type); + if (parser) + gst_bin_add(GST_BIN(receive_output_bin), parser); + g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "videorepair1_%u", stream_id); videorepair1 = gst_element_factory_make("videorepair", name); + gst_bin_add(GST_BIN(receive_output_bin), videorepair1); pad = gst_element_get_static_pad(videorepair1, "src"); session_data = g_slice_new(SessionData); @@ -2885,34 +2864,18 @@ static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, gu gst_object_unref(pad); pad = NULL; - g_object_get(payload, "codec-type", &codec_type, NULL); - parser = _owr_payload_create_parser(payload); - decoder = _owr_payload_create_decoder(payload); + _owr_bin_link_and_sync_elements(GST_BIN(receive_output_bin), &link_ok, &sync_ok, &first, &last); + g_warn_if_fail(link_ok && sync_ok); - gst_bin_add_many(GST_BIN(receive_output_bin), rtpdepay, - videorepair1, decoder, /*decoded_tee,*/ NULL); - depay_sink_pad = gst_element_get_static_pad(rtpdepay, "sink"); - if (parser) { - gst_bin_add(GST_BIN(receive_output_bin), parser); - link_ok &= gst_element_link_many(rtpdepay, parser, videorepair1, decoder, NULL); - } else - link_ok &= gst_element_link_many(rtpdepay, videorepair1, decoder, NULL); - - ghost_pad = ghost_pad_and_add_to_bin(depay_sink_pad, receive_output_bin, "sink"); + sink_pad = gst_element_get_static_pad(first, "sink"); + ghost_pad = ghost_pad_and_add_to_bin(sink_pad, receive_output_bin, "sink"); link_res = gst_pad_link(new_pad, ghost_pad); - gst_object_unref(depay_sink_pad); + gst_object_unref(sink_pad); ghost_pad = NULL; - g_warn_if_fail(link_ok && (link_res == GST_PAD_LINK_OK)); - - sync_ok &= gst_element_sync_state_with_parent(decoder); - if (parser) - sync_ok &= gst_element_sync_state_with_parent(parser); - sync_ok &= gst_element_sync_state_with_parent(videorepair1); - sync_ok &= gst_element_sync_state_with_parent(rtpdepay); - g_warn_if_fail(sync_ok); + g_warn_if_fail(link_res == GST_PAD_LINK_OK); - pad = gst_element_get_static_pad(decoder, "src"); - g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "video_src_%u_%u_%u", OWR_CODEC_TYPE_NONE, + pad = gst_element_get_static_pad(last, "src"); + g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "video_src_%u_%u_%u", codec_type, session_id, stream_id); add_pads_to_bin_and_transport_bin(pad, receive_output_bin, transport_agent->priv->transport_bin, name); gst_object_unref(pad); @@ -2946,8 +2909,8 @@ static void setup_audio_receive_elements(GstPad *new_pad, guint32 session_id, gu rtpdepay = _owr_payload_create_payload_depacketizer(payload); - parser = _owr_payload_create_parser(payload); - decoder = _owr_payload_create_decoder(payload); + parser = _owr_create_parser(_owr_payload_get_codec_type(payload)); + decoder = _owr_create_decoder(_owr_payload_get_codec_type(payload)); gst_bin_add_many(GST_BIN(receive_output_bin), rtp_capsfilter, rtpdepay, decoder, NULL); From 05aad04ca9770ff804a4afd1c055e41f7c483c75 Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Tue, 9 Aug 2016 17:33:47 +0200 Subject: [PATCH 14/26] owr_transport_agent: force PLI request on incoming stream --- transport/owr_transport_agent.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/transport/owr_transport_agent.c b/transport/owr_transport_agent.c index 8985c8eb..66514feb 100644 --- a/transport/owr_transport_agent.c +++ b/transport/owr_transport_agent.c @@ -64,6 +64,7 @@ #include #include #include +#include #include #include #include @@ -2820,6 +2821,18 @@ static GstPadProbeReturn check_for_keyframe(GstPad *pad, GstPadProbeInfo *info, return GST_PAD_PROBE_OK; } +#if TARGET_RPI +static gboolean force_key_unit_event(gpointer data) +{ + GstElement* videorepair = (GstElement*) data; + GstPad* sink_pad = gst_element_get_static_pad(videorepair, "sink"); + gst_pad_push_event(sink_pad, gst_video_event_new_upstream_force_key_unit(GST_CLOCK_TIME_NONE, FALSE, 0)); + gst_object_unref(sink_pad); + gst_object_unref(videorepair); + return G_SOURCE_REMOVE; +} +#endif + static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, guint32 stream_id, OwrPayload *payload, OwrTransportAgent *transport_agent) { GstPad *sink_pad = NULL, *ghost_pad = NULL; @@ -2864,6 +2877,13 @@ static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, gu gst_object_unref(pad); pad = NULL; +#if TARGET_RPI + // The OMX video decoder doesn't seem to handle very well the incoming + // streams from Chrome, not taking into account the first intra frame + // received. So for now, force a PLI request towards the sender after 250ms. + g_timeout_add(250, force_key_unit_event, gst_object_ref(videorepair1)); +#endif + _owr_bin_link_and_sync_elements(GST_BIN(receive_output_bin), &link_ok, &sync_ok, &first, &last); g_warn_if_fail(link_ok && sync_ok); From f8bdd7b1779204710e3e09f51d51ade28f21cdaf Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Wed, 23 Nov 2016 13:10:17 +0100 Subject: [PATCH 15/26] owr_transport_agent: fix build warnings --- transport/owr_transport_agent.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/transport/owr_transport_agent.c b/transport/owr_transport_agent.c index 66514feb..c6521786 100644 --- a/transport/owr_transport_agent.c +++ b/transport/owr_transport_agent.c @@ -1604,12 +1604,12 @@ static void link_rtpbin_to_send_output_bin(OwrTransportAgent *transport_agent, g static void prepare_transport_bin_send_elements(OwrTransportAgent *transport_agent, guint session_id, guint stream_id, gboolean rtcp_mux, PendingSessionInfo *pending_session_info) { - GstElement *nice_element, *dtls_srtp_bin_rtp, *dtls_srtp_bin_rtcp = NULL; + GstElement *nice_element, *dtls_srtp_bin_rtp = NULL, *dtls_srtp_bin_rtcp = NULL; GstElement *scream_queue = NULL; gboolean linked_ok, synced_ok; GstElement *send_output_bin = NULL; SendBinInfo *send_bin_info; - gchar *bin_name, *dtls_srtp_pad_name, *queue_name; + gchar *bin_name, *dtls_srtp_pad_name = NULL, *queue_name; OwrMediaSession *media_session; AgentAndSessionIdPair *agent_and_session_id_pair; @@ -1696,9 +1696,12 @@ static void prepare_transport_bin_send_elements(OwrTransportAgent *transport_age dtls_srtp_pad_name = g_strdup_printf("rtp_sink_%u", session_id); } } - linked_ok = gst_element_link_pads(scream_queue, "src", dtls_srtp_bin_rtp, dtls_srtp_pad_name); - g_warn_if_fail(linked_ok); - g_free(dtls_srtp_pad_name); + + if (dtls_srtp_bin_rtp && dtls_srtp_pad_name) { + linked_ok = gst_element_link_pads(scream_queue, "src", dtls_srtp_bin_rtp, dtls_srtp_pad_name); + g_warn_if_fail(linked_ok); + g_free(dtls_srtp_pad_name); + } if (nice_element) { synced_ok = gst_element_sync_state_with_parent(nice_element); From c58e6c94cfa136232a8f9114f53a126488411b0e Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Thu, 24 Nov 2016 15:37:02 +0100 Subject: [PATCH 16/26] media_renderer: remove reconfigure_element vfunc Replaced with a notify::source handler in the video renderer. --- local/owr_media_renderer.c | 27 ++++++++++++++------------- local/owr_media_renderer.h | 2 +- local/owr_video_renderer.c | 12 +++++++++++- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/local/owr_media_renderer.c b/local/owr_media_renderer.c index 25853295..e62ee001 100644 --- a/local/owr_media_renderer.c +++ b/local/owr_media_renderer.c @@ -58,6 +58,7 @@ enum { PROP_0, PROP_MEDIA_TYPE, PROP_DISABLED, + PROP_SOURCE, N_PROPERTIES }; @@ -115,10 +116,6 @@ static void owr_media_renderer_finalize(GObject *object) G_OBJECT_CLASS(owr_media_renderer_parent_class)->finalize(object); } -static void owr_media_renderer_reconfigure_element_default(G_GNUC_UNUSED OwrMediaRenderer *renderer) -{ -} - static void owr_media_renderer_class_init(OwrMediaRendererClass *klass) { GObjectClass *gobject_class = G_OBJECT_CLASS(klass); @@ -134,13 +131,15 @@ static void owr_media_renderer_class_init(OwrMediaRendererClass *klass) "Whether this renderer is disabled or not", DEFAULT_DISABLED, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + obj_properties[PROP_SOURCE] = g_param_spec_object("source", "Source", + "Current Media Source being rendered", OWR_TYPE_MEDIA_SOURCE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + gobject_class->set_property = owr_media_renderer_set_property; gobject_class->get_property = owr_media_renderer_get_property; gobject_class->finalize = owr_media_renderer_finalize; g_object_class_install_properties(gobject_class, N_PROPERTIES, obj_properties); - - klass->reconfigure_element = owr_media_renderer_reconfigure_element_default; } static gpointer owr_media_renderer_get_bus_set(OwrMessageOrigin *origin) @@ -269,6 +268,10 @@ static void owr_media_renderer_set_property(GObject *object, guint property_id, priv->disabled = g_value_get_boolean(value); break; + case PROP_SOURCE: + priv->source = g_value_get_object(value); + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -292,6 +295,10 @@ static void owr_media_renderer_get_property(GObject *object, guint property_id, g_value_set_boolean(value, priv->disabled); break; + case PROP_SOURCE: + g_value_set_object(value, priv->source); + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -395,8 +402,8 @@ static gboolean set_source(GHashTable *args) } priv->source = g_object_ref(source); + g_object_notify_by_pspec(G_OBJECT(renderer), obj_properties[PROP_SOURCE]); - _owr_media_renderer_reconfigure_element(renderer); maybe_start_renderer(renderer); g_mutex_unlock(&priv->media_renderer_lock); @@ -482,12 +489,6 @@ OwrMediaSource* _owr_media_renderer_get_source(OwrMediaRenderer *renderer) return renderer->priv->source; } -void _owr_media_renderer_reconfigure_element(OwrMediaRenderer *renderer) -{ - g_return_if_fail(OWR_IS_MEDIA_RENDERER(renderer)); - OWR_MEDIA_RENDERER_GET_CLASS(renderer)->reconfigure_element(renderer); -} - gchar * owr_media_renderer_get_dot_data(OwrMediaRenderer *renderer) { g_return_val_if_fail(OWR_IS_MEDIA_RENDERER(renderer), NULL); diff --git a/local/owr_media_renderer.h b/local/owr_media_renderer.h index 69974c86..a67878a2 100644 --- a/local/owr_media_renderer.h +++ b/local/owr_media_renderer.h @@ -63,7 +63,7 @@ struct _OwrMediaRendererClass { /*< private >*/ void *(*get_caps)(OwrMediaRenderer *renderer); void *(*get_sink)(OwrMediaRenderer *renderer); - void (*reconfigure_element)(OwrMediaRenderer *renderer); + }; GType owr_media_renderer_get_type(void) G_GNUC_CONST; diff --git a/local/owr_video_renderer.c b/local/owr_video_renderer.c index adbea545..fccc0a05 100644 --- a/local/owr_video_renderer.c +++ b/local/owr_video_renderer.c @@ -93,6 +93,7 @@ static void owr_video_renderer_reconfigure_element(OwrMediaRenderer *renderer); static GstElement *owr_video_renderer_get_element_with_window_handle(OwrMediaRenderer *renderer, guintptr window_handle); static GstCaps *owr_video_renderer_get_caps(OwrMediaRenderer *renderer); static GstElement *owr_video_renderer_get_sink(OwrMediaRenderer *renderer); +static void _owr_video_renderer_notify_source_changed(OwrMediaRenderer *renderer, GParamSpec *pspec, gpointer user_data); struct _OwrVideoRendererPrivate { guint width; @@ -171,7 +172,7 @@ static void owr_video_renderer_class_init(OwrVideoRendererClass *klass) media_renderer_class->get_caps = (void *(*)(OwrMediaRenderer *))owr_video_renderer_get_caps; media_renderer_class->get_sink = (void *(*)(OwrMediaRenderer *))owr_video_renderer_get_sink; - media_renderer_class->reconfigure_element = (void (*)(OwrMediaRenderer *))owr_video_renderer_reconfigure_element; + g_object_class_install_properties(gobject_class, N_PROPERTIES, obj_properties); } @@ -190,6 +191,8 @@ static void owr_video_renderer_init(OwrVideoRenderer *renderer) g_mutex_init(&priv->closure_mutex); priv->request_context = NULL; priv->renderer_bin = NULL; + + g_signal_connect(renderer, "notify::source", G_CALLBACK(_owr_video_renderer_notify_source_changed), NULL); } static void owr_video_renderer_set_property(GObject *object, guint property_id, @@ -602,6 +605,13 @@ static GstElement *owr_video_renderer_get_sink(OwrMediaRenderer *renderer) return gst_element_factory_make(VIDEO_SINK, "video-renderer-sink"); } +static void _owr_video_renderer_notify_source_changed(OwrMediaRenderer *renderer, GParamSpec *pspec, gpointer user_data) +{ + OWR_UNUSED(pspec); + OWR_UNUSED(user_data); + owr_video_renderer_reconfigure_element(renderer); +} + void _owr_video_renderer_notify_tag_changed(OwrVideoRenderer *video_renderer, const gchar *tag, gboolean have_handle, guintptr new_handle) { OwrVideoRendererPrivate *priv; From 46c9b0aafc8d7448eae1881dcadcd88943090946 Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Mon, 5 Dec 2016 10:20:22 +0100 Subject: [PATCH 17/26] owr_audio_renderer: fix build warnings --- local/owr_audio_renderer.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/local/owr_audio_renderer.c b/local/owr_audio_renderer.c index 9157de7e..6690920b 100644 --- a/local/owr_audio_renderer.c +++ b/local/owr_audio_renderer.c @@ -123,7 +123,7 @@ OwrAudioRenderer *owr_audio_renderer_new(void) static void setup_sink_for_aec(GstElement *sink) { -#if defined(__linux__) && !defined(__ANDROID__) +#if defined(__linux__) && !defined(__ANDROID__) && !TARGET_RPI /* pulsesink */ GstStructure *s; @@ -136,7 +136,8 @@ setup_sink_for_aec(GstElement *sink) #elif defined(__APPLE__) && !TARGET_IPHONE_SIMULATOR /* osxaudiosink */ - +#else + OWR_UNUSED(sink); #endif } From edc27cf0f069543893ed29f3e008a31b927631fe Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Mon, 5 Dec 2016 10:20:45 +0100 Subject: [PATCH 18/26] owr_device_list: skip v4l2 devices on RPi --- local/owr_device_list.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/local/owr_device_list.c b/local/owr_device_list.c index 3d421d28..652a14c8 100644 --- a/local/owr_device_list.c +++ b/local/owr_device_list.c @@ -387,6 +387,12 @@ static gboolean enumerate_video_source_devices(GClosure *callback) OWR_MEDIA_SOURCE_SUPPORTS_VIDEO_ORIENTATION | OWR_MEDIA_SOURCE_SUPPORTS_COLOR_BALANCE); _owr_media_source_set_codec(OWR_MEDIA_SOURCE(source), OWR_CODEC_TYPE_H264); sources = g_list_prepend(sources, source); + + // Skip v4l2 devices. + _owr_utils_call_closure_with_list(callback, sources); + g_list_free_full(sources, g_object_unref); + + return FALSE; #endif dev_dir = g_dir_open("/dev", 0, &error); From b24b5eaa9aaa9bd8333fe8e88bb9f4567adba577 Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Mon, 5 Dec 2016 10:30:59 +0100 Subject: [PATCH 19/26] video: fix raw video rendering --- local/owr_video_renderer.c | 23 +++++++++++------------ transport/owr_payload.c | 2 +- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/local/owr_video_renderer.c b/local/owr_video_renderer.c index fccc0a05..a8c7725f 100644 --- a/local/owr_video_renderer.c +++ b/local/owr_video_renderer.c @@ -380,8 +380,8 @@ static void owr_video_renderer_reconfigure_element(OwrMediaRenderer *renderer) { OwrVideoRenderer *video_renderer; OwrVideoRendererPrivate *priv; - GstElement *parser; - GstElement *decoder; + GstElement *parser = NULL; + GstElement *decoder = NULL; GstElement *balance = NULL; GstElement *upload, *sink, *flip = NULL; GstPad *ghostpad, *sinkpad; @@ -398,12 +398,14 @@ static void owr_video_renderer_reconfigure_element(OwrMediaRenderer *renderer) source = _owr_media_renderer_get_source(renderer); codec_type = _owr_media_source_get_codec(source); - parser = _owr_create_parser(codec_type); - decoder = _owr_create_decoder(codec_type); - if (parser) - gst_bin_add(GST_BIN(priv->renderer_bin), parser); - if (decoder) - gst_bin_add(GST_BIN(priv->renderer_bin), decoder); + if (!_owr_codec_type_is_raw(codec_type)) { + parser = _owr_create_parser(codec_type); + decoder = _owr_create_decoder(codec_type); + if (parser) + gst_bin_add(GST_BIN(priv->renderer_bin), parser); + if (decoder) + gst_bin_add(GST_BIN(priv->renderer_bin), decoder); + } upload = gst_element_factory_make("glupload", "video-renderer-upload"); gst_bin_add(GST_BIN(priv->renderer_bin), upload); @@ -434,7 +436,7 @@ static void owr_video_renderer_reconfigure_element(OwrMediaRenderer *renderer) _owr_update_flip_method(G_OBJECT(renderer), NULL, flip); gst_bin_add(GST_BIN(priv->renderer_bin), flip); } else - g_warning("no suitable flipping element"); + g_warning("the glvideoflip element isn't available, video rotation support is now disabled"); } g_signal_connect_object(renderer, "notify::rotation", G_CALLBACK(update_flip_method), flip, 0); g_signal_connect_object(renderer, "notify::mirror", G_CALLBACK(update_flip_method), flip, 0); @@ -449,9 +451,6 @@ static void owr_video_renderer_reconfigure_element(OwrMediaRenderer *renderer) disable_last_sample_on_sink(&value, NULL); g_value_unset(&value); - // FIXME: huhhhhh really? - g_object_set(sink, "sync", FALSE, NULL); - gst_bin_add(GST_BIN(priv->renderer_bin), sink); _owr_bin_link_and_sync_elements(GST_BIN(priv->renderer_bin), &link_ok, NULL, &first, NULL); diff --git a/transport/owr_payload.c b/transport/owr_payload.c index d4559665..fc9a946b 100644 --- a/transport/owr_payload.c +++ b/transport/owr_payload.c @@ -579,7 +579,7 @@ GstCaps * _owr_payload_create_raw_caps(OwrPayload *payload) "framerate", &framerate, NULL); } - caps = gst_caps_new_empty_simple(_owr_codec_type_to_caps_mime(media_type, priv->codec_type)); + caps = gst_caps_new_empty_simple("video/x-raw"); #ifdef __APPLE__ if (priv->codec_type == OWR_CODEC_TYPE_H264) gst_caps_set_features(caps, 0, gst_caps_features_new_any()); From a3f3ddab4489c5465dcbd947adf1b06b79018737 Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Mon, 5 Dec 2016 10:32:37 +0100 Subject: [PATCH 20/26] transport_agent: fix audio support --- transport/owr_transport_agent.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transport/owr_transport_agent.c b/transport/owr_transport_agent.c index c6521786..cc874a76 100644 --- a/transport/owr_transport_agent.c +++ b/transport/owr_transport_agent.c @@ -824,7 +824,7 @@ static gboolean link_source_to_transport_bin(GstPad *srcpad, GstElement *pipelin if (media_type == OWR_MEDIA_TYPE_VIDEO) g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "video_sink_%u_%u", codec_type, stream_id); else if (media_type == OWR_MEDIA_TYPE_AUDIO) - g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "audio_raw_sink_%u", stream_id); + g_snprintf(name, OWR_OBJECT_NAME_LENGTH_MAX, "audio_sink_%u_%u", codec_type, stream_id); sinkpad = gst_element_get_static_pad(transport_bin, name); ret = gst_pad_link(srcpad, sinkpad) == GST_PAD_LINK_OK; @@ -987,7 +987,7 @@ static void remove_existing_send_source_and_payload(OwrTransportAgent *transport if (media_type == OWR_MEDIA_TYPE_VIDEO) pad_name = g_strdup_printf("video_sink_%u_%u", codec_type, stream_id); else - pad_name = g_strdup_printf("audio_raw_sink_%u", stream_id); + pad_name = g_strdup_printf("audio_sink_%u_%u", codec_type, stream_id); sinkpad = gst_element_get_static_pad(transport_agent->priv->transport_bin, pad_name); g_assert(sinkpad); g_free(pad_name); From 164b79ea00c522c6eb431b65f1e658ec6e87327b Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Mon, 5 Dec 2016 11:31:11 +0100 Subject: [PATCH 21/26] owr_payload: new owr_payload_supported API Can be used to probe the platform for a specific codec support. --- transport/owr_payload.c | 34 ++++++++++++++++++++++++++++++++++ transport/owr_payload.h | 2 ++ 2 files changed, 36 insertions(+) diff --git a/transport/owr_payload.c b/transport/owr_payload.c index fc9a946b..a43ce3bb 100644 --- a/transport/owr_payload.c +++ b/transport/owr_payload.c @@ -627,6 +627,40 @@ GstCaps * _owr_payload_create_encoded_caps(OwrPayload *payload) return caps; } +gboolean owr_payload_supported(OwrCodecType codec_type) +{ + gboolean supported = FALSE; + GstElement* encoder = NULL; + GstElement* decoder = _owr_create_decoder(codec_type); + + switch (codec_type) { + case OWR_CODEC_TYPE_H264: + encoder = _owr_try_codecs(_owr_get_detected_h264_encoders(), NULL); + break; + case OWR_CODEC_TYPE_VP8: + encoder = _owr_try_codecs(_owr_get_detected_vp8_encoders(), NULL); + break; + case OWR_CODEC_TYPE_VP9: + encoder = _owr_try_codecs(_owr_get_detected_vp9_encoders(), NULL); + break; + default: + encoder = gst_element_factory_make(_owr_get_encoder_name(codec_type), NULL); + } + + supported = encoder && decoder; + + if (encoder) { + gst_element_set_state(encoder, GST_STATE_NULL); + gst_object_unref(encoder); + } + + if (decoder) { + gst_element_set_state(decoder, GST_STATE_NULL); + gst_object_unref(decoder); + } + + return supported; +} /* local functions */ diff --git a/transport/owr_payload.h b/transport/owr_payload.h index 6de7d524..66d64090 100644 --- a/transport/owr_payload.h +++ b/transport/owr_payload.h @@ -65,6 +65,8 @@ struct _OwrPayloadClass { GType owr_payload_get_type(void) G_GNUC_CONST; +gboolean owr_payload_supported(OwrCodecType codec_type); + G_END_DECLS #endif /* __OWR_PAYLOAD_H__ */ From 4fe30f4abcc2b9405a7c1fee22a8f73e0391cb8c Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Wed, 7 Dec 2016 09:34:54 +0100 Subject: [PATCH 22/26] local/transport: fix build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit error: ‘for’ loop initial declarations are only allowed in C99 or C11 mode --- local/owr_video_renderer.c | 3 ++- transport/owr_transport_agent.c | 26 ++++++++++++++++++-------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/local/owr_video_renderer.c b/local/owr_video_renderer.c index a8c7725f..5a2c092d 100644 --- a/local/owr_video_renderer.c +++ b/local/owr_video_renderer.c @@ -308,7 +308,8 @@ static void renderer_disabled(OwrMediaRenderer *renderer, G_GNUC_UNUSED GParamSp const GList* controls = gst_color_balance_list_channels(color_balance); gint index = 0; - for (const GList* item = controls; item != NULL; item = item->next, ++index) { + const GList* item; + for (item = controls; item != NULL; item = item->next, ++index) { GstColorBalanceChannel* channel = item->data; if (g_strcmp0(channel->label, "SATURATION") == 0 || g_strcmp0(channel->label, "BRIGHTNESS") == 0) { gint new_value = disabled ? channel->min_value : ((channel->min_value + channel->max_value) / 2); diff --git a/transport/owr_transport_agent.c b/transport/owr_transport_agent.c index cc874a76..5617357a 100644 --- a/transport/owr_transport_agent.c +++ b/transport/owr_transport_agent.c @@ -1676,8 +1676,9 @@ static void prepare_transport_bin_send_elements(OwrTransportAgent *transport_age } else { GSList *sessions = get_sessions_from_stream_id(transport_agent, stream_id); guint previous_session_id = 0; + GSList *walk; // Look for the session for which the global encoder was created. - for (GSList *walk = sessions; walk && previous_session_id == 0; walk = g_slist_next(walk)) { + for (walk = sessions; walk && previous_session_id == 0; walk = g_slist_next(walk)) { OwrMediaSession *current = OWR_MEDIA_SESSION(walk->data); guint current_session_id; @@ -2002,6 +2003,7 @@ static gboolean emit_new_candidate(GHashTable *args) gchar *ufrag = NULL, *password = NULL; gboolean got_credentials; GSList *sessions; + GSList *walk; transport_agent = OWR_TRANSPORT_AGENT(g_hash_table_lookup(args, "transport_agent")); g_return_val_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent), FALSE); @@ -2011,7 +2013,7 @@ static gboolean emit_new_candidate(GHashTable *args) g_return_val_if_fail(nice_candidate, FALSE); sessions = get_sessions_from_stream_id(transport_agent, nice_candidate->stream_id); - for (GSList *walk = sessions; walk; walk = g_slist_next(walk)) { + for (walk = sessions; walk; walk = g_slist_next(walk)) { OwrSession *session = OWR_SESSION(walk->data); if (!nice_candidate->username || !nice_candidate->password) { got_credentials = nice_agent_get_local_credentials(priv->nice_agent, @@ -2071,12 +2073,13 @@ static gboolean emit_candidate_gathering_done(GHashTable *args) guint stream_id; int i; GSList *sessions; + GSList *walk; transport_agent = g_hash_table_lookup(args, "transport-agent"); stream_id = GPOINTER_TO_UINT(g_hash_table_lookup(args, "stream-id")); sessions = get_sessions_from_stream_id(transport_agent, stream_id); - for (GSList *walk = sessions; walk; walk = g_slist_next(walk)) { + for (walk = sessions; walk; walk = g_slist_next(walk)) { session = OWR_SESSION(walk->data); g_signal_emit_by_name(session, "on-candidate-gathering-done", NULL); @@ -2131,6 +2134,7 @@ static gboolean emit_ice_state_changed(GHashTable *args) OwrComponentType component_type; OwrIceState state; GSList *sessions; + GSList *walk; transport_agent = g_hash_table_lookup(args, "transport-agent"); stream_id = GPOINTER_TO_UINT(g_hash_table_lookup(args, "stream-id")); @@ -2138,7 +2142,7 @@ static gboolean emit_ice_state_changed(GHashTable *args) state = GPOINTER_TO_UINT(g_hash_table_lookup(args, "ice-state")); sessions = get_sessions_from_stream_id(transport_agent, stream_id); - for (GSList *walk = sessions; walk; walk = g_slist_next(walk)) { + for (walk = sessions; walk; walk = g_slist_next(walk)) { OwrSession *session = OWR_SESSION(walk->data); _owr_session_emit_ice_state_changed(session, stream_id, component_type, state); @@ -2173,6 +2177,7 @@ static void on_new_selected_pair(NiceAgent *nice_agent, OwrTransportAgent *transport_agent) { GSList *sessions; + GSList *walk; OWR_UNUSED(nice_agent); OWR_UNUSED(lcandidate); @@ -2181,7 +2186,7 @@ static void on_new_selected_pair(NiceAgent *nice_agent, g_return_if_fail(OWR_IS_TRANSPORT_AGENT(transport_agent)); sessions = get_sessions_from_stream_id(transport_agent, stream_id); - for (GSList *walk = sessions; walk; walk = g_slist_next(walk)) { + for (walk = sessions; walk; walk = g_slist_next(walk)) { OwrSession *session = OWR_SESSION(walk->data); guint session_id = get_session_id(transport_agent, session); PendingSessionInfo *pending_session_info; @@ -3095,8 +3100,9 @@ static GstElement * on_rtpbin_request_aux_receiver(G_GNUC_UNUSED GstElement *rtp g_object_unref(media_session); if ((transport_agent->priv->bundle_policy == OWR_BUNDLE_POLICY_TYPE_MAX_BUNDLE) && !pt_map) { + GSList* walk; GST_DEBUG("no valid pt_map found, looking for one in the staged sessions"); - for (GSList* walk = transport_agent->priv->unstarted_sessions; walk; walk = g_slist_next(walk)) { + for (walk = transport_agent->priv->unstarted_sessions; walk; walk = g_slist_next(walk)) { OwrSession* session = OWR_SESSION(walk->data); if (!OWR_IS_MEDIA_SESSION(session)) continue; @@ -4497,9 +4503,12 @@ static gboolean dump_bin(gpointer data) void owr_transport_agent_start(OwrTransportAgent *agent) { + GSList *walk; + if (!agent->priv->unstarted_sessions) return; - for (GSList *walk = agent->priv->unstarted_sessions; walk; walk = g_slist_next(walk)) { + + for (walk = agent->priv->unstarted_sessions; walk; walk = g_slist_next(walk)) { GHashTable *args; args = _owr_create_schedule_table(OWR_MESSAGE_ORIGIN(agent)); g_hash_table_insert(args, "transport_agent", agent); @@ -4637,7 +4646,8 @@ static GstPadProbeReturn probe_rtp_info(GstPad *srcpad, GstPadProbeInfo *info, S rx_payload = _owr_media_session_get_receive_payload(media_session, pt); } else { GSList *sessions = get_sessions_from_stream_id(transport_agent, stream_id); - for (GSList *walk = sessions; walk; walk = g_slist_next(walk)) { + GSList *walk; + for (walk = sessions; walk; walk = g_slist_next(walk)) { media_session = OWR_MEDIA_SESSION(walk->data); rx_payload = _owr_media_session_get_receive_payload(media_session, pt); if (rx_payload) From cf174d29679826d2af176d6eb3942f24d7e14050 Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Wed, 7 Dec 2016 12:59:34 +0100 Subject: [PATCH 23/26] transport_agent: request key-frame on all platforms This seems to be an issue on linux with vp8dec too. --- transport/owr_transport_agent.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/transport/owr_transport_agent.c b/transport/owr_transport_agent.c index 5617357a..74a94a18 100644 --- a/transport/owr_transport_agent.c +++ b/transport/owr_transport_agent.c @@ -2829,17 +2829,15 @@ static GstPadProbeReturn check_for_keyframe(GstPad *pad, GstPadProbeInfo *info, return GST_PAD_PROBE_OK; } -#if TARGET_RPI static gboolean force_key_unit_event(gpointer data) { - GstElement* videorepair = (GstElement*) data; - GstPad* sink_pad = gst_element_get_static_pad(videorepair, "sink"); + GstElement *videorepair = GST_ELEMENT_CAST(data); + GstPad *sink_pad = gst_element_get_static_pad(videorepair, "sink"); gst_pad_push_event(sink_pad, gst_video_event_new_upstream_force_key_unit(GST_CLOCK_TIME_NONE, FALSE, 0)); gst_object_unref(sink_pad); gst_object_unref(videorepair); return G_SOURCE_REMOVE; } -#endif static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, guint32 stream_id, OwrPayload *payload, OwrTransportAgent *transport_agent) { @@ -2885,12 +2883,10 @@ static void setup_video_receive_elements(GstPad *new_pad, guint32 session_id, gu gst_object_unref(pad); pad = NULL; -#if TARGET_RPI - // The OMX video decoder doesn't seem to handle very well the incoming + // The OMX video decoder and vp8dec element don't seem to handle very well the incoming // streams from Chrome, not taking into account the first intra frame // received. So for now, force a PLI request towards the sender after 250ms. g_timeout_add(250, force_key_unit_event, gst_object_ref(videorepair1)); -#endif _owr_bin_link_and_sync_elements(GST_BIN(receive_output_bin), &link_ok, &sync_ok, &first, &last); g_warn_if_fail(link_ok && sync_ok); From 173776e702f72b6626b66079afa6a448242cdbbf Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Wed, 7 Dec 2016 09:35:36 +0100 Subject: [PATCH 24/26] transport: port to OpenSSL 1.1.0 --- transport/owr_crypto_utils.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/transport/owr_crypto_utils.c b/transport/owr_crypto_utils.c index 785fde84..60872a47 100644 --- a/transport/owr_crypto_utils.c +++ b/transport/owr_crypto_utils.c @@ -148,8 +148,22 @@ gpointer _create_crypto_worker_run(gpointer data) key_pair = EVP_PKEY_new(); + // RSA_generate_key was deprecated in OpenSSL 0.9.8. +#if OPENSSL_VERSION_NUMBER < 0x10100001L rsa = RSA_generate_key(2048, RSA_F4, NULL, NULL); - +#else + rsa = RSA_new (); + if (rsa != NULL) { + BIGNUM *e = BN_new (); + if (e == NULL || !BN_set_word(e, RSA_F4) + || !RSA_generate_key_ex(rsa, 2048, e, NULL)) { + RSA_free(rsa); + rsa = NULL; + } + if (e) + BN_free(e); + } +#endif EVP_PKEY_assign_RSA(key_pair, rsa); X509_set_version(cert, 2); From 09eda7252ceed89ddfa85e775f0849b5cd9bd44e Mon Sep 17 00:00:00 2001 From: Carlos Alberto Lopez Perez Date: Mon, 23 Jan 2017 19:15:50 +0100 Subject: [PATCH 25/26] Workaround issue with the RPiCam and resolutions higher than 720p. * When the RPi is configured with a resolution higher than 720p like 1080p, and some application requests OWRTC a full-screen video the requested source video resolution on the RPiCam can cause firmware fatal errors. * This happens with WebKit on AppRTC with the RPi configured on a 1080p display: mmal: mmal_vc_port_enable: failed to enable port vc.ril.video_encode:in:0(OPQV): EINVAL mmal: mmal_port_enable: failed to enable connected port (vc.ril.video_encode:in:0(OPQV))0x59c350 (EINVAL) mmal: mmal_connection_enable: output port couldn't be enabled ==== Error message start ==== Error in element video-source. Error: Internal data flow error. Debugging info: gstbasesrc.c(2948): gst_base_src_loop (): /GstPipeline:local-video-capture-source-bin-0/GstRpiCamSrc:video-source: streaming task paused, reason error (-5) * With this patch, AppRTC works fine on a 1080p screen with the RPiCam. The following GStreamer logs happen: DEBUG [...] RPiCam: Asked source video resolution of 2160x1080 WARN [...] RPiCam: The asked source video resolution was capped to 1440x720 to void mmal firmware related errors --- local/owr_local_media_source.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/local/owr_local_media_source.c b/local/owr_local_media_source.c index d9023cac..93b051e3 100644 --- a/local/owr_local_media_source.c +++ b/local/owr_local_media_source.c @@ -840,6 +840,37 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s gst_caps_set_simple(source_caps, "format", G_TYPE_STRING, "NV12", NULL); #endif +#if defined(TARGET_RPI) && TARGET_RPI +#define MAX_RPI_CAMERA_HEIGHT 720 +#define MAX_RPI_CAMERA_WIDTH 1440 + /* The requested video source resolution can cause firmware mmal related errors. + * This happens for example with WebKit and AppRTC when the screen resolution is set to 1080p. + * We work-around this by capping the maximum allowed resolution to 720p. + * We try to keep the aspect ratio. */ + gint width, height; + gdouble ratio; + const GstStructure *str; + str = gst_caps_get_structure (source_caps, 0); + if (gst_structure_get_int (str, "width", &width) && + gst_structure_get_int (str, "height", &height)) { + GST_DEBUG_OBJECT(local_source, "RPiCam: Asked source video resolution of %dx%d\n", width, height); + if ((height > MAX_RPI_CAMERA_HEIGHT) || + (width > MAX_RPI_CAMERA_WIDTH)) { + ratio = (gdouble) width / height; + if (height > MAX_RPI_CAMERA_HEIGHT) { + height = MAX_RPI_CAMERA_HEIGHT; + width = height * ratio; + } + if (width > MAX_RPI_CAMERA_WIDTH) { + width = MAX_RPI_CAMERA_WIDTH; + height = width / ratio; + } + gst_caps_set_simple(source_caps, "height", G_TYPE_INT, height, NULL); + gst_caps_set_simple(source_caps, "width", G_TYPE_INT, width, NULL); + GST_WARNING_OBJECT(local_source, "RPiCam: The asked source video resolution was capped to %dx%d to avoid mmal firmware related errors\n", width, height); + } + } +#endif //defined(TARGET_RPI) && TARGET_RPI CREATE_ELEMENT(capsfilter, "capsfilter", "video-source-capsfilter"); g_object_set(capsfilter, "caps", source_caps, NULL); gst_caps_unref(source_caps); From c8d6dae59a3fd4e76f8ae77d0cbd7b15c8358238 Mon Sep 17 00:00:00 2001 From: Carlos Alberto Lopez Perez Date: Wed, 17 May 2017 16:38:55 +0200 Subject: [PATCH 26/26] Allow to use a test video source that outputs H264 video This is enabled by setting the value of the environment variable OWR_USE_TEST_SOURCES to 'h264' or 'H264'. Any other value of the environment variable will enable the previous test sources with raw video format. --- local/owr_local.c | 3 +++ local/owr_local_media_source.c | 29 +++++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/local/owr_local.c b/local/owr_local.c index 353bbdd1..de339d75 100644 --- a/local/owr_local.c +++ b/local/owr_local.c @@ -65,6 +65,7 @@ static GList *get_test_sources(OwrMediaType types) OwrMediaType media_type; GList *result_list = NULL; GList *elem; + gboolean useh264 = g_ascii_strcasecmp (g_getenv("OWR_USE_TEST_SOURCES"),"H264") == 0; if (g_once_init_enter(&cached_sources)) { GList *sources = NULL; @@ -73,6 +74,8 @@ static GList *get_test_sources(OwrMediaType types) sources = g_list_append(sources, OWR_MEDIA_SOURCE(source)); source = _owr_local_media_source_new_cached(-1, "Video test source", OWR_MEDIA_TYPE_VIDEO, OWR_SOURCE_TYPE_TEST, OWR_MEDIA_SOURCE_SUPPORTS_NONE); + if (useh264) + _owr_media_source_set_codec(OWR_MEDIA_SOURCE(source), OWR_CODEC_TYPE_H264); sources = g_list_append(sources, OWR_MEDIA_SOURCE(source)); g_once_init_leave(&cached_sources, sources); diff --git a/local/owr_local_media_source.c b/local/owr_local_media_source.c index 93b051e3..14d596e3 100644 --- a/local/owr_local_media_source.c +++ b/local/owr_local_media_source.c @@ -767,8 +767,14 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s #endif break; case OWR_SOURCE_TYPE_TEST: { - GstElement *src, *time; + GstElement *src, *time, *h264enc = NULL; GstPad *srcpad; + gboolean useh264 = g_ascii_strcasecmp (g_getenv("OWR_USE_TEST_SOURCES"),"H264") == 0; + + if (useh264) + printf("video-source encoding: video/x-h264\n"); + else + printf("video-source encoding: video/x-raw\n"); source = gst_bin_new("video-source"); @@ -781,9 +787,24 @@ static GstElement *owr_local_media_source_request_source(OwrMediaSource *media_s g_object_set(time, "font-desc", "Sans 60", NULL); gst_bin_add(GST_BIN(source), time); gst_element_link(src, time); - srcpad = gst_element_get_static_pad(time, "src"); - } else - srcpad = gst_element_get_static_pad(src, "src"); + if (!useh264) + srcpad = gst_element_get_static_pad(time, "src"); + } else if (!useh264) + srcpad = gst_element_get_static_pad(src, "src"); + + if (useh264) { + h264enc = gst_element_factory_make("openh264enc", "openh264enc"); + if (!h264enc) { + GST_ERROR_OBJECT(source, "Failed to create openh264enc element!"); + printf("Failed to create openh264enc element!\n"); + } + gst_bin_add(GST_BIN(source), h264enc); + if (time) + gst_element_link(time, h264enc); + else + gst_element_link(src, h264enc); + srcpad = gst_element_get_static_pad(h264enc, "src"); + } gst_element_add_pad(source, gst_ghost_pad_new("src", srcpad)); gst_object_unref(srcpad);