(Draft) Backport fixes for player suspend

Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp

@@ -495,11 +495,12 @@ void MediaPlayerPrivateGStreamer::play()
         return;
     }
 
-    if (changePipelineState(GST_STATE_PLAYING) == ChangePipelineStateResult::Ok) {
+    auto res = changePipelineState(GST_STATE_PLAYING);
+    if (res == ChangePipelineStateResult::Ok || res == ChangePipelineStateResult::SavedUntilResume) {
         m_isEndReached = false;
         m_isDelayingLoad = false;
         m_preload = MediaPlayer::Preload::Auto;
-        GST_INFO_OBJECT(pipeline(), "Play");
+        GST_INFO_OBJECT(pipeline(), "Play%s", (res == ChangePipelineStateResult::Ok) ? "" : " (state change saved due to suspend)");
 #if ENABLE(MEDIA_TELEMETRY)
         MediaTelemetryReport::singleton().reportPlaybackState(MediaTelemetryReport::AVPipelineState::Play);
 #endif
@@ -1088,10 +1089,16 @@ MediaPlayerPrivateGStreamer::ChangePipelineStateResult MediaPlayerPrivateGStream
 {
     ASSERT(m_pipeline);
 
-    if (m_isPausedByViewport && newState > GST_STATE_PAUSED) {
-        GST_DEBUG_OBJECT(pipeline(), "Saving state for when player becomes visible: %s", gst_element_state_get_name(newState));
-        m_invisiblePlayerState = newState;
-        return ChangePipelineStateResult::Ok;
+    if (playerIsSuspended()) {
+        // Save requests to play until we resume.
+        if (newState > GST_STATE_PAUSED) {
+            GST_DEBUG_OBJECT(pipeline(), "Saving state for when player is resumed: %s", gst_element_state_get_name(newState));
+            m_stateToResume = newState;
+            return ChangePipelineStateResult::SavedUntilResume;
+        }
+
+        // Any other state changes are ok to execute right away.
+        m_stateToResume = GST_STATE_VOID_PENDING;
     }
 
     GstState currentState, pending;
@@ -1363,8 +1370,14 @@ MediaTime MediaPlayerPrivateGStreamer::platformDuration() const
 
 bool MediaPlayerPrivateGStreamer::isMuted() const
 {
-    GST_INFO_OBJECT(pipeline(), "Player is muted: %s", boolForPrinting(m_isMuted));
-    return m_isMuted;
+    bool isMuted = false;
+
+    auto streamVolume = GST_STREAM_VOLUME(m_pipeline.get());
+    if (streamVolume)
+        isMuted = gst_stream_volume_get_mute(streamVolume);
+
+    GST_INFO_OBJECT(pipeline(), "Player is muted: %s", boolForPrinting(isMuted));
+    return isMuted;
 }
 
 void MediaPlayerPrivateGStreamer::commitLoad()
@@ -1964,6 +1977,7 @@ void MediaPlayerPrivateGStreamer::setMuted(bool shouldMute)
     GST_INFO_OBJECT(pipeline(), "Setting muted state to %s", boolForPrinting(shouldMute));
     g_object_set(m_volumeElement.get(), "mute", static_cast<gboolean>(shouldMute), nullptr);
     configureMediaStreamAudioTracks();
+    managePlayerSuspend();
 }
 
 void MediaPlayerPrivateGStreamer::notifyPlayerOfMute()
@@ -3441,6 +3455,7 @@ void MediaPlayerPrivateGStreamer::createGSTPlayBin(const URL& url)
             player->videoSinkCapsChanged(videoSinkPad);
         }), this);
 
+    managePlayerSuspend();
 #if ENABLE(MEDIA_TELEMETRY)
     MediaTelemetryReport::singleton().reportDrmInfo(getDrm());
     MediaTelemetryReport::singleton().reportPlaybackState(MediaTelemetryReport::AVPipelineState::Create);
@@ -4264,44 +4279,56 @@ void MediaPlayerPrivateGStreamer::flushCurrentBuffer()
 
 void MediaPlayerPrivateGStreamer::setVisibleInViewport(bool isVisible)
 {
-    if (isMediaStreamPlayer())
-        return;
+    GST_DEBUG_OBJECT(pipeline(), "Player is now %svisible in the viewport", isVisible ? "" : "not ");
 
-    // Some layout tests (webgl) expect playback of invisible videos to not be suspended, so allow
-    // this using an environment variable, set from the webkitpy glib port sub-classes.
-    const char* allowPlaybackOfInvisibleVideos = g_getenv("WEBKIT_GST_ALLOW_PLAYBACK_OF_INVISIBLE_VIDEOS");
-    if (!isVisible && allowPlaybackOfInvisibleVideos && !strcmp(allowPlaybackOfInvisibleVideos, "1"))
+    if (isMediaStreamPlayer() || isVisible == m_isVisibleInViewport)
         return;
 
+    m_isVisibleInViewport = isVisible;
+    managePlayerSuspend();
+}
+
+void MediaPlayerPrivateGStreamer::managePlayerSuspend()
+{
     if (!m_pipeline)
         return;
 
     RefPtr player = m_player.get();
 
-    GST_INFO_OBJECT(m_pipeline.get(), "%s %s player %svisible in viewport", m_isMuted ? "Muted" : "Un-muted", (player && player->isVideoPlayer()) ? "video" : "audio", isVisible ? "" : "no longer ");
-    if ((player && !player->isVideoPlayer()) || !m_isMuted)
-        return;
+    // Some layout tests (webgl) expect playback of invisible videos to not be suspended, so allow
+    // this using an environment variable, set from the webkitpy glib port sub-classes.
+    const char* allowPlaybackOfInvisibleVideos = g_getenv("WEBKIT_GST_ALLOW_PLAYBACK_OF_INVISIBLE_VIDEOS");
+    bool muted = isMuted();
+    bool shouldBeSuspended = (player && player->isVideoPlayer()) && muted && !m_isVisibleInViewport && allowPlaybackOfInvisibleVideos && !strcmp(allowPlaybackOfInvisibleVideos, "1");
+    GST_INFO_OBJECT(m_pipeline.get(), "%s %s player %svisible in viewport", muted ? "Muted" : "Un-muted", (player && player->isVideoPlayer()) ? "video" : "audio", m_isVisibleInViewport ? "" : "not ");
 
-    if (!isVisible) {
+    if (shouldBeSuspended && !playerIsSuspended()) {
         GstState currentState, pendingState;
         gst_element_get_state(m_pipeline.get(), &currentState, &pendingState, 0);
         GstState targetState = (pendingState != GST_STATE_VOID_PENDING ? pendingState : currentState);
-        if (targetState > GST_STATE_NULL)
-            m_invisiblePlayerState = targetState;
-        m_isPausedByViewport = true;
+        m_playerIsSuspended = true;
+        if (targetState == GST_STATE_NULL) {
+            GST_DEBUG_OBJECT(pipeline(), "Pipeline is already in NULL state, no point in pausing the player.");
+            return;
+        }
+        m_stateToResume = targetState;
         GST_DEBUG_OBJECT(pipeline(), "Media element is muted and not visible in viewport, pausing it to save resources. Will resume afterwards to %s state.",
-            gst_element_state_get_name(m_invisiblePlayerState));
+            gst_element_state_get_name(m_stateToResume));
         gst_element_set_state(m_pipeline.get(), GST_STATE_PAUSED);
         gst_element_get_state(m_pipeline.get(), &currentState, &pendingState, 0);
         GST_DEBUG_OBJECT(pipeline(), "Now pipeline is in %s state with %s pending", gst_element_state_get_name(currentState), gst_element_state_get_name(pendingState));
         m_isPipelinePlaying = false;
-    } else {
-        m_isPausedByViewport = false;
-        if (m_invisiblePlayerState != GST_STATE_VOID_PENDING) {
-            GST_DEBUG_OBJECT(pipeline(), "Element in viewport again, resuming playback via state change to %s.",
-                gst_element_state_get_name(m_invisiblePlayerState));
-            changePipelineState(m_invisiblePlayerState);
-        }
+    } else if (!shouldBeSuspended && playerIsSuspended()) {
+        m_playerIsSuspended = false;
+
+        if (m_stateToResume == GST_STATE_VOID_PENDING)
+            return;
+
+        GstState resumeState = m_stateToResume;
+        m_stateToResume = GST_STATE_VOID_PENDING;
+        GST_DEBUG_OBJECT(pipeline(), "Element is either unmuted or in viewport again, resuming playback via state change to %s.",
+            gst_element_state_get_name(resumeState));
+        changePipelineState(resumeState);
     }
 }
 
@@ -4315,7 +4342,7 @@ void MediaPlayerPrivateGStreamer::paint(GraphicsContext& context, const FloatRec
     if (context.paintingDisabled())
         return;
 
-    if (!m_visible || m_isPausedByViewport)
+    if (!m_pageIsVisible || playerIsSuspended())
         return;
 
     // Keep a reference to the sample to avoid keeping the sampleMutex locked, which would be prone
@@ -4934,7 +4961,7 @@ void MediaPlayerPrivateGStreamer::setVideoRectangle(const IntRect& rect)
 
     Locker locker { m_holePunchLock };
 
-    if (!m_visible || m_suspended)
+    if (!m_pageIsVisible || m_pageIsSuspended)
         return;
 
     if (m_quirksManagerForTesting) {
@@ -4948,18 +4975,18 @@ void MediaPlayerPrivateGStreamer::setVideoRectangle(const IntRect& rect)
 
 void MediaPlayerPrivateGStreamer::setPageIsVisible(bool visible)
 {
-    if (m_visible == visible)
+    if (m_pageIsVisible == visible)
         return;
 
     if (!isHolePunchRenderingEnabled() || !m_videoSink) {
-        m_visible = visible;
+        m_pageIsVisible = visible;
         return;
     }
 
     Locker locker { m_holePunchLock };
-    m_visible = visible;
+    m_pageIsVisible = visible;
 
-    if (!m_visible) {
+    if (!m_pageIsVisible) {
         if (m_quirksManagerForTesting) {
             m_quirksManagerForTesting->setHolePunchVideoRectangle(m_videoSink.get(), IntRect());
             return;
@@ -4972,18 +4999,18 @@ void MediaPlayerPrivateGStreamer::setPageIsVisible(bool visible)
 
 void MediaPlayerPrivateGStreamer::setPageIsSuspended(bool suspended)
 {
-    if (m_suspended == suspended)
+    if (m_pageIsSuspended == suspended)
         return;
 
     if (!isHolePunchRenderingEnabled() || !m_videoSink) {
-        m_suspended = suspended;
+        m_pageIsSuspended = suspended;
         return;
     }
 
     Locker locker { m_holePunchLock };
-    m_suspended = suspended;
+    m_pageIsSuspended = suspended;
 
-    if (m_suspended) {
+    if (m_pageIsSuspended) {
         if (m_quirksManagerForTesting) {
             m_quirksManagerForTesting->setHolePunchVideoRectangle(m_videoSink.get(), IntRect());
             return;

Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.h

@@ -332,6 +332,8 @@ class MediaPlayerPrivateGStreamer
         Ok,
         Rejected,
         Failed,
+        // Pipeline is suspended, and the requested state change was saved to be executed when it resumes.
+        SavedUntilResume,
     };
     ChangePipelineStateResult changePipelineState(GstState);
 
@@ -450,8 +452,8 @@ class MediaPlayerPrivateGStreamer
     GRefPtr<GstElement> m_source { nullptr };
     bool m_areVolumeAndMuteInitialized { false };
 
-    // Reflects whether the pipeline was paused due to the HTMLMediaElement being both muted and invisible in the viewport.
-    bool m_isPausedByViewport { false };
+    // Reflects whether the pipeline was suspended due to the HTMLMediaElement being both muted and invisible in the viewport.
+    bool playerIsSuspended() const { return m_playerIsSuspended; };
 
 #if USE(TEXTURE_MAPPER)
     OptionSet<TextureMapperFlags> m_textureMapperFlags;
@@ -564,6 +566,8 @@ class MediaPlayerPrivateGStreamer
     void finishSeek();
     virtual void didPreroll() { }
 
+    void managePlayerSuspend();
+
     void createGSTPlayBin(const URL&);
 
     bool loadNextLocation();
@@ -665,9 +669,17 @@ class MediaPlayerPrivateGStreamer
     RefPtr<MediaStreamPrivate> m_streamPrivate;
 #endif
 
+    // Only notifyPlayerOfMute uses this to avoid sending redundant notifications.
+    // Since it's updated by a callback, this will be incorrect right after un/muting the player,
+    // use isMuted() instead.
     bool m_isMuted { false };
-    bool m_visible { false };
-    bool m_suspended { false };
+
+    bool m_isVisibleInViewport { true };
+
+    // Whether the page containing the HTMLMediaElement is visible, reflects: setPageIsVisible()
+    bool m_pageIsVisible { false };
+
+    bool m_pageIsSuspended { false };
 
     // playbin3 only:
     bool m_waitingForStreamsSelectedEvent { true };
@@ -721,8 +733,9 @@ class MediaPlayerPrivateGStreamer
 
     bool m_didTryToRecoverPlayingState { false };
 
-    // The state the pipeline should be set back to after the player becomes visible in the viewport again.
-    GstState m_invisiblePlayerState { GST_STATE_VOID_PENDING };
+    bool m_playerIsSuspended { false };
+    // The state the pipeline should be set back to after the player is resumed.
+    GstState m_stateToResume { GST_STATE_VOID_PENDING };
 
     // Specific to MediaStream playback.
     MediaTime m_startTime;

Source/WebCore/platform/graphics/gstreamer/mse/MediaPlayerPrivateGStreamerMSE.cpp

@@ -459,15 +459,15 @@ void MediaPlayerPrivateGStreamerMSE::updateStates()
 {
     bool isWaitingPreroll = isPipelineWaitingPreroll();
     bool shouldUpdatePlaybackState = false;
-    bool shouldBePlaying = (!m_isPaused && !m_isPausedByViewport && readyState() >= MediaPlayer::ReadyState::HaveFutureData && m_playbackRatePausedState != PlaybackRatePausedState::RatePaused)
+    bool shouldBePlaying = (!m_isPaused && readyState() >= MediaPlayer::ReadyState::HaveFutureData && m_playbackRatePausedState != PlaybackRatePausedState::RatePaused)
         || m_playbackRatePausedState == PlaybackRatePausedState::ShouldMoveToPlaying;
     GST_DEBUG_OBJECT(pipeline(), "shouldBePlaying = %s, m_isPipelinePlaying = %s, is seeking %s", boolForPrinting(shouldBePlaying),
         boolForPrinting(m_isPipelinePlaying), boolForPrinting(isWaitingPreroll));
     if (!isWaitingPreroll && shouldBePlaying && !m_isPipelinePlaying) {
         auto result = changePipelineState(GST_STATE_PLAYING);
         if (result == ChangePipelineStateResult::Failed)
             GST_ERROR_OBJECT(pipeline(), "Setting the pipeline to PLAYING failed");
-        else if (result == ChangePipelineStateResult::Ok) {
+        else if (result == ChangePipelineStateResult::Ok || result == ChangePipelineStateResult::SavedUntilResume) {
             m_playbackRatePausedState = PlaybackRatePausedState::Playing;
             shouldUpdatePlaybackState = true;
         }