refactor mpegts h26x packets

common/src/main/java/com/pedro/common/Extensions.kt

@@ -286,13 +286,15 @@ fun Long.compare(l: Long): Int {
 }
 
 fun SurfaceTexture.tryClear() {
-  val surface = Surface(this)
+  //Surface constructor fail if the SurfaceTexture is already released
+  var surface: Surface? = null
   try {
+    surface = Surface(this)
     val canvas = surface.lockCanvas(null)
     canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR)
     surface.unlockCanvasAndPost(canvas)
   } catch (_: Exception) {} finally {
-    surface.release()
+    surface?.release()
   }
 }
 

library/src/main/java/com/pedro/library/base/StreamBase.kt

@@ -411,7 +411,7 @@ abstract class StreamBase(
     }
     videoSource.stop()
     videoSource.release()
-    glInterface.surfaceTexture.tryClear()
+    if (glInterface.isRunning) glInterface.surfaceTexture.tryClear()
     if (wasRunning) source.start(glInterface.surfaceTexture)
     glInterface.setOrientationConfig(source.getOrientationConfig())
     videoSource = source
@@ -544,7 +544,7 @@ abstract class StreamBase(
     stopSources()
     videoSource.release()
     audioSource.release()
-    glInterface.surfaceTexture.tryClear()
+    if (glInterface.isRunning) glInterface.surfaceTexture.tryClear()
   }
 
   /**

library/src/main/java/com/pedro/library/util/FlvMuxerRecordController.kt

@@ -71,6 +71,7 @@ class FlvMuxerRecordController: AsyncBaseRecordController() {
             VideoCodec.VP8 -> Vp8Packet()
             VideoCodec.VP9 -> Vp9Packet()
         }
+        sendInfo = false
         outputStream?.let {
             try {
                 it.write(createFlvFileHeader())
@@ -217,6 +218,11 @@ class FlvMuxerRecordController: AsyncBaseRecordController() {
                 }
             }
             is Vp8Packet -> sendInfo = true
+            //vp9 video info is not in the MediaFormat, only extracted from keyframes
+            is Vp9Packet -> {}
+            else -> {
+                Log.e(TAG, "Unsupported codec: ${videoPacket?.javaClass?.name ?: "null"}")
+            }
         }
         if (sendInfo && recordStatus == RecordController.Status.STARTED) {
             myRequestKeyFrame = null

library/src/main/java/com/pedro/library/util/Mpeg2TsMuxerRecordController.kt

@@ -15,10 +15,10 @@ import com.pedro.library.base.recording.RecordController.RecordTracks
 import com.pedro.srt.mpeg2ts.MpegTsPacket
 import com.pedro.srt.mpeg2ts.MpegTsPacketizer
 import com.pedro.srt.mpeg2ts.MpegType
-import com.pedro.srt.mpeg2ts.Pid
 import com.pedro.srt.mpeg2ts.packets.AacPacket
 import com.pedro.srt.mpeg2ts.packets.BasePacket
-import com.pedro.srt.mpeg2ts.packets.H26XPacket
+import com.pedro.srt.mpeg2ts.packets.H264Packet
+import com.pedro.srt.mpeg2ts.packets.H265Packet
 import com.pedro.srt.mpeg2ts.packets.OpusPacket
 import com.pedro.srt.mpeg2ts.psi.Psi
 import com.pedro.srt.mpeg2ts.psi.PsiManager
@@ -51,7 +51,7 @@ class Mpeg2TsMuxerRecordController : AsyncBaseRecordController() {
 
   private val mpegTsPacketizer = MpegTsPacketizer(psiManager)
   private var audioPacket: BasePacket = AacPacket(limitSize, psiManager)
-  private val videoPacket = H26XPacket(limitSize, psiManager)
+  private var videoPacket: BasePacket = H264Packet(limitSize, psiManager)
   private val chunkSize = limitSize / MpegTsPacketizer.packetSize
   private var sampleRate = 0
   private var isStereo = true
@@ -86,11 +86,14 @@ class Mpeg2TsMuxerRecordController : AsyncBaseRecordController() {
         throw IOException("Unsupported AudioCodec: " + getAudioCodec().name)
       }
     }
-    if (getVideoCodec() == VideoCodec.AV1 || getVideoCodec() == VideoCodec.VP8 || getVideoCodec() == VideoCodec.VP9) {
-      throw IOException("Unsupported VideoCodec: " + getVideoCodec().name)
+    videoPacket = when (getVideoCodec()) {
+      VideoCodec.H264 -> H264Packet(limitSize, psiManager)
+      VideoCodec.H265 -> H265Packet(limitSize, psiManager)
+      else -> {
+        throw IOException("Unsupported VideoCodec: " + getVideoCodec().name)
+      }
     }
-    audioPacket.setLimitSize(limitSize)
-    videoPacket.setLimitSize(limitSize)
+    sendInfo = false
     outputStream?.let {
       val videoEnabled = tracks == RecordTracks.VIDEO || tracks == RecordTracks.ALL
       val audioEnabled = tracks == RecordTracks.AUDIO || tracks == RecordTracks.ALL
@@ -182,37 +185,37 @@ class Mpeg2TsMuxerRecordController : AsyncBaseRecordController() {
   private fun getVideoInfo(buffer: ByteBuffer, info: MediaCodec.BufferInfo) {
     if (info.isKeyframe() || isKeyFrame(buffer)) {
       if (!sendInfo) {
-        when (getVideoCodec()) {
-          VideoCodec.H264 -> {
+        when (videoPacket) {
+          is H264Packet -> {
             val buffers =
               VideoEncoderHelper.decodeSpsPpsFromBuffer(buffer.duplicate(), info.size)
             if (buffers != null) {
               Log.i(TAG, "manual sps/pps extraction success")
               val oldSps = buffers.first
               val oldPps = buffers.second
-              videoPacket.sendVideoInfo(oldSps, oldPps, null)
+              (videoPacket as H264Packet).sendVideoInfo(oldSps, oldPps)
               sendInfo = true
             } else {
               Log.e(TAG, "manual sps/pps extraction failed")
             }
           }
 
-          VideoCodec.H265 -> {
+          is H265Packet -> {
             val byteBufferList = VideoEncoderHelper.extractVpsSpsPpsFromH265(buffer.duplicate())
             if (byteBufferList.size == 3) {
               Log.i(TAG, "manual vps/sps/pps extraction success")
               val oldSps = byteBufferList[1]
               val oldPps = byteBufferList[2]
               val oldVps = byteBufferList[0]
-              videoPacket.sendVideoInfo(oldSps, oldPps, oldVps)
+              (videoPacket as H265Packet).sendVideoInfo(oldSps, oldPps, oldVps)
               sendInfo = true
             } else {
               Log.e(TAG, "manual vps/sps/pps extraction failed")
             }
           }
 
           else -> {
-            Log.e(TAG, "Unsupported codec: ${getVideoCodec()}")
+            Log.e(TAG, "Unsupported codec: ${videoPacket.javaClass.name}")
           }
         }
       }
@@ -228,31 +231,33 @@ class Mpeg2TsMuxerRecordController : AsyncBaseRecordController() {
   }
 
   override fun setVideoFormat(videoFormat: MediaFormat) {
-    when (getVideoCodec()) {
-      VideoCodec.H264 -> {
+    when (videoPacket) {
+      is H264Packet -> {
         val sps = videoFormat.getByteBuffer("csd-0")
         val pps = videoFormat.getByteBuffer("csd-1")
         if (sps != null && pps != null) {
-          videoPacket.sendVideoInfo(sps.duplicate(), pps.duplicate(), null)
+          (videoPacket as H264Packet).sendVideoInfo(sps.duplicate(), pps.duplicate())
           sendInfo = true
         }
       }
 
-      VideoCodec.H265 -> {
+      is H265Packet -> {
         val bufferInfo = videoFormat.getByteBuffer("csd-0")
         if (bufferInfo != null) {
           val byteBufferList = VideoEncoderHelper.extractVpsSpsPpsFromH265(bufferInfo.duplicate())
           if (byteBufferList.size == 3) {
             val sps = byteBufferList[1]
             val pps = byteBufferList[2]
             val vps = byteBufferList[0]
-            videoPacket.sendVideoInfo(sps, pps, vps)
+            (videoPacket as H265Packet).sendVideoInfo(sps, pps, vps)
             sendInfo = true
           }
         }
       }
 
-      else -> {}
+      else -> {
+        Log.e(TAG, "Unsupported codec: ${videoPacket.javaClass.name}")
+      }
     }
     if (sendInfo && recordStatus == RecordController.Status.STARTED) {
       myRequestKeyFrame = null

srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H264Packet.kt

@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2024 pedroSG94.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.pedro.srt.mpeg2ts.packets
+
+import android.util.Log
+import com.pedro.common.VideoCodec
+import com.pedro.common.frame.MediaFrame
+import com.pedro.common.getData
+import com.pedro.common.nal.NalReader
+import com.pedro.common.removeInfo
+import com.pedro.srt.mpeg2ts.MpegTsPacket
+import com.pedro.srt.mpeg2ts.MpegType
+import com.pedro.srt.mpeg2ts.Pes
+import com.pedro.srt.mpeg2ts.PesType
+import com.pedro.srt.mpeg2ts.psi.PsiManager
+import com.pedro.srt.srt.packets.data.PacketPosition
+import com.pedro.srt.utils.chunkPackets
+import com.pedro.srt.utils.getPayload
+import java.nio.ByteBuffer
+
+/**
+ * Created by pedro on 20/8/23.
+ *
+ * Used for H264/H265
+ */
+class H264Packet(
+  limitSize: Int,
+  psiManager: PsiManager,
+): BasePacket(psiManager, limitSize) {
+
+  private val TAG = "H264Packet"
+
+  private var sps: ByteBuffer? = null
+  private var pps: ByteBuffer? = null
+
+  override suspend fun createAndSendPacket(
+    mediaFrame: MediaFrame,
+    callback: suspend (List<MpegTsPacket>) -> Unit
+  ) {
+    val fixedBuffer = mediaFrame.data.removeInfo(mediaFrame.info)
+    val isKeyFrame = mediaFrame.info.isKeyFrame
+    val nals = NalReader.extractNals(fixedBuffer, VideoCodec.H264, false)
+    if (nals.isEmpty()) return
+
+    val sps = this.sps
+    val pps = this.pps
+    if (sps == null || pps == null) {
+      Log.e(TAG, "waiting for a valid video info")
+      return
+    }
+
+    if (isKeyFrame) {
+      if (!nals.contains(pps)) nals.add(0, pps.duplicate())
+      if (!nals.contains(sps)) nals.add(0, sps.duplicate())
+    }
+
+    val payload = nals.getPayload {
+      ByteBuffer.allocate(6).apply {
+        putInt(0x00000001) //annex-b header
+        put(0x09.toByte())
+        put(0xf0.toByte())
+        flip()
+      }
+    }
+    val pes = Pes(psiManager.getVideoPid().toInt(), isKeyFrame, PesType.VIDEO, mediaFrame.info.timestamp, payload)
+    val mpeg2tsPackets = mpegTsPacketizer.write(listOf(pes)).chunkPackets(chunkSize).map { buffer ->
+      MpegTsPacket(buffer, MpegType.VIDEO, PacketPosition.SINGLE, isKeyFrame)
+    }
+    if (mpeg2tsPackets.isNotEmpty()) callback(mpeg2tsPackets)
+  }
+
+  override fun resetPacket(resetInfo: Boolean) {
+    if (resetInfo) {
+      sps = null
+      pps = null
+    }
+  }
+
+  fun sendVideoInfo(sps: ByteBuffer, pps: ByteBuffer) {
+    this.sps = ByteBuffer.wrap(sps.getData())
+    this.pps = ByteBuffer.wrap(pps.getData())
+  }
+}

srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H26XPacket.kt → srt/src/main/java/com/pedro/srt/mpeg2ts/packets/H265Packet.kt

@@ -29,49 +29,57 @@ import com.pedro.srt.mpeg2ts.PesType
 import com.pedro.srt.mpeg2ts.psi.PsiManager
 import com.pedro.srt.srt.packets.data.PacketPosition
 import com.pedro.srt.utils.chunkPackets
+import com.pedro.srt.utils.getPayload
 import java.nio.ByteBuffer
 
 /**
  * Created by pedro on 20/8/23.
  *
  * Used for H264/H265
  */
-class H26XPacket(
+class H265Packet(
   limitSize: Int,
   psiManager: PsiManager,
 ): BasePacket(psiManager, limitSize) {
 
-  private val TAG = "H26XPacket"
+  private val TAG = "H265Packet"
 
   private var sps: ByteBuffer? = null
   private var pps: ByteBuffer? = null
   private var vps: ByteBuffer? = null
-  private var codec = VideoCodec.H264
 
   override suspend fun createAndSendPacket(
     mediaFrame: MediaFrame,
     callback: suspend (List<MpegTsPacket>) -> Unit
   ) {
     val fixedBuffer = mediaFrame.data.removeInfo(mediaFrame.info)
     val isKeyFrame = mediaFrame.info.isKeyFrame
-    val nals = NalReader.extractNals(fixedBuffer, codec, false)
+    val nals = NalReader.extractNals(fixedBuffer, VideoCodec.H265, false)
     if (nals.isEmpty()) return
 
     val sps = this.sps
     val pps = this.pps
     val vps = this.vps
-    if (sps == null || pps == null || (codec == VideoCodec.H265 && vps == null)) {
+    if (sps == null || pps == null || vps == null) {
       Log.e(TAG, "waiting for a valid video info")
       return
     }
 
     if (isKeyFrame) {
       if (!nals.contains(pps)) nals.add(0, pps.duplicate())
       if (!nals.contains(sps)) nals.add(0, sps.duplicate())
-      if (vps != null) if (!nals.contains(vps)) nals.add(0, vps.duplicate())
+      if (!nals.contains(vps)) nals.add(0, vps.duplicate())
     }
 
-    val payload = getPayload(nals, isKeyFrame)
+    val payload = nals.getPayload {
+      ByteBuffer.allocate(7).apply {
+        putInt(0x00000001) //annex-b header
+        put(0x46.toByte())
+        put(0x01.toByte())
+        put(0x50.toByte())
+        flip()
+      }
+    }
     val pes = Pes(psiManager.getVideoPid().toInt(), isKeyFrame, PesType.VIDEO, mediaFrame.info.timestamp, payload)
     val mpeg2tsPackets = mpegTsPacketizer.write(listOf(pes)).chunkPackets(chunkSize).map { buffer ->
       MpegTsPacket(buffer, MpegType.VIDEO, PacketPosition.SINGLE, isKeyFrame)
@@ -87,50 +95,9 @@ class H26XPacket(
     }
   }
 
-  fun setVideoCodec(codec: VideoCodec) {
-    if (codec != VideoCodec.H265 && codec != VideoCodec.H264) {
-      throw IllegalArgumentException("This packet only support H264 and H265")
-    }
-    this.codec = codec
-  }
-
-  fun sendVideoInfo(sps: ByteBuffer, pps: ByteBuffer?, vps: ByteBuffer?) {
+  fun sendVideoInfo(sps: ByteBuffer, pps: ByteBuffer, vps: ByteBuffer) {
     this.sps = ByteBuffer.wrap(sps.getData())
-    this.pps = pps?.let { ByteBuffer.wrap(pps.getData()) }
-    this.vps = vps?.let { ByteBuffer.wrap(vps.getData()) }
-  }
-
-  private fun getPayload(nals: List<ByteBuffer>, isKeyFrame: Boolean): ByteBuffer {
-    val nalsSize = nals.sumOf { it.remaining() }
-    val bufferSize = if (isKeyFrame) {
-      val audSize = when (codec) {
-        VideoCodec.H265 -> 7
-        else -> 6
-      }
-      audSize + nalsSize + (nals.size * 4)
-    } else nalsSize + (nals.size * 4)
-
-    val payload = ByteBuffer.allocate(bufferSize)
-    //add AUD nal
-    if (isKeyFrame) {
-      payload.putInt(0x00000001) //annex-b header
-      when (codec) {
-        VideoCodec.H265 -> {
-          payload.put(0x46.toByte())
-          payload.put(0x01.toByte())
-          payload.put(0x50.toByte())
-        }
-        else -> {
-          payload.put(0x09.toByte())
-          payload.put(0xf0.toByte())
-        }
-      }
-    }
-    nals.forEach {
-      payload.putInt(0x00000001) //annex-b header
-      payload.put(it)
-    }
-    payload.flip()
-    return payload
+    this.pps = ByteBuffer.wrap(pps.getData())
+    this.vps = ByteBuffer.wrap(vps.getData())
   }
 }