diff --git a/common/src/main/java/com/pedro/common/nal/NalReader.kt b/common/src/main/java/com/pedro/common/nal/NalReader.kt index 4d66441f6..c318d01e1 100644 --- a/common/src/main/java/com/pedro/common/nal/NalReader.kt +++ b/common/src/main/java/com/pedro/common/nal/NalReader.kt @@ -52,7 +52,8 @@ object NalReader { val duplicate = buffer.duplicate() duplicate.position(payloadStart - offset) duplicate.limit(limit - offset) - units.add(duplicate.slice()) + val nal = duplicate.slice() + if (shouldKeepNal(nal, codec, shouldDiscardVideoInfo)) units.add(nal) } if (units.isEmpty()) units.add(buffer) return units diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt index bb4536b58..278b461ef 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt @@ -99,7 +99,7 @@ class H265Packet: BasePacket() { val size = nals.sumOf { it.capacity() } buffer = ByteArray(header.size + size + naluSize * nals.size) - val type: Int = nals[0].get(0).toInt().shr(1 and 0x3f) + val type: Int = nals[0].get(0).toInt().shr(1) and 0x3F var nalType = VideoDataType.INTER_FRAME.value if (type == VideoNalType.IDR_N_LP.value || type == VideoNalType.IDR_W_DLP.value || mediaFrame.info.isKeyFrame) { nalType = VideoDataType.KEYFRAME.value diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt index 987e8cf86..932524b2a 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.kt @@ -16,17 +16,14 @@ package com.pedro.rtsp.rtp.packets -import android.util.Log import com.pedro.common.VideoCodec import com.pedro.common.frame.MediaFrame import com.pedro.common.nal.NalReader import com.pedro.common.removeInfo -import com.pedro.common.toByteArray import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants -import com.pedro.rtsp.utils.getVideoStartCodeSize -import java.nio.ByteBuffer import kotlin.experimental.and +import kotlin.experimental.or /** * Created by pedro on 27/11/18. @@ -37,29 +34,15 @@ class H264Packet: BasePacket(RtpConstants.clockVideoFrequency, RtpConstants.payloadType + RtpConstants.trackVideo ) { - private var stapA: ByteArray? = null - private var sendKeyFrame = false - private var videoInfo: Set? = null - private val header = ByteArray(2) - init { channelIdentifier = RtpConstants.trackVideo } - fun sendVideoInfo(sps: ByteBuffer, pps: ByteBuffer) { - videoInfo = setOf(sps, pps) - setSpsPps(sps.toByteArray(), pps.toByteArray()) - } override suspend fun createAndSendPacket( mediaFrame: MediaFrame, callback: suspend (List) -> Unit ) { - val videoInfo = this.videoInfo - if (videoInfo == null) { - Log.e(TAG, "waiting for a valid sps and pps") - return - } val fixedBuffer = mediaFrame.data.removeInfo(mediaFrame.info) // We read a NAL units from ByteBuffer and we send them // NAL units are preceded with 0x00000001 @@ -69,98 +52,55 @@ class H264Packet: BasePacket(RtpConstants.clockVideoFrequency, val ts = mediaFrame.info.timestamp * 1000L val frames = mutableListOf() - val nalType = nals[0].get(0) - val type: Int = (nalType and 0x1F).toInt() - if (type == RtpConstants.IDR || mediaFrame.info.isKeyFrame) { - stapA?.let { - val buffer = getBuffer(it.size + RtpConstants.RTP_HEADER_LENGTH) + nals.forEachIndexed { index, data -> + val nalType = data.get() + val nalSize = data.remaining() + // Small NAL unit => Single NAL unit + if (nalSize <= maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 1) { + val buffer = getBuffer(nalSize + RtpConstants.RTP_HEADER_LENGTH + 1) + buffer[RtpConstants.RTP_HEADER_LENGTH] = nalType + data.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 1, nalSize) val rtpTs = updateTimeStamp(buffer, ts) - markPacket(buffer) //mark end frame - System.arraycopy(it, 0, buffer, RtpConstants.RTP_HEADER_LENGTH, it.size) + if (index == nals.size - 1) markPacket(buffer) //mark end frame updateSeq(buffer) - val rtpFrame = RtpFrame(buffer, rtpTs, it.size + RtpConstants.RTP_HEADER_LENGTH, channelIdentifier) + val rtpFrame = RtpFrame(buffer, rtpTs, buffer.size, channelIdentifier) frames.add(rtpFrame) - sendKeyFrame = true - } ?: run { - Log.i(TAG, "can't create key frame because setSpsPps was not called") - } - } - if (sendKeyFrame) { - nals.forEachIndexed { index, data -> - val nalType = data.get() - val nalSize = data.remaining() - // Small NAL unit => Single NAL unit - if (nalSize <= maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 1) { - val buffer = getBuffer(nalSize + RtpConstants.RTP_HEADER_LENGTH + 1) - buffer[RtpConstants.RTP_HEADER_LENGTH] = nalType - data.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 1, nalSize) - val rtpTs = updateTimeStamp(buffer, ts) - if (index == nals.size - 1) markPacket(buffer) //mark end frame - updateSeq(buffer) - val rtpFrame = RtpFrame(buffer, rtpTs, buffer.size, channelIdentifier) - frames.add(rtpFrame) - } else { - // Set FU-A header - header[1] = nalType and 0x1F // FU header type - header[1] = header[1].plus(0x80).toByte() // set start bit to 1 - // Set FU-A indicator - header[0] = nalType and 0x60 and 0xFF.toByte() // FU indicator NRI - header[0] = header[0].plus(28).toByte() + } else { + // Set FU-A header + val fuHeader = nalType and 0x1F // FU header type + // Set FU-A indicator + val fuIndicator = (nalType and 0x60).plus(28).toByte() // FU indicator NRI - var sum = 0 - while (sum < nalSize) { - val length = if (nalSize - sum > maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 2) { - maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 2 - } else { - data.remaining() - } - val buffer = getBuffer(length + RtpConstants.RTP_HEADER_LENGTH + 2) - buffer[RtpConstants.RTP_HEADER_LENGTH] = header[0] - // Switch start bit - buffer[RtpConstants.RTP_HEADER_LENGTH + 1] = if (sum > 0) header[1] and 0x7F else header[1] - val rtpTs = updateTimeStamp(buffer, ts) - data.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 2, length) - sum += length - // Last packet before next NAL - if (sum >= nalSize) { - // End bit on - buffer[RtpConstants.RTP_HEADER_LENGTH + 1] = buffer[RtpConstants.RTP_HEADER_LENGTH + 1].plus(0x40).toByte() - if (index == nals.size - 1) markPacket(buffer) //mark end frame - } - updateSeq(buffer) - val rtpFrame = RtpFrame(buffer, rtpTs, buffer.size, channelIdentifier) - frames.add(rtpFrame) - } + var sum = 0 + while (sum < nalSize) { + val length = if (nalSize - sum > maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 2) { + maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 2 + } else { + data.remaining() } + val buffer = getBuffer(length + RtpConstants.RTP_HEADER_LENGTH + 2) + buffer[RtpConstants.RTP_HEADER_LENGTH] = fuIndicator + // Switch start bit + buffer[RtpConstants.RTP_HEADER_LENGTH + 1] = if (sum > 0) fuHeader else (fuHeader or 0x80.toByte()) + val rtpTs = updateTimeStamp(buffer, ts) + data.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 2, length) + sum += length + // Last packet before next NAL + if (sum >= nalSize) { + // End bit on + buffer[RtpConstants.RTP_HEADER_LENGTH + 1] = buffer[RtpConstants.RTP_HEADER_LENGTH + 1].plus(0x40).toByte() + if (index == nals.size - 1) markPacket(buffer) //mark end frame + } + updateSeq(buffer) + val rtpFrame = RtpFrame(buffer, rtpTs, buffer.size, channelIdentifier) + frames.add(rtpFrame) + } } - } else { - Log.i(TAG, "waiting for keyframe") } if (frames.isNotEmpty()) callback(frames) } - private fun setSpsPps(sps: ByteArray, pps: ByteArray) { - stapA = ByteArray(sps.size + pps.size + 5) - stapA?.let { - // STAP-A NAL header is 24 - it[0] = 24 - - // Write NALU 1 size into the array (NALU 1 is the SPS). - it[1] = (sps.size shr 8).toByte() - it[2] = (sps.size and 0xFF).toByte() - - // Write NALU 2 size into the array (NALU 2 is the PPS). - it[sps.size + 3] = (pps.size shr 8).toByte() - it[sps.size + 4] = (pps.size and 0xFF).toByte() - - // Write NALU 1 into the array, then write NALU 2 into the array. - System.arraycopy(sps, 0, it, 3, sps.size) - System.arraycopy(pps, 0, it, 5 + sps.size, pps.size) - } - } - override fun reset() { super.reset() - sendKeyFrame = false } } \ No newline at end of file diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt index 6932cad2d..4b1444555 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H265Packet.kt @@ -16,15 +16,13 @@ package com.pedro.rtsp.rtp.packets -import android.util.Log import com.pedro.common.VideoCodec import com.pedro.common.frame.MediaFrame import com.pedro.common.nal.NalReader import com.pedro.common.removeInfo import com.pedro.rtsp.rtsp.RtpFrame import com.pedro.rtsp.utils.RtpConstants -import java.nio.ByteBuffer -import kotlin.experimental.and +import kotlin.experimental.or /** * Created by pedro on 28/11/18. @@ -36,30 +34,18 @@ class H265Packet : BasePacket( RtpConstants.payloadType + RtpConstants.trackVideo ) { - private var videoInfo: Set? = null - private val header = ByteArray(3) - init { channelIdentifier = RtpConstants.trackVideo } - fun sendVideoInfo(sps: ByteBuffer, pps: ByteBuffer, vps: ByteBuffer) { - videoInfo = setOf(sps, pps, vps) - } - override suspend fun createAndSendPacket( mediaFrame: MediaFrame, callback: suspend (List) -> Unit ) { - val videoInfo = this.videoInfo - if (videoInfo == null) { - Log.e(TAG, "waiting for a valid sps, pps and vps") - return - } val fixedBuffer = mediaFrame.data.removeInfo(mediaFrame.info) // We read a NAL units from ByteBuffer and we send them // NAL units are preceded with 0x00000001 - val nals = NalReader.extractNals(fixedBuffer, VideoCodec.H265, true) + val nals = NalReader.extractNals(fixedBuffer, VideoCodec.H265, false) if (nals.isEmpty()) return val ts = mediaFrame.info.timestamp * 1000L @@ -82,18 +68,14 @@ class H265Packet : BasePacket( val rtpFrame = RtpFrame(buffer, rtpTs, buffer.size, channelIdentifier) frames.add(rtpFrame) } else { - val type: Int = nalType.toInt().shr(1 and 0x3f) - //Set PayloadHdr (16bit type=49) - header[0] = (49 shl 1).toByte() - header[1] = 1 // Set FU header // +---------------+ // |0|1|2|3|4|5|6|7| // +-+-+-+-+-+-+-+-+ // |S|E| FuType | // +---------------+ - header[2] = type.toByte() // FU header type - header[2] = header[2].plus(0x80).toByte() // Start bit + val type: Int = nalType.toInt().shr(1) and 0x3F + val fuHeader = type.toByte() // FU header type var sum = 0 while (sum < nalSize) { @@ -103,10 +85,11 @@ class H265Packet : BasePacket( data.remaining() } val buffer = getBuffer(length + RtpConstants.RTP_HEADER_LENGTH + 3) - buffer[RtpConstants.RTP_HEADER_LENGTH] = header[0] - buffer[RtpConstants.RTP_HEADER_LENGTH + 1] = header[1] + //Set PayloadHdr (16bit type=49) + buffer[RtpConstants.RTP_HEADER_LENGTH] = (49 shl 1).toByte() + buffer[RtpConstants.RTP_HEADER_LENGTH + 1] = 1 // Switch start bit - buffer[RtpConstants.RTP_HEADER_LENGTH + 2] = if (sum > 0) header[2] and 0x7F else header[2] + buffer[RtpConstants.RTP_HEADER_LENGTH + 2] = if (sum > 0) fuHeader else (fuHeader or 0x80.toByte()) val rtpTs = updateTimeStamp(buffer, ts) data.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 3, length) sum += length diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt index 873188bfd..b3b63ed2a 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt +++ b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspSender.kt @@ -78,11 +78,11 @@ class RtspSender( videoPacket = when (commandsManager.videoCodec) { VideoCodec.H264 -> { if (pps == null) throw IllegalArgumentException("pps can't be null with h264") - H264Packet().apply { sendVideoInfo(sps, pps) } + H264Packet() } VideoCodec.H265 -> { if (vps == null || pps == null) throw IllegalArgumentException("pps or vps can't be null with h265") - H265Packet().apply { sendVideoInfo(sps, pps, vps) } + H265Packet() } VideoCodec.AV1 -> Av1Packet() } diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt index 0231f5d92..9f4c08203 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/H264PacketTest.kt @@ -39,25 +39,20 @@ class H264PacketTest { val fakeH264 = header.plus(ByteArray(300) { 0x00 }) val info = MediaFrame.Info(0, fakeH264.size, timestamp, true) - val fakeSps = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04) - val fakePps = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x0A, 0x0B, 0x0C) val mediaFrame = MediaFrame(ByteBuffer.wrap(fakeH264), info, MediaFrame.Type.VIDEO) - val h264Packet = H264Packet().apply { sendVideoInfo(ByteBuffer.wrap(fakeSps), ByteBuffer.wrap(fakePps)) } + val h264Packet = H264Packet() h264Packet.setSSRC(123456789) val frames = mutableListOf() h264Packet.createAndSendPacket(mediaFrame) { frames.addAll(it) } - val expectedRtp = byteArrayOf(-128, -32, 0, 2, 0, -87, -118, -57, 7, 91, -51, 21, 5).plus(fakeH264.copyOfRange(header.size, fakeH264.size)) - val expectedStapA = byteArrayOf(-128, -32, 0, 1, 0, -87, -118, -57, 7, 91, -51, 21, 24, 0, 7, 0, 0, 0, 1, 2, 3, 4, 0, 7, 0, 0, 0, 1, 10, 11, 12) + val expectedRtp = byteArrayOf(-128, -32, 0, 1, 0, -87, -118, -57, 7, 91, -51, 21, 5).plus(fakeH264.copyOfRange(header.size, fakeH264.size)) val expectedTimeStamp = 11111111L val expectedSize = RtpConstants.RTP_HEADER_LENGTH + 1 + info.size - header.size - val expectedStapAResult = RtpFrame(expectedStapA, expectedTimeStamp, fakePps.size + fakePps.size + 5 + RtpConstants.RTP_HEADER_LENGTH, RtpConstants.trackVideo) val expectedPacketResult = RtpFrame(expectedRtp, expectedTimeStamp, expectedSize, RtpConstants.trackVideo) assertNotEquals(0, frames.size) - assertTrue(frames.size == 2) - assertEquals(expectedStapAResult, frames[0]) - assertEquals(expectedPacketResult, frames[1]) + assertTrue(frames.size == 1) + assertEquals(expectedPacketResult, frames[0]) } @Test @@ -67,10 +62,8 @@ class H264PacketTest { val fakeH264 = header.plus(ByteArray(2500) { 0x00 }) val info = MediaFrame.Info(0, fakeH264.size, timestamp, true) - val fakeSps = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04) - val fakePps = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x0A, 0x0B, 0x0C) val mediaFrame = MediaFrame(ByteBuffer.wrap(fakeH264), info, MediaFrame.Type.VIDEO) - val h264Packet = H264Packet().apply { sendVideoInfo(ByteBuffer.wrap(fakeSps), ByteBuffer.wrap(fakePps)) } + val h264Packet = H264Packet() h264Packet.setSSRC(123456789) val frames = mutableListOf() h264Packet.createAndSendPacket(mediaFrame) { frames.addAll(it) } @@ -78,22 +71,19 @@ class H264PacketTest { val packet1Size = RtpConstants.MTU - 28 - RtpConstants.RTP_HEADER_LENGTH - 2 val chunk1 = fakeH264.copyOfRange(header.size, header.size + packet1Size) val chunk2 = fakeH264.copyOfRange(header.size + packet1Size, fakeH264.size) - val expectedRtp = byteArrayOf(-128, 96, 0, 2, 0, -87, -118, -57, 7, 91, -51, 21, 28, -123).plus(chunk1) - val expectedRtp2 = byteArrayOf(-128, -32, 0, 3, 0, -87, -118, -57, 7, 91, -51, 21, 28, 69).plus(chunk2) + val expectedRtp = byteArrayOf(-128, 96, 0, 1, 0, -87, -118, -57, 7, 91, -51, 21, 28, -123).plus(chunk1) + val expectedRtp2 = byteArrayOf(-128, -32, 0, 2, 0, -87, -118, -57, 7, 91, -51, 21, 28, 69).plus(chunk2) - val expectedStapA = byteArrayOf(-128, -32, 0, 1, 0, -87, -118, -57, 7, 91, -51, 21, 24, 0, 7, 0, 0, 0, 1, 2, 3, 4, 0, 7, 0, 0, 0, 1, 10, 11, 12) val expectedTimeStamp = 11111111L val expectedSize = chunk1.size + RtpConstants.RTP_HEADER_LENGTH + 2 val expectedSize2 = chunk2.size + RtpConstants.RTP_HEADER_LENGTH + 2 - val expectedStapAResult = RtpFrame(expectedStapA, expectedTimeStamp, fakePps.size + fakePps.size + 5 + RtpConstants.RTP_HEADER_LENGTH, RtpConstants.trackVideo) val expectedPacketResult = RtpFrame(expectedRtp, expectedTimeStamp, expectedSize, RtpConstants.trackVideo) val expectedPacketResult2 = RtpFrame(expectedRtp2, expectedTimeStamp, expectedSize2, RtpConstants.trackVideo) assertNotEquals(0, frames.size) - assertTrue(frames.size == 3) - assertEquals(expectedStapAResult, frames[0]) - assertEquals(expectedPacketResult, frames[1]) - assertEquals(expectedPacketResult2, frames[2]) + assertTrue(frames.size == 2) + assertEquals(expectedPacketResult, frames[0]) + assertEquals(expectedPacketResult2, frames[1]) } } \ No newline at end of file diff --git a/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt b/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt index 3bd698d6f..3e40a2719 100644 --- a/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt +++ b/rtsp/src/test/java/com/pedro/rtsp/rtp/H265PacketTest.kt @@ -37,13 +37,10 @@ class H265PacketTest { val timestamp = 123456789L val header = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x05, 0x00) val fakeH265 = header.plus(ByteArray(300) { 0x00 }) - val fakeSps = ByteBuffer.wrap(byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04)) - val fakePps = ByteBuffer.wrap(byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x0A, 0x0B, 0x0C)) - val fakeVps = ByteBuffer.wrap(byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x0D, 0x0E, 0x0F)) val info = MediaFrame.Info(0, fakeH265.size, timestamp, true) val mediaFrame = MediaFrame(ByteBuffer.wrap(fakeH265), info, MediaFrame.Type.VIDEO) - val h265Packet = H265Packet().apply { sendVideoInfo(fakeSps, fakePps, fakeVps) } + val h265Packet = H265Packet() h265Packet.setSSRC(123456789) val frames = mutableListOf() h265Packet.createAndSendPacket(mediaFrame) { frames.addAll(it) } @@ -63,13 +60,10 @@ class H265PacketTest { val timestamp = 123456789L val header = byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x05, 0x00) val fakeH265 = header.plus(ByteArray(2500) { 0x00 }) - val fakeSps = ByteBuffer.wrap(byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04)) - val fakePps = ByteBuffer.wrap(byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x0A, 0x0B, 0x0C)) - val fakeVps = ByteBuffer.wrap(byteArrayOf(0x00, 0x00, 0x00, 0x01, 0x0D, 0x0E, 0x0F)) val info = MediaFrame.Info(0, fakeH265.size, timestamp, true) val mediaFrame = MediaFrame(ByteBuffer.wrap(fakeH265), info, MediaFrame.Type.VIDEO) - val h265Packet = H265Packet().apply { sendVideoInfo(fakeSps, fakePps, fakeVps) } + val h265Packet = H265Packet() h265Packet.setSSRC(123456789) val frames = mutableListOf() h265Packet.createAndSendPacket(mediaFrame) { frames.addAll(it) }