​技术背景

我们在做Android平台RTSP、RTMP播放器的时候，经常遇到这样的技术诉求，开发者希望拿到播放器解码后的YUV或RGB数据，投递给视觉算法，做AI分析，本文以ffmpeg和大牛直播SDK的SmartPlayer为例，介绍下相关的技术实现。

FFmpeg

FFmpeg 是一个开源的跨平台多媒体处理工具库，广泛应用于音视频处理领域。

格式转换：

编码与解码：

视频处理：

音频处理：

流媒体处理：

集成 FFmpeg

利用 FFmpeg 解码视频并获取 YUV 数据

SmartPlayer

SmartPlayer是大牛直播SDK旗下全自研内核，行业内一致认可的跨平台RTSP、RTMP直播播放器SDK，功能齐全、高稳定、超低延迟，超低资源占用，适用于安防、教育、单兵指挥等行业。

功能设计如下：

播放之前，设置YUV数据回调：

/*
 * SmartPlayer.java
 * Copyright © 2014~2024 daniusdk.com All rights reserved.
 */
private boolean StartPlay()
{
	if(isPlaying)
		return false;
	if(!isPulling)
	{
		if (!OpenPullHandle())
			return false;
	}
	// 如果第二个参数设置为null，则播放纯音频
	libPlayer.SmartPlayerSetSurface(player_handle_, sSurfaceView);
	//libPlayer.SmartPlayerSetSurface(player_handle_, null);
	libPlayer.SmartPlayerSetRenderScaleMode(player_handle_, 1);
	if(video_opt_ == 3)
	{
		libPlayer.SmartPlayerSetExternalRender(player_handle_, new I420ExternalRender(publisher_array_));
	}
	//libPlayer.SmartPlayerSetExternalAudioOutput(player_handle_, new PlayerExternalPCMOutput(stream_publisher_));
	libPlayer.SmartPlayerSetFastStartup(player_handle_, isFastStartup ? 1 : 0);
	libPlayer.SmartPlayerSetAudioOutputType(player_handle_, 1);
	if (isMute) {
		libPlayer.SmartPlayerSetMute(player_handle_, isMute ? 1	: 0);
	}
	if (isHardwareDecoder)
	{
		int isSupportH264HwDecoder = libPlayer.SetSmartPlayerVideoHWDecoder(player_handle_, 1);
		int isSupportHevcHwDecoder = libPlayer.SetSmartPlayerVideoHevcHWDecoder(player_handle_, 1);
		Log.i(TAG, "isSupportH264HwDecoder: " + isSupportH264HwDecoder + ", isSupportHevcHwDecoder: " + isSupportHevcHwDecoder);
	}
	libPlayer.SmartPlayerSetLowLatencyMode(player_handle_, isLowLatency ? 1	: 0);
	libPlayer.SmartPlayerSetRotation(player_handle_, rotate_degrees);
	int iPlaybackRet = libPlayer.SmartPlayerStartPlay(player_handle_);
	if (iPlaybackRet != 0 && !isPulling) {
		Log.e(TAG, "StartPlay failed!");
		releasePlayerHandle();
		return false;
	}
	isPlaying = true;
	return true;
}

YUV数据回调上层实现：

private static class I420ExternalRender implements NTExternalRender {
	// public static final int NT_FRAME_FORMAT_RGBA = 1;
	// public static final int NT_FRAME_FORMAT_ABGR = 2;
	// public static final int NT_FRAME_FORMAT_I420 = 3;
	private WeakReference publisher_;
	private ArrayList > publisher_list_;
	private int width_;
	private int height_;
	private int y_row_bytes_;
	private int u_row_bytes_;
	private int v_row_bytes_;
	private ByteBuffer y_buffer_;
	private ByteBuffer u_buffer_;
	private ByteBuffer v_buffer_;
	public I420ExternalRender(LibPublisherWrapper publisher) {
		if (publisher != null)
			publisher_ = new WeakReference<>(publisher);
	}
	public I420ExternalRender(LibPublisherWrapper[] publisher_list) {
		if (publisher_list != null && publisher_list.length > 0) {
			for (LibPublisherWrapper i : publisher_list) {
				if (i != null) {
					if (null == publisher_list_)
						publisher_list_ = new ArrayList<>();
					publisher_list_.add(new WeakReference<>(i));
				}
			}
		}
	}
	private final List get_publisher_list() {
		if (null == publisher_list_ || publisher_list_.isEmpty())
			return null;
		ArrayList list = new ArrayList<>(publisher_list_.size());
		for (WeakReference i : publisher_list_) {
			if (i != null) {
				LibPublisherWrapper o = i.get();
				if (o != null && !o.empty())
					list.add(o);
			}
		}
		return list;
	}
	@Override
	public int getNTFrameFormat() {
		Log.i(TAG, "I420ExternalRender::getNTFrameFormat return "
				+ NT_FRAME_FORMAT_I420);
		return NT_FRAME_FORMAT_I420;
	}
	private static int align(int d, int a) { return (d + (a - 1)) & ~(a - 1); }
	@Override
	public void onNTFrameSizeChanged(int width, int height) {
		width_ = width;
		height_ = height;
		y_row_bytes_ = width;
		u_row_bytes_ = (width+1)/2;
		v_row_bytes_ = (width+1)/2;
		y_buffer_ = ByteBuffer.allocateDirect(y_row_bytes_*height_);
		u_buffer_ = ByteBuffer.allocateDirect(u_row_bytes_*((height_ + 1) / 2));
		v_buffer_ = ByteBuffer.allocateDirect(v_row_bytes_*((height_ + 1) / 2));
	}
	@Override
	public ByteBuffer getNTPlaneByteBuffer(int index) {
		switch (index) {
			case 0:
				return y_buffer_;
			case 1:
				return u_buffer_;
			case 2:
				return v_buffer_;
			default:
				Log.e(TAG, "I420ExternalRender::getNTPlaneByteBuffer index error:" + index);
				return null;
		}
	}
	@Override
	public int getNTPlanePerRowBytes(int index) {
		switch (index) {
			case 0:
				return y_row_bytes_;
			case 1:
				return u_row_bytes_;
			case 2:
				return v_row_bytes_;
			default:
				Log.e(TAG, "I420ExternalRender::getNTPlanePerRowBytes index error:" + index);
				return 0;
		}
	}
	public void onNTRenderFrame(int width, int height, long timestamp)
	{
		if (null == y_buffer_ || null == u_buffer_ || null == v_buffer_)
			return;
		Log.i(TAG, "I420ExternalRender::onNTRenderFrame " + width + "*" + height + ", t:" + timestamp);
		y_buffer_.rewind();
		u_buffer_.rewind();
		v_buffer_.rewind();
		List publisher_list = get_publisher_list();
		if (null == publisher_list || publisher_list.isEmpty())
			return;
		for (LibPublisherWrapper i : publisher_list) {
			i.PostLayerImageI420ByteBuffer(0, 0, 0,
					y_buffer_, 0, y_row_bytes_,
					u_buffer_, 0, u_row_bytes_,
					v_buffer_, 0, v_row_bytes_,
					width_, height_, 0, 0,
					0,0, 0,0);
		}
	}
}

总结

Android平台RTSP、RTMP播放器回调yuv数据，意义非常重大，既保证了低延迟传输解码，又可以通过回调解码后数据，高效率的投递给AI算法，实现视觉处理。ffmpeg实现还是SmartPlayer，各有利弊，感兴趣的开发者，可以单独跟我探讨。

​