591 lines
19 KiB
C#
591 lines
19 KiB
C#
/*----------------------------------------------------------------
|
||
// Copyright (C) 2025 Beijing All rights reserved.
|
||
//
|
||
// Author: huachangmiao
|
||
// Create Date: 2025/04/11
|
||
// Module Describe:
|
||
//----------------------------------------------------------------*/
|
||
|
||
//******************************************************************
|
||
// CPU软解码
|
||
// 适用平台
|
||
// Editor
|
||
// Windows
|
||
// MacOS
|
||
// Android
|
||
// iOS
|
||
// WebGL
|
||
//******************************************************************
|
||
|
||
// #define ENABLE_COPY_NATIVE_DATA //复制Native数据到托管内存
|
||
|
||
#if UNITY_WEBGL
|
||
#else
|
||
#define ENABLE_AV_MALLOC //使用AV_MALLOC分配内存
|
||
#endif
|
||
|
||
using System;
|
||
using System.Threading;
|
||
using LeviathanVideo.Abstractions;
|
||
using LeviathanVideo.Bindings.Linked;
|
||
using Unity.Collections;
|
||
using Unity.Collections.LowLevel.Unsafe;
|
||
using UnityEngine;
|
||
using System.Runtime.InteropServices;
|
||
using AOT;
|
||
|
||
internal unsafe class LeviathanSoftwareDecoder : ILeviathanDecoder
|
||
{
|
||
// 日志辅助方法
|
||
private static void Log(string message)
|
||
{
|
||
LeviathanLogConfig.Log("SoftwareDecoder", message);
|
||
}
|
||
|
||
private static void LogWarning(string message)
|
||
{
|
||
LeviathanLogConfig.LogWarning("SoftwareDecoder", message);
|
||
}
|
||
|
||
private static void LogError(string message)
|
||
{
|
||
LeviathanLogConfig.LogError("SoftwareDecoder", message);
|
||
}
|
||
|
||
// 事件实现
|
||
public event Action OnFirstFrameDecoded;
|
||
private AVFormatContext* _pFormatContext;
|
||
private AVCodecContext* _pVideoContext;
|
||
|
||
private int _videoStreamIndex;
|
||
private AVFrame* _videoFrame;
|
||
private AVPacket* _packet;
|
||
|
||
private AVIOContext* _pAvioCtx;
|
||
private byte* _avioCtxBuffer;
|
||
private BufferData _bufferData;
|
||
private GCHandle _bdHandle;
|
||
|
||
#if ENABLE_AV_MALLOC
|
||
#else
|
||
private GCHandle _hAvioBuffer;
|
||
#endif
|
||
#if ENABLE_COPY_NATIVE_DATA
|
||
private IntPtr _unmanagedPtr;
|
||
#endif
|
||
|
||
// 私有字段
|
||
private long _frameCount;
|
||
private double _frameInterval;
|
||
private string _codecName = "";
|
||
private long _duration;
|
||
private long _decodedFrameIndex;
|
||
private bool _firstFrameDecoded = false;
|
||
|
||
// 线程安全锁 - 保护 Seek 和 Decode 操作不并发执行
|
||
private readonly object _codecLock = new object();
|
||
|
||
// 实现接口属性
|
||
public long FrameCount => _frameCount;
|
||
public double FrameInterval => _frameInterval;
|
||
public string CodecName => _codecName;
|
||
public long Duration => _duration;
|
||
public long DecodedFrameIndex => _decodedFrameIndex;
|
||
public AVFrame* VideoFrame => _videoFrame;
|
||
|
||
// 循环播放属性（软件解码器不需要特殊处理）
|
||
public bool IsLooping { get; set; } = false;
|
||
|
||
// 快速解码模式（跳过环路滤波器）
|
||
private bool _fastDecode = true;
|
||
|
||
|
||
[StructLayout(LayoutKind.Sequential)]
|
||
private struct BufferData
|
||
{
|
||
public byte* ptr;
|
||
public byte* originPtr;
|
||
public ulong size;
|
||
public ulong totalSize;
|
||
}
|
||
|
||
[MonoPInvokeCallback(typeof(avio_alloc_context_read_packet))]
|
||
private static int ReadPacket(void* opaque, byte* buf, int bufSize)
|
||
{
|
||
BufferData* bd = (BufferData*)opaque;
|
||
bufSize = (int)Math.Min(bd->size, (ulong)bufSize);
|
||
|
||
if (bufSize <= 0)
|
||
return leviathan.AVERROR_EOF;
|
||
|
||
// Buffer.MemoryCopy(bd->ptr, buf, bufSize, bufSize);
|
||
UnsafeUtility.MemCpy(buf, bd->ptr, bufSize);
|
||
bd->ptr += bufSize;
|
||
bd->size -= (ulong)bufSize;
|
||
|
||
return bufSize;
|
||
}
|
||
|
||
[MonoPInvokeCallback(typeof(avio_alloc_context_seek))]
|
||
private static long SeekPacket(void* opaque, long offset, int whence)
|
||
{
|
||
BufferData* bd = (BufferData*)opaque;
|
||
switch (whence)
|
||
{
|
||
case 0: //SEEK_SET
|
||
if (offset < 0 || (ulong)offset > bd->totalSize)
|
||
return leviathan.AVERROR_Enum(leviathan.EINVAL);
|
||
bd->ptr = bd->originPtr + offset;
|
||
bd->size = bd->totalSize - (ulong)offset;
|
||
return offset;
|
||
|
||
case 1: //SEEK_CUR
|
||
long newPos = bd->ptr - bd->originPtr + offset;
|
||
if (newPos < 0 || (ulong)newPos > bd->totalSize)
|
||
return leviathan.AVERROR_Enum(leviathan.EINVAL);
|
||
bd->ptr += offset;
|
||
bd->size = bd->totalSize - (ulong)newPos;
|
||
return newPos;
|
||
|
||
case 2: //SEEK_END
|
||
long endPos = (long)bd->totalSize + offset;
|
||
if (endPos < 0 || endPos > (long)bd->totalSize)
|
||
return leviathan.AVERROR_Enum(leviathan.EINVAL);
|
||
bd->ptr = bd->originPtr + endPos;
|
||
bd->size = bd->totalSize - (ulong)endPos;
|
||
return endPos;
|
||
|
||
default:
|
||
return leviathan.AVERROR_Enum(leviathan.EINVAL);
|
||
}
|
||
}
|
||
|
||
|
||
public int Init(NativeArray<byte> fileData, int beginFrameIndex = 1, bool syncDecodeFirstFrame = true, bool fastDecode = true)
|
||
{
|
||
LinkedBindings.Initialize();
|
||
_fastDecode = fastDecode;
|
||
var ret = 0;
|
||
var src = fileData.GetUnsafeReadOnlyPtr();
|
||
|
||
#if ENABLE_COPY_NATIVE_DATA
|
||
_unmanagedPtr = Marshal.AllocHGlobal(fileData.Length);
|
||
var dst = _unmanagedPtr.ToPointer();
|
||
UnsafeUtility.MemCpy(dst, src, fileData.Length);
|
||
src = dst;
|
||
#endif
|
||
|
||
ulong bufferSize = (ulong)fileData.Length;
|
||
|
||
_bufferData = new BufferData
|
||
{
|
||
originPtr = (byte*)src, // 初始化原始指针
|
||
ptr = (byte*)src,
|
||
size = bufferSize,
|
||
totalSize = bufferSize
|
||
};
|
||
_bdHandle = GCHandle.Alloc(_bufferData, GCHandleType.Pinned);
|
||
|
||
avio_alloc_context_read_packet readCallback = ReadPacket;
|
||
avio_alloc_context_seek seekCallback = SeekPacket;
|
||
|
||
// 为AVIOContext分配缓冲区
|
||
const int avioCtxBufferSize = 4096;
|
||
#if ENABLE_AV_MALLOC
|
||
_avioCtxBuffer = (byte*)leviathan.av_malloc(avioCtxBufferSize);
|
||
|
||
// 分配AVIOContext并将其与AVFormatContext关联
|
||
_pAvioCtx = leviathan.avio_alloc_context(
|
||
_avioCtxBuffer,
|
||
avioCtxBufferSize,
|
||
0, // 不可写
|
||
(void*)_bdHandle.AddrOfPinnedObject(),
|
||
readCallback,
|
||
null, // 无写回调
|
||
seekCallback
|
||
);
|
||
#else
|
||
// wasm无法直接使用av_malloc
|
||
int alignment = 64;
|
||
byte[] avioBufferManaged = new byte[avioCtxBufferSize + alignment - 1];
|
||
_hAvioBuffer = GCHandle.Alloc(avioBufferManaged, GCHandleType.Pinned);
|
||
IntPtr avioBufferPtr = _hAvioBuffer.AddrOfPinnedObject();
|
||
// 调整指针到对齐地址
|
||
IntPtr alignedPtr = new IntPtr((avioBufferPtr.ToInt64() + alignment - 1) & ~(alignment - 1));
|
||
|
||
// 分配AVIOContext并将其与AVFormatContext关联
|
||
_pAvioCtx = leviathan.avio_alloc_context(
|
||
(byte*)alignedPtr,
|
||
avioCtxBufferSize,
|
||
0, // 不可写
|
||
(void*)_bdHandle.AddrOfPinnedObject(),
|
||
readCallback,
|
||
null, // 无写回调
|
||
seekCallback
|
||
);
|
||
#endif
|
||
if (_pAvioCtx == null)
|
||
{
|
||
ret = leviathan.AVERROR_Enum(leviathan.ENOMEM);
|
||
LogError("无法分配AVIO上下文");
|
||
return ret;
|
||
}
|
||
|
||
_pFormatContext = leviathan.avformat_alloc_context();
|
||
_pFormatContext->pb = _pAvioCtx;
|
||
var pFormatContext = _pFormatContext;
|
||
ret = leviathan.avformat_open_input(&pFormatContext, null, null, null);
|
||
if (ret != 0)
|
||
{
|
||
LogError($"avformat_open_input failed: {ret}");
|
||
return ret;
|
||
}
|
||
if (_pFormatContext == null)
|
||
{
|
||
return ret;
|
||
}
|
||
|
||
return InternalPlay(beginFrameIndex, syncDecodeFirstFrame);
|
||
}
|
||
|
||
public int InternalPlay(int beginFrameIndex = 1, bool syncDecodeFirstFrame = true)
|
||
{
|
||
leviathan.avformat_find_stream_info(_pFormatContext, null);
|
||
AVCodec* videoCodec = null;
|
||
_videoStreamIndex = leviathan.av_find_best_stream(_pFormatContext, AVMediaType.AVMEDIA_TYPE_VIDEO, -1, -1, &videoCodec, 0);
|
||
// 获取视频流信息
|
||
AVStream* videoStream = _pFormatContext->streams[_videoStreamIndex];
|
||
// 计算帧间隔
|
||
AVRational frameRate = videoStream->r_frame_rate;
|
||
double fps = leviathan.av_q2d(frameRate);
|
||
_frameInterval = 1.0 / fps;
|
||
|
||
_pVideoContext = leviathan.avcodec_alloc_context3(videoCodec);
|
||
|
||
#if UNITY_EDITOR
|
||
_codecName = leviathan.avcodec_get_name(videoCodec->id);
|
||
#endif
|
||
if (videoCodec == null)
|
||
{
|
||
LogError("videoCodec is null");
|
||
return 1;
|
||
}
|
||
_pVideoContext->thread_count = 0;
|
||
_pVideoContext->thread_type = leviathan.FF_THREAD_FRAME | leviathan.FF_THREAD_SLICE;
|
||
|
||
// 根据 fastDecode 参数决定是否跳过环路滤波器以提升解码速度
|
||
if (_fastDecode)
|
||
{
|
||
_pVideoContext->skip_loop_filter = (AVDiscard)48; //AVDISCARD_ALL
|
||
}
|
||
|
||
// Debug.Log($"{_pVideoContext->codec_id} {_frameInterval} {_pFormatContext->duration}");
|
||
leviathan.avcodec_open2(_pVideoContext, videoCodec, null);
|
||
|
||
_duration = _pFormatContext->duration;
|
||
|
||
if (_pFormatContext->streams[_videoStreamIndex]->nb_frames > 0)
|
||
{
|
||
_frameCount = _pFormatContext->streams[_videoStreamIndex]->nb_frames;
|
||
}
|
||
else
|
||
{
|
||
var dur = _duration / 1000.0 * leviathan.av_q2d(videoStream->time_base);
|
||
int frNum = frameRate.num;
|
||
int frDen = frameRate.den;
|
||
if (dur > 0 && frNum > 0 && frDen > 0)
|
||
{
|
||
_frameCount = (long)(dur * frNum / frDen);
|
||
}
|
||
}
|
||
// Debug.Log($"frameCount: {_frameCount}");
|
||
|
||
_videoFrame = leviathan.av_frame_alloc();
|
||
_packet = leviathan.av_packet_alloc();
|
||
|
||
if (beginFrameIndex > 1)
|
||
{
|
||
SeekToFrame(beginFrameIndex);
|
||
}
|
||
|
||
if (syncDecodeFirstFrame)
|
||
{
|
||
// 第一帧同步解码
|
||
int ret = DecodeNextFrame();
|
||
if (ret != 0)
|
||
{
|
||
LogError($"DecodeNextFrame ret: {ret}");
|
||
}
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
|
||
public int DecodeNextFrame()
|
||
{
|
||
lock (_codecLock)
|
||
{
|
||
if (_pVideoContext == null || _pFormatContext == null)
|
||
return -1;
|
||
|
||
return DecodeNextFrameInternal();
|
||
}
|
||
}
|
||
|
||
private int DecodeNextFrameInternal()
|
||
{
|
||
var flushSent = false;
|
||
while (true)
|
||
{
|
||
var error = leviathan.avcodec_receive_frame(_pVideoContext, _videoFrame);
|
||
if (error == 0)
|
||
{
|
||
long frameIndex = 1;
|
||
if (_frameInterval > 0)
|
||
{
|
||
var timeBase = _pFormatContext->streams[_videoStreamIndex]->time_base;
|
||
double frameTime = _videoFrame->pts * leviathan.av_q2d(timeBase);
|
||
frameIndex = (long)(frameTime / _frameInterval + 0.1) + 1;
|
||
}
|
||
Interlocked.Exchange(ref _decodedFrameIndex, frameIndex);
|
||
|
||
// 触发第一帧回调
|
||
if (!_firstFrameDecoded)
|
||
{
|
||
_firstFrameDecoded = true;
|
||
OnFirstFrameDecoded?.Invoke();
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
if (error == leviathan.AVERROR_Enum(leviathan.EAGAIN))
|
||
{
|
||
if (flushSent)
|
||
{
|
||
// 已发送过flush，无更多数据
|
||
return 1;
|
||
}
|
||
|
||
error = leviathan.av_read_frame(_pFormatContext, _packet);
|
||
if (error < 0)
|
||
{
|
||
if (error == leviathan.AVERROR_EOF)
|
||
{
|
||
// 发送flush packet以取出解码器中剩余帧
|
||
error = leviathan.avcodec_send_packet(_pVideoContext, null);
|
||
flushSent = true;
|
||
if (error < 0 && error != leviathan.AVERROR_Enum(leviathan.EAGAIN))
|
||
{
|
||
return 2;
|
||
}
|
||
continue;
|
||
}
|
||
else
|
||
{
|
||
// 其他错误，继续尝试读取
|
||
continue;
|
||
}
|
||
}
|
||
|
||
if (_packet->stream_index == _videoStreamIndex)
|
||
{
|
||
error = leviathan.avcodec_send_packet(_pVideoContext, _packet);
|
||
if (error < 0)
|
||
{
|
||
leviathan.av_packet_unref(_packet);
|
||
if (error == leviathan.AVERROR_Enum(leviathan.EAGAIN))
|
||
{
|
||
// 解码器需要先输出帧，继续循环处理
|
||
continue;
|
||
}
|
||
else
|
||
{
|
||
return 3;
|
||
}
|
||
}
|
||
}
|
||
leviathan.av_packet_unref(_packet);
|
||
}
|
||
else if (error == leviathan.AVERROR_EOF)
|
||
{
|
||
// 解码器已无更多帧
|
||
return 4;
|
||
}
|
||
else
|
||
{
|
||
// 其他错误
|
||
return error;
|
||
}
|
||
}
|
||
}
|
||
|
||
public void InternalStop()
|
||
{
|
||
lock (_codecLock)
|
||
{
|
||
// Release resources
|
||
var pVideoContext = _pVideoContext;
|
||
var pFormatContext = _pFormatContext;
|
||
var videoFrame = _videoFrame;
|
||
var packet = _packet;
|
||
if (_pVideoContext != null) leviathan.avcodec_free_context(&pVideoContext);
|
||
if (_pFormatContext != null) leviathan.avformat_close_input(&pFormatContext);
|
||
if (_videoFrame != null) leviathan.av_frame_free(&videoFrame);
|
||
if (_packet != null) leviathan.av_packet_free(&packet);
|
||
|
||
_pVideoContext = null;
|
||
_pFormatContext = null;
|
||
_videoFrame = null;
|
||
_packet = null;
|
||
}
|
||
|
||
if (_pAvioCtx != null)
|
||
{
|
||
#if ENABLE_AV_MALLOC
|
||
leviathan.av_free(_pAvioCtx->buffer);
|
||
#else
|
||
_pAvioCtx->buffer = null; // 关键：阻止 FFmpeg 调用 av_free
|
||
if (_hAvioBuffer.IsAllocated)
|
||
{
|
||
_hAvioBuffer.Free();
|
||
}
|
||
#endif
|
||
var pAvioCtx = _pAvioCtx;
|
||
leviathan.avio_context_free(&pAvioCtx);
|
||
_pAvioCtx = null;
|
||
}
|
||
|
||
if (_bdHandle.IsAllocated)
|
||
{
|
||
_bdHandle.Free();
|
||
}
|
||
_bdHandle = default;
|
||
#if ENABLE_COPY_NATIVE_DATA
|
||
if (_unmanagedPtr != IntPtr.Zero)
|
||
{
|
||
Marshal.FreeHGlobal(_unmanagedPtr);
|
||
}
|
||
_unmanagedPtr = default;
|
||
#endif
|
||
}
|
||
|
||
public void Dispose()
|
||
{
|
||
InternalStop();
|
||
}
|
||
|
||
// 跳转到指定时间点
|
||
public void Seek(double ms)
|
||
{
|
||
lock (_codecLock)
|
||
{
|
||
if (_pVideoContext == null || _pFormatContext == null)
|
||
return;
|
||
|
||
long targetFrame = (long)(_frameInterval * ms);
|
||
// Debug.Log("Seek to " + targetFrame + "(" + ms + "ms)");
|
||
leviathan.avformat_seek_file(_pFormatContext, _videoStreamIndex, Int64.MinValue, targetFrame, Int64.MaxValue, leviathan.AVSEEK_FLAG_FRAME);
|
||
leviathan.avcodec_flush_buffers(_pVideoContext);
|
||
}
|
||
}
|
||
|
||
// 跳转到指定帧
|
||
public void SeekToFrame(long frame)
|
||
{
|
||
lock (_codecLock)
|
||
{
|
||
if (_pVideoContext == null || _pFormatContext == null)
|
||
return;
|
||
|
||
long pts = _pFormatContext->streams[_videoStreamIndex]->duration / _frameCount * frame - 1;
|
||
leviathan.avformat_seek_file(_pFormatContext, _videoStreamIndex, Int64.MinValue, pts, Int64.MaxValue, leviathan.AVSEEK_FLAG_FRAME);
|
||
leviathan.avcodec_flush_buffers(_pVideoContext);
|
||
}
|
||
}
|
||
|
||
// 获取decodedFrameIndex的引用，用于Interlocked操作
|
||
public ref long GetDecodedFrameIndexRef()
|
||
{
|
||
return ref _decodedFrameIndex;
|
||
}
|
||
|
||
// 暂停/恢复播放的空方法（软件解码器不需要特殊处理）
|
||
public void Pause(bool pause)
|
||
{
|
||
// 软件解码器的暂停由LeviathanVideoDecoder的_isPause标志控制
|
||
// 这里不需要额外的实现
|
||
}
|
||
|
||
// 微信平台特有方法的空实现
|
||
public int GetDecoderStatus()
|
||
{
|
||
// 软件解码器始终返回已启动状态
|
||
return 2;
|
||
}
|
||
|
||
// 将视频帧数据复制到纹理（需要在信号保护下执行，访问_videoFrame）
|
||
public void CopyFrameDataToTextures(UnityEngine.Texture2D[] textures)
|
||
{
|
||
const int TextureCount = 3;
|
||
|
||
if (textures == null || textures.Length < TextureCount) return;
|
||
|
||
// 更新YUV纹理 - 只做SetPixelData，不做Apply
|
||
for (uint i = 0; i < TextureCount; i++)
|
||
{
|
||
if (_videoFrame->data[i] == null || textures[i] == null) continue;
|
||
|
||
var nativeArray = Unity.Collections.LowLevel.Unsafe.NativeArrayUnsafeUtility.ConvertExistingDataToNativeArray<byte>(
|
||
_videoFrame->data[i],
|
||
textures[i].width * textures[i].height,
|
||
Unity.Collections.Allocator.None
|
||
);
|
||
|
||
try
|
||
{
|
||
#if ENABLE_UNITY_COLLECTIONS_CHECKS
|
||
Unity.Collections.LowLevel.Unsafe.NativeArrayUnsafeUtility.SetAtomicSafetyHandle(ref nativeArray, Unity.Collections.LowLevel.Unsafe.AtomicSafetyHandle.Create());
|
||
#endif
|
||
// 只复制数据到纹理的CPU缓冲区，不上传到GPU
|
||
textures[i].SetPixelData(nativeArray, 0);
|
||
#if ENABLE_UNITY_COLLECTIONS_CHECKS
|
||
Unity.Collections.LowLevel.Unsafe.AtomicSafetyHandle.Release(Unity.Collections.LowLevel.Unsafe.NativeArrayUnsafeUtility.GetAtomicSafetyHandle(nativeArray));
|
||
#endif
|
||
}
|
||
catch (UnityException)
|
||
{
|
||
|
||
}
|
||
finally
|
||
{
|
||
nativeArray.Dispose();
|
||
}
|
||
}
|
||
}
|
||
|
||
// 将纹理数据应用到GPU（可以和下一帧解码并行执行）
|
||
public void ApplyTextures(UnityEngine.Texture2D[] textures)
|
||
{
|
||
const int TextureCount = 3;
|
||
|
||
if (textures == null || textures.Length < TextureCount) return;
|
||
|
||
// 将CPU数据上传到GPU - 不需要访问_videoFrame
|
||
for (uint i = 0; i < TextureCount; i++)
|
||
{
|
||
if (textures[i] == null) continue;
|
||
|
||
try
|
||
{
|
||
textures[i].Apply(false, false);
|
||
}
|
||
catch (UnityException)
|
||
{
|
||
|
||
}
|
||
}
|
||
}
|
||
}
|