/*---------------------------------------------------------------- // Copyright (C) 2025 Beijing All rights reserved. // // Author: huachangmiao // Create Date: 2025/04/11 // Module Describe: //----------------------------------------------------------------*/ //****************************************************************** // CPU软解码 // 适用平台 // Editor // Windows // MacOS // Android // iOS // WebGL //****************************************************************** // #define ENABLE_COPY_NATIVE_DATA //复制Native数据到托管内存 #if UNITY_WEBGL #else #define ENABLE_AV_MALLOC //使用AV_MALLOC分配内存 #endif using System; using System.Threading; using LeviathanVideo.Abstractions; using LeviathanVideo.Bindings.Linked; using Unity.Collections; using Unity.Collections.LowLevel.Unsafe; using UnityEngine; using System.Runtime.InteropServices; using AOT; internal unsafe class LeviathanSoftwareDecoder : ILeviathanDecoder { // 日志辅助方法 private static void Log(string message) { LeviathanLogConfig.Log("SoftwareDecoder", message); } private static void LogWarning(string message) { LeviathanLogConfig.LogWarning("SoftwareDecoder", message); } private static void LogError(string message) { LeviathanLogConfig.LogError("SoftwareDecoder", message); } // 事件实现 public event Action OnFirstFrameDecoded; private AVFormatContext* _pFormatContext; private AVCodecContext* _pVideoContext; private int _videoStreamIndex; private AVFrame* _videoFrame; private AVPacket* _packet; private AVIOContext* _pAvioCtx; private byte* _avioCtxBuffer; private BufferData _bufferData; private GCHandle _bdHandle; #if ENABLE_AV_MALLOC #else private GCHandle _hAvioBuffer; #endif #if ENABLE_COPY_NATIVE_DATA private IntPtr _unmanagedPtr; #endif // 私有字段 private long _frameCount; private double _frameInterval; private string _codecName = ""; private long _duration; private long _decodedFrameIndex; private bool _firstFrameDecoded = false; // 线程安全锁 - 保护 Seek 和 Decode 操作不并发执行 private readonly object _codecLock = new object(); // 实现接口属性 public long FrameCount => _frameCount; public double FrameInterval => _frameInterval; public string CodecName => _codecName; public long Duration => _duration; public long DecodedFrameIndex => _decodedFrameIndex; public AVFrame* VideoFrame => _videoFrame; // 循环播放属性(软件解码器不需要特殊处理) public bool IsLooping { get; set; } = false; // 快速解码模式(跳过环路滤波器) private bool _fastDecode = true; [StructLayout(LayoutKind.Sequential)] private struct BufferData { public byte* ptr; public byte* originPtr; public ulong size; public ulong totalSize; } [MonoPInvokeCallback(typeof(avio_alloc_context_read_packet))] private static int ReadPacket(void* opaque, byte* buf, int bufSize) { BufferData* bd = (BufferData*)opaque; bufSize = (int)Math.Min(bd->size, (ulong)bufSize); if (bufSize <= 0) return leviathan.AVERROR_EOF; // Buffer.MemoryCopy(bd->ptr, buf, bufSize, bufSize); UnsafeUtility.MemCpy(buf, bd->ptr, bufSize); bd->ptr += bufSize; bd->size -= (ulong)bufSize; return bufSize; } [MonoPInvokeCallback(typeof(avio_alloc_context_seek))] private static long SeekPacket(void* opaque, long offset, int whence) { BufferData* bd = (BufferData*)opaque; switch (whence) { case 0: //SEEK_SET if (offset < 0 || (ulong)offset > bd->totalSize) return leviathan.AVERROR_Enum(leviathan.EINVAL); bd->ptr = bd->originPtr + offset; bd->size = bd->totalSize - (ulong)offset; return offset; case 1: //SEEK_CUR long newPos = bd->ptr - bd->originPtr + offset; if (newPos < 0 || (ulong)newPos > bd->totalSize) return leviathan.AVERROR_Enum(leviathan.EINVAL); bd->ptr += offset; bd->size = bd->totalSize - (ulong)newPos; return newPos; case 2: //SEEK_END long endPos = (long)bd->totalSize + offset; if (endPos < 0 || endPos > (long)bd->totalSize) return leviathan.AVERROR_Enum(leviathan.EINVAL); bd->ptr = bd->originPtr + endPos; bd->size = bd->totalSize - (ulong)endPos; return endPos; default: return leviathan.AVERROR_Enum(leviathan.EINVAL); } } public int Init(NativeArray fileData, int beginFrameIndex = 1, bool syncDecodeFirstFrame = true, bool fastDecode = true) { LinkedBindings.Initialize(); _fastDecode = fastDecode; var ret = 0; var src = fileData.GetUnsafeReadOnlyPtr(); #if ENABLE_COPY_NATIVE_DATA _unmanagedPtr = Marshal.AllocHGlobal(fileData.Length); var dst = _unmanagedPtr.ToPointer(); UnsafeUtility.MemCpy(dst, src, fileData.Length); src = dst; #endif ulong bufferSize = (ulong)fileData.Length; _bufferData = new BufferData { originPtr = (byte*)src, // 初始化原始指针 ptr = (byte*)src, size = bufferSize, totalSize = bufferSize }; _bdHandle = GCHandle.Alloc(_bufferData, GCHandleType.Pinned); avio_alloc_context_read_packet readCallback = ReadPacket; avio_alloc_context_seek seekCallback = SeekPacket; // 为AVIOContext分配缓冲区 const int avioCtxBufferSize = 4096; #if ENABLE_AV_MALLOC _avioCtxBuffer = (byte*)leviathan.av_malloc(avioCtxBufferSize); // 分配AVIOContext并将其与AVFormatContext关联 _pAvioCtx = leviathan.avio_alloc_context( _avioCtxBuffer, avioCtxBufferSize, 0, // 不可写 (void*)_bdHandle.AddrOfPinnedObject(), readCallback, null, // 无写回调 seekCallback ); #else // wasm无法直接使用av_malloc int alignment = 64; byte[] avioBufferManaged = new byte[avioCtxBufferSize + alignment - 1]; _hAvioBuffer = GCHandle.Alloc(avioBufferManaged, GCHandleType.Pinned); IntPtr avioBufferPtr = _hAvioBuffer.AddrOfPinnedObject(); // 调整指针到对齐地址 IntPtr alignedPtr = new IntPtr((avioBufferPtr.ToInt64() + alignment - 1) & ~(alignment - 1)); // 分配AVIOContext并将其与AVFormatContext关联 _pAvioCtx = leviathan.avio_alloc_context( (byte*)alignedPtr, avioCtxBufferSize, 0, // 不可写 (void*)_bdHandle.AddrOfPinnedObject(), readCallback, null, // 无写回调 seekCallback ); #endif if (_pAvioCtx == null) { ret = leviathan.AVERROR_Enum(leviathan.ENOMEM); LogError("无法分配AVIO上下文"); return ret; } _pFormatContext = leviathan.avformat_alloc_context(); _pFormatContext->pb = _pAvioCtx; var pFormatContext = _pFormatContext; ret = leviathan.avformat_open_input(&pFormatContext, null, null, null); if (ret != 0) { LogError($"avformat_open_input failed: {ret}"); return ret; } if (_pFormatContext == null) { return ret; } return InternalPlay(beginFrameIndex, syncDecodeFirstFrame); } public int InternalPlay(int beginFrameIndex = 1, bool syncDecodeFirstFrame = true) { leviathan.avformat_find_stream_info(_pFormatContext, null); AVCodec* videoCodec = null; _videoStreamIndex = leviathan.av_find_best_stream(_pFormatContext, AVMediaType.AVMEDIA_TYPE_VIDEO, -1, -1, &videoCodec, 0); // 获取视频流信息 AVStream* videoStream = _pFormatContext->streams[_videoStreamIndex]; // 计算帧间隔 AVRational frameRate = videoStream->r_frame_rate; double fps = leviathan.av_q2d(frameRate); _frameInterval = 1.0 / fps; _pVideoContext = leviathan.avcodec_alloc_context3(videoCodec); #if UNITY_EDITOR _codecName = leviathan.avcodec_get_name(videoCodec->id); #endif if (videoCodec == null) { LogError("videoCodec is null"); return 1; } _pVideoContext->thread_count = 0; _pVideoContext->thread_type = leviathan.FF_THREAD_FRAME | leviathan.FF_THREAD_SLICE; // 根据 fastDecode 参数决定是否跳过环路滤波器以提升解码速度 if (_fastDecode) { _pVideoContext->skip_loop_filter = (AVDiscard)48; //AVDISCARD_ALL } // Debug.Log($"{_pVideoContext->codec_id} {_frameInterval} {_pFormatContext->duration}"); leviathan.avcodec_open2(_pVideoContext, videoCodec, null); _duration = _pFormatContext->duration; if (_pFormatContext->streams[_videoStreamIndex]->nb_frames > 0) { _frameCount = _pFormatContext->streams[_videoStreamIndex]->nb_frames; } else { var dur = _duration / 1000.0 * leviathan.av_q2d(videoStream->time_base); int frNum = frameRate.num; int frDen = frameRate.den; if (dur > 0 && frNum > 0 && frDen > 0) { _frameCount = (long)(dur * frNum / frDen); } } // Debug.Log($"frameCount: {_frameCount}"); _videoFrame = leviathan.av_frame_alloc(); _packet = leviathan.av_packet_alloc(); if (beginFrameIndex > 1) { SeekToFrame(beginFrameIndex); } if (syncDecodeFirstFrame) { // 第一帧同步解码 int ret = DecodeNextFrame(); if (ret != 0) { LogError($"DecodeNextFrame ret: {ret}"); } } return 0; } public int DecodeNextFrame() { lock (_codecLock) { if (_pVideoContext == null || _pFormatContext == null) return -1; return DecodeNextFrameInternal(); } } private int DecodeNextFrameInternal() { var flushSent = false; while (true) { var error = leviathan.avcodec_receive_frame(_pVideoContext, _videoFrame); if (error == 0) { long frameIndex = 1; if (_frameInterval > 0) { var timeBase = _pFormatContext->streams[_videoStreamIndex]->time_base; double frameTime = _videoFrame->pts * leviathan.av_q2d(timeBase); frameIndex = (long)(frameTime / _frameInterval + 0.1) + 1; } Interlocked.Exchange(ref _decodedFrameIndex, frameIndex); // 触发第一帧回调 if (!_firstFrameDecoded) { _firstFrameDecoded = true; OnFirstFrameDecoded?.Invoke(); } return 0; } if (error == leviathan.AVERROR_Enum(leviathan.EAGAIN)) { if (flushSent) { // 已发送过flush,无更多数据 return 1; } error = leviathan.av_read_frame(_pFormatContext, _packet); if (error < 0) { if (error == leviathan.AVERROR_EOF) { // 发送flush packet以取出解码器中剩余帧 error = leviathan.avcodec_send_packet(_pVideoContext, null); flushSent = true; if (error < 0 && error != leviathan.AVERROR_Enum(leviathan.EAGAIN)) { return 2; } continue; } else { // 其他错误,继续尝试读取 continue; } } if (_packet->stream_index == _videoStreamIndex) { error = leviathan.avcodec_send_packet(_pVideoContext, _packet); if (error < 0) { leviathan.av_packet_unref(_packet); if (error == leviathan.AVERROR_Enum(leviathan.EAGAIN)) { // 解码器需要先输出帧,继续循环处理 continue; } else { return 3; } } } leviathan.av_packet_unref(_packet); } else if (error == leviathan.AVERROR_EOF) { // 解码器已无更多帧 return 4; } else { // 其他错误 return error; } } } public void InternalStop() { lock (_codecLock) { // Release resources var pVideoContext = _pVideoContext; var pFormatContext = _pFormatContext; var videoFrame = _videoFrame; var packet = _packet; if (_pVideoContext != null) leviathan.avcodec_free_context(&pVideoContext); if (_pFormatContext != null) leviathan.avformat_close_input(&pFormatContext); if (_videoFrame != null) leviathan.av_frame_free(&videoFrame); if (_packet != null) leviathan.av_packet_free(&packet); _pVideoContext = null; _pFormatContext = null; _videoFrame = null; _packet = null; } if (_pAvioCtx != null) { #if ENABLE_AV_MALLOC leviathan.av_free(_pAvioCtx->buffer); #else _pAvioCtx->buffer = null; // 关键:阻止 FFmpeg 调用 av_free if (_hAvioBuffer.IsAllocated) { _hAvioBuffer.Free(); } #endif var pAvioCtx = _pAvioCtx; leviathan.avio_context_free(&pAvioCtx); _pAvioCtx = null; } if (_bdHandle.IsAllocated) { _bdHandle.Free(); } _bdHandle = default; #if ENABLE_COPY_NATIVE_DATA if (_unmanagedPtr != IntPtr.Zero) { Marshal.FreeHGlobal(_unmanagedPtr); } _unmanagedPtr = default; #endif } public void Dispose() { InternalStop(); } // 跳转到指定时间点 public void Seek(double ms) { lock (_codecLock) { if (_pVideoContext == null || _pFormatContext == null) return; long targetFrame = (long)(_frameInterval * ms); // Debug.Log("Seek to " + targetFrame + "(" + ms + "ms)"); leviathan.avformat_seek_file(_pFormatContext, _videoStreamIndex, Int64.MinValue, targetFrame, Int64.MaxValue, leviathan.AVSEEK_FLAG_FRAME); leviathan.avcodec_flush_buffers(_pVideoContext); } } // 跳转到指定帧 public void SeekToFrame(long frame) { lock (_codecLock) { if (_pVideoContext == null || _pFormatContext == null) return; long pts = _pFormatContext->streams[_videoStreamIndex]->duration / _frameCount * frame - 1; leviathan.avformat_seek_file(_pFormatContext, _videoStreamIndex, Int64.MinValue, pts, Int64.MaxValue, leviathan.AVSEEK_FLAG_FRAME); leviathan.avcodec_flush_buffers(_pVideoContext); } } // 获取decodedFrameIndex的引用,用于Interlocked操作 public ref long GetDecodedFrameIndexRef() { return ref _decodedFrameIndex; } // 暂停/恢复播放的空方法(软件解码器不需要特殊处理) public void Pause(bool pause) { // 软件解码器的暂停由LeviathanVideoDecoder的_isPause标志控制 // 这里不需要额外的实现 } // 微信平台特有方法的空实现 public int GetDecoderStatus() { // 软件解码器始终返回已启动状态 return 2; } // 将视频帧数据复制到纹理(需要在信号保护下执行,访问_videoFrame) public void CopyFrameDataToTextures(UnityEngine.Texture2D[] textures) { const int TextureCount = 3; if (textures == null || textures.Length < TextureCount) return; // 更新YUV纹理 - 只做SetPixelData,不做Apply for (uint i = 0; i < TextureCount; i++) { if (_videoFrame->data[i] == null || textures[i] == null) continue; var nativeArray = Unity.Collections.LowLevel.Unsafe.NativeArrayUnsafeUtility.ConvertExistingDataToNativeArray( _videoFrame->data[i], textures[i].width * textures[i].height, Unity.Collections.Allocator.None ); try { #if ENABLE_UNITY_COLLECTIONS_CHECKS Unity.Collections.LowLevel.Unsafe.NativeArrayUnsafeUtility.SetAtomicSafetyHandle(ref nativeArray, Unity.Collections.LowLevel.Unsafe.AtomicSafetyHandle.Create()); #endif // 只复制数据到纹理的CPU缓冲区,不上传到GPU textures[i].SetPixelData(nativeArray, 0); #if ENABLE_UNITY_COLLECTIONS_CHECKS Unity.Collections.LowLevel.Unsafe.AtomicSafetyHandle.Release(Unity.Collections.LowLevel.Unsafe.NativeArrayUnsafeUtility.GetAtomicSafetyHandle(nativeArray)); #endif } catch (UnityException) { } finally { nativeArray.Dispose(); } } } // 将纹理数据应用到GPU(可以和下一帧解码并行执行) public void ApplyTextures(UnityEngine.Texture2D[] textures) { const int TextureCount = 3; if (textures == null || textures.Length < TextureCount) return; // 将CPU数据上传到GPU - 不需要访问_videoFrame for (uint i = 0; i < TextureCount; i++) { if (textures[i] == null) continue; try { textures[i].Apply(false, false); } catch (UnityException) { } } } }