audio_core: hle: mf: make DetectMediaType return a struct

src/audio_core/hle/wmf_decoder.cpp

@@ -163,7 +163,7 @@ std::optional<BinaryResponse> WMFDecoder::Impl::Decode(const BinaryRequest& requ
 
     if (!initalized) {
         LOG_DEBUG(Audio_DSP, "Decoder not initalized");
-        // This is a hack to continue games that are not compiled with the aac codec
+        // This is a hack to continue games when decoder failed to initialize
         return response;
     }
 
@@ -176,21 +176,21 @@ std::optional<BinaryResponse> WMFDecoder::Impl::Decode(const BinaryRequest& requ
 
     std::array<std::vector<u8>, 2> out_streams;
     unique_mfptr<IMFSample> sample;
-    ADTSData adts_header;
-    char* aac_tag = (char*)calloc(1, 14);
     MFInputState input_status = MFInputState::OK;
+    std::optional<ADTSMeta> adts_meta = DetectMediaType((char*)data, request.size);
 
-    if (DetectMediaType((char*)data, request.size, &adts_header, &aac_tag) != 0) {
+    if (!adts_meta) {
         LOG_ERROR(Audio_DSP, "Unable to deduce decoding parameters from ADTS stream");
         return response;
     }
 
-    response.num_channels = adts_header.channels;
+    response.num_channels = adts_meta->ADTSHeader.channels;
 
     if (!selected) {
         LOG_DEBUG(Audio_DSP, "New ADTS stream: channels = {}, sample rate = {}",
-                  adts_header.channels, adts_header.samplerate);
-        SelectInputMediaType(transform.get(), in_stream_id, adts_header, (UINT8*)aac_tag, 14);
+                  adts_meta->ADTSHeader.channels, adts_meta->ADTSHeader.samplerate);
+        SelectInputMediaType(transform.get(), in_stream_id, adts_meta->ADTSHeader,
+                             adts_meta->AACTag, 14);
         SelectOutputMediaType(transform.get(), out_stream_id);
         SendSample(transform.get(), in_stream_id, nullptr);
         // cache the result from detect_mediatype and call select_*_mediatype only once
@@ -205,7 +205,7 @@ std::optional<BinaryResponse> WMFDecoder::Impl::Decode(const BinaryRequest& requ
     while (true) {
         input_status = SendSample(transform.get(), in_stream_id, sample.get());
 
-        if (DecodingLoop(adts_header, out_streams) == MFOutputState::FatalError) {
+        if (DecodingLoop(adts_meta->ADTSHeader, out_streams) == MFOutputState::FatalError) {
             // if the decode issues are caused by MFT not accepting new samples, try again
             // NOTICE: you are required to check the output even if you already knew/guessed
             // MFT didn't accept the input sample

src/audio_core/hle/wmf_decoder_utils.cpp

@@ -209,17 +209,18 @@ bool SelectOutputMediaType(IMFTransform* transform, int out_stream_id, GUID audi
     return false;
 }
 
-int DetectMediaType(char* buffer, size_t len, ADTSData* output, char** aac_tag) {
+std::optional<ADTSMeta> DetectMediaType(char* buffer, size_t len) {
     if (len < 7) {
-        return -1;
+        return std::nullopt;
     }
 
     ADTSData tmp;
+    ADTSMeta result;
     // see https://docs.microsoft.com/en-us/windows/desktop/api/mmreg/ns-mmreg-heaacwaveinfo_tag
     // for the meaning of the byte array below
 
     // it might be a good idea to wrap the parameters into a struct
-    // and pass that struct into the function but this will lead to messier code
+    // and pass that struct into the function but doing that will lead to messier code
     // const UINT8 aac_data[] = { 0x01, 0x00, 0xfe, 00, 00, 00, 00, 00, 00, 00, 00, 00, 0x11, 0x90
     // }; first byte: 0: raw aac 1: adts 2: adif 3: latm/laos
     UINT8 aac_tmp[] = {0x01, 0x00, 0xfe, 00, 00, 00, 00, 00, 00, 00, 00, 00, 0x00, 0x00};
@@ -227,15 +228,15 @@ int DetectMediaType(char* buffer, size_t len, ADTSData* output, char** aac_tag)
 
     tmp = ParseADTS(buffer);
     if (tmp.length == 0) {
-        return -1;
+        return std::nullopt;
     }
 
     tag = MFGetAACTag(tmp);
     aac_tmp[12] |= (tag & 0xff00) >> 8;
     aac_tmp[13] |= (tag & 0x00ff);
-    std::memcpy(*aac_tag, aac_tmp, 14);
-    std::memcpy(output, &tmp, sizeof(ADTSData));
-    return 0;
+    std::memcpy(&(result.ADTSHeader), &tmp, sizeof(ADTSData));
+    std::memcpy(&(result.AACTag), aac_tmp, 14);
+    return result;
 }
 
 void MFFlush(IMFTransform* transform) {

src/audio_core/hle/wmf_decoder_utils.h

@@ -58,6 +58,12 @@ auto Amp(SmartPtr& smart_ptr) {
 // convient function for formatting error messages
 void ReportError(std::string msg, HRESULT hr);
 
+// data type for transferring ADTS metadata between functions
+struct ADTSMeta {
+    ADTSData ADTSHeader;
+    u8 AACTag[14];
+};
+
 // exported functions
 bool MFCoInit();
 unique_mfptr<IMFTransform> MFDecoderInit(GUID audio_format = MFAudioFormat_AAC);
@@ -67,7 +73,7 @@ unique_mfptr<IMFSample> CreateSample(void* data, DWORD len, DWORD alignment = 1,
 bool SelectInputMediaType(IMFTransform* transform, int in_stream_id, const ADTSData& adts,
                           UINT8* user_data, UINT32 user_data_len,
                           GUID audio_format = MFAudioFormat_AAC);
-int DetectMediaType(char* buffer, size_t len, ADTSData* output, char** aac_tag);
+std::optional<ADTSMeta> DetectMediaType(char* buffer, size_t len);
 bool SelectOutputMediaType(IMFTransform* transform, int out_stream_id,
                            GUID audio_format = MFAudioFormat_PCM);
 void MFFlush(IMFTransform* transform);