Use libopus for decoding Audio

CMakeLists.txt

@@ -42,6 +42,7 @@ find_package(Ogg REQUIRED)    #TODO: Not required if we aren't building the clie
 find_package(Vorbis REQUIRED) #TODO: Not required if we aren't building the client
 find_package(Speex REQUIRED)  #TODO: Not required if we aren't building the client
 find_package(VPX)
+find_package(Opus)
 find_package(CURL REQUIRED)
 find_package(Freetype)
 

Sources/Plasma/Apps/plClient/CMakeLists.txt

@@ -161,6 +161,7 @@ target_link_libraries(plClient ${Ogg_LIBRARIES})
 target_link_libraries(plClient ${Vorbis_LIBRARIES})
 target_link_libraries(plClient ${DirectX_LIBRARIES})
 target_link_libraries(plClient ${CURL_LIBRARY})
+target_link_libraries(plClient ${Opus_LIBRARIES})
 
 if(USE_VLD)
     target_link_libraries(plClient ${VLD_LIBRARY})

Sources/Plasma/FeatureLib/pfMoviePlayer/CMakeLists.txt

@@ -5,6 +5,7 @@ include_directories(../../PubUtilLib)
 
 if (VPX_AVAILABLE)
     include_directories(${VPX_INCLUDE_DIR})
+    include_directories(${Opus_INCLUDE_DIR})
 endif (VPX_AVAILABLE)
 
 set(pfMoviePlayer_SOURCES

Sources/Plasma/FeatureLib/pfMoviePlayer/plMoviePlayer.cpp

@@ -42,14 +42,17 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com
 
 #include "plMoviePlayer.h"
 #include <tuple>
+#include <memory>
 
 #ifdef VPX_AVAILABLE
 #   define VPX_CODEC_DISABLE_COMPAT 1
 #   include <vpx/vpx_decoder.h>
 #   include <vpx/vp8dx.h>
-#   define iface (vpx_codec_vp8_dx())
+#   define iface (vpx_codec_vp9_dx())
 #endif
 
+#include <opus.h>
+
 #include "plGImage/plMipmap.h"
 #include "pnKeyedObject/plUoid.h"
 #include "plPipeline/hsGDeviceRef.h"
@@ -57,6 +60,13 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com
 #include "plPlanarImage.h"
 #include "hsResMgr.h"
 #include "hsTimer.h"
+#include "plAudio/plWin32VideoSound.h"
+#include "../Apps/plClient/plClient.h"
+#include "plScene/plSceneNode.h"
+#include "pnSceneObject/plSceneObject.h"
+#include "pnSceneObject/plAudioInterface.h"
+#include "pnMessage/plSoundMsg.h"
+#include "plgDispatch.h"
 
 #include "webm/mkvreader.hpp"
 #include "webm/mkvparser.hpp"
@@ -248,13 +258,37 @@ plMoviePlayer::plMoviePlayer() :
     fTexture(nullptr),
     fReader(nullptr),
     fTimeScale(0), 
-    fStartTime(0)
+    fStartTime(0),
+    fAudioPlayer(),
+    fPosition(hsPoint2()),
+    fAudioInterface(),
+    fPlaying(true),
+    fOpusDecoder(nil)
 {
     fScale.Set(1.0f, 1.0f);
+
+    
+    fAudioSound = std::shared_ptr<plWin32VideoSound>(new plWin32VideoSound());
+    fAudioPlayer.SetSound(fAudioSound);
+    plSceneNode* sceneNode = plClient::GetInstance()->GetCurrentScene();
+    if (sceneNode != nullptr)
+    {
+        hsTArray<plSceneObject*>& sceneObjects = sceneNode->GetSceneObjects();
+        for (int i = 0; i < sceneObjects.GetCount(); ++i)
+        {
+            if (sceneObjects[i]->GetAudioInterface() == nullptr)
+            {
+                fAudioInterface.ISetAudible(&fAudioPlayer);
+                sceneObjects[i]->SetAudioInterface(&fAudioInterface);
+                break;
+            }
+        }
+    }
 }
 
 plMoviePlayer::~plMoviePlayer()
 {
+    opus_decoder_destroy(fOpusDecoder);
     if (fPlate)
         // The plPlate owns the Mipmap Texture, so it destroys it for us
         plPlateManager::Instance().DestroyPlate(fPlate);
@@ -363,6 +397,10 @@ bool plMoviePlayer::IProcessVideoFrame(const std::vector<blkbuf_t>& frames)
 
 bool plMoviePlayer::Start()
 {
+    plSceneNode* sceneNode = plClient::GetInstance()->GetCurrentScene();
+    if (sceneNode != nullptr)
+        sceneNode->GetKey();
+
 #ifdef VPX_AVAILABLE
     if (!IOpenMovie())
         return false;
@@ -374,6 +412,12 @@ bool plMoviePlayer::Start()
     else
         return false;
 
+    //initialize opus
+    int error;
+    fOpusDecoder = opus_decoder_create(48000, 1, &error);
+    if (error != OPUS_OK)
+        hsAssert(false, "Error occured initalizing opus");
+
     // Need to figure out scaling based on pipe size.
     plPlateManager& plateMgr = plPlateManager::Instance();
     const mkvparser::VideoTrack* video = static_cast<const mkvparser::VideoTrack*>(fSegment->GetTracks()->GetTrackByNumber(fVideoTrack->number));
@@ -383,6 +427,9 @@ bool plMoviePlayer::Start()
     plateMgr.CreatePlate(&fPlate, fPosition.fX, fPosition.fY, width, height);
     fPlate->SetVisible(true);
     fTexture = fPlate->CreateMaterial(static_cast<uint32_t>(video->GetWidth()), static_cast<uint32_t>(video->GetHeight()), nullptr);
+
+    fPlaying = true;
+
     return true;
 #else
     return false;
@@ -391,45 +438,82 @@ bool plMoviePlayer::Start()
 
 bool plMoviePlayer::NextFrame()
 {
-#ifdef VPX_AVAILABLE
+    if (fPlaying)
-    // Get our current timecode
-    int64_t movieTime = 0;
-    if (fStartTime == 0)
-        fStartTime = static_cast<int64_t>(hsTimer::GetMilliSeconds());
-    else
-        movieTime = GetMovieTime();
-
-    std::vector<blkbuf_t> audio;
-    std::vector<blkbuf_t> video;
     {
-        uint8_t tracksWithData = 0;
+#ifdef VPX_AVAILABLE
-        if (fAudioTrack)
+        // Get our current timecode
-        {
+        int64_t movieTime = 0;
-            if (fAudioTrack->GetFrames(this, movieTime, audio))
+        if (fStartTime == 0)
-                tracksWithData++;
+            fStartTime = static_cast<int64_t>(hsTimer::GetMilliSeconds());
-        }
+        else
-        if (fVideoTrack)
+            movieTime = GetMovieTime();
-        {
-            if (fVideoTrack->GetFrames(this, movieTime, video))
-                tracksWithData++;
-        }
-        if (tracksWithData == 0)
-            return false;
-    }
 
-    // Show our mess
+        std::vector<blkbuf_t> audio;
-    IProcessVideoFrame(video);
+        std::vector<blkbuf_t> video;
+        {
+            uint8_t tracksWithData = 0;
+            if (fAudioTrack)
+            {
+                if (fAudioTrack->GetFrames(this, movieTime, audio))
+                    tracksWithData++;
+            }
+            if (fVideoTrack)
+            {
+                if (fVideoTrack->GetFrames(this, movieTime, video))
+                    tracksWithData++;
+            }
+            if (tracksWithData == 0)
+            {
+                Stop();
+                return false;
+            }
+        }
 
-    return true;
+        // Show our mess
+        IProcessVideoFrame(video);
+        IProcessAudioFrame(audio);
+
+        return true;
 #else
-    return false;
+        return false;
 #endif // VPX_AVAILABLE
+    }
+    return false;
 }
 
 bool plMoviePlayer::Stop()
 {
-    for (int i = 0; i < fCallbacks.GetCount(); i++)
+    fPlaying = false;
+    for (int i = 0; i < fCallbacks.size(); i++)
         fCallbacks[i]->Send();
-    fCallbacks.Reset();
+    fCallbacks.clear();
     return false;
 }
+
+bool plMoviePlayer::IProcessAudioFrame(const std::vector<blkbuf_t>& frames)
+{
+    const unsigned char* data = NULL;
+    int32_t size = 0;
+    for (auto it = frames.begin(); it != frames.end(); ++it)
+    {
+        const std::unique_ptr<uint8_t>& buf = std::get<0>(*it);
+        data = buf.get();
+        size = std::get<1>(*it);
+
+        int error;
+        const int frameSize = 5760; //max packet duration at 48kHz
+        opus_int16* pcm = new opus_int16[frameSize * 1 * sizeof(opus_int16)];
+        error = opus_decode(fOpusDecoder, data, size, pcm, frameSize, 0);
+        if (error < 0)
+            hsAssert(false, "opus error");
+        fAudioSound->UpdateSoundBuffer(reinterpret_cast<unsigned char*>(pcm), error);
+        if (plClient::GetInstance()->GetCurrentScene() != nullptr)
+        {
+            plSoundMsg* soundMsg = new plSoundMsg();
+            soundMsg->SetCmd(plSoundMsg::kPlay);
+            soundMsg->SetBCastFlag(plMessage::kBCastByType);
+            fAudioInterface.MsgReceive(soundMsg);
+        }
+    }
+    return true;
+}

Sources/Plasma/FeatureLib/pfMoviePlayer/plMoviePlayer.h

@@ -48,16 +48,22 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com
 #include "hsPoint2.h"
 #include "hsColorRGBA.h"
 #include "plMessage/plMovieMsg.h"
+#include "plAudio/plWin32VideoSound.h"
+#include "plAudible/plVideoAudible.h"
+#include "pnSceneObject/plAudioInterface.h"
 
 #include <memory>
 #include <vector>
 #include <tuple>
 
+#include <opus.h>
+
 namespace mkvparser
 {
     class BlockEntry;
     class MkvReader;
     class Segment;
+    class Track;
 }
 
 typedef std::tuple<std::unique_ptr<uint8_t>, int32_t> blkbuf_t;
@@ -79,9 +85,17 @@ protected:
     hsPoint2 fPosition, fScale;
     plFileName fMoviePath;
 
+    OpusDecoder* fOpusDecoder;
+    std::shared_ptr<plWin32VideoSound> fAudioSound;
+    plVideoAudible fAudioPlayer;
+    plAudioInterface fAudioInterface;
+
+    bool fPlaying;
+
     int64_t GetMovieTime() const;
     bool IOpenMovie();
     bool IProcessVideoFrame(const std::vector<blkbuf_t>& frames);
+    bool IProcessAudioFrame(const std::vector<blkbuf_t>& frames);
 
 public:
     plMoviePlayer();
@@ -92,7 +106,7 @@ public:
     bool Stop();
     bool NextFrame();
 
-    void AddCallback(plMessage* msg) { hsRefCnt_SafeRef(msg); fCallbacks.Append(msg); }
+    void AddCallback(plMessage* msg) { hsRefCnt_SafeRef(msg); fCallbacks.push_back(msg); }
     uint32_t GetNumCallbacks() const { return 0; }
     plMessage* GetCallback(int i) const { return nullptr; }
 
@@ -118,7 +132,7 @@ public:
     void SetFadeToColor(hsColorRGBA c) { }
 
 private:
-    hsTArray<plMessage*> fCallbacks;
+    std::vector<plMessage*> fCallbacks;
 };
 
 #endif // _plMoviePlayer_inc

Sources/Plasma/PubUtilLib/plAudio/CMakeLists.txt

@@ -23,6 +23,7 @@ set(plAudio_SOURCES
     plWin32StaticSound.cpp
     plWin32StreamingSound.cpp
     plWinMicLevel.cpp
+    plWin32VideoSound.cpp
 )
 
 set(plAudio_HEADERS
@@ -40,6 +41,7 @@ set(plAudio_HEADERS
     plWin32StaticSound.h
     plWin32StreamingSound.h
     plWinMicLevel.h
+    plWin32VideoSound.h
 )
 
 add_library(plAudio STATIC ${plAudio_SOURCES} ${plAudio_HEADERS})

Sources/Plasma/PubUtilLib/plAudio/plWin32VideoSound.cpp

@@ -0,0 +1,69 @@
+#include <iostream>
+#include <fstream>
+
+#include "hsResMgr.h"
+#include "plFormat.h"
+
+#include "plDSoundBuffer.h"
+
+#include "plWin32VideoSound.h"
+
+static int uniqueID = 0;
+plWin32VideoSound::plWin32VideoSound() : plWin32Sound()
+{
+    plWAVHeader header;
+    header.fFormatTag = 0x1;
+    header.fNumChannels = 1;
+    header.fBitsPerSample = 16;
+    fDSoundBuffer = new plDSoundBuffer(0, header, false, false, false, true);
+    fDSoundBuffer->SetupVoiceSource();
+    uniqueID++;
+    hsgResMgr::ResMgr()->NewKey(plFormat("videosound#{}", uniqueID), this, plLocation::kGlobalFixedLoc);
+    fSoftVolume = 1.0f;
+}
+
+plWin32VideoSound::~plWin32VideoSound()
+{
+    delete fDSoundBuffer;
+}
+
+void plWin32VideoSound::UpdateSoundBuffer(unsigned char* buffer, size_t size)
+{
+    unsigned int bufferID = 0;
+
+    if (fDSoundBuffer->GetAvailableBufferId(&bufferID))
+    {
+        fDSoundBuffer->VoiceFillBuffer(buffer, size, bufferID);
+        IActuallyPlay();
+    }
+}
+
+void plWin32VideoSound::IDerivedActuallyPlay()
+{
+    if (!fDSoundBuffer->IsPlaying())
+        fDSoundBuffer->Play();
+}
+
+bool plWin32VideoSound::LoadSound(bool is3D)
+{
+    hsAssert(false, "unimplemented cause unnecessary for this class");
+    return false;
+}
+
+void plWin32VideoSound::SetStartPos(unsigned bytes)
+{
+    
+    //do nothing
+    hsAssert(false, "unimplemented cause unnecessary for this class");
+}
+
+float plWin32VideoSound::GetActualTimeSec()
+{
+    hsAssert(false, "unimplemented cause unnecessary for this class");
+    return 0;
+}
+
+void plWin32VideoSound::ISetActualTime(double t)
+{
+    hsAssert(false, "unimplemented cause unnecessary for this class");
+}

Sources/Plasma/PubUtilLib/plAudio/plWin32VideoSound.h

@@ -0,0 +1,63 @@
+/*==LICENSE==*
+
+CyanWorlds.com Engine - MMOG client, server and tools
+Copyright (C) 2011  Cyan Worlds, Inc.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Additional permissions under GNU GPL version 3 section 7
+
+If you modify this Program, or any covered work, by linking or
+combining it with any of RAD Game Tools Bink SDK, Autodesk 3ds Max SDK,
+NVIDIA PhysX SDK, Microsoft DirectX SDK, OpenSSL library, Independent
+JPEG Group JPEG library, Microsoft Windows Media SDK, or Apple QuickTime SDK
+(or a modified version of those libraries),
+containing parts covered by the terms of the Bink SDK EULA, 3ds Max EULA,
+PhysX SDK EULA, DirectX SDK EULA, OpenSSL and SSLeay licenses, IJG
+JPEG Library README, Windows Media SDK EULA, or QuickTime SDK EULA, the
+licensors of this Program grant you additional
+permission to convey the resulting work. Corresponding Source for a
+non-source form of such a combination shall include the source code for
+the parts of OpenSSL and IJG JPEG Library used as well as that of the covered
+work.
+
+You can contact Cyan Worlds, Inc. by email legal@cyan.com
+or by snail mail at:
+Cyan Worlds, Inc.
+14617 N Newport Hwy
+Mead, WA   99021
+
+*==LICENSE==*/
+#ifndef plWin32VideoSound_h
+#define plWin32VideoSound_h
+
+#include "plWin32Sound.h"
+
+class plWin32VideoSound : public plWin32Sound
+{
+public:
+    plWin32VideoSound();
+    virtual ~plWin32VideoSound();
+
+    void UpdateSoundBuffer(unsigned char* buffer, size_t size);
+
+protected:
+    void IDerivedActuallyPlay(void);
+    bool LoadSound(bool is3D);
+    void SetStartPos(unsigned bytes);
+    float GetActualTimeSec();
+    void  ISetActualTime(double t);
+
+};
+#endif

cmake/FindOpus.cmake

@@ -0,0 +1,38 @@
+if(Opus_INCLUDE_DIR AND Opus_LIBRARY)
+  set(Opus_FIND_QUIETLY TRUE)
+endif()
+
+
+find_path(Opus_INCLUDE_DIR opus.h
+          /usr/local/include
+          /usr/include
+)
+
+find_library(Opus_LIBRARY NAMES opus
+             PATHS /usr/local/lib /usr/lib)
+
+find_library(Celt_LIBRARY NAMES celt
+             PATHS /usr/local/lib /usr/lib)
+
+find_library(Silk_LIBRARY NAMES silk_common
+             PATHS /usr/local/lib /usr/lib)
+
+set(Opus_LIBRARIES
+    ${Opus_LIBRARY}
+    ${Celt_LIBRARY}
+    ${Silk_LIBRARY})
+
+
+if(Opus_INCLUDE_DIR AND Opus_LIBRARY)
+  set(Opus_FOUND TRUE)
+endif()
+
+if (Opus_FOUND)
+    if(NOT Opus_FIND_QUIETLY)
+        message(STATUS "Found libopus: ${Opus_INCLUDE_DIR}")
+    endif()
+else()
+    if(Opus_FIND_REQUIRED)
+        message(FATAL_ERROR "Could not find libopus")
+    endif()
+endif()