diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ba60e61..b3028e18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,7 @@ find_package(Ogg REQUIRED) #TODO: Not required if we aren't building the clie find_package(Vorbis REQUIRED) #TODO: Not required if we aren't building the client find_package(Speex REQUIRED) #TODO: Not required if we aren't building the client find_package(VPX) +find_package(Opus) find_package(CURL REQUIRED) find_package(Freetype) diff --git a/Sources/Plasma/Apps/plClient/CMakeLists.txt b/Sources/Plasma/Apps/plClient/CMakeLists.txt index 1909110e..11d44a82 100644 --- a/Sources/Plasma/Apps/plClient/CMakeLists.txt +++ b/Sources/Plasma/Apps/plClient/CMakeLists.txt @@ -161,6 +161,7 @@ target_link_libraries(plClient ${Ogg_LIBRARIES}) target_link_libraries(plClient ${Vorbis_LIBRARIES}) target_link_libraries(plClient ${DirectX_LIBRARIES}) target_link_libraries(plClient ${CURL_LIBRARY}) +target_link_libraries(plClient ${Opus_LIBRARIES}) if(USE_VLD) target_link_libraries(plClient ${VLD_LIBRARY}) diff --git a/Sources/Plasma/FeatureLib/pfMoviePlayer/CMakeLists.txt b/Sources/Plasma/FeatureLib/pfMoviePlayer/CMakeLists.txt index 97e3f685..9e0e2f34 100644 --- a/Sources/Plasma/FeatureLib/pfMoviePlayer/CMakeLists.txt +++ b/Sources/Plasma/FeatureLib/pfMoviePlayer/CMakeLists.txt @@ -5,6 +5,7 @@ include_directories(../../PubUtilLib) if (VPX_AVAILABLE) include_directories(${VPX_INCLUDE_DIR}) + include_directories(${Opus_INCLUDE_DIR}) endif (VPX_AVAILABLE) set(pfMoviePlayer_SOURCES diff --git a/Sources/Plasma/FeatureLib/pfMoviePlayer/plMoviePlayer.cpp b/Sources/Plasma/FeatureLib/pfMoviePlayer/plMoviePlayer.cpp index bbec9581..f0efa687 100644 --- a/Sources/Plasma/FeatureLib/pfMoviePlayer/plMoviePlayer.cpp +++ b/Sources/Plasma/FeatureLib/pfMoviePlayer/plMoviePlayer.cpp @@ -42,14 +42,17 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include "plMoviePlayer.h" #include +#include #ifdef VPX_AVAILABLE # define VPX_CODEC_DISABLE_COMPAT 1 # include # include -# define iface (vpx_codec_vp8_dx()) +# define iface (vpx_codec_vp9_dx()) #endif +#include + #include "plGImage/plMipmap.h" #include "pnKeyedObject/plUoid.h" #include "plPipeline/hsGDeviceRef.h" @@ -57,6 +60,13 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include "plPlanarImage.h" #include "hsResMgr.h" #include "hsTimer.h" +#include "plAudio/plWin32VideoSound.h" +#include "../Apps/plClient/plClient.h" +#include "plScene/plSceneNode.h" +#include "pnSceneObject/plSceneObject.h" +#include "pnSceneObject/plAudioInterface.h" +#include "pnMessage/plSoundMsg.h" +#include "plgDispatch.h" #include "webm/mkvreader.hpp" #include "webm/mkvparser.hpp" @@ -248,13 +258,37 @@ plMoviePlayer::plMoviePlayer() : fTexture(nullptr), fReader(nullptr), fTimeScale(0), - fStartTime(0) + fStartTime(0), + fAudioPlayer(), + fPosition(hsPoint2()), + fAudioInterface(), + fPlaying(true), + fOpusDecoder(nil) { fScale.Set(1.0f, 1.0f); + + + fAudioSound = std::shared_ptr(new plWin32VideoSound()); + fAudioPlayer.SetSound(fAudioSound); + plSceneNode* sceneNode = plClient::GetInstance()->GetCurrentScene(); + if (sceneNode != nullptr) + { + hsTArray& sceneObjects = sceneNode->GetSceneObjects(); + for (int i = 0; i < sceneObjects.GetCount(); ++i) + { + if (sceneObjects[i]->GetAudioInterface() == nullptr) + { + fAudioInterface.ISetAudible(&fAudioPlayer); + sceneObjects[i]->SetAudioInterface(&fAudioInterface); + break; + } + } + } } plMoviePlayer::~plMoviePlayer() { + opus_decoder_destroy(fOpusDecoder); if (fPlate) // The plPlate owns the Mipmap Texture, so it destroys it for us plPlateManager::Instance().DestroyPlate(fPlate); @@ -363,6 +397,10 @@ bool plMoviePlayer::IProcessVideoFrame(const std::vector& frames) bool plMoviePlayer::Start() { + plSceneNode* sceneNode = plClient::GetInstance()->GetCurrentScene(); + if (sceneNode != nullptr) + sceneNode->GetKey(); + #ifdef VPX_AVAILABLE if (!IOpenMovie()) return false; @@ -374,6 +412,12 @@ bool plMoviePlayer::Start() else return false; + //initialize opus + int error; + fOpusDecoder = opus_decoder_create(48000, 1, &error); + if (error != OPUS_OK) + hsAssert(false, "Error occured initalizing opus"); + // Need to figure out scaling based on pipe size. plPlateManager& plateMgr = plPlateManager::Instance(); const mkvparser::VideoTrack* video = static_cast(fSegment->GetTracks()->GetTrackByNumber(fVideoTrack->number)); @@ -383,6 +427,9 @@ bool plMoviePlayer::Start() plateMgr.CreatePlate(&fPlate, fPosition.fX, fPosition.fY, width, height); fPlate->SetVisible(true); fTexture = fPlate->CreateMaterial(static_cast(video->GetWidth()), static_cast(video->GetHeight()), nullptr); + + fPlaying = true; + return true; #else return false; @@ -391,45 +438,82 @@ bool plMoviePlayer::Start() bool plMoviePlayer::NextFrame() { + if (fPlaying) + { #ifdef VPX_AVAILABLE - // Get our current timecode - int64_t movieTime = 0; - if (fStartTime == 0) - fStartTime = static_cast(hsTimer::GetMilliSeconds()); - else - movieTime = GetMovieTime(); + // Get our current timecode + int64_t movieTime = 0; + if (fStartTime == 0) + fStartTime = static_cast(hsTimer::GetMilliSeconds()); + else + movieTime = GetMovieTime(); - std::vector audio; - std::vector video; - { - uint8_t tracksWithData = 0; - if (fAudioTrack) + std::vector audio; + std::vector video; { - if (fAudioTrack->GetFrames(this, movieTime, audio)) - tracksWithData++; - } - if (fVideoTrack) - { - if (fVideoTrack->GetFrames(this, movieTime, video)) - tracksWithData++; + uint8_t tracksWithData = 0; + if (fAudioTrack) + { + if (fAudioTrack->GetFrames(this, movieTime, audio)) + tracksWithData++; + } + if (fVideoTrack) + { + if (fVideoTrack->GetFrames(this, movieTime, video)) + tracksWithData++; + } + if (tracksWithData == 0) + { + Stop(); + return false; + } } - if (tracksWithData == 0) - return false; - } - // Show our mess - IProcessVideoFrame(video); + // Show our mess + IProcessVideoFrame(video); + IProcessAudioFrame(audio); - return true; + return true; #else - return false; + return false; #endif // VPX_AVAILABLE + } + return false; } bool plMoviePlayer::Stop() { - for (int i = 0; i < fCallbacks.GetCount(); i++) + fPlaying = false; + for (int i = 0; i < fCallbacks.size(); i++) fCallbacks[i]->Send(); - fCallbacks.Reset(); + fCallbacks.clear(); return false; } + +bool plMoviePlayer::IProcessAudioFrame(const std::vector& frames) +{ + const unsigned char* data = NULL; + int32_t size = 0; + for (auto it = frames.begin(); it != frames.end(); ++it) + { + const std::unique_ptr& buf = std::get<0>(*it); + data = buf.get(); + size = std::get<1>(*it); + + int error; + const int frameSize = 5760; //max packet duration at 48kHz + opus_int16* pcm = new opus_int16[frameSize * 1 * sizeof(opus_int16)]; + error = opus_decode(fOpusDecoder, data, size, pcm, frameSize, 0); + if (error < 0) + hsAssert(false, "opus error"); + fAudioSound->UpdateSoundBuffer(reinterpret_cast(pcm), error); + if (plClient::GetInstance()->GetCurrentScene() != nullptr) + { + plSoundMsg* soundMsg = new plSoundMsg(); + soundMsg->SetCmd(plSoundMsg::kPlay); + soundMsg->SetBCastFlag(plMessage::kBCastByType); + fAudioInterface.MsgReceive(soundMsg); + } + } + return true; +} diff --git a/Sources/Plasma/FeatureLib/pfMoviePlayer/plMoviePlayer.h b/Sources/Plasma/FeatureLib/pfMoviePlayer/plMoviePlayer.h index 2c53a4b3..e774c4f1 100644 --- a/Sources/Plasma/FeatureLib/pfMoviePlayer/plMoviePlayer.h +++ b/Sources/Plasma/FeatureLib/pfMoviePlayer/plMoviePlayer.h @@ -48,16 +48,22 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include "hsPoint2.h" #include "hsColorRGBA.h" #include "plMessage/plMovieMsg.h" +#include "plAudio/plWin32VideoSound.h" +#include "plAudible/plVideoAudible.h" +#include "pnSceneObject/plAudioInterface.h" #include #include #include +#include + namespace mkvparser { class BlockEntry; class MkvReader; class Segment; + class Track; } typedef std::tuple, int32_t> blkbuf_t; @@ -79,9 +85,17 @@ protected: hsPoint2 fPosition, fScale; plFileName fMoviePath; + OpusDecoder* fOpusDecoder; + std::shared_ptr fAudioSound; + plVideoAudible fAudioPlayer; + plAudioInterface fAudioInterface; + + bool fPlaying; + int64_t GetMovieTime() const; bool IOpenMovie(); bool IProcessVideoFrame(const std::vector& frames); + bool IProcessAudioFrame(const std::vector& frames); public: plMoviePlayer(); @@ -92,7 +106,7 @@ public: bool Stop(); bool NextFrame(); - void AddCallback(plMessage* msg) { hsRefCnt_SafeRef(msg); fCallbacks.Append(msg); } + void AddCallback(plMessage* msg) { hsRefCnt_SafeRef(msg); fCallbacks.push_back(msg); } uint32_t GetNumCallbacks() const { return 0; } plMessage* GetCallback(int i) const { return nullptr; } @@ -118,7 +132,7 @@ public: void SetFadeToColor(hsColorRGBA c) { } private: - hsTArray fCallbacks; + std::vector fCallbacks; }; #endif // _plMoviePlayer_inc diff --git a/Sources/Plasma/PubUtilLib/plAudio/CMakeLists.txt b/Sources/Plasma/PubUtilLib/plAudio/CMakeLists.txt index d29ffb16..bdbfdf1d 100644 --- a/Sources/Plasma/PubUtilLib/plAudio/CMakeLists.txt +++ b/Sources/Plasma/PubUtilLib/plAudio/CMakeLists.txt @@ -23,6 +23,7 @@ set(plAudio_SOURCES plWin32StaticSound.cpp plWin32StreamingSound.cpp plWinMicLevel.cpp + plWin32VideoSound.cpp ) set(plAudio_HEADERS @@ -40,6 +41,7 @@ set(plAudio_HEADERS plWin32StaticSound.h plWin32StreamingSound.h plWinMicLevel.h + plWin32VideoSound.h ) add_library(plAudio STATIC ${plAudio_SOURCES} ${plAudio_HEADERS}) diff --git a/Sources/Plasma/PubUtilLib/plAudio/plWin32VideoSound.cpp b/Sources/Plasma/PubUtilLib/plAudio/plWin32VideoSound.cpp new file mode 100644 index 00000000..20ae005d --- /dev/null +++ b/Sources/Plasma/PubUtilLib/plAudio/plWin32VideoSound.cpp @@ -0,0 +1,69 @@ +#include +#include + +#include "hsResMgr.h" +#include "plFormat.h" + +#include "plDSoundBuffer.h" + +#include "plWin32VideoSound.h" + +static int uniqueID = 0; +plWin32VideoSound::plWin32VideoSound() : plWin32Sound() +{ + plWAVHeader header; + header.fFormatTag = 0x1; + header.fNumChannels = 1; + header.fBitsPerSample = 16; + fDSoundBuffer = new plDSoundBuffer(0, header, false, false, false, true); + fDSoundBuffer->SetupVoiceSource(); + uniqueID++; + hsgResMgr::ResMgr()->NewKey(plFormat("videosound#{}", uniqueID), this, plLocation::kGlobalFixedLoc); + fSoftVolume = 1.0f; +} + +plWin32VideoSound::~plWin32VideoSound() +{ + delete fDSoundBuffer; +} + +void plWin32VideoSound::UpdateSoundBuffer(unsigned char* buffer, size_t size) +{ + unsigned int bufferID = 0; + + if (fDSoundBuffer->GetAvailableBufferId(&bufferID)) + { + fDSoundBuffer->VoiceFillBuffer(buffer, size, bufferID); + IActuallyPlay(); + } +} + +void plWin32VideoSound::IDerivedActuallyPlay() +{ + if (!fDSoundBuffer->IsPlaying()) + fDSoundBuffer->Play(); +} + +bool plWin32VideoSound::LoadSound(bool is3D) +{ + hsAssert(false, "unimplemented cause unnecessary for this class"); + return false; +} + +void plWin32VideoSound::SetStartPos(unsigned bytes) +{ + + //do nothing + hsAssert(false, "unimplemented cause unnecessary for this class"); +} + +float plWin32VideoSound::GetActualTimeSec() +{ + hsAssert(false, "unimplemented cause unnecessary for this class"); + return 0; +} + +void plWin32VideoSound::ISetActualTime(double t) +{ + hsAssert(false, "unimplemented cause unnecessary for this class"); +} \ No newline at end of file diff --git a/Sources/Plasma/PubUtilLib/plAudio/plWin32VideoSound.h b/Sources/Plasma/PubUtilLib/plAudio/plWin32VideoSound.h new file mode 100644 index 00000000..0884152d --- /dev/null +++ b/Sources/Plasma/PubUtilLib/plAudio/plWin32VideoSound.h @@ -0,0 +1,63 @@ +/*==LICENSE==* + +CyanWorlds.com Engine - MMOG client, server and tools +Copyright (C) 2011 Cyan Worlds, Inc. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Additional permissions under GNU GPL version 3 section 7 + +If you modify this Program, or any covered work, by linking or +combining it with any of RAD Game Tools Bink SDK, Autodesk 3ds Max SDK, +NVIDIA PhysX SDK, Microsoft DirectX SDK, OpenSSL library, Independent +JPEG Group JPEG library, Microsoft Windows Media SDK, or Apple QuickTime SDK +(or a modified version of those libraries), +containing parts covered by the terms of the Bink SDK EULA, 3ds Max EULA, +PhysX SDK EULA, DirectX SDK EULA, OpenSSL and SSLeay licenses, IJG +JPEG Library README, Windows Media SDK EULA, or QuickTime SDK EULA, the +licensors of this Program grant you additional +permission to convey the resulting work. Corresponding Source for a +non-source form of such a combination shall include the source code for +the parts of OpenSSL and IJG JPEG Library used as well as that of the covered +work. + +You can contact Cyan Worlds, Inc. by email legal@cyan.com +or by snail mail at: +Cyan Worlds, Inc. +14617 N Newport Hwy +Mead, WA 99021 + +*==LICENSE==*/ +#ifndef plWin32VideoSound_h +#define plWin32VideoSound_h + +#include "plWin32Sound.h" + +class plWin32VideoSound : public plWin32Sound +{ +public: + plWin32VideoSound(); + virtual ~plWin32VideoSound(); + + void UpdateSoundBuffer(unsigned char* buffer, size_t size); + +protected: + void IDerivedActuallyPlay(void); + bool LoadSound(bool is3D); + void SetStartPos(unsigned bytes); + float GetActualTimeSec(); + void ISetActualTime(double t); + +}; +#endif diff --git a/cmake/FindOpus.cmake b/cmake/FindOpus.cmake new file mode 100644 index 00000000..6864a304 --- /dev/null +++ b/cmake/FindOpus.cmake @@ -0,0 +1,38 @@ +if(Opus_INCLUDE_DIR AND Opus_LIBRARY) + set(Opus_FIND_QUIETLY TRUE) +endif() + + +find_path(Opus_INCLUDE_DIR opus.h + /usr/local/include + /usr/include +) + +find_library(Opus_LIBRARY NAMES opus + PATHS /usr/local/lib /usr/lib) + +find_library(Celt_LIBRARY NAMES celt + PATHS /usr/local/lib /usr/lib) + +find_library(Silk_LIBRARY NAMES silk_common + PATHS /usr/local/lib /usr/lib) + +set(Opus_LIBRARIES + ${Opus_LIBRARY} + ${Celt_LIBRARY} + ${Silk_LIBRARY}) + + +if(Opus_INCLUDE_DIR AND Opus_LIBRARY) + set(Opus_FOUND TRUE) +endif() + +if (Opus_FOUND) + if(NOT Opus_FIND_QUIETLY) + message(STATUS "Found libopus: ${Opus_INCLUDE_DIR}") + endif() +else() + if(Opus_FIND_REQUIRED) + message(FATAL_ERROR "Could not find libopus") + endif() +endif() \ No newline at end of file