diff --git a/Sources/Plasma/Apps/plClient/plClient.cpp b/Sources/Plasma/Apps/plClient/plClient.cpp index 2ffacb11..39b5b5a4 100644 --- a/Sources/Plasma/Apps/plClient/plClient.cpp +++ b/Sources/Plasma/Apps/plClient/plClient.cpp @@ -317,10 +317,9 @@ hsBool plClient::Shutdown() plAgeLoader::SetInstance(nil); } - if (pfSecurePreloader::IsInstanced()) + if (pfSecurePreloader::GetInstance()) { - pfSecurePreloader::GetInstance()->Shutdown(); - // pfSecurePreloader handles its own fixed key unregistration + pfSecurePreloader::GetInstance()->Shutdown(); // will unregister itself } if (fInputManager) @@ -2517,6 +2516,8 @@ void plClient::ICompleteInit () { void plClient::IHandlePreloaderMsg (plPreloaderMsg * msg) { plgDispatch::Dispatch()->UnRegisterForExactType(plPreloaderMsg::Index(), GetKey()); + if (pfSecurePreloader* sp = pfSecurePreloader::GetInstance()) + sp->Shutdown(); if (!msg->fSuccess) { char str[1024]; @@ -2555,7 +2556,5 @@ void plClient::IHandleNetCommAuthMsg (plNetCommAuthMsg * msg) { plgDispatch::Dispatch()->RegisterForExactType(plPreloaderMsg::Index(), GetKey()); // Precache our secure files - pfSecurePreloader::GetInstance()->RequestFileGroup(L"Python", L"pak"); - pfSecurePreloader::GetInstance()->RequestFileGroup(L"SDL", L"sdl"); pfSecurePreloader::GetInstance()->Start(); } diff --git a/Sources/Plasma/CoreLib/hsStream.cpp b/Sources/Plasma/CoreLib/hsStream.cpp index 32cd5317..8593ca4b 100644 --- a/Sources/Plasma/CoreLib/hsStream.cpp +++ b/Sources/Plasma/CoreLib/hsStream.cpp @@ -1054,10 +1054,7 @@ hsBool hsRAMStream::AtEnd() UInt32 hsRAMStream::Read(UInt32 byteCount, void * buffer) { if (fBytesRead + byteCount > fAppender.Count() * fAppender.ElemSize()) - { - hsThrow("Attempting to read past end of stream"); byteCount = (fAppender.Count() * fAppender.ElemSize()) - fBytesRead; - } fBytesRead += byteCount; fPosition += byteCount; diff --git a/Sources/Plasma/FeatureLib/pfSecurePreloader/pfSecurePreloader.cpp b/Sources/Plasma/FeatureLib/pfSecurePreloader/pfSecurePreloader.cpp index 158d12a9..982d90d7 100644 --- a/Sources/Plasma/FeatureLib/pfSecurePreloader/pfSecurePreloader.cpp +++ b/Sources/Plasma/FeatureLib/pfSecurePreloader/pfSecurePreloader.cpp @@ -39,608 +39,401 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com Mead, WA 99021 *==LICENSE==*/ -#include "hsSTLStream.h" -#include "hsResMgr.h" + +#include "pfSecurePreloader.h" + +#include "hsStream.h" #include "plgDispatch.h" -#include "pnUtils/pnUtils.h" -#include "pnNetBase/pnNetBase.h" -#include "pnAsyncCore/pnAsyncCore.h" -#include "pnNetCli/pnNetCli.h" -#include "plNetGameLib/plNetGameLib.h" +#include "plCompression/plZlibStream.h" +#include "plEncryption/plChecksum.h" #include "plFile/plFileUtils.h" +#include "plFile/plSecureStream.h" #include "plFile/plStreamSource.h" -#include "plNetCommon/plNetCommon.h" -#include "plProgressMgr/plProgressMgr.h" -#include "plMessage/plPreloaderMsg.h" #include "plMessage/plNetCommMsgs.h" -#include "pfSecurePreloader.h" +#include "plMessage/plPreloaderMsg.h" +#include "plProgressMgr/plProgressMgr.h" -#include "plNetClientComm/plNetClientComm.h" +extern hsBool gDataServerLocal; +pfSecurePreloader* pfSecurePreloader::fInstance = nil; -extern hsBool gDataServerLocal; +///////////////////////////////////////////////////////////////////// +typedef std::pair WcharPair; -// Max number of concurrent file downloads -static const unsigned kMaxConcurrency = 1; +struct AuthRequestParams +{ + pfSecurePreloader* fThis; + std::queue fFileGroups; -pfSecurePreloader * pfSecurePreloader::fInstance; + AuthRequestParams(pfSecurePreloader* parent) + : fThis(parent) { } +}; -/////////////////////////////////////////////////////////////////////////////// -// Callback routines for the network code +///////////////////////////////////////////////////////////////////// -// Called when a file's info is retrieved from the server -static void DefaultFileListRequestCallback(ENetError result, void* param, const NetCliAuthFileInfo infoArr[], unsigned infoCount) -{ - bool success = !IS_NET_ERROR(result); - - std::vector filenames; - std::vector sizes; - if (success) +void ProcAuthDownloadParams(AuthRequestParams* params); + +void GotAuthSrvManifest( + ENetError result, + void* param, + const NetCliAuthFileInfo infoArr[], + UInt32 infoCount +) { + AuthRequestParams* arp = (AuthRequestParams*)param; + if (IS_NET_ERROR(result)) { - filenames.reserve(infoCount); - sizes.reserve(infoCount); - for (unsigned curFile = 0; curFile < infoCount; curFile++) - { - filenames.push_back(infoArr[curFile].filename); - sizes.push_back(infoArr[curFile].filesize); - } + FATAL("Failed to get AuthSrv manifest!"); + arp->fThis->Terminate(); + delete arp; + } else { + arp->fThis->PreloadManifest(infoArr, infoCount); + ProcAuthDownloadParams(arp); } - ((pfSecurePreloader*)param)->RequestFinished(filenames, sizes, success); } -// Called when a file download is either finished, or failed -static void DefaultFileRequestCallback(ENetError result, void* param, const wchar filename[], hsStream* stream) -{ - // Retry download unless shutting down or file not found - switch (result) { - case kNetSuccess: - ((pfSecurePreloader*)param)->FinishedDownload(filename, true); - break; - - case kNetErrFileNotFound: - case kNetErrRemoteShutdown: - ((pfSecurePreloader*)param)->FinishedDownload(filename, false); - break; - - default: - stream->Rewind(); - NetCliAuthFileRequest( - filename, - stream, - &DefaultFileRequestCallback, - param - ); - break; +void GotFileSrvManifest( + ENetError result, + void* param, + const wchar_t group[], + const NetCliFileManifestEntry manifest[], + UInt32 entryCount +) { + pfSecurePreloader* sp = (pfSecurePreloader*)param; + if (result == kNetErrFileNotFound) + { + AuthRequestParams* params = new AuthRequestParams(sp); + params->fFileGroups.push(WcharPair(L"Python", L"pak")); + params->fFileGroups.push(WcharPair(L"SDL", L"sdl")); + ProcAuthDownloadParams(params); + return; + } else if (!entryCount) { + FATAL("SecurePreloader manifest empty!"); + sp->Terminate(); + return; } + + sp->PreloadManifest(manifest, entryCount); } +void FileDownloaded( + ENetError result, + void* param, + const wchar_t filename[], + hsStream* writer +) { + pfSecurePreloader* sp = (pfSecurePreloader*)param; + if (IS_NET_ERROR(result)) + { + FATAL("SecurePreloader download failed"); + sp->Terminate(); + } else { + sp->FilePreloaded(filename, writer); + } +} -/////////////////////////////////////////////////////////////////////////////// -// Our custom stream for writing directly to disk securely, and updating the -// progress bar. Does NOT support reading (cause it doesn't need to) -class Direct2DiskStream : public hsUNIXStream +void ProcAuthDownloadParams(AuthRequestParams* params) { -protected: - wchar * fWriteFileName; + // Request the "manifests" until there are none left, then download the files + if (params->fFileGroups.empty()) + { + params->fThis->PreloadNextFile(); + delete params; + } else { + WcharPair wp = params->fFileGroups.front(); + params->fFileGroups.pop(); + NetCliAuthFileListRequest(wp.first, wp.second, GotAuthSrvManifest, params); + } +} - pfSecurePreloader* fPreloader; +///////////////////////////////////////////////////////////////////// + +class pfSecurePreloaderStream : public plZlibStream +{ + plOperationProgress* fProgress; + bool fIsZipped; public: - Direct2DiskStream(pfSecurePreloader* preloader); - ~Direct2DiskStream(); - - virtual hsBool Open(const char* name, const char* mode = "wb"); - virtual hsBool Open(const wchar* name, const wchar* mode = L"wb"); - virtual hsBool Close(); - virtual UInt32 Read(UInt32 byteCount, void* buffer); - virtual UInt32 Write(UInt32 byteCount, const void* buffer); -}; + pfSecurePreloaderStream(plOperationProgress* prog, bool zipped) + : fProgress(prog), fIsZipped(zipped), plZlibStream() + { + fOutput = new hsRAMStream; + } -Direct2DiskStream::Direct2DiskStream(pfSecurePreloader* preloader) : -fWriteFileName(nil), -fPreloader(preloader) -{} + ~pfSecurePreloaderStream() + { + delete fOutput; + fOutput = nil; + plZlibStream::Close(); + } -Direct2DiskStream::~Direct2DiskStream() -{ - Close(); -} + hsBool AtEnd() { return fOutput->AtEnd(); } + UInt32 GetEOF() { return fOutput->GetEOF(); } + UInt32 GetPosition() const { return fOutput->GetPosition(); } + UInt32 GetSizeLeft() const { return fOutput->GetSizeLeft(); } + UInt32 Read(UInt32 count, void* buf) { return fOutput->Read(count, buf); } + void Rewind() { fOutput->Rewind(); } + void SetPosition(UInt32 pos) { fOutput->SetPosition(pos); } + void Skip(UInt32 deltaByteCount) { fOutput->Skip(deltaByteCount); } -hsBool Direct2DiskStream::Open(const char* name, const char* mode) -{ - wchar* wName = hsStringToWString(name); - wchar* wMode = hsStringToWString(mode); - hsBool ret = Open(wName, wMode); - delete [] wName; - delete [] wMode; - return ret; -} + UInt32 Write(UInt32 count, const void* buf) + { + if (fProgress) + fProgress->Increment((hsScalar)count); + if (fIsZipped) + return plZlibStream::Write(count, buf); + else + return fOutput->Write(count, buf); + } +}; + +///////////////////////////////////////////////////////////////////// -hsBool Direct2DiskStream::Open(const wchar* name, const wchar* mode) +pfSecurePreloader::pfSecurePreloader() + : fProgress(nil), fLegacyMode(false) +{ } + +pfSecurePreloader::~pfSecurePreloader() { - if (0 != wcscmp(mode, L"wb")) { - hsAssert(0, "Unsupported open mode"); - return false; + while (fDownloadEntries.size()) + { + free((void*)fDownloadEntries.front()); + fDownloadEntries.pop(); + } + + while (fManifestEntries.size()) + { + free((void*)fManifestEntries.front()); + fManifestEntries.pop(); } - - fWriteFileName = TRACKED_NEW(wchar[wcslen(name) + 1]); - wcscpy(fWriteFileName, name); - -// LogMsg(kLogPerf, L"Opening disk file %S", fWriteFileName); - return hsUNIXStream::Open(name, mode); } -hsBool Direct2DiskStream::Close() +hsRAMStream* pfSecurePreloader::LoadToMemory(const wchar_t* file) const { - delete [] fWriteFileName; - fWriteFileName = nil; - return hsUNIXStream::Close(); + if (!plFileUtils::FileExists(file)) + return nil; + + hsUNIXStream s; + hsRAMStream* ram = new hsRAMStream; + s.Open(file); + + UInt32 loadLen = 1024 * 1024; + UInt8* buf = new UInt8[loadLen]; + while (UInt32 read = s.Read(loadLen, buf)) + ram->Write(read, buf); + delete[] buf; + + s.Close(); + ram->Rewind(); + return ram; } -UInt32 Direct2DiskStream::Read(UInt32 bytes, void* buffer) +void pfSecurePreloader::SaveFile(hsStream* file, const wchar_t* name) const { - hsAssert(0, "not implemented"); - return 0; // we don't read + hsUNIXStream s; + s.Open(name, L"wb"); + UInt32 pos = file->GetPosition(); + file->Rewind(); + + UInt32 loadLen = 1024 * 1024; + UInt8* buf = new UInt8[loadLen]; + while (UInt32 read = file->Read(loadLen, buf)) + s.Write(read, buf); + file->SetPosition(pos); + s.Close(); + delete[] buf; } -UInt32 Direct2DiskStream::Write(UInt32 bytes, const void* buffer) +bool pfSecurePreloader::IsZipped(const wchar_t* filename) const { -// LogMsg(kLogPerf, L"Writing %u bytes to disk file %S", bytes, fWriteFileName); - fPreloader->UpdateProgressBar(bytes); - return hsUNIXStream::Write(bytes, buffer); + return wcscmp(plFileUtils::GetFileExt(filename), L"gz") == 0; } - -/////////////////////////////////////////////////////////////////////////////// -// secure preloader class implementation - -// closes and deletes all streams -void pfSecurePreloader::ICleanupStreams() +void pfSecurePreloader::PreloadNextFile() { - if (fD2DStreams.size() > 0) + if (fManifestEntries.empty()) { - std::map::iterator curStream; - for (curStream = fD2DStreams.begin(); curStream != fD2DStreams.end(); curStream++) - { - curStream->second->Close(); - delete curStream->second; - curStream->second = nil; - } - fD2DStreams.clear(); + Finish(); + return; } -} - -// queues a single file to be preloaded (does nothing if already preloaded) -void pfSecurePreloader::RequestSingleFile(std::wstring filename) -{ - fileRequest request; - ZERO(request); - request.fType = fileRequest::kSingleFile; - request.fPath = filename; - request.fExt = L""; - fRequests.push_back(request); + const wchar_t* filename = fDownloadEntries.front(); + hsStream* s = new pfSecurePreloaderStream(fProgress, IsZipped(filename)); + + // Thankfully, both callbacks have the same arguments + if (fLegacyMode) + NetCliAuthFileRequest(filename, s, FileDownloaded, this); + else + NetCliFileDownloadRequest(filename, s, FileDownloaded, this); } -// queues a group of files to be preloaded (does nothing if already preloaded) -void pfSecurePreloader::RequestFileGroup(std::wstring dir, std::wstring ext) +void pfSecurePreloader::Init() { - fileRequest request; - ZERO(request); - request.fType = fileRequest::kFileList; - request.fPath = dir; - request.fExt = ext; - - fRequests.push_back(request); + RegisterAs(kSecurePreloader_KEY); + // TODO: If we're going to support reconnects, then let's do it right. + // Later... + //plgDispatch::Dispatch()->RegisterForExactType(plNetCommAuthConnectedMsg::Index(), GetKey()); } -// preloads all requested files from the server (does nothing if already preloaded) void pfSecurePreloader::Start() { - if (gDataServerLocal) { - // using local data, don't do anything - plPreloaderMsg * msg = TRACKED_NEW plPreloaderMsg(); - msg->fSuccess = true; - msg->Send(); +#ifndef PLASMA_EXTERNAL_RELEASE + // Using local data? Move along, move along... + if (gDataServerLocal) + { + Finish(); return; } +#endif - NetCliAuthGetEncryptionKey(fEncryptionKey, 4); // grab the encryption key from the server - - fNetError = false; - - // make sure we are all cleaned up - ICleanupStreams(); - fTotalDataReceived = 0; - - // update the progress bar for downloading - if (!fProgressBar) - fProgressBar = plProgressMgr::GetInstance()->RegisterOperation((hsScalar)(fRequests.size()), "Getting file info...", plProgressMgr::kUpdateText, false, true); + NetCliAuthGetEncryptionKey(fEncryptionKey, 4); - for (unsigned curRequest = 0; curRequest < fRequests.size(); curRequest++) - { - fNumInfoRequestsRemaining++; // increment the counter - if (fRequests[curRequest].fType == fileRequest::kSingleFile) - { -#ifndef PLASMA_EXTERNAL_RELEASE - // in internal releases, we can use on-disk files if they exist - if (plFileUtils::FileExists(fRequests[curRequest].fPath.c_str())) - { - fileInfo info; - info.fOriginalNameAndPath = fRequests[curRequest].fPath; - info.fSizeInBytes = plFileUtils::GetFileSize(info.fOriginalNameAndPath.c_str()); - info.fDownloading = false; - info.fDownloaded = false; - info.fLocal = true; - - // generate garbled name - wchar_t pathBuffer[MAX_PATH + 1]; - wchar_t filename[arrsize(pathBuffer)]; - GetTempPathW(arrsize(pathBuffer), pathBuffer); - GetTempFileNameW(pathBuffer, L"CYN", 0, filename); - info.fGarbledNameAndPath = filename; - - fTotalDataDownload += info.fSizeInBytes; - - fFileInfoMap[info.fOriginalNameAndPath] = info; - } - // internal client will still request it, even if it exists locally, - // so that things get updated properly -#endif // PLASMA_EXTERNAL_RELEASE - NetCliAuthFileListRequest( - fRequests[curRequest].fPath.c_str(), - nil, - &DefaultFileListRequestCallback, - (void*)this - ); - } - else - { -#ifndef PLASMA_EXTERNAL_RELEASE - // in internal releases, we can use on-disk files if they exist - // Build the search string as "dir\\*.ext" - wchar searchStr[MAX_PATH]; - - PathAddFilename(searchStr, fRequests[curRequest].fPath.c_str(), L"*", arrsize(searchStr)); - PathSetExtension(searchStr, searchStr, fRequests[curRequest].fExt.c_str(), arrsize(searchStr)); - - ARRAY(PathFind) paths; - PathFindFiles(&paths, searchStr, kPathFlagFile); // find all files that match - - // convert it to our little file info array - PathFind* curFile = paths.Ptr(); - PathFind* lastFile = paths.Term(); - while (curFile != lastFile) { - fileInfo info; - info.fOriginalNameAndPath = curFile->name; - info.fSizeInBytes = (UInt32)curFile->fileLength; - info.fDownloading = false; - info.fDownloaded = false; - info.fLocal = true; - - // generate garbled name - wchar_t pathBuffer[MAX_PATH + 1]; - wchar_t filename[arrsize(pathBuffer)]; - GetTempPathW(arrsize(pathBuffer), pathBuffer); - GetTempFileNameW(pathBuffer, L"CYN", 0, filename); - info.fGarbledNameAndPath = filename; - - fTotalDataDownload += info.fSizeInBytes; - - fFileInfoMap[info.fOriginalNameAndPath] = info; - curFile++; - } -#endif // PLASMA_EXTERNAL_RELEASE - - NetCliAuthFileListRequest( - fRequests[curRequest].fPath.c_str(), - fRequests[curRequest].fExt.c_str(), - &DefaultFileListRequestCallback, - (void*)this - ); - } - } + // TODO: Localize + fProgress = plProgressMgr::GetInstance()->RegisterOperation(0.0f, "Checking for Updates", plProgressMgr::kUpdateText, false, true); + + // Now, we need to fetch the "SecurePreloader" manifest from the file server, which will contain the python and SDL files. + // We're basically reimplementing plResPatcher here, except preferring to keep everything in memory, then flush to disk + // when we're done. If this fails, then we shall download everything from the AuthSrv like in the old days. + NetCliFileManifestRequest(GotFileSrvManifest, this, L"SecurePreloader"); } -// closes all file pointers and cleans up after itself -void pfSecurePreloader::Cleanup() +void pfSecurePreloader::Terminate() { - ICleanupStreams(); - - fRequests.clear(); - fFileInfoMap.clear(); + FATAL("pfSecurePreloader failure"); - fNumInfoRequestsRemaining = 0; - fTotalDataDownload = 0; - fTotalDataReceived = 0; + plPreloaderMsg* msg = new plPreloaderMsg; + msg->fSuccess = false; + plgDispatch::Dispatch()->MsgSend(msg); +} - DEL(fProgressBar); - fProgressBar = nil; +void pfSecurePreloader::Finish() +{ + plPreloaderMsg* msg = new plPreloaderMsg; + msg->fSuccess = true; + plgDispatch::Dispatch()->MsgSend(msg); } -//============================================================================ -void pfSecurePreloader::RequestFinished(const std::vector & filenames, const std::vector & sizes, bool succeeded) +void pfSecurePreloader::Shutdown() { - fNetError |= !succeeded; - - if (succeeded) + SetInstance(nil); + if (fProgress) { - unsigned count = 0; - for (int curFile = 0; curFile < filenames.size(); curFile++) - { - if (fFileInfoMap.find(filenames[curFile]) != fFileInfoMap.end()) - continue; // if it is a duplicate, ignore it (the duplicate is probably one we found locally) - - fileInfo info; - info.fOriginalNameAndPath = filenames[curFile]; - info.fSizeInBytes = sizes[curFile]; - info.fDownloading = false; - info.fDownloaded = false; - info.fLocal = false; // if we get here, it was retrieved remotely - - // generate garbled name - wchar_t pathBuffer[MAX_PATH + 1]; - wchar_t filename[arrsize(pathBuffer)]; - GetTempPathW(arrsize(pathBuffer), pathBuffer); - GetTempFileNameW(pathBuffer, L"CYN", 0, filename); - info.fGarbledNameAndPath = filename; - - fTotalDataDownload += info.fSizeInBytes; - - fFileInfoMap[info.fOriginalNameAndPath] = info; - ++count; - } - LogMsg(kLogPerf, "Added %u files to secure download queue", count); + delete fProgress; + fProgress = nil; } - if (fProgressBar) - fProgressBar->Increment(1.f); - - --fNumInfoRequestsRemaining; // even if we fail, decrement the counter - if (succeeded) { - DEL(fProgressBar); - fProgressBar = plProgressMgr::GetInstance()->RegisterOperation((hsScalar)(fTotalDataDownload), "Downloading...", plProgressMgr::kUpdateText, false, true); - - // Issue some file download requests (up to kMaxConcurrency) - IIssueDownloadRequests(); - } - else { - IPreloadComplete(); - } + // Takes care of UnReffing us + UnRegisterAs(kSecurePreloader_KEY); } -//============================================================================ -void pfSecurePreloader::IIssueDownloadRequests () { +void pfSecurePreloader::PreloadManifest(const NetCliAuthFileInfo manifestEntries[], UInt32 entryCount) +{ + UInt32 totalBytes = 0; + if (fProgress) + totalBytes = (UInt32)fProgress->GetMax(); + fLegacyMode = true; - std::map::iterator curFile; - for (curFile = fFileInfoMap.begin(); curFile != fFileInfoMap.end(); curFile++) + for (UInt32 i = 0; i < entryCount; ++i) { - // Skip files already downloaded or currently downloading - if (curFile->second.fDownloaded || curFile->second.fDownloading) - continue; - - std::wstring filename = curFile->second.fOriginalNameAndPath; -#ifndef PLASMA_EXTERNAL_RELEASE - // in internal releases, we can use on-disk files if they exist - if (plFileUtils::FileExists(filename.c_str())) + const NetCliAuthFileInfo mfs = manifestEntries[i]; + fDownloadEntries.push(wcsdup(mfs.filename)); + if (IsZipped(mfs.filename)) { - // don't bother streaming, just make the secure stream using the local file - - // a local key overrides the server-downloaded key - UInt32 localKey[4]; - bool hasLocalKey = plFileUtils::GetSecureEncryptionKey(filename.c_str(), localKey, arrsize(localKey)); - hsStream* stream = nil; - if (hasLocalKey) - stream = plSecureStream::OpenSecureFile(filename.c_str(), 0, localKey); - else - stream = plSecureStream::OpenSecureFile(filename.c_str(), 0, fEncryptionKey); - - // add it to the stream source - bool added = plStreamSource::GetInstance()->InsertFile(filename.c_str(), stream); - if (!added) - DEL(stream); // wasn't added, so nuke our local copy + wchar_t* name = wcsdup(mfs.filename); + plFileUtils::StripExt(name); + fManifestEntries.push(name); + + } else + fManifestEntries.push(wcsdup(mfs.filename)); - // and make sure the vars are set up right - curFile->second.fDownloaded = true; - curFile->second.fLocal = true; - } - else -#endif - { - // Enforce concurrency limit - if (fNumDownloadRequestsRemaining >= kMaxConcurrency) - break; - - curFile->second.fDownloading = true; - curFile->second.fDownloaded = false; - curFile->second.fLocal = false; - - // create and setup the stream - Direct2DiskStream* fileStream = TRACKED_NEW Direct2DiskStream(this); - fileStream->Open(curFile->second.fGarbledNameAndPath.c_str(), L"wb"); - fD2DStreams[filename] = (hsStream*)fileStream; - - // request the file from the server - LogMsg(kLogPerf, L"Requesting secure file:%s", filename.c_str()); - ++fNumDownloadRequestsRemaining; - NetCliAuthFileRequest( - filename.c_str(), - (hsStream*)fileStream, - &DefaultFileRequestCallback, - this - ); - } + totalBytes += mfs.filesize; } - - if (!fNumDownloadRequestsRemaining) - IPreloadComplete(); -} - -void pfSecurePreloader::UpdateProgressBar(UInt32 bytesReceived) -{ - fTotalDataReceived += bytesReceived; - if (fTotalDataReceived > fTotalDataDownload) - fTotalDataReceived = fTotalDataDownload; // shouldn't happen... but just in case - if (fProgressBar) - fProgressBar->Increment((hsScalar)bytesReceived); + if (fProgress) + { + fProgress->SetLength((hsScalar)totalBytes); + fProgress->SetTitle("Downloading..."); + } } -void pfSecurePreloader::FinishedDownload(std::wstring filename, bool succeeded) +void pfSecurePreloader::PreloadManifest(const NetCliFileManifestEntry manifestEntries[], UInt32 entryCount) { - for (;;) + UInt32 totalBytes = 0; + for (UInt32 i = 0; i < entryCount; ++i) { - if (fFileInfoMap.find(filename) == fFileInfoMap.end()) + const NetCliFileManifestEntry mfs = manifestEntries[i]; + bool fetchMe = true; + hsRAMStream* s = nil; + + if (plFileUtils::FileExists(mfs.clientName)) { - // file doesn't exist... abort - succeeded = false; - break; + s = LoadToMemory(mfs.clientName); + if (s) + { + // Damn this + const char* md5 = hsWStringToString(mfs.md5); + plMD5Checksum srvHash; + srvHash.SetFromHexString(md5); + delete[] md5; + + // Now actually copare the hashes + plMD5Checksum lclHash; + lclHash.CalcFromStream(s); + fetchMe = (srvHash != lclHash); + } } - fFileInfoMap[filename].fDownloading = false; - - // close and delete the writer stream (even if we failed) - fD2DStreams[filename]->Close(); - delete fD2DStreams[filename]; - fD2DStreams.erase(fD2DStreams.find(filename)); - - if (succeeded) + if (fetchMe) { - // open a secure stream to that file - hsStream* stream = plSecureStream::OpenSecureFile( - fFileInfoMap[filename].fGarbledNameAndPath.c_str(), - plSecureStream::kRequireEncryption | plSecureStream::kDeleteOnExit, // force delete and encryption - fEncryptionKey - ); - - bool addedToSource = plStreamSource::GetInstance()->InsertFile(filename.c_str(), stream); - if (!addedToSource) - DEL(stream); // cleanup if it wasn't added - - fFileInfoMap[filename].fDownloaded = true; - break; + fManifestEntries.push(wcsdup(mfs.clientName)); + fDownloadEntries.push(wcsdup(mfs.downloadName)); + if (IsZipped(mfs.downloadName)) + totalBytes += mfs.zipSize; + else + totalBytes += mfs.fileSize; + } else { + plSecureStream* ss = new plSecureStream(s, fEncryptionKey); + plStreamSource::GetInstance()->InsertFile(mfs.clientName, ss); } - - // file download failed, clean up after it - - // delete the temporary file - if (plFileUtils::FileExists(fFileInfoMap[filename].fGarbledNameAndPath.c_str())) - plFileUtils::RemoveFile(fFileInfoMap[filename].fGarbledNameAndPath.c_str(), true); - // and remove it from the info map - fFileInfoMap.erase(fFileInfoMap.find(filename)); - break; + if (s) + delete s; } - - fNetError |= !succeeded; - --fNumDownloadRequestsRemaining; - LogMsg(kLogPerf, L"Received secure file:%s, success:%s", filename.c_str(), succeeded ? L"Yep" : L"Nope"); - if (!succeeded) - IPreloadComplete(); - else - // Issue some file download requests (up to kMaxConcurrency) - IIssueDownloadRequests(); -} - -//============================================================================ -void pfSecurePreloader::INotifyAuthReconnected () { - - // The secure file download network protocol will now just pick up downloading - // where it left off before the reconnect, so no need to reset in-progress files. - - /* - std::map::iterator curFile; - for (curFile = fFileInfoMap.begin(); curFile != fFileInfoMap.end(); curFile++) { - - // Reset files that were currently downloading - if (curFile->second.fDownloading) - curFile->second.fDownloading = false; - } - - if (fNumDownloadRequestsRemaining > 0) { - - LogMsg(kLogPerf, L"pfSecurePreloader: Auth reconnected, resetting in-progress file downloads"); - - // Issue some file download requests (up to kMaxConcurrency) - IIssueDownloadRequests(); - } - */ -} - -//============================================================================ -void pfSecurePreloader::IPreloadComplete () { - DEL(fProgressBar); - fProgressBar = nil; - - plPreloaderMsg * msg = TRACKED_NEW plPreloaderMsg(); - msg->fSuccess = !fNetError; - msg->Send(); -} - -//============================================================================ -hsBool pfSecurePreloader::MsgReceive (plMessage * msg) { - - if (plNetCommAuthConnectedMsg * authMsg = plNetCommAuthConnectedMsg::ConvertNoRef(msg)) { - - INotifyAuthReconnected(); - return true; - } - - return hsKeyedObject::MsgReceive(msg); -} - -//============================================================================ -pfSecurePreloader * pfSecurePreloader::GetInstance () { - - if (!fInstance) { - - fInstance = NEWZERO(pfSecurePreloader); - fInstance->RegisterAs(kSecurePreloader_KEY); + if (totalBytes && fProgress) + { + fProgress->SetLength((hsScalar)totalBytes); + fProgress->SetTitle("Downloading..."); } - return fInstance; -} - -//============================================================================ -bool pfSecurePreloader::IsInstanced () { - - return fInstance != nil; -} - -//============================================================================ -void pfSecurePreloader::Init () { - - if (!fInitialized) { - - fInitialized = true; - plgDispatch::Dispatch()->RegisterForExactType(plNetCommAuthConnectedMsg::Index(), GetKey()); - } + // This method uses only one manifest, so we're good to go now! + PreloadNextFile(); } -//============================================================================ -void pfSecurePreloader::Shutdown () { +void pfSecurePreloader::FilePreloaded(const wchar_t* file, hsStream* stream) +{ + // Clear out queue + fDownloadEntries.pop(); + const wchar_t* clientName = fManifestEntries.front(); // Stolen by plStreamSource + fManifestEntries.pop(); - if (fInitialized) { - - fInitialized = false; - plgDispatch::Dispatch()->UnRegisterForExactType(plNetCommAuthConnectedMsg::Index(), GetKey()); + if (!fLegacyMode) // AuthSrv data caching is useless + { + plFileUtils::EnsureFilePathExists(clientName); + SaveFile(stream, clientName); } - if (fInstance) { - - fInstance->UnRegister(); - fInstance = nil; - } -} + plSecureStream* ss = new plSecureStream(stream, fEncryptionKey); + plStreamSource::GetInstance()->InsertFile(clientName, ss); + delete stream; // SecureStream holds its own decrypted buffer -//============================================================================ -pfSecurePreloader::pfSecurePreloader () { + // Continue down the warpath + PreloadNextFile(); } -//============================================================================ -pfSecurePreloader::~pfSecurePreloader () { - - Cleanup(); +pfSecurePreloader* pfSecurePreloader::GetInstance() +{ + if (!fInstance) + fInstance = new pfSecurePreloader; + return fInstance; } diff --git a/Sources/Plasma/FeatureLib/pfSecurePreloader/pfSecurePreloader.h b/Sources/Plasma/FeatureLib/pfSecurePreloader/pfSecurePreloader.h index 99c32ee2..56c9d962 100644 --- a/Sources/Plasma/FeatureLib/pfSecurePreloader/pfSecurePreloader.h +++ b/Sources/Plasma/FeatureLib/pfSecurePreloader/pfSecurePreloader.h @@ -42,15 +42,13 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #ifndef __pfSecurePreloader_h__ #define __pfSecurePreloader_h__ -#include "hsTypes.h" -#include "hsStlUtils.h" -#include "hsCritSect.h" -#include "hsStream.h" -#include "plFile/plSecureStream.h" +#include "HeadSpin.h" #include "pnKeyedObject/hsKeyedObject.h" - +#include "plNetGameLib/plNetGameLib.h" +#include class plOperationProgress; +class hsRAMStream; /////////////////////////////////////////////////////////////////////////////// // pfSecurePreloader - a class for handling files we want downloaded from the @@ -60,75 +58,40 @@ class plOperationProgress; class pfSecurePreloader : public hsKeyedObject { private: - static pfSecurePreloader * fInstance; - - struct fileRequest - { - enum requestType {kSingleFile, kFileList}; - requestType fType; - std::wstring fPath; // filename if kSingleFile, path if kFileList - std::wstring fExt; // blank if kSingleFile, extension if kFileList - }; - std::vector fRequests; - - struct fileInfo - { - std::wstring fOriginalNameAndPath; // the human-readable name - std::wstring fGarbledNameAndPath; // the garbled temp name of the file on disk - UInt32 fSizeInBytes; // the total size of the file - bool fDownloading; // is this file currently downloading? - bool fDownloaded; // is this file completely downloaded? - bool fLocal; // is the file a local copy? - }; - std::map fFileInfoMap; // key is human-readable name - std::map fD2DStreams; // direct-to-disk streams, only used while downloading from the server - - UInt32 fNumInfoRequestsRemaining; // the number of file info requests that are still pending - UInt32 fNumDownloadRequestsRemaining; // the number of file download requests that are still pending - UInt32 fTotalDataDownload; // the amount of data we need to download, for progress bar tracking - UInt32 fTotalDataReceived; // the amount of data we have already preloaded, for progress bar tracking - bool fNetError; - bool fInitialized; - - UInt32 fEncryptionKey[4]; // encryption key for all the secure files - - plOperationProgress* fProgressBar; - - void IIssueDownloadRequests (); - void IPreloadComplete (); - - void ICleanupStreams(); // closes and deletes all streams - - void INotifyAuthReconnected (); - - pfSecurePreloader (); - -public: - CLASSNAME_REGISTER(pfSecurePreloader); - GETINTERFACE_ANY(pfSecurePreloader, hsKeyedObject); - ~pfSecurePreloader (); - - void Init (); - void Shutdown (); + static pfSecurePreloader* fInstance; + std::queue fManifestEntries; + std::queue fDownloadEntries; + plOperationProgress* fProgress; + UInt32 fEncryptionKey[4]; + bool fLegacyMode; - // Client interface functions - void RequestSingleFile(std::wstring filename); // queues a single file to be preloaded (does nothing if already preloaded) - void RequestFileGroup(std::wstring dir, std::wstring ext); // queues a group of files to be preloaded (does nothing if already preloaded) - void Start(); // sends all queued requests (does nothing if already preloaded) - void Cleanup(); // closes all file pointers and cleans up after itself + hsRAMStream* LoadToMemory(const wchar_t* file) const; + void SaveFile(hsStream* file, const wchar_t* name) const; + bool IsZipped(const wchar_t* filename) const; - // Functions for the network callbacks - void RequestFinished(const std::vector & filenames, const std::vector & sizes, bool succeeded); - void UpdateProgressBar(UInt32 bytesReceived); - void FinishedDownload(std::wstring filename, bool succeeded); +public: + pfSecurePreloader(); + ~pfSecurePreloader(); - // Instance handling - static pfSecurePreloader * GetInstance (); - static bool IsInstanced (); + CLASSNAME_REGISTER(pfSecurePreloader); + GETINTERFACE_ANY(pfSecurePreloader, hsKeyedObject); - // hsKeyedObject - hsBool MsgReceive (plMessage * msg); + void Init(); + void Start(); + void Terminate(); + void Finish(); + void Shutdown(); + + void PreloadManifest(const NetCliFileManifestEntry manifestEntries[], UInt32 entryCount); + void PreloadManifest(const NetCliAuthFileInfo manifestEntries[], UInt32 entryCount); + void PreloadNextFile(); + void FilePreloaded(const wchar_t* filename, hsStream* stream); + + plOperationProgress* GetProgressBar() { return fProgress; } + + static pfSecurePreloader* GetInstance(); + static void SetInstance(pfSecurePreloader* instance) { fInstance = instance; } }; #endif // __pfSecurePreloader_h__ diff --git a/Sources/Plasma/PubUtilLib/plEncryption/plChecksum.cpp b/Sources/Plasma/PubUtilLib/plEncryption/plChecksum.cpp index aa5fb562..1da87266 100644 --- a/Sources/Plasma/PubUtilLib/plEncryption/plChecksum.cpp +++ b/Sources/Plasma/PubUtilLib/plEncryption/plChecksum.cpp @@ -91,31 +91,43 @@ plMD5Checksum::plMD5Checksum( const char *fileName ) CalcFromFile( fileName ); } +plMD5Checksum::plMD5Checksum( hsStream* stream ) +{ + CalcFromStream(stream); +} + void plMD5Checksum::Clear() { memset( fChecksum, 0, sizeof( fChecksum ) ); fValid = false; } -void plMD5Checksum::CalcFromFile( const char *fileName) +void plMD5Checksum::CalcFromFile( const char *fileName ) { - FILE *fp; + hsUNIXStream s; fValid = false; - if( fp = fopen(fileName, "rb" ) ) + if( s.Open(fileName) ) { - unsigned loadLen = 1024 * 1024; - Start(); + CalcFromStream(&s); + s.Close(); + } +} - UInt8 *buf = TRACKED_NEW UInt8[loadLen]; +void plMD5Checksum::CalcFromStream( hsStream* stream ) +{ + UInt32 sPos = stream->GetPosition(); + unsigned loadLen = 1024 * 1024; + Start(); + + UInt8 *buf = TRACKED_NEW UInt8[loadLen]; - while(int read = fread(buf, sizeof(UInt8), loadLen, fp)) - AddTo( read, buf ); - delete[] buf; + while(int read = stream->Read(loadLen, buf)) + AddTo( read, buf ); + delete[] buf; - Finish(); - fclose(fp); - } + Finish(); + stream->SetPosition(sPos); } void plMD5Checksum::Start( void ) diff --git a/Sources/Plasma/PubUtilLib/plEncryption/plChecksum.h b/Sources/Plasma/PubUtilLib/plEncryption/plChecksum.h index 2f91f6cc..04d3290e 100644 --- a/Sources/Plasma/PubUtilLib/plEncryption/plChecksum.h +++ b/Sources/Plasma/PubUtilLib/plEncryption/plChecksum.h @@ -58,6 +58,8 @@ public: SumStorage GetChecksum() { return fSum; } }; +class hsStream; + class plMD5Checksum { protected: @@ -74,11 +76,13 @@ class plMD5Checksum plMD5Checksum(); plMD5Checksum( const plMD5Checksum &rhs ); plMD5Checksum( const char *fileName ); + plMD5Checksum( hsStream* stream ); hsBool IsValid( void ) const { return fValid; } void Clear(); void CalcFromFile( const char *fileName ); + void CalcFromStream( hsStream* stream ); void Start( void ); void AddTo( UInt32 size, const UInt8 *buffer ); diff --git a/Sources/Plasma/PubUtilLib/plFile/plFileUtils.cpp b/Sources/Plasma/PubUtilLib/plFile/plFileUtils.cpp index 9140f8c7..a1e9d96d 100644 --- a/Sources/Plasma/PubUtilLib/plFile/plFileUtils.cpp +++ b/Sources/Plasma/PubUtilLib/plFile/plFileUtils.cpp @@ -413,6 +413,13 @@ void plFileUtils::StripExt(char* fileName) *(ext-1) = '\0'; } +void plFileUtils::StripExt(wchar* fileName) +{ + wchar* ext = (wchar*)GetFileExt(fileName); + if (ext) + *(ext-1) = L'\0'; +} + const char* plFileUtils::GetFileExt(const char* pathAndName) { const char* fileName = GetFileName(pathAndName); diff --git a/Sources/Plasma/PubUtilLib/plFile/plFileUtils.h b/Sources/Plasma/PubUtilLib/plFile/plFileUtils.h index c3718ec7..ff4ce5a6 100644 --- a/Sources/Plasma/PubUtilLib/plFile/plFileUtils.h +++ b/Sources/Plasma/PubUtilLib/plFile/plFileUtils.h @@ -100,6 +100,7 @@ namespace plFileUtils void StripFile(char* pathAndName); void StripFile(wchar* pathAndName); void StripExt(char* fileName); + void StripExt(wchar* fileName); // Get the size of the given file in bytes UInt32 GetFileSize( const char *path ); diff --git a/Sources/Plasma/PubUtilLib/plFile/plSecureStream.cpp b/Sources/Plasma/PubUtilLib/plFile/plSecureStream.cpp index 2c0279f2..335b1bc7 100644 --- a/Sources/Plasma/PubUtilLib/plFile/plSecureStream.cpp +++ b/Sources/Plasma/PubUtilLib/plFile/plSecureStream.cpp @@ -77,6 +77,22 @@ fDeleteOnExit(deleteOnExit) memcpy(&fKey, &kDefaultKey, sizeof(kDefaultKey)); } +plSecureStream::plSecureStream(hsStream* base, UInt32* key) : +fRef(INVALID_HANDLE_VALUE), +fActualFileSize(0), +fBufferedStream(false), +fRAMStream(nil), +fWriteFileName(nil), +fOpenMode(kOpenFail), +fDeleteOnExit(false) +{ + if (key) + memcpy(&fKey, key, sizeof(kDefaultKey)); + else + memcpy(&fKey, &kDefaultKey, sizeof(kDefaultKey)); + Open(base); +} + plSecureStream::~plSecureStream() { } @@ -232,6 +248,39 @@ hsBool plSecureStream::Open(const wchar* name, const wchar* mode) } } +hsBool plSecureStream::Open(hsStream* stream) +{ + UInt32 pos = stream->GetPosition(); + stream->Rewind(); + if (!ICheckMagicString(stream)) + return false; + + fActualFileSize = stream->ReadLE32(); + UInt32 trimSize = kMagicStringLen + sizeof(UInt32) + fActualFileSize; + fRAMStream = new hsRAMStream; + while (!stream->AtEnd()) + { + // Don't write out any garbage + UInt32 size; + if ((trimSize - stream->GetPosition()) < kEncryptChunkSize) + size = (trimSize - stream->GetPosition()); + else + size = kEncryptChunkSize; + + UInt8 buf[kEncryptChunkSize]; + stream->Read(kEncryptChunkSize, &buf); + IDecipher((UInt32*)&buf, kEncryptChunkSize / sizeof(UInt32)); + fRAMStream->Write(size, &buf); + } + + stream->SetPosition(pos); + fRAMStream->Rewind(); + fPosition = 0; + fBufferedStream = true; + fOpenMode = kOpenRead; + return true; +} + hsBool plSecureStream::Close() { int rtn = false; @@ -598,6 +647,14 @@ bool plSecureStream::FileDecrypt(const wchar* fileName, UInt32* key /* = nil */) return true; } +bool plSecureStream::ICheckMagicString(hsStream* s) +{ + char magicString[kMagicStringLen+1]; + s->Read(kMagicStringLen, &magicString); + magicString[kMagicStringLen] = '\0'; + return (hsStrEQ(magicString, kMagicString) != 0); +} + bool plSecureStream::ICheckMagicString(hsFD fp) { char magicString[kMagicStringLen+1]; diff --git a/Sources/Plasma/PubUtilLib/plFile/plSecureStream.h b/Sources/Plasma/PubUtilLib/plFile/plSecureStream.h index 1354618a..f2f5a846 100644 --- a/Sources/Plasma/PubUtilLib/plFile/plSecureStream.h +++ b/Sources/Plasma/PubUtilLib/plFile/plSecureStream.h @@ -85,13 +85,16 @@ protected: bool IWriteEncrypted(hsStream* sourceStream, const wchar* outputFile); static bool ICheckMagicString(hsFD fp); + static bool ICheckMagicString(hsStream* s); public: plSecureStream(hsBool deleteOnExit = false, UInt32* key = nil); // uses default key if you don't pass one in + plSecureStream(hsStream* base, UInt32* key = nil); ~plSecureStream(); virtual hsBool Open(const char* name, const char* mode = "rb"); virtual hsBool Open(const wchar* name, const wchar* mode = L"rb"); + hsBool Open(hsStream* stream); virtual hsBool Close(); virtual UInt32 Read(UInt32 byteCount, void* buffer); diff --git a/Sources/Plasma/PubUtilLib/plFile/plStreamSource.cpp b/Sources/Plasma/PubUtilLib/plFile/plStreamSource.cpp index 4f58db81..e6a21c5a 100644 --- a/Sources/Plasma/PubUtilLib/plFile/plStreamSource.cpp +++ b/Sources/Plasma/PubUtilLib/plFile/plStreamSource.cpp @@ -128,7 +128,12 @@ hsStream* plStreamSource::GetFile(std::wstring filename) if (plSecureStream::IsSecureFile(sFilename.c_str())) { UInt32 encryptionKey[4]; - plFileUtils::GetSecureEncryptionKey(sFilename.c_str(), encryptionKey, 4); + if (!plFileUtils::GetSecureEncryptionKey(sFilename.c_str(), encryptionKey, 4)) + { + FATAL("Hey camper... You need an NTD key file!"); + return nil; + } + fFileData[filename].fStream = plSecureStream::OpenSecureFile(sFilename.c_str(), 0, encryptionKey); } else // otherwise it is an encrypted or plain stream, this call handles both