Browse Source

Merge pull request #115 from Hoikas/securepreloader

pfSecurePreloader refactor
Branan Purvine-Riley 13 years ago
parent
commit
0e11edf24e
  1. 9
      Sources/Plasma/Apps/plClient/plClient.cpp
  2. 3
      Sources/Plasma/CoreLib/hsStream.cpp
  3. 807
      Sources/Plasma/FeatureLib/pfSecurePreloader/pfSecurePreloader.cpp
  4. 103
      Sources/Plasma/FeatureLib/pfSecurePreloader/pfSecurePreloader.h
  5. 36
      Sources/Plasma/PubUtilLib/plEncryption/plChecksum.cpp
  6. 4
      Sources/Plasma/PubUtilLib/plEncryption/plChecksum.h
  7. 7
      Sources/Plasma/PubUtilLib/plFile/plFileUtils.cpp
  8. 1
      Sources/Plasma/PubUtilLib/plFile/plFileUtils.h
  9. 57
      Sources/Plasma/PubUtilLib/plFile/plSecureStream.cpp
  10. 3
      Sources/Plasma/PubUtilLib/plFile/plSecureStream.h
  11. 7
      Sources/Plasma/PubUtilLib/plFile/plStreamSource.cpp

9
Sources/Plasma/Apps/plClient/plClient.cpp

@ -317,10 +317,9 @@ hsBool plClient::Shutdown()
plAgeLoader::SetInstance(nil);
}
if (pfSecurePreloader::IsInstanced())
if (pfSecurePreloader::GetInstance())
{
pfSecurePreloader::GetInstance()->Shutdown();
// pfSecurePreloader handles its own fixed key unregistration
pfSecurePreloader::GetInstance()->Shutdown(); // will unregister itself
}
if (fInputManager)
@ -2517,6 +2516,8 @@ void plClient::ICompleteInit () {
void plClient::IHandlePreloaderMsg (plPreloaderMsg * msg) {
plgDispatch::Dispatch()->UnRegisterForExactType(plPreloaderMsg::Index(), GetKey());
if (pfSecurePreloader* sp = pfSecurePreloader::GetInstance())
sp->Shutdown();
if (!msg->fSuccess) {
char str[1024];
@ -2555,7 +2556,5 @@ void plClient::IHandleNetCommAuthMsg (plNetCommAuthMsg * msg) {
plgDispatch::Dispatch()->RegisterForExactType(plPreloaderMsg::Index(), GetKey());
// Precache our secure files
pfSecurePreloader::GetInstance()->RequestFileGroup(L"Python", L"pak");
pfSecurePreloader::GetInstance()->RequestFileGroup(L"SDL", L"sdl");
pfSecurePreloader::GetInstance()->Start();
}

3
Sources/Plasma/CoreLib/hsStream.cpp

@ -1054,10 +1054,7 @@ hsBool hsRAMStream::AtEnd()
UInt32 hsRAMStream::Read(UInt32 byteCount, void * buffer)
{
if (fBytesRead + byteCount > fAppender.Count() * fAppender.ElemSize())
{
hsThrow("Attempting to read past end of stream");
byteCount = (fAppender.Count() * fAppender.ElemSize()) - fBytesRead;
}
fBytesRead += byteCount;
fPosition += byteCount;

807
Sources/Plasma/FeatureLib/pfSecurePreloader/pfSecurePreloader.cpp

@ -39,608 +39,401 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com
Mead, WA 99021
*==LICENSE==*/
#include "hsSTLStream.h"
#include "hsResMgr.h"
#include "pfSecurePreloader.h"
#include "hsStream.h"
#include "plgDispatch.h"
#include "pnUtils/pnUtils.h"
#include "pnNetBase/pnNetBase.h"
#include "pnAsyncCore/pnAsyncCore.h"
#include "pnNetCli/pnNetCli.h"
#include "plNetGameLib/plNetGameLib.h"
#include "plCompression/plZlibStream.h"
#include "plEncryption/plChecksum.h"
#include "plFile/plFileUtils.h"
#include "plFile/plSecureStream.h"
#include "plFile/plStreamSource.h"
#include "plNetCommon/plNetCommon.h"
#include "plProgressMgr/plProgressMgr.h"
#include "plMessage/plPreloaderMsg.h"
#include "plMessage/plNetCommMsgs.h"
#include "pfSecurePreloader.h"
#include "plMessage/plPreloaderMsg.h"
#include "plProgressMgr/plProgressMgr.h"
#include "plNetClientComm/plNetClientComm.h"
extern hsBool gDataServerLocal;
pfSecurePreloader* pfSecurePreloader::fInstance = nil;
extern hsBool gDataServerLocal;
/////////////////////////////////////////////////////////////////////
typedef std::pair<const wchar_t*, const wchar_t*> WcharPair;
// Max number of concurrent file downloads
static const unsigned kMaxConcurrency = 1;
struct AuthRequestParams
{
pfSecurePreloader* fThis;
std::queue<WcharPair> fFileGroups;
pfSecurePreloader * pfSecurePreloader::fInstance;
AuthRequestParams(pfSecurePreloader* parent)
: fThis(parent) { }
};
///////////////////////////////////////////////////////////////////////////////
// Callback routines for the network code
/////////////////////////////////////////////////////////////////////
// Called when a file's info is retrieved from the server
static void DefaultFileListRequestCallback(ENetError result, void* param, const NetCliAuthFileInfo infoArr[], unsigned infoCount)
{
bool success = !IS_NET_ERROR(result);
std::vector<std::wstring> filenames;
std::vector<UInt32> sizes;
if (success)
void ProcAuthDownloadParams(AuthRequestParams* params);
void GotAuthSrvManifest(
ENetError result,
void* param,
const NetCliAuthFileInfo infoArr[],
UInt32 infoCount
) {
AuthRequestParams* arp = (AuthRequestParams*)param;
if (IS_NET_ERROR(result))
{
filenames.reserve(infoCount);
sizes.reserve(infoCount);
for (unsigned curFile = 0; curFile < infoCount; curFile++)
{
filenames.push_back(infoArr[curFile].filename);
sizes.push_back(infoArr[curFile].filesize);
}
FATAL("Failed to get AuthSrv manifest!");
arp->fThis->Terminate();
delete arp;
} else {
arp->fThis->PreloadManifest(infoArr, infoCount);
ProcAuthDownloadParams(arp);
}
((pfSecurePreloader*)param)->RequestFinished(filenames, sizes, success);
}
// Called when a file download is either finished, or failed
static void DefaultFileRequestCallback(ENetError result, void* param, const wchar filename[], hsStream* stream)
{
// Retry download unless shutting down or file not found
switch (result) {
case kNetSuccess:
((pfSecurePreloader*)param)->FinishedDownload(filename, true);
break;
case kNetErrFileNotFound:
case kNetErrRemoteShutdown:
((pfSecurePreloader*)param)->FinishedDownload(filename, false);
break;
default:
stream->Rewind();
NetCliAuthFileRequest(
filename,
stream,
&DefaultFileRequestCallback,
param
);
break;
void GotFileSrvManifest(
ENetError result,
void* param,
const wchar_t group[],
const NetCliFileManifestEntry manifest[],
UInt32 entryCount
) {
pfSecurePreloader* sp = (pfSecurePreloader*)param;
if (result == kNetErrFileNotFound)
{
AuthRequestParams* params = new AuthRequestParams(sp);
params->fFileGroups.push(WcharPair(L"Python", L"pak"));
params->fFileGroups.push(WcharPair(L"SDL", L"sdl"));
ProcAuthDownloadParams(params);
return;
} else if (!entryCount) {
FATAL("SecurePreloader manifest empty!");
sp->Terminate();
return;
}
sp->PreloadManifest(manifest, entryCount);
}
void FileDownloaded(
ENetError result,
void* param,
const wchar_t filename[],
hsStream* writer
) {
pfSecurePreloader* sp = (pfSecurePreloader*)param;
if (IS_NET_ERROR(result))
{
FATAL("SecurePreloader download failed");
sp->Terminate();
} else {
sp->FilePreloaded(filename, writer);
}
}
///////////////////////////////////////////////////////////////////////////////
// Our custom stream for writing directly to disk securely, and updating the
// progress bar. Does NOT support reading (cause it doesn't need to)
class Direct2DiskStream : public hsUNIXStream
void ProcAuthDownloadParams(AuthRequestParams* params)
{
protected:
wchar * fWriteFileName;
// Request the "manifests" until there are none left, then download the files
if (params->fFileGroups.empty())
{
params->fThis->PreloadNextFile();
delete params;
} else {
WcharPair wp = params->fFileGroups.front();
params->fFileGroups.pop();
NetCliAuthFileListRequest(wp.first, wp.second, GotAuthSrvManifest, params);
}
}
pfSecurePreloader* fPreloader;
/////////////////////////////////////////////////////////////////////
class pfSecurePreloaderStream : public plZlibStream
{
plOperationProgress* fProgress;
bool fIsZipped;
public:
Direct2DiskStream(pfSecurePreloader* preloader);
~Direct2DiskStream();
virtual hsBool Open(const char* name, const char* mode = "wb");
virtual hsBool Open(const wchar* name, const wchar* mode = L"wb");
virtual hsBool Close();
virtual UInt32 Read(UInt32 byteCount, void* buffer);
virtual UInt32 Write(UInt32 byteCount, const void* buffer);
};
pfSecurePreloaderStream(plOperationProgress* prog, bool zipped)
: fProgress(prog), fIsZipped(zipped), plZlibStream()
{
fOutput = new hsRAMStream;
}
Direct2DiskStream::Direct2DiskStream(pfSecurePreloader* preloader) :
fWriteFileName(nil),
fPreloader(preloader)
{}
~pfSecurePreloaderStream()
{
delete fOutput;
fOutput = nil;
plZlibStream::Close();
}
Direct2DiskStream::~Direct2DiskStream()
{
Close();
}
hsBool AtEnd() { return fOutput->AtEnd(); }
UInt32 GetEOF() { return fOutput->GetEOF(); }
UInt32 GetPosition() const { return fOutput->GetPosition(); }
UInt32 GetSizeLeft() const { return fOutput->GetSizeLeft(); }
UInt32 Read(UInt32 count, void* buf) { return fOutput->Read(count, buf); }
void Rewind() { fOutput->Rewind(); }
void SetPosition(UInt32 pos) { fOutput->SetPosition(pos); }
void Skip(UInt32 deltaByteCount) { fOutput->Skip(deltaByteCount); }
hsBool Direct2DiskStream::Open(const char* name, const char* mode)
{
wchar* wName = hsStringToWString(name);
wchar* wMode = hsStringToWString(mode);
hsBool ret = Open(wName, wMode);
delete [] wName;
delete [] wMode;
return ret;
}
UInt32 Write(UInt32 count, const void* buf)
{
if (fProgress)
fProgress->Increment((hsScalar)count);
if (fIsZipped)
return plZlibStream::Write(count, buf);
else
return fOutput->Write(count, buf);
}
};
/////////////////////////////////////////////////////////////////////
hsBool Direct2DiskStream::Open(const wchar* name, const wchar* mode)
pfSecurePreloader::pfSecurePreloader()
: fProgress(nil), fLegacyMode(false)
{ }
pfSecurePreloader::~pfSecurePreloader()
{
if (0 != wcscmp(mode, L"wb")) {
hsAssert(0, "Unsupported open mode");
return false;
while (fDownloadEntries.size())
{
free((void*)fDownloadEntries.front());
fDownloadEntries.pop();
}
while (fManifestEntries.size())
{
free((void*)fManifestEntries.front());
fManifestEntries.pop();
}
fWriteFileName = TRACKED_NEW(wchar[wcslen(name) + 1]);
wcscpy(fWriteFileName, name);
// LogMsg(kLogPerf, L"Opening disk file %S", fWriteFileName);
return hsUNIXStream::Open(name, mode);
}
hsBool Direct2DiskStream::Close()
hsRAMStream* pfSecurePreloader::LoadToMemory(const wchar_t* file) const
{
delete [] fWriteFileName;
fWriteFileName = nil;
return hsUNIXStream::Close();
if (!plFileUtils::FileExists(file))
return nil;
hsUNIXStream s;
hsRAMStream* ram = new hsRAMStream;
s.Open(file);
UInt32 loadLen = 1024 * 1024;
UInt8* buf = new UInt8[loadLen];
while (UInt32 read = s.Read(loadLen, buf))
ram->Write(read, buf);
delete[] buf;
s.Close();
ram->Rewind();
return ram;
}
UInt32 Direct2DiskStream::Read(UInt32 bytes, void* buffer)
void pfSecurePreloader::SaveFile(hsStream* file, const wchar_t* name) const
{
hsAssert(0, "not implemented");
return 0; // we don't read
hsUNIXStream s;
s.Open(name, L"wb");
UInt32 pos = file->GetPosition();
file->Rewind();
UInt32 loadLen = 1024 * 1024;
UInt8* buf = new UInt8[loadLen];
while (UInt32 read = file->Read(loadLen, buf))
s.Write(read, buf);
file->SetPosition(pos);
s.Close();
delete[] buf;
}
UInt32 Direct2DiskStream::Write(UInt32 bytes, const void* buffer)
bool pfSecurePreloader::IsZipped(const wchar_t* filename) const
{
// LogMsg(kLogPerf, L"Writing %u bytes to disk file %S", bytes, fWriteFileName);
fPreloader->UpdateProgressBar(bytes);
return hsUNIXStream::Write(bytes, buffer);
return wcscmp(plFileUtils::GetFileExt(filename), L"gz") == 0;
}
///////////////////////////////////////////////////////////////////////////////
// secure preloader class implementation
// closes and deletes all streams
void pfSecurePreloader::ICleanupStreams()
void pfSecurePreloader::PreloadNextFile()
{
if (fD2DStreams.size() > 0)
if (fManifestEntries.empty())
{
std::map<std::wstring, hsStream*>::iterator curStream;
for (curStream = fD2DStreams.begin(); curStream != fD2DStreams.end(); curStream++)
{
curStream->second->Close();
delete curStream->second;
curStream->second = nil;
}
fD2DStreams.clear();
Finish();
return;
}
}
// queues a single file to be preloaded (does nothing if already preloaded)
void pfSecurePreloader::RequestSingleFile(std::wstring filename)
{
fileRequest request;
ZERO(request);
request.fType = fileRequest::kSingleFile;
request.fPath = filename;
request.fExt = L"";
fRequests.push_back(request);
const wchar_t* filename = fDownloadEntries.front();
hsStream* s = new pfSecurePreloaderStream(fProgress, IsZipped(filename));
// Thankfully, both callbacks have the same arguments
if (fLegacyMode)
NetCliAuthFileRequest(filename, s, FileDownloaded, this);
else
NetCliFileDownloadRequest(filename, s, FileDownloaded, this);
}
// queues a group of files to be preloaded (does nothing if already preloaded)
void pfSecurePreloader::RequestFileGroup(std::wstring dir, std::wstring ext)
void pfSecurePreloader::Init()
{
fileRequest request;
ZERO(request);
request.fType = fileRequest::kFileList;
request.fPath = dir;
request.fExt = ext;
fRequests.push_back(request);
RegisterAs(kSecurePreloader_KEY);
// TODO: If we're going to support reconnects, then let's do it right.
// Later...
//plgDispatch::Dispatch()->RegisterForExactType(plNetCommAuthConnectedMsg::Index(), GetKey());
}
// preloads all requested files from the server (does nothing if already preloaded)
void pfSecurePreloader::Start()
{
if (gDataServerLocal) {
// using local data, don't do anything
plPreloaderMsg * msg = TRACKED_NEW plPreloaderMsg();
msg->fSuccess = true;
msg->Send();
#ifndef PLASMA_EXTERNAL_RELEASE
// Using local data? Move along, move along...
if (gDataServerLocal)
{
Finish();
return;
}
#endif
NetCliAuthGetEncryptionKey(fEncryptionKey, 4); // grab the encryption key from the server
fNetError = false;
// make sure we are all cleaned up
ICleanupStreams();
fTotalDataReceived = 0;
// update the progress bar for downloading
if (!fProgressBar)
fProgressBar = plProgressMgr::GetInstance()->RegisterOperation((hsScalar)(fRequests.size()), "Getting file info...", plProgressMgr::kUpdateText, false, true);
NetCliAuthGetEncryptionKey(fEncryptionKey, 4);
for (unsigned curRequest = 0; curRequest < fRequests.size(); curRequest++)
{
fNumInfoRequestsRemaining++; // increment the counter
if (fRequests[curRequest].fType == fileRequest::kSingleFile)
{
#ifndef PLASMA_EXTERNAL_RELEASE
// in internal releases, we can use on-disk files if they exist
if (plFileUtils::FileExists(fRequests[curRequest].fPath.c_str()))
{
fileInfo info;
info.fOriginalNameAndPath = fRequests[curRequest].fPath;
info.fSizeInBytes = plFileUtils::GetFileSize(info.fOriginalNameAndPath.c_str());
info.fDownloading = false;
info.fDownloaded = false;
info.fLocal = true;
// generate garbled name
wchar_t pathBuffer[MAX_PATH + 1];
wchar_t filename[arrsize(pathBuffer)];
GetTempPathW(arrsize(pathBuffer), pathBuffer);
GetTempFileNameW(pathBuffer, L"CYN", 0, filename);
info.fGarbledNameAndPath = filename;
fTotalDataDownload += info.fSizeInBytes;
fFileInfoMap[info.fOriginalNameAndPath] = info;
}
// internal client will still request it, even if it exists locally,
// so that things get updated properly
#endif // PLASMA_EXTERNAL_RELEASE
NetCliAuthFileListRequest(
fRequests[curRequest].fPath.c_str(),
nil,
&DefaultFileListRequestCallback,
(void*)this
);
}
else
{
#ifndef PLASMA_EXTERNAL_RELEASE
// in internal releases, we can use on-disk files if they exist
// Build the search string as "dir\\*.ext"
wchar searchStr[MAX_PATH];
PathAddFilename(searchStr, fRequests[curRequest].fPath.c_str(), L"*", arrsize(searchStr));
PathSetExtension(searchStr, searchStr, fRequests[curRequest].fExt.c_str(), arrsize(searchStr));
ARRAY(PathFind) paths;
PathFindFiles(&paths, searchStr, kPathFlagFile); // find all files that match
// convert it to our little file info array
PathFind* curFile = paths.Ptr();
PathFind* lastFile = paths.Term();
while (curFile != lastFile) {
fileInfo info;
info.fOriginalNameAndPath = curFile->name;
info.fSizeInBytes = (UInt32)curFile->fileLength;
info.fDownloading = false;
info.fDownloaded = false;
info.fLocal = true;
// generate garbled name
wchar_t pathBuffer[MAX_PATH + 1];
wchar_t filename[arrsize(pathBuffer)];
GetTempPathW(arrsize(pathBuffer), pathBuffer);
GetTempFileNameW(pathBuffer, L"CYN", 0, filename);
info.fGarbledNameAndPath = filename;
fTotalDataDownload += info.fSizeInBytes;
fFileInfoMap[info.fOriginalNameAndPath] = info;
curFile++;
}
#endif // PLASMA_EXTERNAL_RELEASE
NetCliAuthFileListRequest(
fRequests[curRequest].fPath.c_str(),
fRequests[curRequest].fExt.c_str(),
&DefaultFileListRequestCallback,
(void*)this
);
}
}
// TODO: Localize
fProgress = plProgressMgr::GetInstance()->RegisterOperation(0.0f, "Checking for Updates", plProgressMgr::kUpdateText, false, true);
// Now, we need to fetch the "SecurePreloader" manifest from the file server, which will contain the python and SDL files.
// We're basically reimplementing plResPatcher here, except preferring to keep everything in memory, then flush to disk
// when we're done. If this fails, then we shall download everything from the AuthSrv like in the old days.
NetCliFileManifestRequest(GotFileSrvManifest, this, L"SecurePreloader");
}
// closes all file pointers and cleans up after itself
void pfSecurePreloader::Cleanup()
void pfSecurePreloader::Terminate()
{
ICleanupStreams();
fRequests.clear();
fFileInfoMap.clear();
FATAL("pfSecurePreloader failure");
fNumInfoRequestsRemaining = 0;
fTotalDataDownload = 0;
fTotalDataReceived = 0;
plPreloaderMsg* msg = new plPreloaderMsg;
msg->fSuccess = false;
plgDispatch::Dispatch()->MsgSend(msg);
}
DEL(fProgressBar);
fProgressBar = nil;
void pfSecurePreloader::Finish()
{
plPreloaderMsg* msg = new plPreloaderMsg;
msg->fSuccess = true;
plgDispatch::Dispatch()->MsgSend(msg);
}
//============================================================================
void pfSecurePreloader::RequestFinished(const std::vector<std::wstring> & filenames, const std::vector<UInt32> & sizes, bool succeeded)
void pfSecurePreloader::Shutdown()
{
fNetError |= !succeeded;
if (succeeded)
SetInstance(nil);
if (fProgress)
{
unsigned count = 0;
for (int curFile = 0; curFile < filenames.size(); curFile++)
{
if (fFileInfoMap.find(filenames[curFile]) != fFileInfoMap.end())
continue; // if it is a duplicate, ignore it (the duplicate is probably one we found locally)
fileInfo info;
info.fOriginalNameAndPath = filenames[curFile];
info.fSizeInBytes = sizes[curFile];
info.fDownloading = false;
info.fDownloaded = false;
info.fLocal = false; // if we get here, it was retrieved remotely
// generate garbled name
wchar_t pathBuffer[MAX_PATH + 1];
wchar_t filename[arrsize(pathBuffer)];
GetTempPathW(arrsize(pathBuffer), pathBuffer);
GetTempFileNameW(pathBuffer, L"CYN", 0, filename);
info.fGarbledNameAndPath = filename;
fTotalDataDownload += info.fSizeInBytes;
fFileInfoMap[info.fOriginalNameAndPath] = info;
++count;
}
LogMsg(kLogPerf, "Added %u files to secure download queue", count);
delete fProgress;
fProgress = nil;
}
if (fProgressBar)
fProgressBar->Increment(1.f);
--fNumInfoRequestsRemaining; // even if we fail, decrement the counter
if (succeeded) {
DEL(fProgressBar);
fProgressBar = plProgressMgr::GetInstance()->RegisterOperation((hsScalar)(fTotalDataDownload), "Downloading...", plProgressMgr::kUpdateText, false, true);
// Issue some file download requests (up to kMaxConcurrency)
IIssueDownloadRequests();
}
else {
IPreloadComplete();
}
// Takes care of UnReffing us
UnRegisterAs(kSecurePreloader_KEY);
}
//============================================================================
void pfSecurePreloader::IIssueDownloadRequests () {
void pfSecurePreloader::PreloadManifest(const NetCliAuthFileInfo manifestEntries[], UInt32 entryCount)
{
UInt32 totalBytes = 0;
if (fProgress)
totalBytes = (UInt32)fProgress->GetMax();
fLegacyMode = true;
std::map<std::wstring, fileInfo>::iterator curFile;
for (curFile = fFileInfoMap.begin(); curFile != fFileInfoMap.end(); curFile++)
for (UInt32 i = 0; i < entryCount; ++i)
{
// Skip files already downloaded or currently downloading
if (curFile->second.fDownloaded || curFile->second.fDownloading)
continue;
std::wstring filename = curFile->second.fOriginalNameAndPath;
#ifndef PLASMA_EXTERNAL_RELEASE
// in internal releases, we can use on-disk files if they exist
if (plFileUtils::FileExists(filename.c_str()))
const NetCliAuthFileInfo mfs = manifestEntries[i];
fDownloadEntries.push(wcsdup(mfs.filename));
if (IsZipped(mfs.filename))
{
// don't bother streaming, just make the secure stream using the local file
// a local key overrides the server-downloaded key
UInt32 localKey[4];
bool hasLocalKey = plFileUtils::GetSecureEncryptionKey(filename.c_str(), localKey, arrsize(localKey));
hsStream* stream = nil;
if (hasLocalKey)
stream = plSecureStream::OpenSecureFile(filename.c_str(), 0, localKey);
else
stream = plSecureStream::OpenSecureFile(filename.c_str(), 0, fEncryptionKey);
// add it to the stream source
bool added = plStreamSource::GetInstance()->InsertFile(filename.c_str(), stream);
if (!added)
DEL(stream); // wasn't added, so nuke our local copy
wchar_t* name = wcsdup(mfs.filename);
plFileUtils::StripExt(name);
fManifestEntries.push(name);
} else
fManifestEntries.push(wcsdup(mfs.filename));
// and make sure the vars are set up right
curFile->second.fDownloaded = true;
curFile->second.fLocal = true;
}
else
#endif
{
// Enforce concurrency limit
if (fNumDownloadRequestsRemaining >= kMaxConcurrency)
break;
curFile->second.fDownloading = true;
curFile->second.fDownloaded = false;
curFile->second.fLocal = false;
// create and setup the stream
Direct2DiskStream* fileStream = TRACKED_NEW Direct2DiskStream(this);
fileStream->Open(curFile->second.fGarbledNameAndPath.c_str(), L"wb");
fD2DStreams[filename] = (hsStream*)fileStream;
// request the file from the server
LogMsg(kLogPerf, L"Requesting secure file:%s", filename.c_str());
++fNumDownloadRequestsRemaining;
NetCliAuthFileRequest(
filename.c_str(),
(hsStream*)fileStream,
&DefaultFileRequestCallback,
this
);
}
totalBytes += mfs.filesize;
}
if (!fNumDownloadRequestsRemaining)
IPreloadComplete();
}
void pfSecurePreloader::UpdateProgressBar(UInt32 bytesReceived)
{
fTotalDataReceived += bytesReceived;
if (fTotalDataReceived > fTotalDataDownload)
fTotalDataReceived = fTotalDataDownload; // shouldn't happen... but just in case
if (fProgressBar)
fProgressBar->Increment((hsScalar)bytesReceived);
if (fProgress)
{
fProgress->SetLength((hsScalar)totalBytes);
fProgress->SetTitle("Downloading...");
}
}
void pfSecurePreloader::FinishedDownload(std::wstring filename, bool succeeded)
void pfSecurePreloader::PreloadManifest(const NetCliFileManifestEntry manifestEntries[], UInt32 entryCount)
{
for (;;)
UInt32 totalBytes = 0;
for (UInt32 i = 0; i < entryCount; ++i)
{
if (fFileInfoMap.find(filename) == fFileInfoMap.end())
const NetCliFileManifestEntry mfs = manifestEntries[i];
bool fetchMe = true;
hsRAMStream* s = nil;
if (plFileUtils::FileExists(mfs.clientName))
{
// file doesn't exist... abort
succeeded = false;
break;
s = LoadToMemory(mfs.clientName);
if (s)
{
// Damn this
const char* md5 = hsWStringToString(mfs.md5);
plMD5Checksum srvHash;
srvHash.SetFromHexString(md5);
delete[] md5;
// Now actually copare the hashes
plMD5Checksum lclHash;
lclHash.CalcFromStream(s);
fetchMe = (srvHash != lclHash);
}
}
fFileInfoMap[filename].fDownloading = false;
// close and delete the writer stream (even if we failed)
fD2DStreams[filename]->Close();
delete fD2DStreams[filename];
fD2DStreams.erase(fD2DStreams.find(filename));
if (succeeded)
if (fetchMe)
{
// open a secure stream to that file
hsStream* stream = plSecureStream::OpenSecureFile(
fFileInfoMap[filename].fGarbledNameAndPath.c_str(),
plSecureStream::kRequireEncryption | plSecureStream::kDeleteOnExit, // force delete and encryption
fEncryptionKey
);
bool addedToSource = plStreamSource::GetInstance()->InsertFile(filename.c_str(), stream);
if (!addedToSource)
DEL(stream); // cleanup if it wasn't added
fFileInfoMap[filename].fDownloaded = true;
break;
fManifestEntries.push(wcsdup(mfs.clientName));
fDownloadEntries.push(wcsdup(mfs.downloadName));
if (IsZipped(mfs.downloadName))
totalBytes += mfs.zipSize;
else
totalBytes += mfs.fileSize;
} else {
plSecureStream* ss = new plSecureStream(s, fEncryptionKey);
plStreamSource::GetInstance()->InsertFile(mfs.clientName, ss);
}
// file download failed, clean up after it
// delete the temporary file
if (plFileUtils::FileExists(fFileInfoMap[filename].fGarbledNameAndPath.c_str()))
plFileUtils::RemoveFile(fFileInfoMap[filename].fGarbledNameAndPath.c_str(), true);
// and remove it from the info map
fFileInfoMap.erase(fFileInfoMap.find(filename));
break;
if (s)
delete s;
}
fNetError |= !succeeded;
--fNumDownloadRequestsRemaining;
LogMsg(kLogPerf, L"Received secure file:%s, success:%s", filename.c_str(), succeeded ? L"Yep" : L"Nope");
if (!succeeded)
IPreloadComplete();
else
// Issue some file download requests (up to kMaxConcurrency)
IIssueDownloadRequests();
}
//============================================================================
void pfSecurePreloader::INotifyAuthReconnected () {
// The secure file download network protocol will now just pick up downloading
// where it left off before the reconnect, so no need to reset in-progress files.
/*
std::map<std::wstring, fileInfo>::iterator curFile;
for (curFile = fFileInfoMap.begin(); curFile != fFileInfoMap.end(); curFile++) {
// Reset files that were currently downloading
if (curFile->second.fDownloading)
curFile->second.fDownloading = false;
}
if (fNumDownloadRequestsRemaining > 0) {
LogMsg(kLogPerf, L"pfSecurePreloader: Auth reconnected, resetting in-progress file downloads");
// Issue some file download requests (up to kMaxConcurrency)
IIssueDownloadRequests();
}
*/
}
//============================================================================
void pfSecurePreloader::IPreloadComplete () {
DEL(fProgressBar);
fProgressBar = nil;
plPreloaderMsg * msg = TRACKED_NEW plPreloaderMsg();
msg->fSuccess = !fNetError;
msg->Send();
}
//============================================================================
hsBool pfSecurePreloader::MsgReceive (plMessage * msg) {
if (plNetCommAuthConnectedMsg * authMsg = plNetCommAuthConnectedMsg::ConvertNoRef(msg)) {
INotifyAuthReconnected();
return true;
}
return hsKeyedObject::MsgReceive(msg);
}
//============================================================================
pfSecurePreloader * pfSecurePreloader::GetInstance () {
if (!fInstance) {
fInstance = NEWZERO(pfSecurePreloader);
fInstance->RegisterAs(kSecurePreloader_KEY);
if (totalBytes && fProgress)
{
fProgress->SetLength((hsScalar)totalBytes);
fProgress->SetTitle("Downloading...");
}
return fInstance;
}
//============================================================================
bool pfSecurePreloader::IsInstanced () {
return fInstance != nil;
}
//============================================================================
void pfSecurePreloader::Init () {
if (!fInitialized) {
fInitialized = true;
plgDispatch::Dispatch()->RegisterForExactType(plNetCommAuthConnectedMsg::Index(), GetKey());
}
// This method uses only one manifest, so we're good to go now!
PreloadNextFile();
}
//============================================================================
void pfSecurePreloader::Shutdown () {
void pfSecurePreloader::FilePreloaded(const wchar_t* file, hsStream* stream)
{
// Clear out queue
fDownloadEntries.pop();
const wchar_t* clientName = fManifestEntries.front(); // Stolen by plStreamSource
fManifestEntries.pop();
if (fInitialized) {
fInitialized = false;
plgDispatch::Dispatch()->UnRegisterForExactType(plNetCommAuthConnectedMsg::Index(), GetKey());
if (!fLegacyMode) // AuthSrv data caching is useless
{
plFileUtils::EnsureFilePathExists(clientName);
SaveFile(stream, clientName);
}
if (fInstance) {
fInstance->UnRegister();
fInstance = nil;
}
}
plSecureStream* ss = new plSecureStream(stream, fEncryptionKey);
plStreamSource::GetInstance()->InsertFile(clientName, ss);
delete stream; // SecureStream holds its own decrypted buffer
//============================================================================
pfSecurePreloader::pfSecurePreloader () {
// Continue down the warpath
PreloadNextFile();
}
//============================================================================
pfSecurePreloader::~pfSecurePreloader () {
Cleanup();
pfSecurePreloader* pfSecurePreloader::GetInstance()
{
if (!fInstance)
fInstance = new pfSecurePreloader;
return fInstance;
}

103
Sources/Plasma/FeatureLib/pfSecurePreloader/pfSecurePreloader.h

@ -42,15 +42,13 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com
#ifndef __pfSecurePreloader_h__
#define __pfSecurePreloader_h__
#include "hsTypes.h"
#include "hsStlUtils.h"
#include "hsCritSect.h"
#include "hsStream.h"
#include "plFile/plSecureStream.h"
#include "HeadSpin.h"
#include "pnKeyedObject/hsKeyedObject.h"
#include "plNetGameLib/plNetGameLib.h"
#include <queue>
class plOperationProgress;
class hsRAMStream;
///////////////////////////////////////////////////////////////////////////////
// pfSecurePreloader - a class for handling files we want downloaded from the
@ -60,75 +58,40 @@ class plOperationProgress;
class pfSecurePreloader : public hsKeyedObject
{
private:
static pfSecurePreloader * fInstance;
struct fileRequest
{
enum requestType {kSingleFile, kFileList};
requestType fType;
std::wstring fPath; // filename if kSingleFile, path if kFileList
std::wstring fExt; // blank if kSingleFile, extension if kFileList
};
std::vector<fileRequest> fRequests;
struct fileInfo
{
std::wstring fOriginalNameAndPath; // the human-readable name
std::wstring fGarbledNameAndPath; // the garbled temp name of the file on disk
UInt32 fSizeInBytes; // the total size of the file
bool fDownloading; // is this file currently downloading?
bool fDownloaded; // is this file completely downloaded?
bool fLocal; // is the file a local copy?
};
std::map<std::wstring, fileInfo> fFileInfoMap; // key is human-readable name
std::map<std::wstring, hsStream*> fD2DStreams; // direct-to-disk streams, only used while downloading from the server
UInt32 fNumInfoRequestsRemaining; // the number of file info requests that are still pending
UInt32 fNumDownloadRequestsRemaining; // the number of file download requests that are still pending
UInt32 fTotalDataDownload; // the amount of data we need to download, for progress bar tracking
UInt32 fTotalDataReceived; // the amount of data we have already preloaded, for progress bar tracking
bool fNetError;
bool fInitialized;
UInt32 fEncryptionKey[4]; // encryption key for all the secure files
plOperationProgress* fProgressBar;
void IIssueDownloadRequests ();
void IPreloadComplete ();
void ICleanupStreams(); // closes and deletes all streams
void INotifyAuthReconnected ();
pfSecurePreloader ();
public:
CLASSNAME_REGISTER(pfSecurePreloader);
GETINTERFACE_ANY(pfSecurePreloader, hsKeyedObject);
~pfSecurePreloader ();
void Init ();
void Shutdown ();
static pfSecurePreloader* fInstance;
std::queue<const wchar_t*> fManifestEntries;
std::queue<const wchar_t*> fDownloadEntries;
plOperationProgress* fProgress;
UInt32 fEncryptionKey[4];
bool fLegacyMode;
// Client interface functions
void RequestSingleFile(std::wstring filename); // queues a single file to be preloaded (does nothing if already preloaded)
void RequestFileGroup(std::wstring dir, std::wstring ext); // queues a group of files to be preloaded (does nothing if already preloaded)
void Start(); // sends all queued requests (does nothing if already preloaded)
void Cleanup(); // closes all file pointers and cleans up after itself
hsRAMStream* LoadToMemory(const wchar_t* file) const;
void SaveFile(hsStream* file, const wchar_t* name) const;
bool IsZipped(const wchar_t* filename) const;
// Functions for the network callbacks
void RequestFinished(const std::vector<std::wstring> & filenames, const std::vector<UInt32> & sizes, bool succeeded);
void UpdateProgressBar(UInt32 bytesReceived);
void FinishedDownload(std::wstring filename, bool succeeded);
public:
pfSecurePreloader();
~pfSecurePreloader();
// Instance handling
static pfSecurePreloader * GetInstance ();
static bool IsInstanced ();
CLASSNAME_REGISTER(pfSecurePreloader);
GETINTERFACE_ANY(pfSecurePreloader, hsKeyedObject);
// hsKeyedObject
hsBool MsgReceive (plMessage * msg);
void Init();
void Start();
void Terminate();
void Finish();
void Shutdown();
void PreloadManifest(const NetCliFileManifestEntry manifestEntries[], UInt32 entryCount);
void PreloadManifest(const NetCliAuthFileInfo manifestEntries[], UInt32 entryCount);
void PreloadNextFile();
void FilePreloaded(const wchar_t* filename, hsStream* stream);
plOperationProgress* GetProgressBar() { return fProgress; }
static pfSecurePreloader* GetInstance();
static void SetInstance(pfSecurePreloader* instance) { fInstance = instance; }
};
#endif // __pfSecurePreloader_h__

36
Sources/Plasma/PubUtilLib/plEncryption/plChecksum.cpp

@ -91,31 +91,43 @@ plMD5Checksum::plMD5Checksum( const char *fileName )
CalcFromFile( fileName );
}
plMD5Checksum::plMD5Checksum( hsStream* stream )
{
CalcFromStream(stream);
}
void plMD5Checksum::Clear()
{
memset( fChecksum, 0, sizeof( fChecksum ) );
fValid = false;
}
void plMD5Checksum::CalcFromFile( const char *fileName)
void plMD5Checksum::CalcFromFile( const char *fileName )
{
FILE *fp;
hsUNIXStream s;
fValid = false;
if( fp = fopen(fileName, "rb" ) )
if( s.Open(fileName) )
{
unsigned loadLen = 1024 * 1024;
Start();
CalcFromStream(&s);
s.Close();
}
}
UInt8 *buf = TRACKED_NEW UInt8[loadLen];
void plMD5Checksum::CalcFromStream( hsStream* stream )
{
UInt32 sPos = stream->GetPosition();
unsigned loadLen = 1024 * 1024;
Start();
UInt8 *buf = TRACKED_NEW UInt8[loadLen];
while(int read = fread(buf, sizeof(UInt8), loadLen, fp))
AddTo( read, buf );
delete[] buf;
while(int read = stream->Read(loadLen, buf))
AddTo( read, buf );
delete[] buf;
Finish();
fclose(fp);
}
Finish();
stream->SetPosition(sPos);
}
void plMD5Checksum::Start( void )

4
Sources/Plasma/PubUtilLib/plEncryption/plChecksum.h

@ -58,6 +58,8 @@ public:
SumStorage GetChecksum() { return fSum; }
};
class hsStream;
class plMD5Checksum
{
protected:
@ -74,11 +76,13 @@ class plMD5Checksum
plMD5Checksum();
plMD5Checksum( const plMD5Checksum &rhs );
plMD5Checksum( const char *fileName );
plMD5Checksum( hsStream* stream );
hsBool IsValid( void ) const { return fValid; }
void Clear();
void CalcFromFile( const char *fileName );
void CalcFromStream( hsStream* stream );
void Start( void );
void AddTo( UInt32 size, const UInt8 *buffer );

7
Sources/Plasma/PubUtilLib/plFile/plFileUtils.cpp

@ -413,6 +413,13 @@ void plFileUtils::StripExt(char* fileName)
*(ext-1) = '\0';
}
void plFileUtils::StripExt(wchar* fileName)
{
wchar* ext = (wchar*)GetFileExt(fileName);
if (ext)
*(ext-1) = L'\0';
}
const char* plFileUtils::GetFileExt(const char* pathAndName)
{
const char* fileName = GetFileName(pathAndName);

1
Sources/Plasma/PubUtilLib/plFile/plFileUtils.h

@ -100,6 +100,7 @@ namespace plFileUtils
void StripFile(char* pathAndName);
void StripFile(wchar* pathAndName);
void StripExt(char* fileName);
void StripExt(wchar* fileName);
// Get the size of the given file in bytes
UInt32 GetFileSize( const char *path );

57
Sources/Plasma/PubUtilLib/plFile/plSecureStream.cpp

@ -77,6 +77,22 @@ fDeleteOnExit(deleteOnExit)
memcpy(&fKey, &kDefaultKey, sizeof(kDefaultKey));
}
plSecureStream::plSecureStream(hsStream* base, UInt32* key) :
fRef(INVALID_HANDLE_VALUE),
fActualFileSize(0),
fBufferedStream(false),
fRAMStream(nil),
fWriteFileName(nil),
fOpenMode(kOpenFail),
fDeleteOnExit(false)
{
if (key)
memcpy(&fKey, key, sizeof(kDefaultKey));
else
memcpy(&fKey, &kDefaultKey, sizeof(kDefaultKey));
Open(base);
}
plSecureStream::~plSecureStream()
{
}
@ -232,6 +248,39 @@ hsBool plSecureStream::Open(const wchar* name, const wchar* mode)
}
}
hsBool plSecureStream::Open(hsStream* stream)
{
UInt32 pos = stream->GetPosition();
stream->Rewind();
if (!ICheckMagicString(stream))
return false;
fActualFileSize = stream->ReadLE32();
UInt32 trimSize = kMagicStringLen + sizeof(UInt32) + fActualFileSize;
fRAMStream = new hsRAMStream;
while (!stream->AtEnd())
{
// Don't write out any garbage
UInt32 size;
if ((trimSize - stream->GetPosition()) < kEncryptChunkSize)
size = (trimSize - stream->GetPosition());
else
size = kEncryptChunkSize;
UInt8 buf[kEncryptChunkSize];
stream->Read(kEncryptChunkSize, &buf);
IDecipher((UInt32*)&buf, kEncryptChunkSize / sizeof(UInt32));
fRAMStream->Write(size, &buf);
}
stream->SetPosition(pos);
fRAMStream->Rewind();
fPosition = 0;
fBufferedStream = true;
fOpenMode = kOpenRead;
return true;
}
hsBool plSecureStream::Close()
{
int rtn = false;
@ -598,6 +647,14 @@ bool plSecureStream::FileDecrypt(const wchar* fileName, UInt32* key /* = nil */)
return true;
}
bool plSecureStream::ICheckMagicString(hsStream* s)
{
char magicString[kMagicStringLen+1];
s->Read(kMagicStringLen, &magicString);
magicString[kMagicStringLen] = '\0';
return (hsStrEQ(magicString, kMagicString) != 0);
}
bool plSecureStream::ICheckMagicString(hsFD fp)
{
char magicString[kMagicStringLen+1];

3
Sources/Plasma/PubUtilLib/plFile/plSecureStream.h

@ -85,13 +85,16 @@ protected:
bool IWriteEncrypted(hsStream* sourceStream, const wchar* outputFile);
static bool ICheckMagicString(hsFD fp);
static bool ICheckMagicString(hsStream* s);
public:
plSecureStream(hsBool deleteOnExit = false, UInt32* key = nil); // uses default key if you don't pass one in
plSecureStream(hsStream* base, UInt32* key = nil);
~plSecureStream();
virtual hsBool Open(const char* name, const char* mode = "rb");
virtual hsBool Open(const wchar* name, const wchar* mode = L"rb");
hsBool Open(hsStream* stream);
virtual hsBool Close();
virtual UInt32 Read(UInt32 byteCount, void* buffer);

7
Sources/Plasma/PubUtilLib/plFile/plStreamSource.cpp

@ -128,7 +128,12 @@ hsStream* plStreamSource::GetFile(std::wstring filename)
if (plSecureStream::IsSecureFile(sFilename.c_str()))
{
UInt32 encryptionKey[4];
plFileUtils::GetSecureEncryptionKey(sFilename.c_str(), encryptionKey, 4);
if (!plFileUtils::GetSecureEncryptionKey(sFilename.c_str(), encryptionKey, 4))
{
FATAL("Hey camper... You need an NTD key file!");
return nil;
}
fFileData[filename].fStream = plSecureStream::OpenSecureFile(sFilename.c_str(), 0, encryptionKey);
}
else // otherwise it is an encrypted or plain stream, this call handles both

Loading…
Cancel
Save