Asset Loading from Network at run-time

[Distraught 280]

M2 Download Center

Download Here ( Internal )

Hey guys,

 

I just programmed this feature for my server but I thought it can be really useful for everyone so now I release it.

This stuff is about how you can load images, etc. in the game without directly packing it into the client but uploading them to a web-server. In this tutorial we will make it work for images, but you can extend it to any type of file you want.

There are not much requirements we only use up to C++11 features and you have to have libcurl library.

 

Open up EterLib/ResourceManager.h and add add the following to the end of the class (don't forget to include <future> and <utility>):

private:
	std::list<std::future<CResource*>>		ongoingDownloads;
public:
	void AddDownload(std::future<CResource*>&& f)
	{
		ongoingDownloads.emplace_back(std::forward<std::future<CResource*>>(f));
	}

Go to EterLib/ResourceManager.cpp and find the CResourceManager::Update function, add the following to the end of it:

	for (auto it = ongoingDownloads.begin(); it != ongoingDownloads.end();)
	{
		if (it->wait_for(std::chrono::seconds(0)) == std::future_status::ready)
		{
			it->get()->LoadDownloadedData();
			it = ongoingDownloads.erase(it);
		}
		else
		{
			++it;
		}
	}

Next, open EterLib/Resource.h, find the constructor and modify it like:

CResource(const char* c_szFileName, bool _loadFromNetwork = false);

After add the following to the end of the class:

protected:
	bool loadFromNetwork;
private:
	std::vector<BYTE> downloadedData;
public:
	void LoadDownloadedData();

 

Then go to EterLib/Resource.cpp, find the constructor and also modify it like:

CResource::CResource(const char* c_szFileName, bool _loadFromNetwork) 
	: me_state(STATE_EMPTY)
	, loadFromNetwork(_loadFromNetwork)
{
	SetFileName(c_szFileName);
}

In the same file, add this to the beginning right after the includes:

#include <curl/curl.h>
#include "ResourceManager.h"
  
#define ASSET_SERVER "http://assets.my-website.com/"
  
static size_t CurlWriteCallback(void* contents, size_t size, size_t nmemb, void* userp)
{
	if (nullptr != userp)
	{
		std::vector<BYTE>& vec = *((std::vector<BYTE>*)userp);
		vec.reserve(vec.size() + (size * nmemb));
		for (size_t i = 0; i < size * nmemb; ++i)
		{
			vec.push_back(((BYTE*)contents)[i]);
		}
	}
	return size * nmemb;
}

void CResource::LoadDownloadedData()
{
	if (downloadedData.empty())
		return;

	Clear();

	if (OnLoad(downloadedData.size(), downloadedData.data()))
	{
		me_state = STATE_EXIST;
	}
	else
	{
		Tracef("CResource::Load Error %s\n", GetFileName());
		me_state = STATE_ERROR;
	}

	downloadedData.clear();
}

 

Now, - still in the same file - find CResource::Load function and modify it like this:

void CResource::Load()
{
	if (me_state != STATE_EMPTY)
		return;

	std::string fileName = GetFileName();

	if (loadFromNetwork && downloadedData.empty())
	{
		CResourceManager::instance().AddDownload(std::move(std::async(std::launch::async | std::launch::deferred, [this, fileName]()
			{
				std::string url = ASSET_SERVER;
				url += fileName;

				CURL* curl = curl_easy_init();
				if (curl)
				{
					curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
					curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWriteCallback);
					curl_easy_setopt(curl, CURLOPT_WRITEDATA, &downloadedData);
					curl_easy_perform(curl);
					curl_easy_cleanup(curl);
				}

				return this;
			})));

		fileName = "d:/ymir work/ui/placeholder.tga";
	}

	DWORD		dwStart = ELTimer_GetMSec();
	CMappedFile	file;
	LPCVOID		fileData;

	//Tracenf("Load %s", c_szFileName);

	if (CEterPackManager::Instance().Get(file, fileName.c_str(), &fileData))
	{
		m_dwLoadCostMiliiSecond = ELTimer_GetMSec() - dwStart;
		//Tracef("CResource::Load %s (%d bytes) in %d ms\n", c_szFileName, file.Size(), m_dwLoadCostMiliiSecond);

		if (OnLoad(file.Size(), fileData))
		{
			me_state = STATE_EXIST;
		}
		else
		{
			Tracef("CResource::Load Error %s\n", fileName.c_str());
			me_state = STATE_ERROR;
			return;
		}
	}
	else
	{
		if (OnLoad(0, NULL))
			me_state = STATE_EXIST;
		else
		{
			Tracef("CResource::Load file not exist %s\n", fileName.c_str());
			me_state = STATE_ERROR;
		}
	}
}

 

Still in Resource.cpp, find the CResource::Reload function and modify like:

void CResource::Reload()
{
	Tracef("CResource::Reload %s\n", GetFileName());

	if (loadFromNetwork)
	{
		if (downloadedData.empty())
		{
			std::string fileName = GetFileName();
			CResourceManager::instance().AddDownload(std::move(std::async(std::launch::async | std::launch::deferred, [this, fileName]()
				{
					std::string url = ASSET_SERVER;
					url += "/";
					url += fileName;

					CURL* curl = curl_easy_init();
					if (curl)
					{
						curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
						curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWriteCallback);
						curl_easy_setopt(curl, CURLOPT_WRITEDATA, &downloadedData);
						curl_easy_perform(curl);
						curl_easy_cleanup(curl);
					}

					return this;
				})));
		}
	}
	else
	{
		Clear();

		CMappedFile	file;
		LPCVOID		fileData;

		if (CEterPackManager::Instance().Get(file, GetFileName(), &fileData))
		{
			if (OnLoad(file.Size(), fileData))
			{
				me_state = STATE_EXIST;
			}
			else
			{
				me_state = STATE_ERROR;
				return;
			}
		}
		else
		{
			if (OnLoad(0, NULL))
				me_state = STATE_EXIST;
			else
			{
				me_state = STATE_ERROR;
			}
		}
	}
}

 

Open EterLib/GrpImage.h and modify the constructor:

CGraphicImage(const char* c_szFileName, DWORD dwFilter = D3DX_FILTER_LINEAR, bool loadFromNetwork = false);

In EterLib/GrpImage.cpp also modify it:

CGraphicImage::CGraphicImage(const char * c_szFileName, DWORD dwFilter, bool loadFromNetwork) 
	: CResource(c_szFileName, loadFromNetwork)
	, m_dwFilter(dwFilter)
{
	m_rect.bottom = m_rect.right = m_rect.top = m_rect.left = 0;
}

 

Finally open ScriptLib/Resource.cpp and add this somewhere the beginning:

CResource* NewOnlineImage(const char* c_szFileName)
{
	return new CGraphicImage(c_szFileName, D3DX_FILTER_LINEAR, true);
}

Go down where you see m_resManager.RegisterResourceNewFunctionPointer("jpg", NewImage); and add after:

m_resManager.RegisterResourceNewFunctionPointer("oimg", NewOnlineImage);

 

We're done! Now if you use *.oimg extension anywhere it will load them from what you define as ASSET_SERVER in EterLib/Resource.cpp (http://assets.my-website.com/filename.oimg). You have to rename the image you upload from the original extension to oimg!

 

Put a placeholder image at "d:\ymir work\ui\placeholder.tga" that it will load while waiting for bigger images.

 

GIF in action (normal size image, the pic in the right bottom corner):

 

GIF of loading a big (10MB image) that takes more time:

 

Hope you like it!

[Distraught 280]

Fixed reload function and std::async launch policy. It was using std::launch::async before but to let the operating system use thread pooling (msvc implementation) it had to be std::launch::async | std::launch::deferred.

Edited December 2, 2020 by Distraught

[Helia01 207]

wow

[Kafa 37]

Incredible, thank you!

[Cunoo 52]

This is cool!