fix(userdata): Support non-ASCII Documents paths and survive unreadable user files

Core/GameEngine/Include/Common/UserPreferences.h

@@ -68,6 +68,7 @@ class UserPreferences : public PreferenceMap
 
 protected:
 	AsciiString m_filename;
+	Bool m_loadFailed;	///< the file exists but could not be read; write() must not replace it
 };
 
 //-----------------------------------------------------------------------------

Core/GameEngine/Include/GameClient/MapUtil.h

@@ -118,7 +118,7 @@ class MapCache : public std::map<AsciiString, MapMetaData>
 private:
 	void prepareUnseenMaps(const AsciiString &mapDir);
 	Bool clearUnseenMaps(const AsciiString &mapDir);
-	void loadMapsFromMapCacheINI(const AsciiString &mapDir);
+	Bool loadMapsFromMapCacheINI(const AsciiString &mapDir); // returns false if the cache file was unreadable
 	Bool loadMapsFromDisk(const AsciiString &mapDir, Bool isOfficial, Bool filterByAllowedMaps = FALSE); // returns true if we needed to (re)parse a map
 	Bool addMap(const AsciiString &mapDir, const AsciiString &fname, const AsciiString &lowerFname, FileInfo &fileInfo, Bool isOfficial); ///< returns true if it had to (re)parse the map
 	void writeCacheINI(const AsciiString &mapDir);

Core/GameEngine/Source/Common/INI/INI.cpp

@@ -297,6 +297,12 @@ void INI::prepFile( AsciiString filename, INILoadType loadType )
 	m_readBufferNext = 0;
 	m_readBufferUsed = file->size();
 	m_readBuffer = file->readEntireAndClose();
+	if (m_readBuffer == nullptr)
+	{
+		m_readBufferUsed = 0;
+		DEBUG_CRASH(( "INI::load, cannot read file '%s'", filename.str() ));
+		throw INI_CANT_OPEN_FILE;
+	}
 
 	// save our filename
 	m_filename = filename;

Core/GameEngine/Source/Common/System/AsciiString.cpp

@@ -46,6 +46,9 @@
 
 #include "Common/CriticalSection.h"
 
+#if defined(_WIN32) && !defined(WC_NO_BEST_FIT_CHARS)
+#define WC_NO_BEST_FIT_CHARS 0x00000400
+#endif
 
 // -----------------------------------------------------
 
@@ -307,9 +310,43 @@ char* AsciiString::getBufferForRead(Int len)
 void AsciiString::translate(const UnicodeString& stringSrc)
 {
 	validate();
-	/// @todo srj put in a real translation here; this will only work for 7-bit ascii
-	clear();
 	Int len = stringSrc.getLength();
+#ifdef _WIN32
+	// non-ASCII text becomes process code page bytes (UTF-8 on modern Windows); pure ASCII takes the loop below
+	Bool isAscii = true;
+	for (Int i = 0; i < len; i++)
+	{
+		if (stringSrc.getCharAt(i) >= 0x80)
+		{
+			isAscii = false;
+			break;
+		}
+	}
+	if (!isAscii)
+	{
+		// a legacy code page can't hold everything; substitute '_' so generated file names stay valid
+		const Bool isUtf8 = GetACP() == CP_UTF8;
+		const UINT codePage = isUtf8 ? CP_UTF8 : CP_ACP;
+		const DWORD flags = isUtf8 ? 0 : WC_NO_BEST_FIT_CHARS;
+		const char* defaultChar = isUtf8 ? nullptr : "_";
+		const Int narrowLen = WideCharToMultiByte(codePage, flags, stringSrc.str(), len, nullptr, 0, defaultChar, nullptr);
+		if (narrowLen > 0)
+		{
+			char* narrowBuf = new char[narrowLen + 1];
+			const Int written = WideCharToMultiByte(codePage, flags, stringSrc.str(), len, narrowBuf, narrowLen, defaultChar, nullptr);
+			if (written > 0)
+			{
+				narrowBuf[written] = 0;
+				set(narrowBuf);
+				delete[] narrowBuf;
+				validate();
+				return;
+			}
+			delete[] narrowBuf;
+		}
+	}
+#endif
+	clear();
 	for (Int i = 0; i < len; i++)
 		concat((char)stringSrc.getCharAt(i));
 	validate();

Core/GameEngine/Source/Common/System/LocalFile.cpp

@@ -731,9 +731,16 @@ char* LocalFile::readEntireAndClose()
 	UnsignedInt fileSize = size();
 	char* buffer = NEW char[fileSize];
 
-	read(buffer, fileSize);
+	const Int bytesRead = read(buffer, fileSize);
 
 	close();
 
+	// a short read leaves uninitialized bytes in the buffer, which callers would parse as file content
+	if (bytesRead < 0 || (UnsignedInt)bytesRead != fileSize)
+	{
+		delete[] buffer;
+		return nullptr;
+	}
+
 	return buffer;
 }

Core/GameEngine/Source/Common/System/UnicodeString.cpp

@@ -221,9 +221,38 @@ WideChar* UnicodeString::getBufferForRead(Int len)
 void UnicodeString::translate(const AsciiString& stringSrc)
 {
 	validate();
-	/// @todo srj put in a real translation here; this will only work for 7-bit ascii
-	clear();
 	Int len = stringSrc.getLength();
+#ifdef _WIN32
+	// bytes >= 0x80 are process code page text (UTF-8 on modern Windows); pure ASCII takes the loop below
+	Bool isAscii = true;
+	for (Int i = 0; i < len; i++)
+	{
+		if ((unsigned char)stringSrc.getCharAt(i) >= 0x80)
+		{
+			isAscii = false;
+			break;
+		}
+	}
+	if (!isAscii)
+	{
+		const Int wideLen = MultiByteToWideChar(CP_ACP, 0, stringSrc.str(), len, nullptr, 0);
+		if (wideLen > 0)
+		{
+			WideChar* wideBuf = new WideChar[wideLen + 1];
+			const Int written = MultiByteToWideChar(CP_ACP, 0, stringSrc.str(), len, wideBuf, wideLen);
+			if (written > 0)
+			{
+				wideBuf[written] = 0;
+				set(wideBuf);
+				delete[] wideBuf;
+				validate();
+				return;
+			}
+			delete[] wideBuf;
+		}
+	}
+#endif
+	clear();
 	for (Int i = 0; i < len; i++)
 		concat((WideChar)stringSrc.getCharAt(i));
 	validate();

Core/GameEngine/Source/Common/UserPreferences.cpp

@@ -110,7 +110,7 @@ static AsciiString realAsStr(Real val)
 // UserPreferences Class
 //-----------------------------------------------------------------------------
 
-UserPreferences::UserPreferences()
+UserPreferences::UserPreferences() : m_loadFailed(FALSE)
 {
 }
 
@@ -126,8 +126,14 @@ Bool UserPreferences::load(AsciiString fname)
 
 	m_filename = TheGlobalData->getPath_UserData();
 	m_filename.concat(fname);
+	m_loadFailed = FALSE;
 
 	FILE *fp = fopen(m_filename.str(), "r");
+	if (fp == nullptr && errno != ENOENT)
+	{
+		DEBUG_LOG(("UserPreferences::load - could not open '%s', errno %d", m_filename.str(), errno));
+		m_loadFailed = TRUE;
+	}
 	if (fp)
 	{
 		char buf[LINE_LEN];
@@ -138,6 +144,8 @@ Bool UserPreferences::load(AsciiString fname)
 
 			AsciiString key, val;
 			line.nextToken(&key, "=");
+			if (line.isEmpty())	// no '=' in the line, nothing follows the key
+				continue;
 			val = line.str() + 1;
 
 			key.trim();
@@ -148,7 +156,14 @@ Bool UserPreferences::load(AsciiString fname)
 
 			(*this)[key] = val;
 		}
+		const Bool readFailed = ferror(fp) != 0;
 		fclose(fp);
+		if (readFailed)
+		{
+			DEBUG_LOG(("UserPreferences::load - read error in '%s'", m_filename.str()));
+			m_loadFailed = TRUE;
+			return false;
+		}
 		return true;
 	}
 	return false;
@@ -159,17 +174,43 @@ Bool UserPreferences::write()
 	if (m_filename.isEmpty())
 		return false;
 
-	FILE *fp = fopen(m_filename.str(), "w");
+	// the existing file is unreadable, replacing it with what we hold would lose the user's settings
+	if (m_loadFailed)
+		return false;
+
+	// write to a temp file and swap it in so a failed write never leaves a truncated preference file
+	AsciiString tempFilename = m_filename;
+	tempFilename.concat(".tmp");
+
+	FILE *fp = fopen(tempFilename.str(), "w");
 	if (fp)
 	{
+		Bool ok = TRUE;
 		PreferenceMap::const_iterator it = begin();
 		while (it != end())
 		{
-			fprintf(fp, "%s = %s\n", it->first.str(), it->second.str());
+			if (fprintf(fp, "%s = %s\n", it->first.str(), it->second.str()) < 0)
+				ok = FALSE;
 			++it;
 		}
-		fclose(fp);
-		return true;
+		if (fflush(fp) != 0 || ferror(fp))
+			ok = FALSE;
+		if (fclose(fp) != 0)
+			ok = FALSE;
+
+#ifdef _WIN32
+		if (ok && !MoveFileExA(tempFilename.str(), m_filename.str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH))
+			ok = FALSE;
+#else
+		if (ok && rename(tempFilename.str(), m_filename.str()) != 0)
+			ok = FALSE;
+#endif
+		if (!ok)
+		{
+			DEBUG_LOG(("UserPreferences::write - failed to write '%s'", m_filename.str()));
+			remove(tempFilename.str());
+		}
+		return ok;
 	}
 	return false;
 }

Core/GameEngine/Source/GameClient/GUI/IMEManager.cpp

@@ -673,14 +673,15 @@ Bool IMEManager::serviceIMEMessage(	void *windowsHandle, UnsignedInt message,	In
 			// --------------------------------------------------------------------
 			case WM_IME_CHAR:
 			{
-				WideChar wchar = convertCharToWide(wParam);
+				// Unicode windows get UTF-16, ANSI windows get the character in the ANSI code page
+				WideChar wchar = IsWindowUnicode( (HWND)windowsHandle ) ? (WideChar) (wParam & 0xffff) : convertCharToWide(wParam);
 				#ifdef DEBUG_IME
 				DEBUG_LOG(("IMM: WM_IME_CHAR - '%hc'0x%04x", wchar, wchar ));
 				#endif
 
 				if ( m_window && (wchar > 32 || wchar == VK_RETURN ))
 				{
-					TheWindowManager->winSendInputMsg( m_window, GWM_IME_CHAR, (wParam & 0xffff), lParam );
+					TheWindowManager->winSendInputMsg( m_window, GWM_IME_CHAR, wchar, lParam );
 					m_result = 0;
 					return TRUE;
 				}

Core/GameEngine/Source/GameClient/MapUtil.cpp

@@ -342,8 +342,12 @@ void MapCache::writeCacheINI( const AsciiString &mapDir )
 	TheFileSystem->createDirectory(mapDir);
 
 	filepath.concat(m_mapCacheName);
-	FILE *fp = fopen(filepath.str(), "w");
-	DEBUG_ASSERTCRASH(fp != nullptr, ("Failed to create %s", filepath.str()));
+
+	// write to a temp file and swap it in so a failed write never leaves a truncated MapCache.ini
+	AsciiString tempFilepath = filepath;
+	tempFilepath.concat(".tmp");
+	FILE *fp = fopen(tempFilepath.str(), "w");
+	DEBUG_ASSERTCRASH(fp != nullptr, ("Failed to create %s", tempFilepath.str()));
 	if (fp == nullptr) {
 		return;
 	}
@@ -406,7 +410,22 @@ void MapCache::writeCacheINI( const AsciiString &mapDir )
 		}
 	}
 
-	fclose(fp);
+	Bool ok = fflush(fp) == 0 && !ferror(fp);
+	if (fclose(fp) != 0)
+		ok = FALSE;
+
+#ifdef _WIN32
+	if (ok && !MoveFileExA(tempFilepath.str(), filepath.str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH))
+		ok = FALSE;
+#else
+	if (ok && rename(tempFilepath.str(), filepath.str()) != 0)
+		ok = FALSE;
+#endif
+	if (!ok)
+	{
+		DEBUG_LOG(("MapCache::writeCacheINI - failed to write '%s'", filepath.str()));
+		remove(tempFilepath.str());
+	}
 }
 
 void MapCache::updateCache()
@@ -501,16 +520,26 @@ Bool MapCache::clearUnseenMaps( const AsciiString &mapDir )
 	return erasedSomething;
 }
 
-void MapCache::loadMapsFromMapCacheINI( const AsciiString &mapDir )
+Bool MapCache::loadMapsFromMapCacheINI( const AsciiString &mapDir )
 {
 	INI ini;
 	AsciiString fname;
 	fname.format("%s\\%s", mapDir.str(), m_mapCacheName);
 
 	if (TheFileSystem->doesFileExist(fname.str()))
 	{
-		ini.load( fname, INI_LOAD_OVERWRITE, nullptr );
+		// a partial or unreadable cache must not abort startup, the disk scan re-caches the maps and rewrites it
+		try
+		{
+			ini.load( fname, INI_LOAD_OVERWRITE, nullptr );
+		}
+		catch (...)
+		{
+			DEBUG_LOG(("MapCache::loadMapsFromMapCacheINI - ignoring unreadable '%s'", fname.str()));
+			return FALSE;
+		}
 	}
+	return TRUE;
 }
 
 Bool MapCache::loadMapsFromDisk( const AsciiString &mapDir, Bool isOfficial, Bool filterByAllowedMaps )
@@ -634,7 +663,25 @@ Bool MapCache::addMap(
 
 	DEBUG_LOG(("MapCache::addMap(): caching '%s' because '%s' was not found", fname.str(), lowerFname.str()));
 
-	loadMap(fname); // Just load for querying the data, since we aren't playing this map.
+	// Just load for querying the data, since we aren't playing this map.
+	Bool loaded = FALSE;
+	try
+	{
+		loaded = loadMap(fname);
+	}
+	catch (...)
+	{
+		loaded = FALSE;
+	}
+
+	const UnsignedInt mapCRC = loaded ? calcCRC(fname) : 0;
+	if (!loaded || mapCRC == 0)
+	{
+		// never cache a map we could not read; drop a stale entry so writeCacheINI does not persist it
+		DEBUG_LOG(("MapCache::addMap(): could not read '%s', not caching it", fname.str()));
+		resetMap();
+		return erase(lowerFname) > 0;
+	}
 
 	// The map is now loaded.  Pick out what we need.
 	MapMetaData md;
@@ -649,7 +696,7 @@ Bool MapCache::addMap(
 	md.m_timestamp.m_lowTimeStamp = fileInfo.timestampLow;
 	md.m_supplyPositions = m_supplyPositions;
 	md.m_techPositions = m_techPositions;
-	md.m_CRC = calcCRC(fname);
+	md.m_CRC = mapCRC;
 
 	Bool exists = false;
 	AsciiString nameLookupTag = worldDict.getAsciiString(TheKey_mapName, &exists);

Core/GameEngine/Source/GameClient/System/Image.cpp

@@ -253,8 +253,10 @@ void ImageCollection::load( Int textureSize )
 	if(TheGlobalData)
 	{
 		userDataPath.format("%sINI\\MappedImages\\*.ini",TheGlobalData->getPath_UserData().str());
-		if(FindFirstFile(userDataPath.str(), &findData) !=INVALID_HANDLE_VALUE)
+		HANDLE hFind = FindFirstFile(userDataPath.str(), &findData);
+		if(hFind != INVALID_HANDLE_VALUE)
 		{
+			FindClose(hFind);
 			userDataPath.format("%sINI\\MappedImages",TheGlobalData->getPath_UserData().str());
 			ini.loadDirectory(userDataPath, INI_LOAD_OVERWRITE, nullptr );
 		}