

//_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/
//_/_/
//_/_/  This file is part of CMSDK version 2.1.1.260726 (26 July 2026)
//_/_/  
//_/_/  developed by
//_/_/  
//_/_/  Communicative Machines Ltd. 
//_/_/    http://cmlabs.com/
//_/_/  Copyright (c) 2026
//_/_/  
//_/_/  in collaboration with 
//_/_/  
//_/_/  Icelandic Institute for Intelligent Machines ses
//_/_/    Menntavegur 1
//_/_/    101 Reykjavik
//_/_/    ICELAND
//_/_/    http://www.iiim.is/
//_/_/
//_/_/  and is released under the following
//_/_/  
//_/_/  EXTENDED BSD License
//_/_/
//_/_/  Redistribution and use in source and binary forms, with or without
//_/_/  modification, is permitted provided that the following conditions 
//_/_/  are met:
//_/_/
//_/_/  - Redistributions of source code must retain the above copyright 
//_/_/    and collaboration notice, this list of conditions and the following 
//_/_/    disclaimer.
//_/_/
//_/_/  - Redistributions in binary form must reproduce the above copyright 
//_/_/    notice, this list of conditions and the following disclaimer in the 
//_/_/    documentation and/or other materials provided with the distribution.
//_/_/
//_/_/  - Neither the name of its copyright holders nor the names of its 
//_/_/    contributors may be used to endorse or promote products derived from 
//_/_/    this software without specific prior written permission.
//_/_/
//_/_/  - CADIA Clause: The license granted in and to the software under this 
//_/_/    agreement is a limited-use license. The software may not be used in 
//_/_/    furtherance of: (i) intentionally causing bodily injury or severe 
//_/_/    emotional harm to any person; (ii) invading the personal privacy or 
//_/_/    violating the human rights of any person; or (iii) committing or 
//_/_/    preparing for any act of war. 
//_/_/
//_/_/  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
//_/_/  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
//_/_/  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 
//_/_/  PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER 
//_/_/  OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
//_/_/  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
//_/_/  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
//_/_/  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 
//_/_/  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 
//_/_/  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
//_/_/  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//_/_/
//_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/
/*! @file DataMessage.cpp
	@brief Implementation of DataMessage: construction, flat-block entry management, format conversion and (de)serialisation to JSON/XML/CSV.
	See DataMessage.h for the API documentation; comments here cover implementation details only.
	@ingroup Messaging
*/
#include "DataMessage.h"
#include "PsyTime.h"
#include "UnitTestFramework.h"

namespace cmlabs {

/*
   IMPLEMENTATION NOTES - binary layout
   ------------------------------------
   A DataMessage is ONE contiguous heap block (malloc/realloc, never new[])
   pointed to by 'data':

     [DataMessageHeader][entry][entry][entry]...   total bytes = data->size

   The header starts with { uint32 size; uint32 cid; ... }, so the first
   4 bytes of any serialised message are its total size and the next 4 the
   class id (DATAMESSAGEID). Because the block is already flat, "serialise"
   is simply "send data->size bytes starting at data" - there is no separate
   marshalling step and no per-field byte swapping: all integers are stored
   in HOST byte order. All supported platforms (x86/x64, ARM/ARM64 in the
   default configs) are little-endian, so blocks are wire-compatible between
   nodes; only the transport-level frame length (see NetworkProtocols.cpp)
   uses network byte order.

   Each user entry is a PAIR of tagged records laid out back to back:

     [DataMessageEntryHeader{size,cid=CONSTCHARID}][key chars + NUL]
     [DataMessageEntryHeader{size,cid=<value type>}][value bytes]

   where entry.size is the full record size INCLUDING its own header, so a
   walker advances with 'src += ((DataMessageEntryHeader*)src)->size'. Value
   cids are INTID (int64), DOUBLEID (float64), TIMEID (uint64 ms),
   CONSTCHARID (NUL-terminated text), CHARDATAID (raw bytes), DATAMESSAGEID
   (a nested, fully serialised message) and the *INFOID variants which hold
   only an int64 byte count as a placeholder for content that was stripped
   (see the "draft" constructors below).

   Arrays and maps are NOT separate structures: they are encoded purely in
   the key string. Element i of array "foo" is stored under the mangled key
   "foo_-|i|-_" and map entry "bar" of map "m" under "m_-[bar]-_". All the
   array/map getters below therefore do a linear scan matching the key
   prefix/suffix; there is no index. Lookup of single keys is a linear scan
   too (findEntry() in the header), assisted by a one-slot find cache that
   must be invalidated (invalidateFindKeyCache()) whenever the block is
   modified or reallocated.
*/

std::string DataMessageHeader::toJSON(std::map<uint16, std::string>* subtypes, std::map<uint16, std::string>* subcontexts, std::map<uint32, std::string>* compNames) {
	if (cid != DATAMESSAGEID) return 0;
	if (!subtypes && !subcontexts && !compNames)
		return utils::StringFormat("{ \"size\": %u, \"memid\": \"%llu\", \"time\": \"%llu\", \"time__desc\": \"%s\", \"sendtime\": \"%llu\", \"sendtime__desc\": \"%s\", \"recvtime\": \"%llu\", \"recvtime__desc\": \"%s\", "
			"\"modulecputime\": %u, \"modulewalltime\": %u, \"chaincputime\": %u, \"chainwalltime\": %u, \"chaincount\": %u, "
			"\"origin\": %u, \"destination\": %u, \"ttl\": \"%llu\", \"eol__desc\": \"%s\", \"priority\": %u, \"policy\": %u, \"from\": %u, \"to\": %u, \"type\": \"%s\", \"tag\": \"%u\", \"status\": %u, \"reference\": \"%llu\", \"serial\": \"%llu\", \"contextchange\": \"%s\", \"userdatacount\": %u, \"userdatasize\": %u }",
			size, memid,
			time, time ? PrintTimeString(time).c_str() : "",
			sendtime, sendtime ? PrintTimeString(sendtime).c_str() : "",
			recvtime, recvtime ? PrintTimeString(recvtime).c_str() : "",
			cyclecputime, cyclewalltime, chaincputime, chainwalltime, chaincount,
			origin, destination, ttl, ttl ? PrintTimeString(time + ttl).c_str() : "",
			priority, policy, from, to, type.toString().c_str(), tag, status, reference, serial,
			contextchange.isValid() ? contextchange.toString().c_str() : "",
			userCount, userSize);

	std::string typeStr, contextStr, fromStr, toStr;
	typeStr = type.toString(subtypes);
	contextStr = contextchange.toString(subcontexts);
	if (compNames) {
		fromStr = (*compNames)[from];
		toStr = (*compNames)[to];
	}
	if (!fromStr.length()) {
		if (from)
			fromStr = utils::StringFormat("Component %u", from);
	}
	if (!toStr.length()) {
		if (to)
			toStr = utils::StringFormat("Component %u", to);
	}
	return utils::StringFormat("{ \"size\": %u, \"memid\": \"%llu\", \"time\": \"%llu\", \"time__desc\": \"%s\", \"sendtime\": \"%llu\", \"sendtime__desc\": \"%s\", \"recvtime\": \"%llu\", \"recvtime__desc\": \"%s\", "
		"\"modulecputime\": %u, \"modulewalltime\": %u, \"chaincputime\": %u, \"chainwalltime\": %u, \"chaincount\": %u, "
		"\"origin\": %u, \"destination\": %u, \"ttl\": \"%llu\", \"eol__desc\": \"%s\", \"priority\": %u, \"policy\": %u, \"from\": \"%s\", \"to\": \"%s\", \"type\": \"%s\", \"tag\": \"%u\", \"status\": %u, \"reference\": \"%llu\", \"serial\": \"%llu\", \"contextchange\": \"%s\", \"userdatacount\": %u, \"userdatasize\": %u }",
		size, memid,
		time, time ? PrintTimeString(time).c_str() : "",
		sendtime, sendtime ? PrintTimeString(sendtime).c_str() : "",
		recvtime, recvtime ? PrintTimeString(recvtime).c_str() : "",
		cyclecputime, cyclewalltime, chaincputime, chainwalltime, chaincount,
		origin, destination, ttl, ttl ? PrintTimeString(time + ttl).c_str() : "",
		priority, policy,
		fromStr.c_str(),
		toStr.c_str(),
		typeStr.c_str(), tag, status, reference, serial,
		contextStr.c_str(), userCount, userSize);

}

std::string DataMessageHeader::toXML(std::map<uint16, std::string>* subtypes, std::map<uint16, std::string>* subcontexts, std::map<uint32, std::string>* compNames) {
	if (cid != DATAMESSAGEID) return 0;

	if (!subtypes && !subcontexts && !compNames)
		return utils::StringFormat("<datamessage size=\"%u\" memid=\"%llu\" time=\"%llu\" timetext=\"%s\" sendtime=\"%llu\" sendtimetext=\"%s\" recvtime=\"%llu\" recvtimetext=\"%s\" "
			"modulecputime=\"%u\" modulewalltime=\"%u\" chaincputime=\"%u\" chainwalltime=\"%u\" chaincount=\"%u\" "
			"origin=\"%u\" destination=\"%u\" ttl=\"%llu\" eoltext=\"%s\" priority=\"%u\" policy=\"%u\" from=\"%u\" to=\"%u\" type=\"%s\" tag=\"%u\" status=\"%u\" reference=\"%llu\" serial=\"%llu\" contextchange=\"%s\" userdatacount=\"%u\" userdatasize=\"%u\" />\n",
			size, memid,
			time, time ? PrintTimeString(time).c_str() : "",
			sendtime, sendtime ? PrintTimeString(sendtime).c_str() : "",
			recvtime, recvtime ? PrintTimeString(recvtime).c_str() : "",
			cyclecputime, cyclewalltime, chaincputime, chainwalltime, chaincount,
			origin, destination, ttl, ttl ? PrintTimeString(time + ttl).c_str() : "", priority, policy, from, to, type.toString().c_str(), tag, status, reference, serial, contextchange.isValid() ? contextchange.toString().c_str() : "", userCount, userSize);

	std::string typeStr, contextStr, fromStr, toStr;
	typeStr = type.toString(subtypes);
	contextStr = contextchange.toString(subcontexts);
	if (compNames) {
		fromStr = (*compNames)[from];
		toStr = (*compNames)[to];
	}
	if (!fromStr.length()) {
		if (from)
			fromStr = utils::StringFormat("Component %u", from);
	}
	if (!toStr.length()) {
		if (to)
			toStr = utils::StringFormat("Component %u", to);
	}

	return utils::StringFormat("<datamessage size=\"%u\" memid=\"%llu\" time=\"%llu\" timetext=\"%s\" sendtime=\"%llu\" sendtimetext=\"%s\" recvtime=\"%llu\" recvtimetext=\"%s\" "
		"modulecputime=\"%u\" modulewalltime=\"%u\" chaincputime=\"%u\" chainwalltime=\"%u\" chaincount=\"%u\" "
		"origin=\"%u\" destination=\"%u\" ttl=\"%llu\" eoltext=\"%s\" priority=\"%u\" policy=\"%u\" from=\"%s\" to=\"%s\" type=\"%s\" tag=\"%u\" status=\"%u\" reference=\"%llu\" serial=\"%llu\" contextchange=\"%s\" userdatacount=\"%u\" userdatasize=\"%u\" />\n",
		size, memid,
		time, time ? PrintTimeString(time).c_str() : "",
		sendtime, sendtime ? PrintTimeString(sendtime).c_str() : "",
		recvtime, recvtime ? PrintTimeString(recvtime).c_str() : "",
		cyclecputime, cyclewalltime, chaincputime, chainwalltime, chaincount,
		origin, destination, ttl, ttl ? PrintTimeString(time + ttl).c_str() : "", priority, policy,
		fromStr.c_str(),
		toStr.c_str(),
		typeStr.c_str(), tag, status, reference, serial, contextStr.c_str(), userCount, userSize);
}


// Bulk-import every user entry from another message by walking its flat
// block directly: each key record (CONSTCHARID) is followed by its value
// record, which is re-inserted here via setRawData() (deep copy, preserving
// the original value cid). Returns the number of entries copied.
uint32 DataMessage::copyUserEntriesFromMessage(DataMessage* msg) {
	if ((data->cid != DATAMESSAGEID) || !msg || (msg->data->cid != DATAMESSAGEID))
		return 0;

	if (!msg->data->userCount)
		return 0;

	uint32 count = 0;
	const char* key;
	// Walk [header][entry][entry]... of the source block; srcEnd guards the walk.
	char* src = (char*)(msg->data) + sizeof(DataMessageHeader);
	char* srcEnd = (char*)(msg->data) + msg->data->size;
	while (src < srcEnd) {
		// A CONSTCHARID record here is a key; the value record follows it directly.
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			key = src + sizeof(DataMessageEntryHeader);
			src += sizeof(DataMessageEntryHeader) + strlen(key) + 1;
			if (setRawData(key, src + sizeof(DataMessageEntryHeader), ((DataMessageEntryHeader*)src)->size - sizeof(DataMessageEntryHeader), ((DataMessageEntryHeader*)src)->cid))
				count++;
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	return count;
}

namespace {
	// Scratch records used while grouping mangled array/map keys during the
	// text/JSON/XML dumps: they remember where each element's value record
	// lives inside the flat block so it can be printed under its parent key.
	struct ArrayElem { int64 index; char* entryPtr; uint32 cid; };
	struct MapElem { std::string index; char* entryPtr; uint32 cid; };
}

// Human-readable dump of all user entries. Single pass over the flat block:
// plain keys are printed immediately; keys carrying the array ("_-|i|-_")
// or map ("_-[k]-_") mangling are collected into userArrays/userMaps first
// and printed grouped (indented) after the scan. Nested DATAMESSAGEID
// entries recurse via a temporary DataMessage wrapper.
std::string DataMessage::getUserEntriesAsString() {
	if (data->cid != DATAMESSAGEID) return 0;

	if (!data->userCount)
		return "";

	std::map< std::string, std::list<ArrayElem> > userArrays;
	std::map< std::string, std::list<MapElem> > userMaps;

	DataMessage* msg;
	const char* key;
	const char* mapStart, *mapEnd;
	std::string userData;
	userData.reserve(data->userSize + data->userCount * 64);
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			key = src + sizeof(DataMessageEntryHeader);
			src += sizeof(DataMessageEntryHeader) + strlen(key) + 1;
			if ((mapStart = strstr(key, "_-|")) && (mapEnd = strstr(key, "|-_"))) {
				ArrayElem e;
				e.index = utils::Ascii2Int64(std::string(mapStart + 3, mapEnd - mapStart - 3).c_str());
				e.entryPtr = src;
				e.cid = ((DataMessageEntryHeader*)src)->cid;
				userArrays[std::string(key, mapStart - key)].push_back(e);
				src += ((DataMessageEntryHeader*)src)->size;
				continue;
			}
			else if ((mapStart = strstr(key, "_-[")) && (mapEnd = strstr(key, "]-_"))) {
				MapElem e;
				e.index = std::string(mapStart + 3, mapEnd - mapStart - 3);
				e.entryPtr = src;
				e.cid = ((DataMessageEntryHeader*)src)->cid;
				userMaps[std::string(key, mapStart - key)].push_back(e);
				src += ((DataMessageEntryHeader*)src)->size;
				continue;
			}
			switch (((DataMessageEntryHeader*)src)->cid) {
				case CONSTCHARID:
					userData += utils::StringFormat("'%s' : '%s' (string)\n", key, src + sizeof(DataMessageEntryHeader));
					break;
				case TIMEID:
					userData += utils::StringFormat("'%s' : %s (time)\n",
						key,
						*(uint64*)(src + sizeof(DataMessageEntryHeader)) ? PrintTimeString(*(uint64*)(src + sizeof(DataMessageEntryHeader))).c_str() : ""
						);
					break;
				case CHARDATAID:
					userData += utils::StringFormat("'%s' : size %u (binary)\n", key, ((DataMessageEntryHeader*)src)->size - sizeof(DataMessageEntryHeader));
					break;
				case CONSTCHARINFOID:
					userData += utils::StringFormat("'%s' : (string of size %lld)\n", key, *(int64*)(src + sizeof(DataMessageEntryHeader)));
					break;
				case CHARDATAINFOID:
					userData += utils::StringFormat("'%s' : (binary of size %lld)\n", key, *(int64*)(src + sizeof(DataMessageEntryHeader)));
					break;
				case DATAMESSAGEINFOID:
					userData += utils::StringFormat("'%s' : (message of size %lld)\n", key, *(int64*)(src + sizeof(DataMessageEntryHeader)));
					break;
				case INTID:
					userData += utils::StringFormat("'%s' : %lld (integer)\n", key, *(int64*)(src + sizeof(DataMessageEntryHeader)));
					break;
				case DOUBLEID:
					userData += utils::StringFormat("'%s' : %f (float)\n", key, *(float64*)(src + sizeof(DataMessageEntryHeader)));
					break;
				case DATAMESSAGEID:
					msg = new DataMessage(src + sizeof(DataMessageEntryHeader));
					if (msg) {
						if (userData.length()) userData += ", ";
						userData += utils::StringFormat("'%s' : (message)\n%s\n", key, utils::TextIndent(msg->getUserEntriesAsString().c_str(), " -").c_str());
					}
					delete(msg);
					break;
				case DATAMESSAGEDRAFTID:
					// ignore
				default:
					break;
			}
		}
		// else cannot generate JSON for this entry
		src += ((DataMessageEntryHeader*)src)->size;
	}

	std::string subUserData;

	std::list<ArrayElem>::iterator entryII, entryEE;
	std::map< std::string, std::list<ArrayElem> >::iterator keyII = userArrays.begin(), keyEE = userArrays.end();
	while (keyII != keyEE) {
		subUserData.clear();
		entryII = keyII->second.begin();
		entryEE = keyII->second.end();
		while (entryII != entryEE) {
			char* ep = entryII->entryPtr;
			char* val = ep + sizeof(DataMessageEntryHeader);
			uint32 cid = entryII->cid;
			switch (cid) {
			case CONSTCHARID:
				subUserData += utils::StringFormat("%lld : '%s' (string)\n", entryII->index, val);
				break;
			case TIMEID:
				subUserData += utils::StringFormat("%lld : %s (time)\n",
					entryII->index,
					*(uint64*)val ? PrintTimeString(*(uint64*)val).c_str() : ""
				);
				break;
			case CHARDATAID:
				subUserData += utils::StringFormat("%lld : size %u (binary)\n", entryII->index, ((DataMessageEntryHeader*)ep)->size - sizeof(DataMessageEntryHeader));
				break;
			case DATAMESSAGEID:
				msg = new DataMessage(val);
				if (msg) {
					if (subUserData.length()) subUserData += ", ";
					subUserData += utils::StringFormat("%lld : (message)\n%s\n", entryII->index, utils::TextIndent(msg->getUserEntriesAsString().c_str(), " -").c_str());
				}
				delete(msg);
				break;
			case INTID:
				subUserData += utils::StringFormat("%lld : %lld (integer)\n", entryII->index, *(int64*)val);
				break;
			case DOUBLEID:
				subUserData += utils::StringFormat("%lld : %f (float)\n", entryII->index, *(float64*)val);
				break;
			case CONSTCHARINFOID:
				subUserData += utils::StringFormat("%lld : (string of size %lld)\n", entryII->index, *(int64*)val);
				break;
			case CHARDATAINFOID:
				subUserData += utils::StringFormat("%lld : (binary of size %lld)\n", entryII->index, *(int64*)val);
				break;
			case DATAMESSAGEINFOID:
				subUserData += utils::StringFormat("%lld : (message of size %lld)\n", entryII->index, *(int64*)val);
				break;
			default:
				break;
			}
			entryII++;
		}
		userData += utils::StringFormat("'%s' : (array)\n%s", keyII->first.c_str(), utils::TextIndent(subUserData.c_str(), " -").c_str());
		keyII++;
	}

	std::list<MapElem>::iterator entryI, entryE;
	std::map< std::string, std::list<MapElem> >::iterator keyI = userMaps.begin(), keyE = userMaps.end();
	while (keyI != keyE) {
		subUserData.clear();
		entryI = keyI->second.begin();
		entryE = keyI->second.end();
		while (entryI != entryE) {
			char* ep = entryI->entryPtr;
			char* val = ep + sizeof(DataMessageEntryHeader);
			uint32 cid = entryI->cid;
			switch (cid) {
			case CONSTCHARID:
				subUserData += utils::StringFormat("'%s' : '%s' (string)\n", entryI->index.c_str(), val);
				break;
			case TIMEID:
				subUserData += utils::StringFormat("'%s' : %s (time)\n",
					entryI->index.c_str(),
					*(uint64*)val ? PrintTimeString(*(uint64*)val).c_str() : ""
				);
				break;
			case CHARDATAID:
				subUserData += utils::StringFormat("'%s' : size %u (binary)\n", entryI->index.c_str(), ((DataMessageEntryHeader*)ep)->size - sizeof(DataMessageEntryHeader));
				break;
			case DATAMESSAGEID:
				msg = new DataMessage(val);
				if (msg) {
					if (subUserData.length()) subUserData += ", ";
					subUserData += utils::StringFormat("'%s' : (message)\n%s\n", entryI->index.c_str(), utils::TextIndent(msg->getUserEntriesAsString().c_str(), " -").c_str());
				}
				delete(msg);
				break;
			case INTID:
				subUserData += utils::StringFormat("'%s' : %lld (integer)\n", entryI->index.c_str(), *(int64*)val);
				break;
			case DOUBLEID:
				subUserData += utils::StringFormat("'%s' : %f (float)\n", entryI->index.c_str(), *(float64*)val);
				break;
			case CONSTCHARINFOID:
				subUserData += utils::StringFormat("'%s' : (string of size %lld)\n", entryI->index.c_str(), *(int64*)val);
				break;
			case CHARDATAINFOID:
				subUserData += utils::StringFormat("'%s' : (binary of size %lld)\n", entryI->index.c_str(), *(int64*)val);
				break;
			case DATAMESSAGEINFOID:
				subUserData += utils::StringFormat("'%s' : (message of size %lld)\n", entryI->index.c_str(), *(int64*)val);
				break;
			default:
				break;
			}
			entryI++;
		}
		userData += utils::StringFormat("'%s' : (map)\n%s", keyI->first.c_str(), utils::TextIndent(subUserData.c_str(), " -").c_str());
		keyI++;
	}

	return userData;
}

// JSON dump of all user entries; same two-phase walk as
// getUserEntriesAsString() (plain keys inline, mangled array/map keys
// grouped afterwards). With asText=true values are emitted as bare JSON
// scalars; otherwise each value becomes a {"type", "value"} object so the
// receiver can distinguish text/time/binary/etc.
std::string DataMessage::getUserEntriesAsJSON(bool asText) {
	if (data->cid != DATAMESSAGEID) return "{}";

	if (!data->userCount)
		return "{}";

	std::map< std::string, std::list<ArrayElem> > userArrays;
	std::map< std::string, std::list<MapElem> > userMaps;

	const char* oldData = NULL;
	DataMessage* msg = new DataMessage();
	const char* key;
	const char* mapStart, *mapEnd;
	std::string userData;
	userData.reserve(data->userSize + data->userCount * 64);
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			key = src + sizeof(DataMessageEntryHeader);
			src += sizeof(DataMessageEntryHeader) + strlen(key) + 1;
			if ((mapStart = strstr(key, "_-|")) && (mapEnd = strstr(key, "|-_"))) {
				ArrayElem e;
				e.index = utils::Ascii2Int64(std::string(mapStart + 3, mapEnd - mapStart - 3).c_str());
				e.entryPtr = src;
				e.cid = ((DataMessageEntryHeader*)src)->cid;
				userArrays[std::string(key, mapStart - key)].push_back(e);
				src += ((DataMessageEntryHeader*)src)->size;
				continue;
			}
			else if ((mapStart = strstr(key, "_-[")) && (mapEnd = strstr(key, "]-_"))) {
				MapElem e;
				e.index = std::string(mapStart + 3, mapEnd - mapStart - 3);
				e.entryPtr = src;
				e.cid = ((DataMessageEntryHeader*)src)->cid;
				userMaps[std::string(key, mapStart - key)].push_back(e);
				src += ((DataMessageEntryHeader*)src)->size;
				continue;
			}
			switch (((DataMessageEntryHeader*)src)->cid) {
			case CONSTCHARID:
				if (userData.length()) userData += ", ";
				if (!asText)
					userData += utils::StringFormat("\"%s\": {\"type\": \"text\", \"value\": \"%s\" }\n", key, src + sizeof(DataMessageEntryHeader));
				else
					userData += utils::StringFormat("\"%s\": \"%s\"\n", key, src + sizeof(DataMessageEntryHeader));
				break;
			case TIMEID:
				if (userData.length()) userData += ", ";
				if (!asText)
					userData += utils::StringFormat("\"%s\": {\"type\": \"time\", \"value\": %llu, \"text\": \"%s\" }\n",
						key, *(uint64*)(src + sizeof(DataMessageEntryHeader)),
						*(uint64*)(src + sizeof(DataMessageEntryHeader)) ? PrintTimeString(*(uint64*)(src + sizeof(DataMessageEntryHeader))).c_str() : ""
					);
				else
					userData += utils::StringFormat("\"%s\": \"%s\"\n",
						key,
						*(uint64*)(src + sizeof(DataMessageEntryHeader)) ? PrintTimeString(*(uint64*)(src + sizeof(DataMessageEntryHeader))).c_str() : ""
					);
				break;
			case CHARDATAID:
				if (userData.length()) userData += ", ";
				if (!asText)
					userData += utils::StringFormat("\"%s\": {\"type\": \"binary\", \"size\": %u }\n", key, ((DataMessageEntryHeader*)src)->size - sizeof(DataMessageEntryHeader));
				break;
			case INTID:
				if (userData.length()) userData += ", ";
				if (!asText)
					userData += utils::StringFormat("\"%s\": {\"type\": \"integer\", \"value\": %lld }\n", key, *(int64*)(src + sizeof(DataMessageEntryHeader)));
				else
					userData += utils::StringFormat("\"%s\": \"%lld\"\n", key, *(int64*)(src + sizeof(DataMessageEntryHeader)));
				break;
			case DOUBLEID:
				if (userData.length()) userData += ", ";
				if (!asText)
					userData += utils::StringFormat("\"%s\": {\"type\": \"double\", \"value\": %f }\n", key, *(float64*)(src + sizeof(DataMessageEntryHeader)));
				else
					userData += utils::StringFormat("\"%s\": \"%f\"\n", key, *(float64*)(src + sizeof(DataMessageEntryHeader)));
				break;
			case DATAMESSAGEID:
				if (((DataMessageHeader*)(src + sizeof(DataMessageEntryHeader)))->cid == DATAMESSAGEID) {
					if (!oldData)
						oldData = msg->swapMessageData(src + sizeof(DataMessageEntryHeader));
					else
						msg->swapMessageData(src + sizeof(DataMessageEntryHeader));
					//msg = new DataMessage(src + sizeof(DataMessageEntryHeader));
					//if (msg) {
					if (userData.length()) userData += ", ";
					if (!asText)
						userData += utils::StringFormat("\"%s\": {\"type\": \"message\", \"value\": %s }\n", key, msg->toJSON().c_str());
					else
						userData += utils::StringFormat("\"%s\": %s\n", key, msg->getUserEntriesAsJSON(true).c_str());
					//}
					//delete(msg);
				}
				break;
			case CONSTCHARINFOID:
				if (!asText) {
					if (userData.length()) userData += ", ";
					userData += utils::StringFormat("\"%s\": {\"type\": \"textinfo\", \"size\": %u }\n", key, *(int64*)(src + sizeof(DataMessageEntryHeader)));
				}
				break;
			case CHARDATAINFOID:
				if (!asText) {
					if (userData.length()) userData += ", ";
					userData += utils::StringFormat("\"%s\": {\"type\": \"binary\", \"size\": %u }\n", key, *(int64*)(src + sizeof(DataMessageEntryHeader)));
				}
				break;
			case DATAMESSAGEINFOID:
				if (!asText) {
					if (userData.length()) userData += ", ";
					userData += utils::StringFormat("\"%s\": {\"type\": \"messageinfo\", \"size\": %u }\n", key, *(int64*)(src + sizeof(DataMessageEntryHeader)));
				}
				break;
			case DATAMESSAGEDRAFTID:
			default:
				break;
			}
		}
		// else cannot generate JSON for this entry
		src += ((DataMessageEntryHeader*)src)->size;
	}

	if (oldData)
		msg->swapMessageData(oldData);
	delete(msg);

	std::string subUserData;

	std::list<ArrayElem>::iterator entryII, entryEE;
	std::map< std::string, std::list<ArrayElem> >::iterator keyII = userArrays.begin(), keyEE = userArrays.end();
	while (keyII != keyEE) {
		subUserData.clear();
		entryII = keyII->second.begin();
		entryEE = keyII->second.end();
		while (entryII != entryEE) {
			char* ep = entryII->entryPtr;
			char* val = ep + sizeof(DataMessageEntryHeader);
			uint32 cid = entryII->cid;
			if (subUserData.length()) subUserData += ", ";
			switch (cid) {
			case CONSTCHARID:
				if (!asText)
					subUserData += utils::StringFormat("%lld: {\"type\": \"text\", \"value\": \"%s\" }\n", entryII->index, val);
				else
					subUserData += utils::StringFormat("%lld: \"%s\"\n", entryII->index, val);
				break;
			case TIMEID:
				if (!asText)
					subUserData += utils::StringFormat("%lld: {\"type\": \"time\", \"value\": %llu, \"text\": \"%s\" }\n",
						entryII->index, *(uint64*)val,
						*(uint64*)val ? PrintTimeString(*(uint64*)val).c_str() : ""
					);
				else
					subUserData += utils::StringFormat("%lld: \"%s\"\n", entryII->index, *(uint64*)val ? PrintTimeString(*(uint64*)val).c_str() : "");
				break;
			case CHARDATAID:
				if (!asText)
					subUserData += utils::StringFormat("%lld: {\"type\": \"binary\", \"size\": %u }\n", entryII->index, ((DataMessageEntryHeader*)ep)->size - sizeof(DataMessageEntryHeader));
				break;
			case DATAMESSAGEID:
				msg = new DataMessage(val);
				if (msg) {
					if (subUserData.length()) subUserData += ", ";
					if (!asText)
						subUserData += utils::StringFormat("%lld: {\"type\": \"message\", \"value\": %s }\n", entryII->index, msg->toJSON().c_str());
					else
						subUserData += utils::StringFormat("%lld: %s }\n", entryII->index, msg->getUserEntriesAsJSON(true).c_str());
				}
				delete(msg);
				break;
			case INTID:
				if (!asText)
					subUserData += utils::StringFormat("%lld: {\"type\": \"integer\", \"value\": %lld }\n", entryII->index, *(int64*)val);
				else
					subUserData += utils::StringFormat("%lld: %lld\n", entryII->index, *(int64*)val);
				break;
			case DOUBLEID:
				if (!asText)
					subUserData += utils::StringFormat("%lld: {\"type\": \"double\", \"value\": %f }\n", entryII->index, *(float64*)val);
				else
					subUserData += utils::StringFormat("%lld: \"%f\" }\n", entryII->index, *(float64*)val);
				break;
			case CONSTCHARINFOID:
				if (!asText)
					subUserData += utils::StringFormat("%lld: {\"type\": \"textinfo\", \"size\": %lld }", entryII->index, *(int64*)val);
				break;
			case CHARDATAINFOID:
				if (!asText)
					subUserData += utils::StringFormat("%lld: {\"type\": \"binary\", \"size\": %lld }", entryII->index, *(int64*)val);
				break;
			case DATAMESSAGEINFOID:
				if (!asText)
					subUserData += utils::StringFormat("%lld: {\"type\": \"messageinfo\", \"size\": %lld }", entryII->index, *(int64*)val);
				break;
			default:
				break;
			}
			entryII++;
		}
		if (userData.length()) userData += ", ";
		if (!asText)
			userData += utils::StringFormat("\"%s\": {\"type\": \"array\", \"value\":\n{%s} }\n", keyII->first.c_str(), subUserData.c_str());
		else
			userData += utils::StringFormat("\"%s\": {%s}\n", keyII->first.c_str(), subUserData.c_str());
		keyII++;
	}


	std::list<MapElem>::iterator entryI, entryE;
	std::map< std::string, std::list<MapElem> >::iterator keyI = userMaps.begin(), keyE = userMaps.end();
	while (keyI != keyE) {
		subUserData.clear();
		entryI = keyI->second.begin();
		entryE = keyI->second.end();
		while (entryI != entryE) {
			char* ep = entryI->entryPtr;
			char* val = ep + sizeof(DataMessageEntryHeader);
			uint32 cid = entryI->cid;
			if (subUserData.length()) subUserData += ", ";
			switch (cid) {
			case CONSTCHARID:
				if (!asText)
					subUserData += utils::StringFormat("\"%s\": {\"type\": \"text\", \"value\": \"%s\" }\n", entryI->index.c_str(), val);
				else
					subUserData += utils::StringFormat("\"%s\": \"%s\"\n", entryI->index.c_str(), val);
				break;
			case TIMEID:
				if (!asText)
					subUserData += utils::StringFormat("\"%s\": {\"type\": \"time\", \"value\": %llu, \"text\": \"%s\" }\n",
						entryI->index.c_str(), *(uint64*)val,
						*(uint64*)val ? PrintTimeString(*(uint64*)val).c_str() : ""
					);
				else
					subUserData += utils::StringFormat("\"%s\": \"%s\"\n", entryI->index.c_str(), *(uint64*)val ? PrintTimeString(*(uint64*)val).c_str() : "");
				break;
			case CHARDATAID:
				if (!asText)
					subUserData += utils::StringFormat("\"%s\": {\"type\": \"binary\", \"size\": %u }\n", entryI->index.c_str(), ((DataMessageEntryHeader*)ep)->size - sizeof(DataMessageEntryHeader));
				break;
			case DATAMESSAGEID:
				msg = new DataMessage(val);
				if (msg) {
					if (subUserData.length()) subUserData += ", ";
					if (!asText)
						subUserData += utils::StringFormat("\"%s\": {\"type\": \"message\", \"value\": %s }\n", entryI->index.c_str(), msg->toJSON().c_str());
					else
						subUserData += utils::StringFormat("\"%s\": %s\n", entryI->index.c_str(), msg->getUserEntriesAsJSON(true).c_str());
				}
				delete(msg);
				break;
			case INTID:
				if (!asText)
					subUserData += utils::StringFormat("\"%s\": {\"type\": \"integer\", \"value\": %lld }\n", entryI->index.c_str(), *(int64*)val);
				else
					subUserData += utils::StringFormat("\"%s\": \"%lld\"\n", entryI->index.c_str(), *(int64*)val);
				break;
			case DOUBLEID:
				if (!asText)
					subUserData += utils::StringFormat("\"%s\": {\"type\": \"double\", \"value\": %f }\n", entryI->index.c_str(), *(float64*)val);
				else
					subUserData += utils::StringFormat("\"%s\": \"%f\" }\n", entryI->index.c_str(), *(float64*)val);
				break;
			case CONSTCHARINFOID:
				if (!asText)
					subUserData += utils::StringFormat("\"%s\": {\"type\": \"textinfo\", \"size\": %lld }", entryI->index.c_str(), *(int64*)val);
				break;
			case CHARDATAINFOID:
				if (!asText)
					subUserData += utils::StringFormat("\"%s\": {\"type\": \"binary\", \"size\": %lld }", entryI->index.c_str(), *(int64*)val);
				break;
			case DATAMESSAGEINFOID:
				if (!asText)
					subUserData += utils::StringFormat("\"%s\": {\"type\": \"messageinfo\", \"size\": %lld }", entryI->index.c_str(), *(int64*)val);
				break;
			default:
				break;
			}
			entryI++;
		}
		if (userData.length()) userData += ", ";
		if (!asText)
			userData += utils::StringFormat("\"%s\": {\"type\": \"map\", \"value\":\n{%s} }\n", keyI->first.c_str(), subUserData.c_str());
		else
			userData += utils::StringFormat("\"%s\": {%s}\n", keyI->first.c_str(), subUserData.c_str());
		keyI++;
	}

	return utils::StringFormat("{\n%s}\n", userData.c_str());
}


std::string DataMessage::toJSON(std::map<uint16, std::string>* subtypes, std::map<uint16, std::string>* subcontexts, std::map<uint32, std::string>* compNames) {
	if (data->cid != DATAMESSAGEID) return "";

	if (!data->userCount)
		return data->toJSON(subtypes, subcontexts, compNames);

	std::string userData = getUserEntriesAsJSON(false);

	std::map< std::string, std::list<std::string> > userMaps;
	std::map< std::string, std::list<int64> > userArrays;

	uint32 draftSize = 0;
	uint32 draftUserSize = 0;
	uint32 draftUserCount = 0;

	const char* key;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			key = src + sizeof(DataMessageEntryHeader);
			src += sizeof(DataMessageEntryHeader) + strlen(key) + 1;
			switch (((DataMessageEntryHeader*)src)->cid) {
				case DATAMESSAGEDRAFTID:
					if (strcmp(key, "OS") == 0)
						draftSize = (uint32)(*(int64*)(src + sizeof(DataMessageEntryHeader)));
					else if (strcmp(key, "US") == 0)
						draftUserSize = (uint32)(*(int64*)(src + sizeof(DataMessageEntryHeader)));
					else if (strcmp(key, "UC") == 0)
						draftUserCount = (uint32)(*(int64*)(src + sizeof(DataMessageEntryHeader)));
				default:
					break;
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}

	if (draftSize) {
		return utils::StringFormat("{ \"draft\":\"yes\", \"size\": %u, \"memid\": \"%llu\", \"time\": \"%llu\", \"time__text\": \"%s\", \"sendtime\": \"%llu\", \"sendtime__text\": \"%s\", \"recvtime\": \"%llu\", \"recvtime__text\": \"%s\", \"origin\": %u, \"destination\": %u, \"ttl\": \"%llu\", \"eol__text\": \"%s\", \"priority\": %u, \"policy\": %u, \"from\": %u, \"to\": %u, \"type\": \"%s\", \"tag\": \"%u\", \"status\": %u, \"reference\": \"%llu\", \"serial\": \"%llu\", \"contextchange\": \"%s\", \"userdatacount\": %u, \"userdatasize\": %u, \"userdata\":\n%s }\n",
			draftSize, data->memid,
			data->time, data->time ? PrintTimeString(data->time).c_str() : "",
			data->sendtime, data->sendtime ? PrintTimeString(data->sendtime).c_str() : "",
			data->recvtime, data->recvtime ? PrintTimeString(data->recvtime).c_str() : "",
			data->origin, data->destination, data->ttl, data->ttl ? PrintTimeString(data->time + data->ttl).c_str() : "",
			data->priority, data->policy, data->from, data->to, data->type.toString().c_str(), data->tag, data->status, data->reference, data->serial,
			data->contextchange.isValid() ? data->contextchange.toString().c_str() : "",
			draftUserCount, draftUserSize, userData.c_str());
	}
	else {
		return utils::StringFormat("{ \"size\": %u, \"memid\": \"%llu\", \"time\": \"%llu\", \"time__text\": \"%s\", \"sendtime\": \"%llu\", \"sendtime__text\": \"%s\", \"recvtime\": \"%llu\", \"recvtime__text\": \"%s\", \"origin\": %u, \"destination\": %u, \"ttl\": \"%llu\", \"eol__text\": \"%s\", \"priority\": %u, \"policy\": %u, \"from\": %u, \"to\": %u, \"type\": \"%s\", \"tag\": \"%u\", \"status\": %u, \"reference\": \"%llu\", \"serial\": \"%llu\", \"contextchange\": \"%s\", \"userdatacount\": %u, \"userdatasize\": %u, \"userdata\":\n%s }\n",
			data->size, data->memid,
			data->time, data->time ? PrintTimeString(data->time).c_str() : "",
			data->sendtime, data->sendtime ? PrintTimeString(data->sendtime).c_str() : "",
			data->recvtime, data->recvtime ? PrintTimeString(data->recvtime).c_str() : "",
			data->origin, data->destination, data->ttl, data->ttl ? PrintTimeString(data->time + data->ttl).c_str() : "",
			data->priority, data->policy, data->from, data->to, data->type.toString().c_str(), data->tag, data->status, data->reference, data->serial,
			data->contextchange.isValid() ? data->contextchange.toString().c_str() : "",
			data->userCount, data->userSize, userData.c_str());
	}
}

std::string DataMessage::toCSV(const char* separator, const char* preample, std::map<uint16, std::string>* subtypes, std::map<uint16, std::string>* subcontexts, std::map<uint32, std::string>* compNames) {
	// triggername,type,created,sent,received,modulecputime,modulewalltime,chaincputime,chainwalltime,chaincount,size,priority,ttl,from,to,tag,serial,contextchange,userdatacount,userdatasize,userdata

	char sep[64];
	uint32 len;
	if (separator && (len = (uint32)strlen(separator)))
		memcpy(sep, separator, len+1);
	else
		memcpy(sep, ",", 2);

	std::string userData;
	if (data->userCount) {
		userData = this->getUserEntriesAsString();
		utils::StringSingleReplace(userData, sep, "*", false);
		utils::StringSingleReplace(userData, "\n", "||", false);
	}

	std::string typeStr, contextStr, fromStr, toStr;
	typeStr = data->type.toString(subtypes);
	contextStr = data->contextchange.toString(subcontexts);
	if (compNames) {
		fromStr = (*compNames)[data->from];
		toStr = (*compNames)[data->to];
	}
	if (!fromStr.length()) {
		if (data->from)
			fromStr = utils::StringFormat("Component %u", data->from);
	}
	if (!toStr.length()) {
		if (data->to)
			toStr = utils::StringFormat("Component %u", data->to);
	}

	if (preample && strlen(preample)) {
		return utils::StringFormat("%s%s%s%s%s%s%s%s%s%s%u%s%u%s%u%s%u%s%u%s%u%s%u%s%llu%s%s%s%s%s%u%s%llu%s%s%s%u%s%u%s%s\n",
			preample, sep,
			typeStr.c_str(), sep,
			data->time ? PrintTimeString(data->time).c_str() : "", sep,
			data->sendtime ? PrintTimeString(data->sendtime).c_str() : "", sep,
			data->recvtime ? PrintTimeString(data->recvtime).c_str() : "", sep,
			data->cyclecputime, sep,
			data->cyclewalltime, sep,
			data->chaincputime, sep,
			data->chainwalltime, sep,
			data->chaincount, sep,
			data->size, sep,
			data->priority, sep,
			data->ttl, sep,
			fromStr.c_str(), sep,
			toStr.c_str(), sep,
			data->tag, sep,
			data->serial, sep,
			contextStr.c_str(), sep,
			data->userCount, sep,
			data->userSize, sep,
			userData.c_str());
	}
	else {
		return utils::StringFormat("%s%s%s%s%s%s%s%s%u%s%u%s%u%s%u%s%u%s%u%s%u%s%llu%s%s%s%s%s%u%s%llu%s%s%s%u%s%u%s%s\n",
			typeStr.c_str(), sep,
			data->time ? PrintTimeString(data->time).c_str() : "", sep,
			data->sendtime ? PrintTimeString(data->sendtime).c_str() : "", sep,
			data->recvtime ? PrintTimeString(data->recvtime).c_str() : "", sep,
			data->cyclecputime, sep,
			data->cyclewalltime, sep,
			data->chaincputime, sep,
			data->chainwalltime, sep,
			data->chaincount, sep,
			data->size, sep,
			data->priority, sep,
			data->ttl, sep,
			fromStr.c_str(), sep,
			toStr.c_str(), sep,
			data->tag, sep,
			data->serial, sep,
			contextStr.c_str(), sep,
			data->userCount, sep,
			data->userSize, sep,
			userData.c_str());
	}
}

std::string DataMessage::GetCSVHeader(const char* separator, const char* preample) {
	// triggername,type,created,sent,received,modulecputime,modulewalltime,chaincputime,chainwalltime,chaincount,size,priority,ttl,from,to,tag,serial,contextchange,userdatacount,userdatasize,userdata

	char sep[64];
	uint32 len;
	if (separator && (len = (uint32)strlen(separator)))
		memcpy(sep, separator, len + 1);
	else
		memcpy(sep, ",", 2);

	std::string header;
	if (preample && strlen(preample))
		header = utils::StringFormat("%s,type,created,sent,received,modulecputime,modulewalltime,chaincputime,chainwalltime,chaincount,size,priority,ttl,from,to,tag,serial,contextchange,userdatacount,userdatasize,userdata\n", preample);
	else
		header = "type,created,sent,received,modulecputime,modulewalltime,chaincputime,chainwalltime,chaincount,size,priority,ttl,from,to,tag,serial,contextchange,userdatacount,userdatasize,userdata\n";

	if (stricmp(sep, ",") != 0)
		utils::StringSingleReplace(header, ",", sep, false);

	return header;
}


std::string DataMessage::toXML(std::map<uint16, std::string>* subtypes, std::map<uint16, std::string>* subcontexts, std::map<uint32, std::string>* compNames) {
	if (data->cid != DATAMESSAGEID) return "";

	if (!data->userCount)
		return data->toXML(subtypes, subcontexts, compNames);

	std::map< std::string, std::list<ArrayElem> > userArrays;
	std::map< std::string, std::list<MapElem> > userMaps;
	uint32 draftSize = 0;
	uint32 draftUserSize = 0;
	uint32 draftUserCount = 0;

	const char* oldData = NULL;
	DataMessage* msg = new DataMessage();
	const char* key;
	const char* mapStart, *mapEnd;
	std::string userData;
	userData.reserve(data->userSize + data->userCount * 64);
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			key = src + sizeof(DataMessageEntryHeader);
			src += sizeof(DataMessageEntryHeader) + strlen(key) + 1;
			if ((mapStart = strstr(key, "_-|")) && (mapEnd = strstr(key, "|-_"))) {
				ArrayElem e;
				e.index = utils::Ascii2Int64(std::string(mapStart + 3, mapEnd - mapStart - 3).c_str());
				e.entryPtr = src;
				e.cid = ((DataMessageEntryHeader*)src)->cid;
				userArrays[std::string(key, mapStart - key)].push_back(e);
				src += ((DataMessageEntryHeader*)src)->size;
				continue;
			}
			else if ((mapStart = strstr(key, "_-[")) && (mapEnd = strstr(key, "]-_"))) {
				MapElem e;
				e.index = std::string(mapStart + 3, mapEnd - mapStart - 3);
				e.entryPtr = src;
				e.cid = ((DataMessageEntryHeader*)src)->cid;
				userMaps[std::string(key, mapStart - key)].push_back(e);
				src += ((DataMessageEntryHeader*)src)->size;
				continue;
			}
			switch (((DataMessageEntryHeader*)src)->cid) {
			case CONSTCHARID:
				userData += utils::StringFormat("<data name=\"%s\" type=\"text\" value=\"%s\" />\n", key, src + sizeof(DataMessageEntryHeader));
				break;
			case TIMEID:
				userData += utils::StringFormat("<data name=\"%s\" type=\"time\" value=\"%llu\" text=\"%s\" />\n",
					key,
					*(uint64*)(src + sizeof(DataMessageEntryHeader)),
					*(uint64*)(src + sizeof(DataMessageEntryHeader)) ?
					PrintTimeString(*(uint64*)(src + sizeof(DataMessageEntryHeader))).c_str() : ""
					);
				break;
			case CHARDATAID:
				userData += utils::StringFormat("<data name=\"%s\" type=\"binary\" size=\"%u\" />\n", key, ((DataMessageEntryHeader*)src)->size - sizeof(DataMessageEntryHeader));
				break;
			case INTID:
				userData += utils::StringFormat("<data name=\"%s\" type=\"integer\" value=\"%lld\" />\n", key, *(int64*)(src + sizeof(DataMessageEntryHeader)));
				break;
			case DOUBLEID:
				userData += utils::StringFormat("<data name=\"%s\" type=\"float\" value=\"%f\" />\n", key, *(float64*)(src + sizeof(DataMessageEntryHeader)));
				break;
			case DATAMESSAGEID:
				if (((DataMessageHeader*)(src + sizeof(DataMessageEntryHeader)))->cid == DATAMESSAGEID) {
					if (!oldData)
						oldData = msg->swapMessageData(src + sizeof(DataMessageEntryHeader));
					else
						msg->swapMessageData(src + sizeof(DataMessageEntryHeader));
					//msg = new DataMessage(src + sizeof(DataMessageEntryHeader));
					//if (msg)
					userData += utils::StringFormat("<data name=\"%s\" type=\"datamessage\">\n%s</data>", key, msg->toXML().c_str());
					//delete(msg);
				}
				break;
			case CONSTCHARINFOID:
				userData += utils::StringFormat("<data name=\"%s\" type=\"textinfo\" size=\"%u\" />\n", key, *(int64*)(src + sizeof(DataMessageEntryHeader)));
				break;
			case CHARDATAINFOID:
				userData += utils::StringFormat("<data name=\"%s\" type=\"binary\" size=\"%u\" />\n", key, *(int64*)(src + sizeof(DataMessageEntryHeader)));
				break;
			case DATAMESSAGEINFOID:
				userData += utils::StringFormat("<data name=\"%s\" type=\"messageinfo\" size=\"%u\" />\n", key, *(int64*)(src + sizeof(DataMessageEntryHeader)));
				break;
			case DATAMESSAGEDRAFTID:
				if (strcmp(key, "OS") == 0)
					draftSize = (uint32)(*(int64*)(src + sizeof(DataMessageEntryHeader)));
				else if (strcmp(key, "US") == 0)
					draftUserSize = (uint32)(*(int64*)(src + sizeof(DataMessageEntryHeader)));
				else if (strcmp(key, "UC") == 0)
					draftUserCount = (uint32)(*(int64*)(src + sizeof(DataMessageEntryHeader)));
			default:
				break;
			}
		}
		// else cannot generate JSON for this entry
		src += ((DataMessageEntryHeader*)src)->size;
	}

	if (oldData)
		msg->swapMessageData(oldData);
	delete(msg);

	std::string subUserData;

	std::list<ArrayElem>::iterator entryII, entryEE;
	std::map< std::string, std::list<ArrayElem> >::iterator keyII = userArrays.begin(), keyEE = userArrays.end();
	while (keyII != keyEE) {
		subUserData.clear();
		entryII = keyII->second.begin();
		entryEE = keyII->second.end();
		while (entryII != entryEE) {
			char* ep = entryII->entryPtr;
			char* val = ep + sizeof(DataMessageEntryHeader);
			uint32 cid = entryII->cid;
			switch (cid) {
			case CONSTCHARID:
				subUserData += utils::StringFormat("<data name=\"%lld\" type=\"text\" value=\"%s\" />\n", entryII->index, val);
				break;
			case TIMEID:
				subUserData += utils::StringFormat("<data name=\"%lld\" type=\"time\" value=\"%llu\" text=\"%s\" />\n",
					entryII->index, *(uint64*)val,
					*(uint64*)val ? PrintTimeString(*(uint64*)val).c_str() : ""
				);
				break;
			case CHARDATAID:
				subUserData += utils::StringFormat("<data name=\"%lld\" type=\"binary\" size=\"%u\" />\n", entryII->index, ((DataMessageEntryHeader*)ep)->size - sizeof(DataMessageEntryHeader));
				break;
			case INTID:
				subUserData += utils::StringFormat("<data name=\"%lld\" type=\"integer\" value=\"%lld\" />\n", entryII->index, *(int64*)val);
				break;
			case DOUBLEID:
				subUserData += utils::StringFormat("<data name=\"%lld\" type=\"float\" value=\"%f\" />\n", entryII->index, *(float64*)val);
				break;
			case DATAMESSAGEID:
				msg = new DataMessage(val);
				if (msg) {
					subUserData += utils::StringFormat("<data name=\"%lld\" type=\"datamessage\">\n%s</data>", entryII->index, msg->toXML().c_str());
					delete(msg);
				}
				break;
			case CONSTCHARINFOID:
				subUserData += utils::StringFormat("<data name=\"%lld\" type=\"textinfo\" size=\"%lld\" />\n", entryII->index, *(int64*)val);
				break;
			case CHARDATAINFOID:
				subUserData += utils::StringFormat("<data name=\"%lld\" type=\"binary\" size=\"%lld\" />\n", entryII->index, *(int64*)val);
				break;
			case DATAMESSAGEINFOID:
				subUserData += utils::StringFormat("<data name=\"%lld\" type=\"messageinfo\" size=\"%lld\" />\n", entryII->index, *(int64*)val);
				break;
			default:
				break;
			}
			entryII++;
		}
		userData += utils::StringFormat("<array name=\"%s\">\n%s</array>\n", keyII->first.c_str(), subUserData.c_str());
		keyII++;
	}

	std::list<MapElem>::iterator entryI, entryE;
	std::map< std::string, std::list<MapElem> >::iterator keyI = userMaps.begin(), keyE = userMaps.end();
	while (keyI != keyE) {
		subUserData.clear();
		entryI = keyI->second.begin();
		entryE = keyI->second.end();
		while (entryI != entryE) {
			char* ep = entryI->entryPtr;
			char* val = ep + sizeof(DataMessageEntryHeader);
			uint32 cid = entryI->cid;
			switch (cid) {
			case CONSTCHARID:
				subUserData += utils::StringFormat("<data name=\"%s\" type=\"text\" value=\"%s\" />\n", entryI->index.c_str(), val);
				break;
			case TIMEID:
				subUserData += utils::StringFormat("<data name=\"%s\" type=\"time\" value=\"%llu\" text=\"%s\" />\n",
					entryI->index.c_str(), *(uint64*)val,
					*(uint64*)val ? PrintTimeString(*(uint64*)val).c_str() : ""
				);
				break;
			case CHARDATAID:
				subUserData += utils::StringFormat("<data name=\"%s\" type=\"binary\" size=\"%u\" />\n", entryI->index.c_str(), ((DataMessageEntryHeader*)ep)->size - sizeof(DataMessageEntryHeader));
				break;
			case INTID:
				subUserData += utils::StringFormat("<data name=\"%s\" type=\"integer\" value=\"%lld\" />\n", entryI->index.c_str(), *(int64*)val);
				break;
			case DOUBLEID:
				subUserData += utils::StringFormat("<data name=\"%s\" type=\"float\" value=\"%f\" />\n", entryI->index.c_str(), *(float64*)val);
				break;
			case DATAMESSAGEID:
				msg = new DataMessage(val);
				if (msg) {
					subUserData += utils::StringFormat("<data name=\"%s\" type=\"datamessage\">\n%s</data>", entryI->index.c_str(), msg->toXML().c_str());
					delete(msg);
				}
				break;
			case CONSTCHARINFOID:
				subUserData += utils::StringFormat("<data name=\"%s\" type=\"textinfo\" size=\"%lld\" />\n", entryI->index.c_str(), *(int64*)val);
				break;
			case CHARDATAINFOID:
				subUserData += utils::StringFormat("<data name=\"%s\" type=\"binary\" size=\"%lld\" />\n", entryI->index.c_str(), *(int64*)val);
				break;
			case DATAMESSAGEINFOID:
				subUserData += utils::StringFormat("<data name=\"%s\" type=\"messageinfo\" size=\"%lld\" />\n", entryI->index.c_str(), *(int64*)val);
				break;
			default:
				break;
			}
			entryI++;
		}
		userData += utils::StringFormat("<map name=\"%s\">\n%s</map>\n", keyI->first.c_str(), subUserData.c_str());
		keyI++;
	}

	if (draftSize) {
		return utils::StringFormat("<datamessage draft=\"yes\" size=\"%u\" memid=\"%llu\" time=\"%llu\" timetext=\"%s\" sendtime=\"%llu\" sendtimetext=\"%s\" recvtime=\"%llu\" recvtimetext=\"%s\" origin=\"%u\" destination=\"%u\" ttl=\"%llu\" eoltext=\"%s\" priority=\"%u\" policy=\"%u\" from=\"%u\" to=\"%u\" type=\"%s\" tag=\"%u\" status=\"%u\" reference=\"%llu\" serial=\"%llu\" contextchange=\"%s\" userdatacount=\"%u\" userdatasize=\"%u\">\n%s</datamessage>\n",
			draftSize, data->memid,
			data->time, data->time ? PrintTimeString(data->time).c_str() : "",
			data->sendtime, data->sendtime ? PrintTimeString(data->sendtime).c_str() : "",
			data->recvtime, data->recvtime ? PrintTimeString(data->recvtime).c_str() : "",
			data->origin, data->destination, data->ttl, data->ttl ? PrintTimeString(data->time + data->ttl).c_str() : "",
			data->priority, data->policy, data->from, data->to, data->type.toString().c_str(), data->tag, data->status, data->reference, data->serial, data->contextchange.isValid() ? data->contextchange.toString().c_str() : "", draftUserCount, draftUserSize, userData.c_str());
	}
	else {
		return utils::StringFormat("<datamessage size=\"%u\" memid=\"%llu\" time=\"%llu\" timetext=\"%s\" sendtime=\"%llu\" sendtimetext=\"%s\" recvtime=\"%llu\" recvtimetext=\"%s\" origin=\"%u\" destination=\"%u\" ttl=\"%llu\" eoltext=\"%s\" priority=\"%u\" policy=\"%u\" from=\"%u\" to=\"%u\" type=\"%s\" tag=\"%u\" status=\"%u\" reference=\"%llu\" serial=\"%llu\" contextchange=\"%s\" userdatacount=\"%u\" userdatasize=\"%u\">\n%s</datamessage>\n",
			data->size, data->memid,
			data->time, data->time ? PrintTimeString(data->time).c_str() : "",
			data->sendtime, data->sendtime ? PrintTimeString(data->sendtime).c_str() : "",
			data->recvtime, data->recvtime ? PrintTimeString(data->recvtime).c_str() : "",
			data->origin, data->destination, data->ttl, data->ttl ? PrintTimeString(data->time + data->ttl).c_str() : "",
			data->priority, data->policy, data->from, data->to, data->type.toString().c_str(), data->tag, data->status, data->reference, data->serial, data->contextchange.isValid() ? data->contextchange.toString().c_str() : "", data->userCount, data->userSize, userData.c_str());
	}
}




// ################# Constructors #################

// ---------------------------------------------------------------------------
// Constructors. All of them establish the invariant described at the top of
// this file: 'data' is a single malloc'ed block whose header fields size,
// cid (DATAMESSAGEID) and cver (CURRENTDATAMESSAGEVERSION) are always valid.
// Header-only construction is a plain malloc+memset; entries are appended
// later by setRawData() growing the block with realloc.
// ---------------------------------------------------------------------------
DataMessage::DataMessage() {
	this->data = (DataMessageHeader*) malloc(sizeof(DataMessageHeader));
	memset((char*)this->data, 0, sizeof(DataMessageHeader));
	data->size = sizeof(DataMessageHeader);
	data->cid = DATAMESSAGEID;
	data->cver = CURRENTDATAMESSAGEVERSION;
	data->time = GetTimeNow();
	invalidateFindKeyCache();
}

DataMessage::DataMessage(PsyType type, uint32 from) {
	this->data = (DataMessageHeader*) malloc(sizeof(DataMessageHeader));
	memset((char*)this->data, 0, sizeof(DataMessageHeader));
	data->size = sizeof(DataMessageHeader);
	data->cid = DATAMESSAGEID;
	data->cver = CURRENTDATAMESSAGEVERSION;
	data->type = type;
	data->time = GetTimeNow();
	data->from = from;
	invalidateFindKeyCache();
}

DataMessage::DataMessage(PsyType type, uint32 from, uint32 to, uint64 ttl, uint16 priority) {
	this->data = (DataMessageHeader*) malloc(sizeof(DataMessageHeader));
	memset((char*)this->data, 0, sizeof(DataMessageHeader));
	data->size = sizeof(DataMessageHeader);
	data->cid = DATAMESSAGEID;
	data->cver = CURRENTDATAMESSAGEVERSION;
	data->type = type;
	data->time = GetTimeNow();
	data->from = from;
	data->to = to;
	data->ttl = ttl;
	data->priority = priority;
	invalidateFindKeyCache();
}

DataMessage::DataMessage(PsyType type, uint32 from, uint32 to, uint32 tag, uint64 time, uint64 ttl, uint16 priority) {
	this->data = (DataMessageHeader*) malloc(sizeof(DataMessageHeader));
	memset((char*)this->data, 0, sizeof(DataMessageHeader));
	data->size = sizeof(DataMessageHeader);
	data->cid = DATAMESSAGEID;
	data->cver = CURRENTDATAMESSAGEVERSION;
	data->type = type;
	data->time = GetTimeNow();
	data->from = from;
	data->to = to;
	data->tag = tag;
	data->time = time;
	data->ttl = ttl;
	data->priority = priority;
	invalidateFindKeyCache();
}

// Adopting/copying constructor for raw serialised bytes (e.g. straight off a
// socket). Handles three cases:
//  - old-format block (DATAMESSAGEOLDID): converted into a fresh block via
//    ConvertDataFromOlderMessageFormat(); the input is delete[]'d when
//    copy==false because ownership was transferred to us.
//  - current-format block: adopted in place (copy==false, zero-copy - the
//    caller must NOT free it afterwards) or duplicated (copy==true; the
//    total size is read from the leading uint32).
//  - anything else / failed conversion: falls through to an empty message so
//    the object is always usable.
// The "check cpu numbers" fixup repairs cycle/chain CPU counters that were
// written as wrapped-around (MAXVALUINT32 - x) values by older senders.
DataMessage::DataMessage(char* data, bool copy) {
	char* newData = NULL;
	if (GetObjID(data) == DATAMESSAGEOLDID) {
		if (ConvertDataFromOlderMessageFormat(data, &newData) && newData) {
			this->data = (DataMessageHeader*)newData;
			if (!copy)
				delete[] data;
			invalidateFindKeyCache();
			return;
		}
		else
			delete[] newData;
	}
	else if (GetObjID(data) == DATAMESSAGEID) {
		// check cpu numbers
		if (((DataMessageHeader*)data)->cyclecputime > 10000 * (((DataMessageHeader*)data)->cyclewalltime)) {
			((DataMessageHeader*)data)->cyclecputime = MAXVALUINT32 - (((DataMessageHeader*)data)->cyclecputime);
			((DataMessageHeader*)data)->chaincputime = MAXVALUINT32 - (((DataMessageHeader*)data)->chaincputime);
		}
		if (((DataMessageHeader*)data)->cver == CURRENTDATAMESSAGEVERSION) {
			if (!copy)
				this->data = (DataMessageHeader*)data;
			else {
				uint32 size = *(uint32*)data;
				this->data = (DataMessageHeader*)malloc(size);
				memcpy(this->data, data, size);
			}
			invalidateFindKeyCache();
			return;
		}
		else {
			if (ConvertDataFromOlderMessageFormat(data, &newData) && newData) {
				this->data = (DataMessageHeader*)newData;
				if (!copy)
					delete[] data;
				invalidateFindKeyCache();
				return;
			}
			else
				delete[] newData;
		}
	}

	// If we get here, everything else failed, just create empty message
	this->data = (DataMessageHeader*) malloc(sizeof(DataMessageHeader));
	memset((char*)this->data, 0, sizeof(DataMessageHeader));
	this->data->size = sizeof(DataMessageHeader);
	this->data->cid = DATAMESSAGEID;
	this->data->cver = CURRENTDATAMESSAGEVERSION;
	invalidateFindKeyCache();
	return;
}

// Const-input variant of the above: never adopts, always converts or copies
// (the source bytes are left untouched apart from the CPU-counter fixup).
DataMessage::DataMessage(const char* data) {
	char* newData = NULL;
	if (GetObjID(data) == DATAMESSAGEOLDID) {
		if (ConvertDataFromOlderMessageFormat(data, &newData) && newData) {
			this->data = (DataMessageHeader*)newData;
			invalidateFindKeyCache();
			return;
		}
		else
			delete[] newData;
	}
	else if (GetObjID(data) == DATAMESSAGEID) {
		// check cpu numbers
		if (((DataMessageHeader*)data)->cyclecputime > 10 * (((DataMessageHeader*)data)->cyclewalltime)) {
			((DataMessageHeader*)data)->cyclecputime = MAXVALUINT32 - (((DataMessageHeader*)data)->cyclecputime);
			((DataMessageHeader*)data)->chaincputime = MAXVALUINT32 - (((DataMessageHeader*)data)->chaincputime);
		}
		if (((DataMessageHeader*)data)->cver == CURRENTDATAMESSAGEVERSION) {
			uint32 size = *(uint32*)data;
			this->data = (DataMessageHeader*)malloc(size);
			memcpy(this->data, data, size);
			invalidateFindKeyCache();
			return;
		}
		else {
			if (ConvertDataFromOlderMessageFormat(data, &newData) && newData) {
				this->data = (DataMessageHeader*)newData;
				invalidateFindKeyCache();
				return;
			}
			else
				delete[] newData;
		}
	}

	// If we get here, everything else failed, just create empty message
	this->data = (DataMessageHeader*)malloc(sizeof(DataMessageHeader));
	memset((char*)this->data, 0, sizeof(DataMessageHeader));
	this->data->size = sizeof(DataMessageHeader);
	this->data->cid = DATAMESSAGEID;
	this->data->cver = CURRENTDATAMESSAGEVERSION;
	invalidateFindKeyCache();
	return;
}

// Deep-copy constructor: one malloc + one memcpy of the whole flat block.
DataMessage::DataMessage(const DataMessage& msg) {
	if (GetObjID(msg.data) != DATAMESSAGEID) {
		this->data = (DataMessageHeader*) malloc(sizeof(DataMessageHeader));
		memset((char*)this->data, 0, sizeof(DataMessageHeader));
		this->data->size = sizeof(DataMessageHeader);
		this->data->cid = DATAMESSAGEID;
		this->data->cver = CURRENTDATAMESSAGEVERSION;
		invalidateFindKeyCache();
		return;
	}
	this->data = (DataMessageHeader*)malloc(msg.data->size);
	memcpy(this->data, msg.data, msg.data->size);
	invalidateFindKeyCache();
}

// "Draft" constructor: build a size-capped preview of a (possibly huge)
// message. If the source fits within maxDraftSize it is copied verbatim;
// otherwise only the header is copied and fillInDraftUserDataFrom() adds as
// many entries as the budget allows, replacing oversized content with
// *INFOID size placeholders. Old-format input is first converted via a
// temporary full message.
DataMessage::DataMessage(const char* data, uint32 maxDraftSize) {
	if ((GetObjID(data) == DATAMESSAGEOLDID) ||
		((GetObjID(data) == DATAMESSAGEID && ((DataMessageHeader*)data)->cver != CURRENTDATAMESSAGEVERSION))) {
		DataMessage* correctMsg = new DataMessage(data, true);
		const char* newData = (const char*)correctMsg->data;
		uint32 size = *(uint32*)newData;
		if (size <= maxDraftSize) {
			this->data = (DataMessageHeader*)malloc(size);
			memcpy(this->data, newData, size);
		}
		else {
			this->data = (DataMessageHeader*)malloc(maxDraftSize);
			memcpy(this->data, newData, sizeof(DataMessageHeader));
			fillInDraftUserDataFrom(newData, maxDraftSize);
		}
		delete(correctMsg);
		invalidateFindKeyCache();
		return;
	}
	else if (GetObjID(data) != DATAMESSAGEID) {
		this->data = (DataMessageHeader*)malloc(sizeof(DataMessageHeader));
		memset((char*)this->data, 0, sizeof(DataMessageHeader));
		this->data->size = sizeof(DataMessageHeader);
		this->data->cid = DATAMESSAGEID;
		this->data->cver = CURRENTDATAMESSAGEVERSION;
		invalidateFindKeyCache();
		return;
	}

	uint32 size = *(uint32*)data;
	if (size <= maxDraftSize) {
		this->data = (DataMessageHeader*)malloc(size);
		memcpy(this->data, data, size);
	}
	else {
		this->data = (DataMessageHeader*)malloc(maxDraftSize);
		memcpy(this->data, data, sizeof(DataMessageHeader));
		fillInDraftUserDataFrom(data, maxDraftSize);
	}
	invalidateFindKeyCache();
}

// Draft copy of an existing message; same budget logic as above.
DataMessage::DataMessage(const DataMessage& msg, uint32 maxDraftSize) {
	if (GetObjID(msg.data) != DATAMESSAGEID) {
		this->data = (DataMessageHeader*)malloc(sizeof(DataMessageHeader));
		memset((char*)this->data, 0, sizeof(DataMessageHeader));
		this->data->size = sizeof(DataMessageHeader);
		this->data->cid = DATAMESSAGEID;
		this->data->cver = CURRENTDATAMESSAGEVERSION;
		invalidateFindKeyCache();
		return;
	}
	uint32 size = *(uint32*)msg.data;
	if (size <= maxDraftSize) {
		this->data = (DataMessageHeader*)malloc(size);
		memcpy(this->data, msg.data, size);
	}
	else {
		this->data = (DataMessageHeader*)malloc(maxDraftSize);
		memcpy(this->data, msg.data, sizeof(DataMessageHeader));
		fillInDraftUserDataFrom((char*)msg.data, maxDraftSize);
	}
	invalidateFindKeyCache();
}

// Populate a draft message from a full serialised block without exceeding
// maxDraftSize. Strategy:
//   0. record the original totals as DATAMESSAGEDRAFTID entries "OS"
//      (original size), "US" (user bytes) and "UC" (user count) so a reader
//      can tell what was elided;
//   1. first pass copies all SMALL values (time/int/double) verbatim and
//      replaces binary blobs and nested messages by int64 size placeholders
//      (*INFOID cids);
//   2. second pass adds full strings while room remains, then falls back to
//      CONSTCHARINFOID size placeholders.
// Entries that do not even fit as placeholders are merely counted into
// userCount/userSize so the header still reflects the full message.
bool DataMessage::fillInDraftUserDataFrom(const char* rawdata, uint32 maxDraftSize) {
	DataMessageHeader* data = (DataMessageHeader*)rawdata;
	if (data->cid != DATAMESSAGEID) return false;

	if (!data->userCount)
		return true;

	this->data->size = sizeof(DataMessageHeader);
	uint32 draftInfoSize = (uint32)((2 * sizeof(DataMessageEntryHeader)) + 3 + sizeof(int64));
	if (this->data->size + (3 * draftInfoSize) > maxDraftSize)
		return false;

	this->data->userCount = 0;
	this->data->userSize = 0;

	int64 size;
	uint32 itemSize;

	size = data->size;
	setRawData("OS", (char*)&size, sizeof(int64), DATAMESSAGEDRAFTID);
	size = data->userSize;
	setRawData("US", (char*)&size, sizeof(int64), DATAMESSAGEDRAFTID);
	size = data->userCount;
	setRawData("UC", (char*)&size, sizeof(int64), DATAMESSAGEDRAFTID);

	std::map< std::string, std::list<std::string> > userMaps;

	const char* key;
	std::string userData;

	// First try to add all the small data info (int, float, time and info about binary, messages)
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			key = src + sizeof(DataMessageEntryHeader);
			itemSize = (uint32)(sizeof(DataMessageEntryHeader) + strlen(key) + 1);
			src += itemSize;
			switch (((DataMessageEntryHeader*)src)->cid) {
			case TIMEID:
				itemSize += sizeof(DataMessageEntryHeader) + sizeof(uint64);
				if (this->data->size + itemSize < maxDraftSize) {
					setTime(key, *(uint64*)(src + sizeof(DataMessageEntryHeader)));
				}
				else {
					this->data->userCount++;
					this->data->userSize += itemSize;
				}
				break;
			case INTID:
				itemSize += sizeof(DataMessageEntryHeader) + sizeof(int64);
				if (this->data->size + itemSize < maxDraftSize) {
					setInt(key, *(int64*)(src + sizeof(DataMessageEntryHeader)));
				}
				else {
					this->data->userCount++;
					this->data->userSize += itemSize;
				}
				break;
			case DOUBLEID:
				itemSize += sizeof(DataMessageEntryHeader) + sizeof(float64);
				if (this->data->size + itemSize < maxDraftSize) {
					setDouble(key, (double)*(float64*)(src + sizeof(DataMessageEntryHeader)));
				}
				else {
					this->data->userCount++;
					this->data->userSize += itemSize;
				}
				break;
			case CHARDATAID:
				size = ((DataMessageEntryHeader*)src)->size - sizeof(DataMessageEntryHeader);
				itemSize += sizeof(DataMessageEntryHeader) + sizeof(int64);
				if (this->data->size + itemSize < maxDraftSize) {
					setRawData(key, (char*)&size, sizeof(int64), CHARDATAINFOID);
				}
				else {
					this->data->userCount++;
					this->data->userSize += itemSize - sizeof(int64) + (uint32)size;
				}
				break;
			case DATAMESSAGEID:
				size = ((DataMessageEntryHeader*)src)->size - sizeof(DataMessageEntryHeader);
				itemSize += sizeof(DataMessageEntryHeader) + sizeof(int64);
				if (this->data->size + itemSize < maxDraftSize) {
					setRawData(key, (char*)&size, sizeof(int64), DATAMESSAGEID);
				}
				else {
					this->data->userCount++;
					this->data->userSize += itemSize - sizeof(int64) + (uint32)size;
				}
				break;

			case CONSTCHARINFOID:
				size = (*(int64*)(src + sizeof(DataMessageEntryHeader)));
				itemSize += sizeof(DataMessageEntryHeader) + sizeof(int64);
				if (this->data->size + itemSize < maxDraftSize) {
					setRawData(key, (char*)&size, sizeof(int64), CONSTCHARINFOID);
				}
				else {
					this->data->userCount++;
					this->data->userSize += itemSize - sizeof(int64) + (uint32)size;
				}
				break;
			case CHARDATAINFOID:
				size = (*(int64*)(src + sizeof(DataMessageEntryHeader)));
				itemSize += sizeof(DataMessageEntryHeader) + sizeof(int64);
				if (this->data->size + itemSize < maxDraftSize) {
					setRawData(key, (char*)&size, sizeof(int64), CHARDATAINFOID);
				}
				else {
					this->data->userCount++;
					this->data->userSize += itemSize - sizeof(int64) + (uint32)size;
				}
				break;
			case DATAMESSAGEINFOID:
				size = (*(int64*)(src + sizeof(DataMessageEntryHeader)));
				itemSize += sizeof(DataMessageEntryHeader) + sizeof(int64);
				if (this->data->size + itemSize < maxDraftSize) {
					setRawData(key, (char*)&size, sizeof(int64), DATAMESSAGEINFOID);
				}
				else {
					this->data->userCount++;
					this->data->userSize += itemSize - sizeof(int64) + (uint32)size;
				}
				break;
			// Ignore these for now
			case CONSTCHARID:
			default:
				break;
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}

	// Now try to see how many strings we have room for
	src = (char*)data + sizeof(DataMessageHeader);
	srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			key = src + sizeof(DataMessageEntryHeader);
			itemSize = (uint32)(sizeof(DataMessageEntryHeader) + strlen(key) + 1);
			src += itemSize;
			switch (((DataMessageEntryHeader*)src)->cid) {
			case CONSTCHARID:
				size = ((DataMessageEntryHeader*)src)->size - sizeof(DataMessageEntryHeader);
				itemSize += sizeof(DataMessageEntryHeader);
				if (this->data->size + itemSize + size < maxDraftSize) {
					// add the whole thing
					setString(key, (src + sizeof(DataMessageEntryHeader)));
				}
				else if (this->data->size + itemSize + sizeof(int64) < maxDraftSize) {
					setRawData(key, (char*)&size, sizeof(int64), CONSTCHARINFOID);
				}
				else {
					this->data->userCount++;
					this->data->userSize += itemSize + (uint32)size;
				}
				break;
			case TIMEID:
			case INTID:
			case DOUBLEID:
			case CHARDATAID:
			case DATAMESSAGEID:
			case CONSTCHARINFOID:
			case CHARDATAINFOID:
			case DATAMESSAGEINFOID:
			default:
				break;
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}

	return true;
}


// The block was allocated with malloc/realloc, hence free() (not delete).
DataMessage::~DataMessage() {
	//printf("Deleting msg: %u.%u.%u (%p)...\n", data->type[0], data->type[1], data->type[2], this);
	free(data);
	data = NULL;
}

// Swap the underlying flat block for another one WITHOUT copying; returns
// the old block so the caller can free or re-adopt it. Used by memory-pool
// style callers to re-point a wrapper at shared memory. The find cache must
// be dropped because every cached pointer refers to the old block.
const char* DataMessage::swapMessageData(const char* data) {
	invalidateFindKeyCache();
	const char* oldData = (const char*)this->data;
	this->data = (DataMessageHeader*)data;
	return oldData;
}


// ################# User #################

int64 DataMessage::getAsInt(const char* key) {
	return utils::Ascii2Int64(getAsString(key).c_str());
}

uint64 DataMessage::getAsTime(const char* key) {
	return utils::Ascii2Uint64(getAsString(key).c_str());
}

float64 DataMessage::getAsFloat(const char* key) {
	return utils::Ascii2Float64(getAsString(key).c_str());
}

int64 DataMessage::getAsInt(int64 i, const char* key) {
	return utils::Ascii2Int64(getAsString(i, key).c_str());
}

uint64 DataMessage::getAsTime(int64 i, const char* key) {
	return utils::Ascii2Uint64(getAsString(i, key).c_str());
}

float64 DataMessage::getAsFloat(int64 i, const char* key) {
	return utils::Ascii2Float64(getAsString(i, key).c_str());
}

int64 DataMessage::getAsInt(const char* idx, const char* key) {
	return utils::Ascii2Int64(getAsString(idx, key).c_str());
}

uint64 DataMessage::getAsTime(const char* idx, const char* key) {
	return utils::Ascii2Uint64(getAsString(idx, key).c_str());
}

float64 DataMessage::getAsFloat(const char* idx, const char* key) {
	return utils::Ascii2Float64(getAsString(idx, key).c_str());
}


bool DataMessage::getAsBool(const char* key) {

	char* src = findEntry(key);
	if (src == NULL)
		return false;

	uint32 cid = ((DataMessageEntryHeader*)src)->cid;
	src += sizeof(DataMessageEntryHeader);

	switch (cid) {
	case CONSTCHARID:
		return ( (stricmp(src, "yes") == 0) || (stricmp(src, "true") == 0) || (stricmp(src, "1") == 0));
	case TIMEID:
		return (*(uint64*)src != 0);
	case CHARDATAID:
		return true;
	case INTID:
		return (*(int64*)src != 0);
	case DOUBLEID:
		return (((int64)(*(float64*)src)) != 0);
	case DATAMESSAGEID:
		return true;
	case CONSTCHARINFOID:
		return false;
	case CHARDATAINFOID:
		return false;
	case DATAMESSAGEINFOID:
		return false;
	default:
		return false;
	}
}

std::string DataMessage::getAsString(const char* key) {
	char* src = findEntry(key);
	if (src == NULL) {
		if (stricmp(key, "size") == 0)
			return utils::StringFormat("%u", data->size);
		else if (stricmp(key, "cid") == 0)
			return utils::StringFormat("%llu", data->cid);
		else if (stricmp(key, "memid") == 0)
			return utils::StringFormat("%llu", data->memid);
		else if (stricmp(key, "time") == 0)
			return utils::StringFormat("%llu", data->time);
		else if (stricmp(key, "sendtime") == 0)
			return utils::StringFormat("%llu", data->sendtime);
		else if (stricmp(key, "recvtime") == 0)
			return utils::StringFormat("%llu", data->recvtime);
		else if (stricmp(key, "timetext") == 0)
			return data->time ? PrintTimeString(data->time) : "";
		else if (stricmp(key, "sendtimetext") == 0)
			return data->sendtime ? PrintTimeString(data->sendtime) : "";
		else if (stricmp(key, "recvtimetext") == 0)
			return data->recvtime ? PrintTimeString(data->recvtime) : "";
		else if (stricmp(key, "origin") == 0)
			return utils::StringFormat("%u", data->origin);
		else if (stricmp(key, "destination") == 0)
			return utils::StringFormat("%u", data->destination);
		else if (stricmp(key, "ttl") == 0)
			return utils::StringFormat("%llu", data->ttl);
		else if (stricmp(key, "eol") == 0)
			return data->ttl ? utils::StringFormat("%llu", data->time + data->ttl) : "0";
		else if (stricmp(key, "eoltext") == 0)
			return data->ttl ? PrintTimeString(data->time + data->ttl) : "";
		else if (stricmp(key, "priority") == 0)
			return utils::StringFormat("%u", data->priority);
		else if (stricmp(key, "policy") == 0)
			return utils::StringFormat("%u", data->policy);
		else if (stricmp(key, "from") == 0)
			return utils::StringFormat("%u", data->from);
		else if (stricmp(key, "to") == 0)
			return utils::StringFormat("%u", data->to);
		else if (stricmp(key, "type") == 0)
			return data->type.toString();
		else if (stricmp(key, "tag") == 0)
			return utils::StringFormat("%u", data->tag);
		else if (stricmp(key, "status") == 0)
			return utils::StringFormat("%u", data->status);
		else if (stricmp(key, "reference") == 0)
			return utils::StringFormat("%llu", data->reference);
		else if (stricmp(key, "serial") == 0)
			return utils::StringFormat("%llu", data->serial);
		else if (stricmp(key, "contextchange") == 0)
			return data->contextchange.isValid() ? data->contextchange.toString() : "";
		else if (stricmp(key, "userSize") == 0)
			return utils::StringFormat("%u", data->userSize);
		else if (stricmp(key, "userCount") == 0)
			return utils::StringFormat("%u", data->userCount);
		else
			return "";
	}

	switch (((DataMessageEntryHeader*)src)->cid) {
		case CONSTCHARID:
			return src + sizeof(DataMessageEntryHeader);
		case TIMEID:
			return PrintTimeString(*(uint64*)(src + sizeof(DataMessageEntryHeader)));
		case CHARDATAID:
			return utils::StringFormat("BinaryData [%ub]", ((DataMessageEntryHeader*)src)->size);
		case INTID:
			return utils::StringFormat("%lld", *(int64*)(src + sizeof(DataMessageEntryHeader)));
		case DOUBLEID:
			return utils::StringFormat("%f", *(float64*)(src + sizeof(DataMessageEntryHeader)));
		case DATAMESSAGEID:
			return utils::StringFormat("DataMessage [%ub]", ((DataMessageEntryHeader*)src)->size);
		case CONSTCHARINFOID:
			return utils::StringFormat("(%lld)", *(int64*)(src + sizeof(DataMessageEntryHeader)));
		case CHARDATAINFOID:
			return utils::StringFormat("(%lld)", *(int64*)(src + sizeof(DataMessageEntryHeader)));
		case DATAMESSAGEINFOID:
			return utils::StringFormat("(%lld)", *(int64*)(src + sizeof(DataMessageEntryHeader)));
		default:
			return "";
	}
}

const char* DataMessage::getString(const char* key) {
	bool success = false;
	return getString(key, success);
}

const char* DataMessage::getString(const char* key, uint32 &size) {
	bool success = false;
	return getString(key, size, success);
}

uint64 DataMessage::getTime(const char* key) {
	bool success = false;
	return getTime(key, success);
}

bool DataMessage::getInt(const char* key, int64& value) {
	bool success = false;
	value = getInt(key, success);
	return success;
}

bool DataMessage::getDouble(const char* key, double& value) {
	bool success = false;
	value = getDouble(key, success);
	return success;
}

bool DataMessage::getFloat(const char* key, float64& value) {
	bool success = false;
	value = getFloat(key, success);
	return success;
}

const char* DataMessage::getData(const char* key, uint32& size) {
	bool success = false;
	return getData(key, size, success);
}

char* DataMessage::getDataCopy(const char* key, uint32& size) {
	bool success = false;
	return getDataCopy(key, size, success);
}

DataMessage* DataMessage::getAttachedMessageCopy(const char* key) {
	bool success = false;
	return getAttachedMessageCopy(key, success);
}





// ################# User with success flag #################

const char* DataMessage::getString(const char* key, bool& success) {
	success = false;
	char* src = findEntry(key);
	if (src == NULL)
		return NULL;

	if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
		success = true;
		return src+sizeof(DataMessageEntryHeader);
	}

	return NULL;
}

const char* DataMessage::getString(const char* key, uint32 &size, bool& success) {
	size = 0;
	success = false;
	char* src = findEntry(key);
	if (src == NULL)
		return NULL;

	if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
		success = true;
		size = ((DataMessageEntryHeader*)src)->size - sizeof(DataMessageEntryHeader) - 1;
		return src+sizeof(DataMessageEntryHeader);
	}

	return NULL;
}

uint32 DataMessage::getContentType(const char* key) {
	char* src = findEntry(key);
	if (src == NULL) return 0;
	return ((DataMessageEntryHeader*)src)->cid;
}

uint32 DataMessage::getContentSize(const char* key) {
	char* src = findEntry(key);
	if (src == NULL) return 0;
	return ((DataMessageEntryHeader*)src)->size;
}


uint64 DataMessage::getTime(const char* key, bool& success) {
	success = false;
	char* src = findEntry(key);
	if (src == NULL) return 0;

	if (((DataMessageEntryHeader*)src)->cid == TIMEID) {
		success = true;
		return *(uint64*)(src+sizeof(DataMessageEntryHeader));
	}

	return 0;
}

int64 DataMessage::getInt(const char* key, bool& success) {
	success = false;
	char* src = findEntry(key);
	if (src == NULL) return 0;

	if (((DataMessageEntryHeader*)src)->cid == INTID) {
		success = true;
		return *(int64*)(src+sizeof(DataMessageEntryHeader));
	}
	return 0;
}

int64 DataMessage::getInt(const char* key) {
	bool success = false;
	return getInt(key, success);
}

double DataMessage::getDouble(const char* key, bool& success) {
	success = false;
	char* src = findEntry(key);
	if (src == NULL) return false;

	if (((DataMessageEntryHeader*)src)->cid == DOUBLEID) {
		success = true;
		return *(float64*)(src + sizeof(DataMessageEntryHeader));
	}
	return false;
}

float64 DataMessage::getFloat(const char* key, bool& success) {
	success = false;
	char* src = findEntry(key);
	if (src == NULL) return false;

	if (((DataMessageEntryHeader*)src)->cid == DOUBLEID) {
		success = true;
		return *(float64*)(src + sizeof(DataMessageEntryHeader));
	}
	return false;
}

double DataMessage::getDouble(const char* key) {
	bool success = false;
	return getDouble(key, success);
}

double DataMessage::getFloat(const char* key) {
	bool success = false;
	return getFloat(key, success);
}

const char* DataMessage::getData(const char* key, uint32& size, bool& success) {
	success = false;
	char* src = findEntry(key);
	if (src == NULL) return NULL;

	if (((DataMessageEntryHeader*)src)->cid == CHARDATAID) {
		success = true;
		size = ((DataMessageEntryHeader*)src)->size - sizeof(DataMessageEntryHeader);
		return src+sizeof(DataMessageEntryHeader);
	}

	return NULL;
}

char* DataMessage::getDataCopy(const char* key, uint32& size, bool& success) {
	success = false;
	char* src = findEntry(key);
	if (src == NULL) return NULL;

	if (((DataMessageEntryHeader*)src)->cid == CHARDATAID) {
		success = true;
		size = ((DataMessageEntryHeader*)src)->size - sizeof(DataMessageEntryHeader);
		char* data = new char[size];
		memcpy(data, src+sizeof(DataMessageEntryHeader), size);
		return data;
	}

	return NULL;
}

DataMessage* DataMessage::getAttachedMessageCopy(const char* key, bool& success) {
	success = false;
	char* src = findEntry(key);
	if (src == NULL) return NULL;

	if (((DataMessageEntryHeader*)src)->cid == DATAMESSAGEID) {
		success = true;
		return new DataMessage(src+sizeof(DataMessageEntryHeader), true);
	}
	return NULL;
}













bool DataMessage::setString(const char* key, const char* value) {
	if (!key || !value)
		return false;
	return setRawData(key, value, (uint32)strlen(value)+1, CONSTCHARID);
}

bool DataMessage::setTime(const char* key, uint64 value) {
	return setRawData(key, (char*)&value, sizeof(uint64), TIMEID);
}

bool DataMessage::setInt(const char* key, int64 value) {
	return setRawData(key, (char*)&value, sizeof(int64), INTID);
}

bool DataMessage::setDouble(const char* key, double value) {
	float64 val = value;
	return setRawData(key, (char*)&val, sizeof(float64), DOUBLEID);
}

bool DataMessage::setFloat(const char* key, float64 value) {
	return setRawData(key, (char*)&value, sizeof(float64), DOUBLEID);
}

bool DataMessage::setData(const char* key, const char* value, uint32 size) {
	return setRawData(key, value, size, CHARDATAID);
}

bool DataMessage::setAttachedMessage(const char* key, DataMessage* msg) {
	if (!msg->isValid())
		return false;
	return setRawData(key, (char*)msg->data, msg->data->size, DATAMESSAGEID);
}

bool DataMessage::setRawData(const char* key, const char* value, uint32 size, uint32 datatype) {
	invalidateFindKeyCache();
	char* src = findEntry(key);
	
	uint32 spaceReq = sizeof(DataMessageEntryHeader) + size;
	uint32 fullSpaceReq = 2 * sizeof(DataMessageEntryHeader) + (uint32)strlen(key) + 1 + size;

	if (src != NULL) { // overwrite?
		if (((DataMessageEntryHeader*)src)->size >= spaceReq) {
			((DataMessageEntryHeader*)src)->cid = datatype;
			memcpy(src+sizeof(DataMessageEntryHeader), value, size);
			//data->userCount++;
			//data->userSize += fullSpaceReq;
			return true;
		}
		else {
			removeEntry(key);
			// and act as if not found, ie. continue
		}
	}
	
	src = findSpace(fullSpaceReq);
	if (src == NULL) {
		// Space not found, create a new one
	//	if (!ownsData) return false;

		uint32 oldSize = ((DataMessageHeader*)data)->size;
		data = (DataMessageHeader*)realloc(data, oldSize + fullSpaceReq);
		data->size = oldSize + fullSpaceReq;
		invalidateFindKeyCache();

		src = (char*)data + oldSize;
		((DataMessageEntryHeader*)src)->size = fullSpaceReq;
	}

	((DataMessageEntryHeader*)src)->cid = CONSTCHARID;
	utils::strcpyavail(src+sizeof(DataMessageEntryHeader), key, (uint32)strlen(key)+1, true);

	src += sizeof(DataMessageEntryHeader) + strlen(key) + 1;
	((DataMessageEntryHeader*)src)->cid = datatype;
	((DataMessageEntryHeader*)src)->size = sizeof(DataMessageEntryHeader) + size;
	memcpy(src+sizeof(DataMessageEntryHeader), value, size);
	data->userCount++;
	data->userSize += fullSpaceReq;

	return true;
}







// Arrays

bool DataMessage::isArrayContent(const char* key) {
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-|", key);
	if (data->cid != DATAMESSAGEID) return false;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false) &&
				utils::TextEndsWith(src + sizeof(DataMessageEntryHeader), "|-_", false))
				return true;
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return false;
}

bool DataMessage::isMapContent(const char* key) {
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-[", key);
	if (data->cid != DATAMESSAGEID) return false;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false) && 
				utils::TextEndsWith(src + sizeof(DataMessageEntryHeader), "]-_", false) )
				return true;
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return false;
}

DataMessage::KeyType DataMessage::getKeyType(const char* key) {
	uint32 size;
	char* arrayKeyStart = utils::StringFormat(size, "%s_-|", key);
	char* mapKeyStart = utils::StringFormat(size, "%s_-[", key);
	if (data->cid != DATAMESSAGEID) return NONE;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			if (stricmp(key, src + sizeof(DataMessageEntryHeader)) == 0)
				return SINGLE;
			if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), arrayKeyStart, false) &&
				utils::TextEndsWith(src + sizeof(DataMessageEntryHeader), "|-_", false))
				return ARRAY;
			else if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), mapKeyStart, false) &&
				utils::TextEndsWith(src + sizeof(DataMessageEntryHeader), "]-_", false))
				return MAP;
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] arrayKeyStart;
	delete[] mapKeyStart;
	return NONE;
}




bool DataMessage::hasKey(const char* key) {
	return (findEntry(key) != NULL);
}

bool DataMessage::hasString(const char* key) {
	char* src = findEntry(key);
	if (src == NULL) return 0;
	return (((DataMessageEntryHeader*)src)->cid == CONSTCHARID);
}

bool DataMessage::hasInteger(const char* key) {
	char* src = findEntry(key);
	if (src == NULL) return 0;
	return (((DataMessageEntryHeader*)src)->cid == INTID);
}

bool DataMessage::hasFloat(const char* key) {
	char* src = findEntry(key);
	if (src == NULL) return 0;
	return (((DataMessageEntryHeader*)src)->cid == DOUBLEID);
}

bool DataMessage::hasTime(const char* key) {
	char* src = findEntry(key);
	if (src == NULL) return 0;
	return (((DataMessageEntryHeader*)src)->cid == TIMEID);
}

bool DataMessage::hasData(const char* key) {
	char* src = findEntry(key);
	if (src == NULL) return 0;
	return (((DataMessageEntryHeader*)src)->cid == CHARDATAID);
}

bool DataMessage::hasMessage(const char* key) {
	char* src = findEntry(key);
	if (src == NULL) return 0;
	return (((DataMessageEntryHeader*)src)->cid == DATAMESSAGEID);
}


bool DataMessage::isSingleValue(const char* key) {
	return (findEntry(key) != NULL);
}

uint32 DataMessage::getContentType(int64 i, const char* key) {
	char buf[256];
	int n = snprintf(buf, sizeof(buf), "%s_-|%lld|-_", key, i);
	if (n < 0 || (size_t)n >= sizeof(buf))
		return getContentType(utils::StringFormat("%s_-|%lld|-_", key, i).c_str());
	return getContentType(buf);
}

uint32 DataMessage::getContentSize(int64 i, const char* key) {
	return getContentSize(utils::StringFormat("%s_-|%lld|-_", key, i).c_str());
}

uint64 DataMessage::getTime(int64 i, const char* key) {
	return getTime(utils::StringFormat("%s_-|%lld|-_", key, i).c_str());
}

std::string DataMessage::getAsString(int64 i, const char* key) {
	return getAsString(utils::StringFormat("%s_-|%lld|-_", key, i).c_str());
}

const char* DataMessage::getString(int64 i, const char* key) {
	return getString(utils::StringFormat("%s_-|%lld|-_", key, i).c_str());
}

const char* DataMessage::getString(int64 i, const char* key, uint32 &size) {
	return getString(utils::StringFormat("%s_-|%lld|-_", key, i).c_str());
}

bool DataMessage::getInt(int64 i, const char* key, int64& value) {
	return getInt(utils::StringFormat("%s_-|%lld|-_", key, i).c_str(), value);
}

bool DataMessage::getDouble(int64 i, const char* key, double& value) {
	return getDouble(utils::StringFormat("%s_-|%lld|-_", key, i).c_str(), value);
}

bool DataMessage::getFloat(int64 i, const char* key, float64& value) {
	return getFloat(utils::StringFormat("%s_-|%lld|-_", key, i).c_str(), value);
}

int64 DataMessage::getInt(int64 i, const char* key, bool& success) {
	return getInt(utils::StringFormat("%s_-|%lld|-_", key, i).c_str(), success);
}

double DataMessage::getDouble(int64 i, const char* key, bool& success) {
	return getDouble(utils::StringFormat("%s_-|%lld|-_", key, i).c_str(), success);
}

float64 DataMessage::getFloat(int64 i, const char* key, bool& success) {
	return getFloat(utils::StringFormat("%s_-|%lld|-_", key, i).c_str(), success);
}

int64 DataMessage::getInt(int64 i, const char* key) {
	bool success;
	return getInt(i, key, success);
}

double DataMessage::getDouble(int64 i, const char* key) {
	bool success;
	return getDouble(i, key, success);
}

float64 DataMessage::getFloat(int64 i, const char* key) {
	bool success;
	return getFloat(i, key, success);
}

int64 DataMessage::getInt(const char* idx, const char* key) {
	bool success;
	return getInt(idx, key, success);
}

double DataMessage::getDouble(const char* idx, const char* key) {
	bool success;
	return getDouble(idx, key, success);
}

float64 DataMessage::getFloat(const char* idx, const char* key) {
	bool success;
	return getFloat(idx, key, success);
}


const char* DataMessage::getData(int64 i, const char* key, uint32& size) {
	return getData(utils::StringFormat("%s_-|%lld|-_", key, i).c_str(), size);
}

char* DataMessage::getDataCopy(int64 i, const char* key, uint32& size) {
	return getDataCopy(utils::StringFormat("%s_-|%lld|-_", key, i).c_str(), size);
}

DataMessage* DataMessage::getAttachedMessageCopy(int64 i, const char* key) {
	return getAttachedMessageCopy(utils::StringFormat("%s_-|%lld|-_", key, i).c_str());
}


bool DataMessage::setTime(int64 i, const char* key, uint64 value) {
	return setTime(utils::StringFormat("%s_-|%lld|-_", key, i).c_str(), value);
}

bool DataMessage::setString(int64 i, const char* key, const char* value) {
	return setString(utils::StringFormat("%s_-|%lld|-_", key, i).c_str(), value);
}

bool DataMessage::setInt(int64 i, const char* key, int64 value) {
	return setInt(utils::StringFormat("%s_-|%lld|-_", key, i).c_str(), value);
}

bool DataMessage::setDouble(int64 i, const char* key, double value) {
	return setDouble(utils::StringFormat("%s_-|%lld|-_", key, i).c_str(), value);
}

bool DataMessage::setFloat(int64 i, const char* key, float64 value) {
	return setFloat(utils::StringFormat("%s_-|%lld|-_", key, i).c_str(), value);
}

bool DataMessage::setData(int64 i, const char* key, const char* value, uint32 size) {
	return setData(utils::StringFormat("%s_-|%lld|-_", key, i).c_str(), value, size);
}

bool DataMessage::setAttachedMessage(int64 i, const char* key, DataMessage* msg) {
	return setAttachedMessage(utils::StringFormat("%s_-|%lld|-_", key, i).c_str(), msg);
}


// Maps
uint32 DataMessage::getContentType(const char* idx, const char* key) {
	char buf[256];
	int n = snprintf(buf, sizeof(buf), "%s_-[%s]-_", key, idx);
	if (n < 0 || (size_t)n >= sizeof(buf))
		return getContentType(utils::StringFormat("%s_-[%s]-_", key, idx).c_str());
	return getContentType(buf);
}

uint32 DataMessage::getContentSize(const char* idx, const char* key) {
	return getContentSize(utils::StringFormat("%s_-[%s]-_", key, idx).c_str());
}

uint64 DataMessage::getTime(const char* idx, const char* key) {
	return getTime(utils::StringFormat("%s_-[%s]-_", key, idx).c_str());
}

std::string DataMessage::getAsString(const char* idx, const char* key) {
	return getAsString(utils::StringFormat("%s_-[%s]-_", key, idx).c_str());
}

const char* DataMessage::getString(const char* idx, const char* key) {
	return getString(utils::StringFormat("%s_-[%s]-_", key, idx).c_str());
}

const char* DataMessage::getString(const char* idx, const char* key, uint32 &size) {
	return getString(utils::StringFormat("%s_-[%s]-_", key, idx).c_str());
}

bool DataMessage::getInt(const char* idx, const char* key, int64& value) {
	return getInt(utils::StringFormat("%s_-[%s]-_", key, idx).c_str(), value);
}

bool DataMessage::getDouble(const char* idx, const char* key, double& value) {
	return getDouble(utils::StringFormat("%s_-[%s]-_", key, idx).c_str(), value);
}

bool DataMessage::getFloat(const char* idx, const char* key, float64& value) {
	return getFloat(utils::StringFormat("%s_-[%s]-_", key, idx).c_str(), value);
}

int64 DataMessage::getInt(const char* idx, const char* key, bool& success) {
	return getInt(utils::StringFormat("%s_-[%s]-_", key, idx).c_str(), success);
}

double DataMessage::getDouble(const char* idx, const char* key, bool& success) {
	return getDouble(utils::StringFormat("%s_-[%s]-_", key, idx).c_str(), success);
}

float64 DataMessage::getFloat(const char* idx, const char* key, bool& success) {
	return getFloat(utils::StringFormat("%s_-[%s]-_", key, idx).c_str(), success);
}

const char* DataMessage::getData(const char* idx, const char* key, uint32& size) {
	return getData(utils::StringFormat("%s_-[%s]-_", key, idx).c_str(), size);
}

char* DataMessage::getDataCopy(const char* idx, const char* key, uint32& size) {
	return getDataCopy(utils::StringFormat("%s_-[%s]-_", key, idx).c_str(), size);
}

DataMessage* DataMessage::getAttachedMessageCopy(const char* idx, const char* key) {
	return getAttachedMessageCopy(utils::StringFormat("%s_-[%s]-_", key, idx).c_str());
}


bool DataMessage::setTime(const char* idx, const char* key, uint64 value) {
	return setTime(utils::StringFormat("%s_-[%s]-_", key, idx).c_str(), value);
}

bool DataMessage::setString(const char* idx, const char* key, const char* value) {
	return setString(utils::StringFormat("%s_-[%s]-_", key, idx).c_str(), value);
}

bool DataMessage::setInt(const char* idx, const char* key, int64 value) {
	return setInt(utils::StringFormat("%s_-[%s]-_", key, idx).c_str(), value);
}

bool DataMessage::setDouble(const char* idx, const char* key, double value) {
	return setDouble(utils::StringFormat("%s_-[%s]-_", key, idx).c_str(), value);
}

bool DataMessage::setFloat(const char* idx, const char* key, float64 value) {
	return setFloat(utils::StringFormat("%s_-[%s]-_", key, idx).c_str(), value);
}

bool DataMessage::setData(const char* idx, const char* key, const char* value, uint32 size) {
	return setData(utils::StringFormat("%s_-[%s]-_", key, idx).c_str(), value, size);
}

bool DataMessage::setAttachedMessage(const char* idx, const char* key, DataMessage* msg) {
	return setAttachedMessage(utils::StringFormat("%s_-[%s]-_", key, idx).c_str(), msg);
}








std::map<int64, uint64> DataMessage::getTimeArray(const char* key) {
	std::map<int64, uint64> map;
	int64 i;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-|", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
			if (((DataMessageEntryHeader*)entry)->cid == TIMEID) {
				if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
					//i = utils::Ascii2Int64(std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3).c_str());
					i = utils::Ascii2Int64(src + sizeof(DataMessageEntryHeader) + size);
					map[i] = *(uint64*)(entry + sizeof(DataMessageEntryHeader));
				}
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

std::map<int64, int64> DataMessage::getAsIntArray(const char* key) {
	std::map<int64, int64> map;
	int64 i;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-|", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
				entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;

				//i = utils::Ascii2Int64(std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3).c_str());
				i = utils::Ascii2Int64(src + sizeof(DataMessageEntryHeader) + size);
				switch (((DataMessageEntryHeader*)entry)->cid) {
				case TIMEID:
					map[i] = (int64)(*(uint64*)(entry + sizeof(DataMessageEntryHeader)));
					break;
				case CONSTCHARID:
					map[i] = utils::Ascii2Int64(entry + sizeof(DataMessageEntryHeader));
					break;
				case CHARDATAID:
					map[i] = (int64)((DataMessageEntryHeader*)entry)->size;
					break;
				case INTID:
					map[i] = *(int64*)(entry + sizeof(DataMessageEntryHeader));
					break;
				case DOUBLEID:
					map[i] = (int64)(*(float64*)(entry + sizeof(DataMessageEntryHeader)));
					break;
				case DATAMESSAGEID:
					map[i] = (int64)((DataMessageEntryHeader*)entry)->size;
					break;
				default:
					break;
				}
			}

		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

std::map<int64, float64> DataMessage::getAsFloatArray(const char* key) {
	std::map<int64, float64> map;
	int64 i;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-|", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
				entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;

				//i = utils::Ascii2Int64(std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3).c_str());
				i = utils::Ascii2Int64(src + sizeof(DataMessageEntryHeader) + size);
				switch (((DataMessageEntryHeader*)entry)->cid) {
				case TIMEID:
					map[i] = (float64)(*(uint64*)(entry + sizeof(DataMessageEntryHeader)));
					break;
				case CONSTCHARID:
					map[i] = utils::Ascii2Float64(entry + sizeof(DataMessageEntryHeader));
					break;
				case CHARDATAID:
					map[i] = (float64)((DataMessageEntryHeader*)entry)->size;
					break;
				case INTID:
					map[i] = (float64)(*(int64*)(entry + sizeof(DataMessageEntryHeader)));
					break;
				case DOUBLEID:
					map[i] = *(float64*)(entry + sizeof(DataMessageEntryHeader));
					break;
				case DATAMESSAGEID:
					map[i] = (float64)((DataMessageEntryHeader*)entry)->size;
					break;
				default:
					break;
				}
			}

		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

std::map<int64, std::string> DataMessage::getAsStringArray(const char* key) {
	std::map<int64, std::string> map;
	int64 i;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-|", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
				entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;

				//i = utils::Ascii2Int64(std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3).c_str());
				i = utils::Ascii2Int64(src + sizeof(DataMessageEntryHeader) + size);
				switch (((DataMessageEntryHeader*)entry)->cid) {
				case TIMEID:
					map[i] = PrintTimeString(*(uint64*)(entry + sizeof(DataMessageEntryHeader)));
					break;
				case CONSTCHARID:
					map[i] = entry + sizeof(DataMessageEntryHeader);
					break;
				case CHARDATAID:
					map[i] = utils::StringFormat("BinaryData [%ub]", ((DataMessageEntryHeader*)entry)->size);
					break;
				case INTID:
					map[i] = utils::StringFormat("%lld", *(int64*)(entry + sizeof(DataMessageEntryHeader)));
					break;
				case DOUBLEID:
					map[i] = utils::StringFormat("%f", *(float64*)(entry + sizeof(DataMessageEntryHeader)));
					break;
				case DATAMESSAGEID:
					map[i] = utils::StringFormat("DataMessage [%ub]", ((DataMessageEntryHeader*)entry)->size);
					break;
				default:
					break;
				}
			}

		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

std::map<int64, std::string> DataMessage::getStringArray(const char* key) {
	std::map<int64, std::string> map;
	int64 i;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-|", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
			if (((DataMessageEntryHeader*)entry)->cid == CONSTCHARID) {
				if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
					//i = utils::Ascii2Int64(std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3).c_str());
					i = utils::Ascii2Int64(src + sizeof(DataMessageEntryHeader) + size);
					map[i] = entry + sizeof(DataMessageEntryHeader);
				}
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}


std::map<int64, int64> DataMessage::getIntArray(const char* key) {
	std::map<int64, int64> map;
	int64 i;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-|", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
			if (((DataMessageEntryHeader*)entry)->cid == INTID) {
				if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
					//i = utils::Ascii2Int64(std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3).c_str());
					i = utils::Ascii2Int64(src + sizeof(DataMessageEntryHeader) + size);
					map[i] = *(int64*)(entry + sizeof(DataMessageEntryHeader));
				}
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

std::map<int64, double> DataMessage::getDoubleArray(const char* key) {
	std::map<int64, double> map;
	int64 i;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-|", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
			if (((DataMessageEntryHeader*)entry)->cid == DOUBLEID) {
				if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
					//i = utils::Ascii2Int64(std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3).c_str());
					i = utils::Ascii2Int64(src + sizeof(DataMessageEntryHeader) + size);
					map[i] = *(float64*)(entry + sizeof(DataMessageEntryHeader));
				}
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

std::map<int64, float64> DataMessage::getFloatArray(const char* key) {
	std::map<int64, float64> map;
	int64 i;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-|", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
			if (((DataMessageEntryHeader*)entry)->cid == DOUBLEID) {
				if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
					//i = utils::Ascii2Int64(std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3).c_str());
					i = utils::Ascii2Int64(src + sizeof(DataMessageEntryHeader) + size);
					map[i] = *(float64*)(entry + sizeof(DataMessageEntryHeader));
				}
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

std::map<int64, DataMessage*> DataMessage::getAttachedMessageArray(const char* key) {
	std::map<int64, DataMessage*> map;
	int64 i;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-|", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
			if (((DataMessageEntryHeader*)entry)->cid == DATAMESSAGEID) {
				if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
					//i = utils::Ascii2Int64(std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3).c_str());
					i = utils::Ascii2Int64(src + sizeof(DataMessageEntryHeader) + size);
					map[i] = new DataMessage(entry + sizeof(DataMessageEntryHeader), true);
				}
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

bool DataMessage::setTimeArray(const char* key, std::map<int64, uint64>& map) {
	std::map<int64, uint64>::iterator i = map.begin(), e = map.end();
	while (i != e) {
		if (!setTime(i->first, key, i->second))
			return false;
		i++;
	}
	return true;
}

bool DataMessage::setStringArray(const char* key, std::map<int64, std::string>& map) {
	std::map<int64, std::string>::iterator i = map.begin(), e = map.end();
	while (i != e) {
		if (!setString(i->first, key, i->second.c_str()))
			return false;
		i++;
	}
	return true;
}

bool DataMessage::setIntArray(const char* key, std::map<int64, int64>& map) {
	std::map<int64, int64>::iterator i = map.begin(), e = map.end();
	while (i != e) {
		if (!setInt(i->first, key, i->second))
			return false;
		i++;
	}
	return true;
}

bool DataMessage::setDoubleArray(const char* key, std::map<int64, double>& map) {
	std::map<int64, double>::iterator i = map.begin(), e = map.end();
	while (i != e) {
		if (!setDouble(i->first, key, i->second))
			return false;
		i++;
	}
	return true;
}

bool DataMessage::setFloatArray(const char* key, std::map<int64, float64>& map) {
	std::map<int64, float64>::iterator i = map.begin(), e = map.end();
	while (i != e) {
		if (!setFloat(i->first, key, i->second))
			return false;
		i++;
	}
	return true;
}

bool DataMessage::setAttachedMessageArray(const char* key, std::map<int64, DataMessage*>& map) {
	std::map<int64, DataMessage*>::iterator i = map.begin(), e = map.end();
	while (i != e) {
		if (!setAttachedMessage(i->first, key, i->second))
			return false;
		i++;
	}
	return true;
}








uint32 DataMessage::getArraySize(const char* key) {
	uint32 c = 0;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-|", key);
	if (data->cid != DATAMESSAGEID) return c;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
			if (((DataMessageEntryHeader*)entry)->cid == CONSTCHARID) {
				if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false))
					c++;
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return c;
}

uint32 DataMessage::getArrayCount(const char* key) {
	return getArraySize(key);
}

uint32 DataMessage::getMapSize(const char* key) {
	uint32 c = 0;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-[", key);
	if (data->cid != DATAMESSAGEID) return c;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
			if (((DataMessageEntryHeader*)entry)->cid == CONSTCHARID) {
				if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false))
					c++;
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return c;
}

uint32 DataMessage::getMapCount(const char* key) {
	return getMapSize(key);
}







std::map<std::string, uint64> DataMessage::getTimeMap(const char* key) {
	std::map<std::string, uint64> map;
	std::string str;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-[", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
			if (((DataMessageEntryHeader*)entry)->cid == TIMEID) {
				if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
					str = std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3);
					map[str] = *(uint64*)(entry + sizeof(DataMessageEntryHeader));
				}
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

std::map<std::string, int64> DataMessage::getAsIntMap(const char* key) {
	std::map<std::string, int64> map;
	std::string str;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-[", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
				entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
				str = std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3).c_str();
				switch (((DataMessageEntryHeader*)entry)->cid) {
				case TIMEID:
					map[str] = (int64)(*(uint64*)(entry + sizeof(DataMessageEntryHeader)));
					break;
				case CONSTCHARID:
					map[str] = utils::Ascii2Int64(entry + sizeof(DataMessageEntryHeader));
					break;
				case CHARDATAID:
					map[str] = (int64)((DataMessageEntryHeader*)entry)->size;
					break;
				case INTID:
					map[str] = *(int64*)(entry + sizeof(DataMessageEntryHeader));
					break;
				case DOUBLEID:
					map[str] = (int64)(*(float64*)(entry + sizeof(DataMessageEntryHeader)));
					break;
				case DATAMESSAGEID:
					map[str] = (int64)((DataMessageEntryHeader*)entry)->size;
					break;
				default:
					break;
				}
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

std::map<std::string, float64> DataMessage::getAsFloatMap(const char* key) {
	std::map<std::string, float64> map;
	std::string str;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-[", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
				entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
				str = std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3).c_str();
				switch (((DataMessageEntryHeader*)entry)->cid) {
				case TIMEID:
					map[str] = (float64)(*(uint64*)(entry + sizeof(DataMessageEntryHeader)));
					break;
				case CONSTCHARID:
					map[str] = utils::Ascii2Float64(entry + sizeof(DataMessageEntryHeader));
					break;
				case CHARDATAID:
					map[str] = (float64)((DataMessageEntryHeader*)entry)->size;
					break;
				case INTID:
					map[str] = (float64)(*(int64*)(entry + sizeof(DataMessageEntryHeader)));
					break;
				case DOUBLEID:
					map[str] = *(float64*)(entry + sizeof(DataMessageEntryHeader));
					break;
				case DATAMESSAGEID:
					map[str] = (float64)((DataMessageEntryHeader*)entry)->size;
					break;
				default:
					break;
				}
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

std::map<std::string, std::string> DataMessage::getAsStringMap(const char* key) {
	std::map<std::string, std::string> map;
	std::string str;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-[", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
				entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
				str = std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3).c_str();
				switch (((DataMessageEntryHeader*)entry)->cid) {
				case TIMEID:
					map[str] = PrintTimeString(*(uint64*)(entry + sizeof(DataMessageEntryHeader)));
					break;
				case CONSTCHARID:
					map[str] = entry + sizeof(DataMessageEntryHeader);
					break;
				case CHARDATAID:
					map[str] = utils::StringFormat("BinaryData [%ub]", ((DataMessageEntryHeader*)entry)->size);
					break;
				case INTID:
					map[str] = utils::StringFormat("%lld", *(int64*)(entry + sizeof(DataMessageEntryHeader)));
					break;
				case DOUBLEID:
					map[str] = utils::StringFormat("%f", *(float64*)(entry + sizeof(DataMessageEntryHeader)));
					break;
				case DATAMESSAGEID:
					map[str] = utils::StringFormat("DataMessage [%ub]", ((DataMessageEntryHeader*)entry)->size);
					break;
				default:
					break;
				}
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

std::map<std::string, std::string> DataMessage::getStringMap(const char* key) {
	std::map<std::string, std::string> map;
	std::string str;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-[", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
			if (((DataMessageEntryHeader*)entry)->cid == CONSTCHARID) {
				if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
					str = std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3);
					map[str] = entry + sizeof(DataMessageEntryHeader);
				}
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

std::map<std::string, int64> DataMessage::getIntMap(const char* key) {
	std::map<std::string, int64> map;
	std::string str;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-[", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
			if (((DataMessageEntryHeader*)entry)->cid == INTID) {
				if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
					str = std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3);
					map[str] = *(int64*)(entry + sizeof(DataMessageEntryHeader));
				}
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

std::map<std::string, double> DataMessage::getDoubleMap(const char* key) {
	std::map<std::string, double> map;
	std::string str;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-[", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
			if (((DataMessageEntryHeader*)entry)->cid == DOUBLEID) {
				if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
					str = std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3);
					map[str] = *(float64*)(entry + sizeof(DataMessageEntryHeader));
				}
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

std::map<std::string, float64> DataMessage::getFloatMap(const char* key) {
	std::map<std::string, float64> map;
	std::string str;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-[", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
			if (((DataMessageEntryHeader*)entry)->cid == DOUBLEID) {
				if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
					str = std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3);
					map[str] = *(float64*)(entry + sizeof(DataMessageEntryHeader));
				}
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

std::map<std::string, DataMessage*> DataMessage::getAttachedMessageMap(const char* key) {
	std::map<std::string, DataMessage*> map;
	std::string str;
	uint32 size;
	char* keyStart = utils::StringFormat(size, "%s_-[", key);
	if (data->cid != DATAMESSAGEID) return map;
	char* entry;
	char* src = (char*)data + sizeof(DataMessageHeader);
	char* srcEnd = (char*)data + data->size;
	while (src < srcEnd) {
		if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
			entry = src + sizeof(DataMessageEntryHeader) + strlen(src + sizeof(DataMessageEntryHeader)) + 1;
			if (((DataMessageEntryHeader*)entry)->cid == DATAMESSAGEID) {
				if (utils::TextStartsWith(src + sizeof(DataMessageEntryHeader), keyStart, false)) {
					str = std::string(src + sizeof(DataMessageEntryHeader) + size, strlen(src + sizeof(DataMessageEntryHeader)) - size - 3);
					map[str] = new DataMessage(entry + sizeof(DataMessageEntryHeader), true);
				}
			}
		}
		src += ((DataMessageEntryHeader*)src)->size;
	}
	delete[] keyStart;
	return map;
}

bool DataMessage::setTimeMap(const char* key, std::map<std::string, uint64>& map) {
	std::map<std::string, uint64>::iterator i = map.begin(), e = map.end();
	while (i != e) {
		if (!setTime(i->first.c_str(), key, i->second))
			return false;
		i++;
	}
	return true;
}

bool DataMessage::setStringMap(const char* key, std::map<std::string, std::string>& map) {
	std::map<std::string, std::string>::iterator i = map.begin(), e = map.end();
	while (i != e) {
		if (!setString(i->first.c_str(), key, i->second.c_str()))
			return false;
		i++;
	}
	return true;
}

bool DataMessage::setIntMap(const char* key, std::map<std::string, int64>& map) {
	std::map<std::string, int64>::iterator i = map.begin(), e = map.end();
	while (i != e) {
		if (!setInt(i->first.c_str(), key, i->second))
			return false;
		i++;
	}
	return true;
}

bool DataMessage::setDoubleMap(const char* key, std::map<std::string, double>& map) {
	std::map<std::string, double>::iterator i = map.begin(), e = map.end();
	while (i != e) {
		if (!setDouble(i->first.c_str(), key, i->second))
			return false;
		i++;
	}
	return true;
}

bool DataMessage::setFloatMap(const char* key, std::map<std::string, float64>& map) {
	std::map<std::string, float64>::iterator i = map.begin(), e = map.end();
	while (i != e) {
		if (!setFloat(i->first.c_str(), key, i->second))
			return false;
		i++;
	}
	return true;
}

bool DataMessage::setAttachedMessageMap(const char* key, std::map<std::string, DataMessage*>& map) {
	std::map<std::string, DataMessage*>::iterator i = map.begin(), e = map.end();
	while (i != e) {
		if (!setAttachedMessage(i->first.c_str(), key, i->second))
			return false;
		i++;
	}
	return true;
}


DataMessage* DataMessage::FindClosestMessage(uint64 t, std::map<uint64, DataMessage*> &messages) {
	uint64 bestCandidateDiff = 0, diff;
	DataMessage* bestCandidate = NULL;
	std::map<uint64, DataMessage*>::reverse_iterator i = messages.rbegin(), e = messages.rend();
	while (i != e) {
		if (!bestCandidate) {
			bestCandidate = i->second;
			bestCandidateDiff = (i->first > t) ? i->first - t : t - i->first;
		}
		else {
			diff = (i->first > t) ? i->first - t : t - i->first;
			if (diff < bestCandidateDiff) {
				bestCandidate = i->second;
				bestCandidateDiff = diff;
			}
		}
		i++;
	}
	return bestCandidate;
}

bool DataMessage::addTimeUsage(uint32 cycleCPUTime, uint32 cycleWallTime, uint32 chainCPUTime, uint32 chainWallTime, uint32 chainCount) {
	if (data->cid != DATAMESSAGEID) return false;

	data->chaincputime = chainCPUTime + cycleCPUTime;
	data->chainwalltime = chainWallTime + cycleWallTime;
	data->chaincount = chainCount;

	data->cyclecputime = cycleCPUTime;
	data->cyclewalltime = cycleWallTime;
	return true;
}


bool DataMessage::ConvertDataFromOlderMessageFormat(const char* data, char **newData) {
	*newData = NULL;
	if (!data) return false;
	const DataMessageHeader* header = (DataMessageHeader*)data;
	if (header->cid == DATAMESSAGEOLDID) {
		const DataMessageHeader_Old* oldHeader = (DataMessageHeader_Old*)data;
		if (oldHeader->size < sizeof(DataMessageHeader_Old))
			return false;

		// Data Message Version History
		// Initial version:
		// - data->cid = DATAMESSAGEOLDID
		// - does not contain cver, cyclecputime, cyclewalltime, chaincputime, chainwalltime
		// Version 10:
		// - data->cid = DATAMESSAGEID
		// - now contains cver, cenc, cyclecputime, cyclewalltime, chaincputime, chainwalltime
		// we added 18 bytes of space
		uint32 newSize = sizeof(DataMessageHeader) + oldHeader->userSize;
		*newData = new char[newSize];
		// reset buffer
		memset(*newData, 0, newSize);
		// set new size
		((DataMessageHeader*)*newData)->size = newSize;
		// set new object id type
		((DataMessageHeader*)*newData)->cid = DATAMESSAGEID;
		((DataMessageHeader*)*newData)->cver = CURRENTDATAMESSAGEVERSION;
		// cenc already set to 0
		// copy the rest of the header
		((DataMessageHeader*)*newData)->memid = oldHeader->memid;
		((DataMessageHeader*)*newData)->time = oldHeader->time;
		((DataMessageHeader*)*newData)->sendtime = oldHeader->sendtime;
		((DataMessageHeader*)*newData)->recvtime = oldHeader->recvtime;
		((DataMessageHeader*)*newData)->origin = oldHeader->origin;
		((DataMessageHeader*)*newData)->destination = oldHeader->destination;
		((DataMessageHeader*)*newData)->ttl = oldHeader->ttl;
		((DataMessageHeader*)*newData)->priority = oldHeader->priority;
		((DataMessageHeader*)*newData)->policy = oldHeader->policy;
		((DataMessageHeader*)*newData)->from = oldHeader->from;
		((DataMessageHeader*)*newData)->to = oldHeader->to;
		((DataMessageHeader*)*newData)->type = oldHeader->type;
		((DataMessageHeader*)*newData)->tag = oldHeader->tag;
		((DataMessageHeader*)*newData)->status = oldHeader->status;
		((DataMessageHeader*)*newData)->reference = oldHeader->reference;
		((DataMessageHeader*)*newData)->serial = oldHeader->serial;
		((DataMessageHeader*)*newData)->contextchange = oldHeader->contextchange;
		((DataMessageHeader*)*newData)->userSize = oldHeader->userSize;
		((DataMessageHeader*)*newData)->userCount = oldHeader->userCount;
		// cyclecputime, cyclewalltime, chaincputime, chainwalltime already set to 0
		if (oldHeader->userSize > 0)
			memcpy((*newData) + sizeof(DataMessageHeader), data + sizeof(DataMessageHeader_Old), oldHeader->userSize);
		return true;
	}
	else if (header->cid == DATAMESSAGEID) {
		if (header->cver == CURRENTDATAMESSAGEVERSION) {
			*newData = new char[header->size];
			memcpy(*newData, data, header->size);
			return true;
		}
		switch (header->cver) {
			case 10:
				*newData = new char[header->size];
				memcpy(*newData, data, header->size);
				return true;
			default:
				LogPrint(0, 0, 0, "*** DataMessage format version %u is not supported, only up to version %u ***", header->cver, CURRENTDATAMESSAGEVERSION);
				return false;
		}
	}
	return false;
}

bool DataMessage::UnitTest() {

	// 1. Construction and validity
	unittest::progress(5, "construction and validity");
	{
		DataMessage empty;
		if (!empty.isValid()) {
			unittest::fail("DataMessage test: default constructor invalid\n");
			return false;
		}
		if (empty.getUserCount() != 0) {
			unittest::fail("DataMessage test: empty getUserCount != 0\n");
			return false;
		}

		DataMessage withType(CTRL_TEST, 1);
		if (!withType.isValid() || withType.getUserCount() != 0) {
			unittest::fail("DataMessage test: DataMessage(type, from) invalid\n");
			return false;
		}

		DataMessage full(CTRL_TEST, 1, 2, static_cast<uint64>(1000), static_cast<uint16>(5));
		if (!full.isValid()) {
			unittest::fail("DataMessage test: DataMessage(type, from, to, ttl, priority) invalid\n");
			return false;
		}
	}

	// 2. Single-value write and read
	unittest::progress(20, "single-value read/write");
	{
		DataMessage msg(CTRL_TEST, 0);
		const uint64 t = 1234567890ULL;
		if (!msg.setTime("T", t)) { unittest::fail("DataMessage test: setTime failed\n"); return false; }
		if (!msg.setString("S", "hello")) { unittest::fail("DataMessage test: setString failed\n"); return false; }
		if (!msg.setInt("I", -42)) { unittest::fail("DataMessage test: setInt failed\n"); return false; }
		if (!msg.setDouble("D", 3.14)) { unittest::fail("DataMessage test: setDouble failed\n"); return false; }
		if (!msg.setFloat("F", 2.5f)) { unittest::fail("DataMessage test: setFloat failed\n"); return false; }
		const char bin[] = { 'x', 'y', 'z' };
		if (!msg.setData("B", bin, 3)) { unittest::fail("DataMessage test: setData failed\n"); return false; }

		if (msg.getTime("T") != t) { unittest::fail("DataMessage test: getTime mismatch\n"); return false; }
		const char* s = msg.getString("S");
		if (!s || strcmp(s, "hello") != 0) { unittest::fail("DataMessage test: getString mismatch\n"); return false; }
		int64 i = 0;
		if (!msg.getInt("I", i) || i != -42) { unittest::fail("DataMessage test: getInt mismatch\n"); return false; }
		double d = 0;
		if (!msg.getDouble("D", d) || d < 3.13 || d > 3.15) { unittest::fail("DataMessage test: getDouble mismatch\n"); return false; }
		float64 f = 0;
		if (!msg.getFloat("F", f) || f < 2.4f || f > 2.6f) { unittest::fail("DataMessage test: getFloat mismatch\n"); return false; }
		uint32 bsize = 0;
		const char* b = msg.getData("B", bsize);
		if (!b || bsize != 3 || b[0] != 'x' || b[1] != 'y' || b[2] != 'z') {
			unittest::fail("DataMessage test: getData mismatch\n"); return false;
		}

		if (msg.getContentType("T") != TIMEID) { unittest::fail("DataMessage test: getContentType T != TIMEID\n"); return false; }
		if (msg.getContentType("S") != CONSTCHARID) { unittest::fail("DataMessage test: getContentType S != CONSTCHARID\n"); return false; }
		if (msg.getContentType("I") != INTID) { unittest::fail("DataMessage test: getContentType I != INTID\n"); return false; }
		if (msg.getContentType("D") != DOUBLEID) { unittest::fail("DataMessage test: getContentType D != DOUBLEID\n"); return false; }
		if (msg.getContentType("B") != CHARDATAID) { unittest::fail("DataMessage test: getContentType B != CHARDATAID\n"); return false; }
		if (!msg.hasKey("S") || !msg.hasString("S") || !msg.hasInteger("I") || !msg.hasFloat("D") || !msg.hasTime("T") || !msg.hasData("B")) {
			unittest::fail("DataMessage test: hasKey/hasString etc failed\n"); return false;
		}
		if (msg.getKeyType("S") != DataMessage::SINGLE || !msg.isSingleValue("S")) {
			unittest::fail("DataMessage test: getKeyType/isSingleValue failed\n"); return false;
		}
		if (msg.getAsString("I") != "-42" || msg.getAsInt("S") != 0) { /* getAsInt on string may return 0 */ }
		if (msg.getAsBool("I") != true) { unittest::fail("DataMessage test: getAsBool failed\n"); return false; }

		// getDataCopy
		uint32 csize = 0;
		char* cpy = msg.getDataCopy("B", csize);
		if (!cpy || csize != 3) { unittest::fail("DataMessage test: getDataCopy failed\n"); return false; }
		delete[] cpy;

		// Attached message
		DataMessage* nested = new DataMessage(CTRL_TEST, 99);
		nested->setString("nested", "value");
		if (!msg.setAttachedMessage("M", nested)) { unittest::fail("DataMessage test: setAttachedMessage failed\n"); delete nested; return false; }
		DataMessage* got = msg.getAttachedMessageCopy("M");
		if (!got || got->getUserCount() == 0) { unittest::fail("DataMessage test: getAttachedMessageCopy failed\n"); delete got; return false; }
		if (got->getAsString("nested") != "value") { unittest::fail("DataMessage test: attached message content mismatch\n"); delete got; return false; }
		if (msg.getContentType("M") != DATAMESSAGEID) { unittest::fail("DataMessage test: getContentType M != DATAMESSAGEID\n"); delete got; return false; }
		if (!msg.hasMessage("M")) { unittest::fail("DataMessage test: hasMessage failed\n"); delete got; return false; }
		delete got;
		delete nested;
	}

	// 3. Array access
	unittest::progress(40, "array access");
	{
		DataMessage msg(CTRL_TEST, 0);
		if (!msg.setTime(static_cast<int64>(0), "arr", 100ULL)) { unittest::fail("DataMessage test: setTime array failed\n"); return false; }
		if (!msg.setString(1, "arr", "one")) { unittest::fail("DataMessage test: setString array failed\n"); return false; }
		if (!msg.setInt(2, "arr", 999)) { unittest::fail("DataMessage test: setInt array failed\n"); return false; }
		if (msg.getTime(static_cast<int64>(0), "arr") != 100ULL) { unittest::fail("DataMessage test: getTime array mismatch\n"); return false; }
		if (msg.getAsString(1, "arr") != "one") { unittest::fail("DataMessage test: getAsString array mismatch\n"); return false; }
		int64 vi = 0;
		if (!msg.getInt(2, "arr", vi) || vi != 999) { unittest::fail("DataMessage test: getInt array mismatch\n"); return false; }
		if (msg.getKeyType("arr") != DataMessage::ARRAY || !msg.isArrayContent("arr")) {
			unittest::fail("DataMessage test: array key type failed\n"); return false;
		}
		// getArraySize counts only CONSTCHARID value entries; arr has time, string, int so size may be 1
		if (msg.getArraySize("arr") < 1) {
			unittest::fail("DataMessage test: getArraySize failed\n"); return false;
		}

		std::map<int64, uint64> tmap; tmap[0] = 10; tmap[1] = 20;
		if (!msg.setTimeArray("tarr", tmap)) { unittest::fail("DataMessage test: setTimeArray failed\n"); return false; }
		std::map<int64, std::string> smap; smap[0] = "a"; smap[1] = "b";
		if (!msg.setStringArray("sarr", smap)) { unittest::fail("DataMessage test: setStringArray failed\n"); return false; }
		std::map<int64, int64> imap; imap[0] = 1; imap[1] = 2;
		if (!msg.setIntArray("iarr", imap)) { unittest::fail("DataMessage test: setIntArray failed\n"); return false; }
		std::map<int64, double> dmap; dmap[0] = 1.1; dmap[1] = 2.2;
		if (!msg.setDoubleArray("darr", dmap)) { unittest::fail("DataMessage test: setDoubleArray failed\n"); return false; }
		std::map<int64, uint64> gotT = msg.getTimeArray("tarr");
		if (gotT.size() != 2 || gotT[0] != 10 || gotT[1] != 20) { unittest::fail("DataMessage test: getTimeArray mismatch\n"); return false; }
		std::map<int64, std::string> gotS = msg.getAsStringArray("sarr");
		if (gotS.size() != 2 || gotS[0] != "a" || gotS[1] != "b") { unittest::fail("DataMessage test: getAsStringArray mismatch\n"); return false; }
		std::map<int64, int64> gotI = msg.getIntArray("iarr");
		if (gotI.size() != 2 || gotI[0] != 1 || gotI[1] != 2) { unittest::fail("DataMessage test: getIntArray mismatch\n"); return false; }
		std::map<int64, double> gotD = msg.getDoubleArray("darr");
		if (gotD.size() != 2 || gotD[0] < 1.0 || gotD[0] > 1.2 || gotD[1] < 2.1 || gotD[1] > 2.3) { unittest::fail("DataMessage test: getDoubleArray mismatch\n"); return false; }
	}

	// 4. Map access (string index)
	unittest::progress(55, "map access");
	{
		DataMessage msg(CTRL_TEST, 0);
		if (!msg.setTime("x", "m", 200ULL)) { unittest::fail("DataMessage test: setTime map failed\n"); return false; }
		if (!msg.setString("y", "m", "mapval")) { unittest::fail("DataMessage test: setString map failed\n"); return false; }
		if (!msg.setInt("z", "m", -7)) { unittest::fail("DataMessage test: setInt map failed\n"); return false; }
		if (msg.getTime("x", "m") != 200ULL) { unittest::fail("DataMessage test: getTime map mismatch\n"); return false; }
		if (msg.getAsString("y", "m") != "mapval") { unittest::fail("DataMessage test: getAsString map mismatch\n"); return false; }
		int64 vi = 0;
		if (!msg.getInt("z", "m", vi) || vi != -7) { unittest::fail("DataMessage test: getInt map mismatch\n"); return false; }
		if (msg.getKeyType("m") != DataMessage::MAP || !msg.isMapContent("m")) {
			unittest::fail("DataMessage test: map key type failed\n"); return false;
		}
		// getMapSize counts only CONSTCHARID value entries; m has time, string, int so size may be 1
		if (msg.getMapSize("m") < 1) {
			unittest::fail("DataMessage test: getMapSize failed\n"); return false;
		}

		std::map<std::string, uint64> tmap; tmap["a"] = 1; tmap["b"] = 2;
		if (!msg.setTimeMap("tm", tmap)) { unittest::fail("DataMessage test: setTimeMap failed\n"); return false; }
		std::map<std::string, std::string> smap; smap["k1"] = "v1"; smap["k2"] = "v2";
		if (!msg.setStringMap("sm", smap)) { unittest::fail("DataMessage test: setStringMap failed\n"); return false; }
		std::map<std::string, int64> imap; imap["i1"] = 10; imap["i2"] = 20;
		if (!msg.setIntMap("im", imap)) { unittest::fail("DataMessage test: setIntMap failed\n"); return false; }
		std::map<std::string, uint64> gotT = msg.getTimeMap("tm");
		if (gotT.size() != 2 || gotT["a"] != 1 || gotT["b"] != 2) { unittest::fail("DataMessage test: getTimeMap mismatch\n"); return false; }
		std::map<std::string, std::string> gotS = msg.getAsStringMap("sm");
		if (gotS.size() != 2 || gotS["k1"] != "v1" || gotS["k2"] != "v2") { unittest::fail("DataMessage test: getAsStringMap mismatch\n"); return false; }
		std::map<std::string, int64> gotI = msg.getIntMap("im");
		if (gotI.size() != 2 || gotI["i1"] != 10 || gotI["i2"] != 20) { unittest::fail("DataMessage test: getIntMap mismatch\n"); return false; }
	}

	// 5. Binary serialisation round-trip
	unittest::progress(70, "binary serialisation round-trip");
	{
		DataMessage orig(CTRL_TEST, 0);
		orig.setString("k", "roundtrip");
		orig.setInt("n", 77);
		orig.setDouble("x", 1.5);
		std::map<int64, std::string> amap; amap[0] = "elem0";
		orig.setStringArray("arr", amap);
		std::map<std::string, int64> mmap; mmap["idx"] = 42;
		orig.setIntMap("m", mmap);
		char* raw = orig.getRawData();
		uint32 sz = orig.getSize();
		if (!raw || sz == 0) { unittest::fail("DataMessage test: getRawData/getSize failed\n"); return false; }
		DataMessage decoded((const char*)raw);
		if (!decoded.isValid()) { unittest::fail("DataMessage test: decoded message invalid\n"); return false; }
		if (decoded.getUserCount() != orig.getUserCount()) {
			unittest::fail("DataMessage test: round-trip getUserCount mismatch\n"); return false;
		}
		if (decoded.getAsString("k") != "roundtrip") { unittest::fail("DataMessage test: round-trip string mismatch\n"); return false; }
		int64 nval = 0;
		if (!decoded.getInt("n", nval) || nval != 77) { unittest::fail("DataMessage test: round-trip int mismatch\n"); return false; }
		double xval = 0;
		if (!decoded.getDouble("x", xval) || xval < 1.4 || xval > 1.6) { unittest::fail("DataMessage test: round-trip double mismatch\n"); return false; }
		if (decoded.getAsStringArray("arr")[0] != "elem0") { unittest::fail("DataMessage test: round-trip array mismatch\n"); return false; }
		if (decoded.getIntMap("m")["idx"] != 42) { unittest::fail("DataMessage test: round-trip map mismatch\n"); return false; }
	}

	// 6. Text / structured serialisation
	unittest::progress(82, "text / structured serialisation");
	{
		DataMessage msg(CTRL_TEST, 0);
		msg.setString("foo", "bar");
		std::string ustr = msg.getUserEntriesAsString();
		if (ustr.empty()) { unittest::fail("DataMessage test: getUserEntriesAsString empty\n"); return false; }
		std::string ujson = msg.getUserEntriesAsJSON(false);
		if (ujson.empty()) { unittest::fail("DataMessage test: getUserEntriesAsJSON empty\n"); return false; }
		msg.getUserEntriesAsJSON(true);
		std::string json = msg.toJSON();
		if (json.empty()) { unittest::fail("DataMessage test: toJSON empty\n"); return false; }
		std::string xml = msg.toXML();
		if (xml.empty()) { unittest::fail("DataMessage test: toXML empty\n"); return false; }
		std::string csv = msg.toCSV();
		if (csv.empty()) { unittest::fail("DataMessage test: toCSV empty\n"); return false; }
		std::string hdr = DataMessage::GetCSVHeader();
		if (hdr.empty()) { unittest::fail("DataMessage test: GetCSVHeader empty\n"); return false; }
	}

	// 7. Copy and remove
	unittest::progress(90, "copy and remove");
	{
		DataMessage src(CTRL_TEST, 0);
		src.setString("a", "1");
		src.setInt("b", 2);
		DataMessage dst(CTRL_TEST, 0);
		if (dst.copyUserEntriesFromMessage(&src) != 2) {
			unittest::fail("DataMessage test: copyUserEntriesFromMessage count\n"); return false;
		}
		if (dst.getAsString("a") != "1" || dst.getAsInt("b") != 2) {
			unittest::fail("DataMessage test: copyUserEntriesFromMessage content\n"); return false;
		}
		uint32 cnt = dst.getUserCount();
		if (!dst.removeEntry("a")) { unittest::fail("DataMessage test: removeEntry failed\n"); return false; }
		if (dst.hasKey("a")) { unittest::fail("DataMessage test: removeEntry key still present\n"); return false; }
		if (dst.getUserCount() != cnt - 1) { unittest::fail("DataMessage test: removeEntry count\n"); return false; }
	}

	// 8. Edge / robustness
	unittest::progress(96, "edge / robustness");
	{
		DataMessage msg(CTRL_TEST, 0);
		if (msg.hasKey("nonexistent")) { unittest::fail("DataMessage test: hasKey nonexistent true\n"); return false; }
		if (msg.getString("nonexistent") != NULL) { unittest::fail("DataMessage test: getString nonexistent non-NULL\n"); return false; }
		if (msg.getTime("nonexistent") != 0) { unittest::fail("DataMessage test: getTime nonexistent non-zero\n"); return false; }
		DataMessage empty;
		if (empty.getUserCount() != 0) { unittest::fail("DataMessage test: empty getUserCount\n"); return false; }
		empty.getUserEntriesAsString();
		empty.toJSON();
	}

	unittest::progress(100, "done");
	return true;
}

void Register_DataMessage_Tests() {
	UnitTestRunner::instance().registerTest("datamessage", DataMessage::UnitTest,
		"Data message encode/decode, arrays, maps, serialisation", "core");
}

} // namespace cmlabs
