

//_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/
//_/_/
//_/_/  This file is part of CMSDK version 2.1.1.260726 (26 July 2026)
//_/_/  
//_/_/  developed by
//_/_/  
//_/_/  Communicative Machines Ltd. 
//_/_/    http://cmlabs.com/
//_/_/  Copyright (c) 2026
//_/_/  
//_/_/  in collaboration with 
//_/_/  
//_/_/  Icelandic Institute for Intelligent Machines ses
//_/_/    Menntavegur 1
//_/_/    101 Reykjavik
//_/_/    ICELAND
//_/_/    http://www.iiim.is/
//_/_/
//_/_/  and is released under the following
//_/_/  
//_/_/  EXTENDED BSD License
//_/_/
//_/_/  Redistribution and use in source and binary forms, with or without
//_/_/  modification, is permitted provided that the following conditions 
//_/_/  are met:
//_/_/
//_/_/  - Redistributions of source code must retain the above copyright 
//_/_/    and collaboration notice, this list of conditions and the following 
//_/_/    disclaimer.
//_/_/
//_/_/  - Redistributions in binary form must reproduce the above copyright 
//_/_/    notice, this list of conditions and the following disclaimer in the 
//_/_/    documentation and/or other materials provided with the distribution.
//_/_/
//_/_/  - Neither the name of its copyright holders nor the names of its 
//_/_/    contributors may be used to endorse or promote products derived from 
//_/_/    this software without specific prior written permission.
//_/_/
//_/_/  - CADIA Clause: The license granted in and to the software under this 
//_/_/    agreement is a limited-use license. The software may not be used in 
//_/_/    furtherance of: (i) intentionally causing bodily injury or severe 
//_/_/    emotional harm to any person; (ii) invading the personal privacy or 
//_/_/    violating the human rights of any person; or (iii) committing or 
//_/_/    preparing for any act of war. 
//_/_/
//_/_/  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
//_/_/  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
//_/_/  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 
//_/_/  PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER 
//_/_/  OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
//_/_/  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
//_/_/  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
//_/_/  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 
//_/_/  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 
//_/_/  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
//_/_/  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//_/_/
//_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/
/*! @file DataMessage.h
	@brief The binary DataMessage container — the central data-exchange object of Psyclone/CMSDK.

	A DataMessage is a single contiguous block of memory holding a fixed-size
	header (DataMessageHeader) followed by a variable number of self-describing
	user entries (strings, integers, floats, times, raw binary blocks, attached
	messages, plus arrays and maps of these). Because the whole message lives in
	one flat allocation it can be copied verbatim into shared memory, sent over
	the network, or persisted to disk without any serialisation step — the
	in-memory layout IS the wire format. For that reason the header structs in
	this file must never change size or field order without bumping
	CURRENTDATAMESSAGEVERSION and adding a converter (see
	DataMessage::ConvertDataFromOlderMessageFormat()).

	@note All multi-byte fields are stored in native host byte order; nodes in a
	Psyclone system are assumed to share endianness (x86/ARM little-endian on
	Windows, Linux and macOS builds alike).

	@ingroup Messaging
*/

/*! @defgroup Messaging Messaging
	@brief Message containers, subscriptions, indexing and record/replay.

	Everything that flows between Psyclone components is a message. DataMessage
	carries user payloads, ControlMessage carries internal control commands,
	Subscriptions.h defines the trigger/retrieve/post specifications, and
	MessageIndex / MessagePlayer provide in-memory indexing and record/replay.

	@par Why these objects exist
	A Psyclone system is a mesh of components living in different threads,
	processes and machines. They cannot share pointers or C++ objects directly,
	so every piece of information that crosses a boundary — sensor readings,
	commands, results, log records — is packed into a DataMessage — a single
	flat block of memory containing a fixed header (type, sender, receiver,
	timestamps, TTL, priority) followed by any number of self-describing typed
	entries (strings, integers, floats, times, raw binary, attached
	sub-messages, and arrays/maps of these) addressed by string key.

	@par Why a custom container instead of JSON/protobuf/std::map
	Because the in-memory layout IS the wire format, a DataMessage needs @b no
	serialisation step at any boundary: the same bytes are memcpy'd into a
	shared-memory queue, written to a socket by the Networking layer, or saved
	to disk for replay. There is no schema compiler, no external dependency,
	and behaviour is identical on Windows, Linux and macOS. Entries are
	self-describing, so readers and writers do not need to agree on a schema at
	compile time — a component can read the two fields it cares about and
	forward the rest untouched. Format evolution is handled explicitly via
	CURRENTDATAMESSAGEVERSION and
	DataMessage::ConvertDataFromOlderMessageFormat().

	@par How the pieces work together
	A producer builds a DataMessage and posts it; subscriptions
	(Subscriptions.h) decide which components receive it; MessageIndex can keep
	a queryable in-memory index of recent messages, and MessagePlayer can
	record a stream to disk and replay it later at original or accelerated
	speed.

	@code
	// Producer: build a message and fill in typed fields
	DataMessage* msg = new DataMessage(myType, myComponentID); // PsyType + sender id
	msg->setString("camera", "front-left");
	msg->setInt("width", 1920);
	msg->setDouble("exposure", 0.0125);
	msg->setTime("captured", GetTimeNow());
	msg->setData("pixels", pixelBuffer, pixelBytes);   // raw binary block

	// The flat block can now cross any boundary without serialisation,
	// e.g. via a shared-memory queue (see the Memory group):
	MessageQueue::AddMessageToQueue(qid, msg);          // copied into shared memory
	delete msg;                                         // caller keeps ownership

	// Consumer (possibly another process): read the fields back
	DataMessage* in = MessageQueue::WaitForMessageQueue(qid, 100); // 100 ms wait
	if (in) {
		const char* cam = in->getString("camera");      // pointer into the message
		int64 w         = in->getInt("width");
		uint64 when     = in->getTime("captured");
		uint32 n; const char* px = in->getData("pixels", n);
		// ... use values before deleting the message ...
		delete in;
	}
	@endcode

	@note Pointers returned by getString()/getData() point @e into the
	message's own memory block — they are valid only while the message is alive
	and unmodified; copy the data out if you need it longer.

	@warning The header structs in DataMessage.h are a de-facto wire/shared-
	memory ABI. Never change their size or field order without bumping
	CURRENTDATAMESSAGEVERSION and providing a converter, and always rebuild all
	binaries together (see the @ref Memory group for the same rule on
	shared-memory structs).

	@see DataMessage, ControlMessage, MessageIndex, MessagePlayer, @ref Requests, @ref Memory
*/

#if !defined(_DATAMESSAGE_H_)
#define _DATAMESSAGE_H_

// Data Message Version History
// Initial version:
// - data->cid = DATAMESSAGEOLDID
// - does not contain cver, cenc, cyclecputime, cyclewalltime, chaincputime, chainwalltime
// Version 10:
// - data->cid = DATAMESSAGEID
// - now contains cver, cenc, cyclecputime, cyclewalltime, chaincputime, chainwalltime

#define CURRENTDATAMESSAGEVERSION 10

#include "Types.h"
#include "ObjectIDs.h"
//#include "Collections.h"
#include "Utils.h"

namespace cmlabs {

/*! @brief Header preceding every user entry inside a DataMessage's flat memory block.

	Each user entry (key/value pair) inside a DataMessage starts with this
	8-byte header, followed by the NUL-terminated key string and then the value
	payload. Entries are walked sequentially by advancing @c size bytes; a
	deleted entry keeps its slot but has its @c cid set to UNUSEDID so the space
	can later be reused by an entry of exactly the same size (see
	DataMessage::findSpace()).
	@ingroup Messaging
*/
struct DataMessageEntryHeader {
	uint32	size;	//!< Total size of this entry in bytes, including this header, the key and the payload. Used as the skip distance when scanning entries.
	uint32	cid;	//!< Content type id of the payload (CONSTCHARID, TIMEID, INTID, DOUBLEID, CHARDATAID, DATAMESSAGEID, UNUSEDID, ...), see ObjectIDs.h.
};

/*! @brief Legacy (pre-version-10) DataMessage wire header, kept only for format conversion.

	Messages recorded or transmitted by older Psyclone builds carry
	cid == DATAMESSAGEOLDID and use this layout, which lacks the version/encoding
	fields and the cycle/chain profiling counters of DataMessageHeader.
	DataMessage::ConvertDataFromOlderMessageFormat() upgrades such blocks to the
	current format. @warning Do not modify this struct — it must match the byte
	layout of historical recordings exactly.
	@ingroup Messaging
*/
struct DataMessageHeader_Old {
	uint32		size;			// auto
	uint32		cid;			// auto
	uint64		memid;			// auto
	uint64		time;			// auto
	uint64		sendtime;		// api
	uint64		recvtime;		// api
	uint16		origin;			// NodeID, api
	uint16		destination;	// NodeID, api

	uint64		ttl;			// spec
	uint16		priority;		// spec
	uint8		policy;			// spec

	uint32		from;			// user
	uint32		to;				// user

	PsyType		type;			// api/user
	uint32		tag;			// api/user
	uint16		status;			// api/user
	uint64		reference;		// api
	uint64		serial;			// api/user
	PsyContext	contextchange;	// api/user

	uint32 userSize;	// size of user data
	uint32 userCount;	// number of user data entries
};

/*! @brief The current (version 10) DataMessage wire/shared-memory header.

	This struct is the first thing in every DataMessage memory block; the user
	entries follow immediately after it. It is written verbatim into shared
	memory and onto the network, so its size, alignment and field order form a
	binary contract between all nodes and all recorded data.
	@warning Never reorder, remove or resize fields. To extend the format,
	append fields, bump CURRENTDATAMESSAGEVERSION, change DATAMESSAGEID, and
	provide a converter (as was done for DataMessageHeader_Old, version < 10).

	Field provenance conventions used in the trailing comments:
	- "auto": maintained by the messaging core itself
	- "api": set by the PsyAPI/runtime while routing
	- "spec": copied from the posting TriggerSpec/PostSpec
	- "user": freely settable by user code
	@ingroup Messaging
*/
struct DataMessageHeader {
	uint32		size;			//!< Total size of the whole message block in bytes (header + all entries). auto
	uint32		cid;			//!< Object id, always DATAMESSAGEID for a valid current-format message; used by isValid(). auto
	uint16		cver;			//!< Message format version (CURRENTDATAMESSAGEVERSION). auto
	uint16		cenc;			//!< Encoding flags (reserved for compression/encryption of the payload). auto
	uint64		memid;			//!< Shared-memory id assigned when the message is stored in a memory map. auto
	uint64		time;			//!< Creation time in microseconds since epoch (see PsyTime). auto
	uint64		sendtime;		//!< Time the message was posted/sent, in microseconds. api
	uint64		recvtime;		//!< Time the message was received by the current component, in microseconds. api
	uint16		origin;			//!< NodeID of the node the message originated on. api
	uint16		destination;	//!< NodeID of the target node (0 = local/any). api

	uint64		ttl;			//!< Time-to-live in microseconds; 0 means deliver-and-forget (not retained in memory). spec
	uint16		priority;		//!< Scheduling priority; higher values are processed first by whiteboards and modules. spec
	uint8		policy;			//!< Delivery policy flags, e.g. MESSAGE_NON_GUARANTEED. spec

	uint32		from;			//!< Component id of the sender. user
	uint32		to;				//!< Component id of the addressee; 0 for publish/subscribe distribution. user

	PsyType		type;			//!< Hierarchical message type (e.g. "2.201.3") used for subscription matching. api/user
	uint32		tag;			//!< User tag id for temporal/semantic grouping of related messages. api/user
	uint16		status;			//!< User/application status code. api/user
	uint64		reference;		//!< Reference id linking this message to another (e.g. request/reply correlation). api
	uint64		serial;			//!< Serial number, e.g. sequence position in a recording. api/user
	PsyContext	contextchange;	//!< Context switch this message announces (NOCONTEXT if none). api/user

	uint32 userSize;	//!< Total size in bytes of all user entries currently in use.
	uint32 userCount;	//!< Number of user entries currently in use.

	uint32		cyclecputime;	//!< CPU time spent processing the triggering message in this cycle, in microseconds (profiling).
	uint32		cyclewalltime;	//!< Wall-clock time spent in this cycle, in microseconds (profiling).
	uint32		chaincputime;	//!< Accumulated CPU time across the whole causal message chain, in microseconds.
	uint32		chainwalltime;	//!< Accumulated wall time across the whole causal message chain, in microseconds.
	uint32		chaincount;		//!< Number of messages in the causal chain leading to (and including) this one.

	uint32		systemid;		//!< Id of the Psyclone system instance that sent the message.

	/*! Render the header fields as a JSON object (header only, no user entries).
	@param subtypes optional map translating type level ids to readable names
	@param subcontexts optional map translating context level ids to readable names
	@param compNames optional map translating component ids to readable names
	@return JSON string */
	std::string toJSON(std::map<uint16, std::string>* subtypes = NULL, std::map<uint16, std::string>* subcontexts = NULL, std::map<uint32, std::string>* compNames = NULL);
	/*! Render the header fields as an XML fragment (header only, no user entries).
	@param subtypes optional map translating type level ids to readable names
	@param subcontexts optional map translating context level ids to readable names
	@param compNames optional map translating component ids to readable names
	@return XML string */
	std::string toXML(std::map<uint16, std::string>* subtypes = NULL, std::map<uint16, std::string>* subcontexts = NULL, std::map<uint32, std::string>* compNames = NULL);
};

/*! @brief The central Psyclone data container: a self-contained binary message with typed, named user entries.

	DataMessage is how components exchange data in Psyclone. Conceptually it is
	a routed envelope (type, from/to, tag, timestamps, TTL, priority, context)
	plus an ordered set of named user entries. Entries can be single values
	(string, integer, float, time, raw data, attached DataMessage), arrays
	(indexed by int64) or maps (indexed by string), all accessed through typed
	get/set overloads keyed by name.

	The entire message lives in one flat memory block (pointed to by #data), so
	it can be moved between processes via shared memory or over the network with
	a single copy — there is no separate serialisation format.

	Typical use:
	@code
	DataMessage* msg = new DataMessage(myType, myComponentID);
	msg->setString("name", "camera1");
	msg->setInt("width", 640);
	msg->setData("pixels", pixelBuffer, bufferSize);
	// ... post via the API; the receiver then reads:
	int64 w = msg->getInt("width");
	uint32 size; const char* pixels = msg->getData("pixels", size);
	@endcode

	Ownership and lifetime rules:
	- Pointers returned by getString()/getData() point INTO the message's own
	  memory: they are valid only while the message exists and no entry is
	  added/changed (which may reallocate the block). Do not free them.
	- getDataCopy()/getAttachedMessageCopy() and copy() return heap allocations
	  the CALLER must delete (delete[] for data copies, delete for messages).
	- set...() calls copy the supplied values into the message; the caller keeps
	  ownership of its own buffers (including the DataMessage passed to
	  setAttachedMessage()).

	@warning DataMessage is not thread-safe: concurrent set...() calls, or a
	get...() concurrent with a set...(), must be externally synchronised. The
	internal find-key cache also makes lookups stateful (mutable members).
	@ingroup Messaging
*/
class DataMessage {
public:
	/*! FindClosestMessage(uint64 t, std::map<uint64, DataMessage*> &messages)
	In a map of time -> DataMessage, find the message closes to the time t
	@param t timestamp to find the closest match to
	@param messages map of time -> DataMessage
	@return Closest DataMessage to time t or NULL if messages is empty
	*/
	static DataMessage* FindClosestMessage(uint64 t, std::map<uint64, DataMessage*> &messages);

	/*! Convert a binary block in the legacy (pre-version-10, DATAMESSAGEOLDID) format
	to the current DataMessageHeader layout.
	@param data legacy binary message block (not modified)
	@param newData receives a newly allocated block in the current format; caller takes ownership (delete[])
	@return true if the block was recognised and converted successfully */
	static bool ConvertDataFromOlderMessageFormat(const char* data, char **newData);

	/*! Run the built-in self test for the DataMessage implementation.
	@return true if all tests pass */
	static bool UnitTest();

	/*! DataMessage()
	Creates an empty DataMessage
	*/
	DataMessage();
	/*! DataMessage(PsyType type, uint32 from)
	Creates a DataMessage with type and from fields set
	@param type set the message type
	@param from set the message sender id
	*/
	DataMessage(PsyType type, uint32 from);
	/*! DataMessage(PsyType type, uint32 from, uint32 to, uint64 ttl = 0, uint16 priority = 0)
	Creates a DataMessage with multiple fields set
	@param type set the message type
	@param from set the message sender id
	@param to set the message receiver id
	@param ttl set the time-to-live for the message in microseconds, after which it will have reached its end-of-life (eol)
	@param priority set the message priority
	*/
	DataMessage(PsyType type, uint32 from, uint32 to, uint64 ttl = 0, uint16 priority = 0);
	/*! DataMessage(PsyType type, uint32 from, uint32 to, uint32 tag, uint64 time, uint64 ttl = 0, uint16 priority = 0)
	Creates a DataMessage with multiple fields set
	@param type set the message type
	@param from set the message sender id
	@param to set the message receiver id
	@param tag set the message tag id
	@param time set the message creation time in microseconds
	@param ttl set the time-to-live for the message in microseconds, after which it will have reached its end-of-life (eol)
	@param priority set the message priority
	*/
	DataMessage(PsyType type, uint32 from, uint32 to, uint32 tag, uint64 time, uint64 ttl = 0, uint16 priority = 0);
	/*! DataMessage(char* data, bool copy = false)
	Creates a DataMessage as a copy of the binary message data provided
	@param data the binary message data to copy from
	@param copy should the data be copied or can it be taken
	*/
	DataMessage(char* data, bool copy = false);
	/*! DataMessage(const char* data)
	@param data the binary message data to copy from
	Creates a DataMessage as a copy of the binary message data provided, will always copy the data
	*/
	DataMessage(const char* data);
	/*! DataMessage(const DataMessage& msg)
	@param msg the message to copy from
	Creates a DataMessage as a copy of another DataMessage
	*/
	DataMessage(const DataMessage& msg);
	/*! DataMessage(const char* data, uint32 maxDraftSize)
	Creates a DataMessage as a copy of the binary message data provided, will always copy the data
	If the binary size is below maxDraftSize the full message is copied
	If not some larger user entries will be provided as information only, i.e. name and size, but not content
	@param data the binary message data to copy from
	@param maxDraftSize the maximum binary size of the newly created message
	*/
	DataMessage(const char* data, uint32 maxDraftSize);
	/*! DataMessage(const DataMessage& msg, uint32 maxDraftSize)
	Creates a DataMessage as a copy of the message provided
	If the binary size is below maxDraftSize the full message is copied
	If not some larger user entries will be provided as information only, i.e. name and size, but not content
	@param msg the message to copy from
	@param maxDraftSize the maximum binary size of the newly created message
	*/
	DataMessage(const DataMessage& msg, uint32 maxDraftSize);
	/*! ~DataMessage()
	Message destructor
	*/
	~DataMessage();

	/*! copy()
	Creates a full deep copy of the message and returns it.
	@return new DataMessage owned by the caller (delete when done), or NULL if data is invalid
	*/
	DataMessage* copy() {
		return new DataMessage((const char*)(this->data));
	}

    /*! swapMessageData(const char* data)

        Internal function: replaces this message's memory block with @p data,
        returning the previous block so the caller can dispose of or reuse it.
        Used by the runtime to move message ownership without copying.

        @param data the new binary message block to adopt
        @return the previous message block (caller becomes responsible for it)
     */
	
	const char* swapMessageData(const char* data);
    
    /*! fillInDraftUserDataFrom(const char* data, uint32 maxDraftSize)

        Internal function: for a "draft" message (created with a maxDraftSize
        limit, where large entries were replaced by name+size placeholders),
        copy the missing full entry payloads back in from the complete source
        block @p data.

        @param data the complete original binary message block
        @param maxDraftSize the draft size limit originally used
        @return true if successful
     */
	bool fillInDraftUserDataFrom(const char* data, uint32 maxDraftSize);

	/*! isValid()
	Checks that the message memory block exists and carries the current-format
	object id (DATAMESSAGEID). All accessors call the equivalent check and
	return defaults (0, NULL, false) on an invalid message rather than crashing.
	@return true if the message is valid, false otherwise
	*/
	bool isValid() {return data && GetObjID(data) == DATAMESSAGEID; }

	// User
    
    /*! getContentType(const char* key)
        
        returns the type of content with this key as CONSTCHARID, TIMEID, CHARDATAID, INTID, DOUBLEID, etc
     
        @param key
     */
	
	uint32 getContentType(const char* key);
    
    /*! getContentSize(const char* key)
     
        returns the binary size of the content with this key
     
        @param key
     */
    
	uint32 getContentSize(const char* key);

	/*! getAsBool(const char* key)

	returns the interpretation of the entry as a Boolean value
	i.e. if the value is integer/float != 0, string "yes", etc.

	@param key

	*/

	bool getAsBool(const char* key);

    /*! getAsString(const char* key)
     
        returns the user entry as a string regardless of what type it is

        @param key
     
     */
    
	std::string getAsString(const char* key);

	/*! getAsInt(const char* key)

	returns the user entry as an integer regardless of what type it is

	@param key

	*/

	int64 getAsInt(const char* key);

	/*! getAsTime(const char* key)

	returns the user entry as a time regardless of what type it is

	@param key

	*/

	uint64 getAsTime(const char* key);

	/*! getAsFloat(const char* key)

	returns the user entry as a float regardless of what type it is

	@param key

	*/

	float64 getAsFloat(const char* key);

    /*! getTime(const char* key)
     
        returns the user time entry with this key
     
        @param key
     */
    
	uint64 getTime(const char* key);
	
    /*! getString(const char* key)
        
        returns the user string entry with this key
     
        @param key
     */
    
	const char* getString(const char* key);
	
    /*! getString(const char* key, uint32 &size)
     
        returns the user string entry with this key and provides the size
     
        @param key
        @param size
     */
    
	const char* getString(const char* key, uint32 &size);
	
    /*! getInt(const char* key, int64& value)
     
        returns the user integer entry with this key

        @param key
        @param value
     */
    
	bool getInt(const char* key, int64& value);
	
    /*! getDouble(const char* key, double& value)
     
        returns the user float entry with this key
     
        @param key
        @param value
     
     */
    
	bool getDouble(const char* key, double& value);
	
    /*! getFloat(const char* key, float64& value)
        
        returns the user float entry with this key
     
        @param key
        @param value
     
     */
    
	bool getFloat(const char* key, float64& value);
	
    /*! getData(const char* key, uint32& size)
     
        returns the user binary data entry with this key + the size
     
        @param key
        @param size
     */
    
	const char* getData(const char* key, uint32& size);
	
    /*! getDataCopy(const char* key, uint32& size)
     
        returns a copy of the user binary data entry with this key + the size
     
        @param key
        @param size
     
     */
    
	char* getDataCopy(const char* key, uint32& size);
	
    /*! getAttachedMessageCopy(const char* key)
     
        returns a copy of the user message entry with this key
     
        @param key
     
     */
    
	DataMessage* getAttachedMessageCopy(const char* key);

   
    
    
    /*! getTime(const char* key, bool& success)
     
     returns the user time entry with this key / returning the value and success/fail as a parameter
     
     @param key
     @param success
     */
    
	uint64 getTime(const char* key, bool& success);
   
    /*! getString(const char* key)
     
     returns the user string entry with this key / returning the value and success/fail as a parameter
     
     @param key
     @param success
     */
    
    
	const char* getString(const char* key, bool& success);
    
    /*! getString(const char* key, uint32 &size)
     
     returns the user string entry with this key and provides the size / returning the value and success/fail as a parameter
     
     @param key
     @param size
     @param success
     */
    
	const char* getString(const char* key, uint32 &size, bool& success);
    
    /*! getInt(const char* key, int64& value)
     
     returns the user integer entry with this key / returning the value and success/fail as a parameter

     
     @param key entry key
     @param success set to true if the entry was found
     */
    
    
	int64 getInt(const char* key, bool& success);
    
    /*! getDouble(const char* key, double& value)
     
     returns the user float entry with this key / returning the value and success/fail as a parameter
     

     
     @param key entry key
     @param success set to true if the entry was found
     */
    
	double getDouble(const char* key, bool& success);
    
    /*! getFloat(const char* key, float64& value)
     
     returns the user float entry with this key / returning the value and success/fail as a parameter
     
     @param key entry key
     @param success set to true if the entry was found
     
     */
    
	float64 getFloat(const char* key, bool& success);
    
    /*! getData(const char* key, uint32& size)
     
     returns the user binary data entry with this key + the size / returning the value and success/fail as a parameter
     
     @param key
     @param size
     @param success
     */
    
	const char* getData(const char* key, uint32& size, bool& success);
    
    /*! getDataCopy(const char* key, uint32& size)
     
     returns a copy of the user binary data entry with this key + the size / returning the value and success/fail as a parameter

     
     @param key
     @param size
     @param success
     
     */
    
	char* getDataCopy(const char* key, uint32& size, bool& success);
    
    /*! getAttachedMessageCopy(const char* key)
     
     returns a copy of the user message entry with this key / returning the value and success/fail as a parameter

     
     @param key
     @param success
     */
    
	DataMessage* getAttachedMessageCopy(const char* key, bool& success);

    
    /*! getInt(const char* key)
     
     returns the user integer entry with this key / returning the value with no indication of the success/fail except value will be default (0, "", NULL, etc.)
     
     
     @param key
   
     */
    
	int64 getInt(const char* key);
    
    /*! getDouble(const char* key)
     
     returns the user double entry with this key/ returning the value with no indication of the success/fail except value will be default (0, "", NULL, etc.)
     
     
     @param key
     
     
     */
    
	double getDouble(const char* key);
    
   
    /*! getFloat(const char* key)
     
     returns the user float entry with this key/ returning the value with no indication of the success/fail except value will be default (0, "", NULL, etc.)
     
     
     @param key
     
     
     */
    
	float64 getFloat(const char* key);

	// *** setting all of the above, returns success or fail (usually because the message is not valid)
    
    /*! setTime(const char* key, uint64 value)

     Sets (creating or overwriting) the user time entry named @p key.
     Times are microseconds since epoch (see PsyTime).

        @param key entry name
        @param value time value in microseconds
        @return true if successful, false otherwise (usually because the message is not valid)
     */
	bool setTime(const char* key, uint64 value);
	
    /*! setString(const char* key, const char* value)

     Sets (creating or overwriting) the user string entry named @p key.
     The string is copied into the message; the caller keeps ownership of @p value.

     @param key entry name
     @param value NUL-terminated string to store
     @return true if successful, false otherwise (usually because the message is not valid)
     */
	bool setString(const char* key, const char* value);
	
    /*! setInt(const char* key, int64 value)

     Sets (creating or overwriting) the user integer entry named @p key.

     @param key entry name
     @param value integer value to store
     @return true if successful, false otherwise (usually because the message is not valid)
     */
	bool setInt(const char* key, int64 value);
    
    /*! setDouble(const char* key, double value)

     Sets (creating or overwriting) the user floating-point entry named @p key.

     @param key entry name
     @param value value to store
     @return true if successful, false otherwise (usually because the message is not valid)
     */
	bool setDouble(const char* key, double value);
	
    /*! setFloat(const char* key, float64 value)

     Sets (creating or overwriting) the user floating-point entry named @p key.
     float64 is a typedef for double, so this is equivalent to setDouble().

     @param key entry name
     @param value value to store
     @return true if successful, false otherwise (usually because the message is not valid)
     */
	bool setFloat(const char* key, float64 value);
	
    /*! setData(const char* key, const char* value, uint32 size)

     Sets (creating or overwriting) a raw binary user entry named @p key.
     The bytes are copied into the message; the caller keeps ownership of @p value.

     @param key entry name
     @param value pointer to the bytes to store
     @param size number of bytes to store
     @return true if successful, false otherwise (usually because the message is not valid)
     */
    bool setData(const char* key, const char* value, uint32 size);
	
    /*! setAttachedMessage(const char* key, DataMessage* msg)

     Attaches a full copy of another DataMessage as a user entry named @p key.
     The message content is copied; the caller retains ownership of @p msg.
     Retrieve it later with getAttachedMessageCopy().

     @param key entry name
     @param msg the message to attach (not modified, not taken over)
     @return true if successful, false otherwise (usually because the message is not valid)
     */
	bool setAttachedMessage(const char* key, DataMessage* msg);

	/*! getUserSize()

	Return the memory usage in bytes of all user entries stored in the message

	*/
	uint32 getUserSize() {
		return data->userSize;
	}

	/*! getUserCount()

	Return the number of user entries stored in the message

	*/
	uint32 getUserCount() {
		return data->userCount;
	}

	// Arrays
	// *** Same as above for single values, but now getting and setting entries in arrays
	// myval[0] = val, myval[1] = val, myval[2] = val, etc.

	enum KeyType {NONE, SINGLE, ARRAY, MAP};

	/*! getKeyType(const char* key)

	Return if the key points to a SINGLE, ARRAY or MAP value

	@param key
	*/

	KeyType getKeyType(const char* key);

	/*! isArrayContent(const char* key)

	Check if key variable is an array or not

	@param key
	*/
	bool isArrayContent(const char* key);

	/*! isMapContent(const char* key)

	Check if key variable is a map or not

	@param key
	*/
	bool isMapContent(const char* key);

	/*! isSingleValue(const char* key)

	Check if key variable is not an array nor a map

	@param key
	*/
	bool isSingleValue(const char* key);


	/*!

	Check if key variable exists

	@param key
	*/
	bool hasKey(const char* key);

	/*!

	Check if key variable exists and is a string

	@param key
	*/
	bool hasString(const char* key);

	/*!

	Check if key variable exists and is an integer

	@param key
	*/
	bool hasInteger(const char* key);

	/*!

	Check if key variable exists and is a float

	@param key
	*/
	bool hasFloat(const char* key);

	/*!

	Check if key variable exists and is a time

	@param key
	*/
	bool hasTime(const char* key);

	/*!

	Check if key variable exists and is a data entry

	@param key
	*/
	bool hasData(const char* key);

	/*!

	Check if key variable exists and is a message

	@param key
	*/
	bool hasMessage(const char* key);


    /*! getContentType(int64 i, const char* key)
     
     @param i
     @param key
     */
	uint32 getContentType(int64 i, const char* key);
	
    /*! getContentSize(int64 i, const char* key)
     
     @param i
     @param key
     */
	uint32 getContentSize(int64 i, const char* key);
	 
    /*! getTime(int64 i, const char* key)
     
     
     returns the user time with this key, getting and setting entries in arrays
     
     @param i
     @param key
     */
	uint64 getTime(int64 i, const char* key);
	
    /*! getAsString(int64 i, const char* key)
     
     @param i
     @param key
     */
	std::string getAsString(int64 i, const char* key);
	
	/*! getAsInt(int64 i, const char* key)

	@param i
	@param key
	*/
	int64 getAsInt(int64 i, const char* key);

	/*! getAsTime(int64 i, const char* key)

	@param i
	@param key
	*/
	uint64 getAsTime(int64 i, const char* key);

	/*! getAsFloat(int64 i, const char* key)

	@param i
	@param key
	*/
	float64 getAsFloat(int64 i, const char* key);

	/*! getString(int64 i, const char* key)
     
     @param i
     @param key
     */
	const char* getString(int64 i, const char* key);
	
    /*! getString(int64 i, const char* key, uint32 &size)
     
     @param i
     @param key
     @param size
     */
	const char* getString(int64 i, const char* key, uint32 &size);
	
    /*! getInt(int64 i, const char* key, int64& value)
     
     @param i
     @param key
     @param value
     */
	bool getInt(int64 i, const char* key, int64& value);
	
    /*! getDouble(int64 i, const char* key, double& value)
     
     @param i
     @param key
     @param value
     */
	bool getDouble(int64 i, const char* key, double& value);
    
    /*! getFloat(int64 i, const char* key, float64& value)
     
     @param i
     @param key
     @param value
     */
	bool getFloat(int64 i, const char* key, float64& value);
	
    /*! getInt(int64 i, const char* key, bool& success)
     
     @param i
     @param key
     @param success
     */
	int64 getInt(int64 i, const char* key, bool& success);
    /*! getDouble(int64 i, const char* key, bool& success)
     
     @param i
     @param key
     @param success
     */
	double getDouble(int64 i, const char* key, bool& success);
    /*! getFloat(int64 i, const char* key, bool& success)
     
     @param i
     @param key
     @param success
     */
	float64 getFloat(int64 i, const char* key, bool& success);
    /*! getInt(int64 i, const char* key)
     
     @param i
     @param key
     
     */
	int64 getInt(int64 i, const char* key);
    /*! getDouble(int64 i, const char* key)
     
     @param i
     @param key
     */
	double getDouble(int64 i, const char* key);
    /*! getFloat(int64 i, const char* key)
     
     @param i
     @param key
     */
	float64 getFloat(int64 i, const char* key);
    
    /*! getData(int64 i, const char* key, uint32& size)
     
     @param i
     @param key
     @param size
     */
	const char* getData(int64 i, const char* key, uint32& size);
    
    /*! getDataCopy(int64 i, const char* key, uint32& size)
     
     @param i
     @param key
     @param size
     */
	char* getDataCopy(int64 i, const char* key, uint32& size);
    
    /*! getAttachedMessageCopy(int64 i, const char* key)
     
     @param i
     @param key
    
     */
	DataMessage* getAttachedMessageCopy(int64 i, const char* key);

    /*! setTime(int64 i, const char* key, uint64 value)
     
     @param i
     @param key
     @param value
     */
	
	bool setTime(int64 i, const char* key, uint64 value);
    
    /*! setString(int64 i, const char* key, const char* value)
     
     @param i
     @param key
     @param value
     */
	bool setString(int64 i, const char* key, const char* value);
    
    /*! setInt(int64 i, const char* key, int64 value)
     
     @param i
     @param key
     @param value
     */
	bool setInt(int64 i, const char* key, int64 value);
    
    /*! setDouble(int64 i, const char* key, double value)
     
     @param i
     @param key
     @param value
     */
	bool setDouble(int64 i, const char* key, double value);
    
    /*! setFloat(int64 i, const char* key, float64 value)
     
     @param i
     @param key
     @param value
     */
	bool setFloat(int64 i, const char* key, float64 value);
/*! setData(int64 i, const char* key, const char* value, uint32 size)
     
     @param i
     @param key
     @param value
     @param size
     */
	bool setData(int64 i, const char* key, const char* value, uint32 size);
	//    bool setData(int64 i, const char* key, const void* value, uint32 size);
    
    /*! setAttachedMessage(int64 i, const char* key, DataMessage* msg)
     
     @param i
     @param key
     @param msg
     */
	bool setAttachedMessage(int64 i, const char* key, DataMessage* msg);

	// *** Same as above but reading whole arrays in one go
	std::map<int64, uint64> getTimeArray(const char* key);
    /*! getAsStringArray(const char* key)
     
      @param key
     */
	std::map<int64, std::string> getAsStringArray(const char* key);
	/*! getAsIntArray(const char* key)

	@param key
	*/
	std::map<int64, int64> getAsIntArray(const char* key);
	/*! getAsFloatArray(const char* key)

	@param key
	*/
	std::map<int64, float64> getAsFloatArray(const char* key);
	/*! getStringArray(const char* key)
     
      @param key
     */
	std::map<int64, std::string> getStringArray(const char* key);
    /*! getIntArray(const char* key)
     
      @param key
     
     */
	std::map<int64, int64> getIntArray(const char* key);
    /*!  getDoubleArray(const char* key)
     
      @param key
     */
	std::map<int64, double> getDoubleArray(const char* key);
    /*! getFloatArray(const char* key)
     
      @param key
     */
	std::map<int64, float64> getFloatArray(const char* key);
    /*! getAttachedMessageArray(const char* key)
     
      @param key
     */
	std::map<int64, DataMessage*> getAttachedMessageArray(const char* key);

	/*! getArraySize(const char* key)
	Returns the number of elements in array with name '&lt;key&gt;'
	@param key entry key
	*/
	uint32 getArraySize(const char* key);

	/*! getArrayCount(const char* key)
	Returns the number of elements in array with name '&lt;key&gt;'
	@param key entry key
	*/
	uint32 getArrayCount(const char* key);

	/*! getMapSize(const char* key)
	Returns the number of elements in map with name '&lt;key&gt;'
	@param key entry key
	*/
	uint32 getMapSize(const char* key);

	/*! getMapCount(const char* key)
	Returns the number of elements in map with name '&lt;key&gt;'
	@param key entry key
	*/
	uint32 getMapCount(const char* key);


    /*!
     */
	// *** Same as above but setting whole arrays in one go
    /*! setTimeArray(const char* key, std::map<int64, uint64>& map)
     Sets the whole time array named @p key in one call.
     @param key entry name of the array
     @param map index → time (microseconds) values to store
     @return true if successful
     */
	bool setTimeArray(const char* key, std::map<int64, uint64>& map);
    
    /*! setStringArray(const char* key, std::map<int64, std::string>& map)
     Sets the whole string array named @p key in one call.
     @param key entry name of the array
     @param map index → string values to store
     @return true if successful
     */
	bool setStringArray(const char* key, std::map<int64, std::string>& map);
    
    /*! setIntArray(const char* key, std::map<int64, int64>& map)
     Sets the whole integer array named @p key in one call.
     @param key entry name of the array
     @param map index → integer values to store
     @return true if successful
     */
	bool setIntArray(const char* key, std::map<int64, int64>& map);
    
    /*! setDoubleArray(const char* key, std::map<int64, double>& map)
     Sets the whole double array named @p key in one call.
     @param key entry name of the array
     @param map index → double values to store
     @return true if successful
     */
	bool setDoubleArray(const char* key, std::map<int64, double>& map);
    
    /*! setFloatArray(const char* key, std::map<int64, float64>& map)
     Sets the whole float array named @p key in one call.
     @param key entry name of the array
     @param map index → float values to store
     @return true if successful
     */
	bool setFloatArray(const char* key, std::map<int64, float64>& map);
    
    /*! setAttachedMessageArray(const char* key, std::map<int64, DataMessage*>& map)
     Sets the whole array of attached messages named @p key in one call.
     Each message is copied in; the caller retains ownership of the originals.
     @param key entry name of the array
     @param map index → message values to store
     @return true if successful
     */
	bool setAttachedMessageArray(const char* key, std::map<int64, DataMessage*>& map);

	// Maps
    /*!
     */
    
	// *** Same as for arrays, but now with maps
	// myval["up"] = val, myval["down"] = val, myval["left"] = val, etc.
    
    /*!
     
     returns the type of content with this key as CONSTCHARID, TIMEID, CHARDATAID, INTID, DOUBLEID, etc
     @param idx textual index for the maps
     @param key
     @return contentType as int
     
     */
	uint32 getContentType(const char* idx, const char* key);
    
    /*!
     
     returns the binary size of the content with this key
     @param idx textual index for the maps
     @param key
     @return  binary size of the content
     */
	uint32 getContentSize(const char* idx, const char* key);
    /*!
     
     returns the user time entry with this key
     @param idx textual index for the maps
     @param key
     @return user time
     */
    
	uint64 getTime(const char* idx, const char* key);
    /*!
     
     returns the user entry as a string regardless of what type it is
     @param idx textual index for the maps
     @param key
     @return  user entry as a string
     
     */
	std::string getAsString(const char* idx, const char* key);
	/*!

	returns the user entry as an integer regardless of what type it is
	@param idx textual index for the maps
	@param key
	@return  user entry as a string

	*/
	int64 getAsInt(const char* idx, const char* key);
	/*!

	returns the user entry as a time regardless of what type it is
	@param idx textual index for the maps
	@param key
	@return  user entry as a string

	*/
	uint64 getAsTime(const char* idx, const char* key);
	/*!

	returns the user entry as a float regardless of what type it is
	@param idx textual index for the maps
	@param key
	@return  user entry as a string

	*/
	float64 getAsFloat(const char* idx, const char* key);
	/*!
     
     returns the user string entry with this key
     @param idx textual index for the maps
     @param key
     @return user string entry
     */
	const char* getString(const char* idx, const char* key);
    /*!
     
     returns the user string entry with this key
     @param idx textual index for the maps
     @param key
     @param size the size of the memory chunk for the String
     @return user string entry
     */
	const char* getString(const char* idx, const char* key, uint32 &size);
    /*!
     
     returns the user integer entry with this key
     @param idx textual index for the maps
     @param key
     @param value
     @return true if successfull
     */
    
	bool getInt(const char* idx, const char* key, int64& value);
    /*!
     
     returns the user float entry with this key
     @param idx textual index for the maps
     @param key
     @param value
     @return true if successfull
     
     */
	bool getDouble(const char* idx, const char* key, double& value);
    /*!
     
     returns the user float entry with this key
     @param idx textual index for the maps
     @param key
     @param value
     @return true if successfull
     
     */
	bool getFloat(const char* idx, const char* key, float64& value);
    /*!
     returns the user integer entry with this key
     @param idx textual index for the maps
     @param key
     @param success
     @return  user integer entry
     */
	int64 getInt(const char* idx, const char* key, bool& success);
    /*!
     
     returns the user double entry with this key
     @param idx textual index for the maps
     @param key
     @param success
     @return user double entry
     */
	double getDouble(const char* idx, const char* key, bool& success);
    /*!
     
     returns the user float entry with this key
     @param idx textual index for the maps
     @param key
     @param success
     @return
     */
	float64 getFloat(const char* idx, const char* key, bool& success);
    /*!
     
     returns the user integer entry with this key
     @param idx textual index for the maps
     @param key
     @return user integer entry
     */
	int64 getInt(const char* idx, const char* key);
    /*!
     
     returns the user double entry with this key
     @param idx textual index for the maps
     @param key
     @return user double entry
     
     */
	double getDouble(const char* idx, const char* key);
    /*!
     returns the user float entry with this key
     @param idx textual index for the maps
     @param key
     @return user float entry
     
     */
	float64 getFloat(const char* idx, const char* key);
    /*!
     
     returns the user binary data entry with this key + the size
     @param idx textual index for the maps
     @param key
     @param size
     @return user binary data
     */
	const char* getData(const char* idx, const char* key, uint32& size);
    
    /*!
     
     returns a copy of the user binary data entry with this key + the size
     @param idx textual index for the maps
     @param key
     @param size
     @return copy of the user binary data
     */
	char* getDataCopy(const char* idx, const char* key, uint32& size);
    /*!
     
     returns a copy of the user message entry with this key
     @param idx textual index for the maps
     @param key
     @return copy of the user message entry
     */
	DataMessage* getAttachedMessageCopy(const char* idx, const char* key);
    /*!
     
     sets the user time entry with this key to the value provided
     @param idx textual index for the maps
     @param key
     @param value the value of the time entry
     @return true if successful
     */
	bool setTime(const char* idx, const char* key, uint64 value);
    /*!
     
     sets the user string entry with this key to the value provided
     @param idx textual index for the maps
     @param key
     @param value the new value of the string entry
     @return true if successful
     */
	bool setString(const char* idx, const char* key, const char* value);
    /*!
     
     sets the user integer entry with this key to the value provided
     @param idx textual index for the maps
     @param key
     @param value the new value of the integer entry
     @return true if successful
     */
	bool setInt(const char* idx, const char* key, int64 value);
    /*!
     
     sets the user double entry with this key to the value provided
     @param idx textual index for the maps
     @param key
     @param value the new value of the double entry
     @return true if successful
     
     */
	bool setDouble(const char* idx, const char* key, double value);
    /*!
     
     sets the user float entry with this key to the value provided
     @param idx textual index for the maps
     @param key
     @param value the new value of the float entry
     @return true if successful
     
     */
	bool setFloat(const char* idx, const char* key, float64 value);
    /*!
     
     sets the user binary data entry with this key + the size
     @param idx textual index for the maps
     @param key
     @param value the new value for the data entry
     @param size
     @return true if successful
     */
    bool setData(const char* idx, const char* key, const char* value, uint32 size);
    /*!
     
     sets  the user message entry with this key to the provided message
     @param idx textual index for the maps
     @param key
     @param msg the new message
     @return true if successful
     */
	bool setAttachedMessage(const char* idx, const char* key, DataMessage* msg);

    /*!
     
     returns the user time entry with this key
     @param key
     @return user time
     */
	std::map<std::string, uint64> getTimeMap(const char* key);
    /*!
     
     returns the user entry as a string regardless of what type it is
     @param key
     @return  user entry as a string
     
     */
	std::map<std::string, std::string> getAsStringMap(const char* key);
	/*!

	returns the user entry as an integer map regardless of what type it is
	@param key
	@return  user entry as a string

	*/
	std::map<std::string, int64> getAsIntMap(const char* key);
	/*!

	returns the user entry as a float map regardless of what type it is
	@param key
	@return  user entry as a string

	*/
	std::map<std::string, float64> getAsFloatMap(const char* key);
	/*!
     
     returns the user string entry with this key
     @param key
     @return user string entry
     */
	std::map<std::string, std::string> getStringMap(const char* key);
    /*!
     
     returns the user integer entry with this key
     @param key
     @return user integer entry
     */
	std::map<std::string, int64> getIntMap(const char* key);
    /*!
     
     returns the user double entry with this key
     @param key
     @return user double entry
     
     */
	std::map<std::string, double> getDoubleMap(const char* key);
    /*!
     returns the user float entry with this key
     @param key
     @return user float entry
     
     */
	std::map<std::string, float64> getFloatMap(const char* key);
    /*!
     
     returns a copy of the user message entry with this key
     @param key
     @return copy of the user message entry
     */
	std::map<std::string, DataMessage*> getAttachedMessageMap(const char* key);
    /*!
     
     sets the user time entry with this key
     @param key
     @param map 
     @return true if successful
     */
	bool setTimeMap(const char* key, std::map<std::string, uint64>& map);
    /*!
     
     sets the user string entry with this key 
     @param key
     @param map
     @return true if successful
     */
	bool setStringMap(const char* key, std::map<std::string, std::string>& map);
    /*!
     
     sets the user integer entry with this key 
     @param key
     @param map
     @return true if successful
     */
	bool setIntMap(const char* key, std::map<std::string, int64>& map);
    /*!
     
     sets the user double entry with this key to the value provided
     @param key entry key
     @param map map of values to store
     @return true if successful
     
     */
	bool setDoubleMap(const char* key, std::map<std::string, double>& map);
    /*!
     
     sets the user float entry with this key to the value provided
     @param key entry key
     @param map map of values to store
     @return true if successful
     
     */
	bool setFloatMap(const char* key, std::map<std::string, float64>& map);
    /*!
     
     sets  the user message entry with this key to the provided message
     @param key
     @param map
     @return true if successful
     */
	bool setAttachedMessageMap(const char* key, std::map<std::string, DataMessage*>& map);

    /*! toJSON()
     
        Returns the JSON representation of the full message including all user entries, arrays and maps - and including the JSON representation of any attached messages
     
     */
    
	
	std::string toJSON(std::map<uint16, std::string>* subtypes = NULL, std::map<uint16, std::string>* subcontexts = NULL, std::map<uint32, std::string>* compNames = NULL);
	
    /*! toXML()
     
        Returns the XML representation of the full message including all user entries, arrays and maps - and including the XML representation of any attached messages
     */
    
	std::string toXML(std::map<uint16, std::string>* subtypes = NULL, std::map<uint16, std::string>* subcontexts = NULL, std::map<uint32, std::string>* compNames = NULL);

	/*! toCSV()
	@param separator the separator character, defaults to comma
	@param preample string with entries to add before the standard list
		&lt;preample&gt;,type,created,sent,received,...
	@param subtypes optional map of subtype id to name used for textual output
	@param subcontexts optional map of subcontext id to name used for textual output
	@param compNames optional map of component id to name used for textual output
	Returns the CSV representation of the full message including a textual description of all user entries
	*/

	std::string toCSV(const char* separator = NULL, const char* preample = NULL, std::map<uint16, std::string>* subtypes = NULL, std::map<uint16, std::string>* subcontexts = NULL, std::map<uint32, std::string>* compNames = NULL);

	/*! toCSVHeader()
	@param separator the separator character, defaults to comma
	@param preample string with entries to add before the standard list
		&lt;preample&gt;,type,created,sent,received,...
	Returns the static CSV header line for the output of toCSV()
	*/

	static std::string GetCSVHeader(const char* separator = NULL, const char* preample = NULL);


    /*! copyUserEntriesFromMessage(DataMessage* msg)
     Copies all user data from a message into this message, overwriting any existing entries with the same key
	 @param msg the Message to copy the user data from
	 @return the number of data entries copied
	 */
	uint32 copyUserEntriesFromMessage(DataMessage* msg);

	/*! getUserEntriesAsString()
	Returns the textual representation of the full message including all user entries, arrays and maps - and including the textual representation of any attached messages
	*/
	std::string getUserEntriesAsString();

	/*! getUserEntriesAsJSON(bool asText)
	Returns the JSON representation of the full message including all user entries, arrays and maps - and including the textual representation of any attached messages
	@param asText if true write all entries as human readable text
	@return JSON string
	*/
	std::string getUserEntriesAsJSON(bool asText);


	// API - inline

	/*! getRawData()
		Retrieve a pointer to the raw binary block of the message (header + entries).
		The block is data->size bytes long and remains owned by the message; it can
		be memcpy'd elsewhere to clone or transmit the message verbatim.
		@return NULL if the message is not valid, otherwise pointer to the block
	 */
	char* getRawData() {
		if (data->cid != DATAMESSAGEID) return NULL;
		return (char*)data;
	}

	/*! getOrigin()
		Get the NodeID of the node the message originated on
		@return 0 if the message is not valid, otherwise the origin NodeID
	 */

	uint16 getOrigin() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->origin;
	}

	/*! getDestination()
		Get the NodeID of the destination node
		@return 0 if the message is not valid, otherwise the destination NodeID
	 */
	uint16 getDestination() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->destination;
	}

	/*! setOrigin(uint16 addr)
		Set origin to address addr and return new origin
		@param addr define origin
		@return true if successful, false otherwise
	 */
	bool setOrigin(uint16 addr) {
		if (data->cid != DATAMESSAGEID) return false;
		data->origin = addr;
		return true;
	}

	/*! setDestination(uint16 addr)
		Set destination to address addr and return new destination
		@param addr define destination
		@return true if successful, false otherwise
	 */
	bool setDestination(uint16 addr) {
		if (data->cid != DATAMESSAGEID) return false;
		data->destination = addr;
		return true;
	}

	/*! getSize()
		Get message size
		Many types of data of any size can be put into a message as user entries; the data will be kept in a single block of memory for efficiency
		@return 0 if the message is not valid
		@return size else
	 */
	uint32 getSize() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->size;
	}

	/*! setMultiple(uint32 from, uint32 to, uint32 tag, uint64 ttl, uint8 policy, uint64 sendtime)
		Set multiple information for a message at the same time
		@param from the sender id
		@param to the receiver id
		@param tag the message's tag id
		@param ttl the message's time-to-live in microseconds
		@param policy the policy id
		@param sendtime the timestamp of the sending of the message
		@return true if successful, false otherwise
	 */
	bool setMultiple(uint32 from, uint32 to, uint32 tag, uint64 ttl, uint8 policy, uint64 sendtime) {
		if (data->cid != DATAMESSAGEID) return false;
		if (from) data->from = from;
		if (to) data->to = to;
		if (tag) data->tag = tag;
		if (ttl) data->ttl = ttl;
		if (policy) data->policy = policy;
		if (sendtime) data->sendtime = sendtime;
		return true;
	}

	/*! getTag()
		Retrieve and return message tag id
		Tags can be used to segment data temporally, i.e. a module may only be interested in camera data referencing a specific object or event.
		@return 0 if the message is not valid
		@return tag returns message tag
	 */
	uint32 getTag() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->tag;
	}

	/*! setTag(uint32 tag)

		Sets message tag to tag
		Tags can be used to segment data temporally, i.e. a module may only be interested in camera data referencing a specific object or event.

		@param tag defines message tag

		@return true if successful, false otherwise

	 */
	bool setTag(uint32 tag) {
		if (data->cid != DATAMESSAGEID) return false;
		data->tag = tag;
		return true;
	}

	/*! getReference()
		Get and return message reference id
		@return reference
	 */
	uint64 getReference() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->reference;
	}

	/*! setReference(uint64 ref)
		Set message reference
		@param ref reference
		@return true if successful, false otherwise
	 */
	bool setReference(uint64 ref) {
		if (data->cid != DATAMESSAGEID) return false;
		data->reference = ref;
		return true;
	}


	/*! getSystemID()
	Get and return message system id
	@return id
	*/
	uint32 getSystemID() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->systemid;
	}

	/*! setSystemID(uint32 id)
	Set message system id
	@param id
	@return true if successful, false otherwise
	*/
	bool setSystemID(uint32 id) {
		if (data->cid != DATAMESSAGEID) return false;
		data->systemid = id;
		return true;
	}

	/*! getMemID()
		Get and return memory ID
		@return memid memory ID
	 */
	uint64 getMemID() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->memid;
	}

	/*! setMemID(uint64 id)
		Set memory ID to id
		@param id memory id
		@return true if successful, false otherwise

	 */
	bool setMemID(uint64 id) {
		if (data->cid != DATAMESSAGEID) return false;
		data->memid = id;
		return true;
	}

	/*! getFrom()
		Get the sender id
		@return from
	 */
	uint32 getFrom() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->from;
	}

	/*! setFrom(uint32 from)

		Set the sender id

		@param from

		@return false if message is not valid


	 */

	bool setFrom(uint32 from) {
		if (data->cid != DATAMESSAGEID) return false;
		data->from = from;
		return true;
	}

	/*! getTo()

		Set the receiver id

		@return to

	 */

	uint32 getTo() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->to;
	}

	/*! setTo(uint32 to)

		Get the receiver id

		@param to

		@return

	 */

	bool setTo(uint32 to) {
		if (data->cid != DATAMESSAGEID) return false;
		data->to = to;
		return true;
	}

	/*! getType()

		Get message type

		@return NOTYPE if message is not valid or if type is not set
		@return type else
	 */

	PsyType getType() {
		PsyType t = NOTYPE;
		if (data->cid != DATAMESSAGEID) return t;
		return data->type;
	}

	/*! setType(PsyType &type)

		Set message type to type

		@param type the message type
		@return false message is not valid
		@return true if success

	 */

	bool setType(PsyType &type) {
		if (data->cid != DATAMESSAGEID) return false;
		data->type = type;
		return true;
	}

	/*! getContextChange()

		Get and return context change entry

		@return NOCONTEXT is message is not valid or if context change is not set

		@return contextchange else

	 */

	PsyContext getContextChange() {
		PsyContext c = NOCONTEXT;
		if (data->cid != DATAMESSAGEID) return c;
		return data->contextchange;
	}

	/*! setContextChange(PsyContext& context)


		Set and return context change entry

		@param context to change to

		@return false if message is not valid

		@return true else

	 */

	bool setContextChange(PsyContext& context) {
		if (data->cid != DATAMESSAGEID) return false;
		data->contextchange = context;
		return true;
	}

	/*! getTTL()

		Gets message's "time to live"
		"TTL is a mechanism that limits the lifespan or lifetime of data in a computer or network"

		Messages in Psyclone 2.0 live for a finite amount of time inside the shared memory,
		If ttl = 0 the message will only be delivered to whoever has subscribed to it and after this seize to exist.

		@return 0 if the message is not valid or the ttl is not set

		@return ttl else

	 */

	uint64 getTTL() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->ttl;
	}

	/*! getEOL()

		Get End of life of message

		@return 0 if message is not valid or the ttl is not set

		@return time that the message will expire

	 */

	uint64 getEOL() {
		if (data->cid != DATAMESSAGEID) return 0;
		if (!data->ttl)
			return 0;
		return data->time + data->ttl;
	}

	/*! setTTL(uint64 ttl)

		Sets time to live for a message in microseconds
		Messages in Psyclone 2.0 live for a finite amount of time inside the shared memory,
		If ttl = 0 the message will only be delivered to whoever has subscribed to it and after this seize to exist.


		@param ttl time-to-live in microseconds

		@return false if message is not valid
		@return true else


	 */

	bool setTTL(uint64 ttl) {
		if (data->cid != DATAMESSAGEID) return false;
		data->ttl = ttl;
		return true;
	}

	/*! setEOL(uint64 eol)

		Calculates and sets the equivalent ttl for the message
		
		@param eol the time at which the message is no longer supposed to exist

		@return false if the message is not valid
		@return true


	 */

	bool setEOL(uint64 eol) {
		if (data->cid != DATAMESSAGEID) return false;
		int64 ttl = eol - data->time;
		if (ttl > 0) {
			data->ttl = (uint64)ttl;
			return true;
		}
		else
			return false;
	}

	/*! getPriority()

		Get message priority
		Messages have a prority that determines their relative order of scheduling, both in the whiteboards and in the modules.

		@return 0 if the message is not valid
		@return priority else

	 */

	uint16 getPriority() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->priority;
	}

	/*! setPriority(uint16 priority)

		Set message priority to priority
		Messages have a prority that determines their relative order of scheduling



		@param priority

		@return false if the message is not valid
		@return true else

	 */

	bool setPriority(uint16 priority) {
		if (data->cid != DATAMESSAGEID) return false;
		data->priority = priority;
		return true;
	}

	/*! getPolicy()

		Gets the policy id of the message

		@return 0 if the message is not valid or if no policy is set
		@return policy id else


	 */

	uint8 getPolicy() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->policy;
	}

	/*! setPolicy(uint8 policy)

		Sets the policy id of the message

		@param policy

		@return false if the message is not valid
		@return true else

	 */

	bool setPolicy(uint8 policy) {
		if (data->cid != DATAMESSAGEID) return false;
		data->policy = policy;
		return true;
	}

	/*! getSendTime()

		Get time message was sent

		@return 0 if the message is not valid or the time was not set
		@return sendtime else

	 */

	uint64 getSendTime() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->sendtime;
	}

	/*! setSendTime(uint64 time)

		Set message sendtime to time

		@param time Sendtime

		@return false if the message is not valid
		@return true else

	 */

	bool setSendTime(uint64 time) {
		if (data->cid != DATAMESSAGEID) return false;
		data->sendtime = time;
		return true;
	}

	/*! getRecvTime()

		Get and return time message was received

		@return 0 if the message is not valid or the time was not set
		@return recvtime else

	 */

	uint64 getRecvTime() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->recvtime;
	}

	/*! setRecvTime(uint64 time)

		Set new receiving time time

		@param time receiving time

		@return false if the message is not valid
		@return true else

	 */

	bool setRecvTime(uint64 time) {
		if (data->cid != DATAMESSAGEID) return false;
		data->recvtime = time;
		return true;
	}

	/*! getCreatedTime()

		Get and return time message was created

		@return false if the message is not valid
		@return time else


	 */

	uint64 getCreatedTime() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->time;
	}

	/*! setCreatedTime(uint64 t)

		Set time message was created to t

		@param t time message was created

		@return 0 if the message is not valid
		@return time=t else

	 */

	bool setCreatedTime(uint64 t) {
		if (data->cid != DATAMESSAGEID) return 0;
		data->time = t;
		return true;
	}

	/*! getStatus()

		Get current message status

		@return 0 if the message is not valid
		@return status else

	 */

	uint16 getStatus() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->status;
	}

	/*! setStatus(uint16 status)

		Set message status

		@param status message status

		@return false if the message is not valid
		@return true else

	 */

	bool setStatus(uint16 status) {
		if (data->cid != DATAMESSAGEID) return false;
		data->status = status;
		return true;
	}

	/*! getSerial()

		Get message serial number

		@return 0 if the message is not valid
		@return serial else
	 */

	uint64 getSerial() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->serial;
	}

	/*! setSerial(uint64 serial)

		Set message serial number

		@param serial

		@return false if the message is not valid
		@return true else

	 */

	bool setSerial(uint64 serial) {
		if (data->cid != DATAMESSAGEID) return false;
		data->serial = serial;
		return true;
	}









	/*! addTimeUsage()

	Update the CPU and Wall time usage since trigger message arrived and add up the chain usage

	@param cycleCPUTime cycle CPU time in us
	@param cycleWallTime cycle wall time in us
	@param chainCPUTime chain CPU time in us
	@param chainWallTime chain wall time in us
	@param chainCount number of chain entries to add

	@return false if the message is not valid
	@return true otherwise

	*/

	bool addTimeUsage(uint32 cycleCPUTime, uint32 cycleWallTime, uint32 chainCPUTime = 0, uint32 chainWallTime = 0, uint32 chainCount = 0);


	/*! getCycleCPUTime()

	Get cpu time in us it took to process the trigger message that caused this message to be posted

	@return 0 if the message is not valid
	@return cpu time in us

	*/

	uint16 getCycleCPUTime() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->cyclecputime;
	}

	/*! setCycleCPUTime(uint32 t)

	Set cpu time in us it took to process the trigger message that caused this message to be posted

	@param t time in us

	@return false if the message is not valid
	@return true otherwise

	*/

	bool setCycleCPUTime(uint32 t) {
		if (data->cid != DATAMESSAGEID) return false;
		data->cyclecputime = t;
		return true;
	}




	/*! getCycleWallTime()

	Get wall time in us it took to process the trigger message that caused this message to be posted

	@return 0 if the message is not valid
	@return cpu time in us

	*/

	uint16 getCycleWallTime() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->cyclewalltime;
	}

	/*! setCycleWallTime(uint32 t)

	Set wall time in us it took to process the trigger message that caused this message to be posted

	@param t time in us

	@return false if the message is not valid
	@return true otherwise

	*/

	bool setCycleWallTime(uint32 t) {
		if (data->cid != DATAMESSAGEID) return false;
		data->cyclewalltime = t;
		return true;
	}





	/*! getChainCPUTime()

	Get cpu time in us it took to process the trigger message that caused this message to be posted

	@return 0 if the message is not valid
	@return cpu time in us

	*/

	uint16 getChainCPUTime() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->chaincputime;
	}

	/*! setChainCPUTime(uint32 t)

	Set cpu time in us it took to process the trigger message that caused this message to be posted

	@param t time in us

	@return false if the message is not valid
	@return true otherwise

	*/

	bool setChainCPUTime(uint32 t) {
		if (data->cid != DATAMESSAGEID) return false;
		data->chaincputime = t;
		return true;
	}




	/*! getChainWallTime()

	Get wall time in us it took to process the trigger message that caused this message to be posted

	@return 0 if the message is not valid
	@return cpu time in us

	*/

	uint16 getChainWallTime() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->chainwalltime;
	}

	/*! setChainWallTime(uint32 t)

	Set wall time in us it took to process the trigger message that caused this message to be posted

	@param t time in us

	@return false if the message is not valid
	@return true otherwise

	*/

	bool setChainWallTime(uint32 t) {
		if (data->cid != DATAMESSAGEID) return false;
		data->chainwalltime = t;
		return true;
	}



	/*! getChainCount()

	Get number of messages it took to process the trigger message that caused this message to be posted

	@return 0 if the message is not valid
	@return count

	*/

	uint32 getChainCount() {
		if (data->cid != DATAMESSAGEID) return 0;
		return data->chaincount;
	}

	/*! setChainCount(uint32 count)

	Set number of messages it took to process the trigger message that caused this message to be posted

	@param count number of messages

	@return false if the message is not valid
	@return true otherwise

	*/

	bool setChainCount(uint32 count) {
		if (data->cid != DATAMESSAGEID) return false;
		data->chaincount= count;
		return true;
	}

	/*! deleteEntry(const char* key)

	Marks the entry as unused

	@param key keyword

	@return true if successfull

	*/

	bool deleteEntry(const char* key) {
		if (data->cid != DATAMESSAGEID) return false;
		return removeEntry(key);
	}

















	/*! Pointer to the message's flat memory block (header + user entries).
	This is the actual message: copying these data->size bytes reproduces the
	message exactly. May be reallocated whenever entries are added or resized,
	invalidating any pointers previously returned by getString()/getData(). */
	DataMessageHeader* data;

	mutable std::string _lastFindKey;			//!< Internal: last key looked up (single-entry lookup cache; mutable so const-style getters can update it).
	mutable char* _lastFindKeyResult = NULL;	//!< Internal: cached location of the last key's entry inside the block; NULL if not found.

	/*! Internal: clears the single-entry key-lookup cache. Must be called after
	any operation that moves or invalidates entries (reallocation, removal). */
	void invalidateFindKeyCache() {
		_lastFindKey.clear();
		_lastFindKeyResult = NULL;
	}

protected:
	/*! Internal: low-level entry writer used by all typed set...() overloads.
	@param key entry name
	@param value payload bytes to copy in
	@param size payload size in bytes
	@param datatype content type id for the entry (see ObjectIDs.h)
	@return true if successful */
	bool setRawData(const char* key, const char* value, uint32 size, uint32 datatype);

	/*! findKey(const char* key)
		Internal function: linear scan of the entry list for an entry whose key
		matches @p key (case-insensitive), with a one-slot cache of the last
		lookup to make repeated access to the same key O(1).
		@param key entry name to find
		@return pointer to the start of the entry (its DataMessageEntryHeader),
		or NULL if not found or the message is invalid
	*/
	char* findKey(const char* key) {
		if (data->cid != DATAMESSAGEID) return NULL;
		if (!_lastFindKey.empty() && stricmp(key, _lastFindKey.c_str()) == 0)
			return _lastFindKeyResult;
		char* src = (char*)data + sizeof(DataMessageHeader);
		char* srcEnd = (char*)data + data->size;
		while (src < srcEnd) {
			if (((DataMessageEntryHeader*)src)->cid == CONSTCHARID) {
				if (stricmp(key, src+sizeof(DataMessageEntryHeader)) == 0) {
					_lastFindKey = key;
					_lastFindKeyResult = src;
					return src;
				}
			}
			src += ((DataMessageEntryHeader*)src)->size;
		}
		_lastFindKey = key;
		_lastFindKeyResult = NULL;
		return NULL;
	}

	/*! findEntry(const char* key)

		Internal function: like findKey() but returns a pointer to the entry's
		value payload (just past the entry header and the key string).

		@param key entry name to find
		@return pointer to the payload, or NULL if the key does not exist
	 */

	char* findEntry(const char* key) {
		char* src = findKey(key);
		if (src == NULL) return NULL;
		return src + sizeof(DataMessageEntryHeader) + strlen(key) + 1;
	}

	/*! removeEntry(const char* key)

		Internal function: marks the entry as unused (cid = UNUSEDID) and updates
		userCount/userSize. The slot is not compacted — it remains in the block
		and may be reused by a later entry of exactly the same size (findSpace()).

		@param key entry name
		@return true (also when the key did not exist)
	 */

	bool removeEntry(const char* key) {
		char* src = findKey(key);
		if (src == NULL) return true;
		((DataMessageEntryHeader*)src)->cid = UNUSEDID;
		data->userCount--;
		data->userSize -= ((DataMessageEntryHeader*)src)->size;
		invalidateFindKeyCache();
		return true;
	}

	/*! findSpace(uint32 space)

		Internal function: scans for a previously deleted (UNUSEDID) entry slot
		of EXACTLY @p space bytes that can be reused for a new entry, avoiding a
		reallocation of the whole block.
		@note Only exact-size matches are reused; allowing smaller reuse would
		require slot splitting, which is not implemented (see code comment).

		@param space required slot size in bytes (header + key + payload)
		@return pointer to a reusable slot, or NULL if none exists or the message is invalid
	 */

	char* findSpace(uint32 space) {
		if (data->cid != DATAMESSAGEID) return NULL;
		char* src = (char*)data + sizeof(DataMessageHeader);
		char* srcEnd = (char*)data + data->size;
		while (src < srcEnd) {
			if ( (((DataMessageEntryHeader*)src)->cid == UNUSEDID) &&
				(((DataMessageEntryHeader*)src)->size == space)) {
				// Used to allow anything smaller, but that needs fixing permanently first
				// (((DataMessageEntryHeader*)src)->size >= space) ) {
					return src;
			}
			src += ((DataMessageEntryHeader*)src)->size;
		}
		return NULL;
	}

};

} // namespace cmlabs

#endif //_DATAMESSAGE_H_

