

//_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/
//_/_/
//_/_/  This file is part of CMSDK version 2.1.1.260726 (26 July 2026)
//_/_/  
//_/_/  developed by
//_/_/  
//_/_/  Communicative Machines Ltd. 
//_/_/    http://cmlabs.com/
//_/_/  Copyright (c) 2026
//_/_/  
//_/_/  in collaboration with 
//_/_/  
//_/_/  Icelandic Institute for Intelligent Machines ses
//_/_/    Menntavegur 1
//_/_/    101 Reykjavik
//_/_/    ICELAND
//_/_/    http://www.iiim.is/
//_/_/
//_/_/  and is released under the following
//_/_/  
//_/_/  EXTENDED BSD License
//_/_/
//_/_/  Redistribution and use in source and binary forms, with or without
//_/_/  modification, is permitted provided that the following conditions 
//_/_/  are met:
//_/_/
//_/_/  - Redistributions of source code must retain the above copyright 
//_/_/    and collaboration notice, this list of conditions and the following 
//_/_/    disclaimer.
//_/_/
//_/_/  - Redistributions in binary form must reproduce the above copyright 
//_/_/    notice, this list of conditions and the following disclaimer in the 
//_/_/    documentation and/or other materials provided with the distribution.
//_/_/
//_/_/  - Neither the name of its copyright holders nor the names of its 
//_/_/    contributors may be used to endorse or promote products derived from 
//_/_/    this software without specific prior written permission.
//_/_/
//_/_/  - CADIA Clause: The license granted in and to the software under this 
//_/_/    agreement is a limited-use license. The software may not be used in 
//_/_/    furtherance of: (i) intentionally causing bodily injury or severe 
//_/_/    emotional harm to any person; (ii) invading the personal privacy or 
//_/_/    violating the human rights of any person; or (iii) committing or 
//_/_/    preparing for any act of war. 
//_/_/
//_/_/  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
//_/_/  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
//_/_/  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 
//_/_/  PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER 
//_/_/  OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
//_/_/  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
//_/_/  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
//_/_/  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 
//_/_/  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 
//_/_/  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
//_/_/  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//_/_/
//_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/
/*! @file UnitTestFramework.cpp
    @brief Implementation of the CMSDK unit test harness (UnitTestRunner and
           the unittest:: reporting API). See UnitTestFramework.h for the
           design overview: registration, per-test process forking, progress
           rendering, perf JSON output and compare-mode deltas.
    @ingroup UnitTest */
// ============================================================================

#include "UnitTestFramework.h"
#include "PsyTime.h"
#include "jsmn.h"

#include <stdio.h>
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>

#ifdef WINDOWS
	#include <io.h>
	#define ISATTY(fd) _isatty(fd)
	#define FILENO(f)  _fileno(f)
#else
	#include <unistd.h>
	#include <dirent.h>
	#include <sys/wait.h>
	#include <signal.h>
	#include <execinfo.h>
	#include <fcntl.h>
	#define ISATTY(fd) isatty(fd)
	#define FILENO(f)  fileno(f)
	#define UT_HAVE_FORK 1
#endif

// Hard per-test wall-clock cap when running each test in its own process. A
// test that exceeds this is killed and reported as a timeout (rather than
// hanging the whole suite).
#define UT_TEST_TIMEOUT_US (30ULL * 1000000ULL)

namespace cmlabs {

// Best-effort removal of stale CMSDK shared-memory segments left behind by a
// previous test process that was killed (kill -9) mid-operation. Such a process
// can leave a PROCESS_SHARED pthread mutex locked inside a persisted segment,
// which would deadlock the next run that re-opens it. Tests run under system id
// 0, so clearing these test artifacts before a run makes the suite self-healing
// and safely repeatable (e.g. for an agent looping to chase a perf target).
// This assumes no live Psyclone node is using system id 0 on this machine while
// tests run (which is already required by the "another node running" guard).
static void ut_clearSegmentsIn(const char* dir, const char* prefix) {
#ifndef WINDOWS
	DIR* d = opendir(dir);
	if (!d) return;
	size_t plen = strlen(prefix);
	struct dirent* e;
	while ((e = readdir(d)) != NULL) {
		if (strncmp(e->d_name, prefix, plen) == 0) {
			std::string path = std::string(dir) + "/" + e->d_name;
			unlink(path.c_str());
		}
	}
	closedir(d);
#endif
}

static void ClearStaleTestSegments() {
#ifndef WINDOWS
	// macOS file-backed segments: /tmp/Psyclone_Shmem_<name>
	ut_clearSegmentsIn("/tmp", "Psyclone_Shmem_");
	// Linux / Cygwin POSIX shm: shm_open("/Shmem_<name>") -> /dev/shm/Shmem_<name>
	ut_clearSegmentsIn("/dev/shm", "Shmem_");
#endif
}

#ifdef UT_HAVE_FORK
// ---- Timeout diagnostics for forked tests --------------------------------
// When a test hangs, the parent's only signal is the wall-clock timeout, and
// the child's buffered output is lost when it is killed. To make a hang
// diagnosable we (a) have the child record its current phase to a file on every
// progress() call (read back by the parent on timeout) and (b) on timeout ask
// the child, via SIGUSR1, to dump a native stack trace of the stuck thread to a
// file before we kill it. backtrace_symbols_fd() is async-signal-safe, so the
// handler is safe to run from a signal.
static int ut_traceFd = -1;
static void ut_timeoutTraceHandler(int) {
	if (ut_traceFd >= 0) {
		void* frames[128];
		int n = backtrace(frames, 128);
		backtrace_symbols_fd(frames, n, ut_traceFd);
	}
	_exit(3);
}
static void ut_installTimeoutTraceHandler(const std::string& tracePath) {
	ut_traceFd = open(tracePath.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
	signal(SIGUSR1, ut_timeoutTraceHandler);
}

// Read back the child's last-recorded phase ("percent\taction") as a short
// human-readable note for the status line.
static std::string ut_readProgressNote(const std::string& path) {
	FILE* f = fopen(path.c_str(), "r");
	if (!f) return "";
	char buf[512];
	size_t r = fread(buf, 1, sizeof(buf) - 1, f);
	fclose(f);
	buf[r] = 0;
	std::string s(buf);
	std::string::size_type tab = s.find('\t');
	if (tab == std::string::npos) return s;
	return s.substr(0, tab) + "% \"" + s.substr(tab + 1) + "\"";
}

// Print the captured stack trace (if any) as indented diagnostic lines.
static void ut_printTraceFile(const std::string& path) {
	FILE* f = fopen(path.c_str(), "r");
	if (!f) return;
	char line[1024];
	bool header = false;
	while (fgets(line, sizeof(line), f)) {
		if (!header) { printf("           stack trace where the test hung:\n"); header = true; }
		printf("             %s", line);
	}
	if (header) { printf("\n"); fflush(stdout); }
	fclose(f);
}
#endif

// ----------------------------------------------------------------------------
//  Layout constants for consistent output
// ----------------------------------------------------------------------------
#define UT_NAME_WIDTH    28   // width of the test-name column on the status line
#define UT_INDENT        "           "  // 11 spaces, lines up after "[ PASS ] "
#define UT_METRIC_WIDTH  26   // width of the metric-name column on a subline

// ----------------------------------------------------------------------------
//  Singleton
// ----------------------------------------------------------------------------
UnitTestRunner& UnitTestRunner::instance() {
	static UnitTestRunner theInstance;
	return theInstance;
}

UnitTestRunner::UnitTestRunner()
	: current(NULL), verboseFlag(false), writeJSONFlag(true),
	  forkPerTest(true), isTTY(false), lastProgressLen(0), outputDir("."),
	  compareLoaded(false) {
	isTTY = ISATTY(FILENO(stdout)) ? true : false;
}

// ----------------------------------------------------------------------------
//  Registration & configuration
// ----------------------------------------------------------------------------
void UnitTestRunner::registerTest(const char* name, UnitTestFunc func,
                                  const char* description, const char* category,
                                  bool inDefaultRun) {
	if (!name || !func) return;
	// Replace if a test with this name was already registered.
	for (size_t i = 0; i < tests.size(); i++) {
		if (tests[i].name == name) {
			tests[i].func = func;
			tests[i].description = description ? description : "";
			tests[i].category = category ? category : "";
			tests[i].inDefaultRun = inDefaultRun;
			return;
		}
	}
	UnitTestRecord rec;
	rec.name = name;
	rec.func = func;
	rec.description = description ? description : "";
	rec.category = category ? category : "";
	rec.inDefaultRun = inDefaultRun;
	tests.push_back(rec);
}

void UnitTestRunner::setVerbose(bool v)             { verboseFlag = v; }
void UnitTestRunner::setCompareFile(const char* p)  { if (p) compareFile = p; }
void UnitTestRunner::setOutputFile(const char* p)   { if (p) outputFile = p; }
void UnitTestRunner::setOutputDir(const char* d)    { if (d) outputDir = d; }
void UnitTestRunner::setWriteJSON(bool v)           { writeJSONFlag = v; }
void UnitTestRunner::setForkPerTest(bool v)         { forkPerTest = v; }

bool UnitTestRunner::hasTest(const char* name) const {
	if (!name) return false;
	for (size_t i = 0; i < tests.size(); i++)
		if (stricmp(tests[i].name.c_str(), name) == 0) return true;
	return false;
}

std::vector<std::string> UnitTestRunner::testNames() const {
	std::vector<std::string> names;
	for (size_t i = 0; i < tests.size(); i++) names.push_back(tests[i].name);
	return names;
}

void UnitTestRunner::listTests() const {
	printf("\nAvailable CMSDK unit tests (%u):\n\n", (uint32)tests.size());
	std::string lastCat = "\x01";
	for (size_t i = 0; i < tests.size(); i++) {
		if (tests[i].category != lastCat) {
			lastCat = tests[i].category;
			printf("  [%s]\n", lastCat.empty() ? "general" : lastCat.c_str());
		}
		printf("    %-22s %s\n", tests[i].name.c_str(), tests[i].description.c_str());
	}
	printf("\nRun all with: test=cmsdk    Run one with: test=<name>\n\n");
}

// ----------------------------------------------------------------------------
//  Hooks used by the unittest:: API for the currently running test
// ----------------------------------------------------------------------------
bool UnitTestRunner::hookVerbose() const { return verboseFlag; }

void UnitTestRunner::hookProgress(int percent, const char* action) {
	if (!current) return;
	if (percent < 0)   percent = 0;
	if (percent > 100) percent = 100;

	// Record the current phase to a file (forked child only) so that, if the test
	// hangs and is killed by the timeout, the parent can still report which phase
	// it was in. progress() is called only a handful of times per test, so this is
	// cheap, and it happens regardless of whether stdout is a terminal.
	if (!progressPath.empty()) {
		FILE* pf = fopen(progressPath.c_str(), "w");
		if (pf) { fprintf(pf, "%d\t%s", percent, action ? action : ""); fclose(pf); }
	}

	if (!isTTY) return;            // keep piped/redirected logs clean

	char buf[512];
	int n = snprintf(buf, sizeof(buf), "  [%3d%%] %-*.*s %s",
	                 percent, UT_NAME_WIDTH, UT_NAME_WIDTH,
	                 current->name.c_str(), action ? action : "");
	if (n < 0) return;

	// Pad to clear any longer text drawn previously.
	int pad = lastProgressLen - n;
	fputc('\r', stdout);
	fputs(buf, stdout);
	for (int i = 0; i < pad; i++) fputc(' ', stdout);
	fputc('\r', stdout);
	fflush(stdout);
	lastProgressLen = (n > lastProgressLen) ? n : lastProgressLen;
	// (we keep the max so the next pad clears the longest line)
	lastProgressLen = n;
}

void UnitTestRunner::clearProgressLine() {
	if (!isTTY || lastProgressLen <= 0) return;
	fputc('\r', stdout);
	for (int i = 0; i < lastProgressLen; i++) fputc(' ', stdout);
	fputc('\r', stdout);
	fflush(stdout);
	lastProgressLen = 0;
}

void UnitTestRunner::hookMetric(const char* name, double value, const char* unit, bool higherIsBetter) {
	if (!current || !name) return;
	lock.enter();
	UnitTestMetric m;
	m.name = name;
	m.value = value;
	m.unit = unit ? unit : "";
	m.higherIsBetter = higherIsBetter;
	current->metrics.push_back(m);
	lock.leave();
}

void UnitTestRunner::hookDetail(const char* text) {
	if (!verboseFlag || !current) return;
	clearProgressLine();
	printf(UT_INDENT "%s\n", text ? text : "");
	fflush(stdout);
}

void UnitTestRunner::hookFail(const char* text) {
	if (!current) return;
	std::string reason = text ? text : "";
	// Trim trailing whitespace/newlines so the reason renders on one line.
	while (!reason.empty() &&
	       (reason[reason.size() - 1] == '\n' || reason[reason.size() - 1] == '\r' ||
	        reason[reason.size() - 1] == ' '  || reason[reason.size() - 1] == '\t'))
		reason.erase(reason.size() - 1);
	current->failReason = reason;

	// Print the reason RIGHT NOW (always, regardless of verbose). The status
	// line is printed later by the parent process, and if the test goes on to
	// hang or crash before then, the per-test timeout/crash note would be all
	// that survives - so emit the reason immediately while we still can.
	if (!reason.empty()) {
		clearProgressLine();
		printf(UT_INDENT "reason: %s\n", reason.c_str());
		fflush(stdout);
	}
}

// ----------------------------------------------------------------------------
//  Output of results
// ----------------------------------------------------------------------------
void UnitTestRunner::printStatusLine(const UnitTestRecord* rec) {
	const char* tag = rec->passed ? "PASS" : "FAIL";
	double ms = rec->durationUs / 1000.0;
	// The test's own fail reason was already printed immediately by hookFail();
	// here we only append the framework outcome note (timed out / crashed / no
	// test function) so a hung or crashed test still shows why on the status line.
	if (rec->passed || rec->endNote.empty())
		printf("[ %s ] %-*s %10.2f ms\n", tag, UT_NAME_WIDTH, rec->name.c_str(), ms);
	else
		printf("[ %s ] %-*s %10.2f ms   %s\n", tag, UT_NAME_WIDTH, rec->name.c_str(), ms,
		       rec->endNote.c_str());
	fflush(stdout);
}

void UnitTestRunner::printMetricLines(const UnitTestRecord* rec) {
	for (size_t i = 0; i < rec->metrics.size(); i++) {
		const UnitTestMetric& m = rec->metrics[i];
		char line[512];
		snprintf(line, sizeof(line), UT_INDENT "%-*s %14.3f %-8s",
		         UT_METRIC_WIDTH, m.name.c_str(), m.value, m.unit.c_str());
		printf("%s", line);

		// Comparison against a previous run
		double prev = 0.0;
		std::string key = rec->name + "/" + m.name;
		if (compareLoaded && comparisonValue(key, prev)) {
			if (prev != 0.0) {
				double pct = (m.value - prev) / prev * 100.0;
				const char* dir;
				bool improved = m.higherIsBetter ? (m.value > prev) : (m.value < prev);
				bool worse    = m.higherIsBetter ? (m.value < prev) : (m.value > prev);
				dir = improved ? "better" : (worse ? "worse" : "same");
				printf("   prev %.3f   %+.1f%% (%s)", prev, pct, dir);
			} else {
				printf("   prev %.3f", prev);
			}
		}
		printf("\n");
	}
	fflush(stdout);
}

// ----------------------------------------------------------------------------
//  Running
// ----------------------------------------------------------------------------
void UnitTestRunner::beginTest(UnitTestRecord* rec) {
	current = rec;
	lastProgressLen = 0;
	rec->ran = true;
	rec->passed = false;
	rec->metrics.clear();
	rec->failReason.clear();
	rec->endNote.clear();
}

void UnitTestRunner::finishTest(UnitTestRecord* rec) {
	clearProgressLine();
	printStatusLine(rec);
	printMetricLines(rec);
	current = NULL;
}

// ---- child-result serialisation (used by the forked-execution path) --------
// The child runs one test and records its metrics / fail reason into its own
// copy of the record; it then serialises them to a small file the parent reads
// back (fork copies memory, so the parent cannot see the child's heap directly).
static std::string ut_escape_field(const std::string& s) {
	std::string out;
	for (size_t i = 0; i < s.size(); i++) {
		char c = s[i];
		if (c == '\n' || c == '\r' || c == '\t') out += ' ';
		else out += c;
	}
	return out;
}

static void ut_writeChildResult(const std::string& path, const UnitTestRecord* rec) {
	std::string s;
	s += "fail\t" + ut_escape_field(rec->failReason) + "\n";
	for (size_t i = 0; i < rec->metrics.size(); i++) {
		const UnitTestMetric& m = rec->metrics[i];
		char num[64];
		snprintf(num, sizeof(num), "%.10g", m.value);
		s += std::string("metric\t") + ut_escape_field(m.name) + "\t" + num + "\t" +
		     ut_escape_field(m.unit) + "\t" + (m.higherIsBetter ? "1" : "0") + "\n";
	}
	utils::WriteAFile(path.c_str(), s.c_str(), (uint32)s.size());
}

static void ut_readChildResult(const std::string& path, UnitTestRecord* rec) {
	std::string data = utils::ReadAFileString(path);
	if (data.empty()) return;
	size_t pos = 0;
	while (pos < data.size()) {
		size_t eol = data.find('\n', pos);
		if (eol == std::string::npos) eol = data.size();
		std::string line = data.substr(pos, eol - pos);
		pos = eol + 1;
		if (line.empty()) continue;
		// split on tabs
		std::vector<std::string> f;
		size_t p = 0;
		while (true) {
			size_t t = line.find('\t', p);
			if (t == std::string::npos) { f.push_back(line.substr(p)); break; }
			f.push_back(line.substr(p, t - p));
			p = t + 1;
		}
		if (f[0] == "fail" && f.size() >= 2) {
			rec->failReason = f[1];
		} else if (f[0] == "metric" && f.size() >= 5) {
			UnitTestMetric m;
			m.name = f[1];
			m.value = utils::Ascii2Float64(f[2].c_str());
			m.unit = f[3];
			m.higherIsBetter = (f[4] == "1");
			rec->metrics.push_back(m);
		}
	}
}

// Run a single test in its own process so a crash or hang in the test cannot
// take down the whole suite, and so global/shared state (thread manager,
// shared memory, singletons) never leaks between tests. Returns true on pass.
// On non-fork platforms falls back to running inline.
bool UnitTestRunner::executeRecord(UnitTestRecord* rec) {
	// Clean any stale shared-memory segments left by a previously crashed/killed
	// test before starting this one, so it always begins from a clean slate.
	ClearStaleTestSegments();

#ifdef UT_HAVE_FORK
	if (forkPerTest) {
		std::string dir = outputDir.size() ? outputDir : ".";
		std::string resultPath = dir + "/.cmsdk_result_" + rec->name;
		std::string progPath   = resultPath + ".progress";
		std::string tracePath  = resultPath + ".trace";
		unlink(resultPath.c_str());
		unlink(progPath.c_str());
		unlink(tracePath.c_str());

		fflush(stdout);
		fflush(stderr);
		uint64 t0 = GetTimeNow();
		pid_t pid = fork();
		if (pid < 0) {
			// fork failed - run inline as a fallback
			bool ok = rec->func ? rec->func() : false;
			rec->durationUs = GetTimeNow() - t0;
			return ok;
		}
		if (pid == 0) {
			// ---- child ----
			current = rec;
			progressPath = progPath;                      // record current phase for timeout diagnostics
			ut_installTimeoutTraceHandler(tracePath);     // SIGUSR1 -> dump stack trace, then exit
			lastProgressLen = 0;
			if (isTTY) hookProgress(0, "starting...");
			bool ok = false;
			if (rec->func) ok = rec->func();
			else rec->failReason = "no test function registered";
			clearProgressLine();   // erase our progress line so the parent's status line is clean
			ut_writeChildResult(resultPath, rec);
			fflush(stdout);
			_exit(ok ? 0 : 1);
		}

		// ---- parent ----
		int status = 0;
		bool timedOut = false;
		while (true) {
			pid_t r = waitpid(pid, &status, WNOHANG);
			if (r == pid) break;
			if (r < 0) break;
			if (GetTimeNow() - t0 > UT_TEST_TIMEOUT_US) {
				// Ask the stuck child to dump a stack trace of where it hung, give
				// it a brief grace period to do so, then force-kill if necessary.
				kill(pid, SIGUSR1);
				bool reaped = false;
				uint64 graceStart = GetTimeNow();
				while (GetTimeNow() - graceStart < 2000000ULL) {   // up to 2s
					pid_t rr = waitpid(pid, &status, WNOHANG);
					if (rr == pid || rr < 0) { reaped = true; break; }
					utils::Sleep(15);
				}
				if (!reaped) {
					kill(pid, SIGKILL);
					waitpid(pid, &status, 0);
				}
				timedOut = true;
				break;
			}
			utils::Sleep(15);
		}
		rec->durationUs = GetTimeNow() - t0;

		bool ok = false;
		if (timedOut) {
			// Report which phase the test was in when it hung, and print the stack
			// trace the child captured (if any) so the hang is diagnosable.
			std::string note = utils::StringFormat("timed out after %llus", UT_TEST_TIMEOUT_US / 1000000ULL);
			std::string lastPhase = ut_readProgressNote(progPath);
			if (!lastPhase.empty()) note += " (last phase: " + lastPhase + ")";
			rec->endNote = note;
			ut_readChildResult(resultPath, rec);   // recover any reason/metrics gathered pre-hang
			ut_printTraceFile(tracePath);
		} else if (WIFSIGNALED(status)) {
			rec->endNote = utils::StringFormat("crashed (signal %d)", WTERMSIG(status));
			ut_readChildResult(resultPath, rec);  // recover the reason/metrics gathered pre-crash
		} else {
			ut_readChildResult(resultPath, rec);
			ok = (WIFEXITED(status) && WEXITSTATUS(status) == 0);
		}
		unlink(resultPath.c_str());
		unlink(progPath.c_str());
		unlink(tracePath.c_str());
		return ok;
	}
#endif

	// Inline fallback (no fork available, or fork disabled).
	current = rec;
	lastProgressLen = 0;
	if (isTTY) hookProgress(0, "starting...");
	uint64 t0 = GetTimeNow();
	bool ok = rec->func ? rec->func() : false;
	rec->durationUs = GetTimeNow() - t0;
	if (!rec->func) rec->endNote = "no test function registered";
	return ok;
}

bool UnitTestRunner::runRecords(std::vector<UnitTestRecord*>& records) {
	// Start from a clean shared-memory slate so a previously killed run can't
	// deadlock this one on a stale process-shared mutex.
	ClearStaleTestSegments();

	if (compareFile.size() && !compareLoaded) loadComparison();

	// In non-verbose mode, silence the logging system's console output so the
	// only thing on stdout is the consistent status/metric lines. Tests still
	// report failures via unittest::fail(). Verbose mode leaves logging on.
	bool savedLogToStdOut = true;
	if (!verboseFlag) {
		LogSystem::SetLogReceiver(NULL);            // ensures the singleton exists
		if (LogSystem::LogSingleton) {
			savedLogToStdOut = LogSystem::LogSingleton->printToStdOut;
			LogSystem::LogSingleton->printToStdOut = false;
		}
	}

	uint64 startAll = GetTimeNow();
	int passed = 0, failed = 0;

	for (size_t i = 0; i < records.size(); i++) {
		UnitTestRecord* rec = records[i];
		beginTest(rec);
		bool ok = executeRecord(rec);
		rec->passed = ok;
		finishTest(rec);
		if (ok) passed++; else failed++;
	}

	uint64 totalUs = GetTimeNow() - startAll;

	printf("\n----------------------------------------------------------------\n");
	printf("CMSDK tests: %d passed, %d failed, %d total in %.2f ms\n",
	       passed, failed, (int)records.size(), totalUs / 1000.0);

	if (writeJSONFlag) writeResults(records, totalUs, passed, failed);
	printf("\n");

	if (!verboseFlag && LogSystem::LogSingleton)
		LogSystem::LogSingleton->printToStdOut = savedLogToStdOut;

	return failed == 0;
}

bool UnitTestRunner::runAll() {
	std::vector<UnitTestRecord*> records;
	int excluded = 0;
	for (size_t i = 0; i < tests.size(); i++) {
		if (tests[i].inDefaultRun) records.push_back(&tests[i]);
		else excluded++;
	}
	if (records.empty()) {
		printf("No CMSDK unit tests are registered.\n");
		return false;
	}
	printf("\nRunning %u CMSDK unit tests", (uint32)records.size());
	if (excluded)
		printf(" (%d excluded from the default run; run them by name)", excluded);
	printf("...\n\n");
	return runRecords(records);
}

bool UnitTestRunner::runOne(const char* name) {
	if (!name) return false;
	UnitTestRecord* found = NULL;
	for (size_t i = 0; i < tests.size(); i++) {
		if (stricmp(tests[i].name.c_str(), name) == 0) { found = &tests[i]; break; }
	}
	if (!found) {
		printf("No CMSDK unit test named '%s'.\n", name);
		listTests();
		return false;
	}
	std::vector<UnitTestRecord*> records;
	records.push_back(found);
	printf("\nRunning CMSDK unit test '%s'...\n\n", found->name.c_str());
	return runRecords(records);
}

// ----------------------------------------------------------------------------
//  Comparison file loading
// ----------------------------------------------------------------------------
bool UnitTestRunner::comparisonValue(const std::string& key, double& out) const {
	std::map<std::string, double>::const_iterator it = compareMetrics.find(key);
	if (it == compareMetrics.end()) return false;
	out = it->second;
	return true;
}

void UnitTestRunner::loadComparison() {
	compareLoaded = false;
	compareMetrics.clear();

	std::string json = utils::ReadAFileString(compareFile);
	if (json.empty()) {
		printf("Warning: could not read comparison file '%s'\n", compareFile.c_str());
		return;
	}

	// Count tokens first, then parse.
	jsmn_parser parser;
	jsmn_init(&parser);
	int needed = jsmn_parse(&parser, json.c_str(), json.size(), NULL, 0);
	if (needed <= 0) {
		printf("Warning: comparison file '%s' is not valid JSON\n", compareFile.c_str());
		return;
	}
	jsmntok_t* tokens = new jsmntok_t[needed + 1];
	jsmn_init(&parser);
	int count = jsmn_parse(&parser, json.c_str(), json.size(), tokens, needed + 1);
	if (count <= 0) {
		delete[] tokens;
		printf("Warning: failed to parse comparison file '%s'\n", compareFile.c_str());
		return;
	}

	// Pull every "test/metric" -> number pair from the flat section. Flat keys
	// are the only keys containing '/', so they are unambiguous.
	for (int n = 0; n < count - 1; n++) {
		if (tokens[n].type != JSMN_STRING) continue;
		int klen = tokens[n].end - tokens[n].start;
		if (klen <= 0) continue;
		const char* kstart = json.c_str() + tokens[n].start;
		bool hasSlash = false;
		for (int c = 0; c < klen; c++) { if (kstart[c] == '/') { hasSlash = true; break; } }
		if (!hasSlash) continue;
		if (tokens[n + 1].type != JSMN_PRIMITIVE) continue;
		std::string key(kstart, klen);
		std::string val(json.c_str() + tokens[n + 1].start, tokens[n + 1].end - tokens[n + 1].start);
		compareMetrics[key] = utils::Ascii2Float64(val.c_str());
	}

	delete[] tokens;
	compareLoaded = true;
	printf("Comparing against '%s' (%u recorded metrics)\n\n",
	       compareFile.c_str(), (uint32)compareMetrics.size());
}

// ----------------------------------------------------------------------------
//  Results JSON writing
// ----------------------------------------------------------------------------
static std::string jsonEscape(const std::string& s) {
	std::string out;
	for (size_t i = 0; i < s.size(); i++) {
		char c = s[i];
		switch (c) {
			case '\"': out += "\\\""; break;
			case '\\': out += "\\\\"; break;
			case '\n': out += "\\n";  break;
			case '\r': out += "\\r";  break;
			case '\t': out += "\\t";  break;
			default:   out += c;      break;
		}
	}
	return out;
}

std::string UnitTestRunner::timestampForFilename(uint64 t) {
	struct PsyDateAndTime d = GetDateAndTime(t);
	char buf[40];   // worst-case uint16 fields (see writeResults) => keep truncation-warning-free
	snprintf(buf, sizeof(buf), "%04u%02u%02u-%02u%02u%02u",
	         d.year, d.mon, d.day, d.hour, d.min, d.sec);
	return std::string(buf);
}

void UnitTestRunner::writeResults(const std::vector<UnitTestRecord*>& records,
                                  uint64 totalUs, int passed, int failed) {
	uint64 now = GetTimeNow();
	struct PsyDateAndTime d = GetDateAndTime(now);
	// Sized for the theoretical worst case of the format below: the calendar
	// fields are uint16, so the compiler must assume up to 5 digits each
	// ("%04u-%02u-%02u %02u:%02u:%02u" => up to 35 chars + NUL). 40 keeps
	// -Wformat-truncation quiet without relying on the values staying small.
	char dt[40];
	snprintf(dt, sizeof(dt), "%04u-%02u-%02u %02u:%02u:%02u",
	         d.year, d.mon, d.day, d.hour, d.min, d.sec);

	const char* platform =
#if defined(WINDOWS)
		"win"
#elif defined(DARWIN) || defined(OSX) || defined(__APPLE__)
		"macosx"
#elif defined(LINUX) || defined(__linux)
		"linux"
#else
		"unknown"
#endif
#if defined(ARCH_64)
		"64"
#else
		"32"
#endif
		;

	std::string json;
	json += "{\n";
	json += "  \"meta\": {\n";
	json += utils::StringFormat("    \"version\": 1,\n");
	json += utils::StringFormat("    \"timestamp_us\": %llu,\n", (unsigned long long)now);
	json += utils::StringFormat("    \"datetime\": \"%s\",\n", dt);
	json += utils::StringFormat("    \"platform\": \"%s\",\n", platform);
	json += utils::StringFormat("    \"total\": %d,\n", (int)records.size());
	json += utils::StringFormat("    \"passed\": %d,\n", passed);
	json += utils::StringFormat("    \"failed\": %d,\n", failed);
	json += utils::StringFormat("    \"duration_us\": %llu\n", (unsigned long long)totalUs);
	json += "  },\n";

	// Human-readable structured section
	json += "  \"tests\": {\n";
	for (size_t i = 0; i < records.size(); i++) {
		const UnitTestRecord* rec = records[i];
		json += utils::StringFormat("    \"%s\": {\n", jsonEscape(rec->name).c_str());
		json += utils::StringFormat("      \"status\": \"%s\",\n", rec->passed ? "pass" : "fail");
		json += utils::StringFormat("      \"duration_us\": %llu,\n", (unsigned long long)rec->durationUs);
		json += "      \"metrics\": {\n";
		for (size_t j = 0; j < rec->metrics.size(); j++) {
			const UnitTestMetric& m = rec->metrics[j];
			json += utils::StringFormat(
				"        \"%s\": { \"value\": %.6f, \"unit\": \"%s\", \"higher_is_better\": %s }%s\n",
				jsonEscape(m.name).c_str(), m.value, jsonEscape(m.unit).c_str(),
				m.higherIsBetter ? "true" : "false",
				(j + 1 < rec->metrics.size()) ? "," : "");
		}
		json += "      }\n";
		json += utils::StringFormat("    }%s\n", (i + 1 < records.size()) ? "," : "");
	}
	json += "  },\n";

	// Flat machine-comparison section ("test/metric" -> number)
	json += "  \"flat\": {\n";
	std::vector<std::string> flat;
	for (size_t i = 0; i < records.size(); i++) {
		const UnitTestRecord* rec = records[i];
		flat.push_back(utils::StringFormat("    \"%s/__duration_us\": %llu",
			jsonEscape(rec->name).c_str(), (unsigned long long)rec->durationUs));
		for (size_t j = 0; j < rec->metrics.size(); j++) {
			const UnitTestMetric& m = rec->metrics[j];
			flat.push_back(utils::StringFormat("    \"%s/%s\": %.6f",
				jsonEscape(rec->name).c_str(), jsonEscape(m.name).c_str(), m.value));
		}
	}
	for (size_t i = 0; i < flat.size(); i++)
		json += flat[i] + ((i + 1 < flat.size()) ? ",\n" : "\n");
	json += "  }\n";
	json += "}\n";

	// Decide output paths.
	std::vector<std::string> paths;
	if (outputFile.size()) {
		paths.push_back(outputFile);
	} else {
		std::string dir = outputDir.size() ? outputDir : ".";
		paths.push_back(dir + "/cmsdk_perf_" + timestampForFilename(now) + ".json");
		paths.push_back(dir + "/cmsdk_perf_latest.json");
	}

	for (size_t i = 0; i < paths.size(); i++) {
		if (utils::WriteAFile(paths[i].c_str(), json.c_str(), (uint32)json.size()))
			printf("Performance data written to %s\n", paths[i].c_str());
		else
			printf("Warning: could not write performance data to %s\n", paths[i].c_str());
	}
}

// ----------------------------------------------------------------------------
//  unittest:: free function API
// ----------------------------------------------------------------------------
namespace unittest {

// Format a va_list into a std::string using CMSDK's portable formatter
// (handles the snprintf/_vsnprintf_s differences across MSVC/Cygwin/POSIX).
static std::string ut_vformat(const char* fmt, va_list args) {
	uint32 sz = 0;
	char* s = utils::StringFormatVA(sz, fmt, args);
	std::string out = s ? s : "";
	if (s) free(s);
	return out;
}

void progress(int percent, const char* action) {
	UnitTestRunner::instance().hookProgress(percent, action);
}

void progressf(int percent, const char* fmt, ...) {
	va_list args;
	va_start(args, fmt);
	std::string s = ut_vformat(fmt, args);
	va_end(args);
	UnitTestRunner::instance().hookProgress(percent, s.c_str());
}

void metric(const char* name, double value, const char* unit, bool higherIsBetter) {
	UnitTestRunner::instance().hookMetric(name, value, unit, higherIsBetter);
}

void detail(const char* fmt, ...) {
	if (!UnitTestRunner::instance().hookVerbose()) return;
	va_list args;
	va_start(args, fmt);
	std::string s = ut_vformat(fmt, args);
	va_end(args);
	UnitTestRunner::instance().hookDetail(s.c_str());
}

void fail(const char* fmt, ...) {
	va_list args;
	va_start(args, fmt);
	std::string s = ut_vformat(fmt, args);
	va_end(args);
	UnitTestRunner::instance().hookFail(s.c_str());
}

bool verbose() {
	return UnitTestRunner::instance().hookVerbose();
}

} // namespace unittest

} // namespace cmlabs
