diff options
Diffstat (limited to 'longbow/src/LongBow/longBow_TestCase.c')
| -rw-r--r-- | longbow/src/LongBow/longBow_TestCase.c | 643 |
1 files changed, 643 insertions, 0 deletions
diff --git a/longbow/src/LongBow/longBow_TestCase.c b/longbow/src/LongBow/longBow_TestCase.c new file mode 100644 index 00000000..3b487905 --- /dev/null +++ b/longbow/src/LongBow/longBow_TestCase.c @@ -0,0 +1,643 @@ +/* + * Copyright (c) 2017 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + */ +#include <config.h> + +#include <stdlib.h> +#include <stdio.h> +#include <signal.h> +#include <sys/types.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <sys/wait.h> +#include <unistd.h> +#include <setjmp.h> +#include <assert.h> +#include <errno.h> + +#include <LongBow/unit-test.h> + +#include <LongBow/Reporting/longBowReport_Testing.h> + +#include <LongBow/longBow_Config.h> +#include <LongBow/longBow_TestCaseClipBoard.h> +#include <LongBow/private/longBow_Memory.h> +#include <LongBow/private/longBow_String.h> + + +/** + * @struct longbow_testcase + * @brief The specification of a LongBow Test Case. + * + */ +struct longbow_testcase { + /** + * The name of the test case as a C string. + */ + const char *testCaseName; + + const LongBowTestCaseMetaData *metaData; + + char *fullName; + + /** + * The fixture that is running this test case. + */ + const LongBowTestFixture *fixture; + + /** + * The function to execute the test case. + */ + LongBowTestCaseFunction *testCase; + + /** + * The runtime characteristics consisting of the expected and actual results. + */ + LongBowRuntime *runtime; +}; + + +void +longBowTestCase_ConfigHelp(void) +{ + printf("Test Case options:\n"); + printf(" --set <runnerName>/<fixtureName>/iterations=<integer> Run the named test case <integer> times.\n"); +} + +LongBowTestCase * +longBowTestCase_Create(const char *testCaseName, + const LongBowTestFixture *testFixture, + LongBowTestCaseFunction *testCase, + const LongBowTestCaseMetaData *metaData) +{ + assert(testCaseName != NULL); + + LongBowTestCase *result = longBowMemory_Allocate(sizeof(LongBowTestCase)); + if (result != NULL) { + result->testCaseName = testCaseName; + int status = asprintf(&result->fullName, "%s/%s", longBowTestFixture_GetFullName(testFixture), testCaseName); + assert(status != -1); + result->fixture = testFixture; + result->testCase = testCase; + result->runtime = longBowRuntime_Create(&metaData->expectedResult, + longBowTestRunner_GetConfiguration(longBowTestFixture_GetRunner(testFixture))); + result->metaData = metaData; + } + return result; +} + +const char * +longBowTestCase_GetFullName(const LongBowTestCase *testCase) +{ + assert(testCase != NULL); + return testCase->fullName; +} + +void +longBowTestCase_Destroy(LongBowTestCase **testCaseP) +{ + LongBowTestCase *testCase = *testCaseP; + + longBowRuntime_Destroy(&(testCase->runtime)); + + longBowMemory_Deallocate((void **) testCaseP); +} + +/* + * Given the exit status of a test as returned by wait(2), return the corresponding {@link LongBowStatus}. + * + * @param [in] waitStatus + * @return LongBowStatus + */ +static LongBowStatus +_longBowTestCase_ParseWaitStatus(int waitStatus) +{ + LongBowStatus result = LongBowStatus_WARNED; + if (WIFSIGNALED(waitStatus)) { + int exitSignal = WTERMSIG(waitStatus); + if (exitSignal == SIGABRT) { + result = LONGBOW_STATUS_FAILED; + } else { + result = LongBowStatus_SIGNALLED + exitSignal; + } + } else if (WIFEXITED(waitStatus)) { + result = (LongBowStatus) WEXITSTATUS(waitStatus); + } else { + result = LongBowStatus_STOPPED; + } + + return result; +} + +LongBowLocation * +longBowTestCase_CreateLocation(const LongBowTestCase *testCase) +{ + LongBowLocation *result = longBowLocation_Create(testCase->metaData->fileName, testCase->fullName, testCase->metaData->lineNumber); + return result; +} + +const char * +longBowTestCase_GetName(const LongBowTestCase *testCase) +{ + return testCase->testCaseName; +} + +LongBowTestFixture * +longBowTestCase_GetFixture(const LongBowTestCase *testCase) +{ + return (LongBowTestFixture *) testCase->fixture; +} + +LongBowRuntimeResult * +longBowTestCase_GetExpectedResult(const LongBowTestCase *testCase) +{ + return longBowRuntime_GetExpectedTestCaseResult(testCase->runtime); +} + +LongBowRuntimeResult * +longBowTestCase_GetActualResult(const LongBowTestCase *testCase) +{ + return longBowRuntime_GetActualTestCaseResult(testCase->runtime); +} + +size_t +longBowTestCase_GetEventEvaluationCount(const LongBowTestCase *testCase) +{ + return longBowRuntime_GetActualEventEvaluationCount(testCase->runtime); +} + +static jmp_buf longBowTestCaseAbort; + +/* + * The process running the Test Case receives a signal. + * + * A regular, passing Test Case induces no signal and as a result does not pass through this function. + * A Test Case that fails an assertion will induce the SIGABRT signal which does pass through this function. + * Any other signal is because the Test Case either purposefully sent itself a signal, (including calling abort()), + * or it induced one through some implicit behaviour (like a SIGSEGV). + * + * In all remaining cases, encode the signal received into a return value for the + * corresponding setjmp() and invoke the longjmp(). + */ +__attribute__((noreturn)) +static void +_longBowTestCase_ReceiveSignal(int signal, siginfo_t *siginfo __attribute__((unused)), void *data __attribute__((unused))) +{ + if (signal == SIGINT) { + printf("Howdy\n"); + } + longjmp(longBowTestCaseAbort, signal); +} + +/** + * Return true or false, if LongBow should capture a signal, or not. + * + * LongBow captures signals in order to report on the success or failure of a test. + * Some signals do not indicate that a test failed, + * only that the environment changed, + * or some other event that is unrelated to success or failure. + * Unless, of course, the test is checking for the event (this is not yet supported). + */ +static bool +_longBowTestCase_MustCaptureSignal(const int signal) +{ + switch (signal) { + case SIGTRAP: + return false; + + case SIGCHLD: + return false; + + case SIGKILL: + return false; + + case SIGSTOP: + return false; + + case SIGWINCH: + return false; + + default: + return true; + } +} + +/** + * Setup signals to point to the testCaseSignalAction handler for all signals possible. + */ +static void +_longBowTestCase_TestInitSignals(void) +{ + struct sigaction signalAction; + signalAction.sa_sigaction = _longBowTestCase_ReceiveSignal; + signalAction.sa_flags = SA_SIGINFO; + sigemptyset(&signalAction.sa_mask); + + struct sigaction oldSignals[NSIG]; + for (int i = 1; i < NSIG; i++) { + if (_longBowTestCase_MustCaptureSignal(i)) { + sigaction(i, &signalAction, &oldSignals[i]); + } + } +} + +/** + * Setup signals to their default behaviour. + */ +static void +_longBowTestCase_TestFiniSignals(void) +{ + struct sigaction signalAction; + signalAction.sa_handler = SIG_DFL; + signalAction.sa_flags = SA_SIGINFO; + sigemptyset(&signalAction.sa_mask); + + for (int i = 1; i < NSIG; i++) { + if (_longBowTestCase_MustCaptureSignal(i)) { + sigaction(i, &signalAction, NULL); + } + } +} + +/* + * Determine the status of the given LongBowTestCase. + * + * If the actual event recorded in the test case is equal to the expected event, + * then return LONGBOW_STATUS_SUCCEEDED. + * If the actual event is NULL and the expected event is not, then return LONGBOW_STATUS_FAILED. + * Otherwise, the result is the actual event. + */ +static LongBowStatus +_longBowTestCase_ExpectedVsActualEvent(const LongBowTestCase *testCase) +{ + LongBowStatus result = LONGBOW_STATUS_FAILED; + LongBowEventType *expectedEvent = longBowRuntimeResult_GetEvent(longBowTestCase_GetExpectedResult(testCase)); + LongBowEventType *actualEvent = longBowRuntimeResult_GetEvent(longBowTestCase_GetActualResult(testCase)); + + if (longBowEventType_Equals(expectedEvent, actualEvent)) { + result = LONGBOW_STATUS_SUCCEEDED; + } else if (actualEvent == NULL && expectedEvent != NULL) { + LongBowString *messageString = longBowString_CreateFormat("failed to induce an expected %s event.", + longBowEventType_GetName(expectedEvent)); + + LongBowLocation *location = longBowTestCase_CreateLocation(testCase); + LongBowEvent *event = longBowEvent_Create(expectedEvent, location, "", longBowString_ToString(messageString), NULL); + + longBowReportRuntime_Event(event); + longBowEvent_Destroy(&event); + longBowString_Destroy(&messageString); + + result = LONGBOW_STATUS_FAILED; + } else { + result = longBowEventType_GetStatus(actualEvent); + } + + return result; +} + +/* + * Invoke the test case function and see if it succeeds. + * + * The technique here is to assume the test case will succeed, + * setup a longjmp back to this function + * and then setup signal handling and invoke the test case. + * This essentially wraps the test function an catches and handles the abort() + * (SIGABRT) that results from the LongBow assertion or trap. + * + * If the test case simply returns (ie. the longjmp was never executed), + * then it was successful. + * Otherwise, the longjmp contains a code indicating the status captured by + * the signal handler that was invoked when the test code executed the abort() function. + * Extract relevant information from the current runtime context + * (which is always present whether in a unit test case or not) + * + * Note that it's actually not necessary for the test code to execute an abort(), + * it could invoke the longjmp() directly. + * Something that might be advantagous in the future. + */ +static LongBowStatus +_longBowTestCase_Execute(LongBowTestCase *testCase) +{ + LongBowStatus result = LONGBOW_STATUS_SUCCEEDED; + + if (longBowConfig_IsTrace(longBowTestCase_GetConfiguration(testCase))) { + longBowReportTesting_Trace(" %s/%s/%s", + longBowTestRunner_GetName(longBowTestFixture_GetRunner(longBowTestCase_GetFixture(testCase))), + longBowTestFixture_GetName(longBowTestCase_GetFixture(testCase)), + longBowTestCase_GetName(testCase)); + } + + int receivedSignal = setjmp(longBowTestCaseAbort); + if (receivedSignal == 0) { + _longBowTestCase_TestInitSignals(); + + errno = 0; + + LongBowClipBoard *testClipBoard = longBowTestFixture_GetClipBoard(longBowTestCase_GetFixture(testCase)); + + (testCase->testCase)(longBowTestFixture_GetRunner(longBowTestCase_GetFixture(testCase)), + longBowTestCase_GetFixture(testCase), + testCase, + testClipBoard, + longBowTestCaseAbort); + } else { + // We get here as the result of an extraordinary abort from the testCase invoked just above. + // Sort out the meaning of the received signal here. + // + // If an SIGABRT is received, we expect that the Global Runtime contains a valid LongBowEventType. + // If not, then an old-timey <assert.h> assert() was used and the programmer should be warned, + // or the system under test actually uses SIGABRT and the programmer is warned that this + // incompatible with LongBow. + + LongBowEventType *event = NULL; + if (receivedSignal == SIGABRT) { + if (longBowRuntime_GetActualEventType(longBowRuntime_GetCurrentRuntime()) == NULL) { + char *testCaseString = longBowTestCase_ToString(testCase); + longBowReportRuntime_Warning("Warning: %s commingling LongBow with assert(3) or with SIGABRT will not work.\n", testCaseString); + free(testCaseString); + } + + event = longBowRuntime_GetActualEventType(longBowRuntime_GetCurrentRuntime()); + } else if (receivedSignal == SIGTERM) { + char *testCaseString = longBowTestCase_ToString(testCase); + longBowReportRuntime_Warning("\nWarning: %s premature termination.\n", testCaseString); + free(testCaseString); + event = longBowEventType_GetEventTypeForSignal(receivedSignal); + } else if (receivedSignal == SIGINT) { + char *testCaseString = longBowTestCase_ToString(testCase); + longBowReportRuntime_Warning("\nWarning: %s interrupted.\n", testCaseString); + free(testCaseString); + event = longBowEventType_GetEventTypeForSignal(receivedSignal); + } else { + event = longBowEventType_GetEventTypeForSignal(receivedSignal); + } + + longBowRuntimeResult_SetEvent(longBowTestCase_GetActualResult(testCase), event); + } + + result = _longBowTestCase_ExpectedVsActualEvent(testCase); + + if (result == LONGBOW_STATUS_SUCCEEDED) { + if (longBowTestCase_GetEventEvaluationCount(testCase) == 0) { + result = LongBowStatus_IMPOTENT; + } + } else if (longBowStatus_IsFailed(result)) { + ; + } + + memset(&longBowTestCaseAbort, 0, sizeof(longBowTestCaseAbort)); + + _longBowTestCase_TestFiniSignals(); + + return result; +} + +static LongBowStatus +_longBowTestCase_ExecuteOnce(LongBowTestCase *testCase) +{ + LongBowStatus result = LONGBOW_STATUS_FAILED; + + LongBowTestFixture *fixture = longBowTestCase_GetFixture(testCase); + + LongBowStatus setupStatus = longBowTestFixture_Setup(fixture, testCase); + + if (longBowStatus_IsSuccessful(setupStatus) == true) { + LongBowStatus testCaseStatus = _longBowTestCase_Execute(testCase); + + LongBowStatus tearDownStatus = longBowTestFixture_TearDown(fixture, testCase); + + // Ensure that things only go from "bad" to "worse." If a test case is indicating a failure + // and yet the tear-down is also indicating something NOT successful (like a warning), don't + // demote the status from LONGBOW_STATUS_FAILED to LONGBOW_STATUS_TEARDOWN_WARNING. + // This is messy. Perhaps a better approach would be to structure status as an ordered arrangement of status. + + result = testCaseStatus; + + if (testCaseStatus == LONGBOW_STATUS_SUCCEEDED) { + if (tearDownStatus != LONGBOW_STATUS_SUCCEEDED) { + result = tearDownStatus; + } + } + } + + return result; +} + +static LongBowStatus +_longBowTestCase_Iterate(LongBowTestCase *testCase) +{ + LongBowStatus result = LONGBOW_STATUS_SUCCEEDED; + + LongBowConfig *config = longBowTestCase_GetConfiguration(testCase); + + uint32_t iterations = longBowConfig_GetUint32(config, 1, "%s/iterations", longBowTestCase_GetFullName(testCase)); + + for (uint32_t i = 0; i < iterations && longBowStatus_IsSuccessful(result); i++) { + LongBowRuntime *previousRuntime = longBowRuntime_SetCurrentRuntime(testCase->runtime); + result = _longBowTestCase_ExecuteOnce(testCase); + longBowRuntime_SetCurrentRuntime(previousRuntime); + } + + return result; +} + +/* + * Run a LongBowTestCase in a forked process. + * + * @param testCase The LongBowTestCase to run. + * @return 0 + */ +static int +_longBowTestCase_RunForked(LongBowTestCase *testCase) +{ + struct timeval startTime; + gettimeofday(&startTime, NULL); + + pid_t pid = fork(); + if (pid == 0) { + exit(_longBowTestCase_Iterate(testCase)); + } else { + // Rummage around in various things to obtain the post-mortem + // results of the test that was run in a separate process. + int waitStatus; + struct rusage rusage; +#ifndef __ANDROID__ + wait3(&waitStatus, 0, &rusage); +#else + wait4(-1, &waitStatus, 0, &rusage); +#endif + struct timeval endTime; + gettimeofday(&endTime, NULL); + + struct timeval elapsedTime; + timersub(&endTime, &startTime, &elapsedTime); + + longBowRuntimeResult_SetElapsedTime(longBowTestCase_GetActualResult(testCase), &elapsedTime); + longBowRuntimeResult_SetRUsage(longBowTestCase_GetActualResult(testCase), &rusage); + longBowRuntimeResult_SetStatus(longBowTestCase_GetActualResult(testCase), _longBowTestCase_ParseWaitStatus(waitStatus)); + } + return 0; +} + +/* + * Run a LongBowTestCase in this address space (ie. not a forked process). + * + * @param testCase The LongBowTestCase to run. + * @return 0 + */ +static int +_longBowTestCase_RunNonForked(LongBowTestCase *testCase) +{ + struct timeval startTime; + gettimeofday(&startTime, NULL); + + LongBowStatus status = _longBowTestCase_Iterate(testCase); + + struct timeval endTime; + gettimeofday(&endTime, NULL); + + struct timeval elapsedTime; + timersub(&endTime, &startTime, &elapsedTime); + + longBowRuntimeResult_SetElapsedTime(longBowTestCase_GetActualResult(testCase), &elapsedTime); + longBowRuntimeResult_SetStatus(longBowTestCase_GetActualResult(testCase), status); + + return 0; +} + +LongBowStatus +longBowTestCase_GetExpectedStatus(const LongBowTestCase *testCase) +{ + return longBowRuntimeResult_GetStatus(longBowTestCase_GetExpectedResult(testCase)); +} + +LongBowTestCase * +longBowTestCase_Run(const char *testCaseName, + const LongBowTestFixture *fixture, + LongBowTestCaseFunction *testCase, + const LongBowTestCaseMetaData *testCaseMetaData) +{ + LongBowTestCase *result = longBowTestCase_Create(testCaseName, fixture, testCase, testCaseMetaData); + + if (result != NULL) { + if (longBowConfig_IsRunForked(longBowTestRunner_GetConfiguration(longBowTestFixture_GetRunner(fixture)))) { + _longBowTestCase_RunForked(result); + } else { + _longBowTestCase_RunNonForked(result); + } + + longBowTestFixture_AddTestCase(fixture, result); + longBowReportTesting_DisplayTestCaseResult(result); + } + return result; +} + +LongBowStatus +longBowTestCase_GetStatus(const LongBowTestCase *testCase) +{ + return longBowRuntimeResult_GetStatus(longBowTestCase_GetActualResult(testCase)); +} + +static LongBowClipBoard * +_longBowTestCase_GetClipBoard(const LongBowTestCase *testCase) +{ + return longBowTestFixture_GetClipBoard(testCase->fixture); +} + +void * +longBowTestCase_SetClipBoardData(const LongBowTestCase *testCase, void *data) +{ + return longBowClipBoard_Set(_longBowTestCase_GetClipBoard(testCase), "testCase", data); +} + +void * +longBowTestCase_GetClipBoardData(const LongBowTestCase *testCase) +{ + return longBowClipBoard_Get(_longBowTestCase_GetClipBoard(testCase), "testCase"); +} + +void * +longBowTestCase_Set(const LongBowTestCase *testCase, const char *name, void *value) +{ + return longBowClipBoard_Set(_longBowTestCase_GetClipBoard(testCase), name, value); +} + +void * +longBowTestCase_Get(const LongBowTestCase *testCase, const char *name) +{ + return longBowClipBoard_Get(_longBowTestCase_GetClipBoard(testCase), name); +} + +char * +longBowClipBoard_GetCString(const LongBowTestCase *testCase, const char *name) +{ + return longBowClipBoard_GetAsCString(_longBowTestCase_GetClipBoard(testCase), name); +} + +void * +longBowTestCase_SetInt(const LongBowTestCase *testCase, const char *name, int value) +{ + return longBowClipBoard_SetInt(_longBowTestCase_GetClipBoard(testCase), name, (uint64_t) value); +} + +void * +longBowTestCase_SetCString(const LongBowTestCase *testCase, const char *name, char *value) +{ + return longBowClipBoard_SetCString(_longBowTestCase_GetClipBoard(testCase), name, value); +} + +int +longBowTestCase_GetInt(const LongBowTestCase *testCase, const char *name) +{ + return (intptr_t) longBowTestCase_Get(testCase, name); +} + +LongBowConfig * +longBowTestCase_GetConfiguration(const LongBowTestCase *testCase) +{ + return longBowTestRunner_GetConfiguration(longBowTestFixture_GetRunner(longBowTestCase_GetFixture(testCase))); +} + +char * +longBowTestCase_ToString(const LongBowTestCase *testCase) +{ + return strdup(longBowTestCase_GetFullName(testCase)); +} + +bool +longBowTestCase_IsSuccessful(const LongBowTestCase *testCase) +{ + return longBowStatus_IsSuccessful(longBowTestCase_GetStatus(testCase)); +} + +bool +longBowTestCase_IsFailed(const LongBowTestCase *testCase) +{ + return longBowStatus_IsFailed(longBowTestCase_GetStatus(testCase)); +} + +bool +longBowTestCase_IsWarning(const LongBowTestCase *testCase) +{ + return longBowStatus_IsWarning(longBowTestCase_GetStatus(testCase)); +} + +bool +longBowTestCase_IsIncomplete(const LongBowTestCase *testCase) +{ + return longBowStatus_IsIncomplete(longBowTestCase_GetStatus(testCase)); +} |