diff options
Diffstat (limited to 'longbow/src/LongBow/longBow_TestCase.c')
| -rw-r--r-- | longbow/src/LongBow/longBow_TestCase.c | 643 |
1 files changed, 0 insertions, 643 deletions
diff --git a/longbow/src/LongBow/longBow_TestCase.c b/longbow/src/LongBow/longBow_TestCase.c deleted file mode 100644 index 3b487905..00000000 --- a/longbow/src/LongBow/longBow_TestCase.c +++ /dev/null @@ -1,643 +0,0 @@ -/* - * Copyright (c) 2017 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - */ -#include <config.h> - -#include <stdlib.h> -#include <stdio.h> -#include <signal.h> -#include <sys/types.h> -#include <sys/time.h> -#include <sys/resource.h> -#include <sys/wait.h> -#include <unistd.h> -#include <setjmp.h> -#include <assert.h> -#include <errno.h> - -#include <LongBow/unit-test.h> - -#include <LongBow/Reporting/longBowReport_Testing.h> - -#include <LongBow/longBow_Config.h> -#include <LongBow/longBow_TestCaseClipBoard.h> -#include <LongBow/private/longBow_Memory.h> -#include <LongBow/private/longBow_String.h> - - -/** - * @struct longbow_testcase - * @brief The specification of a LongBow Test Case. - * - */ -struct longbow_testcase { - /** - * The name of the test case as a C string. - */ - const char *testCaseName; - - const LongBowTestCaseMetaData *metaData; - - char *fullName; - - /** - * The fixture that is running this test case. - */ - const LongBowTestFixture *fixture; - - /** - * The function to execute the test case. - */ - LongBowTestCaseFunction *testCase; - - /** - * The runtime characteristics consisting of the expected and actual results. - */ - LongBowRuntime *runtime; -}; - - -void -longBowTestCase_ConfigHelp(void) -{ - printf("Test Case options:\n"); - printf(" --set <runnerName>/<fixtureName>/iterations=<integer> Run the named test case <integer> times.\n"); -} - -LongBowTestCase * -longBowTestCase_Create(const char *testCaseName, - const LongBowTestFixture *testFixture, - LongBowTestCaseFunction *testCase, - const LongBowTestCaseMetaData *metaData) -{ - assert(testCaseName != NULL); - - LongBowTestCase *result = longBowMemory_Allocate(sizeof(LongBowTestCase)); - if (result != NULL) { - result->testCaseName = testCaseName; - int status = asprintf(&result->fullName, "%s/%s", longBowTestFixture_GetFullName(testFixture), testCaseName); - assert(status != -1); - result->fixture = testFixture; - result->testCase = testCase; - result->runtime = longBowRuntime_Create(&metaData->expectedResult, - longBowTestRunner_GetConfiguration(longBowTestFixture_GetRunner(testFixture))); - result->metaData = metaData; - } - return result; -} - -const char * -longBowTestCase_GetFullName(const LongBowTestCase *testCase) -{ - assert(testCase != NULL); - return testCase->fullName; -} - -void -longBowTestCase_Destroy(LongBowTestCase **testCaseP) -{ - LongBowTestCase *testCase = *testCaseP; - - longBowRuntime_Destroy(&(testCase->runtime)); - - longBowMemory_Deallocate((void **) testCaseP); -} - -/* - * Given the exit status of a test as returned by wait(2), return the corresponding {@link LongBowStatus}. - * - * @param [in] waitStatus - * @return LongBowStatus - */ -static LongBowStatus -_longBowTestCase_ParseWaitStatus(int waitStatus) -{ - LongBowStatus result = LongBowStatus_WARNED; - if (WIFSIGNALED(waitStatus)) { - int exitSignal = WTERMSIG(waitStatus); - if (exitSignal == SIGABRT) { - result = LONGBOW_STATUS_FAILED; - } else { - result = LongBowStatus_SIGNALLED + exitSignal; - } - } else if (WIFEXITED(waitStatus)) { - result = (LongBowStatus) WEXITSTATUS(waitStatus); - } else { - result = LongBowStatus_STOPPED; - } - - return result; -} - -LongBowLocation * -longBowTestCase_CreateLocation(const LongBowTestCase *testCase) -{ - LongBowLocation *result = longBowLocation_Create(testCase->metaData->fileName, testCase->fullName, testCase->metaData->lineNumber); - return result; -} - -const char * -longBowTestCase_GetName(const LongBowTestCase *testCase) -{ - return testCase->testCaseName; -} - -LongBowTestFixture * -longBowTestCase_GetFixture(const LongBowTestCase *testCase) -{ - return (LongBowTestFixture *) testCase->fixture; -} - -LongBowRuntimeResult * -longBowTestCase_GetExpectedResult(const LongBowTestCase *testCase) -{ - return longBowRuntime_GetExpectedTestCaseResult(testCase->runtime); -} - -LongBowRuntimeResult * -longBowTestCase_GetActualResult(const LongBowTestCase *testCase) -{ - return longBowRuntime_GetActualTestCaseResult(testCase->runtime); -} - -size_t -longBowTestCase_GetEventEvaluationCount(const LongBowTestCase *testCase) -{ - return longBowRuntime_GetActualEventEvaluationCount(testCase->runtime); -} - -static jmp_buf longBowTestCaseAbort; - -/* - * The process running the Test Case receives a signal. - * - * A regular, passing Test Case induces no signal and as a result does not pass through this function. - * A Test Case that fails an assertion will induce the SIGABRT signal which does pass through this function. - * Any other signal is because the Test Case either purposefully sent itself a signal, (including calling abort()), - * or it induced one through some implicit behaviour (like a SIGSEGV). - * - * In all remaining cases, encode the signal received into a return value for the - * corresponding setjmp() and invoke the longjmp(). - */ -__attribute__((noreturn)) -static void -_longBowTestCase_ReceiveSignal(int signal, siginfo_t *siginfo __attribute__((unused)), void *data __attribute__((unused))) -{ - if (signal == SIGINT) { - printf("Howdy\n"); - } - longjmp(longBowTestCaseAbort, signal); -} - -/** - * Return true or false, if LongBow should capture a signal, or not. - * - * LongBow captures signals in order to report on the success or failure of a test. - * Some signals do not indicate that a test failed, - * only that the environment changed, - * or some other event that is unrelated to success or failure. - * Unless, of course, the test is checking for the event (this is not yet supported). - */ -static bool -_longBowTestCase_MustCaptureSignal(const int signal) -{ - switch (signal) { - case SIGTRAP: - return false; - - case SIGCHLD: - return false; - - case SIGKILL: - return false; - - case SIGSTOP: - return false; - - case SIGWINCH: - return false; - - default: - return true; - } -} - -/** - * Setup signals to point to the testCaseSignalAction handler for all signals possible. - */ -static void -_longBowTestCase_TestInitSignals(void) -{ - struct sigaction signalAction; - signalAction.sa_sigaction = _longBowTestCase_ReceiveSignal; - signalAction.sa_flags = SA_SIGINFO; - sigemptyset(&signalAction.sa_mask); - - struct sigaction oldSignals[NSIG]; - for (int i = 1; i < NSIG; i++) { - if (_longBowTestCase_MustCaptureSignal(i)) { - sigaction(i, &signalAction, &oldSignals[i]); - } - } -} - -/** - * Setup signals to their default behaviour. - */ -static void -_longBowTestCase_TestFiniSignals(void) -{ - struct sigaction signalAction; - signalAction.sa_handler = SIG_DFL; - signalAction.sa_flags = SA_SIGINFO; - sigemptyset(&signalAction.sa_mask); - - for (int i = 1; i < NSIG; i++) { - if (_longBowTestCase_MustCaptureSignal(i)) { - sigaction(i, &signalAction, NULL); - } - } -} - -/* - * Determine the status of the given LongBowTestCase. - * - * If the actual event recorded in the test case is equal to the expected event, - * then return LONGBOW_STATUS_SUCCEEDED. - * If the actual event is NULL and the expected event is not, then return LONGBOW_STATUS_FAILED. - * Otherwise, the result is the actual event. - */ -static LongBowStatus -_longBowTestCase_ExpectedVsActualEvent(const LongBowTestCase *testCase) -{ - LongBowStatus result = LONGBOW_STATUS_FAILED; - LongBowEventType *expectedEvent = longBowRuntimeResult_GetEvent(longBowTestCase_GetExpectedResult(testCase)); - LongBowEventType *actualEvent = longBowRuntimeResult_GetEvent(longBowTestCase_GetActualResult(testCase)); - - if (longBowEventType_Equals(expectedEvent, actualEvent)) { - result = LONGBOW_STATUS_SUCCEEDED; - } else if (actualEvent == NULL && expectedEvent != NULL) { - LongBowString *messageString = longBowString_CreateFormat("failed to induce an expected %s event.", - longBowEventType_GetName(expectedEvent)); - - LongBowLocation *location = longBowTestCase_CreateLocation(testCase); - LongBowEvent *event = longBowEvent_Create(expectedEvent, location, "", longBowString_ToString(messageString), NULL); - - longBowReportRuntime_Event(event); - longBowEvent_Destroy(&event); - longBowString_Destroy(&messageString); - - result = LONGBOW_STATUS_FAILED; - } else { - result = longBowEventType_GetStatus(actualEvent); - } - - return result; -} - -/* - * Invoke the test case function and see if it succeeds. - * - * The technique here is to assume the test case will succeed, - * setup a longjmp back to this function - * and then setup signal handling and invoke the test case. - * This essentially wraps the test function an catches and handles the abort() - * (SIGABRT) that results from the LongBow assertion or trap. - * - * If the test case simply returns (ie. the longjmp was never executed), - * then it was successful. - * Otherwise, the longjmp contains a code indicating the status captured by - * the signal handler that was invoked when the test code executed the abort() function. - * Extract relevant information from the current runtime context - * (which is always present whether in a unit test case or not) - * - * Note that it's actually not necessary for the test code to execute an abort(), - * it could invoke the longjmp() directly. - * Something that might be advantagous in the future. - */ -static LongBowStatus -_longBowTestCase_Execute(LongBowTestCase *testCase) -{ - LongBowStatus result = LONGBOW_STATUS_SUCCEEDED; - - if (longBowConfig_IsTrace(longBowTestCase_GetConfiguration(testCase))) { - longBowReportTesting_Trace(" %s/%s/%s", - longBowTestRunner_GetName(longBowTestFixture_GetRunner(longBowTestCase_GetFixture(testCase))), - longBowTestFixture_GetName(longBowTestCase_GetFixture(testCase)), - longBowTestCase_GetName(testCase)); - } - - int receivedSignal = setjmp(longBowTestCaseAbort); - if (receivedSignal == 0) { - _longBowTestCase_TestInitSignals(); - - errno = 0; - - LongBowClipBoard *testClipBoard = longBowTestFixture_GetClipBoard(longBowTestCase_GetFixture(testCase)); - - (testCase->testCase)(longBowTestFixture_GetRunner(longBowTestCase_GetFixture(testCase)), - longBowTestCase_GetFixture(testCase), - testCase, - testClipBoard, - longBowTestCaseAbort); - } else { - // We get here as the result of an extraordinary abort from the testCase invoked just above. - // Sort out the meaning of the received signal here. - // - // If an SIGABRT is received, we expect that the Global Runtime contains a valid LongBowEventType. - // If not, then an old-timey <assert.h> assert() was used and the programmer should be warned, - // or the system under test actually uses SIGABRT and the programmer is warned that this - // incompatible with LongBow. - - LongBowEventType *event = NULL; - if (receivedSignal == SIGABRT) { - if (longBowRuntime_GetActualEventType(longBowRuntime_GetCurrentRuntime()) == NULL) { - char *testCaseString = longBowTestCase_ToString(testCase); - longBowReportRuntime_Warning("Warning: %s commingling LongBow with assert(3) or with SIGABRT will not work.\n", testCaseString); - free(testCaseString); - } - - event = longBowRuntime_GetActualEventType(longBowRuntime_GetCurrentRuntime()); - } else if (receivedSignal == SIGTERM) { - char *testCaseString = longBowTestCase_ToString(testCase); - longBowReportRuntime_Warning("\nWarning: %s premature termination.\n", testCaseString); - free(testCaseString); - event = longBowEventType_GetEventTypeForSignal(receivedSignal); - } else if (receivedSignal == SIGINT) { - char *testCaseString = longBowTestCase_ToString(testCase); - longBowReportRuntime_Warning("\nWarning: %s interrupted.\n", testCaseString); - free(testCaseString); - event = longBowEventType_GetEventTypeForSignal(receivedSignal); - } else { - event = longBowEventType_GetEventTypeForSignal(receivedSignal); - } - - longBowRuntimeResult_SetEvent(longBowTestCase_GetActualResult(testCase), event); - } - - result = _longBowTestCase_ExpectedVsActualEvent(testCase); - - if (result == LONGBOW_STATUS_SUCCEEDED) { - if (longBowTestCase_GetEventEvaluationCount(testCase) == 0) { - result = LongBowStatus_IMPOTENT; - } - } else if (longBowStatus_IsFailed(result)) { - ; - } - - memset(&longBowTestCaseAbort, 0, sizeof(longBowTestCaseAbort)); - - _longBowTestCase_TestFiniSignals(); - - return result; -} - -static LongBowStatus -_longBowTestCase_ExecuteOnce(LongBowTestCase *testCase) -{ - LongBowStatus result = LONGBOW_STATUS_FAILED; - - LongBowTestFixture *fixture = longBowTestCase_GetFixture(testCase); - - LongBowStatus setupStatus = longBowTestFixture_Setup(fixture, testCase); - - if (longBowStatus_IsSuccessful(setupStatus) == true) { - LongBowStatus testCaseStatus = _longBowTestCase_Execute(testCase); - - LongBowStatus tearDownStatus = longBowTestFixture_TearDown(fixture, testCase); - - // Ensure that things only go from "bad" to "worse." If a test case is indicating a failure - // and yet the tear-down is also indicating something NOT successful (like a warning), don't - // demote the status from LONGBOW_STATUS_FAILED to LONGBOW_STATUS_TEARDOWN_WARNING. - // This is messy. Perhaps a better approach would be to structure status as an ordered arrangement of status. - - result = testCaseStatus; - - if (testCaseStatus == LONGBOW_STATUS_SUCCEEDED) { - if (tearDownStatus != LONGBOW_STATUS_SUCCEEDED) { - result = tearDownStatus; - } - } - } - - return result; -} - -static LongBowStatus -_longBowTestCase_Iterate(LongBowTestCase *testCase) -{ - LongBowStatus result = LONGBOW_STATUS_SUCCEEDED; - - LongBowConfig *config = longBowTestCase_GetConfiguration(testCase); - - uint32_t iterations = longBowConfig_GetUint32(config, 1, "%s/iterations", longBowTestCase_GetFullName(testCase)); - - for (uint32_t i = 0; i < iterations && longBowStatus_IsSuccessful(result); i++) { - LongBowRuntime *previousRuntime = longBowRuntime_SetCurrentRuntime(testCase->runtime); - result = _longBowTestCase_ExecuteOnce(testCase); - longBowRuntime_SetCurrentRuntime(previousRuntime); - } - - return result; -} - -/* - * Run a LongBowTestCase in a forked process. - * - * @param testCase The LongBowTestCase to run. - * @return 0 - */ -static int -_longBowTestCase_RunForked(LongBowTestCase *testCase) -{ - struct timeval startTime; - gettimeofday(&startTime, NULL); - - pid_t pid = fork(); - if (pid == 0) { - exit(_longBowTestCase_Iterate(testCase)); - } else { - // Rummage around in various things to obtain the post-mortem - // results of the test that was run in a separate process. - int waitStatus; - struct rusage rusage; -#ifndef __ANDROID__ - wait3(&waitStatus, 0, &rusage); -#else - wait4(-1, &waitStatus, 0, &rusage); -#endif - struct timeval endTime; - gettimeofday(&endTime, NULL); - - struct timeval elapsedTime; - timersub(&endTime, &startTime, &elapsedTime); - - longBowRuntimeResult_SetElapsedTime(longBowTestCase_GetActualResult(testCase), &elapsedTime); - longBowRuntimeResult_SetRUsage(longBowTestCase_GetActualResult(testCase), &rusage); - longBowRuntimeResult_SetStatus(longBowTestCase_GetActualResult(testCase), _longBowTestCase_ParseWaitStatus(waitStatus)); - } - return 0; -} - -/* - * Run a LongBowTestCase in this address space (ie. not a forked process). - * - * @param testCase The LongBowTestCase to run. - * @return 0 - */ -static int -_longBowTestCase_RunNonForked(LongBowTestCase *testCase) -{ - struct timeval startTime; - gettimeofday(&startTime, NULL); - - LongBowStatus status = _longBowTestCase_Iterate(testCase); - - struct timeval endTime; - gettimeofday(&endTime, NULL); - - struct timeval elapsedTime; - timersub(&endTime, &startTime, &elapsedTime); - - longBowRuntimeResult_SetElapsedTime(longBowTestCase_GetActualResult(testCase), &elapsedTime); - longBowRuntimeResult_SetStatus(longBowTestCase_GetActualResult(testCase), status); - - return 0; -} - -LongBowStatus -longBowTestCase_GetExpectedStatus(const LongBowTestCase *testCase) -{ - return longBowRuntimeResult_GetStatus(longBowTestCase_GetExpectedResult(testCase)); -} - -LongBowTestCase * -longBowTestCase_Run(const char *testCaseName, - const LongBowTestFixture *fixture, - LongBowTestCaseFunction *testCase, - const LongBowTestCaseMetaData *testCaseMetaData) -{ - LongBowTestCase *result = longBowTestCase_Create(testCaseName, fixture, testCase, testCaseMetaData); - - if (result != NULL) { - if (longBowConfig_IsRunForked(longBowTestRunner_GetConfiguration(longBowTestFixture_GetRunner(fixture)))) { - _longBowTestCase_RunForked(result); - } else { - _longBowTestCase_RunNonForked(result); - } - - longBowTestFixture_AddTestCase(fixture, result); - longBowReportTesting_DisplayTestCaseResult(result); - } - return result; -} - -LongBowStatus -longBowTestCase_GetStatus(const LongBowTestCase *testCase) -{ - return longBowRuntimeResult_GetStatus(longBowTestCase_GetActualResult(testCase)); -} - -static LongBowClipBoard * -_longBowTestCase_GetClipBoard(const LongBowTestCase *testCase) -{ - return longBowTestFixture_GetClipBoard(testCase->fixture); -} - -void * -longBowTestCase_SetClipBoardData(const LongBowTestCase *testCase, void *data) -{ - return longBowClipBoard_Set(_longBowTestCase_GetClipBoard(testCase), "testCase", data); -} - -void * -longBowTestCase_GetClipBoardData(const LongBowTestCase *testCase) -{ - return longBowClipBoard_Get(_longBowTestCase_GetClipBoard(testCase), "testCase"); -} - -void * -longBowTestCase_Set(const LongBowTestCase *testCase, const char *name, void *value) -{ - return longBowClipBoard_Set(_longBowTestCase_GetClipBoard(testCase), name, value); -} - -void * -longBowTestCase_Get(const LongBowTestCase *testCase, const char *name) -{ - return longBowClipBoard_Get(_longBowTestCase_GetClipBoard(testCase), name); -} - -char * -longBowClipBoard_GetCString(const LongBowTestCase *testCase, const char *name) -{ - return longBowClipBoard_GetAsCString(_longBowTestCase_GetClipBoard(testCase), name); -} - -void * -longBowTestCase_SetInt(const LongBowTestCase *testCase, const char *name, int value) -{ - return longBowClipBoard_SetInt(_longBowTestCase_GetClipBoard(testCase), name, (uint64_t) value); -} - -void * -longBowTestCase_SetCString(const LongBowTestCase *testCase, const char *name, char *value) -{ - return longBowClipBoard_SetCString(_longBowTestCase_GetClipBoard(testCase), name, value); -} - -int -longBowTestCase_GetInt(const LongBowTestCase *testCase, const char *name) -{ - return (intptr_t) longBowTestCase_Get(testCase, name); -} - -LongBowConfig * -longBowTestCase_GetConfiguration(const LongBowTestCase *testCase) -{ - return longBowTestRunner_GetConfiguration(longBowTestFixture_GetRunner(longBowTestCase_GetFixture(testCase))); -} - -char * -longBowTestCase_ToString(const LongBowTestCase *testCase) -{ - return strdup(longBowTestCase_GetFullName(testCase)); -} - -bool -longBowTestCase_IsSuccessful(const LongBowTestCase *testCase) -{ - return longBowStatus_IsSuccessful(longBowTestCase_GetStatus(testCase)); -} - -bool -longBowTestCase_IsFailed(const LongBowTestCase *testCase) -{ - return longBowStatus_IsFailed(longBowTestCase_GetStatus(testCase)); -} - -bool -longBowTestCase_IsWarning(const LongBowTestCase *testCase) -{ - return longBowStatus_IsWarning(longBowTestCase_GetStatus(testCase)); -} - -bool -longBowTestCase_IsIncomplete(const LongBowTestCase *testCase) -{ - return longBowStatus_IsIncomplete(longBowTestCase_GetStatus(testCase)); -} |