diff --git a/.github/workflows-disabled/ci.yaml b/.github/workflows-disabled/ci.yaml new file mode 100644 index 0000000..bd8f0b6 --- /dev/null +++ b/.github/workflows-disabled/ci.yaml @@ -0,0 +1,73 @@ +name: Build Tests + +on: + push: + branches: + - main + - master + pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-24.04 + + strategy: + fail-fast: false + matrix: + cxx: + - g++ + - g++-13 + + env: + CXX: ${{ matrix.cxx }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y g++ g++-13 make + + - name: Show compiler version + run: $CXX --version + + - name: Run Preprocessor Unit Tests + run: | + cd "$GITHUB_WORKSPACE/preprocessor" + python3 ./UnitTests.py + + - name: Run Library Unit Tests + run: | + cd "$GITHUB_WORKSPACE/runtime" + make test -j2 + ./test --gtest_filter=-NanoLogTest.StagingBuffer_finishReservation_asserts:PackerTest.nibbler_assert + + - name: Build Sample App + run: | + cd "$GITHUB_WORKSPACE/sample" + make clean-all + make depend -C ../runtime + make -j"$(nproc)" + + - name: Build Preprocessor Sample App + run: | + cd "$GITHUB_WORKSPACE/sample_preprocessor" + make clean-all + make depend -C ../runtime + make -j"$(nproc)" + + - name: Run Integration Test + run: | + cd "$GITHUB_WORKSPACE/integrationTest" + make clean-all + ./run.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index 79796be..0000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: Build Tests - -on: [push, pull_request] - -jobs: - build: - - runs-on: ubuntu-latest - - strategy: - matrix: - cxx: [g++, g++-7, g++-8, g++-9] - - env: - CXX: ${{ matrix.cxx }} - - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-python@v1 - with: - python-version: '3.x' - architecture: 'x64' - - - name: Run Preprocessor Unit Tests - run: | - cd ${GITHUB_WORKSPACE}/preprocessor - python ./UnitTests.py - - - name: Run Library Unit Tests - run: | - cd ${GITHUB_WORKSPACE} - git submodule update --init - cd runtime - make test -j2 - # Excludes assert/death tests - ./test --gtest_filter=-NanoLogTest.StagingBuffer_finishReservation_asserts:PackerTest.nibbler_assert - - - name: Build Sample App - run: | - cd ${GITHUB_WORKSPACE}/sample - make clean-all - make depend -C ../runtime - make -j10 > /dev/null - - - name: Build Preprocessor Sample App - run: | - cd ${GITHUB_WORKSPACE}/sample_preprocessor - make clean-all - make depend -C ../runtime - make -j10 > /dev/null - - - name: Run Integration Test - run: | - cd ${GITHUB_WORKSPACE}/integrationTest - make clean-all > /dev/null - ./run.sh diff --git a/.github/workflows/ubuntu-clang-matrix.yml b/.github/workflows/ubuntu-clang-matrix.yml new file mode 100644 index 0000000..95500ce --- /dev/null +++ b/.github/workflows/ubuntu-clang-matrix.yml @@ -0,0 +1,87 @@ +name: Ubuntu Build And Test + +on: + push: + branches: + - main + - master + pull_request: + workflow_dispatch: + +jobs: + build-and-run-sample: + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - name: gcc + install_cmd: sudo apt-get install -y cmake ninja-build g++ python3 + cc: gcc + cxx: g++ + build_cmd: cmake --build build --parallel --target sampleApplication decompressor PackerTest RuntimeTest + - name: clang-21 + install_cmd: | + sudo apt-get install -y ca-certificates cmake gnupg ninja-build python3 wget + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | \ + sudo gpg --dearmor -o /usr/share/keyrings/llvm-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] http://apt.llvm.org/noble/ llvm-toolchain-noble-21 main" | \ + sudo tee /etc/apt/sources.list.d/llvm.list + sudo apt-get update + sudo apt-get install -y clang-21 lld-21 + cc: clang-21 + cxx: clang++-21 + build_cmd: cmake --build build --parallel --verbose --target sampleApplication decompressor PackerTest RuntimeTest + - name: clang-22 + install_cmd: | + sudo apt-get install -y ca-certificates cmake gnupg ninja-build python3 wget + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | \ + sudo gpg --dearmor -o /usr/share/keyrings/llvm-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] http://apt.llvm.org/noble/ llvm-toolchain-noble-22 main" | \ + sudo tee /etc/apt/sources.list.d/llvm.list + sudo apt-get update + sudo apt-get install -y clang-22 lld-22 + cc: clang-22 + cxx: clang++-22 + build_cmd: cmake --build build --parallel --verbose --target sampleApplication decompressor PackerTest RuntimeTest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install dependencies + run: | + sudo apt-get update + ${{ matrix.install_cmd }} + + - name: Show compiler version + if: ${{ matrix.cxx != '' }} + run: | + ${{ matrix.cc }} --version + ${{ matrix.cxx }} --version + + - name: Configure + env: + CC: ${{ matrix.cc }} + CXX: ${{ matrix.cxx }} + run: cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DNANOLOG_BUILD_EXPERIMENTAL_TESTS=ON + + - name: Build + run: ${{ matrix.build_cmd }} + + - name: Run PackerTest + run: ./build/runtime/PackerTest + + - name: Run RuntimeTest + run: ./build/runtime/RuntimeTest + + - name: Run sample application + run: ./build/sample/sampleApplication + + - name: Decompress log and verify output + run: | + ./build/runtime/decompressor decompress /tmp/logFile | tee decompressed.log + grep -F "Simple log message with 0 parameters" decompressed.log + grep -F "A string, pointer, number, and float:" decompressed.log diff --git a/.gitignore b/.gitignore index e4a7470..a7e8bac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ .idea/ -CMakeLists.txt + cmake-build-debug/ make-all.sh -._* \ No newline at end of file +._* +/build +/build-cmake +/build-cmake2 diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..6012af2 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.10) + +if(WIN32) + set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) +endif() + +project(NanoLog LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +option(NANOLOG_BUILD_EXPERIMENTAL_TESTS "Build experimental GoogleTest-based runtime tests" OFF) + +if(NANOLOG_BUILD_EXPERIMENTAL_TESTS) + enable_testing() +endif() + +add_subdirectory(runtime) +add_subdirectory(sample) +add_subdirectory(sample_preprocessor) diff --git a/preprocessor/FunctionGenerator.py b/preprocessor/FunctionGenerator.py index 45e8892..635ae9e 100644 --- a/preprocessor/FunctionGenerator.py +++ b/preprocessor/FunctionGenerator.py @@ -192,10 +192,6 @@ def outputCompilationFiles(outputFileName="BufferStuffer.h", inputFiles=[]): {logLevelEnum} logLevel; }}; -// Start an empty namespace to enclose all the record(debug)/compress/decompress -// and support functions -namespace {{ - using namespace NanoLog::LogLevels; """.format(logLevelEnum=LOG_LEVEL_ENUM)) for logId, code in sorted(mergedCode.items()): @@ -207,8 +203,6 @@ def outputCompilationFiles(outputFileName="BufferStuffer.h", inputFiles=[]): oFile.write(code["decompressFnDef"] + "\n") oFile.write(""" -} // end empty namespace - // Assignment of numerical ids to format NANO_LOG occurrences """) @@ -230,7 +224,7 @@ def outputCompilationFiles(outputFileName="BufferStuffer.h", inputFiles=[]): code["logLevel"] )) - oFile.write("extern const int %s = %d; // %s:%d \"%s\"\n" % ( + oFile.write("extern const uint32_t %s = %d; // %s:%d \"%s\"\n" % ( generateIdVariableNameFromLogId(logId), count, code["filename"], @@ -465,6 +459,8 @@ def generateLogFunctions(self, logLevel, fmtString, if (level > {getLogLevelFn}()) return; + (void) fmtStr; + uint64_t timestamp = PerfUtils::Cycles::rdtsc(); {strlen_declaration}; size_t allocSize = {primitive_size_sum} {strlen_sum} sizeof({entry}); @@ -664,6 +660,8 @@ def generateLogFunctions(self, logLevel, fmtString, else: enumType = "NONE" + enumType = "NanoLogInternal::Log::" + enumType + dictionaryFragment += """ // Fragment {count} if (buffer + sizeof(PrintFragment) diff --git a/preprocessor/parser.py b/preprocessor/parser.py index ff03dba..c157768 100644 --- a/preprocessor/parser.py +++ b/preprocessor/parser.py @@ -322,7 +322,7 @@ def peekNextMeaningfulChar(lines, filePos): # def processFile(inputFile, mapOutputFilename): functionGenerator = FunctionGenerator() - directiveRegex = re.compile("^# (\d+) \"(.*)\"(.*)") + directiveRegex = re.compile(r'^# (\d+) "(.*)"(.*)') with open(inputFile) as f, open(inputFile + "i", 'w') as output: try: diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt new file mode 100644 index 0000000..08ace77 --- /dev/null +++ b/runtime/CMakeLists.txt @@ -0,0 +1,206 @@ +set(NANOLOG_RUNTIME_SOURCES + Cycles.cc + GeneratedCodeStub.cc + Util.cc + Log.cc + NanoLog.cc + RuntimeLogger.cc + TimeTrace.cc +) + +option(NANOLOG_BUILD_EXPERIMENTAL_TESTS "Build experimental GoogleTest-based runtime tests" OFF) + +add_library(NanoLog STATIC ${NANOLOG_RUNTIME_SOURCES}) + +target_include_directories(NanoLog + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} +) + +if(MSVC) + target_compile_options(NanoLog PRIVATE /W4) +elseif(WIN32) + target_compile_options(NanoLog PRIVATE -Wall -Wextra) +else() + target_compile_options(NanoLog PRIVATE -Wall -Wextra -Wformat -Werror=format) +endif() + +if(NOT WIN32) + find_package(Threads REQUIRED) + target_link_libraries(NanoLog PUBLIC Threads::Threads) +endif() + +if(UNIX AND NOT APPLE) + target_link_libraries(NanoLog PUBLIC rt) +endif() + +add_executable(decompressor + Cycles.cc + GeneratedCodeStub.cc + Util.cc + Log.cc + LogDecompressor.cc +) + +target_include_directories(decompressor + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} +) + +if(MSVC) + target_compile_options(decompressor PRIVATE /W4) +elseif(WIN32) + target_compile_options(decompressor PRIVATE -Wall -Wextra) +else() + target_compile_options(decompressor PRIVATE -Wall -Wextra) +endif() + +if(NOT WIN32) + target_link_libraries(decompressor + PRIVATE + Threads::Threads + ) +endif() + +if(UNIX AND NOT APPLE) + target_link_libraries(decompressor PRIVATE rt) +endif() + +if(NANOLOG_BUILD_EXPERIMENTAL_TESTS) + if(EXISTS "${PROJECT_SOURCE_DIR}/googletest/CMakeLists.txt") + enable_testing() + set(CMAKE_POLICY_VERSION_MINIMUM 3.5) + add_subdirectory(${PROJECT_SOURCE_DIR}/googletest + ${CMAKE_BINARY_DIR}/googletest + EXCLUDE_FROM_ALL) + + add_executable(PackerTest + PackerTest.cc + ) + + target_include_directories(PackerTest + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ) + + if(MSVC) + target_compile_options(PackerTest PRIVATE /W4) + elseif(WIN32) + target_compile_options(PackerTest PRIVATE -Wall -Wextra) + else() + target_compile_options(PackerTest PRIVATE -Wall -Wextra) + endif() + + target_link_libraries(PackerTest + PRIVATE + gtest + gtest_main + ) + + add_test(NAME PackerTest COMMAND PackerTest) + + find_package(Python3 REQUIRED COMPONENTS Interpreter) + + set(NANOLOG_TESTHELPER_DIR ${CMAKE_CURRENT_SOURCE_DIR}/testHelper) + set(NANOLOG_TESTHELPER_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/testHelper) + set(NANOLOG_TESTHELPER_PREPROCESSED ${NANOLOG_TESTHELPER_BUILD_DIR}/client.cc.i) + set(NANOLOG_TESTHELPER_INJECTED ${NANOLOG_TESTHELPER_PREPROCESSED}i) + set(NANOLOG_TESTHELPER_MAP ${NANOLOG_TESTHELPER_BUILD_DIR}/client.map) + set(NANOLOG_TESTHELPER_GENERATED ${NANOLOG_TESTHELPER_BUILD_DIR}/GeneratedCode.cc) + + file(MAKE_DIRECTORY ${NANOLOG_TESTHELPER_BUILD_DIR}) + + add_custom_command( + OUTPUT ${NANOLOG_TESTHELPER_PREPROCESSED} + COMMAND ${CMAKE_COMMAND} -E make_directory ${NANOLOG_TESTHELPER_BUILD_DIR} + COMMAND ${CMAKE_CXX_COMPILER} + -E + -I . + testHelper/client.cc + -o ${NANOLOG_TESTHELPER_PREPROCESSED} + -std=c++11 + DEPENDS ${NANOLOG_TESTHELPER_DIR}/client.cc + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + VERBATIM + ) + + add_custom_command( + OUTPUT ${NANOLOG_TESTHELPER_INJECTED} ${NANOLOG_TESTHELPER_MAP} + COMMAND ${Python3_EXECUTABLE} + ${PROJECT_SOURCE_DIR}/preprocessor/parser.py + --mapOutput=${NANOLOG_TESTHELPER_MAP} + ${NANOLOG_TESTHELPER_PREPROCESSED} + DEPENDS + ${NANOLOG_TESTHELPER_PREPROCESSED} + ${PROJECT_SOURCE_DIR}/preprocessor/parser.py + ${PROJECT_SOURCE_DIR}/preprocessor/FunctionGenerator.py + ${PROJECT_SOURCE_DIR}/preprocessor/docopt.py + VERBATIM + ) + + add_custom_command( + OUTPUT ${NANOLOG_TESTHELPER_GENERATED} + COMMAND ${Python3_EXECUTABLE} + ${PROJECT_SOURCE_DIR}/preprocessor/parser.py + --combinedOutput=${NANOLOG_TESTHELPER_GENERATED} + ${NANOLOG_TESTHELPER_MAP} + DEPENDS + ${NANOLOG_TESTHELPER_MAP} + ${PROJECT_SOURCE_DIR}/preprocessor/parser.py + ${PROJECT_SOURCE_DIR}/preprocessor/FunctionGenerator.py + ${PROJECT_SOURCE_DIR}/preprocessor/docopt.py + VERBATIM + ) + + set(NANOLOG_TEST_RUNTIME_SOURCES + Cycles.cc + Util.cc + Log.cc + NanoLog.cc + RuntimeLogger.cc + TimeTrace.cc + ) + + add_executable(RuntimeTest + LogTest.cc + NanoLogTest.cc + NanoLogCpp17Test.cc + ${NANOLOG_TEST_RUNTIME_SOURCES} + ${NANOLOG_TESTHELPER_GENERATED} + ) + + target_compile_definitions(RuntimeTest PRIVATE PREPROCESSOR_NANOLOG) + + target_include_directories(RuntimeTest + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${NANOLOG_TESTHELPER_BUILD_DIR} + ) + + if(MSVC) + target_compile_options(RuntimeTest PRIVATE /W4) + elseif(WIN32) + target_compile_options(RuntimeTest PRIVATE -Wall -Wextra) + else() + target_compile_options(RuntimeTest PRIVATE -Wall -Wextra -Wno-unused-parameter) + endif() + + target_link_libraries(RuntimeTest + PRIVATE + gtest + gtest_main + ) + + if(NOT WIN32) + target_link_libraries(RuntimeTest PRIVATE Threads::Threads) + endif() + + if(UNIX AND NOT APPLE) + target_link_libraries(RuntimeTest PRIVATE rt) + endif() + + add_test(NAME RuntimeTest COMMAND RuntimeTest) + else() + message(STATUS "NANOLOG_BUILD_EXPERIMENTAL_TESTS=ON, but googletest submodule is missing; skipping PackerTest") + endif() +endif() diff --git a/runtime/Config.h b/runtime/Config.h index 3ee6d87..524b622 100644 --- a/runtime/Config.h +++ b/runtime/Config.h @@ -25,7 +25,11 @@ namespace NanoLogConfig { // Controls in what mode the compressed log file will be opened +#ifdef _WIN32 + static const int FILE_PARAMS = O_APPEND | O_RDWR | O_CREAT | O_BINARY; +#else static const int FILE_PARAMS = O_APPEND|O_RDWR|O_CREAT|O_NOATIME|O_DSYNC; +#endif // Location of the initial log file static const char DEFAULT_LOG_FILE[] = "./compressedLog"; diff --git a/runtime/GeneratedCodeStub.cc b/runtime/GeneratedCodeStub.cc new file mode 100644 index 0000000..c2c4d0e --- /dev/null +++ b/runtime/GeneratedCodeStub.cc @@ -0,0 +1,31 @@ +#include "GeneratedCode.h" + +namespace GeneratedFunctions { + +size_t numLogIds = 0; + +LogMetadata logId2Metadata[] = { + {nullptr, nullptr, 0, NanoLog::SILENT_LOG_LEVEL}, +}; + +ssize_t +(*compressFnArray[])(NanoLogInternal::Log::UncompressedEntry *re, char *out) = { + nullptr, +}; + +void +(*decompressAndPrintFnArray[])(const char **in, + FILE *outputFd, + void (*aggFn)(const char*, ...)) = { + nullptr, +}; + +long int +writeDictionary(char *buffer, char *endOfBuffer) +{ + (void)buffer; + (void)endOfBuffer; + return 0; +} + +} // namespace GeneratedFunctions diff --git a/runtime/Log.cc b/runtime/Log.cc index d941041..3c3d50a 100644 --- a/runtime/Log.cc +++ b/runtime/Log.cc @@ -14,8 +14,7 @@ */ #include - -#include +#include #include #include @@ -226,7 +225,7 @@ Log::Encoder::encodeLogMsgs(char *from, while (remaining > 0) { auto *entry = reinterpret_cast(from); - if (entry->entrySize > remaining) { + if (entry->entrySize > static_cast(remaining)) { if (entry->entrySize < (NanoLogConfig::STAGING_BUFFER_SIZE/2)) break; @@ -333,9 +332,9 @@ Log::Encoder::encodeLogMsgs(char *from, "log message (id=%u) during compression. If " "you are using Preprocessor NanoLog, there is " "be a problem with your integration (static " - "logs detected=%lu).\r\n", + "logs detected=%zu).\r\n", entry->fmtId, - GeneratedFunctions::numLogIds); + GeneratedFunctions::numLogIds); } break; @@ -349,7 +348,7 @@ Log::Encoder::encodeLogMsgs(char *from, printf("\t%s\r\n", dictionary.at(entry->fmtId).formatString); #endif - if (entry->entrySize > remaining) { + if (entry->entrySize > static_cast(remaining)) { if (entry->entrySize < (NanoLogConfig::STAGING_BUFFER_SIZE/2)) break; @@ -688,7 +687,7 @@ Log::Decoder::readDictionary(FILE *fd, bool flushOldDictionary) { if (newEnd != endOfRawMetadata) { fprintf(stderr, "Error: Log dictionary is inconsistent; " - "expected %lu bytes, but read %lu bytes\r\n", + "expected %td bytes, but read %td bytes\r\n", newEnd - start, endOfRawMetadata - start); return false; @@ -696,7 +695,7 @@ Log::Decoder::readDictionary(FILE *fd, bool flushOldDictionary) { if (fmtId2metadata.size() != checkpoint.totalMetadataEntries) { fprintf(stderr, "Error: Missing log metadata detected; " - "expected %u messages, but only found %lu\r\n", + "expected %u messages, but only found %zu\r\n", checkpoint.totalMetadataEntries, fmtId2metadata.size()); return false; @@ -839,7 +838,9 @@ Log::Decoder::createMicroCode(char **microCode, fm->logLevel = severity; fm->lineNumber = linenum; fm->filenameLength = static_cast(strlen(filename) + 1); - *microCode = stpcpy(*microCode, filename) + 1; + size_t filenameBytes = strlen(filename) + 1; + memcpy(*microCode, filename, filenameBytes); + *microCode += filenameBytes; fm->numNibbles = 0; fm->numPrintFragments = 0; @@ -992,12 +993,11 @@ Log::Decoder::readDictionaryFragment(FILE *fd) { // These buffers start us off with some statically allocated space. // Should we need more, we will malloc it. size_t bufferSize = 10*1024; - char filenameBuffer[bufferSize]; - char formatBuffer[bufferSize]; + std::vector filenameBuffer(bufferSize); + std::vector formatBuffer(bufferSize); - bool newBuffersAllocated = false; - char *filename = filenameBuffer; - char *format = formatBuffer; + char *filename = filenameBuffer.data(); + char *format = formatBuffer.data(); DictionaryFragment df; size_t bytesRead = fread(&df, 1, sizeof(DictionaryFragment), fd); @@ -1021,21 +1021,16 @@ Log::Decoder::readDictionaryFragment(FILE *fd) { if (bufferSize < cli.filenameLength || bufferSize < cli.formatStringLength) { - if (newBuffersAllocated) { - free(filename); - free(format); - filename = format = nullptr; - } - - newBuffersAllocated = true; bufferSize = 2*std::max(cli.filenameLength, cli.formatStringLength); - filename = static_cast(malloc(bufferSize)); - format = static_cast(malloc(bufferSize)); + filenameBuffer.resize(bufferSize); + formatBuffer.resize(bufferSize); + filename = filenameBuffer.data(); + format = formatBuffer.data(); if (filename == nullptr || format == nullptr) { fprintf(stderr, "Internal Error: Could not allocate enough " "memory to store the filename/format strings " - "in the dictionary. Tried to allocate %lu bytes " + "in the dictionary. Tried to allocate %zu bytes " "to store the %u and %u byte filename and " "format string lengths respectively", bufferSize, @@ -1066,11 +1061,6 @@ Log::Decoder::readDictionaryFragment(FILE *fd) { cli.severity); } - if (newBuffersAllocated) { - free(filename); - free(format); - } - return true; } @@ -1321,6 +1311,10 @@ Log::Decoder::BufferFragment::decompressNextLogStatement(FILE *outputFd, long aggregationFilterId, void (*aggregationFn)(const char*, ...)) { +#ifndef PREPROCESSOR_NANOLOG + (void)aggregationFilterId; + (void)aggregationFn; +#endif double secondsSinceCheckpoint, nanos = 0.0; char timeString[32]; @@ -1382,14 +1376,15 @@ Log::Decoder::BufferFragment::decompressNextLogStatement(FILE *outputFd, } void (*aggFn)(const char*, ...) = nullptr; - if (aggregationFilterId == nextLogId) + if (aggregationFilterId >= 0 && + static_cast(aggregationFilterId) == nextLogId) aggFn = aggregationFn; if (nextLogId >= GeneratedFunctions::numLogIds) { fprintf(stderr, "Log message id=%u not found in the generated " "functions list for Preprocessor NanoLog.\r\n" "This indicates either a corrupt log file or " - "a mismatched decompressor as we only have %lu " + "a mismatched decompressor as we only have %zu " "generated functions\r\n" , nextLogId , GeneratedFunctions::numLogIds); @@ -1776,7 +1771,7 @@ Log::Decoder::internalDecompressUnordered(FILE* outputFd, if (outputFd) fprintf(outputFd, "\r\n\r\n# Decompression Complete after printing " - "%lu log messages\r\n", logMsgsPrinted); + "%" PRIu64 " log messages\r\n", logMsgsPrinted); freeBufferFragment(bf); return good; diff --git a/runtime/Log.h b/runtime/Log.h index abf2873..f8d48a1 100644 --- a/runtime/Log.h +++ b/runtime/Log.h @@ -822,6 +822,7 @@ namespace Log { */ inline void push(long double phony) { + (void) phony; push(-1); } @@ -843,6 +844,7 @@ namespace Log { inline typename std::enable_if::value, T>::type get(int argNum) { + (void) argNum; fprintf(stderr, "**ERROR** Aggregating on Long Doubles is " "currently unsupported\r\n"); #ifndef TESTUTIL_H diff --git a/runtime/LogDecompressor.cc b/runtime/LogDecompressor.cc index 5ac00a4..aa8f653 100644 --- a/runtime/LogDecompressor.cc +++ b/runtime/LogDecompressor.cc @@ -14,6 +14,7 @@ */ #include +#include #include #include @@ -61,7 +62,7 @@ printLogMetadataContainingSubstring(std::string searchString) for (auto id : matchingLogIds) { GeneratedFunctions::LogMetadata lm = GeneratedFunctions::logId2Metadata[id]; - printf("%4lu | %-20s | %-4u | %s\r\n", id, lm.fileName, lm.lineNumber, + printf("%4zu | %-20s | %-4u | %s\r\n", id, lm.fileName, lm.lineNumber, lm.fmtString); } } @@ -250,7 +251,7 @@ int main(int argc, char** argv) { double rcdfTime = PerfUtils::Cycles::toSeconds(stop - start); double totalTime = reserveTime + decodeTime + rcdfTime; - printf("# Took %0.2lf seconds to aggregate %lu time entries " + printf("# Took %0.2lf seconds to aggregate %zu time entries " "(%0.2lf ns/event avg)\r\n", totalTime, interLogTimes.size(), 1.0e9 * totalTime / double(interLogTimes.size())); @@ -270,7 +271,7 @@ int main(int argc, char** argv) { if (outputFd) fprintf(outputFd, "\r\n\r\n# Decompression Complete after printing " - "%ld log messages\r\n", numLogMsgs); + "%" PRId64 " log messages\r\n", numLogMsgs); return 0; } @@ -282,7 +283,7 @@ int main(int argc, char** argv) { if (outputFd) fprintf(outputFd, "\r\n\r\n# Decompression Complete after printing " - "%ld log messages\r\n", numLogMsgs); + "%" PRId64 " log messages\r\n", numLogMsgs); return 0; } @@ -321,7 +322,7 @@ int main(int argc, char** argv) { GeneratedFunctions::logId2Metadata[filterId].fmtString, filterId); #endif // PREPROCESSOR_NANOLOG - printf("Logs Encountered: %lu\r\n", numLogs); + printf("Logs Encountered: %" PRIu64 "\r\n", numLogs); printf("Matching Logs: %ld (%0.2lf%%)\r\n", count, (100.0*(double)count)/(double)numLogs); printf("Min: %ld\r\n", min); diff --git a/runtime/LogTest.cc b/runtime/LogTest.cc index 235103b..ea543e4 100644 --- a/runtime/LogTest.cc +++ b/runtime/LogTest.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,16 @@ #include "Log.h" #include "GeneratedCode.h" +#ifdef _WIN32 +#define TEST_FILE_PATH "testFile" +#define DECOMP_FILE_PATH "testFile2" +#define NULL_DEVICE_PATH "NUL" +#else +#define TEST_FILE_PATH "/tmp/testFile" +#define DECOMP_FILE_PATH "/tmp/testFile2" +#define NULL_DEVICE_PATH "/dev/null" +#endif + extern int __fmtId__Simple32log32message32with32032parameters__testHelper47client46cc__20__; extern int __fmtId__This32is32a32string3237s__testHelper47client46cc__21__; @@ -122,7 +133,7 @@ TEST_F(LogTest, maxSizeOfHeader) { } TEST_F(LogTest, peekEntryType) { - const char *testFile = "/tmp/testFile"; + const char *testFile = TEST_FILE_PATH; char buffer[100]; UnknownHeader *header = reinterpret_cast(buffer); @@ -142,7 +153,7 @@ TEST_F(LogTest, peekEntryType) { oFile.write(buffer, 4); oFile.close(); - FILE *in = fopen(testFile, "r"); + FILE *in = fopen(testFile, "rb"); ASSERT_NE(nullptr, in); EXPECT_EQ(EntryType::LOG_MSGS_OR_DIC, peekEntryType(in)); @@ -305,7 +316,7 @@ TEST_F(LogTest, insertCheckpoint_end2end) { char backing_buffer[2048]; char *writePos = backing_buffer; char *endOfBuffer = backing_buffer + 2048; - const char *testFile = "/tmp/testFile"; + const char *testFile = TEST_FILE_PATH; // Write the buffer to a file and read it back std::ofstream oFile; @@ -313,7 +324,7 @@ TEST_F(LogTest, insertCheckpoint_end2end) { oFile.write(backing_buffer, 0); oFile.close(); - FILE *in = fopen(testFile, "r"); + FILE *in = fopen(testFile, "rb"); ASSERT_NE(nullptr, in); ASSERT_FALSE(readCheckpoint(cp1, in)); @@ -323,11 +334,11 @@ TEST_F(LogTest, insertCheckpoint_end2end) { // Write the buffer to a file and read it back fclose(in); - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(backing_buffer, writePos - backing_buffer); oFile.close(); - in = fopen(testFile, "r"); + in = fopen(testFile, "rb"); ASSERT_NE(nullptr, in); ASSERT_TRUE(readCheckpoint(cp1, in)); @@ -807,12 +818,12 @@ TEST_F(LogTest, swapBuffer) { TEST_F(LogTest, Decoder_open) { char buffer[1000]; - const char *testFile = "/tmp/testFile"; + const char *testFile = TEST_FILE_PATH; Decoder dc; // Open an invalid file testing::internal::CaptureStderr(); - EXPECT_FALSE(dc.open("/dev/null")); + EXPECT_FALSE(dc.open(NULL_DEVICE_PATH)); EXPECT_TRUE(dc.filename.empty()); EXPECT_EQ(0U, dc.inputFd); EXPECT_STREQ("Error: Could not read initial checkpoint, " @@ -821,7 +832,7 @@ TEST_F(LogTest, Decoder_open) { // Write back an empty file and try to open it. std::ofstream oFile; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(buffer, 0); oFile.close(); @@ -834,14 +845,14 @@ TEST_F(LogTest, Decoder_open) { testing::internal::GetCapturedStderr().c_str()); // Write back an invalid, but not-empty file - oFile.open(testFile); - bzero(buffer, 1000); + oFile.open(testFile, std::ios::binary); + std::memset(buffer, 0, 1000); oFile.write(buffer, 100); oFile.close(); // Finally, open a file that at least has a valid checkpoint Encoder encoder(buffer, 1000); - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(buffer, encoder.getEncodedBytes()); oFile.close(); @@ -902,7 +913,7 @@ TEST_F(LogTest, decoder_insertCheckpoint) { TEST_F(LogTest, decoder_readDictionary) { - const char *testFile = "/tmp/testFile"; + const char *testFile = TEST_FILE_PATH; char backing_buffer[4096]; char *writePos = backing_buffer; char *endOfBuffer = backing_buffer + sizeof(backing_buffer); @@ -967,7 +978,7 @@ TEST_F(LogTest, decoder_readDictionary) { ck->newMetadataBytes = dictionaryBytes; std::ofstream oFile; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(backing_buffer, writePos - backing_buffer); oFile.close(); @@ -999,7 +1010,7 @@ TEST_F(LogTest, decoder_readDictionary) { ck->newMetadataBytes = dictionaryBytes; std::ofstream oFile; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(backing_buffer, writePos - backing_buffer); oFile.close(); @@ -1026,7 +1037,7 @@ TEST_F(LogTest, decoder_readDictionary) { // Second read ck->totalMetadataEntries = 4; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(backing_buffer, writePos - backing_buffer); oFile.close(); @@ -1064,7 +1075,7 @@ TEST_F(LogTest, decoder_readDictionary) { ck->totalMetadataEntries = 4; ck->newMetadataBytes = 0; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(backing_buffer, sizeof(Checkpoint)); oFile.close(); @@ -1095,7 +1106,7 @@ TEST_F(LogTest, decoder_readDictionary) { ck->newMetadataBytes = dictionaryBytes; ck->totalMetadataEntries = 2; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(backing_buffer, sizeof(Checkpoint) + dictionaryBytes); oFile.close(); @@ -1126,7 +1137,7 @@ TEST_F(LogTest, decoder_readDictionary) { ck->newMetadataBytes = dictionaryBytes; std::ofstream oFile; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(backing_buffer, writePos - backing_buffer - 10); oFile.close(); @@ -1192,13 +1203,13 @@ TEST_F(LogTest, decoder_readDictionary) { } TEST_F(LogTest, decoder_destructor) { - const char *testFile = "/tmp/testFile"; + const char *testFile = TEST_FILE_PATH; char buffer[1000]; // Write back an invalid, but not-empty file std::ofstream oFile; Encoder encoder(buffer, 1000); - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(buffer, encoder.getEncodedBytes()); oFile.close(); @@ -1239,7 +1250,7 @@ TEST_F(LogTest, decoder_allocate_free) { } TEST_F(LogTest, Decoder_readBufferExtent_end2end) { - const char *testFile = "/tmp/testFile"; + const char *testFile = TEST_FILE_PATH; char inputBuffer[100], outputBuffer1[1000]; // Here we use encoder to prefill the output file @@ -1267,7 +1278,7 @@ TEST_F(LogTest, Decoder_readBufferExtent_end2end) { EXPECT_EQ(5U, e.lastBufferIdEncoded); std::ofstream oFile; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(outputBuffer1, e.getEncodedBytes()); oFile.close(); @@ -1295,7 +1306,7 @@ TEST_F(LogTest, Decoder_readBufferExtent_end2end) { } TEST_F(LogTest, Decoder_readBufferExtent_notEnoughSpace) { - const char *testFile = "/tmp/testFile"; + const char *testFile = TEST_FILE_PATH; char inputBuffer[100], goodBuffer[1000], badBuffer[100]; // Here we use encoder to prefill the output file @@ -1327,7 +1338,7 @@ TEST_F(LogTest, Decoder_readBufferExtent_notEnoughSpace) { std::ofstream oFile; // Test a file that is too small - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(badBuffer, 2); oFile.close(); FILE *in = fopen(testFile, "rb"); @@ -1336,8 +1347,8 @@ TEST_F(LogTest, Decoder_readBufferExtent_notEnoughSpace) { fclose(in); // Test a file that contains invalid data - oFile.open(testFile); - bzero(badBuffer, 100); + oFile.open(testFile, std::ios::binary); + std::memset(badBuffer, 0, 100); oFile.write(badBuffer, 100); oFile.close(); in = fopen(testFile, "rb"); @@ -1353,7 +1364,7 @@ TEST_F(LogTest, Decoder_readBufferExtent_notEnoughSpace) { be->threadIdOrPackNibble = 1; be->wrapAround = false; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(badBuffer, sizeof(badBuffer)); oFile.close(); in = fopen(testFile, "rb"); @@ -1362,7 +1373,7 @@ TEST_F(LogTest, Decoder_readBufferExtent_notEnoughSpace) { fclose(in); // Test a file that contains partially correct data - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(goodBuffer, e.getEncodedBytes() - 1); oFile.close(); in = fopen(testFile, "rb"); @@ -1380,7 +1391,7 @@ TEST_F(LogTest, Decoder_readBufferExtent_notEnoughSpace) { ++be; be->entryType = EntryType::BUFFER_EXTENT; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(badBuffer, sizeof(badBuffer)); oFile.close(); in = fopen(testFile, "rb"); @@ -1398,7 +1409,7 @@ void aggregation(const char*, ...) { } TEST_F(LogTest, decompressNextLogStatement) { - const char *testFile = "/tmp/testFile"; + const char *testFile = TEST_FILE_PATH; char inputBuffer[100], goodBuffer[1000]; // Here we use encoder to prefill the output file @@ -1427,7 +1438,7 @@ TEST_F(LogTest, decompressNextLogStatement) { // Write it out and read it back in. std::ofstream oFile; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(goodBuffer, e.getEncodedBytes()); oFile.close(); @@ -1487,8 +1498,8 @@ TEST_F(LogTest, decompressNextLogStatement) { TEST_F(LogTest, Decoder_internalDecompress_end2end) { // First we have to create a log file with encoder. - const char *testFile = "/tmp/testFile"; - const char *decomp = "/tmp/testFile2"; + const char *testFile = TEST_FILE_PATH; + const char *decomp = DECOMP_FILE_PATH; char inputBuffer[1000], buffer[1000]; Encoder encoder(buffer, 1000, false); @@ -1649,7 +1660,7 @@ TEST_F(LogTest, Decoder_internalDecompress_end2end) { // Write it out and read it back in. std::ofstream oFile; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(buffer, encoder.getEncodedBytes()); oFile.close(); @@ -1778,8 +1789,8 @@ TEST_F(LogTest, Decoder_internalDecompress_fileBreaks) { // This test is what happens if we have a break in a file. It should // in theory output up to that point and no more. char inputBuffer[1000], outputBuffer[1000]; - const char *testFile = "/tmp/testFile"; - const char *decomp = "/tmp/testFile2"; + const char *testFile = TEST_FILE_PATH; + const char *decomp = DECOMP_FILE_PATH; UncompressedEntry* ue = reinterpret_cast(inputBuffer); for (int i = 0; i < 5; ++i) { @@ -1809,7 +1820,7 @@ TEST_F(LogTest, Decoder_internalDecompress_fileBreaks) { EXPECT_EQ(5*sizeof(UncompressedEntry), bytesRead); std::ofstream oFile; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(outputBuffer, encoder.getEncodedBytes()); // Encoder 2 does nothing except output a checkpoint @@ -1934,8 +1945,8 @@ TEST_F(LogTest, Decoder_internalDecompress_fileBreaks2) { // This test is what happens if we have a break in a file. It should // in theory output up to that point and no more. char inputBuffer[1000], outputBuffer[1000]; - const char *testFile = "/tmp/testFile"; - const char *decomp = "/tmp/testFile2"; + const char *testFile = TEST_FILE_PATH; + const char *decomp = DECOMP_FILE_PATH; UncompressedEntry* ue = reinterpret_cast(inputBuffer); for (int i = 0; i < 5; ++i) { @@ -1982,7 +1993,7 @@ TEST_F(LogTest, Decoder_internalDecompress_fileBreaks2) { EXPECT_EQ(2*sizeof(UncompressedEntry), bytesRead); std::ofstream oFile; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(outputBuffer, encoder.getEncodedBytes()); Encoder encoder3(outputBuffer, 1000); @@ -2047,8 +2058,8 @@ TEST_F(LogTest, Decoder_internalDecompress_fileBreaks2) { TEST_F(LogTest, Decoder_decompressNextLogStatement_timeTravel) { // Tests what happen when the checkpoint is newer than the log message. char inputBuffer[1000], outputBuffer[1000]; - const char *testFile = "/tmp/testFile"; - const char *decomp = "/tmp/testFile2"; + const char *testFile = TEST_FILE_PATH; + const char *decomp = DECOMP_FILE_PATH; LogMessage logMsg; UncompressedEntry* ue = reinterpret_cast(inputBuffer); @@ -2074,7 +2085,7 @@ TEST_F(LogTest, Decoder_decompressNextLogStatement_timeTravel) { EXPECT_EQ(sizeof(UncompressedEntry), bytesRead); std::ofstream oFile; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(outputBuffer, encoder.getEncodedBytes()); oFile.close(); @@ -2154,8 +2165,8 @@ TEST_F(LogTest, Decoder_decompressNextLogStatement_timeTravel) { TEST_F(LogTest, Decoder_getNextLogStatement) { // First we have to create a log file with encoder. - const char *testFile = "/tmp/testFile"; - const char *decomp = "/tmp/testFile2"; + const char *testFile = TEST_FILE_PATH; + const char *decomp = DECOMP_FILE_PATH; char inputBuffer[1000], buffer[1000]; Encoder encoder(buffer, 1000, false, true); @@ -2234,7 +2245,7 @@ TEST_F(LogTest, Decoder_getNextLogStatement) { // Write it out and read it back in. std::ofstream oFile; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(buffer, encoder.getEncodedBytes()); oFile.close(); @@ -2294,7 +2305,7 @@ TEST_F(LogTest, Decoder_getNextLogStatement) { EXPECT_EQ(uint64_tParamId, logMsg.getLogId()); EXPECT_EQ(30, logMsg.getTimestamp()); EXPECT_EQ(1, logMsg.getNumArgs()); - EXPECT_EQ(3U, logMsg.get(0)); + EXPECT_EQ(3UL, logMsg.get(0)); ASSERT_TRUE(dc.getNextLogStatement(logMsg)); ASSERT_TRUE(logMsg.valid()); @@ -2348,8 +2359,8 @@ getCount() { TEST_F(LogTest, Decoder_internalDecompress_aggregationFn) { // First we have to create a log file with encoder. - const char *testFile = "/tmp/testFile"; - const char *decomp = "/tmp/testFile2"; + const char *testFile = TEST_FILE_PATH; + const char *decomp = DECOMP_FILE_PATH; char inputBuffer[1000], buffer[1000]; Encoder encoder(buffer, 1000, false); @@ -2510,7 +2521,7 @@ TEST_F(LogTest, Decoder_internalDecompress_aggregationFn) { // Write it out and read it back in. std::ofstream oFile; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(buffer, encoder.getEncodedBytes()); oFile.close(); @@ -2842,7 +2853,7 @@ TEST_F(LogTest, readDictionaryFragment) { df->newMetadataBytes = 0; std::ofstream oFile; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(buffer, sizeof(DictionaryFragment) - 1); oFile.close(); @@ -2857,7 +2868,7 @@ TEST_F(LogTest, readDictionaryFragment) { std::remove(testFile); // Just enough, but header only - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(buffer, sizeof(DictionaryFragment)); oFile.close(); @@ -2899,7 +2910,7 @@ TEST_F(LogTest, readDictionaryFragment) { df->newMetadataBytes = writePos - buffer; df->totalMetadataEntries = 2; - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(buffer, sizeof(DictionaryFragment) + sizeof(CompressedLogInfo) - 1); oFile.close(); @@ -2913,7 +2924,7 @@ TEST_F(LogTest, readDictionaryFragment) { std::remove(testFile); // Header and complete CompressedInfo, but incomplete filenames - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(buffer, sizeof(DictionaryFragment) + sizeof(CompressedLogInfo) + strlen(filename) @@ -2930,7 +2941,7 @@ TEST_F(LogTest, readDictionaryFragment) { std::remove(testFile); // Finally, one that works okay - oFile.open(testFile); + oFile.open(testFile, std::ios::binary); oFile.write(buffer, writePos - buffer); oFile.close(); @@ -3075,4 +3086,4 @@ TEST_F(LogTest, LogMessage_reset) { EXPECT_EQ(nullptr, la.rawArgsExtension); } -}; //namespace \ No newline at end of file +}; //namespace diff --git a/runtime/NanoLogCpp17.h b/runtime/NanoLogCpp17.h index ffb063e..f92491f 100644 --- a/runtime/NanoLogCpp17.h +++ b/runtime/NanoLogCpp17.h @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -465,7 +466,7 @@ store_argument(char **storage, * \param rest * Rest of the remaining variable number of arguments */ -template +template inline void store_arguments(const std::array& paramTypes, size_t (&stringBytes)[M], @@ -482,7 +483,7 @@ store_arguments(const std::array& paramTypes, * Specialization of store_arguments that processes no arguments, i.e. this * is the end of the head/rest recursion. See above for full documentation. */ -template +template inline void store_arguments(const std::array&, size_t (&stringSizes)[M], @@ -696,7 +697,7 @@ getArgSize(const ParamType fmtType, * Total number of bytes needed to represent all arguments with no * compression in the NanoLog system. */ -template +template inline size_t getArgSizes(const std::array& argFmtTypes, uint64_t &previousPrecision, @@ -713,7 +714,7 @@ getArgSizes(const std::array& argFmtTypes, * Specialization for getArgSizes when there are no arguments, i.e. it is * the end of the recursion. (See above for documentation) */ -template +template inline size_t getArgSizes(const std::array&, uint64_t &, size_t (&)[M]) { @@ -787,7 +788,7 @@ compressSingle(BufferUtils::TwoNibbles* nibbles, }(); #pragma GCC diagnostic pop - bzero(*out, characterWidth); + std::memset(*out, 0, characterWidth); *out += characterWidth; return; } @@ -991,7 +992,7 @@ compress(int numNibbles, const ParamType *paramTypes, char **input, char **outpu * \param args * Argument pack for all the arguments for the log invocation */ -template +template inline void log(int &logId, const char *filename, diff --git a/runtime/NanoLogCpp17Test.cc b/runtime/NanoLogCpp17Test.cc index 88e1be9..60b00da 100644 --- a/runtime/NanoLogCpp17Test.cc +++ b/runtime/NanoLogCpp17Test.cc @@ -762,7 +762,7 @@ TEST_F(NanoLogCpp17Test, compressSingle) { } TEST_F(NanoLogCpp17Test, compress_internal) { - BufferUtils::TwoNibbles nibbles[10]; + BufferUtils::TwoNibbles nibbles[10] {}; const ParamType isArgString[] = {NON_STRING, STRING_WITH_NO_PRECISION, STRING, @@ -843,10 +843,10 @@ TEST_F(NanoLogCpp17Test, compress_internal) { // Second string EXPECT_EQ(0, memcmp(out, wString, wStringBytes)); out += wStringBytes; - EXPECT_EQ(0, *out); ++out; - EXPECT_EQ(0, *out); ++out; - EXPECT_EQ(0, *out); ++out; - EXPECT_EQ(0, *out); ++out; + for (size_t i = 0; i < sizeof(wchar_t); ++i) { + EXPECT_EQ(0, *out); + ++out; + } } TEST_F(NanoLogCpp17Test, compress) { @@ -924,10 +924,10 @@ TEST_F(NanoLogCpp17Test, compress) { out += wStringBytes; // Make sure the terminal to wchar_t is right - EXPECT_EQ(0, *out); ++out; - EXPECT_EQ(0, *out); ++out; - EXPECT_EQ(0, *out); ++out; - EXPECT_EQ(0, *out); ++out; + for (size_t i = 0; i < sizeof(wchar_t); ++i) { + EXPECT_EQ(0, *out); + ++out; + } } -}; //namespace \ No newline at end of file +}; //namespace diff --git a/runtime/NanoLogTest.cc b/runtime/NanoLogTest.cc index 66310f0..d1cf4cf 100644 --- a/runtime/NanoLogTest.cc +++ b/runtime/NanoLogTest.cc @@ -167,7 +167,9 @@ TEST_F(NanoLogTest, StagingBuffer_finishReservation) { } TEST_F(NanoLogTest, StagingBuffer_finishReservation_asserts) { - +#ifdef NDEBUG + printf("Skipping death assertions in release builds because bounds-check asserts are disabled.\n"); +#else // Case 1a: Ran out of space and didn't reserve (Artificial) EXPECT_EQ(bufferSize, sb->minFreeSpace); sb->minFreeSpace = 10; @@ -203,6 +205,7 @@ TEST_F(NanoLogTest, StagingBuffer_finishReservation_asserts) { sb->producerPos = sb->storage + halfSize - 50; sb->consumerPos = sb->storage + halfSize + 100; sb->reserveSpaceInternal(100); +#endif } TEST_F(NanoLogTest, StagingBuffer_peek) { @@ -267,4 +270,4 @@ TEST_F(NanoLogTest, StagingBuffer_peek) { sb->peek(&bytesAvailable); EXPECT_EQ(10U, bytesAvailable); } -}; //namespace \ No newline at end of file +}; //namespace diff --git a/runtime/Packer.h b/runtime/Packer.h index b936afa..5ec6b98 100644 --- a/runtime/Packer.h +++ b/runtime/Packer.h @@ -102,19 +102,19 @@ pack(char **buffer, T val) { //TODO(syang0) Is this too costly vs. a simple for loop? int numBytes; - if (val < (1UL << 8)) { + if (val < (1ULL << 8)) { numBytes = 1; - } else if (val < (1UL << 16)) { + } else if (val < (1ULL << 16)) { numBytes = 2; - } else if (val < (1UL << 24)) { + } else if (val < (1ULL << 24)) { numBytes = 3; - } else if (val < (1UL << 32)) { + } else if (val < (1ULL << 32)) { numBytes = 4; - } else if (val < (1UL << 40)) { + } else if (val < (1ULL << 40)) { numBytes = 5; - } else if (val < (1UL << 48)) { + } else if (val < (1ULL << 48)) { numBytes = 6; - } else if (val < (1UL << 56)) { + } else if (val < (1ULL << 56)) { numBytes = 7; } else { numBytes = 8; @@ -162,6 +162,7 @@ pack(char **buffer, int64_t val) //TODO(syang0) we should measure the performance of doing it this way // vs taking both the negated and non-negated versions and encoding the smaller +#if !defined(_WIN32) inline int pack(char **buffer, long long int val) { @@ -170,6 +171,7 @@ pack(char **buffer, long long int val) else return 8 + pack(buffer, static_cast(-val)); } +#endif // The following pack functions that specialize on smaller signed types don't // make sense in the context of NanoLog since printf doesn't allow the @@ -340,9 +342,6 @@ class Nibbler { // Indicates whether whether to use the first nibble or second bool onFirstNibble; - // Number of nibbles in this stream - int numNibbles; - // Position in the stream marking the next packed value const char *currPackedValue; @@ -361,7 +360,6 @@ class Nibbler { Nibbler(const char *nibbleStart, int numNibbles) : nibblePosition(reinterpret_cast(nibbleStart)) , onFirstNibble(true) - , numNibbles(numNibbles) , currPackedValue(nibbleStart + (numNibbles + 1)/2) , endOfValues(nullptr) { diff --git a/runtime/PackerTest.cc b/runtime/PackerTest.cc index 7c9ce28..a529c90 100644 --- a/runtime/PackerTest.cc +++ b/runtime/PackerTest.cc @@ -404,13 +404,13 @@ TEST_F(PackerTest, getSizeOfPackedValues) { EXPECT_EQ(18, getSizeOfPackedValues(nibbles, nibbleCntr)); nibbles[nibbleCntr++/2].second = pack(&buffer, uint64_t(1<<24)); EXPECT_EQ(22, getSizeOfPackedValues(nibbles, nibbleCntr)); - nibbles[nibbleCntr++/2].first = pack(&buffer, uint64_t(1UL<<32)); + nibbles[nibbleCntr++/2].first = pack(&buffer, uint64_t(1ULL << 32)); EXPECT_EQ(27, getSizeOfPackedValues(nibbles, nibbleCntr)); - nibbles[nibbleCntr++/2].second = pack(&buffer, uint64_t(1UL<<40)); + nibbles[nibbleCntr++/2].second = pack(&buffer, uint64_t(1ULL << 40)); EXPECT_EQ(33, getSizeOfPackedValues(nibbles, nibbleCntr)); - nibbles[nibbleCntr++/2].first = pack(&buffer, uint64_t(1UL<<48)); + nibbles[nibbleCntr++/2].first = pack(&buffer, uint64_t(1ULL << 48)); EXPECT_EQ(40, getSizeOfPackedValues(nibbles, nibbleCntr)); - nibbles[nibbleCntr++/2].second = pack(&buffer, uint64_t(1UL<<56)); + nibbles[nibbleCntr++/2].second = pack(&buffer, uint64_t(1ULL << 56)); EXPECT_EQ(48, getSizeOfPackedValues(nibbles, nibbleCntr)); // Try signed numbers @@ -448,10 +448,10 @@ TEST_F(PackerTest, nibbler) { nibbles[nibbleCntr++/2].second = pack(&buffer, uint64_t(1<<8)); nibbles[nibbleCntr++/2].first = pack(&buffer, uint64_t(1<<16)); nibbles[nibbleCntr++/2].second = pack(&buffer, uint64_t(1<<24)); - nibbles[nibbleCntr++/2].first = pack(&buffer, uint64_t(1UL<<32)); - nibbles[nibbleCntr++/2].second = pack(&buffer, uint64_t(1UL<<40)); - nibbles[nibbleCntr++/2].first = pack(&buffer, uint64_t(1UL<<48)); - nibbles[nibbleCntr++/2].second = pack(&buffer, uint64_t(1UL<<56)); + nibbles[nibbleCntr++/2].first = pack(&buffer, uint64_t(1ULL << 32)); + nibbles[nibbleCntr++/2].second = pack(&buffer, uint64_t(1ULL << 40)); + nibbles[nibbleCntr++/2].first = pack(&buffer, uint64_t(1ULL << 48)); + nibbles[nibbleCntr++/2].second = pack(&buffer, uint64_t(1ULL << 56)); // Try signed numbers nibbles[nibbleCntr++/2].first = pack(&buffer, int64_t(-(1<<8))); @@ -480,10 +480,10 @@ TEST_F(PackerTest, nibbler) { EXPECT_EQ(1<<8, nb.getNext()); EXPECT_EQ(1<<16, nb.getNext()); EXPECT_EQ(1<<24, nb.getNext()); - EXPECT_EQ(1UL<<32, nb.getNext()); - EXPECT_EQ(1UL<<40, nb.getNext()); - EXPECT_EQ(1UL<<48, nb.getNext()); - EXPECT_EQ(1UL<<56, nb.getNext()); + EXPECT_EQ(1ULL << 32, nb.getNext()); + EXPECT_EQ(1ULL << 40, nb.getNext()); + EXPECT_EQ(1ULL << 48, nb.getNext()); + EXPECT_EQ(1ULL << 56, nb.getNext()); EXPECT_EQ(int64_t(-(1<<8)), nb.getNext()); EXPECT_EQ(int64_t(-(1<<16)), nb.getNext()); @@ -517,7 +517,11 @@ TEST_F(PackerTest, nibbler_assert) { // Consume all the data and prepare for death EXPECT_EQ((float)0.1, nb.getNext()); EXPECT_EQ((double)0.2, nb.getNext()); +#ifdef NDEBUG + printf("Skipping death assertion in release builds because bounds-check asserts are disabled.\n"); +#else EXPECT_DEATH(nb.getNext(), ""); +#endif } } // namespace diff --git a/runtime/Portability.h b/runtime/Portability.h index 129c051..28dcc43 100644 --- a/runtime/Portability.h +++ b/runtime/Portability.h @@ -1,6 +1,36 @@ #ifndef NANOLOG_PORTABILITY_H #define NANOLOG_PORTABILITY_H +#include +#include + +#ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#include +#include +#include +#include +#include +#ifdef ERROR +#undef ERROR +#endif +#ifndef O_DIRECT +#define O_DIRECT 0 +#endif +#else +#include +#include +#include +#include +#endif + #ifdef _MSC_VER #define NANOLOG_ALWAYS_INLINE __forceinline #elif defined(__GNUC__) @@ -49,4 +79,168 @@ #define NANOLOG_PRINTF_FORMAT_ATTR(string_index, first_to_check) #endif -#endif /* NANOLOG_PORTABILITY_H */ \ No newline at end of file +#ifdef _WIN32 +typedef SSIZE_T ssize_t; + +inline char * +stpcpy(char *dest, const char *src) +{ + while (*src != '\0') { + *dest++ = *src++; + } + *dest = '\0'; + return dest; +} + +struct cpu_set_t { + DWORD_PTR mask; +}; + +#ifndef CPU_ZERO +#define CPU_ZERO(set) ((set)->mask = 0) +#endif + +#ifndef CPU_SET +#define CPU_SET(cpu, set) ((set)->mask |= (DWORD_PTR(1) << (cpu))) +#endif +#endif + +namespace NanoLogPlatform { + +#ifdef _WIN32 +struct AioCb { + int aio_fildes = -1; + void *aio_buf = nullptr; + std::size_t aio_nbytes = 0; + int errorCode = 0; + ssize_t result = 0; + bool pending = false; +}; +#else +using AioCb = aiocb; +#endif + +inline int +openFile(const char *filename, int flags, int mode) +{ +#ifdef _WIN32 + return _open(filename, flags, mode); +#else + return open(filename, flags, mode); +#endif +} + +inline int +closeFile(int fd) +{ +#ifdef _WIN32 + return _close(fd); +#else + return close(fd); +#endif +} + +inline int +accessFile(const char *filename, int mode) +{ +#ifdef _WIN32 + return _access(filename, mode); +#else + return access(filename, mode); +#endif +} + +inline int +flushFile(int fd) +{ +#ifdef _WIN32 + return _commit(fd); +#else + return fdatasync(fd); +#endif +} + +inline int +alignedAlloc(void **ptr, std::size_t alignment, std::size_t size) +{ +#ifdef _WIN32 + *ptr = _aligned_malloc(size, alignment); + return (*ptr == nullptr) ? ENOMEM : 0; +#else + return posix_memalign(ptr, alignment, size); +#endif +} + +inline void +alignedFree(void *ptr) +{ +#ifdef _WIN32 + _aligned_free(ptr); +#else + free(ptr); +#endif +} + +inline int +currentCpu() +{ +#ifdef _WIN32 + return static_cast(GetCurrentProcessorNumber()); +#else + return sched_getcpu(); +#endif +} + +inline int +aioWrite(AioCb *cb) +{ +#ifdef _WIN32 + cb->pending = true; + errno = 0; + cb->result = _write(cb->aio_fildes, + cb->aio_buf, + static_cast(cb->aio_nbytes)); + cb->errorCode = (cb->result < 0) ? errno : 0; + cb->pending = false; + return (cb->result < 0) ? -1 : 0; +#else + return aio_write(cb); +#endif +} + +inline int +aioError(AioCb *cb) +{ +#ifdef _WIN32 + return cb->pending ? EINPROGRESS : cb->errorCode; +#else + return aio_error(cb); +#endif +} + +inline ssize_t +aioReturn(AioCb *cb) +{ +#ifdef _WIN32 + return cb->result; +#else + return aio_return(cb); +#endif +} + +inline int +aioSuspend(const AioCb *const list[], int nent, const struct timespec *timeout) +{ +#ifdef _WIN32 + (void)list; + (void)nent; + (void)timeout; + return 0; +#else + return aio_suspend(list, nent, timeout); +#endif +} + +} // namespace NanoLogPlatform + +#endif /* NANOLOG_PORTABILITY_H */ diff --git a/runtime/RuntimeLogger.cc b/runtime/RuntimeLogger.cc index 56b7c0f..3292383 100644 --- a/runtime/RuntimeLogger.cc +++ b/runtime/RuntimeLogger.cc @@ -15,13 +15,13 @@ #include +#include #include #include #include #include #include #include -#include #include "Cycles.h" /* Cycles::rdtsc() */ #include "RuntimeLogger.h" @@ -73,7 +73,7 @@ RuntimeLogger::RuntimeLogger() stagingBufferPeekDist[i] = 0; const char *filename = NanoLogConfig::DEFAULT_LOG_FILE; - outputFd = open(filename, NanoLogConfig::FILE_PARAMS, 0666); + outputFd = NanoLogPlatform::openFile(filename, NanoLogConfig::FILE_PARAMS, 0666); if (outputFd < 0) { fprintf(stderr, "NanoLog could not open the default file location " "for the log file (\"%s\").\r\n Please check the permissions " @@ -84,16 +84,20 @@ RuntimeLogger::RuntimeLogger() memset(&aioCb, 0, sizeof(aioCb)); - int err = posix_memalign(reinterpret_cast(&compressingBuffer), - 512, NanoLogConfig::OUTPUT_BUFFER_SIZE); + int err = NanoLogPlatform::alignedAlloc( + reinterpret_cast(&compressingBuffer), + 512, + NanoLogConfig::OUTPUT_BUFFER_SIZE); if (err) { perror("The NanoLog system was not able to allocate enough memory " "to support its operations. Quitting...\r\n"); std::exit(-1); } - err = posix_memalign(reinterpret_cast(&outputDoubleBuffer), - 512, NanoLogConfig::OUTPUT_BUFFER_SIZE); + err = NanoLogPlatform::alignedAlloc( + reinterpret_cast(&outputDoubleBuffer), + 512, + NanoLogConfig::OUTPUT_BUFFER_SIZE); if (err) { perror("The NanoLog system was not able to allocate enough memory " "to support its operations. Quitting...\r\n"); @@ -121,17 +125,17 @@ RuntimeLogger::~RuntimeLogger() { // Free all the data structures if (compressingBuffer) { - free(compressingBuffer); + NanoLogPlatform::alignedFree(compressingBuffer); compressingBuffer = nullptr; } if (outputDoubleBuffer) { - free(outputDoubleBuffer); + NanoLogPlatform::alignedFree(outputDoubleBuffer); outputDoubleBuffer = nullptr; } if (outputFd > 0) - close(outputFd); + NanoLogPlatform::closeFile(outputFd); outputFd = 0; } @@ -143,7 +147,7 @@ RuntimeLogger::getStats() { char buffer[1024]; // Leaks abstraction, but basically flush so we get all the time uint64_t start = PerfUtils::Cycles::rdtsc(); - fdatasync(nanoLogSingleton.outputFd); + NanoLogPlatform::flushFile(nanoLogSingleton.outputFd); uint64_t stop = PerfUtils::Cycles::rdtsc(); nanoLogSingleton.cyclesDiskIO_upperBound += (stop - start); @@ -163,7 +167,7 @@ RuntimeLogger::getStats() { nanoLogSingleton.logsProcessed); snprintf(buffer, 1024, - "\r\nWrote %lu events (%0.2lf MB) in %0.3lf seconds " + "\r\nWrote %" PRIu64 " events (%0.2lf MB) in %0.3lf seconds " "(%0.3lf seconds spent compressing)\r\n", nanoLogSingleton.logsProcessed, totalBytesWrittenDouble / 1.0e6, @@ -219,7 +223,8 @@ RuntimeLogger::getStats() { out << buffer; snprintf(buffer, 1024, "The compression ratio was %0.2lf-%0.2lfx " - "(%lu bytes in, %lu bytes out, %lu pad bytes)\n", + "(%" PRIu64 " bytes in, %" PRIu64 " bytes out, %" PRIu64 + " pad bytes)\n", 1.0 * totalBytesReadDouble / (totalBytesWrittenDouble + padBytesWrittenDouble), 1.0 * totalBytesReadDouble / totalBytesWrittenDouble, @@ -252,7 +257,7 @@ RuntimeLogger::getHistograms() Util::arraySize(nanoLogSingleton.stagingBufferPeekDist); for (size_t i = 0; i < numIntervals; ++i) { snprintf(buffer, 1024 - , "\t%02lu - %02lu%%: %lu\r\n" + , "\t%02zu - %02zu%%: %" PRIu64 "\r\n" , i*100/numIntervals , (i+1)*100/numIntervals , nanoLogSingleton.stagingBufferPeekDist[i]); @@ -268,7 +273,7 @@ RuntimeLogger::getHistograms() out << buffer; snprintf(buffer, 1024, - "\tAllocations : %lu\r\n" + "\tAllocations : %" PRIu64 "\r\n" "\tTimes Blocked : %u\r\n", sb->numAllocations, sb->numTimesProducerBlocked); @@ -319,16 +324,16 @@ RuntimeLogger::preallocate() { void RuntimeLogger::waitForAIO() { if (hasOutstandingOperation) { - if (aio_error(&aioCb) == EINPROGRESS) { - const struct aiocb *const aiocb_list[] = {&aioCb}; - int err = aio_suspend(aiocb_list, 1, NULL); + if (NanoLogPlatform::aioError(&aioCb) == EINPROGRESS) { + const NanoLogPlatform::AioCb *const aiocb_list[] = {&aioCb}; + int err = NanoLogPlatform::aioSuspend(aiocb_list, 1, NULL); if (err != 0) perror("LogCompressor's Posix AIO suspend operation failed"); } - int err = aio_error(&aioCb); - ssize_t ret = aio_return(&aioCb); + int err = NanoLogPlatform::aioError(&aioCb); + ssize_t ret = NanoLogPlatform::aioReturn(&aioCb); if (err != 0) { fprintf(stderr, "LogCompressor's POSIX AIO failed with %d: %s\r\n", @@ -383,7 +388,7 @@ RuntimeLogger::compressionThreadMain() { while (!compressionThreadShouldExit || encoder.getEncodedBytes() > 0 || hasOutstandingOperation) { - coreId = sched_getcpu(); + coreId = NanoLogPlatform::currentCpu(); // Indicates how many bytes we have consumed from the StagingBuffers // in a single iteration of the while above. A value of 0 means we @@ -535,13 +540,13 @@ RuntimeLogger::compressionThreadMain() { } if (hasOutstandingOperation) { - if (aio_error(&aioCb) == EINPROGRESS) { - const struct aiocb *const aiocb_list[] = {&aioCb}; + if (NanoLogPlatform::aioError(&aioCb) == EINPROGRESS) { + const NanoLogPlatform::AioCb *const aiocb_list[] = {&aioCb}; if (outputBufferFull) { // If the output buffer is full and we're not done, // wait for completion cyclesActive += PerfUtils::Cycles::rdtsc() - cyclesAwakeStart; - int err = aio_suspend(aiocb_list, 1, NULL); + int err = NanoLogPlatform::aioSuspend(aiocb_list, 1, NULL); cyclesAwakeStart = PerfUtils::Cycles::rdtsc(); if (err != 0) perror("LogCompressor's Posix AIO " @@ -559,14 +564,14 @@ RuntimeLogger::compressionThreadMain() { cyclesAwakeStart = PerfUtils::Cycles::rdtsc(); } - if (aio_error(&aioCb) == EINPROGRESS) + if (NanoLogPlatform::aioError(&aioCb) == EINPROGRESS) continue; } } // Finishing up the IO - int err = aio_error(&aioCb); - ssize_t ret = aio_return(&aioCb); + int err = NanoLogPlatform::aioError(&aioCb); + ssize_t ret = NanoLogPlatform::aioReturn(&aioCb); if (err != 0) { fprintf(stderr, "LogCompressor's POSIX AIO failed" @@ -612,7 +617,7 @@ RuntimeLogger::compressionThreadMain() { totalBytesWritten += bytesToWrite; cyclesAtLastAIOStart = PerfUtils::Cycles::rdtsc(); - if (aio_write(&aioCb) == -1) + if (NanoLogPlatform::aioWrite(&aioCb) == -1) fprintf(stderr, "Error at aio_write(): %s\n", strerror(errno)); hasOutstandingOperation = true; @@ -632,14 +637,15 @@ RuntimeLogger::compressionThreadMain() { void RuntimeLogger::setLogFile_internal(const char *filename) { // Check if it exists and is readable/writeable - if (access(filename, F_OK) == 0 && access(filename, R_OK | W_OK) != 0) { + if (NanoLogPlatform::accessFile(filename, F_OK) == 0 + && NanoLogPlatform::accessFile(filename, R_OK | W_OK) != 0) { std::string err = "Unable to read/write from new log file: "; err.append(filename); throw std::ios_base::failure(err); } // Try to open the file - int newFd = open(filename, NanoLogConfig::FILE_PARAMS, 0666); + int newFd = NanoLogPlatform::openFile(filename, NanoLogConfig::FILE_PARAMS, 0666); if (newFd < 0) { std::string err = "Unable to open file new log file: '"; err.append(filename); @@ -662,7 +668,7 @@ RuntimeLogger::setLogFile_internal(const char *filename) { compressionThread.join(); if (outputFd > 0) - close(outputFd); + NanoLogPlatform::closeFile(outputFd); outputFd = newFd; // Relaunch thread @@ -843,4 +849,4 @@ RuntimeLogger::StagingBuffer::peek(uint64_t *bytesAvailable) { return consumerPos; } -}; // namespace NanoLog Internal \ No newline at end of file +}; // namespace NanoLog Internal diff --git a/runtime/RuntimeLogger.h b/runtime/RuntimeLogger.h index 03ba2b9..07de2b9 100644 --- a/runtime/RuntimeLogger.h +++ b/runtime/RuntimeLogger.h @@ -16,7 +16,6 @@ #ifndef RUNTIME_NANOLOG_H #define RUNTIME_NANOLOG_H -#include #include #include @@ -231,7 +230,7 @@ using namespace NanoLog; int outputFd; // POSIX AIO structure used to communicate async IO requests - struct aiocb aioCb; + NanoLogPlatform::AioCb aioCb; // Used to stage the compressed log messages before passing it on to the // POSIX AIO library. @@ -439,7 +438,7 @@ using namespace NanoLog; // Number of cycles producer was blocked while waiting for space to // free up in the StagingBuffer for an allocation. - uint64_t cyclesProducerBlocked; + [[maybe_unused]] uint64_t cyclesProducerBlocked; // Number of times the producer was blocked while waiting for space // to free up in the StagingBuffer for an allocation @@ -461,7 +460,7 @@ using namespace NanoLog; // An extra cache-line to separate the variables that are primarily // updated/read by the producer (above) from the ones by the // consumer(below) - char cacheLineSpacer[2*Util::BYTES_PER_CACHE_LINE]; + [[maybe_unused]] char cacheLineSpacer[2*Util::BYTES_PER_CACHE_LINE]; // Position within the storage buffer where the consumer will consume // the next bytes from. This value is only updated by the consumer. diff --git a/runtime/Util.cc b/runtime/Util.cc index b73d9af..c6be58d 100644 --- a/runtime/Util.cc +++ b/runtime/Util.cc @@ -13,7 +13,9 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include +#include #include "Util.h" #include @@ -53,14 +55,14 @@ vformat(const char* format, va_list ap) // Try 1K, if not the return value will tell us how much is necessary. int bufSize = 1024; while (true) { - char buf[bufSize]; + std::vector buf(static_cast(bufSize)); // vsnprintf trashes the va_list, so copy it first va_list aq; __va_copy(aq, ap); - int r = vsnprintf(buf, bufSize, format, aq); + int r = vsnprintf(buf.data(), static_cast(bufSize), format, aq); assert(r >= 0); // old glibc versions returned -1 if (r < bufSize) { - s = buf; + s = buf.data(); break; } bufSize = r + 1; @@ -86,7 +88,8 @@ hexDump(const void *buf, uint64_t bytes) char hex[16][3]; char ascii[17]; - snprintf(offset, sizeof(offset), "%016lx", i); + snprintf(offset, sizeof(offset), "%016llx", + static_cast(i)); offset[sizeof(offset) - 1] = '\0'; for (j = 0; j < 16; j++) { diff --git a/runtime/Util.h b/runtime/Util.h index 87a1350..3d570be 100644 --- a/runtime/Util.h +++ b/runtime/Util.h @@ -18,11 +18,7 @@ #include #include -#include #include -#include -#include -#include #include #include @@ -31,6 +27,12 @@ #include "Portability.h" +#ifndef _WIN32 +#include +#include +#include +#endif + namespace NanoLogInternal { /** @@ -76,10 +78,14 @@ rdpmc(int ecx) * resources */ static -pid_t FORCE_INLINE +uint32_t FORCE_INLINE gettid() { - return static_cast(syscall( __NR_gettid )); +#ifdef _WIN32 + return static_cast(GetCurrentThreadId()); +#else + return static_cast(syscall(__NR_gettid)); +#endif } /** @@ -94,8 +100,12 @@ void pinThreadToCore(int id) { cpu_set_t cpuset; CPU_ZERO(&cpuset); - CPU_SET(id, &cpuset); - assert(sched_setaffinity(0, sizeof(cpuset), &cpuset) == 0); + CPU_SET(id, &cpuset); +#ifdef _WIN32 + assert(SetThreadAffinityMask(GetCurrentThread(), cpuset.mask) != 0); +#else + assert(sched_setaffinity(0, sizeof(cpuset), &cpuset) == 0); +#endif } /** @@ -108,7 +118,15 @@ cpu_set_t getCpuAffinity() { cpu_set_t cpuset; CPU_ZERO(&cpuset); +#ifdef _WIN32 + DWORD_PTR processMask = 0; + [[maybe_unused]] DWORD_PTR systemMask = 0; + (void)systemMask; // silence unused variable warning on Windows + assert(GetProcessAffinityMask(GetCurrentProcess(), &processMask, &systemMask)); + cpuset.mask = processMask; +#else assert(sched_getaffinity(0, sizeof(cpuset), &cpuset) == 0); +#endif return cpuset; } @@ -122,8 +140,12 @@ cpu_set_t getCpuAffinity() { * current thread is permitted to run on. */ static FORCE_INLINE -void setCpuAffinity(cpu_set_t cpuset) { +void setCpuAffinity([[maybe_unused]] cpu_set_t cpuset) { +#ifdef _WIN32 + assert(SetThreadAffinityMask(GetCurrentThread(), cpuset.mask) != 0); +#else assert(sched_setaffinity(0, sizeof(cpuset), &cpuset) == 0); +#endif } /** diff --git a/sample/CMakeLists.txt b/sample/CMakeLists.txt new file mode 100644 index 0000000..4197f9c --- /dev/null +++ b/sample/CMakeLists.txt @@ -0,0 +1,19 @@ +add_executable(sampleApplication main.cc) + +target_link_libraries(sampleApplication + PRIVATE + NanoLog +) + +target_include_directories(sampleApplication + PRIVATE + ${PROJECT_SOURCE_DIR}/runtime +) + +if(MSVC) + target_compile_options(sampleApplication PRIVATE /W4) +elseif(WIN32) + target_compile_options(sampleApplication PRIVATE -DNDEBUG -O3 -g) +else() + target_compile_options(sampleApplication PRIVATE -Werror=format -DNDEBUG -O3 -g) +endif() diff --git a/sample/main.cc b/sample/main.cc index af67390..5935a97 100644 --- a/sample/main.cc +++ b/sample/main.cc @@ -33,7 +33,11 @@ using namespace NanoLog::LogLevels; int main(int argc, char** argv) { // Optional: Set the output location for the NanoLog system. By default // the log will be output to ./compressedLog +#ifdef _WIN32 + NanoLog::setLogFile("logFile"); +#else NanoLog::setLogFile("/tmp/logFile"); +#endif // Optional optimization: pre-allocates thread-local data structures // needed by NanoLog. This can be invoked once per new @@ -91,9 +95,11 @@ void runBenchmark() { time_span = std::chrono::duration_cast>( stop - start).count(); - printf("The total time spent invoking NANO_LOG with no parameters %lu " + printf("The total time spent invoking NANO_LOG with no parameters %llu " "times took %0.2lf seconds (%0.2lf ns/message average)\r\n", - RECORDS, time_span, (time_span/RECORDS)*1e9); + static_cast(RECORDS), + time_span, + (time_span/RECORDS)*1e9); start = std::chrono::high_resolution_clock::now(); // Flush all pending log messages to disk diff --git a/sample_preprocessor/CMakeLists.txt b/sample_preprocessor/CMakeLists.txt new file mode 100644 index 0000000..87343bb --- /dev/null +++ b/sample_preprocessor/CMakeLists.txt @@ -0,0 +1,195 @@ +find_package(Python3 REQUIRED COMPONENTS Interpreter) + +if(NOT WIN32) + find_package(Threads REQUIRED) +endif() + +set(SAMPLE_PREPROC_RUNTIME_SOURCES + ${PROJECT_SOURCE_DIR}/runtime/Cycles.cc + ${PROJECT_SOURCE_DIR}/runtime/NanoLog.cc + ${PROJECT_SOURCE_DIR}/runtime/Util.cc + ${PROJECT_SOURCE_DIR}/runtime/Log.cc + ${PROJECT_SOURCE_DIR}/runtime/RuntimeLogger.cc + ${PROJECT_SOURCE_DIR}/runtime/TimeTrace.cc +) + +set(SAMPLE_PREPROC_RUNTIME_FLAGS + -std=c++11 + -O3 + -DNDEBUG + -g +) + +set(SAMPLE_PREPROC_APP_FLAGS + -std=c++17 + -DNDEBUG + -O3 + -g +) + +set(SAMPLE_PREPROC_GENERATED_DIR ${CMAKE_CURRENT_BINARY_DIR}/generated) +set(SAMPLE_PREPROC_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/main.cc) +set(SAMPLE_PREPROC_PREPROCESSED ${SAMPLE_PREPROC_GENERATED_DIR}/main.cc.i) +set(SAMPLE_PREPROC_INJECTED ${SAMPLE_PREPROC_PREPROCESSED}i) +set(SAMPLE_PREPROC_MAP ${SAMPLE_PREPROC_GENERATED_DIR}/main.cc.map) +set(SAMPLE_PREPROC_GENERATED_CODE ${SAMPLE_PREPROC_GENERATED_DIR}/GeneratedCode.cc) +set(SAMPLE_PREPROC_USER_OBJECT ${SAMPLE_PREPROC_GENERATED_DIR}/main.cc.o) + +file(MAKE_DIRECTORY ${SAMPLE_PREPROC_GENERATED_DIR}) + +add_custom_command( + OUTPUT ${SAMPLE_PREPROC_PREPROCESSED} + COMMAND ${CMAKE_COMMAND} -E make_directory ${SAMPLE_PREPROC_GENERATED_DIR} + COMMAND ${CMAKE_CXX_COMPILER} + -E + -I ${PROJECT_SOURCE_DIR}/runtime + ${SAMPLE_PREPROC_SOURCE} + -o ${SAMPLE_PREPROC_PREPROCESSED} + -std=c++11 + DEPENDS ${SAMPLE_PREPROC_SOURCE} + VERBATIM +) + +add_custom_command( + OUTPUT ${SAMPLE_PREPROC_INJECTED} ${SAMPLE_PREPROC_MAP} + COMMAND ${Python3_EXECUTABLE} + ${PROJECT_SOURCE_DIR}/preprocessor/parser.py + --mapOutput=${SAMPLE_PREPROC_MAP} + ${SAMPLE_PREPROC_PREPROCESSED} + DEPENDS + ${SAMPLE_PREPROC_PREPROCESSED} + ${PROJECT_SOURCE_DIR}/preprocessor/parser.py + ${PROJECT_SOURCE_DIR}/preprocessor/FunctionGenerator.py + ${PROJECT_SOURCE_DIR}/preprocessor/docopt.py + VERBATIM +) + +add_custom_command( + OUTPUT ${SAMPLE_PREPROC_USER_OBJECT} + COMMAND ${CMAKE_CXX_COMPILER} + -c + -o ${SAMPLE_PREPROC_USER_OBJECT} + ${SAMPLE_PREPROC_INJECTED} + ${SAMPLE_PREPROC_APP_FLAGS} + DEPENDS ${SAMPLE_PREPROC_INJECTED} + VERBATIM +) + +add_custom_command( + OUTPUT ${SAMPLE_PREPROC_GENERATED_CODE} + COMMAND ${Python3_EXECUTABLE} + ${PROJECT_SOURCE_DIR}/preprocessor/parser.py + --combinedOutput=${SAMPLE_PREPROC_GENERATED_CODE} + ${SAMPLE_PREPROC_MAP} + DEPENDS + ${SAMPLE_PREPROC_MAP} + ${PROJECT_SOURCE_DIR}/preprocessor/parser.py + ${PROJECT_SOURCE_DIR}/preprocessor/FunctionGenerator.py + ${PROJECT_SOURCE_DIR}/preprocessor/docopt.py + VERBATIM +) + +add_library(samplePreprocessorNanoLog STATIC + ${SAMPLE_PREPROC_RUNTIME_SOURCES} + ${SAMPLE_PREPROC_GENERATED_CODE} +) + +target_compile_definitions(samplePreprocessorNanoLog PRIVATE PREPROCESSOR_NANOLOG) + +target_include_directories(samplePreprocessorNanoLog + PUBLIC + ${PROJECT_SOURCE_DIR}/runtime + PRIVATE + ${SAMPLE_PREPROC_GENERATED_DIR} +) + +if(MSVC) + target_compile_options(samplePreprocessorNanoLog PRIVATE /W4) +elseif(WIN32) + target_compile_options(samplePreprocessorNanoLog PRIVATE ${SAMPLE_PREPROC_RUNTIME_FLAGS} -Wall -Wextra) +else() + target_compile_options(samplePreprocessorNanoLog PRIVATE + ${SAMPLE_PREPROC_RUNTIME_FLAGS} + -Wall + -Wformat=2 + -Wextra + -Wwrite-strings + -Wno-unused-parameter + -Wmissing-format-attribute + -Wno-non-template-friend + -Woverloaded-virtual + -Wcast-qual + -Wcast-align + -Wno-address-of-packed-member + -Wconversion + -Weffc++ + ) +endif() + +if(NOT WIN32) + target_link_libraries(samplePreprocessorNanoLog PUBLIC Threads::Threads) +endif() + +if(UNIX AND NOT APPLE) + target_link_libraries(samplePreprocessorNanoLog PUBLIC rt) +endif() + +add_executable(samplePreprocessorDecompressor + ${SAMPLE_PREPROC_RUNTIME_SOURCES} + ${PROJECT_SOURCE_DIR}/runtime/LogDecompressor.cc + ${SAMPLE_PREPROC_GENERATED_CODE} +) + +set_target_properties(samplePreprocessorDecompressor PROPERTIES OUTPUT_NAME decompressor) + +target_compile_definitions(samplePreprocessorDecompressor PRIVATE PREPROCESSOR_NANOLOG) + +target_include_directories(samplePreprocessorDecompressor + PRIVATE + ${PROJECT_SOURCE_DIR}/runtime + ${SAMPLE_PREPROC_GENERATED_DIR} +) + +if(MSVC) + target_compile_options(samplePreprocessorDecompressor PRIVATE /W4) +elseif(WIN32) + target_compile_options(samplePreprocessorDecompressor PRIVATE ${SAMPLE_PREPROC_RUNTIME_FLAGS} -Wall -Wextra) +else() + target_compile_options(samplePreprocessorDecompressor PRIVATE + ${SAMPLE_PREPROC_RUNTIME_FLAGS} + -Wall + -Wformat=2 + -Wextra + -Wwrite-strings + -Wno-unused-parameter + -Wmissing-format-attribute + -Wno-non-template-friend + -Woverloaded-virtual + -Wcast-qual + -Wcast-align + -Wno-address-of-packed-member + -Wconversion + -Weffc++ + ) +endif() + +if(NOT WIN32) + target_link_libraries(samplePreprocessorDecompressor PRIVATE Threads::Threads) +endif() + +if(UNIX AND NOT APPLE) + target_link_libraries(samplePreprocessorDecompressor PRIVATE rt) +endif() + +set_source_files_properties(${SAMPLE_PREPROC_USER_OBJECT} + PROPERTIES + EXTERNAL_OBJECT TRUE + GENERATED TRUE +) + +add_executable(samplePreprocessorApplication ${SAMPLE_PREPROC_USER_OBJECT}) +set_target_properties(samplePreprocessorApplication PROPERTIES OUTPUT_NAME sampleApplication) + +target_link_libraries(samplePreprocessorApplication PRIVATE samplePreprocessorNanoLog) + +add_dependencies(samplePreprocessorApplication samplePreprocessorDecompressor) diff --git a/sample_preprocessor/main.cc b/sample_preprocessor/main.cc index 7e6d5d8..7451262 100644 --- a/sample_preprocessor/main.cc +++ b/sample_preprocessor/main.cc @@ -37,7 +37,11 @@ int main(int argc, char** argv) { // Optional: Set the output location for the NanoLog system. By default // the log will be output to ./compressedLog +#ifdef _WIN32 + NanoLog::setLogFile("logFile"); +#else NanoLog::setLogFile("/tmp/logFile"); +#endif // Optional optimization: pre-allocates thread-local data structures // needed by NanoLog. This can be invoked once per new @@ -64,9 +68,11 @@ int main(int argc, char** argv) { time_span = std::chrono::duration_cast>( stop - start).count(); - printf("The total time spent invoking NANO_LOG with no parameters %lu " + printf("The total time spent invoking NANO_LOG with no parameters %llu " "times took %0.2lf seconds (%0.2lf ns/message average)\r\n", - RECORDS, time_span, (time_span/RECORDS)*1e9); + static_cast(RECORDS), + time_span, + (time_span/RECORDS)*1e9); start = std::chrono::high_resolution_clock::now(); // Flush all pending log messages to disk