diff --git a/internal/cld2_dynamic_compat.h b/internal/cld2_dynamic_compat.h new file mode 100644 index 0000000..fcded50 --- /dev/null +++ b/internal/cld2_dynamic_compat.h @@ -0,0 +1,37 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CLD2_INTERNAL_CLD2_DYNAMIC_COMPAT_H_ +#define CLD2_INTERNAL_CLD2_DYNAMIC_COMPAT_H_ + +// open(), close(), mmap() and munmap() are not available in vanilla win32. +// This header provides compatibility for different operating systems using +// standard preprocessor definitions. +// Note that _WIN32 is also defined on 64-bit platforms :) +// +// For more information see https://code.google.com/p/cld2/issues/detail?id=19 + +#ifdef _WIN32 + #include + #define OPEN _open + #define CLOSE _close +#else // E.g., POSIX. We don't try to support Mac versions prior to OSX. + #include + #include + #include + #define OPEN open + #define CLOSE close +#endif + +#endif // CLD2_INTERNAL_CLD2_DYNAMIC_COMPAT_H_ diff --git a/internal/cld2_dynamic_data_loader.cc b/internal/cld2_dynamic_data_loader.cc index 7227b8e..8764c3e 100644 --- a/internal/cld2_dynamic_data_loader.cc +++ b/internal/cld2_dynamic_data_loader.cc @@ -18,8 +18,8 @@ #include #include #include -#include +#include "cld2_dynamic_compat.h" // for win32/posix compatibility #include "cld2_dynamic_data.h" #include "cld2_dynamic_data_loader.h" #include "integral_types.h" @@ -140,10 +140,16 @@ CLD2DynamicData::FileHeader* loadInternal(FILE* inFile, const void* basePointer, void unloadDataFile(CLD2::ScoringTables** scoringTables, void** mmapAddress, uint32_t* mmapLength) { +#ifdef _WIN32 + // See https://code.google.com/p/cld2/issues/detail?id=20 + fprintf(stderr, "dynamic data unloading from file is not currently supported on win32, use raw mode instead."); + return; +#else // i.e., is POSIX (no support for Mac prior to OSX) CLD2DynamicDataLoader::unloadDataRaw(scoringTables); munmap(*mmapAddress, *mmapLength); *mmapAddress = NULL; *mmapLength = 0; +#endif // ifdef _WIN32 } void unloadDataRaw(CLD2::ScoringTables** scoringTables) { @@ -157,21 +163,28 @@ void unloadDataRaw(CLD2::ScoringTables** scoringTables) { CLD2::ScoringTables* loadDataFile(const char* fileName, void** mmapAddressOut, uint32_t* mmapLengthOut) { + +#ifdef _WIN32 + // See https://code.google.com/p/cld2/issues/detail?id=20 + fprintf(stderr, "dynamic data loading from file is not currently supported on win32, use raw mode instead."); + return NULL; +#else // i.e., is POSIX (no support for Mac prior to OSX) CLD2DynamicData::FileHeader* header = loadHeaderFromFile(fileName); if (header == NULL) { return NULL; } // Initialize the memory map - int inFileHandle = open(fileName, O_RDONLY); + int inFileHandle = OPEN(fileName, O_RDONLY); void* mapped = mmap(NULL, header->totalFileSizeBytes, PROT_READ, MAP_PRIVATE, inFileHandle, 0); // Record the map address. This allows callers to unmap *mmapAddressOut=mapped; *mmapLengthOut=header->totalFileSizeBytes; - close(inFileHandle); + CLOSE(inFileHandle); return loadDataInternal(header, mapped, header->totalFileSizeBytes); +#endif // ifdef _WIN32 } CLD2::ScoringTables* loadDataRaw(const void* basePointer, const uint32_t length) { diff --git a/internal/cld2_unittest.cc b/internal/cld2_unittest.cc index 185051c..ee85ab7 100644 --- a/internal/cld2_unittest.cc +++ b/internal/cld2_unittest.cc @@ -26,6 +26,7 @@ #include #include +#include "cld2_dynamic_compat.h" #include "../public/compact_lang_det.h" #include "../public/encodings.h" #include "unittest_data.h" @@ -265,6 +266,7 @@ int RunTests (int flags, bool get_vector) { bool any_fail = false; #ifdef CLD2_DYNAMIC_MODE +#ifndef _WIN32 fprintf(stdout, "[DYNAMIC] Test running in dynamic data mode!\n"); if (!CLD2::isDataDynamic()) { fprintf(stderr, "[DYNAMIC] *** Error: CLD2::isDataDynamic() returned false in a dynamic build!\n"); @@ -285,6 +287,7 @@ int RunTests (int flags, bool get_vector) { any_fail = true; } fprintf(stdout, "[DYNAMIC] Data loaded, file-based tests commencing\n"); +#endif // ifndef _WIN32 #else // CLD2_DYNAMIC_MODE is not defined if (CLD2::isDataDynamic()) { fprintf(stderr, "*** Error: CLD2::isDataDynamic() returned true in a non-dynamic build!\n"); @@ -307,6 +310,7 @@ int RunTests (int flags, bool get_vector) { } #ifdef CLD2_DYNAMIC_MODE +#ifndef _WIN32 fprintf(stdout, "[DYNAMIC] File-based tests complete, attempting to unload file data\n"); CLD2::unloadData(); dataLoaded = CLD2::isDataLoaded(); @@ -325,10 +329,10 @@ int RunTests (int flags, bool get_vector) { const int actualSize = ftell(inFile); fclose(inFile); - int inFileHandle = open(data_file, O_RDONLY); + int inFileHandle = OPEN(data_file, O_RDONLY); void* mapped = mmap(NULL, actualSize, PROT_READ, MAP_PRIVATE, inFileHandle, 0); - close(inFileHandle); + CLOSE(inFileHandle); fprintf(stdout, "[DYNAMIC] mmap'ed successfully, attempting data load.\n"); CLD2::loadDataFromRawAddress(mapped, actualSize); @@ -360,7 +364,8 @@ int RunTests (int flags, bool get_vector) { fprintf(stdout, "[DYNAMIC] Attempting translation after unloading map data\n"); any_fail |= !OneTest(flags, get_vector, UNKNOWN_LANGUAGE, kTeststr_en, strlen(kTeststr_en)); - fprintf(stdout, "[DYNAMIC] All dynamic-mode tests complete\n"); + fprintf(stdout, "[DYNAMIC] All dynamic-mode tests complete\n"); +#endif // ifndef _WIN32 #else // CLD2_DYNAMIC_MODE is not defined // These functions should do nothing, and shouldn't cause a crash. A warning is output to STDERR. fprintf(stderr, "Checking that non-dynamic implementations of dynamic data methods are no-ops (ignore the warnings).\n"); diff --git a/internal/compact_lang_det_impl.cc b/internal/compact_lang_det_impl.cc index fc1c333..3277b18 100644 --- a/internal/compact_lang_det_impl.cc +++ b/internal/compact_lang_det_impl.cc @@ -98,7 +98,12 @@ extern const short kAvgDeltaOctaScore[]; if (isDataLoaded()) { unloadData(); } - dynamicTables = CLD2DynamicDataLoader::loadDataFile(fileName, &mmapAddress, &mmapLength); + ScoringTables* result = CLD2DynamicDataLoader::loadDataFile(fileName, &mmapAddress, &mmapLength); + if (result == NULL) { + fprintf(stderr, "WARNING: Dynamic data loading failed.\n"); + return; + } + dynamicTables = result; kScoringtables = *dynamicTables; dataSourceIsFile = true; dynamicDataLoaded = true; @@ -108,7 +113,12 @@ extern const short kAvgDeltaOctaScore[]; if (isDataLoaded()) { unloadData(); } - dynamicTables = CLD2DynamicDataLoader::loadDataRaw(rawAddress, length); + ScoringTables* result = CLD2DynamicDataLoader::loadDataRaw(rawAddress, length); + if (result == NULL) { + fprintf(stderr, "WARNING: Dynamic data loading failed.\n"); + return; + } + dynamicTables = result; kScoringtables = *dynamicTables; dataSourceIsFile = false; dynamicDataLoaded = true;