Add SGML↔Unicode conversion

Signed-off-by: Simon Rozman <simon@rozman.si>
2023-03-09 13:57:16 +01:00
parent 9ce8e6bff9
commit b43b853235
10 changed files with 3716 additions and 0 deletions
--- a/UnitTests/.gitignore
+++ b/UnitTests/.gitignore
@@ -0,0 +1,5 @@
+/.vs
+/*.user
+/Debug
+/Release
+/x64
--- a/UnitTests/UnitTests.sln
+++ b/UnitTests/UnitTests.sln
@@ -0,0 +1,31 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.32126.315
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "UnitTests", "UnitTests.vcxproj", "{9AFC377D-C32D-4D42-82C2-09FC818020A2}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Win32 = Debug|Win32
+		Debug|x64 = Debug|x64
+		Release|Win32 = Release|Win32
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{9AFC377D-C32D-4D42-82C2-09FC818020A2}.Debug|Win32.ActiveCfg = Debug|Win32
+		{9AFC377D-C32D-4D42-82C2-09FC818020A2}.Debug|Win32.Build.0 = Debug|Win32
+		{9AFC377D-C32D-4D42-82C2-09FC818020A2}.Debug|x64.ActiveCfg = Debug|x64
+		{9AFC377D-C32D-4D42-82C2-09FC818020A2}.Debug|x64.Build.0 = Debug|x64
+		{9AFC377D-C32D-4D42-82C2-09FC818020A2}.Release|Win32.ActiveCfg = Release|Win32
+		{9AFC377D-C32D-4D42-82C2-09FC818020A2}.Release|Win32.Build.0 = Release|Win32
+		{9AFC377D-C32D-4D42-82C2-09FC818020A2}.Release|x64.ActiveCfg = Release|x64
+		{9AFC377D-C32D-4D42-82C2-09FC818020A2}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {BBDB843D-98C3-46EF-BDE8-0E80FD851852}
+	EndGlobalSection
+EndGlobal
--- a/UnitTests/UnitTests.vcxproj
+++ b/UnitTests/UnitTests.vcxproj
@@ -0,0 +1,129 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <VCProjectVersion>16.0</VCProjectVersion>
+    <ProjectGuid>{9AFC377D-C32D-4D42-82C2-09FC818020A2}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>UnitTests</RootNamespace>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+    <ProjectSubType>NativeUnitTestProject</ProjectSubType>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>$(DefaultPlatformToolset)</PlatformToolset>
+    <CharacterSet>Unicode</CharacterSet>
+    <UseOfMfc>false</UseOfMfc>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration" />
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="Shared">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+    <CodeAnalysisRuleSet>NativeRecommendedRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <LinkIncremental>true</LinkIncremental>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <LinkIncremental>false</LinkIncremental>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <PrecompiledHeader>Use</PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <EnablePREfast>true</EnablePREfast>
+      <SDLCheck>true</SDLCheck>
+      <AdditionalIncludeDirectories>..\include;$(VCInstallDir)UnitTest\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <UseFullPaths>true</UseFullPaths>
+      <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
+    </ClCompile>
+    <Link>
+      <SubSystem>Windows</SubSystem>
+      <AdditionalLibraryDirectories>$(VCInstallDir)UnitTest\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>Advapi32.lib;Shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+    </ClCompile>
+    <Link>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'">
+    <ClCompile>
+      <PreprocessorDefinitions>WIN32;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+  <ItemGroup>
+    <ClCompile Include="pch.cpp">
+      <PrecompiledHeader>Create</PrecompiledHeader>
+    </ClCompile>
+    <ClCompile Include="sgml.cpp" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="pch.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
--- a/UnitTests/UnitTests.vcxproj.filters
+++ b/UnitTests/UnitTests.vcxproj.filters
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <Filter Include="Source Files">
+      <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+      <Extensions>cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+    </Filter>
+    <Filter Include="Header Files">
+      <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+      <Extensions>h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd</Extensions>
+    </Filter>
+    <Filter Include="Resource Files">
+      <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+      <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+    </Filter>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="pch.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="sgml.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="pch.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+  </ItemGroup>
+</Project>
--- a/UnitTests/pch.cpp
+++ b/UnitTests/pch.cpp
@@ -0,0 +1,6 @@
+/*
+	SPDX-License-Identifier: MIT
+	Copyright © 2023 Amebis
+*/
+
+#include "pch.h"
--- a/UnitTests/pch.h
+++ b/UnitTests/pch.h
@@ -0,0 +1,25 @@
+/*
+	SPDX-License-Identifier: MIT
+	Copyright © 2023 Amebis
+*/
+
+#pragma once
+
+#define SECURITY_WIN32
+#define _WINSOCKAPI_	// Prevent inclusion of winsock.h in windows.h
+
+#include <stdex/base64.h>
+#include <stdex/errno.h>
+#include <stdex/exception.h>
+#include <stdex/hex.h>
+#include <stdex/idrec.h>
+#include <stdex/interval.h>
+#include <stdex/mapping.h>
+//#include <stdex/parser.h>
+#include <stdex/progress.h>
+#include <stdex/sal.h>
+#include <stdex/sgml.h>
+#include <stdex/string.h>
+#include <stdex/vector_queue.h>
+
+#include <CppUnitTest.h>
--- a/UnitTests/sgml.cpp
+++ b/UnitTests/sgml.cpp
@@ -0,0 +1,59 @@
+/*
+	SPDX-License-Identifier: MIT
+	Copyright © 2023 Amebis
+*/
+
+#include "pch.h"
+
+using namespace std;
+using namespace Microsoft::VisualStudio::CppUnitTestFramework;
+
+namespace UnitTests
+{
+	TEST_CLASS(sgml)
+	{
+	public:
+		TEST_METHOD(sgml2str)
+		{
+			Assert::AreEqual(L"This is a test.", stdex::sgml2str("This is a test.", (size_t)-1).c_str());
+			Assert::AreEqual(L"Th\u00ed\u0161 i\u22c5 a te\u0073\u0304t.&unknown;😀😅", stdex::sgml2str("Th&iacute;&scaron; i&sdot; &#97; te&smacr;t.&unknown;&#x1F600;&#X1f605;", (size_t)-1).c_str());
+			Assert::AreEqual(L"This", stdex::sgml2str("This is a test.", 4).c_str());
+			Assert::AreEqual(L"T\u0068\u0301", stdex::sgml2str("T&hacute;is is a test.", 9).c_str());
+			Assert::AreEqual(L"T&hac", stdex::sgml2str("T&hacute;is is a test.", 5).c_str());
+			Assert::AreEqual(L"The &quot;quoted&quot; &amp; text.", stdex::sgml2str("The &quot;quoted&quot; &amp; text.", (size_t)-1, stdex::sgml_c).c_str());
+
+			stdex::mapping_vector<size_t> map;
+			constexpr size_t i = 0;
+			constexpr size_t j = 0;
+			stdex::sgml2str("Th&iacute;&scaron; i&sdot; &#97; te&smacr;t.&unknown;&#x1F600;&#X1f605;", (size_t)-1, 0, stdex::mapping<size_t>(i, j), &map);
+			Assert::IsTrue(stdex::mapping_vector<size_t>{
+				{ i + 2, j + 2 },
+				{ i + 10, j + 3 },
+				{ i + 10, j + 3 },
+				{ i + 18, j + 4 },
+				{ i + 20, j + 6 },
+				{ i + 26, j + 7 },
+				{ i + 27, j + 8 },
+				{ i + 32, j + 9 },
+				{ i + 35, j + 12 },
+				{ i + 42, j + 14 },
+				{ i + 53, j + 25 },
+				{ i + 62, j + 27 },
+				{ i + 62, j + 27 },
+				{ i + 71, j + 29 },
+			} == map);
+		}
+
+		TEST_METHOD(str2sgml)
+		{
+			Assert::AreEqual("This is a test.", stdex::str2sgml(L"This is a test.", (size_t)-1).c_str());
+			Assert::AreEqual("Th&iacute;&scaron; i&sdot; a te&smacr;t.&amp;unknown;&#x1f600;&#x1f605;", stdex::str2sgml(L"Th\u00ed\u0161 i\u22c5 a te\u0073\u0304t.&unknown;😀😅", (size_t)-1).c_str());
+			Assert::AreEqual("This", stdex::str2sgml(L"This is a test.", 4).c_str());
+			Assert::AreEqual("te&smacr;", stdex::str2sgml(L"te\u0073\u0304t", 4).c_str());
+			Assert::AreEqual("tes", stdex::str2sgml(L"te\u0073\u0304t", 3).c_str());
+			Assert::AreEqual("&#x2318;&permil;&#x362;", stdex::str2sgml(L"⌘‰͢", (size_t)-1).c_str());
+			Assert::AreEqual("$\"<>&amp;", stdex::str2sgml(L"$\"<>&", (size_t)-1).c_str());
+			Assert::AreEqual("$&quot;<>&amp;", stdex::str2sgml(L"$\"<>&", (size_t)-1, stdex::sgml_c).c_str());
+		}
+	};
+}