ZRColaWS: Initial working version

Signed-off-by: Simon Rozman <simon@rozman.si>

ZRColaWS/Makefile

@@ -14,6 +14,13 @@ zrcolad: ../lib/libZRCola/lib/libZRCola.a $(OBJS)
 ../lib/libZRCola/lib/libZRCola.a:
 	$(MAKE) $(MFLAGS) -C ../lib/libZRCola/build
 
+.PHONY: install
+install: zrcolad ../output/data/ZRCola.zrcdb
+	install -d $(PREFIX)/bin/
+	install -m 755 zrcolad $(PREFIX)/bin/
+	install -d $(PREFIX)/share/zrcola/
+	install -m 644 ../output/data/ZRCola.zrcdb $(PREFIX)/share/zrcola/
+
 .PHONY: clean
 clean:
 	-rm -r *.{d,o} zrcolad

ZRColaWS/controller.h

@@ -6,6 +6,9 @@
 #pragma once
 
 #include "dto.h"
+#include "iconverter.h"
+#include "zrcolaws.h"
+#include <zrcola/translate.h>
 #include <oatpp/core/macro/codegen.hpp>
 #include <oatpp/core/macro/component.hpp>
 #include <oatpp/web/server/api/ApiController.hpp>
@@ -17,26 +20,88 @@ class MyController : public oatpp::web::server::api::ApiController
 public:
     MyController(OATPP_COMPONENT(std::shared_ptr<ObjectMapper>, objectMapper)) : oatpp::web::server::api::ApiController(objectMapper) {}
 
-    ENDPOINT("GET", "/hello", hello)
+    ADD_CORS(getAbout)
+    ENDPOINT("GET", "/about", getAbout)
     {
-        auto dto = MessageDto::createShared();
+        return createDtoResponse(Status::CODE_200, aboutDto::createShared());
-        dto->statusCode = 0;
-        dto->message = "Hello World!";
-        return createDtoResponse(Status::CODE_200, dto);
     }
 
-    ADD_CORS(postUsers)
+    ADD_CORS(getTranset)
-    ENDPOINT("POST", "/users", postUsers, BODY_DTO(Object<UserDto>, user))
+    ENDPOINT("GET", "/transet", getTranset)
     {
-        auto dto = MessageDto::createShared();
+        try {
-        dto->statusCode = 0;
+            utf16toutf8 c;
-        std::string msg;
+            auto result = oatpp::Vector<oatpp::Object<transetDto>>::createShared();
-        msg  = "Hello ";
+            auto dto = transetDto::createShared();
-        msg += user->name ? user->name->c_str() : "unknown";
+            dto->set = ZRCOLA_TRANSETID_DEFAULT;
-        msg += "!";
+            dto->src = "ZRCola Decomposed";
-        dto->message = msg;
+            dto->dst = "ZRCola Composed";
-        return createDtoResponse(Status::CODE_200, dto);
+            result->push_back(dto);
+            for (size_t i = 0, n = ts_db.idxTranSet.size(); i < n; i++) {
+                const auto &ts = ts_db.idxTranSet[i];
+                dto = transetDto::createShared();
+                dto->set = ts.set;
+                dto->src = c.convert(ts.src(), ts.src_len());
+                dto->dst = c.convert(ts.dst(), ts.dst_len());
+                result->push_back(dto);
             }
+            dto = transetDto::createShared();
+            dto->set = ZRCOLA_TRANSETID_UNICODE;
+            dto->src = "ZRCola Decomposed";
+            dto->dst = "Unicode";
+            result->push_back(dto);
+            return createDtoResponse(Status::CODE_200, result);
+        } catch (std::exception &ex) {
+            OATPP_LOGE("ZRColaWS", "%s: %s", typeid(ex).name(), ex.what());
+            return ResponseFactory::createResponse(Status::CODE_500, ex.what());
+        }
+    }
+
+    ADD_CORS(postTranslate)
+    ENDPOINT("POST", "/translate", postTranslate, BODY_DTO(Object<translateInDto>, input))
+    {
+        try {
+            utf8toutf16 cIn;
+            std::u16string dst, dst2;
+            if (input->text)
+                dst = cIn.convert(*input->text);
+            std::vector<ZRCola::mapping_vector> mapping;
+            if (input->transet) {
+                ZRCola::mapping_vector map;
+                auto ts = input->transet->cbegin();
+                const auto ts_end = input->transet->cend();
+                for (; ts != ts_end; ++ts) {
+                    switch (*ts) {
+                    case ZRCOLA_TRANSETID_DEFAULT:
+                    case ZRCOLA_TRANSETID_UNICODE:
+                        // Decompose first, then re-compose.
+                        t_db.TranslateInv(*ts, dst.data(), dst.size(), dst2, &map);
+                        mapping.push_back(std::move(map));
+                        t_db.Translate(*ts, dst2.data(), dst2.size(), dst, &map);
+                        mapping.push_back(std::move(map));
+                        break;
+
+                    default:
+                        t_db.Translate(*ts, dst.data(), dst.size(), dst2, &map);
+                        mapping.push_back(std::move(map));
+                        dst = std::move(dst2);
+                    }
+                }
+            }
+
+            // TODO: Flatten mapping and return along with the translated text.
+
+            utf16toutf8 cOut;
+            auto dto = translateOutDto::createShared();
+            dto->text = cOut.convert(dst);
+            return createDtoResponse(Status::CODE_200, dto);
+        } catch (std::exception &ex) {
+            OATPP_LOGE("ZRColaWS", "%s: %s", typeid(ex).name(), ex.what());
+            return ResponseFactory::createResponse(Status::CODE_500, ex.what());
+        }
+    }
+
+    // TODO: Add POST handler for inverse translation.
 };
 
 #include OATPP_CODEGEN_END(ApiController)

ZRColaWS/dto.h

@@ -5,24 +5,40 @@
 
 #pragma once
 
+#include "../include/version.h"
 #include <oatpp/core/data/mapping/type/Object.hpp>
 #include <oatpp/core/macro/codegen.hpp>
 #include <oatpp/core/Types.hpp>
 
 #include OATPP_CODEGEN_BEGIN(DTO)
 
-class MessageDto : public oatpp::DTO
+class aboutDto : public oatpp::DTO
 {
-    DTO_INIT(MessageDto, DTO)
+    DTO_INIT(aboutDto, DTO)
-    DTO_FIELD(Int32, statusCode);
+    DTO_FIELD(String, vendor) = PRODUCT_CFG_VENDOR;
-    DTO_FIELD(String, message);
+    DTO_FIELD(String, application) = PRODUCT_CFG_APPLICATION;
+    DTO_FIELD(String, version) = PRODUCT_VERSION_STR;
 };
 
-class UserDto : public oatpp::DTO
+class transetDto : public oatpp::DTO
 {
-    DTO_INIT(UserDto, DTO)
+    DTO_INIT(transetDto, DTO)
-    DTO_FIELD(Int64, id);
+    DTO_FIELD(UInt16, set);
-    DTO_FIELD(String, name);
+    DTO_FIELD(String, src);
+    DTO_FIELD(String, dst);
+};
+
+class translateInDto : public oatpp::DTO
+{
+    DTO_INIT(translateInDto, DTO)
+    DTO_FIELD(Vector<UInt16>, transet);
+    DTO_FIELD(String, text);
+};
+
+class translateOutDto : public oatpp::DTO
+{
+    DTO_INIT(translateOutDto, DTO)
+    DTO_FIELD(String, text);
 };
 
 #include OATPP_CODEGEN_END(DTO)

ZRColaWS/iconverter.h

@@ -0,0 +1,75 @@
+/*
+    SPDX-License-Identifier: GPL-3.0-or-later
+    Copyright © 2022 Amebis
+*/
+
+#pragma once
+
+#include <stdex/sal.h>
+#include <iconv.h>
+#include <cstring>
+#include <stdexcept>
+#include <string>
+
+inline static std::runtime_error errno_error(_In_z_ const char *file, _In_ int line, _In_z_ const char *func)
+{
+    int _errno = errno;
+    return std::runtime_error(
+        std::string(file) + ":" + std::to_string(line) +
+        std::string(func) + " error " + std::to_string(_errno) + ": " +
+        std::strerror(_errno));
+}
+
+template <typename T_from, typename T_to>
+class iconverter
+{
+public:
+    iconverter(_In_z_ const char* from, _In_z_ const char* to)
+    {
+        m_handle = iconv_open(to, from);
+        if (m_handle == (iconv_t)-1)
+            throw errno_error(__FILE__, __LINE__, __FUNCTION__);
+    }
+
+    ~iconverter()
+    {
+        iconv_close(m_handle);
+    }
+
+    std::basic_string<T_to> convert(_In_z_count_(count) const T_from* input, _In_ size_t count) const
+    {
+        T_to buf[0x100];
+        std::basic_string<T_to> result;
+        size_t inSize = sizeof(T_from) * count;
+        do {
+            T_to* output = &buf[0];
+            size_t outSize = sizeof(buf);
+            errno = 0;
+            iconv(m_handle, (char**)&input, &inSize, (char**)&output, &outSize);
+            if (errno)
+                throw errno_error(__FILE__, __LINE__, __FUNCTION__);
+            result.insert(result.end(), buf, (T_to*)((char*)buf + sizeof(buf) - outSize));
+        } while (inSize);
+        return result;
+    }
+
+    std::basic_string<T_to> convert(_In_ const std::basic_string<T_from>& input)
+    {
+        return convert(input.c_str(), input.length());
+    }
+
+protected:
+    iconv_t m_handle;
+};
+
+class utf16toutf8 : public iconverter<char16_t, char>
+{
+public:
+    utf16toutf8() : iconverter("UTF-16LE", "UTF-8") {}
+};
+
+class utf8toutf16 : public iconverter<char, char16_t>
+{
+public:
+    utf8toutf16() : iconverter("UTF-8", "UTF-16LE") {}
+};

ZRColaWS/test.html

@@ -2,29 +2,44 @@
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
 <meta http-equiv="Content-Language" content="sl">
 </head>
+<style>
+textarea, input[type="text"]
+{
+	font-family: ZRCola;
+	font-size: 24px;
+}	
+</style>
+<body>
+	<form method="GET" name="about" action="http://localhost:8000/about">
+		<h1>about</h1>
+		<p><input type="submit" value="Submit"></p>
+	</form>
 
-<script type="text/javascript">
+	<form method="GET" name="transet" action="http://localhost:8000/transet">
-function submitJSON() {
+		<h1>transet</h1>
-	var form = document.users;
+		<p><input type="submit" value="Submit"></p>
+	</form>
+
+	<script type="text/javascript">
+	function submitJSON() {
+		var form = document.translate;
 		var xhr = new XMLHttpRequest();
 		xhr.onreadystatechange = function() {
 			if (xhr.readyState == XMLHttpRequest.DONE)
 				form["res"].value = xhr.responseText;
 		}
-	xhr.open("POST", "http://localhost:8000/users", true);
+		xhr.open("POST", "http://localhost:8000/translate", true);
 		xhr.setRequestHeader('Content-Type', 'application/json; charset=UTF-8');
 		xhr.send(form["req"].value);
-}
+	}
-</script>
+	</script>
-<body>
+	<form name="translate">
-<form name="users">
+		<h1>translate</h1>
-	<h1>users</h1>
 		<p>
-		<textarea rows="10" name="req" cols="63">{"id":0, "name":"joe"}</textarea>
+			<textarea rows="10" name="req" cols="40">{"transet":[0], "text":"To je test."}</textarea>
 			>>
-		<textarea rows="10" name="res" cols="63"></textarea>
+			<textarea rows="10" name="res" cols="40"></textarea>
 		</p>
 		<p><input type="submit" value="Submit" onclick="submitJSON(); return false"><input type="reset" value="Reset"></p>
-</form>
+	</form>
-
 </body>

ZRColaWS/zrcolaws.cpp

@@ -3,16 +3,121 @@
     Copyright © 2022 Amebis
 */
 
-#include "../include/version.h"
 #include "appcomponent.h"
 #include "controller.h"
+#include "zrcolaws.h"
 #include <oatpp/network/Server.hpp>
+#include <fstream>
 
 using namespace std;
+using namespace ZRCola;
+
+translation_db t_db;
+transet_db ts_db;
+// transeq_db tsq_db;
+// langchar_db lc_db;
+// language_db lang_db;
+// character_db chr_db;
+// chrcat_db cc_db;
+// chrtag_db ct_db;
+// tagname_db tn_db;
+// highlight_db h_db;
+
+static void load_database()
+{
+    fstream dat(PREFIX "/share/zrcola/ZRCola.zrcdb", ios_base::in | ios_base::binary);
+    if (!dat.good())
+        throw runtime_error(PREFIX "/share/zrcola/ZRCola.zrcdb not found or cannot be opened.");
+
+    if (!stdex::idrec::find<recordid_t, recordsize_t, ZRCOLA_RECORD_ALIGN>(dat, ZRCOLA_DB_ID, sizeof(recordid_t)))
+        throw runtime_error(PREFIX "/share/zrcola/ZRCola.zrcdb is not a valid ZRCola database.");
+
+    recordsize_t size;
+    dat.read((char*)&size, sizeof(recordsize_t));
+    if (dat.good()) {
+        bool has_translation_data = false;
+
+        for (;;) {
+            recordid_t id;
+            if (!stdex::idrec::read_id(dat, id, size)) break;
+
+            if (id == translation_rec::id()) {
+                dat >> translation_rec(t_db);
+                if (dat.good()) {
+                    has_translation_data = true;
+                } else {
+                    OATPP_LOGE("ZRColaWS", "ZRColaWS", "Error reading translation data from " PREFIX "/share/zrcola/ZRCola.zrcdb.");
+                    t_db.clear();
+                }
+            } else if (id == transet_rec::id()) {
+                dat >> transet_rec(ts_db);
+                if (!dat.good()) {
+                    OATPP_LOGE("ZRColaWS", "Error reading translation set data from " PREFIX "/share/zrcola/ZRCola.zrcdb.");
+                    ts_db.clear();
+                }
+            // } else if (id == transeq_rec::id()) {
+            //     dat >> transeq_rec(tsq_db);
+            //     if (!dat.good()) {
+            //         OATPP_LOGE("ZRColaWS", "Error reading translation sequence data from " PREFIX "/share/zrcola/ZRCola.zrcdb.");
+            //         tsq_db.clear();
+            //     }
+            // } else if (id == langchar_rec::id()) {
+            //     dat >> langchar_rec(lc_db);
+            //     if (!dat.good()) {
+            //         OATPP_LOGE("ZRColaWS", "Error reading language character data from " PREFIX "/share/zrcola/ZRCola.zrcdb.");
+            //         lc_db.clear();
+            //     }
+            // } else if (id == language_rec::id()) {
+            //     dat >> language_rec(lang_db);
+            //     if (!dat.good()) {
+            //         OATPP_LOGE("ZRColaWS", "Error reading language character data from " PREFIX "/share/zrcola/ZRCola.zrcdb.");
+            //         lang_db.clear();
+            //     }
+            // } else if (id == character_rec::id()) {
+            //     dat >> character_rec(chr_db);
+            //     if (!dat.good()) {
+            //         OATPP_LOGE("ZRColaWS", "Error reading character data from " PREFIX "/share/zrcola/ZRCola.zrcdb.");
+            //         chr_db.clear();
+            //     }
+            // } else if (id == chrcat_rec::id()) {
+            //     dat >> chrcat_rec(cc_db);
+            //     if (!dat.good()) {
+            //         OATPP_LOGE("ZRColaWS", "Error reading character category data from " PREFIX "/share/zrcola/ZRCola.zrcdb.");
+            //         cc_db.clear();
+            //     }
+            // } else if (id == chrtag_rec::id()) {
+            //     dat >> chrtag_rec(ct_db);
+            //     if (!dat.good()) {
+            //         OATPP_LOGE("ZRColaWS", "Error reading character tag data from " PREFIX "/share/zrcola/ZRCola.zrcdb.");
+            //         ct_db.clear();
+            //     }
+            // } else if (id == tagname_rec::id()) {
+            //     dat >> tagname_rec(tn_db);
+            //     if (!dat.good()) {
+            //         OATPP_LOGE("ZRColaWS", "Error reading tag name data from " PREFIX "/share/zrcola/ZRCola.zrcdb.");
+            //         tn_db.clear();
+            //     }
+            // } else if (id == highlight_rec::id()) {
+            //     dat >> highlight_rec(h_db);
+            //     if (!dat.good()) {
+            //         OATPP_LOGE("ZRColaWS", "Error reading highlight data from " PREFIX "/share/zrcola/ZRCola.zrcdb.");
+            //         h_db.clear();
+            //     }
+            } else
+                stdex::idrec::ignore<recordsize_t, ZRCOLA_RECORD_ALIGN>(dat);
+        }
+
+        if (!has_translation_data)
+            throw runtime_error(PREFIX "/share/zrcola/ZRCola.zrcdb has no translation data.");
+    }
+}
 
 int main()
 {
     oatpp::base::Environment::init();
+    try {
+        load_database();
+
         AppComponent components;
         OATPP_COMPONENT(std::shared_ptr<oatpp::web::server::HttpRouter>, router);
         auto myController = std::make_shared<MyController>();
@@ -24,5 +129,9 @@ int main()
             connectionProvider->getProperty("host").getData(), connectionProvider->getProperty("port").getData());
         server.run();
         oatpp::base::Environment::destroy();
+    } catch (exception &ex) {
+        OATPP_LOGE("ZRColaWS", "%s: %s", typeid(ex).name(), ex.what());
+        return 1;
+    }
     return 0;
 }

ZRColaWS/zrcolaws.h

@@ -0,0 +1,24 @@
+/*
+    SPDX-License-Identifier: GPL-3.0-or-later
+    Copyright © 2022 Amebis
+*/
+
+#pragma once
+
+#include "../include/version.h"
+#include <zrcola/idrec.h>
+
+#ifndef PREFIX
+#define PREFIX "/usr/local"
+#endif
+
+extern ZRCola::translation_db t_db;
+extern ZRCola::transet_db ts_db;
+// extern ZRCola::transeq_db tsq_db;
+// extern ZRCola::langchar_db lc_db;
+// extern ZRCola::language_db lang_db;
+// extern ZRCola::character_db chr_db;
+// extern ZRCola::chrcat_db cc_db;
+// extern ZRCola::chrtag_db ct_db;
+// extern ZRCola::tagname_db tn_db;
+// extern ZRCola::highlight_db h_db;

include/props.mak

@@ -1,4 +1,7 @@
-CPPFLAGS := $(CPPFLAGS) -MMD -MP
+ifeq ($(PREFIX),)
+PREFIX := /usr/local
+endif
+CPPFLAGS := $(CPPFLAGS) -MMD -MP -DPREFIX='"$(PREFIX)"'
 ifeq ($(CFG),Debug)
 CPPFLAGS := $(CPPFLAGS) -D_DEBUG
 CFLAGS := $(CFLAGS) -Og -g