stdex
Additional custom or not Standard C++ covered algorithms
Loading...
Searching...
No Matches
parser.cpp
1/*
2 SPDX-License-Identifier: MIT
3 Copyright © 2023 Amebis
4*/
5
6#include "pch.h"
7
8using namespace std;
9using namespace stdex;
10using namespace stdex::parser;
11#ifdef _WIN32
12using namespace Microsoft::VisualStudio::CppUnitTestFramework;
13
14namespace Microsoft {
15 namespace VisualStudio {
16 namespace CppUnitTestFramework {
17 static std::wstring ToString(const stdex::interval<size_t>& q)
18 {
19 return stdex::sprintf(L"<%zu, %zu>", nullptr, q.start, q.end);
20 }
21 }
22 }
23}
24#endif
25
26namespace UnitTests
27{
28 TEST_CLASS(parser)
29 {
30 public:
31 TEST_METHOD(wtest)
32 {
33 static const wchar_t text[] = L"This is a test.\nSecond line.";
34
35 {
36 wnoop p;
37 Assert::IsTrue(p.match(text));
38 Assert::AreEqual((size_t)0, p.interval.start);
39 Assert::AreEqual((size_t)0, p.interval.end);
40 }
41
42 {
43 wcu p(L't');
44 Assert::IsFalse(p.match(text));
45 Assert::IsTrue(p.match(text, 0, _countof(text), match_case_insensitive));
46 Assert::AreEqual((size_t)0, p.interval.start);
47 Assert::AreEqual((size_t)1, p.interval.end);
48 }
49
50 {
51 wspace_cu p;
52 Assert::IsFalse(p.match(text));
53 Assert::IsTrue(p.match(text, 4));
54 Assert::AreEqual((size_t)4, p.interval.start);
55 Assert::AreEqual((size_t)5, p.interval.end);
56 }
57
58 {
59 wpunct_cu p;
60 Assert::IsFalse(p.match(text));
61 Assert::IsTrue(p.match(text, 14));
62 Assert::AreEqual((size_t)14, p.interval.start);
63 Assert::AreEqual((size_t)15, p.interval.end);
64 }
65
66 {
67 wspace_or_punct_cu p;
68 Assert::IsFalse(p.match(text));
69 Assert::IsTrue(p.match(text, 4));
70 Assert::AreEqual((size_t)4, p.interval.start);
71 Assert::AreEqual((size_t)5, p.interval.end);
72 Assert::IsTrue(p.match(text, 14));
73 Assert::AreEqual((size_t)14, p.interval.start);
74 Assert::AreEqual((size_t)15, p.interval.end);
75 }
76
77 {
78 wbol p;
79 Assert::IsTrue(p.match(text));
80 Assert::AreEqual((size_t)0, p.interval.start);
81 Assert::AreEqual((size_t)0, p.interval.end);
82 Assert::IsFalse(p.match(text, 1));
83 Assert::IsFalse(p.match(text, 15));
84 Assert::IsTrue(p.match(text, 16));
85 Assert::AreEqual((size_t)16, p.interval.start);
86 Assert::AreEqual((size_t)16, p.interval.end);
87 }
88
89 {
90 weol p;
91 Assert::IsFalse(p.match(text));
92 Assert::IsFalse(p.match(text, 1));
93 Assert::IsTrue(p.match(text, 15));
94 Assert::AreEqual((size_t)15, p.interval.start);
95 Assert::AreEqual((size_t)15, p.interval.end);
96 Assert::IsFalse(p.match(text, 16));
97 }
98
99 {
100 wcu_set p(L"abcD");
101 Assert::IsFalse(p.match(text));
102 Assert::IsTrue(p.match(text, 8));
103 Assert::AreEqual((size_t)8, p.interval.start);
104 Assert::AreEqual((size_t)9, p.interval.end);
105 Assert::AreEqual((size_t)0, p.hit_offset);
106 Assert::IsFalse(p.match(text, 21));
107 Assert::IsTrue(p.match(text, 21, _countof(text), match_case_insensitive));
108 Assert::AreEqual((size_t)21, p.interval.start);
109 Assert::AreEqual((size_t)22, p.interval.end);
110 Assert::AreEqual((size_t)3, p.hit_offset);
111 }
112
113 {
114 stdex::parser::wstring p(L"this");
115 Assert::IsFalse(p.match(text));
116 Assert::IsTrue(p.match(text, 0, sizeof(text), match_case_insensitive));
117 Assert::AreEqual((size_t)0, p.interval.start);
118 Assert::AreEqual((size_t)4, p.interval.end);
119 }
120
121 {
122 wany_cu chr;
123 witerations p(make_shared_no_delete(&chr), 1, 5);
124 Assert::IsTrue(p.match(text));
125 Assert::AreEqual((size_t)0, p.interval.start);
126 Assert::AreEqual((size_t)5, p.interval.end);
127 }
128
129 {
130 wspace_cu nospace(true);
131 witerations p(make_shared_no_delete(&nospace), 1);
132 Assert::IsTrue(p.match(text));
133 Assert::AreEqual((size_t)0, p.interval.start);
134 Assert::AreEqual((size_t)4, p.interval.end);
135 }
136
137 {
138 wcu chr_t(L't'), chr_h(L'h'), chr_i(L'i'), chr_s(L's');
139 wspace_cu space;
140 wsequence p({
141 make_shared_no_delete(&chr_t),
142 make_shared_no_delete(&chr_h),
143 make_shared_no_delete(&chr_i),
144 make_shared_no_delete(&chr_s),
145 make_shared_no_delete(&space) });
146 Assert::IsFalse(p.match(text));
147 Assert::IsTrue(p.match(text, 0, _countof(text), match_case_insensitive));
148 Assert::AreEqual((size_t)0, p.interval.start);
149 Assert::AreEqual((size_t)5, p.interval.end);
150 }
151
152 {
153 stdex::parser::wstring apple(L"apple"), orange(L"orange"), _this(L"this");
154 wspace_cu space;
155 wbranch p({
156 make_shared_no_delete(&apple),
157 make_shared_no_delete(&orange),
158 make_shared_no_delete(&_this),
159 make_shared_no_delete(&space) });
160 Assert::IsFalse(p.match(text));
161 Assert::IsTrue(p.match(text, 0, _countof(text), match_case_insensitive));
162 Assert::AreEqual((size_t)2, p.hit_offset);
163 Assert::AreEqual((size_t)0, p.interval.start);
164 Assert::AreEqual((size_t)4, p.interval.end);
165 }
166
167 {
168 wstring_branch p(L"apple", L"orange", L"this", nullptr);
169 Assert::IsFalse(p.match(text));
170 Assert::IsTrue(p.match(text, 0, _countof(text), match_case_insensitive));
171 Assert::AreEqual((size_t)2, p.hit_offset);
172 Assert::AreEqual((size_t)0, p.interval.start);
173 Assert::AreEqual((size_t)4, p.interval.end);
174 }
175
176 {
177 wcu chr_s(L's'), chr_h(L'h'), chr_i(L'i'), chr_t(L't');
178 wpermutation p({
179 make_shared_no_delete(&chr_s),
180 make_shared_no_delete(&chr_h),
181 make_shared_no_delete(&chr_i),
182 make_shared_no_delete(&chr_t) });
183 Assert::IsFalse(p.match(text));
184 Assert::IsTrue(p.match(text, 0, _countof(text), match_case_insensitive));
185 Assert::AreEqual((size_t)0, p.interval.start);
186 Assert::AreEqual((size_t)4, p.interval.end);
187 }
188
189 {
190 wspace_cu space;
191 wiban p(make_shared_no_delete(&space));
192 Assert::IsTrue(p.match(L"SI56023120015226972", 0, SIZE_MAX));
193 Assert::IsTrue(p.is_valid);
194 Assert::AreEqual(L"SI", p.country);
195 Assert::AreEqual(L"56", p.check_digits);
196 Assert::AreEqual(L"023120015226972", p.bban);
197 Assert::IsTrue(p.match(L"SI56 0231 2001 5226 972", 0, SIZE_MAX));
198 Assert::IsTrue(p.is_valid);
199 Assert::AreEqual(L"SI", p.country);
200 Assert::AreEqual(L"56", p.check_digits);
201 Assert::AreEqual(L"023120015226972", p.bban);
202 Assert::IsFalse(p.match(L"si56 0231 2001 5226 972", 0, SIZE_MAX));
203 Assert::IsFalse(p.is_valid);
204 Assert::IsTrue(p.match(L"si56 0231 2001 5226 972", 0, SIZE_MAX, match_case_insensitive));
205 Assert::IsTrue(p.is_valid);
206 Assert::IsTrue(p.match(L"SI56 0231 2001 5226 9720", 0, SIZE_MAX));
207 Assert::IsFalse(p.is_valid);
208 Assert::IsTrue(p.match(L"...SI56 0231 2001 5226 972...", 3, SIZE_MAX));
209 Assert::IsTrue(p.is_valid);
210 Assert::IsTrue(p.match(L"SI56 0231 2001 5226 972", 0, SIZE_MAX)); // no-break space
211 Assert::IsTrue(p.is_valid);
212 }
213
214 {
215 wspace_cu space;
216 wcreditor_reference p(make_shared_no_delete(&space));
217 Assert::IsTrue(p.match(L"RF18539007547034", 0, SIZE_MAX));
218 Assert::IsTrue(p.is_valid);
219 Assert::AreEqual(L"18", p.check_digits);
220 Assert::AreEqual(L"000000000539007547034", p.reference);
221 Assert::IsTrue(p.match(L"RF18 5390 0754 7034", 0, SIZE_MAX));
222 Assert::IsTrue(p.is_valid);
223 Assert::AreEqual(L"18", p.check_digits);
224 Assert::AreEqual(L"000000000539007547034", p.reference);
225 Assert::IsFalse(p.match(L"rf18 5390 0754 7034", 0, SIZE_MAX));
226 Assert::IsFalse(p.is_valid);
227 Assert::IsTrue(p.match(L"rf18 5390 0754 7034", 0, SIZE_MAX, match_case_insensitive));
228 Assert::IsTrue(p.is_valid);
229 Assert::IsTrue(p.match(L"RF18 5390 0754 70340", 0, SIZE_MAX));
230 Assert::IsFalse(p.is_valid);
231 Assert::IsTrue(p.match(L"...RF18 5390 0754 7034...", 3, SIZE_MAX));
232 Assert::IsTrue(p.is_valid);
233 Assert::IsTrue(p.match(L"RF18 5390 0754 7034", 0, SIZE_MAX)); // no-break space
234 Assert::IsTrue(p.is_valid);
235 }
236
237 {
238 wspace_cu space;
239 wsi_reference p(make_shared_no_delete(&space));
240 Assert::IsTrue(p.match(L"SI121234567890120", 0, SIZE_MAX));
241 Assert::IsTrue(p.is_valid);
242 Assert::AreEqual(L"12", p.model);
243 Assert::AreEqual(stdex::interval<size_t>(4, 17), p.part1.interval);
244 Assert::IsTrue(p.match(L"SI12 1234567890120", 0, SIZE_MAX));
245 Assert::IsTrue(p.is_valid);
246 Assert::AreEqual(L"12", p.model);
247 Assert::AreEqual(stdex::interval<size_t>(5, 18), p.part1.interval);
248 Assert::IsFalse(p.match(L"si12 1234567890120", 0, SIZE_MAX));
249 Assert::IsTrue(p.match(L"si12 1234567890120", 0, SIZE_MAX, match_case_insensitive));
250 Assert::IsTrue(p.match(L"...SI12 1234567890120...", 3, SIZE_MAX));
251 Assert::IsTrue(p.match(L"SI12 1234567890120", 0, SIZE_MAX)); // no-break space
252 }
253 }
254
255 TEST_METHOD(sgml_test)
256 {
257 std::locale locale_slSI("sl_SI");
258 static const char text[] = "V ko&zcaron;u&scaron;&ccaron;ku zlobnega mizarja stopiclja fant\nin kli&ccaron;e&nbsp;1234567890.";
259
260 {
261 sgml_noop p;
262 Assert::IsTrue(p.match(text));
263 Assert::AreEqual((size_t)0, p.interval.start);
264 Assert::AreEqual((size_t)0, p.interval.end);
265 }
266
267 {
268 sgml_cp p("v");
269 Assert::IsFalse(p.match(text));
270 Assert::IsTrue(p.match(text, 0, _countof(text), match_case_insensitive));
271 Assert::AreEqual((size_t)0, p.interval.start);
272 Assert::AreEqual((size_t)1, p.interval.end);
273 }
274
275 {
276 sgml_cp p("&Zcaron;", SIZE_MAX, false, locale_slSI);
277 Assert::IsFalse(p.match(text, 4));
278 Assert::IsTrue(p.match(text, 4, _countof(text), match_case_insensitive));
279 Assert::AreEqual((size_t)4, p.interval.start);
280 Assert::AreEqual((size_t)12, p.interval.end);
281 }
282
283 {
284 sgml_space_cp p(false, locale_slSI);
285 Assert::IsFalse(p.match(text));
286 Assert::IsTrue(p.match(text, 1));
287 Assert::AreEqual((size_t)1, p.interval.start);
288 Assert::AreEqual((size_t)2, p.interval.end);
289 Assert::IsTrue(p.match(text, 79));
290 Assert::AreEqual((size_t)79, p.interval.start);
291 Assert::AreEqual((size_t)85, p.interval.end);
292 }
293
294 {
295 sgml_string_branch p(locale_slSI, "apple", "orange", "Ko&Zcaron;u&Scaron;&ccaron;Ku", nullptr);
296 Assert::IsFalse(p.match(text, 2));
297 Assert::IsTrue(p.match(text, 2, _countof(text), match_case_insensitive));
298 Assert::AreEqual((size_t)2, p.hit_offset);
299 Assert::AreEqual((size_t)2, p.interval.start);
300 Assert::AreEqual((size_t)31, p.interval.end);
301 }
302
303 {
304 sgml_space_cp space;
305 sgml_iban p(make_shared_no_delete(&space));
306 Assert::IsTrue(p.match("SI56023120015226972", 0, SIZE_MAX));
307 Assert::IsTrue(p.is_valid);
308 Assert::AreEqual("SI", p.country);
309 Assert::AreEqual("56", p.check_digits);
310 Assert::AreEqual("023120015226972", p.bban);
311 Assert::IsTrue(p.match("SI56 0231 2001 5226 972", 0, SIZE_MAX));
312 Assert::IsTrue(p.is_valid);
313 Assert::AreEqual("SI", p.country);
314 Assert::AreEqual("56", p.check_digits);
315 Assert::AreEqual("023120015226972", p.bban);
316 Assert::IsFalse(p.match("si56 0231 2001 5226 972", 0, SIZE_MAX));
317 Assert::IsFalse(p.is_valid);
318 Assert::IsTrue(p.match("si56 0231 2001 5226 972", 0, SIZE_MAX, match_case_insensitive));
319 Assert::IsTrue(p.is_valid);
320 Assert::IsTrue(p.match("SI56 0231 2001 5226 9720", 0, SIZE_MAX));
321 Assert::IsFalse(p.is_valid);
322 Assert::IsTrue(p.match("...SI56 0231 2001 5226 972...", 3, SIZE_MAX));
323 Assert::IsTrue(p.is_valid);
324 Assert::IsTrue(p.match("SI56&nbsp;0231&nbsp;2001&nbsp;5226&nbsp;972", 0, SIZE_MAX));
325 Assert::IsTrue(p.is_valid);
326 }
327
328 {
329 sgml_space_cp space;
330 sgml_creditor_reference p(make_shared_no_delete(&space));
331 Assert::IsTrue(p.match("RF18539007547034", 0, SIZE_MAX));
332 Assert::IsTrue(p.is_valid);
333 Assert::AreEqual("18", p.check_digits);
334 Assert::AreEqual("000000000539007547034", p.reference);
335 Assert::IsTrue(p.match("RF18 5390 0754 7034", 0, SIZE_MAX));
336 Assert::IsTrue(p.is_valid);
337 Assert::AreEqual("18", p.check_digits);
338 Assert::AreEqual("000000000539007547034", p.reference);
339 Assert::IsFalse(p.match("rf18 5390 0754 7034", 0, SIZE_MAX));
340 Assert::IsFalse(p.is_valid);
341 Assert::IsTrue(p.match("rf18 5390 0754 7034", 0, SIZE_MAX, match_case_insensitive));
342 Assert::IsTrue(p.is_valid);
343 Assert::IsTrue(p.match("RF18 5390 0754 70340", 0, SIZE_MAX));
344 Assert::IsFalse(p.is_valid);
345 Assert::IsTrue(p.match("...RF18 5390 0754 7034...", 3, SIZE_MAX));
346 Assert::IsTrue(p.is_valid);
347 Assert::IsTrue(p.match("RF18&nbsp;5390&nbsp;0754&nbsp;7034", 0, SIZE_MAX));
348 Assert::IsTrue(p.is_valid);
349 }
350
351 {
352 sgml_space_cp space;
353 sgml_si_reference p(make_shared_no_delete(&space));
354 Assert::IsTrue(p.match("SI121234567890120", 0, SIZE_MAX));
355 Assert::IsTrue(p.is_valid);
356 Assert::AreEqual("12", p.model);
357 Assert::AreEqual(stdex::interval<size_t>(4, 17), p.part1.interval);
358 Assert::IsTrue(p.match("SI12 1234567890120", 0, SIZE_MAX));
359 Assert::IsTrue(p.is_valid);
360 Assert::AreEqual("12", p.model);
361 Assert::AreEqual(stdex::interval<size_t>(5, 18), p.part1.interval);
362 Assert::IsFalse(p.match("si12 1234567890120", 0, SIZE_MAX));
363 Assert::IsTrue(p.match("si12 1234567890120", 0, SIZE_MAX, match_case_insensitive));
364 Assert::IsTrue(p.match("...SI12 1234567890120...", 3, SIZE_MAX));
365 Assert::IsTrue(p.match("SI12&nbsp;1234567890120", 0, SIZE_MAX));
366 }
367 }
368
369 TEST_METHOD(http_test)
370 {
371 static const std::locale locale("en_US.UTF-8");
372 static const char request[] =
373 "GET / HTTP/2\r\n"
374 "Host: stackoverflow.com\r\n"
375 "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/110.0\r\n"
376 "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8\r\n"
377 "Accept-Language: sl,en-US;q=0.8,en;q=0.6,de-DE;q=0.4,de;q=0.2\r\n"
378 "Accept-Encoding: gzip, deflate, br\r\n"
379 "DNT: 1\r\n"
380 "Connection: keep-alive\r\n"
381 "Cookie: prov=00000000-0000-0000-0000-000000000000; acct=t=00000000000000000%2f%2f0000%2b0000%2b000&s=00000000000000000000000000000000; OptanonConsent=isGpcEnabled=0&datestamp=Fri+Feb+03+2023+11%3A11%3A08+GMT%2B0100+(Srednjeevropski+standardni+%C4%8Das)&version=6.37.0&isIABGlobal=false&hosts=&consentId=00000000-0000-0000-0000-000000000000&interactionCount=1&landingPath=NotLandingPage&groups=00000%3A0%2C00000%3A0%2C00000%3A0%2C00000%3A0; OptanonAlertBoxClosed=2023-02-03T10:11:08.683Z\r\n"
382 "Upgrade-Insecure-Requests: 1\r\n"
383 "Sec-Fetch-Dest: document\r\n"
384 "Sec-Fetch-Mode: navigate\r\n"
385 "Sec-Fetch-Site: none\r\n"
386 "Sec-Fetch-User: ?1\r\n"
387 "Pragma: no-cache\r\n"
388 "Cache-Control: no-cache\r\n"
389 "\r\n";
390
391 {
392 http_request p(locale);
393 Assert::IsTrue(p.match(request));
394 Assert::AreEqual((size_t)0, p.interval.start);
395 Assert::AreEqual((size_t)14, p.interval.end);
396 Assert::AreEqual((size_t)0, p.verb.start);
397 Assert::AreEqual((size_t)3, p.verb.end);
398 Assert::AreEqual((size_t)4, p.url.interval.start);
399 Assert::AreEqual((size_t)5, p.url.interval.end);
400 Assert::AreEqual((size_t)6, p.protocol.interval.start);
401 Assert::AreEqual((size_t)12, p.protocol.interval.end);
402 Assert::AreEqual((uint16_t)0x200, p.protocol.version);
403 }
404
405 {
406 std::list<http_header> hdrs;
407 size_t offset = 14;
408 for (;;) {
409 http_header h;
410 if (h.match(request, offset)) {
411 offset = h.interval.end;
412 hdrs.push_back(std::move(h));
413 }
414 else
415 break;
416 }
417 Assert::AreEqual((size_t)15, hdrs.size());
418 http_weighted_collection<http_weighted_value<http_language>> langs;
419 for (const auto& h : hdrs)
420 if (strnicmp(request + h.name.start, h.name.size(), "Accept-Language", (size_t)-1, locale) == 0)
421 langs.insert(request, h.value.start, h.value.end);
422 Assert::IsTrue(!langs.empty());
423 {
424 const vector<std::string> control = {
425 "sl", "en-US", "en", "de-DE", "de"
426 };
427 auto c = control.cbegin();
428 auto l = langs.cbegin();
429 for (; c != control.cend() && l != langs.cend(); ++c, ++l)
430 Assert::IsTrue(strnicmp(request + l->value.interval.start, l->value.interval.size(), c->c_str(), c->size(), locale) == 0);
431 Assert::IsTrue(c == control.cend());
432 Assert::IsTrue(l == langs.cend());
433 }
434 }
435
436 //static const char response[] =
437 // "HTTP/2 200 OK\r\n"
438 // "cache-control: private\r\n"
439 // "content-type: text/html; charset=utf-8\r\n"
440 // "content-encoding: gzip\r\n"
441 // "strict-transport-security: max-age=15552000\r\n"
442 // "x-frame-options: SAMEORIGIN\r\n"
443 // "set-cookie: acct=t=00000000000000000%2f%2f0000%2b0000%2b000&s=00000000000000000000000000000000; expires=Sat, 16 Sep 2023 10:23:00 GMT; domain=.stackoverflow.com; path=/; secure; samesite=none; httponly\r\n"
444 // "set-cookie: prov_tgt=; expires=Tue, 14 Mar 2023 10:23:00 GMT; domain=.stackoverflow.com; path=/; secure; samesite=none; httponly\r\n"
445 // "x-request-guid: a6536a49-b473-4c6f-b313-c1e7c0d8f600\r\n"
446 // "feature-policy: microphone 'none'; speaker 'none'\r\n"
447 // "content-security-policy: upgrade-insecure-requests; frame-ancestors 'self' https://stackexchange.com\r\n"
448 // "accept-ranges: bytes\r\n"
449 // "date: Thu, 16 Mar 2023 10:23:00 GMT\r\n"
450 // "via: 1.1 varnish\r\n"
451 // "x-served-by: cache-vie6354-VIE\r\n"
452 // "x-cache: MISS\r\n"
453 // "x-cache-hits: 0\r\n"
454 // "x-timer: S1678962181.533907,VS0,VE144\r\n"
455 // "vary: Accept-Encoding,Fastly-SSL\r\n"
456 // "x-dns-prefetch-control: off\r\n"
457 // "X-Firefox-Spdy: h2\r\n"
458 // "\r\n";
459 }
460 };
461}
stdex::parser::basic_parser< char >
stdex::parser::basic_parser::interval
interval< size_t > interval
Region of the last match.
Definition parser.hpp:170
stdex::parser::basic_string
Test for given string.
Definition parser.hpp:816
stdex::parser::http_header
Test for HTTP header.
Definition parser.hpp:7157
stdex::parser::http_request
Test for HTTP request.
Definition parser.hpp:7019
stdex::parser::sgml_cp
Test for specific SGML code point.
Definition parser.hpp:341
stdex::parser::sgml_space_cp
Test for any SGML space code point.
Definition parser.hpp:428
stdex::interval
Numerical interval.
Definition interval.hpp:18
stdex::interval::size
T size() const
Returns interval size.
Definition interval.hpp:47
stdex::interval::end
T end
interval end
Definition interval.hpp:20
stdex::interval::start
T start
interval start
Definition interval.hpp:19
Generated on Tue Sep 19 2023 14:53:37 for stdex by doxygen 1.9.8