From 17edd0435c93c49887a83605117683eee335356a Mon Sep 17 00:00:00 2001 From: Adam Hojka Date: Thu, 9 Jun 2022 23:55:21 +0200 Subject: [PATCH 1/5] [HXCPP][HL] UNICODE fixes (clipboard, window title, file dialogs, paths, font glyphs, ...) --- project/src/ExternalInterface.cpp | 153 ++++++++++++++---------------- src/lime/text/Font.hx | 7 +- src/lime/ui/FileDialog.hx | 6 +- 3 files changed, 79 insertions(+), 87 deletions(-) diff --git a/project/src/ExternalInterface.cpp b/project/src/ExternalInterface.cpp index 08195e7eb..371d03fc5 100644 --- a/project/src/ExternalInterface.cpp +++ b/project/src/ExternalInterface.cpp @@ -48,12 +48,14 @@ #include #include -#ifdef HX_WINDOWS #include +#ifndef ANDROID #include #endif +#include #include +#include DEFINE_KIND (k_finalizer); @@ -133,15 +135,38 @@ namespace lime { } + std::string wstring_utf8 (const std::wstring& val) { + + #ifdef ANDROID + struct codecvt : public std::codecvt { + ~codecvt () = default; + }; + return std::wstring_convert ().to_bytes (val); + #else + return std::wstring_convert> ().to_bytes (val); + #endif + + } + + + vbyte* hl_wstring_to_utf8_bytes (const std::wstring& val) { + + const std::string utf8 (wstring_utf8 (val)); + vbyte* const bytes = hl_alloc_bytes (utf8.size () + 1); + std::memcpy(bytes, utf8.c_str (), utf8.size () + 1); + return bytes; + + } + + std::wstring* hxstring_to_wstring (HxString val) { if (val.c_str ()) { - std::string _val = std::string (val.c_str ()); #ifdef HX_WINDOWS - std::wstring_convert> converter; - return new std::wstring (converter.from_bytes (_val)); + return new std::wstring (hxs_wchar (val, nullptr)); #else + const std::string _val (hxs_utf8 (val, nullptr)); return new std::wstring (_val.begin (), _val.end ()); #endif @@ -473,7 +498,7 @@ namespace lime { value lime_bytes_read_file (HxString path, value bytes) { Bytes data (bytes); - data.ReadFile (path.c_str ()); + data.ReadFile (hxs_utf8 (path, nullptr)); return data.Value (bytes); } @@ -573,7 +598,7 @@ namespace lime { void lime_clipboard_set_text (HxString text) { - Clipboard::SetText (text.c_str ()); + Clipboard::SetText (hxs_utf8 (text, nullptr)); } @@ -720,13 +745,9 @@ namespace lime { if (path) { - int size = std::wcslen (path->c_str ()); - char* result = (char*)malloc (size + 1); - std::wcstombs (result, path->c_str (), size); - result[size] = '\0'; + vbyte* const result = hl_wstring_to_utf8_bytes (*path); delete path; - - return (vbyte*)result; + return result; } else { @@ -790,13 +811,9 @@ namespace lime { if (path) { - int size = std::wcslen (path->c_str ()); - char* result = (char*)malloc (size + 1); - std::wcstombs (result, path->c_str (), size); - result[size] = '\0'; + vbyte* const result = hl_wstring_to_utf8_bytes (*path); delete path; - - return (vbyte*)result; + return result; } else { @@ -864,12 +881,7 @@ namespace lime { for (int i = 0; i < files.size (); i++) { - int size = std::wcslen (files[i]->c_str ()); - char* _file = (char*)malloc (size + 1); - std::wcstombs (_file, files[i]->c_str (), size); - _file[size] = '\0'; - - *resultData++ = (vbyte*)_file; + *resultData++ = hl_wstring_to_utf8_bytes (*files[i]); delete files[i]; } @@ -932,13 +944,9 @@ namespace lime { if (path) { - int size = std::wcslen (path->c_str ()); - char* result = (char*)malloc (size + 1); - std::wcstombs (result, path->c_str (), size); - result[size] = '\0'; + vbyte* const result = hl_wstring_to_utf8_bytes (*path); delete path; - - return (vbyte*)result; + return result; } else { @@ -1109,12 +1117,11 @@ namespace lime { #ifdef LIME_FREETYPE Font *font = (Font*)fontHandle->ptr; wchar_t *name = font->GetFamilyName (); - int size = std::wcslen (name); - char* result = (char*)malloc (size + 1); - std::wcstombs (result, name, size); - result[size] = '\0'; + if (!name) + return nullptr; + vbyte* const result = hl_wstring_to_utf8_bytes (name); delete name; - return (vbyte*)result; + return result; #else return 0; #endif @@ -1126,7 +1133,10 @@ namespace lime { #ifdef LIME_FREETYPE Font *font = (Font*)val_data (fontHandle); - return font->GetGlyphIndex ((char*)character.c_str ()); + const std::string str (hxs_utf8 (character, nullptr)); + const std::unique_ptr buff (new char[str.size() + 1]); + std::memcpy (buff.get (), str.c_str (), str.size () + 1); + return font->GetGlyphIndex (buff.get ()); #else return -1; #endif @@ -1150,7 +1160,10 @@ namespace lime { #ifdef LIME_FREETYPE Font *font = (Font*)val_data (fontHandle); - return (value)font->GetGlyphIndices (true, (char*)characters.c_str ()); + const std::string str (hxs_utf8 (characters, nullptr)); + const std::unique_ptr buff (new char[str.size() + 1]); + std::memcpy (buff.get (), str.c_str (), str.size () + 1); + return (value)font->GetGlyphIndices (true, buff.get ()); #else return alloc_null (); #endif @@ -2327,7 +2340,7 @@ namespace lime { value lime_jpeg_decode_file (HxString path, bool decodeData, value buffer) { ImageBuffer imageBuffer (buffer); - Resource resource = Resource (path.c_str ()); + Resource resource = Resource (hxs_utf8 (path, nullptr)); #ifdef LIME_JPEG if (JPEG::Decode (&resource, &imageBuffer, decodeData)) { @@ -2565,7 +2578,7 @@ namespace lime { value lime_png_decode_file (HxString path, bool decodeData, value buffer) { ImageBuffer imageBuffer (buffer); - Resource resource = Resource (path.c_str ()); + Resource resource = Resource (hxs_utf8 (path, nullptr)); #ifdef LIME_PNG if (PNG::Decode (&resource, &imageBuffer, decodeData)) { @@ -2670,13 +2683,9 @@ namespace lime { if (model) { - int size = std::wcslen (model->c_str ()); - char* result = (char*)malloc (size + 1); - std::wcstombs (result, model->c_str (), size); - result[size] = '\0'; + vbyte* const result = hl_wstring_to_utf8_bytes (*model); delete model; - - return (vbyte*)result; + return result; } @@ -2714,13 +2723,9 @@ namespace lime { if (vendor) { - int size = std::wcslen (vendor->c_str ()); - char* result = (char*)malloc (size + 1); - std::wcstombs (result, vendor->c_str (), size); - result[size] = '\0'; + vbyte* const result = hl_wstring_to_utf8_bytes (*vendor); delete vendor; - - return (vbyte*)result; + return result; } @@ -2733,7 +2738,7 @@ namespace lime { value lime_system_get_directory (int type, HxString company, HxString title) { - std::wstring* path = System::GetDirectory ((SystemDirectory)type, company.c_str (), title.c_str ()); + std::wstring* path = System::GetDirectory ((SystemDirectory)type, hxs_utf8 (company, nullptr), hxs_utf8 (title, nullptr)); if (path) { @@ -2758,13 +2763,9 @@ namespace lime { if (path) { - int size = std::wcslen (path->c_str ()); - char* result = (char*)malloc (size + 1); - std::wcstombs (result, path->c_str (), size); - result[size] = '\0'; + vbyte* const result = hl_wstring_to_utf8_bytes (*path); delete path; - - return (vbyte*)result; + return result; } @@ -2852,13 +2853,9 @@ namespace lime { if (label) { - int size = std::wcslen (label->c_str ()); - char* result = (char*)malloc (size + 1); - std::wcstombs (result, label->c_str (), size); - result[size] = '\0'; + vbyte* const result = hl_wstring_to_utf8_bytes (*label); delete label; - - return (vbyte*)result; + return result; } @@ -2896,13 +2893,9 @@ namespace lime { if (name) { - int size = std::wcslen (name->c_str ()); - char* result = (char*)malloc (size + 1); - std::wcstombs (result, name->c_str (), size); - result[size] = '\0'; + vbyte* const result = hl_wstring_to_utf8_bytes (*name); delete name; - - return (vbyte*)result; + return result; } @@ -2940,14 +2933,9 @@ namespace lime { if (version) { - int size = std::wcslen (version->c_str ()); - char* result = (char*)malloc (size + 1); - std::wcstombs (result, version->c_str (), size); - result[size] = '\0'; + vbyte* const result = hl_wstring_to_utf8_bytes (*version); delete version; - - return (vbyte*)result; - + return result; } #endif @@ -3100,7 +3088,7 @@ namespace lime { void lime_window_alert (value window, HxString message, HxString title) { Window* targetWindow = (Window*)val_data (window); - targetWindow->Alert (message.c_str (), title.c_str ()); + targetWindow->Alert (hxs_utf8 (message, nullptr), hxs_utf8 (title, nullptr)); } @@ -3108,7 +3096,7 @@ namespace lime { HL_PRIM void HL_NAME(hl_window_alert) (HL_CFFIPointer* window, hl_vstring* message, hl_vstring* title) { Window* targetWindow = (Window*)window->ptr; - targetWindow->Alert (message ? (const char*)hl_to_utf8 ((const uchar*)message) : NULL, title ? (const char*)hl_to_utf8 ((const uchar*)title) : NULL); + targetWindow->Alert (message ? hl_to_utf8 (message->bytes) : nullptr, title ? hl_to_utf8 (title->bytes) : nullptr); } @@ -3187,7 +3175,7 @@ namespace lime { value lime_window_create (value application, int width, int height, int flags, HxString title) { - Window* window = CreateWindow ((Application*)val_data (application), width, height, flags, title.c_str ()); + Window* window = CreateWindow ((Application*)val_data (application), width, height, flags, hxs_utf8 (title, nullptr)); return CFFIPointer (window, gc_window); } @@ -3688,13 +3676,14 @@ namespace lime { value lime_window_set_title (value window, HxString title) { Window* targetWindow = (Window*)val_data (window); - const char* result = targetWindow->SetTitle (title.c_str ()); + const char* titleUtf8 = hxs_utf8 (title, nullptr); + const char* result = targetWindow->SetTitle (titleUtf8); if (result) { value _result = alloc_string (result); - if (result != title.c_str ()) { + if (result != titleUtf8) { free ((char*) result); diff --git a/src/lime/text/Font.hx b/src/lime/text/Font.hx index 30681e3d7..88dff1f05 100644 --- a/src/lime/text/Font.hx +++ b/src/lime/text/Font.hx @@ -188,8 +188,11 @@ class Font public function getGlyphs(characters:String = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^`'\"/\\&*()[]{}<>|:;_-+=?,. "):Array { #if (lime_cffi && !macro) - var glyphs:Dynamic = NativeCFFI.lime_font_get_glyph_indices(src, characters); - return glyphs; + #if hl + return [for (index in NativeCFFI.lime_font_get_glyph_indices(src, characters)) new Glyph(index)]; + #else + return NativeCFFI.lime_font_get_glyph_indices(src, characters); + #end #else return null; #end diff --git a/src/lime/ui/FileDialog.hx b/src/lime/ui/FileDialog.hx index 559cc912c..ea3f0b4fb 100644 --- a/src/lime/ui/FileDialog.hx +++ b/src/lime/ui/FileDialog.hx @@ -112,7 +112,7 @@ class FileDialog var path = null; #if hl var bytes = NativeCFFI.lime_file_dialog_open_file(title, filter, defaultPath); - path = @:privateAccess String.fromUTF8(cast bytes); + path = bytes != null ? @:privateAccess String.fromUTF8(cast bytes) : null; #else path = NativeCFFI.lime_file_dialog_open_file(title, filter, defaultPath); #end @@ -149,7 +149,7 @@ class FileDialog var path = null; #if hl var bytes = NativeCFFI.lime_file_dialog_open_directory(title, filter, defaultPath); - path = @:privateAccess String.fromUTF8(cast bytes); + path = bytes != null ? @:privateAccess String.fromUTF8(cast bytes) : null; #else path = NativeCFFI.lime_file_dialog_open_directory(title, filter, defaultPath); #end @@ -311,7 +311,7 @@ class FileDialog var path = null; #if hl var bytes = NativeCFFI.lime_file_dialog_save_file(title, filter, defaultPath); - path = @:privateAccess String.fromUTF8(cast bytes); + path = bytes != null ? @:privateAccess String.fromUTF8(cast bytes) : null; #else path = NativeCFFI.lime_file_dialog_save_file(title, filter, defaultPath); #end From 981d02e4112645fe6bb7a586a868f00a8aeac0ea Mon Sep 17 00:00:00 2001 From: Adam Hojka Date: Sat, 11 Jun 2022 18:30:26 +0200 Subject: [PATCH 2/5] Custom wstring to UTF-8 string conversion function (codecvt may not be included in some toolchains and was deprecated in C++17 anyway). --- project/src/ExternalInterface.cpp | 62 ++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 9 deletions(-) diff --git a/project/src/ExternalInterface.cpp b/project/src/ExternalInterface.cpp index 371d03fc5..1b70d43fa 100644 --- a/project/src/ExternalInterface.cpp +++ b/project/src/ExternalInterface.cpp @@ -48,8 +48,8 @@ #include #include +#ifdef HX_WINDOWS #include -#ifndef ANDROID #include #endif #include @@ -137,14 +137,58 @@ namespace lime { std::string wstring_utf8 (const std::wstring& val) { - #ifdef ANDROID - struct codecvt : public std::codecvt { - ~codecvt () = default; - }; - return std::wstring_convert ().to_bytes (val); - #else - return std::wstring_convert> ().to_bytes (val); - #endif + std::string out; + unsigned int codepoint = 0; + + for (const wchar_t chr : val) { + + if (chr >= 0xd800 && chr <= 0xdbff) { + + codepoint = ((chr - 0xd800) << 10) + 0x10000; + + } else { + + if (chr >= 0xdc00 && chr <= 0xdfff) { + + codepoint |= chr - 0xdc00; + + } else { + + codepoint = chr; + + } + + if (codepoint <= 0x7f) { + + out.append (1, static_cast (codepoint)); + + } else if (codepoint <= 0x7ff) { + + out.append (1, static_cast (0xc0 | ((codepoint >> 6) & 0x1f))); + out.append (1, static_cast (0x80 | (codepoint & 0x3f))); + + } else if (codepoint <= 0xffff) { + + out.append (1, static_cast (0xe0 | ((codepoint >> 12) & 0x0f))); + out.append (1, static_cast (0x80 | ((codepoint >> 6) & 0x3f))); + out.append (1, static_cast (0x80 | (codepoint & 0x3f))); + + } else { + + out.append (1, static_cast (0xf0 | ((codepoint >> 18) & 0x07))); + out.append (1, static_cast (0x80 | ((codepoint >> 12) & 0x3f))); + out.append (1, static_cast (0x80 | ((codepoint >> 6) & 0x3f))); + out.append (1, static_cast (0x80 | (codepoint & 0x3f))); + + } + + codepoint = 0; + + } + + } + + return out; } From 1517bb3557d3228b52d0322a8cc1d2c278be324e Mon Sep 17 00:00:00 2001 From: Tobi Laskowski Date: Fri, 5 Jan 2024 11:26:17 +0000 Subject: [PATCH 3/5] Clean up --- project/src/ExternalInterface.cpp | 32 +++---------------------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/project/src/ExternalInterface.cpp b/project/src/ExternalInterface.cpp index 522255dfd..0f5503533 100644 --- a/project/src/ExternalInterface.cpp +++ b/project/src/ExternalInterface.cpp @@ -264,34 +264,6 @@ namespace lime { } - vbyte* wstring_to_vbytes (std::wstring* val) { - - if (val) { - - #ifdef HX_WINDOWS - int size = std::wcslen (val->c_str ()); - char* result = (char*)malloc (size + 1); - std::wcstombs (result, val->c_str (), size); - result[size] = '\0'; - return (vbyte*)result; - #else - std::string _val = std::string (val->begin (), val->end ()); - int size = std::strlen (_val.c_str ()); - char* result = (char*)malloc (size + 1); - std::strncpy (result, _val.c_str (), size); - result[size] = '\0'; - return (vbyte*)result; - #endif - - } else { - - return 0; - - } - - } - - value lime_application_create () { Application* application = CreateApplication (); @@ -3189,7 +3161,9 @@ namespace lime { HL_PRIM void HL_NAME(hl_window_alert) (HL_CFFIPointer* window, hl_vstring* message, hl_vstring* title) { Window* targetWindow = (Window*)window->ptr; - targetWindow->Alert (message ? hl_to_utf8 (message->bytes) : nullptr, title ? hl_to_utf8 (title->bytes) : nullptr); + const char *cmessage = message ? hl_to_utf8(message->bytes) : nullptr; + const char *ctitle = title ? hl_to_utf8(title->bytes) : nullptr; + targetWindow->Alert (cmessage, ctitle); } From 590be2bbe6e33d832bc20e6b33c26221824b1b04 Mon Sep 17 00:00:00 2001 From: player-03 Date: Wed, 31 Jan 2024 13:46:09 -0500 Subject: [PATCH 4/5] Remove unnecessary string copy. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Casting `const char*` → `char*` is bad form, and we'd rather copy the string than do that. However, `GetGlyphIndex()` now takes a const, so neither workaround is needed. --- project/src/ExternalInterface.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/project/src/ExternalInterface.cpp b/project/src/ExternalInterface.cpp index 0f5503533..85987e569 100644 --- a/project/src/ExternalInterface.cpp +++ b/project/src/ExternalInterface.cpp @@ -1198,10 +1198,7 @@ namespace lime { #ifdef LIME_FREETYPE Font *font = (Font*)val_data (fontHandle); - const std::string str (hxs_utf8 (character, nullptr)); - const std::unique_ptr buff (new char[str.size() + 1]); - std::memcpy (buff.get (), str.c_str (), str.size () + 1); - return font->GetGlyphIndex (buff.get ()); + return font->GetGlyphIndex (character.c_str ()); #else return -1; #endif From 7c50f6e9e9276018b62a39032de40b5c8d1d84b6 Mon Sep 17 00:00:00 2001 From: player-03 Date: Wed, 31 Jan 2024 14:02:50 -0500 Subject: [PATCH 5/5] Remove another unnecessary buffer copy. Also, use `hxs_utf8()` for consistency. --- project/src/ExternalInterface.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/project/src/ExternalInterface.cpp b/project/src/ExternalInterface.cpp index 85987e569..33a2f9c6f 100644 --- a/project/src/ExternalInterface.cpp +++ b/project/src/ExternalInterface.cpp @@ -1198,7 +1198,7 @@ namespace lime { #ifdef LIME_FREETYPE Font *font = (Font*)val_data (fontHandle); - return font->GetGlyphIndex (character.c_str ()); + return font->GetGlyphIndex (hxs_utf8 (character, nullptr)); #else return -1; #endif @@ -1222,10 +1222,7 @@ namespace lime { #ifdef LIME_FREETYPE Font *font = (Font*)val_data (fontHandle); - const std::string str (hxs_utf8 (characters, nullptr)); - const std::unique_ptr buff (new char[str.size() + 1]); - std::memcpy (buff.get (), str.c_str (), str.size () + 1); - return (value)font->GetGlyphIndices (true, buff.get ()); + return (value)font->GetGlyphIndices (true, hxs_utf8 (characters, nullptr)); #else return alloc_null (); #endif