Imported Upstream version 7.3.3

Change-Id: I19b8eba9412d6377bfa3daa54e965fbed99940b2
Signed-off-by: DongHun Kwak <dh0128.kwak@samsung.com>

28 files changed, 105 insertions, 30 deletions

diff --git a/test/Makefile b/test/Makefile
index e2a55e9..e2a55e9 100755..100644
--- a/test/Makefile
+++ b/test/Makefile
diff --git a/test/chard7.txt b/test/chard7.txt
index 3b3c9fc..3b3c9fc 100755..100644
--- a/test/chard7.txt
+++ b/test/chard7.txt
diff --git a/test/chardos.txt b/test/chardos.txt
index ff85358..ff85358 100755..100644
--- a/test/chardos.txt
+++ b/test/chardos.txt
diff --git a/test/charu7.txt b/test/charu7.txt
index 0073580..0073580 100755..100644
--- a/test/charu7.txt
+++ b/test/charu7.txt
diff --git a/test/dos_bin.txt b/test/dos_bin.txt
index 82c9864..82c9864 100755..100644
--- a/test/dos_bin.txt
+++ b/test/dos_bin.txt
diff --git a/test/gb18030.txt b/test/gb18030.txt
index 9e294aa..9e294aa 100755..100644
--- a/test/gb18030.txt
+++ b/test/gb18030.txt
diff --git a/test/gb18030b.txt b/test/gb18030b.txt
index 2b08b78..2b08b78 100755..100644
--- a/test/gb18030b.txt
+++ b/test/gb18030b.txt
diff --git a/test/gb18030u.txt b/test/gb18030u.txt
index 7687092..7687092 100755..100644
--- a/test/gb18030u.txt
+++ b/test/gb18030u.txt
diff --git a/test/invalhig.txt b/test/invalhig.txt
index da9ce0f..da9ce0f 100755..100644
--- a/test/invalhig.txt
+++ b/test/invalhig.txt
diff --git a/test/invallow.txt b/test/invallow.txt
index c870871..c870871 100755..100644
--- a/test/invallow.txt
+++ b/test/invallow.txt
diff --git a/test/setlocale.c b/test/setlocale.c
new file mode 100644
index 0000000..005f0fb
--- /dev/null
+++ b/test/setlocale.c
@@ -0,0 +1,89 @@
+#include <stdio.h>

+#include <fcntl.h>

+#include <windows.h>

+#include <locale.h>

+

+/*

+

+ This program proves that when you set the locale to "", the Chinese ANSI CP936

+ encoded text is printed wrongly in a simplified Chinese regional setting.

+ UTF-8 is also printed wrongly.

+

+To test this program you first need to change the Windows locale setting to

+simplified Chinese. There is no problem doing that, because the "display

+language" will stay the original language, and you can still use your Windows.

+

+Control Panel > Region and Language > Administrative > Change system locale

+

+Select simplified Chinese and reboot your PC.

+

+For example output see setlocale.png.

+

+ */

+

+print_string(const char* str, const char* ustr, const wchar_t *wstr) {

+  int prevmode;

+  UINT outputCP;

+  int utf8 = 0;

+

+  /* Set utf8 to 1 to print UTF-8 text.

+     If you print both UTF-16 en UTF-8 in one program the console gets

+     mixed up. When the UTF-8 is printed after UTF-16 the UTF-16 text

+     is displayed wrongly.

+   */ 

+

+  if ( ! utf8 ) {

+    /* When the locale is set to "" the following line will produce wrong output */

+      printf ("ANSI CP936 %s\n",str);

+

+    /* UTF-16 will produce correct output in all cases. */

+      prevmode = _setmode(_fileno(stdout), _O_U16TEXT);

+      wprintf(L"UTF-16    %ls\n",wstr);

+      _setmode(_fileno(stdout), prevmode);

+   } else {

+

+    /* UTF-8 will produce wrong output when the locale is "".

+       When the locale is "C" wrong output with raster font, and correct output

+       with TrueType font. */

+      outputCP = GetConsoleOutputCP();

+      SetConsoleOutputCP(CP_UTF8);

+      wprintf(L"UTF-8     %S\n",ustr);

+      SetConsoleOutputCP(outputCP);

+

+    /* The code below produces wrong output in all cases */

+    //  prevmode = _setmode(_fileno(stdout), _O_U8TEXT);

+    //  wprintf(L"UTF-8     %S",ustr);

+    //  _setmode(_fileno(stdout), prevmode);

+   }

+

+}

+

+int main() {

+

+  char str[5];      /* ANSI CP936 */

+  char ustr[15];    /* UTF-8 */

+  wchar_t wstr[10]; /* UTF-16 */

+

+/* Create ANSI CP936 string (meaning: Western-European). */

+  str[0] = 0xce;

+  str[1] = 0xf7;

+  str[2] = 0xc5;

+  str[3] = 0xb7;

+  str[4] = '\0';

+/* Convert CP936 to UTF-16. */

+  MultiByteToWideChar(936, 0, str, -1, wstr, sizeof(wstr));

+/* Convert UTF-16 to UTF-8 */

+  WideCharToMultiByte(CP_UTF8, 0, wstr, -1, ustr, sizeof(ustr), NULL, NULL);

+

+

+

+  setlocale (LC_ALL, "");

+  printf("==> setlocale (LC_ALL, \"\");\n");

+  print_string(str, ustr, wstr);

+

+  setlocale (LC_ALL, "C");

+  printf("\n==> setlocale (LC_ALL, \"C\");\n");

+  print_string(str, ustr, wstr);

+

+  return  0;

+}

diff --git a/test/setlocale.png b/test/setlocale.png
new file mode 100644
index 0000000..9920216
--- /dev/null
+++ b/test/setlocale.png
diff --git a/test/testcmd.bat b/test/testcmd.bat
index 9431352..8acea08 100755
--- a/test/testcmd.bat
+++ b/test/testcmd.bat
@@ -8,7 +8,7 @@ REM This script will only run on Windows 7 and higher.
 

 REM switch to UTF-8 code page

 chcp 65001

-dos2unix -i uni_el_αρχείο.txt uni_zh_文件.txt

+dos2unix -D unicode -i uni_el_αρχείο.txt uni_zh_文件.txt

 

 REM set code page back to original value

 chcp 850

diff --git a/test/testps16.ps1 b/test/testps16.ps1
index 059ec03..c927442 100755
--- a/test/testps16.ps1
+++ b/test/testps16.ps1
diff --git a/test/testps8.ps1 b/test/testps8.ps1
index 7bf72db..dc2bcfe 100755
--- a/test/testps8.ps1
+++ b/test/testps8.ps1
@@ -8,5 +8,8 @@ dos2unix -i uni_el_αρχείο.txt uni_zh_文件.txt
 

 echo "test select-string:"

 

-# The following is not working. Why?

-dos2unix -i uni* | select-string -encoding utf8 -pattern αρχ
\ No newline at end of file
+# select-string requires a BOM.

+$env:DOS2UNIX_DISPLAY_ENC = "utf8bom"

+dos2unix -i uni* | select-string -encoding utf8 -pattern αρχ

+

+$env:DOS2UNIX_DISPLAY_ENC = ""

diff --git a/test/testu16.c b/test/testu16.c
index 8ed706a..d0a8f9e 100755..100644
--- a/test/testu16.c
+++ b/test/testu16.c
@@ -3,44 +3,27 @@
 #include <fcntl.h>

 #include <io.h>

 

-/* This program demonstrates that when Unicode printed text is redirected to

-   a file, the file is not in correct UTF-16.

+/* This program demonstrates Unicode UTF-16 printed text redirected to a

+   correct UTF-16 file.

+

+   .\testu16.exe > out.txt

+

 */

 

 int main () {

 

   int prevmode;

 

-/*

-  When the output of this program is redirected to a file in Windows Command

-  Prompt you get for line breaks 0d0a 00, while it should be 0d00 0a00.

-

-c:\test>.\testu16.exe > o.txt

-

-c:\test>xxd o.txt

-0000000: 6f00 6e00 6500 0d0a 0074 0077 006f 000d  o.n.e....t.w.o..

-0000010: 0a00 7400 6800 7200 6500 6500 0d0a 00    ..t.h.r.e.e....

-

-

-

-  When the output is redirected to a file in PowerShell, null characters

-  0000 are inserted. It looks like UTF-32 with an UTF-16 BOM and UTF-16 line

-  breaks.

-

-PS C:\test> .\testu16.exe > p.txt

-PS C:\test> xxd p.txt

-0000000: fffe 6f00 0000 6e00 0000 6500 0000 0d00  ..o...n...e.....

-0000010: 0a00 0000 7400 0000 7700 0000 6f00 0000  ....t...w...o...

-0000020: 0d00 0a00 0000 7400 0000 6800 0000 7200  ......t...h...r.

-0000030: 0000 6500 0000 6500 0000 0d00 0a00 0000  ..e...e.........

-0000040: 0d00 0a00                                ....

-

-*/

 

   prevmode = _setmode(_fileno(stdout), _O_U16TEXT);

+  /* We need to print an UTF-16 BOM for correct redirection in PowerShell. */

+  fwprintf(stdout, L"\xfeff");

   fwprintf(stdout,L"one\n");

   fwprintf(stdout,L"two\n");

   fwprintf(stdout,L"three\n");

+  /* Flushing stdout is required to get correct UTF-16.

+     This is required for both CMD.exe and PowerShell. */

+  fflush(stdout);

   _setmode(_fileno(stdout), prevmode);

 

 

diff --git a/test/uni_nl_ëäï.txt b/test/uni_nl_ëäï.txt
index a9cd879..a9cd879 100755..100644
--- a/test/uni_nl_ëäï.txt
+++ b/test/uni_nl_ëäï.txt
diff --git a/test/unix_bin.txt b/test/unix_bin.txt
index b2cafbc..b2cafbc 100755..100644
--- a/test/unix_bin.txt
+++ b/test/unix_bin.txt
diff --git a/test/utf16.txt b/test/utf16.txt
index 868b5af..868b5af 100755..100644
--- a/test/utf16.txt
+++ b/test/utf16.txt
diff --git a/test/utf16be.txt b/test/utf16be.txt
index 1be5db9..1be5db9 100755..100644
--- a/test/utf16be.txt
+++ b/test/utf16be.txt
diff --git a/test/utf16bin.txt b/test/utf16bin.txt
index 6c4216b..6c4216b 100755..100644
--- a/test/utf16bin.txt
+++ b/test/utf16bin.txt
diff --git a/test/utf16le.txt b/test/utf16le.txt
index db9f535..db9f535 100755..100644
--- a/test/utf16le.txt
+++ b/test/utf16le.txt
diff --git a/test/utf16m.txt b/test/utf16m.txt
index cd523f8..cd523f8 100755..100644
--- a/test/utf16m.txt
+++ b/test/utf16m.txt
diff --git a/test/utf16u.txt b/test/utf16u.txt
index 2bd9b3b..2bd9b3b 100755..100644
--- a/test/utf16u.txt
+++ b/test/utf16u.txt
diff --git a/test/utf8dos.txt b/test/utf8dos.txt
index 76afa4f..76afa4f 100755..100644
--- a/test/utf8dos.txt
+++ b/test/utf8dos.txt
diff --git a/test/utf8dosn.txt b/test/utf8dosn.txt
index 154aa88..154aa88 100755..100644
--- a/test/utf8dosn.txt
+++ b/test/utf8dosn.txt
diff --git a/test/utf8unix.txt b/test/utf8unix.txt
index 8b93c0c..8b93c0c 100755..100644
--- a/test/utf8unix.txt
+++ b/test/utf8unix.txt
diff --git a/test/utf8unxb.txt b/test/utf8unxb.txt
index f670e9b..f670e9b 100755..100644
--- a/test/utf8unxb.txt
+++ b/test/utf8unxb.txt