avutil/wchar_filename: propagate MultiByteToWideChar() and WideCharToMultiByte() failures

Don't return success if the string could not be converted.

Signed-off-by: James Almer <jamrial@gmail.com>
(cherry picked from commit 92885f2681)

libavutil/wchar_filename.h

@@ -32,7 +32,8 @@ static inline int utf8towchar(const char *filename_utf8, wchar_t **filename_w)
     num_chars = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, filename_utf8, -1, NULL, 0);
     if (num_chars <= 0) {
         *filename_w = NULL;
-        return 0;
+        errno = EINVAL;
+        return -1;
     }
     *filename_w = (wchar_t *)av_calloc(num_chars, sizeof(wchar_t));
     if (!*filename_w) {
@@ -52,7 +53,8 @@ static inline int wchartocp(unsigned int code_page, const wchar_t *filename_w,
                                         NULL, 0, NULL, NULL);
     if (num_chars <= 0) {
         *filename = NULL;
-        return 0;
+        errno = EINVAL;
+        return -1;
     }
     *filename = (char*)av_malloc_array(num_chars, sizeof *filename);
     if (!*filename) {