Merge pull request #5044 from laszlocsomor/expand-wildcards

Windows: expand path names on command line

BUILD

@@ -405,25 +405,7 @@ cc_library(
     ],
     copts = COPTS,
     includes = ["src/"],
-    linkopts = LINK_OPTS + select({
+    linkopts = LINK_OPTS,
-        ":msvc": [
-            # Linking to setargv.obj makes the default command line argument
-            # parser expand wildcards, so the main method's argv will contain the
-            # expanded list instead of the wildcards.
-            #
-            # Adding dummy "-DEFAULTLIB:kernel32.lib", because:
-            # - Microsoft ships this object file next to default libraries
-            # - but this file is not a library, just a precompiled object
-            # - "-WHOLEARCHIVE" and "-DEFAULTLIB" only accept library,
-            #   not precompiled object.
-            # - Bazel would assume linkopt that does not start with "-" or "$"
-            #   as a label to a target, so we add a harmless "-DEFAULTLIB:kernel32.lib"
-            #   before "setargv.obj".
-            # See https://msdn.microsoft.com/en-us/library/8bch7bkk.aspx
-            "-DEFAULTLIB:kernel32.lib setargv.obj",
-        ],
-        "//conditions:default": [],
-    }),
     visibility = ["//visibility:public"],
     deps = [":protobuf"],
 )

src/google/protobuf/compiler/command_line_interface.cc

@@ -1427,7 +1427,32 @@ CommandLineInterface::InterpretArgument(const std::string& name,
       return PARSE_ARGUMENT_FAIL;
     }
 
+#if defined(_WIN32)
+    // On Windows, the shell (typically cmd.exe) does not expand wildcards in
+    // file names (e.g. foo\*.proto), so we do it ourselves.
+    switch (google::protobuf::io::win32::ExpandWildcards(
+          value,
+          [this](const string& path) {
+            this->input_files_.push_back(path);
+          })) {
+      case google::protobuf::io::win32::ExpandWildcardsResult::kSuccess:
+        break;
+      case google::protobuf::io::win32::ExpandWildcardsResult::kErrorNoMatchingFile:
+        // Path does not exist, is not a file, or it's longer than MAX_PATH and
+        // long path handling is disabled.
+        std::cerr << "Invalid file name pattern or missing input file \""
+                  << value << "\"" << std::endl;
+        return PARSE_ARGUMENT_FAIL;
+      default:
+        std::cerr << "Cannot convert path \"" << value
+                  << "\" to or from Windows style" << std::endl;
+        return PARSE_ARGUMENT_FAIL;
+    }
+#else  // not _WIN32
+    // On other platforms than Windows (e.g. Linux, Mac OS) the shell (typically
+    // Bash) expands wildcards.
     input_files_.push_back(value);
+#endif  // _WIN32
 
   } else if (name == "-I" || name == "--proto_path") {
     // Java's -classpath (and some other languages) delimits path components

src/google/protobuf/io/io_win32.cc

@@ -59,6 +59,11 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <wctype.h>
+
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN 1
+#endif
+
 #include <windows.h>
 
 #include <memory>
@@ -356,6 +361,56 @@ wstring testonly_utf8_to_winpath(const char* path) {
   return as_windows_path(path, &wpath) ? wpath : wstring();
 }
 
+ExpandWildcardsResult ExpandWildcards(
+    const string& path, std::function<void(const string&)> consume) {
+  if (path.find_first_of("*?") == string::npos) {
+    // There are no wildcards in the path, we don't need to expand it.
+    consume(path);
+    return ExpandWildcardsResult::kSuccess;
+  }
+
+  wstring wpath;
+  if (!as_windows_path(path.c_str(), &wpath)) {
+    return ExpandWildcardsResult::kErrorInputPathConversion;
+  }
+
+  static const wstring kDot = L".";
+  static const wstring kDotDot = L"..";
+  WIN32_FIND_DATAW metadata;
+  HANDLE handle = ::FindFirstFileW(wpath.c_str(), &metadata);
+  if (handle == INVALID_HANDLE_VALUE) {
+    // The pattern does not match any files (or directories).
+    return ExpandWildcardsResult::kErrorNoMatchingFile;
+  }
+
+  string::size_type pos = path.find_last_of("\\/");
+  string dirname;
+  if (pos != string::npos) {
+    dirname = path.substr(0, pos + 1);
+  }
+
+  ExpandWildcardsResult matched = ExpandWildcardsResult::kErrorNoMatchingFile;
+  do {
+    // Ignore ".", "..", and directories.
+    if ((metadata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0
+        && kDot != metadata.cFileName && kDotDot != metadata.cFileName) {
+      matched = ExpandWildcardsResult::kSuccess;
+      string filename;
+      if (!strings::wcs_to_utf8(metadata.cFileName, &filename)) {
+        return ExpandWildcardsResult::kErrorOutputPathConversion;
+      }
+
+      if (dirname.empty()) {
+        consume(filename);
+      } else {
+        consume(dirname + filename);
+      }
+    }
+  } while (::FindNextFileW(handle, &metadata));
+  FindClose(handle);
+  return matched;
+}
+
 namespace strings {
 
 bool wcs_to_mbs(const WCHAR* s, string* out, bool outUtf8) {

src/google/protobuf/io/io_win32.h

@@ -49,6 +49,7 @@
 
 #if defined(_WIN32)
 
+#include <functional>
 #include <string>
 
 #include <google/protobuf/port.h>
@@ -76,6 +77,24 @@ PROTOBUF_EXPORT int stat(const char* path, struct _stat* buffer);
 PROTOBUF_EXPORT int write(int fd, const void* buffer, size_t size);
 PROTOBUF_EXPORT std::wstring testonly_utf8_to_winpath(const char* path);
 
+enum class ExpandWildcardsResult {
+  kSuccess = 0,
+  kErrorNoMatchingFile = 1,
+  kErrorInputPathConversion = 2,
+  kErrorOutputPathConversion = 3,
+};
+
+// Expand wildcards in a path pattern, feed the result to a consumer function.
+//
+// `path` must be a valid, Windows-style path. It may be absolute, or relative
+// to the current working directory, and it may contain wildcards ("*" and "?")
+// in the last path segment. This function passes all matching file names to
+// `consume`. The resulting paths may not be absolute nor normalized.
+//
+// The function returns a value from `ExpandWildcardsResult`.
+LIBPROTOBUF_EXPORT ExpandWildcardsResult ExpandWildcards(
+    const std::string& path, std::function<void(const std::string&)> consume);
+
 namespace strings {
 
 // Convert from UTF-16 to Active-Code-Page-encoded or to UTF-8-encoded text.

src/google/protobuf/io/io_win32_unittest.cc

@@ -54,6 +54,7 @@
 #include <memory>
 #include <sstream>
 #include <string>
+#include <vector>
 
 #include <gtest/gtest.h>
 
@@ -85,6 +86,7 @@ const wchar_t kUtf16Text[] = {
 };
 
 using std::string;
+using std::vector;
 using std::wstring;
 
 class IoWin32Test : public ::testing::Test {
@@ -146,12 +148,24 @@ bool GetCwdAsUtf8(string* result) {
   }
 }
 
+bool CreateEmptyFile(const wstring& path) {
+  HANDLE h = CreateFileW(path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS,
+                         FILE_ATTRIBUTE_NORMAL, NULL);
+  if (h == INVALID_HANDLE_VALUE) {
+    return false;
+  }
+  CloseHandle(h);
+  return true;
+}
+
 }  // namespace
 
 void IoWin32Test::SetUp() {
   test_tmpdir.clear();
   wtest_tmpdir.clear();
-  EXPECT_GT(::GetCurrentDirectoryW(MAX_PATH, working_directory), 0);
+  DWORD size = ::GetCurrentDirectoryW(MAX_PATH, working_directory);
+  EXPECT_GT(size, 0);
+  EXPECT_LT(size, MAX_PATH);
 
   string tmp;
   bool ok = false;
@@ -354,7 +368,7 @@ TEST_F(IoWin32Test, MkdirTestNonAscii) {
   ASSERT_INITIALIZED;
 
   // Create a non-ASCII path.
-  // Ensure that we can create the directory using SetCurrentDirectoryW.
+  // Ensure that we can create the directory using CreateDirectoryW.
   EXPECT_TRUE(CreateDirectoryW((wtest_tmpdir + L"\\1").c_str(), nullptr));
   EXPECT_TRUE(CreateDirectoryW((wtest_tmpdir + L"\\1\\" + kUtf16Text).c_str(), nullptr));
   // Ensure that we can create a very similarly named directory using mkdir.
@@ -402,6 +416,179 @@ TEST_F(IoWin32Test, ChdirTestNonAscii) {
   ASSERT_EQ(wNonAscii, cwd);
 }
 
+TEST_F(IoWin32Test, ExpandWildcardsInRelativePathTest) {
+  wstring wNonAscii(wtest_tmpdir + L"\\" + kUtf16Text);
+  EXPECT_TRUE(CreateDirectoryW(wNonAscii.c_str(), nullptr));
+  // Create mock files we will test pattern matching on.
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\foo_a.proto"));
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\foo_b.proto"));
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\bar.proto"));
+  // `cd` into `wtest_tmpdir`.
+  EXPECT_TRUE(SetCurrentDirectoryW(wtest_tmpdir.c_str()));
+
+  int found_a = 0;
+  int found_b = 0;
+  vector<string> found_bad;
+  // Assert matching a relative path pattern. Results should also be relative.
+  ExpandWildcardsResult result = 
+      ExpandWildcards(
+          string(kUtf8Text) + "\\foo*.proto",
+          [&found_a, &found_b, &found_bad](const string& p) {
+            if (p == string(kUtf8Text) + "\\foo_a.proto") {
+              found_a++;
+            } else if (p == string(kUtf8Text) + "\\foo_b.proto") {
+              found_b++;
+            } else {
+              found_bad.push_back(p);
+            }
+          });
+  EXPECT_EQ(result, ExpandWildcardsResult::kSuccess);
+  EXPECT_EQ(found_a, 1);
+  EXPECT_EQ(found_b, 1);
+  if (!found_bad.empty()) {
+    FAIL() << found_bad[0];
+  }
+
+  // Assert matching the exact filename.
+  found_a = 0;
+  found_bad.clear();
+  result = 
+      ExpandWildcards(
+          string(kUtf8Text) + "\\foo_a.proto",
+          [&found_a, &found_bad](const string& p) {
+            if (p == string(kUtf8Text) + "\\foo_a.proto") {
+              found_a++;
+            } else {
+              found_bad.push_back(p);
+            }
+          });
+  EXPECT_EQ(result, ExpandWildcardsResult::kSuccess);
+  EXPECT_EQ(found_a, 1);
+  if (!found_bad.empty()) {
+    FAIL() << found_bad[0];
+  }
+}
+
+TEST_F(IoWin32Test, ExpandWildcardsInAbsolutePathTest) {
+  wstring wNonAscii(wtest_tmpdir + L"\\" + kUtf16Text);
+  EXPECT_TRUE(CreateDirectoryW(wNonAscii.c_str(), nullptr));
+  // Create mock files we will test pattern matching on.
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\foo_a.proto"));
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\foo_b.proto"));
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\bar.proto"));
+
+  int found_a = 0;
+  int found_b = 0;
+  vector<string> found_bad;
+  // Assert matching an absolute path. The results should also use absolute
+  // path.
+  ExpandWildcardsResult result = 
+      ExpandWildcards(
+          string(test_tmpdir) + "\\" + kUtf8Text + "\\foo*.proto",
+          [this, &found_a, &found_b, &found_bad](const string& p) {
+            if (p == string(this->test_tmpdir)
+                         + "\\"
+                         + kUtf8Text
+                         + "\\foo_a.proto") {
+              found_a++;
+            } else if (p == string(this->test_tmpdir)
+                                + "\\"
+                                + kUtf8Text
+                                + "\\foo_b.proto") {
+              found_b++;
+            } else {
+              found_bad.push_back(p);
+            }
+          });
+  EXPECT_EQ(result, ExpandWildcardsResult::kSuccess);
+  EXPECT_EQ(found_a, 1);
+  EXPECT_EQ(found_b, 1);
+  if (!found_bad.empty()) {
+    FAIL() << found_bad[0];
+  }
+
+  // Assert matching the exact filename.
+  found_a = 0;
+  found_bad.clear();
+  result = 
+      ExpandWildcards(
+          string(test_tmpdir) + "\\" + kUtf8Text + "\\foo_a.proto",
+          [this, &found_a, &found_bad](const string& p) {
+            if (p == string(this->test_tmpdir)
+                         + "\\"
+                         + kUtf8Text
+                         + "\\foo_a.proto") {
+              found_a++;
+            } else {
+              found_bad.push_back(p);
+            }
+          });
+  EXPECT_EQ(result, ExpandWildcardsResult::kSuccess);
+  EXPECT_EQ(found_a, 1);
+  if (!found_bad.empty()) {
+    FAIL() << found_bad[0];
+  }
+}
+
+TEST_F(IoWin32Test, ExpandWildcardsIgnoresDirectoriesTest) {
+  wstring wNonAscii(wtest_tmpdir + L"\\" + kUtf16Text);
+  EXPECT_TRUE(CreateDirectoryW(wNonAscii.c_str(), nullptr));
+  // Create mock files we will test pattern matching on.
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\foo_a.proto"));
+  EXPECT_TRUE(CreateDirectoryW((wNonAscii + L"\\foo_b.proto").c_str(), nullptr));
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\foo_c.proto"));
+  // `cd` into `wtest_tmpdir`.
+  EXPECT_TRUE(SetCurrentDirectoryW(wtest_tmpdir.c_str()));
+
+  int found_a = 0;
+  int found_c = 0;
+  vector<string> found_bad;
+  // Assert that the pattern matches exactly the expected files, and using the
+  // absolute path as did the input pattern.
+  ExpandWildcardsResult result = 
+      ExpandWildcards(
+          string(kUtf8Text) + "\\foo*.proto",
+          [&found_a, &found_c, &found_bad](const string& p) {
+            if (p == string(kUtf8Text) + "\\foo_a.proto") {
+              found_a++;
+            } else if (p == string(kUtf8Text) + "\\foo_c.proto") {
+              found_c++;
+            } else {
+              found_bad.push_back(p);
+            }
+          });
+  EXPECT_EQ(result, ExpandWildcardsResult::kSuccess);
+  EXPECT_EQ(found_a, 1);
+  EXPECT_EQ(found_c, 1);
+  if (!found_bad.empty()) {
+    FAIL() << found_bad[0];
+  }
+}
+
+TEST_F(IoWin32Test, ExpandWildcardsFailsIfNoFileMatchesTest) {
+  wstring wNonAscii(wtest_tmpdir + L"\\" + kUtf16Text);
+  EXPECT_TRUE(CreateDirectoryW(wNonAscii.c_str(), nullptr));
+  // Create mock files we will test pattern matching on.
+  EXPECT_TRUE(CreateEmptyFile(wNonAscii + L"\\foo_a.proto"));
+  // `cd` into `wtest_tmpdir`.
+  EXPECT_TRUE(SetCurrentDirectoryW(wtest_tmpdir.c_str()));
+
+  // Control test: should match foo*.proto
+  ExpandWildcardsResult result = ExpandWildcards(
+      string(kUtf8Text) + "\\foo*.proto", [](const string&) {});
+  EXPECT_EQ(result, ExpandWildcardsResult::kSuccess);
+
+  // Control test: should match foo_a.proto
+  result = ExpandWildcards(
+      string(kUtf8Text) + "\\foo_a.proto", [](const string&) {});
+  EXPECT_EQ(result, ExpandWildcardsResult::kSuccess);
+
+  // Actual test: should not match anything.
+  result = ExpandWildcards(
+      string(kUtf8Text) + "\\bar*.proto", [](const string&) {});
+  ASSERT_EQ(result, ExpandWildcardsResult::kErrorNoMatchingFile);
+}
+
 TEST_F(IoWin32Test, AsWindowsPathTest) {
   DWORD size = GetCurrentDirectoryW(0, nullptr);
   std::unique_ptr<wchar_t[]> cwd_str(new wchar_t[size]);