diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am
index f0ed489937..1e162eb1ef 100644
--- a/benchmarks/Makefile.am
+++ b/benchmarks/Makefile.am
@@ -22,30 +22,20 @@ generate_datasets_LDADD = $(top_srcdir)/src/libprotobuf.la
 generate_datasets_SOURCES = generate_datasets.cc
 generate_datasets_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)
 nodist_generate_datasets_SOURCES =                             \
-  google_message1.h                                            \
-  google_message2.h                                            \
   $(benchmarks_protoc_outputs)                                 \
   $(benchmarks_protoc_outputs_proto2)
 
 # Explicit deps because BUILT_SOURCES are only done before a "make all/check"
 # so a direct "make test_cpp" could fail if parallel enough.
 # See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually
-generate_datasets-generate_datasets.$(OBJEXT): benchmarks.pb.h google_message1.h google_message2.h
+generate_datasets-generate_datasets.$(OBJEXT): benchmarks.pb.h
 
 $(benchmarks_protoc_outputs): protoc_middleman
 $(benchmarks_protoc_outputs_proto2): protoc_middleman2
 
-google_message1.h: google_message1.dat
-	xxd -i $< $@
-
-google_message2.h: google_message2.dat
-	xxd -i $< $@
-
 CLEANFILES =                                                   \
   $(benchmarks_protoc_outputs)                                 \
   $(benchmarks_protoc_outputs_proto2)                          \
-  google_message1.h                                            \
-  google_message2.h                                            \
   protoc_middleman                                             \
   protoc_middleman2                                            \
   dataset.*
diff --git a/benchmarks/generate_datasets.cc b/benchmarks/generate_datasets.cc
index f6f30cd85b..8e9b441cf7 100644
--- a/benchmarks/generate_datasets.cc
+++ b/benchmarks/generate_datasets.cc
@@ -34,8 +34,6 @@ const char *file_suffix = ".pb";
 #include <fstream>
 #include <iostream>
 #include "benchmarks.pb.h"
-#include "google_message1.h"
-#include "google_message2.h"
 
 using benchmarks::BenchmarkDataset;
 using google::protobuf::Descriptor;
@@ -102,13 +100,23 @@ void WriteFile(const std::string& name, const std::string& message_name,
   WriteFileWithPayloads(name, message_name, payloads);
 }
 
+std::string ReadFile(const std::string& name) {
+  std::ifstream file(name);
+  GOOGLE_CHECK(file.is_open()) << "Couldn't find file '" << name <<
+                                  "', please make sure you are running "
+                                  "this command from the benchmarks/ "
+                                  "directory.\n";
+  return std::string((std::istreambuf_iterator<char>(file)),
+                     std::istreambuf_iterator<char>());
+}
+
 int main() {
   WriteFile("google_message1_proto3", "benchmarks.p3.GoogleMessage1",
-            ARRAY_TO_STRING(google_message1_dat));
+            ReadFile("google_message1.dat"));
   WriteFile("google_message1_proto2", "benchmarks.p2.GoogleMessage1",
-            ARRAY_TO_STRING(google_message1_dat));
+            ReadFile("google_message1.dat"));
 
   // Not in proto3 because it has a group, which is not supported.
   WriteFile("google_message2", "benchmarks.p2.GoogleMessage2",
-            ARRAY_TO_STRING(google_message2_dat));
+            ReadFile("google_message2.dat"));
 }
diff --git a/tests.sh b/tests.sh
index 75bf9001c6..6a9439a551 100755
--- a/tests.sh
+++ b/tests.sh
@@ -38,7 +38,7 @@ build_cpp() {
   cd conformance && make test_cpp && cd ..
 
   # Verify benchmarking code can build successfully.
-  cd benchmarks && make && cd ..
+  cd benchmarks && make && ./generate-datasets && cd ..
 }
 
 build_cpp_distcheck() {