fix trailing comment attribution when file has no final nweline

src/google/protobuf/io/tokenizer.cc

@@ -942,7 +942,7 @@ bool Tokenizer::NextWithComments(std::string* prev_trailing_comments,
             // makes no sense to attach a comment to the following token.
             collector.Flush();
           }
-          if (prev_line == line_ || trailing_comment_end_line == line_) {
+          if (result && (prev_line == line_ || trailing_comment_end_line == line_)) {
             // When previous token and this one are on the same line, or
             // even if a multi-line trailing comment ends on the same line
             // as this token, it's unclear to what token the comment

src/google/protobuf/io/tokenizer_unittest.cc

@@ -650,6 +650,18 @@ DocCommentCase kDocCommentCases[] = {
      {},
      ""},
 
+    {"prev // no next token\n",
+
+     " no next token\n",
+     {},
+     ""},
+
+    {"prev // no next token and no trailing newline",
+
+     " no next token and no trailing newline",
+     {},
+     ""},
+
     {"prev /* detached */ next",
 
      "",
@@ -780,7 +792,7 @@ DocCommentCase kDocCommentCases[] = {
      prev /* a single block comment
          that spans multiple lines
          is detached if it ends
-         on the same line as next */ next"
+         on the same line as next */ next
      )pb",
 
      "",
@@ -791,7 +803,7 @@ DocCommentCase kDocCommentCases[] = {
      ""},
 
     {R"pb(
-     prev /* trailing */ /* leading */ next"
+     prev /* trailing */ /* leading */ next
      )pb",
 
      " trailing ",
@@ -802,12 +814,25 @@ DocCommentCase kDocCommentCases[] = {
      prev /* multi-line
           trailing */ /* an oddly
                       placed detached */ /* an oddly
-                                         placed leading */ next"
+                                         placed leading */ next
      )pb",
 
      " multi-line\ntrailing ",
      {" an oddly\nplaced detached "},
      " an oddly\nplaced leading "},
+
+    {R"pb(
+     prev // trailing with newline
+     // detached
+     /* another detached */
+     // leading but no next token to attach it to
+     )pb",
+
+     " trailing with newline\n",
+     {" detached\n",
+      " another detached ",
+      " leading but no next token to attach it to\n"},
+     ""},
 };
 
 TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) {
@@ -822,8 +847,8 @@ TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) {
                          kDocCommentCases_case.input.size(), kBlockSizes_case);
   Tokenizer tokenizer2(&input2, &error_collector);
 
-  tokenizer.Next();
-  tokenizer2.Next();
+  EXPECT_TRUE(tokenizer.Next());
+  EXPECT_TRUE(tokenizer2.Next());
 
   EXPECT_EQ("prev", tokenizer.current().text);
   EXPECT_EQ("prev", tokenizer2.current().text);
@@ -831,11 +856,14 @@ TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) {
   std::string prev_trailing_comments;
   std::vector<std::string> detached_comments;
   std::string next_leading_comments;
-  tokenizer.NextWithComments(&prev_trailing_comments, &detached_comments,
-                             &next_leading_comments);
-  tokenizer2.NextWithComments(NULL, NULL, NULL);
-  EXPECT_EQ("next", tokenizer.current().text);
-  EXPECT_EQ("next", tokenizer2.current().text);
+  bool has_next = tokenizer.NextWithComments(&prev_trailing_comments,
+                                            &detached_comments,
+                                            &next_leading_comments);
+  EXPECT_EQ(has_next, tokenizer2.NextWithComments(NULL, NULL, NULL));
+  if (has_next) {
+    EXPECT_EQ("next", tokenizer.current().text);
+    EXPECT_EQ("next", tokenizer2.current().text);
+  }
 
   EXPECT_EQ(kDocCommentCases_case.prev_trailing_comments,
             prev_trailing_comments);