grpc/tools/release/release_notes.py

# Copyright 2019 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Generate draft and release notes in Markdown from Github PRs.

You'll need a github API token to avoid being rate-limited. See
https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/

This script collects PRs using "git log X..Y" from local repo where X and Y are
tags or release branch names of previous and current releases respectively.
Typically, notes are generated before the release branch is labelled so Y is
almost always the name of the release branch. X is the previous release branch
if this is not a patch release. Otherwise, it is the previous release tag.
For example, for release v1.17.0, X will be origin/v1.16.x and for release v1.17.3,
X will be v1.17.2. In both cases Y will be origin/v1.17.x.

"""

from collections import defaultdict
import json
import logging
import re
import subprocess

import urllib3

logging.basicConfig(level=logging.WARNING)

content_header = """Draft Release Notes For {version}
--
Final release notes will be generated from the PR titles that have *"release notes:yes"* label. If you have any additional notes please add them below. These will be appended to auto generated release notes. Previous release notes are [here](https://github.com/grpc/grpc/releases).

**Also, look at the PRs listed below against your name.** Please apply the missing labels and make necessary corrections (like fixing the title) to the PR in Github. Final release notes will be generated just before the release on {date}.

Add additional notes not in PRs
--

Core
-


C++
-


C#
-


Objective-C
-


PHP
-


Python
-


Ruby
-


"""

rl_header = """This is release {version} ([{name}](https://github.com/grpc/grpc/blob/master/doc/g_stands_for.md)) of gRPC Core.

For gRPC documentation, see [grpc.io](https://grpc.io/). For previous releases, see [Releases](https://github.com/grpc/grpc/releases).

This release contains refinements, improvements, and bug fixes, with highlights listed below.


"""

HTML_URL = "https://github.com/grpc/grpc/pull/"
API_URL = "https://api.github.com/repos/grpc/grpc/pulls/"


def print_commits_wo_pr(commits_wo_pr):
    """Print commit and CL info for the commits that are submitted with CL-first workflow and warn the release manager to check manually."""
    print("***WARNING***")
    print(
        "The following commits are submitted with the CL-first workflow and does not have a PR number available in its commit info!"
    )
    print(
        "Release manager needs to use the following info to go to the CL and gets the PR info from the Copybara:copybara_presubmit info on the CL and manually verify if the PR has the `release notes: yes` label!"
    )
    print("\n")

    for commit in commits_wo_pr:
        glg_command = [
            "git",
            "log",
            "-n 1",
            "%s" % commit,
        ]
        output = subprocess.check_output(glg_command).decode("utf-8", "ignore")
        matches = re.search("PiperOrigin-RevId: ([0-9]+)$", output)
        print("Commit: https://github.com/grpc/grpc/commit/%s" % commit)
        print(
            "CL:     https://critique.corp.google.com/cl/%s" % matches.group(1)
        )
        print("\n")

    print("***WARNING***")


def get_commit_log(prevRelLabel, relBranch):
    """Return the output of 'git log prevRelLabel..relBranch'"""

    import subprocess

    glg_command = [
        "git",
        "log",
        "--pretty=oneline",
        "%s..%s" % (prevRelLabel, relBranch),
    ]
    print(("Running ", " ".join(glg_command)))
    return subprocess.check_output(glg_command).decode("utf-8", "ignore")


def get_pr_data(pr_num):
    """Get the PR data from github. Return 'error' on exception"""
    http = urllib3.PoolManager(
        retries=urllib3.Retry(total=7, backoff_factor=1), timeout=4.0
    )
    url = API_URL + pr_num
    try:
        response = http.request(
            "GET", url, headers={"Authorization": "token %s" % TOKEN}
        )
    except urllib3.exceptions.HTTPError as e:
        print("Request error:", e.reason)
        return "error"
    return json.loads(response.data.decode("utf-8"))


def get_pr_titles(gitLogs):
    import re

    # All commits
    match_commit = "^([a-fA-F0-9]+) "
    all_commits_set = set(re.findall(match_commit, gitLogs, re.MULTILINE))

    error_count = 0
    # PRs with merge commits
    match_merge_pr = "^([a-fA-F0-9]+) .*Merge pull request #(\d+)"
    matches = re.findall(match_merge_pr, gitLogs, re.MULTILINE)
    merge_commits = []
    prlist_merge_pr = []
    if matches:
        merge_commits, prlist_merge_pr = zip(*matches)
    merge_commits_set = set(merge_commits)
    print("\nPRs matching 'Merge pull request #<num>':")
    print(prlist_merge_pr)
    print("\n")

    # PRs using Github's squash & merge feature
    match_sq = "^([a-fA-F0-9]+) .*\(#(\d+)\)$"
    matches = re.findall(match_sq, gitLogs, re.MULTILINE)
    if matches:
        sq_commits, prlist_sq = zip(*matches)
    sq_commits_set = set(sq_commits)
    print("\nPRs matching '[PR Description](#<num>)$'")
    print(prlist_sq)
    print("\n")
    prlist = list(prlist_merge_pr) + list(prlist_sq)
    langs_pr = defaultdict(list)
    for pr_num in prlist:
        pr_num = str(pr_num)
        print(("---------- getting data for PR " + pr_num))
        pr = get_pr_data(pr_num)
        if pr == "error":
            print(
                ("\n***ERROR*** Error in getting data for PR " + pr_num + "\n")
            )
            error_count += 1
            continue
        rl_no_found = False
        rl_yes_found = False
        lang_found = False
        for label in pr["labels"]:
            if label["name"] == "release notes: yes":
                rl_yes_found = True
            elif label["name"] == "release notes: no":
                rl_no_found = True
            elif label["name"].startswith("lang/"):
                lang_found = True
                lang = label["name"].split("/")[1].lower()
                # lang = lang[0].upper() + lang[1:]
        body = pr["title"]
        if not body.endswith("."):
            body = body + "."

        prline = (
            "-  " + body + " ([#" + pr_num + "](" + HTML_URL + pr_num + "))"
        )
        detail = "- " + pr["user"]["login"] + "@ " + prline
        print(detail)
        # if no RL label
        if not rl_no_found and not rl_yes_found:
            print(("Release notes label missing for " + pr_num))
            langs_pr["nolabel"].append(detail)
        elif rl_yes_found and not lang_found:
            print(("Lang label missing for " + pr_num))
            langs_pr["nolang"].append(detail)
        elif rl_no_found:
            print(("'Release notes:no' found for " + pr_num))
            langs_pr["notinrel"].append(detail)
        elif rl_yes_found:
            print(
                (
                    "'Release notes:yes' found for "
                    + pr_num
                    + " with lang "
                    + lang
                )
            )
            langs_pr["inrel"].append(detail)
            langs_pr[lang].append(prline)

    # Warn the release manager to manually check the commits that do not have PR
    # info in its commit message.
    commits_wo_pr = all_commits_set - merge_commits_set - sq_commits_set
    if commits_wo_pr:
        print_commits_wo_pr(commits_wo_pr)

    return langs_pr, error_count


def write_draft(langs_pr, file, version, date):
    file.write(content_header.format(version=version, date=date))
    file.write("PRs with missing release notes label - please fix in Github\n")
    file.write("---\n")
    file.write("\n")
    if langs_pr["nolabel"]:
        langs_pr["nolabel"].sort()
        file.write("\n".join(langs_pr["nolabel"]))
    else:
        file.write("- None")
    file.write("\n")
    file.write("\n")
    file.write("PRs with missing lang label - please fix in Github\n")
    file.write("---\n")
    file.write("\n")
    if langs_pr["nolang"]:
        langs_pr["nolang"].sort()
        file.write("\n".join(langs_pr["nolang"]))
    else:
        file.write("- None")
    file.write("\n")
    file.write("\n")
    file.write(
        "PRs going into release notes - please check title and fix in Github."
        " Do not edit here.\n"
    )
    file.write("---\n")
    file.write("\n")
    if langs_pr["inrel"]:
        langs_pr["inrel"].sort()
        file.write("\n".join(langs_pr["inrel"]))
    else:
        file.write("- None")
    file.write("\n")
    file.write("\n")
    file.write("PRs not going into release notes\n")
    file.write("---\n")
    file.write("\n")
    if langs_pr["notinrel"]:
        langs_pr["notinrel"].sort()
        file.write("\n".join(langs_pr["notinrel"]))
    else:
        file.write("- None")
    file.write("\n")
    file.write("\n")


def write_rel_notes(langs_pr, file, version, name):
    file.write(rl_header.format(version=version, name=name))
    if langs_pr["core"]:
        file.write("Core\n---\n\n")
        file.write("\n".join(langs_pr["core"]))
        file.write("\n")
        file.write("\n")
    if langs_pr["c++"]:
        file.write("C++\n---\n\n")
        file.write("\n".join(langs_pr["c++"]))
        file.write("\n")
        file.write("\n")
    if langs_pr["c#"]:
        file.write("C#\n---\n\n")
        file.write("\n".join(langs_pr["c#"]))
        file.write("\n")
        file.write("\n")
    if langs_pr["go"]:
        file.write("Go\n---\n\n")
        file.write("\n".join(langs_pr["go"]))
        file.write("\n")
        file.write("\n")
    if langs_pr["Java"]:
        file.write("Java\n---\n\n")
        file.write("\n".join(langs_pr["Java"]))
        file.write("\n")
        file.write("\n")
    if langs_pr["node"]:
        file.write("Node\n---\n\n")
        file.write("\n".join(langs_pr["node"]))
        file.write("\n")
        file.write("\n")
    if langs_pr["objc"]:
        file.write("Objective-C\n---\n\n")
        file.write("\n".join(langs_pr["objc"]))
        file.write("\n")
        file.write("\n")
    if langs_pr["php"]:
        file.write("PHP\n---\n\n")
        file.write("\n".join(langs_pr["php"]))
        file.write("\n")
        file.write("\n")
    if langs_pr["python"]:
        file.write("Python\n---\n\n")
        file.write("\n".join(langs_pr["python"]))
        file.write("\n")
        file.write("\n")
    if langs_pr["ruby"]:
        file.write("Ruby\n---\n\n")
        file.write("\n".join(langs_pr["ruby"]))
        file.write("\n")
        file.write("\n")
    if langs_pr["other"]:
        file.write("Other\n---\n\n")
        file.write("\n".join(langs_pr["other"]))
        file.write("\n")
        file.write("\n")


def build_args_parser():
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "release_version", type=str, help="New release version e.g. 1.14.0"
    )
    parser.add_argument(
        "release_name", type=str, help="New release name e.g. gladiolus"
    )
    parser.add_argument(
        "release_date", type=str, help="Release date e.g. 7/30/18"
    )
    parser.add_argument(
        "previous_release_label",
        type=str,
        help="Previous release branch/tag e.g. v1.13.x",
    )
    parser.add_argument(
        "release_branch",
        type=str,
        help="Current release branch e.g. origin/v1.14.x",
    )
    parser.add_argument(
        "draft_filename", type=str, help="Name of the draft file e.g. draft.md"
    )
    parser.add_argument(
        "release_notes_filename",
        type=str,
        help="Name of the release notes file e.g. relnotes.md",
    )
    parser.add_argument(
        "--token",
        type=str,
        default="",
        help="GitHub API token to avoid being rate limited",
    )
    return parser


def main():
    import os

    global TOKEN

    parser = build_args_parser()
    args = parser.parse_args()
    version, name, date = (
        args.release_version,
        args.release_name,
        args.release_date,
    )
    start, end = args.previous_release_label, args.release_branch

    TOKEN = args.token
    if TOKEN == "":
        try:
            TOKEN = os.environ["GITHUB_TOKEN"]
        except:
            pass
    if TOKEN == "":
        print(
            "Error: Github API token required. Either include param"
            " --token=<your github token> or set environment variable"
            " GITHUB_TOKEN to your github token"
        )
        return

    langs_pr, error_count = get_pr_titles(get_commit_log(start, end))

    draft_file, rel_file = args.draft_filename, args.release_notes_filename
    filename = os.path.abspath(draft_file)
    if os.path.exists(filename):
        file = open(filename, "r+")
    else:
        file = open(filename, "w")

    file.seek(0)
    write_draft(langs_pr, file, version, date)
    file.truncate()
    file.close()
    print(("\nDraft notes written to " + filename))

    filename = os.path.abspath(rel_file)
    if os.path.exists(filename):
        file = open(filename, "r+")
    else:
        file = open(filename, "w")

    file.seek(0)
    write_rel_notes(langs_pr, file, version, name)
    file.truncate()
    file.close()
    print(("\nRelease notes written to " + filename))
    if error_count > 0:
        print("\n\n*** Errors were encountered. See log. *********\n")


if __name__ == "__main__":
    main()