mirror of https://github.com/grpc/grpc.git
Migrate Infrastructure Scripts to Python 3 (#27135)
* Run 2to3 on tools directory
* Delete github_stats_tracking
* Re-run 2to3
* Remove unused script
* Remove unused script
* Remove unused line count utility
* Yapf. Isort
* Remove accidentally included file
* Migrate tools/distrib directory to python 3
* Remove unnecessary shebang
* Restore line_count directory
* Immediately convert subprocess.check_output output to string
* Take care of Python 2 shebangs
* Invoke scripts using a Python 3 interpreter
* Yapf. Isort
* Try installing Python 3 first
* See if we have any Python 3 versions installed
* Add Python 3.7 to Windows path
* Try adding a symlink
* Try to symlink differently
* Install six for Python 3
* Run run_interop_tests with python 3
* Try installing six in python3.7 explicitly
* Revert "Try installing six in python3.7 explicitly"
This reverts commit 2cf60d72f3.
* And debug some more
* Fix issue with jobset.py
* Add debug for CI failure
* Revert microbenchmark changes
pull/27969/head
Richard Belleville
3 years ago
committed by
GitHub
parent
c1089d2964
commit
59693b7e72
79 changed files with 483 additions and 648 deletions
@ -1,93 +0,0 @@ |
||||
#!/usr/bin/env python2.7 |
||||
# Copyright 2015 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
import argparse |
||||
import os |
||||
import shutil |
||||
import subprocess |
||||
|
||||
parser = argparse.ArgumentParser( |
||||
description='Submit the package to a PyPI repository.') |
||||
parser.add_argument('--repository', |
||||
'-r', |
||||
metavar='r', |
||||
type=str, |
||||
default='pypi', |
||||
help='The repository to push the package to. ' |
||||
'Ensure the value appears in your .pypirc file. ' |
||||
'Defaults to "pypi".') |
||||
parser.add_argument('--identity', |
||||
'-i', |
||||
metavar='i', |
||||
type=str, |
||||
help='GPG identity to sign the files with.') |
||||
parser.add_argument( |
||||
'--username', |
||||
'-u', |
||||
metavar='u', |
||||
type=str, |
||||
help='Username to authenticate with the repository. Not needed if you have ' |
||||
'configured your .pypirc to include your username.') |
||||
parser.add_argument( |
||||
'--password', |
||||
'-p', |
||||
metavar='p', |
||||
type=str, |
||||
help='Password to authenticate with the repository. Not needed if you have ' |
||||
'configured your .pypirc to include your password.') |
||||
parser.add_argument( |
||||
'--bdist', |
||||
'-b', |
||||
action='store_true', |
||||
help='Generate a binary distribution (wheel) for the current OS.') |
||||
parser.add_argument( |
||||
'--dist-args', |
||||
type=str, |
||||
help='Additional arguments to pass to the *dist setup.py command.') |
||||
args = parser.parse_args() |
||||
|
||||
# Move to the root directory of Python GRPC. |
||||
pkgdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../') |
||||
# Remove previous distributions; they somehow confuse twine. |
||||
try: |
||||
shutil.rmtree(os.path.join(pkgdir, 'dist/')) |
||||
except: |
||||
pass |
||||
|
||||
# Build the Cython C files |
||||
build_env = os.environ.copy() |
||||
build_env['GRPC_PYTHON_BUILD_WITH_CYTHON'] = "1" |
||||
cmd = ['python', 'setup.py', 'build_ext', '--inplace'] |
||||
subprocess.call(cmd, cwd=pkgdir, env=build_env) |
||||
|
||||
# Make the push. |
||||
if args.bdist: |
||||
cmd = ['python', 'setup.py', 'bdist_wheel'] |
||||
else: |
||||
cmd = ['python', 'setup.py', 'sdist'] |
||||
if args.dist_args: |
||||
cmd += args.dist_args.split() |
||||
subprocess.call(cmd, cwd=pkgdir) |
||||
|
||||
cmd = ['twine', 'upload', '-r', args.repository] |
||||
if args.identity is not None: |
||||
cmd.extend(['-i', args.identity]) |
||||
if args.username is not None: |
||||
cmd.extend(['-u', args.username]) |
||||
if args.password is not None: |
||||
cmd.extend(['-p', args.password]) |
||||
cmd.append('dist/*') |
||||
|
||||
subprocess.call(cmd, cwd=pkgdir) |
||||
@ -1,13 +0,0 @@ |
||||
runtime: python27 |
||||
api_version: 1 |
||||
threadsafe: true |
||||
|
||||
service: github-stats-tracking |
||||
|
||||
handlers: |
||||
- url: /.* |
||||
script: main.app |
||||
|
||||
libraries: |
||||
- name: ssl |
||||
version: latest |
||||
@ -1,19 +0,0 @@ |
||||
# Copyright 2019 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# appengine_config.py |
||||
from google.appengine.ext import vendor |
||||
|
||||
# Add any libraries install in the "lib" folder. |
||||
vendor.add('lib') |
||||
@ -1,4 +0,0 @@ |
||||
cron: |
||||
- description: "daily github stats tracking job" |
||||
url: /daily |
||||
schedule: every 24 hours |
||||
@ -1,96 +0,0 @@ |
||||
# Copyright 2019 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
from datetime import datetime |
||||
from datetime import timedelta |
||||
from time import time |
||||
|
||||
from github import Github |
||||
from github import Label |
||||
from google.cloud import bigquery |
||||
|
||||
ACCESS_TOKEN = "" |
||||
|
||||
|
||||
def get_stats_from_github(): |
||||
# Please set the access token properly before deploying. |
||||
assert ACCESS_TOKEN |
||||
g = Github(ACCESS_TOKEN) |
||||
print g.rate_limiting |
||||
repo = g.get_repo('grpc/grpc') |
||||
|
||||
LABEL_LANG = set(label for label in repo.get_labels() |
||||
if label.name.split('/')[0] == 'lang') |
||||
LABEL_KIND_BUG = repo.get_label('kind/bug') |
||||
LABEL_PRIORITY_P0 = repo.get_label('priority/P0') |
||||
LABEL_PRIORITY_P1 = repo.get_label('priority/P1') |
||||
LABEL_PRIORITY_P2 = repo.get_label('priority/P2') |
||||
|
||||
def is_untriaged(issue): |
||||
key_labels = set() |
||||
for label in issue.labels: |
||||
label_kind = label.name.split('/')[0] |
||||
if label_kind in ('lang', 'kind', 'priority'): |
||||
key_labels.add(label_kind) |
||||
return len(key_labels) < 3 |
||||
|
||||
untriaged_open_issues = [ |
||||
issue for issue in repo.get_issues(state='open') |
||||
if issue.pull_request is None and is_untriaged(issue) |
||||
] |
||||
total_bugs = [ |
||||
issue for issue in repo.get_issues(state='all', labels=[LABEL_KIND_BUG]) |
||||
if issue.pull_request is None |
||||
] |
||||
|
||||
lang_to_stats = {} |
||||
for lang in LABEL_LANG: |
||||
lang_bugs = filter(lambda bug: lang in bug.labels, total_bugs) |
||||
closed_bugs = filter(lambda bug: bug.state == 'closed', lang_bugs) |
||||
open_bugs = filter(lambda bug: bug.state == 'open', lang_bugs) |
||||
open_p0_bugs = filter(lambda bug: LABEL_PRIORITY_P0 in bug.labels, |
||||
open_bugs) |
||||
open_p1_bugs = filter(lambda bug: LABEL_PRIORITY_P1 in bug.labels, |
||||
open_bugs) |
||||
open_p2_bugs = filter(lambda bug: LABEL_PRIORITY_P2 in bug.labels, |
||||
open_bugs) |
||||
lang_to_stats[lang] = [ |
||||
len(lang_bugs), |
||||
len(closed_bugs), |
||||
len(open_bugs), |
||||
len(open_p0_bugs), |
||||
len(open_p1_bugs), |
||||
len(open_p2_bugs) |
||||
] |
||||
return len(untriaged_open_issues), lang_to_stats |
||||
|
||||
|
||||
def insert_stats_to_db(untriaged_open_issues, lang_to_stats): |
||||
timestamp = time() |
||||
client = bigquery.Client() |
||||
dataset_ref = client.dataset('github_issues') |
||||
table_ref = dataset_ref.table('untriaged_issues') |
||||
table = client.get_table(table_ref) |
||||
errors = client.insert_rows(table, [(timestamp, untriaged_open_issues)]) |
||||
table_ref = dataset_ref.table('bug_stats') |
||||
table = client.get_table(table_ref) |
||||
rows = [] |
||||
for lang, stats in lang_to_stats.iteritems(): |
||||
rows.append((timestamp, lang.name[5:]) + tuple(stats)) |
||||
errors = client.insert_rows(table, rows) |
||||
|
||||
|
||||
def fetch(): |
||||
untriaged_open_issues, lang_to_stats = get_stats_from_github() |
||||
insert_stats_to_db(untriaged_open_issues, lang_to_stats) |
||||
@ -1,28 +0,0 @@ |
||||
# Copyright 2019 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
from fetch_data import fetch |
||||
import webapp2 |
||||
|
||||
|
||||
class DailyCron(webapp2.RequestHandler): |
||||
|
||||
def get(self): |
||||
fetch() |
||||
self.response.status = 204 |
||||
|
||||
|
||||
app = webapp2.WSGIApplication([ |
||||
('/daily', DailyCron), |
||||
], debug=True) |
||||
Loading…
Reference in new issue