Sanitize before bigquery upload

pull/10108/head
Craig Tiller 8 years ago
parent 56cea8f2ee
commit bbfb25bf70
  1. 19
      tools/profiling/microbenchmarks/bm2bq.py

@ -73,6 +73,14 @@ columns = [
('framing_bytes_per_iteration', 'float'), ('framing_bytes_per_iteration', 'float'),
] ]
SANITIZE = {
'integer': int,
'float': float,
'boolean': bool,
'string': str,
'timestamp': str,
}
if sys.argv[1] == '--schema': if sys.argv[1] == '--schema':
print ',\n'.join('%s:%s' % (k, t.upper()) for k, t in columns) print ',\n'.join('%s:%s' % (k, t.upper()) for k, t in columns)
sys.exit(0) sys.exit(0)
@ -89,7 +97,10 @@ else:
writer = csv.DictWriter(sys.stdout, [c for c,t in columns]) writer = csv.DictWriter(sys.stdout, [c for c,t in columns])
for row in bm_json.expand_json(js, js2): for row in bm_json.expand_json(js, js2):
if 'label' in row: sane_row = {}
del row['label'] for name, sql_type in columns:
del row['cpp_name'] if name in row:
writer.writerow(row) if row[name] == '': continue
sane_row[name] = SANITIZE[sql_type](row[name])
writer.writerow(sane_row)

Loading…
Cancel
Save