No issue: Add telemetry data renewal helper
parent
09101eb787
commit
38bfb96342
@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python3
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
"""
|
||||
A script to help generate telemetry renewal csv and request template.
|
||||
This script also modifies metrics.yaml to mark soon to expired telemetry entries.
|
||||
"""
|
||||
|
||||
import os
|
||||
import csv
|
||||
import yaml
|
||||
import json
|
||||
import sys
|
||||
|
||||
from yaml.loader import FullLoader
|
||||
|
||||
METRICS_FILENAME = "../app/metrics.yaml"
|
||||
NEW_METRICS_FILENAME = "../app/metrics_new.yaml"
|
||||
|
||||
# This is to make sure we only write headers for the csv file once
|
||||
write_header = True
|
||||
# The number of soon to expired telemetry detected
|
||||
total_count = 0
|
||||
|
||||
USAGE="""usage: ./{script_name} future_fenix_version_number"""
|
||||
|
||||
# list of values that we care about
|
||||
_KEY_FILTER = [
|
||||
"type",
|
||||
"description",
|
||||
"bugs",
|
||||
"data_reviews",
|
||||
"data_sensitivity",
|
||||
"notification_emails",
|
||||
"notification",
|
||||
"expires",
|
||||
]
|
||||
|
||||
def response(last_key, content, expire_version, writer, renewal):
|
||||
global write_header
|
||||
global total_count
|
||||
for key, value in content.items():
|
||||
if (key == "$schema") or (key == "no_lint"):
|
||||
continue
|
||||
|
||||
if ("expires" in value) and ((value["expires"] == "never") or (not value["expires"] <= expire_version)):
|
||||
continue
|
||||
|
||||
if (key == "type"):
|
||||
remove_keys = []
|
||||
for key in content.keys():
|
||||
if (key not in _KEY_FILTER):
|
||||
remove_keys.append(key)
|
||||
|
||||
for key in remove_keys:
|
||||
content.pop(key)
|
||||
|
||||
total_count += 1
|
||||
|
||||
# name of the telemtry
|
||||
result = {"#": total_count, "name" : last_key.lstrip('.')}
|
||||
result.update(content)
|
||||
|
||||
# add columns for product to fille out, these should always be added at the end
|
||||
result.update({"keep(Y/N)" : ""})
|
||||
result.update({"new expiry version" : ""})
|
||||
result.update({"reason to extend" : ""})
|
||||
|
||||
# output data-renewal request template
|
||||
if (write_header):
|
||||
header = result.keys()
|
||||
writer.writerow(header)
|
||||
write_header = False
|
||||
renewal.write("# Request for Data Collection Renewal\n")
|
||||
renewal.write("### Renew for 1 year\n")
|
||||
renewal.write("Total: TBD\n")
|
||||
renewal.write("———\n")
|
||||
|
||||
writer.writerow(result.values())
|
||||
|
||||
renewal.write("`" + last_key.lstrip('.') + "`:\n")
|
||||
renewal.write("1) Provide a link to the initial Data Collection Review Request for this collection.\n")
|
||||
renewal.write(" - " + content["data_reviews"][0] + "\n")
|
||||
renewal.write("\n")
|
||||
renewal.write("2) When will this collection now expire?\n")
|
||||
renewal.write(" - TBD\n")
|
||||
renewal.write("\n")
|
||||
renewal.write("3) Why was the initial period of collection insufficient?\n")
|
||||
renewal.write(" - TBD\n")
|
||||
renewal.write("\n")
|
||||
renewal.write("———\n")
|
||||
return
|
||||
|
||||
if type(value) is dict:
|
||||
response(last_key + "." + key, value, expire_version, writer, renewal)
|
||||
|
||||
with open(METRICS_FILENAME, 'r') as f:
|
||||
try:
|
||||
arg1 = sys.argv[1]
|
||||
except:
|
||||
print ("usage is to include argument of the form `100`")
|
||||
quit()
|
||||
|
||||
# parse metrics.yaml to json
|
||||
write_header = True
|
||||
data = yaml.load(f, Loader=FullLoader)
|
||||
json_data = json.dumps(data)
|
||||
content = json.loads(str(json_data))
|
||||
csv_filename = arg1 + "_expiry_list.csv"
|
||||
renewal_filename = arg1 + "_renewal_request.txt"
|
||||
current_version = int(arg1)
|
||||
|
||||
# remove files created by last run if exists
|
||||
if os.path.exists(csv_filename):
|
||||
print("remove old csv file")
|
||||
os.remove(csv_filename)
|
||||
|
||||
# remove files created by last run if exists
|
||||
if os.path.exists(renewal_filename):
|
||||
print("remove old renewal request template file")
|
||||
os.remove(renewal_filename)
|
||||
|
||||
# remove files created by last run if exists
|
||||
if os.path.exists(NEW_METRICS_FILENAME):
|
||||
print("remove old metrics yaml file")
|
||||
os.remove(NEW_METRICS_FILENAME)
|
||||
|
||||
data_file = open(csv_filename, 'w')
|
||||
csv_writer = csv.writer(data_file)
|
||||
renewal_file = open(renewal_filename, 'w')
|
||||
|
||||
response("", content, current_version, csv_writer, renewal_file)
|
||||
renewal_file.close()
|
||||
print("Completed")
|
||||
print("Total count: " + str(total_count))
|
||||
|
||||
# Go through the metrics.yaml file to mark expired telemetry
|
||||
verify_count = 0
|
||||
f.seek(0, 0)
|
||||
data = f.readlines()
|
||||
with open(NEW_METRICS_FILENAME, 'w') as f2:
|
||||
for line in data:
|
||||
if (line.lstrip(' ').startswith("expires: ") and not(line.lstrip(' ').startswith("expires: never"))):
|
||||
start_pos = len("expires: ")
|
||||
version = int(line.lstrip(' ')[start_pos:])
|
||||
if (version <= current_version):
|
||||
verify_count += 1
|
||||
f2.writelines(line.rstrip('\n') + " /* TODO <" + str(verify_count) + "> require renewal */\n")
|
||||
else:
|
||||
f2.writelines(line)
|
||||
else:
|
||||
f2.writelines(line)
|
||||
f2.close()
|
||||
|
||||
print ("\n==============================")
|
||||
if (total_count != verify_count):
|
||||
print("!!! Count check failed !!!")
|
||||
else:
|
||||
print("Count check passed")
|
||||
print ("==============================")
|
||||
|
||||
os.remove(METRICS_FILENAME)
|
||||
os.rename(NEW_METRICS_FILENAME, METRICS_FILENAME)
|
Loading…
Reference in New Issue