Repository with assignments using the Test Informed Learning with Examples (TILE) method to integrate testing into existing programming courses for free.
Join our LinkedIN Community.
Use the following BibTeX entry to cite this work:
@INPROCEEDINGS{DVMB2023, author={Doorn, Niels and Vos, Tanja and Marín, Beatriz and Barendsen, Erik}, booktitle={2023 IEEE Conference on Software Testing, Verification and Validation (ICST)}, title={Set the right example when teaching programming: Test Informed Learning with Examples (TILE)}, year={2023}, volume={}, number={}, pages={269-280}, doi={10.1109/ICST57152.2023.00033} }
#file generate_test_report_all
import json
import xlwt
from xlwt import Workbook
from lark import Lark
from lark import Transformer
def get_failed_testcases(filename):
"""
Expects filename to be a file that contains the output of a !pytest run.
Returns the list of testcases that have failed.
Throws FileNotFoundError exception if file does not exist.
"""
#1.Open the file and name the file-handle fhand
fhand = open(filename, 'r')
#2.Copy the content of the file in variable content
content = fhand.read()
#3: Close the file
fhand.close()
#Look for the failed test cases
if not ("= FAILURES" in content):
return [] #There are no failed test cases
else:
# Find the testcases that have failed, they
# start with "testcase = " in the file
lss_lines = content.splitlines()
testcases = []
for l in lss_lines:
if "testcase =" in l:
testcases.append(l)
return testcases
def get_test_signature(filename):
"""
Given a Python file containing "@pytest.mark.parametrize", it returns a list that
represents the signature of the test. If there are no pytests in the file, it returns
the empty list
Throws FileNotFoundError exception if file does not exist.
"""
#1: Open the file and name the file-handle fhand
python_file = open(filename, "r")
#2: Read through the file to find the line that indicates that
# the test cases start (i.e. @pytest.mark.parametrize)
line = python_file.readline()
while not (line.startswith("@pytest.mark.parametrize") or line==''):
line = python_file.readline()
#3: Close the file
python_file.close()
#line now is the "@pytest.mark.parametrize" line
#Now, we need to know what the structure of the test cases is,
#i.e. how many inputs. So we first filter the characters that we do not need.
filter_out = [',', "@pytest.mark.parametrize", "(", ")", "[", '"']
for f in filter_out:
line = line.replace(f, "")
#Then we split, such that we get a list like
#['testcase', input1, ..., inputn, output]
test_signature = line.split()
return test_signature
# Below is the grammar describing test cases.
# test case lines look like: '(num, i1, i2,...,in o), #any type of comments'
# - starts with (
# - ends with ),
# - the first argument is a number, the ID of the test case
# - after the end test case ), commenst starting with #can be discarded
# - different parts of the test case are separated by ", "
# - i1, i2, ..., in and o can be of any Python type (int, float, bool, strings, lists, tuples, variables, sets)
# - the exercise explicity indicate that we assume there are no operators (unary, binary operators), variable names, dictionaries, function calls
testcase_parser = Lark(r"""
testcase : "(" DEC_NUMBER "," value ("," value)* ")" [","] [SH_COMMENT]
value: list
| tuple
| emptyset
| set
| string
| number
| "True" -> true
| "False" -> false
| "None" -> none
list : "[" [value ("," value)*] "]"
tuple: "(" [value ("," value)*] ")"
set : "{" value ("," value)* "}"
emptyset: "set()"
number: DEC_NUMBER | FLOAT_NUMBER
string: /[ubf]?r?("(?!"").*?(?<!\\)(\\\\)*?"|'(?!'').*?(?<!\\)(\\\\)*?')/i
DEC_NUMBER: /0|[1-9][\d_]*/i
FLOAT_NUMBER: /((\d+\.[\d_]*|\.[\d_]+)([Ee][-+]?\d+)?|\d+([Ee][-+]?\d+))/
%import common.ESCAPED_STRING
%import common.SH_COMMENT
%import common.CNAME
%import common.SIGNED_NUMBER
%import common.WS
%ignore WS
""", start='testcase')
# Evaluate the tree, using a Transformer.
# A transformer is a class with methods corresponding to branch names.
# For each branch, the appropriate method will be called with the children
# of the branch as its argument, and its return value will replace the branch
# in the tree. We want to transform the parse tree into a tuple containing the
# test case values.
class MyTransformer(Transformer):
def testcase(self, items):
*vs, c = items
if c==None: #it means it is a comment (see SH_COMMENT), so we can discard
return tuple(vs)
else:
return tuple(items)
def SH_COMMENT(self,n):
return None
def value(self, items):
[res] = items
return res
def pair(self, key_value):
k, v = key_value
return k, v
def string(self, s):
(s,) = s
return s[1:-1]
def number (self, n):
(n,) = n
return n
def FLOAT_NUMBER (self, n):
return float(n)
def DEC_NUMBER(self, n):
return int(n)
def emptyset(self, items):
return set()
def set(self, items):
res = set()
for i in items:
res.add(i)
return res
list = list
tuple = tuple
dict = dict
none = lambda self, _: None
true = lambda self, _: True
false = lambda self, _: False
def get_test_cases(filename):
"""
This function returns a list of the test cases that are defined in the
file with "@pytest.mark.parametrize". If it is not a pytest file it returns
the empty list
Throws FileNotFoundError exception if file does not exist.
"""
#1: Open the file
python_file = open(filename, "r")
#2: Read the file until you encounter the line where the testcases
# start (that is @pytest.mark.parametrize)
line = python_file.readline()
while not (line.startswith("@pytest.mark.parametrize") or line==''):
line = python_file.readline()
#read one more line, to point line to the first test case
line = python_file.readline()
test_cases = []
while (line.startswith("(")): #each test case starts with "("
#parse the line
tc_tree = testcase_parser.parse(line)
#reduce the parse tree to a tc tuple like (num, i1, i2,...,in o)
tc = MyTransformer().transform(tc_tree)
#add the testcase to the list of test cases
test_cases.append(tc)
line = python_file.readline() #go to next line in file
return test_cases
#3: Close the file
python_file.close()
def fill_excell_headers(test_signature, wb):
"""
# This function fills the headers of a test report with number_of_inputs input values
"""
#We know the structure we need to create for the excell file from the test_signature
number_of_inputs = len(test_signature)-2
# add_sheet is used to create sheet for Test Report
sheet = wb.add_sheet('Test Report')
# add test case ID colum at 0,0
sheet.write(0, 0, 'test case ID')
# add input columns for each test case input
for i in range (1, number_of_inputs+1):
sheet.write(0, i, 'input'+str(i))
# add input columns for the expected outcome and the result
sheet.write(0, number_of_inputs+1 , 'expected outcome')
sheet.write(0, number_of_inputs+2 , 'result')
return sheet
def generate_excell_test_report(filenameTest, filenameTestRes):
""" filenameTest es el nombre de fichero .py de testing y
filenameTestRes es el nombre de fichero .txt con test results
"""
try:
test_signature = get_test_signature(filenameTest)
if (test_signature == []):
print("This is not a pytest file")
else:
test_cases = get_test_cases(filenameTest)
failed_test_cases = get_failed_testcases(filenameTestRes)
failed_test_cases_numbers = []
for f in failed_test_cases:
failed_test_cases_numbers.append(int(f.split()[2].replace(",","")))
# Workbook is created
wb = Workbook()
#fill with headers for the columns
sheet = fill_excell_headers(test_signature, wb)
#write ID, inputs y output in excell
for i in range(len(test_cases)):
for j in range(len(test_cases[i])):
sheet.write(i+1, j , str(test_cases[i][j]))
if test_cases[i][0] in failed_test_cases_numbers:
sheet.write(i+1, len(test_cases[i]) , "FAILED")
else:
sheet.write(i+1, len(test_cases[i]) , "PASSED")
report_name = filenameTest.replace(".py", "")
# Save the Workbook
wb.save(report_name + 'TestReport.xls')
except FileNotFoundError:
print("El fichero no existe" + filenameTest + " o " + filenameTestRes)
def generate_JSON_test_report(filenameTest, filenameTestRes):
""" filenameTest es el nombre de fichero .py de testing y
filenameTestRes es el nombre de fichero .txt con test results
"""
try:
test_signature = get_test_signature(filenameTest)
if (test_signature == []):
print("This is not a pytest file")
else:
test_cases = get_test_cases(filenameTest)
failed_test_cases = get_failed_testcases(filenameTestRes)
failed_test_cases_numbers = []
for f in failed_test_cases:
failed_test_cases_numbers.append(int(f.split()[2].replace(",","")))
test_cases_dicts = []
for tc in test_cases:
tc_dict = {"id":tc[0]}
out = tc[-1]
tc_inputs = tc[1:len(tc)-1]
inputs = []
for t in tc:
inputs.append(t)
tc_dict["inputs"]=inputs
tc_dict["output esperado"]=out
if tc[0] in failed_test_cases_numbers:
tc_dict["resultado"]= "FAILED"
else:
tc_dict["resultado"]= "PASSED"
test_cases_dicts.append(tc_dict)
report_name = filenameTest.replace(".py", "")
fhand_write = open(report_name + "test_case_report.json", "w")
fhand_write.write(json.dumps(test_cases_dicts))
fhand_write.close()
except FileNotFoundError:
print("El fichero no existe" + filenameTest)
def main():
#test the report generatiom
#you need to check the output manualy as follows:
# 1) see if the files were generated in the directory
# 2) check the data in the files corresponds to the testcases in the .py file,
# and the outputs in the .txt file
file1_test = "pytests-for_testing_reports/union_test.py"
file1_testres = "pytests-for_testing_reports/output_union_test.txt"
file2_test = "pytests-for_testing_reports/min_max_list_test.py"
file2_testres = "pytests-for_testing_reports/output_min_max_list_test.txt"
file3_test = "pytests-for_testing_reports/interseccion_test.py"
file3_testres = "pytests-for_testing_reports/output_interseccion_test.txt"
file4_test = "pytests-for_testing_reports/filtrar_impares_test.py"
file4_testres = "pytests-for_testing_reports/output_filtrar_impares_test.txt"
generate_excell_test_report(file1_test, file1_testres)
generate_JSON_test_report(file1_test, file1_testres)
generate_excell_test_report(file2_test, file2_testres)
generate_JSON_test_report(file2_test, file2_testres)
generate_excell_test_report(file3_test, file3_testres)
generate_JSON_test_report(file3_test, file3_testres)
generate_excell_test_report(file4_test, file4_testres)
generate_JSON_test_report(file4_test, file4_testres)