diff --git a/tests/bugfixes/github/test_CVE_2017_1000126.py b/tests/bugfixes/github/test_CVE_2017_1000126.py index 414da55e..a4c8341b 100644 --- a/tests/bugfixes/github/test_CVE_2017_1000126.py +++ b/tests/bugfixes/github/test_CVE_2017_1000126.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/175" - filename = "{data_path}/cve_2017_1000126_stack-oob-read.webp" - commands = ["{exiv2} " + filename] + filename = "$data_path/cve_2017_1000126_stack-oob-read.webp" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerCorruptedMetadata} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerCorruptedMetadata """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_1000127.py b/tests/bugfixes/github/test_CVE_2017_1000127.py index ed22da37..bdb5642b 100644 --- a/tests/bugfixes/github/test_CVE_2017_1000127.py +++ b/tests/bugfixes/github/test_CVE_2017_1000127.py @@ -3,14 +3,14 @@ import system_tests -class TestPoC(system_tests.Case): +class TestPoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/176" - filename = "{data_path}/heap-oob-write.tiff" - commands = ["{exiv2} " + filename] + filename = "$data_path/heap-oob-write.tiff" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11336.py b/tests/bugfixes/github/test_CVE_2017_11336.py index 95741a36..91466757 100644 --- a/tests/bugfixes/github/test_CVE_2017_11336.py +++ b/tests/bugfixes/github/test_CVE_2017_11336.py @@ -3,15 +3,15 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/49" - filename = "{data_path}/POC2" - commands = ["{exiv2} " + filename] + filename = "$data_path/POC2" + commands = ["$exiv2 " + filename] retval = [1] stdout = [""] stderr = [ - """{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + """$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] diff --git a/tests/bugfixes/github/test_CVE_2017_11337.py b/tests/bugfixes/github/test_CVE_2017_11337.py index c2ebeac9..dc3cfca5 100644 --- a/tests/bugfixes/github/test_CVE_2017_11337.py +++ b/tests/bugfixes/github/test_CVE_2017_11337.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/50" - filename = "{data_path}/POC3" - commands = ["{exiv2} " + filename] + filename = "$data_path/POC3" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11338.py b/tests/bugfixes/github/test_CVE_2017_11338.py index 74c852ff..9b6d1153 100644 --- a/tests/bugfixes/github/test_CVE_2017_11338.py +++ b/tests/bugfixes/github/test_CVE_2017_11338.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/51" - filename = "{data_path}/POC4" - commands = ["{exiv2} " + filename] + filename = "$data_path/POC4" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11339.py b/tests/bugfixes/github/test_CVE_2017_11339.py index 257619b0..20fdd682 100644 --- a/tests/bugfixes/github/test_CVE_2017_11339.py +++ b/tests/bugfixes/github/test_CVE_2017_11339.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/52" - filename = "{data_path}/POC5" - commands = ["{exiv2} " + filename] + filename = "$data_path/POC5" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11340.py b/tests/bugfixes/github/test_CVE_2017_11340.py index 2b289c77..497b7871 100644 --- a/tests/bugfixes/github/test_CVE_2017_11340.py +++ b/tests/bugfixes/github/test_CVE_2017_11340.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/53" - filename = "{data_path}/POC6" - commands = ["{exiv2} " + filename] + filename = "$data_path/POC6" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11553.py b/tests/bugfixes/github/test_CVE_2017_11553.py index f57d85e8..17d56e87 100644 --- a/tests/bugfixes/github/test_CVE_2017_11553.py +++ b/tests/bugfixes/github/test_CVE_2017_11553.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/54" - filename = "{data_path}/POC7" - commands = ["{exiv2} " + filename] + filename = "$data_path/POC7" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11591.py b/tests/bugfixes/github/test_CVE_2017_11591.py index 772c5240..83ca8d9f 100644 --- a/tests/bugfixes/github/test_CVE_2017_11591.py +++ b/tests/bugfixes/github/test_CVE_2017_11591.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/55" - filename = "{data_path}/POC8" - commands = ["{exiv2} " + filename] + filename = "$data_path/POC8" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11592.py b/tests/bugfixes/github/test_CVE_2017_11592.py index 233d38ea..49273031 100644 --- a/tests/bugfixes/github/test_CVE_2017_11592.py +++ b/tests/bugfixes/github/test_CVE_2017_11592.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/56" - filename = "{data_path}/POC9" - commands = ["{exiv2} " + filename] + filename = "$data_path/POC9" + commands = ["$exiv2 " + filename] stdout = [""""""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_11683.py b/tests/bugfixes/github/test_CVE_2017_11683.py index fac4a5ec..e00a8be3 100644 --- a/tests/bugfixes/github/test_CVE_2017_11683.py +++ b/tests/bugfixes/github/test_CVE_2017_11683.py @@ -3,15 +3,15 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/57" - filename = "{data_path}/POC" - commands = ["{exiv2} " + filename] + filename = "$data_path/POC" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{kerInvalidTypeValue}: 0 -{exiv2_exception_message} """ + filename + """: -{kerInvalidTypeValue} + stderr = ["""$kerInvalidTypeValue: 0 +$exiv2_exception_message """ + filename + """: +$kerInvalidTypeValue """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_12955.py b/tests/bugfixes/github/test_CVE_2017_12955.py index 5b40d148..80e42013 100644 --- a/tests/bugfixes/github/test_CVE_2017_12955.py +++ b/tests/bugfixes/github/test_CVE_2017_12955.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/58" - filename = "{data_path}/POC11" - commands = ["{exiv2} " + filename] + filename = "$data_path/POC11" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_12956.py b/tests/bugfixes/github/test_CVE_2017_12956.py index 74f73d65..5a956ea6 100644 --- a/tests/bugfixes/github/test_CVE_2017_12956.py +++ b/tests/bugfixes/github/test_CVE_2017_12956.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/59" - filename = "{data_path}/POC12" - commands = ["{exiv2} " + filename] + filename = "$data_path/POC12" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_12957.py b/tests/bugfixes/github/test_CVE_2017_12957.py index acaaa07c..1b8e4a01 100644 --- a/tests/bugfixes/github/test_CVE_2017_12957.py +++ b/tests/bugfixes/github/test_CVE_2017_12957.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/60" - filename = "{data_path}/POC13" - commands = ["{exiv2} " + filename] + filename = "$data_path/POC13" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14857.py b/tests/bugfixes/github/test_CVE_2017_14857.py index 895a7e54..84a07c42 100644 --- a/tests/bugfixes/github/test_CVE_2017_14857.py +++ b/tests/bugfixes/github/test_CVE_2017_14857.py @@ -3,15 +3,15 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/76" - filename = "{data_path}/010_bad_free" - commands = ["{exiv2} " + filename] + filename = "$data_path/010_bad_free" + commands = ["$exiv2 " + filename] retval = [1] stdout = [""] stderr = [ - """{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + """$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] diff --git a/tests/bugfixes/github/test_CVE_2017_14858.py b/tests/bugfixes/github/test_CVE_2017_14858.py index bada8574..09572ee6 100644 --- a/tests/bugfixes/github/test_CVE_2017_14858.py +++ b/tests/bugfixes/github/test_CVE_2017_14858.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/138" - filename = "{data_path}/007-heap-buffer-over" - commands = ["{exiv2} " + filename] + filename = "$data_path/007-heap-buffer-over" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14859.py b/tests/bugfixes/github/test_CVE_2017_14859.py index a156895a..ff7073d0 100644 --- a/tests/bugfixes/github/test_CVE_2017_14859.py +++ b/tests/bugfixes/github/test_CVE_2017_14859.py @@ -3,15 +3,15 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/74" - filename = "{data_path}/005-invalid-mem" - commands = ["{exiv2} " + filename] + filename = "$data_path/005-invalid-mem" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerCorruptedMetadata} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerCorruptedMetadata """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14860.py b/tests/bugfixes/github/test_CVE_2017_14860.py index abae772a..74b5711b 100644 --- a/tests/bugfixes/github/test_CVE_2017_14860.py +++ b/tests/bugfixes/github/test_CVE_2017_14860.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/73" - filename = "{data_path}/003-heap-buffer-over" - commands = ["{exiv2} " + filename] + filename = "$data_path/003-heap-buffer-over" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerCorruptedMetadata} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerCorruptedMetadata """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14861.py b/tests/bugfixes/github/test_CVE_2017_14861.py index 432a0a17..5e4cf362 100644 --- a/tests/bugfixes/github/test_CVE_2017_14861.py +++ b/tests/bugfixes/github/test_CVE_2017_14861.py @@ -3,17 +3,17 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = [ "https://github.com/Exiv2/exiv2/issues/139", "https://bugzilla.redhat.com/show_bug.cgi?id=1494787" ] - filename = "{data_path}/009-stack-over" - commands = ["{exiv2} " + filename] + filename = "$data_path/009-stack-over" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14862.py b/tests/bugfixes/github/test_CVE_2017_14862.py index b915c348..6d6205dc 100644 --- a/tests/bugfixes/github/test_CVE_2017_14862.py +++ b/tests/bugfixes/github/test_CVE_2017_14862.py @@ -3,15 +3,15 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/75" - filename = "{data_path}/008-invalid-mem" - commands = ["{exiv2} " + filename] + filename = "$data_path/008-invalid-mem" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerCorruptedMetadata} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerCorruptedMetadata """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14863.py b/tests/bugfixes/github/test_CVE_2017_14863.py index 4cba3a92..78b4c101 100644 --- a/tests/bugfixes/github/test_CVE_2017_14863.py +++ b/tests/bugfixes/github/test_CVE_2017_14863.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/132" - filename = "{data_path}/01-Null-exiv2-poc" - commands = ["{exiv2} " + filename] + filename = "$data_path/01-Null-exiv2-poc" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14864.py b/tests/bugfixes/github/test_CVE_2017_14864.py index 26a46273..f0deef97 100644 --- a/tests/bugfixes/github/test_CVE_2017_14864.py +++ b/tests/bugfixes/github/test_CVE_2017_14864.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/73" - filename = "{data_path}/02-Invalid-mem-def" - commands = ["{exiv2} " + filename] + filename = "$data_path/02-Invalid-mem-def" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerCorruptedMetadata} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerCorruptedMetadata """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14865.py b/tests/bugfixes/github/test_CVE_2017_14865.py index e0f4da61..9392c051 100644 --- a/tests/bugfixes/github/test_CVE_2017_14865.py +++ b/tests/bugfixes/github/test_CVE_2017_14865.py @@ -3,15 +3,15 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/134" - filename = "{data_path}/004-heap-buffer-over" - commands = ["{exiv2} " + filename] + filename = "$data_path/004-heap-buffer-over" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{kerInvalidTypeValue}: 250 -{exiv2_exception_message} """ + filename + """: -{kerInvalidTypeValue} + stderr = ["""$kerInvalidTypeValue: 250 +$exiv2_exception_message """ + filename + """: +$kerInvalidTypeValue """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_14866.py b/tests/bugfixes/github/test_CVE_2017_14866.py index 659aa617..da4fdb81 100644 --- a/tests/bugfixes/github/test_CVE_2017_14866.py +++ b/tests/bugfixes/github/test_CVE_2017_14866.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/140" - filename = "{data_path}/006-heap-buffer-over" - commands = ["{exiv2} " + filename] + filename = "$data_path/006-heap-buffer-over" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_17669.py b/tests/bugfixes/github/test_CVE_2017_17669.py index bf6a46b3..c733b0c3 100644 --- a/tests/bugfixes/github/test_CVE_2017_17669.py +++ b/tests/bugfixes/github/test_CVE_2017_17669.py @@ -3,16 +3,16 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/187" - filename = "{data_path}/issue_187" - commands = ["{exiv2} " + filename] + filename = "$data_path/issue_187" + commands = ["$exiv2 " + filename] retval = [1] stdout = [""] stderr = [ - """{exiv2_exception_message} """ + filename + """: -{kerFailedToReadImageData} + """$exiv2_exception_message """ + filename + """: +$kerFailedToReadImageData """ ] diff --git a/tests/bugfixes/github/test_CVE_2017_17722.py b/tests/bugfixes/github/test_CVE_2017_17722.py index fd5391ce..65f2d825 100644 --- a/tests/bugfixes/github/test_CVE_2017_17722.py +++ b/tests/bugfixes/github/test_CVE_2017_17722.py @@ -3,15 +3,15 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/208" - filename = "{data_path}/2018-01-09-exiv2-crash-001.tiff" - commands = ["{exiv2} " + filename] + filename = "$data_path/2018-01-09-exiv2-crash-001.tiff" + commands = ["$exiv2 " + filename] retval = [1] stdout = [""] stderr = [ - """{exiv2_exception_message} """ + filename + """: -{kerCorruptedMetadata} + """$exiv2_exception_message """ + filename + """: +$kerCorruptedMetadata """] diff --git a/tests/bugfixes/github/test_CVE_2017_17725.py b/tests/bugfixes/github/test_CVE_2017_17725.py index da8f48de..1127b980 100644 --- a/tests/bugfixes/github/test_CVE_2017_17725.py +++ b/tests/bugfixes/github/test_CVE_2017_17725.py @@ -3,15 +3,15 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/188" found_by = ["Wei You", "@youwei1988"] - filename = "{data_path}/poc_2017-12-12_issue188" - commands = ["{exiv2} " + filename] + filename = "$data_path/poc_2017-12-12_issue188" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_overflow_exception_message} """ + filename + """: -{addition_overflow_message} + stderr = ["""$exiv2_overflow_exception_message """ + filename + """: +$addition_overflow_message """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2017_18005.py b/tests/bugfixes/github/test_CVE_2017_18005.py index e7ae3f7a..bb0314a0 100644 --- a/tests/bugfixes/github/test_CVE_2017_18005.py +++ b/tests/bugfixes/github/test_CVE_2017_18005.py @@ -3,7 +3,7 @@ import system_tests -class TestPoC(system_tests.Case): +class TestPoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/168" @@ -12,20 +12,20 @@ Error: Offset of directory Image, entry 0x0117 is out of bounds: Offset = 0x3030 """ + 12 * """Error: Offset of directory Image, entry 0x3030 is out of bounds: Offset = 0x30303030; truncating the entry """ - filename = "{data_path}/cve_2017_18005_reproducer.tiff" + filename = "$data_path/cve_2017_18005_reproducer.tiff" commands = [ - "{exiv2} -v pr -P EIXxgklnycsvth " + filename, - "{exiv2json} " + filename + "$exiv2 -v pr -P EIXxgklnycsvth " + filename, + "$exiv2json " + filename ] stdout = ["""File 1/1: """ + filename + """ 0x0117 Image Exif.Image.StripByteCounts StripByteCounts Strip Byte Count SByte 0 0 """, - """{{ - "Exif": {{ - "Image": {{ + """{ + "Exif": { + "Image": { "StripByteCounts": 0, "0x3030": 0, "0x3030": "", @@ -40,9 +40,9 @@ Error: Offset of directory Image, entry 0x0117 is out of bounds: Offset = 0x3030 "0x3030": 0, "0x3030": 0, "0x3030": 0 - }} - }} -}} + } + } +} """ ] stderr = [ diff --git a/tests/bugfixes/github/test_CVE_2017_9953.py b/tests/bugfixes/github/test_CVE_2017_9953.py index 79070b6e..22af56d3 100644 --- a/tests/bugfixes/github/test_CVE_2017_9953.py +++ b/tests/bugfixes/github/test_CVE_2017_9953.py @@ -3,14 +3,14 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/144" - filename = "{data_path}/POC1" - commands = ["{exiv2} " + filename] + filename = "$data_path/POC1" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerInvalidMalloc} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerInvalidMalloc """] retval = [1] diff --git a/tests/bugfixes/github/test_CVE_2018_4868.py b/tests/bugfixes/github/test_CVE_2018_4868.py index 86fbd22c..28a42b5e 100644 --- a/tests/bugfixes/github/test_CVE_2018_4868.py +++ b/tests/bugfixes/github/test_CVE_2018_4868.py @@ -3,16 +3,16 @@ import system_tests -class TestCvePoC(system_tests.Case): +class TestCvePoC(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/202" cve_url = "http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-4868" found_by = ["afl", "topsecLab", "xcainiao"] - filename = "{data_path}/exiv2-memorymmap-error" - commands = ["{exiv2} " + filename] + filename = "$data_path/exiv2-memorymmap-error" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerCorruptedMetadata} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerCorruptedMetadata """] retval = [1] diff --git a/tests/bugfixes/github/test_issue_159.py b/tests/bugfixes/github/test_issue_159.py index be4bf998..820d32ab 100644 --- a/tests/bugfixes/github/test_issue_159.py +++ b/tests/bugfixes/github/test_issue_159.py @@ -3,18 +3,18 @@ import system_tests -class TestFirstPoC(system_tests.Case): +class TestFirstPoC(metaclass=system_tests.CaseMeta): """ Regression test for the first bug described in: https://github.com/Exiv2/exiv2/issues/159 """ url = "https://github.com/Exiv2/exiv2/issues/159" - filename = "{data_path}/printStructure" - commands = ["{exiv2} " + filename] + filename = "$data_path/printStructure" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerCorruptedMetadata} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerCorruptedMetadata """] retval = [1] diff --git a/tests/bugfixes/github/test_issue_170.py b/tests/bugfixes/github/test_issue_170.py index a67e8a4b..99527f26 100644 --- a/tests/bugfixes/github/test_issue_170.py +++ b/tests/bugfixes/github/test_issue_170.py @@ -3,15 +3,15 @@ import system_tests -class DecodeIHDRChunkOutOfBoundsRead(system_tests.Case): +class DecodeIHDRChunkOutOfBoundsRead(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/170" - filename = "{data_path}/issue_170_poc" + filename = "$data_path/issue_170_poc" - commands = ["{exiv2} " + filename] + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerFailedToReadImageData} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerFailedToReadImageData """] retval = [1] diff --git a/tests/bugfixes/github/test_issue_20.py b/tests/bugfixes/github/test_issue_20.py index d3014a25..1733dc02 100644 --- a/tests/bugfixes/github/test_issue_20.py +++ b/tests/bugfixes/github/test_issue_20.py @@ -3,7 +3,7 @@ import system_tests -class TamronSupport(system_tests.Case): +class TamronSupport(metaclass=system_tests.CaseMeta): description = "Added support for 'Tamron SP 15-30mm f/2.8 Di VC USD A012' and 'Tamron SP 90mm f/2.8 Di VC USD MACRO1:1'" @@ -14,9 +14,9 @@ class TamronSupport(system_tests.Case): "TamronSP90mmF2.8DiVCUSDMacroF004.exv", "TamronSP90mmF2.8DiVCUSDMacroF017.exv" ] - commands = ["{exiv2} -pa --grep lens/i ../../../test/data/" + files[0]] \ + commands = ["$exiv2 -pa --grep lens/i ../../../test/data/" + files[0]] \ + list(map( - lambda fname: "{exiv2} -pa --grep lenstype/i ../../../test/data/" + fname, + lambda fname: "$exiv2 -pa --grep lenstype/i ../../../test/data/" + fname, files[1:] )) retval = [0] * len(files) diff --git a/tests/bugfixes/github/test_issue_227.py b/tests/bugfixes/github/test_issue_227.py index b0f756ff..5d3fabac 100644 --- a/tests/bugfixes/github/test_issue_227.py +++ b/tests/bugfixes/github/test_issue_227.py @@ -3,7 +3,7 @@ import system_tests -class SigmaLenses(system_tests.Case): +class SigmaLenses(metaclass=system_tests.CaseMeta): files = [ "Sigma_120-300_DG_OS_HSM_Sport_lens.exv", @@ -12,7 +12,7 @@ class SigmaLenses(system_tests.Case): ] commands = list( - map(lambda fname: "{exiv2} -pa --grep lens/i {data_path}/" + fname, files) + map(lambda fname: "$exiv2 -pa --grep lens/i $data_path/" + fname, files) ) retval = 3 * [0] diff --git a/tests/bugfixes/github/test_issue_246.py b/tests/bugfixes/github/test_issue_246.py index 02eb908b..62c73133 100644 --- a/tests/bugfixes/github/test_issue_246.py +++ b/tests/bugfixes/github/test_issue_246.py @@ -2,15 +2,16 @@ import system_tests -class TestFirstPoC(system_tests.Case): + +class TestFirstPoC(metaclass=system_tests.CaseMeta): """ Regression test for the first bug described in: https://github.com/Exiv2/exiv2/issues/246 """ url = "https://github.com/Exiv2/exiv2/issues/246" - filename = "{data_path}/1-string-format.jpg" - commands = ["{exiv2} -pS " + filename] + filename = "$data_path/1-string-format.jpg" + commands = ["$exiv2 -pS " + filename] stdout = [ """STRUCTURE OF JPEG FILE: """ + filename + """ address | marker | length | data @@ -19,7 +20,7 @@ class TestFirstPoC(system_tests.Case): """] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerNoImageInInputData} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerNoImageInInputData """] retval = [1] diff --git a/tests/bugfixes/github/test_issue_247.py b/tests/bugfixes/github/test_issue_247.py index b5d4ef56..39844a17 100644 --- a/tests/bugfixes/github/test_issue_247.py +++ b/tests/bugfixes/github/test_issue_247.py @@ -2,15 +2,15 @@ import system_tests -class TestFirstPoC(system_tests.Case): +class TestFirstPoC(metaclass=system_tests.CaseMeta): """ Regression test for the first bug described in: https://github.com/Exiv2/exiv2/issues/247 """ url = "https://github.com/Exiv2/exiv2/issues/247" - filename = "{data_path}/2-invalid-memory-access" - commands = ["{exiv2} -pt " + filename] + filename = "$data_path/2-invalid-memory-access" + commands = ["$exiv2 -pt " + filename] stdout = [ """Exif.Image.Make Ascii 6 Canon Exif.Image.Orientation Short 1 top, left diff --git a/tests/bugfixes/github/test_issue_253.py b/tests/bugfixes/github/test_issue_253.py index d0aad448..910f0ddf 100644 --- a/tests/bugfixes/github/test_issue_253.py +++ b/tests/bugfixes/github/test_issue_253.py @@ -2,17 +2,18 @@ import system_tests -class TestFirstPoC(system_tests.Case): + +class TestFirstPoC(metaclass=system_tests.CaseMeta): """ Regression test for the first bug described in: https://github.com/Exiv2/exiv2/issues/253 """ url = "https://github.com/Exiv2/exiv2/issues/253" - filename = "{data_path}/3-stringformat-outofbound-read" - commands = ["{exiv2} " + filename] + filename = "$data_path/3-stringformat-outofbound-read" + commands = ["$exiv2 " + filename] stdout = [""] - stderr = ["""{exiv2_exception_message} """ + filename + """: -{kerNotAJpeg} + stderr = ["""$exiv2_exception_message """ + filename + """: +$kerNotAJpeg """] retval = [1] diff --git a/tests/bugfixes/github/test_issue_45.py b/tests/bugfixes/github/test_issue_45.py index 216a81b0..16727b26 100644 --- a/tests/bugfixes/github/test_issue_45.py +++ b/tests/bugfixes/github/test_issue_45.py @@ -3,12 +3,12 @@ import system_tests -class Sigma24_105mmRecognization(system_tests.Case): +class Sigma24_105mmRecognization(metaclass=system_tests.CaseMeta): url = "https://github.com/Exiv2/exiv2/issues/45" - filename = "{data_path}/exiv2-g45.exv" - commands = ["{exiv2} -pa --grep lens/i " + filename] + filename = "$data_path/exiv2-g45.exv" + commands = ["$exiv2 -pa --grep lens/i " + filename] stdout = ["""Exif.CanonCs.LensType Short 1 Sigma 24-105mm F4 DG OS HSM [Art 013] Exif.CanonCs.Lens Short 3 24.0 - 105.0 mm Exif.CanonCf.LensAFStopButton Short 1 0 diff --git a/tests/bugfixes/github/test_regression_issue_201.py b/tests/bugfixes/github/test_regression_issue_201.py index f203619f..967c162b 100644 --- a/tests/bugfixes/github/test_regression_issue_201.py +++ b/tests/bugfixes/github/test_regression_issue_201.py @@ -3,9 +3,9 @@ import system_tests -class ShadowingError(system_tests.Case): +class ShadowingError(metaclass=system_tests.CaseMeta): - commands = ["{exiv2} -PE {data_path}/IMGP0020.exv"] + commands = ["$exiv2 -PE $data_path/IMGP0020.exv"] stdout = [""] stderr = [""] retval = [0] diff --git a/tests/bugfixes/redmine/test_1305.py b/tests/bugfixes/redmine/test_1305.py index 8c6a06dd..ade18af2 100644 --- a/tests/bugfixes/redmine/test_1305.py +++ b/tests/bugfixes/redmine/test_1305.py @@ -3,7 +3,7 @@ import system_tests -class Issue1305Test(system_tests.Case): +class Issue1305Test(metaclass=system_tests.CaseMeta): err_msg_dir_img = """Warning: Directory Image, entry 0x3030 has unknown Exif (TIFF) type 12336; setting type size 1. Error: Directory Image, entry 0x3030 has invalid size 808464432*1; skipping entry. """ @@ -19,8 +19,8 @@ Error: Directory Pentax, entry 0x3030 has invalid size 808464432*1; skipping ent name = "regression test for issue 1305" url = "http://dev.exiv2.org/issues/1305" - filename = "{data_path}/IMGP0006-min.jpg" - commands = ["{exiv2} " + filename] + filename = "$data_path/IMGP0006-min.jpg" + commands = ["$exiv2 " + filename] stdout = ["""File name : """ + filename + """ File size : 12341 Bytes MIME type : image/jpeg diff --git a/tests/doc.md b/tests/doc.md index fff7ba5f..5f9c887a 100644 --- a/tests/doc.md +++ b/tests/doc.md @@ -184,51 +184,44 @@ An example test case file would look like this: import system_tests -class AnInformativeName(system_tests.Case): +class AnInformativeName(metaclass=system_tests.CaseMeta): filename = "invalid_input_file" commands = [ - "{binary} -c {import_file} -i {filename}" + "$binary -c $import_file -i $filename" ] - retval = ["{abort_exit_value}"] - stdout = ["Reading {filename}"] + retval = ["$abort_exit_value"] + stdout = ["Reading $filename"] stderr = [ - """{abort_error} -error in {filename} + """$abort_error +error in $filename """ ] ``` The first 6 lines are necessary boilerplate to pull in the necessary routines to run the actual tests (these are implemented in the module `system_tests` with -the class `system_tests.Case` extending `unittest.TestCase`). When adding new -tests one should choose a new class name that briefly summarizes the test. Note -that the file name (without the extension) with the directory structure is -interpreted as the module by Python and pre-pended to the class name when -reporting about the tests. E.g. the file `regression/crashes/test_bug_15.py` -with the class `OutOfBoundsRead` gets reported as -`regression.crashes.test_bug_15.OutOfBoundsRead** already including a brief -summary of this test. - -**Caution:** Always import `system_tests` in the aforementioned syntax and don't -use `from system_tests import Case`. This will not work, as the `system_tests` -module stores the suite's config internally which will not be available if you -perform a `from system_tests import Case` (this causes Python to create a copy -of the class `system_tests.Case` for your module, without reading the -configuration file). +the meta-class `system_tests.CaseMeta` which performs the necessary preparations +for the tests to run). When adding new tests one should choose a new class name +that briefly summarizes the test. Note that the file name (without the +extension) with the directory structure is interpreted as the module by Python +and pre-pended to the class name when reporting about the tests. E.g. the file +`regression/crashes/test_bug_15.py` with the class `OutOfBoundsRead` gets +reported as `regression.crashes.test_bug_15.OutOfBoundsRead` already including +a brief summary of this test. In the following lines the lists `commands`, `retval`, `stdout` and `stderr` -should be defined. These are lists of strings and must all have the same amount +should be defined. These are lists of strings and must all have the same number of elements. -The test suite at first takes all these strings and substitutes all values in -curly braces with variables either defined in this class alongside (like +The test suite at first takes all these strings and substitutes all values +following a `$` with variables either defined in this class alongside (like `filename` in the above example) or with the values defined in the test suite's configuration file. Please note that defining a variable with the same name as a variable in the suite's configuration file will result in an error (otherwise one of the variables would take precedence leading to unexpected results). The -substitution of values in performed using Python's string `format()` method and -more elaborate format strings can be used when necessary. +substitution of values is performed using the template module from Python's +string library via `safe_substitute`. In the above example the command would thus expand to: ``` shell @@ -281,6 +274,45 @@ This section describes more advanced features that are probably not necessary the "standard" usage of the test suite. +### Using a different output encoding + +The test suite will try to interpret the program's output as utf-8 encoded +strings and if that fails it will try the `iso-8859-1` encoding (also know as +`latin-1`). + +If the tested program outputs characters in another encoding then it can be +supplied as the `encodings` parameter in each test case: +``` python +# -*- coding: utf-8 -*- + +import system_tests + + +class AnInformativeName(metaclass=system_tests.CaseMeta): + + encodings = ['ascii'] + + filename = "invalid_input_file" + commands = [ + "$binary -c $import_file -i $filename" + ] + retval = ["$abort_exit_value"] + stdout = ["Reading $filename"] + stderr = [ + """$abort_error +error in $filename +""" + ] +``` + +The test suite will try to decode the program's output with the provided +encodings in the order that they appear in the list. It will select the first +encoding that can decode the output successfully. If no encoding is able to +decode the program's output, then an error is raised. The list of all supported +encodings can be found +[here](https://docs.python.org/3/library/codecs.html#standard-encodings). + + ### Creating file copies For tests that modify their input file it is useful to run these with a @@ -295,26 +327,26 @@ Example: import system_tests -@system_tests.CopyFiles("{filename}", "{some_path}/another_file.txt") -class AnInformativeName(system_tests.Case): +@system_tests.CopyFiles("$filename", "$some_path/another_file.txt") +class AnInformativeName(metaclass=system_tests.CaseMeta): filename = "invalid_input_file" commands = [ - "{binary} -c {import_file} -i {filename}" + "$binary -c $import_file -i $filename" ] - retval = ["{abort_exit_value}"] - stdout = ["Reading {filename}"] + retval = ["$abort_exit_value"] + stdout = ["Reading $filename"] stderr = [ - """{abort_error} -error in {filename} + """$abort_error +error in $filename """ ] ``` In this example, the test suite would automatically create a copy of the files -`invalid_input_file` and `{some_path}/another_file.txt` (`some_path` would be of +`invalid_input_file` and `$some_path/another_file.txt` (`some_path` would be of course expanded too) named `invalid_input_file_copy` and -`{some_path}/another_file_copy.txt`. After the test ran, the copies are +`$some_path/another_file_copy.txt`. After the test ran, the copies are deleted. Please note that variable expansion in the filenames is possible. @@ -356,12 +388,12 @@ by the test suite. It can be used in the following way: import system_tests -class AnInformativeName(system_tests.Case): +class AnInformativeName(metaclass=system_tests.CaseMeta): filename = "invalid_input_file" - commands = ["{binary} -c {import_file} -i {filename}"] - retval = ["{abort_exit_value}"] - stdout = ["Reading {filename}"] + commands = ["$binary -c $import_file -i $filename"] + retval = ["$abort_exit_value"] + stdout = ["Reading $filename"] stderr = ["""A huge amount of error messages would be here that we absolutely do not care about. Actually everything in this string gets ignored, so we can just leave it empty. """ ] @@ -380,12 +412,12 @@ variable substitution using the test suite's configuration file. Unfortunately, it has to run in a class member function. The `setUp()` function can be used for this, as it is run before each test. For example like this: ``` python -class SomeName(system_tests.Case): +class SomeName(metaclass=system_tests.CaseMeta): def setUp(self): - self.commands = [self.expand_variables("{some_var}/foo.txt")] + self.commands = [self.expand_variables("$some_var/foo.txt")] self.stderr = [""] - self.stdout = [self.expand_variables("{success_message}")] + self.stdout = [self.expand_variables("$success_message")] self.retval = [0] ``` @@ -394,11 +426,11 @@ This example will work, as the test runner reads the data for `commands`, work is creating a new member in `setUp()` and trying to use it as a variable for expansion, like this: ``` python -class SomeName(system_tests.Case): +class SomeName(metaclass=system_tests.CaseMeta): def setUp(self): self.new_var = "foo" - self.another_string = self.expand_variables("{new_var}") + self.another_string = self.expand_variables("$new_var") ``` This example fails in `self.expand_variables` because the expansion uses only @@ -407,13 +439,13 @@ class member in `setUp()` the changed version will **not** be used for variable expansion, as the variables are saved in a new dictionary **before** `setUp()` runs. Thus this: ``` python -class SomeName(system_tests.Case): +class SomeName(metaclass=system_tests.CaseMeta): new_var = "foo" def setUp(self): self.new_var = "bar" - self.another_string = self.expand_variables("{new_var}") + self.another_string = self.expand_variables("$new_var") ``` will result in `another_string` being "foo" and not "bar". @@ -425,27 +457,27 @@ will result in `another_string` being "foo" and not "bar". cases. `setUpClass()` is used by `system_tests.Case` to store the variables for expansion. -- Keep in mind that the variable expansion uses Python's `format()` - function. This can make it more cumbersome to include formatted strings into - variables like `commands` which will likely contain other variables from the - test suite. E.g.: `commands = ["{binary} {:s}".format(f) for f in files]` will - not work as `format()` will expect a value for binary. This can be worked - around using either the old Python formatting via `%` or by formatting first - and then concatenating the problematic parts. - ## Running the test suite -The test suite is written for Python 3 but is in principle also compatible with -Python 2, albeit it is not regularly tested, so its functionality is not -guaranteed with Python 2. +The test suite is written for Python 3 and is not compatible with Python 2, thus +it must be run with `python3` and not with `python` (which is usually an alias +for Python 2). Then navigate to the `tests/` subdirectory and run: ``` shell python3 runner.py ``` +One can supply the script with a directory where the suite should look for the +tests (it will search the directory recursively). If omitted, the runner will +look in the directory where the configuration file is located. + The runner script also supports the optional arguments `--config_file` which allows to provide a different test suite configuration file than the default `suite.conf`. It also forwards the verbosity setting via the `-v`/`--verbose` flags to Python's unittest module. + +Optionally one can provide the `--debug` flag which will instruct test suite to +print all command invocations and all expected and obtained outputs to the +standard output. diff --git a/tests/runner.py b/tests/runner.py index 745dcebd..63b82810 100644 --- a/tests/runner.py +++ b/tests/runner.py @@ -15,16 +15,34 @@ if __name__ == '__main__': "--config_file", type=str, nargs=1, + help="Path to the suite's configuration file", default=['suite.conf'] ) parser.add_argument( "--verbose", "-v", action='count', + help="verbosity level", default=1 ) + parser.add_argument( + "--debug", + help="enable debugging output", + action='store_true' + ) + + parser.add_argument( + "dir", + help="directory where the test are searched for (defaults to the config" + "file's location)", + default=None, + type=str, + nargs='?' + ) + args = parser.parse_args() conf_file = args.config_file[0] - discovery_root = os.path.dirname(conf_file) + discovery_root = os.path.dirname(conf_file if args.dir is None else args.dir) + system_tests.set_debug_mode(args.debug) system_tests.configure_suite(conf_file) diff --git a/tests/suite.conf b/tests/suite.conf index e15b4724..45a10eaf 100644 --- a/tests/suite.conf +++ b/tests/suite.conf @@ -12,7 +12,8 @@ exiv2_path: ../build/bin exiv2: ${ENV:exiv2_path}/exiv2${ENV:binary_extension} exiv2json: ${ENV:exiv2_path}/exiv2json${ENV:binary_extension} data_path: ../test/data -tiff-test: ${ENV:exiv2_path}/tiff-test${ENV:binary_extension} +tiff_test: ${ENV:exiv2_path}/tiff-test${ENV:binary_extension} + [variables] kerFailedToReadImageData: Failed to read image data diff --git a/tests/system_tests.py b/tests/system_tests.py index c494681b..f870714b 100644 --- a/tests/system_tests.py +++ b/tests/system_tests.py @@ -8,6 +8,7 @@ import threading import shlex import sys import shutil +import string import unittest @@ -75,9 +76,24 @@ class CasePreservingConfigParser(configparser.ConfigParser): return option +#: global parameters extracted from the test suite's configuration file _parameters = {} +#: setting whether debug mode is enabled or not +_debug_mode = False + + +def set_debug_mode(debug): + """ Enable or disable debug mode + + In debug mode the test suite will print out all commands that it runs, the + expected output and the actually obtained output + """ + global _debug_mode + _debug_mode = debug + + def configure_suite(config_file): """ Populates a global datastructure with the parameters from the suite's @@ -406,42 +422,120 @@ class CopyFiles(FileDecoratorBase): return shutil.copyfile(expanded_file_name, new_name) +def test_run(self): + """ + This function reads in the members commands, retval, stdout, stderr and runs + the `expand_variables` function on each. The resulting commands are then run + using the subprocess module and compared against the expected values that + were provided in the static members via `compare_stdout` and + `compare_stderr`. Furthermore a threading.Timer is used to abort the + execution if a configured timeout is reached. + + It is automatically added as a member function to each system test by the + CaseMeta metaclass. This ensures that it is run by each system test + **after** setUp() and setUpClass() were run. + """ + if not (len(self.commands) == len(self.retval) + == len(self.stdout) == len(self.stderr)): + raise ValueError( + "commands, retval, stdout and stderr don't have the same length" + ) + for i, command, retval, stdout, stderr in zip(range(len(self.commands)), + self.commands, + self.retval, + self.stdout, + self.stderr): + command, retval, stdout, stderr = map( + self.expand_variables, [command, retval, stdout, stderr] + ) + retval = int(retval) + timeout = {"flag": False} + + if _debug_mode: + print( + '', "="*80, "will run: " + command, "expected stdout:", stdout, + "expected stderr:", stderr, + "expected return value: {:d}".format(retval), + sep='\n' + ) + + proc = subprocess.Popen( + _cmd_splitter(command), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=self.work_dir + ) + + def timeout_reached(timeout): + timeout["flag"] = True + proc.kill() + + t = threading.Timer( + _parameters["timeout"], timeout_reached, args=[timeout] + ) + t.start() + got_stdout, got_stderr = proc.communicate() + t.cancel() + + processed_stdout = None + processed_stderr = None + for encoding in self.encodings: + try: + processed_stdout = _process_output_post( + got_stdout.decode(encoding) + ) + processed_stderr = _process_output_post( + got_stderr.decode(encoding) + ) + except UnicodeError: + pass + else: + break + if processed_stdout is None or processed_stderr is None: + raise ValueError( + "Could not decode the output of the command '{!s}' with the " + "following encodings: {!s}" + .format(command, ','.join(self.encodings)) + ) + + if _debug_mode: + print( + "got stdout:", processed_stdout, "got stderr:", + processed_stderr, "got return value: {:d}" + .format(proc.returncode), + sep='\n' + ) + + self.assertFalse(timeout["flag"], msg="Timeout reached") + self.compare_stdout(i, command, processed_stdout, stdout) + self.compare_stderr(i, command, processed_stderr, stderr) + self.assertEqual( + retval, proc.returncode, msg="Return value does not match" + ) + + class Case(unittest.TestCase): """ System test case base class, provides the functionality to interpret static - class members as system tests and runs them. - - This class reads in the members commands, retval, stdout, stderr and runs - the format function on each, where format is called with the kwargs being a - merged dictionary of all variables that were extracted from the suite's - configuration file and all static members of the current class. - - The resulting commands are then run using the subprocess module and compared - against the expected values that were provided in the static - members. Furthermore a threading.Timer is used to abort the execution if a - configured timeout is reached. - - The class itself must be inherited from, otherwise it is not useful at all, - as it does not provide any static members that could be used to run system - tests. However, a class that inherits from this class needn't provide any - member functions at all, the inherited test_run() function performs all - required functionality in child classes. + class members as system tests. + + The class itself only provides utility functions and system tests need not + inherit from it, as it is automatically added via the CaseMeta metaclass. """ - """ maxDiff set so that arbitrarily large diffs will be shown """ + #: maxDiff set so that arbitrarily large diffs will be shown maxDiff = None + #: list of encodings that are used to decode the test program's output + #: the first encoding that does not raise a UnicodeError is used + encodings = ['utf-8', 'iso-8859-1'] + @classmethod def setUpClass(cls): """ - This function adds the variables variable_dict & work_dir to the class. - - work_dir - set to the file where the current class is defined - variable_dict - a merged dictionary of all static members of the current - class and all variables extracted from the suite's - configuration file + This function adds the variable work_dir to the class, which is the path + to the directory where the python source file is located. """ - cls.variable_dict = _disjoint_dict_merge(cls.__dict__, _parameters) cls.work_dir = os.path.dirname(inspect.getfile(cls)) def compare_stdout(self, i, command, got_stdout, expected_stdout): @@ -460,71 +554,89 @@ class Case(unittest.TestCase): unittest.TestCase. This function can be overridden in a child class to implement a custom check. """ - self.assertMultiLineEqual(expected_stdout, got_stdout) + self.assertMultiLineEqual( + expected_stdout, got_stdout, msg="Standard output does not match" + ) def compare_stderr(self, i, command, got_stderr, expected_stderr): - """ - Same as compare_stdout only for standard-error. - """ - self.assertMultiLineEqual(expected_stderr, got_stderr) + """ Same as compare_stdout only for standard-error. """ + self.assertMultiLineEqual( + expected_stderr, got_stderr, msg="Standard error does not match" + ) - def expand_variables(self, string): + def expand_variables(self, unexpanded_string): """ - Expands all variables in curly braces in the given string using the - dictionary variable_dict. + Expands all variables of the form ``$var`` in the given string using the + dictionary `variable_dict`. - The expansion itself is performed by the builtin string method format(). - A KeyError indicates that the supplied string contains a variable - in curly braces that is missing from self.variable_dict + The expansion itself is performed by the string's template module using + via `safe_substitute`. """ - return str(string).format(**self.variable_dict) + return string.Template(str(unexpanded_string))\ + .safe_substitute(**self.variable_dict) - def test_run(self): - """ - Actual system test function which runs the provided commands, - pre-processes all variables and post processes the output before passing - it on to compare_stderr() & compare_stdout(). - """ - for i, command, retval, stdout, stderr in zip(range(len(self.commands)), - self.commands, - self.retval, - self.stdout, - self.stderr): - command, retval, stdout, stderr = map( - self.expand_variables, [command, retval, stdout, stderr] - ) - retval = int(retval) - timeout = {"flag": False} - - proc = subprocess.Popen( - _cmd_splitter(command), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=self.work_dir - ) +class CaseMeta(type): + """ System tests generation metaclass. - def timeout_reached(timeout): - timeout["flag"] = True - proc.kill() + This metaclass is performs the following tasks: - t = threading.Timer( - _parameters["timeout"], timeout_reached, args=[timeout] - ) - t.start() - got_stdout, got_stderr = proc.communicate() - t.cancel() - - self.assertFalse(timeout["flag"] and "Timeout reached") - self.compare_stdout( - i, command, - _process_output_post(got_stdout.decode('utf-8')), stdout - ) - self.compare_stderr( - i, command, - _process_output_post(got_stderr.decode('utf-8')), stderr - ) - self.assertEqual(retval, proc.returncode) + 1. Add the `test_run` function as a member of the test class + 2. Add the `Case` class as the parent class + 3. Expand all variables already defined in the class, so that any additional + code does not have to perform this task + + Using a metaclass instead of inheriting from case has the advantage, that we + can expand all variables in the strings before any test case or even the + class constructor is run! Thus users will immediately see the expanded + result. Also adding the `test_run` function as a direct member and not via + inheritance enforces that it is being run **after** the test cases setUp & + setUpClass (which oddly enough seems not to be the case in the unittest + module where test functions of the parent class run before setUpClass of the + child class). + """ + + def __new__(mcs, clsname, bases, dct): + + changed = False + + # expand all non-private variables by brute force + # => try all expanding all elements defined in the current class until + # there is no change in them any more + keys = [key for key in list(dct.keys()) if not key.startswith('_')] + while not changed: + for key in keys: + + old_value = dct[key] + + # only try expanding strings and lists + if isinstance(old_value, str): + new_value = string.Template(old_value).safe_substitute( + **_disjoint_dict_merge(dct, _parameters) + ) + elif isinstance(old_value, list): + # do not try to expand anything but strings in the list + new_value = [ + string.Template(elem).safe_substitute( + **_disjoint_dict_merge(dct, _parameters) + ) + if isinstance(elem, str) else elem + for elem in old_value + ] + else: + continue + + if old_value != new_value: + changed = True + dct[key] = new_value + + dct['variable_dict'] = _disjoint_dict_merge(dct, _parameters) + dct['test_run'] = test_run + + if Case not in bases: + bases += (Case,) + + return super(CaseMeta, mcs).__new__(mcs, clsname, bases, dct) def check_no_ASAN_UBSAN_errors(self, i, command, got_stderr, expected_stderr): diff --git a/tests/tiff_test/test_tag_compare.py b/tests/tiff_test/test_tag_compare.py index 0812795f..b50dd196 100644 --- a/tests/tiff_test/test_tag_compare.py +++ b/tests/tiff_test/test_tag_compare.py @@ -3,7 +3,7 @@ import system_tests -class OutputTagExtract(system_tests.Case): +class OutputTagExtract(metaclass=system_tests.CaseMeta): """ Test whether exiv2 -pa $file and exiv2 -pS $file produces the same output. """ @@ -107,7 +107,7 @@ class OutputTagExtract(system_tests.Case): self.compare_pS_pa() commands = [ - "{exiv2} %s {data_path}/mini9.tif" % (opt) for opt in ["-pa", "-pS"] + "$exiv2 %s $data_path/mini9.tif" % (opt) for opt in ["-pa", "-pS"] ] stderr = [""] * 2 @@ -131,7 +131,7 @@ Exif.Image.YResolution Rational 1 72 Exif.Image.PlanarConfiguration Short 1 1 Exif.Image.ResolutionUnit Short 1 inch """, - """STRUCTURE OF TIFF FILE (II): {data_path}/mini9.tif + """STRUCTURE OF TIFF FILE (II): $data_path/mini9.tif address | tag | type | count | offset | value 254 | 0x00fe NewSubfileType | LONG | 1 | | 0 266 | 0x0100 ImageWidth | SHORT | 1 | | 9 @@ -150,5 +150,5 @@ Exif.Image.ResolutionUnit Short 1 inch 422 | 0x011b YResolution | RATIONAL | 1 | 518 | 1207959552/16777216 434 | 0x011c PlanarConfiguration | SHORT | 1 | | 1 446 | 0x0128 ResolutionUnit | SHORT | 1 | | 2 -END {data_path}/mini9.tif +END $data_path/mini9.tif """] diff --git a/tests/tiff_test/test_tiff_test_program.py b/tests/tiff_test/test_tiff_test_program.py index c9b31b11..a75523d6 100644 --- a/tests/tiff_test/test_tiff_test_program.py +++ b/tests/tiff_test/test_tiff_test_program.py @@ -3,10 +3,10 @@ import system_tests -@system_tests.CopyFiles("{data_path}/mini9.tif") -class TestTiffTestProg(system_tests.Case): +@system_tests.CopyFiles("$data_path/mini9.tif") +class TestTiffTestProg(metaclass=system_tests.CaseMeta): - commands = ["{tiff-test} {data_path}/mini9_copy.tif"] + commands = ["$tiff_test $data_path/mini9_copy.tif"] stdout = [ """Test 1: Writing empty Exif data without original binary data: ok. diff --git a/tests/writing_tests.md b/tests/writing_tests.md index e3e9cfe9..1f9a2377 100644 --- a/tests/writing_tests.md +++ b/tests/writing_tests.md @@ -11,13 +11,13 @@ The simplest test has the following structure: import system_tests -class GoodTestName(system_tests.Case): +class GoodTestName(metaclass=system_tests.CaseMeta): - filename = "{data_path}/test_file" - commands = ["{exiv2} " + filename, "{exiv2} " + filename + '_2'] + filename = "$data_path/test_file" + commands = ["$exiv2 $filename", "$exiv2 $filename" + '_2'] stdout = [""] * 2 - stderr = ["""{exiv2_exception_msg} """ + filename + """: -{error_58_message} + stderr = ["""$exiv2_exception_msg $filename: +$kerFailedToReadImageData """] * 2 retval = [1] * 2 ``` @@ -25,8 +25,8 @@ class GoodTestName(system_tests.Case): The test suite will run the provided commands in `commands` and compare them to the output in `stdout` and `stderr` and it will compare the return values. -The strings in curly braces are variables either defined in this test's class or -are taken from the suite's configuration file (see `doc.md` for a complete +The strings after a `$` are variables either defined in this test's class or are +taken from the suite's configuration file (see `doc.md` for a complete explanation). When creating new tests, follow roughly these steps: