harness: Handle assertion failure case for compiler-rt tests which default to pass
[barrelfish] / tools / harness / tests / compiler-rt_builtins.py
1 ##########################################################################
2 # Copyright (c) 2009, ETH Zurich.
3 # All rights reserved.
4 #
5 # This file is distributed under the terms in the attached LICENSE file.
6 # If you do not find this file, copies can be found by writing to:
7 # ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
8 ##########################################################################
9
10 import tests
11 from common import TestCommon
12 from results import PassFailMultiResult
13
14 class CompilerRTBuiltinsAbstract(TestCommon):
15
16     def get_finish_string(self):
17         return "usleeptest_done"
18
19     def process_data(self, testdir, rawiter):
20         # the test passed if no error occurred
21         errors = []
22         for line in rawiter:
23             if "error in" in line:
24                 errors.append(line)
25             if line.startswith("Assertion failed on core"):
26                 errors.append(line)
27
28         return PassFailMultiResult(self.name, errors)
29
30 # list of tests to run for compiler-rt
31 all_tests = [
32         "$BUILD/compiler-rt/test/builtins/Unit/absvdi2_test",
33         "$BUILD/compiler-rt/test/builtins/Unit/absvsi2_test",
34         "$BUILD/compiler-rt/test/builtins/Unit/absvti2_test",
35         "$BUILD/compiler-rt/test/builtins/Unit/adddf3vfp_test",
36         "$BUILD/compiler-rt/test/builtins/Unit/addsf3vfp_test",
37         "$BUILD/compiler-rt/test/builtins/Unit/addtf3_test",
38         "$BUILD/compiler-rt/test/builtins/Unit/addvdi3_test",
39         "$BUILD/compiler-rt/test/builtins/Unit/addvsi3_test",
40         "$BUILD/compiler-rt/test/builtins/Unit/addvti3_test",
41         "$BUILD/compiler-rt/test/builtins/Unit/ashldi3_test",
42         "$BUILD/compiler-rt/test/builtins/Unit/ashlti3_test",
43         "$BUILD/compiler-rt/test/builtins/Unit/ashrdi3_test",
44         "$BUILD/compiler-rt/test/builtins/Unit/ashrti3_test",
45         "$BUILD/compiler-rt/test/builtins/Unit/bswapdi2_test",
46         "$BUILD/compiler-rt/test/builtins/Unit/bswapsi2_test",
47 #       "$BUILD/compiler-rt/test/builtins/Unit/clear_cache_test",
48         "$BUILD/compiler-rt/test/builtins/Unit/clzdi2_test",
49         "$BUILD/compiler-rt/test/builtins/Unit/clzsi2_test",
50         "$BUILD/compiler-rt/test/builtins/Unit/clzti2_test",
51         "$BUILD/compiler-rt/test/builtins/Unit/cmpdi2_test",
52         "$BUILD/compiler-rt/test/builtins/Unit/cmpti2_test",
53         "$BUILD/compiler-rt/test/builtins/Unit/comparedf2_test",
54         "$BUILD/compiler-rt/test/builtins/Unit/comparesf2_test",
55         "$BUILD/compiler-rt/test/builtins/Unit/ctzdi2_test",
56         "$BUILD/compiler-rt/test/builtins/Unit/ctzsi2_test",
57         "$BUILD/compiler-rt/test/builtins/Unit/ctzti2_test",
58         "$BUILD/compiler-rt/test/builtins/Unit/divdc3_test",
59         "$BUILD/compiler-rt/test/builtins/Unit/divdf3vfp_test",
60         "$BUILD/compiler-rt/test/builtins/Unit/divdi3_test",
61         "$BUILD/compiler-rt/test/builtins/Unit/divmodsi4_test",
62         "$BUILD/compiler-rt/test/builtins/Unit/divsc3_test",
63         "$BUILD/compiler-rt/test/builtins/Unit/divsf3vfp_test",
64         "$BUILD/compiler-rt/test/builtins/Unit/divsi3_test",
65 #       "$BUILD/compiler-rt/test/builtins/Unit/divtc3_test",
66         "$BUILD/compiler-rt/test/builtins/Unit/divtf3_test",
67         "$BUILD/compiler-rt/test/builtins/Unit/divti3_test",
68         "$BUILD/compiler-rt/test/builtins/Unit/divxc3_test",
69 #       "$BUILD/compiler-rt/test/builtins/Unit/enable_execute_stack_test",
70         "$BUILD/compiler-rt/test/builtins/Unit/eqdf2vfp_test",
71         "$BUILD/compiler-rt/test/builtins/Unit/eqsf2vfp_test",
72         "$BUILD/compiler-rt/test/builtins/Unit/eqtf2_test",
73         "$BUILD/compiler-rt/test/builtins/Unit/extebdsfdf2vfp_test",
74         "$BUILD/compiler-rt/test/builtins/Unit/extenddftf2_test",
75 #       "$BUILD/compiler-rt/test/builtins/Unit/extendhfsf2_test",
76         "$BUILD/compiler-rt/test/builtins/Unit/extendsftf2_test",
77         "$BUILD/compiler-rt/test/builtins/Unit/ffsdi2_test",
78         "$BUILD/compiler-rt/test/builtins/Unit/ffsti2_test",
79         "$BUILD/compiler-rt/test/builtins/Unit/fixdfdi_test",
80         "$BUILD/compiler-rt/test/builtins/Unit/fixdfsivfp_test",
81         "$BUILD/compiler-rt/test/builtins/Unit/fixdfti_test",
82         "$BUILD/compiler-rt/test/builtins/Unit/fixsfdi_test",
83         "$BUILD/compiler-rt/test/builtins/Unit/fixsfsivfp_test",
84         "$BUILD/compiler-rt/test/builtins/Unit/fixsfti_test",
85         "$BUILD/compiler-rt/test/builtins/Unit/fixtfdi_test",
86         "$BUILD/compiler-rt/test/builtins/Unit/fixtfsi_test",
87         "$BUILD/compiler-rt/test/builtins/Unit/fixtfti_test",
88         # this errors
89         "$BUILD/compiler-rt/test/builtins/Unit/fixunsdfdi_test",
90         "$BUILD/compiler-rt/test/builtins/Unit/fixunsdfsi_test",
91         "$BUILD/compiler-rt/test/builtins/Unit/fixunsdfsivfp_test",
92         "$BUILD/compiler-rt/test/builtins/Unit/fixunsdfti_test",
93         # this errors
94         "$BUILD/compiler-rt/test/builtins/Unit/fixunssfdi_test",
95         "$BUILD/compiler-rt/test/builtins/Unit/fixunssfsi_test",
96         "$BUILD/compiler-rt/test/builtins/Unit/fixunssfsivfp_test",
97         "$BUILD/compiler-rt/test/builtins/Unit/fixunssfti_test",
98         "$BUILD/compiler-rt/test/builtins/Unit/fixunstfdi_test",
99         "$BUILD/compiler-rt/test/builtins/Unit/fixunstfsi_test",
100         "$BUILD/compiler-rt/test/builtins/Unit/fixunstfti_test",
101         "$BUILD/compiler-rt/test/builtins/Unit/fixunsxfdi_test",
102         "$BUILD/compiler-rt/test/builtins/Unit/fixunsxfsi_test",
103         "$BUILD/compiler-rt/test/builtins/Unit/fixunsxfti_test",
104         "$BUILD/compiler-rt/test/builtins/Unit/fixxfdi_test",
105         "$BUILD/compiler-rt/test/builtins/Unit/fixxfti_test",
106         "$BUILD/compiler-rt/test/builtins/Unit/floatdidf_test",
107         "$BUILD/compiler-rt/test/builtins/Unit/floatdisf_test",
108         "$BUILD/compiler-rt/test/builtins/Unit/floatditf_test",
109         "$BUILD/compiler-rt/test/builtins/Unit/floatdixf_test",
110         "$BUILD/compiler-rt/test/builtins/Unit/floatsidfvfp_test",
111         "$BUILD/compiler-rt/test/builtins/Unit/floatsisfvfp_test",
112         "$BUILD/compiler-rt/test/builtins/Unit/floatsitf_test",
113         "$BUILD/compiler-rt/test/builtins/Unit/floattidf_test",
114         "$BUILD/compiler-rt/test/builtins/Unit/floattisf_test",
115         "$BUILD/compiler-rt/test/builtins/Unit/floattixf_test",
116         "$BUILD/compiler-rt/test/builtins/Unit/floatundidf_test",
117         "$BUILD/compiler-rt/test/builtins/Unit/floatundisf_test",
118         "$BUILD/compiler-rt/test/builtins/Unit/floatunditf_test",
119         "$BUILD/compiler-rt/test/builtins/Unit/floatundixf_test",
120         "$BUILD/compiler-rt/test/builtins/Unit/floatunsitf_test",
121         "$BUILD/compiler-rt/test/builtins/Unit/floatunssidfvfp_test",
122         "$BUILD/compiler-rt/test/builtins/Unit/floatunssisfvfp_test",
123         "$BUILD/compiler-rt/test/builtins/Unit/floatuntidf_test",
124         "$BUILD/compiler-rt/test/builtins/Unit/floatuntisf_test",
125         "$BUILD/compiler-rt/test/builtins/Unit/floatuntixf_test",
126 #       "$BUILD/compiler-rt/test/builtins/Unit/gcc_personality_test",
127         "$BUILD/compiler-rt/test/builtins/Unit/gedf2vfp_test",
128         "$BUILD/compiler-rt/test/builtins/Unit/gesf2vfp_test",
129         "$BUILD/compiler-rt/test/builtins/Unit/getf2_test",
130         "$BUILD/compiler-rt/test/builtins/Unit/gtdf2vfp_test",
131         "$BUILD/compiler-rt/test/builtins/Unit/gtsf2vfp_test",
132         "$BUILD/compiler-rt/test/builtins/Unit/gttf2_test",
133         "$BUILD/compiler-rt/test/builtins/Unit/ledf2vfp_test",
134         "$BUILD/compiler-rt/test/builtins/Unit/lesf2vfp_test",
135         "$BUILD/compiler-rt/test/builtins/Unit/letf2_test",
136         "$BUILD/compiler-rt/test/builtins/Unit/lshrdi3_test",
137         "$BUILD/compiler-rt/test/builtins/Unit/lshrti3_test",
138         "$BUILD/compiler-rt/test/builtins/Unit/ltdf2vfp_test",
139         "$BUILD/compiler-rt/test/builtins/Unit/ltsf2vfp_test",
140         "$BUILD/compiler-rt/test/builtins/Unit/lttf2_test",
141         "$BUILD/compiler-rt/test/builtins/Unit/moddi3_test",
142         "$BUILD/compiler-rt/test/builtins/Unit/modsi3_test",
143         "$BUILD/compiler-rt/test/builtins/Unit/modti3_test",
144         "$BUILD/compiler-rt/test/builtins/Unit/muldc3_test",
145         "$BUILD/compiler-rt/test/builtins/Unit/muldf3vfp_test",
146         "$BUILD/compiler-rt/test/builtins/Unit/muldi3_test",
147         "$BUILD/compiler-rt/test/builtins/Unit/mulodi4_test",
148         "$BUILD/compiler-rt/test/builtins/Unit/mulosi4_test",
149         "$BUILD/compiler-rt/test/builtins/Unit/muloti4_test",
150         "$BUILD/compiler-rt/test/builtins/Unit/mulsc3_test",
151         "$BUILD/compiler-rt/test/builtins/Unit/mulsf3vfp_test",
152         "$BUILD/compiler-rt/test/builtins/Unit/multc3_test",
153         "$BUILD/compiler-rt/test/builtins/Unit/multf3_test",
154         "$BUILD/compiler-rt/test/builtins/Unit/multi3_test",
155         "$BUILD/compiler-rt/test/builtins/Unit/mulvdi3_test",
156         "$BUILD/compiler-rt/test/builtins/Unit/mulvsi3_test",
157         "$BUILD/compiler-rt/test/builtins/Unit/mulvti3_test",
158         "$BUILD/compiler-rt/test/builtins/Unit/mulxc3_test",
159         "$BUILD/compiler-rt/test/builtins/Unit/nedf2vfp_test",
160         "$BUILD/compiler-rt/test/builtins/Unit/negdf2vfp_test",
161         "$BUILD/compiler-rt/test/builtins/Unit/negdi2_test",
162         "$BUILD/compiler-rt/test/builtins/Unit/negsf2vfp_test",
163         "$BUILD/compiler-rt/test/builtins/Unit/negti2_test",
164         "$BUILD/compiler-rt/test/builtins/Unit/negvdi2_test",
165         "$BUILD/compiler-rt/test/builtins/Unit/negvsi2_test",
166         "$BUILD/compiler-rt/test/builtins/Unit/negvti2_test",
167         "$BUILD/compiler-rt/test/builtins/Unit/nesf2vfp_test",
168         "$BUILD/compiler-rt/test/builtins/Unit/netf2_test",
169         "$BUILD/compiler-rt/test/builtins/Unit/paritydi2_test",
170         "$BUILD/compiler-rt/test/builtins/Unit/paritysi2_test",
171         "$BUILD/compiler-rt/test/builtins/Unit/parityti2_test",
172         "$BUILD/compiler-rt/test/builtins/Unit/popcountdi2_test",
173         "$BUILD/compiler-rt/test/builtins/Unit/popcountsi2_test",
174         "$BUILD/compiler-rt/test/builtins/Unit/popcountti2_test",
175         "$BUILD/compiler-rt/test/builtins/Unit/powidf2_test",
176         "$BUILD/compiler-rt/test/builtins/Unit/powisf2_test",
177         "$BUILD/compiler-rt/test/builtins/Unit/powitf2_test",
178         "$BUILD/compiler-rt/test/builtins/Unit/powixf2_test",
179         "$BUILD/compiler-rt/test/builtins/Unit/subdf3vfp_test",
180         "$BUILD/compiler-rt/test/builtins/Unit/subsf3vfp_test",
181         "$BUILD/compiler-rt/test/builtins/Unit/subtf3_test",
182         "$BUILD/compiler-rt/test/builtins/Unit/subvdi3_test",
183         "$BUILD/compiler-rt/test/builtins/Unit/subvsi3_test",
184         "$BUILD/compiler-rt/test/builtins/Unit/subvti3_test",
185 #       "$BUILD/compiler-rt/test/builtins/Unit/trampoline_setup_test",
186 #       "$BUILD/compiler-rt/test/builtins/Unit/truncdfhf2_test",
187         "$BUILD/compiler-rt/test/builtins/Unit/truncdfsf2_test",
188         "$BUILD/compiler-rt/test/builtins/Unit/truncdfsf2vfp_test",
189 #       "$BUILD/compiler-rt/test/builtins/Unit/truncsfhf2_test",
190         "$BUILD/compiler-rt/test/builtins/Unit/trunctfdf2_test",
191         "$BUILD/compiler-rt/test/builtins/Unit/trunctfsf2_test",
192         "$BUILD/compiler-rt/test/builtins/Unit/ucmpdi2_test",
193         "$BUILD/compiler-rt/test/builtins/Unit/ucmpti2_test",
194         "$BUILD/compiler-rt/test/builtins/Unit/udivdi3_test",
195         "$BUILD/compiler-rt/test/builtins/Unit/udivmoddi4_test",
196         "$BUILD/compiler-rt/test/builtins/Unit/udivmodsi4_test",
197         "$BUILD/compiler-rt/test/builtins/Unit/udivmodti4_test",
198         "$BUILD/compiler-rt/test/builtins/Unit/udivsi3_test",
199         "$BUILD/compiler-rt/test/builtins/Unit/udivti3_test",
200         "$BUILD/compiler-rt/test/builtins/Unit/umoddi3_test",
201         "$BUILD/compiler-rt/test/builtins/Unit/umodsi3_test",
202         "$BUILD/compiler-rt/test/builtins/Unit/umodti3_test",
203         "$BUILD/compiler-rt/test/builtins/Unit/unorddf2vfp_test",
204         "$BUILD/compiler-rt/test/builtins/Unit/unordsf2vfp_test",
205         "$BUILD/compiler-rt/test/builtins/Unit/unordtf2_test",
206 ]
207
208 def get_modules_tpl(ts, self, build, machine):
209     '''Function template for get_modules() for each compiler-rt test case'''
210     modules = super(CompilerRTBuiltinsAbstract, self).get_modules(build, machine)
211     for m in ts:
212         modules.add_module(m)
213     modules.add_module("usleeptest", [ "5" ])
214     return modules
215
216 def chunker(seq, size):
217     '''Helper function: this takes a sequence `seq` and splits it up into
218     `size`-sized chunks, except for the last chunk which is just the <= size
219     long remainder of the sequence'''
220     return (seq[pos:pos+size] for pos in xrange(0, len(seq), size))
221
222 # generate test-cases with <=CHUNK_SIZE compiler-rt tests each
223 CHUNK_SIZE=35
224 # array just to keep the class objects somewhere
225 compiler_rt_tests_classes = []
226 for i, ts in enumerate(chunker(all_tests, CHUNK_SIZE)):
227     # append new class to our array
228     compiler_rt_tests_classes.append(
229         # this is essentially the decorator @tests.add_test
230         tests.add_test(
231             # type is the (built-in) base-class for python classes, here we
232             # construct classes by calling its constructor
233             # signature of type constructor:
234             #   type(classname, baseclass tuple, dict with methods/attributes)
235             type('CompilerRTBuiltins%d' % (i+1),
236                  (CompilerRTBuiltinsAbstract,),
237                  { 'name': 'compiler-rt%d' % (i+1),
238                     # partially bind the get_modules() template to select the
239                     # right set of tests. Note the ts=ts in the lambda
240                     # arguments, this prevents python's default late-binding
241                     # for closure arguments.
242                      'get_modules':
243                          lambda s, b, m, ts=ts: get_modules_tpl(ts, s, b, m)})))