1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 '''
23 Automatic Biskit module testing
24 ===============================
25
26 Module tests are extremely important to keep the Biskit package
27 manageable and functional. Every Biskit module should contain a class
28 derrived from L{BiskitTest} (conventionally called C{Test}) that
29 comprises one or more test functions. L{BiskitTestLoader} then
30 automatically extracts all BiskitTest child classes from the whole
31 package and bundles them into a L{FilteredTestSuite}.
32
33 Test cases that are L{LONG}, depend on L{PVM} or external programs
34 (L{EXE}) or are obsolete (L{OLD}) should be marked with the
35 appropriate tag(s) by overriding L{BiskitTest.TAGS} on the class
36 level. Test cases can then be filtered out or included into the
37 L{FilteredTestSuite} according to their TAGS.
38
39 Originally, testing code was simply in the __main__ section of each
40 module where we could execute it directly from emacs (or with python
41 -i) for interactive debugging. By comparison, unittest test fixtures
42 are less easy to execute stand-alone and intermediate variables remain
43 hidden within the test instance. The L{localTest}() removes this
44 hurdle and runs the test code of a single module as if it would be
45 executed directly in __main__. Simply putting the localTest() function
46 without parameters into the __main__ section of your module is
47 enough. Test.test_* methods should assign intermediate and final
48 results to self.|something| variables -- L{localTest} will then push
49 all self.* fields into the global namespace for interactive debugging.
50
51 Usage
52 =====
53
54 By way of example, a Test case for MyModule would look like this::
55
56 class MyClass:
57 ...
58
59 ### Module testing ###
60 import Biskit.test as BT
61
62 class Test(BT.BiskitTest):
63 """Test MyModule"""
64
65 TAGS = [ BT.LONG ]
66
67 def test_veryLongComputation( self ):
68 """MyModule.veryLongComputation test"""
69
70 self.m = MyClass()
71 self.result = self.m.veryLongComputation()
72
73 if self.local: ## only if the module is executed directly
74 print self.result
75
76 self.assertEqual( self.result, 42, 'unexpected result' )
77
78
79 if __name__ == '__main__':
80
81 ## run Test and push self.* fields into global namespace
82 BT.localTest( )
83
84 print result ## works thanks to some namespace magic in localTest
85
86
87 Note:
88 - If TAG is not given, the test will have the default NORMAL tag.
89 - Names of test functions **must** start with C{test}.
90 - The doc string of test_* will be reported as id of this test.
91 '''
92
93 import unittest as U
94 import glob
95 import os.path
96
97 import Biskit
98 from LogFile import StdLog, LogFile
99 import tools as T
100
101
102 NORMAL = 0
103 LONG = 1
104 PVM = 2
105 EXE = 3
106 OLD = 5
107
109 pass
110
112 """
113 Base class for Biskit test cases.
114
115 BiskitTest adds some functionality over the standard
116 C{unittest.TestCase}:
117
118 - self.local reflects whether the Test is performed in the
119 __main__ scope of the module it belongs to -- or as part of
120 the whole test suite.
121
122 - Each test case can be classified by assigning flags to its
123 static TAGS field -- this will be used by the test runner to
124 filter out, e.g. very long tests or tests that require PVM.
125
126 - self.log should capture all the output, by default it goes to
127 the TESTLOG instance defined at the module level.
128
129 Usage:
130 ======
131
132 Biskit test cases should be created by sub-classing BiskitTest
133 and by adding one or more test_* methods (each method starting
134 with 'test_' is treated as a separate test). The doc string of a
135 test_* method becomes the id reported by the TextTestRunner.
136
137 L{prepare} should be overriden for the definition of permanent
138 input and temporary output files. L{cleanUp} should be overriden
139 for clean up actions and to remove temporary files (use
140 L{Biskit.tools.tryRemove}). L{cleanUp} is not called if
141 BiskitTest is set into debugging mode (see L{BiskitTest.DEBUG}).
142
143 The L{TAGS} field should be overriden to reflect any special categories
144 of the test case (L{LONG}, L{PVM}, L{EXE} or L{OLD}).
145 """
146
147 TAGS = [ NORMAL ]
148
149
150 DEBUG = False
151
152
153 TESTLOG = StdLog()
154
155
156 VERBOSITY= 2
157
159 """Hook for test setup."""
160 pass
161
163 """Hook for post-test clean up; skipped if DEBUG==True."""
164 pass
165
172
176
177
179 """
180 Collection of BiskitTests filtered by category tags.
181 FilteredTestSuite silently ignores Test cases that are either
182
183 * classified into any of the forbidden categories
184
185 * not classified into any of the allowed categories
186
187 By default (if initialized without parameters), all groups are allowed
188 and no groups are forbidden.
189 """
190
191 - def __init__( self, tests=(), allowed=[], forbidden=[] ):
192 """
193 @param tests: iterable of TestCases
194 @type tests: ( BiskitTest, )
195 @param allowed: list of allowed tags
196 @type allowed: [ int ]
197 @param forbidden : list of forbidden tags
198 @type forbidden : [ int ]
199 """
200 self._allowed = allowed
201 self._forbidden = forbidden
202 U.TestSuite.__init__( self, tests=tests )
203
204
206 """
207 Add Biskit test case if it is matching the allowed and disallowed
208 groups.
209 @param test: test case
210 @type test: BiskitTest
211 """
212 assert isinstance( test, Biskit.test.BiskitTest ), \
213 'FilteredTestSuite only accepts BiskitTest instances not %r' \
214 % test
215
216 matches = [ g for g in test.TAGS if g in self._forbidden ]
217 if len( matches ) > 0:
218 return
219
220 matches = [ g for g in test.TAGS if g in self._allowed ]
221
222 if not self._allowed or (self._allowed and len(matches) > 0):
223 U.TestSuite.addTest( self, test )
224
225
226 -class Flushing_TextTestResult( U._TextTestResult ):
227 """
228 Helper class for (Flushing)TextTestRunner.
229 Customize _TextTestResult so that the reported test id is flushed
230 B{before} the test starts. Otherwise the 'sometest.id ...' is only
231 printed together with the '...ok' after the test is finished.
232 """
233
234 - def startTest(self, test):
235 """print id at start of test... and flush it"""
236 super( self.__class__, self ).startTest( test )
237 self.stream.flush()
238
239 -class FlushingTextTestRunner( U.TextTestRunner ):
240 """
241 Convince TextTestRunner to use the flushing text output rather
242 than the default one.
243 """
244
245 - def _makeResult(self):
246 return Flushing_TextTestResult(self.stream, self.descriptions,
247 self.verbosity)
248
249
251 """
252 A replacement for the unittest TestLoaders. It automatically
253 collects all BiskitTests from a whole package (that means a
254 folder with python files).
255 """
256
257 - def __init__( self, log=StdLog(),
258 allowed=[], forbidden=[], verbosity=2, debug=False ):
259 """
260 @param log: log output target [default: L{Biskit.StdLog}]
261 @type log: Biskit.LogFile
262 @param allowed: tags required for test to be considered, default: []
263 @type allowed: [ int ]
264 @param forbidden: tags leading to the exclusion of test, default: []
265 @type forbidden: [ int ]
266 @param verbosity: verbosity level for unittest.TextTestRunner
267 @type verbosity: int
268 """
269
270 self.allowed = allowed
271 self.forbidden= forbidden
272 self.log = log
273 self.verbosity = verbosity
274 self.debugging = debug
275 self.suite = FilteredTestSuite( allowed=allowed, forbidden=forbidden )
276 self.modules_untested = []
277 self.modules_tested = []
278 self.result = U.TestResult()
279
280
282 """
283 Import all python files of a package as modules. Sub-packages
284 are ignored and have to be collected separately.
285
286 @param path: single search path for a package
287 @type path: str
288 @param module: name of the python package [default: Biskit]
289 @type module: str
290
291 @return: list of imported python modules, see also __import__
292 @rtype : [ module ]
293
294 @raise ImportError: if a python file cannot be imported
295 """
296 module_folder = module.replace('.', os.path.sep)
297
298 files = glob.glob( os.path.join( path, module_folder,'*.py' ) )
299
300 files = map( T.stripFilename, files )
301 files = [ f for f in files if f[0] != '_' ]
302
303 r = []
304
305 for f in files:
306 try:
307 r += [ __import__( '.'.join([module, f]), globals(),
308 None, [module]) ]
309 except:
310 pass
311
312 return r
313
314
316 """
317 Extract all test cases from a list of python modules and add them to
318 the internal test suite.
319 @param modules: list of modules to be checked for BiskitTest classes
320 @type modules: [ module ]
321 """
322 for m in modules:
323 tested = 0
324
325 for i in m.__dict__.values():
326
327 if type(i) is type and \
328 issubclass( i, Biskit.test.BiskitTest ) and \
329 i.__name__ != 'BiskitTest':
330
331 suite = U.defaultTestLoader.loadTestsFromTestCase( i )
332 self.suite.addTests( suite )
333 tested = 1
334
335 if tested:
336 self.modules_tested += [m]
337 else:
338 self.modules_untested += [m]
339
340
342 """
343 Add all BiskitTests found in a given module to the internal test suite.
344 @param path: single search path for a package
345 @type path: str
346 @param module: name of the python package
347 @type module: str
348 """
349 modules = self.modulesFromPath( path=path, module=module )
350 self.addTestsFromModules( modules )
351
352
354
355 modules = [ m.__name__ for m in modules ]
356 modules = [ m.replace('.',' . ') for m in modules ]
357
358 return modules
359
360
362 """
363 Report how things went to stdout.
364 """
365 print '\nThe test log file has been saved to: %r'% self.log.fname
366 total = self.result.testsRun
367 failed = len(self.result.failures) + len(self.result.errors)
368
369 m_tested = len( self.modules_tested )
370 m_untested= len( self.modules_untested)
371 m_total = m_tested + m_untested
372
373
374 print '\nTest Coverage:\n=============\n'
375 print '%i out of %i modules had no test case:' % (m_untested,m_total)
376 for m in self.__moduleNames( self.modules_untested) :
377 print '\t', m
378
379
380 print '\nSUMMARY:\n=======\n'
381 print 'A total of %i tests from %i modules were run.' %(total,m_tested)
382 print ' - %i passed'% (total - failed)
383 print ' - %i failed'% failed
384
385
386 if failed:
387
388 for test, ftrace in self.result.failures:
389 print ' - failed: %s'% test.id()
390
391 for test, ftrace in self.result.errors:
392 print ' - error : %s'% test.id()
393
394
395 - def run( self, dry=False ):
396 """
397 @param dry: do not actually run the test but just set it up [False]
398 @type dry: bool
399 """
400
401 for testclass in self.suite:
402 testclass.DEBUG = self.debugging
403 testclass.VERBOSITY = self.verbosity
404 testclass.TESTLOG = self.log
405
406 runner = FlushingTextTestRunner(self.log.f(), verbosity=self.verbosity)
407 if not dry:
408 self.result = runner.run( self.suite )
409
410
411
412
413
415 """
416 Fetch the namespace of the module/script running as __main__.
417 @return: the namespace of the outermost calling stack frame
418 @rtype: dict
419 """
420 import inspect
421
422 try:
423 frames = inspect.stack()
424 f = frames[-1][0]
425 r = f.f_globals
426 finally:
427 del frames, f
428
429 return r
430
431
433 """
434 @return: all BisktTest child classes found in given namespace
435 @rtype: [ class ]
436 @raise BiskitTestError: if there is no BiskitTest child
437 """
438 r =[]
439
440 for i in namespace.values():
441
442 if type(i) is type \
443 and issubclass( i, Biskit.test.BiskitTest )\
444 and i.__name__ != 'BiskitTest':
445
446 r += [i]
447
448 if not r:
449 raise BiskitTestError, 'no BiskitTest class found in namespace'
450
451 return r
452
453
456 """
457 Perform the BiskitTest(s) found in the scope of the calling module.
458 After the test run, all fields of the BiskitTest instance are
459 pushed into the global namespace so that they can be inspected in the
460 interactive interpreter. The BiskitTest instance itself is also
461 put into the calling namespace as variable 'self', so that test code
462 fragments referring to it can be executed interactively.
463
464 @param testclass: BiskitTest-derived class [default: first one found]
465 @type testclass: class
466 @param verbosity: verbosity level of TextTestRunner
467 @type verbosity: int
468 @param debug: don't delete temporary files (skipp cleanUp) [0]
469 @type debug: int
470
471 @return: the test result object
472 @rtype: unittest.TestResult
473
474 @raise BiskitTestError: if there is no BiskitTest-derived class defined
475 """
476
477 outer = getOuterNamespace()
478 if testclass:
479 testclasses = [testclass]
480 else:
481 testclasses = extractTestCases( outer )
482
483 suite = U.TestSuite()
484 for test in testclasses:
485 suite.addTests( U.TestLoader().loadTestsFromTestCase( test ) )
486
487 for test in suite:
488 test.DEBUG = debug
489 test.VERBOSITY = verbosity
490 test.TESTLOG = log
491
492 runner= U.TextTestRunner(verbosity=verbosity)
493 r = runner.run( suite )
494
495 for t in suite._tests:
496 outer.update( t.__dict__ )
497 outer.update( {'self':t })
498
499 return r
500
501
502
503
504 -class Test(BiskitTest):
505 """Mock test, test doesn't test itself"""
506 pass
507
508
509
510
511 -def _use( defaults ):
512 print """
513 Run unittest tests for biskit.
514
515 test.py [-i |include tag1 tag2..| -e |exclude tag1 tag2..|
516 -p |package1 package2..|
517 -v |verbosity| -log |log-file| -nox ]
518
519 i - include tags, only run tests with at least one of these tags [All]
520 e - exclude tags, do not run tests labeled with one of these tags [old]
521 valid tags are:
522 long - long running test case
523 pvm - depends on PVM
524 exe - depends on external application
525 old - is obsolete
526 (If no tags are given to -i this means all tests are included)
527
528 p - packages to test, e.g. Biskit Biskit.Dock Biskit.Mod [All]
529 v - int, verbosity level, 3 switches on several graphical plots [2]
530 log - path to logfile (overriden); empty -log means STDOUT [STDOUT]
531 nox - suppress test plots [False]
532 dry - do not actually run the test but just collect tests [False]
533
534 Examples:
535
536 * Run all but long or obsolete tests from Biskit and Biskit.Dock:
537 test.py -e old long -p Biskit Biskit.Dock
538
539 * Run only PVM-dependent tests of the Biskit.Mod sub-package:
540 test.py -i pvm -p Biskit.Mod
541
542
543 Default options:
544 """
545 for key, value in defaults.items():
546 print "\t-",key, "\t",value
547
548 sys.exit(0)
549
550
560
573
574 if __name__ == '__main__':
575
576 from Biskit import EHandler
577 import sys
578
579
580 defaults = {'i':'',
581 'e':'old',
582 'p':['Biskit', 'Biskit.Dock', 'Biskit.Mod', 'Biskit.PVM',
583 'Biskit.Statistics'],
584 'v':'2',
585 'log': '',
586 }
587
588 o = T.cmdDict( defaults )
589
590 if len( sys.argv ) == 1 and 'test.py' in sys.argv[0]:
591 _use( defaults )
592
593 _convertOptions( o )
594
595 BiskitTest.VERBOSITY = o['v']
596 BiskitTest.DEBUG = o['debug']
597
598 l = BiskitTestLoader( allowed=o['i'], forbidden=o['e'],
599 verbosity=o['v'], log=o['log'], debug=o['debug'])
600
601
602 for package in o['p']:
603 print 'collecting ', repr( package )
604 l.collectTests( module=package )
605
606 l.run( dry=o['dry'] )
607 l.report()
608
609 print "DONE"
610