Source Code for Module Biskit.Mod.AlignerMaster

  1  ## 
  2  ## Biskit, a toolkit for the manipulation of macromolecular structures 
  3  ## Copyright (C) 2004-2006 Raik Gruenberg & Johan Leckner 
  4  ## 
  5  ## This program is free software; you can redistribute it and/or 
  6  ## modify it under the terms of the GNU General Public License as 
  7  ## published by the Free Software Foundation; either version 2 of the 
  8  ## License, or any later version. 
  9  ## 
 10  ## This program is distributed in the hope that it will be useful, 
 11  ## but WITHOUT ANY WARRANTY; without even the implied warranty of 
 12  ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 
 13  ## General Public License for more details. 
 14  ## 
 15  ## You find a copy of the GNU General Public License in the file 
 16  ## license.txt along with this program; if not, write to the Free 
 17  ## Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 
 18   
 19  ## Contributions: Olivier PERIN, Raik Gruenberg 
 20  ## last $Author: graik $ 
 21  ## last $Date: 2007/03/02 17:18:52 $ 
 22  ## $Revision: 2.11 $ 
 23  """ 
 24  Parallelise Sequence Alignment 
 25  """ 
 26   
 27  from Biskit.PVM.TrackingJobMaster import TrackingJobMaster 
 28   
 29  import Biskit.hosts as hosts 
 30  from Biskit.tools import projectRoot 
 31  import Biskit.tools as T 
 32  import os, glob 
 33   
 34 -class AlignerMaster(TrackingJobMaster): 
 35   
 36      slave_script =  projectRoot() + '/Biskit/Mod/AlignerSlave.py' 
 37   
 38 -    def __init__(self, hosts, folders, pdbFolder=None, fastaTemplates=None, 
 39                   fastaSequences=None, fastaTarget=None, ferror=None, 
 40                   verbose=1, **kw): 
 41          """ 
 42          @param hosts: list of host-names 
 43          @type  hosts: [str] 
 44          @param folders: list of project directories (path) 
 45          @type  folders: [str] 
 46          @param pdbFolder: pdb directory (*.alpha for Aligner) 
 47          @type  pdbFolder: str 
 48          @param fastaTemplates: path to find 'templates.fasta' 
 49          @type  fastaTemplates: str 
 50          @param fastaSequences: path to find 'nr.fasta' 
 51          @type  fastaSequences: str 
 52          @param fastaTarget: path to find 'target.fasta' 
 53          @type  fastaTarget: str 
 54          @param ferror: filename to output errors from the Slave 
 55          @type  ferror: str 
 56          @param verbose: verbosity level (default: 1) 
 57          @type  verbose: 1|0  
 58          @param kw: additional TrackingJobMaster arguments:: 
 59              chunk_size   - int, number of items that are processed per job 
 60              niceness     - {str_host-name: int_niceness} 
 61              slave_script - str, absolute path to slave-script 
 62              show_output  - 1|0, display one xterm per slave [0] 
 63              add_hosts    - 1|0, add hosts to PVM before starting [1] 
 64          @type kw: param=value 
 65          """ 
 66          self.verbose = verbose 
 67          self.folders = folders 
 68          self.pdbFolder = pdbFolder  
 69          self.fastaTemplates = fastaTemplates  
 70          self.fastaSequences = fastaSequences  
 71          self.fastaTarget = fastaTarget 
 72          self.ferror = ferror or 'AlignSlaveErrors.out' 
 73   
 74          data = self.setupJobs() 
 75   
 76          TrackingJobMaster.__init__(self, data=data, chunk_size=1, 
 77                                     hosts=hosts, 
 78                                     slave_script=self.slave_script, 
 79                                     redistribute=0, 
 80                                     verbose=verbose, **kw) 
 81   
 82   
 83 -    def __dir_or_none( self, folder, filename ): 
 84          if filename is None: 
 85              return None 
 86          return os.path.join( folder, filename ) 
 87   
 88   
 89 -    def setupJobs(self): 
 90          """ 
 91          Prepare the job dictionnary for 'AlignerSlave' 
 92           
 93          @return: input informations for aligner for each project 
 94          @rtype: {{str}} 
 95          """ 
 96          r = {} 
 97   
 98          for f in self.folders: 
 99              aligner_input = {} 
100              aligner_input['outFolder'] = T.absfile(f) 
101              aligner_input['fastaTemplates'] = \ 
102                                    self.__dir_or_none( f, self.fastaTemplates ) 
103              aligner_input['fastaSequences'] = \ 
104                                    self.__dir_or_none( f, self.fastaSequences ) 
105              aligner_input['fastaTarget'] = \ 
106                                    self.__dir_or_none( f, self.fastaTarget ) 
107   
108              pdb_list = [] 
109              if self.pdbFolder is None: 
110                  pdb_list = None 
111              else: 
112                  pdbfiles = os.listdir(f + self.pdbFolder) 
113                  for pdb in pdbfiles: 
114                      pdb_list.append(f + self.pdbFolder + '/%s'%pdb) 
115   
116              aligner_input['pdbFiles'] = pdb_list 
117   
118              r[T.absfile(f)] = aligner_input 
119   
120          return r 
121   
122   
123 -    def getInitParameters(self, slave_tid): 
124          """ 
125          Hand over parameters to slave once. 
126   
127          @param slave_tid: slave task id 
128          @type  slave_tid: int 
129   
130          @return: dictionary with init parameters 
131          @rtype: {param:value}         
132          """ 
133          return {'progress_str':'slave calculating..', 
134                  'ferror':self.ferror, 'os.environ':os.environ } 
135   
136   
137 -    def cleanup( self ): 
138          print "Cleaning up..." 
139   
140   
141 -    def done( self ): 
142          print "Done aligning." 
143   
144   
145   
146  ############# 
147  ##  TESTING         
148  ############# 
149  import Biskit.test as BT 
150   
151 -class FullProjectTest(BT.BiskitTest): 
152      """ 
153      Performs full cross-validation alignments on the test project. 
154      Requires files that are not checked in but can be generated 
155      from the sequence in test/Mod/project. 
156      """ 
157   
158      TAGS = [ BT.LONG, BT.PVM, BT.EXE ] 
159   
160      ## rename to test_fullProject to perform this test 
161 -    def t_fullProject( self ): 
162          """ 
163          Mod.AlignerMaster full project test 
164          """ 
165          ## a full test case that demands a valid testRoot project 
166          projRoot =  T.testRoot()+'/Mod/project' 
167          self.projects = glob.glob( projRoot + '/validation/*' ) 
168   
169          self.master = AlignerMaster(folders = self.projects, 
170                                 ferror = projRoot+'/AlignErrors.out', 
171                                 hosts = hosts.cpus_all[ : 10 ], 
172                                 show_output = self.local) 
173   
174          self.r = self.master.calculateResult() 
175   
176   
177 -class TestBase( BT.BiskitTest ): 
178   
179 -    def prepare(self): 
180          import tempfile 
181          import shutil 
182   
183          ## collect the input files needed 
184          self.outfolder = tempfile.mkdtemp( '_test_AlignerMaster' ) 
185          os.mkdir( self.outfolder +'/templates' ) 
186          os.mkdir( self.outfolder +'/sequences' ) 
187   
188          shutil.copytree( T.testRoot() + '/Mod/project/templates/t_coffee', 
189                           self.outfolder + '/templates/t_coffee' ) 
190   
191          shutil.copy( T.testRoot() + '/Mod/project/templates/templates.fasta', 
192                       self.outfolder + '/templates' ) 
193   
194          shutil.copy( T.testRoot() + '/Mod/project/sequences/nr.fasta', 
195                       self.outfolder + '/sequences/' ) 
196   
197          shutil.copy( T.testRoot() + '/Mod/project/target.fasta', 
198                       self.outfolder  ) 
199   
200   
201 -    def t_AlignerMaster(self, run=True): 
202   
203          nodes = hosts.cpus_all[ : 5 ] 
204           
205          self.master = AlignerMaster( folders=[self.outfolder], 
206                                       ferror=self.outfolder+'/AlignErrors.out', 
207                                       hosts=nodes, 
208                                       show_output=self.local, 
209                                       verbose=self.local ) 
210   
211          if run: 
212              assert len(nodes) > 0, 'master needs at least 1 pvm node.' 
213   
214              self.r = self.master.calculateResult() 
215              if self.local and self.DEBUG: 
216                  self.log.add('The alignment result is in %s/t_coffee'%\ 
217                               self.outfolder) 
218               
219 -    def cleanUp(self): 
220          T.tryRemove( self.outfolder, tree=1 ) 
221           
222   
223 -class TestDry( TestBase ): 
224      """Dry run test case that does not actually run t-coffee""" 
225   
226      TAGS = [ BT.PVM ] 
227   
228 -    def test_AlignerMaster(self): 
229          """Mod.AlignerMaster limited dry run test""" 
230          return self.t_AlignerMaster(run=False) 
231   
232 -class TestReal( TestBase ): 
233      """Full AlignerMaster test case""" 
234   
235      TAGS = [BT.PVM, BT.EXE, BT.LONG] 
236   
237 -    def test_AlignerMaster(self): 
238          """Mod.AlignerMaster full test""" 
239          return self.t_AlignerMaster(run=True) 
240       
241   
242  if __name__ == '__main__': 
243   
244      BT.localTest() 
245