12357Sktlim@umich.edu# Copyright (c) 2006 The Regents of The University of Michigan 22357Sktlim@umich.edu# All rights reserved. 32357Sktlim@umich.edu# 42357Sktlim@umich.edu# Redistribution and use in source and binary forms, with or without 52357Sktlim@umich.edu# modification, are permitted provided that the following conditions are 62357Sktlim@umich.edu# met: redistributions of source code must retain the above copyright 72357Sktlim@umich.edu# notice, this list of conditions and the following disclaimer; 82357Sktlim@umich.edu# redistributions in binary form must reproduce the above copyright 92357Sktlim@umich.edu# notice, this list of conditions and the following disclaimer in the 102357Sktlim@umich.edu# documentation and/or other materials provided with the distribution; 112357Sktlim@umich.edu# neither the name of the copyright holders nor the names of its 122357Sktlim@umich.edu# contributors may be used to endorse or promote products derived from 132357Sktlim@umich.edu# this software without specific prior written permission. 142357Sktlim@umich.edu# 152357Sktlim@umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 162357Sktlim@umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 172357Sktlim@umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 182357Sktlim@umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 192357Sktlim@umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 202357Sktlim@umich.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 212357Sktlim@umich.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 222357Sktlim@umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 232357Sktlim@umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 242357Sktlim@umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 252357Sktlim@umich.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 262357Sktlim@umich.edu# 272357Sktlim@umich.edu# Authors: Kevin Lim 282357Sktlim@umich.edu 292357Sktlim@umich.eduimport os, popen2, re, sys 302357Sktlim@umich.edu 312357Sktlim@umich.educlass MyPOpen(object): 322357Sktlim@umich.edu def __init__(self, cmd, input = None, output = None, bufsize = -1): 332357Sktlim@umich.edu self.status = -1 342357Sktlim@umich.edu 352357Sktlim@umich.edu if input is None: 362357Sktlim@umich.edu p2c_read, p2c_write = os.pipe() 372357Sktlim@umich.edu self.tochild = os.fdopen(p2c_write, 'w', bufsize) 382357Sktlim@umich.edu else: 392357Sktlim@umich.edu p2c_write = None 402357Sktlim@umich.edu if isinstance(input, file): 412357Sktlim@umich.edu p2c_read = input.fileno() 422357Sktlim@umich.edu elif isinstance(input, str): 432357Sktlim@umich.edu input = file(input, 'r') 442357Sktlim@umich.edu p2c_read = input.fileno() 452357Sktlim@umich.edu elif isinstance(input, int): 462357Sktlim@umich.edu p2c_read = input 472357Sktlim@umich.edu else: 482357Sktlim@umich.edu raise AttributeError 492357Sktlim@umich.edu 502357Sktlim@umich.edu if output is None: 512357Sktlim@umich.edu c2p_read, c2p_write = os.pipe() 522357Sktlim@umich.edu self.fromchild = os.fdopen(c2p_read, 'r', bufsize) 532357Sktlim@umich.edu else: 542357Sktlim@umich.edu c2p_read = None 552357Sktlim@umich.edu if isinstance(output, file): 562357Sktlim@umich.edu c2p_write = output.fileno() 572357Sktlim@umich.edu elif isinstance(output, str): 582357Sktlim@umich.edu output = file(output, 'w') 592357Sktlim@umich.edu c2p_write = output.fileno() 602357Sktlim@umich.edu elif isinstance(output, int): 612357Sktlim@umich.edu c2p_write = output 622357Sktlim@umich.edu else: 632357Sktlim@umich.edu raise AttributeError 642357Sktlim@umich.edu 652357Sktlim@umich.edu self.pid = os.fork() 662357Sktlim@umich.edu if self.pid == 0: 672357Sktlim@umich.edu os.dup2(p2c_read, sys.stdin.fileno()) 682357Sktlim@umich.edu os.dup2(c2p_write, sys.stdout.fileno()) 692357Sktlim@umich.edu os.dup2(c2p_write, sys.stderr.fileno()) 702357Sktlim@umich.edu try: 712357Sktlim@umich.edu os.execvp(cmd[0], cmd) 722357Sktlim@umich.edu finally: 732357Sktlim@umich.edu os._exit(1) 742357Sktlim@umich.edu 752357Sktlim@umich.edu os.close(p2c_read) 762357Sktlim@umich.edu os.close(c2p_write) 772357Sktlim@umich.edu 782357Sktlim@umich.edu def poll(self): 792357Sktlim@umich.edu if self.status < 0: 802357Sktlim@umich.edu pid, status = os.waitpid(self.pid, os.WNOHANG) 812357Sktlim@umich.edu if pid == self.pid: 822357Sktlim@umich.edu self.status = status 832357Sktlim@umich.edu return self.status 842357Sktlim@umich.edu 852357Sktlim@umich.edu def wait(self): 862357Sktlim@umich.edu if self.status < 0: 872357Sktlim@umich.edu pid, status = os.waitpid(self.pid, 0) 882357Sktlim@umich.edu if pid == self.pid: 892357Sktlim@umich.edu self.status = status 902357Sktlim@umich.edu return self.status 912357Sktlim@umich.edu 922357Sktlim@umich.edu 932357Sktlim@umich.educlass oarsub: 942357Sktlim@umich.edu def __init__(self): 952357Sktlim@umich.edu self.walltime = None 962357Sktlim@umich.edu self.queue = None 972357Sktlim@umich.edu self.properties = None 982357Sktlim@umich.edu 992357Sktlim@umich.edu # OAR 2.0 parameters only! 1002357Sktlim@umich.edu self.name = None 1012357Sktlim@umich.edu self.afterok = None 1022357Sktlim@umich.edu self.notify = None 1032357Sktlim@umich.edu self.stderr = None 1042357Sktlim@umich.edu self.stdout = None 1052357Sktlim@umich.edu 1062357Sktlim@umich.edu 1072357Sktlim@umich.edu self.oarhost = None 1082357Sktlim@umich.edu self.oarsub = 'oarsub' 1092357Sktlim@umich.edu 1102357Sktlim@umich.edu self.jobid = re.compile('IdJob = (\S+)') 1112357Sktlim@umich.edu #self.outfile = open("jobnames.dat", "a+") 1122357Sktlim@umich.edu 1132357Sktlim@umich.edu def build(self, script, args = []): 1142357Sktlim@umich.edu self.cmd = [ self.oarsub ] 1152357Sktlim@umich.edu 1162357Sktlim@umich.edu print "args:", args 1172357Sktlim@umich.edu print "script:", script 1182357Sktlim@umich.edu if self.properties: 1192357Sktlim@umich.edu self.cmd.append('-p"%s"' % self.properties ) 1202357Sktlim@umich.edu 1212357Sktlim@umich.edu if self.queue: 1222357Sktlim@umich.edu self.cmd.append('-q "%s"' % self.queue) 1232357Sktlim@umich.edu 1242357Sktlim@umich.edu if self.walltime: 1252357Sktlim@umich.edu self.cmd.append('-l walltime=%s' % self.walltime) 1262357Sktlim@umich.edu 1272357Sktlim@umich.edu if script[0] != "/": 1282357Sktlim@umich.edu self.script = os.getcwd() 1292357Sktlim@umich.edu else: 1302357Sktlim@umich.edu self.script = script 1312357Sktlim@umich.edu 1322357Sktlim@umich.edu self.cmd.extend(args) 1332357Sktlim@umich.edu self.cmd.append(self.script) 1342357Sktlim@umich.edu #cmd = [ 'ssh', '-x', self.oarhost, '"cd %s; %s"' % (os.getcwd(), self.command) ] 1352357Sktlim@umich.edu self.command = ' '.join(self.cmd) 1362357Sktlim@umich.edu 1372357Sktlim@umich.edu print "command: [%s]" % self.command 1382357Sktlim@umich.edu 1392357Sktlim@umich.edu def do(self): 1402357Sktlim@umich.edu oar = MyPOpen(self.cmd) 1412357Sktlim@umich.edu self.result = oar.fromchild.read() 1422357Sktlim@umich.edu ec = oar.wait() 1432357Sktlim@umich.edu 1442357Sktlim@umich.edu if ec != 0 and self.oarhost: 1452357Sktlim@umich.edu pstdin, pstdout = os.popen4(self.command) 1462357Sktlim@umich.edu self.result = pstdout.read() 1472357Sktlim@umich.edu 1482357Sktlim@umich.edu jobid = self.jobid.match(self.result) 1492357Sktlim@umich.edu if jobid == None: 1502357Sktlim@umich.edu print "Couldn't get jobid from [%s]" % self.result 1512357Sktlim@umich.edu sys.exit(1) 1522357Sktlim@umich.edu else: 1532357Sktlim@umich.edu #self.outfile.write("%d %s\n" %(int(jobid.group(1)), self.name)); 1542357Sktlim@umich.edu #self.outfile.flush() 1552357Sktlim@umich.edu self.result = jobid.group(1) 1562357Sktlim@umich.edu 1572357Sktlim@umich.edu return 0 1582357Sktlim@umich.edu 1592357Sktlim@umich.educlass qsub: 1602357Sktlim@umich.edu def __init__(self): 1612357Sktlim@umich.edu self.afterok = None 1622357Sktlim@umich.edu self.hold = False 1632357Sktlim@umich.edu self.join = False 1642357Sktlim@umich.edu self.keep_stdout = False 1652357Sktlim@umich.edu self.keep_stderr = False 1662357Sktlim@umich.edu self.node_type = None 1672357Sktlim@umich.edu self.mail_abort = False 1682357Sktlim@umich.edu self.mail_begin = False 1692357Sktlim@umich.edu self.mail_end = False 1702357Sktlim@umich.edu self.name = None 1712357Sktlim@umich.edu self.stdout = None 1722357Sktlim@umich.edu self.priority = None 1732357Sktlim@umich.edu self.queue = None 1742357Sktlim@umich.edu self.pbshost = None 1752357Sktlim@umich.edu self.qsub = 'qsub' 1762357Sktlim@umich.edu self.env = {} 1772357Sktlim@umich.edu 1782357Sktlim@umich.edu def build(self, script, args = []): 1792357Sktlim@umich.edu self.cmd = [ self.qsub ] 1802357Sktlim@umich.edu 1812357Sktlim@umich.edu if self.env: 1822357Sktlim@umich.edu arg = '-v' 1832357Sktlim@umich.edu arg += ','.join([ '%s=%s' % i for i in self.env.iteritems() ]) 1842357Sktlim@umich.edu self.cmd.append(arg) 1852357Sktlim@umich.edu 1862357Sktlim@umich.edu if self.hold: 1872357Sktlim@umich.edu self.cmd.append('-h') 1882357Sktlim@umich.edu 1892357Sktlim@umich.edu if self.stdout: 1902357Sktlim@umich.edu self.cmd.append('-olocalhost:' + self.stdout) 1912357Sktlim@umich.edu 1922357Sktlim@umich.edu if self.keep_stdout and self.keep_stderr: 1932357Sktlim@umich.edu self.cmd.append('-koe') 1942357Sktlim@umich.edu elif self.keep_stdout: 1952357Sktlim@umich.edu self.cmd.append('-ko') 1962357Sktlim@umich.edu elif self.keep_stderr: 1972357Sktlim@umich.edu self.cmd.append('-ke') 1982357Sktlim@umich.edu else: 1992357Sktlim@umich.edu self.cmd.append('-kn') 2002357Sktlim@umich.edu 2012357Sktlim@umich.edu if self.join: 2022357Sktlim@umich.edu self.cmd.append('-joe') 2032357Sktlim@umich.edu 2042357Sktlim@umich.edu if self.node_type: 2052357Sktlim@umich.edu self.cmd.append('-lnodes=' + self.node_type) 2062357Sktlim@umich.edu 2072357Sktlim@umich.edu if self.mail_abort or self.mail_begin or self.mail_end: 2082357Sktlim@umich.edu flags = '' 2092357Sktlim@umich.edu if self.mail_abort: 2102357Sktlim@umich.edu flags.append('a') 2112357Sktlim@umich.edu if self.mail_begin: 2122357Sktlim@umich.edu flags.append('b') 2132357Sktlim@umich.edu if self.mail_end: 2142357Sktlim@umich.edu flags.append('e') 2152357Sktlim@umich.edu if len(flags): 2162357Sktlim@umich.edu self.cmd.append('-m ' + flags) 2172357Sktlim@umich.edu else: 2182357Sktlim@umich.edu self.cmd.append('-mn') 2192357Sktlim@umich.edu 2202357Sktlim@umich.edu if self.name: 2212357Sktlim@umich.edu self.cmd.append("-N%s" % self.name) 2222357Sktlim@umich.edu 2232357Sktlim@umich.edu if self.priority: 2242357Sktlim@umich.edu self.cmd.append('-p' + self.priority) 2252357Sktlim@umich.edu 2262357Sktlim@umich.edu if self.queue: 2272357Sktlim@umich.edu self.cmd.append('-q' + self.queue) 2282357Sktlim@umich.edu 2292357Sktlim@umich.edu if self.afterok: 2302357Sktlim@umich.edu self.cmd.append('-Wdepend=afterok:%s' % self.afterok) 2312357Sktlim@umich.edu 2322357Sktlim@umich.edu self.cmd.extend(args) 2332357Sktlim@umich.edu self.script = script 2342357Sktlim@umich.edu self.command = ' '.join(self.cmd + [ self.script ]) 2352357Sktlim@umich.edu 2362357Sktlim@umich.edu def do(self): 2372357Sktlim@umich.edu pbs = MyPOpen(self.cmd + [ self.script ]) 2382357Sktlim@umich.edu self.result = pbs.fromchild.read() 2392357Sktlim@umich.edu ec = pbs.wait() 2402357Sktlim@umich.edu 2412357Sktlim@umich.edu if ec != 0 and self.pbshost: 2422357Sktlim@umich.edu cmd = ' '.join(self.cmd + [ '-' ]) 2432357Sktlim@umich.edu cmd = [ 'ssh', '-x', self.pbshost, cmd ] 2442357Sktlim@umich.edu self.command = ' '.join(cmd) 2452357Sktlim@umich.edu ssh = MyPOpen(cmd, input = self.script) 2462357Sktlim@umich.edu self.result = ssh.fromchild.read() 2472357Sktlim@umich.edu ec = ssh.wait() 2482357Sktlim@umich.edu 2492357Sktlim@umich.edu return ec 250