113540Sandrea.mondelli@ucf.edu#!/usr/bin/env python2.7 22357Sktlim@umich.edu# Copyright (c) 2006 The Regents of The University of Michigan 32357Sktlim@umich.edu# All rights reserved. 42357Sktlim@umich.edu# 52357Sktlim@umich.edu# Redistribution and use in source and binary forms, with or without 62357Sktlim@umich.edu# modification, are permitted provided that the following conditions are 72357Sktlim@umich.edu# met: redistributions of source code must retain the above copyright 82357Sktlim@umich.edu# notice, this list of conditions and the following disclaimer; 92357Sktlim@umich.edu# redistributions in binary form must reproduce the above copyright 102357Sktlim@umich.edu# notice, this list of conditions and the following disclaimer in the 112357Sktlim@umich.edu# documentation and/or other materials provided with the distribution; 122357Sktlim@umich.edu# neither the name of the copyright holders nor the names of its 132357Sktlim@umich.edu# contributors may be used to endorse or promote products derived from 142357Sktlim@umich.edu# this software without specific prior written permission. 152357Sktlim@umich.edu# 162357Sktlim@umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 172357Sktlim@umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 182357Sktlim@umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 192357Sktlim@umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 202357Sktlim@umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 212357Sktlim@umich.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 222357Sktlim@umich.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 232357Sktlim@umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 242357Sktlim@umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 252357Sktlim@umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 262357Sktlim@umich.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 272357Sktlim@umich.edu# 282357Sktlim@umich.edu# Authors: Kevin Lim 292357Sktlim@umich.edu 302357Sktlim@umich.eduimport os, os.path, re, socket, sys 312357Sktlim@umich.edufrom os import environ as env, listdir 322357Sktlim@umich.edufrom os.path import basename, isdir, isfile, islink, join as joinpath, normpath 332357Sktlim@umich.edufrom filecmp import cmp as filecmp 342357Sktlim@umich.edufrom shutil import copy 352357Sktlim@umich.edu 362357Sktlim@umich.edudef nfspath(dir): 372357Sktlim@umich.edu if dir.startswith('/.automount/'): 382357Sktlim@umich.edu dir = '/n/%s' % dir[12:] 392357Sktlim@umich.edu elif not dir.startswith('/n/'): 402357Sktlim@umich.edu dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir) 412357Sktlim@umich.edu return dir 422357Sktlim@umich.edu 432357Sktlim@umich.edudef syncdir(srcdir, destdir): 442357Sktlim@umich.edu srcdir = normpath(srcdir) 452357Sktlim@umich.edu destdir = normpath(destdir) 462357Sktlim@umich.edu if not isdir(destdir): 472357Sktlim@umich.edu sys.exit('destination directory "%s" does not exist' % destdir) 482357Sktlim@umich.edu 492357Sktlim@umich.edu for root, dirs, files in os.walk(srcdir): 502357Sktlim@umich.edu root = normpath(root) 512357Sktlim@umich.edu prefix = os.path.commonprefix([root, srcdir]) 522357Sktlim@umich.edu root = root[len(prefix):] 532357Sktlim@umich.edu if root.startswith('/'): 542357Sktlim@umich.edu root = root[1:] 552357Sktlim@umich.edu for rem in [ d for d in dirs if d.startswith('.') or d == 'SCCS']: 562357Sktlim@umich.edu dirs.remove(rem) 572357Sktlim@umich.edu 582357Sktlim@umich.edu for entry in dirs: 592357Sktlim@umich.edu newdir = joinpath(destdir, root, entry) 602357Sktlim@umich.edu if not isdir(newdir): 612357Sktlim@umich.edu os.mkdir(newdir) 622357Sktlim@umich.edu print 'mkdir', newdir 632357Sktlim@umich.edu 642357Sktlim@umich.edu for i,d in enumerate(dirs): 652357Sktlim@umich.edu if islink(joinpath(srcdir, root, d)): 662357Sktlim@umich.edu dirs[i] = joinpath(d, '.') 672357Sktlim@umich.edu 682357Sktlim@umich.edu for entry in files: 692357Sktlim@umich.edu dest = normpath(joinpath(destdir, root, entry)) 702357Sktlim@umich.edu src = normpath(joinpath(srcdir, root, entry)) 712357Sktlim@umich.edu if not isfile(dest) or not filecmp(src, dest): 722357Sktlim@umich.edu print 'copy %s %s' % (dest, src) 732357Sktlim@umich.edu copy(src, dest) 742357Sktlim@umich.edu 752357Sktlim@umich.eduprogpath = nfspath(sys.path[0]) 762357Sktlim@umich.eduprogname = basename(sys.argv[0]) 772357Sktlim@umich.eduusage = """\ 782357Sktlim@umich.eduUsage: 792357Sktlim@umich.edu %(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp> 802357Sktlim@umich.edu -c clean directory if job can be run 812357Sktlim@umich.edu -C submit the checkpointing runs 822357Sktlim@umich.edu -d Make jobs be dependent on the completion of the checkpoint runs 832357Sktlim@umich.edu -e only echo pbs command info, don't actually send the job 842357Sktlim@umich.edu -f force the job to run regardless of state 852357Sktlim@umich.edu -q <queue> submit job to the named queue 862357Sktlim@umich.edu -j <jobfile> specify the jobfile (default is <rootdir>/Test.py) 872357Sktlim@umich.edu -v be verbose 882357Sktlim@umich.edu 892357Sktlim@umich.edu %(progname)s [-j <jobfile>] -l [-v] <regexp> 902357Sktlim@umich.edu -j <jobfile> specify the jobfile (default is <rootdir>/Test.py) 912357Sktlim@umich.edu -l list job names, don't submit 922357Sktlim@umich.edu -v be verbose (list job parameters) 932357Sktlim@umich.edu 942357Sktlim@umich.edu %(progname)s -h 952357Sktlim@umich.edu -h display this help 962357Sktlim@umich.edu""" % locals() 972357Sktlim@umich.edu 982357Sktlim@umich.edutry: 992357Sktlim@umich.edu import getopt 1002357Sktlim@umich.edu opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lnq:Rt:v') 1012357Sktlim@umich.eduexcept getopt.GetoptError: 1022357Sktlim@umich.edu sys.exit(usage) 1032357Sktlim@umich.edu 1042357Sktlim@umich.edudepend = False 1052357Sktlim@umich.educlean = False 1062357Sktlim@umich.eduonlyecho = False 1072357Sktlim@umich.eduexprs = [] 1082357Sktlim@umich.eduforce = False 1092357Sktlim@umich.edulistonly = False 1102357Sktlim@umich.eduqueue = '' 1112357Sktlim@umich.eduverbose = False 1122357Sktlim@umich.edujfile = 'Test.py' 1132357Sktlim@umich.edudocpts = False 1142357Sktlim@umich.edudoruns = True 1152357Sktlim@umich.edurunflag = False 1162357Sktlim@umich.edunode_type = 'FAST' 1172357Sktlim@umich.eduupdate = True 1182357Sktlim@umich.edu 1192357Sktlim@umich.edufor opt,arg in opts: 1202357Sktlim@umich.edu if opt == '-C': 1212357Sktlim@umich.edu docpts = True 1222357Sktlim@umich.edu if opt == '-c': 1232357Sktlim@umich.edu clean = True 1242357Sktlim@umich.edu if opt == '-d': 1252357Sktlim@umich.edu depend = True 1262357Sktlim@umich.edu if opt == '-e': 1272357Sktlim@umich.edu onlyecho = True 1282357Sktlim@umich.edu if opt == '-f': 1292357Sktlim@umich.edu force = True 1302357Sktlim@umich.edu if opt == '-h': 1312357Sktlim@umich.edu print usage 1322357Sktlim@umich.edu sys.exit(0) 1332357Sktlim@umich.edu if opt == '-j': 1342357Sktlim@umich.edu jfile = arg 1352357Sktlim@umich.edu if opt == '-l': 1362357Sktlim@umich.edu listonly = True 1372357Sktlim@umich.edu if opt == '-n': 1382357Sktlim@umich.edu update = False 1392357Sktlim@umich.edu if opt == '-q': 1402357Sktlim@umich.edu queue = arg 1412357Sktlim@umich.edu if opt == '-R': 1422357Sktlim@umich.edu runflag = True 1432357Sktlim@umich.edu if opt == '-t': 1442357Sktlim@umich.edu node_type = arg 1452357Sktlim@umich.edu if opt == '-v': 1462357Sktlim@umich.edu verbose = True 1472357Sktlim@umich.edu 1482357Sktlim@umich.eduif docpts: 1492357Sktlim@umich.edu doruns = runflag 1502357Sktlim@umich.edu 1512357Sktlim@umich.edufor arg in args: 1522357Sktlim@umich.edu exprs.append(re.compile(arg)) 1532357Sktlim@umich.edu 1542357Sktlim@umich.eduimport jobfile, batch 1552357Sktlim@umich.edufrom job import JobDir, date 1562357Sktlim@umich.edu 1572357Sktlim@umich.educonf = jobfile.JobFile(jfile) 1582357Sktlim@umich.edu 1592357Sktlim@umich.eduif update and not listonly and not onlyecho and isdir(conf.linkdir): 1602357Sktlim@umich.edu if verbose: 1612357Sktlim@umich.edu print 'Checking for outdated files in Link directory' 1622357Sktlim@umich.edu if not isdir(conf.basedir): 1632357Sktlim@umich.edu os.mkdir(conf.basedir) 1642357Sktlim@umich.edu syncdir(conf.linkdir, conf.basedir) 1652357Sktlim@umich.edu 1662357Sktlim@umich.edujobnames = {} 1672357Sktlim@umich.edujoblist = [] 1682357Sktlim@umich.edu 1692357Sktlim@umich.eduif docpts and doruns: 1702357Sktlim@umich.edu gen = conf.alljobs() 1712357Sktlim@umich.eduelif docpts: 1722357Sktlim@umich.edu gen = conf.checkpoints() 1732357Sktlim@umich.eduelif doruns: 1742357Sktlim@umich.edu gen = conf.jobs() 1752357Sktlim@umich.edu 1762357Sktlim@umich.edufor job in gen: 1772357Sktlim@umich.edu if job.name in jobnames: 1782357Sktlim@umich.edu continue 1792357Sktlim@umich.edu 1802357Sktlim@umich.edu if exprs: 1812357Sktlim@umich.edu for expr in exprs: 1822357Sktlim@umich.edu if expr.match(job.name): 1832357Sktlim@umich.edu joblist.append(job) 1842357Sktlim@umich.edu break 1852357Sktlim@umich.edu else: 1862357Sktlim@umich.edu joblist.append(job) 1872357Sktlim@umich.edu 1882357Sktlim@umich.eduif listonly: 1892357Sktlim@umich.edu if verbose: 1902357Sktlim@umich.edu for job in joblist: 1912357Sktlim@umich.edu job.printinfo() 1922357Sktlim@umich.edu else: 1932357Sktlim@umich.edu for job in joblist: 1942357Sktlim@umich.edu print job.name 1952357Sktlim@umich.edu sys.exit(0) 1962357Sktlim@umich.edu 1972357Sktlim@umich.eduif not onlyecho: 1982357Sktlim@umich.edu newlist = [] 1992357Sktlim@umich.edu for job in joblist: 2002357Sktlim@umich.edu jobdir = JobDir(joinpath(conf.rootdir, job.name)) 2012357Sktlim@umich.edu if jobdir.exists(): 2022357Sktlim@umich.edu if not force: 2032357Sktlim@umich.edu status = jobdir.getstatus() 2042357Sktlim@umich.edu if status == 'queued': 2052357Sktlim@umich.edu continue 2062357Sktlim@umich.edu 2072357Sktlim@umich.edu if status == 'running': 2082357Sktlim@umich.edu continue 2092357Sktlim@umich.edu 2102357Sktlim@umich.edu if status == 'success': 2112357Sktlim@umich.edu continue 2122357Sktlim@umich.edu 2132357Sktlim@umich.edu if not clean: 2142357Sktlim@umich.edu sys.exit('job directory %s not clean!' % jobdir) 2152357Sktlim@umich.edu 2162357Sktlim@umich.edu jobdir.clean() 2172357Sktlim@umich.edu newlist.append(job) 2182357Sktlim@umich.edu joblist = newlist 2192357Sktlim@umich.edu 2202357Sktlim@umich.educlass NameHack(object): 2212357Sktlim@umich.edu def __init__(self, host='pbs.pool', port=24465): 2222357Sktlim@umich.edu self.host = host 2232357Sktlim@umich.edu self.port = port 2242357Sktlim@umich.edu self.socket = None 2252357Sktlim@umich.edu 2262357Sktlim@umich.edu def setname(self, jobid, jobname): 2272357Sktlim@umich.edu try: 2282357Sktlim@umich.edu jobid = int(jobid) 2292357Sktlim@umich.edu except ValueError: 2302357Sktlim@umich.edu jobid = int(jobid.strip().split('.')[0]) 2312357Sktlim@umich.edu 2322357Sktlim@umich.edu jobname = jobname.strip() 2332357Sktlim@umich.edu # since pbs can handle jobnames of 15 characters or less, 2342357Sktlim@umich.edu # don't use the raj hack. 2352357Sktlim@umich.edu if len(jobname) <= 15: 2362357Sktlim@umich.edu return 2372357Sktlim@umich.edu 2382357Sktlim@umich.edu if self.socket is None: 2392357Sktlim@umich.edu import socket 2402357Sktlim@umich.edu self.socket = socket.socket() 2412357Sktlim@umich.edu # Connect to pbs.pool and send the jobid/jobname pair to port 2422357Sktlim@umich.edu # 24465 (Raj didn't realize that there are only 64k ports and 2432357Sktlim@umich.edu # setup inetd to point to port 90001) 2442357Sktlim@umich.edu self.socket.connect((self.host, self.port)) 2452357Sktlim@umich.edu 2462357Sktlim@umich.edu self.socket.send("%s %s\n" % (jobid, jobname)) 2472357Sktlim@umich.edu 2482357Sktlim@umich.edunamehack = NameHack() 2492357Sktlim@umich.edu 2502357Sktlim@umich.edurootdir = conf.rootdir 2512357Sktlim@umich.eduscript = joinpath(rootdir, 'Base', 'job.py') 2522357Sktlim@umich.edu 2532357Sktlim@umich.edufor job in joblist: 2542357Sktlim@umich.edu jobdir = JobDir(joinpath(rootdir, job.name)) 2552357Sktlim@umich.edu if depend: 2562357Sktlim@umich.edu cptdir = JobDir(joinpath(rootdir, job.checkpoint.name)) 2572357Sktlim@umich.edu path = str(cptdir) 2582357Sktlim@umich.edu if not isdir(path) or not isfile(joinpath(path, '.success')): 2592357Sktlim@umich.edu continue 2602357Sktlim@umich.edu 2612357Sktlim@umich.edu cptjob = cptdir.readval('.batch_jobid') 2622357Sktlim@umich.edu 2632357Sktlim@umich.edu if not onlyecho: 2642357Sktlim@umich.edu jobdir.create() 2652357Sktlim@umich.edu os.chdir(str(jobdir)) 2662357Sktlim@umich.edu os.environ['PWD'] = str(jobdir) 2672357Sktlim@umich.edu 2682357Sktlim@umich.edu print 'Job name: %s' % job.name 2692357Sktlim@umich.edu print 'Job directory: %s' % jobdir 2702357Sktlim@umich.edu 2712357Sktlim@umich.edu 2722357Sktlim@umich.edu qsub = batch.oarsub() 2732357Sktlim@umich.edu qsub.oarhost = 'poolfs.eecs.umich.edu' 2742357Sktlim@umich.edu #qsub.stdout = jobdir.file('jobout') 2752357Sktlim@umich.edu qsub.name = job.name 2762357Sktlim@umich.edu qsub.walltime = '50' 2772357Sktlim@umich.edu #qsub.join = True 2782357Sktlim@umich.edu #qsub.node_type = node_type 2792357Sktlim@umich.edu #qsub.env['ROOTDIR'] = conf.rootdir 2802357Sktlim@umich.edu #qsub.env['JOBNAME'] = job.name 2812357Sktlim@umich.edu #if depend: 2822357Sktlim@umich.edu # qsub.afterok = cptjob 2832357Sktlim@umich.edu #if queue: 2842357Sktlim@umich.edu # qsub.queue = queue 2852357Sktlim@umich.edu qsub.properties = "64bit = 'Yes' or 64bit = 'No'" 2862357Sktlim@umich.edu qsub.build(script) 2872357Sktlim@umich.edu 2882357Sktlim@umich.edu if verbose: 2892357Sktlim@umich.edu print 'cwd: %s' % qsub.command 2902357Sktlim@umich.edu print 'PBS Command: %s' % qsub.command 2912357Sktlim@umich.edu 2922357Sktlim@umich.edu if not onlyecho: 2932357Sktlim@umich.edu ec = qsub.do() 2942357Sktlim@umich.edu if ec == 0: 2952357Sktlim@umich.edu jobid = qsub.result 2962357Sktlim@umich.edu print 'OAR Jobid: %s' % jobid 2972357Sktlim@umich.edu #namehack.setname(jobid, job.name) 2982357Sktlim@umich.edu queued = date() 2992357Sktlim@umich.edu jobdir.echofile('.batch_jobid', jobid) 3002357Sktlim@umich.edu jobdir.echofile('.batch_jobname', job.name) 3012357Sktlim@umich.edu jobdir.echofile('.queued', queued) 3022357Sktlim@umich.edu jobdir.setstatus('queued on %s' % queued) 3032357Sktlim@umich.edu else: 3042357Sktlim@umich.edu print 'OAR Failed' 3052357Sktlim@umich.edu print 3062357Sktlim@umich.edu print 307