send.py revision 13540
11049Sbinkertn@umich.edu#!/usr/bin/env python2.7 21049Sbinkertn@umich.edu# Copyright (c) 2005 The Regents of The University of Michigan 31049Sbinkertn@umich.edu# All rights reserved. 41049Sbinkertn@umich.edu# 51049Sbinkertn@umich.edu# Redistribution and use in source and binary forms, with or without 61049Sbinkertn@umich.edu# modification, are permitted provided that the following conditions are 71049Sbinkertn@umich.edu# met: redistributions of source code must retain the above copyright 81049Sbinkertn@umich.edu# notice, this list of conditions and the following disclaimer; 91049Sbinkertn@umich.edu# redistributions in binary form must reproduce the above copyright 101049Sbinkertn@umich.edu# notice, this list of conditions and the following disclaimer in the 111049Sbinkertn@umich.edu# documentation and/or other materials provided with the distribution; 121049Sbinkertn@umich.edu# neither the name of the copyright holders nor the names of its 131049Sbinkertn@umich.edu# contributors may be used to endorse or promote products derived from 141049Sbinkertn@umich.edu# this software without specific prior written permission. 151049Sbinkertn@umich.edu# 161049Sbinkertn@umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 171049Sbinkertn@umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 181049Sbinkertn@umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 191049Sbinkertn@umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 201049Sbinkertn@umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 211049Sbinkertn@umich.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 221049Sbinkertn@umich.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 231049Sbinkertn@umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 241049Sbinkertn@umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 251049Sbinkertn@umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 261049Sbinkertn@umich.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 271049Sbinkertn@umich.edu# 281049Sbinkertn@umich.edu# Authors: Ali Saidi 291049Sbinkertn@umich.edu# Nathan Binkert 301049Sbinkertn@umich.edu 311049Sbinkertn@umich.eduimport os, os.path, re, socket, sys 321049Sbinkertn@umich.edufrom os import environ as env, listdir 331049Sbinkertn@umich.edufrom os.path import basename, isdir, isfile, islink, join as joinpath, normpath 341049Sbinkertn@umich.edufrom filecmp import cmp as filecmp 351049Sbinkertn@umich.edufrom shutil import copy 361049Sbinkertn@umich.edu 371049Sbinkertn@umich.edudef nfspath(dir): 381049Sbinkertn@umich.edu if dir.startswith('/.automount/'): 391049Sbinkertn@umich.edu dir = '/n/%s' % dir[12:] 401049Sbinkertn@umich.edu elif not dir.startswith('/n/'): 411049Sbinkertn@umich.edu dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir) 421307Sbinkertn@umich.edu return dir 431307Sbinkertn@umich.edu 441307Sbinkertn@umich.edudef syncdir(srcdir, destdir): 451307Sbinkertn@umich.edu srcdir = normpath(srcdir) 461307Sbinkertn@umich.edu destdir = normpath(destdir) 471307Sbinkertn@umich.edu if not isdir(destdir): 481307Sbinkertn@umich.edu sys.exit('destination directory "%s" does not exist' % destdir) 491307Sbinkertn@umich.edu 501307Sbinkertn@umich.edu for root, dirs, files in os.walk(srcdir): 511307Sbinkertn@umich.edu root = normpath(root) 521307Sbinkertn@umich.edu prefix = os.path.commonprefix([root, srcdir]) 531307Sbinkertn@umich.edu root = root[len(prefix):] 541307Sbinkertn@umich.edu if root.startswith('/'): 551307Sbinkertn@umich.edu root = root[1:] 561307Sbinkertn@umich.edu for rem in [ d for d in dirs if d.startswith('.') or d == 'SCCS']: 571307Sbinkertn@umich.edu dirs.remove(rem) 581307Sbinkertn@umich.edu 591307Sbinkertn@umich.edu for entry in dirs: 601307Sbinkertn@umich.edu newdir = joinpath(destdir, root, entry) 611307Sbinkertn@umich.edu if not isdir(newdir): 621307Sbinkertn@umich.edu os.mkdir(newdir) 631307Sbinkertn@umich.edu print 'mkdir', newdir 641307Sbinkertn@umich.edu 651307Sbinkertn@umich.edu for i,d in enumerate(dirs): 661307Sbinkertn@umich.edu if islink(joinpath(srcdir, root, d)): 671307Sbinkertn@umich.edu dirs[i] = joinpath(d, '.') 681307Sbinkertn@umich.edu 691307Sbinkertn@umich.edu for entry in files: 701307Sbinkertn@umich.edu dest = normpath(joinpath(destdir, root, entry)) 711307Sbinkertn@umich.edu src = normpath(joinpath(srcdir, root, entry)) 721307Sbinkertn@umich.edu if not isfile(dest) or not filecmp(src, dest): 731307Sbinkertn@umich.edu print 'copy %s %s' % (dest, src) 741307Sbinkertn@umich.edu copy(src, dest) 751307Sbinkertn@umich.edu 761307Sbinkertn@umich.eduprogpath = nfspath(sys.path[0]) 771307Sbinkertn@umich.eduprogname = basename(sys.argv[0]) 781307Sbinkertn@umich.eduusage = """\ 791307Sbinkertn@umich.eduUsage: 801307Sbinkertn@umich.edu %(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp> 811307Sbinkertn@umich.edu -c clean directory if job can be run 821307Sbinkertn@umich.edu -C submit the checkpointing runs 831307Sbinkertn@umich.edu -d Make jobs be dependent on the completion of the checkpoint runs 841307Sbinkertn@umich.edu -e only echo pbs command info, don't actually send the job 851307Sbinkertn@umich.edu -f force the job to run regardless of state 861307Sbinkertn@umich.edu -q <queue> submit job to the named queue 871307Sbinkertn@umich.edu -j <jobfile> specify the jobfile (default is <rootdir>/Test.py) 881307Sbinkertn@umich.edu -v be verbose 891307Sbinkertn@umich.edu 901307Sbinkertn@umich.edu %(progname)s [-j <jobfile>] -l [-v] <regexp> 911307Sbinkertn@umich.edu -j <jobfile> specify the jobfile (default is <rootdir>/Test.py) 921307Sbinkertn@umich.edu -l list job names, don't submit 931307Sbinkertn@umich.edu -v be verbose (list job parameters) 941307Sbinkertn@umich.edu 951307Sbinkertn@umich.edu %(progname)s -h 961307Sbinkertn@umich.edu -h display this help 971307Sbinkertn@umich.edu""" % locals() 981307Sbinkertn@umich.edu 991307Sbinkertn@umich.edutry: 1001307Sbinkertn@umich.edu import getopt 1011307Sbinkertn@umich.edu opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lnq:Rt:v') 1021307Sbinkertn@umich.eduexcept getopt.GetoptError: 1031307Sbinkertn@umich.edu sys.exit(usage) 1041269Sbinkertn@umich.edu 1051307Sbinkertn@umich.edudepend = False 1061307Sbinkertn@umich.educlean = False 1071307Sbinkertn@umich.eduonlyecho = False 1081307Sbinkertn@umich.eduexprs = [] 1091049Sbinkertn@umich.eduforce = False 1101269Sbinkertn@umich.edulistonly = False 1111269Sbinkertn@umich.eduqueue = '' 1121269Sbinkertn@umich.eduverbose = False 1131269Sbinkertn@umich.edujfile = 'Test.py' 1141269Sbinkertn@umich.edudocpts = False 1151269Sbinkertn@umich.edudoruns = True 1161162Sbinkertn@umich.edurunflag = False 1171049Sbinkertn@umich.edunode_type = 'FAST' 1181049Sbinkertn@umich.eduupdate = True 1191269Sbinkertn@umich.edu 1201269Sbinkertn@umich.edufor opt,arg in opts: 1211269Sbinkertn@umich.edu if opt == '-C': 1221269Sbinkertn@umich.edu docpts = True 1231269Sbinkertn@umich.edu if opt == '-c': 1241269Sbinkertn@umich.edu clean = True 1251269Sbinkertn@umich.edu if opt == '-d': 1261269Sbinkertn@umich.edu depend = True 1271269Sbinkertn@umich.edu if opt == '-e': 1281269Sbinkertn@umich.edu onlyecho = True 1291269Sbinkertn@umich.edu if opt == '-f': 1301269Sbinkertn@umich.edu force = True 1311269Sbinkertn@umich.edu if opt == '-h': 1321269Sbinkertn@umich.edu print usage 1331269Sbinkertn@umich.edu sys.exit(0) 1341269Sbinkertn@umich.edu if opt == '-j': 1351269Sbinkertn@umich.edu jfile = arg 1361269Sbinkertn@umich.edu if opt == '-l': 1371269Sbinkertn@umich.edu listonly = True 1381269Sbinkertn@umich.edu if opt == '-n': 1391269Sbinkertn@umich.edu update = False 1401269Sbinkertn@umich.edu if opt == '-q': 1411269Sbinkertn@umich.edu queue = arg 1421049Sbinkertn@umich.edu if opt == '-R': 1431269Sbinkertn@umich.edu runflag = True 1441269Sbinkertn@umich.edu if opt == '-t': 1451269Sbinkertn@umich.edu node_type = arg 1461049Sbinkertn@umich.edu if opt == '-v': 1471049Sbinkertn@umich.edu verbose = True 1481269Sbinkertn@umich.edu 1491049Sbinkertn@umich.eduif docpts: 1501269Sbinkertn@umich.edu doruns = runflag 1511269Sbinkertn@umich.edu 1521165Sbinkertn@umich.edufor arg in args: 1531049Sbinkertn@umich.edu exprs.append(re.compile(arg)) 1541307Sbinkertn@umich.edu 1551049Sbinkertn@umich.eduimport jobfile, pbs 1561307Sbinkertn@umich.edufrom job import JobDir, date 1571307Sbinkertn@umich.edu 1581049Sbinkertn@umich.educonf = jobfile.JobFile(jfile) 1591049Sbinkertn@umich.edu 1601049Sbinkertn@umich.eduif update and not listonly and not onlyecho and isdir(conf.linkdir): 1611049Sbinkertn@umich.edu if verbose: 1621269Sbinkertn@umich.edu print 'Checking for outdated files in Link directory' 1631049Sbinkertn@umich.edu if not isdir(conf.basedir): 1641049Sbinkertn@umich.edu os.mkdir(conf.basedir) 1651049Sbinkertn@umich.edu syncdir(conf.linkdir, conf.basedir) 1661049Sbinkertn@umich.edu 1671049Sbinkertn@umich.edujobnames = {} 1681049Sbinkertn@umich.edujoblist = [] 1691049Sbinkertn@umich.edu 1701049Sbinkertn@umich.eduif docpts and doruns: 1711049Sbinkertn@umich.edu gen = conf.alljobs() 1721049Sbinkertn@umich.eduelif docpts: 1731049Sbinkertn@umich.edu gen = conf.checkpoints() 1741049Sbinkertn@umich.eduelif doruns: 1751049Sbinkertn@umich.edu gen = conf.jobs() 1761049Sbinkertn@umich.edu 1771049Sbinkertn@umich.edufor job in gen: 1781049Sbinkertn@umich.edu if job.name in jobnames: 1791049Sbinkertn@umich.edu continue 1801049Sbinkertn@umich.edu 1811049Sbinkertn@umich.edu if exprs: 1821049Sbinkertn@umich.edu for expr in exprs: 1831049Sbinkertn@umich.edu if expr.match(job.name): 1841049Sbinkertn@umich.edu joblist.append(job) 1851049Sbinkertn@umich.edu break 1861049Sbinkertn@umich.edu else: 1871049Sbinkertn@umich.edu joblist.append(job) 1881049Sbinkertn@umich.edu 1891049Sbinkertn@umich.eduif listonly: 1901049Sbinkertn@umich.edu if verbose: 1911049Sbinkertn@umich.edu for job in joblist: 1921049Sbinkertn@umich.edu job.printinfo() 1931049Sbinkertn@umich.edu else: 1941049Sbinkertn@umich.edu for job in joblist: 1951049Sbinkertn@umich.edu print job.name 1961049Sbinkertn@umich.edu sys.exit(0) 1971049Sbinkertn@umich.edu 1981049Sbinkertn@umich.eduif not onlyecho: 1991049Sbinkertn@umich.edu newlist = [] 2001049Sbinkertn@umich.edu for job in joblist: 2011049Sbinkertn@umich.edu jobdir = JobDir(joinpath(conf.rootdir, job.name)) 2021049Sbinkertn@umich.edu if jobdir.exists(): 2031049Sbinkertn@umich.edu if not force: 2041049Sbinkertn@umich.edu status = jobdir.getstatus() 2051049Sbinkertn@umich.edu if status == 'queued': 2061049Sbinkertn@umich.edu continue 2071049Sbinkertn@umich.edu 2081049Sbinkertn@umich.edu if status == 'running': 2091049Sbinkertn@umich.edu continue 2101049Sbinkertn@umich.edu 2111049Sbinkertn@umich.edu if status == 'success': 2121049Sbinkertn@umich.edu continue 2131049Sbinkertn@umich.edu 2141049Sbinkertn@umich.edu if not clean: 2151049Sbinkertn@umich.edu sys.exit('job directory %s not clean!' % jobdir) 2161049Sbinkertn@umich.edu 2171049Sbinkertn@umich.edu jobdir.clean() 2181049Sbinkertn@umich.edu newlist.append(job) 2191049Sbinkertn@umich.edu joblist = newlist 2201049Sbinkertn@umich.edu 2211049Sbinkertn@umich.educlass NameHack(object): 2221049Sbinkertn@umich.edu def __init__(self, host='pbs.pool', port=24465): 2231049Sbinkertn@umich.edu self.host = host 2241049Sbinkertn@umich.edu self.port = port 2251049Sbinkertn@umich.edu self.socket = None 2261049Sbinkertn@umich.edu 2271049Sbinkertn@umich.edu def setname(self, jobid, jobname): 2281049Sbinkertn@umich.edu try: 2291049Sbinkertn@umich.edu jobid = int(jobid) 2301049Sbinkertn@umich.edu except ValueError: 2311049Sbinkertn@umich.edu jobid = int(jobid.strip().split('.')[0]) 2321049Sbinkertn@umich.edu 2331049Sbinkertn@umich.edu jobname = jobname.strip() 2341049Sbinkertn@umich.edu # since pbs can handle jobnames of 15 characters or less, 2351049Sbinkertn@umich.edu # don't use the raj hack. 2361049Sbinkertn@umich.edu if len(jobname) <= 15: 2371049Sbinkertn@umich.edu return 2381049Sbinkertn@umich.edu 2391049Sbinkertn@umich.edu if self.socket is None: 2401049Sbinkertn@umich.edu import socket 2411049Sbinkertn@umich.edu self.socket = socket.socket() 2421049Sbinkertn@umich.edu # Connect to pbs.pool and send the jobid/jobname pair to port 2431049Sbinkertn@umich.edu # 24465 (Raj didn't realize that there are only 64k ports and 2441049Sbinkertn@umich.edu # setup inetd to point to port 90001) 2451049Sbinkertn@umich.edu self.socket.connect((self.host, self.port)) 2461049Sbinkertn@umich.edu 2471049Sbinkertn@umich.edu self.socket.send("%s %s\n" % (jobid, jobname)) 2481049Sbinkertn@umich.edu 2491049Sbinkertn@umich.edunamehack = NameHack() 2501049Sbinkertn@umich.edu 2511049Sbinkertn@umich.edufor job in joblist: 2521309Ssaidi@eecs.umich.edu jobdir = JobDir(joinpath(conf.rootdir, job.name)) 2531309Ssaidi@eecs.umich.edu if depend: 2541309Ssaidi@eecs.umich.edu cptdir = JobDir(joinpath(conf.rootdir, job.checkpoint.name)) 2551309Ssaidi@eecs.umich.edu cptjob = cptdir.readval('.pbs_jobid') 2561309Ssaidi@eecs.umich.edu 2571309Ssaidi@eecs.umich.edu if not onlyecho: 2581309Ssaidi@eecs.umich.edu jobdir.create() 2591317Ssaidi@eecs.umich.edu 2601317Ssaidi@eecs.umich.edu print 'Job name: %s' % job.name 2611317Ssaidi@eecs.umich.edu print 'Job directory: %s' % jobdir 2621317Ssaidi@eecs.umich.edu 2631317Ssaidi@eecs.umich.edu qsub = pbs.qsub() 2641309Ssaidi@eecs.umich.edu qsub.pbshost = 'simpool.eecs.umich.edu' 2651309Ssaidi@eecs.umich.edu qsub.stdout = jobdir.file('jobout') 2661309Ssaidi@eecs.umich.edu qsub.name = job.name[:15] 2671309Ssaidi@eecs.umich.edu qsub.join = True 2681309Ssaidi@eecs.umich.edu qsub.node_type = node_type 2691309Ssaidi@eecs.umich.edu qsub.env['ROOTDIR'] = conf.rootdir 2701317Ssaidi@eecs.umich.edu qsub.env['JOBNAME'] = job.name 2711317Ssaidi@eecs.umich.edu if depend: 2721317Ssaidi@eecs.umich.edu qsub.afterok = cptjob 2731317Ssaidi@eecs.umich.edu if queue: 2741309Ssaidi@eecs.umich.edu qsub.queue = queue 2751309Ssaidi@eecs.umich.edu qsub.build(joinpath(progpath, 'job.py')) 2761309Ssaidi@eecs.umich.edu 2771309Ssaidi@eecs.umich.edu if verbose: 2781309Ssaidi@eecs.umich.edu print 'PBS Command: %s' % qsub.command 2791309Ssaidi@eecs.umich.edu 2801317Ssaidi@eecs.umich.edu if not onlyecho: 2811317Ssaidi@eecs.umich.edu ec = qsub.do() 2821309Ssaidi@eecs.umich.edu if ec == 0: 2831317Ssaidi@eecs.umich.edu jobid = qsub.result 2841317Ssaidi@eecs.umich.edu print 'PBS Jobid: %s' % jobid 2851317Ssaidi@eecs.umich.edu namehack.setname(jobid, job.name) 2861317Ssaidi@eecs.umich.edu queued = date() 2871317Ssaidi@eecs.umich.edu jobdir.echofile('.pbs_jobid', jobid) 2881317Ssaidi@eecs.umich.edu jobdir.echofile('.pbs_jobname', job.name) 2891317Ssaidi@eecs.umich.edu jobdir.echofile('.queued', queued) 2901309Ssaidi@eecs.umich.edu jobdir.setstatus('queued on %s' % queued) 2911317Ssaidi@eecs.umich.edu else: 2921317Ssaidi@eecs.umich.edu print 'PBS Failed' 2931317Ssaidi@eecs.umich.edu