send.py revision 1908
11376Sbinkertn@umich.edu#!/usr/bin/env python 21376Sbinkertn@umich.edu# Copyright (c) 2005 The Regents of The University of Michigan 31376Sbinkertn@umich.edu# All rights reserved. 41376Sbinkertn@umich.edu# 51376Sbinkertn@umich.edu# Redistribution and use in source and binary forms, with or without 61376Sbinkertn@umich.edu# modification, are permitted provided that the following conditions are 71376Sbinkertn@umich.edu# met: redistributions of source code must retain the above copyright 81376Sbinkertn@umich.edu# notice, this list of conditions and the following disclaimer; 91376Sbinkertn@umich.edu# redistributions in binary form must reproduce the above copyright 101376Sbinkertn@umich.edu# notice, this list of conditions and the following disclaimer in the 111376Sbinkertn@umich.edu# documentation and/or other materials provided with the distribution; 121376Sbinkertn@umich.edu# neither the name of the copyright holders nor the names of its 131376Sbinkertn@umich.edu# contributors may be used to endorse or promote products derived from 141376Sbinkertn@umich.edu# this software without specific prior written permission. 151376Sbinkertn@umich.edu# 161376Sbinkertn@umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 171376Sbinkertn@umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 181376Sbinkertn@umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 191376Sbinkertn@umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 201376Sbinkertn@umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 211376Sbinkertn@umich.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 221376Sbinkertn@umich.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 231376Sbinkertn@umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 241376Sbinkertn@umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 251376Sbinkertn@umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 261376Sbinkertn@umich.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 271376Sbinkertn@umich.edu# 281376Sbinkertn@umich.edu# Authors: Ali Saidi 291376Sbinkertn@umich.edu# Nathan Binkert 301376Sbinkertn@umich.edu 311385Sbinkertn@umich.eduimport os, os.path, re, socket, sys 321376Sbinkertn@umich.edufrom os import environ as env, listdir 331816Sbinkertn@umich.edufrom os.path import basename, isdir, isfile, islink, join as joinpath, normpath 341376Sbinkertn@umich.edufrom filecmp import cmp as filecmp 351816Sbinkertn@umich.edufrom shutil import copy 361376Sbinkertn@umich.edu 371385Sbinkertn@umich.edudef nfspath(dir): 381385Sbinkertn@umich.edu if dir.startswith('/.automount/'): 391385Sbinkertn@umich.edu dir = '/n/%s' % dir[12:] 401385Sbinkertn@umich.edu elif not dir.startswith('/n/'): 411385Sbinkertn@umich.edu dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir) 421385Sbinkertn@umich.edu return dir 431385Sbinkertn@umich.edu 441816Sbinkertn@umich.edudef syncdir(srcdir, destdir): 451816Sbinkertn@umich.edu srcdir = normpath(srcdir) 461816Sbinkertn@umich.edu destdir = normpath(destdir) 471816Sbinkertn@umich.edu if not isdir(destdir): 481816Sbinkertn@umich.edu sys.exit('destination directory "%s" does not exist' % destdir) 491816Sbinkertn@umich.edu 501816Sbinkertn@umich.edu for root, dirs, files in os.walk(srcdir): 511816Sbinkertn@umich.edu root = normpath(root) 521816Sbinkertn@umich.edu prefix = os.path.commonprefix([root, srcdir]) 531816Sbinkertn@umich.edu root = root[len(prefix):] 541816Sbinkertn@umich.edu if root.startswith('/'): 551816Sbinkertn@umich.edu root = root[1:] 561816Sbinkertn@umich.edu for rem in [ d for d in dirs if d.startswith('.') or d == 'SCCS']: 571816Sbinkertn@umich.edu dirs.remove(rem) 581816Sbinkertn@umich.edu 591816Sbinkertn@umich.edu for entry in dirs: 601816Sbinkertn@umich.edu newdir = joinpath(destdir, root, entry) 611816Sbinkertn@umich.edu if not isdir(newdir): 621816Sbinkertn@umich.edu os.mkdir(newdir) 631816Sbinkertn@umich.edu print 'mkdir', newdir 641816Sbinkertn@umich.edu 651816Sbinkertn@umich.edu for i,d in enumerate(dirs): 661816Sbinkertn@umich.edu if islink(joinpath(srcdir, root, d)): 671816Sbinkertn@umich.edu dirs[i] = joinpath(d, '.') 681816Sbinkertn@umich.edu 691816Sbinkertn@umich.edu for entry in files: 701816Sbinkertn@umich.edu dest = normpath(joinpath(destdir, root, entry)) 711816Sbinkertn@umich.edu src = normpath(joinpath(srcdir, root, entry)) 721816Sbinkertn@umich.edu if not isfile(dest) or not filecmp(src, dest): 731816Sbinkertn@umich.edu print 'copy %s %s' % (dest, src) 741816Sbinkertn@umich.edu copy(src, dest) 751816Sbinkertn@umich.edu 761385Sbinkertn@umich.eduprogpath = nfspath(sys.path[0]) 771376Sbinkertn@umich.eduprogname = basename(sys.argv[0]) 781376Sbinkertn@umich.eduusage = """\ 791376Sbinkertn@umich.eduUsage: 801602Sbinkertn@umich.edu %(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp> 811376Sbinkertn@umich.edu -c clean directory if job can be run 821376Sbinkertn@umich.edu -e only echo pbs command info, don't actually send the job 831376Sbinkertn@umich.edu -f force the job to run regardless of state 841376Sbinkertn@umich.edu -q <queue> submit job to the named queue 851602Sbinkertn@umich.edu -j <jobfile> specify the jobfile (default is <basedir>/test.py) 861376Sbinkertn@umich.edu -v be verbose 871376Sbinkertn@umich.edu 881602Sbinkertn@umich.edu %(progname)s [-j <jobfile>] -l [-v] <regexp> 891602Sbinkertn@umich.edu -j <jobfile> specify the jobfile (default is <basedir>/test.py) 901376Sbinkertn@umich.edu -l list job names, don't submit 911376Sbinkertn@umich.edu -v be verbose (list job parameters) 921376Sbinkertn@umich.edu 931376Sbinkertn@umich.edu %(progname)s -h 941376Sbinkertn@umich.edu -h display this help 951376Sbinkertn@umich.edu""" % locals() 961376Sbinkertn@umich.edu 971376Sbinkertn@umich.edutry: 981376Sbinkertn@umich.edu import getopt 991881Sbinkertn@umich.edu opts, args = getopt.getopt(sys.argv[1:], '-CRcd:efhj:lq:v') 1001376Sbinkertn@umich.eduexcept getopt.GetoptError: 1011376Sbinkertn@umich.edu sys.exit(usage) 1021376Sbinkertn@umich.edu 1031376Sbinkertn@umich.educlean = False 1041376Sbinkertn@umich.eduonlyecho = False 1051376Sbinkertn@umich.eduexprs = [] 1061376Sbinkertn@umich.eduforce = False 1071376Sbinkertn@umich.edulistonly = False 1081376Sbinkertn@umich.eduqueue = '' 1091376Sbinkertn@umich.eduverbose = False 1101881Sbinkertn@umich.edujfile = 'Base/test.py' 1111881Sbinkertn@umich.edudocpts = False 1121881Sbinkertn@umich.edudoruns = True 1131881Sbinkertn@umich.edurunflag = False 1141881Sbinkertn@umich.edu 1151381Sbinkertn@umich.edufor opt,arg in opts: 1161881Sbinkertn@umich.edu if opt == '-C': 1171881Sbinkertn@umich.edu docpts = True 1181881Sbinkertn@umich.edu if opt == '-R': 1191881Sbinkertn@umich.edu runflag = True 1201381Sbinkertn@umich.edu if opt == '-c': 1211376Sbinkertn@umich.edu clean = True 1221381Sbinkertn@umich.edu if opt == '-e': 1231376Sbinkertn@umich.edu onlyecho = True 1241381Sbinkertn@umich.edu if opt == '-f': 1251376Sbinkertn@umich.edu force = True 1261381Sbinkertn@umich.edu if opt == '-h': 1271376Sbinkertn@umich.edu print usage 1281376Sbinkertn@umich.edu sys.exit(0) 1291602Sbinkertn@umich.edu if opt == '-j': 1301602Sbinkertn@umich.edu jfile = arg 1311381Sbinkertn@umich.edu if opt == '-l': 1321376Sbinkertn@umich.edu listonly = True 1331381Sbinkertn@umich.edu if opt == '-q': 1341381Sbinkertn@umich.edu queue = arg 1351381Sbinkertn@umich.edu if opt == '-v': 1361376Sbinkertn@umich.edu verbose = True 1371376Sbinkertn@umich.edu 1381881Sbinkertn@umich.eduif docpts: 1391881Sbinkertn@umich.edu doruns = runflag 1401381Sbinkertn@umich.edu 1411376Sbinkertn@umich.edufor arg in args: 1421376Sbinkertn@umich.edu exprs.append(re.compile(arg)) 1431376Sbinkertn@umich.edu 1441881Sbinkertn@umich.eduimport jobfile, pbs 1451881Sbinkertn@umich.edufrom job import JobDir, date 1461881Sbinkertn@umich.edu 1471881Sbinkertn@umich.educonf = jobfile.JobFile(jfile) 1481881Sbinkertn@umich.edu 1491881Sbinkertn@umich.eduif not listonly and not onlyecho and isdir(conf.linkdir): 1501385Sbinkertn@umich.edu if verbose: 1511385Sbinkertn@umich.edu print 'Checking for outdated files in Link directory' 1521881Sbinkertn@umich.edu syncdir(conf.linkdir, conf.basedir) 1531376Sbinkertn@umich.edu 1541881Sbinkertn@umich.edujobnames = {} 1551881Sbinkertn@umich.edujoblist = [] 1561376Sbinkertn@umich.edu 1571881Sbinkertn@umich.eduif docpts and doruns: 1581881Sbinkertn@umich.edu gen = conf.alljobs() 1591881Sbinkertn@umich.eduelif docpts: 1601881Sbinkertn@umich.edu gen = conf.checkpoints() 1611881Sbinkertn@umich.eduelif doruns: 1621881Sbinkertn@umich.edu gen = conf.jobs() 1631376Sbinkertn@umich.edu 1641881Sbinkertn@umich.edufor job in gen: 1651881Sbinkertn@umich.edu if job.name in jobnames: 1661376Sbinkertn@umich.edu continue 1671376Sbinkertn@umich.edu 1681881Sbinkertn@umich.edu if exprs: 1691881Sbinkertn@umich.edu for expr in exprs: 1701881Sbinkertn@umich.edu if expr.match(job.name): 1711881Sbinkertn@umich.edu joblist.append(job) 1721881Sbinkertn@umich.edu break 1731881Sbinkertn@umich.edu else: 1741881Sbinkertn@umich.edu joblist.append(job) 1751376Sbinkertn@umich.edu 1761376Sbinkertn@umich.eduif listonly: 1771376Sbinkertn@umich.edu if verbose: 1781881Sbinkertn@umich.edu for job in joblist: 1791881Sbinkertn@umich.edu job.printinfo() 1801376Sbinkertn@umich.edu else: 1811881Sbinkertn@umich.edu for job in joblist: 1821881Sbinkertn@umich.edu print job.name 1831376Sbinkertn@umich.edu sys.exit(0) 1841376Sbinkertn@umich.edu 1851376Sbinkertn@umich.eduif not onlyecho: 1861881Sbinkertn@umich.edu newlist = [] 1871881Sbinkertn@umich.edu for job in joblist: 1881881Sbinkertn@umich.edu jobdir = JobDir(joinpath(conf.rootdir, job.name)) 1891881Sbinkertn@umich.edu if jobdir.exists(): 1901376Sbinkertn@umich.edu if not force: 1911881Sbinkertn@umich.edu status = jobdir.getstatus() 1921881Sbinkertn@umich.edu if status == 'queued': 1931376Sbinkertn@umich.edu continue 1941376Sbinkertn@umich.edu 1951881Sbinkertn@umich.edu if status == 'running': 1961881Sbinkertn@umich.edu continue 1971881Sbinkertn@umich.edu 1981881Sbinkertn@umich.edu if status == 'success': 1991376Sbinkertn@umich.edu continue 2001376Sbinkertn@umich.edu 2011376Sbinkertn@umich.edu if not clean: 2021881Sbinkertn@umich.edu sys.exit('job directory %s not clean!' % jobdir) 2031376Sbinkertn@umich.edu 2041881Sbinkertn@umich.edu jobdir.clean() 2051881Sbinkertn@umich.edu newlist.append(job) 2061881Sbinkertn@umich.edu joblist = newlist 2071376Sbinkertn@umich.edu 2081881Sbinkertn@umich.educlass NameHack(object): 2091881Sbinkertn@umich.edu def __init__(self, host='pbs.pool', port=24465): 2101881Sbinkertn@umich.edu self.host = host 2111881Sbinkertn@umich.edu self.port = port 2121881Sbinkertn@umich.edu self.socket = None 2131816Sbinkertn@umich.edu 2141881Sbinkertn@umich.edu def setname(self, jobid, jobname): 2151881Sbinkertn@umich.edu try: 2161881Sbinkertn@umich.edu jobid = int(jobid) 2171881Sbinkertn@umich.edu except ValueError: 2181881Sbinkertn@umich.edu jobid = int(jobid.strip().split('.')[0]) 2191816Sbinkertn@umich.edu 2201881Sbinkertn@umich.edu jobname = jobname.strip() 2211881Sbinkertn@umich.edu # since pbs can handle jobnames of 15 characters or less, 2221881Sbinkertn@umich.edu # don't use the raj hack. 2231881Sbinkertn@umich.edu if len(jobname) <= 15: 2241881Sbinkertn@umich.edu return 2251376Sbinkertn@umich.edu 2261881Sbinkertn@umich.edu if self.socket is None: 2271881Sbinkertn@umich.edu import socket 2281881Sbinkertn@umich.edu self.socket = socket.socket() 2291881Sbinkertn@umich.edu # Connect to pbs.pool and send the jobid/jobname pair to port 2301881Sbinkertn@umich.edu # 24465 (Raj didn't realize that there are only 64k ports and 2311881Sbinkertn@umich.edu # setup inetd to point to port 90001) 2321881Sbinkertn@umich.edu self.socket.connect((self.host, self.port)) 2331376Sbinkertn@umich.edu 2341881Sbinkertn@umich.edu self.socket.send("%s %s\n" % (jobid, jobname)) 2351881Sbinkertn@umich.edu 2361881Sbinkertn@umich.edunamehack = NameHack() 2371881Sbinkertn@umich.edu 2381881Sbinkertn@umich.edufor job in joblist: 2391881Sbinkertn@umich.edu jobdir = JobDir(joinpath(conf.rootdir, job.name)) 2401881Sbinkertn@umich.edu 2411881Sbinkertn@umich.edu if not onlyecho: 2421881Sbinkertn@umich.edu jobdir.create() 2431881Sbinkertn@umich.edu 2441881Sbinkertn@umich.edu print 'Job name: %s' % job.name 2451385Sbinkertn@umich.edu print 'Job directory: %s' % jobdir 2461376Sbinkertn@umich.edu 2471376Sbinkertn@umich.edu qsub = pbs.qsub() 2481376Sbinkertn@umich.edu qsub.pbshost = 'simpool.eecs.umich.edu' 2491881Sbinkertn@umich.edu qsub.stdout = jobdir.file('jobout') 2501881Sbinkertn@umich.edu qsub.name = job.name[:15] 2511376Sbinkertn@umich.edu qsub.join = True 2521376Sbinkertn@umich.edu qsub.node_type = 'FAST' 2531881Sbinkertn@umich.edu qsub.env['ROOTDIR'] = conf.rootdir 2541881Sbinkertn@umich.edu qsub.env['JOBNAME'] = job.name 2551376Sbinkertn@umich.edu if len(queue): 2561376Sbinkertn@umich.edu qsub.queue = queue 2571385Sbinkertn@umich.edu qsub.build(joinpath(progpath, 'job.py')) 2581376Sbinkertn@umich.edu 2591385Sbinkertn@umich.edu if verbose: 2601385Sbinkertn@umich.edu print 'PBS Command: %s' % qsub.command 2611385Sbinkertn@umich.edu 2621385Sbinkertn@umich.edu if not onlyecho: 2631385Sbinkertn@umich.edu ec = qsub.do() 2641385Sbinkertn@umich.edu if ec == 0: 2651816Sbinkertn@umich.edu jobid = qsub.result 2661816Sbinkertn@umich.edu print 'PBS Jobid: %s' % jobid 2671881Sbinkertn@umich.edu namehack.setname(jobid, job.name) 2681881Sbinkertn@umich.edu queued = date() 2691908Sbinkertn@umich.edu jobdir.echofile('.pbs_jobid', jobid) 2701908Sbinkertn@umich.edu jobdir.echofile('.pbs_jobname', job.name) 2711881Sbinkertn@umich.edu jobdir.echofile('.queued', queued) 2721881Sbinkertn@umich.edu jobdir.setstatus('queued on %s' % queued) 2731385Sbinkertn@umich.edu else: 2741385Sbinkertn@umich.edu print 'PBS Failed' 275