send.py revision 1916
11376Sbinkertn@umich.edu#!/usr/bin/env python 21376Sbinkertn@umich.edu# Copyright (c) 2005 The Regents of The University of Michigan 31376Sbinkertn@umich.edu# All rights reserved. 41376Sbinkertn@umich.edu# 51376Sbinkertn@umich.edu# Redistribution and use in source and binary forms, with or without 61376Sbinkertn@umich.edu# modification, are permitted provided that the following conditions are 71376Sbinkertn@umich.edu# met: redistributions of source code must retain the above copyright 81376Sbinkertn@umich.edu# notice, this list of conditions and the following disclaimer; 91376Sbinkertn@umich.edu# redistributions in binary form must reproduce the above copyright 101376Sbinkertn@umich.edu# notice, this list of conditions and the following disclaimer in the 111376Sbinkertn@umich.edu# documentation and/or other materials provided with the distribution; 121376Sbinkertn@umich.edu# neither the name of the copyright holders nor the names of its 131376Sbinkertn@umich.edu# contributors may be used to endorse or promote products derived from 141376Sbinkertn@umich.edu# this software without specific prior written permission. 151376Sbinkertn@umich.edu# 161376Sbinkertn@umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 171376Sbinkertn@umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 181376Sbinkertn@umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 191376Sbinkertn@umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 201376Sbinkertn@umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 211376Sbinkertn@umich.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 221376Sbinkertn@umich.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 231376Sbinkertn@umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 241376Sbinkertn@umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 251376Sbinkertn@umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 261376Sbinkertn@umich.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 271376Sbinkertn@umich.edu# 281376Sbinkertn@umich.edu# Authors: Ali Saidi 291376Sbinkertn@umich.edu# Nathan Binkert 301376Sbinkertn@umich.edu 311385Sbinkertn@umich.eduimport os, os.path, re, socket, sys 321376Sbinkertn@umich.edufrom os import environ as env, listdir 331816Sbinkertn@umich.edufrom os.path import basename, isdir, isfile, islink, join as joinpath, normpath 341376Sbinkertn@umich.edufrom filecmp import cmp as filecmp 351816Sbinkertn@umich.edufrom shutil import copy 361376Sbinkertn@umich.edu 371385Sbinkertn@umich.edudef nfspath(dir): 381385Sbinkertn@umich.edu if dir.startswith('/.automount/'): 391385Sbinkertn@umich.edu dir = '/n/%s' % dir[12:] 401385Sbinkertn@umich.edu elif not dir.startswith('/n/'): 411385Sbinkertn@umich.edu dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir) 421385Sbinkertn@umich.edu return dir 431385Sbinkertn@umich.edu 441816Sbinkertn@umich.edudef syncdir(srcdir, destdir): 451816Sbinkertn@umich.edu srcdir = normpath(srcdir) 461816Sbinkertn@umich.edu destdir = normpath(destdir) 471816Sbinkertn@umich.edu if not isdir(destdir): 481816Sbinkertn@umich.edu sys.exit('destination directory "%s" does not exist' % destdir) 491816Sbinkertn@umich.edu 501816Sbinkertn@umich.edu for root, dirs, files in os.walk(srcdir): 511816Sbinkertn@umich.edu root = normpath(root) 521816Sbinkertn@umich.edu prefix = os.path.commonprefix([root, srcdir]) 531816Sbinkertn@umich.edu root = root[len(prefix):] 541816Sbinkertn@umich.edu if root.startswith('/'): 551816Sbinkertn@umich.edu root = root[1:] 561816Sbinkertn@umich.edu for rem in [ d for d in dirs if d.startswith('.') or d == 'SCCS']: 571816Sbinkertn@umich.edu dirs.remove(rem) 581816Sbinkertn@umich.edu 591816Sbinkertn@umich.edu for entry in dirs: 601816Sbinkertn@umich.edu newdir = joinpath(destdir, root, entry) 611816Sbinkertn@umich.edu if not isdir(newdir): 621816Sbinkertn@umich.edu os.mkdir(newdir) 631816Sbinkertn@umich.edu print 'mkdir', newdir 641816Sbinkertn@umich.edu 651816Sbinkertn@umich.edu for i,d in enumerate(dirs): 661816Sbinkertn@umich.edu if islink(joinpath(srcdir, root, d)): 671816Sbinkertn@umich.edu dirs[i] = joinpath(d, '.') 681816Sbinkertn@umich.edu 691816Sbinkertn@umich.edu for entry in files: 701816Sbinkertn@umich.edu dest = normpath(joinpath(destdir, root, entry)) 711816Sbinkertn@umich.edu src = normpath(joinpath(srcdir, root, entry)) 721816Sbinkertn@umich.edu if not isfile(dest) or not filecmp(src, dest): 731816Sbinkertn@umich.edu print 'copy %s %s' % (dest, src) 741816Sbinkertn@umich.edu copy(src, dest) 751816Sbinkertn@umich.edu 761385Sbinkertn@umich.eduprogpath = nfspath(sys.path[0]) 771376Sbinkertn@umich.eduprogname = basename(sys.argv[0]) 781376Sbinkertn@umich.eduusage = """\ 791376Sbinkertn@umich.eduUsage: 801602Sbinkertn@umich.edu %(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp> 811376Sbinkertn@umich.edu -c clean directory if job can be run 821376Sbinkertn@umich.edu -e only echo pbs command info, don't actually send the job 831376Sbinkertn@umich.edu -f force the job to run regardless of state 841376Sbinkertn@umich.edu -q <queue> submit job to the named queue 851916Sbinkertn@umich.edu -j <jobfile> specify the jobfile (default is <rootdir>/Test.py) 861376Sbinkertn@umich.edu -v be verbose 871376Sbinkertn@umich.edu 881602Sbinkertn@umich.edu %(progname)s [-j <jobfile>] -l [-v] <regexp> 891916Sbinkertn@umich.edu -j <jobfile> specify the jobfile (default is <rootdir>/Test.py) 901376Sbinkertn@umich.edu -l list job names, don't submit 911376Sbinkertn@umich.edu -v be verbose (list job parameters) 921376Sbinkertn@umich.edu 931376Sbinkertn@umich.edu %(progname)s -h 941376Sbinkertn@umich.edu -h display this help 951376Sbinkertn@umich.edu""" % locals() 961376Sbinkertn@umich.edu 971376Sbinkertn@umich.edutry: 981376Sbinkertn@umich.edu import getopt 991916Sbinkertn@umich.edu opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lq:Rt:v') 1001376Sbinkertn@umich.eduexcept getopt.GetoptError: 1011376Sbinkertn@umich.edu sys.exit(usage) 1021376Sbinkertn@umich.edu 1031916Sbinkertn@umich.edudepend = False 1041376Sbinkertn@umich.educlean = False 1051376Sbinkertn@umich.eduonlyecho = False 1061376Sbinkertn@umich.eduexprs = [] 1071376Sbinkertn@umich.eduforce = False 1081376Sbinkertn@umich.edulistonly = False 1091376Sbinkertn@umich.eduqueue = '' 1101376Sbinkertn@umich.eduverbose = False 1111916Sbinkertn@umich.edujfile = 'Test.py' 1121881Sbinkertn@umich.edudocpts = False 1131881Sbinkertn@umich.edudoruns = True 1141881Sbinkertn@umich.edurunflag = False 1151916Sbinkertn@umich.edunode_type = 'FAST' 1161881Sbinkertn@umich.edu 1171381Sbinkertn@umich.edufor opt,arg in opts: 1181881Sbinkertn@umich.edu if opt == '-C': 1191881Sbinkertn@umich.edu docpts = True 1201381Sbinkertn@umich.edu if opt == '-c': 1211376Sbinkertn@umich.edu clean = True 1221916Sbinkertn@umich.edu if opt == '-d': 1231916Sbinkertn@umich.edu depend = True 1241381Sbinkertn@umich.edu if opt == '-e': 1251376Sbinkertn@umich.edu onlyecho = True 1261381Sbinkertn@umich.edu if opt == '-f': 1271376Sbinkertn@umich.edu force = True 1281381Sbinkertn@umich.edu if opt == '-h': 1291376Sbinkertn@umich.edu print usage 1301376Sbinkertn@umich.edu sys.exit(0) 1311602Sbinkertn@umich.edu if opt == '-j': 1321602Sbinkertn@umich.edu jfile = arg 1331381Sbinkertn@umich.edu if opt == '-l': 1341376Sbinkertn@umich.edu listonly = True 1351381Sbinkertn@umich.edu if opt == '-q': 1361381Sbinkertn@umich.edu queue = arg 1371916Sbinkertn@umich.edu if opt == '-R': 1381916Sbinkertn@umich.edu runflag = True 1391916Sbinkertn@umich.edu if opt == '-t': 1401916Sbinkertn@umich.edu node_type = arg 1411381Sbinkertn@umich.edu if opt == '-v': 1421376Sbinkertn@umich.edu verbose = True 1431376Sbinkertn@umich.edu 1441881Sbinkertn@umich.eduif docpts: 1451881Sbinkertn@umich.edu doruns = runflag 1461381Sbinkertn@umich.edu 1471376Sbinkertn@umich.edufor arg in args: 1481376Sbinkertn@umich.edu exprs.append(re.compile(arg)) 1491376Sbinkertn@umich.edu 1501881Sbinkertn@umich.eduimport jobfile, pbs 1511881Sbinkertn@umich.edufrom job import JobDir, date 1521881Sbinkertn@umich.edu 1531881Sbinkertn@umich.educonf = jobfile.JobFile(jfile) 1541881Sbinkertn@umich.edu 1551881Sbinkertn@umich.eduif not listonly and not onlyecho and isdir(conf.linkdir): 1561385Sbinkertn@umich.edu if verbose: 1571385Sbinkertn@umich.edu print 'Checking for outdated files in Link directory' 1581916Sbinkertn@umich.edu if not isdir(conf.basedir): 1591916Sbinkertn@umich.edu os.mkdir(conf.basedir) 1601881Sbinkertn@umich.edu syncdir(conf.linkdir, conf.basedir) 1611376Sbinkertn@umich.edu 1621881Sbinkertn@umich.edujobnames = {} 1631881Sbinkertn@umich.edujoblist = [] 1641376Sbinkertn@umich.edu 1651881Sbinkertn@umich.eduif docpts and doruns: 1661881Sbinkertn@umich.edu gen = conf.alljobs() 1671881Sbinkertn@umich.eduelif docpts: 1681881Sbinkertn@umich.edu gen = conf.checkpoints() 1691881Sbinkertn@umich.eduelif doruns: 1701881Sbinkertn@umich.edu gen = conf.jobs() 1711376Sbinkertn@umich.edu 1721881Sbinkertn@umich.edufor job in gen: 1731881Sbinkertn@umich.edu if job.name in jobnames: 1741376Sbinkertn@umich.edu continue 1751376Sbinkertn@umich.edu 1761881Sbinkertn@umich.edu if exprs: 1771881Sbinkertn@umich.edu for expr in exprs: 1781881Sbinkertn@umich.edu if expr.match(job.name): 1791881Sbinkertn@umich.edu joblist.append(job) 1801881Sbinkertn@umich.edu break 1811881Sbinkertn@umich.edu else: 1821881Sbinkertn@umich.edu joblist.append(job) 1831376Sbinkertn@umich.edu 1841376Sbinkertn@umich.eduif listonly: 1851376Sbinkertn@umich.edu if verbose: 1861881Sbinkertn@umich.edu for job in joblist: 1871881Sbinkertn@umich.edu job.printinfo() 1881376Sbinkertn@umich.edu else: 1891881Sbinkertn@umich.edu for job in joblist: 1901881Sbinkertn@umich.edu print job.name 1911376Sbinkertn@umich.edu sys.exit(0) 1921376Sbinkertn@umich.edu 1931376Sbinkertn@umich.eduif not onlyecho: 1941881Sbinkertn@umich.edu newlist = [] 1951881Sbinkertn@umich.edu for job in joblist: 1961881Sbinkertn@umich.edu jobdir = JobDir(joinpath(conf.rootdir, job.name)) 1971881Sbinkertn@umich.edu if jobdir.exists(): 1981376Sbinkertn@umich.edu if not force: 1991881Sbinkertn@umich.edu status = jobdir.getstatus() 2001881Sbinkertn@umich.edu if status == 'queued': 2011376Sbinkertn@umich.edu continue 2021376Sbinkertn@umich.edu 2031881Sbinkertn@umich.edu if status == 'running': 2041881Sbinkertn@umich.edu continue 2051881Sbinkertn@umich.edu 2061881Sbinkertn@umich.edu if status == 'success': 2071376Sbinkertn@umich.edu continue 2081376Sbinkertn@umich.edu 2091376Sbinkertn@umich.edu if not clean: 2101881Sbinkertn@umich.edu sys.exit('job directory %s not clean!' % jobdir) 2111376Sbinkertn@umich.edu 2121881Sbinkertn@umich.edu jobdir.clean() 2131881Sbinkertn@umich.edu newlist.append(job) 2141881Sbinkertn@umich.edu joblist = newlist 2151376Sbinkertn@umich.edu 2161881Sbinkertn@umich.educlass NameHack(object): 2171881Sbinkertn@umich.edu def __init__(self, host='pbs.pool', port=24465): 2181881Sbinkertn@umich.edu self.host = host 2191881Sbinkertn@umich.edu self.port = port 2201881Sbinkertn@umich.edu self.socket = None 2211816Sbinkertn@umich.edu 2221881Sbinkertn@umich.edu def setname(self, jobid, jobname): 2231881Sbinkertn@umich.edu try: 2241881Sbinkertn@umich.edu jobid = int(jobid) 2251881Sbinkertn@umich.edu except ValueError: 2261881Sbinkertn@umich.edu jobid = int(jobid.strip().split('.')[0]) 2271816Sbinkertn@umich.edu 2281881Sbinkertn@umich.edu jobname = jobname.strip() 2291881Sbinkertn@umich.edu # since pbs can handle jobnames of 15 characters or less, 2301881Sbinkertn@umich.edu # don't use the raj hack. 2311881Sbinkertn@umich.edu if len(jobname) <= 15: 2321881Sbinkertn@umich.edu return 2331376Sbinkertn@umich.edu 2341881Sbinkertn@umich.edu if self.socket is None: 2351881Sbinkertn@umich.edu import socket 2361881Sbinkertn@umich.edu self.socket = socket.socket() 2371881Sbinkertn@umich.edu # Connect to pbs.pool and send the jobid/jobname pair to port 2381881Sbinkertn@umich.edu # 24465 (Raj didn't realize that there are only 64k ports and 2391881Sbinkertn@umich.edu # setup inetd to point to port 90001) 2401881Sbinkertn@umich.edu self.socket.connect((self.host, self.port)) 2411376Sbinkertn@umich.edu 2421881Sbinkertn@umich.edu self.socket.send("%s %s\n" % (jobid, jobname)) 2431881Sbinkertn@umich.edu 2441881Sbinkertn@umich.edunamehack = NameHack() 2451881Sbinkertn@umich.edu 2461881Sbinkertn@umich.edufor job in joblist: 2471881Sbinkertn@umich.edu jobdir = JobDir(joinpath(conf.rootdir, job.name)) 2481916Sbinkertn@umich.edu if depend: 2491916Sbinkertn@umich.edu cptdir = JobDir(joinpath(conf.rootdir, job.checkpoint.name)) 2501916Sbinkertn@umich.edu cptjob = cptdir.readval('.pbs_jobid') 2511881Sbinkertn@umich.edu 2521881Sbinkertn@umich.edu if not onlyecho: 2531881Sbinkertn@umich.edu jobdir.create() 2541881Sbinkertn@umich.edu 2551881Sbinkertn@umich.edu print 'Job name: %s' % job.name 2561385Sbinkertn@umich.edu print 'Job directory: %s' % jobdir 2571376Sbinkertn@umich.edu 2581376Sbinkertn@umich.edu qsub = pbs.qsub() 2591376Sbinkertn@umich.edu qsub.pbshost = 'simpool.eecs.umich.edu' 2601881Sbinkertn@umich.edu qsub.stdout = jobdir.file('jobout') 2611881Sbinkertn@umich.edu qsub.name = job.name[:15] 2621376Sbinkertn@umich.edu qsub.join = True 2631916Sbinkertn@umich.edu qsub.node_type = node_type 2641881Sbinkertn@umich.edu qsub.env['ROOTDIR'] = conf.rootdir 2651881Sbinkertn@umich.edu qsub.env['JOBNAME'] = job.name 2661916Sbinkertn@umich.edu if depend: 2671916Sbinkertn@umich.edu qsub.afterok = cptjob 2681916Sbinkertn@umich.edu if queue: 2691376Sbinkertn@umich.edu qsub.queue = queue 2701385Sbinkertn@umich.edu qsub.build(joinpath(progpath, 'job.py')) 2711376Sbinkertn@umich.edu 2721385Sbinkertn@umich.edu if verbose: 2731385Sbinkertn@umich.edu print 'PBS Command: %s' % qsub.command 2741385Sbinkertn@umich.edu 2751385Sbinkertn@umich.edu if not onlyecho: 2761385Sbinkertn@umich.edu ec = qsub.do() 2771385Sbinkertn@umich.edu if ec == 0: 2781816Sbinkertn@umich.edu jobid = qsub.result 2791816Sbinkertn@umich.edu print 'PBS Jobid: %s' % jobid 2801881Sbinkertn@umich.edu namehack.setname(jobid, job.name) 2811881Sbinkertn@umich.edu queued = date() 2821908Sbinkertn@umich.edu jobdir.echofile('.pbs_jobid', jobid) 2831908Sbinkertn@umich.edu jobdir.echofile('.pbs_jobname', job.name) 2841881Sbinkertn@umich.edu jobdir.echofile('.queued', queued) 2851881Sbinkertn@umich.edu jobdir.setstatus('queued on %s' % queued) 2861385Sbinkertn@umich.edu else: 2871385Sbinkertn@umich.edu print 'PBS Failed' 288