113540Sandrea.mondelli@ucf.edu#!/usr/bin/env python2.7 21376Sbinkertn@umich.edu# Copyright (c) 2005 The Regents of The University of Michigan 31376Sbinkertn@umich.edu# All rights reserved. 41376Sbinkertn@umich.edu# 51376Sbinkertn@umich.edu# Redistribution and use in source and binary forms, with or without 61376Sbinkertn@umich.edu# modification, are permitted provided that the following conditions are 71376Sbinkertn@umich.edu# met: redistributions of source code must retain the above copyright 81376Sbinkertn@umich.edu# notice, this list of conditions and the following disclaimer; 91376Sbinkertn@umich.edu# redistributions in binary form must reproduce the above copyright 101376Sbinkertn@umich.edu# notice, this list of conditions and the following disclaimer in the 111376Sbinkertn@umich.edu# documentation and/or other materials provided with the distribution; 121376Sbinkertn@umich.edu# neither the name of the copyright holders nor the names of its 131376Sbinkertn@umich.edu# contributors may be used to endorse or promote products derived from 141376Sbinkertn@umich.edu# this software without specific prior written permission. 151376Sbinkertn@umich.edu# 161376Sbinkertn@umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 171376Sbinkertn@umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 181376Sbinkertn@umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 191376Sbinkertn@umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 201376Sbinkertn@umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 211376Sbinkertn@umich.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 221376Sbinkertn@umich.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 231376Sbinkertn@umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 241376Sbinkertn@umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 251376Sbinkertn@umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 261376Sbinkertn@umich.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 271376Sbinkertn@umich.edu# 281376Sbinkertn@umich.edu# Authors: Ali Saidi 291376Sbinkertn@umich.edu# Nathan Binkert 301376Sbinkertn@umich.edu 311385Sbinkertn@umich.eduimport os, os.path, re, socket, sys 321376Sbinkertn@umich.edufrom os import environ as env, listdir 331816Sbinkertn@umich.edufrom os.path import basename, isdir, isfile, islink, join as joinpath, normpath 341376Sbinkertn@umich.edufrom filecmp import cmp as filecmp 351816Sbinkertn@umich.edufrom shutil import copy 361376Sbinkertn@umich.edu 371385Sbinkertn@umich.edudef nfspath(dir): 381385Sbinkertn@umich.edu if dir.startswith('/.automount/'): 391385Sbinkertn@umich.edu dir = '/n/%s' % dir[12:] 401385Sbinkertn@umich.edu elif not dir.startswith('/n/'): 411385Sbinkertn@umich.edu dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir) 421385Sbinkertn@umich.edu return dir 431385Sbinkertn@umich.edu 441816Sbinkertn@umich.edudef syncdir(srcdir, destdir): 451816Sbinkertn@umich.edu srcdir = normpath(srcdir) 461816Sbinkertn@umich.edu destdir = normpath(destdir) 471816Sbinkertn@umich.edu if not isdir(destdir): 481816Sbinkertn@umich.edu sys.exit('destination directory "%s" does not exist' % destdir) 491816Sbinkertn@umich.edu 501816Sbinkertn@umich.edu for root, dirs, files in os.walk(srcdir): 511816Sbinkertn@umich.edu root = normpath(root) 521816Sbinkertn@umich.edu prefix = os.path.commonprefix([root, srcdir]) 531816Sbinkertn@umich.edu root = root[len(prefix):] 541816Sbinkertn@umich.edu if root.startswith('/'): 551816Sbinkertn@umich.edu root = root[1:] 561816Sbinkertn@umich.edu for rem in [ d for d in dirs if d.startswith('.') or d == 'SCCS']: 571816Sbinkertn@umich.edu dirs.remove(rem) 581816Sbinkertn@umich.edu 591816Sbinkertn@umich.edu for entry in dirs: 601816Sbinkertn@umich.edu newdir = joinpath(destdir, root, entry) 611816Sbinkertn@umich.edu if not isdir(newdir): 621816Sbinkertn@umich.edu os.mkdir(newdir) 631816Sbinkertn@umich.edu print 'mkdir', newdir 641816Sbinkertn@umich.edu 651816Sbinkertn@umich.edu for i,d in enumerate(dirs): 661816Sbinkertn@umich.edu if islink(joinpath(srcdir, root, d)): 671816Sbinkertn@umich.edu dirs[i] = joinpath(d, '.') 681816Sbinkertn@umich.edu 691816Sbinkertn@umich.edu for entry in files: 701816Sbinkertn@umich.edu dest = normpath(joinpath(destdir, root, entry)) 711816Sbinkertn@umich.edu src = normpath(joinpath(srcdir, root, entry)) 721816Sbinkertn@umich.edu if not isfile(dest) or not filecmp(src, dest): 731816Sbinkertn@umich.edu print 'copy %s %s' % (dest, src) 741816Sbinkertn@umich.edu copy(src, dest) 751816Sbinkertn@umich.edu 761385Sbinkertn@umich.eduprogpath = nfspath(sys.path[0]) 771376Sbinkertn@umich.eduprogname = basename(sys.argv[0]) 781376Sbinkertn@umich.eduusage = """\ 791376Sbinkertn@umich.eduUsage: 801602Sbinkertn@umich.edu %(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp> 811376Sbinkertn@umich.edu -c clean directory if job can be run 821956Ssaidi@eecs.umich.edu -C submit the checkpointing runs 831956Ssaidi@eecs.umich.edu -d Make jobs be dependent on the completion of the checkpoint runs 841376Sbinkertn@umich.edu -e only echo pbs command info, don't actually send the job 851376Sbinkertn@umich.edu -f force the job to run regardless of state 861376Sbinkertn@umich.edu -q <queue> submit job to the named queue 871916Sbinkertn@umich.edu -j <jobfile> specify the jobfile (default is <rootdir>/Test.py) 881376Sbinkertn@umich.edu -v be verbose 891376Sbinkertn@umich.edu 901602Sbinkertn@umich.edu %(progname)s [-j <jobfile>] -l [-v] <regexp> 911916Sbinkertn@umich.edu -j <jobfile> specify the jobfile (default is <rootdir>/Test.py) 921376Sbinkertn@umich.edu -l list job names, don't submit 931376Sbinkertn@umich.edu -v be verbose (list job parameters) 941376Sbinkertn@umich.edu 951376Sbinkertn@umich.edu %(progname)s -h 961376Sbinkertn@umich.edu -h display this help 971376Sbinkertn@umich.edu""" % locals() 981376Sbinkertn@umich.edu 991376Sbinkertn@umich.edutry: 1001376Sbinkertn@umich.edu import getopt 1011948Sbinkertn@umich.edu opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lnq:Rt:v') 1021376Sbinkertn@umich.eduexcept getopt.GetoptError: 1031376Sbinkertn@umich.edu sys.exit(usage) 1041376Sbinkertn@umich.edu 1051916Sbinkertn@umich.edudepend = False 1061376Sbinkertn@umich.educlean = False 1071376Sbinkertn@umich.eduonlyecho = False 1081376Sbinkertn@umich.eduexprs = [] 1091376Sbinkertn@umich.eduforce = False 1101376Sbinkertn@umich.edulistonly = False 1111376Sbinkertn@umich.eduqueue = '' 1121376Sbinkertn@umich.eduverbose = False 1131916Sbinkertn@umich.edujfile = 'Test.py' 1141881Sbinkertn@umich.edudocpts = False 1151881Sbinkertn@umich.edudoruns = True 1161881Sbinkertn@umich.edurunflag = False 1171916Sbinkertn@umich.edunode_type = 'FAST' 1181948Sbinkertn@umich.eduupdate = True 1191881Sbinkertn@umich.edu 1201381Sbinkertn@umich.edufor opt,arg in opts: 1211881Sbinkertn@umich.edu if opt == '-C': 1221881Sbinkertn@umich.edu docpts = True 1231381Sbinkertn@umich.edu if opt == '-c': 1241376Sbinkertn@umich.edu clean = True 1251916Sbinkertn@umich.edu if opt == '-d': 1261916Sbinkertn@umich.edu depend = True 1271381Sbinkertn@umich.edu if opt == '-e': 1281376Sbinkertn@umich.edu onlyecho = True 1291381Sbinkertn@umich.edu if opt == '-f': 1301376Sbinkertn@umich.edu force = True 1311381Sbinkertn@umich.edu if opt == '-h': 1321376Sbinkertn@umich.edu print usage 1331376Sbinkertn@umich.edu sys.exit(0) 1341602Sbinkertn@umich.edu if opt == '-j': 1351602Sbinkertn@umich.edu jfile = arg 1361381Sbinkertn@umich.edu if opt == '-l': 1371376Sbinkertn@umich.edu listonly = True 1381948Sbinkertn@umich.edu if opt == '-n': 1391948Sbinkertn@umich.edu update = False 1401381Sbinkertn@umich.edu if opt == '-q': 1411381Sbinkertn@umich.edu queue = arg 1421916Sbinkertn@umich.edu if opt == '-R': 1431916Sbinkertn@umich.edu runflag = True 1441916Sbinkertn@umich.edu if opt == '-t': 1451916Sbinkertn@umich.edu node_type = arg 1461381Sbinkertn@umich.edu if opt == '-v': 1471376Sbinkertn@umich.edu verbose = True 1481376Sbinkertn@umich.edu 1491881Sbinkertn@umich.eduif docpts: 1501881Sbinkertn@umich.edu doruns = runflag 1511381Sbinkertn@umich.edu 1521376Sbinkertn@umich.edufor arg in args: 1531376Sbinkertn@umich.edu exprs.append(re.compile(arg)) 1541376Sbinkertn@umich.edu 1551881Sbinkertn@umich.eduimport jobfile, pbs 1561881Sbinkertn@umich.edufrom job import JobDir, date 1571881Sbinkertn@umich.edu 1581881Sbinkertn@umich.educonf = jobfile.JobFile(jfile) 1591881Sbinkertn@umich.edu 1601948Sbinkertn@umich.eduif update and not listonly and not onlyecho and isdir(conf.linkdir): 1611385Sbinkertn@umich.edu if verbose: 1621385Sbinkertn@umich.edu print 'Checking for outdated files in Link directory' 1631916Sbinkertn@umich.edu if not isdir(conf.basedir): 1641916Sbinkertn@umich.edu os.mkdir(conf.basedir) 1651881Sbinkertn@umich.edu syncdir(conf.linkdir, conf.basedir) 1661376Sbinkertn@umich.edu 1671881Sbinkertn@umich.edujobnames = {} 1681881Sbinkertn@umich.edujoblist = [] 1691376Sbinkertn@umich.edu 1701881Sbinkertn@umich.eduif docpts and doruns: 1711881Sbinkertn@umich.edu gen = conf.alljobs() 1721881Sbinkertn@umich.eduelif docpts: 1731881Sbinkertn@umich.edu gen = conf.checkpoints() 1741881Sbinkertn@umich.eduelif doruns: 1751881Sbinkertn@umich.edu gen = conf.jobs() 1761376Sbinkertn@umich.edu 1771881Sbinkertn@umich.edufor job in gen: 1781881Sbinkertn@umich.edu if job.name in jobnames: 1791376Sbinkertn@umich.edu continue 1801376Sbinkertn@umich.edu 1811881Sbinkertn@umich.edu if exprs: 1821881Sbinkertn@umich.edu for expr in exprs: 1831881Sbinkertn@umich.edu if expr.match(job.name): 1841881Sbinkertn@umich.edu joblist.append(job) 1851881Sbinkertn@umich.edu break 1861881Sbinkertn@umich.edu else: 1871881Sbinkertn@umich.edu joblist.append(job) 1881376Sbinkertn@umich.edu 1891376Sbinkertn@umich.eduif listonly: 1901376Sbinkertn@umich.edu if verbose: 1911881Sbinkertn@umich.edu for job in joblist: 1921881Sbinkertn@umich.edu job.printinfo() 1931376Sbinkertn@umich.edu else: 1941881Sbinkertn@umich.edu for job in joblist: 1951881Sbinkertn@umich.edu print job.name 1961376Sbinkertn@umich.edu sys.exit(0) 1971376Sbinkertn@umich.edu 1981376Sbinkertn@umich.eduif not onlyecho: 1991881Sbinkertn@umich.edu newlist = [] 2001881Sbinkertn@umich.edu for job in joblist: 2011881Sbinkertn@umich.edu jobdir = JobDir(joinpath(conf.rootdir, job.name)) 2021881Sbinkertn@umich.edu if jobdir.exists(): 2031376Sbinkertn@umich.edu if not force: 2041881Sbinkertn@umich.edu status = jobdir.getstatus() 2051881Sbinkertn@umich.edu if status == 'queued': 2061376Sbinkertn@umich.edu continue 2071376Sbinkertn@umich.edu 2081881Sbinkertn@umich.edu if status == 'running': 2091881Sbinkertn@umich.edu continue 2101881Sbinkertn@umich.edu 2111881Sbinkertn@umich.edu if status == 'success': 2121376Sbinkertn@umich.edu continue 2131376Sbinkertn@umich.edu 2141376Sbinkertn@umich.edu if not clean: 2151881Sbinkertn@umich.edu sys.exit('job directory %s not clean!' % jobdir) 2161376Sbinkertn@umich.edu 2171881Sbinkertn@umich.edu jobdir.clean() 2181881Sbinkertn@umich.edu newlist.append(job) 2191881Sbinkertn@umich.edu joblist = newlist 2201376Sbinkertn@umich.edu 2211881Sbinkertn@umich.educlass NameHack(object): 2221881Sbinkertn@umich.edu def __init__(self, host='pbs.pool', port=24465): 2231881Sbinkertn@umich.edu self.host = host 2241881Sbinkertn@umich.edu self.port = port 2251881Sbinkertn@umich.edu self.socket = None 2261816Sbinkertn@umich.edu 2271881Sbinkertn@umich.edu def setname(self, jobid, jobname): 2281881Sbinkertn@umich.edu try: 2291881Sbinkertn@umich.edu jobid = int(jobid) 2301881Sbinkertn@umich.edu except ValueError: 2311881Sbinkertn@umich.edu jobid = int(jobid.strip().split('.')[0]) 2321816Sbinkertn@umich.edu 2331881Sbinkertn@umich.edu jobname = jobname.strip() 2341881Sbinkertn@umich.edu # since pbs can handle jobnames of 15 characters or less, 2351881Sbinkertn@umich.edu # don't use the raj hack. 2361881Sbinkertn@umich.edu if len(jobname) <= 15: 2371881Sbinkertn@umich.edu return 2381376Sbinkertn@umich.edu 2391881Sbinkertn@umich.edu if self.socket is None: 2401881Sbinkertn@umich.edu import socket 2411881Sbinkertn@umich.edu self.socket = socket.socket() 2421881Sbinkertn@umich.edu # Connect to pbs.pool and send the jobid/jobname pair to port 2431881Sbinkertn@umich.edu # 24465 (Raj didn't realize that there are only 64k ports and 2441881Sbinkertn@umich.edu # setup inetd to point to port 90001) 2451881Sbinkertn@umich.edu self.socket.connect((self.host, self.port)) 2461376Sbinkertn@umich.edu 2471881Sbinkertn@umich.edu self.socket.send("%s %s\n" % (jobid, jobname)) 2481881Sbinkertn@umich.edu 2491881Sbinkertn@umich.edunamehack = NameHack() 2501881Sbinkertn@umich.edu 2511881Sbinkertn@umich.edufor job in joblist: 2521881Sbinkertn@umich.edu jobdir = JobDir(joinpath(conf.rootdir, job.name)) 2531916Sbinkertn@umich.edu if depend: 2541916Sbinkertn@umich.edu cptdir = JobDir(joinpath(conf.rootdir, job.checkpoint.name)) 2551916Sbinkertn@umich.edu cptjob = cptdir.readval('.pbs_jobid') 2561881Sbinkertn@umich.edu 2571881Sbinkertn@umich.edu if not onlyecho: 2581881Sbinkertn@umich.edu jobdir.create() 2591881Sbinkertn@umich.edu 2601881Sbinkertn@umich.edu print 'Job name: %s' % job.name 2611385Sbinkertn@umich.edu print 'Job directory: %s' % jobdir 2621376Sbinkertn@umich.edu 2631376Sbinkertn@umich.edu qsub = pbs.qsub() 2641376Sbinkertn@umich.edu qsub.pbshost = 'simpool.eecs.umich.edu' 2651881Sbinkertn@umich.edu qsub.stdout = jobdir.file('jobout') 2661881Sbinkertn@umich.edu qsub.name = job.name[:15] 2671376Sbinkertn@umich.edu qsub.join = True 2681916Sbinkertn@umich.edu qsub.node_type = node_type 2691881Sbinkertn@umich.edu qsub.env['ROOTDIR'] = conf.rootdir 2701881Sbinkertn@umich.edu qsub.env['JOBNAME'] = job.name 2711916Sbinkertn@umich.edu if depend: 2721916Sbinkertn@umich.edu qsub.afterok = cptjob 2731916Sbinkertn@umich.edu if queue: 2741376Sbinkertn@umich.edu qsub.queue = queue 2751385Sbinkertn@umich.edu qsub.build(joinpath(progpath, 'job.py')) 2761376Sbinkertn@umich.edu 2771385Sbinkertn@umich.edu if verbose: 2781385Sbinkertn@umich.edu print 'PBS Command: %s' % qsub.command 2791385Sbinkertn@umich.edu 2801385Sbinkertn@umich.edu if not onlyecho: 2811385Sbinkertn@umich.edu ec = qsub.do() 2821385Sbinkertn@umich.edu if ec == 0: 2831816Sbinkertn@umich.edu jobid = qsub.result 2841816Sbinkertn@umich.edu print 'PBS Jobid: %s' % jobid 2851881Sbinkertn@umich.edu namehack.setname(jobid, job.name) 2861881Sbinkertn@umich.edu queued = date() 2871908Sbinkertn@umich.edu jobdir.echofile('.pbs_jobid', jobid) 2881908Sbinkertn@umich.edu jobdir.echofile('.pbs_jobname', job.name) 2891881Sbinkertn@umich.edu jobdir.echofile('.queued', queued) 2901881Sbinkertn@umich.edu jobdir.setstatus('queued on %s' % queued) 2911385Sbinkertn@umich.edu else: 2921385Sbinkertn@umich.edu print 'PBS Failed' 293