send.py revision 1602
11376Sbinkertn@umich.edu#!/usr/bin/env python 21376Sbinkertn@umich.edu# Copyright (c) 2005 The Regents of The University of Michigan 31376Sbinkertn@umich.edu# All rights reserved. 41376Sbinkertn@umich.edu# 51376Sbinkertn@umich.edu# Redistribution and use in source and binary forms, with or without 61376Sbinkertn@umich.edu# modification, are permitted provided that the following conditions are 71376Sbinkertn@umich.edu# met: redistributions of source code must retain the above copyright 81376Sbinkertn@umich.edu# notice, this list of conditions and the following disclaimer; 91376Sbinkertn@umich.edu# redistributions in binary form must reproduce the above copyright 101376Sbinkertn@umich.edu# notice, this list of conditions and the following disclaimer in the 111376Sbinkertn@umich.edu# documentation and/or other materials provided with the distribution; 121376Sbinkertn@umich.edu# neither the name of the copyright holders nor the names of its 131376Sbinkertn@umich.edu# contributors may be used to endorse or promote products derived from 141376Sbinkertn@umich.edu# this software without specific prior written permission. 151376Sbinkertn@umich.edu# 161376Sbinkertn@umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 171376Sbinkertn@umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 181376Sbinkertn@umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 191376Sbinkertn@umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 201376Sbinkertn@umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 211376Sbinkertn@umich.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 221376Sbinkertn@umich.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 231376Sbinkertn@umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 241376Sbinkertn@umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 251376Sbinkertn@umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 261376Sbinkertn@umich.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 271376Sbinkertn@umich.edu# 281376Sbinkertn@umich.edu# Authors: Ali Saidi 291376Sbinkertn@umich.edu# Nathan Binkert 301376Sbinkertn@umich.edu 311385Sbinkertn@umich.eduimport os, os.path, re, socket, sys 321376Sbinkertn@umich.edufrom os import environ as env, listdir 331376Sbinkertn@umich.edufrom os.path import basename, isdir, isfile, islink, join as joinpath 341376Sbinkertn@umich.edufrom filecmp import cmp as filecmp 351376Sbinkertn@umich.edufrom shutil import copyfile 361376Sbinkertn@umich.edu 371385Sbinkertn@umich.edudef nfspath(dir): 381385Sbinkertn@umich.edu if dir.startswith('/.automount/'): 391385Sbinkertn@umich.edu dir = '/n/%s' % dir[12:] 401385Sbinkertn@umich.edu elif not dir.startswith('/n/'): 411385Sbinkertn@umich.edu dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir) 421385Sbinkertn@umich.edu return dir 431385Sbinkertn@umich.edu 441385Sbinkertn@umich.eduprogpath = nfspath(sys.path[0]) 451376Sbinkertn@umich.eduprogname = basename(sys.argv[0]) 461376Sbinkertn@umich.eduusage = """\ 471376Sbinkertn@umich.eduUsage: 481602Sbinkertn@umich.edu %(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp> 491376Sbinkertn@umich.edu -c clean directory if job can be run 501376Sbinkertn@umich.edu -e only echo pbs command info, don't actually send the job 511376Sbinkertn@umich.edu -f force the job to run regardless of state 521376Sbinkertn@umich.edu -q <queue> submit job to the named queue 531602Sbinkertn@umich.edu -j <jobfile> specify the jobfile (default is <basedir>/test.py) 541376Sbinkertn@umich.edu -v be verbose 551376Sbinkertn@umich.edu 561602Sbinkertn@umich.edu %(progname)s [-j <jobfile>] -l [-v] <regexp> 571602Sbinkertn@umich.edu -j <jobfile> specify the jobfile (default is <basedir>/test.py) 581376Sbinkertn@umich.edu -l list job names, don't submit 591376Sbinkertn@umich.edu -v be verbose (list job parameters) 601376Sbinkertn@umich.edu 611376Sbinkertn@umich.edu %(progname)s -h 621376Sbinkertn@umich.edu -h display this help 631376Sbinkertn@umich.edu""" % locals() 641376Sbinkertn@umich.edu 651376Sbinkertn@umich.edutry: 661376Sbinkertn@umich.edu import getopt 671602Sbinkertn@umich.edu opts, args = getopt.getopt(sys.argv[1:], '-cd:efhj:lq:v') 681376Sbinkertn@umich.eduexcept getopt.GetoptError: 691376Sbinkertn@umich.edu sys.exit(usage) 701376Sbinkertn@umich.edu 711376Sbinkertn@umich.educlean = False 721376Sbinkertn@umich.eduonlyecho = False 731376Sbinkertn@umich.eduexprs = [] 741376Sbinkertn@umich.eduforce = False 751376Sbinkertn@umich.edulistonly = False 761376Sbinkertn@umich.eduqueue = '' 771376Sbinkertn@umich.eduverbose = False 781385Sbinkertn@umich.edurootdir = nfspath(os.getcwd()) 791602Sbinkertn@umich.edujfile = 'test.py' 801381Sbinkertn@umich.edufor opt,arg in opts: 811381Sbinkertn@umich.edu if opt == '-c': 821376Sbinkertn@umich.edu clean = True 831381Sbinkertn@umich.edu if opt == '-d': 841381Sbinkertn@umich.edu rootdir = arg 851381Sbinkertn@umich.edu if opt == '-e': 861376Sbinkertn@umich.edu onlyecho = True 871381Sbinkertn@umich.edu if opt == '-f': 881376Sbinkertn@umich.edu force = True 891381Sbinkertn@umich.edu if opt == '-h': 901376Sbinkertn@umich.edu print usage 911376Sbinkertn@umich.edu sys.exit(0) 921602Sbinkertn@umich.edu if opt == '-j': 931602Sbinkertn@umich.edu jfile = arg 941381Sbinkertn@umich.edu if opt == '-l': 951376Sbinkertn@umich.edu listonly = True 961381Sbinkertn@umich.edu if opt == '-q': 971381Sbinkertn@umich.edu queue = arg 981381Sbinkertn@umich.edu if opt == '-v': 991376Sbinkertn@umich.edu verbose = True 1001376Sbinkertn@umich.edu 1011381Sbinkertn@umich.edubasedir = joinpath(rootdir, 'Base') 1021381Sbinkertn@umich.edulinkdir = joinpath(rootdir, 'Link') 1031381Sbinkertn@umich.edu 1041376Sbinkertn@umich.edufor arg in args: 1051376Sbinkertn@umich.edu exprs.append(re.compile(arg)) 1061376Sbinkertn@umich.edu 1071381Sbinkertn@umich.eduif not listonly and not onlyecho and isdir(linkdir): 1081385Sbinkertn@umich.edu if verbose: 1091385Sbinkertn@umich.edu print 'Checking for outdated files in Link directory' 1101381Sbinkertn@umich.edu entries = listdir(linkdir) 1111376Sbinkertn@umich.edu for entry in entries: 1121381Sbinkertn@umich.edu link = joinpath(linkdir, entry) 1131381Sbinkertn@umich.edu if not islink(link) or not isfile(link): 1141376Sbinkertn@umich.edu continue 1151376Sbinkertn@umich.edu 1161381Sbinkertn@umich.edu base = joinpath(basedir, entry) 1171376Sbinkertn@umich.edu if not isfile(base) or not filecmp(link, base): 1181381Sbinkertn@umich.edu print 'Base/%s is different than Link/%s: copying' % (entry, entry) 1191376Sbinkertn@umich.edu copyfile(link, base) 1201376Sbinkertn@umich.edu 1211376Sbinkertn@umich.eduimport job, jobfile, pbs 1221376Sbinkertn@umich.edu 1231602Sbinkertn@umich.edutest = jobfile.JobFile(joinpath(basedir, jfile)) 1241376Sbinkertn@umich.edu 1251376Sbinkertn@umich.edujoblist = [] 1261376Sbinkertn@umich.edufor jobname in test.jobs: 1271376Sbinkertn@umich.edu if not exprs: 1281376Sbinkertn@umich.edu joblist.append(jobname) 1291376Sbinkertn@umich.edu continue 1301376Sbinkertn@umich.edu 1311376Sbinkertn@umich.edu for expr in exprs: 1321376Sbinkertn@umich.edu if expr.match(jobname): 1331376Sbinkertn@umich.edu joblist.append(jobname) 1341376Sbinkertn@umich.edu break 1351376Sbinkertn@umich.edu 1361376Sbinkertn@umich.eduif listonly: 1371376Sbinkertn@umich.edu if verbose: 1381376Sbinkertn@umich.edu for jobname in joblist: 1391376Sbinkertn@umich.edu test.printinfo(jobname) 1401376Sbinkertn@umich.edu else: 1411376Sbinkertn@umich.edu for jobname in joblist: 1421376Sbinkertn@umich.edu print jobname 1431376Sbinkertn@umich.edu sys.exit(0) 1441376Sbinkertn@umich.edu 1451376Sbinkertn@umich.eduif not onlyecho: 1461376Sbinkertn@umich.edu jl = [] 1471376Sbinkertn@umich.edu for jobname in joblist: 1481383Sbinkertn@umich.edu jobdir = joinpath(rootdir, jobname) 1491376Sbinkertn@umich.edu if os.path.exists(jobname): 1501376Sbinkertn@umich.edu if not force: 1511383Sbinkertn@umich.edu if os.path.isfile(joinpath(jobdir, '.success')): 1521376Sbinkertn@umich.edu continue 1531376Sbinkertn@umich.edu 1541383Sbinkertn@umich.edu if os.path.isfile(joinpath(jobdir, '.start')) and \ 1551383Sbinkertn@umich.edu not os.path.isfile(joinpath(jobdir, '.stop')): 1561376Sbinkertn@umich.edu continue 1571376Sbinkertn@umich.edu 1581376Sbinkertn@umich.edu if not clean: 1591376Sbinkertn@umich.edu sys.exit('job directory not clean!') 1601376Sbinkertn@umich.edu 1611383Sbinkertn@umich.edu job.cleandir(jobdir) 1621376Sbinkertn@umich.edu else: 1631383Sbinkertn@umich.edu os.mkdir(jobdir) 1641393Sbinkertn@umich.edu jl.append(jobname) 1651376Sbinkertn@umich.edu joblist = jl 1661376Sbinkertn@umich.edu 1671376Sbinkertn@umich.edufor jobname in joblist: 1681376Sbinkertn@umich.edu jobdir = joinpath(rootdir, jobname) 1691376Sbinkertn@umich.edu 1701376Sbinkertn@umich.edu if not onlyecho and not os.path.isdir(jobdir): 1711376Sbinkertn@umich.edu sys.exit('%s is not a directory. Cannot build job' % jobdir) 1721376Sbinkertn@umich.edu 1731385Sbinkertn@umich.edu print 'Job name: %s' % jobname 1741385Sbinkertn@umich.edu print 'Job directory: %s' % jobdir 1751376Sbinkertn@umich.edu 1761376Sbinkertn@umich.edu qsub = pbs.qsub() 1771376Sbinkertn@umich.edu qsub.pbshost = 'simpool.eecs.umich.edu' 1781376Sbinkertn@umich.edu qsub.stdout = joinpath(jobdir, 'jobout') 1791376Sbinkertn@umich.edu qsub.name = jobname 1801376Sbinkertn@umich.edu qsub.join = True 1811376Sbinkertn@umich.edu qsub.node_type = 'FAST' 1821376Sbinkertn@umich.edu qsub.env['ROOTDIR'] = rootdir 1831376Sbinkertn@umich.edu if len(queue): 1841376Sbinkertn@umich.edu qsub.queue = queue 1851385Sbinkertn@umich.edu qsub.build(joinpath(progpath, 'job.py')) 1861376Sbinkertn@umich.edu 1871385Sbinkertn@umich.edu if verbose: 1881385Sbinkertn@umich.edu print 'PBS Command: %s' % qsub.command 1891385Sbinkertn@umich.edu 1901385Sbinkertn@umich.edu if not onlyecho: 1911385Sbinkertn@umich.edu ec = qsub.do() 1921385Sbinkertn@umich.edu if ec == 0: 1931385Sbinkertn@umich.edu print 'PBS Jobid: %s' % qsub.result 1941385Sbinkertn@umich.edu else: 1951385Sbinkertn@umich.edu print 'PBS Failed' 196