1#!/usr/bin/env python 2# Copyright (c) 2005 The Regents of The University of Michigan 3# All rights reserved. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer; 9# redistributions in binary form must reproduce the above copyright 10# notice, this list of conditions and the following disclaimer in the 11# documentation and/or other materials provided with the distribution; 12# neither the name of the copyright holders nor the names of its 13# contributors may be used to endorse or promote products derived from 14# this software without specific prior written permission. 15# 16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27# 28# Authors: Ali Saidi 29# Nathan Binkert 30
|
31import os, os.path, re, sys
|
31import os, os.path, re, socket, sys |
32from os import environ as env, listdir 33from os.path import basename, isdir, isfile, islink, join as joinpath 34from filecmp import cmp as filecmp 35from shutil import copyfile 36
|
37def nfspath(dir): 38 if dir.startswith('/.automount/'): 39 dir = '/n/%s' % dir[12:] 40 elif not dir.startswith('/n/'): 41 dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir) 42 return dir 43 44progpath = nfspath(sys.path[0]) |
45progname = basename(sys.argv[0]) 46usage = """\ 47Usage: 48 %(progname)s [-c] [-e] [-f] [-q queue] [-v] <regexp> 49 -c clean directory if job can be run 50 -e only echo pbs command info, don't actually send the job 51 -f force the job to run regardless of state 52 -q <queue> submit job to the named queue 53 -v be verbose 54 55 %(progname)s -l [-v] <regexp> 56 -l list job names, don't submit 57 -v be verbose (list job parameters) 58 59 %(progname)s -h 60 -h display this help 61""" % locals() 62 63try: 64 import getopt 65 opts, args = getopt.getopt(sys.argv[1:], '-cd:efhlq:v') 66except getopt.GetoptError: 67 sys.exit(usage) 68 69clean = False 70onlyecho = False 71exprs = [] 72force = False 73listonly = False 74queue = '' 75verbose = False
|
68rootdir = re.sub(r'^/\.automount/', r'/n/', os.getcwd())
|
76rootdir = nfspath(os.getcwd()) |
77for opt,arg in opts: 78 if opt == '-c': 79 clean = True 80 if opt == '-d': 81 rootdir = arg 82 if opt == '-e': 83 onlyecho = True 84 if opt == '-f': 85 force = True 86 if opt == '-h': 87 print usage 88 sys.exit(0) 89 if opt == '-l': 90 listonly = True 91 if opt == '-q': 92 queue = arg 93 if opt == '-v': 94 verbose = True 95 96basedir = joinpath(rootdir, 'Base') 97linkdir = joinpath(rootdir, 'Link') 98 99for arg in args: 100 exprs.append(re.compile(arg)) 101 102if not listonly and not onlyecho and isdir(linkdir):
|
95 print 'Checking for outdated files in Link directory'
|
103 if verbose: 104 print 'Checking for outdated files in Link directory' |
105 entries = listdir(linkdir) 106 for entry in entries: 107 link = joinpath(linkdir, entry) 108 if not islink(link) or not isfile(link): 109 continue 110 111 base = joinpath(basedir, entry) 112 if not isfile(base) or not filecmp(link, base): 113 print 'Base/%s is different than Link/%s: copying' % (entry, entry) 114 copyfile(link, base) 115 116import job, jobfile, pbs 117 118test = jobfile.JobFile(joinpath(basedir, 'test.py')) 119 120joblist = [] 121for jobname in test.jobs: 122 if not exprs: 123 joblist.append(jobname) 124 continue 125 126 for expr in exprs: 127 if expr.match(jobname): 128 joblist.append(jobname) 129 break 130 131if listonly: 132 if verbose: 133 for jobname in joblist: 134 test.printinfo(jobname) 135 else: 136 for jobname in joblist: 137 print jobname 138 sys.exit(0) 139 140if not onlyecho: 141 jl = [] 142 for jobname in joblist: 143 jobdir = joinpath(rootdir, jobname) 144 if os.path.exists(jobname): 145 if not force: 146 if os.path.isfile(joinpath(jobdir, '.success')): 147 continue 148 149 if os.path.isfile(joinpath(jobdir, '.start')) and \ 150 not os.path.isfile(joinpath(jobdir, '.stop')): 151 continue 152 153 if not clean: 154 sys.exit('job directory not clean!') 155 156 job.cleandir(jobdir) 157 else: 158 os.mkdir(jobdir) 159 jl.append(jobname) 160 joblist = jl 161 162for jobname in joblist: 163 jobdir = joinpath(rootdir, jobname) 164 165 if not onlyecho and not os.path.isdir(jobdir): 166 sys.exit('%s is not a directory. Cannot build job' % jobdir) 167
|
159 print >>sys.stderr, 'Job name: %s' % jobname
160 print >>sys.stderr, 'Job directory: %s' % jobdir
|
168 print 'Job name: %s' % jobname 169 print 'Job directory: %s' % jobdir |
170 171 qsub = pbs.qsub() 172 qsub.pbshost = 'simpool.eecs.umich.edu' 173 qsub.stdout = joinpath(jobdir, 'jobout') 174 qsub.name = jobname 175 qsub.join = True 176 qsub.node_type = 'FAST'
|
168 qsub.onlyecho = onlyecho
|
177 qsub.env['ROOTDIR'] = rootdir
|
170 qsub.verbose = verbose
|
178 if len(queue): 179 qsub.queue = queue
|
180 qsub.build(joinpath(progpath, 'job.py')) |
181
|
174 qsub.do(joinpath(basedir, 'job.py'))
175 print >>sys.stderr, ''
|
182 if verbose: 183 print 'PBS Command: %s' % qsub.command 184 185 if not onlyecho: 186 ec = qsub.do() 187 if ec == 0: 188 print 'PBS Jobid: %s' % qsub.result 189 else: 190 print 'PBS Failed' |
|