send.py revision 1602
1#!/usr/bin/env python 2# Copyright (c) 2005 The Regents of The University of Michigan 3# All rights reserved. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer; 9# redistributions in binary form must reproduce the above copyright 10# notice, this list of conditions and the following disclaimer in the 11# documentation and/or other materials provided with the distribution; 12# neither the name of the copyright holders nor the names of its 13# contributors may be used to endorse or promote products derived from 14# this software without specific prior written permission. 15# 16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27# 28# Authors: Ali Saidi 29# Nathan Binkert 30 31import os, os.path, re, socket, sys 32from os import environ as env, listdir 33from os.path import basename, isdir, isfile, islink, join as joinpath 34from filecmp import cmp as filecmp 35from shutil import copyfile 36 37def nfspath(dir): 38 if dir.startswith('/.automount/'): 39 dir = '/n/%s' % dir[12:] 40 elif not dir.startswith('/n/'): 41 dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir) 42 return dir 43 44progpath = nfspath(sys.path[0]) 45progname = basename(sys.argv[0]) 46usage = """\ 47Usage: 48 %(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp> 49 -c clean directory if job can be run 50 -e only echo pbs command info, don't actually send the job 51 -f force the job to run regardless of state 52 -q <queue> submit job to the named queue 53 -j <jobfile> specify the jobfile (default is <basedir>/test.py) 54 -v be verbose 55 56 %(progname)s [-j <jobfile>] -l [-v] <regexp> 57 -j <jobfile> specify the jobfile (default is <basedir>/test.py) 58 -l list job names, don't submit 59 -v be verbose (list job parameters) 60 61 %(progname)s -h 62 -h display this help 63""" % locals() 64 65try: 66 import getopt 67 opts, args = getopt.getopt(sys.argv[1:], '-cd:efhj:lq:v') 68except getopt.GetoptError: 69 sys.exit(usage) 70 71clean = False 72onlyecho = False 73exprs = [] 74force = False 75listonly = False 76queue = '' 77verbose = False 78rootdir = nfspath(os.getcwd()) 79jfile = 'test.py' 80for opt,arg in opts: 81 if opt == '-c': 82 clean = True 83 if opt == '-d': 84 rootdir = arg 85 if opt == '-e': 86 onlyecho = True 87 if opt == '-f': 88 force = True 89 if opt == '-h': 90 print usage 91 sys.exit(0) 92 if opt == '-j': 93 jfile = arg 94 if opt == '-l': 95 listonly = True 96 if opt == '-q': 97 queue = arg 98 if opt == '-v': 99 verbose = True 100 101basedir = joinpath(rootdir, 'Base') 102linkdir = joinpath(rootdir, 'Link') 103 104for arg in args: 105 exprs.append(re.compile(arg)) 106 107if not listonly and not onlyecho and isdir(linkdir): 108 if verbose: 109 print 'Checking for outdated files in Link directory' 110 entries = listdir(linkdir) 111 for entry in entries: 112 link = joinpath(linkdir, entry) 113 if not islink(link) or not isfile(link): 114 continue 115 116 base = joinpath(basedir, entry) 117 if not isfile(base) or not filecmp(link, base): 118 print 'Base/%s is different than Link/%s: copying' % (entry, entry) 119 copyfile(link, base) 120 121import job, jobfile, pbs 122 123test = jobfile.JobFile(joinpath(basedir, jfile)) 124 125joblist = [] 126for jobname in test.jobs: 127 if not exprs: 128 joblist.append(jobname) 129 continue 130 131 for expr in exprs: 132 if expr.match(jobname): 133 joblist.append(jobname) 134 break 135 136if listonly: 137 if verbose: 138 for jobname in joblist: 139 test.printinfo(jobname) 140 else: 141 for jobname in joblist: 142 print jobname 143 sys.exit(0) 144 145if not onlyecho: 146 jl = [] 147 for jobname in joblist: 148 jobdir = joinpath(rootdir, jobname) 149 if os.path.exists(jobname): 150 if not force: 151 if os.path.isfile(joinpath(jobdir, '.success')): 152 continue 153 154 if os.path.isfile(joinpath(jobdir, '.start')) and \ 155 not os.path.isfile(joinpath(jobdir, '.stop')): 156 continue 157 158 if not clean: 159 sys.exit('job directory not clean!') 160 161 job.cleandir(jobdir) 162 else: 163 os.mkdir(jobdir) 164 jl.append(jobname) 165 joblist = jl 166 167for jobname in joblist: 168 jobdir = joinpath(rootdir, jobname) 169 170 if not onlyecho and not os.path.isdir(jobdir): 171 sys.exit('%s is not a directory. Cannot build job' % jobdir) 172 173 print 'Job name: %s' % jobname 174 print 'Job directory: %s' % jobdir 175 176 qsub = pbs.qsub() 177 qsub.pbshost = 'simpool.eecs.umich.edu' 178 qsub.stdout = joinpath(jobdir, 'jobout') 179 qsub.name = jobname 180 qsub.join = True 181 qsub.node_type = 'FAST' 182 qsub.env['ROOTDIR'] = rootdir 183 if len(queue): 184 qsub.queue = queue 185 qsub.build(joinpath(progpath, 'job.py')) 186 187 if verbose: 188 print 'PBS Command: %s' % qsub.command 189 190 if not onlyecho: 191 ec = qsub.do() 192 if ec == 0: 193 print 'PBS Jobid: %s' % qsub.result 194 else: 195 print 'PBS Failed' 196