send.py revision 1381
1#!/usr/bin/env python 2# Copyright (c) 2005 The Regents of The University of Michigan 3# All rights reserved. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer; 9# redistributions in binary form must reproduce the above copyright 10# notice, this list of conditions and the following disclaimer in the 11# documentation and/or other materials provided with the distribution; 12# neither the name of the copyright holders nor the names of its 13# contributors may be used to endorse or promote products derived from 14# this software without specific prior written permission. 15# 16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27# 28# Authors: Ali Saidi 29# Nathan Binkert 30 31import os, os.path, re, sys 32from os import environ as env, listdir 33from os.path import basename, isdir, isfile, islink, join as joinpath 34from filecmp import cmp as filecmp 35from shutil import copyfile 36 37progname = basename(sys.argv[0]) 38usage = """\ 39Usage: 40 %(progname)s [-c] [-e] [-f] [-q queue] [-v] <regexp> 41 -c clean directory if job can be run 42 -e only echo pbs command info, don't actually send the job 43 -f force the job to run regardless of state 44 -q <queue> submit job to the named queue 45 -v be verbose 46 47 %(progname)s -l [-v] <regexp> 48 -l list job names, don't submit 49 -v be verbose (list job parameters) 50 51 %(progname)s -h 52 -h display this help 53""" % locals() 54 55try: 56 import getopt 57 opts, args = getopt.getopt(sys.argv[1:], '-cd:efhlq:v') 58except getopt.GetoptError: 59 sys.exit(usage) 60 61clean = False 62onlyecho = False 63exprs = [] 64force = False 65listonly = False 66queue = '' 67verbose = False 68rootdir = re.sub(r'^/\.automount/', r'/n/', os.getcwd()) 69for opt,arg in opts: 70 if opt == '-c': 71 clean = True 72 if opt == '-d': 73 rootdir = arg 74 if opt == '-e': 75 onlyecho = True 76 if opt == '-f': 77 force = True 78 if opt == '-h': 79 print usage 80 sys.exit(0) 81 if opt == '-l': 82 listonly = True 83 if opt == '-q': 84 queue = arg 85 if opt == '-v': 86 verbose = True 87 88basedir = joinpath(rootdir, 'Base') 89linkdir = joinpath(rootdir, 'Link') 90 91for arg in args: 92 exprs.append(re.compile(arg)) 93 94if not listonly and not onlyecho and isdir(linkdir): 95 print 'Checking for outdated files in Link directory' 96 entries = listdir(linkdir) 97 for entry in entries: 98 link = joinpath(linkdir, entry) 99 if not islink(link) or not isfile(link): 100 continue 101 102 base = joinpath(basedir, entry) 103 if not isfile(base) or not filecmp(link, base): 104 print 'Base/%s is different than Link/%s: copying' % (entry, entry) 105 copyfile(link, base) 106 107import job, jobfile, pbs 108 109test = jobfile.JobFile(joinpath(basedir, 'test.py')) 110 111joblist = [] 112for jobname in test.jobs: 113 if not exprs: 114 joblist.append(jobname) 115 continue 116 117 for expr in exprs: 118 if expr.match(jobname): 119 joblist.append(jobname) 120 break 121 122if listonly: 123 if verbose: 124 for jobname in joblist: 125 test.printinfo(jobname) 126 else: 127 for jobname in joblist: 128 print jobname 129 sys.exit(0) 130 131if not onlyecho: 132 jl = [] 133 for jobname in joblist: 134 if os.path.exists(jobname): 135 if not force: 136 if os.path.isfile(joinpath(jobname, '.success')): 137 continue 138 139 if os.path.isfile(joinpath(jobname, '.start')) and \ 140 not os.path.isfile(joinpath(jobname, '.stop')): 141 continue 142 143 if not clean: 144 sys.exit('job directory not clean!') 145 146 job.cleandir(jobname) 147 else: 148 os.mkdir(jobname) 149 jl.append(jobname) 150 joblist = jl 151 152for jobname in joblist: 153 jobdir = joinpath(rootdir, jobname) 154 155 if not onlyecho and not os.path.isdir(jobdir): 156 sys.exit('%s is not a directory. Cannot build job' % jobdir) 157 158 print >>sys.stderr, 'Job name: %s' % jobname 159 print >>sys.stderr, 'Job directory: %s' % jobdir 160 161 qsub = pbs.qsub() 162 qsub.pbshost = 'simpool.eecs.umich.edu' 163 qsub.stdout = joinpath(jobdir, 'jobout') 164 qsub.name = jobname 165 qsub.join = True 166 qsub.node_type = 'FAST' 167 qsub.onlyecho = onlyecho 168 qsub.env['ROOTDIR'] = rootdir 169 qsub.verbose = verbose 170 if len(queue): 171 qsub.queue = queue 172 173 qsub.do(joinpath(basedir, 'job.py')) 174 print >>sys.stderr, '' 175