send.py revision 1376
1#!/usr/bin/env python 2# Copyright (c) 2005 The Regents of The University of Michigan 3# All rights reserved. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer; 9# redistributions in binary form must reproduce the above copyright 10# notice, this list of conditions and the following disclaimer in the 11# documentation and/or other materials provided with the distribution; 12# neither the name of the copyright holders nor the names of its 13# contributors may be used to endorse or promote products derived from 14# this software without specific prior written permission. 15# 16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27# 28# Authors: Ali Saidi 29# Nathan Binkert 30 31import os, os.path, re, sys 32from os import environ as env, listdir 33from os.path import basename, isdir, isfile, islink, join as joinpath 34from filecmp import cmp as filecmp 35from shutil import copyfile 36 37progname = basename(sys.argv[0]) 38usage = """\ 39Usage: 40 %(progname)s [-c] [-e] [-f] [-q queue] [-v] <regexp> 41 -c clean directory if job can be run 42 -e only echo pbs command info, don't actually send the job 43 -f force the job to run regardless of state 44 -q <queue> submit job to the named queue 45 -v be verbose 46 47 %(progname)s -l [-v] <regexp> 48 -l list job names, don't submit 49 -v be verbose (list job parameters) 50 51 %(progname)s -h 52 -h display this help 53""" % locals() 54 55try: 56 import getopt 57 opts, args = getopt.getopt(sys.argv[1:], '-cefhlq:v') 58except getopt.GetoptError: 59 sys.exit(usage) 60 61clean = False 62onlyecho = False 63exprs = [] 64force = False 65listonly = False 66queue = '' 67verbose = False 68for o,a in opts: 69 if o == '-c': 70 clean = True 71 if o == '-e': 72 onlyecho = True 73 if o == '-f': 74 force = True 75 if o == '-h': 76 print usage 77 sys.exit(0) 78 if o == '-l': 79 listonly = True 80 if o == '-q': 81 queue = a 82 if o == '-v': 83 verbose = True 84 85for arg in args: 86 exprs.append(re.compile(arg)) 87 88if not listonly and not onlyecho and isdir('Link'): 89 print 'Checking for outdated files in Link directory' 90 entries = listdir('Link') 91 for entry in entries: 92 link = joinpath('Link', entry) 93 if not islink(link): 94 continue 95 96 base = joinpath('Base', entry) 97 if not isfile(base) or not filecmp(link, base): 98 print '%s is different than source %s...copying' % (base, link) 99 copyfile(link, base) 100 101import job, jobfile, pbs 102 103test = jobfile.JobFile(joinpath('Base', 'test.py')) 104 105joblist = [] 106for jobname in test.jobs: 107 if not exprs: 108 joblist.append(jobname) 109 continue 110 111 for expr in exprs: 112 if expr.match(jobname): 113 joblist.append(jobname) 114 break 115 116if listonly: 117 if verbose: 118 for jobname in joblist: 119 test.printinfo(jobname) 120 else: 121 for jobname in joblist: 122 print jobname 123 sys.exit(0) 124 125if not onlyecho: 126 jl = [] 127 for jobname in joblist: 128 if os.path.exists(jobname): 129 if not force: 130 if os.path.isfile(joinpath(jobname, '.success')): 131 continue 132 133 if os.path.isfile(joinpath(jobname, '.start')) and \ 134 not os.path.isfile(joinpath(jobname, '.stop')): 135 continue 136 137 if not clean: 138 sys.exit('job directory not clean!') 139 140 job.cleandir(jobname) 141 else: 142 os.mkdir(jobname) 143 jl.append(jobname) 144 joblist = jl 145 146rootdir = re.sub(r'^/\.automount/', r'/n/', os.getcwd()) 147for jobname in joblist: 148 jobdir = joinpath(rootdir, jobname) 149 150 if not onlyecho and not os.path.isdir(jobdir): 151 sys.exit('%s is not a directory. Cannot build job' % jobdir) 152 153 print >>sys.stderr, 'Job name: %s' % jobname 154 print >>sys.stderr, 'Job directory: %s' % jobdir 155 156 qsub = pbs.qsub() 157 qsub.pbshost = 'simpool.eecs.umich.edu' 158 qsub.stdout = joinpath(jobdir, 'jobout') 159 qsub.name = jobname 160 qsub.join = True 161 qsub.node_type = 'FAST' 162 qsub.onlyecho = onlyecho 163 qsub.env['ROOTDIR'] = rootdir 164 qsub.verbose = verbose 165 if len(queue): 166 qsub.queue = queue 167 168 qsub.do(joinpath('Base', 'job.py')) 169 print >>sys.stderr, '' 170