send.py revision 1376:190de61fed5a 
1#!/usr/bin/env python
2# Copyright (c) 2005 The Regents of The University of Michigan
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met: redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer;
9# redistributions in binary form must reproduce the above copyright
10# notice, this list of conditions and the following disclaimer in the
11# documentation and/or other materials provided with the distribution;
12# neither the name of the copyright holders nor the names of its
13# contributors may be used to endorse or promote products derived from
14# this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27#
28# Authors: Ali Saidi
29#          Nathan Binkert
30
31import os, os.path, re, sys
32from os import environ as env, listdir
33from os.path import basename, isdir, isfile, islink, join as joinpath
34from filecmp import cmp as filecmp
35from shutil import copyfile
36
37progname = basename(sys.argv[0])
38usage = """\
39Usage:
40    %(progname)s [-c] [-e] [-f] [-q queue] [-v] <regexp>
41    -c           clean directory if job can be run
42    -e           only echo pbs command info, don't actually send the job
43    -f           force the job to run regardless of state
44    -q <queue>   submit job to the named queue
45    -v           be verbose
46
47    %(progname)s -l [-v] <regexp>
48    -l           list job names, don't submit
49    -v           be verbose (list job parameters)
50
51    %(progname)s -h
52    -h           display this help
53""" % locals()
54
55try:
56    import getopt
57    opts, args = getopt.getopt(sys.argv[1:], '-cefhlq:v')
58except getopt.GetoptError:
59    sys.exit(usage)
60
61clean = False
62onlyecho = False
63exprs = []
64force = False
65listonly = False
66queue = ''
67verbose = False
68for o,a in opts:
69    if o == '-c':
70        clean = True
71    if o == '-e':
72        onlyecho = True
73    if o == '-f':
74        force = True
75    if o == '-h':
76        print usage
77        sys.exit(0)
78    if o == '-l':
79        listonly = True
80    if o == '-q':
81        queue = a
82    if o == '-v':
83        verbose = True
84
85for arg in args:
86    exprs.append(re.compile(arg))
87
88if not listonly and not onlyecho and isdir('Link'):
89    print 'Checking for outdated files in Link directory'
90    entries = listdir('Link')
91    for entry in entries:
92        link = joinpath('Link', entry)
93        if not islink(link):
94            continue
95
96        base = joinpath('Base', entry)
97        if not isfile(base) or not filecmp(link, base):
98            print '%s is different than source %s...copying' % (base, link)
99            copyfile(link, base)
100
101import job, jobfile, pbs
102
103test = jobfile.JobFile(joinpath('Base', 'test.py'))
104
105joblist = []
106for jobname in test.jobs:
107    if not exprs:
108        joblist.append(jobname)
109        continue
110
111    for expr in exprs:
112        if expr.match(jobname):
113            joblist.append(jobname)
114            break
115
116if listonly:
117    if verbose:
118        for jobname in joblist:
119            test.printinfo(jobname)
120    else:
121        for jobname in joblist:
122            print jobname
123    sys.exit(0)
124
125if not onlyecho:
126    jl = []
127    for jobname in joblist:
128        if os.path.exists(jobname):
129            if not force:
130                if os.path.isfile(joinpath(jobname, '.success')):
131                    continue
132
133                if os.path.isfile(joinpath(jobname, '.start')) and \
134                       not os.path.isfile(joinpath(jobname, '.stop')):
135                    continue
136
137            if not clean:
138                sys.exit('job directory not clean!')
139
140            job.cleandir(jobname)
141        else:
142            os.mkdir(jobname)
143    jl.append(jobname)
144    joblist = jl
145
146rootdir = re.sub(r'^/\.automount/', r'/n/', os.getcwd())
147for jobname in joblist:
148    jobdir = joinpath(rootdir, jobname)
149
150    if not onlyecho and not os.path.isdir(jobdir):
151        sys.exit('%s is not a directory.  Cannot build job' % jobdir)
152
153    print >>sys.stderr, 'Job name:       %s' % jobname
154    print >>sys.stderr, 'Job directory:  %s' % jobdir
155
156    qsub = pbs.qsub()
157    qsub.pbshost = 'simpool.eecs.umich.edu'
158    qsub.stdout = joinpath(jobdir, 'jobout')
159    qsub.name = jobname
160    qsub.join = True
161    qsub.node_type = 'FAST'
162    qsub.onlyecho = onlyecho
163    qsub.env['ROOTDIR'] = rootdir
164    qsub.verbose = verbose
165    if len(queue):
166        qsub.queue = queue
167
168    qsub.do(joinpath('Base', 'job.py'))
169    print >>sys.stderr, ''
170