send.py revision 1381:6ede00091fa9 
1#!/usr/bin/env python
2# Copyright (c) 2005 The Regents of The University of Michigan
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met: redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer;
9# redistributions in binary form must reproduce the above copyright
10# notice, this list of conditions and the following disclaimer in the
11# documentation and/or other materials provided with the distribution;
12# neither the name of the copyright holders nor the names of its
13# contributors may be used to endorse or promote products derived from
14# this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27#
28# Authors: Ali Saidi
29#          Nathan Binkert
30
31import os, os.path, re, sys
32from os import environ as env, listdir
33from os.path import basename, isdir, isfile, islink, join as joinpath
34from filecmp import cmp as filecmp
35from shutil import copyfile
36
37progname = basename(sys.argv[0])
38usage = """\
39Usage:
40    %(progname)s [-c] [-e] [-f] [-q queue] [-v] <regexp>
41    -c           clean directory if job can be run
42    -e           only echo pbs command info, don't actually send the job
43    -f           force the job to run regardless of state
44    -q <queue>   submit job to the named queue
45    -v           be verbose
46
47    %(progname)s -l [-v] <regexp>
48    -l           list job names, don't submit
49    -v           be verbose (list job parameters)
50
51    %(progname)s -h
52    -h           display this help
53""" % locals()
54
55try:
56    import getopt
57    opts, args = getopt.getopt(sys.argv[1:], '-cd:efhlq:v')
58except getopt.GetoptError:
59    sys.exit(usage)
60
61clean = False
62onlyecho = False
63exprs = []
64force = False
65listonly = False
66queue = ''
67verbose = False
68rootdir = re.sub(r'^/\.automount/', r'/n/', os.getcwd())
69for opt,arg in opts:
70    if opt == '-c':
71        clean = True
72    if opt == '-d':
73        rootdir = arg
74    if opt == '-e':
75        onlyecho = True
76    if opt == '-f':
77        force = True
78    if opt == '-h':
79        print usage
80        sys.exit(0)
81    if opt == '-l':
82        listonly = True
83    if opt == '-q':
84        queue = arg
85    if opt == '-v':
86        verbose = True
87
88basedir = joinpath(rootdir, 'Base')
89linkdir = joinpath(rootdir, 'Link')
90
91for arg in args:
92    exprs.append(re.compile(arg))
93
94if not listonly and not onlyecho and isdir(linkdir):
95    print 'Checking for outdated files in Link directory'
96    entries = listdir(linkdir)
97    for entry in entries:
98        link = joinpath(linkdir, entry)
99        if not islink(link) or not isfile(link):
100            continue
101
102        base = joinpath(basedir, entry)
103        if not isfile(base) or not filecmp(link, base):
104            print 'Base/%s is different than Link/%s: copying' % (entry, entry)
105            copyfile(link, base)
106
107import job, jobfile, pbs
108
109test = jobfile.JobFile(joinpath(basedir, 'test.py'))
110
111joblist = []
112for jobname in test.jobs:
113    if not exprs:
114        joblist.append(jobname)
115        continue
116
117    for expr in exprs:
118        if expr.match(jobname):
119            joblist.append(jobname)
120            break
121
122if listonly:
123    if verbose:
124        for jobname in joblist:
125            test.printinfo(jobname)
126    else:
127        for jobname in joblist:
128            print jobname
129    sys.exit(0)
130
131if not onlyecho:
132    jl = []
133    for jobname in joblist:
134        if os.path.exists(jobname):
135            if not force:
136                if os.path.isfile(joinpath(jobname, '.success')):
137                    continue
138
139                if os.path.isfile(joinpath(jobname, '.start')) and \
140                       not os.path.isfile(joinpath(jobname, '.stop')):
141                    continue
142
143            if not clean:
144                sys.exit('job directory not clean!')
145
146            job.cleandir(jobname)
147        else:
148            os.mkdir(jobname)
149    jl.append(jobname)
150    joblist = jl
151
152for jobname in joblist:
153    jobdir = joinpath(rootdir, jobname)
154
155    if not onlyecho and not os.path.isdir(jobdir):
156        sys.exit('%s is not a directory.  Cannot build job' % jobdir)
157
158    print >>sys.stderr, 'Job name:       %s' % jobname
159    print >>sys.stderr, 'Job directory:  %s' % jobdir
160
161    qsub = pbs.qsub()
162    qsub.pbshost = 'simpool.eecs.umich.edu'
163    qsub.stdout = joinpath(jobdir, 'jobout')
164    qsub.name = jobname
165    qsub.join = True
166    qsub.node_type = 'FAST'
167    qsub.onlyecho = onlyecho
168    qsub.env['ROOTDIR'] = rootdir
169    qsub.verbose = verbose
170    if len(queue):
171        qsub.queue = queue
172
173    qsub.do(joinpath(basedir, 'job.py'))
174    print >>sys.stderr, ''
175