send.py revision 1383 
1#!/usr/bin/env python
2# Copyright (c) 2005 The Regents of The University of Michigan
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met: redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer;
9# redistributions in binary form must reproduce the above copyright
10# notice, this list of conditions and the following disclaimer in the
11# documentation and/or other materials provided with the distribution;
12# neither the name of the copyright holders nor the names of its
13# contributors may be used to endorse or promote products derived from
14# this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27#
28# Authors: Ali Saidi
29#          Nathan Binkert
30
31import os, os.path, re, sys
32from os import environ as env, listdir
33from os.path import basename, isdir, isfile, islink, join as joinpath
34from filecmp import cmp as filecmp
35from shutil import copyfile
36
37progname = basename(sys.argv[0])
38usage = """\
39Usage:
40    %(progname)s [-c] [-e] [-f] [-q queue] [-v] <regexp>
41    -c           clean directory if job can be run
42    -e           only echo pbs command info, don't actually send the job
43    -f           force the job to run regardless of state
44    -q <queue>   submit job to the named queue
45    -v           be verbose
46
47    %(progname)s -l [-v] <regexp>
48    -l           list job names, don't submit
49    -v           be verbose (list job parameters)
50
51    %(progname)s -h
52    -h           display this help
53""" % locals()
54
55try:
56    import getopt
57    opts, args = getopt.getopt(sys.argv[1:], '-cd:efhlq:v')
58except getopt.GetoptError:
59    sys.exit(usage)
60
61clean = False
62onlyecho = False
63exprs = []
64force = False
65listonly = False
66queue = ''
67verbose = False
68rootdir = re.sub(r'^/\.automount/', r'/n/', os.getcwd())
69for opt,arg in opts:
70    if opt == '-c':
71        clean = True
72    if opt == '-d':
73        rootdir = arg
74    if opt == '-e':
75        onlyecho = True
76    if opt == '-f':
77        force = True
78    if opt == '-h':
79        print usage
80        sys.exit(0)
81    if opt == '-l':
82        listonly = True
83    if opt == '-q':
84        queue = arg
85    if opt == '-v':
86        verbose = True
87
88basedir = joinpath(rootdir, 'Base')
89linkdir = joinpath(rootdir, 'Link')
90
91for arg in args:
92    exprs.append(re.compile(arg))
93
94if not listonly and not onlyecho and isdir(linkdir):
95    print 'Checking for outdated files in Link directory'
96    entries = listdir(linkdir)
97    for entry in entries:
98        link = joinpath(linkdir, entry)
99        if not islink(link) or not isfile(link):
100            continue
101
102        base = joinpath(basedir, entry)
103        if not isfile(base) or not filecmp(link, base):
104            print 'Base/%s is different than Link/%s: copying' % (entry, entry)
105            copyfile(link, base)
106
107import job, jobfile, pbs
108
109test = jobfile.JobFile(joinpath(basedir, 'test.py'))
110
111joblist = []
112for jobname in test.jobs:
113    if not exprs:
114        joblist.append(jobname)
115        continue
116
117    for expr in exprs:
118        if expr.match(jobname):
119            joblist.append(jobname)
120            break
121
122if listonly:
123    if verbose:
124        for jobname in joblist:
125            test.printinfo(jobname)
126    else:
127        for jobname in joblist:
128            print jobname
129    sys.exit(0)
130
131if not onlyecho:
132    jl = []
133    for jobname in joblist:
134        jobdir = joinpath(rootdir, jobname)
135        if os.path.exists(jobname):
136            if not force:
137                if os.path.isfile(joinpath(jobdir, '.success')):
138                    continue
139
140                if os.path.isfile(joinpath(jobdir, '.start')) and \
141                       not os.path.isfile(joinpath(jobdir, '.stop')):
142                    continue
143
144            if not clean:
145                sys.exit('job directory not clean!')
146
147            job.cleandir(jobdir)
148        else:
149            os.mkdir(jobdir)
150    jl.append(jobname)
151    joblist = jl
152
153for jobname in joblist:
154    jobdir = joinpath(rootdir, jobname)
155
156    if not onlyecho and not os.path.isdir(jobdir):
157        sys.exit('%s is not a directory.  Cannot build job' % jobdir)
158
159    print >>sys.stderr, 'Job name:       %s' % jobname
160    print >>sys.stderr, 'Job directory:  %s' % jobdir
161
162    qsub = pbs.qsub()
163    qsub.pbshost = 'simpool.eecs.umich.edu'
164    qsub.stdout = joinpath(jobdir, 'jobout')
165    qsub.name = jobname
166    qsub.join = True
167    qsub.node_type = 'FAST'
168    qsub.onlyecho = onlyecho
169    qsub.env['ROOTDIR'] = rootdir
170    qsub.verbose = verbose
171    if len(queue):
172        qsub.queue = queue
173
174    qsub.do(joinpath(basedir, 'job.py'))
175    print >>sys.stderr, ''
176