send.py revision 1602:0fad8b5c6d2f 
1#!/usr/bin/env python
2# Copyright (c) 2005 The Regents of The University of Michigan
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met: redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer;
9# redistributions in binary form must reproduce the above copyright
10# notice, this list of conditions and the following disclaimer in the
11# documentation and/or other materials provided with the distribution;
12# neither the name of the copyright holders nor the names of its
13# contributors may be used to endorse or promote products derived from
14# this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27#
28# Authors: Ali Saidi
29#          Nathan Binkert
30
31import os, os.path, re, socket, sys
32from os import environ as env, listdir
33from os.path import basename, isdir, isfile, islink, join as joinpath
34from filecmp import cmp as filecmp
35from shutil import copyfile
36
37def nfspath(dir):
38    if dir.startswith('/.automount/'):
39        dir = '/n/%s' % dir[12:]
40    elif not dir.startswith('/n/'):
41        dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir)
42    return dir
43
44progpath = nfspath(sys.path[0])
45progname = basename(sys.argv[0])
46usage = """\
47Usage:
48    %(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp>
49    -c           clean directory if job can be run
50    -e           only echo pbs command info, don't actually send the job
51    -f           force the job to run regardless of state
52    -q <queue>   submit job to the named queue
53    -j <jobfile> specify the jobfile (default is <basedir>/test.py)
54    -v           be verbose
55
56    %(progname)s [-j <jobfile>] -l [-v] <regexp>
57    -j <jobfile> specify the jobfile (default is <basedir>/test.py)
58    -l           list job names, don't submit
59    -v           be verbose (list job parameters)
60
61    %(progname)s -h
62    -h           display this help
63""" % locals()
64
65try:
66    import getopt
67    opts, args = getopt.getopt(sys.argv[1:], '-cd:efhj:lq:v')
68except getopt.GetoptError:
69    sys.exit(usage)
70
71clean = False
72onlyecho = False
73exprs = []
74force = False
75listonly = False
76queue = ''
77verbose = False
78rootdir = nfspath(os.getcwd())
79jfile = 'test.py'
80for opt,arg in opts:
81    if opt == '-c':
82        clean = True
83    if opt == '-d':
84        rootdir = arg
85    if opt == '-e':
86        onlyecho = True
87    if opt == '-f':
88        force = True
89    if opt == '-h':
90        print usage
91        sys.exit(0)
92    if opt == '-j':
93        jfile = arg
94    if opt == '-l':
95        listonly = True
96    if opt == '-q':
97        queue = arg
98    if opt == '-v':
99        verbose = True
100
101basedir = joinpath(rootdir, 'Base')
102linkdir = joinpath(rootdir, 'Link')
103
104for arg in args:
105    exprs.append(re.compile(arg))
106
107if not listonly and not onlyecho and isdir(linkdir):
108    if verbose:
109        print 'Checking for outdated files in Link directory'
110    entries = listdir(linkdir)
111    for entry in entries:
112        link = joinpath(linkdir, entry)
113        if not islink(link) or not isfile(link):
114            continue
115
116        base = joinpath(basedir, entry)
117        if not isfile(base) or not filecmp(link, base):
118            print 'Base/%s is different than Link/%s: copying' % (entry, entry)
119            copyfile(link, base)
120
121import job, jobfile, pbs
122
123test = jobfile.JobFile(joinpath(basedir, jfile))
124
125joblist = []
126for jobname in test.jobs:
127    if not exprs:
128        joblist.append(jobname)
129        continue
130
131    for expr in exprs:
132        if expr.match(jobname):
133            joblist.append(jobname)
134            break
135
136if listonly:
137    if verbose:
138        for jobname in joblist:
139            test.printinfo(jobname)
140    else:
141        for jobname in joblist:
142            print jobname
143    sys.exit(0)
144
145if not onlyecho:
146    jl = []
147    for jobname in joblist:
148        jobdir = joinpath(rootdir, jobname)
149        if os.path.exists(jobname):
150            if not force:
151                if os.path.isfile(joinpath(jobdir, '.success')):
152                    continue
153
154                if os.path.isfile(joinpath(jobdir, '.start')) and \
155                       not os.path.isfile(joinpath(jobdir, '.stop')):
156                    continue
157
158            if not clean:
159                sys.exit('job directory not clean!')
160
161            job.cleandir(jobdir)
162        else:
163            os.mkdir(jobdir)
164        jl.append(jobname)
165    joblist = jl
166
167for jobname in joblist:
168    jobdir = joinpath(rootdir, jobname)
169
170    if not onlyecho and not os.path.isdir(jobdir):
171        sys.exit('%s is not a directory.  Cannot build job' % jobdir)
172
173    print 'Job name:       %s' % jobname
174    print 'Job directory:  %s' % jobdir
175
176    qsub = pbs.qsub()
177    qsub.pbshost = 'simpool.eecs.umich.edu'
178    qsub.stdout = joinpath(jobdir, 'jobout')
179    qsub.name = jobname
180    qsub.join = True
181    qsub.node_type = 'FAST'
182    qsub.env['ROOTDIR'] = rootdir
183    if len(queue):
184        qsub.queue = queue
185    qsub.build(joinpath(progpath, 'job.py'))
186
187    if verbose:
188        print 'PBS Command:    %s' % qsub.command
189
190    if not onlyecho:
191        ec = qsub.do()
192        if ec == 0:
193            print 'PBS Jobid:      %s' % qsub.result
194        else:
195            print 'PBS Failed'
196