1#!/usr/bin/env python
2# Copyright (c) 2005 The Regents of The University of Michigan
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met: redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer;
9# redistributions in binary form must reproduce the above copyright
10# notice, this list of conditions and the following disclaimer in the
11# documentation and/or other materials provided with the distribution;
12# neither the name of the copyright holders nor the names of its
13# contributors may be used to endorse or promote products derived from
14# this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27#
28# Authors: Ali Saidi
29# Nathan Binkert
30
| 1#!/usr/bin/env python
2# Copyright (c) 2005 The Regents of The University of Michigan
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met: redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer;
9# redistributions in binary form must reproduce the above copyright
10# notice, this list of conditions and the following disclaimer in the
11# documentation and/or other materials provided with the distribution;
12# neither the name of the copyright holders nor the names of its
13# contributors may be used to endorse or promote products derived from
14# this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27#
28# Authors: Ali Saidi
29# Nathan Binkert
30
|
31import os, os.path, re, sys
| 31import os, os.path, re, socket, sys
|
32from os import environ as env, listdir
33from os.path import basename, isdir, isfile, islink, join as joinpath
34from filecmp import cmp as filecmp
35from shutil import copyfile
36
| 32from os import environ as env, listdir
33from os.path import basename, isdir, isfile, islink, join as joinpath
34from filecmp import cmp as filecmp
35from shutil import copyfile
36
|
| 37def nfspath(dir):
38 if dir.startswith('/.automount/'):
39 dir = '/n/%s' % dir[12:]
40 elif not dir.startswith('/n/'):
41 dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir)
42 return dir
43
44progpath = nfspath(sys.path[0])
|
37progname = basename(sys.argv[0])
38usage = """\
39Usage:
40 %(progname)s [-c] [-e] [-f] [-q queue] [-v] <regexp>
41 -c clean directory if job can be run
42 -e only echo pbs command info, don't actually send the job
43 -f force the job to run regardless of state
44 -q <queue> submit job to the named queue
45 -v be verbose
46
47 %(progname)s -l [-v] <regexp>
48 -l list job names, don't submit
49 -v be verbose (list job parameters)
50
51 %(progname)s -h
52 -h display this help
53""" % locals()
54
55try:
56 import getopt
57 opts, args = getopt.getopt(sys.argv[1:], '-cd:efhlq:v')
58except getopt.GetoptError:
59 sys.exit(usage)
60
61clean = False
62onlyecho = False
63exprs = []
64force = False
65listonly = False
66queue = ''
67verbose = False
| 45progname = basename(sys.argv[0])
46usage = """\
47Usage:
48 %(progname)s [-c] [-e] [-f] [-q queue] [-v] <regexp>
49 -c clean directory if job can be run
50 -e only echo pbs command info, don't actually send the job
51 -f force the job to run regardless of state
52 -q <queue> submit job to the named queue
53 -v be verbose
54
55 %(progname)s -l [-v] <regexp>
56 -l list job names, don't submit
57 -v be verbose (list job parameters)
58
59 %(progname)s -h
60 -h display this help
61""" % locals()
62
63try:
64 import getopt
65 opts, args = getopt.getopt(sys.argv[1:], '-cd:efhlq:v')
66except getopt.GetoptError:
67 sys.exit(usage)
68
69clean = False
70onlyecho = False
71exprs = []
72force = False
73listonly = False
74queue = ''
75verbose = False
|
68rootdir = re.sub(r'^/\.automount/', r'/n/', os.getcwd())
| 76rootdir = nfspath(os.getcwd())
|
69for opt,arg in opts:
70 if opt == '-c':
71 clean = True
72 if opt == '-d':
73 rootdir = arg
74 if opt == '-e':
75 onlyecho = True
76 if opt == '-f':
77 force = True
78 if opt == '-h':
79 print usage
80 sys.exit(0)
81 if opt == '-l':
82 listonly = True
83 if opt == '-q':
84 queue = arg
85 if opt == '-v':
86 verbose = True
87
88basedir = joinpath(rootdir, 'Base')
89linkdir = joinpath(rootdir, 'Link')
90
91for arg in args:
92 exprs.append(re.compile(arg))
93
94if not listonly and not onlyecho and isdir(linkdir):
| 77for opt,arg in opts:
78 if opt == '-c':
79 clean = True
80 if opt == '-d':
81 rootdir = arg
82 if opt == '-e':
83 onlyecho = True
84 if opt == '-f':
85 force = True
86 if opt == '-h':
87 print usage
88 sys.exit(0)
89 if opt == '-l':
90 listonly = True
91 if opt == '-q':
92 queue = arg
93 if opt == '-v':
94 verbose = True
95
96basedir = joinpath(rootdir, 'Base')
97linkdir = joinpath(rootdir, 'Link')
98
99for arg in args:
100 exprs.append(re.compile(arg))
101
102if not listonly and not onlyecho and isdir(linkdir):
|
95 print 'Checking for outdated files in Link directory'
| 103 if verbose:
104 print 'Checking for outdated files in Link directory'
|
96 entries = listdir(linkdir)
97 for entry in entries:
98 link = joinpath(linkdir, entry)
99 if not islink(link) or not isfile(link):
100 continue
101
102 base = joinpath(basedir, entry)
103 if not isfile(base) or not filecmp(link, base):
104 print 'Base/%s is different than Link/%s: copying' % (entry, entry)
105 copyfile(link, base)
106
107import job, jobfile, pbs
108
109test = jobfile.JobFile(joinpath(basedir, 'test.py'))
110
111joblist = []
112for jobname in test.jobs:
113 if not exprs:
114 joblist.append(jobname)
115 continue
116
117 for expr in exprs:
118 if expr.match(jobname):
119 joblist.append(jobname)
120 break
121
122if listonly:
123 if verbose:
124 for jobname in joblist:
125 test.printinfo(jobname)
126 else:
127 for jobname in joblist:
128 print jobname
129 sys.exit(0)
130
131if not onlyecho:
132 jl = []
133 for jobname in joblist:
134 jobdir = joinpath(rootdir, jobname)
135 if os.path.exists(jobname):
136 if not force:
137 if os.path.isfile(joinpath(jobdir, '.success')):
138 continue
139
140 if os.path.isfile(joinpath(jobdir, '.start')) and \
141 not os.path.isfile(joinpath(jobdir, '.stop')):
142 continue
143
144 if not clean:
145 sys.exit('job directory not clean!')
146
147 job.cleandir(jobdir)
148 else:
149 os.mkdir(jobdir)
150 jl.append(jobname)
151 joblist = jl
152
153for jobname in joblist:
154 jobdir = joinpath(rootdir, jobname)
155
156 if not onlyecho and not os.path.isdir(jobdir):
157 sys.exit('%s is not a directory. Cannot build job' % jobdir)
158
| 105 entries = listdir(linkdir)
106 for entry in entries:
107 link = joinpath(linkdir, entry)
108 if not islink(link) or not isfile(link):
109 continue
110
111 base = joinpath(basedir, entry)
112 if not isfile(base) or not filecmp(link, base):
113 print 'Base/%s is different than Link/%s: copying' % (entry, entry)
114 copyfile(link, base)
115
116import job, jobfile, pbs
117
118test = jobfile.JobFile(joinpath(basedir, 'test.py'))
119
120joblist = []
121for jobname in test.jobs:
122 if not exprs:
123 joblist.append(jobname)
124 continue
125
126 for expr in exprs:
127 if expr.match(jobname):
128 joblist.append(jobname)
129 break
130
131if listonly:
132 if verbose:
133 for jobname in joblist:
134 test.printinfo(jobname)
135 else:
136 for jobname in joblist:
137 print jobname
138 sys.exit(0)
139
140if not onlyecho:
141 jl = []
142 for jobname in joblist:
143 jobdir = joinpath(rootdir, jobname)
144 if os.path.exists(jobname):
145 if not force:
146 if os.path.isfile(joinpath(jobdir, '.success')):
147 continue
148
149 if os.path.isfile(joinpath(jobdir, '.start')) and \
150 not os.path.isfile(joinpath(jobdir, '.stop')):
151 continue
152
153 if not clean:
154 sys.exit('job directory not clean!')
155
156 job.cleandir(jobdir)
157 else:
158 os.mkdir(jobdir)
159 jl.append(jobname)
160 joblist = jl
161
162for jobname in joblist:
163 jobdir = joinpath(rootdir, jobname)
164
165 if not onlyecho and not os.path.isdir(jobdir):
166 sys.exit('%s is not a directory. Cannot build job' % jobdir)
167
|
159 print >>sys.stderr, 'Job name: %s' % jobname
160 print >>sys.stderr, 'Job directory: %s' % jobdir
| 168 print 'Job name: %s' % jobname
169 print 'Job directory: %s' % jobdir
|
161
162 qsub = pbs.qsub()
163 qsub.pbshost = 'simpool.eecs.umich.edu'
164 qsub.stdout = joinpath(jobdir, 'jobout')
165 qsub.name = jobname
166 qsub.join = True
167 qsub.node_type = 'FAST'
| 170
171 qsub = pbs.qsub()
172 qsub.pbshost = 'simpool.eecs.umich.edu'
173 qsub.stdout = joinpath(jobdir, 'jobout')
174 qsub.name = jobname
175 qsub.join = True
176 qsub.node_type = 'FAST'
|
168 qsub.onlyecho = onlyecho
| |
169 qsub.env['ROOTDIR'] = rootdir
| 177 qsub.env['ROOTDIR'] = rootdir
|
170 qsub.verbose = verbose
| |
171 if len(queue):
172 qsub.queue = queue
| 178 if len(queue):
179 qsub.queue = queue
|
| 180 qsub.build(joinpath(progpath, 'job.py'))
|
173
| 181
|
174 qsub.do(joinpath(basedir, 'job.py'))
175 print >>sys.stderr, ''
| 182 if verbose:
183 print 'PBS Command: %s' % qsub.command
184
185 if not onlyecho:
186 ec = qsub.do()
187 if ec == 0:
188 print 'PBS Jobid: %s' % qsub.result
189 else:
190 print 'PBS Failed'
|
| |