send.py (1602:0fad8b5c6d2f) send.py (1816:ecb6cb1337e8)
1#!/usr/bin/env python
2# Copyright (c) 2005 The Regents of The University of Michigan
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met: redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer;

--- 16 unchanged lines hidden (view full) ---

25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27#
28# Authors: Ali Saidi
29# Nathan Binkert
30
31import os, os.path, re, socket, sys
32from os import environ as env, listdir
1#!/usr/bin/env python
2# Copyright (c) 2005 The Regents of The University of Michigan
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met: redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer;

--- 16 unchanged lines hidden (view full) ---

25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27#
28# Authors: Ali Saidi
29# Nathan Binkert
30
31import os, os.path, re, socket, sys
32from os import environ as env, listdir
33from os.path import basename, isdir, isfile, islink, join as joinpath
33from os.path import basename, isdir, isfile, islink, join as joinpath, normpath
34from filecmp import cmp as filecmp
34from filecmp import cmp as filecmp
35from shutil import copyfile
35from shutil import copy
36
37def nfspath(dir):
38 if dir.startswith('/.automount/'):
39 dir = '/n/%s' % dir[12:]
40 elif not dir.startswith('/n/'):
41 dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir)
42 return dir
43
36
37def nfspath(dir):
38 if dir.startswith('/.automount/'):
39 dir = '/n/%s' % dir[12:]
40 elif not dir.startswith('/n/'):
41 dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir)
42 return dir
43
44def syncdir(srcdir, destdir):
45 srcdir = normpath(srcdir)
46 destdir = normpath(destdir)
47 if not isdir(destdir):
48 sys.exit('destination directory "%s" does not exist' % destdir)
49
50 for root, dirs, files in os.walk(srcdir):
51 root = normpath(root)
52 prefix = os.path.commonprefix([root, srcdir])
53 root = root[len(prefix):]
54 if root.startswith('/'):
55 root = root[1:]
56 for rem in [ d for d in dirs if d.startswith('.') or d == 'SCCS']:
57 dirs.remove(rem)
58
59 for entry in dirs:
60 newdir = joinpath(destdir, root, entry)
61 if not isdir(newdir):
62 os.mkdir(newdir)
63 print 'mkdir', newdir
64
65 for i,d in enumerate(dirs):
66 if islink(joinpath(srcdir, root, d)):
67 dirs[i] = joinpath(d, '.')
68
69 for entry in files:
70 dest = normpath(joinpath(destdir, root, entry))
71 src = normpath(joinpath(srcdir, root, entry))
72 if not isfile(dest) or not filecmp(src, dest):
73 print 'copy %s %s' % (dest, src)
74 copy(src, dest)
75
44progpath = nfspath(sys.path[0])
45progname = basename(sys.argv[0])
46usage = """\
47Usage:
48 %(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp>
49 -c clean directory if job can be run
50 -e only echo pbs command info, don't actually send the job
51 -f force the job to run regardless of state

--- 50 unchanged lines hidden (view full) ---

102linkdir = joinpath(rootdir, 'Link')
103
104for arg in args:
105 exprs.append(re.compile(arg))
106
107if not listonly and not onlyecho and isdir(linkdir):
108 if verbose:
109 print 'Checking for outdated files in Link directory'
76progpath = nfspath(sys.path[0])
77progname = basename(sys.argv[0])
78usage = """\
79Usage:
80 %(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp>
81 -c clean directory if job can be run
82 -e only echo pbs command info, don't actually send the job
83 -f force the job to run regardless of state

--- 50 unchanged lines hidden (view full) ---

134linkdir = joinpath(rootdir, 'Link')
135
136for arg in args:
137 exprs.append(re.compile(arg))
138
139if not listonly and not onlyecho and isdir(linkdir):
140 if verbose:
141 print 'Checking for outdated files in Link directory'
110 entries = listdir(linkdir)
111 for entry in entries:
112 link = joinpath(linkdir, entry)
113 if not islink(link) or not isfile(link):
114 continue
142 syncdir(linkdir, basedir)
115
143
116 base = joinpath(basedir, entry)
117 if not isfile(base) or not filecmp(link, base):
118 print 'Base/%s is different than Link/%s: copying' % (entry, entry)
119 copyfile(link, base)
120
121import job, jobfile, pbs
122
123test = jobfile.JobFile(joinpath(basedir, jfile))
124
125joblist = []
126for jobname in test.jobs:
127 if not exprs:
128 joblist.append(jobname)

--- 30 unchanged lines hidden (view full) ---

159 sys.exit('job directory not clean!')
160
161 job.cleandir(jobdir)
162 else:
163 os.mkdir(jobdir)
164 jl.append(jobname)
165 joblist = jl
166
144import job, jobfile, pbs
145
146test = jobfile.JobFile(joinpath(basedir, jfile))
147
148joblist = []
149for jobname in test.jobs:
150 if not exprs:
151 joblist.append(jobname)

--- 30 unchanged lines hidden (view full) ---

182 sys.exit('job directory not clean!')
183
184 job.cleandir(jobdir)
185 else:
186 os.mkdir(jobdir)
187 jl.append(jobname)
188 joblist = jl
189
190def setname(jobid, jobname):
191 # since pbs can handle jobnames of 15 characters or less, don't
192 # use the raj hack.
193 if len(jobname) <= 15:
194 return
195
196 import socket
197 s = socket.socket()
198 # Connect to pbs.pool and send the jobid/jobname pair to port
199 # 24465 (Raj didn't realize that there are only 64k ports and
200 # setup inetd to point to port 90001)
201 s.connect(("pbs.pool", 24465))
202 s.send("%s %s\n" % (jobid, jobname))
203 s.close()
204
167for jobname in joblist:
168 jobdir = joinpath(rootdir, jobname)
169
170 if not onlyecho and not os.path.isdir(jobdir):
171 sys.exit('%s is not a directory. Cannot build job' % jobdir)
172
173 print 'Job name: %s' % jobname
174 print 'Job directory: %s' % jobdir
175
176 qsub = pbs.qsub()
177 qsub.pbshost = 'simpool.eecs.umich.edu'
178 qsub.stdout = joinpath(jobdir, 'jobout')
205for jobname in joblist:
206 jobdir = joinpath(rootdir, jobname)
207
208 if not onlyecho and not os.path.isdir(jobdir):
209 sys.exit('%s is not a directory. Cannot build job' % jobdir)
210
211 print 'Job name: %s' % jobname
212 print 'Job directory: %s' % jobdir
213
214 qsub = pbs.qsub()
215 qsub.pbshost = 'simpool.eecs.umich.edu'
216 qsub.stdout = joinpath(jobdir, 'jobout')
179 qsub.name = jobname
217 qsub.name = jobname[:15]
180 qsub.join = True
181 qsub.node_type = 'FAST'
182 qsub.env['ROOTDIR'] = rootdir
218 qsub.join = True
219 qsub.node_type = 'FAST'
220 qsub.env['ROOTDIR'] = rootdir
221 qsub.env['JOBNAME'] = jobname
183 if len(queue):
184 qsub.queue = queue
185 qsub.build(joinpath(progpath, 'job.py'))
186
187 if verbose:
188 print 'PBS Command: %s' % qsub.command
189
190 if not onlyecho:
191 ec = qsub.do()
192 if ec == 0:
222 if len(queue):
223 qsub.queue = queue
224 qsub.build(joinpath(progpath, 'job.py'))
225
226 if verbose:
227 print 'PBS Command: %s' % qsub.command
228
229 if not onlyecho:
230 ec = qsub.do()
231 if ec == 0:
193 print 'PBS Jobid: %s' % qsub.result
232 jobid = qsub.result
233 print 'PBS Jobid: %s' % jobid
234 setname(jobid, jobname)
194 else:
195 print 'PBS Failed'
235 else:
236 print 'PBS Failed'