send.py revision 13540
14120Sgblack@eecs.umich.edu#!/usr/bin/env python2.7 24120Sgblack@eecs.umich.edu# Copyright (c) 2006 The Regents of The University of Michigan 34120Sgblack@eecs.umich.edu# All rights reserved. 44120Sgblack@eecs.umich.edu# 57087Snate@binkert.org# Redistribution and use in source and binary forms, with or without 67087Snate@binkert.org# modification, are permitted provided that the following conditions are 77087Snate@binkert.org# met: redistributions of source code must retain the above copyright 87087Snate@binkert.org# notice, this list of conditions and the following disclaimer; 97087Snate@binkert.org# redistributions in binary form must reproduce the above copyright 107087Snate@binkert.org# notice, this list of conditions and the following disclaimer in the 117087Snate@binkert.org# documentation and/or other materials provided with the distribution; 127087Snate@binkert.org# neither the name of the copyright holders nor the names of its 134120Sgblack@eecs.umich.edu# contributors may be used to endorse or promote products derived from 147087Snate@binkert.org# this software without specific prior written permission. 157087Snate@binkert.org# 167087Snate@binkert.org# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 177087Snate@binkert.org# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 187087Snate@binkert.org# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 197087Snate@binkert.org# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 207087Snate@binkert.org# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 217087Snate@binkert.org# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 224120Sgblack@eecs.umich.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 237087Snate@binkert.org# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 244120Sgblack@eecs.umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 254120Sgblack@eecs.umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 264120Sgblack@eecs.umich.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 274120Sgblack@eecs.umich.edu# 284120Sgblack@eecs.umich.edu# Authors: Kevin Lim 294120Sgblack@eecs.umich.edu 304120Sgblack@eecs.umich.eduimport os, os.path, re, socket, sys 314120Sgblack@eecs.umich.edufrom os import environ as env, listdir 324120Sgblack@eecs.umich.edufrom os.path import basename, isdir, isfile, islink, join as joinpath, normpath 334120Sgblack@eecs.umich.edufrom filecmp import cmp as filecmp 344120Sgblack@eecs.umich.edufrom shutil import copy 354120Sgblack@eecs.umich.edu 364120Sgblack@eecs.umich.edudef nfspath(dir): 374120Sgblack@eecs.umich.edu if dir.startswith('/.automount/'): 384120Sgblack@eecs.umich.edu dir = '/n/%s' % dir[12:] 394120Sgblack@eecs.umich.edu elif not dir.startswith('/n/'): 404120Sgblack@eecs.umich.edu dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir) 414120Sgblack@eecs.umich.edu return dir 424120Sgblack@eecs.umich.edu 434166Sgblack@eecs.umich.edudef syncdir(srcdir, destdir): 444166Sgblack@eecs.umich.edu srcdir = normpath(srcdir) 458229Snate@binkert.org destdir = normpath(destdir) 4611854Sbrandon.potter@amd.com if not isdir(destdir): 474166Sgblack@eecs.umich.edu sys.exit('destination directory "%s" does not exist' % destdir) 4810299Salexandru.dutu@amd.com 494120Sgblack@eecs.umich.edu for root, dirs, files in os.walk(srcdir): 505956Sgblack@eecs.umich.edu root = normpath(root) 515956Sgblack@eecs.umich.edu prefix = os.path.commonprefix([root, srcdir]) 524120Sgblack@eecs.umich.edu root = root[len(prefix):] 534120Sgblack@eecs.umich.edu if root.startswith('/'): 547073Sgblack@eecs.umich.edu root = root[1:] 557073Sgblack@eecs.umich.edu for rem in [ d for d in dirs if d.startswith('.') or d == 'SCCS']: 567073Sgblack@eecs.umich.edu dirs.remove(rem) 577073Sgblack@eecs.umich.edu 584166Sgblack@eecs.umich.edu for entry in dirs: 5911851Sbrandon.potter@amd.com newdir = joinpath(destdir, root, entry) 604166Sgblack@eecs.umich.edu if not isdir(newdir): 614166Sgblack@eecs.umich.edu os.mkdir(newdir) 625962Sgblack@eecs.umich.edu print 'mkdir', newdir 635962Sgblack@eecs.umich.edu 645962Sgblack@eecs.umich.edu for i,d in enumerate(dirs): 655956Sgblack@eecs.umich.edu if islink(joinpath(srcdir, root, d)): 665956Sgblack@eecs.umich.edu dirs[i] = joinpath(d, '.') 674166Sgblack@eecs.umich.edu 6811851Sbrandon.potter@amd.com for entry in files: 6911851Sbrandon.potter@amd.com dest = normpath(joinpath(destdir, root, entry)) 705956Sgblack@eecs.umich.edu src = normpath(joinpath(srcdir, root, entry)) 715956Sgblack@eecs.umich.edu if not isfile(dest) or not filecmp(src, dest): 725973Sgblack@eecs.umich.edu print 'copy %s %s' % (dest, src) 7311884Sbrandon.potter@amd.com copy(src, dest) 744166Sgblack@eecs.umich.edu 754166Sgblack@eecs.umich.eduprogpath = nfspath(sys.path[0]) 765962Sgblack@eecs.umich.eduprogname = basename(sys.argv[0]) 775962Sgblack@eecs.umich.eduusage = """\ 7811320Ssteve.reinhardt@amd.comUsage: 795962Sgblack@eecs.umich.edu %(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp> 805962Sgblack@eecs.umich.edu -c clean directory if job can be run 815962Sgblack@eecs.umich.edu -C submit the checkpointing runs 825956Sgblack@eecs.umich.edu -d Make jobs be dependent on the completion of the checkpoint runs 835958Sgblack@eecs.umich.edu -e only echo pbs command info, don't actually send the job 845958Sgblack@eecs.umich.edu -f force the job to run regardless of state 8511886Sbrandon.potter@amd.com -q <queue> submit job to the named queue 8611886Sbrandon.potter@amd.com -j <jobfile> specify the jobfile (default is <rootdir>/Test.py) 8711886Sbrandon.potter@amd.com -v be verbose 8811886Sbrandon.potter@amd.com 8911886Sbrandon.potter@amd.com %(progname)s [-j <jobfile>] -l [-v] <regexp> 9011886Sbrandon.potter@amd.com -j <jobfile> specify the jobfile (default is <rootdir>/Test.py) 9111886Sbrandon.potter@amd.com -l list job names, don't submit 9211886Sbrandon.potter@amd.com -v be verbose (list job parameters) 9311886Sbrandon.potter@amd.com 9411886Sbrandon.potter@amd.com %(progname)s -h 9511886Sbrandon.potter@amd.com -h display this help 9611886Sbrandon.potter@amd.com""" % locals() 9711886Sbrandon.potter@amd.com 9811886Sbrandon.potter@amd.comtry: 9911886Sbrandon.potter@amd.com import getopt 1005956Sgblack@eecs.umich.edu opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lnq:Rt:v') 1014166Sgblack@eecs.umich.eduexcept getopt.GetoptError: 10211851Sbrandon.potter@amd.com sys.exit(usage) 1035956Sgblack@eecs.umich.edu 1045956Sgblack@eecs.umich.edudepend = False 10511851Sbrandon.potter@amd.comclean = False 10611851Sbrandon.potter@amd.comonlyecho = False 1074166Sgblack@eecs.umich.eduexprs = [] 1086709Svince@csl.cornell.eduforce = False 1096709Svince@csl.cornell.edulistonly = False 1106709Svince@csl.cornell.eduqueue = '' 1116709Svince@csl.cornell.eduverbose = False 1126709Svince@csl.cornell.edujfile = 'Test.py' 1136709Svince@csl.cornell.edudocpts = False 1146709Svince@csl.cornell.edudoruns = True 11511886Sbrandon.potter@amd.comrunflag = False 11611886Sbrandon.potter@amd.comnode_type = 'FAST' 11711886Sbrandon.potter@amd.comupdate = True 11811886Sbrandon.potter@amd.com 11911886Sbrandon.potter@amd.comfor opt,arg in opts: 12011886Sbrandon.potter@amd.com if opt == '-C': 12111886Sbrandon.potter@amd.com docpts = True 12211886Sbrandon.potter@amd.com if opt == '-c': 12311886Sbrandon.potter@amd.com clean = True 12411886Sbrandon.potter@amd.com if opt == '-d': 12511886Sbrandon.potter@amd.com depend = True 12611886Sbrandon.potter@amd.com if opt == '-e': 12711886Sbrandon.potter@amd.com onlyecho = True 12811886Sbrandon.potter@amd.com if opt == '-f': 1296709Svince@csl.cornell.edu force = True 1306709Svince@csl.cornell.edu if opt == '-h': 1316709Svince@csl.cornell.edu print usage 1325956Sgblack@eecs.umich.edu sys.exit(0) 13311884Sbrandon.potter@amd.com if opt == '-j': 1347532Ssteve.reinhardt@amd.com jfile = arg 1355958Sgblack@eecs.umich.edu if opt == '-l': 1366701Sgblack@eecs.umich.edu listonly = True 1379552Sandreas.hansson@arm.com if opt == '-n': 13811851Sbrandon.potter@amd.com update = False 1395958Sgblack@eecs.umich.edu if opt == '-q': 14011886Sbrandon.potter@amd.com queue = arg 14111886Sbrandon.potter@amd.com if opt == '-R': 1425956Sgblack@eecs.umich.edu runflag = True 1435956Sgblack@eecs.umich.edu if opt == '-t': 14411851Sbrandon.potter@amd.com node_type = arg 1455956Sgblack@eecs.umich.edu if opt == '-v': 1465956Sgblack@eecs.umich.edu verbose = True 14711851Sbrandon.potter@amd.com 14811851Sbrandon.potter@amd.comif docpts: 1495956Sgblack@eecs.umich.edu doruns = runflag 1505973Sgblack@eecs.umich.edu 1515973Sgblack@eecs.umich.edufor arg in args: 1525973Sgblack@eecs.umich.edu exprs.append(re.compile(arg)) 1535973Sgblack@eecs.umich.edu 1545973Sgblack@eecs.umich.eduimport jobfile, batch 1555973Sgblack@eecs.umich.edufrom job import JobDir, date 1565973Sgblack@eecs.umich.edu 15711886Sbrandon.potter@amd.comconf = jobfile.JobFile(jfile) 15811886Sbrandon.potter@amd.com 15911886Sbrandon.potter@amd.comif update and not listonly and not onlyecho and isdir(conf.linkdir): 16011886Sbrandon.potter@amd.com if verbose: 16111886Sbrandon.potter@amd.com print 'Checking for outdated files in Link directory' 16211886Sbrandon.potter@amd.com if not isdir(conf.basedir): 16311886Sbrandon.potter@amd.com os.mkdir(conf.basedir) 16411886Sbrandon.potter@amd.com syncdir(conf.linkdir, conf.basedir) 16511886Sbrandon.potter@amd.com 16611886Sbrandon.potter@amd.comjobnames = {} 16711886Sbrandon.potter@amd.comjoblist = [] 16811886Sbrandon.potter@amd.com 16911886Sbrandon.potter@amd.comif docpts and doruns: 17011886Sbrandon.potter@amd.com gen = conf.alljobs() 1715973Sgblack@eecs.umich.eduelif docpts: 1725973Sgblack@eecs.umich.edu gen = conf.checkpoints() 1735973Sgblack@eecs.umich.eduelif doruns: 1745956Sgblack@eecs.umich.edu gen = conf.jobs() 17511884Sbrandon.potter@amd.com 1767532Ssteve.reinhardt@amd.comfor job in gen: 1775958Sgblack@eecs.umich.edu if job.name in jobnames: 17811877Sbrandon.potter@amd.com continue 1796701Sgblack@eecs.umich.edu 1806701Sgblack@eecs.umich.edu if exprs: 1815958Sgblack@eecs.umich.edu for expr in exprs: 18211886Sbrandon.potter@amd.com if expr.match(job.name): 18311886Sbrandon.potter@amd.com joblist.append(job) 1844166Sgblack@eecs.umich.edu break 18510299Salexandru.dutu@amd.com else: 18610299Salexandru.dutu@amd.com joblist.append(job) 18710299Salexandru.dutu@amd.com 18810299Salexandru.dutu@amd.comif listonly: 18910299Salexandru.dutu@amd.com if verbose: 19010299Salexandru.dutu@amd.com for job in joblist: 19110299Salexandru.dutu@amd.com job.printinfo() 19210299Salexandru.dutu@amd.com else: 1934166Sgblack@eecs.umich.edu for job in joblist: 1944120Sgblack@eecs.umich.edu print job.name 1954120Sgblack@eecs.umich.edu sys.exit(0) 196 197if not onlyecho: 198 newlist = [] 199 for job in joblist: 200 jobdir = JobDir(joinpath(conf.rootdir, job.name)) 201 if jobdir.exists(): 202 if not force: 203 status = jobdir.getstatus() 204 if status == 'queued': 205 continue 206 207 if status == 'running': 208 continue 209 210 if status == 'success': 211 continue 212 213 if not clean: 214 sys.exit('job directory %s not clean!' % jobdir) 215 216 jobdir.clean() 217 newlist.append(job) 218 joblist = newlist 219 220class NameHack(object): 221 def __init__(self, host='pbs.pool', port=24465): 222 self.host = host 223 self.port = port 224 self.socket = None 225 226 def setname(self, jobid, jobname): 227 try: 228 jobid = int(jobid) 229 except ValueError: 230 jobid = int(jobid.strip().split('.')[0]) 231 232 jobname = jobname.strip() 233 # since pbs can handle jobnames of 15 characters or less, 234 # don't use the raj hack. 235 if len(jobname) <= 15: 236 return 237 238 if self.socket is None: 239 import socket 240 self.socket = socket.socket() 241 # Connect to pbs.pool and send the jobid/jobname pair to port 242 # 24465 (Raj didn't realize that there are only 64k ports and 243 # setup inetd to point to port 90001) 244 self.socket.connect((self.host, self.port)) 245 246 self.socket.send("%s %s\n" % (jobid, jobname)) 247 248namehack = NameHack() 249 250rootdir = conf.rootdir 251script = joinpath(rootdir, 'Base', 'job.py') 252 253for job in joblist: 254 jobdir = JobDir(joinpath(rootdir, job.name)) 255 if depend: 256 cptdir = JobDir(joinpath(rootdir, job.checkpoint.name)) 257 path = str(cptdir) 258 if not isdir(path) or not isfile(joinpath(path, '.success')): 259 continue 260 261 cptjob = cptdir.readval('.batch_jobid') 262 263 if not onlyecho: 264 jobdir.create() 265 os.chdir(str(jobdir)) 266 os.environ['PWD'] = str(jobdir) 267 268 print 'Job name: %s' % job.name 269 print 'Job directory: %s' % jobdir 270 271 272 qsub = batch.oarsub() 273 qsub.oarhost = 'poolfs.eecs.umich.edu' 274 #qsub.stdout = jobdir.file('jobout') 275 qsub.name = job.name 276 qsub.walltime = '50' 277 #qsub.join = True 278 #qsub.node_type = node_type 279 #qsub.env['ROOTDIR'] = conf.rootdir 280 #qsub.env['JOBNAME'] = job.name 281 #if depend: 282 # qsub.afterok = cptjob 283 #if queue: 284 # qsub.queue = queue 285 qsub.properties = "64bit = 'Yes' or 64bit = 'No'" 286 qsub.build(script) 287 288 if verbose: 289 print 'cwd: %s' % qsub.command 290 print 'PBS Command: %s' % qsub.command 291 292 if not onlyecho: 293 ec = qsub.do() 294 if ec == 0: 295 jobid = qsub.result 296 print 'OAR Jobid: %s' % jobid 297 #namehack.setname(jobid, job.name) 298 queued = date() 299 jobdir.echofile('.batch_jobid', jobid) 300 jobdir.echofile('.batch_jobname', job.name) 301 jobdir.echofile('.queued', queued) 302 jobdir.setstatus('queued on %s' % queued) 303 else: 304 print 'OAR Failed' 305 print 306 print 307