send.py revision 1602
112697Santhony.gutierrez@amd.com#!/usr/bin/env python 212697Santhony.gutierrez@amd.com# Copyright (c) 2005 The Regents of The University of Michigan 311308Santhony.gutierrez@amd.com# All rights reserved. 412697Santhony.gutierrez@amd.com# 511308Santhony.gutierrez@amd.com# Redistribution and use in source and binary forms, with or without 612697Santhony.gutierrez@amd.com# modification, are permitted provided that the following conditions are 712697Santhony.gutierrez@amd.com# met: redistributions of source code must retain the above copyright 811308Santhony.gutierrez@amd.com# notice, this list of conditions and the following disclaimer; 912697Santhony.gutierrez@amd.com# redistributions in binary form must reproduce the above copyright 1012697Santhony.gutierrez@amd.com# notice, this list of conditions and the following disclaimer in the 1111308Santhony.gutierrez@amd.com# documentation and/or other materials provided with the distribution; 1212697Santhony.gutierrez@amd.com# neither the name of the copyright holders nor the names of its 1312697Santhony.gutierrez@amd.com# contributors may be used to endorse or promote products derived from 1412697Santhony.gutierrez@amd.com# this software without specific prior written permission. 1511308Santhony.gutierrez@amd.com# 1612697Santhony.gutierrez@amd.com# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1712697Santhony.gutierrez@amd.com# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1812697Santhony.gutierrez@amd.com# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 1911308Santhony.gutierrez@amd.com# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2012697Santhony.gutierrez@amd.com# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2112697Santhony.gutierrez@amd.com# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2212697Santhony.gutierrez@amd.com# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2312697Santhony.gutierrez@amd.com# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2412697Santhony.gutierrez@amd.com# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2512697Santhony.gutierrez@amd.com# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2612697Santhony.gutierrez@amd.com# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2712697Santhony.gutierrez@amd.com# 2812697Santhony.gutierrez@amd.com# Authors: Ali Saidi 2912697Santhony.gutierrez@amd.com# Nathan Binkert 3012697Santhony.gutierrez@amd.com 3111308Santhony.gutierrez@amd.comimport os, os.path, re, socket, sys 3212697Santhony.gutierrez@amd.comfrom os import environ as env, listdir 3311308Santhony.gutierrez@amd.comfrom os.path import basename, isdir, isfile, islink, join as joinpath 3411308Santhony.gutierrez@amd.comfrom filecmp import cmp as filecmp 3511308Santhony.gutierrez@amd.comfrom shutil import copyfile 3611308Santhony.gutierrez@amd.com 3711308Santhony.gutierrez@amd.comdef nfspath(dir): 3813400Sodanrc@yahoo.com.br if dir.startswith('/.automount/'): 3911308Santhony.gutierrez@amd.com dir = '/n/%s' % dir[12:] 4011308Santhony.gutierrez@amd.com elif not dir.startswith('/n/'): 4113951Sodanrc@yahoo.com.br dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir) 4211308Santhony.gutierrez@amd.com return dir 4313400Sodanrc@yahoo.com.br 4413400Sodanrc@yahoo.com.brprogpath = nfspath(sys.path[0]) 4511670Sandreas.hansson@arm.comprogname = basename(sys.argv[0]) 4611670Sandreas.hansson@arm.comusage = """\ 4711308Santhony.gutierrez@amd.comUsage: 4811308Santhony.gutierrez@amd.com %(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp> 4911308Santhony.gutierrez@amd.com -c clean directory if job can be run 5011308Santhony.gutierrez@amd.com -e only echo pbs command info, don't actually send the job 5111308Santhony.gutierrez@amd.com -f force the job to run regardless of state 5211308Santhony.gutierrez@amd.com -q <queue> submit job to the named queue 5311308Santhony.gutierrez@amd.com -j <jobfile> specify the jobfile (default is <basedir>/test.py) 5411308Santhony.gutierrez@amd.com -v be verbose 5511308Santhony.gutierrez@amd.com 5611308Santhony.gutierrez@amd.com %(progname)s [-j <jobfile>] -l [-v] <regexp> 5711308Santhony.gutierrez@amd.com -j <jobfile> specify the jobfile (default is <basedir>/test.py) 5811308Santhony.gutierrez@amd.com -l list job names, don't submit 5911308Santhony.gutierrez@amd.com -v be verbose (list job parameters) 6011308Santhony.gutierrez@amd.com 6111308Santhony.gutierrez@amd.com %(progname)s -h 6211308Santhony.gutierrez@amd.com -h display this help 6311308Santhony.gutierrez@amd.com""" % locals() 6411308Santhony.gutierrez@amd.com 6511308Santhony.gutierrez@amd.comtry: 6611308Santhony.gutierrez@amd.com import getopt 6711308Santhony.gutierrez@amd.com opts, args = getopt.getopt(sys.argv[1:], '-cd:efhj:lq:v') 6811308Santhony.gutierrez@amd.comexcept getopt.GetoptError: 6911308Santhony.gutierrez@amd.com sys.exit(usage) 7011308Santhony.gutierrez@amd.com 7111308Santhony.gutierrez@amd.comclean = False 7211308Santhony.gutierrez@amd.comonlyecho = False 7311308Santhony.gutierrez@amd.comexprs = [] 7411308Santhony.gutierrez@amd.comforce = False 7511308Santhony.gutierrez@amd.comlistonly = False 7611308Santhony.gutierrez@amd.comqueue = '' 7711308Santhony.gutierrez@amd.comverbose = False 7811308Santhony.gutierrez@amd.comrootdir = nfspath(os.getcwd()) 7911308Santhony.gutierrez@amd.comjfile = 'test.py' 8011308Santhony.gutierrez@amd.comfor opt,arg in opts: 8111308Santhony.gutierrez@amd.com if opt == '-c': 8211308Santhony.gutierrez@amd.com clean = True 8311308Santhony.gutierrez@amd.com if opt == '-d': 8411308Santhony.gutierrez@amd.com rootdir = arg 8511308Santhony.gutierrez@amd.com if opt == '-e': 8611308Santhony.gutierrez@amd.com onlyecho = True 8711308Santhony.gutierrez@amd.com if opt == '-f': 8811308Santhony.gutierrez@amd.com force = True 8911308Santhony.gutierrez@amd.com if opt == '-h': 9011308Santhony.gutierrez@amd.com print usage 9111308Santhony.gutierrez@amd.com sys.exit(0) 9211308Santhony.gutierrez@amd.com if opt == '-j': 9311308Santhony.gutierrez@amd.com jfile = arg 9411308Santhony.gutierrez@amd.com if opt == '-l': 9511308Santhony.gutierrez@amd.com listonly = True 9611308Santhony.gutierrez@amd.com if opt == '-q': 9711308Santhony.gutierrez@amd.com queue = arg 9811308Santhony.gutierrez@amd.com if opt == '-v': 9911308Santhony.gutierrez@amd.com verbose = True 10011308Santhony.gutierrez@amd.com 10111308Santhony.gutierrez@amd.combasedir = joinpath(rootdir, 'Base') 10211308Santhony.gutierrez@amd.comlinkdir = joinpath(rootdir, 'Link') 10311308Santhony.gutierrez@amd.com 10411308Santhony.gutierrez@amd.comfor arg in args: 10511308Santhony.gutierrez@amd.com exprs.append(re.compile(arg)) 10611308Santhony.gutierrez@amd.com 10711308Santhony.gutierrez@amd.comif not listonly and not onlyecho and isdir(linkdir): 10811308Santhony.gutierrez@amd.com if verbose: 10911308Santhony.gutierrez@amd.com print 'Checking for outdated files in Link directory' 11011308Santhony.gutierrez@amd.com entries = listdir(linkdir) 11111308Santhony.gutierrez@amd.com for entry in entries: 11211308Santhony.gutierrez@amd.com link = joinpath(linkdir, entry) 11311308Santhony.gutierrez@amd.com if not islink(link) or not isfile(link): 11411308Santhony.gutierrez@amd.com continue 11511308Santhony.gutierrez@amd.com 11611308Santhony.gutierrez@amd.com base = joinpath(basedir, entry) 11711308Santhony.gutierrez@amd.com if not isfile(base) or not filecmp(link, base): 11811308Santhony.gutierrez@amd.com print 'Base/%s is different than Link/%s: copying' % (entry, entry) 11911308Santhony.gutierrez@amd.com copyfile(link, base) 12011308Santhony.gutierrez@amd.com 12111308Santhony.gutierrez@amd.comimport job, jobfile, pbs 12211308Santhony.gutierrez@amd.com 12311308Santhony.gutierrez@amd.comtest = jobfile.JobFile(joinpath(basedir, jfile)) 12411308Santhony.gutierrez@amd.com 12511308Santhony.gutierrez@amd.comjoblist = [] 12611308Santhony.gutierrez@amd.comfor jobname in test.jobs: 12711308Santhony.gutierrez@amd.com if not exprs: 12811308Santhony.gutierrez@amd.com joblist.append(jobname) 12911308Santhony.gutierrez@amd.com continue 13011308Santhony.gutierrez@amd.com 13111308Santhony.gutierrez@amd.com for expr in exprs: 13211308Santhony.gutierrez@amd.com if expr.match(jobname): 13311308Santhony.gutierrez@amd.com joblist.append(jobname) 13411308Santhony.gutierrez@amd.com break 13511308Santhony.gutierrez@amd.com 13611308Santhony.gutierrez@amd.comif listonly: 13711308Santhony.gutierrez@amd.com if verbose: 13811308Santhony.gutierrez@amd.com for jobname in joblist: 13911308Santhony.gutierrez@amd.com test.printinfo(jobname) 14011308Santhony.gutierrez@amd.com else: 14111308Santhony.gutierrez@amd.com for jobname in joblist: 14211308Santhony.gutierrez@amd.com print jobname 14311308Santhony.gutierrez@amd.com sys.exit(0) 14411308Santhony.gutierrez@amd.com 14511308Santhony.gutierrez@amd.comif not onlyecho: 14611308Santhony.gutierrez@amd.com jl = [] 14711308Santhony.gutierrez@amd.com for jobname in joblist: 14811308Santhony.gutierrez@amd.com jobdir = joinpath(rootdir, jobname) 14911308Santhony.gutierrez@amd.com if os.path.exists(jobname): 15011308Santhony.gutierrez@amd.com if not force: 15111308Santhony.gutierrez@amd.com if os.path.isfile(joinpath(jobdir, '.success')): 15211308Santhony.gutierrez@amd.com continue 15311308Santhony.gutierrez@amd.com 15411308Santhony.gutierrez@amd.com if os.path.isfile(joinpath(jobdir, '.start')) and \ 15511308Santhony.gutierrez@amd.com not os.path.isfile(joinpath(jobdir, '.stop')): 15611308Santhony.gutierrez@amd.com continue 15711308Santhony.gutierrez@amd.com 15811308Santhony.gutierrez@amd.com if not clean: 15911308Santhony.gutierrez@amd.com sys.exit('job directory not clean!') 16011308Santhony.gutierrez@amd.com 16111308Santhony.gutierrez@amd.com job.cleandir(jobdir) 16211308Santhony.gutierrez@amd.com else: 16311308Santhony.gutierrez@amd.com os.mkdir(jobdir) 16411308Santhony.gutierrez@amd.com jl.append(jobname) 16511308Santhony.gutierrez@amd.com joblist = jl 16611308Santhony.gutierrez@amd.com 16711308Santhony.gutierrez@amd.comfor jobname in joblist: 16811308Santhony.gutierrez@amd.com jobdir = joinpath(rootdir, jobname) 16911308Santhony.gutierrez@amd.com 17011308Santhony.gutierrez@amd.com if not onlyecho and not os.path.isdir(jobdir): 17111308Santhony.gutierrez@amd.com sys.exit('%s is not a directory. Cannot build job' % jobdir) 17212065Snikos.nikoleris@arm.com 17311308Santhony.gutierrez@amd.com print 'Job name: %s' % jobname 17411308Santhony.gutierrez@amd.com print 'Job directory: %s' % jobdir 17511308Santhony.gutierrez@amd.com 17611308Santhony.gutierrez@amd.com qsub = pbs.qsub() 17712065Snikos.nikoleris@arm.com qsub.pbshost = 'simpool.eecs.umich.edu' 17812065Snikos.nikoleris@arm.com qsub.stdout = joinpath(jobdir, 'jobout') 17911308Santhony.gutierrez@amd.com qsub.name = jobname 18011308Santhony.gutierrez@amd.com qsub.join = True 18111308Santhony.gutierrez@amd.com qsub.node_type = 'FAST' 18211308Santhony.gutierrez@amd.com qsub.env['ROOTDIR'] = rootdir 18311308Santhony.gutierrez@amd.com if len(queue): 18411308Santhony.gutierrez@amd.com qsub.queue = queue 18511308Santhony.gutierrez@amd.com qsub.build(joinpath(progpath, 'job.py')) 18611308Santhony.gutierrez@amd.com 18711308Santhony.gutierrez@amd.com if verbose: 18811308Santhony.gutierrez@amd.com print 'PBS Command: %s' % qsub.command 18911308Santhony.gutierrez@amd.com 19011308Santhony.gutierrez@amd.com if not onlyecho: 19111308Santhony.gutierrez@amd.com ec = qsub.do() 19211308Santhony.gutierrez@amd.com if ec == 0: 19311308Santhony.gutierrez@amd.com print 'PBS Jobid: %s' % qsub.result 19411308Santhony.gutierrez@amd.com else: 19511308Santhony.gutierrez@amd.com print 'PBS Failed' 19611308Santhony.gutierrez@amd.com