send.py revision 1383
11376Sbinkertn@umich.edu#!/usr/bin/env python
21376Sbinkertn@umich.edu# Copyright (c) 2005 The Regents of The University of Michigan
31376Sbinkertn@umich.edu# All rights reserved.
41376Sbinkertn@umich.edu#
51376Sbinkertn@umich.edu# Redistribution and use in source and binary forms, with or without
61376Sbinkertn@umich.edu# modification, are permitted provided that the following conditions are
71376Sbinkertn@umich.edu# met: redistributions of source code must retain the above copyright
81376Sbinkertn@umich.edu# notice, this list of conditions and the following disclaimer;
91376Sbinkertn@umich.edu# redistributions in binary form must reproduce the above copyright
101376Sbinkertn@umich.edu# notice, this list of conditions and the following disclaimer in the
111376Sbinkertn@umich.edu# documentation and/or other materials provided with the distribution;
121376Sbinkertn@umich.edu# neither the name of the copyright holders nor the names of its
131376Sbinkertn@umich.edu# contributors may be used to endorse or promote products derived from
141376Sbinkertn@umich.edu# this software without specific prior written permission.
151376Sbinkertn@umich.edu#
161376Sbinkertn@umich.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
171376Sbinkertn@umich.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
181376Sbinkertn@umich.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
191376Sbinkertn@umich.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
201376Sbinkertn@umich.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
211376Sbinkertn@umich.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
221376Sbinkertn@umich.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
231376Sbinkertn@umich.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
241376Sbinkertn@umich.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
251376Sbinkertn@umich.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
261376Sbinkertn@umich.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
271376Sbinkertn@umich.edu#
281376Sbinkertn@umich.edu# Authors: Ali Saidi
291376Sbinkertn@umich.edu#          Nathan Binkert
301376Sbinkertn@umich.edu
311376Sbinkertn@umich.eduimport os, os.path, re, sys
321376Sbinkertn@umich.edufrom os import environ as env, listdir
331376Sbinkertn@umich.edufrom os.path import basename, isdir, isfile, islink, join as joinpath
341376Sbinkertn@umich.edufrom filecmp import cmp as filecmp
351376Sbinkertn@umich.edufrom shutil import copyfile
361376Sbinkertn@umich.edu
371376Sbinkertn@umich.eduprogname = basename(sys.argv[0])
381376Sbinkertn@umich.eduusage = """\
391376Sbinkertn@umich.eduUsage:
401376Sbinkertn@umich.edu    %(progname)s [-c] [-e] [-f] [-q queue] [-v] <regexp>
411376Sbinkertn@umich.edu    -c           clean directory if job can be run
421376Sbinkertn@umich.edu    -e           only echo pbs command info, don't actually send the job
431376Sbinkertn@umich.edu    -f           force the job to run regardless of state
441376Sbinkertn@umich.edu    -q <queue>   submit job to the named queue
451376Sbinkertn@umich.edu    -v           be verbose
461376Sbinkertn@umich.edu
471376Sbinkertn@umich.edu    %(progname)s -l [-v] <regexp>
481376Sbinkertn@umich.edu    -l           list job names, don't submit
491376Sbinkertn@umich.edu    -v           be verbose (list job parameters)
501376Sbinkertn@umich.edu
511376Sbinkertn@umich.edu    %(progname)s -h
521376Sbinkertn@umich.edu    -h           display this help
531376Sbinkertn@umich.edu""" % locals()
541376Sbinkertn@umich.edu
551376Sbinkertn@umich.edutry:
561376Sbinkertn@umich.edu    import getopt
571381Sbinkertn@umich.edu    opts, args = getopt.getopt(sys.argv[1:], '-cd:efhlq:v')
581376Sbinkertn@umich.eduexcept getopt.GetoptError:
591376Sbinkertn@umich.edu    sys.exit(usage)
601376Sbinkertn@umich.edu
611376Sbinkertn@umich.educlean = False
621376Sbinkertn@umich.eduonlyecho = False
631376Sbinkertn@umich.eduexprs = []
641376Sbinkertn@umich.eduforce = False
651376Sbinkertn@umich.edulistonly = False
661376Sbinkertn@umich.eduqueue = ''
671376Sbinkertn@umich.eduverbose = False
681381Sbinkertn@umich.edurootdir = re.sub(r'^/\.automount/', r'/n/', os.getcwd())
691381Sbinkertn@umich.edufor opt,arg in opts:
701381Sbinkertn@umich.edu    if opt == '-c':
711376Sbinkertn@umich.edu        clean = True
721381Sbinkertn@umich.edu    if opt == '-d':
731381Sbinkertn@umich.edu        rootdir = arg
741381Sbinkertn@umich.edu    if opt == '-e':
751376Sbinkertn@umich.edu        onlyecho = True
761381Sbinkertn@umich.edu    if opt == '-f':
771376Sbinkertn@umich.edu        force = True
781381Sbinkertn@umich.edu    if opt == '-h':
791376Sbinkertn@umich.edu        print usage
801376Sbinkertn@umich.edu        sys.exit(0)
811381Sbinkertn@umich.edu    if opt == '-l':
821376Sbinkertn@umich.edu        listonly = True
831381Sbinkertn@umich.edu    if opt == '-q':
841381Sbinkertn@umich.edu        queue = arg
851381Sbinkertn@umich.edu    if opt == '-v':
861376Sbinkertn@umich.edu        verbose = True
871376Sbinkertn@umich.edu
881381Sbinkertn@umich.edubasedir = joinpath(rootdir, 'Base')
891381Sbinkertn@umich.edulinkdir = joinpath(rootdir, 'Link')
901381Sbinkertn@umich.edu
911376Sbinkertn@umich.edufor arg in args:
921376Sbinkertn@umich.edu    exprs.append(re.compile(arg))
931376Sbinkertn@umich.edu
941381Sbinkertn@umich.eduif not listonly and not onlyecho and isdir(linkdir):
951376Sbinkertn@umich.edu    print 'Checking for outdated files in Link directory'
961381Sbinkertn@umich.edu    entries = listdir(linkdir)
971376Sbinkertn@umich.edu    for entry in entries:
981381Sbinkertn@umich.edu        link = joinpath(linkdir, entry)
991381Sbinkertn@umich.edu        if not islink(link) or not isfile(link):
1001376Sbinkertn@umich.edu            continue
1011376Sbinkertn@umich.edu
1021381Sbinkertn@umich.edu        base = joinpath(basedir, entry)
1031376Sbinkertn@umich.edu        if not isfile(base) or not filecmp(link, base):
1041381Sbinkertn@umich.edu            print 'Base/%s is different than Link/%s: copying' % (entry, entry)
1051376Sbinkertn@umich.edu            copyfile(link, base)
1061376Sbinkertn@umich.edu
1071376Sbinkertn@umich.eduimport job, jobfile, pbs
1081376Sbinkertn@umich.edu
1091381Sbinkertn@umich.edutest = jobfile.JobFile(joinpath(basedir, 'test.py'))
1101376Sbinkertn@umich.edu
1111376Sbinkertn@umich.edujoblist = []
1121376Sbinkertn@umich.edufor jobname in test.jobs:
1131376Sbinkertn@umich.edu    if not exprs:
1141376Sbinkertn@umich.edu        joblist.append(jobname)
1151376Sbinkertn@umich.edu        continue
1161376Sbinkertn@umich.edu
1171376Sbinkertn@umich.edu    for expr in exprs:
1181376Sbinkertn@umich.edu        if expr.match(jobname):
1191376Sbinkertn@umich.edu            joblist.append(jobname)
1201376Sbinkertn@umich.edu            break
1211376Sbinkertn@umich.edu
1221376Sbinkertn@umich.eduif listonly:
1231376Sbinkertn@umich.edu    if verbose:
1241376Sbinkertn@umich.edu        for jobname in joblist:
1251376Sbinkertn@umich.edu            test.printinfo(jobname)
1261376Sbinkertn@umich.edu    else:
1271376Sbinkertn@umich.edu        for jobname in joblist:
1281376Sbinkertn@umich.edu            print jobname
1291376Sbinkertn@umich.edu    sys.exit(0)
1301376Sbinkertn@umich.edu
1311376Sbinkertn@umich.eduif not onlyecho:
1321376Sbinkertn@umich.edu    jl = []
1331376Sbinkertn@umich.edu    for jobname in joblist:
1341383Sbinkertn@umich.edu        jobdir = joinpath(rootdir, jobname)
1351376Sbinkertn@umich.edu        if os.path.exists(jobname):
1361376Sbinkertn@umich.edu            if not force:
1371383Sbinkertn@umich.edu                if os.path.isfile(joinpath(jobdir, '.success')):
1381376Sbinkertn@umich.edu                    continue
1391376Sbinkertn@umich.edu
1401383Sbinkertn@umich.edu                if os.path.isfile(joinpath(jobdir, '.start')) and \
1411383Sbinkertn@umich.edu                       not os.path.isfile(joinpath(jobdir, '.stop')):
1421376Sbinkertn@umich.edu                    continue
1431376Sbinkertn@umich.edu
1441376Sbinkertn@umich.edu            if not clean:
1451376Sbinkertn@umich.edu                sys.exit('job directory not clean!')
1461376Sbinkertn@umich.edu
1471383Sbinkertn@umich.edu            job.cleandir(jobdir)
1481376Sbinkertn@umich.edu        else:
1491383Sbinkertn@umich.edu            os.mkdir(jobdir)
1501376Sbinkertn@umich.edu    jl.append(jobname)
1511376Sbinkertn@umich.edu    joblist = jl
1521376Sbinkertn@umich.edu
1531376Sbinkertn@umich.edufor jobname in joblist:
1541376Sbinkertn@umich.edu    jobdir = joinpath(rootdir, jobname)
1551376Sbinkertn@umich.edu
1561376Sbinkertn@umich.edu    if not onlyecho and not os.path.isdir(jobdir):
1571376Sbinkertn@umich.edu        sys.exit('%s is not a directory.  Cannot build job' % jobdir)
1581376Sbinkertn@umich.edu
1591376Sbinkertn@umich.edu    print >>sys.stderr, 'Job name:       %s' % jobname
1601376Sbinkertn@umich.edu    print >>sys.stderr, 'Job directory:  %s' % jobdir
1611376Sbinkertn@umich.edu
1621376Sbinkertn@umich.edu    qsub = pbs.qsub()
1631376Sbinkertn@umich.edu    qsub.pbshost = 'simpool.eecs.umich.edu'
1641376Sbinkertn@umich.edu    qsub.stdout = joinpath(jobdir, 'jobout')
1651376Sbinkertn@umich.edu    qsub.name = jobname
1661376Sbinkertn@umich.edu    qsub.join = True
1671376Sbinkertn@umich.edu    qsub.node_type = 'FAST'
1681376Sbinkertn@umich.edu    qsub.onlyecho = onlyecho
1691376Sbinkertn@umich.edu    qsub.env['ROOTDIR'] = rootdir
1701376Sbinkertn@umich.edu    qsub.verbose = verbose
1711376Sbinkertn@umich.edu    if len(queue):
1721376Sbinkertn@umich.edu        qsub.queue = queue
1731376Sbinkertn@umich.edu
1741381Sbinkertn@umich.edu    qsub.do(joinpath(basedir, 'job.py'))
1751376Sbinkertn@umich.edu    print >>sys.stderr, ''
176