send.py revision 1956:e9447a75c009
16019Shines@cs.fsu.edu#!/usr/bin/env python
26019Shines@cs.fsu.edu# Copyright (c) 2005 The Regents of The University of Michigan
313120SEdmund.Grimley-Evans@arm.com# All rights reserved.
47178Sgblack@eecs.umich.edu#
57178Sgblack@eecs.umich.edu# Redistribution and use in source and binary forms, with or without
67178Sgblack@eecs.umich.edu# modification, are permitted provided that the following conditions are
77178Sgblack@eecs.umich.edu# met: redistributions of source code must retain the above copyright
87178Sgblack@eecs.umich.edu# notice, this list of conditions and the following disclaimer;
97178Sgblack@eecs.umich.edu# redistributions in binary form must reproduce the above copyright
107178Sgblack@eecs.umich.edu# notice, this list of conditions and the following disclaimer in the
117178Sgblack@eecs.umich.edu# documentation and/or other materials provided with the distribution;
127178Sgblack@eecs.umich.edu# neither the name of the copyright holders nor the names of its
137178Sgblack@eecs.umich.edu# contributors may be used to endorse or promote products derived from
147178Sgblack@eecs.umich.edu# this software without specific prior written permission.
156019Shines@cs.fsu.edu#
166019Shines@cs.fsu.edu# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
176019Shines@cs.fsu.edu# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
186019Shines@cs.fsu.edu# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
196019Shines@cs.fsu.edu# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
206019Shines@cs.fsu.edu# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
216019Shines@cs.fsu.edu# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
226019Shines@cs.fsu.edu# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
236019Shines@cs.fsu.edu# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
246019Shines@cs.fsu.edu# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
256019Shines@cs.fsu.edu# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
266019Shines@cs.fsu.edu# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
276019Shines@cs.fsu.edu#
286019Shines@cs.fsu.edu# Authors: Ali Saidi
296019Shines@cs.fsu.edu#          Nathan Binkert
306019Shines@cs.fsu.edu
316019Shines@cs.fsu.eduimport os, os.path, re, socket, sys
326019Shines@cs.fsu.edufrom os import environ as env, listdir
336019Shines@cs.fsu.edufrom os.path import basename, isdir, isfile, islink, join as joinpath, normpath
346019Shines@cs.fsu.edufrom filecmp import cmp as filecmp
356019Shines@cs.fsu.edufrom shutil import copy
366019Shines@cs.fsu.edu
376019Shines@cs.fsu.edudef nfspath(dir):
386019Shines@cs.fsu.edu    if dir.startswith('/.automount/'):
396019Shines@cs.fsu.edu        dir = '/n/%s' % dir[12:]
406019Shines@cs.fsu.edu    elif not dir.startswith('/n/'):
416019Shines@cs.fsu.edu        dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir)
426019Shines@cs.fsu.edu    return dir
436019Shines@cs.fsu.edu
446019Shines@cs.fsu.edudef syncdir(srcdir, destdir):
456019Shines@cs.fsu.edu    srcdir = normpath(srcdir)
466019Shines@cs.fsu.edu    destdir = normpath(destdir)
476019Shines@cs.fsu.edu    if not isdir(destdir):
487639Sgblack@eecs.umich.edu        sys.exit('destination directory "%s" does not exist' % destdir)
497639Sgblack@eecs.umich.edu
507639Sgblack@eecs.umich.edu    for root, dirs, files in os.walk(srcdir):
517639Sgblack@eecs.umich.edu        root = normpath(root)
527639Sgblack@eecs.umich.edu        prefix = os.path.commonprefix([root, srcdir])
537639Sgblack@eecs.umich.edu        root = root[len(prefix):]
547639Sgblack@eecs.umich.edu        if root.startswith('/'):
557639Sgblack@eecs.umich.edu            root = root[1:]
567639Sgblack@eecs.umich.edu        for rem in [ d for d in dirs if d.startswith('.') or d == 'SCCS']:
577639Sgblack@eecs.umich.edu            dirs.remove(rem)
587639Sgblack@eecs.umich.edu
597639Sgblack@eecs.umich.edu        for entry in dirs:
607639Sgblack@eecs.umich.edu            newdir = joinpath(destdir, root, entry)
617639Sgblack@eecs.umich.edu            if not isdir(newdir):
627639Sgblack@eecs.umich.edu                os.mkdir(newdir)
637639Sgblack@eecs.umich.edu                print 'mkdir', newdir
647639Sgblack@eecs.umich.edu
657639Sgblack@eecs.umich.edu        for i,d in enumerate(dirs):
667639Sgblack@eecs.umich.edu            if islink(joinpath(srcdir, root, d)):
677639Sgblack@eecs.umich.edu                dirs[i] = joinpath(d, '.')
687639Sgblack@eecs.umich.edu
697639Sgblack@eecs.umich.edu        for entry in files:
707639Sgblack@eecs.umich.edu            dest = normpath(joinpath(destdir, root, entry))
717639Sgblack@eecs.umich.edu            src = normpath(joinpath(srcdir, root, entry))
727639Sgblack@eecs.umich.edu            if not isfile(dest) or not filecmp(src, dest):
737639Sgblack@eecs.umich.edu                print 'copy %s %s' % (dest, src)
747639Sgblack@eecs.umich.edu                copy(src, dest)
757639Sgblack@eecs.umich.edu
767639Sgblack@eecs.umich.eduprogpath = nfspath(sys.path[0])
777639Sgblack@eecs.umich.eduprogname = basename(sys.argv[0])
787639Sgblack@eecs.umich.eduusage = """\
797639Sgblack@eecs.umich.eduUsage:
807639Sgblack@eecs.umich.edu    %(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp>
817639Sgblack@eecs.umich.edu    -c           clean directory if job can be run
827639Sgblack@eecs.umich.edu    -C           submit the checkpointing runs
837639Sgblack@eecs.umich.edu    -d           Make jobs be dependent on the completion of the checkpoint runs
847639Sgblack@eecs.umich.edu    -e           only echo pbs command info, don't actually send the job
857639Sgblack@eecs.umich.edu    -f           force the job to run regardless of state
867639Sgblack@eecs.umich.edu    -q <queue>   submit job to the named queue
877639Sgblack@eecs.umich.edu    -j <jobfile> specify the jobfile (default is <rootdir>/Test.py)
887639Sgblack@eecs.umich.edu    -v           be verbose
897639Sgblack@eecs.umich.edu
907639Sgblack@eecs.umich.edu    %(progname)s [-j <jobfile>] -l [-v] <regexp>
917639Sgblack@eecs.umich.edu    -j <jobfile> specify the jobfile (default is <rootdir>/Test.py)
927639Sgblack@eecs.umich.edu    -l           list job names, don't submit
937639Sgblack@eecs.umich.edu    -v           be verbose (list job parameters)
947356Sgblack@eecs.umich.edu
957356Sgblack@eecs.umich.edu    %(progname)s -h
967356Sgblack@eecs.umich.edu    -h           display this help
977435Sgblack@eecs.umich.edu""" % locals()
987435Sgblack@eecs.umich.edu
997435Sgblack@eecs.umich.edutry:
1007435Sgblack@eecs.umich.edu    import getopt
1017435Sgblack@eecs.umich.edu    opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lq:Rt:v')
1027435Sgblack@eecs.umich.eduexcept getopt.GetoptError:
1037435Sgblack@eecs.umich.edu    sys.exit(usage)
1047435Sgblack@eecs.umich.edu
1057435Sgblack@eecs.umich.edudepend = False
1067435Sgblack@eecs.umich.educlean = False
1077435Sgblack@eecs.umich.eduonlyecho = False
1087639Sgblack@eecs.umich.eduexprs = []
1097639Sgblack@eecs.umich.eduforce = False
1107639Sgblack@eecs.umich.edulistonly = False
1117435Sgblack@eecs.umich.eduqueue = ''
1127639Sgblack@eecs.umich.eduverbose = False
1137639Sgblack@eecs.umich.edujfile = 'Test.py'
1147639Sgblack@eecs.umich.edudocpts = False
1157639Sgblack@eecs.umich.edudoruns = True
1167639Sgblack@eecs.umich.edurunflag = False
1177639Sgblack@eecs.umich.edunode_type = 'FAST'
1187639Sgblack@eecs.umich.edu
1197639Sgblack@eecs.umich.edufor opt,arg in opts:
1207639Sgblack@eecs.umich.edu    if opt == '-C':
1217639Sgblack@eecs.umich.edu        docpts = True
1227639Sgblack@eecs.umich.edu    if opt == '-c':
1237639Sgblack@eecs.umich.edu        clean = True
1247639Sgblack@eecs.umich.edu    if opt == '-d':
1257639Sgblack@eecs.umich.edu        depend = True
1267639Sgblack@eecs.umich.edu    if opt == '-e':
1277639Sgblack@eecs.umich.edu        onlyecho = True
1287639Sgblack@eecs.umich.edu    if opt == '-f':
1297639Sgblack@eecs.umich.edu        force = True
1307639Sgblack@eecs.umich.edu    if opt == '-h':
1317639Sgblack@eecs.umich.edu        print usage
1327639Sgblack@eecs.umich.edu        sys.exit(0)
13312595Ssiddhesh.poyarekar@gmail.com    if opt == '-j':
1347639Sgblack@eecs.umich.edu        jfile = arg
1357639Sgblack@eecs.umich.edu    if opt == '-l':
1367639Sgblack@eecs.umich.edu        listonly = True
1377639Sgblack@eecs.umich.edu    if opt == '-q':
1387639Sgblack@eecs.umich.edu        queue = arg
1397639Sgblack@eecs.umich.edu    if opt == '-R':
1407639Sgblack@eecs.umich.edu        runflag = True
1417639Sgblack@eecs.umich.edu    if opt == '-t':
1427639Sgblack@eecs.umich.edu        node_type = arg
1437639Sgblack@eecs.umich.edu    if opt == '-v':
1447639Sgblack@eecs.umich.edu        verbose = True
1457639Sgblack@eecs.umich.edu
1468144SAli.Saidi@ARM.comif docpts:
1477639Sgblack@eecs.umich.edu    doruns = runflag
1487639Sgblack@eecs.umich.edu
1497639Sgblack@eecs.umich.edufor arg in args:
1507639Sgblack@eecs.umich.edu    exprs.append(re.compile(arg))
1517639Sgblack@eecs.umich.edu
1527639Sgblack@eecs.umich.eduimport jobfile, pbs
1537639Sgblack@eecs.umich.edufrom job import JobDir, date
15410037SARM gem5 Developers
1557639Sgblack@eecs.umich.educonf = jobfile.JobFile(jfile)
1567639Sgblack@eecs.umich.edu
1577639Sgblack@eecs.umich.eduif not listonly and not onlyecho and isdir(conf.linkdir):
1587639Sgblack@eecs.umich.edu    if verbose:
1597639Sgblack@eecs.umich.edu        print 'Checking for outdated files in Link directory'
1607639Sgblack@eecs.umich.edu    if not isdir(conf.basedir):
1617639Sgblack@eecs.umich.edu        os.mkdir(conf.basedir)
1627639Sgblack@eecs.umich.edu    syncdir(conf.linkdir, conf.basedir)
1637639Sgblack@eecs.umich.edu
1647639Sgblack@eecs.umich.edujobnames = {}
1657639Sgblack@eecs.umich.edujoblist = []
16610037SARM gem5 Developers
1677639Sgblack@eecs.umich.eduif docpts and doruns:
1687639Sgblack@eecs.umich.edu    gen = conf.alljobs()
1697639Sgblack@eecs.umich.eduelif docpts:
1707639Sgblack@eecs.umich.edu    gen = conf.checkpoints()
1717639Sgblack@eecs.umich.eduelif doruns:
1727639Sgblack@eecs.umich.edu    gen = conf.jobs()
1737639Sgblack@eecs.umich.edu
1747639Sgblack@eecs.umich.edufor job in gen:
17510037SARM gem5 Developers    if job.name in jobnames:
1767639Sgblack@eecs.umich.edu        continue
1777639Sgblack@eecs.umich.edu
17810037SARM gem5 Developers    if exprs:
1797639Sgblack@eecs.umich.edu        for expr in exprs:
1807639Sgblack@eecs.umich.edu            if expr.match(job.name):
18110037SARM gem5 Developers                joblist.append(job)
1827591SAli.Saidi@ARM.com                break
1837639Sgblack@eecs.umich.edu    else:
1847435Sgblack@eecs.umich.edu        joblist.append(job)
1857435Sgblack@eecs.umich.edu
1867639Sgblack@eecs.umich.eduif listonly:
18710037SARM gem5 Developers    if verbose:
1887639Sgblack@eecs.umich.edu        for job in joblist:
1897639Sgblack@eecs.umich.edu            job.printinfo()
1907639Sgblack@eecs.umich.edu    else:
1917639Sgblack@eecs.umich.edu        for job in joblist:
1927639Sgblack@eecs.umich.edu            print job.name
1937639Sgblack@eecs.umich.edu    sys.exit(0)
1947639Sgblack@eecs.umich.edu
1957639Sgblack@eecs.umich.eduif not onlyecho:
1967639Sgblack@eecs.umich.edu    newlist = []
1977639Sgblack@eecs.umich.edu    for job in joblist:
1987639Sgblack@eecs.umich.edu        jobdir = JobDir(joinpath(conf.rootdir, job.name))
1997639Sgblack@eecs.umich.edu        if jobdir.exists():
2007639Sgblack@eecs.umich.edu            if not force:
2017639Sgblack@eecs.umich.edu                status = jobdir.getstatus()
2027639Sgblack@eecs.umich.edu                if status == 'queued':
2037639Sgblack@eecs.umich.edu                    continue
2047639Sgblack@eecs.umich.edu
2057639Sgblack@eecs.umich.edu                if status == 'running':
2067639Sgblack@eecs.umich.edu                    continue
2077639Sgblack@eecs.umich.edu
2087639Sgblack@eecs.umich.edu                if status == 'success':
2097639Sgblack@eecs.umich.edu                    continue
2107639Sgblack@eecs.umich.edu
2117639Sgblack@eecs.umich.edu            if not clean:
2127639Sgblack@eecs.umich.edu                sys.exit('job directory %s not clean!' % jobdir)
2137639Sgblack@eecs.umich.edu
2147639Sgblack@eecs.umich.edu            jobdir.clean()
2157639Sgblack@eecs.umich.edu        newlist.append(job)
2167639Sgblack@eecs.umich.edu    joblist = newlist
2177639Sgblack@eecs.umich.edu
2187639Sgblack@eecs.umich.educlass NameHack(object):
2197639Sgblack@eecs.umich.edu    def __init__(self, host='pbs.pool', port=24465):
22010037SARM gem5 Developers        self.host = host
2217591SAli.Saidi@ARM.com        self.port = port
2227591SAli.Saidi@ARM.com        self.socket = None
2237639Sgblack@eecs.umich.edu
2247639Sgblack@eecs.umich.edu    def setname(self, jobid, jobname):
2257639Sgblack@eecs.umich.edu        try:
22610037SARM gem5 Developers            jobid = int(jobid)
2277639Sgblack@eecs.umich.edu        except ValueError:
2287639Sgblack@eecs.umich.edu            jobid = int(jobid.strip().split('.')[0])
2297639Sgblack@eecs.umich.edu
2307639Sgblack@eecs.umich.edu        jobname = jobname.strip()
2317639Sgblack@eecs.umich.edu        # since pbs can handle jobnames of 15 characters or less,
2327639Sgblack@eecs.umich.edu        # don't use the raj hack.
2337639Sgblack@eecs.umich.edu        if len(jobname) <= 15:
2347639Sgblack@eecs.umich.edu            return
23510037SARM gem5 Developers
2367639Sgblack@eecs.umich.edu        if self.socket is None:
2377639Sgblack@eecs.umich.edu            import socket
2387639Sgblack@eecs.umich.edu            self.socket = socket.socket()
23910037SARM gem5 Developers            # Connect to pbs.pool and send the jobid/jobname pair to port
2407639Sgblack@eecs.umich.edu            # 24465 (Raj didn't realize that there are only 64k ports and
2417639Sgblack@eecs.umich.edu            # setup inetd to point to port 90001)
2427639Sgblack@eecs.umich.edu            self.socket.connect((self.host, self.port))
2437435Sgblack@eecs.umich.edu
2447435Sgblack@eecs.umich.edu        self.socket.send("%s %s\n" % (jobid, jobname))
2457639Sgblack@eecs.umich.edu
2467639Sgblack@eecs.umich.edunamehack = NameHack()
2477639Sgblack@eecs.umich.edu
2487639Sgblack@eecs.umich.edufor job in joblist:
2497639Sgblack@eecs.umich.edu    jobdir = JobDir(joinpath(conf.rootdir, job.name))
2507639Sgblack@eecs.umich.edu    if depend:
2517639Sgblack@eecs.umich.edu        cptdir = JobDir(joinpath(conf.rootdir, job.checkpoint.name))
2527639Sgblack@eecs.umich.edu        cptjob = cptdir.readval('.pbs_jobid')
25310037SARM gem5 Developers
2547639Sgblack@eecs.umich.edu    if not onlyecho:
25510037SARM gem5 Developers        jobdir.create()
2567639Sgblack@eecs.umich.edu
2577639Sgblack@eecs.umich.edu    print 'Job name:       %s' % job.name
2587639Sgblack@eecs.umich.edu    print 'Job directory:  %s' % jobdir
2597639Sgblack@eecs.umich.edu
2607639Sgblack@eecs.umich.edu    qsub = pbs.qsub()
2617639Sgblack@eecs.umich.edu    qsub.pbshost = 'simpool.eecs.umich.edu'
2627639Sgblack@eecs.umich.edu    qsub.stdout = jobdir.file('jobout')
2637639Sgblack@eecs.umich.edu    qsub.name = job.name[:15]
2647639Sgblack@eecs.umich.edu    qsub.join = True
2657639Sgblack@eecs.umich.edu    qsub.node_type = node_type
2667639Sgblack@eecs.umich.edu    qsub.env['ROOTDIR'] = conf.rootdir
2677639Sgblack@eecs.umich.edu    qsub.env['JOBNAME'] = job.name
2687639Sgblack@eecs.umich.edu    if depend:
2697639Sgblack@eecs.umich.edu        qsub.afterok = cptjob
2707639Sgblack@eecs.umich.edu    if queue:
2717639Sgblack@eecs.umich.edu        qsub.queue = queue
2727639Sgblack@eecs.umich.edu    qsub.build(joinpath(progpath, 'job.py'))
2737639Sgblack@eecs.umich.edu
2747639Sgblack@eecs.umich.edu    if verbose:
2757639Sgblack@eecs.umich.edu        print 'PBS Command:    %s' % qsub.command
2767639Sgblack@eecs.umich.edu
2777639Sgblack@eecs.umich.edu    if not onlyecho:
2787639Sgblack@eecs.umich.edu        ec = qsub.do()
2797639Sgblack@eecs.umich.edu        if ec == 0:
2807639Sgblack@eecs.umich.edu            jobid = qsub.result
2817639Sgblack@eecs.umich.edu            print 'PBS Jobid:      %s' % jobid
2827639Sgblack@eecs.umich.edu            namehack.setname(jobid, job.name)
2837639Sgblack@eecs.umich.edu            queued = date()
2847639Sgblack@eecs.umich.edu            jobdir.echofile('.pbs_jobid', jobid)
2857639Sgblack@eecs.umich.edu            jobdir.echofile('.pbs_jobname', job.name)
2867639Sgblack@eecs.umich.edu            jobdir.echofile('.queued', queued)
2877639Sgblack@eecs.umich.edu            jobdir.setstatus('queued on %s' % queued)
2887639Sgblack@eecs.umich.edu        else:
2897639Sgblack@eecs.umich.edu            print 'PBS Failed'
2907639Sgblack@eecs.umich.edu