job.py (1816:ecb6cb1337e8) | job.py (1881:fc205a7edd58) |
---|---|
1#!/usr/bin/env python 2# Copyright (c) 2005 The Regents of The University of Michigan 3# All rights reserved. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer; --- 15 unchanged lines hidden (view full) --- 24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27# 28# Authors: Nathan Binkert 29# Steve Reinhardt 30# Ali Saidi 31 | 1#!/usr/bin/env python 2# Copyright (c) 2005 The Regents of The University of Michigan 3# All rights reserved. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer; --- 15 unchanged lines hidden (view full) --- 24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27# 28# Authors: Nathan Binkert 29# Steve Reinhardt 30# Ali Saidi 31 |
32import os, os.path, shutil, signal, socket, sys, time | 32import os, os.path, shutil, signal, socket, sys |
33from os import environ as env 34from os.path import join as joinpath, expanduser 35 | 33from os import environ as env 34from os.path import join as joinpath, expanduser 35 |
36def date(): 37 import time 38 return time.strftime('%a %b %e %H:%M:%S %Z %Y', time.localtime()) 39 40def cleandir(dir): 41 for root, dirs, files in os.walk(dir, False): 42 for name in files: 43 os.remove(joinpath(root, name)) 44 for name in dirs: 45 os.rmdir(joinpath(root, name)) 46 |
|
36class rsync: 37 def __init__(self): 38 self.sudo = False 39 self.rsync = 'rsync' 40 self.compress = False 41 self.archive = True 42 self.delete = False 43 self.options = '' --- 12 unchanged lines hidden (view full) --- 56 args.append('--delete') 57 if len(self.options): 58 args.append(self.options) 59 args.append(src) 60 args.append(dst) 61 62 return os.spawnvp(os.P_WAIT, args[0], args) 63 | 47class rsync: 48 def __init__(self): 49 self.sudo = False 50 self.rsync = 'rsync' 51 self.compress = False 52 self.archive = True 53 self.delete = False 54 self.options = '' --- 12 unchanged lines hidden (view full) --- 67 args.append('--delete') 68 if len(self.options): 69 args.append(self.options) 70 args.append(src) 71 args.append(dst) 72 73 return os.spawnvp(os.P_WAIT, args[0], args) 74 |
64def cleandir(dir): 65 for root, dirs, files in os.walk(dir, False): 66 for name in files: 67 os.remove(joinpath(root, name)) 68 for name in dirs: 69 os.rmdir(joinpath(root, name)) | 75class JobDir(object): 76 def __init__(self, dir): 77 self.dir = dir |
70 | 78 |
71def date(): 72 return time.strftime('%a %b %e %H:%M:%S %Z %Y', time.localtime()) | 79 def file(self, filename): 80 return joinpath(self.dir, filename) |
73 | 81 |
74def remfile(file): 75 if os.path.isfile(file): 76 os.unlink(file) | 82 def create(self): 83 if os.path.exists(self.dir): 84 if not os.path.isdir(self.dir): 85 sys.exit('%s is not a directory. Cannot build job' % self.dir) 86 else: 87 os.mkdir(self.dir) |
77 | 88 |
78def readval(filename): 79 file = open(filename, 'r') 80 value = file.readline().strip() 81 file.close() 82 return value | 89 def exists(self): 90 return os.path.isdir(self.dir) |
83 | 91 |
92 def clean(self): 93 cleandir(self.dir) 94 95 def hasfile(self, filename): 96 return os.path.isfile(self.file(filename)) 97 98 def echofile(self, filename, string): 99 filename = self.file(filename) 100 try: 101 f = file(filename, 'w') 102 print >>f, string 103 f.flush() 104 f.close() 105 except IOError,e: 106 sys.exit(e) 107 108 def rmfile(self, filename): 109 filename = self.file(filename) 110 if os.path.isfile(filename): 111 os.unlink(filename) 112 113 def readval(self, filename): 114 filename = self.file(filename) 115 f = file(filename, 'r') 116 value = f.readline().strip() 117 f.close() 118 return value 119 120 def setstatus(self, string): 121 filename = self.file('.status') 122 try: 123 f = file(filename, 'a') 124 print >>f, string 125 f.flush() 126 f.close() 127 except IOError,e: 128 sys.exit(e) 129 130 def getstatus(self): 131 filename = self.file('.status') 132 try: 133 f = file(filename, 'r') 134 except IOError, e: 135 return 'none' 136 137 # fast forward to the end 138 for line in f: pass 139 140 # the first word on the last line is the status 141 return line.split(' ')[0] 142 143 def __str__(self): 144 return self.dir 145 |
|
84if __name__ == '__main__': 85 rootdir = env.setdefault('ROOTDIR', os.getcwd()) 86 pbs_jobid = env['PBS_JOBID'] 87 pbs_jobname = env['PBS_JOBNAME'] 88 basedir = joinpath(rootdir, 'Base') 89 jobname = env.setdefault('JOBNAME', pbs_jobname) 90 jobfile = env.setdefault('JOBFILE', joinpath(basedir, 'test.py')) 91 outdir = env.setdefault('OUTPUT_DIR', joinpath(rootdir, jobname)) 92 env['POOLJOB'] = 'True' 93 94 if os.path.isdir("/work"): 95 workbase = "/work" 96 else: 97 workbase = "/tmp/" 98 99 workdir = joinpath(workbase, '%s.%s' % (env['USER'], pbs_jobid)) | 146if __name__ == '__main__': 147 rootdir = env.setdefault('ROOTDIR', os.getcwd()) 148 pbs_jobid = env['PBS_JOBID'] 149 pbs_jobname = env['PBS_JOBNAME'] 150 basedir = joinpath(rootdir, 'Base') 151 jobname = env.setdefault('JOBNAME', pbs_jobname) 152 jobfile = env.setdefault('JOBFILE', joinpath(basedir, 'test.py')) 153 outdir = env.setdefault('OUTPUT_DIR', joinpath(rootdir, jobname)) 154 env['POOLJOB'] = 'True' 155 156 if os.path.isdir("/work"): 157 workbase = "/work" 158 else: 159 workbase = "/tmp/" 160 161 workdir = joinpath(workbase, '%s.%s' % (env['USER'], pbs_jobid)) |
162 host = socket.gethostname() |
|
100 | 163 |
101 def echofile(filename, string): 102 try: 103 f = file(joinpath(outdir, filename), 'w') 104 print >>f, string 105 f.flush() 106 f.close() 107 except IOError,e: 108 sys.exit(e) 109 | |
110 os.umask(0022) 111 | 164 os.umask(0022) 165 |
112 echofile('.start', date()) 113 echofile('.pbs_jobid', pbs_jobid) 114 echofile('.pbs_jobname', pbs_jobid) 115 echofile('.host', socket.gethostname()) | 166 jobdir = JobDir(outdir) |
116 | 167 |
168 started = date() 169 jobdir.echofile('.running', started) 170 jobdir.rmfile('.queued') 171 jobdir.echofile('.pbs_jobid', pbs_jobid) 172 jobdir.echofile('.pbs_jobname', pbs_jobid) 173 jobdir.echofile('.host', host) 174 175 jobdir.setstatus('running on %s on %s' % (host, started)) 176 |
|
117 if os.path.isdir(workdir): 118 cleandir(workdir) 119 else: 120 os.mkdir(workdir) 121 | 177 if os.path.isdir(workdir): 178 cleandir(workdir) 179 else: 180 os.mkdir(workdir) 181 |
122 if os.path.isdir('/z/dist'): | 182 if False and os.path.isdir('/z/dist'): |
123 sync = rsync() 124 sync.delete = True 125 sync.sudo = True 126 sync.do('poolfs::dist/m5/', '/z/dist/m5/') 127 128 try: 129 os.chdir(workdir) 130 except OSError,e: 131 sys.exit(e) 132 | 183 sync = rsync() 184 sync.delete = True 185 sync.sudo = True 186 sync.do('poolfs::dist/m5/', '/z/dist/m5/') 187 188 try: 189 os.chdir(workdir) 190 except OSError,e: 191 sys.exit(e) 192 |
133 os.symlink(joinpath(outdir, 'output'), 'status.out') | 193 os.symlink(jobdir.file('output'), 'status.out') |
134 135 args = [ joinpath(basedir, 'm5'), joinpath(basedir, 'run.py') ] 136 if not len(args): 137 sys.exit("no arguments") 138 | 194 195 args = [ joinpath(basedir, 'm5'), joinpath(basedir, 'run.py') ] 196 if not len(args): 197 sys.exit("no arguments") 198 |
139 print 'starting job... %s' % date() | 199 print 'starting job... %s' % started |
140 print ' '.join(args) 141 print 142 sys.stdout.flush() 143 144 childpid = os.fork() 145 if not childpid: 146 # Execute command 147 sys.stdin.close() | 200 print ' '.join(args) 201 print 202 sys.stdout.flush() 203 204 childpid = os.fork() 205 if not childpid: 206 # Execute command 207 sys.stdin.close() |
148 fd = os.open(joinpath(outdir, "output"), | 208 fd = os.open(jobdir.file("output"), |
149 os.O_WRONLY | os.O_CREAT | os.O_TRUNC) 150 os.dup2(fd, sys.stdout.fileno()) 151 os.dup2(fd, sys.stderr.fileno()) 152 os.execvp(args[0], args) 153 154 def handler(signum, frame): 155 if childpid != 0: 156 os.kill(childpid, signum) --- 8 unchanged lines hidden (view full) --- 165 signal.signal(signal.SIGUSR2, handler) 166 167 done = 0 168 while not done: 169 try: 170 thepid,ec = os.waitpid(childpid, 0) 171 if ec: 172 print 'Exit code ', ec | 209 os.O_WRONLY | os.O_CREAT | os.O_TRUNC) 210 os.dup2(fd, sys.stdout.fileno()) 211 os.dup2(fd, sys.stderr.fileno()) 212 os.execvp(args[0], args) 213 214 def handler(signum, frame): 215 if childpid != 0: 216 os.kill(childpid, signum) --- 8 unchanged lines hidden (view full) --- 225 signal.signal(signal.SIGUSR2, handler) 226 227 done = 0 228 while not done: 229 try: 230 thepid,ec = os.waitpid(childpid, 0) 231 if ec: 232 print 'Exit code ', ec |
173 echofile('.failure', date()) | 233 status = 'failure' |
174 else: | 234 else: |
175 echofile('.success', date()) | 235 status = 'success' |
176 done = 1 177 except OSError: 178 pass 179 | 236 done = 1 237 except OSError: 238 pass 239 |
180 print '\njob complete... %s' % date() 181 echofile('.stop', date()) | 240 complete = date() 241 print '\njob complete... %s' % complete 242 jobdir.echofile('.%s' % status, complete) 243 jobdir.rmfile('.running') 244 jobdir.setstatus('%s on %s' % (status, complete)) |