| 1 | import subprocess |
|---|
| 2 | import re |
|---|
| 3 | import copy |
|---|
| 4 | from StringIO import StringIO |
|---|
| 5 | import os |
|---|
| 6 | |
|---|
| 7 | import doug |
|---|
| 8 | from doug.config import DOUGConfigParser, ControlFile |
|---|
| 9 | |
|---|
| 10 | from scripts import ScriptException |
|---|
| 11 | |
|---|
| 12 | import logging |
|---|
| 13 | LOG = logging.getLogger('doug') |
|---|
| 14 | |
|---|
| 15 | _defaultConfig = None |
|---|
| 16 | def getDefaultConfig(): |
|---|
| 17 | global _defaultConfig |
|---|
| 18 | if _defaultConfig is None: |
|---|
| 19 | _defaultConfig = DOUGConfigParser(name="DOUG default config") |
|---|
| 20 | _defaultConfig.addConfigContents(doug.execution_conf_tmpl) |
|---|
| 21 | return _defaultConfig |
|---|
| 22 | |
|---|
| 23 | def getDefaultControlFile(basedir): |
|---|
| 24 | cf = ControlFile(contents=doug.DOUG_ctl_tmpl, basedir=basedir) |
|---|
| 25 | cf.name = '<doug.DOUG_ctl_tmpl>' |
|---|
| 26 | return cf |
|---|
| 27 | |
|---|
| 28 | class DOUGExecution: |
|---|
| 29 | |
|---|
| 30 | def __init__(self, config, dougControls=None): |
|---|
| 31 | self.workdir = os.path.abspath(config.get('doug', 'workdir')) |
|---|
| 32 | self.config = DOUGConfigParser(name='DOUG execution', basedir=self.workdir) |
|---|
| 33 | # default config |
|---|
| 34 | self.config.addConfig(getDefaultConfig()) |
|---|
| 35 | self.config.addControlFile(getDefaultControlFile(self.workdir)) |
|---|
| 36 | |
|---|
| 37 | # copy controls from control file |
|---|
| 38 | if dougControls is not None: |
|---|
| 39 | self.config.addControlFile(dougControls) |
|---|
| 40 | # copy config |
|---|
| 41 | self.config.addConfig(config) |
|---|
| 42 | |
|---|
| 43 | # output or other files, exception grabs it on exit |
|---|
| 44 | self.files = [] |
|---|
| 45 | |
|---|
| 46 | # how many test results are using this test, files are deleted only after last free() call |
|---|
| 47 | self._inUse = 0 |
|---|
| 48 | |
|---|
| 49 | def setUp(self): |
|---|
| 50 | LOG.debug("Preparing testing environment") |
|---|
| 51 | self.workdir = self.workdir |
|---|
| 52 | if os.path.isdir(self.workdir): |
|---|
| 53 | self.workdirExisted = True |
|---|
| 54 | else: |
|---|
| 55 | self.workdirExisted = False |
|---|
| 56 | os.mkdir(self.workdir) |
|---|
| 57 | LOG.debug("Working directory %s created" % self.workdir) |
|---|
| 58 | self.preserveOutput = self.config.getboolean("doug", "preserveOutput") |
|---|
| 59 | try: |
|---|
| 60 | # create control file |
|---|
| 61 | self.testctrlfname = os.path.abspath(os.path.join(self.workdir, 'DOUG-exec.ctl')) |
|---|
| 62 | controlFile = self.config.getControlFile(self.testctrlfname) |
|---|
| 63 | controlFile.save(self.testctrlfname) |
|---|
| 64 | self.files.append((self.testctrlfname, "Control file")) |
|---|
| 65 | |
|---|
| 66 | # run mpiboot |
|---|
| 67 | mpibootname = self.config.get("doug", "mpiboot") |
|---|
| 68 | outfilename = self.config.get("doug", "mpiboot-outfilename") |
|---|
| 69 | errfilename = self.config.get("doug", "mpiboot-errfilename") |
|---|
| 70 | |
|---|
| 71 | if mpibootname: |
|---|
| 72 | LOG.debug("Setting up mpi") |
|---|
| 73 | mpiboot = subprocess.Popen("%s > %s 2> %s" % (mpibootname, outfilename, errfilename), shell=True) |
|---|
| 74 | res = mpiboot.wait() |
|---|
| 75 | if res: |
|---|
| 76 | raise ScriptException("Error running %s (%d)" |
|---|
| 77 | "inspect output files (%s, %s) for error description." |
|---|
| 78 | % (mpibootname, res, outfilename, errfilename)) |
|---|
| 79 | except: |
|---|
| 80 | self._clean() |
|---|
| 81 | raise |
|---|
| 82 | |
|---|
| 83 | |
|---|
| 84 | def tearDown(self): |
|---|
| 85 | try: |
|---|
| 86 | mpihaltname = self.config.get("doug", "mpihalt") |
|---|
| 87 | if mpihaltname: |
|---|
| 88 | outfilename = self.config.get("doug", "mpihalt-outfilename") |
|---|
| 89 | errfilename = self.config.get("doug", "mpihalt-errfilename") |
|---|
| 90 | LOG.debug("Shutting down mpi") |
|---|
| 91 | mpihalt = subprocess.Popen("%s > %s 2> %s" % (mpihaltname, outfilename, errfilename), shell=True) |
|---|
| 92 | import time |
|---|
| 93 | time.sleep(4) # lamhalt <=7.1.1 does not wait until whole universe is shut down |
|---|
| 94 | res = mpihalt.wait() |
|---|
| 95 | if res: |
|---|
| 96 | LOG.warn("Error running %s (%d)" |
|---|
| 97 | "inspect output files (%s, %s) for error description." |
|---|
| 98 | % (mpihaltname, res, outfilename, errfilename)) |
|---|
| 99 | except Exception, e: |
|---|
| 100 | LOG.warn("Exception running mpihalt: %s" % e) |
|---|
| 101 | |
|---|
| 102 | def _clean(self): |
|---|
| 103 | if not self.preserveOutput and not self.workdirExisted: |
|---|
| 104 | os.system('rm -rf %s' % self.workdir) |
|---|
| 105 | LOG.debug("Temporary directory %s deleted" % self.workdir) |
|---|
| 106 | |
|---|
| 107 | def acquire(self): |
|---|
| 108 | self._inUse += 1 |
|---|
| 109 | |
|---|
| 110 | def free(self): |
|---|
| 111 | self._inUse -= 1 |
|---|
| 112 | if self._inUse == 0: |
|---|
| 113 | self._clean() |
|---|
| 114 | |
|---|
| 115 | def run(self): |
|---|
| 116 | return self.runDOUG() |
|---|
| 117 | |
|---|
| 118 | def runDOUG(self): |
|---|
| 119 | LOG.debug("Running DOUG") |
|---|
| 120 | nproc = self.config.getint('doug', 'nproc') |
|---|
| 121 | solver = self.config.getint('doug-controls', 'solver') |
|---|
| 122 | levels = self.config.getint('doug-controls', 'levels') |
|---|
| 123 | method = self.config.getint('doug-controls', 'method') |
|---|
| 124 | LOG.info("solver=%d, method=%d, levels=%d, nproc=%d" % (solver, method, levels, nproc)) |
|---|
| 125 | mpirun = self.config.get("doug", "mpirun") |
|---|
| 126 | main = self.config.getpath("doug", "executable") |
|---|
| 127 | errfname = self.config.getpath("doug", "errfilename") |
|---|
| 128 | outfname = self.config.getpath("doug", "outfilename") |
|---|
| 129 | solutionfname = self.config.getpath('doug-controls', 'solution_file') |
|---|
| 130 | |
|---|
| 131 | curdir = os.getcwd() |
|---|
| 132 | |
|---|
| 133 | self.result = result = DOUGConfigParser(self.config.defaults(), basedir=self.workdir) |
|---|
| 134 | result.add_section('doug-result') |
|---|
| 135 | try: |
|---|
| 136 | LOG.debug("Changing directory to %s" % self.workdir) |
|---|
| 137 | os.chdir(self.workdir) |
|---|
| 138 | outf = open(outfname, "w") |
|---|
| 139 | errf = open(errfname, "w") |
|---|
| 140 | try: |
|---|
| 141 | args = [mpirun, "-np", "%d"%nproc, main, "-f", self.testctrlfname, "-p"] |
|---|
| 142 | LOG.debug("Running %s" % " ".join(args)) |
|---|
| 143 | doug = subprocess.Popen(args, stdout=outf, stderr=errf) |
|---|
| 144 | |
|---|
| 145 | import time |
|---|
| 146 | maxtime = self.config.getint('doug', 'max-time') |
|---|
| 147 | for i in xrange(maxtime): # ~1 minute |
|---|
| 148 | time.sleep(1) |
|---|
| 149 | doug.poll() |
|---|
| 150 | if doug.returncode != None: |
|---|
| 151 | break |
|---|
| 152 | else: |
|---|
| 153 | LOG.info("Terminating DOUG") |
|---|
| 154 | doug.terminate() |
|---|
| 155 | doug.wait() |
|---|
| 156 | |
|---|
| 157 | value = doug.returncode |
|---|
| 158 | LOG.debug("Finished %s with code %d" % (mpirun, value)) |
|---|
| 159 | self.files.append((outfname, "%s standard output" % mpirun)) |
|---|
| 160 | self.files.append((errfname, "%s standard error" % mpirun)) |
|---|
| 161 | result.setpath('doug-result', 'returnvalue', str(value)) |
|---|
| 162 | result.setpath('doug-result', 'outputfile', outfname) |
|---|
| 163 | result.setpath('doug-result', 'errorfile', errfname) |
|---|
| 164 | |
|---|
| 165 | if value != 0: |
|---|
| 166 | se = ScriptException("Error occured while running doug (value=%d), " |
|---|
| 167 | "inspect output files (%s, %s) for error description." % |
|---|
| 168 | (value, outfname, errfname)) |
|---|
| 169 | raise se |
|---|
| 170 | |
|---|
| 171 | if solutionfname and os.path.isfile(solutionfname): |
|---|
| 172 | result.setpath('doug-result', 'solutionfile', solutionfname) |
|---|
| 173 | if solutionfname and os.path.isfile('aggr1.txt'): |
|---|
| 174 | result.setpath('doug-result', 'fineaggrsfile', 'aggr1.txt') |
|---|
| 175 | #self.files.append(("aggr1.txt", "Fine aggregates")) |
|---|
| 176 | if solutionfname and os.path.isfile('aggr2.txt'): |
|---|
| 177 | result.setpath('doug-result', 'coarseaggrsfile', 'aggr2.txt') |
|---|
| 178 | #self.files.append(("aggr2.txt", "Coarse aggregates")) |
|---|
| 179 | |
|---|
| 180 | files = os.listdir(self.workdir) |
|---|
| 181 | files = filter(lambda name: name.startswith('prof.00'), files) |
|---|
| 182 | if files: |
|---|
| 183 | result.setpath('doug-result', 'profilefile', files[0]) |
|---|
| 184 | self.files.append((os.path.join(self.workdir, files[0]), "Profile info")) |
|---|
| 185 | |
|---|
| 186 | # compare answers |
|---|
| 187 | |
|---|
| 188 | finally: |
|---|
| 189 | outf.close() |
|---|
| 190 | errf.close() |
|---|
| 191 | LOG.debug("Changing directory to %s" % curdir) |
|---|
| 192 | os.chdir(curdir) |
|---|
| 193 | except ScriptException, e: |
|---|
| 194 | for fn in self.files: |
|---|
| 195 | e.addFile(*fn) |
|---|
| 196 | raise |
|---|
| 197 | |
|---|
| 198 | return result |
|---|
| 199 | |
|---|
| 200 | def __str__(self): |
|---|
| 201 | return "%s: solver=%d, method=%d, processors=%d" % \ |
|---|
| 202 | (self.testname, self.solver, self.method, self.nproc) |
|---|
