cmsMake.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771
  1. #!/usr/bin/env python3
  2. # Contest Management System - http://cms-dev.github.io/
  3. # Copyright © 2010-2014 Giovanni Mascellani <mascellani@poisson.phc.unipi.it>
  4. # Copyright © 2010-2017 Stefano Maggiolo <s.maggiolo@gmail.com>
  5. # Copyright © 2010-2012 Matteo Boscariol <boscarim@hotmail.com>
  6. # Copyright © 2013-2017 Luca Wehrstedt <luca.wehrstedt@gmail.com>
  7. # Copyright © 2014-2015 Luca Versari <veluca93@gmail.com>
  8. #
  9. # This program is free software: you can redistribute it and/or modify
  10. # it under the terms of the GNU Affero General Public License as
  11. # published by the Free Software Foundation, either version 3 of the
  12. # License, or (at your option) any later version.
  13. #
  14. # This program is distributed in the hope that it will be useful,
  15. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17. # GNU Affero General Public License for more details.
  18. #
  19. # You should have received a copy of the GNU Affero General Public License
  20. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  21. import argparse
  22. import copy
  23. import functools
  24. import logging
  25. import os
  26. import shutil
  27. import subprocess
  28. import sys
  29. import tempfile
  30. import yaml
  31. from cms import utf8_decoder
  32. from cms.grading.languagemanager import SOURCE_EXTS, filename_to_language
  33. from cmscommon.terminal import move_cursor, add_color_to_string, \
  34. colors, directions
  35. from cmstaskenv.Test import test_testcases, clean_test_env
  36. SOL_DIRNAME = 'sol'
  37. SOL_FILENAME = 'soluzione'
  38. SOL_EXTS = SOURCE_EXTS
  39. CHECK_DIRNAME = 'cor'
  40. CHECK_EXTS = SOL_EXTS
  41. TEXT_DIRNAME = 'testo'
  42. TEXT_TEX = 'testo.tex'
  43. TEXT_PDF = 'testo.pdf'
  44. TEXT_AUX = 'testo.aux'
  45. TEXT_LOG = 'testo.log'
  46. INPUT0_TXT = 'input0.txt'
  47. OUTPUT0_TXT = 'output0.txt'
  48. GEN_DIRNAME = 'gen'
  49. GEN_GEN = 'GEN'
  50. GEN_BASENAME = 'generatore'
  51. GEN_EXTS = ['.py', '.sh', '.cpp', '.c', '.pas']
  52. VALIDATOR_BASENAME = 'valida'
  53. GRAD_BASENAME = 'grader'
  54. STUB_BASENAME = 'stub'
  55. INPUT_DIRNAME = 'input'
  56. OUTPUT_DIRNAME = 'output'
  57. RESULT_DIRNAME = 'result'
  58. DATA_DIRS = [os.path.join('.', 'cmstaskenv', 'data'),
  59. os.path.join('/', 'usr', 'local', 'share', 'cms', 'cmsMake')]
  60. logger = logging.getLogger()
  61. def detect_data_dir():
  62. for _dir in DATA_DIRS:
  63. if os.path.exists(_dir):
  64. return os.path.abspath(_dir)
  65. DATA_DIR = detect_data_dir()
  66. def endswith2(string, suffixes):
  67. """True if string ends with one of the given suffixes.
  68. """
  69. return any(string.endswith(suffix) for suffix in suffixes)
  70. def basename2(string, suffixes):
  71. """If string ends with one of the specified suffixes, returns its
  72. basename (i.e., itself after removing the suffix) and the suffix
  73. packed in a tuple. Otherwise returns None.
  74. """
  75. try:
  76. suffix = next(s for s in suffixes if string.endswith(s))
  77. return string[:-len(suffix)], string[-len(suffix):]
  78. except StopIteration:
  79. return None, None
  80. def call(base_dir, args, stdin=None, stdout=None, stderr=None, env=None):
  81. print("> Executing command %s in dir %s" %
  82. (" ".join(args), base_dir), file=sys.stderr)
  83. if env is None:
  84. env = {}
  85. env2 = copy.copy(os.environ)
  86. env2.update(env)
  87. subprocess.check_call(args, stdin=stdin, stdout=stdout, stderr=stderr,
  88. cwd=base_dir, env=env2)
  89. def detect_task_name(base_dir):
  90. return os.path.split(os.path.abspath(base_dir))[1]
  91. def parse_task_yaml(base_dir):
  92. parent_dir = os.path.split(os.path.abspath(base_dir))[0]
  93. # We first look for the yaml file inside the task folder,
  94. # and eventually fallback to a yaml file in its parent folder.
  95. yaml_path = os.path.join(base_dir, "task.yaml")
  96. try:
  97. with open(yaml_path, "rt", encoding="utf-8") as yaml_file:
  98. conf = yaml.load(yaml_file)
  99. except OSError:
  100. yaml_path = os.path.join(parent_dir, "%s.yaml" %
  101. (detect_task_name(base_dir)))
  102. with open(yaml_path, "rt", encoding="utf-8") as yaml_file:
  103. conf = yaml.load(yaml_file)
  104. return conf
  105. def detect_task_type(base_dir):
  106. sol_dir = os.path.join(base_dir, SOL_DIRNAME)
  107. check_dir = os.path.join(base_dir, CHECK_DIRNAME)
  108. grad_present = os.path.exists(sol_dir) and \
  109. any(x.startswith(GRAD_BASENAME + '.') for x in os.listdir(sol_dir))
  110. stub_present = os.path.exists(sol_dir) and \
  111. any(x.startswith(STUB_BASENAME + '.') for x in os.listdir(sol_dir))
  112. cor_present = os.path.exists(check_dir) and \
  113. any(x.startswith('correttore.') for x in os.listdir(check_dir))
  114. man_present = os.path.exists(check_dir) and \
  115. any(x.startswith('manager.') for x in os.listdir(check_dir))
  116. if not (cor_present or man_present or stub_present or grad_present):
  117. return ["Batch", "Diff"] # TODO Could also be an OutputOnly
  118. elif not (man_present or stub_present or grad_present) and cor_present:
  119. return ["Batch", "Comp"] # TODO Could also be an OutputOnly
  120. elif not (cor_present or man_present or stub_present) and grad_present:
  121. return ["Batch", "Grad"]
  122. elif not (man_present or stub_present) and cor_present and grad_present:
  123. return ["Batch", "GradComp"]
  124. elif not (cor_present or grad_present) and man_present and stub_present:
  125. return ["Communication", ""]
  126. else:
  127. return ["Invalid", ""]
  128. def noop():
  129. pass
  130. def build_sols_list(base_dir, task_type, in_out_files, yaml_conf):
  131. if yaml_conf.get('only_gen', False):
  132. return []
  133. sol_dir = os.path.join(base_dir, SOL_DIRNAME)
  134. actions = []
  135. test_actions = []
  136. for src in (os.path.join(SOL_DIRNAME, x)
  137. for x in os.listdir(sol_dir)
  138. if endswith2(x, SOL_EXTS)):
  139. exe, ext = basename2(src, SOL_EXTS)
  140. lang = filename_to_language(src)
  141. # Delete the dot
  142. ext = ext[1:]
  143. exe_EVAL = "%s_EVAL" % (exe)
  144. # Ignore things known to be auxiliary files
  145. if exe == os.path.join(SOL_DIRNAME, GRAD_BASENAME):
  146. continue
  147. if exe == os.path.join(SOL_DIRNAME, STUB_BASENAME):
  148. continue
  149. if ext == 'pas' and exe.endswith('lib'):
  150. continue
  151. srcs = []
  152. # The grader, when present, must be in the first position of srcs.
  153. if task_type == ['Batch', 'Grad'] or \
  154. task_type == ['Batch', 'GradComp']:
  155. srcs.append(os.path.join(SOL_DIRNAME,
  156. GRAD_BASENAME + '.%s' % (ext)))
  157. if task_type == ['Communication', '']:
  158. srcs.append(os.path.join(SOL_DIRNAME,
  159. STUB_BASENAME + '.%s' % (ext)))
  160. srcs.append(src)
  161. test_deps = [exe_EVAL] + in_out_files
  162. if task_type == ['Batch', 'Comp'] or \
  163. task_type == ['Batch', 'GradComp']:
  164. test_deps.append('cor/correttore')
  165. if task_type == ['Communication', '']:
  166. test_deps.append('cor/manager')
  167. def compile_src(srcs, exe, for_evaluation, lang, assume=None):
  168. # We put everything in a temporary directory to reproduce
  169. # the same conditions that we have when compiling a
  170. # submission.
  171. tempdir = tempfile.mkdtemp()
  172. try:
  173. task_name = detect_task_name(base_dir)
  174. grader_num = 1 if len(srcs) > 1 else 0
  175. new_srcs = []
  176. for grader in srcs[:grader_num]:
  177. grader_name = os.path.basename(grader)
  178. shutil.copyfile(os.path.join(base_dir, grader),
  179. os.path.join(tempdir, grader_name))
  180. new_srcs.append(os.path.join(tempdir, grader_name))
  181. # For now, we assume we only have one non-grader source.
  182. source_name = task_name + lang.source_extension
  183. shutil.copyfile(os.path.join(base_dir, srcs[grader_num]),
  184. os.path.join(tempdir, source_name))
  185. new_srcs.append(source_name)
  186. # Libraries are needed/used only for C/C++ and Pascal
  187. header_extension = lang.header_extension
  188. if header_extension is not None:
  189. lib_template = "%s" + header_extension
  190. lib_filename = lib_template % (task_name)
  191. lib_path = os.path.join(
  192. base_dir, SOL_DIRNAME, lib_filename)
  193. if os.path.exists(lib_path):
  194. shutil.copyfile(lib_path,
  195. os.path.join(tempdir, lib_filename))
  196. new_exe = os.path.join(tempdir, task_name)
  197. compilation_commands = lang.get_compilation_commands(
  198. new_srcs, new_exe, for_evaluation=for_evaluation)
  199. for command in compilation_commands:
  200. call(tempdir, command)
  201. move_cursor(directions.UP, erase=True, stream=sys.stderr)
  202. shutil.copyfile(os.path.join(tempdir, new_exe),
  203. os.path.join(base_dir, exe))
  204. shutil.copymode(os.path.join(tempdir, new_exe),
  205. os.path.join(base_dir, exe))
  206. finally:
  207. shutil.rmtree(tempdir)
  208. def test_src(exe, lang, assume=None):
  209. # Solution names begin with sol/ and end with _EVAL, we strip that
  210. print(
  211. "Testing solution",
  212. add_color_to_string(exe[4:-5], colors.BLACK, bold=True)
  213. )
  214. test_testcases(
  215. base_dir,
  216. exe,
  217. language=lang,
  218. assume=assume)
  219. actions.append(
  220. (srcs,
  221. [exe],
  222. functools.partial(compile_src, srcs, exe, False, lang),
  223. 'compile solution'))
  224. actions.append(
  225. (srcs,
  226. [exe_EVAL],
  227. functools.partial(compile_src, srcs, exe_EVAL, True, lang),
  228. 'compile solution with -DEVAL'))
  229. test_actions.append((test_deps,
  230. ['test_%s' % (os.path.split(exe)[1])],
  231. functools.partial(test_src, exe_EVAL, lang),
  232. 'test solution (compiled with -DEVAL)'))
  233. return actions + test_actions
  234. def build_checker_list(base_dir, task_type):
  235. check_dir = os.path.join(base_dir, CHECK_DIRNAME)
  236. actions = []
  237. if os.path.exists(check_dir):
  238. for src in (os.path.join(CHECK_DIRNAME, x)
  239. for x in os.listdir(check_dir)
  240. if endswith2(x, SOL_EXTS)):
  241. exe, ext = basename2(src, CHECK_EXTS)
  242. lang = filename_to_language(src)
  243. def compile_check(src, exe, assume=None):
  244. commands = lang.get_compilation_commands([src], exe)
  245. for command in commands:
  246. call(base_dir, command)
  247. actions.append(([src], [exe],
  248. functools.partial(compile_check, src, exe),
  249. 'compile checker'))
  250. return actions
  251. def build_text_list(base_dir, task_type):
  252. text_tex = os.path.join(TEXT_DIRNAME, TEXT_TEX)
  253. text_pdf = os.path.join(TEXT_DIRNAME, TEXT_PDF)
  254. text_aux = os.path.join(TEXT_DIRNAME, TEXT_AUX)
  255. text_log = os.path.join(TEXT_DIRNAME, TEXT_LOG)
  256. def make_pdf(assume=None):
  257. call(base_dir,
  258. ['pdflatex', '-output-directory', TEXT_DIRNAME,
  259. '-interaction', 'batchmode', text_tex],
  260. env={'TEXINPUTS': '.:%s:%s/file:' % (TEXT_DIRNAME, TEXT_DIRNAME)})
  261. actions = []
  262. if os.path.exists(text_tex):
  263. actions.append(([text_tex], [text_pdf, text_aux, text_log],
  264. make_pdf, 'compile to PDF'))
  265. return actions
  266. def iter_GEN(name):
  267. st = 0
  268. with open(name, "rt", encoding="utf-8") as f:
  269. for line in f:
  270. line = line.strip()
  271. splitted = line.split('#', 1)
  272. if len(splitted) == 1:
  273. # This line represents a testcase, otherwise
  274. # it's just a blank
  275. if splitted[0] != '':
  276. yield (False, splitted[0], st)
  277. else:
  278. testcase, comment = splitted
  279. is_trivial = comment.startswith(" ")
  280. testcase = testcase.strip()
  281. comment = comment.strip()
  282. testcase_detected = len(testcase) > 0
  283. copy_testcase_detected = comment.startswith("COPY:")
  284. subtask_detected = comment.startswith('ST:')
  285. flags = [testcase_detected,
  286. copy_testcase_detected,
  287. subtask_detected]
  288. flags_count = len([x for x in flags if x])
  289. if flags_count > 1:
  290. raise Exception("No testcase and command in"
  291. " the same line allowed")
  292. if flags_count == 0 and not is_trivial:
  293. raise Exception("Unrecognized non-trivial line")
  294. if testcase_detected:
  295. yield (False, testcase, st)
  296. if copy_testcase_detected:
  297. yield (True, comment[5:].strip(), st)
  298. # This line starts a new subtask
  299. if subtask_detected:
  300. st += 1
  301. def build_gen_list(base_dir, task_type, yaml_conf):
  302. input_dir = os.path.join(base_dir, INPUT_DIRNAME)
  303. output_dir = os.path.join(base_dir, OUTPUT_DIRNAME)
  304. gen_dir = os.path.join(base_dir, GEN_DIRNAME)
  305. gen_exe = None
  306. validator_exe = None
  307. for src in (x for x in os.listdir(gen_dir) if endswith2(x, GEN_EXTS)):
  308. base, ext = basename2(src, GEN_EXTS)
  309. lang = filename_to_language(src)
  310. if base == GEN_BASENAME:
  311. gen_exe = os.path.join(GEN_DIRNAME, base)
  312. gen_src = os.path.join(GEN_DIRNAME, base + ext)
  313. gen_lang = lang
  314. elif base == VALIDATOR_BASENAME:
  315. validator_exe = os.path.join(GEN_DIRNAME, base)
  316. validator_src = os.path.join(GEN_DIRNAME, base + ext)
  317. validator_lang = lang
  318. if gen_exe is None:
  319. raise Exception("Couldn't find generator")
  320. if validator_exe is None:
  321. raise Exception("Couldn't find validator")
  322. gen_GEN = os.path.join(GEN_DIRNAME, GEN_GEN)
  323. sol_exe = os.path.join(SOL_DIRNAME, SOL_FILENAME)
  324. # Count non-trivial lines in GEN and establish which external
  325. # files are needed for input generation
  326. testcases = list(iter_GEN(os.path.join(base_dir, gen_GEN)))
  327. testcase_num = len(testcases)
  328. copy_files = [x[1] for x in testcases if x[0]]
  329. def compile_src(src, exe, lang, assume=None):
  330. if lang.source_extension in ['.cpp', '.c', '.pas']:
  331. commands = lang.get_compilation_commands(
  332. [src], exe, for_evaluation=False)
  333. for command in commands:
  334. call(base_dir, command)
  335. elif lang.source_extension in ['.py', '.sh']:
  336. os.symlink(os.path.basename(src), exe)
  337. else:
  338. raise Exception("Wrong generator/validator language!")
  339. # Question: why, differently from outputs, inputs have to be
  340. # created all together instead of selectively over those that have
  341. # been changed since last execution? This is a waste of time,
  342. # usually generating inputs is a pretty long thing. Answer:
  343. # because cmsMake architecture, which is based on file timestamps,
  344. # doesn't make us able to understand which lines of gen/GEN have
  345. # been changed. Douch! We'll have to think better this thing for
  346. # the new format we're developing.
  347. def make_input(assume=None):
  348. n = 0
  349. try:
  350. os.makedirs(input_dir)
  351. except OSError:
  352. pass
  353. for (is_copy, line, st) in testcases:
  354. print(
  355. "Generating",
  356. add_color_to_string("input # %d" % n, colors.BLACK,
  357. stream=sys.stderr, bold=True),
  358. file=sys.stderr
  359. )
  360. new_input = os.path.join(input_dir, 'input%d.txt' % (n))
  361. if is_copy:
  362. # Copy the file
  363. print("> Copy input file from:", line)
  364. copy_input = os.path.join(base_dir, line)
  365. shutil.copyfile(copy_input, new_input)
  366. else:
  367. # Call the generator
  368. with open(new_input, 'wb') as fout:
  369. call(base_dir,
  370. [gen_exe] + line.split(),
  371. stdout=fout)
  372. command = [validator_exe, new_input]
  373. if st != 0:
  374. command.append("%s" % st)
  375. call(base_dir, command)
  376. n += 1
  377. for _ in range(3):
  378. move_cursor(directions.UP, erase=True, stream=sys.stderr)
  379. def make_output(n, assume=None):
  380. try:
  381. os.makedirs(output_dir)
  382. except OSError:
  383. pass
  384. print(
  385. "Generating",
  386. add_color_to_string("output # %d" % n, colors.BLACK,
  387. stream=sys.stderr, bold=True),
  388. file=sys.stderr
  389. )
  390. temp_dir = tempfile.mkdtemp(prefix=os.path.join(base_dir, "tmp"))
  391. use_stdin = yaml_conf.get("infile") in {None, ""}
  392. use_stdout = yaml_conf.get("outfile") in {None, ""}
  393. # Names of the actual source and destination.
  394. infile = os.path.join(input_dir, 'input%d.txt' % (n))
  395. outfile = os.path.join(output_dir, 'output%d.txt' % (n))
  396. # Names of the input and output in temp directory.
  397. copied_infile = os.path.join(
  398. temp_dir,
  399. "input.txt" if use_stdin else yaml_conf.get("infile"))
  400. copied_outfile = os.path.join(
  401. temp_dir,
  402. "output.txt" if use_stdout else yaml_conf.get("outfile"))
  403. os.symlink(infile, copied_infile)
  404. fin = None
  405. fout = None
  406. try:
  407. if use_stdin:
  408. fin = open(copied_infile, "rb")
  409. if use_stdout:
  410. fout = open(copied_outfile, 'wb')
  411. shutil.copy(sol_exe, temp_dir)
  412. # If the task of of type Communication, then there is
  413. # nothing to put in the output files
  414. if task_type != ['Communication', '']:
  415. call(temp_dir, [os.path.join(temp_dir, SOL_FILENAME)],
  416. stdin=fin, stdout=fout)
  417. move_cursor(directions.UP, erase=True, stream=sys.stderr)
  418. finally:
  419. if fin is not None:
  420. fin.close()
  421. if fout is not None:
  422. fout.close()
  423. os.rename(copied_outfile, outfile)
  424. shutil.rmtree(temp_dir)
  425. move_cursor(directions.UP, erase=True, stream=sys.stderr)
  426. actions = []
  427. actions.append(([gen_src],
  428. [gen_exe],
  429. functools.partial(compile_src, gen_src, gen_exe, gen_lang),
  430. "compile the generator"))
  431. actions.append(([validator_src],
  432. [validator_exe],
  433. functools.partial(compile_src, validator_src,
  434. validator_exe, validator_lang),
  435. "compile the validator"))
  436. actions.append(([gen_GEN, gen_exe, validator_exe] + copy_files,
  437. [os.path.join(INPUT_DIRNAME, 'input%d.txt' % (x))
  438. for x in range(0, testcase_num)],
  439. make_input,
  440. "input generation"))
  441. for n in range(testcase_num):
  442. actions.append(([os.path.join(INPUT_DIRNAME, 'input%d.txt' % (n)),
  443. sol_exe],
  444. [os.path.join(OUTPUT_DIRNAME, 'output%d.txt' % (n))],
  445. functools.partial(make_output, n),
  446. "output generation"))
  447. in_out_files = [os.path.join(INPUT_DIRNAME, 'input%d.txt' % (n))
  448. for n in range(testcase_num)] + \
  449. [os.path.join(OUTPUT_DIRNAME, 'output%d.txt' % (n))
  450. for n in range(testcase_num)]
  451. return actions, in_out_files
  452. def build_action_list(base_dir, task_type, yaml_conf):
  453. """Build a list of actions that cmsMake is able to do here. Each
  454. action is described by a tuple (infiles, outfiles, callable,
  455. description) where:
  456. 1) infiles is a list of files this action depends on;
  457. 2) outfiles is a list of files this action produces; it is
  458. intended that this action can be skipped if all the outfiles is
  459. newer than all the infiles; moreover, the outfiles get deleted
  460. when the action is cleaned;
  461. 3) callable is a callable Python object that, when called,
  462. performs the action;
  463. 4) description is a human-readable description of what this
  464. action does.
  465. """
  466. actions = []
  467. gen_actions, in_out_files = build_gen_list(base_dir, task_type, yaml_conf)
  468. actions += gen_actions
  469. actions += build_sols_list(base_dir, task_type, in_out_files, yaml_conf)
  470. actions += build_checker_list(base_dir, task_type)
  471. actions += build_text_list(base_dir, task_type)
  472. return actions
  473. def clean(base_dir, generated_list):
  474. # Delete all generated files
  475. for f in generated_list:
  476. try:
  477. os.remove(os.path.join(base_dir, f))
  478. except OSError:
  479. pass
  480. # Delete other things
  481. try:
  482. os.rmdir(os.path.join(base_dir, INPUT_DIRNAME))
  483. except OSError:
  484. pass
  485. try:
  486. os.rmdir(os.path.join(base_dir, OUTPUT_DIRNAME))
  487. except OSError:
  488. pass
  489. try:
  490. shutil.rmtree(os.path.join(base_dir, RESULT_DIRNAME))
  491. except OSError:
  492. pass
  493. # Delete compiled and/or backup files
  494. for dirname, _, filenames in os.walk(base_dir):
  495. for filename in filenames:
  496. if any(filename.endswith(ext) for ext in {".o", ".pyc", "~"}):
  497. os.remove(os.path.join(dirname, filename))
  498. def build_execution_tree(actions):
  499. """Given a set of actions as described in the docstring of
  500. build_action_list(), builds an execution tree and the list of all
  501. the buildable files. The execution tree is a dictionary that maps
  502. each builable or source file to the tuple (infiles, callable),
  503. where infiles and callable are as in the docstring of
  504. build_action_list().
  505. """
  506. exec_tree = {}
  507. generated_list = []
  508. src_list = set()
  509. for action in actions:
  510. for exe in action[1]:
  511. if exe in exec_tree:
  512. raise Exception("Target %s not unique" % (exe))
  513. exec_tree[exe] = (action[0], action[2])
  514. generated_list.append(exe)
  515. for src in action[0]:
  516. src_list.add(src)
  517. for src in src_list:
  518. if src not in exec_tree:
  519. exec_tree[src] = ([], noop)
  520. return exec_tree, generated_list
  521. def execute_target(base_dir, exec_tree, target,
  522. already_executed=None, stack=None,
  523. debug=False, assume=None):
  524. # Initialization
  525. if debug:
  526. print(">> Target %s is requested" % (target))
  527. if already_executed is None:
  528. already_executed = set()
  529. if stack is None:
  530. stack = set()
  531. # Get target information
  532. deps = exec_tree[target][0]
  533. action = exec_tree[target][1]
  534. # If this target is already in the stack, we have a circular
  535. # dependency
  536. if target in stack:
  537. raise Exception("Circular dependency detected")
  538. # If the target was already made in another subtree, we have
  539. # nothing to do
  540. if target in already_executed:
  541. if debug:
  542. print(">> Target %s has already been built, ignoring..." %
  543. (target))
  544. return
  545. # Otherwise, do a step of the DFS to make dependencies
  546. if debug:
  547. print(">> Building dependencies for target %s" % (target))
  548. already_executed.add(target)
  549. stack.add(target)
  550. for dep in deps:
  551. execute_target(base_dir, exec_tree, dep,
  552. already_executed, stack, assume=assume)
  553. stack.remove(target)
  554. if debug:
  555. print(">> Dependencies built for target %s" % (target))
  556. # Check if the action really needs to be done (i.e., there is one
  557. # dependency more recent than the generated file)
  558. dep_times = max(
  559. [0] + [os.stat(os.path.join(base_dir, dep)).st_mtime for dep in deps])
  560. try:
  561. gen_time = os.stat(os.path.join(base_dir, target)).st_mtime
  562. except OSError:
  563. gen_time = 0
  564. if gen_time >= dep_times:
  565. if debug:
  566. print(">> Target %s is already new enough, not building" %
  567. (target))
  568. return
  569. # At last: actually make the so long desired action :-)
  570. if debug:
  571. print(">> Acutally building target %s" % (target))
  572. action(assume=assume)
  573. if debug:
  574. print(">> Target %s finished to build" % (target))
  575. def execute_multiple_targets(base_dir, exec_tree, targets,
  576. debug=False, assume=None):
  577. already_executed = set()
  578. for target in targets:
  579. execute_target(base_dir, exec_tree, target,
  580. already_executed, debug=debug, assume=assume)
  581. def main():
  582. # Parse command line options
  583. parser = argparse.ArgumentParser()
  584. group = parser.add_mutually_exclusive_group()
  585. parser.add_argument("-D", "--base-dir", action="store", type=utf8_decoder,
  586. help="base directory for problem to make "
  587. "(CWD by default)")
  588. parser.add_argument("-l", "--list", action="store_true", default=False,
  589. help="list actions that cmsMake is aware of")
  590. parser.add_argument("-c", "--clean", action="store_true", default=False,
  591. help="clean all generated files")
  592. parser.add_argument("-a", "--all", action="store_true", default=False,
  593. help="make all targets")
  594. group.add_argument("-y", "--yes",
  595. dest="assume", action="store_const", const='y',
  596. help="answer yes to all questions")
  597. group.add_argument("-n", "--no",
  598. dest="assume", action="store_const", const='n',
  599. help="answer no to all questions")
  600. parser.add_argument("-d", "--debug", action="store_true", default=False,
  601. help="enable debug messages")
  602. parser.add_argument("targets", action="store", type=utf8_decoder,
  603. nargs="*", metavar="target", help="target to build")
  604. options = parser.parse_args()
  605. base_dir = options.base_dir
  606. if base_dir is None:
  607. base_dir = os.getcwd()
  608. else:
  609. base_dir = os.path.abspath(base_dir)
  610. assume = options.assume
  611. task_type = detect_task_type(base_dir)
  612. yaml_conf = parse_task_yaml(base_dir)
  613. actions = build_action_list(base_dir, task_type, yaml_conf)
  614. exec_tree, generated_list = build_execution_tree(actions)
  615. if [len(options.targets) > 0, options.list, options.clean,
  616. options.all].count(True) > 1:
  617. parser.error("Too many commands")
  618. if options.list:
  619. print("Task name: %s" % (detect_task_name(base_dir)))
  620. print("Task type: %s %s" % (task_type[0], task_type[1]))
  621. print("Available operations:")
  622. for entry in actions:
  623. print(" %s: %s -> %s" %
  624. (entry[3], ", ".join(entry[0]), ", ".join(entry[1])))
  625. elif options.clean:
  626. print("Cleaning")
  627. clean(base_dir, generated_list)
  628. elif options.all:
  629. print("Making all targets")
  630. print()
  631. try:
  632. execute_multiple_targets(base_dir, exec_tree,
  633. generated_list, debug=options.debug,
  634. assume=assume)
  635. # After all work, possibly clean the left-overs of testing
  636. finally:
  637. clean_test_env()
  638. else:
  639. try:
  640. execute_multiple_targets(base_dir, exec_tree,
  641. options.targets, debug=options.debug,
  642. assume=assume)
  643. # After all work, possibly clean the left-overs of testing
  644. finally:
  645. clean_test_env()
  646. if __name__ == '__main__':
  647. main()