RunTimeTest.py 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133
  1. #!/usr/bin/env python3
  2. # Contest Management System - http://cms-dev.github.io/
  3. # Copyright © 2015-2018 Stefano Maggiolo <s.maggiolo@gmail.com>
  4. # Copyright © 2016 Luca Wehrstedt <luca.wehrstedt@gmail.com>
  5. #
  6. # This program is free software: you can redistribute it and/or modify
  7. # it under the terms of the GNU Affero General Public License as
  8. # published by the Free Software Foundation, either version 3 of the
  9. # License, or (at your option) any later version.
  10. #
  11. # This program is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU Affero General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU Affero General Public License
  17. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18. import argparse
  19. import logging
  20. import os
  21. import sys
  22. import cmstestsuite.tasks.batch_50 as batch_50
  23. from cmstestsuite import CONFIG
  24. from cmstestsuite.Test import Test
  25. from cmstestsuite.Tests import LANG_C
  26. from cmstestsuite.functionaltestframework import FunctionalTestFramework
  27. from cmstestsuite.profiling import \
  28. PROFILER_KERNPROF, PROFILER_NONE, PROFILER_YAPPI
  29. from cmstestsuite.testrunner import TestRunner
  30. logger = logging.getLogger(__name__)
  31. class TimeTest:
  32. def __init__(self, name, task, filename, languages, repetitions):
  33. self.framework = FunctionalTestFramework()
  34. self.name = name
  35. self.task_module = task
  36. self.filename = filename
  37. self.languages = languages
  38. self.repetitions = repetitions
  39. submission_format = list(
  40. e.strip() for e in task.task_info["submission_format"].split())
  41. self.submission_format_element = submission_format[0]
  42. self.submission_ids = []
  43. def submit(self, task_id, user_id, language):
  44. # Source files are stored under cmstestsuite/code/.
  45. path = os.path.join(os.path.dirname(__file__), 'code')
  46. # Choose the correct file to submit.
  47. filename = self.filename.replace("%l", language)
  48. full_path = os.path.join(path, filename)
  49. # Submit our code.
  50. self.submission_ids = [
  51. self.framework.cws_submit(
  52. task_id, user_id,
  53. self.submission_format_element, full_path, language)
  54. for _ in range(self.repetitions)]
  55. def wait(self, contest_id, unused_language):
  56. # Wait for evaluation to complete.
  57. for submission_id in self.submission_ids:
  58. self.framework.get_evaluation_result(contest_id, submission_id)
  59. def main():
  60. parser = argparse.ArgumentParser(
  61. description="Runs the CMS functional test suite.")
  62. parser.add_argument(
  63. "-s", "--submissions", action="store", type=int, default=50,
  64. help="set the number of submissions to submit (default 50)")
  65. parser.add_argument(
  66. "-w", "--workers", action="store", type=int, default=4,
  67. help="set the number of workers to use (default 4)")
  68. parser.add_argument(
  69. "-l", "--cpu_limits", action="append", default=[],
  70. help="set maximum CPU percentage for a set of services, for example: "
  71. "'-l .*Server:40' limits servers to use 40%% of a CPU or less; "
  72. "can be specified multiple times (requires cputool)")
  73. parser.add_argument(
  74. "-v", "--verbose", action="count", default=0,
  75. help="print debug information (use multiple times for more)")
  76. parser.add_argument(
  77. "--profiler", choices=[PROFILER_YAPPI, PROFILER_KERNPROF],
  78. default=PROFILER_NONE, help="set profiler")
  79. args = parser.parse_args()
  80. CONFIG["VERBOSITY"] = args.verbose
  81. CONFIG["COVERAGE"] = False
  82. CONFIG["PROFILER"] = args.profiler
  83. test_list = [Test('batch',
  84. task=batch_50, filenames=['correct-stdio.%l'],
  85. languages=(LANG_C, ), checks=[])
  86. for _ in range(args.submissions)]
  87. cpu_limits = []
  88. for l in args.cpu_limits:
  89. if ":" not in l:
  90. parser.error("CPU limit must be in the form <regex>:<limit>.")
  91. regex, _, limit = l.rpartition(":")
  92. try:
  93. limit = int(limit)
  94. except ValueError:
  95. parser.error("CPU limit must be an integer.")
  96. cpu_limits.append((regex, limit))
  97. runner = TestRunner(test_list, workers=args.workers,
  98. cpu_limits=cpu_limits)
  99. runner.submit_tests(concurrent_submit_and_eval=False)
  100. runner.log_elapsed_time()
  101. failures = runner.wait_for_evaluation()
  102. runner.log_elapsed_time()
  103. if failures == []:
  104. logger.info("All tests passed!")
  105. return 0
  106. else:
  107. logger.error("Some test failed!")
  108. return 1
  109. if __name__ == "__main__":
  110. sys.exit(main())