ImportContest.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420
  1. #!/usr/bin/env python3
  2. # Contest Management System - http://cms-dev.github.io/
  3. # Copyright © 2010-2013 Giovanni Mascellani <mascellani@poisson.phc.unipi.it>
  4. # Copyright © 2010-2018 Stefano Maggiolo <s.maggiolo@gmail.com>
  5. # Copyright © 2010-2012 Matteo Boscariol <boscarim@hotmail.com>
  6. # Copyright © 2013 Luca Wehrstedt <luca.wehrstedt@gmail.com>
  7. # Copyright © 2014-2015 William Di Luigi <williamdiluigi@gmail.com>
  8. # Copyright © 2015-2016 Luca Chiodini <luca@chiodini.org>
  9. #
  10. # This program is free software: you can redistribute it and/or modify
  11. # it under the terms of the GNU Affero General Public License as
  12. # published by the Free Software Foundation, either version 3 of the
  13. # License, or (at your option) any later version.
  14. #
  15. # This program is distributed in the hope that it will be useful,
  16. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18. # GNU Affero General Public License for more details.
  19. #
  20. # You should have received a copy of the GNU Affero General Public License
  21. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  22. """This script imports a contest from disk using one of the available
  23. loaders.
  24. The data parsed by the loader is used to create a new Contest in the
  25. database.
  26. """
  27. # We enable monkey patching to make many libraries gevent-friendly
  28. # (for instance, urllib3, used by requests)
  29. import gevent.monkey
  30. gevent.monkey.patch_all() # noqa
  31. import argparse
  32. import datetime
  33. import ipaddress
  34. import logging
  35. import os
  36. import sys
  37. from cms import utf8_decoder
  38. from cms.db import SessionGen, User, Team, Participation, Task, Contest
  39. from cms.db.filecacher import FileCacher
  40. from cmscontrib.importing import ImportDataError, update_contest, update_task
  41. from cmscontrib.loaders import choose_loader, build_epilog
  42. logger = logging.getLogger(__name__)
  43. class ContestImporter:
  44. """This script creates a contest and all its associations to users
  45. and tasks.
  46. """
  47. def __init__(self, path, yes, zero_time, import_tasks,
  48. update_contest, update_tasks, no_statements,
  49. delete_stale_participations, loader_class):
  50. self.yes = yes
  51. self.zero_time = zero_time
  52. self.import_tasks = import_tasks
  53. self.update_contest = update_contest
  54. self.update_tasks = update_tasks
  55. self.no_statements = no_statements
  56. self.delete_stale_participations = delete_stale_participations
  57. self.file_cacher = FileCacher()
  58. self.loader = loader_class(os.path.abspath(path), self.file_cacher)
  59. def do_import(self):
  60. """Get the contest from the Loader and store it."""
  61. # We need to check whether the contest has changed *before* calling
  62. # get_contest() as that method might reset the "has_changed" bit.
  63. contest_has_changed = False
  64. if self.update_contest:
  65. contest_has_changed = self.loader.contest_has_changed()
  66. # Get the contest. The loader should give a bare contest, putting tasks
  67. # and participations only in the other return values. We make sure.
  68. contest, tasks, participations = self.loader.get_contest()
  69. if contest.tasks != []:
  70. contest.tasks = []
  71. logger.warning("Contest loader should not fill tasks.")
  72. if contest.participations != []:
  73. contest.participations = []
  74. logger.warning("Contest loader should not fill participations.")
  75. tasks = tasks if tasks is not None else []
  76. participations = participations if participations is not None else []
  77. # Apply the modification flags
  78. if self.zero_time:
  79. contest.start = datetime.datetime(1970, 1, 1)
  80. contest.stop = datetime.datetime(1970, 1, 1)
  81. with SessionGen() as session:
  82. try:
  83. contest = self._contest_to_db(
  84. session, contest, contest_has_changed)
  85. # Detach all tasks before reattaching them
  86. for t in list(contest.tasks):
  87. t.contest = None
  88. for tasknum, taskname in enumerate(tasks):
  89. self._task_to_db(session, contest, tasknum, taskname)
  90. # Delete stale participations if asked to, then import all
  91. # others.
  92. if self.delete_stale_participations:
  93. self._delete_stale_participations(
  94. session, contest,
  95. set(p["username"] for p in participations))
  96. for p in participations:
  97. self._participation_to_db(session, contest, p)
  98. except ImportDataError as e:
  99. logger.error(str(e))
  100. logger.info("Error while importing, no changes were made.")
  101. return False
  102. session.commit()
  103. contest_id = contest.id
  104. logger.info("Import finished (new contest id: %s).", contest_id)
  105. return True
  106. def _contest_to_db(self, session, new_contest, contest_has_changed):
  107. """Add the new contest to the DB
  108. session (Session): session to use.
  109. new_contest (Contest): contest that has to end up in the DB.
  110. contest_has_changed (bool): whether the loader thinks new_contest has
  111. changed since the last time it was imported.
  112. return (Contest): the contest in the DB.
  113. raise (ImportDataError): if the contest already exists on the DB and
  114. the user did not ask to update any data.
  115. """
  116. contest = session.query(Contest)\
  117. .filter(Contest.name == new_contest.name).first()
  118. if contest is None:
  119. # Contest not present, we import it.
  120. logger.info("Creating contest on the database.")
  121. contest = new_contest
  122. session.add(contest)
  123. else:
  124. if not (self.update_contest or self.update_tasks):
  125. # Contest already present, but user did not ask to update any
  126. # data. We cannot import anything and this is most probably
  127. # not what the user wanted, so we let them know.
  128. raise ImportDataError(
  129. "Contest \"%s\" already exists in database. "
  130. "Use --update-contest to update it." % contest.name)
  131. if self.update_contest:
  132. # Contest already present, user asked us to update it; we do so
  133. # if it has changed.
  134. if contest_has_changed:
  135. logger.info("Contest data has changed, updating it.")
  136. update_contest(contest, new_contest)
  137. else:
  138. logger.info("Contest data has not changed.")
  139. return contest
  140. def _task_to_db(self, session, contest, tasknum, taskname):
  141. """Add the task to the DB and attach it to the contest
  142. session (Session): session to use.
  143. contest (Contest): the contest in the DB.
  144. tasknum (int): num the task should have in the contest.
  145. taskname (string): name of the task.
  146. return (Task): the task in the DB.
  147. raise (ImportDataError): in case of one of these errors:
  148. - if the task is not in the DB and user did not ask to import it;
  149. - if the loader cannot load the task;
  150. - if the task is already in the DB, attached to another contest.
  151. """
  152. task_loader = self.loader.get_task_loader(taskname)
  153. task = session.query(Task).filter(Task.name == taskname).first()
  154. if task is None:
  155. # Task is not in the DB; if the user asked us to import it, we do
  156. # so, otherwise we return an error.
  157. if not self.import_tasks:
  158. raise ImportDataError(
  159. "Task \"%s\" not found in database. "
  160. "Use --import-task to import it." % taskname)
  161. task = task_loader.get_task(get_statement=not self.no_statements)
  162. if task is None:
  163. raise ImportDataError(
  164. "Could not import task \"%s\"." % taskname)
  165. session.add(task)
  166. elif not task_loader.task_has_changed():
  167. # Task is in the DB and has not changed, nothing to do.
  168. logger.info("Task \"%s\" data has not changed.", taskname)
  169. elif self.update_tasks:
  170. # Task is in the DB, but has changed, and the user asked us to
  171. # update it. We do so.
  172. new_task = task_loader.get_task(
  173. get_statement=not self.no_statements)
  174. if new_task is None:
  175. raise ImportDataError(
  176. "Could not reimport task \"%s\"." % taskname)
  177. logger.info("Task \"%s\" data has changed, updating it.", taskname)
  178. update_task(task, new_task, get_statements=not self.no_statements)
  179. else:
  180. # Task is in the DB, has changed, and the user didn't ask to update
  181. # it; we just show a warning.
  182. logger.warning("Not updating task \"%s\", even if it has changed. "
  183. "Use --update-tasks to update it.", taskname)
  184. # Finally we tie the task to the contest, if it is not already used
  185. # elsewhere.
  186. if task.contest is not None and task.contest.name != contest.name:
  187. raise ImportDataError(
  188. "Task \"%s\" is already tied to contest \"%s\"."
  189. % (taskname, task.contest.name))
  190. task.num = tasknum
  191. task.contest = contest
  192. return task
  193. @staticmethod
  194. def _participation_to_db(session, contest, new_p):
  195. """Add the new participation to the DB and attach it to the contest
  196. session (Session): session to use.
  197. contest (Contest): the contest in the DB.
  198. new_p (dict): dictionary with the participation data, including at
  199. least "username"; may contain "team", "hidden", "ip", "password".
  200. return (Participation): the participation in the DB.
  201. raise (ImportDataError): in case of one of these errors:
  202. - the user for this participation does not already exist in the DB;
  203. - the team for this participation does not already exist in the DB.
  204. """
  205. user = session.query(User)\
  206. .filter(User.username == new_p["username"]).first()
  207. if user is None:
  208. # FIXME: it would be nice to automatically try to import.
  209. raise ImportDataError("User \"%s\" not found in database. "
  210. "Use cmsImportUser to import it." %
  211. new_p["username"])
  212. team = session.query(Team)\
  213. .filter(Team.code == new_p.get("team")).first()
  214. if team is None and new_p.get("team") is not None:
  215. # FIXME: it would be nice to automatically try to import.
  216. raise ImportDataError("Team \"%s\" not found in database. "
  217. "Use cmsImportTeam to import it."
  218. % new_p.get("team"))
  219. # Check that the participation is not already defined.
  220. p = session.query(Participation)\
  221. .filter(Participation.user_id == user.id)\
  222. .filter(Participation.contest_id == contest.id)\
  223. .first()
  224. # FIXME: detect if some details of the participation have been updated
  225. # and thus the existing participation needs to be changed.
  226. if p is not None:
  227. logger.warning("Participation of user %s in this contest already "
  228. "exists, not updating it.", new_p["username"])
  229. return p
  230. # Prepare new participation
  231. args = {
  232. "user": user,
  233. "contest": contest,
  234. }
  235. if "team" in new_p:
  236. args["team"] = team
  237. if "hidden" in new_p:
  238. args["hidden"] = new_p["hidden"]
  239. if "ip" in new_p and new_p["ip"] is not None:
  240. args["ip"] = list(map(ipaddress.ip_network, new_p["ip"].split(",")))
  241. if "password" in new_p:
  242. args["password"] = new_p["password"]
  243. new_p = Participation(**args)
  244. session.add(new_p)
  245. return new_p
  246. def _delete_stale_participations(self, session, contest,
  247. usernames_to_keep):
  248. """Delete the stale participations.
  249. Stale participations are those in the contest, with a username not in
  250. usernames_to_keep.
  251. session (Session): SQL session to use.
  252. contest (Contest): the contest to examine.
  253. usernames_to_keep ({str}): usernames of non-stale participations.
  254. """
  255. participations = [p for p in contest.participations
  256. if p.user.username not in usernames_to_keep]
  257. if len(participations) > 0:
  258. ans = "y"
  259. if not self.yes:
  260. ans = input("There are %s stale participations. "
  261. "Are you sure you want to delete them and their "
  262. "associated data, including submissions? [y/N] "
  263. % len(participations))\
  264. .strip().lower()
  265. if ans in ["y", "yes"]:
  266. for p in participations:
  267. logger.info("Deleting participations for user %s.",
  268. p.user.username)
  269. session.delete(p)
  270. def main():
  271. """Parse arguments and launch process."""
  272. parser = argparse.ArgumentParser(
  273. description="""\
  274. Import a contest from disk
  275. If updating a contest already in the DB:
  276. - tasks attached to the contest in the DB but not to the contest to be imported
  277. will be detached;
  278. - participations attached to the contest in the DB but not to the contest to be
  279. imported will be retained, this to avoid deleting submissions.
  280. """,
  281. epilog=build_epilog(),
  282. formatter_class=argparse.RawDescriptionHelpFormatter
  283. )
  284. parser.add_argument(
  285. "-y", "--yes",
  286. action="store_true",
  287. help="don't ask for confirmation before deleting data"
  288. )
  289. parser.add_argument(
  290. "-z", "--zero-time",
  291. action="store_true",
  292. help="set to zero contest start and stop time"
  293. )
  294. parser.add_argument(
  295. "-L", "--loader",
  296. action="store", type=utf8_decoder,
  297. default=None,
  298. help="use the specified loader (default: autodetect)"
  299. )
  300. parser.add_argument(
  301. "-i", "--import-tasks",
  302. action="store_true",
  303. help="import tasks if they do not exist"
  304. )
  305. parser.add_argument(
  306. "-u", "--update-contest",
  307. action="store_true",
  308. help="update an existing contest"
  309. )
  310. parser.add_argument(
  311. "-U", "--update-tasks",
  312. action="store_true",
  313. help="update existing tasks"
  314. )
  315. parser.add_argument(
  316. "-S", "--no-statements",
  317. action="store_true",
  318. help="do not import / update task statements"
  319. )
  320. parser.add_argument(
  321. "--delete-stale-participations",
  322. action="store_true",
  323. help="when updating a contest, delete the participations not in the "
  324. "new contest, including their submissions and other data"
  325. )
  326. parser.add_argument(
  327. "import_directory",
  328. action="store", type=utf8_decoder,
  329. help="source directory from where import"
  330. )
  331. args = parser.parse_args()
  332. loader_class = choose_loader(
  333. args.loader,
  334. args.import_directory,
  335. parser.error
  336. )
  337. importer = ContestImporter(
  338. path=args.import_directory,
  339. yes=args.yes,
  340. zero_time=args.zero_time,
  341. import_tasks=args.import_tasks,
  342. update_contest=args.update_contest,
  343. update_tasks=args.update_tasks,
  344. no_statements=args.no_statements,
  345. delete_stale_participations=args.delete_stale_participations,
  346. loader_class=loader_class)
  347. success = importer.do_import()
  348. return 0 if success is True else 1
  349. if __name__ == "__main__":
  350. sys.exit(main())