def main(args): """Loop thru all the games and parse them.""" if not os.path.isdir(args.dir): print ("The specified folder is not a directory.") sys.exit(1) NUMBER_OF_FILES = len(os.listdir(args.dir)) if args.num_of_files: NUMBER_OF_FILES = args.num_of_files print ("Parsing", NUMBER_OF_FILES, "files") sql = None if not args.stdout: sql = sqlite3.connect(args.database) sql.execute("""PRAGMA foreign_keys = ON;""") sql.execute("""CREATE TABLE airdates( game INTEGER PRIMARY KEY, airdate TEXT );""") sql.execute("""CREATE TABLE documents( id INTEGER PRIMARY KEY AUTOINCREMENT, clue TEXT, answer TEXT );""") sql.execute("""CREATE TABLE categories( id INTEGER PRIMARY KEY AUTOINCREMENT, category TEXT UNIQUE );""") sql.execute("""CREATE TABLE clues( id INTEGER PRIMARY KEY AUTOINCREMENT, game INTEGER, round INTEGER, value INTEGER, FOREIGN KEY(id) REFERENCES documents(id), FOREIGN KEY(game) REFERENCES airdates(game) );""") sql.execute("""CREATE TABLE classifications( clue_id INTEGER, category_id INTEGER, FOREIGN KEY(clue_id) REFERENCES clues(id), FOREIGN KEY(category_id) REFERENCES categories(id) );""") for i, file_name in enumerate(glob(os.path.join(args.dir, "*.html")), 1): with open(os.path.abspath(file_name)) as f: parse_game(f, sql, i) if not args.stdout: sql.commit() print ("All done")