#!/usr/bin/env python3 """ This is the main program for the Zulip archive system. For help: python archive.py -h Note that this actual file mostly does the following: parse command line arguments check some settings from settings.py complain if you haven't made certain directories The actual work is done in two main libraries: lib/html.py lib/populate.py """ # The workflow (timing for the leanprover Zulip chat, on my slow laptop): # - populate_all() builds a json file in `settings.json_directory` for each topic, # containing message data and an index json file mapping streams to their topics. # This uses the Zulip API and takes ~10 minutes to crawl the whole chat. # - populate_incremental() assumes there is already a json cache and collects only new messages. # - build_website() builds the webstie # - See hosting.md for suggestions on hosting. # import sys if sys.version_info < (3, 6): version_error = " Python version must be 3.6 or higher\n\ Your current version of python is {}.{}\n\ Please try again with python3.".format( sys.version_info.major, sys.version_info.minor ) raise Exception(version_error) import argparse import configparser import os import zulip from lib.common import stream_validator, exit_immediately # Most of the heavy lifting is done by the following modules: from lib.populate import populate_all, populate_incremental from lib.website import build_website from lib.sitemap import build_sitemap try: import settings except ModuleNotFoundError: # TODO: Add better instructions. exit_immediately( """ We can't find settings.py. Please copy default_settings.py to settings.py and then edit the settings.py file to fit your use case. For testing, you can often leave the default settings, but you will still want to review them first. """ ) NO_JSON_DIR_ERROR_WRITE = """ We cannot find a place to write JSON files. Please run the below command: mkdir {}""" NO_JSON_DIR_ERROR_READ = """ We cannot find a place to read JSON files. Please run the below command: mkdir {} And then fetch the JSON: python archive.py -t""" NO_HTML_DIR_ERROR = """ We cannot find a place to write HTML files. Please run the below command: mkdir {}""" def get_json_directory(for_writing): json_dir = settings.json_directory if not json_dir.exists(): # I use posix paths here, since even on Windows folks will # probably be using some kinda Unix-y shell to run mkdir. if for_writing: error_msg = NO_JSON_DIR_ERROR_WRITE.format(json_dir.as_posix()) else: error_msg = NO_JSON_DIR_ERROR_READ.format(json_dir.as_posix()) exit_immediately(error_msg) if not json_dir.is_dir(): exit_immediately(str(json_dir) + " needs to be a directory") return settings.json_directory def get_html_directory(): html_dir = settings.html_directory if not html_dir.exists(): error_msg = NO_HTML_DIR_ERROR.format(html_dir.as_posix()) exit_immediately(error_msg) if not html_dir.is_dir(): exit_immediately(str(html_dir) + " needs to be a directory") return settings.html_directory def get_client_info(): config_file = "./zuliprc" client = zulip.Client(config_file=config_file) # It would be convenient if the Zulip client object # had a `site` field, but instead I just re-read the file # directly to get it. config = configparser.RawConfigParser() config.read(config_file) zulip_url = config.get("api", "site") return client, zulip_url def run(): parser = argparse.ArgumentParser( description="Build an html archive of the Zulip chat." ) parser.add_argument( "-b", action="store_true", default=False, help="Build .md files" ) parser.add_argument( "--no-sitemap", action="store_true", default=False, help="Don't build sitemap files", ) parser.add_argument( "-t", action="store_true", default=False, help="Make a clean json archive" ) parser.add_argument( "-i", action="store_true", default=False, help="Incrementally update the json archive", ) results = parser.parse_args() if results.t and results.i: print("Cannot perform both a total and incremental update. Use -t or -i.") exit(1) if not (results.t or results.i or results.b): print("\nERROR!\n\nYou have not specified any work to do.\n") parser.print_help() exit(1) json_root = get_json_directory(for_writing=results.t) # The directory where this archive.py is located repo_root = os.path.dirname(os.path.realpath(__file__)) if results.b: md_root = get_html_directory() if results.t or results.i: is_valid_stream_name = stream_validator(settings) client, zulip_url = get_client_info() if results.t: populate_all( client, json_root, is_valid_stream_name, ) elif results.i: populate_incremental( client, json_root, is_valid_stream_name, ) if results.b: build_website( json_root, md_root, settings.site_url, settings.html_root, settings.title, zulip_url, settings.zulip_icon_url, repo_root, settings.page_head_html, settings.page_footer_html, ) if not results.no_sitemap: build_sitemap(settings.site_url, md_root.as_posix(), md_root.as_posix()) if __name__ == "__main__": run()