| 1234567891011121314151617181920212223242526272829303132333435363738394041 |
- import logging
- import argparse
- from . import S3Concat
- logging.basicConfig(
- level=logging.INFO,
- format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
- )
- def cli():
- parser = argparse.ArgumentParser(description='Convert data files')
- parser.add_argument("--bucket",
- help="base bucket to use",
- required=True,
- )
- parser.add_argument("--folder",
- help="folder whose contents should be combined",
- required=True,
- )
- parser.add_argument("--output",
- help=("output location for resulting merged files,"
- " relative to the specified base bucket"),
- required=True,
- )
- parser.add_argument("--filesize",
- help=("Use to create multiple files if needed."
- " Min filesize of the concatenated files"
- " in [B,KB,MB,GB,TB]. e.x. 5.2GB"),
- default=None,
- )
- parser.add_argument("--small-parts-threads",
- type=int,
- help=("[Advanced Usage] Number of threads to"
- " download small parts with. (Default: 1)"),
- default=1)
- args = parser.parse_args()
- job = S3Concat(args.bucket, args.output, args.filesize)
- job.add_files(args.folder)
- job.concat(small_parts_threads=args.small_parts_threads)
|
