| 1234567891011121314151617181920212223242526272829303132 |
- def fetch_archived_data_upload_to_s3_and_delete(
- archive_host, db_name, table_name, archive_db_name, archive_table_name,
- column_name_to_log_in_file, transaction_size, where_clause):
- no_of_rows_archived = db_utils.get_count_of_rows_archived(
- archive_db_name, archive_table_name)
- if not no_of_rows_archived:
- logging.info(
- f'Archive table {archive_db_name}.{archive_table_name} '
- f'had no rows, dropping archive table')
- db_utils.drop_archive_table(archive_db_name, archive_table_name)
- return None
- local_file_name, s3_path = db_utils.get_file_names(
- db_name, table_name, archive_db_name, archive_table_name,
- column_name_to_log_in_file, where_clause)
- archive_utils.archive_to_file(
- archive_host, archive_db_name, archive_table_name, transaction_size, local_file_name)
- gzip_file_name = compress_to_gzip(local_file_name)
- gzip_s3_path = f'{s3_path}.gz'
- # s3_utils.upload_to_s3(local_file_name, s3_path)
- s3_utils.upload_to_s3(gzip_file_name, gzip_s3_path)
- logging.info(f'Deleting local file: {local_file_name}')
- os.remove(local_file_name)
- os.remove(gzip_file_name)
- db_utils.drop_archive_table(archive_db_name, archive_table_name)
- return None
|
