def archive(self): access_key = config.get('ckanext.s3archive.access_key') secret_key = config.get('ckanext.s3archive.secret_key') bucket_name = config.get('ckanext.s3archive.bucket') if not access_key: print ('ckanext.s3archive.access_key config argument not set') return if not secret_key: print ('ckanext.s3archive.secret_key config argument not set') return if not bucket_name: print ('ckanext.s3archive.bucket config argument not set') return storage_path = get_storage_path() if not storage_path: print ('ckan.storage_path not set in config') return resource_path = os.path.join(storage_path, 'resources') def walk(bucket, dir, files): for file in files: full_path = os.path.join(resource_path, dir, file) if not os.path.isfile(full_path) or full_path.endswith('~'): continue key_name = full_path[len(resource_path):] for key in bucket.list(prefix=key_name.lstrip('/')): key.delete() resource_id = key_name.replace('/', '') resource = model.Resource.get(resource_id) if not resource: continue last_part = resource.url.split('/')[-1] file_name = munge.munge_filename(last_part) key_name = key_name + '/' + file_name key = s3key.Key(bucket) key.key = key_name key.set_contents_from_filename(full_path) print ('Archived %s' % key_name) os.remove(full_path) conn = s3connection.S3Connection(access_key, secret_key) bucket = conn.get_bucket(bucket_name) try: os.path.walk(resource_path, walk, bucket) finally: conn.close()