def upload_files():
adl = core.AzureDLFileSystem(adlCreds, store_name=config.store_name)
uploadedFolders = adl.ls(adls_upload_folder_path)
uploadedFolders = set([folder.replace(adls_upload_folder_path[1:], "")+"/" for folder in uploadedFolders])
local_folders = glob.glob(local_upload_folder_path+"*") # * means all if need specific format then *.csv
local_folders = set([d.replace(local_upload_folder_path, "")+"/" for d in local_folders])
to_upload_folders = local_folders.difference(uploadedFolders)
folder_names = sorted([d.replace(local_upload_folder_path, "") for d in to_upload_folders])
files = []
for folder in folder_names:
path = local_upload_folder_path+folder
for f in listdir(path):
if isfile(join(path, f)):
files.append(folder+f)
print("Uploading the following folders:
{}
Total number of files to upload:
{}".format(", ". join(folder_names), len(files)))
for f in files:
adl.put(local_upload_folder_path+f, adls_upload_folder_path+f)
print("Upload finished.")
time.sleep(2)
global uploaded_files
uploaded_files = True