def upload_files(): adl = core.AzureDLFileSystem(adlCreds, store_name=config.store_name) uploadedFolders = adl.ls(adls_upload_folder_path) uploadedFolders = set([folder.replace(adls_upload_folder_path[1:], "")+"/" for folder in uploadedFolders]) local_folders = glob.glob(local_upload_folder_path+"*") # * means all if need specific format then *.csv local_folders = set([d.replace(local_upload_folder_path, "")+"/" for d in local_folders]) to_upload_folders = local_folders.difference(uploadedFolders) folder_names = sorted([d.replace(local_upload_folder_path, "") for d in to_upload_folders]) files = [] for folder in folder_names: path = local_upload_folder_path+folder for f in listdir(path): if isfile(join(path, f)): files.append(folder+f) print("Uploading the following folders:
{}
Total number of files to upload:
{}".format(", ". join(folder_names), len(files))) for f in files: adl.put(local_upload_folder_path+f, adls_upload_folder_path+f) print("Upload finished.") time.sleep(2) global uploaded_files uploaded_files = True