1234567891011121314151617181920212223242526272829303132 |
- def upload_files():
- adl = core.AzureDLFileSystem(adlCreds, store_name=config.store_name)
- uploadedFolders = adl.ls(adls_upload_folder_path)
-
- uploadedFolders = set([folder.replace(adls_upload_folder_path[1:], "")+"/" for folder in uploadedFolders])
-
- local_folders = glob.glob(local_upload_folder_path+"*") # * means all if need specific format then *.csv
- local_folders = set([d.replace(local_upload_folder_path, "")+"/" for d in local_folders])
- to_upload_folders = local_folders.difference(uploadedFolders)
- folder_names = sorted([d.replace(local_upload_folder_path, "") for d in to_upload_folders])
- files = []
- for folder in folder_names:
- path = local_upload_folder_path+folder
- for f in listdir(path):
- if isfile(join(path, f)):
- files.append(folder+f)
- print("Uploading the following folders:<br>{}<br>Total number of files to upload:<br>{}".format(", ". join(folder_names), len(files)))
-
- for f in files:
- adl.put(local_upload_folder_path+f, adls_upload_folder_path+f)
-
- print("Upload finished.")
- time.sleep(2)
- global uploaded_files
- uploaded_files = True
|