123456789101112131415161718192021222324252627282930313233343536373839404142 |
- def get_unzip_hdfs_file_from_dir(hdfs_dir, save_dir):
-
- hadoop_ls = "hadoop fs -ls %s | grep -i '^-'" % hdfs_dir
-
- save_file_list = []
-
- hdfs_result = exec_sh(hadoop_ls, None)
-
- hdfs_stdout = hdfs_result["stdout"]
-
-
- hdfs_list = []
-
- if hdfs_stdout:
-
- hdfs_lines = hdfs_stdout.split("\n")
-
- for line in hdfs_lines:
-
- line_list = re.split("\s+", line)
-
- if line_list.__len__() == 8:
-
-
- hdfs_list.append(line_list[7])
- else:
- pass
-
- for file in hdfs_list:
- save_filename = get_unzip_hdfs_file(file, save_dir)
- save_file_list.append(save_filename)
- return save_file_list
- else:
- return False
|