GnnForPrivacyScan/preprocess/splitfile.py

import os

from utils.fileio import *


def split_file(file_dir, output_dir, endpoint=".py"):
    for file_path in walk_files(file_dir, endpoint):
        with open(file_path, 'r') as file:
            content = file.read()
            s_index = 0
            e_index = len(content)
            file_id = 1
            while s_index < e_index:
                try:
                    left_index = content.index("def ", s_index, e_index)
                    s_index = left_index + 1
                    right_index = content.index("def ", s_index, e_index)
                    content_tmp = content[left_index:right_index]
                    if left_index > 3 and content[left_index - 4: left_index] == "    ":
                        content_tmp = content_tmp.replace("\n    ", "\n")
                    new_file_name = file_path.replace(file_dir, output_dir).replace(".py", "_" + str(file_id) + ".py")
                    if not os.path.exists(new_file_name[:new_file_name.rindex("/")]):
                        os.makedirs(new_file_name[:new_file_name.rindex("/")])
                    if content[left_index + 4] != '_':
                        with open(new_file_name, 'w') as file_out:
                            file_out.write(content_tmp)
                        file_id += 1
                except ValueError:
                    break
            if s_index < e_index and content[s_index - 1: s_index + 3] == "def ":
                content_tmp = content[s_index - 1:e_index]
                if s_index > 4 and content[s_index - 5: s_index - 1] == "    ":
                    content_tmp = content_tmp.replace("\n    ", "\n")
                new_file_name = file_path.replace(file_dir, output_dir).replace(".py", "_" + str(file_id) + ".py")
                if not os.path.exists(new_file_name[:new_file_name.rindex("/")]):
                    os.makedirs(new_file_name[:new_file_name.rindex("/")])
                if content[s_index + 3] != '_':
                    with open(new_file_name, 'w') as file_out:
                        file_out.write(content_tmp)
            b = "asdfasdfasdf"


if __name__ == '__main__':
    split_file("/Users/liufan/program/PYTHON/sap2nd/GnnForPrivacyScan/data/purposeCombined", "../data/purposeSplit3")