PrivacyScanData/target/S3/new2/django-s3sync-master/s3sync/management/commands/s3sync_media_6.py

def upload_s3(self, arg, dirname, names):
        """
        This is the callback to os.path.walk and where much of the work happens
        """
        bucket, key, bucket_name, root_dir = arg

        # Skip files and directories we don't want to sync
        for pattern in self.EXCLUDE_LIST:
            if fnmatch(os.path.basename(dirname), pattern):
                if self.verbosity > 1:
                    print ('Skipping: %s (rule: %s)' % (names, pattern))
                del names[:]
                return

        # Later we assume the MEDIA_ROOT ends with a trailing slash
        if not root_dir.endswith(os.path.sep):
            root_dir = root_dir + os.path.sep

        list_prefix = dirname[len(root_dir):]
        if self.prefix:
            list_prefix = '%s/%s' % (self.prefix, list_prefix)
        s3_list = bucket_lister(bucket, prefix=list_prefix)

        for name in names:
            bad_name = False
            for pattern in self.EXCLUDE_LIST:
                if fnmatch(name, pattern):
                    bad_name = True  # Skip files we don't want to sync
            if bad_name:
                if self.verbosity > 1:
                    print ('Skipping: %s (rule: %s)' % (names, pattern))
                continue

            filename = os.path.join(dirname, name)
            if os.path.isdir(filename):
                continue  # Don't try to upload directories

            file_key = filename[len(root_dir):]
            if self.prefix:
                file_key = '%s/%s' % (self.prefix, file_key)

            # Check if file on S3 is older than local file, if so, upload
            # TODO: check if hash chunk corresponds
            if not self.do_force:
                s3_key = self.find_key_in_list(s3_list, file_key)
                if s3_key:
                    s3_datetime = datetime.datetime(*time.strptime(
                        s3_key.last_modified, '%Y-%m-%dT%H:%M:%S.000Z')[0:6])
                    local_datetime = datetime.datetime.utcfromtimestamp(
                        os.stat(filename).st_mtime)
                    if local_datetime < s3_datetime:
                        self.skip_count += 1
                        if self.verbosity > 1:
                            print ("File %s hasn't been modified since last " \
                                "being uploaded" % (file_key))
                        if file_key in self.s3_files:
                            self.files_processed.add(file_key)
                            del self.s3_files[file_key]
                        continue
            if file_key in self.s3_files:
                self.files_processed.add(file_key)
                del self.s3_files[file_key]

            # File is newer, let's process and upload
            if self.verbosity > 0:
                print ("Uploading %s..." % file_key)
                if self.dry_run:
                    self.upload_count += 1
                    continue

            try:
                upload_file_to_s3(file_key, filename, key,
                    do_gzip=self.do_gzip, do_expires=self.do_expires,
                    verbosity=self.verbosity)
            except boto.exception.S3CreateError as e:
                # TODO: retry to create a few times
                print ("Failed to upload: %s" % e)
            except Exception as e:
                print (e)
                raise
            else:
                self.upload_count += 1

        # If we don't care about what's missing, wipe this to save memory.
        if not self.remove_missing:
            self.s3_files = {}
        else:
            self.finish_list(s3_list)