PrivacyScanData/Share/source_util.py

# -*- coding: utf-8 -*- #
# Copyright 2018 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""'functions deploy' utilities for function source code."""

from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import os
import random
import re
import string

from apitools.base.py import http_wrapper
from apitools.base.py import transfer


from googlecloudsdk.api_lib.functions import exceptions
from googlecloudsdk.api_lib.functions import util as api_util
from googlecloudsdk.api_lib.storage import storage_util
from googlecloudsdk.command_lib.util import gcloudignore
from googlecloudsdk.core import http as http_utils
from googlecloudsdk.core import log
from googlecloudsdk.core import properties
from googlecloudsdk.core.util import archive
from googlecloudsdk.core.util import files as file_utils
from six.moves import range


def _GcloudIgnoreCreationPredicate(directory):
  return gcloudignore.AnyFileOrDirExists(
      directory, gcloudignore.GIT_FILES + ['node_modules'])


def _GetChooser(path):
  default_ignore_file = gcloudignore.DEFAULT_IGNORE_FILE + '\nnode_modules\n'

  return gcloudignore.GetFileChooserForDir(
      path, default_ignore_file=default_ignore_file,
      gcloud_ignore_creation_predicate=_GcloudIgnoreCreationPredicate)


def _ValidateUnpackedSourceSize(path):
  """Validate size of unpacked source files."""
  chooser = _GetChooser(path)
  predicate = chooser.IsIncluded
  try:
    size_b = file_utils.GetTreeSizeBytes(path, predicate=predicate)
  except OSError as e:
    raise exceptions.FunctionsError(
        'Error building source archive from path [{path}]. '
        'Could not validate source files: [{error}]. '
        'Please ensure that path [{path}] contains function code or '
        'specify another directory with --source'.format(path=path, error=e))
  size_limit_mb = 512
  size_limit_b = size_limit_mb * 2 ** 20
  if size_b > size_limit_b:
    raise exceptions.OversizedDeployment(
        str(size_b) + 'B', str(size_limit_b) + 'B')


def _CreateSourcesZipFile(zip_dir, source_path):
  """Prepare zip file with source of the function to upload.

  Args:
    zip_dir: str, directory in which zip file will be located. Name of the file
             will be `fun.zip`.
    source_path: str, directory containing the sources to be zipped.
  Returns:
    Path to the zip file (str).
  Raises:
    FunctionsError
  """
  api_util.ValidateDirectoryExistsOrRaiseFunctionError(source_path)
  _ValidateUnpackedSourceSize(source_path)
  zip_file_name = os.path.join(zip_dir, 'fun.zip')
  try:
    chooser = _GetChooser(source_path)
    predicate = chooser.IsIncluded
    archive.MakeZipFromDir(zip_file_name, source_path, predicate=predicate)
  except ValueError as e:
    raise exceptions.FunctionsError(
        'Error creating a ZIP archive with the source code '
        'for directory {0}: {1}'.format(source_path, str(e)))
  return zip_file_name


def _GenerateRemoteZipFileName(function_name):
  suffix = ''.join(random.choice(string.ascii_lowercase) for _ in range(12))
  return '{0}-{1}-{2}.zip'.format(
      properties.VALUES.functions.region.Get(), function_name, suffix)


def _UploadFileToGcs(source, function_ref, stage_bucket):
  """Upload local source files to GCS staging bucket."""
  zip_file = _GenerateRemoteZipFileName(function_ref.RelativeName())
  bucket_ref = storage_util.BucketReference.FromArgument(
      stage_bucket)
  gcs_url = storage_util.ObjectReference(bucket_ref, zip_file).ToUrl()
  upload_result = storage_util.RunGsutilCommand('cp', [source, gcs_url])
  if upload_result != 0:
    raise exceptions.FunctionsError(
        'Failed to upload the function source code to the bucket {0}'
        .format(stage_bucket))
  return gcs_url


def _AddDefaultBranch(source_archive_url):
  cloud_repo_pattern = (r'^https://source\.developers\.google\.com'
                        r'/projects/[^/]+'
                        r'/repos/[^/]+$')
  if re.match(cloud_repo_pattern, source_archive_url):
    return source_archive_url + '/moveable-aliases/master'
  return source_archive_url


def _GetUploadUrl(messages, service, function_ref):
  request = (messages.
             CloudfunctionsProjectsLocationsFunctionsGenerateUploadUrlRequest)(
                 parent='projects/{}/locations/{}'.format(
                     function_ref.projectsId, function_ref.locationsId))
  response = service.GenerateUploadUrl(request)
  return response.uploadUrl


def _CheckUploadStatus(status_code):
  """Validates that HTTP status for upload is 2xx."""
  return status_code // 100 == 2


def _UploadFileToGeneratedUrl(source, messages, service, function_ref):
  """Upload function source to URL generated by API."""
  url = _GetUploadUrl(messages, service, function_ref)
  upload = transfer.Upload.FromFile(source,
                                    mime_type='application/zip')
  upload_request = http_wrapper.Request(
      url, http_method='PUT', headers={
          'content-type': 'application/zip',
          # Magic header, request will fail without it.
          # Not documented at the moment this comment was being written.
          'x-goog-content-length-range': '0,104857600',
          'Content-Length': '{0:d}'.format(upload.total_size)})
  upload_request.body = upload.stream.read()
  response = http_wrapper.MakeRequest(
      http_utils.Http(), upload_request, retry_func=upload.retry_func,
      retries=upload.num_retries)
  if not _CheckUploadStatus(response.status_code):
    raise exceptions.FunctionsError(
        'Failed to upload the function source code to signed url: {url}. '
        'Status: [{code}:{detail}]'.format(url=url,
                                           code=response.status_code,
                                           detail=response.content))
  return url


def UploadFile(source, stage_bucket, messages, service, function_ref):
  if stage_bucket:
    return _UploadFileToGcs(source, function_ref, stage_bucket)
  return _UploadFileToGeneratedUrl(source, messages, service, function_ref)


def SetFunctionSourceProps(function, function_ref, source_arg, stage_bucket):
  """Add sources to function.

  Args:
    function: The function to add a source to.
    function_ref: The reference to the function.
    source_arg: Location of source code to deploy.
    stage_bucket: The name of the Google Cloud Storage bucket where source code
        will be stored.
  Returns:
    A list of fields on the function that have been changed.
  """
  function.sourceArchiveUrl = None
  function.sourceRepository = None
  function.sourceUploadUrl = None

  messages = api_util.GetApiMessagesModule()

  if source_arg is None:
    source_arg = '.'
  source_arg = source_arg or '.'
  if source_arg.startswith('gs://'):
    if not source_arg.endswith('.zip'):
      # Users may have .zip archives with unusual names, and we don't want to
      # prevent those from being deployed; the deployment should go through so
      # just warn here.
      log.warning(
          '[{}] does not end with extension `.zip`. '
          'The `--source` argument must designate the zipped source archive '
          'when providing a Google Cloud Storage URI.'.format(source_arg))
    function.sourceArchiveUrl = source_arg
    return ['sourceArchiveUrl']
  elif source_arg.startswith('https://'):
    function.sourceRepository = messages.SourceRepository(
        url=_AddDefaultBranch(source_arg)
    )
    return ['sourceRepository']
  with file_utils.TemporaryDirectory() as tmp_dir:
    zip_file = _CreateSourcesZipFile(tmp_dir, source_arg)
    service = api_util.GetApiClientInstance().projects_locations_functions

    upload_url = UploadFile(
        zip_file, stage_bucket, messages, service, function_ref)
    if upload_url.startswith('gs://'):
      function.sourceArchiveUrl = upload_url
      return ['sourceArchiveUrl']
    else:
      function.sourceUploadUrl = upload_url
      return ['sourceUploadUrl']