PrivacyScanData/ThirdParty/data_export.py

# Copyright (C) 2014 Ion Torrent Systems, Inc. All Rights Reserved
from __future__ import absolute_import

import json
import os
import os.path

import boto
import boto.s3.key
import celery.states
import requests
from celery.result import AsyncResult
from celery.task import task
from celery.utils.log import get_task_logger
from django import forms
from django.conf import settings
from django.http import Http404, HttpResponse
from django.shortcuts import get_object_or_404, redirect, render
from django.views.decorators.http import require_POST
from ion.utils import makeCSA

from iondb.rundb import models
from iondb.rundb.api import SupportUploadResource
from iondb.utils import makePDF

logger = get_task_logger(__name__)


def list_export_uploads(request, tag):

    uploads = models.FileMonitor.objects.filter(tags__contains="upload").order_by(
        "-created"
    )
    if tag:
        uploads = uploads.filter(tags_contains=tag)
    ctx = {"uploads": uploads}
    return render(request, "rundb/data/data_export.html", ctx)


def md5_stats_file(path):
    with open(path, "rb", 8192) as fp:
        digest_hex, diges_64, size = boto.s3.key.compute_md5(fp)
    return digest_hex, diges_64, size


@task
def export_upload_file(monitor_id):
    monitor = models.FileMonitor.objects.get(pk=monitor_id)
    full = monitor.full_path()
    monitor.status = "Checking"
    monitor.save()
    if not os.path.exists(full):
        logger.error("OS Error in file uploader")
        monitor.status = "Error: file does not exist"
        monitor.save()
        return

    digest_hex, diges_64, size = md5_stats_file(full)
    monitor.size = size
    monitor.md5sum = digest_hex
    monitor.url = "{0}:{1}".format(monitor.name, monitor.md5sum)

    monitor.status = "Connecting"
    monitor.save()

    try:
        con = boto.connect_s3(settings.AWS_ACCESS_KEY, settings.AWS_SECRET_KEY)
        bucket = con.get_bucket(settings.AWS_BUCKET_NAME)
        key = bucket.get_key(monitor.url)
        if key is not None:
            monitor.status = "Complete"
            monitor.save()
            return
        key = bucket.new_key(monitor.url)
    except Exception as err:
        logger.exception("Connecting error")
        monitor.status = "Error: {0}".format(err)
        monitor.save()
        return

    monitor.status = "Uploading"
    monitor.save()

    # Rewrite this into a class or a callable object
    # last_time = time.time()
    def get_progress(current, total):
        # now = time.time()
        # if now - last_time >= 0.5:
        monitor.progress = current
        monitor.save()
        # last_time = now

    try:
        key.set_contents_from_filename(
            full, cb=get_progress, num_cb=1000, md5=(digest_hex, diges_64)
        )
    except Exception as err:
        logger.exception("Uploading error")
        monitor.status = "Error: Uploading {0}".format(err)[:60]
        monitor.save()
        return

    monitor.progress = monitor.size
    monitor.status = "Complete"
    monitor.save()


def export_upload_report(request):
    try:
        report_pk = int(request.POST.get("report"))
    except ValueError:
        raise Http404("'{0}' is not a valid report ID".format(report_pk))
    path = request.POST.get("file_path")
    report = get_object_or_404(models.Results, pk=report_pk)
    root = report.get_report_dir()
    full_path = os.path.join(root, path)
    if not os.path.exists(full_path):
        raise Http404(
            "'{0}' does not exist as a file in report {1}".format(path, report_pk)
        )
    tag = "report/{0}/".format(report_pk)
    monitor = models.FileMonitor(
        local_dir=os.path.dirname(full_path),
        name=os.path.basename(full_path),
        tags="upload," + tag,
        status="Queued",
    )

    monitor.save()
    result = export_upload_file.delay(monitor.id)
    monitor.celery_task_id = result.task_id
    monitor.save()

    return redirect("list_export_uploads", tag="")


class SupportUploadForm(forms.Form):

    contact_email = forms.EmailField(required=True)
    description = forms.CharField(required=True)
    result = forms.IntegerField(required=True)


@task
def filemonitor_errback(task_id, monitor_pk):
    try:
        monitor = models.FileMonitor.objects.get(pk=monitor_pk)
    except Exception:
        logger.exception("Monitor error callback failed for pk={0}".format(monitor_pk))
        return
    monitor.status = "Error"
    monitor.save()


@task
def generate_csa(result_pk, monitor_pk=None):
    result = models.Results.objects.get(pk=result_pk)
    report_dir = result.get_report_dir()
    raw_data_dir = result.experiment.expDir

    try:
        monitor = models.FileMonitor.objects.get(pk=monitor_pk)
    except models.FileMonitor.DoesNotExist:
        monitor = models.FileMonitor()
        monitor.tags = "generate_csa"

    csa_file_name = "csa_{0:04d}.zip".format(int(result_pk))
    monitor.status = "Generating"
    monitor.local_dir = report_dir
    monitor.name = csa_file_name
    monitor.save()

    # Generate report PDF file.
    # This will create a file named report.pdf in results directory
    makePDF.write_report_pdf(result_pk)
    csa_path = makeCSA.makeCSA(report_dir, raw_data_dir, monitor.name)

    digest_hex, digest_64, size = md5_stats_file(csa_path)
    monitor.md5sum = digest_hex
    monitor.size = size
    monitor.status = "Generated"
    monitor.save()


def get_ts_info():
    path = "/etc/torrentserver/tsconf.conf"
    d = dict()
    if os.path.exists(path):
        for l in open(path):
            row = map(str.strip, l.split(":", 1))
            if len(row) == 2:
                d[row[0]] = row[1]
    return d


def make_auth_header(account):
    return {"Authorization": "Bearer {0}".format(account.access_token)}


@task
def is_authenticated(account_pk):
    account = models.RemoteAccount.objects.get(pk=account_pk)
    if account.has_access():
        url = settings.SUPPORT_AUTH_URL
        auth_header = make_auth_header(account)
        info = get_ts_info()
        params = {
            "version": info.get("version", "Version Missing"),
            "serial_number": info.get("serialnumber", "Serial Missing"),
        }
        try:
            response = requests.get(
                url, params=params, headers=auth_header, verify=False
            )
        except requests.exceptions.RequestException as err:
            logger.error("Request Exception: {0}".format(err))
        else:
            if response.ok:
                return True
            else:
                logger.error("Authentication failure at {0}: {1}".format(url, response))
    return False


@task
def check_authentication(support_upload_pk):
    upload = models.SupportUpload.objects.get(pk=support_upload_pk)
    upload.local_status = "Authenticating"
    upload.save()
    if is_authenticated(upload.account_id):
        upload.local_status = "Authenticated"
        upload.save()
        return True
    else:
        upload.local_status = "Access Denied"
        upload.save()
        return False


@task
def upload_to_support(support_upload_pk):
    upload = models.SupportUpload.objects.select_related("file", "account").get(
        pk=support_upload_pk
    )
    upload.local_status = "Uploading"
    upload.save()

    url = settings.SUPPORT_UPLOAD_URL
    auth_header = make_auth_header(upload.account)
    info = get_ts_info()
    form_data = {
        "contact_email": upload.contact_email,
        "description": upload.description,
        "version": info.get("version", "Version Missing"),
        "serial_number": info.get("serialnumber", "Serial Missing"),
    }
    path = upload.file.full_path()
    files = {"file": open(path, "rb")}

    try:
        response = requests.post(
            url, data=form_data, files=files, headers=auth_header, verify=False
        )
    except Exception as err:
        upload.local_status = "Error"
        upload.local_message = str(err)
    else:
        if response.ok:
            try:
                tick = response.json()
            except ValueError:
                tick = {}
            upload.local_status = "Complete"
            upload.ticket_id = tick.get("ticket_id", "None")
            upload.ticket_status = tick.get("ticket_status", "Remote Error")
            upload.ticket_message = tick.get(
                "ticket_message",
                "There was an error in the support server.  Your Torrent Server is working fine, and you should contact your support representative.",
            )
        else:
            upload.local_status = "Error"
            upload.local_message = response.reason
    finally:
        upload.save()


@task(max_retries=120)
def check_and_upload(support_upload_pk, auth_task, gen_task):
    auth = AsyncResult(auth_task)
    gen_csa = AsyncResult(gen_task)
    if not (auth.ready() and gen_csa.ready()):
        check_and_upload.retry(countdown=1)
    elif auth.status == "SUCCESS" and gen_csa.status == "SUCCESS":
        if auth.result:
            upload_to_support(support_upload_pk)
    else:
        return


@require_POST
def report_support_upload(request):
    form = SupportUploadForm(request.POST)
    # check for existing support upload
    data = {"created": False}
    account = models.RemoteAccount.objects.get(remote_resource="support.iontorrent")
    if not account.has_access():
        data["error"] = "invalid_auth"
    if not form.is_valid():
        data["error"] = "invalid_form"
        data["form_errors"] = form.errors

    if "error" not in data:
        result_pk = form.cleaned_data["result"]
        support_upload = (
            models.SupportUpload.objects.filter(result=result_pk)
            .order_by("-id")
            .first()
        )
        if not support_upload:
            data["created"] = True
            support_upload = models.SupportUpload(
                account=account,
                result_id=result_pk,
                user=request.user,
                local_status="Preparing",
                contact_email=form.cleaned_data["contact_email"],
                description=form.cleaned_data["description"],
            )
            support_upload.save()

        async_result = AsyncResult(support_upload.celery_task_id)
        if (
            not support_upload.celery_task_id
            or async_result.status in celery.states.READY_STATES
        ):
            if not support_upload.file:
                monitor = models.FileMonitor()
                monitor.save()
                support_upload.file = monitor
                support_upload.save()
            support_upload.file.status = ("Queued",)
            support_upload.file.tags = "support_upload,generate_csa"
            support_upload.file.save()

            generate = generate_csa.s(result_pk, support_upload.file.pk)
            errback = filemonitor_errback.s(support_upload.file.pk)
            gen_result = generate.apply_async(link_error=errback)
            support_upload.file.celery_task_id = gen_result.task_id

            auth_result = check_authentication.delay(support_upload.pk)
            upload_result = check_and_upload.apply_async(
                (support_upload.pk, auth_result.task_id, gen_result.task_id),
                countdown=1,
            )

            support_upload.celery_task_id = upload_result.task_id
            support_upload.save()

        resource = SupportUploadResource()
        uri = resource.get_resource_uri(support_upload)
        data["resource_uri"] = uri
    # make CSA
    # check auth
    # start upload task
    # respond with support upload object for JS polling
    return HttpResponse(
        json.dumps(data, indent=4, sort_keys=True), mimetype="application/json"
    )