def _process_one_package(package_name, version): logger.info("Worker: Package:{} Version:{}".format(package_name, version)) if not package_name or not version: logger.warn("Package_name or version was empty. Moving on as the queue had bad data") return # .6684 seconds to run. 74577 total packages package_info = PyPIPackageInformation.get_package_specific_version_info(package_name, version) if not package_info: logger.error("Worker: Package:{} Version:{} failed to get package info".format(package_name, version)) return supports_python_2 = len([x for x in package_info['classifiers'] if x.startswith('Programming Language :: Python :: 2')]) > 0 supports_python_3 = len([x for x in package_info['classifiers'] if x.startswith('Programming Language :: Python :: 3')]) > 0 uploaded = package_info['uploaded'] try: summary_entity = table_service.get_entity(config.PACKAGE_SUMMARY_TABLENAME, package_name, STATIC_ROW_KEY) except: # we don't have a summary for this entry. summary_entity = { 'PartitionKey':package_name, 'RowKey':STATIC_ROW_KEY, 'First_Published':None, 'Python2_Start':None, 'Python2_End':None, 'Python3_Start':None } table_service.insert_or_replace_entity(config.PACKAGE_SUMMARY_TABLENAME, package_name, STATIC_ROW_KEY, summary_entity) summary_entity = table_service.get_entity(config.PACKAGE_SUMMARY_TABLENAME, package_name, STATIC_ROW_KEY) # set fields using upload. Upload is none if the version has never been uploaded # Basically just filter out packages that never have content from our records. if uploaded is not None: if not hasattr(summary_entity, 'First_Published') or summary_entity.First_Published is None or summary_entity.First_Published > uploaded: # if the published date is empty or later than the current release we # are viewing update summary_entity.First_Published = uploaded if supports_python_2 and \ (not hasattr(summary_entity, 'Python2_Start') or summary_entity.Python2_Start is None or summary_entity.Python2_Start > uploaded): # if the published date is empty or later than the date and it supports # python 2 summary_entity.Python2_Start = uploaded if supports_python_2 and hasattr(summary_entity, 'Python2_End') and summary_entity.Python2_End is not None and summary_entity.Python2_End < uploaded: # we support python2 but it is after the date we thought python 2 # support ended we must not have really ended summary_entity.Python2_End = None elif hasattr(summary_entity, 'Python2_Start') and hasattr(summary_entity, 'Python2_End') and \ summary_entity.Python2_Start is not None and summary_entity.Python2_End is not None and \ (summary_entity.Python2_End > uploaded and summary_entity.Python2_Start < uploaded): # if we don't support python2, and we have started supporting python2 # at some point # and if the date we are saying we ended is after the start summary_entity.Python2_End = uploaded if supports_python_3 and \ (not hasattr(summary_entity, 'Python3_Start') or summary_entity.Python3_Start is None or summary_entity.Python3_Start > uploaded): # if the published date is empty or later than the current release we # are viewing update summary_entity.Python3_Start = uploaded version_entity = _insert_entity_to_package_version_table(package_name, version, supports_python_2, supports_python_3, package_info['downloads'], uploaded) summary_entity = table_service.insert_or_replace_entity(config.PACKAGE_SUMMARY_TABLENAME, package_name, STATIC_ROW_KEY, summary_entity)