| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061 |
- def _process_one_package(package_name, version):
- logger.info("Worker: Package:{} Version:{}".format(package_name, version))
- if not package_name or not version:
- logger.warn("Package_name or version was empty. Moving on as the queue had bad data")
- return
- # .6684 seconds to run. 74577 total packages
- package_info = PyPIPackageInformation.get_package_specific_version_info(package_name, version)
- if not package_info:
- logger.error("Worker: Package:{} Version:{} failed to get package info".format(package_name, version))
- return
- supports_python_2 = len([x for x in package_info['classifiers'] if x.startswith('Programming Language :: Python :: 2')]) > 0
- supports_python_3 = len([x for x in package_info['classifiers'] if x.startswith('Programming Language :: Python :: 3')]) > 0
- uploaded = package_info['uploaded']
- try:
- summary_entity = table_service.get_entity(config.PACKAGE_SUMMARY_TABLENAME, package_name, STATIC_ROW_KEY)
- except:
- # we don't have a summary for this entry.
- summary_entity = {
- 'PartitionKey':package_name, 'RowKey':STATIC_ROW_KEY, 'First_Published':None,
- 'Python2_Start':None, 'Python2_End':None, 'Python3_Start':None
- }
- table_service.insert_or_replace_entity(config.PACKAGE_SUMMARY_TABLENAME, package_name, STATIC_ROW_KEY, summary_entity)
- summary_entity = table_service.get_entity(config.PACKAGE_SUMMARY_TABLENAME, package_name, STATIC_ROW_KEY)
- # set fields using upload. Upload is none if the version has never been uploaded
- # Basically just filter out packages that never have content from our records.
- if uploaded is not None:
- if not hasattr(summary_entity, 'First_Published') or summary_entity.First_Published is None or summary_entity.First_Published > uploaded:
- # if the published date is empty or later than the current release we
- # are viewing update
- summary_entity.First_Published = uploaded
- if supports_python_2 and \
- (not hasattr(summary_entity, 'Python2_Start') or summary_entity.Python2_Start is None or summary_entity.Python2_Start > uploaded):
- # if the published date is empty or later than the date and it supports
- # python 2
- summary_entity.Python2_Start = uploaded
-
- if supports_python_2 and hasattr(summary_entity, 'Python2_End') and summary_entity.Python2_End is not None and summary_entity.Python2_End < uploaded:
- # we support python2 but it is after the date we thought python 2
- # support ended we must not have really ended
- summary_entity.Python2_End = None
- elif hasattr(summary_entity, 'Python2_Start') and hasattr(summary_entity, 'Python2_End') and \
- summary_entity.Python2_Start is not None and summary_entity.Python2_End is not None and \
- (summary_entity.Python2_End > uploaded and summary_entity.Python2_Start < uploaded):
- # if we don't support python2, and we have started supporting python2
- # at some point
- # and if the date we are saying we ended is after the start
- summary_entity.Python2_End = uploaded
- if supports_python_3 and \
- (not hasattr(summary_entity, 'Python3_Start') or summary_entity.Python3_Start is None or summary_entity.Python3_Start > uploaded):
- # if the published date is empty or later than the current release we
- # are viewing update
- summary_entity.Python3_Start = uploaded
- version_entity = _insert_entity_to_package_version_table(package_name, version, supports_python_2, supports_python_3, package_info['downloads'], uploaded)
- summary_entity = table_service.insert_or_replace_entity(config.PACKAGE_SUMMARY_TABLENAME, package_name, STATIC_ROW_KEY, summary_entity)
|
