|
| 1 | +import aiohttp |
| 2 | +import importlib |
| 3 | +import json |
| 4 | + |
| 5 | +from asgiref.sync import sync_to_async |
| 6 | +from queue import Empty, Queue |
| 7 | +from threading import Thread |
| 8 | + |
| 9 | +from pulpcore.plugin.util import get_domain |
| 10 | +from pulpcore.plugin.models import CreatedResource, VulnerabilityReport |
| 11 | +from pulpcore.constants import OSV_QUERY_URL, VULNERABILITY_TASK_THREAD_TIMEOUT |
| 12 | + |
| 13 | + |
| 14 | +# Create a thread-safe queue to share Content units between threads |
| 15 | +content_queue = Queue() |
| 16 | + |
| 17 | + |
| 18 | +async def check_content(func, *args): |
| 19 | + """ |
| 20 | + Start the background_thread and make the API requests (scan) to osv.dev with the packages |
| 21 | + from Queue. |
| 22 | +
|
| 23 | + Args: |
| 24 | + func (callable | str): The function to populate content_queue Queue with the osv.dev |
| 25 | + expected request data format. |
| 26 | + args (tuple): The positional arguments to pass on to the func. |
| 27 | + """ |
| 28 | + module_name, function_name = func.rsplit(".", 1) |
| 29 | + module = importlib.import_module(module_name) |
| 30 | + func = getattr(module, function_name) |
| 31 | + |
| 32 | + background_thread = Thread(target=func, args=args) |
| 33 | + await _scan_packages(background_thread) |
| 34 | + background_thread.join() |
| 35 | + |
| 36 | + |
| 37 | +async def _scan_packages(background_thread): |
| 38 | + """ |
| 39 | + Makes a request to the osv.dev API and store the results in VulnerabilityReport model. |
| 40 | +
|
| 41 | + Args: |
| 42 | + background_thread (Thread): We need to pass the thread object used to populate the queue to |
| 43 | + prevent deadlock issues. |
| 44 | + """ |
| 45 | + scanned_packages = await _scan_packages_from_queue( |
| 46 | + background_thread=background_thread, |
| 47 | + ) |
| 48 | + await _save_vulnerability_report(scanned_packages) |
| 49 | + |
| 50 | + |
| 51 | +async def _scan_packages_from_queue(background_thread, http_client=None): |
| 52 | + """ |
| 53 | + Scans packages from a queue by making HTTP requests to OSV API. |
| 54 | +
|
| 55 | + Args: |
| 56 | + background_thread (Thread): Thread populating the queue |
| 57 | + http_client (Optional[aiohttp.ClientSession]): HTTP client for making requests |
| 58 | +
|
| 59 | + Returns: |
| 60 | + Dict[str, Any]: Dictionary mapping package names to vulnerability data |
| 61 | + """ |
| 62 | + |
| 63 | + # Use provided client or create a new one |
| 64 | + if http_client: |
| 65 | + return await _process_queue_with_client(background_thread, http_client) |
| 66 | + |
| 67 | + async with aiohttp.ClientSession() as session: |
| 68 | + return await _process_queue_with_client(background_thread, session) |
| 69 | + |
| 70 | + |
| 71 | +async def _process_queue_with_client(background_thread, session): |
| 72 | + """ |
| 73 | + Process queue items using the provided HTTP client session. |
| 74 | +
|
| 75 | + Args: |
| 76 | + background_thread (Thread): Thread populating the queue |
| 77 | + session (aiohttp.ClientSession): HTTP client session |
| 78 | +
|
| 79 | + Returns: |
| 80 | + Dict[str, Any]: Dictionary mapping package names to vulnerability data |
| 81 | + """ |
| 82 | + scanned_packages = {} |
| 83 | + try: |
| 84 | + for osv_data in iter( |
| 85 | + lambda: content_queue.get(timeout=VULNERABILITY_TASK_THREAD_TIMEOUT), None |
| 86 | + ): |
| 87 | + if isinstance(osv_data, Exception): |
| 88 | + raise RuntimeError(f"Background vuln report task failed to execute: {osv_data}") |
| 89 | + |
| 90 | + vulnerability_data = await _query_osv_api(session, osv_data) |
| 91 | + package_name = _get_package_name(osv_data) |
| 92 | + |
| 93 | + if vulnerability_data.get("vulns"): |
| 94 | + scanned_packages[package_name] = vulnerability_data["vulns"] |
| 95 | + |
| 96 | + # Handle pagination |
| 97 | + if next_page_token := vulnerability_data.get("next_page_token"): |
| 98 | + osv_data["page_token"] = next_page_token |
| 99 | + content_queue.put(osv_data) |
| 100 | + |
| 101 | + except Empty: |
| 102 | + if not background_thread.is_alive(): |
| 103 | + raise RuntimeError("Vuln report task thread died unexpectedly.") |
| 104 | + else: |
| 105 | + raise RuntimeError("Background vuln report thread took too long.") |
| 106 | + |
| 107 | + return scanned_packages |
| 108 | + |
| 109 | + |
| 110 | +async def _query_osv_api(session, osv_data): |
| 111 | + """ |
| 112 | + Make a single request to the OSV API. |
| 113 | +
|
| 114 | + Args: |
| 115 | + session (aiohttp.ClientSession): HTTP client session |
| 116 | + osv_data (Dict[str, Any]): OSV query data |
| 117 | +
|
| 118 | + Returns: |
| 119 | + Dict[str, Any]: JSON response from OSV API |
| 120 | + """ |
| 121 | + data = json.dumps(osv_data) |
| 122 | + async with session.post(url=OSV_QUERY_URL, data=data) as response: |
| 123 | + response_body = await response.text() |
| 124 | + return json.loads(response_body) |
| 125 | + |
| 126 | + |
| 127 | +def _get_package_name(osv_data): |
| 128 | + """ |
| 129 | + Extract package name from OSV data. |
| 130 | +
|
| 131 | + Args: |
| 132 | + osv_data (Dict[str, Any]): OSV query data |
| 133 | +
|
| 134 | + Returns: |
| 135 | + str: Formatted package name |
| 136 | + """ |
| 137 | + osv_package_name = osv_data["package"]["name"] |
| 138 | + if osv_package_version := osv_data.get("version", ""): |
| 139 | + return f"{osv_package_name}-{osv_package_version}" |
| 140 | + return f"{osv_package_name}" |
| 141 | + |
| 142 | + |
| 143 | +async def _save_vulnerability_report(scanned_packages): |
| 144 | + """ |
| 145 | + Save vulnerability report to the database. |
| 146 | +
|
| 147 | + Args: |
| 148 | + scanned_packages (Dict[str, Any]): Dictionary mapping package names to vulnerability data |
| 149 | + """ |
| 150 | + vuln_report, created = await sync_to_async(VulnerabilityReport.objects.get_or_create)( |
| 151 | + vulns=scanned_packages, pulp_domain=get_domain() |
| 152 | + ) |
| 153 | + if created: |
| 154 | + await CreatedResource.objects.acreate(content_object=vuln_report) |
0 commit comments