@abstractmethod

      else:
          compatible_resources = (

      resource_json = get_resource_json_obj(

  @patch(
      "gem5.resources.client.clientwrapper",

  @patch(
      "gem5.resources.client.clientwrapper",

  def test_get_resources_json_at_path(self) -> None:

Bobby Bruce has submitted this change. ( https://gem5-review.googlesource.com/c/public/gem5/+/71506?usp=email ) Change subject: stdlib: Refactor gem5 Vision/gem5-resources code ...................................................................... stdlib: Refactor gem5 Vision/gem5-resources code This patch includes several changes to the gem5 tools interface to the gem5-resources infrastructure. These are: * The old download and JSON query functions have been removed from the downloader module. These functions were used for directly downloading and inspecting the resource JSON file, hosted at https://resources.gem5.org/resources. This information is now obtained via `gem5.client`. If a resources JSON file is specified as a client, it should conform to the new schema: https//resources.gem5.org/gem5-resources-schema.json. The old schema (pre-v23.0) is no longer valid. Tests have been updated to reflect this change. Those which tested these old functions have been removed. * Unused imports have been removed. * For the resource query functions, and those tasked with obtaining the resources, the parameter `gem5_version` has been added. In all cases it does the same thing: * It will filter results based on compatibility to the `gem5_version` specified. If no resources are compatible the latest version of that resource is chosen (though a warning is thrown). * By default it is set to the current gem5 version. * It is optional. If `None`, this filtering functionality is not carried out. * Tests have been updated to fix the version to “develop” so the they do not break between versions. * The `gem5_version` parameters will filter using a logic which will base compatibility on the specificity of the gem5-version specified in a resource’s data. If a resource has a compatible gem5-version of “v18.4” it will be compatible with any minor/hotfix version within the v18.4 release (this can be seen as matching on “v18.4.*.*”.) Likewise, if a resource has a compatible gem5-version of “v18.4.1” then it’s only compatible with the v18.4.1 release but any of it’s hot fix releases (“v18.4.1.*”). * The ‘list_resources’ function has been updated to use the “gem5.client” APIs to get resource information from the clients (MongoDB or a JSON file). This has been designed to remain backwards compatible to as much as is possible, though, due to schema changes, the function does search across all versions of gem5. * `get_resources` function was added to the `AbstractClient`. This is a more general function than `get_resource_by_id`. It was primarily created to handle the `list_resources` update but is a useful update to the API. The `get_resource_by_id` function has been altered to function as a wrapped to the `get_resources` function. * Removed “GEM5_RESOURCE_JSON” code has been removed. This is no longer used. * Tests have been cleaned up a little bit to be easier to read. * Some docstrings have been updated. Things that are left TODO with this code: * The client_wrapper/client/abstract_client abstractions are rather pointless. In particular the client_wrapper and client classes could be merged. * The downloader module no longer does much and should have its functions merged into other modules. * With the addition of the `get_resources` function, much of the code in the `AbstractClient` could be simplified. Change-Id: I0ce48e88b93a2b9db53d4749861fa0b5f9472053 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71506 Reviewed-by: Kunal Pai <kunpai@ucdavis.edu> Maintainer: Jason Lowe-Power <power.jg@gmail.com> Tested-by: kokoro <noreply+kokoro@google.com> --- M src/python/gem5/resources/client.py M src/python/gem5/resources/client_api/abstract_client.py M src/python/gem5/resources/client_api/atlasclient.py M src/python/gem5/resources/client_api/client_wrapper.py M src/python/gem5/resources/client_api/jsonclient.py M src/python/gem5/resources/downloader.py M src/python/gem5/resources/resource.py M src/python/gem5/resources/workload.py M tests/gem5/configs/download_check.py M tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py R tests/pyunit/stdlib/resources/pyunit_json_client_checks.py M tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py D tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py M tests/pyunit/stdlib/resources/pyunit_resource_specialization.py M tests/pyunit/stdlib/resources/pyunit_workload_checks.py M tests/pyunit/stdlib/resources/refs/mongo-mock.json M tests/pyunit/stdlib/resources/refs/obtain-resource.json M tests/pyunit/stdlib/resources/refs/resource-specialization.json M tests/pyunit/stdlib/resources/refs/resources.json D tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json M tests/pyunit/stdlib/resources/refs/workload-checks.json 21 files changed, 445 insertions(+), 603 deletions(-) Approvals: Kunal Pai: Looks good to me, approved kokoro: Regressions pass Jason Lowe-Power: Looks good to me, approved diff --git a/src/python/gem5/resources/client.py b/src/python/gem5/resources/client.py index bd473eb..ab8262b 100644 --- a/src/python/gem5/resources/client.py +++ b/src/python/gem5/resources/client.py @@ -31,6 +31,7 @@ from .client_api.client_wrapper import ClientWrapper from gem5.gem5_default_config import config from m5.util import inform +from _m5 import core def getFileContent(file_path: Path) -> Dict: @@ -49,17 +50,7 @@ clientwrapper = None -def get_resource_json_obj( - resource_id, - resource_version: Optional[str] = None, - clients: Optional[List[str]] = None, -) -> Dict: - """ - Get the resource json object from the clients wrapper - :param resource_id: The resource id - :param resource_version: The resource version - :param clients: The list of clients to query - """ +def _get_clientwrapper(): global clientwrapper if clientwrapper is None: # First check if the config file path is provided in the environment variable @@ -78,7 +69,42 @@ gem5_config = config inform("Using default config") clientwrapper = ClientWrapper(gem5_config) + return clientwrapper - return clientwrapper.get_resource_json_obj_from_client( - resource_id, resource_version, clients + +def list_resources( + clients: Optional[List[str]] = None, + gem5_version: Optional[str] = core.gem5Version, +) -> Dict[str, List[str]]: + """ + List all the resources available + + :param clients: The list of clients to query + :param gem5_version: The gem5 version of the resource to get. By default, + it is the gem5 version of the current build. If set to none, it will return + all gem5 versions of the resource. + :return: A Python Dict where the key is the resource id and the value is + a list of all the supported resource versions. + """ + return _get_clientwrapper().list_resources(clients, gem5_version) + + +def get_resource_json_obj( + resource_id, + resource_version: Optional[str] = None, + clients: Optional[List[str]] = None, + gem5_version: Optional[str] = core.gem5Version, +) -> Dict: + """ + Get the resource json object from the clients wrapper + :param resource_id: The resource id + :param resource_version: The resource version + :param clients: The list of clients to query + :param gem5_version: The gem5 versions to filter the resources based on + compatibility. By default, it is the gem5 version of the current build. + If None, filtering based on compatibility is not performed. + """ + + return _get_clientwrapper().get_resource_json_obj_from_client( + resource_id, resource_version, clients, gem5_version ) diff --git a/src/python/gem5/resources/client_api/abstract_client.py b/src/python/gem5/resources/client_api/abstract_client.py index 74a513f..7f8ad61 100644 --- a/src/python/gem5/resources/client_api/abstract_client.py +++ b/src/python/gem5/resources/client_api/abstract_client.py @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from abc import ABC, abstractmethod -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import urllib.parse @@ -63,9 +63,61 @@ return False @abstractmethod + def get_resources( + self, + resource_id: Optional[str] = None, + resource_version: Optional[str] = None, + gem5_version: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """ + :param resource_id: The ID of the Resource. Optional, if not set, all + resources will be returned. + :param resource_version: The version of the Resource. Optional, if + not set, all resource versions will be returned. Note: If `resource_id` + is not set, this parameter will be ignored. + :param gem5_version: The version of gem5. Optional, if not set, all + versions will be returned. + :return: A list of all the Resources with the given ID. + """ + raise NotImplementedError + + def filter_incompatible_resources( + self, + resources_to_filter: List[Dict[str, Any]], + gem5_version: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """Returns a filtered list resources based on gem5 version + compatibility. + + Note: This function assumes if the minor component of + a resource's gem5_version is not specified, the resource is compatible + with all minor versions of the same major version. + Likewise, if no hot-fix component is specified, it is assumed that + the resource is compatible with all hot-fix versions of the same + minor version. + + * '20.1' would be compatible with gem5 '20.1.1.0' and '20.1.2.0'. + * '21.5.2' would be compatible with gem5 '21.5.2.0' and '21.5.2.0'. + * '22.3.2.4' would only be compatible with gem5 '22.3.2.4'. + + :param resources_to_filter: The list of resources to filter. + :param gem5_version: The gem5 version in which the filtered resources + should be compatible. If None, no filtering will be done. + : + """ + if not gem5_version: + return resources_to_filter + + filtered_resources = [] + for resource in resources_to_filter: + for version in resource["gem5_versions"]: + if gem5_version.startswith(version): + filtered_resources.append(resource) + return filtered_resources + def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: """ :param resource_id: The ID of the Resource. :return: A list of all the Resources with the given ID. """ - raise NotImplementedError + return self.get_resources(resource_id=resource_id) diff --git a/src/python/gem5/resources/client_api/atlasclient.py b/src/python/gem5/resources/client_api/atlasclient.py index 4a6e5cf..7d2a27c 100644 --- a/src/python/gem5/resources/client_api/atlasclient.py +++ b/src/python/gem5/resources/client_api/atlasclient.py @@ -64,14 +64,26 @@ token = result["access_token"] return token - def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: + def get_resources( + self, + resource_id: Optional[str] = None, + resource_version: Optional[str] = None, + gem5_version: Optional[str] = None, + ) -> List[Dict[str, Any]]: url = f"{self.url}/action/find" data = { "dataSource": self.dataSource, "collection": self.collection, "database": self.database, - "filter": {"id": resource_id}, } + filter = {} + if resource_id: + filter["id"] = resource_id + if resource_version is not None: + filter["resource_version"] = resource_version + + if filter: + data["filter"] = filter data = json.dumps(data).encode("utf-8") headers = { @@ -88,4 +100,8 @@ result = json.loads(response.read().decode("utf-8")) resources = result["documents"] - return resources + # I do this as a lazy post-processing step because I can't figure out + # how to do this via an Atlas query, which may be more efficient. + return self.filter_incompatible_resources( + resources_to_filter=resources, gem5_version=gem5_version + ) diff --git a/src/python/gem5/resources/client_api/client_wrapper.py b/src/python/gem5/resources/client_api/client_wrapper.py index 74ee831..ebf1b8e 100644 --- a/src/python/gem5/resources/client_api/client_wrapper.py +++ b/src/python/gem5/resources/client_api/client_wrapper.py @@ -59,6 +59,38 @@ warn(f"Error creating client {client}: {str(e)}") return clients + def list_resources( + self, + clients: Optional[List[str]] = None, + gem5_version: Optional[str] = core.gem5Version, + ) -> Dict[str, List[str]]: + + clients_to_search = ( + list(self.clients.keys()) if clients is None else clients + ) + # There's some duplications of functionality here (similar code in + # `get_all_resources_by_id`. This code could be refactored to avoid + # this). + resources = [] + for client in clients_to_search: + if client not in self.clients: + raise Exception(f"Client: {client} does not exist") + try: + resources.extend( + self.clients[client].get_resources( + gem5_version=gem5_version + ) + ) + except Exception as e: + warn(f"Error getting resources from client {client}: {str(e)}") + + to_return = {} + for resource in resources: + if resource["id"] not in to_return: + to_return[resource["id"]] = [] + to_return[resource["id"]].append(resource["resource_version"]) + return to_return + def get_all_resources_by_id( self, resource_id: str, @@ -98,6 +130,7 @@ resource_id: str, resource_version: Optional[str] = None, clients: Optional[List[str]] = None, + gem5_version: Optional[str] = core.gem5Version, ) -> Dict: """ This function returns the resource object from the client with the @@ -106,6 +139,9 @@ :param resource_version: The version of the resource to search for. :param clients: A list of clients to search through. If None, all clients are searched. + :param gem5_version: The gem5 version to check compatibility with. If + None, no compatibility check is performed. By default, is the current + version of gem5. :return: The resource object as a Python dictionary if found. If not found, exception is thrown. """ @@ -124,7 +160,9 @@ else: compatible_resources = ( - self._get_resources_compatible_with_gem5_version(resources) + self._get_resources_compatible_with_gem5_version( + resources, gem5_version=gem5_version + ) ) if len(compatible_resources) == 0: resource_to_return = self._sort_resources(resources)[0] @@ -133,7 +171,10 @@ compatible_resources )[0] - self._check_resource_version_compatibility(resource_to_return) + if gem5_version: + self._check_resource_version_compatibility( + resource_to_return, gem5_version=gem5_version + ) return resource_to_return @@ -172,16 +213,31 @@ ) -> List: """ Returns a list of compatible resources with the current gem5 version. + + Note: This function assumes if the minor component of + a resource's gem5_version is not specified, it that the + resource is compatible all minor versions of the same major version. + Likewise, if no hot-fix component is specified, it is assumed that + the resource is compatible with all hot-fix versions of the same + minor version. + + * '20.1' would be compatible with gem5 '20.1.1.0' and '20.1.2.0'. + * '21.5.2' would be compatible with gem5 '21.5.2.0' and '21.5.2.0'. + * '22.3.2.4' would only be compatible with gem5 '22.3.2.4'. + :param resources: A list of resources to filter. :return: A list of compatible resources as Python dictionaries. - If no compatible resources are found, the original list of resources - is returned. + + **Note**: This is a big duplication of code. This functionality already + exists in the `AbstractClient` class. This code should be refactored + to avoid this duplication. """ - compatible_resources = [ - resource - for resource in resources - if gem5_version in resource["gem5_versions"] - ] + + compatible_resources = [] + for resource in resources: + for version in resource["gem5_versions"]: + if gem5_version.startswith(version): + compatible_resources.append(resource) return compatible_resources def _sort_resources(self, resources: List) -> List: @@ -213,7 +269,12 @@ """ if not resource: return False - if gem5_version not in resource["gem5_versions"]: + if ( + gem5_version + and not self._get_resources_compatible_with_gem5_version( + [resource], gem5_version=gem5_version + ) + ): warn( f"Resource {resource['id']} with version " f"{resource['resource_version']} is not known to be compatible" diff --git a/src/python/gem5/resources/client_api/jsonclient.py b/src/python/gem5/resources/client_api/jsonclient.py index 225126e..9e83713 100644 --- a/src/python/gem5/resources/client_api/jsonclient.py +++ b/src/python/gem5/resources/client_api/jsonclient.py @@ -58,13 +58,31 @@ ) self.resources = json.loads(response.read().decode("utf-8")) - def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: - """ - :param resource_id: The ID of the Resource. - :return: A list of all the Resources with the given ID. - """ - return [ - resource - for resource in self.resources - if resource["id"] == resource_id - ] + def get_resources_json(self) -> List[Dict[str, Any]]: + """Returns a JSON representation of the resources.""" + return self.resources + + def get_resources( + self, + resource_id: Optional[str] = None, + resource_version: Optional[str] = None, + gem5_version: Optional[str] = None, + ) -> List[Dict[str, Any]]: + filter = self.resources # Unfiltered. + if resource_id: + filter = [ # Filter by resource_id. + resource + for resource in filter + if resource["id"] == resource_id + ] + if resource_version: + filter = [ # Filter by resource_version. + resource + for resource in filter + if resource["resource_version"] == resource_version + ] + + # Filter by gem5_version. + return self.filter_incompatible_resources( + resources_to_filter=filter, gem5_version=gem5_version + ) diff --git a/src/python/gem5/resources/downloader.py b/src/python/gem5/resources/downloader.py index 0781d9b..bb5ca84 100644 --- a/src/python/gem5/resources/downloader.py +++ b/src/python/gem5/resources/downloader.py @@ -24,24 +24,24 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import json import urllib.request import urllib.parse -import hashlib import os import shutil import gzip -import hashlib -import base64 import time import random from pathlib import Path import tarfile -from tempfile import gettempdir from urllib.error import HTTPError -from typing import List, Dict, Set, Optional +from typing import List, Optional, Dict -from .client import get_resource_json_obj +from _m5 import core + +from .client import ( + get_resource_json_obj, + list_resources as client_list_resources, +) from .md5_utils import md5_file, md5_dir from ..utils.progress_bar import tqdm, progress_hook @@ -53,188 +53,6 @@ """ -def _resources_json_version_required() -> str: - """ - Specifies the version of resources.json to obtain. - """ - return "develop" - - -def _get_resources_json_uri() -> str: - return "https://resources.gem5.org/resources.json" - - -def _url_validator(url): - try: - result = urllib.parse.urlparse(url) - return all([result.scheme, result.netloc, result.path]) - except: - return False - - -def _get_resources_json_at_path(path: str, use_caching: bool = True) -> Dict: - """ - Returns a resource JSON, in the form of a Python Dict. The location - of the JSON must be specified. - - If `use_caching` is True, and a URL is passed, a copy of the JSON will be - cached locally, and used for up to an hour after retrieval. - - :param path: The URL or local path of the JSON file. - :param use_caching: True if a cached file is to be used (up to an hour), - otherwise the file will be retrieved from the URL regardless. True by - default. Only valid in cases where a URL is passed. - """ - - # If a local valid path is passed, just load it. - if Path(path).is_file(): - return json.load(open(path)) - - # If it's not a local path, it should be a URL. We check this here and - # raise an Exception if it's not. - if not _url_validator(path): - raise Exception( - f"Resources location '{path}' is not a valid path or URL." - ) - - download_path = os.path.join( - gettempdir(), - f"gem5-resources-{hashlib.md5(path.encode()).hexdigest()}" - f"-{str(os.getuid())}.json", - ) - - # We apply a lock on the resources file for when it's downloaded, or - # re-downloaded, and read. This stops a corner-case from occuring where - # the file is re-downloaded while being read by another gem5 thread. - # Note the timeout is 120 so the `_download` function is given time to run - # its Truncated Exponential Backoff algorithm - # (maximum of roughly 1 minute). Typically this code will run quickly. - with FileLock(f"{download_path}.lock", timeout=120): - - # The resources.json file can change at any time, but to avoid - # excessive retrieval we cache a version locally and use it for up to - # an hour before obtaining a fresh copy. - # - # `time.time()` and `os.path.getmtime(..)` both return an unix epoch - # time in seconds. Therefore, the value of "3600" here represents an - # hour difference between the two values. `time.time()` gets the - # current time, and `os.path.getmtime(<file>)` gets the modification - # time of the file. This is the most portable solution as other ideas, - # like "file creation time", are not always the same concept between - # operating systems. - if ( - not use_caching - or not os.path.exists(download_path) - or (time.time() - os.path.getmtime(download_path)) > 3600 - ): - _download(path, download_path) - - with open(download_path) as f: - file_contents = f.read() - - try: - to_return = json.loads(file_contents) - except json.JSONDecodeError: - # This is a bit of a hack. If the URL specified exists in a Google - # Source repo (which is the case when on the gem5 develop branch) we - # retrieve the JSON in base64 format. This cannot be loaded directly as - # text. Conversion is therefore needed. - to_return = json.loads(base64.b64decode(file_contents).decode("utf-8")) - - return to_return - - -def _get_resources_json() -> Dict: - """ - Gets the Resources JSON. - - :returns: The Resources JSON (as a Python Dictionary). - """ - - path = os.getenv("GEM5_RESOURCE_JSON", _get_resources_json_uri()) - to_return = _get_resources_json_at_path(path=path) - - # If the current version pulled is not correct, look up the - # "previous-versions" field to find the correct one. - # If the resource JSON file does not have a "version" field or it's - # null/None, then we will use this resource JSON file (this is usefull for - # testing purposes). - version = _resources_json_version_required() - json_version = None if "version" not in to_return else to_return["version"] - - if json_version and json_version != version: - if version in to_return["previous-versions"].keys(): - to_return = _get_resources_json_at_path( - path=to_return["previous-versions"][version] - ) - else: - # This should never happen, but we thrown an exception to explain - # that we can't find the version. - raise Exception( - f"Version '{version}' of resources.json cannot be found." - ) - - return to_return - - -def _get_url_base() -> str: - """ - Obtains the "url_base" string from the resources.json file. - - :returns: The "url_base" string value from the resources.json file. - """ - json = _get_resources_json() - if "url_base" in json.keys(): - return json["url_base"] - return "" - - -def _get_resources( - valid_types: Set[str], resources_group: Optional[Dict] = None -) -> Dict[str, Dict]: - """ - A recursive function to get all the workload/resource of the specified type - in the resources.json file. - - :param valid_types: The type to return (i.e., "resource" or "workload). - :param resource_group: Used for recursion: The current resource group being - iterated through. - - :returns: A dictionary of artifact names to the resource JSON objects. - """ - - if resources_group is None: - resources_group = _get_resources_json()["resources"] - - to_return = {} - for resource in resources_group: - if resource["type"] in valid_types: - # If the type is valid then we add it directly to the map - # after a check that the name is unique. - if resource["name"] in to_return.keys(): - raise Exception( - f"Error: Duplicate resource with name '{resource['name']}'." - ) - to_return[resource["name"]] = resource - elif resource["type"] == "group": - # If it's a group we get recursive. We then check to see if there - # are any duplication of keys. - new_map = _get_resources( - valid_types=valid_types, resources_group=resource["contents"] - ) - intersection = set(new_map.keys()).intersection(to_return.keys()) - if len(intersection) > 0: - # Note: if this error is received it's likely an error with - # the resources.json file. The resources names need to be - # unique keyes. - raise Exception( - f"Error: Duplicate resources with names: {str(intersection)}." - ) - to_return.update(new_map) - - return to_return - - def _download(url: str, download_to: str, max_attempts: int = 6) -> None: """ Downloads a file. @@ -336,61 +154,26 @@ ) -def list_resources() -> List[str]: +def list_resources( + clients: Optional[List] = None, gem5_version: Optional[str] = None +) -> Dict[str, List[str]]: """ - Lists all available resources by name. + Lists all available resources. Returns a dictionary where the key is the + id of the resources and the value is a list of that resource's versions. - :returns: A list of resources by name. + :param clients: A list of clients to use when listing resources. If None, + all clients will be used. None by default. + + :param gem5_version: The gem5 version to which all resources should be + compatible with. If None, compatibility of resources is not considered and + all resources will be returned. + + **Note**: This function is here for legacy reasons. The `list_resources` + function was originally stored here. In order to remain backwards + compatible, this function will call the `client_list_resources` function + """ - from .resource import _get_resource_json_type_map - - return _get_resources( - valid_types=_get_resource_json_type_map.keys() - ).keys() - - -def get_workload_json_obj(workload_name: str) -> Dict: - """ - Get a JSON object of a specified workload. - - :param workload_name: The name of the workload. - - :raises Exception: An exception is raised if the specified workload does - not exit. - """ - workload_map = _get_resources(valid_types={"workload"}) - - if workload_name not in workload_map: - raise Exception( - f"Error: Workload with name {workload_name} does not exist" - ) - - return workload_map[workload_name] - - -def get_resources_json_obj(resource_name: str) -> Dict: - """ - Get a JSON object of a specified resource. - - :param resource_name: The name of the resource. - - :returns: The JSON object (in the form of a dictionary). - - :raises Exception: An exception is raised if the specified resources does - not exist. - """ - from .resource import _get_resource_json_type_map - - resource_map = _get_resources( - valid_types=_get_resource_json_type_map.keys() - ) - - if resource_name not in resource_map: - raise Exception( - f"Error: Resource with name '{resource_name}' does not exist" - ) - - return resource_map[resource_name] + return client_list_resources(clients=clients, gem5_version=gem5_version) def get_resource( @@ -401,6 +184,7 @@ download_md5_mismatch: bool = True, resource_version: Optional[str] = None, clients: Optional[List] = None, + gem5_version: Optional[str] = core.gem5Version, ) -> None: """ Obtains a gem5 resource and stored it to a specified location. If the @@ -429,6 +213,10 @@ :param clients: A list of clients to use when obtaining the resource. If None, all clients will be used. None by default. + :param gem5_version: The gem5 version to use when obtaining the resource. + By default, the version of gem5 being used is used. This is used primarily + for testing purposes. + :raises Exception: An exception is thrown if a file is already present at `to_path` but it does not have the correct md5 sum. An exception will also be thrown is a directory is present at `to_path` @@ -444,6 +232,7 @@ resource_name, resource_version=resource_version, clients=clients, + gem5_version=gem5_version, ) if os.path.exists(to_path): diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index 22adf15..bc9f448 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -28,6 +28,7 @@ import os from pathlib import Path from m5.util import warn, fatal +from _m5 import core from .downloader import get_resource @@ -559,17 +560,15 @@ download_md5_mismatch: bool = True, resource_version: Optional[str] = None, clients: Optional[List] = None, + gem5_version=core.gem5Version, ) -> AbstractResource: """ This function primarily serves as a factory for resources. It will return the correct `AbstractResource` implementation based on the resource - requested, by referencing the "resource.json" file (by default, that hosted - at https://resources.gem5.org/resources.json). In addition to this, this - function will download the resource if not detected in the - `resource_directory`. + requested. :param resource_name: The name of the gem5 resource as it appears under the - "name" field in the `resource.json` file. + "id" field in the `resource.json` file. :param resource_directory: The location of the directory in which the resource is to be stored. If this parameter is not set, it will set to the environment variable `GEM5_RESOURCE_DIR`. If the environment is not @@ -582,11 +581,17 @@ Not a required parameter. None by default. :param clients: A list of clients to search for the resource. If this parameter is not set, it will default search all clients. + :param gem5_version: The gem5 version to use to filter incompatible + resource versions. By default set to the current gem5 version. If None, + this filtering is not performed. """ # Obtain the resource object entry for this resource resource_json = get_resource_json_obj( - resource_id, resource_version=resource_version, clients=clients + resource_id, + resource_version=resource_version, + clients=clients, + gem5_version=gem5_version, ) to_path = None @@ -629,6 +634,7 @@ download_md5_mismatch=download_md5_mismatch, resource_version=resource_version, clients=clients, + gem5_version=gem5_version, ) # Obtain the type from the JSON. From this we will determine what subclass diff --git a/src/python/gem5/resources/workload.py b/src/python/gem5/resources/workload.py index 148ab3f..0798b89 100644 --- a/src/python/gem5/resources/workload.py +++ b/src/python/gem5/resources/workload.py @@ -27,6 +27,8 @@ from .resource import obtain_resource from .client import get_resource_json_obj +from _m5 import core + from typing import Dict, Any, List, Optional @@ -160,6 +162,7 @@ resource_directory: Optional[str] = None, resource_version: Optional[str] = None, clients: Optional[List] = None, + gem5_version: Optional[str] = core.gem5Version, ) -> None: """ This constructor will load the workload details from the workload with @@ -201,12 +204,17 @@ :param resource_directory: An optional parameter that specifies where any resources should be download and accessed from. If None, a default location will be used. None by default. + :param gem5_version: The gem5 version for the Workload to be loaded. + By default, the current gem5 version is used. This will filter + resources which are incompatible with the current gem5 version. If + None, no filtering will be done. """ workload_json = get_resource_json_obj( workload_name, resource_version=resource_version, clients=clients, + gem5_version=gem5_version, ) func = workload_json["function"] @@ -219,7 +227,9 @@ value = workload_json["resources"][key] assert isinstance(value, str) params[key] = obtain_resource( - value, resource_directory=resource_directory + value, + resource_directory=resource_directory, + gem5_version=gem5_version, ) if "additional_params" in workload_json: diff --git a/tests/gem5/configs/download_check.py b/tests/gem5/configs/download_check.py index decc62c..2180f4f 100644 --- a/tests/gem5/configs/download_check.py +++ b/tests/gem5/configs/download_check.py @@ -26,10 +26,11 @@ from gem5.resources.downloader import ( list_resources, - get_resources_json_obj, get_resource, ) +from gem5.resources.client import get_resource_json_obj + from gem5.resources.md5_utils import md5 import os @@ -52,6 +53,15 @@ ) parser.add_argument( + "--gem5-version", + type=str, + required=False, + help="The gem5 version to check the resources against. Resources not " + "compatible with this version will be ignored. If not set, no " + "compatibility tests are performed.", +) + +parser.add_argument( "--download-directory", type=str, required=True, @@ -67,39 +77,59 @@ ids = args.ids +resource_list = list_resources(gem5_version=args.gem5_version) if len(ids) == 0: - ids = list_resources() + ids = resource_list # We log all the errors as they occur then dump them at the end. This means we # can be aware of all download errors in a single failure. errors = str() for id in ids: - if id not in list_resources(): + if id not in resource_list: errors += ( f"Resource with ID '{id}' not found in " + f"`list_resources()`.{os.linesep}" ) continue - resource_json = get_resources_json_obj(id) - download_path = os.path.join(args.download_directory, id) - try: - get_resource(resource_name=id, to_path=download_path) - except Exception as e: - errors += f"Failure to download resource '{id}'.{os.linesep}" - errors += f"Exception message:{os.linesep}{str(e)}" - errors += f"{os.linesep}{os.linesep}" - continue + for resource_version in ids[id]: - if md5(Path(download_path)) != resource_json["md5sum"]: - errors += ( - f"Downloaded resource '{id}' md5 " - + f"({md5(Path(download_path))}) differs to that in the " - + f"JSON ({resource_json['md5sum']}).{os.linesep}" + resource_json = get_resource_json_obj( + resource_id=id, + resource_version=resource_version, + gem5_version=args.gem5_version, ) + if resource_json["category"] == "workload": + # Workloads are not downloaded as part of this test. + continue + download_path = os.path.join( + args.download_directory, f"{id}-v{resource_version}" + ) + try: + get_resource( + resource_name=id, + resource_version=resource_version, + gem5_version=args.gem5_version, + to_path=download_path, + ) + except Exception as e: + errors += ( + f"Failure to download resource '{id}', " + + f"v{resource_version}.{os.linesep}" + ) + errors += f"Exception message:{os.linesep}{str(e)}" + errors += f"{os.linesep}{os.linesep}" + continue - # Remove the downloaded resource. + if md5(Path(download_path)) != resource_json["md5sum"]: + errors += ( + f"Downloaded resource '{id}' md5 " + + f"({md5(Path(download_path))}) differs to that recorded in " + + f" gem5-resources ({resource_json['md5sum']}).{os.linesep}" + ) + + # Remove the downloaded resource. shutil.rmtree(download_path, ignore_errors=True) # If errors exist, raise an exception highlighting them. diff --git a/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py b/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py index 344f67b..f190b1e 100644 --- a/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py @@ -25,13 +25,9 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest -from gem5.isas import ISA from gem5.resources.client import get_resource_json_obj -import gem5.resources.client from gem5.resources.client_api.client_wrapper import ClientWrapper -from typing import Dict from unittest.mock import patch -from unittest import mock import json from urllib.error import HTTPError import io @@ -62,23 +58,8 @@ }, } -mock_config_combined = { - "sources": { - "gem5-resources": { - "dataSource": "gem5-vision", - "database": "gem5-vision", - "collection": "versions_test", - "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", - "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", - "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", - "isMongo": True, - }, - "baba": { - "url": mock_json_path, - "isMongo": False, - }, - }, -} +mock_config_combined = mock_config_mongo +mock_config_combined["sources"]["baba"] = mock_config_json["sources"]["baba"] mock_json = {} @@ -145,12 +126,12 @@ def test_get_resource_json_obj(self): # Test that the resource object is correctly returned resource = "this-is-a-test-resource" - resource = get_resource_json_obj(resource) + resource = get_resource_json_obj(resource, gem5_version="develop") self.assertEqual(resource["id"], "this-is-a-test-resource") - self.assertEqual(resource["resource_version"], "2.0.0") + self.assertEqual(resource["resource_version"], "1.1.0") self.assertEqual(resource["category"], "binary") self.assertEqual( - resource["description"], "This is a test resource but double newer" + resource["description"], "This is a test resource but newer" ) self.assertEqual( resource["source_url"], @@ -167,7 +148,9 @@ resource_id = "test-id" client = "invalid" with self.assertRaises(Exception) as context: - get_resource_json_obj(resource_id, clients=[client]) + get_resource_json_obj( + resource_id, clients=[client], gem5_version="develop" + ) self.assertTrue( f"Client: {client} does not exist" in str(context.exception) ) @@ -181,7 +164,9 @@ resource_id = "this-is-a-test-resource" resource_version = "1.0.0" resource = get_resource_json_obj( - resource_id, resource_version=resource_version + resource_id, + resource_version=resource_version, + gem5_version="develop", ) self.assertEqual(resource["id"], "this-is-a-test-resource") self.assertEqual(resource["resource_version"], "1.0.0") @@ -200,17 +185,18 @@ @patch("urllib.request.urlopen", side_effect=mocked_requests_post) def test_get_resource_json_obj_1(self, mock_get): resource = "x86-ubuntu-18.04-img" - resource = get_resource_json_obj(resource) + resource = get_resource_json_obj(resource, gem5_version="develop") self.assertEqual(resource["id"], "x86-ubuntu-18.04-img") - self.assertEqual(resource["resource_version"], "1.1.0") + self.assertEqual(resource["resource_version"], "2.0.0") self.assertEqual(resource["category"], "disk-image") self.assertEqual( resource["description"], - "A disk image containing Ubuntu 18.04 for x86. This image will run an `m5 readfile` instruction after booting. If no script file is specified an `m5 exit` instruction will be executed.", + "This is a test resource", ) self.assertEqual( resource["source_url"], - "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "https://github.com/gem5/gem5-resources/tree/develop/" + "src/x86-ubuntu", ) self.assertEqual(resource["architecture"], "X86") @@ -227,6 +213,7 @@ resource_id, resource_version=resource_version, clients=["gem5-resources"], + gem5_version="develop", ) self.assertEqual(resource["id"], "x86-ubuntu-18.04-img") self.assertEqual(resource["resource_version"], "1.0.0") @@ -246,7 +233,9 @@ def test_get_resource_json_obj_with_id_invalid_mongodb(self, mock_get): resource_id = "invalid-id" with self.assertRaises(Exception) as context: - get_resource_json_obj(resource_id, clients=["gem5-resources"]) + get_resource_json_obj( + resource_id, clients=["gem5-resources"], gem5_version="develop" + ) self.assertTrue( "Resource with ID 'invalid-id' not found." in str(context.exception) @@ -267,12 +256,13 @@ resource_id, resource_version=resource_version, clients=["gem5-resources"], + gem5_version="develop", ) self.assertTrue( f"Resource x86-ubuntu-18.04-img with version '2.5.0'" " not found.\nResource versions can be found at: " - f"https://resources.gem5.org/resources/x86-ubuntu-18.04-img/versions" - in str(context.exception) + "https://resources.gem5.org/resources/x86-ubuntu-18.04-img/" + "versions" in str(context.exception) ) @patch( @@ -286,12 +276,13 @@ get_resource_json_obj( resource_id, resource_version=resource_version, + gem5_version="develop", ) self.assertTrue( - f"Resource this-is-a-test-resource with version '2.5.0'" + "Resource this-is-a-test-resource with version '2.5.0'" " not found.\nResource versions can be found at: " - f"https://resources.gem5.org/resources/this-is-a-test-resource/versions" - in str(context.exception) + "https://resources.gem5.org/resources/this-is-a-test-resource/" + "versions" in str(context.exception) ) @patch( @@ -308,11 +299,13 @@ resource_id_mongo, resource_version=resource_version_mongo, clients=["gem5-resources"], + gem5_version="develop", ) resource_json = get_resource_json_obj( resource_id_json, resource_version=resource_version_json, clients=["baba"], + gem5_version="develop", ) self.assertEqual(resource_mongo["id"], "x86-ubuntu-18.04-img") self.assertEqual(resource_mongo["resource_version"], "1.0.0") @@ -322,7 +315,8 @@ ) self.assertEqual( resource_mongo["source_url"], - "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "https://github.com/gem5/gem5-resources/tree/develop/src/" + "x86-ubuntu", ) self.assertEqual(resource_mongo["architecture"], "X86") @@ -347,6 +341,7 @@ resource_id = "simpoint-resource" resource = get_resource_json_obj( resource_id, + gem5_version="develop", ) self.assertEqual(resource["id"], resource_id) self.assertEqual(resource["resource_version"], "0.2.0") @@ -371,6 +366,7 @@ resource_id = "x86-ubuntu-18.04-img" resource_json = get_resource_json_obj( resource_id, + gem5_version="develop", ) self.assertEqual(resource_json["id"], "x86-ubuntu-18.04-img") @@ -378,8 +374,7 @@ self.assertEqual(resource_json["category"], "disk-image") resource_json = get_resource_json_obj( - resource_id, - resource_version="1.0.0", + resource_id, resource_version="1.0.0", gem5_version="develop" ) self.assertEqual(resource_json["id"], "x86-ubuntu-18.04-img") @@ -396,6 +391,7 @@ with self.assertRaises(Exception) as context: get_resource_json_obj( resource_id, + gem5_version="develop", ) self.assertTrue( f"Resource {resource_id} has multiple resources with" @@ -428,6 +424,7 @@ with contextlib.redirect_stderr(f): get_resource_json_obj( resource_id, + gem5_version="develop", ) self.assertTrue( "Error getting resources from client gem5-resources:" @@ -440,21 +437,7 @@ @patch( "gem5.resources.client.clientwrapper", - ClientWrapper( - { - "sources": { - "gem5-resources": { - "dataSource": "gem5-vision", - "database": "gem5-vision", - "collection": "versions_test", - "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v", - "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", - "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", - "isMongo": True, - } - }, - } - ), + ClientWrapper(mock_config_mongo), ) @patch("urllib.request.urlopen", side_effect=mocked_requests_post) def test_invalid_url(self, mock_get): @@ -464,6 +447,7 @@ with contextlib.redirect_stderr(f): get_resource_json_obj( resource_id, + gem5_version="develop", ) self.assertTrue( "Error getting resources from client gem5-resources:" @@ -476,21 +460,7 @@ @patch( "gem5.resources.client.clientwrapper", - ClientWrapper( - { - "sources": { - "gem5-resources": { - "dataSource": "gem5-vision", - "database": "gem5-vision", - "collection": "versions_test", - "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", - "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", - "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", - "isMongo": True, - } - }, - } - ), + ClientWrapper(mock_config_mongo), ) @patch("urllib.request.urlopen", side_effect=mocked_requests_post) def test_invalid_url(self, mock_get): @@ -500,6 +470,7 @@ with contextlib.redirect_stderr(f): get_resource_json_obj( resource_id, + gem5_version="develop", ) self.assertTrue( "Error getting resources from client gem5-resources:" diff --git a/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py b/tests/pyunit/stdlib/resources/pyunit_json_client_checks.py similarity index 87% rename from tests/pyunit/stdlib/resources/pyunit_downloader_checks.py rename to tests/pyunit/stdlib/resources/pyunit_json_client_checks.py index 19169e4..88db3d4 100644 --- a/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_json_client_checks.py @@ -30,15 +30,11 @@ from typing import Dict import json -from gem5.resources.downloader import ( - _get_resources_json_at_path, - _get_resources_json, - _resources_json_version_required, -) +from gem5.resources.client_api.jsonclient import JSONClient -class ResourceDownloaderTestSuite(unittest.TestCase): - """Test cases for gem5.resources.downloader""" +class JSONClientTestSuite(unittest.TestCase): + """Test cases for gem5.resources.client_api.jsonclient""" @classmethod def setUpClass(cls) -> str: @@ -142,12 +138,9 @@ file.close() cls.file_path = file.name - os.environ["GEM5_RESOURCE_JSON"] = cls.file_path - @classmethod def tearDownClass(cls) -> None: os.remove(cls.file_path) - del os.environ["GEM5_RESOURCE_JSON"] def verify_json(self, json: Dict) -> None: """ @@ -167,32 +160,22 @@ self.assertEquals("test-version", json[3]["id"]) def test_get_resources_json_at_path(self) -> None: - # Tests the gem5.resources.downloader._get_resources_json_at_path() - # function. + # Tests JSONClient.get_resources_json() - json = _get_resources_json_at_path(path=self.file_path) - self.verify_json(json=json) - - def test_get_resources_json(self) -> None: - # Tests the gem5.resources.downloader._get_resources_json() function. - - json = _get_resources_json() + client = JSONClient(path=self.file_path) + json = client.get_resources_json() self.verify_json(json=json) def test_get_resources_json_invalid_url(self) -> None: - # Tests the gem5.resources.downloader._get_resources_json() function in - # case where an invalid url is passed as the URL/PATH of the - # resources.json file. + # Tests the JSONClient.get_resources_json() function in case where an + # invalid url is passed as the URL/PATH of the resources JSON file. path = "NotAURLorFilePath" - os.environ["GEM5_RESOURCE_JSON"] = path with self.assertRaises(Exception) as context: - _get_resources_json() + client = JSONClient(path=path) + json = client.get_resources_json() self.assertTrue( f"Resources location '{path}' is not a valid path or URL." in str(context.exception) ) - - # Set back to the old path - os.environ["GEM5_RESOURCE_JSON"] = self.file_path diff --git a/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py b/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py index 791d96c..b1eda4e 100644 --- a/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py +++ b/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py @@ -30,12 +30,7 @@ import contextlib from pathlib import Path -from gem5.resources.resource import * - -from gem5.resources.looppoint import ( - LooppointCsvLoader, - LooppointJsonLoader, -) +from gem5.resources.resource import obtain_resource, BinaryResource from gem5.isas import ISA @@ -61,24 +56,6 @@ new=ClientWrapper(mock_config_json), ) class TestObtainResourcesCheck(unittest.TestCase): - @classmethod - def setUpClass(cls): - """Prior to running the suite we set the resource directory to - "ref/resource-specialization.json" - """ - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( - os.path.realpath(os.path.dirname(__file__)), - "refs", - "obtain-resource.json", - ) - - @classmethod - def tearDownClass(cls) -> None: - """After running the suite we unset the gem5-resource JSON file, as to - not interfere with others tests. - """ - del os.environ["GEM5_RESOURCE_JSON"] - def get_resource_dir(cls) -> str: """To ensure the resources are cached to the same directory as all other tests, this function returns the location of the testing @@ -99,26 +76,27 @@ resource = obtain_resource( resource_id="test-binary-resource", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) - self.assertEquals("2.5.0", resource.get_resource_version()) + self.assertEquals("1.7.0", resource.get_resource_version()) self.assertIsInstance(resource, BinaryResource) - # self.assertIn(gem5Version, resource.get_gem5_versions()) - self.assertEquals("test description", resource.get_description()) + self.assertEquals( + "test description v1.7.0", resource.get_description() + ) self.assertEquals("src/test-source", resource.get_source()) self.assertEquals(ISA.ARM, resource.get_architecture()) def test_obtain_resources_with_version_compatible(self): - gem5Version = core.gem5Version resource = obtain_resource( resource_id="test-binary-resource", resource_directory=self.get_resource_dir(), - resource_version="1.7.0", + resource_version="1.5.0", + gem5_version="develop", ) - self.assertEquals("1.7.0", resource.get_resource_version()) + self.assertEquals("1.5.0", resource.get_resource_version()) self.assertIsInstance(resource, BinaryResource) - # self.assertIn(gem5Version, resource.get_gem5_versions()) self.assertEquals( - "test description v1.7.0", resource.get_description() + "test description for 1.5.0", resource.get_description() ) self.assertEquals("src/test-source", resource.get_source()) self.assertEquals(ISA.ARM, resource.get_architecture()) @@ -143,6 +121,7 @@ resource_id="test-binary-resource", resource_directory=self.get_resource_dir(), resource_version="1.5.0", + gem5_version="develop", ) self.assertEquals("1.5.0", resource.get_resource_version()) self.assertIsInstance(resource, BinaryResource) @@ -157,6 +136,7 @@ obtain_resource( resource_id="invalid-id", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertTrue( "Resource with ID 'invalid-id' not found." @@ -169,6 +149,7 @@ resource_id="invalid-id", resource_directory=self.get_resource_dir(), resource_version="1.7.0", + gem5_version="develop", ) self.assertTrue( "Resource with ID 'invalid-id' not found." @@ -182,8 +163,6 @@ resource_directory=self.get_resource_dir(), resource_version="3.0.0", ) - print("context.exception: ", context.exception) - print(str(context.exception)) self.assertTrue( f"Resource test-binary-resource with version '3.0.0'" " not found.\nResource versions can be found at: " diff --git a/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py b/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py deleted file mode 100644 index 8f6674f..0000000 --- a/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright (c) 2023 The Regents of the University of California -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import unittest -import tempfile -import os -from typing import Dict - -from gem5.resources.downloader import ( - get_resources_json_obj, -) - - -class ResourceDownloadTestSuite(unittest.TestCase): - """Test cases for gem5.resources.downloader""" - - @classmethod - def setUpClass(cls) -> str: - pass - - def get_resource_json_by_id(self) -> None: - """Get a resource by its id""" - resources = get_resources_json_obj("test-version") - self.assertEqual(resources["id"], "test-version") - self.assertEqual(resources["resource_version"], "2.0.0") - - def get_resource_json_invalid_id(self) -> None: - """Should throw an exception when trying to get a resource that doesn't exist""" - with self.assertRaises(Exception) as context: - get_resources_json_obj("this-resource-doesnt-exist") - self.assertTrue( - f"Error: Resource with name 'this-resource-doesnt-exist' does not exist" - in str(context.exception) - ) - - def get_resource_json_by_id_and_version(self) -> None: - """Get a resource by its id and version""" - resources = get_resources_json_obj("test-version", "1.0.0") - self.assertEqual(resources["id"], "test-version") - self.assertEqual(resources["resource_version"], "1.0.0") - - def get_resource_json_by_id_and_invalid_version(self) -> None: - """Get a resource by its id and an invalid version (does not exist)""" - with self.assertRaises(Exception) as context: - get_resources_json_obj("test-version", "3.0.0") - self.assertTrue( - f"Specified Version 3.0.0 does not exist for the resource 'test-version'." - in str(context.exception) - ) diff --git a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py index 5c22a73..f2088db 100644 --- a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py +++ b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py @@ -62,24 +62,6 @@ function. """ - @classmethod - def setUpClass(cls): - """Prior to running the suite we set the resource directory to - "ref/resource-specialization.json" - """ - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( - os.path.realpath(os.path.dirname(__file__)), - "refs", - "resource-specialization.json", - ) - - @classmethod - def tearDownClass(cls) -> None: - """After running the suite we unset the gem5-resource JSON file, as to - not interfere with others tests. - """ - del os.environ["GEM5_RESOURCE_JSON"] - def get_resource_dir(cls) -> str: """To ensure the resources are cached to the same directory as all other tests, this function returns the location of the testing @@ -99,6 +81,7 @@ resource = obtain_resource( resource_id="binary-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, BinaryResource) @@ -114,6 +97,7 @@ resource = obtain_resource( resource_id="kernel-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, KernelResource) @@ -129,6 +113,7 @@ resource = obtain_resource( resource_id="bootloader-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, BootloaderResource) @@ -144,6 +129,7 @@ resource = obtain_resource( resource_id="disk-image-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, DiskImageResource) @@ -159,6 +145,7 @@ resource = obtain_resource( resource_id="checkpoint-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, CheckpointResource) @@ -173,6 +160,7 @@ resource = obtain_resource( resource_id="git-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, GitResource) @@ -185,6 +173,7 @@ resource = obtain_resource( resource_id="simpoint-directory-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, SimpointDirectoryResource) @@ -219,6 +208,7 @@ resource = obtain_resource( resource_id="simpoint-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, SimpointResource) @@ -240,6 +230,7 @@ resource_id="file-example", resource_directory=self.get_resource_dir(), resource_version="1.0.0", + gem5_version="develop", ) self.assertIsInstance(resource, FileResource) @@ -268,6 +259,7 @@ resource = obtain_resource( resource_id="looppoint-pinpoint-csv-resource", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, LooppointCsvResource) @@ -289,6 +281,7 @@ resource_id="looppoint-json-restore-resource-region-1", resource_directory=self.get_resource_dir(), resource_version="1.0.0", + gem5_version="develop", ) self.assertIsInstance(resource, LooppointJsonResource) diff --git a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py index b898fae..b59e09d 100644 --- a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py @@ -40,17 +40,7 @@ from unittest.mock import patch from pathlib import Path -mock_config_json1 = { - "sources": { - "baba": { - "url": Path(__file__).parent - / "refs/workload-checks-custom-workload.json", - "isMongo": False, - } - }, -} - -mock_config_json2 = { +mock_config_json = { "sources": { "baba": { "url": Path(__file__).parent / "refs/workload-checks.json", @@ -68,29 +58,19 @@ @classmethod @patch( "gem5.resources.client.clientwrapper", - new=ClientWrapper(mock_config_json1), + new=ClientWrapper(mock_config_json), ) def setUpClass(cls) -> None: - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( - os.path.realpath(os.path.dirname(__file__)), - "refs", - "workload-checks-custom-workload.json", - ) - cls.custom_workload = CustomWorkload( function="set_se_binary_workload", parameters={ - "binary": obtain_resource("x86-hello64-static"), + "binary": obtain_resource( + "x86-hello64-static", gem5_version="develop" + ), "arguments": ["hello", 6], }, ) - @classmethod - def tearDownClass(cls): - # Unset the environment variable so this test does not interfere with - # others. - os.environ["GEM5_RESOURCE_JSON"] - def test_get_function_str(self) -> None: # Tests `CustomResource.get_function_str` @@ -140,7 +120,8 @@ "test", self.custom_workload.get_parameters()["binary"] ) - # We set the overridden parameter back to it's old valu self.custom_workload.set_parameter("binary", old_value) + # We set the overridden parameter back to it's old value + self.custom_workload.set_parameter("binary", old_value) class WorkloadTestSuite(unittest.TestCase): @@ -151,21 +132,10 @@ @classmethod @patch( "gem5.resources.client.clientwrapper", - ClientWrapper(mock_config_json2), + ClientWrapper(mock_config_json), ) def setUpClass(cls): - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( - os.path.realpath(os.path.dirname(__file__)), - "refs", - "workload-checks.json", - ) - cls.workload = Workload("simple-boot") - - @classmethod - def tearDownClass(cls): - # Unset the environment variable so this test does not interfere with - # others. - os.environ["GEM5_RESOURCE_JSON"] + cls.workload = Workload("simple-boot", gem5_version="develop") def test_get_function_str(self) -> None: # Tests `Resource.get_function_str` diff --git a/tests/pyunit/stdlib/resources/refs/mongo-mock.json b/tests/pyunit/stdlib/resources/refs/mongo-mock.json index b6376cc..e2fb058 100644 --- a/tests/pyunit/stdlib/resources/refs/mongo-mock.json +++ b/tests/pyunit/stdlib/resources/refs/mongo-mock.json @@ -22,7 +22,7 @@ "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", "resource_version": "1.0.0", "gem5_versions": [ - "23.0" + "develop" ], "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" }, @@ -49,7 +49,7 @@ "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", "resource_version": "1.1.0", "gem5_versions": [ - "23.0" + "develop" ], "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" } diff --git a/tests/pyunit/stdlib/resources/refs/obtain-resource.json b/tests/pyunit/stdlib/resources/refs/obtain-resource.json index fac95e1..9125bf4 100644 --- a/tests/pyunit/stdlib/resources/refs/obtain-resource.json +++ b/tests/pyunit/stdlib/resources/refs/obtain-resource.json @@ -24,7 +24,7 @@ "source": "src/test-source", "resource_version": "2.0.0", "gem5_versions": [ - "develop" + "23.0" ] }, { @@ -38,7 +38,8 @@ "source": "src/test-source", "resource_version": "1.7.0", "gem5_versions": [ - "develop" + "develop", + "develop-2" ] }, { @@ -52,8 +53,7 @@ "source": "src/test-source", "resource_version": "1.5.0", "gem5_versions": [ - "21.1", - "22.1" + "develop" ] } ] diff --git a/tests/pyunit/stdlib/resources/refs/resource-specialization.json b/tests/pyunit/stdlib/resources/refs/resource-specialization.json index 1129f1b..414bf73 100644 --- a/tests/pyunit/stdlib/resources/refs/resource-specialization.json +++ b/tests/pyunit/stdlib/resources/refs/resource-specialization.json @@ -10,6 +10,7 @@ "source": "src/linux-kernel", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -25,6 +26,7 @@ "root_partition": "1", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -39,6 +41,7 @@ "source": "src/simple", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -51,6 +54,7 @@ "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -66,6 +70,7 @@ "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -75,10 +80,11 @@ "description": null, "is_zipped": false, "is_tar_archive": true, - "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -87,11 +93,12 @@ "id": "file-example", "description": null, "is_zipped": false, - "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "md5sum": "2efd144c11829ab18d54eae6371e120a", "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", "source": null, "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -106,6 +113,7 @@ "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -125,6 +133,7 @@ "workload_name": "Example Workload", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -148,6 +157,7 @@ ], "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -161,6 +171,7 @@ "source": null, "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -170,11 +181,12 @@ "description": "A looppoint json file resource.", "is_zipped": false, "region_id": "1", - "md5sum": "a71ed64908b082ea619b26b940a643c1", + "md5sum": "efb85ebdf90c5cee655bf2e05ae7692a", "url": "http://dist.gem5.org/dist/develop/looppoints/x86-matrix-multiply-omp-100-8-looppoint-json-20230128", "source": null, "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] } diff --git a/tests/pyunit/stdlib/resources/refs/resources.json b/tests/pyunit/stdlib/resources/refs/resources.json index 812caef..56930f3 100644 --- a/tests/pyunit/stdlib/resources/refs/resources.json +++ b/tests/pyunit/stdlib/resources/refs/resources.json @@ -21,7 +21,7 @@ "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", "resource_version": "1.0.0", "gem5_versions": [ - "23.0" + "develop" ], "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" }, @@ -48,7 +48,7 @@ "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", "resource_version": "1.1.0", "gem5_versions": [ - "23.0" + "develop" ], "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" }, @@ -71,7 +71,7 @@ "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", "resource_version": "2.0.0", "gem5_versions": [ - "23.1" + "999.1" ], "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" }, @@ -94,7 +94,7 @@ "source_url": "", "resource_version": "1.0.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -122,7 +122,7 @@ "source_url": "", "resource_version": "0.2.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -150,7 +150,7 @@ "source_url": "", "resource_version": "0.2.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -178,7 +178,7 @@ "source_url": "", "resource_version": "0.2.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -206,7 +206,7 @@ "source_url": "", "resource_version": "0.2.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -234,7 +234,7 @@ "source_url": "", "resource_version": "0.2.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -262,7 +262,7 @@ "source_url": "", "resource_version": "0.2.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -290,7 +290,7 @@ "source_url": "", "resource_version": "0.2.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -322,7 +322,7 @@ "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", "resource_version": "2.0.0", "gem5_versions": [ - "23.0" + "develop" ], "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" } diff --git a/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json b/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json deleted file mode 100644 index a7e9c9d..0000000 --- a/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "category": "binary", - "id": "x86-hello64-static", - "description": "A 'Hello World!' binary.", - "architecture": "X86", - "is_zipped": false, - "md5sum": "dbf120338b37153e3334603970cebd8c", - "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/x86/linux/hello64-static", - "source": "src/simple", - "resource_version": "1.0.0", - "gem5_versions": [ - "23.0" - ] - } -] diff --git a/tests/pyunit/stdlib/resources/refs/workload-checks.json b/tests/pyunit/stdlib/resources/refs/workload-checks.json index d41001d..dcb8577 100644 --- a/tests/pyunit/stdlib/resources/refs/workload-checks.json +++ b/tests/pyunit/stdlib/resources/refs/workload-checks.json @@ -10,7 +10,7 @@ "source": "src/linux-kernel", "resource_version": "1.0.0", "gem5_versions": [ - "23.0" + "develop" ] }, { @@ -25,7 +25,7 @@ "root_partition": "1", "resource_version": "1.0.0", "gem5_versions": [ - "23.0" + "develop" ] }, { @@ -42,7 +42,21 @@ }, "resource_version": "1.0.0", "gem5_versions": [ - "23.0" + "develop" + ] + }, + { + "category": "binary", + "id": "x86-hello64-static", + "description": "A 'Hello World!' binary.", + "architecture": "X86", + "is_zipped": false, + "md5sum": "dbf120338b37153e3334603970cebd8c", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/x86/linux/hello64-static", + "source": "src/simple", + "resource_version": "1.0.0", + "gem5_versions": [ + "develop" ] } ] -- To view, visit https://gem5-review.googlesource.com/c/public/gem5/+/71506?usp=email To unsubscribe, or for help writing mail filters, visit https://gem5-review.googlesource.com/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: public/gem5 Gerrit-Branch: release-staging-v23-0 Gerrit-Change-Id: I0ce48e88b93a2b9db53d4749861fa0b5f9472053 Gerrit-Change-Number: 71506 Gerrit-PatchSet: 4 Gerrit-Owner: Bobby Bruce <bbruce@ucdavis.edu> Gerrit-Reviewer: Bobby Bruce <bbruce@ucdavis.edu> Gerrit-Reviewer: Jason Lowe-Power <jason@lowepower.com> Gerrit-Reviewer: Jason Lowe-Power <power.jg@gmail.com> Gerrit-Reviewer: Kunal Pai <kunpai@ucdavis.edu> Gerrit-Reviewer: kokoro <noreply+kokoro@google.com> Gerrit-CC: kokoro <noreply+kokoro@google.com>