def __init__(
      self,

      self._local_path = local_path

      super().__init__(
          local_path=local_path,

      super().__init__(
          local_path=local_path,

  def __init__(
      self,

      super().__init__(
          local_path=local_path,

      # This is the path to which the resource is to be stored.

  def __init__(

      ```json
      {

          simpoint_file_path = simpoint_directory.get_simpoint_file()
          weight_file_path = simpoint_resource.get_weight_file()

      resource = obtain_resource(

      self.assertEquals(

      resource = obtain_resource(

      self.assertEquals(

  @classmethod

  @classmethod

Bobby Bruce has submitted this change. ( https://gem5-review.googlesource.com/c/public/gem5/+/71278?usp=email ) Change subject: stdlib, tests, configs: Introduce gem5 Vision to resources ...................................................................... stdlib, tests, configs: Introduce gem5 Vision to resources This patch makes changes to the stdlib based on the gem5 Vision project. Firstly, a MongoDB database is supported. A JSON database's support is continued. The JSON can either be a local path or a raw GitHub link. The data for these databases is stored in src/python under "gem5-config.json". This will be used by default. However, the configuration can be overridden: - by providing a path using the GEM5_CONFIG env variable. - by placing a gem5-config.json file in the current working directory. An AbstractClient is an abstract class that implements searching and sorting relevant to the databases. Clients is an optional list that can be passed while defining any Resource class and obtain_resource. These databases can be defined in the config JSON. Resources now have versions. This allows for a single version, e.g., 'x86-ubuntu-boot', to have multiple versions. As such, the key of a resource is its ID and Version (e.g., 'x86-ubuntu-boot/v2.1.0'). Different versions of a resource might be compatible with different versions of gem5. By default, it picks the latest version compatible with the gem5 Version of the user. A gem5 resource schema now has additional fields. These are: - source_url: Stores URL of GitHub Source of the resource. - license: License information of the resource. - tags: Words to identify a resource better, like hello for hello-world - example_usage: How to use the resource in a simulation. - gem5_versions: List of gem5 versions that resource is compatible with. - resource_version: The version of the resource itself. - size: The download size of the resource, if it exists. - code_examples: List of objects. These objects contain the path to where a resource is used in gem5 example config scripts, and if the resource itself is used in tests or not. - category: Category of the resource, as defined by classes in src/python/gem5/resources/resource.py. Some fields have been renamed: - "name" is changed to "id" - "documentation" is changed to "description" Besides these, the schema also supports resource specialization. It adds fields relevant to a specific resource as specified in src/python/gem5/resources/resource.py These changes have been made to better present information on the new gem5 Resources website. But, they do not affect the way resources are used by a gem5 user. This patch is also backwards compatible. Existing code doesn't break with this new infrastructure. Also, refs in the tests have been changed to match this new schema. Tests have been changed to work with the two clients. Change-Id: Ia9bf47f7900763827fd5e873bcd663cc3ecdba40 Co-authored-by: Kunal Pai <kunpai@ucdavis.edu> Co-authored-by: Parth Shah <helloparthshah@gmail.com> Co-authored-by: Harshil Patel <harshilp2107@gmail.com> Co-authored-by: aarsli <arsli@ucdavis.edu> Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71278 Maintainer: Bobby Bruce <bbruce@ucdavis.edu> Tested-by: kokoro <noreply+kokoro@google.com> Reviewed-by: Bobby Bruce <bbruce@ucdavis.edu> --- M src/python/SConscript A src/python/gem5/resources/client.py A src/python/gem5/resources/client_api/__init__.py A src/python/gem5/resources/client_api/abstract_client.py A src/python/gem5/resources/client_api/atlasclient.py A src/python/gem5/resources/client_api/client_wrapper.py A src/python/gem5/resources/client_api/jsonclient.py M src/python/gem5/resources/downloader.py M src/python/gem5/resources/resource.py M src/python/gem5/resources/workload.py M src/python/gem5/utils/simpoint.py A src/python/gem5_default_config.py M tests/gem5/verifier.py A tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py M tests/pyunit/stdlib/resources/pyunit_downloader_checks.py A tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py A tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py M tests/pyunit/stdlib/resources/pyunit_resource_specialization.py M tests/pyunit/stdlib/resources/pyunit_workload_checks.py A tests/pyunit/stdlib/resources/refs/mongo_dup_mock.json A tests/pyunit/stdlib/resources/refs/mongo_mock.json A tests/pyunit/stdlib/resources/refs/obtain-resource.json M tests/pyunit/stdlib/resources/refs/resource-specialization.json A tests/pyunit/stdlib/resources/refs/resources.json M tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json M tests/pyunit/stdlib/resources/refs/workload-checks.json 26 files changed, 2,413 insertions(+), 342 deletions(-) Approvals: Bobby Bruce: Looks good to me, approved; Looks good to me, approved kokoro: Regressions pass diff --git a/src/python/SConscript b/src/python/SConscript index e261bfa..f98b570 100644 --- a/src/python/SConscript +++ b/src/python/SConscript @@ -264,12 +264,24 @@ PySource('gem5.prebuilt.riscvmatched', 'gem5/prebuilt/riscvmatched/riscvmatched_core.py') PySource('gem5.resources', 'gem5/resources/__init__.py') +PySource('gem5.resources', 'gem5/resources/client.py') PySource('gem5.resources', 'gem5/resources/downloader.py') PySource('gem5.resources', 'gem5/resources/md5_utils.py') PySource('gem5.resources', 'gem5/resources/resource.py') PySource('gem5.resources', 'gem5/resources/workload.py') PySource('gem5.resources', 'gem5/resources/looppoint.py') PySource('gem5.resources', 'gem5/resources/elfie.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/__init__.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/jsonclient.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/atlasclient.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/client_wrapper.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/abstract_client.py') +PySource('gem5', 'gem5_default_config.py') PySource('gem5.utils', 'gem5/utils/__init__.py') PySource('gem5.utils', 'gem5/utils/filelock.py') PySource('gem5.utils', 'gem5/utils/override.py') diff --git a/src/python/gem5/resources/client.py b/src/python/gem5/resources/client.py new file mode 100644 index 0000000..bd473eb --- /dev/null +++ b/src/python/gem5/resources/client.py @@ -0,0 +1,84 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +from pathlib import Path +import os +from typing import Optional, Dict, List +from .client_api.client_wrapper import ClientWrapper +from gem5.gem5_default_config import config +from m5.util import inform + + +def getFileContent(file_path: Path) -> Dict: + """ + Get the content of the file at the given path + :param file_path: The path of the file + :return: The content of the file + """ + if file_path.exists(): + with open(file_path, "r") as file: + return json.load(file) + else: + raise Exception(f"File not found at {file_path}") + + +clientwrapper = None + + +def get_resource_json_obj( + resource_id, + resource_version: Optional[str] = None, + clients: Optional[List[str]] = None, +) -> Dict: + """ + Get the resource json object from the clients wrapper + :param resource_id: The resource id + :param resource_version: The resource version + :param clients: The list of clients to query + """ + global clientwrapper + if clientwrapper is None: + # First check if the config file path is provided in the environment variable + if "GEM5_CONFIG" in os.environ: + config_file_path = Path(os.environ["GEM5_CONFIG"]) + gem5_config = getFileContent(config_file_path) + inform("Using config file specified by $GEM5_CONFIG") + inform(f"Using config file at {os.environ['GEM5_CONFIG']}") + # If not, check if the config file is present in the current directory + elif (Path().cwd().resolve() / "gem5-config.json").exists(): + config_file_path = Path().resolve() / "gem5-config.json" + gem5_config = getFileContent(config_file_path) + inform(f"Using config file at {config_file_path}") + # If not, use the default config in the build directory + else: + gem5_config = config + inform("Using default config") + clientwrapper = ClientWrapper(gem5_config) + + return clientwrapper.get_resource_json_obj_from_client( + resource_id, resource_version, clients + ) diff --git a/src/python/gem5/resources/client_api/__init__.py b/src/python/gem5/resources/client_api/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/src/python/gem5/resources/client_api/__init__.py diff --git a/src/python/gem5/resources/client_api/abstract_client.py b/src/python/gem5/resources/client_api/abstract_client.py new file mode 100644 index 0000000..74a513f --- /dev/null +++ b/src/python/gem5/resources/client_api/abstract_client.py @@ -0,0 +1,71 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from abc import ABC, abstractmethod +from typing import Any, Dict, List +import urllib.parse + + +class AbstractClient(ABC): + def verify_status_code(self, status_code: int) -> None: + """ + Verifies that the status code is 200. + :param status_code: The status code to verify. + """ + if status_code == 200: + return + if status_code == 429: + raise Exception("Panic: Too many requests") + if status_code == 401: + raise Exception("Panic: Unauthorized") + if status_code == 404: + raise Exception("Panic: Not found") + if status_code == 400: + raise Exception("Panic: Bad request") + if status_code == 500: + raise Exception("Panic: Internal server error") + + raise Exception(f"Panic: Unknown status code {status_code}") + + def _url_validator(self, url: str) -> bool: + """ + Validates the provided URL. + :param url: The URL to be validated. + :return: True if the URL is valid, False otherwise. + """ + try: + result = urllib.parse.urlparse(url) + return all([result.scheme, result.netloc, result.path]) + except: + return False + + @abstractmethod + def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: + """ + :param resource_id: The ID of the Resource. + :return: A list of all the Resources with the given ID. + """ + raise NotImplementedError diff --git a/src/python/gem5/resources/client_api/atlasclient.py b/src/python/gem5/resources/client_api/atlasclient.py new file mode 100644 index 0000000..4a6e5cf --- /dev/null +++ b/src/python/gem5/resources/client_api/atlasclient.py @@ -0,0 +1,91 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from urllib import request, parse +from urllib.error import HTTPError, URLError +from typing import Optional, Dict, Union, Type, Tuple, List, Any +import json +from .abstract_client import AbstractClient + + +class AtlasClient(AbstractClient): + def __init__(self, config: Dict[str, str]): + """ + Initializes a connection to a MongoDB Atlas database. + :param uri: The URI for connecting to the MongoDB server. + :param db: The name of the database to connect to. + :param collection: The name of the collection within the database. + """ + self.apiKey = config["apiKey"] + self.url = config["url"] + self.collection = config["collection"] + self.database = config["database"] + self.dataSource = config["dataSource"] + self.authUrl = config["authUrl"] + + def get_token(self): + data = {"key": self.apiKey} + data = json.dumps(data).encode("utf-8") + + req = request.Request( + self.authUrl, + data=data, + headers={"Content-Type": "application/json"}, + ) + try: + response = request.urlopen(req) + except HTTPError as e: + self.verify_status_code(e.status) + return None + result = json.loads(response.read().decode("utf-8")) + token = result["access_token"] + return token + + def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: + url = f"{self.url}/action/find" + data = { + "dataSource": self.dataSource, + "collection": self.collection, + "database": self.database, + "filter": {"id": resource_id}, + } + data = json.dumps(data).encode("utf-8") + + headers = { + "Authorization": f"Bearer {self.get_token()}", + "Content-Type": "application/json", + } + + req = request.Request(url, data=data, headers=headers) + try: + response = request.urlopen(req) + except HTTPError as e: + self.verify_status_code(e.status) + return None + result = json.loads(response.read().decode("utf-8")) + resources = result["documents"] + + return resources diff --git a/src/python/gem5/resources/client_api/client_wrapper.py b/src/python/gem5/resources/client_api/client_wrapper.py new file mode 100644 index 0000000..74ee831 --- /dev/null +++ b/src/python/gem5/resources/client_api/client_wrapper.py @@ -0,0 +1,228 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from .jsonclient import JSONClient +from .atlasclient import AtlasClient +from _m5 import core +from typing import Optional, Dict, List +from distutils.version import StrictVersion +import itertools +from m5.util import warn + + +class ClientWrapper: + def __init__(self, config): + self.clients = self.create_clients(config) + + def create_clients( + self, + config: Dict, + ) -> Dict: + """ + This function creates respective client object for each source in the + config file according to the type of source. + Params: config: config file containing the source information + Returns: clients: dictionary of clients for each source + """ + clients = {} + for client in config["sources"]: + client_source = config["sources"][client] + try: + if client_source["isMongo"]: + clients[client] = AtlasClient(client_source) + else: + clients[client] = JSONClient(client_source["url"]) + except Exception as e: + warn(f"Error creating client {client}: {str(e)}") + return clients + + def get_all_resources_by_id( + self, + resource_id: str, + clients: Optional[List[str]] = None, + ) -> List[Dict]: + """ + This function returns all the resources with the given id from all the + sources. + :param resource_id: The id of the resource to search for. + :param clients: A list of clients to search through. If None, all + clients are searched. + :return: A list of resources as Python dictionaries. + """ + resources = [] + if not clients: + clients = list(self.clients.keys()) + for client in clients: + if client not in self.clients: + raise Exception(f"Client: {client} does not exist") + try: + resources.extend( + self.clients[client].get_resources_by_id(resource_id) + ) + except Exception as e: + warn(f"Error getting resources from client {client}: {str(e)}") + # check if no 2 resources have the same id and version + for res1, res2 in itertools.combinations(resources, 2): + if res1["resource_version"] == res2["resource_version"]: + raise Exception( + f"Resource {resource_id} has multiple resources with " + f"the same version: {res1['resource_version']}" + ) + return resources + + def get_resource_json_obj_from_client( + self, + resource_id: str, + resource_version: Optional[str] = None, + clients: Optional[List[str]] = None, + ) -> Dict: + """ + This function returns the resource object from the client with the + given id and version. + :param resource_id: The id of the resource to search for. + :param resource_version: The version of the resource to search for. + :param clients: A list of clients to search through. If None, all + clients are searched. + :return: The resource object as a Python dictionary if found. + If not found, exception is thrown. + """ + # getting all the resources with the given id from the dictionary + resources = self.get_all_resources_by_id(resource_id, clients) + # if no resource with the given id is found, return None + if len(resources) == 0: + raise Exception(f"Resource with ID '{resource_id}' not found.") + + resource_to_return = None + + if resource_version: + resource_to_return = self._search_version_in_resources( + resources, resource_id, resource_version + ) + + else: + compatible_resources = ( + self._get_resources_compatible_with_gem5_version(resources) + ) + if len(compatible_resources) == 0: + resource_to_return = self._sort_resources(resources)[0] + else: + resource_to_return = self._sort_resources( + compatible_resources + )[0] + + self._check_resource_version_compatibility(resource_to_return) + + return resource_to_return + + def _search_version_in_resources( + self, resources: List, resource_id: str, resource_version: str + ) -> Dict: + """ + Searches for the resource with the given version. If the resource is + not found, an exception is thrown. + :param resources: A list of resources to search through. + :param resource_version: The version of the resource to search for. + :return: The resource object as a Python dictionary if found. + If not found, None is returned. + """ + return_resource = next( + iter( + [ + resource + for resource in resources + if resource["resource_version"] == resource_version + ] + ), + None, + ) + if not return_resource: + raise Exception( + f"Resource {resource_id} with version '{resource_version}'" + " not found.\nResource versions can be found at: " + "https://resources.gem5.org/" + f"resources/{resource_id}/versions" + ) + return return_resource + + def _get_resources_compatible_with_gem5_version( + self, resources: List, gem5_version: str = core.gem5Version + ) -> List: + """ + Returns a list of compatible resources with the current gem5 version. + :param resources: A list of resources to filter. + :return: A list of compatible resources as Python dictionaries. + If no compatible resources are found, the original list of resources + is returned. + """ + compatible_resources = [ + resource + for resource in resources + if gem5_version in resource["gem5_versions"] + ] + return compatible_resources + + def _sort_resources(self, resources: List) -> List: + """ + Sorts the resources by ID. + If the IDs are the same, the resources are sorted by version. + :param resources: A list of resources to sort. + :return: A list of sorted resources. + """ + return sorted( + resources, + key=lambda resource: ( + resource["id"].lower(), + StrictVersion(resource["resource_version"]), + ), + reverse=True, + ) + + def _check_resource_version_compatibility( + self, resource: dict, gem5_version: Optional[str] = core.gem5Version + ) -> bool: + """ + Checks if the resource is compatible with the gem5 version. + Prints a warning if the resource is not compatible. + :param resource: The resource to check. + :optional param gem5_version: The gem5 version to check + compatibility with. + :return: True if the resource is compatible, False otherwise. + """ + if not resource: + return False + if gem5_version not in resource["gem5_versions"]: + warn( + f"Resource {resource['id']} with version " + f"{resource['resource_version']} is not known to be compatible" + f" with gem5 version {gem5_version}. " + "This may cause problems with your simulation. " + "This resource's compatibility " + "with different gem5 versions can be found here: " + "https://resources.gem5.org" + f"/resources/{resource['id']}/versions" + ) + return False + return True diff --git a/src/python/gem5/resources/client_api/jsonclient.py b/src/python/gem5/resources/client_api/jsonclient.py new file mode 100644 index 0000000..225126e --- /dev/null +++ b/src/python/gem5/resources/client_api/jsonclient.py @@ -0,0 +1,70 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +from pathlib import Path +from urllib import request +from typing import Optional, Dict, Union, Type, Tuple, List, Any +from .abstract_client import AbstractClient +from urllib.error import URLError +from m5.util import warn + + +class JSONClient(AbstractClient): + def __init__(self, path: str): + """ + Initializes a JSON client. + :param path: The path to the Resource, either URL or local. + """ + self.path = path + self.resources = [] + + if Path(self.path).is_file(): + self.resources = json.load(open(self.path)) + elif not self._url_validator(self.path): + raise Exception( + f"Resources location '{self.path}' is not a valid path or URL." + ) + else: + req = request.Request(self.path) + try: + response = request.urlopen(req) + except URLError as e: + raise Exception( + f"Unable to open Resources location '{self.path}': {e}" + ) + self.resources = json.loads(response.read().decode("utf-8")) + + def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: + """ + :param resource_id: The ID of the Resource. + :return: A list of all the Resources with the given ID. + """ + return [ + resource + for resource in self.resources + if resource["id"] == resource_id + ] diff --git a/src/python/gem5/resources/downloader.py b/src/python/gem5/resources/downloader.py index 16b0147..0781d9b 100644 --- a/src/python/gem5/resources/downloader.py +++ b/src/python/gem5/resources/downloader.py @@ -41,6 +41,7 @@ from urllib.error import HTTPError from typing import List, Dict, Set, Optional +from .client import get_resource_json_obj from .md5_utils import md5_file, md5_dir from ..utils.progress_bar import tqdm, progress_hook @@ -398,6 +399,8 @@ unzip: bool = True, untar: bool = True, download_md5_mismatch: bool = True, + resource_version: Optional[str] = None, + clients: Optional[List] = None, ) -> None: """ Obtains a gem5 resource and stored it to a specified location. If the @@ -419,6 +422,13 @@ will delete this local resource and re-download it if this parameter is True. True by default. + :param resource_version: The version of the resource to be obtained. If + None, the latest version of the resource compatible with the working + directory's gem5 version will be obtained. None by default. + + :param clients: A list of clients to use when obtaining the resource. If + None, all clients will be used. None by default. + :raises Exception: An exception is thrown if a file is already present at `to_path` but it does not have the correct md5 sum. An exception will also be thrown is a directory is present at `to_path` @@ -430,11 +440,13 @@ # minutes.Most resources should be downloaded and decompressed in this # timeframe, even on the most constrained of systems. with FileLock(f"{to_path}.lock", timeout=900): - - resource_json = get_resources_json_obj(resource_name) + resource_json = get_resource_json_obj( + resource_name, + resource_version=resource_version, + clients=clients, + ) if os.path.exists(to_path): - if os.path.isfile(to_path): md5 = md5_file(Path(to_path)) else: @@ -495,9 +507,8 @@ ) ) - # Get the URL. The URL may contain '{url_base}' which needs replaced - # with the correct value. - url = resource_json["url"].format(url_base=_get_url_base()) + # Get the URL. + url = resource_json["url"] _download(url=url, download_to=download_dest) print(f"Finished downloading resource '{resource_name}'.") diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index 9cba9cf..22adf15 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -29,13 +29,15 @@ from pathlib import Path from m5.util import warn, fatal -from .downloader import get_resource, get_resources_json_obj +from .downloader import get_resource from .looppoint import LooppointCsvLoader, LooppointJsonLoader from ..isas import ISA, get_isa_from_str from typing import Optional, Dict, Union, Type, Tuple, List +from .client import get_resource_json_obj + """ Resources are items needed to run a simulation, such as a disk image, kernel, or binary. The gem5 project provides pre-built resources, with sources, at @@ -67,18 +69,20 @@ def __init__( self, + resource_version: Optional[str] = None, local_path: Optional[str] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, ): """ :param local_path: The path on the host system where this resource is located. - :param documentation: Documentation describing this resource. Not a + :param description: Description describing this resource. Not a required parameter. By default is None. :param source: The source (as in "source code") for this resource. This string should navigate users to where the source for this resource may be found. Not a required parameter. By default is None. + :param resource_version: Version of the resource itself. """ if local_path and not os.path.exists(local_path): @@ -88,16 +92,21 @@ ) self._local_path = local_path - self._documentation = documentation + self._description = description self._source = source + self._version = resource_version + + def get_resource_version(self) -> str: + """Returns the version of the resource.""" + return self._version def get_local_path(self) -> Optional[str]: """Returns the local path of the resource.""" return self._local_path - def get_documentation(self) -> Optional[str]: - """Returns documentation associated with this resource.""" - return self._documentation + def get_description(self) -> Optional[str]: + """Returns description associated with this resource.""" + return self._description def get_source(self) -> Optional[str]: """Returns information as to where the source for this resource may be @@ -112,7 +121,8 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): @@ -123,8 +133,9 @@ super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -134,11 +145,11 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): - if not os.path.isdir(local_path): raise Exception( f"DirectoryResource path specified, {local_path}, is not a " @@ -147,8 +158,9 @@ super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -158,15 +170,17 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, root_partition: Optional[str] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) self._root_partition = root_partition @@ -181,15 +195,17 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, architecture: Optional[Union[ISA, str]] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) self._architecture = None @@ -210,16 +226,18 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, architecture: Optional[Union[ISA, str]] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, architecture=architecture, source=source, + resource_version=resource_version, ) @@ -229,14 +247,16 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -246,16 +266,18 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, architecture: Optional[Union[ISA, str]] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, architecture=architecture, + resource_version=resource_version, ) @@ -270,14 +292,16 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -290,12 +314,13 @@ def __init__( self, + resource_version: Optional[str] = None, simpoint_interval: int = None, simpoint_list: List[int] = None, weight_list: List[float] = None, warmup_interval: int = 0, workload_name: Optional[str] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, local_path: Optional[str] = None, **kwargs, @@ -314,8 +339,9 @@ super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) self._weight_list = weight_list @@ -402,15 +428,17 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): FileResource.__init__( self, local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) LooppointCsvLoader.__init__(self, pinpoints_file=Path(local_path)) @@ -419,16 +447,18 @@ def __init__( self, local_path: str, + resource_version: Optional[str] = None, region_id: Optional[Union[str, int]] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): FileResource.__init__( self, local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) LooppointJsonLoader.__init__( self, looppoint_file=local_path, region_id=region_id @@ -446,8 +476,9 @@ weight_file: str, simpoint_interval: int, warmup_interval: int, + resource_version: Optional[str] = None, workload_name: Optional[str] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): @@ -478,8 +509,9 @@ warmup_interval=warmup_interval, workload_name=workload_name, local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) def get_simpoint_file(self) -> Path: @@ -522,9 +554,11 @@ def obtain_resource( - resource_name: str, + resource_id: str, resource_directory: Optional[str] = None, download_md5_mismatch: bool = True, + resource_version: Optional[str] = None, + clients: Optional[List] = None, ) -> AbstractResource: """ This function primarily serves as a factory for resources. It will return @@ -544,10 +578,16 @@ have the correct md5 value, the resoruce will be deleted and re-downloaded if this value is True. Otherwise an exception will be thrown. True by default. + :param resource_version: Version of the resource itself. + Not a required parameter. None by default. + :param clients: A list of clients to search for the resource. If this + parameter is not set, it will default search all clients. """ - # Obtain the JSON resource entry for this resource - resource_json = get_resources_json_obj(resource_name) + # Obtain the resource object entry for this resource + resource_json = get_resource_json_obj( + resource_id, resource_version=resource_version, clients=clients + ) to_path = None # If the "url" field is specified, the resoruce must be downloaded. @@ -580,38 +620,37 @@ os.makedirs(resource_directory, exist_ok=True) # This is the path to which the resource is to be stored. - to_path = os.path.join(resource_directory, resource_name) + to_path = os.path.join(resource_directory, resource_id) # Download the resource if it does not already exist. get_resource( - resource_name=resource_name, - to_path=os.path.join(resource_directory, resource_name), + resource_name=resource_id, + to_path=os.path.join(resource_directory, resource_id), download_md5_mismatch=download_md5_mismatch, + resource_version=resource_version, + clients=clients, ) # Obtain the type from the JSON. From this we will determine what subclass # of `AbstractResource` we are to create and return. - resources_type = resource_json["type"] + resources_category = resource_json["category"] - if resources_type == "resource": + if resources_category == "resource": # This is a stop-gap measure to ensure to work with older versions of # the "resource.json" file. These should be replaced with their # respective specializations ASAP and this case removed. - if ( - "additional_metadata" in resource_json - and "root_partition" in resource_json["additional_metadata"] - ): + if "root_partition" in resource_json: # In this case we should return a DiskImageResource. - root_partition = resource_json["additional_metadata"][ - "root_partition" - ] + root_partition = resource_json["root_partition"] return DiskImageResource( - local_path=to_path, root_partition=root_partition + local_path=to_path, + root_partition=root_partition, + **resource_json, ) return CustomResource(local_path=to_path) - assert resources_type in _get_resource_json_type_map - resource_class = _get_resource_json_type_map[resources_type] + assert resources_category in _get_resource_json_type_map + resource_class = _get_resource_json_type_map[resources_category] # Once we know what AbstractResource subclass we are using, we create it. # The fields in the JSON object are assumed to map like-for-like to the @@ -694,6 +733,7 @@ def __init__( self, local_path: str, + resource_version: Optional[str] = None, root_partition: Optional[str] = None, metadata: Dict = {}, ): @@ -702,6 +742,7 @@ :param root_partition: The root disk partition to use. :param metadata: Metadata for the resource. **Warning:** As of " "v22.1.1, this parameter is not used. + :param resource_version: Version of the resource itself. """ warn( "The `CustomDiskImageResource` class is deprecated. Please use " @@ -713,13 +754,19 @@ "`CustomDiskImageResource` constructor. This parameter is not " "used." ) - super().__init__(local_path=local_path, root_partition=root_partition) + super().__init__( + local_path=local_path, + root_partition=root_partition, + resource_version=resource_version, + ) def Resource( - resource_name: str, + resource_id: str, resource_directory: Optional[str] = None, download_md5_mismatch: bool = True, + resource_version: Optional[str] = None, + clients: Optional[List[str]] = None, ) -> AbstractResource: """ This function was created to maintain backwards compability for v21.1.0 @@ -737,9 +784,11 @@ ) return obtain_resource( - resource_name=resource_name, + resource_id=resource_id, resource_directory=resource_directory, download_md5_mismatch=download_md5_mismatch, + resource_version=resource_version, + clients=clients, ) diff --git a/src/python/gem5/resources/workload.py b/src/python/gem5/resources/workload.py index e0a19d0..148ab3f 100644 --- a/src/python/gem5/resources/workload.py +++ b/src/python/gem5/resources/workload.py @@ -24,10 +24,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from .downloader import get_workload_json_obj from .resource import obtain_resource +from .client import get_resource_json_obj -from typing import Dict, Any, Optional +from typing import Dict, Any, List, Optional class AbstractWorkload: @@ -155,7 +155,11 @@ """ def __init__( - self, workload_name: str, resource_directory: Optional[str] = None + self, + workload_name: str, + resource_directory: Optional[str] = None, + resource_version: Optional[str] = None, + clients: Optional[List] = None, ) -> None: """ This constructor will load the workload details from the workload with @@ -167,13 +171,13 @@ ```json { - "type" : "workload", - "name" : "x86-ubuntu-18.04-echo-hello", - "documentation" : "Description of workload here", + "category" : "workload", + "id" : "x86-ubuntu-18.04-echo-hello", + "description" : "Description of workload here", "function" : "set_kernel_disk_workload", "resources" : { "kernel" : "x86-linux-kernel-5.4.49", - "disk_image" : "x86-ubuntu-18.04-img" + "disk-image" : "x86-ubuntu-18.04-img" }, "additional_params" : { "readfile_contents" : "m5_exit; echo 'hello'; m5_exit" @@ -187,7 +191,7 @@ ```python board.set_kernel_disk_workload( kernel = Resource("x86-linux-kernel-5.4.49"), - disk_image = Resource("x86-ubuntu-18.04-img"), + disk-image = Resource("x86-ubuntu-18.04-img"), readfile_contents = "m5_exit; echo 'hello'; m5_exit", ) ``` @@ -198,7 +202,12 @@ any resources should be download and accessed from. If None, a default location will be used. None by default. """ - workload_json = get_workload_json_obj(workload_name=workload_name) + + workload_json = get_resource_json_obj( + workload_name, + resource_version=resource_version, + clients=clients, + ) func = workload_json["function"] assert isinstance(func, str) diff --git a/src/python/gem5/utils/simpoint.py b/src/python/gem5/utils/simpoint.py index eab92e2..0d1af4b 100644 --- a/src/python/gem5/utils/simpoint.py +++ b/src/python/gem5/utils/simpoint.py @@ -83,15 +83,11 @@ simpoint_file_path = simpoint_directory.get_simpoint_file() weight_file_path = simpoint_resource.get_weight_file() - simpoint_interval = ( - simpoint_resource.get_metadata() - .get("additional_metadata") - .get("simpoint_interval") + simpoint_interval = simpoint_resource.get_metadata().get( + "simpoint_interval" ) - warmup_interval = ( - simpoint_resource.get_metadata() - .get("additional_metadata") - .get("warmup_interval") + warmup_interval = simpoint_resource.get_metadata().get( + "warmup_interval" ) self._simpoint_interval = simpoint_interval diff --git a/src/python/gem5_default_config.py b/src/python/gem5_default_config.py new file mode 100644 index 0000000..980c883 --- /dev/null +++ b/src/python/gem5_default_config.py @@ -0,0 +1,39 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +config = { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + } +} diff --git a/tests/gem5/verifier.py b/tests/gem5/verifier.py index 93d47c8..c725fc6 100644 --- a/tests/gem5/verifier.py +++ b/tests/gem5/verifier.py @@ -172,6 +172,7 @@ re.compile("^Using GPU kernel code file\(s\) "), re.compile("^.* not found locally\. Downloading"), re.compile("^Finished downloading"), + re.compile("^info: Using default config"), ] diff --git a/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py b/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py new file mode 100644 index 0000000..96aadf6 --- /dev/null +++ b/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py @@ -0,0 +1,511 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +from gem5.isas import ISA +from gem5.resources.client import get_resource_json_obj +import gem5.resources.client +from gem5.resources.client_api.client_wrapper import ClientWrapper +from typing import Dict +from unittest.mock import patch +from unittest import mock +import json +from urllib.error import HTTPError +import io +import contextlib +from pathlib import Path + +mock_json_path = Path(__file__).parent / "refs/resources.json" +mock_config_json = { + "sources": { + "baba": { + "url": mock_json_path, + "isMongo": False, + } + }, +} + +mock_config_mongo = { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + }, +} + +mock_config_combined = { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + }, + "baba": { + "url": mock_json_path, + "isMongo": False, + }, + }, +} + +mock_json = {} + +with open(Path(__file__).parent / "refs/mongo_mock.json", "r") as f: + mock_json = json.load(f) + +duplicate_mock_json = {} + +with open(Path(__file__).parent / "refs/mongo_dup_mock.json", "r") as f: + duplicate_mock_json = json.load(f) + + +def mocked_requests_post(*args): + # mokcing urllib.request.urlopen + class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status = status_code + + def read(self): + return json.dumps(self.json_data).encode("utf-8") + + data = json.loads(args[0].data) + if "/api-key/login" in args[0].full_url: + return MockResponse({"access_token": "test-token"}, 200) + if "/endpoint/data/v1/action/find" in args[0].full_url: + if data: + if data["filter"]["id"] == "x86-ubuntu-18.04-img": + return MockResponse( + { + "documents": mock_json, + }, + 200, + ) + if data["filter"]["id"] == "test-duplicate": + return MockResponse( + { + "documents": duplicate_mock_json, + }, + 200, + ) + if data["filter"]["id"] == "test-too-many": + error_file = io.BytesIO() + error_file.status = 429 + raise HTTPError( + args[0].full_url, 429, "Too Many Requests", {}, error_file + ) + return MockResponse( + { + "documents": [], + }, + 200, + ) + error_file = io.BytesIO() + error_file.status = 404 + raise HTTPError(args[0].full_url, 404, "Not Found", {}, error_file) + + +class ClientWrapperTestSuite(unittest.TestCase): + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json), + ) + def test_get_resource_json_obj(self): + # Test that the resource object is correctly returned + resource = "this-is-a-test-resource" + resource = get_resource_json_obj(resource) + self.assertEqual(resource["id"], "this-is-a-test-resource") + self.assertEqual(resource["resource_version"], "2.0.0") + self.assertEqual(resource["category"], "binary") + self.assertEqual( + resource["description"], "This is a test resource but double newer" + ) + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json), + ) + def test_get_resource_json_obj_invalid_client(self): + # Test that an exception is raised when an invalid client is passed + resource_id = "test-id" + client = "invalid" + with self.assertRaises(Exception) as context: + get_resource_json_obj(resource_id, clients=[client]) + self.assertTrue( + f"Client: {client} does not exist" in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json), + ) + def test_get_resource_json_obj_with_version(self): + # Test that the resource object is correctly returned + resource_id = "this-is-a-test-resource" + resource_version = "1.0.0" + resource = get_resource_json_obj( + resource_id, resource_version=resource_version + ) + self.assertEqual(resource["id"], "this-is-a-test-resource") + self.assertEqual(resource["resource_version"], "1.0.0") + self.assertEqual(resource["category"], "binary") + self.assertEqual(resource["description"], "This is a test resource") + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_1(self, mock_get): + resource = "x86-ubuntu-18.04-img" + resource = get_resource_json_obj(resource) + self.assertEqual(resource["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource["resource_version"], "1.1.0") + self.assertEqual(resource["category"], "disk-image") + self.assertEqual( + resource["description"], + "A disk image containing Ubuntu 18.04 for x86. This image will run an `m5 readfile` instruction after booting. If no script file is specified an `m5 exit` instruction will be executed.", + ) + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_with_version_mongodb(self, mock_get): + # Test that the resource object is correctly returned + resource_id = "x86-ubuntu-18.04-img" + resource_version = "1.0.0" + resource = get_resource_json_obj( + resource_id, + resource_version=resource_version, + clients=["gem5-resources"], + ) + self.assertEqual(resource["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource["resource_version"], "1.0.0") + self.assertEqual(resource["category"], "disk-image") + self.assertEqual(resource["description"], "This is a test resource") + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_with_id_invalid_mongodb(self, mock_get): + resource_id = "invalid-id" + with self.assertRaises(Exception) as context: + get_resource_json_obj(resource_id, clients=["gem5-resources"]) + self.assertTrue( + "Resource with ID 'invalid-id' not found." + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_with_version_invalid_mongodb( + self, mock_get + ): + resource_id = "x86-ubuntu-18.04-img" + resource_version = "2.5.0" + with self.assertRaises(Exception) as context: + get_resource_json_obj( + resource_id, + resource_version=resource_version, + clients=["gem5-resources"], + ) + self.assertTrue( + f"Resource x86-ubuntu-18.04-img with version '2.5.0'" + " not found.\nResource versions can be found at: " + f"https://resources.gem5.org/resources/x86-ubuntu-18.04-img/versions" + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json), + ) + def test_get_resource_json_obj_with_version_invalid_json(self): + resource_id = "this-is-a-test-resource" + resource_version = "2.5.0" + with self.assertRaises(Exception) as context: + get_resource_json_obj( + resource_id, + resource_version=resource_version, + ) + self.assertTrue( + f"Resource this-is-a-test-resource with version '2.5.0'" + " not found.\nResource versions can be found at: " + f"https://resources.gem5.org/resources/this-is-a-test-resource/versions" + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_combined), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_combine(self, mock_get): + resource_id_mongo = "x86-ubuntu-18.04-img" + resource_version_mongo = "1.0.0" + resource_id_json = "this-is-a-test-resource" + resource_version_json = "1.0.0" + resource_mongo = get_resource_json_obj( + resource_id_mongo, + resource_version=resource_version_mongo, + clients=["gem5-resources"], + ) + resource_json = get_resource_json_obj( + resource_id_json, + resource_version=resource_version_json, + clients=["baba"], + ) + self.assertEqual(resource_mongo["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource_mongo["resource_version"], "1.0.0") + self.assertEqual(resource_mongo["category"], "disk-image") + self.assertEqual( + resource_mongo["description"], "This is a test resource" + ) + self.assertEqual( + resource_mongo["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + ) + self.assertEqual(resource_mongo["architecture"], "X86") + + self.assertEqual(resource_json["id"], "this-is-a-test-resource") + self.assertEqual(resource_json["resource_version"], "1.0.0") + self.assertEqual(resource_json["category"], "binary") + self.assertEqual( + resource_json["description"], "This is a test resource" + ) + self.assertEqual( + resource_json["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + ) + self.assertEqual(resource_json["architecture"], "X86") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_combined), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_multi_database_second_only(self, mock_get): + resource_id = "simpoint-resource" + resource = get_resource_json_obj( + resource_id, + ) + self.assertEqual(resource["id"], resource_id) + self.assertEqual(resource["resource_version"], "0.2.0") + self.assertEqual(resource["category"], "file") + self.assertEqual( + resource["description"], + ( + "Simpoints for running the 'x86-print-this' resource with" + ' the parameters `"print this" 15000`. This is encapsulated' + " in the 'x86-print-this-15000-with-simpoints' workload." + ), + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_combined), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_same_resource_different_versions( + self, mock_get + ): + resource_id = "x86-ubuntu-18.04-img" + resource_json = get_resource_json_obj( + resource_id, + ) + + self.assertEqual(resource_json["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource_json["resource_version"], "2.0.0") + self.assertEqual(resource_json["category"], "disk-image") + + resource_json = get_resource_json_obj( + resource_id, + resource_version="1.0.0", + ) + + self.assertEqual(resource_json["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource_json["resource_version"], "1.0.0") + self.assertEqual(resource_json["category"], "disk-image") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_combined), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_same_resource_same_version(self, mock_get): + resource_id = "test-duplicate" + with self.assertRaises(Exception) as context: + get_resource_json_obj( + resource_id, + ) + self.assertTrue( + f"Resource {resource_id} has multiple resources with" + f" the same version: 0.2.0" in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper( + { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/logi", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + }, + } + ), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_invalid_auth_url(self, mock_get): + resource_id = "test-resource" + f = io.StringIO() + with self.assertRaises(Exception) as context: + with contextlib.redirect_stderr(f): + get_resource_json_obj( + resource_id, + ) + self.assertTrue( + "Error getting resources from client gem5-resources:" + " Panic: Not found" in str(f.getvalue()) + ) + self.assertTrue( + "Resource with ID 'test-resource' not found." + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper( + { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + }, + } + ), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_invalid_url(self, mock_get): + resource_id = "test-resource" + f = io.StringIO() + with self.assertRaises(Exception) as context: + with contextlib.redirect_stderr(f): + get_resource_json_obj( + resource_id, + ) + self.assertTrue( + "Error getting resources from client gem5-resources:" + " Panic: Not found" in str(f.getvalue()) + ) + self.assertTrue( + "Resource with ID 'test-resource' not found." + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper( + { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + }, + } + ), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_invalid_url(self, mock_get): + resource_id = "test-too-many" + f = io.StringIO() + with self.assertRaises(Exception) as context: + with contextlib.redirect_stderr(f): + get_resource_json_obj( + resource_id, + ) + self.assertTrue( + "Error getting resources from client gem5-resources:" + " Panic: Too many requests" in str(f.getvalue()) + ) + self.assertTrue( + "Resource with ID 'test-too-many' not found." + in str(context.exception) + ) diff --git a/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py b/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py index 08736bb..19169e4 100644 --- a/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 The Regents of the University of California +# Copyright (c) 2023 The Regents of the University of California # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,6 +28,7 @@ import tempfile import os from typing import Dict +import json from gem5.resources.downloader import ( _get_resources_json_at_path, @@ -42,48 +43,102 @@ @classmethod def setUpClass(cls) -> str: """ - This creates a simple resource.json temp file for testing purposes. + This creates a simple resources collection for testing """ - - file_contents = ( - "{" - + f'"version" : "{_resources_json_version_required()}",' - + """ - "url_base" : "http://dist.gem5.org/dist/v21-2", - "previous-versions" : {}, - "resources": [ - { - "type": "resource", - "name" : "riscv-disk-img", - "documentation" : "A simple RISCV disk image based on busybox.", - "architecture": "RISCV", - "is_zipped" : true, - "md5sum" : "d6126db9f6bed7774518ae25aa35f153", - "url": "{url_base}/images/riscv/busybox/riscv-disk.img.gz", - "source" : "src/riscv-fs", - "additional_metadata" : { - "root_partition": null - } - }, - { - "type": "resource", - "name" : "riscv-lupio-busybox-img", - "documentation" : "A RISCV disk image, based on busybox, to ...", - "architecture": "RISCV", - "is_zipped" : true, - "md5sum" : "e5bee8a31f45f4803f87c0d781553ccc", - "url": "{url_base}/images/riscv/busybox/riscv-lupio-busybox.img", - "source" : "src/lupv", - "additional_metadata" : { - "root_partition": "1" - } - } - ] -} - """ - ) + file_contents = [ + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource", + "architecture": "X86", + "size": 13816, + "tags": ["asmtest", "testing", "riscv", "testing"], + "is_zipped": False, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + "example_usage": 'get_resource(resource_name="rv64mi-p-sbreak")', + }, + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource but double newer", + "architecture": "X86", + "size": 13816, + "tags": ["asmtest"], + "is_zipped": False, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "2.0.0", + "gem5_versions": ["23.1"], + "example_usage": 'get_resource(resource_name="rv64mi-p-sbreak")', + }, + { + "category": "simpoint", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": False, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": True, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": 'get_resource(resource_name="x86-print-this-1500-simpoints")', + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint", + ], + }, + { + "category": "file", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": False, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": True, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": ["23.0"], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": 'get_resource(resource_name="x86-print-this-1500-simpoints")', + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint", + ], + }, + ] file = tempfile.NamedTemporaryFile(mode="w", delete=False) - file.write(file_contents) + file.write(json.dumps(file_contents)) file.close() cls.file_path = file.name @@ -100,14 +155,16 @@ "create_temp_resources_json" has been loaded correctly into a Python dictionary. """ - self.assertTrue("resources" in json) - self.assertEquals(2, len(json["resources"])) - self.assertTrue("name" in json["resources"][0]) - self.assertEquals("riscv-disk-img", json["resources"][0]["name"]) - self.assertTrue("name" in json["resources"][1]) - self.assertEquals( - "riscv-lupio-busybox-img", json["resources"][1]["name"] - ) + self.assertEquals(4, len(json)) + self.assertTrue("id" in json[0]) + self.assertEquals("this-is-a-test-resource", json[0]["id"]) + self.assertEquals("binary", json[0]["category"]) + self.assertTrue("id" in json[1]) + self.assertEquals("this-is-a-test-resource", json[1]["id"]) + self.assertTrue("id" in json[2]) + self.assertEquals("test-version", json[2]["id"]) + self.assertTrue("id" in json[3]) + self.assertEquals("test-version", json[3]["id"]) def test_get_resources_json_at_path(self) -> None: # Tests the gem5.resources.downloader._get_resources_json_at_path() diff --git a/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py b/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py new file mode 100644 index 0000000..791d96c --- /dev/null +++ b/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py @@ -0,0 +1,192 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import os +import io +import contextlib +from pathlib import Path + +from gem5.resources.resource import * + +from gem5.resources.looppoint import ( + LooppointCsvLoader, + LooppointJsonLoader, +) + +from gem5.isas import ISA + +from _m5 import core + +from gem5.resources.client_api.client_wrapper import ClientWrapper +from unittest.mock import patch + +mock_json_path = Path(__file__).parent / "refs/obtain-resource.json" + +mock_config_json = { + "sources": { + "baba": { + "url": mock_json_path, + "isMongo": False, + } + }, +} + + +@patch( + "gem5.resources.client.clientwrapper", + new=ClientWrapper(mock_config_json), +) +class TestObtainResourcesCheck(unittest.TestCase): + @classmethod + def setUpClass(cls): + """Prior to running the suite we set the resource directory to + "ref/resource-specialization.json" + """ + os.environ["GEM5_RESOURCE_JSON"] = os.path.join( + os.path.realpath(os.path.dirname(__file__)), + "refs", + "obtain-resource.json", + ) + + @classmethod + def tearDownClass(cls) -> None: + """After running the suite we unset the gem5-resource JSON file, as to + not interfere with others tests. + """ + del os.environ["GEM5_RESOURCE_JSON"] + + def get_resource_dir(cls) -> str: + """To ensure the resources are cached to the same directory as all + other tests, this function returns the location of the testing + directories "resources" directory. + """ + return os.path.join( + os.path.realpath(os.path.dirname(__file__)), + os.pardir, + os.pardir, + os.pardir, + "gem5", + "resources", + ) + + def test_obtain_resources_no_version(self): + """Test that the resource loader returns latest version compatible with that version of gem5 when no version is specified.""" + gem5Version = core.gem5Version + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + ) + self.assertEquals("2.5.0", resource.get_resource_version()) + self.assertIsInstance(resource, BinaryResource) + # self.assertIn(gem5Version, resource.get_gem5_versions()) + self.assertEquals("test description", resource.get_description()) + self.assertEquals("src/test-source", resource.get_source()) + self.assertEquals(ISA.ARM, resource.get_architecture()) + + def test_obtain_resources_with_version_compatible(self): + gem5Version = core.gem5Version + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="1.7.0", + ) + self.assertEquals("1.7.0", resource.get_resource_version()) + self.assertIsInstance(resource, BinaryResource) + # self.assertIn(gem5Version, resource.get_gem5_versions()) + self.assertEquals( + "test description v1.7.0", resource.get_description() + ) + self.assertEquals("src/test-source", resource.get_source()) + self.assertEquals(ISA.ARM, resource.get_architecture()) + + def test_obtain_resources_with_version_incompatible(self): + resource = None + f = io.StringIO() + with contextlib.redirect_stderr(f): + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="1.5.0", + ) + self.assertTrue( + f"warn: Resource test-binary-resource with version 1.5.0 is not known to be compatible with gem5 version {core.gem5Version}. " + "This may cause problems with your simulation. This resource's compatibility with different gem5 versions can be found here: " + f"https://resources.gem5.org/resources/test-binary-resource/versions" + in f.getvalue() + ) + + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="1.5.0", + ) + self.assertEquals("1.5.0", resource.get_resource_version()) + self.assertIsInstance(resource, BinaryResource) + self.assertEquals( + "test description for 1.5.0", resource.get_description() + ) + self.assertEquals("src/test-source", resource.get_source()) + self.assertEquals(ISA.ARM, resource.get_architecture()) + + def test_obtain_resources_no_version_invalid_id(self): + with self.assertRaises(Exception) as context: + obtain_resource( + resource_id="invalid-id", + resource_directory=self.get_resource_dir(), + ) + self.assertTrue( + "Resource with ID 'invalid-id' not found." + in str(context.exception) + ) + + def test_obtain_resources_with_version_invalid_id(self): + with self.assertRaises(Exception) as context: + obtain_resource( + resource_id="invalid-id", + resource_directory=self.get_resource_dir(), + resource_version="1.7.0", + ) + self.assertTrue( + "Resource with ID 'invalid-id' not found." + in str(context.exception) + ) + + def test_obtain_resources_with_version_invalid_version(self): + with self.assertRaises(Exception) as context: + obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="3.0.0", + ) + print("context.exception: ", context.exception) + print(str(context.exception)) + self.assertTrue( + f"Resource test-binary-resource with version '3.0.0'" + " not found.\nResource versions can be found at: " + f"https://resources.gem5.org/resources/test-binary-resource/versions" + in str(context.exception) + ) diff --git a/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py b/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py new file mode 100644 index 0000000..8f6674f --- /dev/null +++ b/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py @@ -0,0 +1,72 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import tempfile +import os +from typing import Dict + +from gem5.resources.downloader import ( + get_resources_json_obj, +) + + +class ResourceDownloadTestSuite(unittest.TestCase): + """Test cases for gem5.resources.downloader""" + + @classmethod + def setUpClass(cls) -> str: + pass + + def get_resource_json_by_id(self) -> None: + """Get a resource by its id""" + resources = get_resources_json_obj("test-version") + self.assertEqual(resources["id"], "test-version") + self.assertEqual(resources["resource_version"], "2.0.0") + + def get_resource_json_invalid_id(self) -> None: + """Should throw an exception when trying to get a resource that doesn't exist""" + with self.assertRaises(Exception) as context: + get_resources_json_obj("this-resource-doesnt-exist") + self.assertTrue( + f"Error: Resource with name 'this-resource-doesnt-exist' does not exist" + in str(context.exception) + ) + + def get_resource_json_by_id_and_version(self) -> None: + """Get a resource by its id and version""" + resources = get_resources_json_obj("test-version", "1.0.0") + self.assertEqual(resources["id"], "test-version") + self.assertEqual(resources["resource_version"], "1.0.0") + + def get_resource_json_by_id_and_invalid_version(self) -> None: + """Get a resource by its id and an invalid version (does not exist)""" + with self.assertRaises(Exception) as context: + get_resources_json_obj("test-version", "3.0.0") + self.assertTrue( + f"Specified Version 3.0.0 does not exist for the resource 'test-version'." + in str(context.exception) + ) diff --git a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py index 660bf5f..5c22a73 100644 --- a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py +++ b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py @@ -37,7 +37,25 @@ from gem5.isas import ISA +from gem5.resources.client_api.client_wrapper import ClientWrapper +from unittest.mock import patch +mock_json_path = Path(__file__).parent / "refs/resource-specialization.json" + +mock_config_json = { + "sources": { + "baba": { + "url": mock_json_path, + "isMongo": False, + } + }, +} + + +@patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json), +) class ResourceSpecializationSuite(unittest.TestCase): """This suite tests that `gem5.resource.resource` casts to the correct `AbstractResource` specialization when using the `obtain_resource` @@ -79,14 +97,14 @@ def test_binary_resource(self) -> None: """Tests the loading of of a BinaryResource""" resource = obtain_resource( - resource_name="binary-example", + resource_id="binary-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, BinaryResource) self.assertEquals( - "binary-example documentation.", resource.get_documentation() + "binary-example documentation.", resource.get_description() ) self.assertEquals("src/simple", resource.get_source()) self.assertEquals(ISA.ARM, resource.get_architecture()) @@ -94,14 +112,14 @@ def test_kernel_resource(self) -> None: """Tests the loading of a KernelResource.""" resource = obtain_resource( - resource_name="kernel-example", + resource_id="kernel-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, KernelResource) self.assertEquals( - "kernel-example documentation.", resource.get_documentation() + "kernel-example documentation.", resource.get_description() ) self.assertEquals("src/linux-kernel", resource.get_source()) self.assertEquals(ISA.RISCV, resource.get_architecture()) @@ -109,14 +127,14 @@ def test_bootloader_resource(self) -> None: """Tests the loading of a BootloaderResource.""" resource = obtain_resource( - resource_name="bootloader-example", + resource_id="bootloader-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, BootloaderResource) self.assertEquals( - "bootloader documentation.", resource.get_documentation() + "bootloader documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) self.assertIsNone(resource.get_architecture()) @@ -124,14 +142,14 @@ def test_disk_image_resource(self) -> None: """Tests the loading of a DiskImageResource.""" resource = obtain_resource( - resource_name="disk-image-example", + resource_id="disk-image-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, DiskImageResource) self.assertEquals( - "disk-image documentation.", resource.get_documentation() + "disk-image documentation.", resource.get_description() ) self.assertEquals("src/x86-ubuntu", resource.get_source()) self.assertEquals("1", resource.get_root_partition()) @@ -139,40 +157,40 @@ def test_checkpoint_resource(self) -> None: """Tests the loading of a CheckpointResource.""" resource = obtain_resource( - resource_name="checkpoint-example", + resource_id="checkpoint-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, CheckpointResource) self.assertEquals( - "checkpoint-example documentation.", resource.get_documentation() + "checkpoint-example documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) def test_git_resource(self) -> None: """Tests the loading of a GitResource.""" resource = obtain_resource( - resource_name="git-example", + resource_id="git-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, GitResource) - self.assertIsNone(resource.get_documentation()) + self.assertIsNone(resource.get_description()) self.assertIsNone(resource.get_source()) def test_simpoint_directory_resource(self) -> None: """Tests the loading of a Simpoint directory resource.""" resource = obtain_resource( - resource_name="simpoint-directory-example", + resource_id="simpoint-directory-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, SimpointDirectoryResource) self.assertEquals( - "simpoint directory documentation.", resource.get_documentation() + "simpoint directory documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) @@ -199,14 +217,14 @@ def test_simpoint_resource(self) -> None: """Tests the loading of a Simpoint resource.""" resource = obtain_resource( - resource_name="simpoint-example", + resource_id="simpoint-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, SimpointResource) self.assertEquals( - "simpoint documentation.", resource.get_documentation() + "simpoint documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) self.assertIsNone(resource.get_local_path()) @@ -219,26 +237,27 @@ def test_file_resource(self) -> None: """Tests the loading of a FileResource.""" resource = obtain_resource( - resource_name="file-example", + resource_id="file-example", resource_directory=self.get_resource_dir(), + resource_version="1.0.0", ) self.assertIsInstance(resource, FileResource) - self.assertIsNone(resource.get_documentation()) + self.assertIsNone(resource.get_description()) self.assertIsNone(resource.get_source()) def test_directory_resource(self) -> None: """Tests the loading of a DirectoryResource.""" resource = obtain_resource( - resource_name="directory-example", + resource_id="directory-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, DirectoryResource) self.assertEquals( - "directory-example documentation.", resource.get_documentation() + "directory-example documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) @@ -247,7 +266,7 @@ pinpoints csv file.""" resource = obtain_resource( - resource_name="looppoint-pinpoint-csv-resource", + resource_id="looppoint-pinpoint-csv-resource", resource_directory=self.get_resource_dir(), ) @@ -258,7 +277,7 @@ self.assertIsInstance(resource, LooppointCsvLoader) self.assertEquals( - "A looppoint pinpoints csv file.", resource.get_documentation() + "A looppoint pinpoints csv file.", resource.get_description() ) self.assertIsNone(resource.get_source()) @@ -267,8 +286,9 @@ Looppoint JSON file.""" resource = obtain_resource( - resource_name="looppoint-json-restore-resource-region-1", + resource_id="looppoint-json-restore-resource-region-1", resource_directory=self.get_resource_dir(), + resource_version="1.0.0", ) self.assertIsInstance(resource, LooppointJsonResource) @@ -278,6 +298,6 @@ self.assertTrue("1" in resource.get_regions()) self.assertEquals( - "A looppoint json file resource.", resource.get_documentation() + "A looppoint json file resource.", resource.get_description() ) self.assertIsNone(resource.get_source()) diff --git a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py index 2bc31f5..b898fae 100644 --- a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 The Regents of the University of California +# Copyright (c) 2023 The Regents of the University of California # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -36,6 +36,29 @@ from typing import Dict +from gem5.resources.client_api.client_wrapper import ClientWrapper +from unittest.mock import patch +from pathlib import Path + +mock_config_json1 = { + "sources": { + "baba": { + "url": Path(__file__).parent + / "refs/workload-checks-custom-workload.json", + "isMongo": False, + } + }, +} + +mock_config_json2 = { + "sources": { + "baba": { + "url": Path(__file__).parent / "refs/workload-checks.json", + "isMongo": False, + } + }, +} + class CustomWorkloadTestSuite(unittest.TestCase): """ @@ -43,8 +66,11 @@ """ @classmethod + @patch( + "gem5.resources.client.clientwrapper", + new=ClientWrapper(mock_config_json1), + ) def setUpClass(cls) -> None: - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( os.path.realpath(os.path.dirname(__file__)), "refs", @@ -114,8 +140,7 @@ "test", self.custom_workload.get_parameters()["binary"] ) - # We set the overridden parameter back to it's old value. - self.custom_workload.set_parameter("binary", old_value) + # We set the overridden parameter back to it's old valu self.custom_workload.set_parameter("binary", old_value) class WorkloadTestSuite(unittest.TestCase): @@ -124,8 +149,11 @@ """ @classmethod + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json2), + ) def setUpClass(cls): - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( os.path.realpath(os.path.dirname(__file__)), "refs", @@ -157,9 +185,9 @@ self.assertTrue("kernel" in parameters) self.assertTrue(isinstance(parameters["kernel"], BinaryResource)) - self.assertTrue("disk_image" in parameters) + self.assertTrue("disk-image" in parameters) self.assertTrue( - isinstance(parameters["disk_image"], DiskImageResource) + isinstance(parameters["disk-image"], DiskImageResource) ) self.assertTrue("readfile_contents" in parameters) diff --git a/tests/pyunit/stdlib/resources/refs/mongo_dup_mock.json b/tests/pyunit/stdlib/resources/refs/mongo_dup_mock.json new file mode 100644 index 0000000..fe87cc8 --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/mongo_dup_mock.json @@ -0,0 +1,30 @@ +[ + { + "category": "file", + "id": "test-duplicate", + "description": "test", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + } +] diff --git a/tests/pyunit/stdlib/resources/refs/mongo_mock.json b/tests/pyunit/stdlib/resources/refs/mongo_mock.json new file mode 100644 index 0000000..b6376cc --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/mongo_mock.json @@ -0,0 +1,56 @@ +[ + { + "category": "disk-image", + "id": "x86-ubuntu-18.04-img", + "description": "This is a test resource", + "architecture": "X86", + "size": 688119691, + "tags": [ + "x86", + "fullsystem" + ], + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "source": "src/x86-ubuntu", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "root_partition": "1", + "code_examples": [], + "license": "", + "author": [ + "Ayaz Akram" + ], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" + }, + { + "category": "disk-image", + "id": "x86-ubuntu-18.04-img", + "description": "A disk image containing Ubuntu 18.04 for x86. This image will run an `m5 readfile` instruction after booting. If no script file is specified an `m5 exit` instruction will be executed.", + "architecture": "X86", + "size": 688119691, + "tags": [ + "x86", + "fullsystem" + ], + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "source": "src/x86-ubuntu", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "root_partition": "1", + "code_examples": [], + "license": "", + "author": [ + "Ayaz Akram" + ], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "resource_version": "1.1.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" + } +] diff --git a/tests/pyunit/stdlib/resources/refs/obtain-resource.json b/tests/pyunit/stdlib/resources/refs/obtain-resource.json new file mode 100644 index 0000000..fac95e1 --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/obtain-resource.json @@ -0,0 +1,59 @@ +[ + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "2.5.0", + "gem5_versions": [ + "25.0" + ] + }, + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "2.0.0", + "gem5_versions": [ + "develop" + ] + }, + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description v1.7.0", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "1.7.0", + "gem5_versions": [ + "develop" + ] + }, + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description for 1.5.0", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "1.5.0", + "gem5_versions": [ + "21.1", + "22.1" + ] + } +] diff --git a/tests/pyunit/stdlib/resources/refs/resource-specialization.json b/tests/pyunit/stdlib/resources/refs/resource-specialization.json index c4d5eb4..1129f1b 100644 --- a/tests/pyunit/stdlib/resources/refs/resource-specialization.json +++ b/tests/pyunit/stdlib/resources/refs/resource-specialization.json @@ -1,132 +1,181 @@ - -{ - "version" : "develop", - "url_base" : "http://dist.gem5.org/dist/v22-1", - "previous-versions" : { - "develop" : "https://gem5.googlesource.com/public/gem5-resources/+/refs/heads/develop/resources.json?format=TEXT", - "21.2" : "http://resources.gem5.org/prev-resources-json/resources-21-2.json" +[ + { + "category": "kernel", + "id": "kernel-example", + "description": "kernel-example documentation.", + "architecture": "RISCV", + "is_zipped": false, + "md5sum": "60a53c7d47d7057436bf4b9df707a841", + "url": "http://dist.gem5.org/dist/develop/kernels/x86/static/vmlinux-5.4.49", + "source": "src/linux-kernel", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] }, - "resources": [ - { - "type" : "kernel", - "name" : "kernel-example", - "documentation" : "kernel-example documentation.", - "architecture" : "RISCV", - "is_zipped" : false, - "md5sum" : "60a53c7d47d7057436bf4b9df707a841", - "url" : "{url_base}/kernels/x86/static/vmlinux-5.4.49", - "source" : "src/linux-kernel" - }, - { - "type" : "disk-image", - "name" : "disk-image-example", - "documentation" : "disk-image documentation.", - "architecture" : "X86", - "is_zipped" : true, - "md5sum" : "90e363abf0ddf22eefa2c7c5c9391c49", - "url" : "{url_base}/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", - "source" : "src/x86-ubuntu", - "root_partition": "1" - }, - { - "type" : "binary", - "name" : "binary-example", - "documentation" : "binary-example documentation.", - "architecture" : "ARM", - "is_zipped" : false, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url" : "{url_base}/test-progs/hello/bin/arm/linux/hello64-static", - "source" : "src/simple" - }, - { - "type" : "bootloader", - "name" : "bootloader-example", - "documentation" : "bootloader documentation.", - "is_zipped" : false, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url" : "{url_base}/test-progs/hello/bin/arm/linux/hello64-static" - }, - { - "type" : "checkpoint", - "name" : "checkpoint-example", - "documentation" : "checkpoint-example documentation.", - "architecture": "RISCV", - "is_zipped" : false, - "md5sum" : "3a57c1bb1077176c4587b8a3bf4f8ace", - "source" : null, - "is_tar_archive" : true, - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar" - }, - { - "type" : "git", - "name" : "git-example", - "documentation" : null, - "is_zipped" : false, - "is_tar_archive" : true, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar" - }, - { - "type" : "file", - "name" : "file-example", - "documentation" : null, - "is_zipped" : false, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar", - "source" : null - }, - { - "type" : "directory", - "name" : "directory-example", - "documentation" : "directory-example documentation.", - "is_zipped" : false, - "md5sum" : "3a57c1bb1077176c4587b8a3bf4f8ace", - "source" : null, - "is_tar_archive" : true, - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar" - }, - { - "type": "simpoint-directory", - "name": "simpoint-directory-example", - "documentation": "simpoint directory documentation.", - "is_zipped" : false, - "md5sum" : "3fcffe3956c8a95e3fb82e232e2b41fb", - "source" : null, - "is_tar_archive" : true, - "url": "{url_base}/simpoints/x86-print-this-15000-simpoints-20221013.tar", - "simpoint_interval": 1000000, - "warmup_interval": 1000000, - "simpoint_file": "simpoint.simpt", - "weight_file": "simpoint.weight", - "workload_name": "Example Workload" - }, - { - "type": "simpoint", - "name": "simpoint-example", - "documentation": "simpoint documentation.", - "simpoint_interval": 1000000, - "warmup_interval": 23445, - "simpoint_list" : [2,3,4,15], - "weight_list" : [0.1, 0.2, 0.4, 0.3] - }, - { - "type": "looppoint-pinpoint-csv", - "name": "looppoint-pinpoint-csv-resource", - "documentation" : "A looppoint pinpoints csv file.", - "is_zipped" : false, - "md5sum" : "199ab22dd463dc70ee2d034bfe045082", - "url": "http://dist.gem5.org/dist/develop/pinpoints/x86-matrix-multiply-omp-100-8-global-pinpoints-20230127", - "source" : null - }, - { - "type": "looppoint-json", - "name": "looppoint-json-restore-resource-region-1", - "documentation" : "A looppoint json file resource.", - "is_zipped" : false, - "region_id" : "1", - "md5sum" : "a71ed64908b082ea619b26b940a643c1", - "url": "http://dist.gem5.org/dist/develop/looppoints/x86-matrix-multiply-omp-100-8-looppoint-json-20230128", - "source" : null - } - ] -} + { + "category": "disk-image", + "id": "disk-image-example", + "description": "disk-image documentation.", + "architecture": "X86", + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "source": "src/x86-ubuntu", + "root_partition": "1", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "binary", + "id": "binary-example", + "description": "binary-example documentation.", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/simple", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "bootloader", + "id": "bootloader-example", + "description": "bootloader documentation.", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "checkpoint", + "id": "checkpoint-example", + "description": "checkpoint-example documentation.", + "architecture": "RISCV", + "is_zipped": false, + "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", + "source": null, + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "git", + "id": "git-example", + "description": null, + "is_zipped": false, + "is_tar_archive": true, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "file", + "id": "file-example", + "description": null, + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "source": null, + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "directory", + "id": "directory-example", + "description": "directory-example documentation.", + "is_zipped": false, + "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", + "source": null, + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "simpoint-directory", + "id": "simpoint-directory-example", + "description": "simpoint directory documentation.", + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "source": null, + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "simpoint_file": "simpoint.simpt", + "weight_file": "simpoint.weight", + "workload_name": "Example Workload", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "simpoint", + "id": "simpoint-example", + "description": "simpoint documentation.", + "simpoint_interval": 1000000, + "warmup_interval": 23445, + "simpoint_list": [ + 2, + 3, + 4, + 15 + ], + "weight_list": [ + 0.1, + 0.2, + 0.4, + 0.3 + ], + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "looppoint-pinpoint-csv", + "id": "looppoint-pinpoint-csv-resource", + "description": "A looppoint pinpoints csv file.", + "is_zipped": false, + "md5sum": "199ab22dd463dc70ee2d034bfe045082", + "url": "http://dist.gem5.org/dist/develop/pinpoints/x86-matrix-multiply-omp-100-8-global-pinpoints-20230127", + "source": null, + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "looppoint-json", + "id": "looppoint-json-restore-resource-region-1", + "description": "A looppoint json file resource.", + "is_zipped": false, + "region_id": "1", + "md5sum": "a71ed64908b082ea619b26b940a643c1", + "url": "http://dist.gem5.org/dist/develop/looppoints/x86-matrix-multiply-omp-100-8-looppoint-json-20230128", + "source": null, + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + } +] diff --git a/tests/pyunit/stdlib/resources/refs/resources.json b/tests/pyunit/stdlib/resources/refs/resources.json new file mode 100644 index 0000000..812caef --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/resources.json @@ -0,0 +1,329 @@ +[ + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource", + "architecture": "X86", + "size": 13816, + "tags": [ + "asmtest", + "testing", + "riscv", + "testing" + ], + "is_zipped": false, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" + }, + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource but newer", + "architecture": "X86", + "size": 13816, + "tags": [ + "asmtest", + "testing", + "riscv", + "testing", + "new" + ], + "is_zipped": false, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "1.1.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" + }, + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource but double newer", + "architecture": "X86", + "size": 13816, + "tags": [ + "asmtest" + ], + "is_zipped": false, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "2.0.0", + "gem5_versions": [ + "23.1" + ], + "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" + }, + { + "category": "simpoint", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "test-duplicate", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "aa", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "asfsaf", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "simpoint-resource", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "bat43f34fman", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "adadadas", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "disk-image", + "id": "x86-ubuntu-18.04-img", + "description": "This is a test resource", + "architecture": "X86", + "size": 688119691, + "tags": [ + "x86", + "fullsystem" + ], + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "source": "src/x86-ubuntu", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "root_partition": "1", + "code_examples": [], + "license": "", + "author": [ + "Ayaz Akram" + ], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "resource_version": "2.0.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" + } +] diff --git a/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json b/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json index a9dd2aa..8349b92 100644 --- a/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json +++ b/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json @@ -1,17 +1,16 @@ -{ - "version" : null, - "url_base" : "http://dist.gem5.org/dist/v22-0", - "previous-versions" : {}, - "resources": [ - { - "type" : "binary", - "name" : "x86-hello64-static", - "documentation" : "A 'Hello World!' binary.", - "architecture" : "X86", - "is_zipped" : false, - "md5sum" : "dbf120338b37153e3334603970cebd8c", - "url" : "{url_base}/test-progs/hello/bin/x86/linux/hello64-static", - "source" : "src/simple" - } - ] -} +[ + { + "category": "binary", + "id": "x86-hello64-static", + "description": "A 'Hello World!' binary.", + "architecture": "X86", + "is_zipped": false, + "md5sum": "dbf120338b37153e3334603970cebd8c", + "url": "{url_base}/test-progs/hello/bin/x86/linux/hello64-static", + "source": "src/simple", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + } +] diff --git a/tests/pyunit/stdlib/resources/refs/workload-checks.json b/tests/pyunit/stdlib/resources/refs/workload-checks.json index 4f7e76b..d19396b 100644 --- a/tests/pyunit/stdlib/resources/refs/workload-checks.json +++ b/tests/pyunit/stdlib/resources/refs/workload-checks.json @@ -1,40 +1,48 @@ -{ - "url_base" : "http://dist.gem5.org/dist/v22-0", - "previous-versions" : {}, - "resources": [ - { - "type" : "kernel", - "name" : "x86-linux-kernel-5.2.3", - "documentation" : "The linux kernel (v5.2.3), compiled to X86.", - "architecture" : "X86", - "is_zipped" : false, - "md5sum" : "4838c99b77d33c8307b939c16624e4ac", - "url" : "{url_base}/kernels/x86/static/vmlinux-5.2.3", - "source" : "src/linux-kernel" +[ + { + "category": "kernel", + "id": "x86-linux-kernel-5.2.3", + "description": "The linux kernel (v5.2.3), compiled to X86.", + "architecture": "X86", + "is_zipped": false, + "md5sum": "4838c99b77d33c8307b939c16624e4ac", + "url": "{url_base}/kernels/x86/static/vmlinux-5.2.3", + "source": "src/linux-kernel", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "disk-image", + "id": "x86-ubuntu-18.04-img", + "description": "A disk image containing Ubuntu 18.04 for x86..", + "architecture": "X86", + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "url": "{url_base}/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "source": "src/x86-ubuntu", + "root_partition": "1", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "workload", + "id": "simple-boot", + "description": "Description of workload here", + "function": "set_kernel_disk_workload", + "resources": { + "kernel": "x86-linux-kernel-5.2.3", + "disk-image": "x86-ubuntu-18.04-img" }, - { - "type" : "disk-image", - "name" : "x86-ubuntu-18.04-img", - "documentation" : "A disk image containing Ubuntu 18.04 for x86..", - "architecture" : "X86", - "is_zipped" : true, - "md5sum" : "90e363abf0ddf22eefa2c7c5c9391c49", - "url" : "{url_base}/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", - "source" : "src/x86-ubuntu", - "root_partition": "1" + "additional_params": { + "readfile_contents": "echo 'Boot successful'; m5 exit" }, - { - "type" : "workload", - "name" : "simple-boot", - "documentation" : "Description of workload here", - "function" : "set_kernel_disk_workload", - "resources" : { - "kernel" : "x86-linux-kernel-5.2.3", - "disk_image" : "x86-ubuntu-18.04-img" - }, - "additional_params" : { - "readfile_contents" : "echo 'Boot successful'; m5 exit" - } - } - ] -} + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + } +] -- To view, visit https://gem5-review.googlesource.com/c/public/gem5/+/71278?usp=email To unsubscribe, or for help writing mail filters, visit https://gem5-review.googlesource.com/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: public/gem5 Gerrit-Branch: release-staging-v23-0 Gerrit-Change-Id: Ia9bf47f7900763827fd5e873bcd663cc3ecdba40 Gerrit-Change-Number: 71278 Gerrit-PatchSet: 2 Gerrit-Owner: Bobby Bruce <bbruce@ucdavis.edu> Gerrit-Reviewer: Bobby Bruce <bbruce@ucdavis.edu> Gerrit-Reviewer: Jason Lowe-Power <jason@lowepower.com> Gerrit-Reviewer: kokoro <noreply+kokoro@google.com>