# The resources.json file can change at any time, but to avoid
      # excessive retrieval we cache a version locally and use it for up  

      run_tar_extract = (
          untar

              def is_within_directory(directory, target):

                  for member in tar.getmembers():
                      member_path = os.path.join(path, member.name)
                      if not is_within_directory(path, member_path):

      self._local_path = local_path

      super().__init__(
          local_path=local_path,

      super().__init__(
          local_path=local_path,

      super().__init__(
          local_path=local_path,

  def __init__(

      ```json
      {

  def _create_temp_directory(self) -> Path:

      resource = obtain_resource(

      self.assertEquals(

      resource = obtain_resource(

      self.assertEquals(

  @classmethod

  @classmethod

Kunal Pai has uploaded this change for review. ( https://gem5-review.googlesource.com/c/public/gem5/+/70858?usp=email ) Change subject: resources, tests, configs: Introduce gem5 Vision to resources ...................................................................... resources, tests, configs: Introduce gem5 Vision to resources This patch makes changes to resources based on the gem5 Vision project. Firstly, a MongoDB database is supported. A JSON database's support is continued. The JSON can either be a local path or a raw GitHub link. The data for these databases is stored in src/python under "gem5-config.json". This will be used by default. However, the configuration can be overridden: - by providing a path using the GEM5_RESOURCES_CONFIG env variable. - by placing a gem5-config.json file in the current working directory. An AbstractClient is an abstract class that implements searching and sorting relevant to the databases. Databases is an optional list that can be passed while defining any Resource class and obtain_resource. These databases can be defined in the config JSON. Resource version is a new feature introduced. It is a string and decouples a resource from the gem5 Version. Example of a version is "1.0.0". It is an optional field that can be passed while defining any Resource class and obtain_resource. By default, it picks the latest version compatible with the gem5 Version of the user. A gem5 resource schema now has additional fields. These are: - source_url: Stores URL of GitHub Source of the resource. - license: License information of the resource. - tags: Words to identify a resource better, like hello for hello-world - example_usage: How to use the resource in a simulation. - gem5_versions: List of gem5 versions that resource is compatible with. - resource_version: The version of the resource itself. - size: The download size of the resource, if it exists. - code_examples: List of objects. These objects contain the path to where a resource is used in gem5 example config scripts, and if the resource itself is used in tests or not. - category: Category of the resource, as defined by classes in src/python/gem5/resources/resource.py. Some fields have been renamed: - "name" is changed to "id" - "documentation" is changed to "description" Besides these, the schema also supports resource specialization. It adds fields relevant to a specific resource as specified in src/python/gem5/resources/resource.py These changes have been made to better present information on the new gem5 Resources website. But, they do not affect the way resources are used by a gem5 user. This patch is also backwards compatible. Existing code doesn't break with this new infrastructure. Also, refs in the tests have been changed to match this new schema. Tests have been changed to work with the two clients. Change-Id: Ia9bf47f7900763827fd5e873bcd663cc3ecdba40 Co-authored-by: Kunal Pai <kunpai@ucdavis.edu> Co-authored-by: Parth Shah <helloparthshah@gmail.com> Co-authored-by: Harshil Patel <harshilp2107@gmail.com> Co-authored-by: aarsli <arsli@ucdavis.edu> --- M .gitignore M src/python/SConscript A src/python/gem5-config.json A src/python/gem5/resources/client_api/__init__.py A src/python/gem5/resources/client_api/client.py A src/python/gem5/resources/client_api/jsonclient.py A src/python/gem5/resources/client_api/mongoclient.py A src/python/gem5/resources/client_wrapper.py M src/python/gem5/resources/downloader.py M src/python/gem5/resources/looppoint.py M src/python/gem5/resources/resource.py M src/python/gem5/resources/workload.py M src/python/gem5/utils/simpoint.py A tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py M tests/pyunit/stdlib/resources/pyunit_downloader_checks.py M tests/pyunit/stdlib/resources/pyunit_md5_utils_check.py A tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py A tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py M tests/pyunit/stdlib/resources/pyunit_resource_specialization.py M tests/pyunit/stdlib/resources/pyunit_workload_checks.py A tests/pyunit/stdlib/resources/refs/mongo_mock.json A tests/pyunit/stdlib/resources/refs/obtain-resource.json M tests/pyunit/stdlib/resources/refs/resource-specialization.json A tests/pyunit/stdlib/resources/refs/resources.json M tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json M tests/pyunit/stdlib/resources/refs/workload-checks.json 26 files changed, 2,545 insertions(+), 649 deletions(-) diff --git a/.gitignore b/.gitignore index 229a0d5..40338fb 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ configs/dram/lowp_sweep.cfg .pyenv .vscode +__pycache__ diff --git a/src/python/SConscript b/src/python/SConscript index 3b00b34..f6f7a3a 100644 --- a/src/python/SConscript +++ b/src/python/SConscript @@ -26,315 +26,441 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -Import('*') +Import("*") +Import("env") +PySource("gem5", "gem5/__init__.py") +PySource("gem5", "gem5/coherence_protocol.py") +PySource("gem5", "gem5/isas.py") +PySource("gem5", "gem5/runtime.py") +PySource("gem5.simulate", "gem5/simulate/__init__.py") +PySource("gem5.simulate", "gem5/simulate/simulator.py") +PySource("gem5.simulate", "gem5/simulate/exit_event.py") +PySource("gem5.simulate", "gem5/simulate/exit_event_generators.py") +PySource("gem5.components", "gem5/components/__init__.py") +PySource("gem5.components.boards", "gem5/components/boards/__init__.py") +PySource("gem5.components.boards", "gem5/components/boards/abstract_board.py") +PySource("gem5.components.boards", "gem5/components/boards/abstract_system_board.py") +PySource("gem5.components.boards", "gem5/components/boards/mem_mode.py") +PySource("gem5.components.boards", "gem5/components/boards/riscv_board.py") +PySource( + "gem5.components.boards.experimental", + "gem5/components/boards/experimental/__init__.py", +) +PySource( + "gem5.components.boards.experimental", + "gem5/components/boards/experimental/lupv_board.py", +) +PySource("gem5.components.boards", "gem5/components/boards/simple_board.py") +PySource("gem5.components.boards", "gem5/components/boards/test_board.py") +PySource("gem5.components.boards", "gem5/components/boards/x86_board.py") +PySource("gem5.components.boards", "gem5/components/boards/arm_board.py") +PySource("gem5.components.boards", "gem5/components/boards/kernel_disk_workload.py") +PySource("gem5.components.boards", "gem5/components/boards/se_binary_workload.py") +PySource( + "gem5.components.cachehierarchies", "gem5/components/cachehierarchies/__init__.py" +) +PySource( + "gem5.components.cachehierarchies", + "gem5/components/cachehierarchies/abstract_cache_hierarchy.py", +) +PySource( + "gem5.components.cachehierarchies", + "gem5/components/cachehierarchies/abstract_two_level_cache_hierarchy.py", +) +PySource( + "gem5.components.cachehierarchies", + "gem5/components/cachehierarchies/abstract_three_level_cache_hierarchy.py", +) +PySource( + "gem5.components.cachehierarchies.chi", + "gem5/components/cachehierarchies/chi/__init__.py", +) +PySource( + "gem5.components.cachehierarchies.chi", + "gem5/components/cachehierarchies/chi/private_l1_cache_hierarchy.py", +) +PySource( + "gem5.components.cachehierarchies.chi.nodes", + "gem5/components/cachehierarchies/chi/nodes/__init__.py", +) +PySource( + "gem5.components.cachehierarchies.chi.nodes", + "gem5/components/cachehierarchies/chi/nodes/abstract_node.py", +) +PySource( + "gem5.components.cachehierarchies.chi.nodes", + "gem5/components/cachehierarchies/chi/nodes/directory.py", +) +PySource( + "gem5.components.cachehierarchies.chi.nodes", + "gem5/components/cachehierarchies/chi/nodes/dma_requestor.py", +) +PySource( + "gem5.components.cachehierarchies.chi.nodes", + "gem5/components/cachehierarchies/chi/nodes/private_l1_moesi_cache.py", +) +PySource( + "gem5.components.cachehierarchies.chi.nodes", + "gem5/components/cachehierarchies/chi/nodes/memory_controller.py", +) +PySource( + "gem5.components.cachehierarchies.classic", + "gem5/components/cachehierarchies/classic/__init__.py", +) +PySource( + "gem5.components.cachehierarchies.classic", + "gem5/components/cachehierarchies/classic/" "abstract_classic_cache_hierarchy.py", +) +PySource( + "gem5.components.cachehierarchies.classic", + "gem5/components/cachehierarchies/classic/no_cache.py", +) +PySource( + "gem5.components.cachehierarchies.classic", + "gem5/components/cachehierarchies/classic/private_l1_cache_hierarchy.py", +) +PySource( + "gem5.components.cachehierarchies.classic", + "gem5/components/cachehierarchies/classic/" + "private_l1_private_l2_cache_hierarchy.py", +) +PySource( + "gem5.components.cachehierarchies.classic", + "gem5/components/cachehierarchies/classic/" + "private_l1_shared_l2_cache_hierarchy.py", +) +PySource( + "gem5.components.cachehierarchies.classic.caches", + "gem5/components/cachehierarchies/classic/caches/__init__.py", +) +PySource( + "gem5.components.cachehierarchies.classic.caches", + "gem5/components/cachehierarchies/classic/caches/l1dcache.py", +) +PySource( + "gem5.components.cachehierarchies.classic.caches", + "gem5/components/cachehierarchies/classic/caches/l1icache.py", +) +PySource( + "gem5.components.cachehierarchies.classic.caches", + "gem5/components/cachehierarchies/classic/caches/l2cache.py", +) +PySource( + "gem5.components.cachehierarchies.classic.caches", + "gem5/components/cachehierarchies/classic/caches/mmu_cache.py", +) +PySource( + "gem5.components.cachehierarchies.ruby", + "gem5/components/cachehierarchies/ruby/__init__.py", +) +PySource( + "gem5.components.cachehierarchies.ruby", + "gem5/components/cachehierarchies/ruby/abstract_ruby_cache_hierarchy.py", +) +PySource( + "gem5.components.cachehierarchies.ruby", + "gem5/components/cachehierarchies/ruby/mesi_two_level_cache_hierarchy.py", +) +PySource( + "gem5.components.cachehierarchies.ruby", + "gem5/components/cachehierarchies/ruby/" "mesi_three_level_cache_hierarchy.py", +) +PySource( + "gem5.components.cachehierarchies.ruby", + "gem5/components/cachehierarchies/ruby/mi_example_cache_hierarchy.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches", + "gem5/components/cachehierarchies/ruby/caches/__init__.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches", + "gem5/components/cachehierarchies/ruby/caches/abstract_directory.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches", + "gem5/components/cachehierarchies/ruby/caches/abstract_dma_controller.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches", + "gem5/components/cachehierarchies/ruby/caches/abstract_l1_cache.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches", + "gem5/components/cachehierarchies/ruby/caches/abstract_l2_cache.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches.mesi_two_level", + "gem5/components/cachehierarchies/ruby/caches/mesi_two_level/__init__.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches.mesi_two_level", + "gem5/components/cachehierarchies/ruby/caches/mesi_two_level/directory.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches.mesi_two_level", + "gem5/components/cachehierarchies/ruby/caches/mesi_two_level/" "dma_controller.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches.mesi_two_level", + "gem5/components/cachehierarchies/ruby/caches/mesi_two_level/l1_cache.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches.mesi_two_level", + "gem5/components/cachehierarchies/ruby/caches/mesi_two_level/l2_cache.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches.mesi_three_level", + "gem5/components/cachehierarchies/ruby/caches/mesi_three_level/" "__init__.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches.mesi_three_level", + "gem5/components/cachehierarchies/ruby/caches/mesi_three_level/" "directory.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches.mesi_three_level", + "gem5/components/cachehierarchies/ruby/caches/mesi_three_level/" + "dma_controller.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches.mesi_three_level", + "gem5/components/cachehierarchies/ruby/caches/mesi_three_level/" "l1_cache.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches.mesi_three_level", + "gem5/components/cachehierarchies/ruby/caches/mesi_three_level/" "l2_cache.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches.mesi_three_level", + "gem5/components/cachehierarchies/ruby/caches/mesi_three_level/" "l3_cache.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches.mi_example", + "gem5/components/cachehierarchies/ruby/caches/mi_example/__init__.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches.mi_example", + "gem5/components/cachehierarchies/ruby/caches/mi_example/directory.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches.mi_example", + "gem5/components/cachehierarchies/ruby/caches/mi_example/" "dma_controller.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.caches.mi_example", + "gem5/components/cachehierarchies/ruby/caches/mi_example/l1_cache.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.topologies", + "gem5/components/cachehierarchies/ruby/topologies/__init__.py", +) +PySource( + "gem5.components.cachehierarchies.ruby.topologies", + "gem5/components/cachehierarchies/ruby/topologies/simple_pt2pt.py", +) +PySource("gem5.components.memory", "gem5/components/memory/__init__.py") +PySource("gem5.components.memory", "gem5/components/memory/abstract_memory_system.py") +PySource("gem5.components.memory", "gem5/components/memory/dramsim_3.py") -PySource('gem5', 'gem5/__init__.py') -PySource('gem5', 'gem5/coherence_protocol.py') -PySource('gem5', 'gem5/isas.py') -PySource('gem5', 'gem5/runtime.py') -PySource('gem5.simulate', 'gem5/simulate/__init__.py') -PySource('gem5.simulate', 'gem5/simulate/simulator.py') -PySource('gem5.simulate', 'gem5/simulate/exit_event.py') -PySource('gem5.simulate', 'gem5/simulate/exit_event_generators.py') -PySource('gem5.components', 'gem5/components/__init__.py') -PySource('gem5.components.boards', 'gem5/components/boards/__init__.py') -PySource('gem5.components.boards', 'gem5/components/boards/abstract_board.py') -PySource('gem5.components.boards', - 'gem5/components/boards/abstract_system_board.py') -PySource('gem5.components.boards', 'gem5/components/boards/mem_mode.py') -PySource('gem5.components.boards', 'gem5/components/boards/riscv_board.py') -PySource('gem5.components.boards.experimental', - 'gem5/components/boards/experimental/__init__.py') -PySource('gem5.components.boards.experimental', - 'gem5/components/boards/experimental/lupv_board.py') -PySource('gem5.components.boards', 'gem5/components/boards/simple_board.py') -PySource('gem5.components.boards', 'gem5/components/boards/test_board.py') -PySource('gem5.components.boards', 'gem5/components/boards/x86_board.py') -PySource('gem5.components.boards', 'gem5/components/boards/arm_board.py') -PySource('gem5.components.boards', - "gem5/components/boards/kernel_disk_workload.py") -PySource('gem5.components.boards', - "gem5/components/boards/se_binary_workload.py") -PySource('gem5.components.cachehierarchies', - 'gem5/components/cachehierarchies/__init__.py') -PySource('gem5.components.cachehierarchies', - 'gem5/components/cachehierarchies/abstract_cache_hierarchy.py') -PySource('gem5.components.cachehierarchies', - 'gem5/components/cachehierarchies/abstract_two_level_cache_hierarchy.py') -PySource('gem5.components.cachehierarchies', - 'gem5/components/cachehierarchies/abstract_three_level_cache_hierarchy.py') -PySource('gem5.components.cachehierarchies.chi', - 'gem5/components/cachehierarchies/chi/__init__.py') -PySource('gem5.components.cachehierarchies.chi', - 'gem5/components/cachehierarchies/chi/private_l1_cache_hierarchy.py') -PySource('gem5.components.cachehierarchies.chi.nodes', - 'gem5/components/cachehierarchies/chi/nodes/__init__.py') -PySource('gem5.components.cachehierarchies.chi.nodes', - 'gem5/components/cachehierarchies/chi/nodes/abstract_node.py') -PySource('gem5.components.cachehierarchies.chi.nodes', - 'gem5/components/cachehierarchies/chi/nodes/directory.py') -PySource('gem5.components.cachehierarchies.chi.nodes', - 'gem5/components/cachehierarchies/chi/nodes/dma_requestor.py') -PySource('gem5.components.cachehierarchies.chi.nodes', - 'gem5/components/cachehierarchies/chi/nodes/private_l1_moesi_cache.py') -PySource('gem5.components.cachehierarchies.chi.nodes', - 'gem5/components/cachehierarchies/chi/nodes/memory_controller.py') -PySource('gem5.components.cachehierarchies.classic', - 'gem5/components/cachehierarchies/classic/__init__.py') -PySource('gem5.components.cachehierarchies.classic', - 'gem5/components/cachehierarchies/classic/' - 'abstract_classic_cache_hierarchy.py') -PySource('gem5.components.cachehierarchies.classic', - 'gem5/components/cachehierarchies/classic/no_cache.py') -PySource('gem5.components.cachehierarchies.classic', - 'gem5/components/cachehierarchies/classic/private_l1_cache_hierarchy.py') -PySource('gem5.components.cachehierarchies.classic', - 'gem5/components/cachehierarchies/classic/' - 'private_l1_private_l2_cache_hierarchy.py') -PySource('gem5.components.cachehierarchies.classic', - 'gem5/components/cachehierarchies/classic/' - 'private_l1_shared_l2_cache_hierarchy.py') -PySource('gem5.components.cachehierarchies.classic.caches', - 'gem5/components/cachehierarchies/classic/caches/__init__.py') -PySource('gem5.components.cachehierarchies.classic.caches', - 'gem5/components/cachehierarchies/classic/caches/l1dcache.py') -PySource('gem5.components.cachehierarchies.classic.caches', - 'gem5/components/cachehierarchies/classic/caches/l1icache.py') -PySource('gem5.components.cachehierarchies.classic.caches', - 'gem5/components/cachehierarchies/classic/caches/l2cache.py') -PySource('gem5.components.cachehierarchies.classic.caches', - 'gem5/components/cachehierarchies/classic/caches/mmu_cache.py') -PySource('gem5.components.cachehierarchies.ruby', - 'gem5/components/cachehierarchies/ruby/__init__.py') -PySource('gem5.components.cachehierarchies.ruby', - 'gem5/components/cachehierarchies/ruby/abstract_ruby_cache_hierarchy.py') -PySource('gem5.components.cachehierarchies.ruby', - 'gem5/components/cachehierarchies/ruby/mesi_two_level_cache_hierarchy.py') -PySource('gem5.components.cachehierarchies.ruby', - 'gem5/components/cachehierarchies/ruby/' - 'mesi_three_level_cache_hierarchy.py') -PySource('gem5.components.cachehierarchies.ruby', - 'gem5/components/cachehierarchies/ruby/mi_example_cache_hierarchy.py') -PySource('gem5.components.cachehierarchies.ruby.caches', - 'gem5/components/cachehierarchies/ruby/caches/__init__.py') -PySource('gem5.components.cachehierarchies.ruby.caches', - 'gem5/components/cachehierarchies/ruby/caches/abstract_directory.py') -PySource('gem5.components.cachehierarchies.ruby.caches', - 'gem5/components/cachehierarchies/ruby/caches/abstract_dma_controller.py') -PySource('gem5.components.cachehierarchies.ruby.caches', - 'gem5/components/cachehierarchies/ruby/caches/abstract_l1_cache.py') -PySource('gem5.components.cachehierarchies.ruby.caches', - 'gem5/components/cachehierarchies/ruby/caches/abstract_l2_cache.py') -PySource('gem5.components.cachehierarchies.ruby.caches.mesi_two_level', - 'gem5/components/cachehierarchies/ruby/caches/mesi_two_level/__init__.py') -PySource('gem5.components.cachehierarchies.ruby.caches.mesi_two_level', - 'gem5/components/cachehierarchies/ruby/caches/mesi_two_level/directory.py') -PySource('gem5.components.cachehierarchies.ruby.caches.mesi_two_level', - 'gem5/components/cachehierarchies/ruby/caches/mesi_two_level/' - 'dma_controller.py') -PySource('gem5.components.cachehierarchies.ruby.caches.mesi_two_level', - 'gem5/components/cachehierarchies/ruby/caches/mesi_two_level/l1_cache.py') -PySource('gem5.components.cachehierarchies.ruby.caches.mesi_two_level', - 'gem5/components/cachehierarchies/ruby/caches/mesi_two_level/l2_cache.py') -PySource('gem5.components.cachehierarchies.ruby.caches.mesi_three_level', - 'gem5/components/cachehierarchies/ruby/caches/mesi_three_level/' - '__init__.py') -PySource('gem5.components.cachehierarchies.ruby.caches.mesi_three_level', - 'gem5/components/cachehierarchies/ruby/caches/mesi_three_level/' - 'directory.py') -PySource('gem5.components.cachehierarchies.ruby.caches.mesi_three_level', - 'gem5/components/cachehierarchies/ruby/caches/mesi_three_level/' - 'dma_controller.py') -PySource('gem5.components.cachehierarchies.ruby.caches.mesi_three_level', - 'gem5/components/cachehierarchies/ruby/caches/mesi_three_level/' - 'l1_cache.py') -PySource('gem5.components.cachehierarchies.ruby.caches.mesi_three_level', - 'gem5/components/cachehierarchies/ruby/caches/mesi_three_level/' - 'l2_cache.py') -PySource('gem5.components.cachehierarchies.ruby.caches.mesi_three_level', - 'gem5/components/cachehierarchies/ruby/caches/mesi_three_level/' - 'l3_cache.py') -PySource('gem5.components.cachehierarchies.ruby.caches.mi_example', - 'gem5/components/cachehierarchies/ruby/caches/mi_example/__init__.py') -PySource('gem5.components.cachehierarchies.ruby.caches.mi_example', - 'gem5/components/cachehierarchies/ruby/caches/mi_example/directory.py') -PySource('gem5.components.cachehierarchies.ruby.caches.mi_example', - 'gem5/components/cachehierarchies/ruby/caches/mi_example/' - 'dma_controller.py') -PySource('gem5.components.cachehierarchies.ruby.caches.mi_example', - 'gem5/components/cachehierarchies/ruby/caches/mi_example/l1_cache.py') -PySource('gem5.components.cachehierarchies.ruby.topologies', - 'gem5/components/cachehierarchies/ruby/topologies/__init__.py') -PySource('gem5.components.cachehierarchies.ruby.topologies', - 'gem5/components/cachehierarchies/ruby/topologies/simple_pt2pt.py') -PySource('gem5.components.memory', 'gem5/components/memory/__init__.py') -PySource('gem5.components.memory', 'gem5/components/memory/abstract_memory_system.py') -PySource('gem5.components.memory', 'gem5/components/memory/dramsim_3.py') +if env["HAVE_DRAMSYS"]: + PySource("gem5.components.memory", "gem5/components/memory/dramsys.py") -if env['HAVE_DRAMSYS']: - PySource('gem5.components.memory', 'gem5/components/memory/dramsys.py') +PySource("gem5.components.memory", "gem5/components/memory/simple.py") +PySource("gem5.components.memory", "gem5/components/memory/memory.py") +PySource("gem5.components.memory", "gem5/components/memory/single_channel.py") +PySource("gem5.components.memory", "gem5/components/memory/multi_channel.py") +PySource("gem5.components.memory", "gem5/components/memory/hbm.py") +PySource( + "gem5.components.memory.dram_interfaces", + "gem5/components/memory/dram_interfaces/__init__.py", +) +PySource( + "gem5.components.memory.dram_interfaces", + "gem5/components/memory/dram_interfaces/ddr3.py", +) +PySource( + "gem5.components.memory.dram_interfaces", + "gem5/components/memory/dram_interfaces/ddr4.py", +) +PySource( + "gem5.components.memory.dram_interfaces", + "gem5/components/memory/dram_interfaces/gddr.py", +) +PySource( + "gem5.components.memory.dram_interfaces", + "gem5/components/memory/dram_interfaces/hbm.py", +) +PySource( + "gem5.components.memory.dram_interfaces", + "gem5/components/memory/dram_interfaces/hmc.py", +) +PySource( + "gem5.components.memory.dram_interfaces", + "gem5/components/memory/dram_interfaces/lpddr2.py", +) +PySource( + "gem5.components.memory.dram_interfaces", + "gem5/components/memory/dram_interfaces/lpddr3.py", +) +PySource( + "gem5.components.memory.dram_interfaces", + "gem5/components/memory/dram_interfaces/lpddr5.py", +) +PySource( + "gem5.components.memory.dram_interfaces", + "gem5/components/memory/dram_interfaces/wideio.py", +) +PySource("gem5.components.processors", "gem5/components/processors/__init__.py") +PySource("gem5.components.processors", "gem5/components/processors/abstract_core.py") +PySource( + "gem5.components.processors", + "gem5/components/processors/abstract_generator_core.py", +) +PySource( + "gem5.components.processors", "gem5/components/processors/abstract_generator.py" +) +PySource( + "gem5.components.processors", "gem5/components/processors/abstract_processor.py" +) +PySource( + "gem5.components.processors", "gem5/components/processors/complex_generator_core.py" +) +PySource( + "gem5.components.processors", "gem5/components/processors/complex_generator.py" +) +PySource("gem5.components.processors", "gem5/components/processors/cpu_types.py") +PySource( + "gem5.components.processors", "gem5/components/processors/gups_generator_core.py" +) +PySource("gem5.components.processors", "gem5/components/processors/gups_generator.py") +PySource( + "gem5.components.processors", "gem5/components/processors/gups_generator_ep.py" +) +PySource( + "gem5.components.processors", "gem5/components/processors/gups_generator_par.py" +) +PySource( + "gem5.components.processors", "gem5/components/processors/linear_generator_core.py" +) +PySource("gem5.components.processors", "gem5/components/processors/linear_generator.py") +PySource( + "gem5.components.processors", "gem5/components/processors/random_generator_core.py" +) +PySource("gem5.components.processors", "gem5/components/processors/random_generator.py") +PySource("gem5.components.processors", "gem5/components/processors/simple_core.py") +PySource("gem5.components.processors", "gem5/components/processors/base_cpu_core.py") +PySource("gem5.components.processors", "gem5/components/processors/simple_processor.py") +PySource( + "gem5.components.processors", "gem5/components/processors/base_cpu_processor.py" +) +PySource( + "gem5.components.processors", + "gem5/components/processors/simple_switchable_processor.py", +) +PySource( + "gem5.components.processors", "gem5/components/processors/switchable_processor.py" +) +PySource("gem5.utils", "gem5/utils/simpoint.py") +PySource( + "gem5.components.processors", "gem5/components/processors/traffic_generator_core.py" +) +PySource( + "gem5.components.processors", "gem5/components/processors/traffic_generator.py" +) +PySource("gem5.prebuilt", "gem5/prebuilt/__init__.py") +PySource("gem5.prebuilt.demo", "gem5/prebuilt/demo/__init__.py") +PySource("gem5.prebuilt.demo", "gem5/prebuilt/demo/x86_demo_board.py") +PySource("gem5.prebuilt.riscvmatched", "gem5/prebuilt/riscvmatched/__init__.py") +PySource( + "gem5.prebuilt.riscvmatched", "gem5/prebuilt/riscvmatched/riscvmatched_board.py" +) +PySource( + "gem5.prebuilt.riscvmatched", "gem5/prebuilt/riscvmatched/riscvmatched_cache.py" +) +PySource( + "gem5.prebuilt.riscvmatched", "gem5/prebuilt/riscvmatched/riscvmatched_processor.py" +) +PySource( + "gem5.prebuilt.riscvmatched", "gem5/prebuilt/riscvmatched/riscvmatched_core.py" +) +PySource("gem5.resources", "gem5/resources/__init__.py") +PySource("gem5.resources", "gem5/resources/client_wrapper.py") +PySource("gem5.resources", "gem5/resources/downloader.py") +PySource("gem5.resources", "gem5/resources/md5_utils.py") +PySource("gem5.resources", "gem5/resources/resource.py") +PySource("gem5.resources", "gem5/resources/workload.py") +PySource("gem5.resources", "gem5/resources/looppoint.py") +PySource("gem5.resources", "gem5/resources/elfie.py") +PySource("gem5.resources.client_api", "gem5/resources/client_api/__init__.py") +PySource("gem5.resources.client_api", "gem5/resources/client_api/jsonclient.py") +PySource("gem5.resources.client_api", "gem5/resources/client_api/mongoclient.py") +PySource("gem5.resources.client_api", "gem5/resources/client_api/client.py") +env.Command( + "gem5-config.json", "gem5-config.json", + Copy("$TARGET", "${SOURCE.srcpath}") +) +PySource("gem5.utils", "gem5/utils/__init__.py") +PySource("gem5.utils", "gem5/utils/filelock.py") +PySource("gem5.utils", "gem5/utils/override.py") +PySource("gem5.utils", "gem5/utils/progress_bar.py") +PySource("gem5.utils", "gem5/utils/requires.py") +PySource("gem5.utils.multiprocessing", "gem5/utils/multiprocessing/__init__.py") +PySource("gem5.utils.multiprocessing", "gem5/utils/multiprocessing/_command_line.py") +PySource("gem5.utils.multiprocessing", "gem5/utils/multiprocessing/context.py") +PySource("gem5.utils.multiprocessing", "gem5/utils/multiprocessing/popen_spawn_gem5.py") -PySource('gem5.components.memory', 'gem5/components/memory/simple.py') -PySource('gem5.components.memory', 'gem5/components/memory/memory.py') -PySource('gem5.components.memory', 'gem5/components/memory/single_channel.py') -PySource('gem5.components.memory', 'gem5/components/memory/multi_channel.py') -PySource('gem5.components.memory', 'gem5/components/memory/hbm.py') -PySource('gem5.components.memory.dram_interfaces', - 'gem5/components/memory/dram_interfaces/__init__.py') -PySource('gem5.components.memory.dram_interfaces', - 'gem5/components/memory/dram_interfaces/ddr3.py') -PySource('gem5.components.memory.dram_interfaces', - 'gem5/components/memory/dram_interfaces/ddr4.py') -PySource('gem5.components.memory.dram_interfaces', - 'gem5/components/memory/dram_interfaces/gddr.py') -PySource('gem5.components.memory.dram_interfaces', - 'gem5/components/memory/dram_interfaces/hbm.py') -PySource('gem5.components.memory.dram_interfaces', - 'gem5/components/memory/dram_interfaces/hmc.py') -PySource('gem5.components.memory.dram_interfaces', - 'gem5/components/memory/dram_interfaces/lpddr2.py') -PySource('gem5.components.memory.dram_interfaces', - 'gem5/components/memory/dram_interfaces/lpddr3.py') -PySource('gem5.components.memory.dram_interfaces', -'gem5/components/memory/dram_interfaces/lpddr5.py') -PySource('gem5.components.memory.dram_interfaces', - 'gem5/components/memory/dram_interfaces/wideio.py') -PySource('gem5.components.processors', - 'gem5/components/processors/__init__.py') -PySource('gem5.components.processors', - 'gem5/components/processors/abstract_core.py') -PySource('gem5.components.processors', - 'gem5/components/processors/abstract_generator_core.py') -PySource('gem5.components.processors', - 'gem5/components/processors/abstract_generator.py') -PySource('gem5.components.processors', - 'gem5/components/processors/abstract_processor.py') -PySource('gem5.components.processors', - 'gem5/components/processors/complex_generator_core.py') -PySource('gem5.components.processors', - 'gem5/components/processors/complex_generator.py') -PySource('gem5.components.processors', - 'gem5/components/processors/cpu_types.py') -PySource('gem5.components.processors', - 'gem5/components/processors/gups_generator_core.py') -PySource('gem5.components.processors', - 'gem5/components/processors/gups_generator.py') -PySource('gem5.components.processors', - 'gem5/components/processors/gups_generator_ep.py') -PySource('gem5.components.processors', - 'gem5/components/processors/gups_generator_par.py') -PySource('gem5.components.processors', - 'gem5/components/processors/linear_generator_core.py') -PySource('gem5.components.processors', - 'gem5/components/processors/linear_generator.py') -PySource('gem5.components.processors', - 'gem5/components/processors/random_generator_core.py') -PySource('gem5.components.processors', - 'gem5/components/processors/random_generator.py') -PySource('gem5.components.processors', - 'gem5/components/processors/simple_core.py') -PySource('gem5.components.processors', - 'gem5/components/processors/base_cpu_core.py') -PySource('gem5.components.processors', - 'gem5/components/processors/simple_processor.py') -PySource('gem5.components.processors', - 'gem5/components/processors/base_cpu_processor.py') -PySource('gem5.components.processors', - 'gem5/components/processors/simple_switchable_processor.py') -PySource('gem5.components.processors', - 'gem5/components/processors/switchable_processor.py') -PySource('gem5.utils', 'gem5/utils/simpoint.py') -PySource('gem5.components.processors', - 'gem5/components/processors/traffic_generator_core.py') -PySource('gem5.components.processors', - 'gem5/components/processors/traffic_generator.py') -PySource('gem5.prebuilt', 'gem5/prebuilt/__init__.py') -PySource('gem5.prebuilt.demo', 'gem5/prebuilt/demo/__init__.py') -PySource('gem5.prebuilt.demo', 'gem5/prebuilt/demo/x86_demo_board.py') -PySource('gem5.prebuilt.riscvmatched', - 'gem5/prebuilt/riscvmatched/__init__.py') -PySource('gem5.prebuilt.riscvmatched', - 'gem5/prebuilt/riscvmatched/riscvmatched_board.py') -PySource('gem5.prebuilt.riscvmatched', - 'gem5/prebuilt/riscvmatched/riscvmatched_cache.py') -PySource('gem5.prebuilt.riscvmatched', - 'gem5/prebuilt/riscvmatched/riscvmatched_processor.py') -PySource('gem5.prebuilt.riscvmatched', - 'gem5/prebuilt/riscvmatched/riscvmatched_core.py') -PySource('gem5.resources', 'gem5/resources/__init__.py') -PySource('gem5.resources', 'gem5/resources/downloader.py') -PySource('gem5.resources', 'gem5/resources/md5_utils.py') -PySource('gem5.resources', 'gem5/resources/resource.py') -PySource('gem5.resources', 'gem5/resources/workload.py') -PySource('gem5.resources', 'gem5/resources/looppoint.py') -PySource('gem5.resources', 'gem5/resources/elfie.py') -PySource('gem5.utils', 'gem5/utils/__init__.py') -PySource('gem5.utils', 'gem5/utils/filelock.py') -PySource('gem5.utils', 'gem5/utils/override.py') -PySource('gem5.utils', 'gem5/utils/progress_bar.py') -PySource('gem5.utils', 'gem5/utils/requires.py') -PySource('gem5.utils.multiprocessing', - 'gem5/utils/multiprocessing/__init__.py') -PySource('gem5.utils.multiprocessing', - 'gem5/utils/multiprocessing/_command_line.py') -PySource('gem5.utils.multiprocessing', - 'gem5/utils/multiprocessing/context.py') -PySource('gem5.utils.multiprocessing', - 'gem5/utils/multiprocessing/popen_spawn_gem5.py') +PySource("", "importer.py") +PySource("m5", "m5/__init__.py") +PySource("m5", "m5/SimObject.py") +PySource("m5", "m5/core.py") +PySource("m5", "m5/debug.py") +PySource("m5", "m5/event.py") +PySource("m5", "m5/main.py") +PySource("m5", "m5/options.py") +PySource("m5", "m5/params.py") +PySource("m5", "m5/proxy.py") +PySource("m5", "m5/simulate.py") +PySource("m5", "m5/ticks.py") +PySource("m5", "m5/trace.py") +PySource("m5.objects", "m5/objects/__init__.py") +PySource("m5.stats", "m5/stats/__init__.py") +PySource("m5.util", "m5/util/__init__.py") +PySource("m5.util", "m5/util/attrdict.py") +PySource("m5.util", "m5/util/convert.py") +PySource("m5.util", "m5/util/dot_writer.py") +PySource("m5.util", "m5/util/dot_writer_ruby.py") +PySource("m5.util", "m5/util/fdthelper.py") +PySource("m5.util", "m5/util/multidict.py") +PySource("m5.util", "m5/util/pybind.py") +PySource("m5.util", "m5/util/terminal.py") +PySource("m5.util", "m5/util/terminal_formatter.py") -PySource('', 'importer.py') -PySource('m5', 'm5/__init__.py') -PySource('m5', 'm5/SimObject.py') -PySource('m5', 'm5/core.py') -PySource('m5', 'm5/debug.py') -PySource('m5', 'm5/event.py') -PySource('m5', 'm5/main.py') -PySource('m5', 'm5/options.py') -PySource('m5', 'm5/params.py') -PySource('m5', 'm5/proxy.py') -PySource('m5', 'm5/simulate.py') -PySource('m5', 'm5/ticks.py') -PySource('m5', 'm5/trace.py') -PySource('m5.objects', 'm5/objects/__init__.py') -PySource('m5.stats', 'm5/stats/__init__.py') -PySource('m5.util', 'm5/util/__init__.py') -PySource('m5.util', 'm5/util/attrdict.py') -PySource('m5.util', 'm5/util/convert.py') -PySource('m5.util', 'm5/util/dot_writer.py') -PySource('m5.util', 'm5/util/dot_writer_ruby.py') -PySource('m5.util', 'm5/util/fdthelper.py') -PySource('m5.util', 'm5/util/multidict.py') -PySource('m5.util', 'm5/util/pybind.py') -PySource('m5.util', 'm5/util/terminal.py') -PySource('m5.util', 'm5/util/terminal_formatter.py') +PySource("m5.internal", "m5/internal/__init__.py") +PySource("m5.internal", "m5/internal/params.py") +PySource("m5.ext", "m5/ext/__init__.py") +PySource("m5.ext.pyfdt", "m5/ext/pyfdt/pyfdt.py") +PySource("m5.ext.pyfdt", "m5/ext/pyfdt/__init__.py") -PySource('m5.internal', 'm5/internal/__init__.py') -PySource('m5.internal', 'm5/internal/params.py') -PySource('m5.ext', 'm5/ext/__init__.py') -PySource('m5.ext.pyfdt', 'm5/ext/pyfdt/pyfdt.py') -PySource('m5.ext.pyfdt', 'm5/ext/pyfdt/__init__.py') +PySource("m5.ext.pystats", "m5/ext/pystats/__init__.py") +PySource("m5.ext.pystats", "m5/ext/pystats/serializable_stat.py") +PySource("m5.ext.pystats", "m5/ext/pystats/abstract_stat.py") +PySource("m5.ext.pystats", "m5/ext/pystats/group.py") +PySource("m5.ext.pystats", "m5/ext/pystats/simstat.py") +PySource("m5.ext.pystats", "m5/ext/pystats/statistic.py") +PySource("m5.ext.pystats", "m5/ext/pystats/storagetype.py") +PySource("m5.ext.pystats", "m5/ext/pystats/timeconversion.py") +PySource("m5.ext.pystats", "m5/ext/pystats/jsonloader.py") +PySource("m5.stats", "m5/stats/gem5stats.py") -PySource('m5.ext.pystats', 'm5/ext/pystats/__init__.py') -PySource('m5.ext.pystats', 'm5/ext/pystats/serializable_stat.py') -PySource('m5.ext.pystats', 'm5/ext/pystats/abstract_stat.py') -PySource('m5.ext.pystats', 'm5/ext/pystats/group.py') -PySource('m5.ext.pystats', 'm5/ext/pystats/simstat.py') -PySource('m5.ext.pystats', 'm5/ext/pystats/statistic.py') -PySource('m5.ext.pystats', 'm5/ext/pystats/storagetype.py') -PySource('m5.ext.pystats', 'm5/ext/pystats/timeconversion.py') -PySource('m5.ext.pystats', 'm5/ext/pystats/jsonloader.py') -PySource('m5.stats', 'm5/stats/gem5stats.py') +Source("embedded.cc", add_tags=["python", "m5_module"]) +Source("importer.cc", add_tags=["python", "m5_module"]) +cc, hh = env.Blob("m5ImporterCode", "importer.py") +Source(cc, add_tags=["python", "m5_module"]) -Source('embedded.cc', add_tags=['python', 'm5_module']) -Source('importer.cc', add_tags=['python', 'm5_module']) -cc, hh = env.Blob('m5ImporterCode', 'importer.py') -Source(cc, add_tags=['python', 'm5_module']) +Source("pybind11/core.cc", add_tags="python") +Source("pybind11/debug.cc", add_tags="python") +Source("pybind11/event.cc", add_tags="python") +Source("pybind11/object_file.cc", add_tags="python") +Source("pybind11/stats.cc", add_tags="python") -Source('pybind11/core.cc', add_tags='python') -Source('pybind11/debug.cc', add_tags='python') -Source('pybind11/event.cc', add_tags='python') -Source('pybind11/object_file.cc', add_tags='python') -Source('pybind11/stats.cc', add_tags='python') - -SimObject('m5/objects/SimObject.py', sim_objects=['SimObject'], - enums=['ByteOrder']) +SimObject("m5/objects/SimObject.py", sim_objects=["SimObject"], enums=["ByteOrder"]) diff --git a/src/python/gem5-config.json b/src/python/gem5-config.json new file mode 100644 index 0000000..077b231 --- /dev/null +++ b/src/python/gem5-config.json @@ -0,0 +1,14 @@ +{ + "schemaUrl": "https://raw.githubusercontent.com/Gem5Vision/json-to-mongodb/main/schema/schema.json", + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "name": "data-ejhjf", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": true + } + } +} diff --git a/src/python/gem5/resources/client_api/__init__.py b/src/python/gem5/resources/client_api/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/src/python/gem5/resources/client_api/__init__.py diff --git a/src/python/gem5/resources/client_api/client.py b/src/python/gem5/resources/client_api/client.py new file mode 100644 index 0000000..6367ba9 --- /dev/null +++ b/src/python/gem5/resources/client_api/client.py @@ -0,0 +1,166 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from abc import abstractmethod, ABC +import urllib.request +import urllib.parse +from _m5 import core +import warnings +from typing import Optional, Dict, Union, Type, Tuple, List, Any +from distutils.version import StrictVersion + + +class AbstractClient(ABC): + @abstractmethod + def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: + """ + Retrieves all the resources with the given ID. + :param resource_id: The ID of the Resource. + :return: A list of resources as Python dictionaries. + """ + raise NotImplementedError + + def get_resource_json_obj_from_client( + self, resource_id: str, resource_version: Optional[str] = None + ) -> dict: + """ + Retrieves the Resource object identified by the given resource ID. + :param resource_id: The ID of the Resource. + :param resource_version: (optional) The version of the Resource. + If not given, the latest version compatible with the current + gem5 version is returned. + """ + # getting all the resources with the given id from the dictionary + resources = self.get_resources_by_id(resource_id) + + # if no resource with the given id is found throw an exception + if len(resources) == 0: + return None + + # sorting the resources by version + resources = self._sort_resources_by_version(resources) + + # if a version is given, search for the resource with the given version + if resource_version: + return self._search_version_in_resources( + resources, resource_version + ) + + # if no version is given, return the compatible + # resource with the latest version + compatible_resources = ( + self._get_resources_compatible_with_gem5_version(resources) + ) + if len(compatible_resources) == 0: + self._check_resource_version_compatibility(resources[0]) + return resources[0] + return compatible_resources[0] + + def _url_validator(self, url: str) -> bool: + """ + Validates the provided URL. + :param url: The URL to be validated. + :return: True if the URL is valid, False otherwise. + """ + try: + result = urllib.parse.urlparse(url) + return all([result.scheme, result.netloc, result.path]) + except: + return False + + def _search_version_in_resources( + self, resources: List, resource_version: str = "1.0.0" + ) -> dict: + """ + Searches for the resource with the given version. If the resource is + not found, an exception is thrown. + :param resources: A list of resources to search through. + :param resource_version: The version of the resource to search for. + :return: The resource object as a Python dictionary if found. + :raises: RuntimeError if the resource with the specified version is + not found. + """ + resource = next( + iter( + [ + resource + for resource in resources + if resource["resource_version"] == resource_version + ] + ), + None, + ) + if resource is None: + return None + self._check_resource_version_compatibility(resource) + return resource + + def _get_resources_compatible_with_gem5_version( + self, resources: List, gem5_version: str = core.gem5Version + ) -> List: + """ + Returns a list of compatible resources with the current gem5 version. + :param resources: A list of resources to filter. + :return: A list of compatible resources as Python dictionaries. + If no compatible resources are found, the original list of resources + is returned. + """ + compatible_resources = [ + resource + for resource in resources + if gem5_version in resource["gem5_versions"] + ] + return compatible_resources + + def _sort_resources_by_version(self, resources: List) -> List: + """ + Sorts the resources by version in descending order. + :param resources: A list of resources to sort. + :return: A list of sorted resources. + """ + return sorted( + resources, + key=lambda resource: ( + resource["id"].lower(), + StrictVersion(resource["resource_version"]), + ), + reverse=True, + ) + + def _check_resource_version_compatibility( + self, resource: dict, gem5_version: str = core.gem5Version + ) -> bool: + if gem5_version not in resource["gem5_versions"]: + warnings.warn( + f"Resource {resource['id']} version {resource['resource_version']} " + f"is not known to be compatible with gem5 version {gem5_version}. " + "This may cause problems with your simulation. This resource's compatibility " + "with different gem5 versions can be found here:" + "https://gem5vision.github.io/gem5-resources-website" + f"/resources/{resource['id']}/versions" + ) + return False + return True diff --git a/src/python/gem5/resources/client_api/jsonclient.py b/src/python/gem5/resources/client_api/jsonclient.py new file mode 100644 index 0000000..37fe0f7 --- /dev/null +++ b/src/python/gem5/resources/client_api/jsonclient.py @@ -0,0 +1,64 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +from pathlib import Path + +from urllib import request +from .client import AbstractClient +from typing import Optional, Dict, Union, Type, Tuple, List, Any + + +class JSONClient(AbstractClient): + def __init__(self, path: str): + """ + Initializes a JSON database. + :param path: The path to the Resource, either URL or local. + """ + self.path = path + self.resources = [] + + if Path(self.path).is_file(): + self.resources = json.load(open(self.path)) + elif not self._url_validator(self.path): + raise Exception( + f"Resources location '{self.path}' is not a valid path or URL." + ) + else: + req = request.Request(self.path) + response = request.urlopen(req) + self.resources = json.loads(response.read().decode("utf-8")) + + def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: + """ + :param resource_id: The ID of the Resource. + :return: A list of all the Resources with the given ID. + """ + return [ + resource + for resource in self.resources + if resource["id"] == resource_id + ] diff --git a/src/python/gem5/resources/client_api/mongoclient.py b/src/python/gem5/resources/client_api/mongoclient.py new file mode 100644 index 0000000..413128a --- /dev/null +++ b/src/python/gem5/resources/client_api/mongoclient.py @@ -0,0 +1,86 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from .client import AbstractClient + +from urllib import request, parse +from typing import Optional, Dict, Union, Type, Tuple, List, Any +import json + + +class MongoClient(AbstractClient): + def __init__(self, config: Dict[str, str]): + """ + Initializes a connection to a MongoDB database. + :param uri: The URI for connecting to the MongoDB server. + :param db: The name of the database to connect to. + :param collection: The name of the collection within the database. + """ + self.apiKey = config["apiKey"] + self.url = config["url"] + self.collection = config["collection"] + self.database = config["database"] + self.dataSource = config["dataSource"] + self.name = config["name"] + + def get_token(self): + url = ( + "https://realm.mongodb.com/api/client/v2.0/app/" + f"{self.name}/auth/providers/api-key/login" + ) + data = {"key": self.apiKey} + data = json.dumps(data).encode("utf-8") + + req = request.Request( + url, data=data, headers={"Content-Type": "application/json"} + ) + response = request.urlopen(req) + result = json.loads(response.read().decode("utf-8")) + token = result["access_token"] + + return token + + def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: + url = f"{self.url}/action/find" + data = { + "dataSource": self.dataSource, + "collection": self.collection, + "database": self.database, + "filter": {"id": resource_id}, + } + data = json.dumps(data).encode("utf-8") + + headers = { + "Authorization": f"Bearer {self.get_token()}", + "Content-Type": "application/json", + } + + req = request.Request(url, data=data, headers=headers) + response = request.urlopen(req) + result = json.loads(response.read().decode("utf-8")) + resources = result["documents"] + + return resources diff --git a/src/python/gem5/resources/client_wrapper.py b/src/python/gem5/resources/client_wrapper.py new file mode 100644 index 0000000..1335667 --- /dev/null +++ b/src/python/gem5/resources/client_wrapper.py @@ -0,0 +1,122 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +from typing import Optional, Dict +import json +from pathlib import Path +from typing import Optional, Dict, List +from .client_api.jsonclient import JSONClient +from .client_api.mongoclient import MongoClient +from python.gem5 import __file__ +import os + + +def create_clients( + config: Dict, +) -> Dict: + """ + This function creates respective client object for each database in the config file according to the type of database. + Params: config: config file containing the database information + Returns: clients: dictionary of clients for each database + """ + clients = {} + for client in config["sources"]: + database = config["sources"][client] + if database["isMongo"]: + clients[client] = MongoClient(database) + else: + clients[client] = JSONClient(database["url"]) + return clients + + +clients = None + +# First check if the config file path is provided in the environment variable +if "GEM5_RESOURCES_CONFIG" in os.environ: + config_file_path = Path(os.environ["GEM5_RESOURCES_CONFIG"]) +# If not, check if the config file is present in the current directory +elif (Path().resolve() / "gem5-config.json").exists(): + config_file_path = Path().resolve() / "gem5-config.json" +# If not, use the default config in the build directory +else: + config_file_path = ( + Path(__file__).resolve().parent.parent / "gem5-config.json" + ) + +# If config file is found, load the config file +if config_file_path.exists(): + with open(config_file_path, "r") as file: + config = json.load(file) +else: + raise Exception(f"Config file not found at {config_file_path}") + +# If clients is not already created, create clients for each database +if clients is None: + clients = create_clients(config) + + +def get_resource_json_obj( + resource_id, + resource_version: Optional[str] = None, + databases: Optional[List[str]] = [], +) -> Dict: + """ + This function returns the resource object from the corresponding database. + + :param resource_id: resource id of the resource + :optional param resource_version: resource version of the resource + :optional param database: database name. If not provided, the first database in the config file is used + :return: resource object + """ + resources = [] + if not databases: + databases = list(clients.keys()) + for database in databases: + if database not in clients: + raise Exception(f"Database: {database} does not exist") + resource = clients[database].get_resource_json_obj_from_client( + resource_id, resource_version + ) + if resource is not None: + resources.append(resource) + + if len(resources) == 0: + if resource_version is None: + raise Exception(f"Resource with ID '{resource_id}' not found.") + else: + raise Exception( + f"Resource {resource_id} with version '{resource_version}'" + " not found.\nResource versions can be found at: " + f"https://gem5vision.github.io/gem5-resources-website/resources/{resource_id}/versions" + ) + if len(resources) > 1: + raise Exception( + f"Resource: {resource_id} exists in multiple databases. " + "Please specify the database name to use." + ) + + return resources[0] diff --git a/src/python/gem5/resources/downloader.py b/src/python/gem5/resources/downloader.py index 16b0147..bc71934 100644 --- a/src/python/gem5/resources/downloader.py +++ b/src/python/gem5/resources/downloader.py @@ -41,6 +41,7 @@ from urllib.error import HTTPError from typing import List, Dict, Set, Optional +from .client_wrapper import get_resource_json_obj from .md5_utils import md5_file, md5_dir from ..utils.progress_bar import tqdm, progress_hook @@ -109,7 +110,6 @@ # its Truncated Exponential Backoff algorithm # (maximum of roughly 1 minute). Typically this code will run quickly. with FileLock(f"{download_path}.lock", timeout=120): - # The resources.json file can change at any time, but to avoid # excessive retrieval we cache a version locally and use it for up to # an hour before obtaining a fresh copy. @@ -398,6 +398,8 @@ unzip: bool = True, untar: bool = True, download_md5_mismatch: bool = True, + resource_version: Optional[str] = None, + databases: Optional[List] = [], ) -> None: """ Obtains a gem5 resource and stored it to a specified location. If the @@ -430,17 +432,19 @@ # minutes.Most resources should be downloaded and decompressed in this # timeframe, even on the most constrained of systems. with FileLock(f"{to_path}.lock", timeout=900): - - resource_json = get_resources_json_obj(resource_name) + resource_obj = get_resource_json_obj( + resource_name, + resource_version=resource_version, + databases=databases, + ) if os.path.exists(to_path): - if os.path.isfile(to_path): md5 = md5_file(Path(to_path)) else: md5 = md5_dir(Path(to_path)) - if md5 == resource_json["md5sum"]: + if md5 == resource_obj["md5sum"]: # In this case, the file has already been download, no need to # do so again. return @@ -461,10 +465,10 @@ # string-based way of doing things. It can be refactored away over # time: # https://gem5-review.googlesource.com/c/public/gem5-resources/+/51168 - if isinstance(resource_json["is_zipped"], str): - run_unzip = unzip and resource_json["is_zipped"].lower() == "true" - elif isinstance(resource_json["is_zipped"], bool): - run_unzip = unzip and resource_json["is_zipped"] + if isinstance(resource_obj["is_zipped"], str): + run_unzip = unzip and resource_obj["is_zipped"].lower() == "true" + elif isinstance(resource_obj["is_zipped"], bool): + run_unzip = unzip and resource_obj["is_zipped"] else: raise Exception( "The resource.json entry for '{}' has a value for the " @@ -475,8 +479,8 @@ run_tar_extract = ( untar - and "is_tar_archive" in resource_json - and resource_json["is_tar_archive"] + and "is_tar_archive" in resource_obj + and resource_obj["is_tar_archive"] ) tar_extension = ".tar" @@ -495,9 +499,8 @@ ) ) - # Get the URL. The URL may contain '{url_base}' which needs replaced - # with the correct value. - url = resource_json["url"].format(url_base=_get_url_base()) + # Get the URL. + url = resource_obj["url"] _download(url=url, download_to=download_dest) print(f"Finished downloading resource '{resource_name}'.") @@ -523,7 +526,6 @@ with tarfile.open(download_dest) as f: def is_within_directory(directory, target): - abs_directory = os.path.abspath(directory) abs_target = os.path.abspath(target) @@ -534,7 +536,6 @@ def safe_extract( tar, path=".", members=None, *, numeric_owner=False ): - for member in tar.getmembers(): member_path = os.path.join(path, member.name) if not is_within_directory(path, member_path): diff --git a/src/python/gem5/resources/looppoint.py b/src/python/gem5/resources/looppoint.py index 684faef..bc5d4a1 100644 --- a/src/python/gem5/resources/looppoint.py +++ b/src/python/gem5/resources/looppoint.py @@ -491,7 +491,7 @@ with open(_path) as file: json_contents = json.load(file) for rid in json_contents: - + start_pc = int(json_contents[rid]["simulation"]["start"]["pc"]) start_globl = int( json_contents[rid]["simulation"]["start"]["global"] diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index 9cba9cf..2939e80 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -29,13 +29,15 @@ from pathlib import Path from m5.util import warn, fatal -from .downloader import get_resource, get_resources_json_obj +from .downloader import get_resource from .looppoint import LooppointCsvLoader, LooppointJsonLoader from ..isas import ISA, get_isa_from_str from typing import Optional, Dict, Union, Type, Tuple, List +from .client_wrapper import get_resource_json_obj + """ Resources are items needed to run a simulation, such as a disk image, kernel, or binary. The gem5 project provides pre-built resources, with sources, at @@ -68,17 +70,20 @@ def __init__( self, local_path: Optional[str] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, + resource_version: str = "1.0.0", ): """ :param local_path: The path on the host system where this resource is located. - :param documentation: Documentation describing this resource. Not a + :param description: Description describing this resource. Not a required parameter. By default is None. :param source: The source (as in "source code") for this resource. This string should navigate users to where the source for this resource may be found. Not a required parameter. By default is None. + :param resource_version: Version of the resource itself. + Not a required parameter. By default is 1.0.0 """ if local_path and not os.path.exists(local_path): @@ -88,16 +93,21 @@ ) self._local_path = local_path - self._documentation = documentation + self._description = description self._source = source + self._version = resource_version + + def get_resource_version(self) -> str: + """Returns the version of the resource.""" + return self._version def get_local_path(self) -> Optional[str]: """Returns the local path of the resource.""" return self._local_path - def get_documentation(self) -> Optional[str]: - """Returns documentation associated with this resource.""" - return self._documentation + def get_description(self) -> Optional[str]: + """Returns description associated with this resource.""" + return self._description def get_source(self) -> Optional[str]: """Returns information as to where the source for this resource may be @@ -112,8 +122,9 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, + resource_version: str = "1.0.0", **kwargs, ): if not os.path.isfile(local_path): @@ -123,8 +134,9 @@ super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -134,11 +146,11 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, + resource_version: str = "1.0.0", **kwargs, ): - if not os.path.isdir(local_path): raise Exception( f"DirectoryResource path specified, {local_path}, is not a " @@ -147,8 +159,9 @@ super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -158,15 +171,17 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, root_partition: Optional[str] = None, + resource_version: str = "1.0.0", **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) self._root_partition = root_partition @@ -181,15 +196,17 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, architecture: Optional[Union[ISA, str]] = None, + resource_version: str = "1.0.0", **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) self._architecture = None @@ -210,16 +227,18 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, architecture: Optional[Union[ISA, str]] = None, + resource_version: str = "1.0.0", **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, architecture=architecture, source=source, + resource_version=resource_version, ) @@ -229,14 +248,16 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, + resource_version: str = "1.0.0", **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -246,16 +267,18 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, architecture: Optional[Union[ISA, str]] = None, + resource_version: str = "1.0.0", **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, architecture=architecture, + resource_version=resource_version, ) @@ -270,14 +293,16 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, + resource_version: str = "1.0.0", **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -295,9 +320,10 @@ weight_list: List[float] = None, warmup_interval: int = 0, workload_name: Optional[str] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, local_path: Optional[str] = None, + resource_version: str = "1.0.0", **kwargs, ): """ @@ -314,8 +340,9 @@ super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) self._weight_list = weight_list @@ -402,15 +429,17 @@ def __init__( self, local_path: str, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, + resource_version: str = "1.0.0", **kwargs, ): FileResource.__init__( self, local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) LooppointCsvLoader.__init__(self, pinpoints_file=Path(local_path)) @@ -420,15 +449,17 @@ self, local_path: str, region_id: Optional[Union[str, int]] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, + resource_version: str = "1.0.0", **kwargs, ): FileResource.__init__( self, local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) LooppointJsonLoader.__init__( self, looppoint_file=local_path, region_id=region_id @@ -447,8 +478,9 @@ simpoint_interval: int, warmup_interval: int, workload_name: Optional[str] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, + resource_version: str = "1.0.0", **kwargs, ): """ @@ -478,8 +510,9 @@ warmup_interval=warmup_interval, workload_name=workload_name, local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) def get_simpoint_file(self) -> Path: @@ -522,9 +555,11 @@ def obtain_resource( - resource_name: str, + resource_id: str, resource_directory: Optional[str] = None, download_md5_mismatch: bool = True, + resource_version: Optional[str] = None, + databases: Optional[List] = [], ) -> AbstractResource: """ This function primarily serves as a factory for resources. It will return @@ -544,15 +579,20 @@ have the correct md5 value, the resoruce will be deleted and re-downloaded if this value is True. Otherwise an exception will be thrown. True by default. + :param resource_version: Version of the resource itself. + Not a required parameter. None by default. + :param database: Database from where the resource information should + be extracted. Not a required parameter. None by default. """ - # Obtain the JSON resource entry for this resource - resource_json = get_resources_json_obj(resource_name) + # Obtain the resource object entry for this resource + resource_obj = get_resource_json_obj( + resource_id, resource_version=resource_version, databases=databases + ) to_path = None # If the "url" field is specified, the resoruce must be downloaded. - if "url" in resource_json and resource_json["url"]: - + if "url" in resource_obj and resource_obj["url"]: # If the `resource_directory` parameter is not set via this function, we # check the "GEM5_RESOURCE_DIR" environment variable. If this too is not # set we call `_get_default_resource_dir()` to determine where the @@ -580,43 +620,42 @@ os.makedirs(resource_directory, exist_ok=True) # This is the path to which the resource is to be stored. - to_path = os.path.join(resource_directory, resource_name) + to_path = os.path.join(resource_directory, resource_id) # Download the resource if it does not already exist. get_resource( - resource_name=resource_name, - to_path=os.path.join(resource_directory, resource_name), + resource_name=resource_id, + to_path=os.path.join(resource_directory, resource_id), download_md5_mismatch=download_md5_mismatch, + resource_version=resource_version, + databases=databases, ) # Obtain the type from the JSON. From this we will determine what subclass # of `AbstractResource` we are to create and return. - resources_type = resource_json["type"] + resources_category = resource_obj["category"] - if resources_type == "resource": + if resources_category == "resource": # This is a stop-gap measure to ensure to work with older versions of # the "resource.json" file. These should be replaced with their # respective specializations ASAP and this case removed. - if ( - "additional_metadata" in resource_json - and "root_partition" in resource_json["additional_metadata"] - ): + if "root_partition" in resource_obj: # In this case we should return a DiskImageResource. - root_partition = resource_json["additional_metadata"][ - "root_partition" - ] + root_partition = resource_obj["root_partition"] return DiskImageResource( - local_path=to_path, root_partition=root_partition + local_path=to_path, + root_partition=root_partition, + **resource_obj, ) return CustomResource(local_path=to_path) - assert resources_type in _get_resource_json_type_map - resource_class = _get_resource_json_type_map[resources_type] + assert resources_category in _get_resource_json_type_map + resource_class = _get_resource_json_type_map[resources_category] # Once we know what AbstractResource subclass we are using, we create it. # The fields in the JSON object are assumed to map like-for-like to the # subclass contructor, so we can pass the resource_json map directly. - return resource_class(local_path=to_path, **resource_json) + return resource_class(local_path=to_path, **resource_obj) def _get_default_resource_dir() -> str: @@ -696,12 +735,15 @@ local_path: str, root_partition: Optional[str] = None, metadata: Dict = {}, + version: str = "1.0.0", ): """ :param local_path: The path of the disk image on the host system. :param root_partition: The root disk partition to use. :param metadata: Metadata for the resource. **Warning:** As of " "v22.1.1, this parameter is not used. + :param resource_version: Version of the resource itself. + Not a required parameter. By default is 1.0.0 """ warn( "The `CustomDiskImageResource` class is deprecated. Please use " @@ -713,13 +755,19 @@ "`CustomDiskImageResource` constructor. This parameter is not " "used." ) - super().__init__(local_path=local_path, root_partition=root_partition) + super().__init__( + local_path=local_path, + root_partition=root_partition, + resource_version=version, + ) def Resource( resource_name: str, resource_directory: Optional[str] = None, download_md5_mismatch: bool = True, + resource_version: Optional[str] = None, + database: Optional[str] = None, ) -> AbstractResource: """ This function was created to maintain backwards compability for v21.1.0 @@ -737,14 +785,16 @@ ) return obtain_resource( - resource_name=resource_name, + resource_id=resource_name, resource_directory=resource_directory, download_md5_mismatch=download_md5_mismatch, + resource_version=resource_version, + databases=database, ) _get_resource_json_type_map = { - "disk-image": DiskImageResource, + "disk_image": DiskImageResource, "binary": BinaryResource, "kernel": KernelResource, "checkpoint": CheckpointResource, diff --git a/src/python/gem5/resources/workload.py b/src/python/gem5/resources/workload.py index e0a19d0..64ed698 100644 --- a/src/python/gem5/resources/workload.py +++ b/src/python/gem5/resources/workload.py @@ -24,10 +24,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from .downloader import get_workload_json_obj from .resource import obtain_resource +from .client_wrapper import get_resource_json_obj -from typing import Dict, Any, Optional +from typing import Dict, Any, List, Optional class AbstractWorkload: @@ -155,7 +155,11 @@ """ def __init__( - self, workload_name: str, resource_directory: Optional[str] = None + self, + workload_name: str, + resource_directory: Optional[str] = None, + resource_version: Optional[str] = None, + databases: Optional[List] = [], ) -> None: """ This constructor will load the workload details from the workload with @@ -167,9 +171,9 @@ ```json { - "type" : "workload", - "name" : "x86-ubuntu-18.04-echo-hello", - "documentation" : "Description of workload here", + "category" : "workload", + "id" : "x86-ubuntu-18.04-echo-hello", + "description" : "Description of workload here", "function" : "set_kernel_disk_workload", "resources" : { "kernel" : "x86-linux-kernel-5.4.49", @@ -198,7 +202,12 @@ any resources should be download and accessed from. If None, a default location will be used. None by default. """ - workload_json = get_workload_json_obj(workload_name=workload_name) + + workload_json = get_resource_json_obj( + workload_name, + resource_version=resource_version, + databases=databases, + ) func = workload_json["function"] assert isinstance(func, str) diff --git a/src/python/gem5/utils/simpoint.py b/src/python/gem5/utils/simpoint.py index eab92e2..c50debc 100644 --- a/src/python/gem5/utils/simpoint.py +++ b/src/python/gem5/utils/simpoint.py @@ -85,12 +85,10 @@ weight_file_path = simpoint_resource.get_weight_file() simpoint_interval = ( simpoint_resource.get_metadata() - .get("additional_metadata") .get("simpoint_interval") ) warmup_interval = ( simpoint_resource.get_metadata() - .get("additional_metadata") .get("warmup_interval") ) diff --git a/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py b/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py new file mode 100644 index 0000000..4039ede --- /dev/null +++ b/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py @@ -0,0 +1,387 @@ +# Copyright (c) 2022 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +from gem5.isas import ISA +from gem5.resources.client_wrapper import ( + get_resource_json_obj, + create_clients, +) +from typing import Dict +from unittest.mock import patch +import json + +mock_config_json = { + "schemaUrl": "https://raw.githubusercontent.com/Gem5Vision/json-to-mongodb/main/schema/schema.json", + "sources": { + "baba": { + "url": "tests/pyunit/stdlib/resources/refs/resources.json", + "isMongo": False, + } + }, +} + +mock_config_mongo = { + "schemaUrl": "https://raw.githubusercontent.com/Gem5Vision/json-to-mongodb/main/schema/schema.json", + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "resources", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "name": "data-ejhjf", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + }, +} + +mock_config_combined = { + "schemaUrl": "https://raw.githubusercontent.com/Gem5Vision/json-to-mongodb/main/schema/schema.json", + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "resources", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "name": "data-ejhjf", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + }, + "baba": { + "url": "tests/pyunit/stdlib/resources/refs/resources.json", + "isMongo": False, + }, + }, +} + +mock_json = {} + +with open("tests/pyunit/stdlib/resources/refs/mongo_mock.json", "r") as f: + mock_json = json.load(f) + + +def mocked_requests_post(*args, **kwargs): + # mokcing urllib.request.urlopen + class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def read(self): + return json.dumps(self.json_data).encode("utf-8") + + data = json.loads(args[0].data) + if "/api-key/login" in args[0].full_url: + return MockResponse({"access_token": "test-token"}, 200) + if "/action/find" in args[0].full_url: + if data: + if data["filter"]["id"] == "invalid-id": + return MockResponse( + { + "documents": [], + }, + 200, + ) + return MockResponse( + { + "documents": mock_json, + }, + 200, + ) + + return MockResponse(None, 404) + + +class ClientWrapperTestSuite(unittest.TestCase): + @patch("gem5.resources.client_wrapper.config", mock_config_json) + @patch( + "gem5.resources.client_wrapper.clients", + create_clients(mock_config_json), + ) + def test_get_resource_json_obj(self): + # Test that the resource object is correctly returned + resource = "this-is-a-test-resource" + resource = get_resource_json_obj(resource) + self.assertEqual(resource["id"], "this-is-a-test-resource") + self.assertEqual(resource["resource_version"], "2.0.0") + self.assertEqual(resource["category"], "binary") + self.assertEqual( + resource["description"], "This is a test resource but double newer" + ) + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch("gem5.resources.client_wrapper.config", mock_config_json) + @patch( + "gem5.resources.client_wrapper.clients", + create_clients(mock_config_json), + ) + def test_get_resource_json_obj_invalid_database(self): + # Test that an exception is raised when an invalid database is passed + resource_id = "test-id" + database = "invalid" + with self.assertRaises(Exception) as context: + get_resource_json_obj(resource_id, databases=[database]) + self.assertTrue( + f"Database: {database} does not exist" in str(context.exception) + ) + + @patch("gem5.resources.client_wrapper.config", mock_config_json) + @patch( + "gem5.resources.client_wrapper.clients", + create_clients(mock_config_json), + ) + def test_get_resource_json_obj_with_version(self): + # Test that the resource object is correctly returned + resource_id = "this-is-a-test-resource" + resource_version = "1.0.0" + resource = get_resource_json_obj( + resource_id, resource_version=resource_version + ) + self.assertEqual(resource["id"], "this-is-a-test-resource") + self.assertEqual(resource["resource_version"], "1.0.0") + self.assertEqual(resource["category"], "binary") + self.assertEqual(resource["description"], "This is a test resource") + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch("gem5.resources.client_wrapper.config", mock_config_mongo) + @patch( + "gem5.resources.client_wrapper.clients", + create_clients(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj(self, mock_get): + resource = "x86-ubuntu-18.04-img" + resource = get_resource_json_obj( + resource, databases=["gem5-resources"] + ) + self.assertEqual(resource["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource["resource_version"], "2.0.0") + self.assertEqual(resource["category"], "disk_image") + self.assertEqual( + resource["description"], + "A disk image containing Ubuntu 18.04 for x86. This image will run an `m5 readfile` instruction after booting. If no script file is specified an `m5 exit` instruction will be executed.", + ) + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch("gem5.resources.client_wrapper.config", mock_config_mongo) + @patch( + "gem5.resources.client_wrapper.clients", + create_clients(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_with_version_mongodb(self, mock_get): + # Test that the resource object is correctly returned + resource_id = "x86-ubuntu-18.04-img" + resource_version = "1.0.0" + resource = get_resource_json_obj( + resource_id, + resource_version=resource_version, + databases=["gem5-resources"], + ) + self.assertEqual(resource["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource["resource_version"], "1.0.0") + self.assertEqual(resource["category"], "disk_image") + self.assertEqual(resource["description"], "This is a test resource") + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch("gem5.resources.client_wrapper.config", mock_config_mongo) + @patch( + "gem5.resources.client_wrapper.clients", + create_clients(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_with_id_invalid_mongodb(self, mock_get): + resource_id = "invalid-id" + with self.assertRaises(Exception) as context: + get_resource_json_obj(resource_id, databases=["gem5-resources"]) + self.assertTrue( + "Resource with ID 'invalid-id' not found." + in str(context.exception) + ) + + @patch("gem5.resources.client_wrapper.config", mock_config_mongo) + @patch( + "gem5.resources.client_wrapper.clients", + create_clients(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_with_version_invalid_mongodb( + self, mock_get + ): + resource_id = "x86-ubuntu-18.04-img" + resource_version = "2.5.0" + with self.assertRaises(Exception) as context: + get_resource_json_obj( + resource_id, + resource_version=resource_version, + databases=["gem5-resources"], + ) + self.assertTrue( + f"Resource x86-ubuntu-18.04-img with version '2.5.0'" + " not found.\nResource versions can be found at: " + f"https://gem5vision.github.io/gem5-resources-website/resources/x86-ubuntu-18.04-img/versions" + in str(context.exception) + ) + + @patch("gem5.resources.client_wrapper.config", mock_config_json) + @patch( + "gem5.resources.client_wrapper.clients", + create_clients(mock_config_json), + ) + def test_get_resource_json_obj_with_version_invalid_json(self): + resource_id = "this-is-a-test-resource" + resource_version = "2.5.0" + with self.assertRaises(Exception) as context: + get_resource_json_obj( + resource_id, + resource_version=resource_version, + ) + self.assertTrue( + f"Resource this-is-a-test-resource with version '2.5.0'" + " not found.\nResource versions can be found at: " + f"https://gem5vision.github.io/gem5-resources-website/resources/this-is-a-test-resource/versions" + in str(context.exception) + ) + + @patch("gem5.resources.client_wrapper.config", mock_config_combined) + @patch( + "gem5.resources.client_wrapper.clients", + create_clients(mock_config_combined), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_combine(self, mock_get): + resource_id_mongo = "x86-ubuntu-18.04-img" + resource_version_mongo = "1.0.0" + resource_id_json = "this-is-a-test-resource" + resource_version_json = "1.0.0" + resource_mongo = get_resource_json_obj( + resource_id_mongo, + resource_version=resource_version_mongo, + databases=["gem5-resources"], + ) + resource_json = get_resource_json_obj( + resource_id_json, + resource_version=resource_version_json, + databases=["baba"], + ) + self.assertEqual(resource_mongo["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource_mongo["resource_version"], "1.0.0") + self.assertEqual(resource_mongo["category"], "disk_image") + self.assertEqual( + resource_mongo["description"], "This is a test resource" + ) + self.assertEqual( + resource_mongo["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + ) + self.assertEqual(resource_mongo["architecture"], "X86") + + self.assertEqual(resource_json["id"], "this-is-a-test-resource") + self.assertEqual(resource_json["resource_version"], "1.0.0") + self.assertEqual(resource_json["category"], "binary") + self.assertEqual( + resource_json["description"], "This is a test resource" + ) + self.assertEqual( + resource_json["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + ) + self.assertEqual(resource_json["architecture"], "X86") + + @patch("gem5.resources.client_wrapper.config", mock_config_combined) + @patch( + "gem5.resources.client_wrapper.clients", + create_clients(mock_config_combined), + ) + def test_get_resource_json_obj_multi_database_second_only(self): + resource_id = "simpoint-resource" + resource = get_resource_json_obj( + resource_id, + ) + self.assertEqual(resource["id"], resource_id) + self.assertEqual(resource["resource_version"], "0.2.0") + self.assertEqual(resource["category"], "file") + self.assertEqual( + resource["description"], + ( + "Simpoints for running the 'x86-print-this' resource with" + ' the parameters `"print this" 15000`. This is encapsulated' + " in the 'x86-print-this-15000-with-simpoints' workload." + ), + ) + + @patch("gem5.resources.client_wrapper.config", mock_config_combined) + @patch( + "gem5.resources.client_wrapper.clients", + create_clients(mock_config_combined), + ) + def test_get_resource_json_same_resource_different_versions(self): + resource_id = "x86-ubuntu-18.04-img" + with self.assertRaises(Exception) as context: + get_resource_json_obj( + resource_id, + ) + self.assertTrue( + f"Resource: {resource_id} exists in multiple databases. " + "Please specify the database name to use." + ) + resource_version_mongo = "1.0.0" + resource_version_json = "2.0.0" + resource_mongo = get_resource_json_obj( + resource_id, + resource_version=resource_version_mongo, + ) + resource_json = get_resource_json_obj( + resource_id, + databases=["baba"], + ) + self.assertEqual(resource_mongo["id"], "x86-ubuntu-18.04-img") + self.assertEqual( + resource_mongo["resource_version"], resource_version_mongo + ) + self.assertEqual(resource_mongo["category"], "disk_image") + self.assertEqual(resource_json["id"], "x86-ubuntu-18.04-img") + self.assertEqual( + resource_json["resource_version"], resource_version_json + ) + self.assertEqual(resource_json["category"], "disk_image") diff --git a/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py b/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py index 08736bb..6e3ea4d 100644 --- a/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py @@ -28,6 +28,7 @@ import tempfile import os from typing import Dict +import json from gem5.resources.downloader import ( _get_resources_json_at_path, @@ -42,48 +43,102 @@ @classmethod def setUpClass(cls) -> str: """ - This creates a simple resource.json temp file for testing purposes. + This creates a simple resources collection for testing """ - - file_contents = ( - "{" - + f'"version" : "{_resources_json_version_required()}",' - + """ - "url_base" : "http://dist.gem5.org/dist/v21-2", - "previous-versions" : {}, - "resources": [ - { - "type": "resource", - "name" : "riscv-disk-img", - "documentation" : "A simple RISCV disk image based on busybox.", - "architecture": "RISCV", - "is_zipped" : true, - "md5sum" : "d6126db9f6bed7774518ae25aa35f153", - "url": "{url_base}/images/riscv/busybox/riscv-disk.img.gz", - "source" : "src/riscv-fs", - "additional_metadata" : { - "root_partition": null - } - }, - { - "type": "resource", - "name" : "riscv-lupio-busybox-img", - "documentation" : "A RISCV disk image, based on busybox, to ...", - "architecture": "RISCV", - "is_zipped" : true, - "md5sum" : "e5bee8a31f45f4803f87c0d781553ccc", - "url": "{url_base}/images/riscv/busybox/riscv-lupio-busybox.img", - "source" : "src/lupv", - "additional_metadata" : { - "root_partition": "1" - } - } - ] -} - """ - ) + file_contents = [ + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource", + "architecture": "X86", + "size": 13816, + "tags": ["asmtest", "testing", "riscv", "testing"], + "is_zipped": False, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + "example_usage": 'get_resource(resource_name="rv64mi-p-sbreak")', + }, + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource but double newer", + "architecture": "X86", + "size": 13816, + "tags": ["asmtest"], + "is_zipped": False, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "2.0.0", + "gem5_versions": ["23.1"], + "example_usage": 'get_resource(resource_name="rv64mi-p-sbreak")', + }, + { + "category": "simpoint", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": False, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": True, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": 'get_resource(resource_name="x86-print-this-1500-simpoints")', + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint", + ], + }, + { + "category": "file", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": False, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": True, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": ["23.0"], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": 'get_resource(resource_name="x86-print-this-1500-simpoints")', + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint", + ], + }, + ] file = tempfile.NamedTemporaryFile(mode="w", delete=False) - file.write(file_contents) + file.write(json.dumps(file_contents)) file.close() cls.file_path = file.name @@ -100,14 +155,16 @@ "create_temp_resources_json" has been loaded correctly into a Python dictionary. """ - self.assertTrue("resources" in json) - self.assertEquals(2, len(json["resources"])) - self.assertTrue("name" in json["resources"][0]) - self.assertEquals("riscv-disk-img", json["resources"][0]["name"]) - self.assertTrue("name" in json["resources"][1]) - self.assertEquals( - "riscv-lupio-busybox-img", json["resources"][1]["name"] - ) + self.assertEquals(4, len(json)) + self.assertTrue("id" in json[0]) + self.assertEquals("this-is-a-test-resource", json[0]["id"]) + self.assertEquals("binary", json[0]["category"]) + self.assertTrue("id" in json[1]) + self.assertEquals("this-is-a-test-resource", json[1]["id"]) + self.assertTrue("id" in json[2]) + self.assertEquals("test-version", json[2]["id"]) + self.assertTrue("id" in json[3]) + self.assertEquals("test-version", json[3]["id"]) def test_get_resources_json_at_path(self) -> None: # Tests the gem5.resources.downloader._get_resources_json_at_path() diff --git a/tests/pyunit/stdlib/resources/pyunit_md5_utils_check.py b/tests/pyunit/stdlib/resources/pyunit_md5_utils_check.py index 65bf335..826d2ae 100644 --- a/tests/pyunit/stdlib/resources/pyunit_md5_utils_check.py +++ b/tests/pyunit/stdlib/resources/pyunit_md5_utils_check.py @@ -75,7 +75,6 @@ """Test cases for gem5.resources.md5_utils.md5_dir()""" def _create_temp_directory(self) -> Path: - dir = tempfile.mkdtemp() with open(os.path.join(dir, "file1"), "w") as f: diff --git a/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py b/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py new file mode 100644 index 0000000..2f9089c --- /dev/null +++ b/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py @@ -0,0 +1,194 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import os + +from pathlib import Path + +from gem5.resources.resource import * + +from gem5.resources.looppoint import ( + LooppointCsvLoader, + LooppointJsonLoader, +) + +from gem5.isas import ISA + +from _m5 import core + +from gem5.resources.client_wrapper import ( + create_clients, + clients, +) +from unittest.mock import patch + +mock_config_json = { + "schemaUrl": "https://raw.githubusercontent.com/Gem5Vision/json-to-mongodb/main/schema/schema.json", + "sources": { + "baba": { + "url": "tests/pyunit/stdlib/resources/refs/obtain-resource.json", + "isMongo": False, + } + }, +} + + +@patch( + "gem5.resources.client_wrapper.clients", + new=create_clients(mock_config_json), +) +class TestObtainResourcesCheck(unittest.TestCase): + @classmethod + def setUpClass(cls): + """Prior to running the suite we set the resource directory to + "ref/resource-specialization.json" + """ + os.environ["GEM5_RESOURCE_JSON"] = os.path.join( + os.path.realpath(os.path.dirname(__file__)), + "refs", + "obtain-resource.json", + ) + + @classmethod + def tearDownClass(cls) -> None: + """After running the suite we unset the gem5-resource JSON file, as to + not interfere with others tests. + """ + del os.environ["GEM5_RESOURCE_JSON"] + + def get_resource_dir(cls) -> str: + """To ensure the resources are cached to the same directory as all + other tests, this function returns the location of the testing + directories "resources" directory. + """ + return os.path.join( + os.path.realpath(os.path.dirname(__file__)), + os.pardir, + os.pardir, + os.pardir, + "gem5", + "resources", + ) + + def test_obtain_resources_no_version(self): + """Test that the resource loader returns latest version compatible with that version of gem5 when no version is specified.""" + gem5Version = core.gem5Version + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + ) + self.assertEquals("2.5.0", resource.get_resource_version()) + self.assertIsInstance(resource, BinaryResource) + # self.assertIn(gem5Version, resource.get_gem5_versions()) + self.assertEquals("test description", resource.get_description()) + self.assertEquals("src/test-source", resource.get_source()) + self.assertEquals(ISA.ARM, resource.get_architecture()) + + def test_obtain_resources_with_version_compatible(self): + gem5Version = core.gem5Version + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="1.7.0", + ) + self.assertEquals("1.7.0", resource.get_resource_version()) + self.assertIsInstance(resource, BinaryResource) + # self.assertIn(gem5Version, resource.get_gem5_versions()) + self.assertEquals( + "test description v1.7.0", resource.get_description() + ) + self.assertEquals("src/test-source", resource.get_source()) + self.assertEquals(ISA.ARM, resource.get_architecture()) + + def test_obtain_resources_with_version_incompatible(self): + resource = None + with self.assertWarns(Warning) as warning: + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="1.5.0", + ) + print(warning.warning.args[0]) + self.assertTrue( + f"Resource test-binary-resource version 1.5.0 is not known to be compatible with gem5 version {core.gem5Version}. " + "This may cause problems with your simulation. This resource's compatibility with different gem5 versions can be found here:" + f"https://gem5vision.github.io/gem5-resources-website/resources/test-binary-resource/versions" + in warning.warning.args[0] + ) + + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="1.5.0", + ) + self.assertEquals("1.5.0", resource.get_resource_version()) + self.assertIsInstance(resource, BinaryResource) + self.assertEquals( + "test description for 1.5.0", resource.get_description() + ) + self.assertEquals("src/test-source", resource.get_source()) + self.assertEquals(ISA.ARM, resource.get_architecture()) + + def test_obtain_resources_no_version_invalid_id(self): + with self.assertRaises(Exception) as context: + obtain_resource( + resource_id="invalid-id", + resource_directory=self.get_resource_dir(), + ) + self.assertTrue( + "Resource with ID 'invalid-id' not found." + in str(context.exception) + ) + + def test_obtain_resources_with_version_invalid_id(self): + with self.assertRaises(Exception) as context: + obtain_resource( + resource_id="invalid-id", + resource_directory=self.get_resource_dir(), + resource_version="1.7.0", + ) + self.assertTrue( + "Resource invalid-id with version '1.7.0'" + " not found.\nResource versions can be found at: " + "https://gem5vision.github.io/gem5-resources-website/resources/invalid-id/versions" + in str(context.exception) + ) + + def test_obtain_resources_with_version_invalid_version(self): + with self.assertRaises(Exception) as context: + obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="3.0.0", + ) + print(str(context.exception)) + self.assertTrue( + f"Resource test-binary-resource with version '3.0.0'" + " not found.\nResource versions can be found at: " + f"https://gem5vision.github.io/gem5-resources-website/resources/test-binary-resource/versions" + in str(context.exception) + ) diff --git a/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py b/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py new file mode 100644 index 0000000..21b48fb --- /dev/null +++ b/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py @@ -0,0 +1,72 @@ +# Copyright (c) 2022 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import tempfile +import os +from typing import Dict + +from gem5.resources.downloader import ( + get_resources_json_obj, +) + + +class ResourceDownloadTestSuite(unittest.TestCase): + """Test cases for gem5.resources.downloader""" + + @classmethod + def setUpClass(cls) -> str: + pass + + def get_resource_json_by_id(self) -> None: + """Get a resource by its id""" + resources = get_resources_json_obj("test-version") + self.assertEqual(resources["id"], "test-version") + self.assertEqual(resources["resource_version"], "2.0.0") + + def get_resource_json_invalid_id(self) -> None: + """Should throw an exception when trying to get a resource that doesn't exist""" + with self.assertRaises(Exception) as context: + get_resources_json_obj("this-resource-doesnt-exist") + self.assertTrue( + f"Error: Resource with name 'this-resource-doesnt-exist' does not exist" + in str(context.exception) + ) + + def get_resource_json_by_id_and_version(self) -> None: + """Get a resource by its id and version""" + resources = get_resources_json_obj("test-version", "1.0.0") + self.assertEqual(resources["id"], "test-version") + self.assertEqual(resources["resource_version"], "1.0.0") + + def get_resource_json_by_id_and_invalid_version(self) -> None: + """Get a resource by its id and an invalid version (does not exist)""" + with self.assertRaises(Exception) as context: + get_resources_json_obj("test-version", "3.0.0") + self.assertTrue( + f"Specified Version 3.0.0 does not exist for the resource 'test-version'." + in str(context.exception) + ) diff --git a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py index 660bf5f..119e078 100644 --- a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py +++ b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py @@ -37,7 +37,25 @@ from gem5.isas import ISA +from python.gem5.resources.client_wrapper import create_clients, clients +from unittest.mock import patch +mock_config_json = { + "schemaUrl": "https://raw.githubusercontent.com/Gem5Vision/json-to-mongodb/main/schema/schema.json", + "sources": { + "baba": { + "url": "tests/pyunit/stdlib/resources/refs/resource-specialization.json", + "isMongo": False, + } + }, +} + + +@patch("gem5.resources.client_wrapper.config", mock_config_json) +@patch( + "gem5.resources.client_wrapper.clients", + create_clients(mock_config_json), +) class ResourceSpecializationSuite(unittest.TestCase): """This suite tests that `gem5.resource.resource` casts to the correct `AbstractResource` specialization when using the `obtain_resource` @@ -79,14 +97,14 @@ def test_binary_resource(self) -> None: """Tests the loading of of a BinaryResource""" resource = obtain_resource( - resource_name="binary-example", + resource_id="binary-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, BinaryResource) self.assertEquals( - "binary-example documentation.", resource.get_documentation() + "binary-example documentation.", resource.get_description() ) self.assertEquals("src/simple", resource.get_source()) self.assertEquals(ISA.ARM, resource.get_architecture()) @@ -94,14 +112,14 @@ def test_kernel_resource(self) -> None: """Tests the loading of a KernelResource.""" resource = obtain_resource( - resource_name="kernel-example", + resource_id="kernel-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, KernelResource) self.assertEquals( - "kernel-example documentation.", resource.get_documentation() + "kernel-example documentation.", resource.get_description() ) self.assertEquals("src/linux-kernel", resource.get_source()) self.assertEquals(ISA.RISCV, resource.get_architecture()) @@ -109,14 +127,14 @@ def test_bootloader_resource(self) -> None: """Tests the loading of a BootloaderResource.""" resource = obtain_resource( - resource_name="bootloader-example", + resource_id="bootloader-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, BootloaderResource) self.assertEquals( - "bootloader documentation.", resource.get_documentation() + "bootloader documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) self.assertIsNone(resource.get_architecture()) @@ -124,14 +142,14 @@ def test_disk_image_resource(self) -> None: """Tests the loading of a DiskImageResource.""" resource = obtain_resource( - resource_name="disk-image-example", + resource_id="disk-image-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, DiskImageResource) self.assertEquals( - "disk-image documentation.", resource.get_documentation() + "disk-image documentation.", resource.get_description() ) self.assertEquals("src/x86-ubuntu", resource.get_source()) self.assertEquals("1", resource.get_root_partition()) @@ -139,40 +157,40 @@ def test_checkpoint_resource(self) -> None: """Tests the loading of a CheckpointResource.""" resource = obtain_resource( - resource_name="checkpoint-example", + resource_id="checkpoint-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, CheckpointResource) self.assertEquals( - "checkpoint-example documentation.", resource.get_documentation() + "checkpoint-example documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) def test_git_resource(self) -> None: """Tests the loading of a GitResource.""" resource = obtain_resource( - resource_name="git-example", + resource_id="git-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, GitResource) - self.assertIsNone(resource.get_documentation()) + self.assertIsNone(resource.get_description()) self.assertIsNone(resource.get_source()) def test_simpoint_directory_resource(self) -> None: """Tests the loading of a Simpoint directory resource.""" resource = obtain_resource( - resource_name="simpoint-directory-example", + resource_id="simpoint-directory-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, SimpointDirectoryResource) self.assertEquals( - "simpoint directory documentation.", resource.get_documentation() + "simpoint directory documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) @@ -199,14 +217,14 @@ def test_simpoint_resource(self) -> None: """Tests the loading of a Simpoint resource.""" resource = obtain_resource( - resource_name="simpoint-example", + resource_id="simpoint-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, SimpointResource) self.assertEquals( - "simpoint documentation.", resource.get_documentation() + "simpoint documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) self.assertIsNone(resource.get_local_path()) @@ -219,26 +237,27 @@ def test_file_resource(self) -> None: """Tests the loading of a FileResource.""" resource = obtain_resource( - resource_name="file-example", + resource_id="file-example", resource_directory=self.get_resource_dir(), + resource_version="1.0.0", ) self.assertIsInstance(resource, FileResource) - self.assertIsNone(resource.get_documentation()) + self.assertIsNone(resource.get_description()) self.assertIsNone(resource.get_source()) def test_directory_resource(self) -> None: """Tests the loading of a DirectoryResource.""" resource = obtain_resource( - resource_name="directory-example", + resource_id="directory-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, DirectoryResource) self.assertEquals( - "directory-example documentation.", resource.get_documentation() + "directory-example documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) @@ -247,7 +266,7 @@ pinpoints csv file.""" resource = obtain_resource( - resource_name="looppoint-pinpoint-csv-resource", + resource_id="looppoint-pinpoint-csv-resource", resource_directory=self.get_resource_dir(), ) @@ -258,7 +277,7 @@ self.assertIsInstance(resource, LooppointCsvLoader) self.assertEquals( - "A looppoint pinpoints csv file.", resource.get_documentation() + "A looppoint pinpoints csv file.", resource.get_description() ) self.assertIsNone(resource.get_source()) @@ -267,8 +286,9 @@ Looppoint JSON file.""" resource = obtain_resource( - resource_name="looppoint-json-restore-resource-region-1", + resource_id="looppoint-json-restore-resource-region-1", resource_directory=self.get_resource_dir(), + resource_version="1.0.0", ) self.assertIsInstance(resource, LooppointJsonResource) @@ -278,6 +298,6 @@ self.assertTrue("1" in resource.get_regions()) self.assertEquals( - "A looppoint json file resource.", resource.get_documentation() + "A looppoint json file resource.", resource.get_description() ) self.assertIsNone(resource.get_source()) diff --git a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py index 2bc31f5..3288666 100644 --- a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py @@ -36,6 +36,29 @@ from typing import Dict +from gem5.resources.client_wrapper import create_clients +from unittest.mock import patch + +mock_config_json1 = { + "schemaUrl": "https://raw.githubusercontent.com/Gem5Vision/json-to-mongodb/main/schema/schema.json", + "sources": { + "baba": { + "url": "tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json", + "isMongo": False, + } + }, +} + +mock_config_json2 = { + "schemaUrl": "https://raw.githubusercontent.com/Gem5Vision/json-to-mongodb/main/schema/schema.json", + "sources": { + "baba": { + "url": "tests/pyunit/stdlib/resources/refs/workload-checks.json", + "isMongo": False, + } + }, +} + class CustomWorkloadTestSuite(unittest.TestCase): """ @@ -43,8 +66,12 @@ """ @classmethod + @patch("gem5.resources.client_wrapper.config", mock_config_json1) + @patch( + "gem5.resources.client_wrapper.clients", + create_clients(mock_config_json1), + ) def setUpClass(cls) -> None: - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( os.path.realpath(os.path.dirname(__file__)), "refs", @@ -114,8 +141,7 @@ "test", self.custom_workload.get_parameters()["binary"] ) - # We set the overridden parameter back to it's old value. - self.custom_workload.set_parameter("binary", old_value) + # We set the overridden parameter back to it's old valu self.custom_workload.set_parameter("binary", old_value) class WorkloadTestSuite(unittest.TestCase): @@ -124,8 +150,12 @@ """ @classmethod + @patch("gem5.resources.client_wrapper.config", mock_config_json2) + @patch( + "gem5.resources.client_wrapper.clients", + create_clients(mock_config_json2), + ) def setUpClass(cls): - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( os.path.realpath(os.path.dirname(__file__)), "refs", diff --git a/tests/pyunit/stdlib/resources/refs/mongo_mock.json b/tests/pyunit/stdlib/resources/refs/mongo_mock.json new file mode 100644 index 0000000..59a8406 --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/mongo_mock.json @@ -0,0 +1,56 @@ +[ + { + "category": "disk_image", + "id": "x86-ubuntu-18.04-img", + "description": "This is a test resource", + "architecture": "X86", + "size": 688119691, + "tags": [ + "x86", + "fullsystem" + ], + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "source": "src/x86-ubuntu", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "root_partition": "1", + "code_examples": [], + "license": "", + "author": [ + "Ayaz Akram" + ], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" + }, + { + "category": "disk_image", + "id": "x86-ubuntu-18.04-img", + "description": "A disk image containing Ubuntu 18.04 for x86. This image will run an `m5 readfile` instruction after booting. If no script file is specified an `m5 exit` instruction will be executed.", + "architecture": "X86", + "size": 688119691, + "tags": [ + "x86", + "fullsystem" + ], + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "source": "src/x86-ubuntu", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "root_partition": "1", + "code_examples": [], + "license": "", + "author": [ + "Ayaz Akram" + ], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "resource_version": "2.0.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" + } +] \ No newline at end of file diff --git a/tests/pyunit/stdlib/resources/refs/obtain-resource.json b/tests/pyunit/stdlib/resources/refs/obtain-resource.json new file mode 100644 index 0000000..633679f --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/obtain-resource.json @@ -0,0 +1,59 @@ +[ + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "2.5.0", + "gem5_versions": [ + "25.0" + ] + }, + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "2.0.0", + "gem5_versions": [ + "develop" + ] + }, + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description v1.7.0", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "1.7.0", + "gem5_versions": [ + "develop" + ] + }, + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description for 1.5.0", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "1.5.0", + "gem5_versions": [ + "21.1", + "22.1" + ] + } +] \ No newline at end of file diff --git a/tests/pyunit/stdlib/resources/refs/resource-specialization.json b/tests/pyunit/stdlib/resources/refs/resource-specialization.json index c4d5eb4..3df7f6b 100644 --- a/tests/pyunit/stdlib/resources/refs/resource-specialization.json +++ b/tests/pyunit/stdlib/resources/refs/resource-specialization.json @@ -1,132 +1,181 @@ - -{ - "version" : "develop", - "url_base" : "http://dist.gem5.org/dist/v22-1", - "previous-versions" : { - "develop" : "https://gem5.googlesource.com/public/gem5-resources/+/refs/heads/develop/resources.json?format=TEXT", - "21.2" : "http://resources.gem5.org/prev-resources-json/resources-21-2.json" +[ + { + "category": "kernel", + "id": "kernel-example", + "description": "kernel-example documentation.", + "architecture": "RISCV", + "is_zipped": false, + "md5sum": "60a53c7d47d7057436bf4b9df707a841", + "url": "http://dist.gem5.org/dist/develop/kernels/x86/static/vmlinux-5.4.49", + "source": "src/linux-kernel", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] }, - "resources": [ - { - "type" : "kernel", - "name" : "kernel-example", - "documentation" : "kernel-example documentation.", - "architecture" : "RISCV", - "is_zipped" : false, - "md5sum" : "60a53c7d47d7057436bf4b9df707a841", - "url" : "{url_base}/kernels/x86/static/vmlinux-5.4.49", - "source" : "src/linux-kernel" - }, - { - "type" : "disk-image", - "name" : "disk-image-example", - "documentation" : "disk-image documentation.", - "architecture" : "X86", - "is_zipped" : true, - "md5sum" : "90e363abf0ddf22eefa2c7c5c9391c49", - "url" : "{url_base}/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", - "source" : "src/x86-ubuntu", - "root_partition": "1" - }, - { - "type" : "binary", - "name" : "binary-example", - "documentation" : "binary-example documentation.", - "architecture" : "ARM", - "is_zipped" : false, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url" : "{url_base}/test-progs/hello/bin/arm/linux/hello64-static", - "source" : "src/simple" - }, - { - "type" : "bootloader", - "name" : "bootloader-example", - "documentation" : "bootloader documentation.", - "is_zipped" : false, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url" : "{url_base}/test-progs/hello/bin/arm/linux/hello64-static" - }, - { - "type" : "checkpoint", - "name" : "checkpoint-example", - "documentation" : "checkpoint-example documentation.", - "architecture": "RISCV", - "is_zipped" : false, - "md5sum" : "3a57c1bb1077176c4587b8a3bf4f8ace", - "source" : null, - "is_tar_archive" : true, - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar" - }, - { - "type" : "git", - "name" : "git-example", - "documentation" : null, - "is_zipped" : false, - "is_tar_archive" : true, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar" - }, - { - "type" : "file", - "name" : "file-example", - "documentation" : null, - "is_zipped" : false, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar", - "source" : null - }, - { - "type" : "directory", - "name" : "directory-example", - "documentation" : "directory-example documentation.", - "is_zipped" : false, - "md5sum" : "3a57c1bb1077176c4587b8a3bf4f8ace", - "source" : null, - "is_tar_archive" : true, - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar" - }, - { - "type": "simpoint-directory", - "name": "simpoint-directory-example", - "documentation": "simpoint directory documentation.", - "is_zipped" : false, - "md5sum" : "3fcffe3956c8a95e3fb82e232e2b41fb", - "source" : null, - "is_tar_archive" : true, - "url": "{url_base}/simpoints/x86-print-this-15000-simpoints-20221013.tar", - "simpoint_interval": 1000000, - "warmup_interval": 1000000, - "simpoint_file": "simpoint.simpt", - "weight_file": "simpoint.weight", - "workload_name": "Example Workload" - }, - { - "type": "simpoint", - "name": "simpoint-example", - "documentation": "simpoint documentation.", - "simpoint_interval": 1000000, - "warmup_interval": 23445, - "simpoint_list" : [2,3,4,15], - "weight_list" : [0.1, 0.2, 0.4, 0.3] - }, - { - "type": "looppoint-pinpoint-csv", - "name": "looppoint-pinpoint-csv-resource", - "documentation" : "A looppoint pinpoints csv file.", - "is_zipped" : false, - "md5sum" : "199ab22dd463dc70ee2d034bfe045082", - "url": "http://dist.gem5.org/dist/develop/pinpoints/x86-matrix-multiply-omp-100-8-global-pinpoints-20230127", - "source" : null - }, - { - "type": "looppoint-json", - "name": "looppoint-json-restore-resource-region-1", - "documentation" : "A looppoint json file resource.", - "is_zipped" : false, - "region_id" : "1", - "md5sum" : "a71ed64908b082ea619b26b940a643c1", - "url": "http://dist.gem5.org/dist/develop/looppoints/x86-matrix-multiply-omp-100-8-looppoint-json-20230128", - "source" : null - } - ] -} + { + "category": "disk_image", + "id": "disk-image-example", + "description": "disk-image documentation.", + "architecture": "X86", + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "source": "src/x86-ubuntu", + "root_partition": "1", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "binary", + "id": "binary-example", + "description": "binary-example documentation.", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/simple", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "bootloader", + "id": "bootloader-example", + "description": "bootloader documentation.", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "checkpoint", + "id": "checkpoint-example", + "description": "checkpoint-example documentation.", + "architecture": "RISCV", + "is_zipped": false, + "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", + "source": null, + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "git", + "id": "git-example", + "description": null, + "is_zipped": false, + "is_tar_archive": true, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "file", + "id": "file-example", + "description": null, + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "source": null, + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "directory", + "id": "directory-example", + "description": "directory-example documentation.", + "is_zipped": false, + "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", + "source": null, + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "simpoint-directory", + "id": "simpoint-directory-example", + "description": "simpoint directory documentation.", + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "source": null, + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "simpoint_file": "simpoint.simpt", + "weight_file": "simpoint.weight", + "workload_name": "Example Workload", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "simpoint", + "id": "simpoint-example", + "description": "simpoint documentation.", + "simpoint_interval": 1000000, + "warmup_interval": 23445, + "simpoint_list": [ + 2, + 3, + 4, + 15 + ], + "weight_list": [ + 0.1, + 0.2, + 0.4, + 0.3 + ], + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "looppoint-pinpoint-csv", + "id": "looppoint-pinpoint-csv-resource", + "description": "A looppoint pinpoints csv file.", + "is_zipped": false, + "md5sum": "199ab22dd463dc70ee2d034bfe045082", + "url": "http://dist.gem5.org/dist/develop/pinpoints/x86-matrix-multiply-omp-100-8-global-pinpoints-20230127", + "source": null, + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "looppoint-json", + "id": "looppoint-json-restore-resource-region-1", + "description": "A looppoint json file resource.", + "is_zipped": false, + "region_id": "1", + "md5sum": "a71ed64908b082ea619b26b940a643c1", + "url": "http://dist.gem5.org/dist/develop/looppoints/x86-matrix-multiply-omp-100-8-looppoint-json-20230128", + "source": null, + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + } +] \ No newline at end of file diff --git a/tests/pyunit/stdlib/resources/refs/resources.json b/tests/pyunit/stdlib/resources/refs/resources.json new file mode 100644 index 0000000..5e7f191 --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/resources.json @@ -0,0 +1,329 @@ +[ + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource", + "architecture": "X86", + "size": 13816, + "tags": [ + "asmtest", + "testing", + "riscv", + "testing" + ], + "is_zipped": false, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" + }, + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource but newer", + "architecture": "X86", + "size": 13816, + "tags": [ + "asmtest", + "testing", + "riscv", + "testing", + "new" + ], + "is_zipped": false, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "1.1.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" + }, + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource but double newer", + "architecture": "X86", + "size": 13816, + "tags": [ + "asmtest" + ], + "is_zipped": false, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "2.0.0", + "gem5_versions": [ + "23.1" + ], + "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" + }, + { + "category": "simpoint", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "dasdasd", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "aa", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "asfsaf", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "simpoint-resource", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "bat43f34fman", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "adadadas", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "disk_image", + "id": "x86-ubuntu-18.04-img", + "description": "This is a test resource", + "architecture": "X86", + "size": 688119691, + "tags": [ + "x86", + "fullsystem" + ], + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "source": "src/x86-ubuntu", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "root_partition": "1", + "code_examples": [], + "license": "", + "author": [ + "Ayaz Akram" + ], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "resource_version": "2.0.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" + } +] diff --git a/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json b/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json index a9dd2aa..5c03ae0 100644 --- a/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json +++ b/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json @@ -1,17 +1,16 @@ -{ - "version" : null, - "url_base" : "http://dist.gem5.org/dist/v22-0", - "previous-versions" : {}, - "resources": [ - { - "type" : "binary", - "name" : "x86-hello64-static", - "documentation" : "A 'Hello World!' binary.", - "architecture" : "X86", - "is_zipped" : false, - "md5sum" : "dbf120338b37153e3334603970cebd8c", - "url" : "{url_base}/test-progs/hello/bin/x86/linux/hello64-static", - "source" : "src/simple" - } - ] -} +[ + { + "category": "binary", + "id": "x86-hello64-static", + "description": "A 'Hello World!' binary.", + "architecture": "X86", + "is_zipped": false, + "md5sum": "dbf120338b37153e3334603970cebd8c", + "url": "{url_base}/test-progs/hello/bin/x86/linux/hello64-static", + "source": "src/simple", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + } +] \ No newline at end of file diff --git a/tests/pyunit/stdlib/resources/refs/workload-checks.json b/tests/pyunit/stdlib/resources/refs/workload-checks.json index 4f7e76b..e1ba6ab 100644 --- a/tests/pyunit/stdlib/resources/refs/workload-checks.json +++ b/tests/pyunit/stdlib/resources/refs/workload-checks.json @@ -1,40 +1,48 @@ -{ - "url_base" : "http://dist.gem5.org/dist/v22-0", - "previous-versions" : {}, - "resources": [ - { - "type" : "kernel", - "name" : "x86-linux-kernel-5.2.3", - "documentation" : "The linux kernel (v5.2.3), compiled to X86.", - "architecture" : "X86", - "is_zipped" : false, - "md5sum" : "4838c99b77d33c8307b939c16624e4ac", - "url" : "{url_base}/kernels/x86/static/vmlinux-5.2.3", - "source" : "src/linux-kernel" +[ + { + "category": "kernel", + "id": "x86-linux-kernel-5.2.3", + "description": "The linux kernel (v5.2.3), compiled to X86.", + "architecture": "X86", + "is_zipped": false, + "md5sum": "4838c99b77d33c8307b939c16624e4ac", + "url": "{url_base}/kernels/x86/static/vmlinux-5.2.3", + "source": "src/linux-kernel", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "disk_image", + "id": "x86-ubuntu-18.04-img", + "description": "A disk image containing Ubuntu 18.04 for x86..", + "architecture": "X86", + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "url": "{url_base}/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "source": "src/x86-ubuntu", + "root_partition": "1", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "workload", + "id": "simple-boot", + "description": "Description of workload here", + "function": "set_kernel_disk_workload", + "resources": { + "kernel": "x86-linux-kernel-5.2.3", + "disk_image": "x86-ubuntu-18.04-img" }, - { - "type" : "disk-image", - "name" : "x86-ubuntu-18.04-img", - "documentation" : "A disk image containing Ubuntu 18.04 for x86..", - "architecture" : "X86", - "is_zipped" : true, - "md5sum" : "90e363abf0ddf22eefa2c7c5c9391c49", - "url" : "{url_base}/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", - "source" : "src/x86-ubuntu", - "root_partition": "1" + "additional_params": { + "readfile_contents": "echo 'Boot successful'; m5 exit" }, - { - "type" : "workload", - "name" : "simple-boot", - "documentation" : "Description of workload here", - "function" : "set_kernel_disk_workload", - "resources" : { - "kernel" : "x86-linux-kernel-5.2.3", - "disk_image" : "x86-ubuntu-18.04-img" - }, - "additional_params" : { - "readfile_contents" : "echo 'Boot successful'; m5 exit" - } - } - ] -} + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + } +] \ No newline at end of file -- To view, visit https://gem5-review.googlesource.com/c/public/gem5/+/70858?usp=email To unsubscribe, or for help writing mail filters, visit https://gem5-review.googlesource.com/settings?usp=email Gerrit-MessageType: newchange Gerrit-Project: public/gem5 Gerrit-Branch: develop Gerrit-Change-Id: Ia9bf47f7900763827fd5e873bcd663cc3ecdba40 Gerrit-Change-Number: 70858 Gerrit-PatchSet: 1 Gerrit-Owner: Kunal Pai <kunpai@ucdavis.edu>