Device.cpp

Go to the documentation of this file.

// -------------------------------------------------------//
//
// SHAMROCK code for hydrodynamics
// Copyright (c) 2021-2026 Timothée David--Cléris <tim.shamrock@proton.me>
// SPDX-License-Identifier: CeCILL Free Software License Agreement v2.1
// Shamrock is licensed under the CeCILL 2.1 License, see LICENSE for more information
//
// -------------------------------------------------------//
 
#include "shambackends/Device.hpp"
#include "shambase/logs/loglevels.hpp"
#include "shambase/memory.hpp"
#include "shambase/numeric_limits.hpp"
#include "shambase/popen.hpp"
#include "shambase/string.hpp"
#include "shambackends/sysinfo.hpp"
#include "shamcmdopt/env.hpp"
#include "shamcomm/logs.hpp"
#include "shamcomm/mpiInfo.hpp"
#include <fmt/ranges.h>
#include <nlohmann/json.hpp>
 
auto SHAM_MAX_ALLOC_SIZE
    = shamcmdopt::getenv_str_register("SHAM_MAX_ALLOC_SIZE", "shamrock max alloc size if set");
 
namespace {
 
    std::optional<std::string> get_cpu_name() {
#ifdef __linux__
        std::string lscpu_json = "";
        try {
            lscpu_json = shambase::popen_fetch_output("lscpu -J");
        } catch (const std::exception &e) {
            return std::nullopt;
        }
 
        nlohmann::json lscpu_json_parsed;
        try {
            lscpu_json_parsed = nlohmann::json::parse(lscpu_json);
        } catch (const std::exception &e) {
            return std::nullopt;
        }
 
        try {
            const auto &entries = lscpu_json_parsed.at("lscpu");
            if (!entries.is_array()) {
                return std::nullopt;
            }
            std::optional<std::string> model_name;
            std::optional<std::string> socket_count;
            for (const auto &entry : entries) {
                const std::string field = entry.at("field").get<std::string>();
                const std::string data  = entry.at("data").get<std::string>();
                if (field == "Model name:") {
                    model_name = data;
                } else if (field == "Socket(s):") {
                    socket_count = data;
                }
            }
            if (!model_name.has_value() || !socket_count.has_value() || model_name->empty()
                || socket_count->empty()) {
                return std::nullopt;
            }
 
            if (*socket_count == "1") {
                return *model_name;
            } else {
                return fmt::format("{} x {}", *model_name, *socket_count);
            }
 
        } catch (const std::exception &e) {
            return std::nullopt;
        }
#elif defined(__APPLE__)
        // in that case the command is simply sysctl -n machdep.cpu.brand_string
        std::string brand_string = "";
        try {
            brand_string = shambase::popen_fetch_output("sysctl -n machdep.cpu.brand_string");
            shambase::replace_all(brand_string, "\n", "");
            return brand_string;
        } catch (const std::exception &e) {
            return std::nullopt;
        }
        return brand_string;
#else
        return std::nullopt;
#endif
    }
 
} // namespace
 
namespace sham {
 
    Backend get_device_backend(const sycl::device &dev) {
        std::string pname = dev.get_platform().get_info<sycl::info::platform::name>();
 
        // The platform name may include information about the device
        // and/or the backend. We look for some keywords to determine
        // the backend.
        if (shambase::contain_substr(pname, "CUDA")) {
            return Backend::CUDA; // NVIDIA CUDA
        }
        if (shambase::contain_substr(pname, "NVIDIA")) {
            return Backend::CUDA;
        }
        if (shambase::contain_substr(pname, "ROCM")) {
            return Backend::ROCM; // AMD ROCm
        }
        if (shambase::contain_substr(pname, "AMD")) {
            return Backend::ROCM;
        }
        if (shambase::contain_substr(pname, "HIP")) {
            return Backend::ROCM; // AMD ROCm
        }
        if (shambase::contain_substr(pname, "OpenMP")) {
            return Backend::OPENMP; // OpenMP
        }
 
        return Backend::UNKNOWN; // Unknown backend
    }
 
    DeviceType get_device_type(const sycl::device &dev) {
        auto DeviceType = dev.get_info<sycl::info::device::device_type>();
        switch (DeviceType) {
        case sycl::info::device_type::cpu: return DeviceType::CPU;
        case sycl::info::device_type::gpu: return DeviceType::GPU;
        default                          : return DeviceType::UNKNOWN;
        }
    }
 
#define FETCH_PROP(info_, info_type)                                                               \
    std::optional<info_type> info_ = [&]() -> std::optional<info_type> {                           \
        try {                                                                                      \
            return {dev.get_info<sycl::info::device::info_>()};                                    \
        } catch (...) {                                                                            \
            warnings.push_back(                                                                    \
                "dev.get_info<sycl::info::device::" #info_ ">() raised an exception for device "   \
                + name);                                                                           \
            return {};                                                                             \
        }                                                                                          \
    }();
 
#define FETCH_PROPN(info_, info_type, n)                                                           \
    std::optional<info_type> n = [&]() -> std::optional<info_type> {                               \
        try {                                                                                      \
            return {dev.get_info<sycl::info::device::info_>()};                                    \
        } catch (...) {                                                                            \
            warnings.push_back(                                                                    \
                "dev.get_info<sycl::info::device::" #info_ ">() raised an exception for device "   \
                + name);                                                                           \
            return {};                                                                             \
        }                                                                                          \
    }();
 
#define FETCH_PROPN_FULL(info_, info_type, n)                                                      \
    std::optional<info_type> n = [&]() -> std::optional<info_type> {                               \
        try {                                                                                      \
            return {dev.get_info<info_>()};                                                        \
        } catch (...) {                                                                            \
            warnings.push_back(                                                                    \
                "dev.get_info<" #info_ ">() raised an exception for device " + name);              \
            return {};                                                                             \
        }                                                                                          \
    }();
 
    DeviceProperties fetch_properties(const sycl::device &dev) {
 
        std::vector<std::string> warnings;
 
        // Just to ensure that this one is not empty
        std::string name = "?";
        FETCH_PROPN(name, std::string, dev_name);
        if (dev_name) {
            name = *dev_name;
        }
 
#if SYCL_COMP_ACPP
        if (get_device_backend(dev) == Backend::OPENMP) {
            auto cpu_name = get_cpu_name();
            if (cpu_name) {
                name = *cpu_name;
            }
        }
#endif
 
        FETCH_PROP(vendor, std::string)
 
        FETCH_PROP(device_type, sycl::info::device_type)
        FETCH_PROP(vendor_id, uint32_t)
        FETCH_PROP(max_compute_units, uint32_t)
        FETCH_PROP(max_work_item_dimensions, uint32_t)
        FETCH_PROPN(max_work_item_sizes<1>, sycl::id<1>, max_work_item_sizes_1d)
        FETCH_PROPN(max_work_item_sizes<2>, sycl::id<2>, max_work_item_sizes_2d)
        FETCH_PROPN(max_work_item_sizes<3>, sycl::id<3>, max_work_item_sizes_3d)
        FETCH_PROP(max_work_group_size, size_t)
        FETCH_PROP(max_num_sub_groups, uint32_t)
        FETCH_PROP(sub_group_independent_forward_progress, bool)
        FETCH_PROP(sub_group_sizes, std::vector<size_t>)
 
        FETCH_PROP(preferred_vector_width_char, uint32_t)
        FETCH_PROP(preferred_vector_width_short, uint32_t)
        FETCH_PROP(preferred_vector_width_int, uint32_t)
        FETCH_PROP(preferred_vector_width_long, uint32_t)
        FETCH_PROP(preferred_vector_width_float, uint32_t)
        FETCH_PROP(preferred_vector_width_double, uint32_t)
        FETCH_PROP(preferred_vector_width_half, uint32_t)
        FETCH_PROP(native_vector_width_char, uint32_t)
        FETCH_PROP(native_vector_width_short, uint32_t)
        FETCH_PROP(native_vector_width_int, uint32_t)
        FETCH_PROP(native_vector_width_long, uint32_t)
        FETCH_PROP(native_vector_width_float, uint32_t)
        FETCH_PROP(native_vector_width_double, uint32_t)
        FETCH_PROP(native_vector_width_half, uint32_t)
 
        FETCH_PROP(max_clock_frequency, uint32_t)
        FETCH_PROP(address_bits, uint32_t)
        FETCH_PROP(max_mem_alloc_size, uint64_t)
 
        // Image a really second class objects in SYCL right now ...
        // FETCH_PROP(max_read_image_args, uint32_t)
        // FETCH_PROP(max_write_image_args, uint32_t)
        // FETCH_PROP(image2d_max_width, size_t)
        // FETCH_PROP(image2d_max_height, size_t)
        // FETCH_PROP(image3d_max_width, size_t)
        // FETCH_PROP(image3d_max_height, size_t)
        // FETCH_PROP(image3d_max_depth, size_t)
        // FETCH_PROP(image_max_buffer_size, size_t)
        // FETCH_PROP(max_samplers, uint32_t)
 
        FETCH_PROP(max_parameter_size, size_t)
        FETCH_PROP(mem_base_addr_align, uint32_t)
        FETCH_PROP(half_fp_config, std::vector<sycl::info::fp_config>)
        FETCH_PROP(single_fp_config, std::vector<sycl::info::fp_config>)
        FETCH_PROP(double_fp_config, std::vector<sycl::info::fp_config>)
        FETCH_PROP(global_mem_cache_type, sycl::info::global_mem_cache_type)
        FETCH_PROP(global_mem_cache_line_size, uint32_t)
        FETCH_PROP(global_mem_cache_size, uint64_t)
        FETCH_PROP(global_mem_size, uint64_t)
        FETCH_PROP(local_mem_type, sycl::info::local_mem_type)
        FETCH_PROP(local_mem_size, uint64_t)
        FETCH_PROP(error_correction_support, bool)
#ifdef SYCL_COMP_INTEL_LLVM
        FETCH_PROP(atomic_memory_order_capabilities, std::vector<sycl::memory_order>)
        FETCH_PROP(atomic_fence_order_capabilities, std::vector<sycl::memory_order>)
        FETCH_PROP(atomic_memory_scope_capabilities, std::vector<sycl::memory_scope>)
        FETCH_PROP(atomic_fence_scope_capabilities, std::vector<sycl::memory_scope>)
#endif
        FETCH_PROP(profiling_timer_resolution, size_t)
        FETCH_PROP(is_available, bool)
        FETCH_PROP(execution_capabilities, std::vector<sycl::info::execution_capability>)
        // FETCH_PROP(built_in_kernel_ids,std::vector<sycl::kernel_id>)
        // FETCH_PROP(built_in_kernels, std::vector<std::string>)
        // FETCH_PROP(platform, sycl::platform)
 
        FETCH_PROP(driver_version, std::string)
        FETCH_PROP(version, std::string)
#ifdef SYCL_COMP_INTEL_LLVM
        FETCH_PROP(backend_version, std::string)
#endif
        // FETCH_PROP(aspects, std::vector<sycl::aspect>)
        // FETCH_PROP(printf_buffer_size, size_t)
#ifdef SYCL_COMP_INTEL_LLVM
        // FETCH_PROP(parent_device, device)
#endif
        FETCH_PROP(partition_max_sub_devices, uint32_t)
        FETCH_PROP(partition_properties, std::vector<sycl::info::partition_property>)
        FETCH_PROP(partition_affinity_domains, std::vector<sycl::info::partition_affinity_domain>)
        FETCH_PROP(partition_type_property, sycl::info::partition_property)
        FETCH_PROP(partition_type_affinity_domain, sycl::info::partition_affinity_domain)
 
        auto physmem = sham::getPhysicalMemory();
 
// On acpp 2^64-1 is returned, so we need to correct it
// see : https://github.com/AdaptiveCpp/AdaptiveCpp/issues/1573
#ifdef SYCL_COMP_ACPP
        if (get_device_backend(dev) == Backend::OPENMP) {
            // Correct memory size
            if (physmem) {
                global_mem_size = {*physmem};
            }
        }
#endif
 
        // with acpp 8 bit is returned for most backends so we default to 8 bytes (64 bits)
        if (*mem_base_addr_align && mem_base_addr_align == 8) {
            warnings.push_back(
                shambase::format(
                    "mem_base_addr_align for is {} bits. I will assume that this is an "
                    "issue and default to 64 bits (8 bytes) instead.",
                    *mem_base_addr_align));
            mem_base_addr_align = CHAR_BIT * 8;
        }
 
        // Some backends do not report sub_group_sizes, so we default to {1}
        u32 default_work_group_size = 1;
        if (!sub_group_sizes) {
            sub_group_sizes = std::vector<size_t>{default_work_group_size};
            warnings.push_back(
                shambase::format(
                    "cannot fetch sub_group_sizes, defaulting to {}", default_work_group_size));
        }
        default_work_group_size = shambase::get_check_ref(sub_group_sizes)[0];
 
        size_t max_alloc_dev  = shambase::get_check_ref(max_mem_alloc_size);
        size_t max_alloc_host = ((physmem) ? *physmem : i64_max);
        if (SHAM_MAX_ALLOC_SIZE) {
            try {
                const auto max_alloc = std::stoull(SHAM_MAX_ALLOC_SIZE.value());
                max_alloc_dev        = max_alloc;
                max_alloc_host       = max_alloc;
            } catch (const std::exception &e) {
                warnings.push_back(
                    shambase::format(
                        "Could not parse SHAM_MAX_ALLOC_SIZE value '{}'. Error: {}. "
                        "Ignoring override.",
                        SHAM_MAX_ALLOC_SIZE.value(),
                        e.what()));
            }
        }
 
        DeviceProperties ret = {
            Vendor::UNKNOWN,         // We cannot determine the vendor
            get_device_backend(dev), // Query the backend based on the platform name
            get_device_type(dev),
            name,
            dev.get_platform().get_info<sycl::info::platform::name>(),
            shambase::get_check_ref(global_mem_size),
            shambase::get_check_ref(global_mem_cache_line_size),
            shambase::get_check_ref(global_mem_cache_size),
            shambase::get_check_ref(local_mem_size),
            shambase::get_check_ref(max_compute_units),
            max_alloc_dev,
            max_alloc_host,
            // the SYCL standard returns the alignment in bits, we convert to bytes for convenience
            shambase::get_check_ref(mem_base_addr_align) / CHAR_BIT,
            shambase::get_check_ref(sub_group_sizes),
            default_work_group_size,
            std::nullopt,
            warnings};
 
        { // PCI id infos
#if defined(SYCL_EXT_INTEL_DEVICE_INFO) && SYCL_EXT_INTEL_DEVICE_INFO >= 5
            FETCH_PROPN_FULL(sycl::ext::intel::info::device::pci_address, std::string, pci_address)
            if (pci_address) {
                ret.pci_address = *pci_address;
            }
#endif
        }
 
        return ret;
    }
 
    DeviceMPIProperties fetch_mpi_properties(
        const sycl::device &dev, const DeviceProperties &prop) {
        bool dgpu_capable = false;
 
        // If CUDA-aware MPI is enabled, and the device is a CUDA device,
        // then we can use it
        if (shamcomm::is_direct_comm_aware(shamcomm::get_mpi_cuda_aware_status())
            && (prop.backend == Backend::CUDA)) {
            dgpu_capable = true;
        }
 
        // Same for ROCm-aware MPI and ROCm devices
        if (shamcomm::is_direct_comm_aware(shamcomm::get_mpi_rocm_aware_status())
            && (prop.backend == Backend::ROCM)) {
            dgpu_capable = true;
        }
 
        // And for OpenMP since the data is on host is it by definition aware
        if (prop.backend == Backend::OPENMP) {
            dgpu_capable = true;
        }
 
        // For other cases we can still force the DGPU state by setting a forced state
        if (auto forcing = shamcomm::should_force_dgpu_state()) {
            dgpu_capable = shamcomm::is_direct_comm_aware(*forcing);
        }
 
        return DeviceMPIProperties{dgpu_capable};
    }
 
    std::vector<sycl::device> get_sycl_device_list() {
        std::vector<sycl::device> devs; // The list of devices to be returned
        const auto &Platforms = sycl::platform::get_platforms();
        for (const auto &Platform : Platforms) {
            const auto &Devices = Platform.get_devices();
            for (const auto &Device : Devices) {
                devs.push_back(Device);
            }
        }
        return devs;
    }
 
    Device sycl_dev_to_sham_dev(usize i, const sycl::device &dev) {
        DeviceProperties prop       = fetch_properties(dev); // Get the properties of the device
        DeviceMPIProperties propmpi = {false};               // Get the MPI properties
        return Device{
            i,      // The index of the device
            dev,    // The SYCL device
            prop,   // The properties of the device
            propmpi // The MPI properties of the device
        };
    }
 
    std::vector<std::unique_ptr<Device>> get_device_list() {
        std::vector<sycl::device> devs = get_sycl_device_list();
        std::vector<std::unique_ptr<Device>> ret; // The return list of unique pointers to Device
        ret.reserve(devs.size());
 
        for (const sycl::device &dev : devs) {
            usize i = ret.size(); // Get the current index of the device
            ret.push_back(std::make_unique<Device>(sycl_dev_to_sham_dev(i, dev)));
        }
 
        return ret;
    }
 
    void Device::update_mpi_prop() { mpi_prop = fetch_mpi_properties(dev, prop); }
 
    void Device::print_info() {
        shamcomm::logs::raw_ln("  Device info :");
        switch (prop.backend) {
        case sham::Backend::OPENMP : shamcomm::logs::raw_ln("   - Backend : OpenMP"); break;
        case sham::Backend::CUDA   : shamcomm::logs::raw_ln("   - Backend : CUDA"); break;
        case sham::Backend::ROCM   : shamcomm::logs::raw_ln("   - Backend : ROCM"); break;
        case sham::Backend::UNKNOWN: shamcomm::logs::raw_ln("   - Backend : Unknown"); break;
        }
        switch (prop.vendor) {
        case sham::Vendor::AMD    : shamcomm::logs::raw_ln("   - Vendor : AMD"); break;
        case sham::Vendor::APPLE  : shamcomm::logs::raw_ln("   - Vendor : Apple"); break;
        case sham::Vendor::INTEL  : shamcomm::logs::raw_ln("   - Vendor : Intel"); break;
        case sham::Vendor::NVIDIA : shamcomm::logs::raw_ln("   - Vendor : Nvidia"); break;
        case sham::Vendor::UNKNOWN: shamcomm::logs::raw_ln("   - Vendor : Unknown"); break;
        }
        logger::raw_ln("   - Global mem size :", shambase::readable_sizeof(prop.global_mem_size));
        logger::raw_ln(
            "   - Cache line size :", shambase::readable_sizeof(prop.global_mem_cache_line_size));
        logger::raw_ln(
            "   - Cache size      :", shambase::readable_sizeof(prop.global_mem_cache_size));
        logger::raw_ln("   - Local mem size  :", shambase::readable_sizeof(prop.local_mem_size));
        logger::raw_ln("   - Direct MPI capable :", mpi_prop.is_mpi_direct_capable);
    }
 
} // namespace sham