data_set.hpp

Go to the documentation of this file.

 
#ifndef PLSSVM_DATA_SET_HPP_
#define PLSSVM_DATA_SET_HPP_
#pragma once
 
#include "plssvm/detail/io/arff_parsing.hpp"             // plssvm::detail::io::{read_libsvm_data, write_libsvm_data}
#include "plssvm/detail/io/file_reader.hpp"              // plssvm::detail::io::file_reader
#include "plssvm/detail/io/libsvm_parsing.hpp"           // plssvm::detail::io::{read_arff_data, write_arff_data}
#include "plssvm/detail/io/scaling_factors_parsing.hpp"  // plssvm::detail::io::{parse_scaling_factors, read_scaling_factors}
#include "plssvm/detail/logger.hpp"                      // plssvm::detail::log, plssvm::verbosity_level
#include "plssvm/detail/performance_tracker.hpp"         // plssvm::detail::tracking_entry
#include "plssvm/detail/string_utility.hpp"              // plssvm::detail::ends_with
#include "plssvm/detail/type_list.hpp"                   // plssvm::detail::{real_type_list, label_type_list, type_list_contains_v}
#include "plssvm/detail/utility.hpp"                     // plssvm::detail::contains
#include "plssvm/exceptions/exceptions.hpp"              // plssvm::data_set_exception
#include "plssvm/file_format_types.hpp"                  // plssvm::file_format_type
 
#include "fmt/chrono.h"                                  // directly output std::chrono times via fmt
#include "fmt/core.h"                                    // fmt::format
#include "fmt/ostream.h"                                 // directly output objects with operator<< overload via fmt
 
#include <algorithm>                                     // std::all_of, std::max, std::min, std::sort, std::adjacent_find
#include <chrono>                                        // std::chrono::{time_point, steady_clock, duration_cast, millisecond}
#include <cstddef>                                       // std::size_t
#include <functional>                                    // std::reference_wrapper, std::cref
#include <iostream>                                      // std::cout, std::endl
#include <limits>                                        // std::numeric_limits::{max, lowest}
#include <map>                                           // std::map
#include <memory>                                        // std::shared_ptr, std::make_shared
#include <optional>                                      // std::optional, std::make_optional, std::nullopt
#include <set>                                           // std::set
#include <string>                                        // std::string
#include <tuple>                                         // std::tie
#include <utility>                                       // std::move, std::pair, std::make_pair
#include <vector>                                        // std::vector
 
namespace plssvm {
 
template <typename T>
using optional_ref = std::optional<std::reference_wrapper<T>>;
 
template <typename T, typename U = int>
class data_set {
    // make sure only valid template types are used
    static_assert(detail::type_list_contains_v<T, detail::real_type_list>, "Illegal real type provided! See the 'real_type_list' in the type_list.hpp header for a list of the allowed types.");
    static_assert(detail::type_list_contains_v<U, detail::label_type_list>, "Illegal label type provided! See the 'label_type_list' in the type_list.hpp header for a list of the allowed types.");
 
    // plssvm::model needs the default constructor
    template <typename, typename>
    friend class model;
    // plssvm::csvm needs the label mapping
    friend class csvm;
 
  public:
    using real_type = T;
    using label_type = U;
    using size_type = std::size_t;
 
    // forward declare the scaling class
    class scaling;
    // forward declare the label_mapper class
    class label_mapper;
 
    explicit data_set(const std::string &filename);
    data_set(const std::string &filename, file_format_type format);
    data_set(const std::string &filename, scaling scale_parameter);
    data_set(const std::string &filename, file_format_type format, scaling scale_parameter);
 
    explicit data_set(std::vector<std::vector<real_type>> data_points);
    data_set(std::vector<std::vector<real_type>> data_points, std::vector<label_type> labels);
    data_set(std::vector<std::vector<real_type>> data_points, scaling scale_parameter);
    data_set(std::vector<std::vector<real_type>> data_points, std::vector<label_type> labels, scaling scale_parameter);
 
    void save(const std::string &filename, file_format_type format) const;
    void save(const std::string &filename) const;
 
    [[nodiscard]] const std::vector<std::vector<real_type>> &data() const noexcept { return *X_ptr_; }
    [[nodiscard]] bool has_labels() const noexcept { return labels_ptr_ != nullptr; }
    [[nodiscard]] optional_ref<const std::vector<label_type>> labels() const noexcept;
    [[nodiscard]] std::optional<std::vector<label_type>> different_labels() const;
 
    [[nodiscard]] size_type num_data_points() const noexcept { return num_data_points_; }
    [[nodiscard]] size_type num_features() const noexcept { return num_features_; }
    [[nodiscard]] size_type num_different_labels() const noexcept { return mapping_ != nullptr ? mapping_->num_mappings() : 0; }
 
    [[nodiscard]] bool is_scaled() const noexcept { return scale_parameters_ != nullptr; }
    [[nodiscard]] optional_ref<const scaling> scaling_factors() const noexcept;
 
  private:
    data_set() :
        X_ptr_{ std::make_shared<std::vector<std::vector<real_type>>>() } {}
 
    void create_mapping();
    void scale();
    void read_file(const std::string &filename, file_format_type format);
 
    std::shared_ptr<std::vector<std::vector<real_type>>> X_ptr_{ nullptr };
    std::shared_ptr<std::vector<label_type>> labels_ptr_{ nullptr };
    std::shared_ptr<std::vector<real_type>> y_ptr_{ nullptr };
 
    size_type num_data_points_{ 0 };
    size_type num_features_{ 0 };
 
    std::shared_ptr<const label_mapper> mapping_{ nullptr };
    std::shared_ptr<scaling> scale_parameters_{ nullptr };
};
 
//*************************************************************************************************************************************//
//                                                         scaling nested-class                                                        //
//*************************************************************************************************************************************//
 
template <typename T, typename U>
class data_set<T, U>::scaling {
  public:
    struct factors {
        factors() = default;
        factors(const size_type feature_index, const real_type lower_bound, const real_type upper_bound) :
            feature{ feature_index }, lower{ lower_bound }, upper{ upper_bound } {}
 
        size_type feature{};
        real_type lower{};
        real_type upper{};
    };
 
    scaling(real_type lower, real_type upper);
    scaling(const std::string &filename);
 
    void save(const std::string &filename) const;
 
    std::pair<real_type, real_type> scaling_interval{};
    std::vector<factors> scaling_factors{};
};
 
template <typename T, typename U>
data_set<T, U>::scaling::scaling(const real_type lower, const real_type upper) :
    scaling_interval{ std::make_pair(lower, upper) } {
    if (lower >= upper) {
        throw data_set_exception{ fmt::format("Inconsistent scaling interval specification: lower ({}) must be less than upper ({})!", lower, upper) };
    }
}
 
template <typename T, typename U>
data_set<T, U>::scaling::scaling(const std::string &filename) {
    // open the file
    detail::io::file_reader reader{ filename };
    reader.read_lines('#');
 
    // read scaling values from file
    std::tie(scaling_interval, scaling_factors) = detail::io::parse_scaling_factors<real_type, factors>(reader);
}
 
template <typename T, typename U>
void data_set<T, U>::scaling::save(const std::string &filename) const {
    const std::chrono::time_point start_time = std::chrono::steady_clock::now();
 
    // write scaling values to file
    detail::io::write_scaling_factors(filename, scaling_interval, scaling_factors);
 
    const std::chrono::time_point end_time = std::chrono::steady_clock::now();
    detail::log(verbosity_level::full | verbosity_level::timing,
                "Write {} scaling factors in {} to the file '{}'.\n",
                detail::tracking_entry{ "scaling_factors_write", "num_scaling_factors", scaling_factors.size() },
                detail::tracking_entry{ "scaling_factors_write", "time", std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time) },
                detail::tracking_entry{ "scaling_factors_write", "filename", filename });
}
 
//*************************************************************************************************************************************//
//                                                      label mapper nested-class                                                      //
//*************************************************************************************************************************************//
 
template <typename T, typename U>
class data_set<T, U>::label_mapper {
  public:
    explicit label_mapper(const std::vector<label_type> &labels);
 
    [[nodiscard]] const real_type &get_mapped_value_by_label(const label_type &label) const;
    [[nodiscard]] const label_type &get_label_by_mapped_value(const real_type &mapped_value) const;
    [[nodiscard]] size_type num_mappings() const noexcept;
    [[nodiscard]] std::vector<label_type> labels() const;
 
  private:
    std::map<label_type, real_type> label_to_mapped_{};
    std::map<real_type, label_type> mapped_to_label_{};
};
 
template <typename T, typename U>
data_set<T, U>::data_set::label_mapper::label_mapper(const std::vector<label_type> &labels) {
    // we are only interested in unique labels
    std::set<label_type> unique_labels(labels.begin(), labels.end());
    // currently, only two different labels are supported
    if (unique_labels.size() != 2) {
        throw data_set_exception{ fmt::format("Currently only binary classification is supported, but {} different labels were given!", unique_labels.size()) };
    }
    // create mapping
    // first label
    auto iter = unique_labels.begin();
    label_to_mapped_[*iter] = -1;
    mapped_to_label_[-1] = *iter;
    // second label
    ++iter;
    label_to_mapped_[*iter] = +1;
    mapped_to_label_[+1] = *iter;
}
 
template <typename T, typename U>
auto data_set<T, U>::label_mapper::get_mapped_value_by_label(const label_type &label) const -> const real_type & {
    if (!detail::contains(label_to_mapped_, label)) {
        throw data_set_exception{ fmt::format("Label \"{}\" unknown in this label mapping!", label) };
    }
    return label_to_mapped_.at(label);
}
 
template <typename T, typename U>
auto data_set<T, U>::label_mapper::get_label_by_mapped_value(const real_type &mapped_value) const -> const label_type & {
    if (!detail::contains(mapped_to_label_, mapped_value)) {
        throw data_set_exception{ fmt::format("Mapped value \"{}\" unknown in this label mapping!", mapped_value) };
    }
    return mapped_to_label_.at(mapped_value);
}
 
template <typename T, typename U>
auto data_set<T, U>::label_mapper::num_mappings() const noexcept -> size_type {
    PLSSVM_ASSERT(label_to_mapped_.size() == mapped_to_label_.size(), "Both maps must contain the same number of values, but {} and {} were given!", label_to_mapped_.size(), mapped_to_label_.size());
    return label_to_mapped_.size();
}
 
template <typename T, typename U>
auto data_set<T, U>::label_mapper::labels() const -> std::vector<label_type> {
    std::vector<label_type> available_labels;
    available_labels.reserve(this->num_mappings());
    for (const auto &[key, value] : label_to_mapped_) {
        available_labels.push_back(key);
    }
    return available_labels;
}
 
//*************************************************************************************************************************************//
//                                                           data set class                                                            //
//*************************************************************************************************************************************//
 
template <typename T, typename U>
data_set<T, U>::data_set(const std::string &filename) {
    // read data set from file
    // if the file doesn't end with .arff, assume a LIBSVM file
    this->read_file(filename, detail::ends_with(filename, ".arff") ? file_format_type::arff : file_format_type::libsvm);
}
 
template <typename T, typename U>
data_set<T, U>::data_set(const std::string &filename, const file_format_type format) {
    // read data set from file
    this->read_file(filename, format);
}
 
template <typename T, typename U>
data_set<T, U>::data_set(const std::string &filename, scaling scale_parameter) :
    data_set{ filename } {
    // initialize scaling
    scale_parameters_ = std::make_shared<scaling>(std::move(scale_parameter));
    // scale data set
    this->scale();
}
 
template <typename T, typename U>
data_set<T, U>::data_set(const std::string &filename, file_format_type format, scaling scale_parameter) :
    data_set{ filename, format } {
    // initialize scaling
    scale_parameters_ = std::make_shared<scaling>(std::move(scale_parameter));
    // scale data set
    this->scale();
}
 
template <typename T, typename U>
data_set<T, U>::data_set(std::vector<std::vector<real_type>> data_points) :
    X_ptr_{ std::make_shared<std::vector<std::vector<real_type>>>(std::move(data_points)) } {
    // the provided data points vector may not be empty
    if (X_ptr_->empty()) {
        throw data_set_exception{ "Data vector is empty!" };
    }
    // check that all data points have the same number of features
    if (!std::all_of(X_ptr_->cbegin(), X_ptr_->cend(), [this](const std::vector<real_type> &point) { return point.size() == X_ptr_->front().size(); })) {
        throw data_set_exception{ "All points in the data vector must have the same number of features!" };
    }
    // check that the data points have at least one feature
    if (X_ptr_->front().size() == 0) {
        throw data_set_exception{ "No features provided for the data points!" };
    }
 
    num_data_points_ = X_ptr_->size();
    num_features_ = X_ptr_->front().size();
}
 
template <typename T, typename U>
data_set<T, U>::data_set(std::vector<std::vector<real_type>> data_points, std::vector<label_type> labels) :
    data_set{ std::move(data_points) } {
    // initialize labels
    labels_ptr_ = std::make_shared<std::vector<label_type>>(std::move(labels));
    // the number of labels must be equal to the number of data points!
    if (X_ptr_->size() != labels_ptr_->size()) {
        throw data_set_exception{ fmt::format("Number of labels ({}) must match the number of data points ({})!", labels_ptr_->size(), X_ptr_->size()) };
    }
 
    // create mapping from labels
    this->create_mapping();
}
 
template <typename T, typename U>
data_set<T, U>::data_set(std::vector<std::vector<real_type>> data_points, scaling scale_parameter) :
    data_set{ std::move(data_points) } {
    // initialize scaling
    scale_parameters_ = std::make_shared<scaling>(std::move(scale_parameter));
    // scale data set
    this->scale();
}
 
template <typename T, typename U>
data_set<T, U>::data_set(std::vector<std::vector<real_type>> data_points, std::vector<label_type> labels, scaling scale_parameter) :
    data_set{ std::move(data_points), std::move(labels) } {
    // initialize scaling
    scale_parameters_ = std::make_shared<scaling>(std::move(scale_parameter));
    // scale data set
    this->scale();
}
 
template <typename T, typename U>
void data_set<T, U>::save(const std::string &filename, const file_format_type format) const {
    const std::chrono::time_point start_time = std::chrono::steady_clock::now();
 
    // save the data set
    if (this->has_labels()) {
        // save data with labels
        switch (format) {
            case file_format_type::libsvm:
                detail::io::write_libsvm_data(filename, *X_ptr_, *labels_ptr_);
                break;
            case file_format_type::arff:
                detail::io::write_arff_data(filename, *X_ptr_, *labels_ptr_);
                break;
        }
    } else {
        // save data without labels
        switch (format) {
            case file_format_type::libsvm:
                detail::io::write_libsvm_data(filename, *X_ptr_);
                break;
            case file_format_type::arff:
                detail::io::write_arff_data(filename, *X_ptr_);
                break;
        }
    }
 
    const std::chrono::time_point end_time = std::chrono::steady_clock::now();
    detail::log(verbosity_level::full | verbosity_level::timing,
                "Write {} data points with {} features in {} to the {} file '{}'.\n",
                detail::tracking_entry{ "data_set_write", "num_data_points", num_data_points_ },
                detail::tracking_entry{ "data_set_write", "num_features", num_features_ },
                detail::tracking_entry{ "data_set_write", "time", std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time) },
                detail::tracking_entry{ "data_set_write", "format", format },
                detail::tracking_entry{ "data_set_write", "filename", filename });
}
 
template <typename T, typename U>
void data_set<T, U>::save(const std::string &filename) const {
    if (detail::ends_with(filename, ".libsvm")) {
        this->save(filename, file_format_type::libsvm);
    } else if (detail::ends_with(filename, ".arff")) {
        this->save(filename, file_format_type::arff);
    } else {
        throw data_set_exception(fmt::format("Unrecognized file extension for file \"{}\" (must be one of: .libsvm or .arff)!", filename));
    }
}
 
template <typename T, typename U>
auto data_set<T, U>::labels() const noexcept -> optional_ref<const std::vector<label_type>> {
    if (this->has_labels()) {
        return std::make_optional(std::cref(*labels_ptr_));
    }
    return std::nullopt;
}
 
template <typename T, typename U>
auto data_set<T, U>::different_labels() const -> std::optional<std::vector<label_type>> {
    if (this->has_labels()) {
        return std::make_optional(mapping_->labels());
    }
    return std::nullopt;
}
 
template <typename T, typename U>
auto data_set<T, U>::scaling_factors() const noexcept -> optional_ref<const scaling> {
    if (this->is_scaled()) {
        return std::make_optional(std::cref(*scale_parameters_));
    }
    return std::nullopt;
}
 
//*************************************************************************************************************************************//
//                                                      PRIVATE MEMBER FUNCTIONS                                                       //
//*************************************************************************************************************************************//
 
template <typename T, typename U>
void data_set<T, U>::create_mapping() {
    PLSSVM_ASSERT(labels_ptr_ != nullptr, "Can't create mapping if no labels are provided!");
 
    // create label mapping
    label_mapper mapper{ *labels_ptr_ };
 
    // convert input labels to now mapped values
    std::vector<real_type> tmp(labels_ptr_->size());
    #pragma omp parallel for default(shared) shared(tmp, mapper)
    for (typename std::vector<real_type>::size_type i = 0; i < tmp.size(); ++i) {
        tmp[i] = mapper.get_mapped_value_by_label((*labels_ptr_)[i]);
    }
    y_ptr_ = std::make_shared<std::vector<real_type>>(std::move(tmp));
    mapping_ = std::make_shared<const label_mapper>(std::move(mapper));
}
 
template <typename T, typename U>
void data_set<T, U>::scale() {
    PLSSVM_ASSERT(this->is_scaled(), "No scaling parameters given for scaling!");
 
    const std::chrono::time_point start_time = std::chrono::steady_clock::now();
 
    // unpack scaling interval pair
    const real_type lower = scale_parameters_->scaling_interval.first;
    const real_type upper = scale_parameters_->scaling_interval.second;
 
    // calculate scaling factors if necessary, use provided once otherwise
    if (scale_parameters_->scaling_factors.empty()) {
        // calculate feature-wise min/max values for scaling
        for (size_type feature = 0; feature < num_features_; ++feature) {
            real_type min_value = std::numeric_limits<real_type>::max();
            real_type max_value = std::numeric_limits<real_type>::lowest();
 
            // calculate min/max values of all data points at the specific feature
            #pragma omp parallel for default(shared) firstprivate(feature) reduction(min : min_value) reduction(max : max_value)
            for (size_type data_point = 0; data_point < num_data_points_; ++data_point) {
                min_value = std::min(min_value, (*X_ptr_)[data_point][feature]);
                max_value = std::max(max_value, (*X_ptr_)[data_point][feature]);
            }
 
            // add scaling factor only if min_value != 0.0 AND max_value != 0.0
            if (!(min_value == real_type{ 0.0 } && max_value == real_type{ 0.0 })) {
                scale_parameters_->scaling_factors.emplace_back(feature, min_value, max_value);
            }
        }
    } else {
        // the number of scaling factors may not exceed the number of features
        if (scale_parameters_->scaling_factors.size() > num_features_) {
            throw data_set_exception{ fmt::format("Need at most as much scaling factors as features in the data set are present ({}), but {} were given!", num_features_, scale_parameters_->scaling_factors.size()) };
        }
        // sort vector
        const auto scaling_factors_comp_less = [](const typename scaling::factors &lhs, const typename scaling::factors &rhs) { return lhs.feature < rhs.feature; };
        std::sort(scale_parameters_->scaling_factors.begin(), scale_parameters_->scaling_factors.end(), scaling_factors_comp_less);
        // check whether the biggest feature index is smaller than the number of features
        if (scale_parameters_->scaling_factors.back().feature >= num_features_) {
            throw data_set_exception{ fmt::format("The maximum scaling feature index most not be greater than {}, but is {}!", num_features_ - 1, scale_parameters_->scaling_factors.back().feature) };
        }
        // check that there are no duplicate entries
        const auto scaling_factors_comp_eq = [](const typename scaling::factors &lhs, const typename scaling::factors &rhs) { return lhs.feature == rhs.feature; };
        const auto iter = std::adjacent_find(scale_parameters_->scaling_factors.begin(), scale_parameters_->scaling_factors.end(), scaling_factors_comp_eq);
        if (iter != scale_parameters_->scaling_factors.end()) {
            throw data_set_exception{ fmt::format("Found more than one scaling factor for the feature index {}!", iter->feature) };
        }
    }
 
    // scale values
    #pragma omp parallel for default(shared) firstprivate(lower, upper)
    for (size_type i = 0; i < scale_parameters_->scaling_factors.size(); ++i) {
        // extract feature-wise min and max values
        const typename scaling::factors factor = scale_parameters_->scaling_factors[i];
        // scale data values
        for (size_type data_point = 0; data_point < num_data_points_; ++data_point) {
            (*X_ptr_)[data_point][factor.feature] = lower + (upper - lower) * ((*X_ptr_)[data_point][factor.feature] - factor.lower) / (factor.upper - factor.lower);
        }
    }
 
    const std::chrono::time_point end_time = std::chrono::steady_clock::now();
    detail::log(verbosity_level::full | verbosity_level::timing,
                "Scaled the data set to the range [{}, {}] in {}.\n",
                detail::tracking_entry{ "data_set_scale", "lower", lower },
                detail::tracking_entry{ "data_set_scale", "upper", upper },
                detail::tracking_entry{ "data_set_scale", "time", std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time) });
}
 
template <typename T, typename U>
void data_set<T, U>::read_file(const std::string &filename, file_format_type format) {
    const std::chrono::time_point start_time = std::chrono::steady_clock::now();
 
    // get the comment character based on the file_format_type
    char comment{ ' ' };
    switch (format) {
        case file_format_type::libsvm:
            comment = '#';
            break;
        case file_format_type::arff:
            comment = '%';
            break;
    }
 
    // open the file
    detail::io::file_reader reader{ filename };
    reader.read_lines(comment);
 
    // create the empty placeholders
    std::vector<std::vector<real_type>> data{};
    std::vector<label_type> label{};
 
    // parse the given file
    switch (format) {
        case file_format_type::libsvm:
            std::tie(num_data_points_, num_features_, data, label) = detail::io::parse_libsvm_data<real_type, label_type>(reader);
            break;
        case file_format_type::arff:
            std::tie(num_data_points_, num_features_, data, label) = detail::io::parse_arff_data<real_type, label_type>(reader);
            break;
    }
 
    // update shared pointer
    X_ptr_ = std::make_shared<decltype(data)>(std::move(data));
    if (label.empty()) {
        labels_ptr_ = nullptr;
    } else {
        labels_ptr_ = std::make_shared<decltype(label)>(std::move(label));
    }
 
    // create label mapping
    if (this->has_labels()) {
        this->create_mapping();
    }
 
    const std::chrono::time_point end_time = std::chrono::steady_clock::now();
    detail::log(verbosity_level::full | verbosity_level::timing,
                "Read {} data points with {} features in {} using the {} parser from file '{}'.\n",
                detail::tracking_entry{ "data_set_read", "num_data_points", num_data_points_ },
                detail::tracking_entry{ "data_set_read", "num_features", num_features_ },
                detail::tracking_entry{ "data_set_read", "time", std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time) },
                detail::tracking_entry{ "data_set_read", "format", format },
                detail::tracking_entry{ "data_set_read", "filename", filename });
}
 
}  // namespace plssvm
 
#endif  // PLSSVM_DATA_SET_HPP_