csv_parser.h

Go to the documentation of this file.

/**************************************************************************
Copyright: (C) 2021 - 2025 Alexander Shaduri
License: Zlib
***************************************************************************/
 
#ifndef CSV_PARSER_H
#define CSV_PARSER_H
 
#include "csv_cell.h"
#include "csv_error.h"
#include "csv_matrix.h"
#include "csv_util.h"
#include "csv_policies.h"
 
#include <string>
#include <string_view>
#include <array>
 
 
namespace Csv {
 
 
template<typename BehaviorPolicy = LocaleAwareBehaviorPolicy>
class Parser {
    public:
 
        constexpr Parser() = default;
 
 
        template<typename StoreCellFunction>
        constexpr void parse(std::string_view data, StoreCellFunction storeCellFunc) const;
 
 
        template<typename Vector2D>
        constexpr void parseTo2DVector(std::string_view data, Vector2D& values) const;
 
 
        template<std::size_t rows, std::size_t columns, typename Cell = CellStringReference>
        constexpr auto parseTo2DArray(std::string_view data) const;
 
 
        template<typename Vector>
        constexpr MatrixInformation parseToVectorRowMajor(std::string_view data, Vector& values) const;
 
 
        template<typename Vector>
        constexpr MatrixInformation parseToVectorRowMajor(std::string_view data, Vector& values,
                std::optional<std::size_t> rows_hint, std::optional<std::size_t> columns) const;
 
 
        template<typename Vector>
        constexpr MatrixInformation parseToVectorColumnMajor(std::string_view data, Vector& values,
                std::size_t rows, std::optional<std::size_t> columns_hint) const;
 
 
        template<std::size_t rows, std::size_t columns, typename Cell = CellStringReference>
        constexpr auto parseToArray(std::string_view data, MatrixOrder order) const;
 
 
 
    private:
 
        enum class MachineState {
            AtCellStart,
            InLeadingWhiteSpace,
            InsideUnquotedValue,
            InsideQuotedValue,
            AfterQuotedValue,
        };
 
 
        struct ParserState {
            MachineState machine_state = MachineState::AtCellStart;
            std::size_t current_row = 0, current_column = 0;
            std::string_view current_value;
            bool escaped_quotes_encountered = false;
 
 
            constexpr void switchToNextColumn()
            {
                current_value = std::string_view();
                escaped_quotes_encountered = false;
                ++current_column;
            }
 
 
            [[nodiscard]] constexpr std::size_t switchToNextLine(const std::string_view& data, std::size_t current_pos)
            {
                current_value = std::string_view();
                escaped_quotes_encountered = false;
                current_column = 0;
                ++current_row;
                // If it's CR, and the next character is LF, skip LF as well.
                if (auto next_char = peek(data, current_pos); data[current_pos] == '\r' && next_char == '\n') {
                    ++current_pos;
                }
                return current_pos;
            }
 
 
            constexpr void increaseCurrentValueSize(std::size_t by_chars)
            {
                current_value = std::string_view(current_value.data(), current_value.size() + by_chars);
            }
 
 
            constexpr void restartCurrentValue(const std::string_view& data, std::size_t current_pos, std::size_t size)
            {
                current_value = data.substr(current_pos, size);
                escaped_quotes_encountered = false;
            }
 
        };
 
 
        [[nodiscard]] static constexpr std::char_traits<char>::int_type peek(const std::string_view& data,
                std::size_t current_pos, std::size_t advance_chars = 1)
        {
            if ((current_pos + advance_chars) < data.size()) {
                return std::char_traits<char>::to_int_type(data[current_pos + advance_chars]);
            }
            return std::char_traits<char>::eof();
        }
 
 
        [[nodiscard]] static constexpr std::char_traits<char>::int_type readChar(
                const std::string_view& data, std::size_t pos)
        {
            return peek(data, pos, 0);
        }
 
 
        template <typename StoreCellFunction>
        constexpr void store(StoreCellFunction storeCellFunc,
                const ParserState& state, CellTypeHint type_hint) const
        {
            if (type_hint == CellTypeHint::Empty && !BehaviorPolicy::useEmptyCellType()) {
                type_hint = CellTypeHint::StringWithoutEscapedQuotes;
            }
            storeCellFunc(state.current_row, state.current_column, state.current_value, type_hint);
        }
 
};
 
 
 
 
// ---- Implementation
 
 
 
template<typename BehaviorPolicy>
template<typename StoreCellFunction>
constexpr void Parser<BehaviorPolicy>::parse(std::string_view data, StoreCellFunction storeCellFunc) const
{
    ParserState state;
 
    for (std::size_t pos = 0; pos <= data.size(); ++pos) {
        auto current_char = readChar(data, pos);
 
        switch (state.machine_state) {
            // Starting the cell
            case MachineState::AtCellStart:
            {
                switch(current_char) {
                    case ' ':
                    case '\t':
                        // Store the whitespace
                        state.machine_state = MachineState::InLeadingWhiteSpace;
                        state.restartCurrentValue(data, pos, 1);
                        break;
                    case '\"':
                        // Start a quoted cell.
                        // Discard the starting quote as well, but provide a "Quoted" hint when the quoted cell
                        // is stored.
                        state.machine_state = MachineState::InsideQuotedValue;
                        state.restartCurrentValue(data, pos + 1, 0);
                        break;
                    case ',':
                        // Empty cell. Store the value.
                        store(storeCellFunc, state, CellTypeHint::Empty);
                        state.switchToNextColumn();
                        break;
                    case '\r':
                    case '\n':
                        // Empty cell (trailing comma; last value on the line). Store the value.
                        store(storeCellFunc, state, CellTypeHint::Empty);
                        state.machine_state = MachineState::AtCellStart;
                        // Handle CRLF if needed and set the state to the next line, cell start.
                        pos = state.switchToNextLine(data, pos);
                        break;
                    case std::char_traits<char>::eof():
                        // If it's in the first column, it's a trailing newline, nothing else to do.
                        // Otherwise, it's a last empty cell on the line after comma (aka trailing comma).
                        if (state.current_column != 0) {
                            store(storeCellFunc, state, CellTypeHint::Empty);
                        }
                        return;
                    default:
                        // Start an unquoted cell
                        state.machine_state = MachineState::InsideUnquotedValue;
                        state.restartCurrentValue(data, pos, 1);
                        break;
                }
                break;
            }
 
            // Only whitespace encountered in the cell so far
            case MachineState::InLeadingWhiteSpace:
            {
                switch(current_char) {
                    case ' ':
                    case '\t':
                        // More whitespace. Append to existing leading whitespace.
                        state.increaseCurrentValueSize(1);
                        break;
                    case '\"':
                        // Quote encountered. Discard the leading whitespace, start a quoted cell.
                        // Discard the starting quote as well, but provide a "Quoted" hint when the quoted cell
                        // is stored.
                        state.machine_state = MachineState::InsideQuotedValue;
                        state.restartCurrentValue(data, pos + 1, 0);
                        break;
                    case ',':
                        // Whitespace-only string cell. Store the value.
                        store(storeCellFunc, state, CellTypeHint::StringWithoutEscapedQuotes);
                        state.machine_state = MachineState::AtCellStart;
                        state.switchToNextColumn();
                        break;
                    case '\r':
                    case '\n':
                        // Whitespace-only string cell (last value on the line). Store the value.
                        store(storeCellFunc, state, CellTypeHint::StringWithoutEscapedQuotes);
                        state.machine_state = MachineState::AtCellStart;
                        // Handle CRLF if needed and set the state to the next line, cell start.
                        pos = state.switchToNextLine(data, pos);
                        break;
                    case std::char_traits<char>::eof():
                        // Store the value, exit.
                        store(storeCellFunc, state, CellTypeHint::StringWithoutEscapedQuotes);
                        return;
                    default:
                        // Continue an unquoted cell
                        state.machine_state = MachineState::InsideUnquotedValue;
                        state.increaseCurrentValueSize(1);
                        break;
                }
                break;
            }
 
            // We encountered non-whitespace characters in a cell, and it didn't start with a quote.
            case MachineState::InsideUnquotedValue:
            {
                switch(current_char) {
                    case '\"':
                        // Make sure the next character is also a quote, otherwise it's a format error.
                        // We don't accept unescaped double-quotes in unquoted strings because it leads
                        // to ambiguity.
                        if (peek(data, pos) != '\"') {
                            throw ParseError(state.current_row, state.current_column);
                        }
                        // Continue unquoted string, consume the second quote as well.
                        ++pos;
                        state.increaseCurrentValueSize(2);
                        state.escaped_quotes_encountered = true;  // used for hints when storing the value
                        break;
                    case ',':
                        // End of cell. Store the value.
                        store(storeCellFunc, state, state.escaped_quotes_encountered ?
                                CellTypeHint::StringWithEscapedQuotes : CellTypeHint::UnquotedData);
                        state.machine_state = MachineState::AtCellStart;
                        state.switchToNextColumn();
                        break;
                    case '\r':
                    case '\n':
                        // End of line. Store the value.
                        store(storeCellFunc, state, state.escaped_quotes_encountered ?
                                CellTypeHint::StringWithEscapedQuotes : CellTypeHint::UnquotedData);
                        state.machine_state = MachineState::AtCellStart;
                        // Handle CRLF if needed and set the state to the next line, cell start.
                        pos = state.switchToNextLine(data, pos);
                        break;
                    case std::char_traits<char>::eof():
                        // Store the value, exit.
                        store(storeCellFunc, state, state.escaped_quotes_encountered ?
                                CellTypeHint::StringWithEscapedQuotes : CellTypeHint::UnquotedData);
                        return;
                    default:
                        // Continue an unquoted cell
                        state.increaseCurrentValueSize(1);
                        break;
                }
                break;
            }
 
            // The cell started with an optional whitespace and a quote, we're past the first quote.
            case MachineState::InsideQuotedValue:
            {
                switch(current_char) {
                    case '\"':
                        // If the next character is also a quote, it's an escaped quote.
                        if (peek(data, pos) == '\"') {
                            // Continue quoted string, consume the second quote as well.
                            ++pos;
                            state.increaseCurrentValueSize(2);
                            state.escaped_quotes_encountered = true;  // used for hints when storing the value
                        } else {
                            // End of quoted value. Store the value, discard the ending quote.
                            store(storeCellFunc, state, state.escaped_quotes_encountered ?
                                    CellTypeHint::StringWithEscapedQuotes : CellTypeHint::StringWithoutEscapedQuotes);
                            state.machine_state = MachineState::AfterQuotedValue;
                        }
                        break;
                    case std::char_traits<char>::eof():
                        // EOF while inside a quoted cell, throw an error.
                        throw ParseError(state.current_row, state.current_column);
                    default:
                        // Continue an unquoted cell
                        state.increaseCurrentValueSize(1);
                        break;
                }
                break;
            }
 
            // The quoted cell just ended
            case MachineState::AfterQuotedValue:
            {
                switch(current_char) {
                    case ' ':
                    case '\t':
                        // Just whitespace, ignore it.
                        break;
                    case ',':
                        // End of cell. The value has been stored already, switch to the next line.
                        state.machine_state = MachineState::AtCellStart;
                        state.switchToNextColumn();
                        break;
                    case '\r':
                    case '\n':
                        // End of line. The value has been stored already, switch to the next line.
                        state.machine_state = MachineState::AtCellStart;
                        pos = state.switchToNextLine(data, pos);
                        break;
                    case std::char_traits<char>::eof():
                        // Nothing more to do, return.
                        return;
                    default:
                        // Anything else is an error
                        throw ParseError(state.current_row, state.current_column);
                }
                break;
            }
        }
    }
}
 
 
 
template<typename BehaviorPolicy>
template<typename Vector2D>
constexpr void Parser<BehaviorPolicy>::parseTo2DVector(std::string_view data, Vector2D& values) const
{
    Vector2D parsed_values;
    parse(data,
            [&parsed_values](std::size_t row, std::size_t column, std::string_view cell_data, CellTypeHint hint)
        {
            if (parsed_values.size() < (column + 1)) {
                parsed_values.resize(column + 1);
            }
            if (parsed_values[column].size() < (row + 1)) {
                parsed_values[column].resize(row + 1);
            }
            parsed_values[column][row] =
                    BehaviorPolicy::template create<typename Vector2D::value_type::value_type>(cell_data, hint);
        }
    );
    std::swap(values, parsed_values);
}
 
 
 
template<typename BehaviorPolicy>
template<std::size_t rows, std::size_t columns, typename Cell>
constexpr auto Parser<BehaviorPolicy>::parseTo2DArray(std::string_view data) const
{
    std::array<std::array<Cell, rows>, columns> matrix;
 
    parse(data,
        [&matrix](std::size_t row, std::size_t column,
                std::string_view cell_data, Csv::CellTypeHint hint)
                constexpr
        {
            matrix[column][row] = BehaviorPolicy::template create<Cell>(cell_data, hint);
        }
    );
 
    return matrix;
}
 
 
 
template<typename BehaviorPolicy>
template<typename Vector>
constexpr MatrixInformation Parser<BehaviorPolicy>::parseToVectorRowMajor(std::string_view data, Vector& values) const
{
    return parseToVectorRowMajor(data, values, std::nullopt, std::nullopt);
}
 
 
 
template<typename BehaviorPolicy>
template<typename Vector>
constexpr MatrixInformation Parser<BehaviorPolicy>::parseToVectorRowMajor(std::string_view data, Vector& values,
        std::optional<std::size_t> rows_hint, std::optional<std::size_t> columns) const
{
    Vector parsed_values;
    if (rows_hint.has_value() && columns.has_value()) {
        parsed_values.reserve(rows_hint.value() * columns.value());
    }
 
    MatrixInformation info;
    info.setOrder(MatrixOrder::RowMajor);
 
    bool unknown_columns = !columns.has_value();
    if (!unknown_columns) {
        info.setColumns(columns.value());
    }
    std::size_t read_columns = 0;
 
    // The lambda is called in row-major order, so we can exploit that to determine the number of columns.
    parse(data,
            [&](std::size_t row, std::size_t column, std::string_view cell_data, [[maybe_unused]] CellTypeHint hint)
        {
            if (unknown_columns) {
                if (row == 0) {
                    read_columns = column + 1;
                    parsed_values.resize(read_columns);
                } else if (row == 1) {
                    unknown_columns = false;
                    info.setColumns(read_columns);
                    if (rows_hint.has_value()) {
                        parsed_values.reserve(rows_hint.value() * info.getColumns());
                    }
                }
            }
 
            if (!unknown_columns) {
                if (parsed_values.size() < ((row + 1) * info.getColumns())) {
                    parsed_values.resize((row + 1) * info.getColumns());
                }
            }
            parsed_values[info.matrixIndex(row, column)] =
                    BehaviorPolicy::template create<typename Vector::value_type>(cell_data, hint);
        }
    );
    std::swap(values, parsed_values);
 
    if (values.empty()) {
        MatrixInformation empty_info;
        empty_info.setOrder(MatrixOrder::RowMajor);
        return empty_info;
    }
 
    info.setRows(info.getColumns() == 0 ? 0 : (values.size() / info.getColumns()));
 
    return info;
}
 
 
 
template<typename BehaviorPolicy>
template<typename Vector>
constexpr MatrixInformation Parser<BehaviorPolicy>::parseToVectorColumnMajor(std::string_view data, Vector& values,
        std::size_t rows, std::optional<std::size_t> columns_hint) const
{
    Vector parsed_values;
    if (columns_hint.has_value()) {
        parsed_values.reserve(columns_hint.value() * rows);
    }
 
    MatrixInformation info;
    info.setOrder(MatrixOrder::ColumnMajor);
    info.setRows(rows);
 
    parse(data,
            [&](std::size_t row, std::size_t column, std::string_view cell_data, [[maybe_unused]] CellTypeHint hint)
        {
            if (parsed_values.size() < ((column + 1) * rows)) {
                parsed_values.resize((column + 1) * rows);
            }
            parsed_values[info.matrixIndex(row, column)] =
                    BehaviorPolicy::template create<typename Vector::value_type>(cell_data, hint);
        }
    );
    std::swap(values, parsed_values);
 
    if (values.empty()) {
        MatrixInformation empty_info;
        empty_info.setOrder(MatrixOrder::ColumnMajor);
        return empty_info;
    }
 
    info.setColumns(info.getRows() == 0 ? 0 : (values.size() / info.getRows()));
 
    return info;
}
 
 
 
template<typename BehaviorPolicy>
template<std::size_t rows, std::size_t columns, typename Cell>
constexpr auto Parser<BehaviorPolicy>::parseToArray(std::string_view data, MatrixOrder order) const
{
    std::array<Cell, rows * columns> matrix;
 
    parse(data,
        [&matrix, order](std::size_t row, std::size_t column,
                std::string_view cell_data, Csv::CellTypeHint hint)
                constexpr
        {
            matrix[MatrixInformation::matrixIndex(row, column, rows, columns, order)] =
                    BehaviorPolicy::template create<Cell>(cell_data, hint);
        }
    );
 
    return matrix;
}
 
 
 
}  // end ns
 
 
 
#endif