/*
 * Copyright (c) 2025, valkey-search contributors
 * All rights reserved.
 * SPDX-License-Identifier: BSD 3-Clause
 *
 */

#ifndef _VALKEY_SEARCH_INDEXES_TEXT_LEXER_H_
#define _VALKEY_SEARCH_INDEXES_TEXT_LEXER_H_

/*

STATELESS LEXER DESIGN

The Lexer is a stateless processor that takes configuration parameters
and produces tokenized output. Configuration is stored in TextIndexSchema
and Text classes, then passed to lexer methods as parameters.

Tokenization Pipeline:
1. Split text on punctuation characters (configurable)
2. Convert to lowercase
3. Stop word removal (filter out common words)
4. Apply stemming based on language and field settings

*/

#include <bitset>
#include <string>
#include <vector>

#include "absl/container/flat_hash_map.h"
#include "absl/container/flat_hash_set.h"
#include "absl/container/inlined_vector.h"
#include "absl/status/statusor.h"
#include "absl/strings/string_view.h"
#include "src/index_schema.pb.h"

struct sb_stemmer;

namespace valkey_search::indexes::text {

// Inline capacity for per-document stem mapping
constexpr size_t kInProgressStemVariantsInlineCapacity = 4;

// Per-document stem mappings: stemmed_word -> list of original words that stem
// to it
using InProgressStemMap = absl::flat_hash_map<
    std::string,
    absl::InlinedVector<std::string, kInProgressStemVariantsInlineCapacity>>;

struct Lexer {
  Lexer(data_model::Language language, const std::string& punctuation,
        const std::vector<std::string>& stop_words);
  ~Lexer() = default;

  absl::StatusOr<std::vector<std::string>> Tokenize(
      absl::string_view text, bool stemming_enabled, uint32_t min_stem_size,
      InProgressStemMap* stem_mappings = nullptr) const;

  bool IsPunctuation(char c) const {
    return punct_bitmap_[static_cast<unsigned char>(c)];
  }

  bool IsStopWord(absl::string_view lowercase_word) const {
    return stop_words_set_.contains(lowercase_word);
  }
  sb_stemmer* GetStemmer() const;
  void NormalizeLowerCaseInPlace(std::string& str) const;
  void StemWordInPlace(std::string& word, sb_stemmer* stemmer,
                       uint32_t min_stem_size = 0) const;
  void UpdateStemMap(absl::string_view original_word, sb_stemmer* stemmer,
                     uint32_t min_stem_size,
                     InProgressStemMap& stem_mappings) const;

 private:
  data_model::Language language_;
  std::bitset<256> punct_bitmap_;
  absl::flat_hash_set<std::string> stop_words_set_;

  // UTF-8 processing helpers
  bool IsValidUtf8(absl::string_view text) const;
  // Common stemming logic
  std::string_view DoStemming(absl::string_view word, sb_stemmer* stemmer,
                              uint32_t min_stem_size) const;
};

}  // namespace valkey_search::indexes::text

#endif
