/*
 * SessionRParser.hpp
 *
 * Copyright (C) 2022 by Posit Software, PBC
 *
 * Unless you have received this program directly from Posit Software pursuant
 * to the terms of a commercial license agreement with Posit Software, then
 * this program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

// This class implements a simple (recursive-descent) R parser -- it does not
// build a full syntax tree, but instead builds a tree structure that we can
// use to efficiently look up and validate that variables within certain scopes
// are properly declared / defined before usage.

#ifndef SESSION_MODULES_RPARSER_HPP
#define SESSION_MODULES_RPARSER_HPP

// #define RSTUDIO_DEBUG_LABEL "parser"
// #define RSTUDIO_ENABLE_DEBUG_MACROS

#include <vector>
#include <map>
#include <set>
#include <gsl/gsl-lite.hpp>

#include <core/Algorithm.hpp>
#include <core/r_util/RTokenizer.hpp>
#include <core/r_util/RTokenCursor.hpp>
#include <core/collection/Position.hpp>
#include <core/collection/Stack.hpp>

#include <r/RSexp.hpp>
#include <r/RExec.hpp>

#include <boost/shared_ptr.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/bind/bind.hpp>
#include <boost/container/flat_set.hpp>

#include <core/Macros.hpp>

using namespace boost::placeholders;

namespace rstudio {
namespace session {
namespace modules {
namespace rparser {

using namespace core::collection;

class ParseOptions
{
public:
   
   explicit ParseOptions(bool lintRFunctions = false,
                         bool checkArgumentsToRFunctionCalls = false,
                         bool checkUnexpectedAssignmentInFunctionCall = false,
                         bool warnIfNoSuchVariableInScope = false,
                         bool warnIfVariableIsDefinedButNotUsed = false,
                         bool recordStyleLint = false)
      : isExplicit_(false),
        lintRFunctions_(lintRFunctions),
        checkArgumentsToRFunctionCalls_(checkArgumentsToRFunctionCalls),
        checkUnexpectedAssignmentInFunctionCall_(checkUnexpectedAssignmentInFunctionCall),
        warnIfNoSuchVariableInScope_(warnIfNoSuchVariableInScope),
        warnIfVariableIsDefinedButNotUsed_(warnIfVariableIsDefinedButNotUsed),
        recordStyleLint_(recordStyleLint)
   {}
   
   void setIsExplicit(bool isExplicit)
   {
      isExplicit_ = isExplicit;
   }
   
   bool isExplicit() const
   {
      return isExplicit_;
   }
   
   void setRecordStyleLint(bool record)
   {
      recordStyleLint_ = record;
   }
   
   bool recordStyleLint() const
   {
      return recordStyleLint_;
   }
   
   void setLintRFunctions(bool lint)
   {
      lintRFunctions_ = lint;
   }
   
   bool lintRFunctions() const
   {
      return lintRFunctions_;
   }
   
   bool checkArgumentsToRFunctionCalls() const
   {
      return checkArgumentsToRFunctionCalls_;
   }
   
   void setCheckArgumentsToRFunctionCalls(bool checkArgumentsToRFunctionCalls)
   {
      checkArgumentsToRFunctionCalls_ = checkArgumentsToRFunctionCalls;
   }
   
   bool checkUnexpectedAssignmentInFunctionCall() const
   {
      return checkUnexpectedAssignmentInFunctionCall_;
   }
   
   void setCheckUnexpectedAssignmentInFunctionCall(bool checkUnexpectedAssignmentInFunctionCall)
   {
      checkUnexpectedAssignmentInFunctionCall_ = checkUnexpectedAssignmentInFunctionCall;
   }
   
   bool warnIfNoSuchVariableInScope() const
   {
      return warnIfNoSuchVariableInScope_;
   }
   
   void setWarnIfNoSuchVariableInScope(bool value)
   {
      warnIfNoSuchVariableInScope_ = value;
   }
   
   bool warnIfVariableIsDefinedButNotUsed() const
   {
      return warnIfVariableIsDefinedButNotUsed_;
   }
   
   void setWarnIfVariableIsDefinedButNotUsed(bool warnIfVariableIsDefinedButNotUsed)
   {
      warnIfVariableIsDefinedButNotUsed_ = warnIfVariableIsDefinedButNotUsed;
   }
   
   void setSyntaxOnly()
   {
      lintRFunctions_ = false;
      checkArgumentsToRFunctionCalls_ = false;
      checkUnexpectedAssignmentInFunctionCall_ = false;
      warnIfNoSuchVariableInScope_ = false;
      warnIfVariableIsDefinedButNotUsed_ = false;
   }
   
   void setCoreDiagnostics()
   {
      lintRFunctions_ = true;
      checkArgumentsToRFunctionCalls_ = false;
      checkUnexpectedAssignmentInFunctionCall_ = false;
      warnIfNoSuchVariableInScope_ = false;
      warnIfVariableIsDefinedButNotUsed_ = false;
   }
   
   void enableAllDiagnostics()
   {
      lintRFunctions_ = true;
      checkArgumentsToRFunctionCalls_ = true;
      checkUnexpectedAssignmentInFunctionCall_ = true;
      warnIfNoSuchVariableInScope_ = true;
      warnIfVariableIsDefinedButNotUsed_ = true;
   }
   
   std::set<std::string>& globals() { return globals_; }
   const std::set<std::string>& globals() const { return globals_; }

private:
   bool isExplicit_;
   bool lintRFunctions_;
   bool checkArgumentsToRFunctionCalls_;
   bool checkUnexpectedAssignmentInFunctionCall_;
   bool warnIfNoSuchVariableInScope_;
   bool warnIfVariableIsDefinedButNotUsed_;
   bool recordStyleLint_;
   
   std::set<std::string> globals_;
};

struct ParseItem;

class ParseNode;

struct ParseItem
{
   explicit ParseItem(const std::string& symbol,
                      const Position& position,
                      const ParseNode* pNode)
      : symbol(symbol),
        position(position),
        pNode(pNode) {}
   
   bool operator <(const ParseItem& other) const
   {
      return position < other.position;
   }
   
   std::string symbol;
   Position position;
   const ParseNode* pNode;
   
};

namespace linter {

enum LintType
{
   LintTypeStyle,
   LintTypeInfo,
   LintTypeWarning,
   LintTypeError
};

inline std::string lintTypeToString(LintType type)
{
   switch (type)
   {
   case LintTypeStyle: return "style";
   case LintTypeInfo: return "info";
   case LintTypeWarning: return "warning";
   case LintTypeError: return "error";
   }
   
   return std::string();
}

using namespace core;
using namespace core::r_util;

struct LintItem
{

   LintItem(int startRow,
            int startColumn,
            int endRow,
            int endColumn,
            LintType type,
            const std::string message)
      : startRow(startRow),
        startColumn(startColumn),
        endRow(endRow),
        endColumn(endColumn),
        type(type),
        message(message)
   {}
   
   LintItem(const RToken& item,
            LintType type,
            const std::string& message)
      : startRow(gsl::narrow_cast<int>(item.row())),
        startColumn(gsl::narrow_cast<int>(item.column())),
        endRow(gsl::narrow_cast<int>(item.row())),
        endColumn(gsl::narrow_cast<int>(item.column() + item.length())),
        type(type),
        message(message)
   {}
   
   LintItem(const ParseItem& item,
            LintType type,
            const std::string& message)
      : startRow(gsl::narrow_cast<int>(item.position.row)),
        startColumn(gsl::narrow_cast<int>(item.position.column)),
        endRow(gsl::narrow_cast<int>(item.position.row)),
        endColumn(gsl::narrow_cast<int>(item.position.column + item.symbol.length())),
        type(type),
        message(message)
   {}
   
   int startRow;
   int startColumn;
   int endRow;
   int endColumn;
   LintType type;
   std::string message;
};

inline bool operator==(const LintItem& lhs, const LintItem& rhs)
{
   return
         lhs.startRow    == rhs.startRow &&
         lhs.startColumn == rhs.startColumn &&
         lhs.endRow      == rhs.endRow &&
         lhs.endColumn   == rhs.endColumn &&
         lhs.type        == rhs.type &&
         lhs.message     == rhs.message;
}

class LintItems
{
public:
   
   LintItems()
      : errorCount_(0)
   {}
   
   explicit LintItems(const ParseOptions& parseOptions)
      : errorCount_(0),
        parseOptions_(parseOptions)
   {}
   
   // default ctors: copyable members
   
   void add(std::size_t startRow,
            std::size_t startColumn,
            std::size_t endRow,
            std::size_t endColumn,
            LintType type,
            const std::string& message)
   {
      LintItem item(gsl::narrow_cast<int>(startRow),
                    gsl::narrow_cast<int>(startColumn),
                    gsl::narrow_cast<int>(endRow),
                    gsl::narrow_cast<int>(endColumn),
                    type,
                    message);
      add(item);
   }
   
   void unexpectedToken(const RToken& token,
                        const std::string& expected = std::string())
   {
      const std::string& content = token.contentAsUtf8();
      std::string message = "unexpected token '" + content + "'";
      if (!expected.empty())
         message = message + ", expected " + expected;
      
      addLintItem(token,
                  LintTypeError,
                  message);
   }
   
   void unexpectedToken(const RToken& token,
                        const std::wstring& expected)
   {
      unexpectedToken(token, string_utils::wideToUtf8(expected));
   }
   
   void unexpectedClosingBracket(const RToken& token)
   {
      addLintItem(token,
                  LintTypeError,
                  "unexpected closing bracket '" + token.contentAsUtf8() + "'");
   }
   
   void expectedMatchForOpenBracket(const RToken& openBracketToken)
   {
      addLintItem(openBracketToken,
                  LintTypeError,
                  "unmatched opening bracket '" + openBracketToken.contentAsUtf8() + "'");
   }
   
   void unexpectedEndOfDocument(const RToken& token)
   {
      addLintItem(token,
                  LintTypeError,
                  "unexpected end of document");
   }
   
   void noSymbolNamed(const core::r_util::token_cursor::RTokenCursor& cursor,
                      const std::string& candidate = std::string())
   {
      ParseItem item(
               cursor.contentAsUtf8(),
               cursor.currentPosition(),
               nullptr);
      noSymbolNamed(item, candidate);
   }
   
   void noSymbolNamed(const ParseItem& item,
                      const std::string& candidate = std::string())
   {
      std::string message("no symbol named '" + item.symbol + "' in scope");
      if (!candidate.empty())
         message += "; did you mean '" + candidate + "'?";
      
      addLintItem(item,
                  LintTypeWarning,
                  message);
   }
   
   void symbolDefinedAfterUsage(const ParseItem& item,
                                const Position& position)
   {
      LintItem lint(gsl::narrow_cast<int>(position.row),
                    gsl::narrow_cast<int>(position.column),
                    gsl::narrow_cast<int>(position.row),
                    gsl::narrow_cast<int>(position.column + item.symbol.length()),
                    LintTypeInfo,
                    "'" + item.symbol + "' is defined after it is used");
      
      lintItems_.push_back(lint);
   }
   
   void packageNotLoaded(const RToken& packageToken,
                         const RToken& symbolToken)
   {
      std::string message = fmt::format(
               "package '{}' is not available; cannot provide argument diagnostics",
               packageToken.contentAsUtf8());
      
      add(
               packageToken.row(),
               packageToken.column(),
               symbolToken.row(),
               symbolToken.column() + symbolToken.length(),
               LintTypeInfo,
               message);
   }
   
   
   void expectedWhitespace(const RToken& token)
   {
      addLintItem(token,
                  LintTypeStyle,
                  "expected whitespace");
   }
   
   void unnecessaryWhitespace(const RToken& token)
   {
      addLintItem(token,
                  LintTypeStyle,
                  "unnecessary whitespace");
   }
   
   void unexpectedWhitespaceAroundOperator(const RToken& token)
   {
      addLintItem(token,
                  LintTypeStyle,
                  "unexpected whitespace around extraction operator");
   }
   
   void expectedWhitespaceAroundOperator(const RToken& token)
   {
      addLintItem(token,
                  LintTypeStyle,
                  "expected whitespace around '" + token.contentAsUtf8() + "' operator");
   }
   
   void inconsistentWhitespaceAroundOperator(const RToken& token)
   {
      addLintItem(token,
                  LintTypeStyle,
                  "inconsistent whitespace around '" + token.contentAsUtf8() + "' operator");
   }

   void tooManyErrors(const Position& position)
   {
      LintItem lint(gsl::narrow_cast<int>(position.row),
                    gsl::narrow_cast<int>(position.column),
                    gsl::narrow_cast<int>(position.row),
                    gsl::narrow_cast<int>(position.column),
                    LintTypeError,
                    "too many errors emitted; stopping now");
      lintItems_.push_back(lint);
      ++errorCount_;
   }
   
   void expectedCommaFollowingToken(const RToken& rToken)
   {
      addLintItem(rToken,
                  LintTypeError,
                  "expected ',' after expression");
   }
   
   void symbolDefinedButNotUsed(const std::string& symbol,
                                const Position& position)
   {
      add(position.row,
          position.column,
          position.row,
          position.column + symbol.length(),
          LintTypeWarning,
          "'" + symbol + "' is defined but not used");
   }
   
   void missingArgumentToFunctionCall(const RToken& rToken)
   {
      addLintItem(rToken,
                  LintTypeWarning,
                  "missing argument to function call");
   }
   
   void incorrectEqualityComparison(
         const std::string& what,
         const std::string& suggestion,
         const Position& start,
         const Position& end)
   {
      std::string message =
            "use '" + suggestion + "' to check whether expression evaluates to " + what;
      
      add(start.row,
          start.column,
          end.row,
          end.column,
          LintTypeWarning,
          message);
   }
   
   void noExistingDefinitionForParentAssignment(const RToken& rToken)
   {
      addLintItem(rToken,
                  LintTypeStyle,
                  "no definition for '" + token_utils::getSymbolName(rToken) + "' in scope");
   }
   
   void unexpectedAssignmentInArgumentList(const RToken& rToken)
   {
      addLintItem(
               rToken,
               LintTypeWarning,
               "unexpected assignment in argument list; did you mean to use '='?");
   }

   void unexpectedAssignmentInConditional(const RToken& rToken,
                                          const std::string& context)
   {
      std::string message = fmt::format(
          "unexpected '=' in '{}' conditional; did you mean to use '=='?",
          context);

      addLintItem(rToken, LintTypeError, message);
   }

   
   void addLintItem(const RToken& rToken,
                    LintType type,
                    const std::string& message)
   {
      add(LintItem(rToken, type, message));
   }
   
   void addLintItem(const ParseItem& item,
                    LintType type,
                    const std::string& message)
   {
      add(LintItem(item, type, message));
   }
   
   const std::vector<LintItem>& get() const
   {
      return lintItems_;
   }
   
   void push_back(const LintItem& item)
   {
      lintItems_.push_back(item);
   }
   
   void push_back(const LintItems& items)
   {
      for (std::size_t i = 0, n = items.size(); i < n; ++i)
         lintItems_.push_back(items.get()[i]);
   }
   
   typedef std::vector<LintItem>::iterator iterator;
   typedef std::vector<LintItem>::const_iterator const_iterator;
   
   const_iterator begin() const { return lintItems_.begin(); }
   const_iterator end() const { return lintItems_.end(); }
   std::size_t size() const { return lintItems_.size(); }
   std::size_t empty() const { return lintItems_.empty(); }
   
   std::size_t errorCount() const { return errorCount_; }
   bool hasErrors() const { return errorCount_ > 0; }
   void dump();
   
private:
   
   void add(const LintItem& item)
   {
      if (!parseOptions_.recordStyleLint() && item.type == LintTypeStyle)
         return;
      
      // Avoid duplicate lint items
      for (auto&& lintItem : lintItems_)
         if (lintItem == item)
            return;
      
      lintItems_.push_back(item);
      errorCount_ += item.type == LintTypeError;
   }

   std::vector<LintItem> lintItems_;
   std::size_t errorCount_;
   ParseOptions parseOptions_;
};

}

using namespace linter;

std::string& complement(const std::string& bracket);

class ParseNode : public boost::noncopyable
{
   
public:
   
   typedef std::vector< boost::shared_ptr<ParseNode> > Children;
   typedef std::vector<Position> Positions;
   typedef std::map<std::string, Positions> SymbolPositions;
   
   typedef std::string PackageName;
   typedef std::set<std::string> Symbols;
   typedef std::map<PackageName, Symbols> PackageSymbols;
   
private:
   
   // private constructor: root node should be created through
   // 'createRootNode()', with future nodes appended to that node;
   // child nodes with 'createChildNode()'.
   //
   // 'start' refers to the location of the opening brace opening
   // the node (or, [0, 0] for the root node)
   ParseNode(ParseNode* pParent,
             const std::string& name,
             Position position)
      : pParent_(pParent), name_(name), position_(position) {}
   
public:
   
   static boost::shared_ptr<ParseNode> createRootNode()
   {
      return boost::shared_ptr<ParseNode>(
               new ParseNode(nullptr, "<root>", Position(0, 0)));
   }
   
   static boost::shared_ptr<ParseNode> createNode(
         const std::string& name)
   {
      return boost::shared_ptr<ParseNode>(
               new ParseNode(nullptr, name, Position(0, 0)));
   }
   
   bool isRootNode() const
   {
      return pParent_ == nullptr;
   }
   
   const SymbolPositions& getDefinedSymbols() const
   {
      return definedSymbols_;
   }
   
   const SymbolPositions& getReferencedSymbols() const
   {
      return referencedSymbols_;
   }
   
   void addDefinedSymbol(int row,
                         int column,
                         const std::string& name)
   {
      definedSymbols_[name].push_back(Position(row, column));
   }
   
   void addDefinedSymbol(const std::string& name,
                         const Position& position)
   {
      definedSymbols_[name].push_back(position);
   }
   
   void addDefinedSymbol(const RToken& rToken,
                         const Position& position)
   {
      std::string name = token_utils::getSymbolName(rToken);
      definedSymbols_[name].push_back(position);
   }
   
   void addDefinedSymbol(const RToken& rToken)
   {
      addDefinedSymbol(rToken, rToken.position());
   }

   void addReferencedSymbol(int row,
                            int column,
                            const std::string& name)
   {
      referencedSymbols_[name].push_back(Position(row, column));
   }

   void addReferencedSymbol(const RToken& rToken)
   {
      std::string name = token_utils::getSymbolName(rToken);
      referencedSymbols_[name].push_back(rToken.position());
   }
   
   void addNseReferencedSymbol(const RToken& rToken)
   {
      std::string name = token_utils::getSymbolName(rToken);
      nseReferencedSymbols_[name].push_back(rToken.position());
   }
   
   void addInternalPackageSymbol(const std::string& package,
                                 const std::string& name)
   {
      internalSymbols_[package].insert(name);
   }
   
   void addExportedPackageSymbol(const std::string& package,
                                 const std::string& name)
   {
      exportedSymbols_[package].insert(name);
   }
   
   const std::string& getName() const
   {
      return name_;
   }
   
   // Tree-related operations
   
   ParseNode* const getParent() const
   {
      return pParent_;
   }
   
   ParseNode* const getRoot() const
   {
      ParseNode* pNode = const_cast<ParseNode*>(this);
      while (pNode->pParent_ != nullptr)
         pNode = pNode->pParent_;
      
      return pNode;
   }
   
   const Children& getChildren() const
   {
      return children_;
   }
   
   void addChild(boost::shared_ptr<ParseNode>& pChild,
                 const Position& position)
   {
      pChild->pParent_ = this;
      pChild->position_ = position;
      children_.push_back(pChild);
   }
   
   void findAllUnresolvedSymbols(std::vector<ParseItem>* pItems) const
   {
      // Get the unresolved symbols at this node
      std::set<ParseItem> unresolved = getUnresolvedSymbols();
      pItems->insert(pItems->end(), unresolved.begin(), unresolved.end());
      
      // Apply this over all children on the node
      Children children = getChildren();
      
      for (Children::iterator it = children.begin();
           it != children.end();
           ++it)
      {
         (**it).findAllUnresolvedSymbols(pItems);
      }
   }
   
   bool symbolExistsInSearchPath(const std::string& symbol,
                                 const std::set<std::string>& searchPathObjects) const
   {
      return searchPathObjects.count(symbol) != 0;
   }
   
private:
   
   static bool findFunctionImpl(const ParseNode* pNode,
                                const std::string& name,
                                const Position& position,
                                const ParseNode** ppFoundNode)
   {
      if (!pNode) return false;
      
      if (pNode->name_ == name && pNode->position_ <= position)
      {
         if (ppFoundNode) *ppFoundNode = pNode;
         return true;
      }
      
      // We search the children in reverse order, to ensure we find
      // the first function with a particular name (in case multiple
      // functions with the same name exist)
      std::size_t n = pNode->children_.size();
      for (std::size_t i = 0; i < n; i++)
      {
         std::size_t index = n - i - 1;
         const boost::shared_ptr<ParseNode>& pChild = pNode->children_[index];
         if (pChild->name_ == name && pChild->position_ <= position)
         {
            if (ppFoundNode) *ppFoundNode = pChild.get();
            return true;
         }
      }
      
      return findFunctionImpl(
               pNode->pParent_,
               name,
               position,
               ppFoundNode);
   }
   
public:
   
   bool findFunction(const std::string& name,
                     const Position& position,
                     const ParseNode** ppFoundNode = nullptr) const
   {
      return findFunctionImpl(
               this,
               name,
               position,
               ppFoundNode);
   }
   
private:
   
   static bool findVariableImpl(const ParseNode* pNode,
                                const std::string& name,
                                const Position& position,
                                bool checkPosition,
                                Position* pFoundPosition)
   {
      if (!pNode) return false;
      
      // First, perform a position-wide search in the current node.
      Positions* pPositions = nullptr;
      core::algorithm::get(pNode->getDefinedSymbols(), name, &pPositions);
      
      if (!pPositions)
         return findVariableImpl(pNode->getParent(),
                                 name,
                                 position,
                                 false,
                                 pFoundPosition);
      
      std::size_t n = pPositions->size();
      if (checkPosition)
      {
         for (std::size_t i = n; i != 0; --i)
         {
            Position& definitionPos = (*pPositions)[i - 1];
            if (definitionPos < position)
            {
               if (pFoundPosition) *pFoundPosition = definitionPos;
               return true;
            }
         }
      }
      else if (n > 0)
      {
         if (pFoundPosition) *pFoundPosition = (*pPositions)[n - 1];
         return true;
      }
      
      return findVariableImpl(
               pNode->getParent(),
               name,
               position,
               false,
               pFoundPosition);
   }
   
public:
   
   bool findVariable(const std::string& name,
                     const Position& position,
                     Position* pFoundPosition = nullptr) const
   {
      return findVariableImpl(this,
                              name,
                              position,
                              true,
                              pFoundPosition);
   }
   
   bool symbolHasDefinitionInTree(const std::string& symbol,
                                  const Position& position) const
   {
      if (definedSymbols_.count(symbol) != 0)
      {
         DEBUG("- Checking for symbol '" << symbol << "' in node");
         Positions& positions = const_cast<ParseNode*>(this)->definedSymbols_[symbol];
         for (Positions::reverse_iterator it = positions.rbegin();
              it != positions.rend();
              ++it)
         {
            // NOTE: '<=' because 'defined' variables are both referenced
            // and defined (at least for now?)
            if (*it <= position)
               return true;
         }
      }
      
      if (pParent_)
         return pParent_->symbolHasDefinitionInTree(symbol, position);
      
      return false;
   }
   
   bool symbolHasDefinitionInRange(const std::string& symbol,
                                   const Position& position) const
   {
      const SymbolRanges& ranges = getRoot()->symbolRanges_;
      for (SymbolRanges::const_iterator it = ranges.begin();
           it != ranges.end();
           ++it)
      {
         if (it->first.contains(position) &&
             it->second.count(symbol))
         {
            return true;
         }
      }
      return false;
   }
   
   std::set<ParseItem> getUnresolvedSymbols() const
   {
      std::set<ParseItem> unresolvedSymbols;
      
      for (SymbolPositions::const_iterator it = referencedSymbols_.begin();
           it != referencedSymbols_.end();
           ++it)
      {
         const std::string& symbol = it->first;
         for (const Position& position : it->second)
         {
            DEBUG("-- Checking for symbol '" << symbol << "' " << position.toString());
            if (!symbolHasDefinitionInTree(symbol, position) &&
                !symbolHasDefinitionInRange(symbol, position))
            {
               DEBUG("--- No definition for symbol '" << symbol << "'");
               unresolvedSymbols.insert(
                        ParseItem(symbol, position, this));
            }
            else
            {
               DEBUG("--- Found definition for symbol '" << symbol << "'");
            }
         }
      }

      return unresolvedSymbols;
   }
   
   bool isSymbolUsedInChildNode(const std::string& symbolName)
   {
      for (const boost::shared_ptr<ParseNode>& pChild : children_)
      {
         if (pChild->getReferencedSymbols().count(symbolName))
            return true;
         
         if (pChild->isSymbolUsedInChildNode(symbolName))
            return true;
      }
      return false;
   }
   
   bool isSymbolDefinedButNotUsed(const std::string& symbolName,
                                  bool checkChildNodes,
                                  bool checkNseCalls)
   {
      if (!definedSymbols_.count(symbolName))
         return false;

      if (checkChildNodes && isSymbolUsedInChildNode(symbolName))
         return false;

      // NOTE: We record a definition at the same position as
      // each reference, so a symbol is effectively defined but
      // not used if every reference position coincides with a
      // definition position (i.e. the only "uses" are the
      // definitions themselves).
      const Positions& defnPositions = definedSymbols_[symbolName];
      boost::container::flat_set<Position> defnSet(
            defnPositions.begin(), defnPositions.end());

      SymbolPositions::const_iterator refsIt =
            referencedSymbols_.find(symbolName);
      if (refsIt != referencedSymbols_.end())
      {
         for (const Position& pos : refsIt->second)
         {
            if (!defnSet.count(pos))
               return false;
         }
      }

      if (checkNseCalls)
      {
         SymbolPositions::const_iterator nseIt =
               nseReferencedSymbols_.find(symbolName);
         if (nseIt != nseReferencedSymbols_.end())
         {
            for (const Position& pos : nseIt->second)
            {
               if (!defnSet.count(pos))
                  return false;
            }
         }
      }

      return true;
   }
   
   std::string suggestSimilarSymbolFor(const ParseItem& item) const
   {
      std::string nameLower = boost::algorithm::to_lower_copy(item.symbol);
      for (SymbolPositions::const_iterator it = definedSymbols_.begin();
           it != definedSymbols_.end();
           ++it)
      {
         DEBUG("-- '" << it->first << "'");
         if (nameLower == boost::algorithm::to_lower_copy(it->first))
         {
            for (const Position& position : it->second)
            {
               if (position < item.position)
               {
                  return it->first;
               }
            }
         }
      }

      if (getParent())
         return getParent()->suggestSimilarSymbolFor(item);

      return std::string();
   }
   
   template <typename Container>
   void makeSymbolsAvailableInRange(
         const Container& symbols,
         const Position& begin,
         const Position& end)
   {
      core::algorithm::insert(
            getRoot()->symbolRanges_[Range(begin, end)],
            symbols.begin(),
            symbols.end());
   }
   
public:
   
   const std::string& name() const { return name_; }
   const Position& position() const { return position_; }
   
private:
   
   // tree reference -- children and parent
   ParseNode* pParent_;
   
   Children children_;
   
   // member variables
   std::string name_; // name of scope (usually function name)
   Position position_; // location of opening '{' for scope
   
   // variables defined in this scope, e.g. with 'x <- ...'.
   // map variable names to locations (row, column)
   SymbolPositions definedSymbols_;
   
   // variables referenced in this scope
   // map variable names to locations(row, column)
   SymbolPositions referencedSymbols_;
   
   // variables referenced within NSE-performing functions
   // currently used to resolve 'is variable used?' lint
   SymbolPositions nseReferencedSymbols_;
   
   // for e.g. <pkg>::<foo>, we keep a cache of those symbols
   // in case we want to verify that e.g. <foo> really is an
   // exported function from <pkg>. TODO: keep position
   
   PackageSymbols internalSymbols_; // <pkg>::<foo>
   PackageSymbols exportedSymbols_; // <pgk>:::<bar>
   
   typedef std::map<Range, std::set<std::string> > SymbolRanges;
   SymbolRanges symbolRanges_;
};

class ParseStatus
{
   
public:
   
   explicit ParseStatus(const FilePath& filePath, const ParseOptions& parseOptions)
      : pRoot_(ParseNode::createRootNode()),
        pNode_(pRoot_.get()),
        lint_(parseOptions),
        parseOptions_(parseOptions),
        filePath_(filePath)
   {
      parseStateStack_.push(ParseStateTopLevel);
      functionNames_.push(std::wstring(L""));
   }
   
   ParseNode* node() { return pNode_; }
   LintItems& lint() { return lint_; }
   boost::shared_ptr<ParseNode> root() { return pRoot_; }
   
   void addChildAndSetAsCurrentNode(boost::shared_ptr<ParseNode> pChild,
                                    const Position& position)
   {
      node()->addChild(pChild, position);
      pNode_ = pChild.get();
   }
   
   void setParentAsCurrent()
   {
      pNode_ = pNode_->getParent();
   }
   
   bool hasLint() const
   {
      return !lint_.get().empty();
   }
   
   enum ParseState {
      
      // top level
      ParseStateTopLevel,
      
      // within () but not an argument list
      ParseStateWithinParens,
      
      // within {}
      ParseStateWithinBraces,
      
      // if '(' <cond> ')' (<statement> | <expr>)
      ParseStateIfCondition,
      ParseStateIfStatement,
      ParseStateIfExpression,
      
      // while '(' <cond> ')' (<statement> | <expr>)
      ParseStateWhileCondition,
      ParseStateWhileStatement,
      ParseStateWhileExpression,
      
      // for '(' <id> in <cond> ')' (<statement> | <expr>)
      ParseStateForCondition,
      ParseStateForStatement,
      ParseStateForExpression,
      
      // repeat (<statement> | <expr>)
      ParseStateRepeatStatement,
      ParseStateRepeatExpression,
      
      // function(<arglist>) <expr>
      ParseStateFunctionArgumentList,
      ParseStateFunctionStatement,
      ParseStateFunctionExpression,
      
      // <id> ( <arglist> )
      ParseStateParenArgumentList,
      ParseStateSingleBracketArgumentList,
      ParseStateDoubleBracketArgumentList,
      
   };
   
   Stack<ParseState>& parseStateStack()
   {
      return parseStateStack_;
   }
   
   void pushFunctionCallState(ParseState state,
                              const std::wstring& functionName,
                              bool isNseFunction)
   {
      DEBUG("Pushing state: " << stateAsString(state));
      parseStateStack_.push(state);
      functionNames_.push(functionName);
      nseCallStack_.push(isNseFunction);
   }
   
   const std::wstring& currentFunctionName() const
   {
      return functionNames_.peek();
   }
   
   void enterFunctionScope(const std::string& name,
                           const Position& position)
   {
      addChildAndSetAsCurrentNode(
               ParseNode::createNode(name),
               position);
      
      DEBUG("Entering function scope: '" << name << "' at " << position);
      pushState(ParseStateFunctionArgumentList);
   }
   
   void exitFunctionScope()
   {
      popState();
   }
   
   void pushState(ParseState state)
   {
      DEBUG("Pushing state: " << stateAsString(state));
      parseStateStack_.push(state);
   }
   
   void popFunctionName()
   {
      if (functionNames_.peek().length())
         functionNames_.pop();
   }
   
   void popState()
   {
      switch (currentState())
      {
      case ParseStateFunctionExpression:
      case ParseStateFunctionStatement:
         setParentAsCurrent();
         break;
      case ParseStateParenArgumentList:
      case ParseStateSingleBracketArgumentList:
      case ParseStateDoubleBracketArgumentList:
         popFunctionName();
         popNseCall();
         break;
         
      // suppress compiler warnings
      default:
         break;
      }

      if (currentState() != ParseStateTopLevel)
      {
         DEBUG("Popping state: " << currentStateAsString());
         parseStateStack_.pop();
      }
      else
      {
         DEBUG("Already at top level; no state to pop");
      }
   }
   
   ParseState peekState(std::size_t depth = 0) const
   {
      std::size_t n = parseStateStack_.size();
      if (depth >= n)
         return ParseStateTopLevel;
      return parseStateStack_[n - depth - 1];
   }
   
   ParseState currentState() const { return parseStateStack_.peek(); }
   
   std::string stateAsString(ParseState state) const
   {
#define CASE(__X__) \
   case __X__: return #__X__
      
      switch (state)
      {
      CASE(ParseStateTopLevel);
      CASE(ParseStateWithinParens);
      CASE(ParseStateWithinBraces);
      CASE(ParseStateIfCondition);
      CASE(ParseStateIfStatement);
      CASE(ParseStateIfExpression);
      CASE(ParseStateWhileCondition);
      CASE(ParseStateWhileStatement);
      CASE(ParseStateWhileExpression);
      CASE(ParseStateForCondition);
      CASE(ParseStateForStatement);
      CASE(ParseStateForExpression);
      CASE(ParseStateRepeatStatement);
      CASE(ParseStateRepeatExpression);
      CASE(ParseStateFunctionArgumentList);
      CASE(ParseStateFunctionStatement);
      CASE(ParseStateFunctionExpression);
      CASE(ParseStateParenArgumentList);
      CASE(ParseStateSingleBracketArgumentList);
      CASE(ParseStateDoubleBracketArgumentList);
      default: return "<unknown>";
      }
#undef CASE
   }
   
   std::string currentStateAsString() const { return stateAsString(currentState()); }
   
   bool isAtTopLevel() const
   {
      return currentState() == ParseStateTopLevel;
   }
   
   bool isInArgumentList() const
   {
      switch (currentState())
      {
      case ParseStateParenArgumentList:
      case ParseStateSingleBracketArgumentList:
      case ParseStateDoubleBracketArgumentList:
      case ParseStateFunctionArgumentList:
         return true;
      default:
         return false;
      }
   }
   
   bool isInIfStatementOrExpression() const
   {
      switch (currentState())
      {
      case ParseStateIfStatement:
      case ParseStateIfExpression:
         return true;
      default:
         return false;
      }
   }
   
   bool isInControlFlowStatement() const
   {
      switch (currentState())
      {
      case ParseStateForStatement:
      case ParseStateWhileStatement:
      case ParseStateIfStatement:
      case ParseStateRepeatStatement:
      case ParseStateFunctionStatement:
         return true;
      default:
         return false;
      }
   }
   
   bool isInControlFlowExpression() const
   {
      switch (currentState())
      {
      case ParseStateForExpression:
      case ParseStateWhileExpression:
      case ParseStateIfExpression:
      case ParseStateRepeatExpression:
      case ParseStateFunctionExpression:
         return true;
      default:
         return false;
      }
   }
   
   bool isInControlFlowCondition() const
   {
      switch (currentState())
      {
      case ParseStateForCondition:
      case ParseStateWhileCondition:
      case ParseStateIfCondition:
         return true;
      default:
         return false;
      }
   }
   
   bool isInParentheticalScope() const
   {
      return isInParentheticalScope(currentState());
   }
   
   bool isInParentheticalScope(ParseState state) const
   {
      switch (state)
      {
      case ParseStateForCondition:
      case ParseStateWhileCondition:
      case ParseStateIfCondition:
         
      case ParseStateParenArgumentList:
      case ParseStateSingleBracketArgumentList:
      case ParseStateDoubleBracketArgumentList:
      case ParseStateFunctionArgumentList:
         
      case ParseStateWithinParens:
         return true;
      default:
         return false;
      }
   }
   
   bool isWithinParenFunctionCall() const
   {
      return currentState() == ParseStateParenArgumentList;
   }
   
   const Stack<std::wstring>& functionNames() const
   {
      return functionNames_;
   }
   
   const ParseOptions& parseOptions() const
   {
      return parseOptions_;
   }
   
   void pushNseCall(bool value)
   {
      nseCallStack_.push(value);
   }
   
   void popNseCall()
   {
      nseCallStack_.pop();
   }
   
   bool isWithinNseCall() const
   {
      return !nseCallStack_.empty() &&
              nseCallStack_.peek();
   }
   
   void pushBracket(const RToken& token)
   {
      DEBUG("===== Pushing bracket: " << token);
      bracketStack_.push(token);
   }
   
   void popBracket(const RToken& rhs)
   {
      DEBUG("===== Popping bracket; cursor: " << rhs);
      
      using namespace token_utils;
      if (bracketStack_.empty())
      {
         DEBUG("===== Attempted to pop bracket from empty stack: " << rhs);
         lint_.unexpectedClosingBracket(rhs);
         return;
      }
      
      const RToken& lhs = bracketStack_.peek();
      if (typeComplement(lhs.type()) != rhs.type())
      {
         DEBUG("===== Incorrect closing bracket [" << lhs << " : " << rhs << "]");
         lint_.unexpectedClosingBracket(rhs);
         lint_.expectedMatchForOpenBracket(lhs);
      }
      
      bracketStack_.pop();
   }
   
   // to be called after parsing finished
   void addLintIfBracketStackNotEmpty()
   {
      while (!bracketStack_.empty())
      {
         const RToken& lhs = bracketStack_.peek();
         lint_.expectedMatchForOpenBracket(lhs);
         bracketStack_.pop();
      }
   }
   
   template <typename Container>
   void makeSymbolsAvailableInRange(
         const Container& symbols,
         const Position& begin,
         const Position& end)
   {
      pNode_->makeSymbolsAvailableInRange(
               symbols,
               begin,
               end);
   }
   
   const FilePath& filePath() const
   {
      return filePath_;
   }

private:
   boost::shared_ptr<ParseNode> pRoot_;
   ParseNode* pNode_;
   LintItems lint_;
   ParseOptions parseOptions_;
   Stack<ParseState> parseStateStack_;
   Stack<std::wstring> functionNames_;
   
   // NOTE: Really prefer 'bool' here but that invokes the
   // std::vector<bool> data member which we want to avoid
   Stack<char> nseCallStack_;
   
   // NOTE: Using 'RToken' here implies that the parse tree
   // is only valid as long as the underlying tokens are valid;
   // this should be the case but should attempt to enforce
   // this.
   Stack<RToken> bracketStack_;
   
   FilePath filePath_;
};

class ParseResults {
   
public:
   
   ParseResults()
      : parseTree_(ParseNode::createRootNode())
   {}
   
   ParseResults(boost::shared_ptr<ParseNode> parseTree,
                const LintItems& lint)
      : parseTree_(parseTree),
        lint_(lint)
   {}
   ParseResults(boost::shared_ptr<ParseNode> parseTree,
                const LintItems& lint,
                const std::set<std::string>& globals)
      : parseTree_(parseTree),
        lint_(lint),
        globals_(globals)
   {}
   
   // copy ctor: copyable members
   
   ParseNode* parseTree() const
   {
      return parseTree_.get();
   }
   
   const LintItems& lint() const { return lint_; }
   LintItems& lint() { return lint_; }
   
   const std::set<std::string>& globals() const { return globals_; }
   std::set<std::string>& globals() { return globals_; }
   
private:
   
   boost::shared_ptr<ParseNode> parseTree_;
   LintItems lint_;
   std::set<std::string> globals_;
};

// Primary method ----
ParseResults parse(const core::FilePath& filePath,
                   const std::wstring& rCode,
                   const ParseOptions& parseOptions = ParseOptions());

// Useful aliases ----
ParseResults parse(const core::FilePath& filePath,
                   const ParseOptions& parseOptions = ParseOptions());

ParseResults parse(const std::string& rCode,
                   const ParseOptions& parseOptions = ParseOptions());

ParseResults parse(const std::wstring& rCode,
                   const ParseOptions& parseOptions = ParseOptions());

} // namespace rparser
} // namespace modules
} // namespace session
} // namespace rstudio


#endif // SESSION_MODULES_RPARSER_HPP
