Lex.hpp

Go to the documentation of this file.

#pragma once
 
#include "../Instancing/ClientMarker.hpp"
#include "../Basics/LogTrace.hpp"
 
#include <iostream>
#include <vector>
#include <string>
#include <unordered_set>
 
namespace iv
{
 
class Instance;
 
class Lex
{
public:
ClientMarker cm;
 
                    Lex( Instance * inst );
    Instance *      instance() const;
    
    //----------------- configuration --------------------------
    void            DefineKeyword( const char * keyword );
    void            DefineOperator( const char * op );
    
    void            numbers_enabled( bool enabled );
    
    //----------------- tokens ---------------------------------
    enum Token
    {
        Keyword,        // keyword must have whitespace characters on both sides to be considered keyword ; can not begin with a number ; can not contain whitespaces and operators
        Operator,       // operator that does not have to be separated with whitespace characters from variables ; can not begin with a number ; can not contain whitespaces ; all prefixes of specified operators are also operators
        Newline,        // newline is skipped if not explicitly requested
        Eof,            // end of input
        
        String,         // anything that has quotes or apostrophes on both sides; quoted strings support escaping with backslash: \" -> ", \n -> newline, \r -> cr, \t -> tab, \\ -> backslash; Other escaped characters yield error
        Name,           // anything not containing whitespace characters that is not anything else ; can not begin with a number ; can not contain whitespaces and operators
        Integer,        // sequence of numbers (0-9); Whenever Double is requested in Lex's interface, Integer can substitute for it. Negative values work under the same conditions as for Double numbers.
        Double          // Examples: 1.3, 1e3, 1e-3, 1.e-5, 5.5e-5, if operator '-' is defined, then this can only be positive (and it is up on user code to parse the minus and negate the number).
    };
    
    //----------------- parsing --------------------------------
    void            Init( std::istream & in, bool single_line = false );
    void            Close();
    
    void            DisableNewlineSkipping();
    void            EnableNewlineSkipping();
    
    bool            IsNext( Token token );
    bool            IsNextKeyword( const char * keyword );
    bool            IsNextOperator( const char * op );
    
    Token           GetNextToken();
    std::string     GetNextTokenValue();
    
    void            Accept( Token token );
    void            AcceptKeyword( const char * keyword );
    void            AcceptOperator( const char * op );
    void            AcceptFail();   
    
    std::string     AcceptString();
    std::string     AcceptName();
    int             AcceptInteger();
    double          AcceptDouble();
    
    void            Recover( Token token );
    void            RecoverKeyword( const char * keyword );
    void            RecoverOperator( const char * op );
    
    int             GetLine() const;
    int             GetColumn() const;
    
    void            LogicFail( const char * message );
    
    void            ParserFail( const char * prefix  = nullptr);
    
    bool            Failed();
    
private:
    void            NextChar();
    void            NextToken();
    void            NextTokenImpl();
    void            Error( const char * msg );
    void            ReadNumber( bool negative );
    void            ReadIdentifier();   
    void            PrecomputeOperators();
    std::string     TokenToStr( Token token, const char * associated_token_str );
    bool            isspace( char c );
    
private:
    Instance * inst;
    bool            _single_line;
 
    std::unordered_set< std::string >   keywords;
    std::unordered_set< std::string >   operators;
    bool                                _numbers_enabled;
    
    std::istream *  in;
    int             line;
    int             column;
    int             prev_line;
    int             prev_column;
    
    bool            failed;
    
    // current token
    int             newlines;       
    bool            newline_skipping;
    Token           token;
    std::string     token_str;      
    
    // current char
    int             c;
    
    // tested tokens
    std::vector< Token >        tested_tokens;
    std::vector< std::string >  tested_keywords;
    std::vector< std::string >  tested_operators;
    
    // precomputed operators
    std::unordered_set< char > operators_begins;
    std::unordered_set< std::string > operators_prefixes;   
};
 
class Lex_LogTrace : public LogTrace
{
public:
    Lex_LogTrace( Lex const & lex ) :
        lex( lex )
    {
    }
    
protected:
    virtual void PrintTraceLine( std::ostream & out ) override
    {
        out << "Line "<< ( lex.GetLine()+1 ) << ":" << ( lex.GetColumn()+1 ) << ":" << std::endl;
    }
    
private:
    Lex const & lex;
};
 
}