5 #include "../Basics/SS.hpp"
6 #include "../Instancing/Instance.hpp"
14 cm( inst, this,
"Lex" ),
16 _single_line( false ),
17 _numbers_enabled( true ),
25 newline_skipping( true ),
37 this->keywords.insert( keyword );
42 this->operators.insert( op );
47 this->newline_skipping =
false;
52 this->newline_skipping =
true;
57 this->_numbers_enabled = enabled;
60 void Lex::Init( std::istream & in,
bool single_line )
64 this->_single_line = single_line;
71 this->PrecomputeOperators();
89 if( this->failed || !this->in || !this->in->good() )
91 this->c = std::char_traits< char >::eof();
95 this->c = this->in->get();
104 if( !this->in->eof() && this->in->fail() )
106 this->c = std::char_traits< char >::eof();
107 this->Error(
"Input error: Error reading input stream." );
113 void Lex::PrecomputeOperators()
115 for(
auto & op : this->operators )
117 this->operators_begins.insert( op[0] );
123 this->operators_prefixes.insert( prefix );
128 void Lex::ReadNumber(
bool negative )
136 while( this->c >=
'0' && this->c <=
'9' )
145 this->token_str = str;
153 while( this->c >=
'0' && this->c <=
'9' )
168 if( this->c < '0' || this->c >
'9' )
170 this->Error(
"Lex error: No number after 'e' in floating point literal." );
176 while( this->c >=
'0' && this->c <=
'9' )
184 this->token_str = str;
187 if( !this->isspace( this->c ) && !this->operators_begins.count( this->c ) )
189 this->Error(
"Lex error: Number must be followed by a whitespace character or by an operator." );
196 void Lex::ReadIdentifier()
198 char first = this->c;
204 if( this->operators_begins.count( first ) )
206 while( this->c != std::char_traits< char >::eof() && this->operators_prefixes.count( str + (
char)this->c ) )
213 this->token_str = str;
218 while( this->c != std::char_traits< char >::eof() && !this->isspace( this->c ) && !this->operators_begins.count( this->c ) )
224 this->token_str = str;
225 if( this->keywords.count( str ) )
233 bool Lex::isspace(
char c )
235 return c==
' ' || c==
'\t' || c==
'\r' || c==
'\n';
238 void Lex::NextToken()
240 this->prev_line = this->line;
241 this->prev_column = this->column;
242 this->NextTokenImpl();
243 this->
cm.
log(
SRC_INFO,
Defs::Log::Lex,
"NextToken: ", this->TokenToStr( this->token, this->token_str.c_str() ),
" ", this->token_str );
246 void Lex::NextTokenImpl()
250 this->tested_tokens.clear();
251 this->tested_keywords.clear();
252 this->tested_operators.clear();
261 else if( this->c == std::char_traits< char >::eof() )
264 this->token_str =
"";
268 else if( this->c ==
'\n' )
271 if( this->_single_line )
282 else if( this->c ==
'#' )
284 while( this->c !=
'\n' && this->c != std::char_traits< char >::eof() )
287 else if( this->isspace( this->c ) )
291 else if( this->_numbers_enabled && !this->operators_begins.count(
'-' ) && this->c ==
'-' )
294 this->ReadNumber(
true );
297 else if( this->_numbers_enabled && this->c >=
'0' && this->c <=
'9' )
299 this->ReadNumber(
false );
302 else if( this->c ==
'"' || this->c ==
'\'' )
307 bool escaped =
false;
308 while( this->c != end || escaped )
313 if( this->c ==
'\\' )
315 else if( this->c ==
'n' )
317 else if( this->c ==
'r' )
319 else if( this->c ==
't' )
321 else if( this->c ==
'"' )
323 else if( this->c != std::char_traits< char >::eof() )
325 this->Error( SS() <<
"Lex error: Invalid escaped character in string: \\" << this->c <<
"." << SS::c_str() );
333 if( this->c == std::char_traits< char >::eof() )
335 this->Error(
"Lex error: Unexpected end of input, string not closed." );
340 else if( this->c ==
'\n' )
342 this->Error(
"Lex error: Unexpected end of line, string not closed." );
347 else if( end ==
'"' && this->c ==
'\\' )
362 this->token_str = str;
367 this->ReadIdentifier();
380 this->tested_tokens.push_back( token );
382 if( !this->newline_skipping && this->newlines && token !=
Newline )
386 return this->newlines;
390 return this->token == token;
398 this->tested_keywords.push_back( keyword );
400 if( !this->newline_skipping && this->newlines )
403 return this->token ==
Keyword && this->token_str == keyword;
411 this->tested_operators.push_back( op );
413 if( !this->newline_skipping && this->newlines )
416 return this->token ==
Operator && this->token_str == op;
421 if( !this->newline_skipping && this->newlines )
429 if( !this->newline_skipping && this->newlines )
432 return this->token_str;
448 if( !this->newline_skipping && token !=
Newline && this->newlines )
450 this->tested_tokens.push_back( token );
455 if( this->token == token )
465 else if( token ==
Newline && this->newlines )
472 this->tested_tokens.push_back( token );
480 if( !this->keywords.count( keyword ) )
482 this->Error(
SS()<<
"Trying to accept keyword '"<<keyword<<
"' that was not registered using method Lex::DefineKeyword."<<
SS::c_str() );
489 if( !this->newline_skipping && this->newlines )
491 this->tested_keywords.push_back( keyword );
496 if( this->token ==
Keyword && this->token_str == keyword )
502 this->tested_keywords.push_back( keyword );
510 if( !this->operators.count( op ) )
512 this->Error(
SS()<<
"Trying to accept operator '"<<op<<
"' that was not registered using method Lex::DefineOperator."<<
SS::c_str() );
519 if( !this->newline_skipping && this->newlines )
521 this->tested_operators.push_back( op );
526 if( this->token ==
Operator && this->token_str == op )
532 this->tested_operators.push_back( op );
543 if( !this->newline_skipping && this->newlines )
545 this->tested_tokens.push_back(
String );
550 if( this->token ==
String )
552 std::string result = this->token_str;
558 this->tested_tokens.push_back(
String );
569 if( !this->newline_skipping && this->newlines )
571 this->tested_tokens.push_back(
Name );
576 if( this->token ==
Name )
578 auto result = this->token_str;
584 this->tested_tokens.push_back(
Name );
595 if( !this->newline_skipping && this->newlines )
597 this->tested_tokens.push_back(
Integer );
604 auto result = atoi( this->token_str.c_str() );
610 this->tested_tokens.push_back(
Integer );
621 if( !this->newline_skipping && this->newlines )
623 this->tested_tokens.push_back(
Double );
628 if( this->token ==
Double )
630 auto result = atof( this->token_str.c_str() );
634 else if( this->token ==
Integer )
636 auto result = atoi( this->token_str.c_str() );
638 return double( result );
642 this->tested_tokens.push_back(
Double );
651 return this->prev_line;
656 return this->prev_column;
659 std::string Lex::TokenToStr( Token token,
const char * str )
664 return SS()<<
"Keyword( "<<str<<
" )"<<
SS::str();
666 return SS()<<
"Operator( "<<str<<
" )"<<
SS::str();
672 return SS()<<
"String \""<<str<<
"\""<<
SS::str();
674 return SS()<<
"Name( "<<str<<
" )"<<
SS::str();
689 this->Error( message );
699 std::string current_token_string;
700 if( !this->newline_skipping && this->newlines )
701 current_token_string = this->TokenToStr(
Newline,
"\n" );
703 current_token_string = this->TokenToStr( this->token, this->token_str.c_str() );
707 ss << prefix <<
": ";
708 ss <<
"Unexpected token " << current_token_string <<
". Expecting one of these tokens: ";
712 for(
Token t : this->tested_tokens )
719 ss << this->TokenToStr( t,
"" );
722 for(
auto & str : this->tested_keywords )
729 ss << this->TokenToStr(
Keyword, str.c_str() );
732 for(
auto & str : this->tested_operators )
739 ss << this->TokenToStr(
Operator, str.c_str() );
753 void Lex::Error(
const char * msg )
766 this->failed =
false;
770 while( this->newlines==0 && this->token !=
Eof )
773 else if( token ==
Double )
775 while( this->token !=
Double && this->token !=
Integer && this->token !=
Eof )
780 while( this->token != token && this->token !=
Eof )
793 this->failed =
false;
795 while( !( this->token ==
Keyword && this->token_str == keyword ) && this->token !=
Eof )
807 this->failed =
false;
809 while( !( this->token ==
Operator && this->token_str == op ) && this->token !=
Eof )