tokenizer.h

#pragma once
 
#include <vector>
 
#include "token.h"
 
#include "lib_begin.h"
 
namespace cgv {
        namespace utils {
 
class CGV_API tokenizer : public token
{
protected:
        std::string separators;
        bool merge_separators;
        std::string begin_skip;
        std::string end_skip;
        std::string escape_skip;
        std::string whitespaces;
        void init();
        bool handle_skip(token& result);
        bool handle_separators(token& result,bool check_skip=true);
        bool reverse_handle_skip(token& result);
        bool reverse_handle_separators(token& result,bool check_skip=true);
public:
        tokenizer();
        tokenizer(const token&);
        tokenizer(const char*);
        tokenizer(const std::string&);
        tokenizer& set_ws(const std::string& ws);
        tokenizer& set_skip(const std::string& open, const std::string& close);
        tokenizer& set_skip(const std::string& open, const std::string& close, const std::string& escape);
        tokenizer& set_sep(const std::string& sep, bool merge);
        tokenizer& set_sep(const std::string& sep);
        tokenizer& set_sep_merge(bool merge);
        token bite();
        token reverse_bite();
        void reverse_skip_whitespaces();
        void skip_whitespaces();
        bool skip_ws_check_empty() { skip_whitespaces(); return empty(); }
        bool reverse_skip_ws_check_empty() { reverse_skip_whitespaces(); return empty(); }
        bool balanced_bite(token& result, const std::string& open_parenthesis, const std::string& close_parenthesis, bool wait_for_sep = false);
        void bite_all(std::vector<token>& result);
};
 
inline void bite_all(tokenizer& t, std::vector<token>& result) { while(!t.skip_ws_check_empty())result.push_back(t.bite()); }
 
        }
}
 
#include <cgv/config/lib_end.h>