Changeset 54676


Ignore:
Timestamp:
Jul 5, 2009, 6:50:29 PM (9 years ago)
Author:
Hartmut Kaiser
Message:

Spirit: Added support for accessing the token value from a lexer semantic action

Location:
trunk/boost/spirit/home/lex
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • trunk/boost/spirit/home/lex/argument.hpp

    r54511 r54676  
    134134
    135135    ///////////////////////////////////////////////////////////////////////////
    136     //  The eoi_getter is used to create the _eoi placeholder, which is a
    137     //  Phoenix actor used to access the end of input iterator pointing to the
    138     //  end of the underlying input sequence.
    139     //
    140     //  This actor is invoked whenever the placeholder '_eoi' is used in a
     136    //  The value_getter is used to create the _value placeholder, which is a
     137    //  Phoenix actor used to access or change the value of the current token.
     138    //
     139    //  This actor is invoked whenever the placeholder '_value' is used in a
    141140    //  lexer semantic action:
    142141    //
    143142    //      lex::token_def<> identifier = "[a-zA-Z_][a-zA-Z0-9_]*";
    144143    //      this->self = identifier
    145     //          [ std::cout << construct_<std::string>(_end, _eoi) ];
    146     //
    147     //  The example shows how to use _eoi to print all remaining input after
    148     //  matching a token 'identifier'.
    149     struct eoi_getter
     144    //          [ _value = construct_<std::string>(_start, _end) ];
     145    //
     146    //  The example shows how to use _value to set the identifier name as the
     147    //  token value.
     148    struct value_getter
    150149    {
    151150        typedef mpl::true_ no_nullary;
     
    160159            context_type;
    161160
     161            typedef typename context_type::token_value_type& type;
     162        };
     163
     164        template <typename Env>
     165        typename result<Env>::type
     166        eval(Env const& env) const
     167        {
     168            return fusion::at_c<4>(env.args()).value();
     169        }
     170    };
     171
     172    ///////////////////////////////////////////////////////////////////////////
     173    //  The eoi_getter is used to create the _eoi placeholder, which is a
     174    //  Phoenix actor used to access the end of input iterator pointing to the
     175    //  end of the underlying input sequence.
     176    //
     177    //  This actor is invoked whenever the placeholder '_eoi' is used in a
     178    //  lexer semantic action:
     179    //
     180    //      lex::token_def<> identifier = "[a-zA-Z_][a-zA-Z0-9_]*";
     181    //      this->self = identifier
     182    //          [ std::cout << construct_<std::string>(_end, _eoi) ];
     183    //
     184    //  The example shows how to use _eoi to print all remaining input after
     185    //  matching a token 'identifier'.
     186    struct eoi_getter
     187    {
     188        typedef mpl::true_ no_nullary;
     189
     190        template <typename Env>
     191        struct result
     192        {
     193            typedef typename
     194                remove_const<
     195                    typename mpl::at_c<typename Env::args_type, 4>::type
     196                >::type
     197            context_type;
     198
    162199            typedef typename context_type::base_iterator_type const& type;
    163200        };
     
    184221    // token
    185222    phoenix::actor<phoenix::argument<3> > const _tokenid = phoenix::argument<3>();
     223
     224    // '_value' may be used to access and change the token value of the current
     225    // token
     226    phoenix::actor<value_getter> const _value = value_getter();
    186227
    187228    // _state may be used to access and change the name of the current lexer
  • trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp

    r54225 r54676  
    5959    ///////////////////////////////////////////////////////////////////////////
    6060    template <typename Token
    61       , template <typename, typename, typename> class FunctorData
     61      , template <typename, typename, typename, typename> class FunctorData
    6262      , typename Iterator = typename Token::iterator_type
    6363      , typename SupportsActors = mpl::false_
     
    7474        // reference, see
    7575        // http://www.open-std.org/JTC1/SC22/WG21/docs/cwg_defects.html#45.
    76         friend class FunctorData<Iterator, SupportsActors, SupportsState>;
     76        typedef typename Token::token_value_type token_value_type;
     77        friend class FunctorData<Iterator, SupportsActors, SupportsState
     78          , token_value_type>;
    7779
    7880        // Helper template allowing to assign a value on exit
     
    111113        typedef Token result_type;
    112114        typedef functor unique;
    113         typedef FunctorData<Iterator, SupportsActors, SupportsState> shared;
     115        typedef FunctorData<Iterator, SupportsActors, SupportsState
     116          , token_value_type> shared;
    114117
    115118        BOOST_SPIRIT_EOF_PREFIX result_type const eof;
     
    191194                    return result = result_type(id, state, data.get_first(), end);
    192195                }
     196                else if (pass_flags::pass_use_value == pass) {
     197                    // return matched token using the token value as set before
     198                    // using data.set_value(), advancing 'data.first_' past the
     199                    // matched sequence
     200                    assign_on_exit<Iterator> on_exit(data.get_first(), end);
     201                    return result = result_type(id, state, data.value());
     202                }
    193203                else if (pass_flags::pass_fail == pass) {
    194204                    // if the data.first_ got adjusted above, revert this adjustment
     
    200210                }
    201211
    202             // if this token needs to be ignored, just repeat the matching
     212            // if this token needs to be ignored, just repeat the matching,
     213            // while starting right after the current match
     214                data.get_first() = end;
    203215
    204216            } while (true);
     
    239251    ///////////////////////////////////////////////////////////////////////////
    240252    template <typename Token
    241       , template <typename, typename, typename> class FunctorData
     253      , template <typename, typename, typename, typename> class FunctorData
    242254      , typename Iterator, typename SupportsActors, typename SupportsState>
    243255    typename functor<Token, FunctorData, Iterator, SupportsActors, SupportsState>::result_type const
  • trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp

    r54225 r54676  
    2424    {
    2525        ///////////////////////////////////////////////////////////////////////
    26         template <typename Iterator, typename HasActors, typename HasState>
     26        template <typename Iterator, typename HasActors, typename HasState
     27          , typename TokenValue>
    2728        struct data;    // no default specialization
    2829
    2930        ///////////////////////////////////////////////////////////////////////
    3031        //  neither supports state, nor actors
    31         template <typename Iterator>
    32         struct data<Iterator, mpl::false_, mpl::false_>
     32        template <typename Iterator, typename TokenValue>
     33        struct data<Iterator, mpl::false_, mpl::false_, TokenValue>
    3334        {
    3435        protected:
     
    3940        public:
    4041            typedef Iterator base_iterator_type;
     42            typedef unused_type token_value_type;
    4143            typedef std::size_t state_type;
    4244            typedef char_type const* state_name_type;
     
    147149            Iterator const& get_last() const { return last_; }
    148150
     151            unused_type value() const { return unused; }
     152
    149153        protected:
    150154            Iterator& first_;
     
    157161        ///////////////////////////////////////////////////////////////////////
    158162        //  doesn't support lexer semantic actions
    159         template <typename Iterator>
    160         struct data<Iterator, mpl::false_, mpl::true_>
    161           : data<Iterator, mpl::false_, mpl::false_>
     163        template <typename Iterator, typename TokenValue>
     164        struct data<Iterator, mpl::false_, mpl::true_, TokenValue>
     165          : data<Iterator, mpl::false_, mpl::false_, TokenValue>
    162166        {
    163167        protected:
    164             typedef data<Iterator, mpl::false_, mpl::false_> base_type;
     168            typedef data<Iterator, mpl::false_, mpl::false_, TokenValue> base_type;
    165169            typedef typename base_type::char_type char_type;
    166170
    167171        public:
    168172            typedef Iterator base_iterator_type;
     173            typedef unused_type token_value_type;
    169174            typedef typename base_type::state_type state_type;
    170175            typedef typename base_type::state_name_type state_name_type;
     
    222227        ///////////////////////////////////////////////////////////////////////
    223228        //  does support lexer semantic actions, may support state
    224         template <typename Iterator, typename HasState>
    225         struct data<Iterator, mpl::true_, HasState>
    226           : data<Iterator, mpl::false_, HasState>
     229        template <typename Iterator, typename HasState, typename TokenValue>
     230        struct data<Iterator, mpl::true_, HasState, TokenValue>
     231          : data<Iterator, mpl::false_, HasState, TokenValue>
    227232        {
    228233        public:
     
    231236
    232237        protected:
    233             typedef data<Iterator, mpl::false_, HasState> base_type;
     238            typedef data<Iterator, mpl::false_, HasState, TokenValue> base_type;
    234239            typedef typename base_type::char_type char_type;
    235240            typedef typename semantic_actions_type::functor_wrapper_type
     
    238243        public:
    239244            typedef Iterator base_iterator_type;
     245            typedef TokenValue token_value_type;
    240246            typedef typename base_type::state_type state_type;
    241247            typedef typename base_type::state_name_type state_name_type;
     
    311317            }
    312318
     319            TokenValue const& value() const
     320            {
     321                return value_;
     322            }
     323            TokenValue& value()
     324            {
     325                return value_;
     326            }
     327            template <typename Value>
     328            void set_value(Value const& val)
     329            {
     330                value_ = val;
     331            }
     332
    313333        protected:
    314334            semantic_actions_type const& actions_;
    315335            Iterator hold_;     // iterator needed to support lex::more()
    316336            bool has_hold_;     // 'true' if hold_ is valid
     337            TokenValue value_;
    317338        };
    318339    }
  • trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp

    r54218 r54676  
    206206    template <typename Token = token<>
    207207      , typename Iterator = typename Token::iterator_type
    208       , typename Functor = functor<Token, lexertl::detail::data, Iterator, mpl::false_>
     208      , typename Functor = functor<Token, lexertl::detail::data, Iterator>
    209209      , typename TokenSet = lex::token_set<token_set<Token, Iterator> > >
    210210    class lexer
     
    363363        // lexertl specific data
    364364        mutable boost::lexer::basic_state_machine<char_type> state_machine_;
    365         std::size_t unique_ids_;
    366365        boost::lexer::regex_flags flags_;
    367366        boost::lexer::basic_rules<char_type> rules_;
  • trunk/boost/spirit/home/lex/lexer/lexertl/token.hpp

    r54279 r54676  
    118118        typedef mpl::false_ has_state;
    119119        typedef std::size_t id_type;
     120        typedef unused_type token_value_type;
    120121
    121122        //  default constructed tokens correspond to EOI tokens
     
    126127
    127128        token(id_type id, std::size_t) : id_(id) {}
     129
     130        token(id_type id, std::size_t, token_value_type)
     131          : id_(id) {}
    128132
    129133        token(id_type id, std::size_t, Iterator const& first
     
    204208        token(id_type id, std::size_t state)
    205209          : base_type(id, boost::lexer::npos), state_(state) {}
     210
     211        token(id_type id, std::size_t state, token_value_type)
     212          : base_type(id, boost::lexer::npos, unused)
     213          , state_(state) {}
    206214
    207215        token(id_type id, std::size_t state
     
    293301        //  accessed for the first time.
    294302        typedef iterator_range<Iterator> iterpair_type;
     303
     304    public:
     305        typedef typename base_type::id_type id_type;
    295306        typedef typename detail::token_value_type<
    296307            iterpair_type, AttributeTypes
    297308        >::type token_value_type;
    298309
    299     public:
    300310        typedef Iterator iterator_type;
    301311
     
    308318          , value_(iterpair_type(iterator_type(), iterator_type())) {}
    309319
    310         token(std::size_t id, std::size_t state, Iterator const& first
     320        token(id_type id, std::size_t state, token_value_type const& value)
     321          : base_type(id, state, value)
     322          , value_(value) {}
     323
     324        token(id_type id, std::size_t state, Iterator const& first
    311325              , Iterator const& last)
    312326          : base_type(id, state, first, last)
  • trunk/boost/spirit/home/lex/lexer/pass_flags.hpp

    r53801 r54676  
    1818    BOOST_SCOPED_ENUM_START(pass_flags)
    1919    {
    20         pass_fail = 0,       // make the current match fail in retrospective
    21         pass_normal = 1,     // continue normal token matching, that's the default
    22         pass_ignore = 2      // ignore the current token and start matching the next
     20        pass_fail = 0,        // make the current match fail in retrospective
     21        pass_normal = 1,      // continue normal token matching, that's the default
     22        pass_ignore = 2,      // ignore the current token and start matching the next
     23        pass_use_value = 3    // continue normal matching but use token value as set before
    2324    };
    2425    BOOST_SCOPED_ENUM_END
  • trunk/boost/spirit/home/lex/tokenize_and_parse.hpp

    r54648 r54676  
    250250    //
    251251    ///////////////////////////////////////////////////////////////////////////
     252    namespace detail
     253    {
     254        template <typename Token>
     255        bool tokenize_callback(Token const& t, void (*f)(Token const&))
     256        {
     257            f(t);
     258            return true;
     259        }
     260
     261        template <typename Token, typename Eval>
     262        bool tokenize_callback(Token const& t, phoenix::actor<Eval> const& f)
     263        {
     264            f(t);
     265            return true;
     266        }
     267
     268        template <typename Token>
     269        bool tokenize_callback(Token const& t, bool (*f)(Token const&))
     270        {
     271            return f(t);
     272        }
     273    }
     274
    252275    template <typename Iterator, typename Lexer, typename F>
    253276    inline bool
     
    261284        for (/**/; iter != end; ++iter)
    262285        {
    263             if (!f(*iter))
     286            if (!detail::tokenize_callback(*iter, f))
    264287                return false;
    265288        }
Note: See TracChangeset for help on using the changeset viewer.