1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
|
/*=============================================================================
Boost.Wave: A Standard compliant C++ preprocessor library
Sample: Print out the preprocessed tokens returned by the Wave iterator
This sample shows, how it is possible to use a custom lexer type and a
custom token type with the Wave library.
http://www.boost.org/
Copyright (c) 2001-2011 Hartmut Kaiser. Distributed under the Boost
Software License, Version 1.0. (See accompanying file
LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
=============================================================================*/
#include "cpp_tokens.hpp" // global configuration
///////////////////////////////////////////////////////////////////////////////
// Include Wave itself
#include <boost/wave.hpp>
///////////////////////////////////////////////////////////////////////////////
// The following files contain the custom lexer type to use
#include "slex_token.hpp"
#include "slex_iterator.hpp"
///////////////////////////////////////////////////////////////////////////////
// include lexer specifics, import lexer names
#if !defined(BOOST_WAVE_SEPARATE_LEXER_INSTANTIATION)
#include "slex/cpp_slex_lexer.hpp"
#endif // !defined(BOOST_WAVE_SEPARATE_LEXER_INSTANTIATION)
///////////////////////////////////////////////////////////////////////////////
// import required names
using namespace boost::spirit::classic;
using std::string;
using std::getline;
using std::ifstream;
using std::cout;
using std::cerr;
using std::endl;
using std::ostream;
///////////////////////////////////////////////////////////////////////////////
// main program
int
main(int argc, char *argv[])
{
if (2 != argc) {
cout << "Usage: cpp_tokens input_file" << endl;
return 1;
}
// read the file to analyse into a std::string
ifstream infile(argv[1]);
string teststr;
if (infile.is_open()) {
infile.unsetf(std::ios::skipws);
#if defined(BOOST_NO_TEMPLATED_ITERATOR_CONSTRUCTORS)
// this is known to be very slow for large files on some systems
copy (std::istream_iterator<char>(infile),
std::istream_iterator<char>(),
std::inserter(teststr, teststr.end()));
#else
teststr = std::string(std::istreambuf_iterator<char>(infile.rdbuf()),
std::istreambuf_iterator<char>());
#endif
}
else {
teststr = argv[1];
}
// The following typedef does the trick. It defines the context type to use,
// which depends on the lexer type (provided by the second template
// parameter). Our lexer type 'slex_iterator<>' depends on a custom token type
// 'slex_token<>'. Our custom token type differs from the original one provided
// by the Wave library only by defining an additional operator<<(), which is
// used to dump the token information carried by a given token (see loop
// below).
typedef boost::wave::cpplexer::slex_token<> token_type;
typedef boost::wave::cpplexer::slex::slex_iterator<token_type> lexer_type;
typedef boost::wave::context<std::string::iterator, lexer_type>
context_type;
// The C++ preprocessor iterator shouldn't be constructed directly. It is to be
// generated through a boost::wave::context<> object. This object is
// additionally to be used to initialize and define different parameters of
// the actual preprocessing.
// The preprocessing of the input stream is done on the fly behind the scenes
// during iteration over the context_type::iterator_type stream.
context_type ctx (teststr.begin(), teststr.end(), argv[1]);
ctx.set_language(boost::wave::support_cpp0x);
ctx.set_language(boost::wave::enable_preserve_comments(ctx.get_language()));
ctx.set_language(boost::wave::enable_prefer_pp_numbers(ctx.get_language()));
ctx.set_language(boost::wave::enable_emit_contnewlines(ctx.get_language()));
context_type::iterator_type first = ctx.begin();
context_type::iterator_type last = ctx.end();
context_type::token_type current_token;
try {
// Traverse over the tokens generated from the input and dump the token
// contents.
while (first != last) {
// retrieve next token
current_token = *first;
// output token info
cout << "matched " << current_token << endl;
++first;
}
}
catch (boost::wave::cpp_exception const& e) {
// some preprocessing error
cerr
<< e.file_name() << "(" << e.line_no() << "): "
<< e.description() << endl;
return 2;
}
catch (std::exception const& e) {
// use last recognized token to retrieve the error position
cerr
<< current_token.get_position().get_file()
<< "(" << current_token.get_position().get_line() << "): "
<< "unexpected exception: " << e.what()
<< endl;
return 3;
}
catch (...) {
// use last recognized token to retrieve the error position
cerr
<< current_token.get_position().get_file()
<< "(" << current_token.get_position().get_line() << "): "
<< "unexpected exception." << endl;
return 4;
}
return 0;
}
|
