@@ -78,32 +78,35 @@ static void testcase(const std::string &name, void (*f)(), int argc, char * cons
7878#define TEST_CASE (F ) (testcase(#F, F, argc, argv))
7979
8080
81+ static simplecpp::TokenList makeTokenList (const char code[], std::vector<std::string> &filenames, const std::string &filename=std::string(), simplecpp::OutputList *outputList=nullptr)
82+ {
83+ std::istringstream istr (code);
84+ return simplecpp::TokenList (istr,filenames,filename,outputList);
85+ }
8186
82- static std::string readfile (const char code[], int sz=- 1 , simplecpp::OutputList *outputList=nullptr )
87+ static simplecpp::TokenList makeTokenList (const char code[], std:: size_t size, std::vector<std::string> &filenames, const std::string &filename=std::string() , simplecpp::OutputList *outputList=nullptr)
8388{
84- std::istringstream istr (sz == -1 ? std::string (code) : std::string (code,sz));
85- std::vector<std::string> files;
86- return simplecpp::TokenList (istr,files,std::string (),outputList).stringify ();
89+ std::istringstream istr (std::string (code, size));
90+ return simplecpp::TokenList (istr,filenames,filename,outputList);
8791}
8892
89- static simplecpp::TokenList makeTokenList (const char code[], std::vector<std::string> &files, const std::string &file )
93+ static std::string readfile (const char code[], simplecpp::OutputList *outputList= nullptr )
9094{
91- std::istringstream istr (code) ;
92- return simplecpp::TokenList (istr ,files,file );
95+ std::vector<std::string> files ;
96+ return makeTokenList (code ,files,std::string (),outputList). stringify ( );
9397}
9498
95- static simplecpp::TokenList makeTokenList (const char code[])
99+ static std::string readfile (const char code[], std:: size_t size, simplecpp::OutputList *outputList= nullptr )
96100{
97101 std::vector<std::string> files;
98- return makeTokenList (code, files, std::string ());
102+ return makeTokenList (code,size, files,std::string (),outputList). stringify ( );
99103}
100104
101105static std::string preprocess (const char code[], const simplecpp::DUI &dui, simplecpp::OutputList *outputList)
102106{
103- std::istringstream istr (code);
104107 std::vector<std::string> files;
105108 std::map<std::string, simplecpp::TokenList*> filedata;
106- simplecpp::TokenList tokens (istr ,files);
109+ simplecpp::TokenList tokens = makeTokenList (code ,files);
107110 tokens.removeComments ();
108111 simplecpp::TokenList tokens2 (files);
109112 simplecpp::preprocess (tokens2, tokens, files, filedata, dui, outputList);
@@ -167,15 +170,15 @@ static void backslash()
167170 // <backslash><space><newline> preprocessed differently
168171 simplecpp::OutputList outputList;
169172
170- readfile (" //123 \\\n 456" , - 1 , &outputList);
173+ readfile (" //123 \\\n 456" , &outputList);
171174 ASSERT_EQUALS (" " , toString (outputList));
172- readfile (" //123 \\ \n 456" , - 1 , &outputList);
175+ readfile (" //123 \\ \n 456" , &outputList);
173176 ASSERT_EQUALS (" file0,1,portability_backslash,Combination 'backslash space newline' is not portable.\n " , toString (outputList));
174177
175178 outputList.clear ();
176- readfile (" #define A \\\n 123" , - 1 , &outputList);
179+ readfile (" #define A \\\n 123" , &outputList);
177180 ASSERT_EQUALS (" " , toString (outputList));
178- readfile (" #define A \\ \n 123" , - 1 , &outputList);
181+ readfile (" #define A \\ \n 123" , &outputList);
179182 ASSERT_EQUALS (" file0,1,portability_backslash,Combination 'backslash space newline' is not portable.\n " , toString (outputList));
180183}
181184
@@ -196,7 +199,8 @@ static void builtin()
196199static std::string testConstFold (const char code[])
197200{
198201 try {
199- simplecpp::TokenList expr = makeTokenList (code);
202+ std::vector<std::string> files;
203+ simplecpp::TokenList expr = makeTokenList (code, files);
200204 expr.constFold ();
201205 return expr.stringify ();
202206 } catch (std::exception &) {
@@ -803,36 +807,35 @@ static void error3()
803807{
804808 const char code[] = " #error \" bla bla\\\n "
805809 " bla bla.\"\n " ;
806- std::istringstream istr (code);
807810 std::vector<std::string> files;
808811 simplecpp::OutputList outputList;
809- simplecpp::TokenList rawtokens (istr , files, " test.c" , &outputList);
812+ const simplecpp::TokenList rawtokens = makeTokenList (code , files, " test.c" , &outputList);
810813 ASSERT_EQUALS (" " , toString (outputList));
811814}
812815
813816static void error4 ()
814817{
815818 // "#error x\n1"
816- const std::string code (" \xFE\xFF\x00\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x31 " , 22 );
817- std::istringstream istr (code);
819+ const char code[] = " \xFE\xFF\x00\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x31 " ;
818820 std::vector<std::string> files;
819821 std::map<std::string, simplecpp::TokenList*> filedata;
820822 simplecpp::OutputList outputList;
821823 simplecpp::TokenList tokens2 (files);
822- simplecpp::preprocess (tokens2, simplecpp::TokenList (istr,files," test.c" ), files, filedata, simplecpp::DUI (), &outputList);
824+ const simplecpp::TokenList rawtoken = makeTokenList (code, sizeof (code),files," test.c" );
825+ simplecpp::preprocess (tokens2, rawtoken, files, filedata, simplecpp::DUI (), &outputList);
823826 ASSERT_EQUALS (" file0,1,#error,#error x\n " , toString (outputList));
824827}
825828
826829static void error5 ()
827830{
828831 // "#error x\n1"
829- const std::string code (" \xFF\xFE\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x78\x00\x31\x00 " , 22 );
830- std::istringstream istr (code);
832+ const char code[] = " \xFF\xFE\x23\x00\x65\x00\x72\x00\x72\x00\x6f\x00\x72\x00\x20\x00\x78\x00\x0a\x00\x78\x00\x31\x00 " ;
831833 std::vector<std::string> files;
832834 std::map<std::string, simplecpp::TokenList*> filedata;
833835 simplecpp::OutputList outputList;
834836 simplecpp::TokenList tokens2 (files);
835- simplecpp::preprocess (tokens2, simplecpp::TokenList (istr,files," test.c" ), files, filedata, simplecpp::DUI (), &outputList);
837+ const simplecpp::TokenList rawtokens = makeTokenList (code, sizeof (code),files," test.c" );
838+ simplecpp::preprocess (tokens2, rawtokens, files, filedata, simplecpp::DUI (), &outputList);
836839 ASSERT_EQUALS (" file0,1,#error,#error x\n " , toString (outputList));
837840}
838841
@@ -1557,13 +1560,13 @@ static void missingHeader1()
15571560static void missingHeader2 ()
15581561{
15591562 const char code[] = " #include \" foo.h\"\n " ; // this file exists
1560- std::istringstream istr (code);
15611563 std::vector<std::string> files;
15621564 std::map<std::string, simplecpp::TokenList*> filedata;
15631565 filedata[" foo.h" ] = nullptr ;
15641566 simplecpp::OutputList outputList;
15651567 simplecpp::TokenList tokens2 (files);
1566- simplecpp::preprocess (tokens2, simplecpp::TokenList (istr,files), files, filedata, simplecpp::DUI (), &outputList);
1568+ const simplecpp::TokenList rawtokens = makeTokenList (code,files);
1569+ simplecpp::preprocess (tokens2, rawtokens, files, filedata, simplecpp::DUI (), &outputList);
15671570 ASSERT_EQUALS (" " , toString (outputList));
15681571}
15691572
@@ -1578,9 +1581,8 @@ static void missingHeader3()
15781581static void nestedInclude ()
15791582{
15801583 const char code[] = " #include \" test.h\"\n " ;
1581- std::istringstream istr (code);
15821584 std::vector<std::string> files;
1583- simplecpp::TokenList rawtokens (istr ,files," test.h" );
1585+ simplecpp::TokenList rawtokens = makeTokenList (code ,files," test.h" );
15841586 std::map<std::string, simplecpp::TokenList*> filedata;
15851587 filedata[" test.h" ] = &rawtokens;
15861588
@@ -1604,9 +1606,8 @@ static void multiline2()
16041606 const char code[] = " #define A /*\\\n "
16051607 " */1\n "
16061608 " A" ;
1607- std::istringstream istr (code);
16081609 std::vector<std::string> files;
1609- simplecpp::TokenList rawtokens (istr ,files);
1610+ simplecpp::TokenList rawtokens = makeTokenList (code ,files);
16101611 ASSERT_EQUALS (" # define A /**/ 1\n\n A" , rawtokens.stringify ());
16111612 rawtokens.removeComments ();
16121613 std::map<std::string, simplecpp::TokenList*> filedata;
@@ -1620,9 +1621,8 @@ static void multiline3() // #28 - macro with multiline comment
16201621 const char code[] = " #define A /*\\\n "
16211622 " */ 1\n "
16221623 " A" ;
1623- std::istringstream istr (code);
16241624 std::vector<std::string> files;
1625- simplecpp::TokenList rawtokens (istr ,files);
1625+ simplecpp::TokenList rawtokens = makeTokenList (code ,files);
16261626 ASSERT_EQUALS (" # define A /* */ 1\n\n A" , rawtokens.stringify ());
16271627 rawtokens.removeComments ();
16281628 std::map<std::string, simplecpp::TokenList*> filedata;
@@ -1637,9 +1637,8 @@ static void multiline4() // #28 - macro with multiline comment
16371637 " /*\\\n "
16381638 " */ 1\n "
16391639 " A" ;
1640- std::istringstream istr (code);
16411640 std::vector<std::string> files;
1642- simplecpp::TokenList rawtokens (istr ,files);
1641+ simplecpp::TokenList rawtokens = makeTokenList (code ,files);
16431642 ASSERT_EQUALS (" # define A /* */ 1\n\n\n A" , rawtokens.stringify ());
16441643 rawtokens.removeComments ();
16451644 std::map<std::string, simplecpp::TokenList*> filedata;
@@ -1652,7 +1651,8 @@ static void multiline5() // column
16521651{
16531652 const char code[] = " #define A\\\n "
16541653 " (" ;
1655- const simplecpp::TokenList rawtokens = makeTokenList (code);
1654+ std::vector<std::string> files;
1655+ const simplecpp::TokenList rawtokens = makeTokenList (code, files);
16561656 ASSERT_EQUALS (" # define A (" , rawtokens.stringify ());
16571657 ASSERT_EQUALS (11 , rawtokens.cback ()->location .col );
16581658}
@@ -1662,7 +1662,8 @@ static void multiline6() // multiline string in macro
16621662 const char code[] = " #define string (\"\\\n "
16631663 " x\" )\n "
16641664 " string\n " ;
1665- const simplecpp::TokenList rawtokens = makeTokenList (code);
1665+ std::vector<std::string> files;
1666+ const simplecpp::TokenList rawtokens = makeTokenList (code, files);
16661667 ASSERT_EQUALS (" # define string ( \" x\" )\n "
16671668 " \n "
16681669 " string" , rawtokens.stringify ());
@@ -1673,7 +1674,8 @@ static void multiline7() // multiline string in macro
16731674 const char code[] = " #define A(X) aaa { f(\"\\\n "
16741675 " a\" ); }\n "
16751676 " A(1)" ;
1676- const simplecpp::TokenList rawtokens = makeTokenList (code);
1677+ std::vector<std::string> files;
1678+ const simplecpp::TokenList rawtokens = makeTokenList (code, files);
16771679 ASSERT_EQUALS (" # define A ( X ) aaa { f ( \" a\" ) ; }\n "
16781680 " \n "
16791681 " A ( 1 )" , rawtokens.stringify ());
@@ -1911,11 +1913,11 @@ static void readfile_char_error()
19111913{
19121914 simplecpp::OutputList outputList;
19131915
1914- readfile (" A = L's" , - 1 , &outputList);
1916+ readfile (" A = L's" , &outputList);
19151917 ASSERT_EQUALS (" file0,1,syntax_error,No pair for character (\' ). Can't process file. File is either invalid or unicode, which is currently not supported.\n " , toString (outputList));
19161918 outputList.clear ();
19171919
1918- readfile (" A = 's\n '" , - 1 , &outputList);
1920+ readfile (" A = 's\n '" , &outputList);
19191921 ASSERT_EQUALS (" file0,1,syntax_error,No pair for character (\' ). Can't process file. File is either invalid or unicode, which is currently not supported.\n " , toString (outputList));
19201922}
19211923
@@ -1969,36 +1971,36 @@ static void readfile_string_error()
19691971{
19701972 simplecpp::OutputList outputList;
19711973
1972- readfile (" A = \" abs" , - 1 , &outputList);
1974+ readfile (" A = \" abs" , &outputList);
19731975 ASSERT_EQUALS (" file0,1,syntax_error,No pair for character (\" ). Can't process file. File is either invalid or unicode, which is currently not supported.\n " , toString (outputList));
19741976 outputList.clear ();
19751977
1976- readfile (" A = u8\" abs\n\" " , - 1 , &outputList);
1978+ readfile (" A = u8\" abs\n\" " , &outputList);
19771979 ASSERT_EQUALS (" file0,1,syntax_error,No pair for character (\" ). Can't process file. File is either invalid or unicode, which is currently not supported.\n " , toString (outputList));
19781980 outputList.clear ();
19791981
1980- readfile (" A = R\" as\n (abc)as\" " , - 1 , &outputList);
1982+ readfile (" A = R\" as\n (abc)as\" " , &outputList);
19811983 ASSERT_EQUALS (" file0,1,syntax_error,Invalid newline in raw string delimiter.\n " , toString (outputList));
19821984 outputList.clear ();
19831985
1984- readfile (" A = u8R\" as\n (abc)as\" " , - 1 , &outputList);
1986+ readfile (" A = u8R\" as\n (abc)as\" " , &outputList);
19851987 ASSERT_EQUALS (" file0,1,syntax_error,Invalid newline in raw string delimiter.\n " , toString (outputList));
19861988 outputList.clear ();
19871989
1988- readfile (" A = R\" as(abc)a\" " , - 1 , &outputList);
1990+ readfile (" A = R\" as(abc)a\" " , &outputList);
19891991 ASSERT_EQUALS (" file0,1,syntax_error,Raw string missing terminating delimiter.\n " , toString (outputList));
19901992 outputList.clear ();
19911993
1992- readfile (" A = LR\" as(abc)a\" " , - 1 , &outputList);
1994+ readfile (" A = LR\" as(abc)a\" " , &outputList);
19931995 ASSERT_EQUALS (" file0,1,syntax_error,Raw string missing terminating delimiter.\n " , toString (outputList));
19941996 outputList.clear ();
19951997
1996- readfile (" #define A \" abs" , - 1 , &outputList);
1998+ readfile (" #define A \" abs" , &outputList);
19971999 ASSERT_EQUALS (" file0,1,syntax_error,No pair for character (\" ). Can't process file. File is either invalid or unicode, which is currently not supported.\n " , toString (outputList));
19982000 outputList.clear ();
19992001
20002002 // Don't warn for a multiline define
2001- readfile (" #define A \" abs\\\n\" " , - 1 , &outputList);
2003+ readfile (" #define A \" abs\\\n\" " , &outputList);
20022004 ASSERT_EQUALS (" " , toString (outputList));
20032005}
20042006
@@ -2010,11 +2012,11 @@ static void readfile_cpp14_number()
20102012static void readfile_unhandled_chars ()
20112013{
20122014 simplecpp::OutputList outputList;
2013- readfile (" // 你好世界" , - 1 , &outputList);
2015+ readfile (" // 你好世界" , &outputList);
20142016 ASSERT_EQUALS (" " , toString (outputList));
2015- readfile (" s=\" 你好世界\" " , - 1 , &outputList);
2017+ readfile (" s=\" 你好世界\" " , &outputList);
20162018 ASSERT_EQUALS (" " , toString (outputList));
2017- readfile (" int 你好世界=0;" , - 1 , &outputList);
2019+ readfile (" int 你好世界=0;" , &outputList);
20182020 ASSERT_EQUALS (" file0,1,unhandled_char_error,The code contains unhandled character(s) (character code=228). Neither unicode nor extended ascii is supported.\n " , toString (outputList));
20192021}
20202022
@@ -2057,9 +2059,9 @@ static void tokenMacro1()
20572059 " A" ;
20582060 std::vector<std::string> files;
20592061 std::map<std::string, simplecpp::TokenList*> filedata;
2060- std::istringstream istr (code);
20612062 simplecpp::TokenList tokenList (files);
2062- simplecpp::preprocess (tokenList, simplecpp::TokenList (istr,files), files, filedata, simplecpp::DUI ());
2063+ const simplecpp::TokenList rawtokens = makeTokenList (code,files);
2064+ simplecpp::preprocess (tokenList, rawtokens, files, filedata, simplecpp::DUI ());
20632065 ASSERT_EQUALS (" A" , tokenList.cback ()->macro );
20642066}
20652067
@@ -2069,9 +2071,9 @@ static void tokenMacro2()
20692071 " ADD(1,2)" ;
20702072 std::vector<std::string> files;
20712073 std::map<std::string, simplecpp::TokenList*> filedata;
2072- std::istringstream istr (code);
20732074 simplecpp::TokenList tokenList (files);
2074- simplecpp::preprocess (tokenList, simplecpp::TokenList (istr,files), files, filedata, simplecpp::DUI ());
2075+ const simplecpp::TokenList rawtokens = makeTokenList (code,files);
2076+ simplecpp::preprocess (tokenList, rawtokens, files, filedata, simplecpp::DUI ());
20752077 const simplecpp::Token *tok = tokenList.cfront ();
20762078 ASSERT_EQUALS (" 1" , tok->str ());
20772079 ASSERT_EQUALS (" " , tok->macro );
@@ -2090,9 +2092,9 @@ static void tokenMacro3()
20902092 " ADD(FRED,2)" ;
20912093 std::vector<std::string> files;
20922094 std::map<std::string, simplecpp::TokenList*> filedata;
2093- std::istringstream istr (code);
20942095 simplecpp::TokenList tokenList (files);
2095- simplecpp::preprocess (tokenList, simplecpp::TokenList (istr,files), files, filedata, simplecpp::DUI ());
2096+ const simplecpp::TokenList rawtokens = makeTokenList (code,files);
2097+ simplecpp::preprocess (tokenList, rawtokens, files, filedata, simplecpp::DUI ());
20962098 const simplecpp::Token *tok = tokenList.cfront ();
20972099 ASSERT_EQUALS (" 1" , tok->str ());
20982100 ASSERT_EQUALS (" FRED" , tok->macro );
@@ -2111,9 +2113,9 @@ static void tokenMacro4()
21112113 " A" ;
21122114 std::vector<std::string> files;
21132115 std::map<std::string, simplecpp::TokenList*> filedata;
2114- std::istringstream istr (code);
21152116 simplecpp::TokenList tokenList (files);
2116- simplecpp::preprocess (tokenList, simplecpp::TokenList (istr,files), files, filedata, simplecpp::DUI ());
2117+ const simplecpp::TokenList rawtokens = makeTokenList (code,files);
2118+ simplecpp::preprocess (tokenList, rawtokens, files, filedata, simplecpp::DUI ());
21172119 const simplecpp::Token *tok = tokenList.cfront ();
21182120 ASSERT_EQUALS (" 1" , tok->str ());
21192121 ASSERT_EQUALS (" A" , tok->macro );
@@ -2126,9 +2128,9 @@ static void tokenMacro5()
21262128 " SET_BPF_JUMP(A | B | C);" ;
21272129 std::vector<std::string> files;
21282130 std::map<std::string, simplecpp::TokenList*> filedata;
2129- std::istringstream istr (code);
21302131 simplecpp::TokenList tokenList (files);
2131- simplecpp::preprocess (tokenList, simplecpp::TokenList (istr,files), files, filedata, simplecpp::DUI ());
2132+ const simplecpp::TokenList rawtokens = makeTokenList (code,files);
2133+ simplecpp::preprocess (tokenList, rawtokens, files, filedata, simplecpp::DUI ());
21322134 const simplecpp::Token *tok = tokenList.cfront ()->next ;
21332135 ASSERT_EQUALS (" D" , tok->str ());
21342136 ASSERT_EQUALS (" SET_BPF_JUMP" , tok->macro );
@@ -2159,13 +2161,34 @@ static void utf8()
21592161
21602162static void unicode ()
21612163{
2162- ASSERT_EQUALS (" 12" , readfile (" \xFE\xFF\x00\x31\x00\x32 " , 6 ));
2163- ASSERT_EQUALS (" 12" , readfile (" \xFF\xFE\x31\x00\x32\x00 " , 6 ));
2164- ASSERT_EQUALS (" //\n 1" , readfile (" \xFE\xFF\x00\x2f\x00\x2f\x00\x0a\x00\x31 " , 10 ));
2165- ASSERT_EQUALS (" //\n 1" , readfile (" \xFF\xFE\x2f\x00\x2f\x00\x0a\x00\x31\x00 " , 10 ));
2166- ASSERT_EQUALS (" \" a\" " , readfile (" \xFE\xFF\x00\x22\x00\x61\x00\x22 " , 8 ));
2167- ASSERT_EQUALS (" \" a\" " , readfile (" \xFF\xFE\x22\x00\x61\x00\x22\x00 " , 8 ));
2168- ASSERT_EQUALS (" \n //1" , readfile (" \xff\xfe\x0d\x00\x0a\x00\x2f\x00\x2f\x00\x31\x00\x0d\x00\x0a\x00 " ,16 ));
2164+ {
2165+ const char code[] = " \xFE\xFF\x00\x31\x00\x32 " ;
2166+ ASSERT_EQUALS (" 12" , readfile (code, sizeof (code)));
2167+ }
2168+ {
2169+ const char code[] = " \xFF\xFE\x31\x00\x32\x00 " ;
2170+ ASSERT_EQUALS (" 12" , readfile (code, sizeof (code)));
2171+ }
2172+ {
2173+ const char code[] = " \xFE\xFF\x00\x2f\x00\x2f\x00\x0a\x00\x31 " ;
2174+ ASSERT_EQUALS (" //\n 1" , readfile (code, sizeof (code)));
2175+ }
2176+ {
2177+ const char code[] = " \xFF\xFE\x2f\x00\x2f\x00\x0a\x00\x31\x00 " ;
2178+ ASSERT_EQUALS (" //\n 1" , readfile (code, sizeof (code)));
2179+ }
2180+ {
2181+ const char code[] = " \xFE\xFF\x00\x22\x00\x61\x00\x22 " ;
2182+ ASSERT_EQUALS (" \" a\" " , readfile (code, sizeof (code)));
2183+ }
2184+ {
2185+ const char code[] = " \xFF\xFE\x22\x00\x61\x00\x22\x00 " ;
2186+ ASSERT_EQUALS (" \" a\" " , readfile (code, sizeof (code)));
2187+ }
2188+ {
2189+ const char code[] = " \xff\xfe\x0d\x00\x0a\x00\x2f\x00\x2f\x00\x31\x00\x0d\x00\x0a\x00 " ;
2190+ ASSERT_EQUALS (" \n //1" , readfile (code, sizeof (code)));
2191+ }
21692192}
21702193
21712194static void warning ()
0 commit comments