tokenizer.c: Support windows newlines.

src/libespeak-ng/tokenizer.c

@@ -112,6 +112,11 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer)
 	switch (c = text_decoder_getc(tokenizer->decoder))
 	{
 	case '\r':
+		if (text_decoder_peekc(tokenizer->decoder) == '\n') {
+			current += utf8_out(c, current);
+			c = text_decoder_getc(tokenizer->decoder);
+		}
+		// fallthrough
 	case '\n':
 		current += utf8_out(c, current);
 		*current = '\0';

tests/tokenizer.c

@@ -233,6 +233,33 @@ test_mac_newline_tokens()
 	destroy_tokenizer(tokenizer);
 }
 
+void
+test_windows_newline_tokens()
+{
+	printf("testing windows newline tokens\n");
+
+	espeak_ng_TOKENIZER *tokenizer = create_tokenizer();
+	espeak_ng_TEXT_DECODER *decoder = create_text_decoder();
+
+	assert(text_decoder_decode_string(decoder, "\r\n\r\n", -1, ESPEAKNG_ENCODING_US_ASCII) == ENS_OK);
+	assert(tokenizer_reset(tokenizer, decoder) == 1);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(strcmp(tokenizer_get_token_text(tokenizer), "\r\n") == 0);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(strcmp(tokenizer_get_token_text(tokenizer), "\r\n") == 0);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(*tokenizer_get_token_text(tokenizer) == '\0');
+
+	destroy_text_decoder(decoder);
+	destroy_tokenizer(tokenizer);
+}
+
 int
 main(int argc, char **argv)
 {
@@ -251,6 +278,7 @@ main(int argc, char **argv)
 	test_unbound_tokenizer();
 	test_linux_newline_tokens();
 	test_mac_newline_tokens();
+	test_windows_newline_tokens();
 
 	printf("done\n");