Added EncodedInputStream, AutoUTFInputStream, AutoUTF

git-svn-id: https://rapidjson.googlecode.com/svn/trunk@40 c5894555-1306-4e8d-425f-1f6f381ee07c

bin/encodings/utf16bebom.json

 {

bin/encodings/utf16lebom.json

 {

bin/encodings/utf8.json

+{
+	"en":"I can eat glass and it doesn't hurt me.",
+	"zh-Hant":"我能吞下玻璃而不傷身體。",
+	"zh-Hans":"我能吞下玻璃而不伤身体。",
+	"ja":"私はガラスを食べられます。それは私を傷つけません。",
+	"ko":"나는 유리를 먹을 수 있어요. 그래도 아프지 않아요"
+}
\ No newline at end of file

bin/encodings/utf8bom.json

+{
+	"en":"I can eat glass and it doesn't hurt me.",
+	"zh-Hant":"我能吞下玻璃而不傷身體。",
+	"zh-Hans":"我能吞下玻璃而不伤身体。",
+	"ja":"私はガラスを食べられます。それは私を傷つけません。",
+	"ko":"나는 유리를 먹을 수 있어요. 그래도 아프지 않아요"
+}
\ No newline at end of file

example/pretty/pretty.cpp

@@ -19,7 +19,7 @@ int main(int argc, char* argv[]) {
 	PrettyWriter<FileWriteStream> writer(os);
 
 	// JSON reader parse from the input stream and let writer generate the output.
-	if (!reader.Parse<0>(is, writer)) {
+	if (!reader.Parse<kParseValidateEncodingFlag>(is, writer)) {
 		fprintf(stderr, "\nError(%u): %s\n", (unsigned)reader.GetErrorOffset(), reader.GetParseError());
 		return 1;
 	}

include/rapidjson/encodedstream.h

+#ifndef RAPIDJSON_ENCODEDSTREAM_H_
+#define RAPIDJSON_ENCODEDSTREAM_H_
+
+#include "rapidjson.h"
+
+namespace rapidjson {
+
+//! Adapts an input byte stream with an specified encoding.
+template <typename Encoding, typename InputStream>
+class EncodedInputStream {
+public:
+	typedef typename Encoding::Ch Ch;
+
+	EncodedInputStream(InputStream& is) : is_(is) { 
+		Encoding::TakeBOM(is_);
+		Read();
+	}
+
+	Ch Peek() const { return current_; }
+	Ch Take() { Ch c = current_; Read(); return c; }
+	size_t Tell() const { is_.Tell(); }
+
+	// Not implemented
+	void Put(Ch c) { RAPIDJSON_ASSERT(false); }
+	void Flush() { RAPIDJSON_ASSERT(false); } 
+	Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
+	size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; }
+
+private:
+	void Read() {
+		current_ = Encoding::Take(is_);
+	}
+	InputStream& is_;
+	Ch current_;
+};
+
+template <typename CharType, typename InputStream>
+class AutoUTFInputStream {
+public:
+	typedef CharType Ch;
+
+	AutoUTFInputStream(InputStream& is, UTFType type = kUTF8) : is_(is), type_(type) {
+		TakeBOM(is);
+		Read();
+	}
+
+	Ch Peek() const { return current_; }
+	Ch Take() { Ch c = current_; Read(); return c; }
+	size_t Tell() const { is_.Tell(); }
+
+	// Not implemented
+	void Put(Ch c) { RAPIDJSON_ASSERT(false); }
+	void Flush() { RAPIDJSON_ASSERT(false); } 
+	Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
+	size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; }
+
+private:
+	friend struct AutoUTF<Ch>;
+
+	void TakeBOM(InputStream& is) {
+#define TAKE() is.Take()
+#define PEEK(x) if ((unsigned char)is.Peek() != x) break
+		switch ((unsigned char)is.Peek()) {
+		case 0x00: TAKE(); PEEK(0x00); TAKE(); PEEK(0xFE); TAKE(); PEEK(0xFF); type_ = kUTF32BE; return;
+		case 0xEF: TAKE(); PEEK(0xBB); TAKE(); PEEK(0xBF); TAKE(); type_ = kUTF8; return;
+		case 0xFE: TAKE(); PEEK(0xFF); TAKE(); type_ = kUTF16BE; return;
+		case 0xFF: TAKE(); PEEK(0xFE); TAKE();
+			if (is.Peek() == 0x00) {
+				TAKE(); PEEK(0x00); TAKE(); type_ = kUTF32LE; return;
+			}
+			type_ = kUTF16LE;
+			return;
+		}
+#undef TAKE
+#undef PEEK
+	}
+
+	void Read() {
+		typedef Ch (*TakeFunc)(InputStream& is);
+		static const TakeFunc f[] = {
+			UTF8<Ch>::Take,
+			UTF16LE<Ch>::Take,
+			UTF16BE<Ch>::Take,
+			UTF32LE<Ch>::Take,
+			UTF32BE<Ch>::Take,
+		};
+
+		current_ = f[type_](is_);
+	}
+
+	InputStream& is_;
+	UTFType type_;
+	Ch current_;
+};
+
+} // namespace rapidjson
+
+#endif // RAPIDJSON_FILESTREAM_H_

include/rapidjson/reader.h

@@ -5,6 +5,7 @@
 // Version 0.1
 
 #include "rapidjson.h"
+#include "encodings.h"
 #include "internal/pow10.h"
 #include "internal/stack.h"
 #include <csetjmp>

test/perftest/perftest.h

@@ -6,7 +6,7 @@
 #define TEST_YAJL		0
 #define TEST_ULTRAJSON  0
 #define TEST_PLATFORM   0
-#define TEST_MISC		1
+#define TEST_MISC		0
 
 #if TEST_RAPIDJSON
 //#define RAPIDJSON_SSE2

test/unittest/filestreamtest.cpp

@@ -2,6 +2,7 @@
 #include "rapidjson/filestream.h"
 #include "rapidjson/filereadstream.h"
 #include "rapidjson/filewritestream.h"
+#include "rapidjson/encodedstream.h"
 
 using namespace rapidjson;
 
@@ -31,22 +32,23 @@ protected:
 	size_t length_;
 };
 
-TEST_F(FileStreamTest, FileStream_Read) {
-	FILE *fp = fopen(filename_, "rb");
-	ASSERT_TRUE(fp != 0);
-	FileStream s(fp);
-
-	for (size_t i = 0; i < length_; i++) {
-		EXPECT_EQ(json_[i], s.Peek());
-		EXPECT_EQ(json_[i], s.Peek());	// 2nd time should be the same
-		EXPECT_EQ(json_[i], s.Take());
-	}
-
-	EXPECT_EQ(length_, s.Tell());
-	EXPECT_EQ('\0', s.Peek());
-
-	fclose(fp);
-}
+// Depreciated
+//TEST_F(FileStreamTest, FileStream_Read) {
+//	FILE *fp = fopen(filename_, "rb");
+//	ASSERT_TRUE(fp != 0);
+//	FileStream s(fp);
+//
+//	for (size_t i = 0; i < length_; i++) {
+//		EXPECT_EQ(json_[i], s.Peek());
+//		EXPECT_EQ(json_[i], s.Peek());	// 2nd time should be the same
+//		EXPECT_EQ(json_[i], s.Take());
+//	}
+//
+//	EXPECT_EQ(length_, s.Tell());
+//	EXPECT_EQ('\0', s.Peek());
+//
+//	fclose(fp);
+//}
 
 TEST_F(FileStreamTest, FileReadStream) {
 	FILE *fp = fopen(filename_, "rb");