Fixed #139

17 files changed, 432 insertions, 58 deletions

diff --git a/Changelog b/Changelog
index c69e1bc..23ed824 100644
--- a/Changelog
+++ b/Changelog
@@ -5,6 +5,7 @@ ejdb (1.2.8) UNRELEASED; urgency=low
   * Fix: $rename can operate on nested json objects #107
   * Fix: $inc doesn't create key if it doesn't exist #120
   * Source code style fixes
+  * A data format version info now stored in the database meta header #139
 
  -- Anton Adamansky <adamansky@gmail.com>  Mon, 27 Apr 2015 21:30:11 +0600
 
diff --git a/ejdb.project b/ejdb.project
index 3c92eca..b16105c 100644
--- a/ejdb.project
+++ b/ejdb.project
@@ -206,7 +206,7 @@
         <LibraryPath Value="Debug"/>
       </Linker>
       <ResourceCompiler Options="" Required="no"/>
-      <General OutputFile="" IntermediateDirectory="./Debug" Command="$(ProjectPath)/build/src/ejdb/tests/ejdbtest2" CommandArguments="" UseSeparateDebugArgs="no" DebugArguments="" WorkingDirectory="$(ProjectPath)/build/src/ejdb/tests" PauseExecWhenProcTerminates="yes" IsGUIProgram="no" IsEnabled="yes"/>
+      <General OutputFile="" IntermediateDirectory="./Debug" Command="$(ProjectPath)/build/src/ejdb/tests/ejdbtest4" CommandArguments="" UseSeparateDebugArgs="no" DebugArguments="" WorkingDirectory="$(ProjectPath)/build/src/ejdb/tests" PauseExecWhenProcTerminates="yes" IsGUIProgram="no" IsEnabled="yes"/>
       <Environment EnvVarSetName="&lt;Use Defaults&gt;" DbgSetName="&lt;Use Defaults&gt;">
         <![CDATA[]]>
       </Environment>
diff --git a/src/bson/bson.c b/src/bson/bson.c
index 7d22a7e..d16512f 100644
--- a/src/bson/bson.c
+++ b/src/bson/bson.c
@@ -52,6 +52,11 @@ const int initialBufferSize = 128;
 /* only need one of these */
 static const int zero = 0;
 
+extern void *(*bson_malloc_func)(size_t);
+extern void *(*bson_realloc_func)(void *, size_t);
+extern void ( *bson_free_func)(void *);
+extern bson_printf_func bson_errprintf;
+
 /* Custom standard function pointers. */
 void *(*bson_malloc_func)(size_t) = MYMALLOC;
 void *(*bson_realloc_func)(void *, size_t) = MYREALLOC;
@@ -64,18 +69,20 @@ bson_printf_func bson_errprintf = _bson_errprintf;
 static int ( *oid_fuzz_func)(void) = NULL;
 static int ( *oid_inc_func)(void) = NULL;
 
-const char* bson_first_errormsg(bson *bson) {
-    if (bson->errstr) {
-        return bson->errstr;
+const char* bson_first_errormsg(bson *b) {
+    if (b->errstr) {
+        return b->errstr;
     }
-    if (bson->err & BSON_FIELD_HAS_DOT) {
+    if (b->err & BSON_FIELD_HAS_DOT) {
         return "BSON key contains '.' character";
-    } else if (bson->err & BSON_FIELD_INIT_DOLLAR) {
+    } else if (b->err & BSON_FIELD_INIT_DOLLAR) {
         return "BSON key starts with '$' character";
-    } else if (bson->err & BSON_ALREADY_FINISHED) {
+    } else if (b->err & BSON_ALREADY_FINISHED) {
         return "Trying to modify a finished BSON object";
-    } else if (bson->err & BSON_NOT_UTF8) {
+    } else if (b->err & BSON_NOT_UTF8) {
         return "A key or a string is not valid UTF-8";
+    } else if (b->err & BSON_NOT_FINISHED) {
+        return "BSON not finished";
     }
     return "Unspecified BSON error";
 }
@@ -2793,3 +2800,44 @@ finish:
     return out;
 }
 
+
+typedef struct {
+    bson *bs;
+    bool checkdots; 
+    bool checkdollar;
+} _BSONVALIDATECTX;
+
+static bson_visitor_cmd_t _bson_validate_visitor(
+        const char *ipath, int ipathlen, 
+        const char *key, int keylen,
+        const bson_iterator *it, 
+        bool after, void *op) {
+    _BSONVALIDATECTX *ctx = op;
+    assert(ctx);
+    if (bson_check_field_name(ctx->bs, key, keylen, 
+                              ctx->checkdots, ctx->checkdollar) == BSON_ERROR) {
+        return BSON_VCMD_TERMINATE;
+    }
+    return (BSON_VCMD_OK | BSON_VCMD_SKIP_AFTER);
+}
+
+
+int bson_validate(bson *bs, bool checkdots, bool checkdollar) {
+    if (!bs) {
+        return BSON_ERROR;
+    }
+    if (!bs->finished) {
+        bs->err |= BSON_NOT_FINISHED;
+        return BSON_ERROR;
+    }
+    bson_iterator it;
+    bson_iterator_init(&it, bs);
+    _BSONVALIDATECTX ctx = {
+        .bs = bs,
+        .checkdots = checkdots,
+        .checkdollar = checkdollar
+    };
+    bson_visit_fields(&it, BSON_TRAVERSE_ARRAYS_EXCLUDED, _bson_validate_visitor, &ctx);
+    return bs->err ? BSON_ERROR : BSON_OK;
+}
+
diff --git a/src/bson/bson.h b/src/bson/bson.h
index 58bc096..f7403f7 100644
--- a/src/bson/bson.h
+++ b/src/bson/bson.h
@@ -53,7 +53,8 @@ enum bson_validity_t {
     BSON_FIELD_HAS_DOT = (1 << 2), /**< Warning: key contains '.' character. */
     BSON_FIELD_INIT_DOLLAR = (1 << 3), /**< Warning: key starts with '$' character. */
     BSON_ALREADY_FINISHED = (1 << 4), /**< Trying to modify a finished BSON object. */
-    BSON_ERROR_ANY = (1 << 5) /**< Unspecified error */
+    BSON_ERROR_ANY = (1 << 5), /**< Unspecified error */
+    BSON_NOT_FINISHED = (1 << 6) /**< BSON object not finished */
 };
 
 enum bson_binary_subtype_t {
@@ -147,7 +148,8 @@ EJDB_EXPORT const char* bson_first_errormsg(bson *bson);
     (_bs_I)->cur = (_bs)->data + 4; \
     (_bs_I)->first = 1;
 
-/* ----------------------------
+
+/* --------------------------------
    READING
    ------------------------------ */
 
@@ -1018,12 +1020,6 @@ EJDB_EXPORT int bson_numstrn(char *str, int maxbuf, int64_t i);
 typedef void( *bson_err_handler)(const char *errmsg);
 typedef int (*bson_printf_func)(const char *, ...);
 
-extern void *(*bson_malloc_func)(size_t);
-extern void *(*bson_realloc_func)(void *, size_t);
-extern void ( *bson_free_func)(void *);
-
-extern bson_printf_func bson_errprintf;
-
 void bson_free(void *ptr);
 
 /**
@@ -1267,5 +1263,17 @@ EJDB_EXPORT int bson2json(const char *bsdata, char **buf, int *sp);
 EJDB_EXPORT bson* json2bson(const char *jsonstr);
 
 
+/**
+ * @brief Validate bson object.
+ * Set the bs->err bitmask as validation result.
+ * 
+ * @param bs Bson object to be validated.
+ * @param checkdots Check what keys contain dot(.) characters
+ * @param checkdollar Check what keys contain dollar($) characters
+ * @return BSON_OK if all checks passed otherwise return BSON_ERROR
+ */
+EJDB_EXPORT int bson_validate(bson *bs, bool checkdots, bool checkdollar);
+
+
 EJDB_EXTERN_C_END
 #endif
diff --git a/src/bson/encoding.c b/src/bson/encoding.c
index 4bf405f..62b9ef1 100644
--- a/src/bson/encoding.c
+++ b/src/bson/encoding.c
@@ -104,7 +104,6 @@ static int isLegalUTF8(const unsigned char *source, int length) {
 /* If the name is part of a db ref ($ref, $db, or $id), then return true. */
 static int bson_string_is_db_ref(const unsigned char *string, const int length) {
     int result = 0;
-
     if (length >= 4) {
         if (string[1] == 'r' && string[2] == 'e' && string[3] == 'f')
             result = 1;
@@ -114,7 +113,6 @@ static int bson_string_is_db_ref(const unsigned char *string, const int length)
         else if (string[1] == 'd' && string[2] == 'b')
             result = 1;
     }
-
     return result;
 }
 
@@ -134,7 +132,6 @@ static int bson_validate_string(bson *b, const unsigned char *string,
         if (check_dot && *(string + position) == '.') {
             b->err |= BSON_FIELD_HAS_DOT;
         }
-
         if (check_utf8) {
             sequence_length = trailingBytesForUTF8[*(string + position)] + 1;
             if ((position + sequence_length) > length) {
@@ -153,13 +150,11 @@ static int bson_validate_string(bson *b, const unsigned char *string,
 }
 
 int bson_check_string(bson *b, const char *string,
-        const int length) {
-
+                      const int length) {
     return bson_validate_string(b, (const unsigned char *) string, length, 1, 0, 0);
 }
 
 int bson_check_field_name(bson *b, const char *string,
-        const int length, int check_dot, int check_dollar) {
-
+                          const int length, int check_dot, int check_dollar) {
     return bson_validate_string(b, (const unsigned char *) string, length, 1, check_dot, check_dollar);
 }
diff --git a/src/ejdb/ejdb.c b/src/ejdb/ejdb.c
index 58ffaad..ec8e7ef 100644
--- a/src/ejdb/ejdb.c
+++ b/src/ejdb/ejdb.c
@@ -30,7 +30,7 @@
 #define JBCUNLOCKMETHOD(JB_col)                         \
     ((JB_col)->mmtx ? _ejcollunlockmethod(JB_col) : true)
 
-#define JBISOPEN(JB_jb) ((JB_jb) && (JB_jb)->metadb && (JB_jb)->metadb->open) ? true : false
+#define JBISOPEN(JB_jb) (((JB_jb) && (JB_jb)->metadb && (JB_jb)->metadb->open) ? true : false)
 
 #define JBISVALCOLNAME(JB_cname) ((JB_cname) && \
                                   strlen(JB_cname) < JBMAXCOLNAMELEN && \
@@ -170,6 +170,31 @@ const char *ejdbversion() {
     return tcversion;
 }
 
+uint32_t ejdbformatversion(EJDB *jb) {
+    return JBISOPEN(jb) ? jb->fversion : 0;
+}
+
+uint8_t ejdbformatversionmajor(EJDB *jb) {
+   return (JBISOPEN(jb) && jb->fversion) ? jb->fversion / 100000 : 0; 
+}
+
+uint16_t ejdbformatversionminor(EJDB *jb) {
+    if (!JBISOPEN(jb) || !jb->fversion) {
+        return 0;
+    }
+    int major = jb->fversion / 100000;
+    return (jb->fversion - major * 100000) / 1000;
+}
+
+uint16_t ejdbformatversionpatch(EJDB *jb) {
+   if (!JBISOPEN(jb) || !jb->fversion) {
+        return 0;
+    }
+    int major = jb->fversion / 100000;
+    int minor = (jb->fversion - major * 100000) / 1000;
+    return (jb->fversion - major * 100000 - minor * 1000);
+}
+
 const char* ejdberrmsg(int ecode) {
     if (ecode > -6 && ecode < 0) { //Hook for negative error codes of utf8proc library
         return utf8proc_errmsg(ecode);
@@ -239,6 +264,7 @@ EJDB* ejdbnew(void) {
     EJDB *jb;
     TCCALLOC(jb, 1, sizeof (*jb));
     jb->metadb = tctdbnew();
+    jb->fversion = 0;
     tctdbsetmutex(jb->metadb);
     tctdbsetcache(jb->metadb, 1024, 0, 0);
     if (!_ejdbsetmutex(jb)) {
@@ -259,6 +285,7 @@ void ejdbdel(EJDB *jb) {
         jb->cdbs[i] = NULL;
     }
     jb->cdbsnum = 0;
+    jb->fversion = 0;
     if (jb->mmtx) {
         pthread_rwlock_destroy(jb->mmtx);
         TCFREE(jb->mmtx);
@@ -281,6 +308,7 @@ bool ejdbclose(EJDB *jb) {
     if (!tctdbclose(jb->metadb)) {
         rv = false;
     }
+    jb->fversion = 0;
     JBUNLOCKMETHOD(jb);
     return rv;
 }
@@ -301,21 +329,57 @@ bool ejdbopen(EJDB *jb, const char *path, int mode) {
     if (!rv) {
         goto finish;
     }
-    jb->cdbsnum = 0;
+    
     TCTDB *mdb = jb->metadb;
-    rv = tctdbiterinit(mdb);
-    if (!rv) {
+    char *colname = NULL;
+    uint64_t mbuf;
+    jb->cdbsnum = 0;
+    
+    if (!(rv = tctdbiterinit(mdb))) {
         goto finish;
     }
-    char *colname = NULL;
-    for (int i = 0; i < mdb->hdb->rnum && (colname = tctdbiternext2(mdb)) != NULL; ++i) {
-        EJCOLL *cdb;
-        EJCOLLOPTS opts;
-        _metagetopts(jb, colname, &opts);
-        _addcoldb0(colname, jb, &opts, &cdb);
+    //check ejdb format version
+    if (tctdbreadopaque(mdb, &mbuf, 0, sizeof(mbuf)) != sizeof(mbuf)) {
+        rv = false;
+        _ejdbsetecode(jb, JBEMETANVALID, __FILE__, __LINE__, __func__);
+        goto finish;
+    }
+    for (int i = 0; rv && i < mdb->hdb->rnum && (colname = tctdbiternext2(mdb)) != NULL; ++i) {
+        if ((rv = tcisvalidutf8str(colname, strlen(colname)))) {
+            EJCOLL *cdb;
+            EJCOLLOPTS opts;
+            if ((rv = _metagetopts(jb, colname, &opts))) {
+                rv = _addcoldb0(colname, jb, &opts, &cdb);
+            }
+        } else {
+            _ejdbsetecode(jb, JBEMETANVALID, __FILE__, __LINE__, __func__);
+        }
         TCFREE(colname);
     }
+    
 finish:
+    if (rv) {
+        mbuf = TCITOHLL(mbuf);
+        uint16_t magic = (uint16_t) mbuf;
+        jb->fversion = (uint32_t) (mbuf >> 16);
+        
+        if (!mbuf && (mode & (JBOWRITER | JBOTRUNC))) { //write ejdb format info opaque data
+            magic = EJDB_MAGIC;
+            jb->fversion = 100000 * EJDB_VERSION_MAJOR + 1000 * EJDB_VERSION_MINOR + EJDB_VERSION_PATCH;
+            mbuf |= jb->fversion;
+            mbuf = (mbuf << 16) | ((uint64_t) magic & 0xffff);
+            mbuf = TCHTOILL(mbuf);
+            if (tctdbwriteopaque(mdb, &mbuf, 0, sizeof(mbuf)) != sizeof(mbuf)) {
+                rv = false;
+                _ejdbsetecode(jb, TCEWRITE, __FILE__, __LINE__, __func__);
+            } else {
+                tctdbsync(jb->metadb);
+            }
+        } else if (magic && (magic != EJDB_MAGIC)) {
+            rv = false;
+           _ejdbsetecode(jb, JBEMETANVALID, __FILE__, __LINE__, __func__); 
+        }
+    }
     JBUNLOCKMETHOD(jb);
     return rv;
 }
@@ -710,7 +774,10 @@ bool ejdbsyncoll(EJCOLL *coll) {
 bool ejdbsyncdb(EJDB *jb) {
     assert(jb);
     JBENSUREOPENLOCK(jb, true, false);
-    bool rv = true;
+    bool rv = tctdbsync(jb->metadb);
+    if (!rv) {
+        return rv;
+    }
     for (int i = 0; i < jb->cdbsnum; ++i) {
         assert(jb->cdbs[i]);
         rv = JBCLOCKMETHOD(jb->cdbs[i], true);
@@ -4549,12 +4616,16 @@ static bool _metasetopts(EJDB *jb, const char *colname, EJCOLLOPTS *opts) {
 
 static bool _metagetopts(EJDB *jb, const char *colname, EJCOLLOPTS *opts) {
     assert(opts);
-    bool rv = true;
     memset(opts, 0, sizeof (*opts));
     bson *bsopts = _metagetbson(jb, colname, strlen(colname), "opts");
     if (!bsopts) {
         return true;
     }
+    if (bson_validate(bsopts, true, true) != BSON_OK) {
+        _ejdbsetecode(jb, JBEMETANVALID, __FILE__, __LINE__, __func__);
+        bson_del(bsopts);
+        return false;
+    }
     bson_iterator it;
     bson_type bt = bson_find(&it, bsopts, "compressed");
     if (bt == BSON_BOOL) {
@@ -4573,7 +4644,7 @@ static bool _metagetopts(EJDB *jb, const char *colname, EJCOLLOPTS *opts) {
         opts->records = bson_iterator_long(&it);
     }
     bson_del(bsopts);
-    return rv;
+    return true;
 }
 
 static bool _metasetbson(EJDB *jb, const char *colname, int colnamesz,
@@ -5702,18 +5773,16 @@ static void _delcoldb(EJCOLL *coll) {
 
 static bool _addcoldb0(const char *cname, EJDB *jb, EJCOLLOPTS *opts, EJCOLL **res) {
     int i;
-    bool rv = true;
     TCTDB *cdb;
-
+    
     for (i = 0; i < EJDB_MAX_COLLECTIONS && jb->cdbs[i]; ++i);
     if (i == EJDB_MAX_COLLECTIONS) {
         _ejdbsetecode(jb, JBEMAXNUMCOLS, __FILE__, __LINE__, __func__);
         return false;
     }
-    rv = _createcoldb(cname, jb, opts, &cdb);
-    if (!rv) {
+    if (!_createcoldb(cname, jb, opts, &cdb)) {
         *res = NULL;
-        return rv;
+        return false;
     }
     EJCOLL *coll;
     TCCALLOC(coll, 1, sizeof (*coll));
@@ -5724,9 +5793,11 @@ static bool _addcoldb0(const char *cname, EJDB *jb, EJCOLLOPTS *opts, EJCOLL **r
     coll->tdb = cdb;
     coll->jb = jb;
     coll->mmtx = NULL;
-    _ejdbcolsetmutex(coll);
+    if (!_ejdbcolsetmutex(coll)) {
+        return false;
+    }
     *res = coll;
-    return rv;
+    return true;
 }
 
 static bool _createcoldb(const char *colname, EJDB *jb, EJCOLLOPTS *opts, TCTDB **res) {
diff --git a/src/ejdb/ejdb.h b/src/ejdb/ejdb.h
index f61150d..c805979 100644
--- a/src/ejdb/ejdb.h
+++ b/src/ejdb/ejdb.h
@@ -97,12 +97,52 @@ enum { /*< Query search mode flags in ejdbqryexecute() */
 };
 
 /**
- * Returns EJDB library version string. Eg: "1.1.13"
+ * Return EJDB library version string. Eg: "1.1.13"
  */
 EJDB_EXPORT const char *ejdbversion();
 
 /**
- * Return true if passed `oid` string cat be converted to valid
+ * Return EJDB database format version.
+ * 
+ * Format version number uses the following convention:
+ *  100000 * major + 1000 * minor + patch.
+ * 
+ * Return `0` 
+ *  - Database is not opened
+ *  - Database was created by libejdb < v1.2.8 and opened in read-only mode.
+ */
+EJDB_EXPORT uint32_t ejdbformatversion(EJDB *jb);
+
+/**
+ * Return EJDB database `major` format version.
+ * 
+ * Return `0` 
+ *  - Database is not opened
+ *  - Database was created by libejdb < v1.2.8 and opened in read-only mode.
+ */
+EJDB_EXPORT uint8_t ejdbformatversionmajor(EJDB *jb);
+
+/**
+ * Return EJDB database `minor` format version.
+ * 
+ * Return `0` 
+ *  - Database is not opened
+ *  - Database was created by libejdb < v1.2.8 and opened in read-only mode.
+ */
+EJDB_EXPORT uint16_t ejdbformatversionminor(EJDB *jb);
+
+/**
+ * Return EJDB database `patch` format version.
+ * 
+ * Return `0` 
+ *  - Database is not opened
+ *  - Database was created by libejdb < v1.2.8 and opened in read-only mode.
+ */
+EJDB_EXPORT uint16_t ejdbformatversionpatch(EJDB *jb);
+
+
+/**
+ * Return true if a passed `oid` string cat be converted to valid
  * 12 bit BSON object identifier (OID).
  * @param oid String
  */
diff --git a/src/ejdb/ejdb_private.h b/src/ejdb/ejdb_private.h
index 4bb03ff..482fa59 100644
--- a/src/ejdb/ejdb_private.h
+++ b/src/ejdb/ejdb_private.h
@@ -31,6 +31,9 @@ EJDB_EXTERN_C_START
                                           atype == BSON_ARRAY || atype == BSON_DATE)
 
 #define EJDB_MAX_COLLECTIONS 1024
+#define EJDB_MAGIC 0xEBB1
+#define EJDB_MAGIC_SZ 2;    //number of bytes to encode magic in TCTDB opaque data
+#define EJDB_VERSION_SZ 4;  //number of bytes to encode version in TCTDB opaque data 
 
 
 struct EJCOLL { /**> EJDB Collection. */
@@ -44,6 +47,7 @@ struct EJCOLL { /**> EJDB Collection. */
 struct EJDB {
     EJCOLL * cdbs[EJDB_MAX_COLLECTIONS]; /*> Collection DBs for JSON collections. */
     int cdbsnum; /*> Count of collection DB. */
+    uint32_t fversion; /*> Database format version */
     TCTDB *metadb; /*> Metadata DB. */
     void *mmtx; /*> Mutex for method */
 };
diff --git a/src/ejdb/tests/ejdbtest1.c b/src/ejdb/tests/ejdbtest1.c
index 77da154..53caa41 100644
--- a/src/ejdb/tests/ejdbtest1.c
+++ b/src/ejdb/tests/ejdbtest1.c
@@ -31,6 +31,38 @@ int clean_suite(void) {
     return 0;
 }
 
+void testVersionMeta() {
+    EJDB *vjb = ejdbnew();
+    bool rv = ejdbopen(vjb, "dbt1meta", JBOWRITER | JBOCREAT | JBOTRUNC);
+    CU_ASSERT_TRUE_FATAL(rv);
+    rv = ejdbclose(vjb);
+    CU_ASSERT_TRUE_FATAL(rv);
+    ejdbdel(vjb);
+    
+    vjb = ejdbnew();
+    rv = ejdbopen(vjb, "dbt1meta", JBOREADER);
+    CU_ASSERT_TRUE_FATAL(rv);
+    uint32_t fversion = ejdbformatversion(vjb);
+    CU_ASSERT_NOT_EQUAL(fversion, 0);
+    
+    //jb->fversion = 100000 * EJDB_VERSION_MAJOR + 1000 * EJDB_VERSION_MINOR + EJDB_VERSION_PATCH;
+    int major = fversion / 100000;
+    CU_ASSERT_EQUAL(major, EJDB_VERSION_MAJOR);
+    CU_ASSERT_EQUAL(major, ejdbformatversionmajor(vjb));
+    
+    int minor = (fversion - major * 100000) / 1000;
+    CU_ASSERT_EQUAL(minor, EJDB_VERSION_MINOR);
+    CU_ASSERT_EQUAL(minor, ejdbformatversionminor(vjb));
+    
+    int patch = (fversion - major * 100000 - minor * 1000);
+    CU_ASSERT_EQUAL(patch, EJDB_VERSION_PATCH);
+    CU_ASSERT_EQUAL(patch, ejdbformatversionpatch(vjb));
+    
+    rv = ejdbclose(vjb);
+    CU_ASSERT_TRUE_FATAL(rv);
+    ejdbdel(vjb);
+}
+
 void testTicket102() {
     const char *json = "[0, 1, 2]";
     bson *ret = json2bson(json);
@@ -281,7 +313,8 @@ int main() {
     }
 
     /* Add the tests to the suite */
-    if ((NULL == CU_add_test(pSuite, "testSaveLoad", testSaveLoad)) ||
+    if (   (NULL == CU_add_test(pSuite, "testVersionMeta", testVersionMeta)) ||
+            (NULL == CU_add_test(pSuite, "testSaveLoad", testSaveLoad)) ||
             (NULL == CU_add_test(pSuite, "testBuildQuery1", testBuildQuery1)) ||
             (NULL == CU_add_test(pSuite, "testDBOptions", testDBOptions)) ||
             (NULL == CU_add_test(pSuite, "testTicket102", testTicket102)) 
diff --git a/src/ejdb/tests/ejdbtest4.c b/src/ejdb/tests/ejdbtest4.c
index 5018fc3..2336eec 100644
--- a/src/ejdb/tests/ejdbtest4.c
+++ b/src/ejdb/tests/ejdbtest4.c
@@ -349,6 +349,17 @@ void testTicket135(void) {
 }
 
 
+// We are trying to open TCHDB which is not actuall ejdb database
+void testOpenOtherTCHDB(void) {
+    EJDB *jb = ejdbnew();
+    bool ret = ejdbopen(jb, "dbt4_export_col1", JBOREADER);
+    CU_ASSERT_FALSE(ret);
+    int ecode = ejdbecode(jb);
+    CU_ASSERT_EQUAL(ecode, JBEMETANVALID);
+    ejdbclose(jb);
+    ejdbdel(jb);
+}
+
 int init_suite(void) {
     return 0;
 }
@@ -377,7 +388,8 @@ int main() {
             (NULL == CU_add_test(pSuite, "testTicket53", testTicket53)) ||
             (NULL == CU_add_test(pSuite, "testBSONExportImport", testBSONExportImport)) ||
             (NULL == CU_add_test(pSuite, "testBSONExportImport2", testBSONExportImport2)) ||
-            (NULL == CU_add_test(pSuite, "testTicket135", testTicket135))
+            (NULL == CU_add_test(pSuite, "testTicket135", testTicket135)) ||
+            (NULL == CU_add_test(pSuite, "testOpenOtherTCHDB", testOpenOtherTCHDB))
             ) {
         CU_cleanup_registry();
         return CU_get_error();
diff --git a/src/tcbdb/tcbdb.c b/src/tcbdb/tcbdb.c
index 7163087..17e9e0b 100644
--- a/src/tcbdb/tcbdb.c
+++ b/src/tcbdb/tcbdb.c
@@ -3338,7 +3338,7 @@ static bool tcbdboptimizeimpl(TCBDB *bdb, int32_t lmemb, int32_t nmemb,
     tbdb->lcnum = BDBLEVELMAX;
     tbdb->ncnum = BDBCACHEOUT * 2;
     if (!tcbdbopen(tbdb, tpath, BDBOWRITER | BDBOCREAT | BDBOTRUNC) ||
-            !tchdbcopyopaque(tbdb->hdb, bdb->hdb, 0, -1)) {
+         tchdbcopyopaque(tbdb->hdb, bdb->hdb, 0, -1) < 0) {
         tcbdbdel(tbdb);
         TCFREE(tpath);
         TCFREE(opath);
diff --git a/src/tchdb/tchdb.c b/src/tchdb/tchdb.c
index c63c125..8ea46fb 100644
--- a/src/tchdb/tchdb.c
+++ b/src/tchdb/tchdb.c
@@ -1653,7 +1653,11 @@ uint8_t tchdbopts(TCHDB *hdb) {
     return hdb->opts;
 }
 
-bool tchdbcopyopaque(TCHDB *dst, TCHDB *src, int off, int rsz) {
+size_t tchdbmaxopaquesz() {
+    return HDBOPAQUESZ;
+}
+
+int tchdbcopyopaque(TCHDB *dst, TCHDB *src, int off, int rsz) {
     assert(dst && src);
     if (rsz == -1) {
         rsz = HDBOPAQUESZ;
@@ -1662,7 +1666,7 @@ bool tchdbcopyopaque(TCHDB *dst, TCHDB *src, int off, int rsz) {
     rsz = tchdbreadopaque(src, odata, off, rsz);
     if (rsz == -1) {
         tchdbsetecode(dst, tchdbecode(src), __FILE__, __LINE__, __func__);
-        return false;
+        return -1;
     }
     return (tchdbwriteopaque(dst, odata, off, rsz) == rsz);
 }
@@ -5066,7 +5070,7 @@ static bool tchdboptimizeimpl(TCHDB *hdb, int64_t bnum, int8_t apow, int8_t fpow
     if (opts == UINT8_MAX) opts = hdb->opts;
     tchdbtune(thdb, bnum, apow, fpow, opts);
     if (!tchdbopen(thdb, tpath, HDBOWRITER | HDBOCREAT | HDBOTRUNC) ||
-            !tchdbcopyopaque(thdb, hdb, 0, HDBOPAQUESZ)) {
+         tchdbcopyopaque(thdb, hdb, 0, HDBOPAQUESZ) < 0) {
         tchdbdel(thdb);
         TCFREE(tpath);
         return false;
diff --git a/src/tchdb/tchdb.h b/src/tchdb/tchdb.h
index 157ce85..0d90e1c 100644
--- a/src/tchdb/tchdb.h
+++ b/src/tchdb/tchdb.h
@@ -748,13 +748,18 @@ EJDB_EXPORT uint8_t tchdbflags(TCHDB *hdb);
    The return value is the options. */
 EJDB_EXPORT uint8_t tchdbopts(TCHDB *hdb);
 
+/**
+ * @brief 
+ * Return a maximum size of opaque data can be stored.
+ */
+EJDB_EXPORT size_t tchdbmaxopaquesz();
 
 /**
  * Get opaque data into specified buffer `dst`
  * `bsiz` Max size to be read.
  *  Return -1 if error, otherwise number of bytes writen in dst.
  */
-int tchdbreadopaque(TCHDB *hdb, void *dst, int off, int bsiz);
+EJDB_EXPORT int tchdbreadopaque(TCHDB *hdb, void *dst, int off, int bsiz);
 
 /**
  * Write opaque data.
@@ -762,12 +767,13 @@ int tchdbreadopaque(TCHDB *hdb, void *dst, int off, int bsiz);
  * can be truncated if it greater than max opaque data size.
  * Return -1 if error, otherwise number of bytes read from src.
  */
-int tchdbwriteopaque(TCHDB *hdb, const void *src, int off, int nb);
+EJDB_EXPORT int tchdbwriteopaque(TCHDB *hdb, const void *src, int off, int nb);
 
 /**
- * Copy opaque data between databases
+ * Copy opaque data between databases.
+ * Return -1 if error, otherwise number of bytes copied.
  */
-bool tchdbcopyopaque(TCHDB *dst, TCHDB *src, int off, int nb);
+EJDB_EXPORT int tchdbcopyopaque(TCHDB *dst, TCHDB *src, int off, int nb);
 
 /* Get the number of used elements of the bucket array of a hash database object.
    `hdb' specifies the hash database object.
diff --git a/src/tctdb/tctdb.c b/src/tctdb/tctdb.c
index 19cb838..d02afbd 100644
--- a/src/tctdb/tctdb.c
+++ b/src/tctdb/tctdb.c
@@ -21,8 +21,9 @@
 #include "tctdb.h"
 #include "myconf.h"
 
-#define TDBOPAQUESIZ   64                // size of using opaque field
-#define TDBLEFTOPQSIZ  64                // size of left opaque field
+#define TDBOPAQUESIZ   16                // size of using opaque field
+#define TDBLEFTOPQSIZ  112               // size of left opaque field
+
 #define TDBPAGEBUFSIZ  32768             // size of a buffer to read each page
 
 #define TDBDEFAPOW     4                 // default alignment power
@@ -1804,6 +1805,42 @@ int tctdbmetastrtosettype(const char *str) {
     return type;
 }
 
+size_t tctdbmaxopaquesz() {
+    return TDBLEFTOPQSIZ;
+}
+
+int tctdbreadopaque(TCTDB *tdb, void *dst, int off, int bsiz) {
+    assert(tdb && dst); 
+    if (bsiz == -1) {
+        bsiz = TDBLEFTOPQSIZ;
+    }
+    return tchdbreadopaque(tdb->hdb, dst, TDBOPAQUESIZ + off, bsiz);    
+}
+
+int tctdbwriteopaque(TCTDB *tdb, const void *src, int off, int nb) {
+    assert(tdb && src); 
+    if (off < 0) {
+        return -1;
+    }
+    if (nb == -1) {
+        nb = TDBLEFTOPQSIZ;
+    }
+    return tchdbwriteopaque(tdb->hdb, src, TDBOPAQUESIZ + off, nb);
+}
+
+
+int tctdbcopyopaque(TCTDB *dst, TCTDB *src, int off, int nb) {
+    assert(dst && dst->hdb);
+    assert(src && src->hdb);
+    if (off < 0) {
+        return -1;
+    }
+    if (nb == -1) {
+        nb = TDBLEFTOPQSIZ;
+    }
+    return tchdbcopyopaque(dst->hdb, src->hdb, TDBOPAQUESIZ + off, nb);
+}
+
 
 
 /*************************************************************************************************
@@ -2209,7 +2246,10 @@ static bool tctdboptimizeimpl(TCTDB *tdb, int64_t bnum, int8_t apow, int8_t fpow
     if (opts & TDBTTCBS) hopts |= HDBTTCBS;
     if (opts & TDBTEXCODEC) hopts |= HDBTEXCODEC;
     tchdbtune(thdb, bnum, apow, fpow, hopts);
-    if (tchdbopen(thdb, tpath, HDBOWRITER | HDBOCREAT | HDBOTRUNC) && tchdbcopyopaque(thdb, hdb, 0, -1)) {
+    
+    if (tchdbopen(thdb, tpath, HDBOWRITER | HDBOCREAT | HDBOTRUNC) && 
+        tchdbcopyopaque(thdb, hdb, 0, -1) >= 0) {
+            
         if (!tchdbiterinit(hdb)) err = true;
         TCXSTR *kxstr = tcxstrnew();
         TCXSTR *vxstr = tcxstrnew();
diff --git a/src/tctdb/tctdb.h b/src/tctdb/tctdb.h
index 82cee8f..ee29a28 100644
--- a/src/tctdb/tctdb.h
+++ b/src/tctdb/tctdb.h
@@ -1094,6 +1094,33 @@ EJDB_EXPORT int tctdbqrystrtoordertype(const char *str);
    The return value is the set operation type or -1 on failure. */
 EJDB_EXPORT int tctdbmetastrtosettype(const char *str);
 
+/**
+ * @brief 
+ * Return a maximum size of opaque data can be stored.
+ */
+EJDB_EXPORT size_t tctdbmaxopaquesz();
+
+/**
+ * Get opaque data into specified buffer `dst`
+ * `bsiz` Max size to be read.
+ *  Return -1 if error, otherwise number of bytes writen in dst.
+ */
+EJDB_EXPORT int tctdbreadopaque(TCTDB *tdb, void *dst, int off, int bsiz);
+
+/**
+ * Write opaque data.
+ * Number of bytes specified bt `nb`
+ * can be truncated if it greater than max opaque data size.
+ * Return -1 if error, otherwise number of bytes read from src.
+ */
+EJDB_EXPORT int tctdbwriteopaque(TCTDB *tdb, const void *src, int off, int nb);
+
+/**
+ * Copy opaque data between databases.
+ * Return -1 if error, otherwise number of bytes copied.
+ */
+EJDB_EXPORT int tctdbcopyopaque(TCTDB *dst, TCTDB *src, int off, int nb);
+
 
 /* Add a record into indices of a table database object.
    `tdb' specifies the table database object.
diff --git a/src/tcutil/tcutil.c b/src/tcutil/tcutil.c
index 22e49ce..1f7bf44 100644
--- a/src/tcutil/tcutil.c
+++ b/src/tcutil/tcutil.c
@@ -4825,6 +4825,83 @@ char *tcstrsqzspc(char *str) {
     return str;
 }
 
+/*
+ * Index into the table below with the first byte of a UTF-8 sequence to
+ * get the number of trailing bytes that are supposed to follow it.
+ */
+static const char trailingBytesForUTF8[256] = {
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
+};
+
+static bool isvalidutf8seq(const unsigned char *seq, int length) {
+    unsigned char a;
+    const unsigned char *srcptr = seq + length;
+    switch (length) {
+        default:
+            return false;
+            /* Everything else falls through when "true"... */
+        case 4:
+            if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+        case 3:
+            if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+        case 2:
+            if ((a = (*--srcptr)) > 0xBF) return false;
+            switch (*seq) {
+                /* no fall-through in this inner switch */
+                case 0xE0:
+                    if (a < 0xA0) return false;
+                    break;
+                case 0xF0:
+                    if (a < 0x90) return false;
+                    break;
+                case 0xF4:
+                    if (a > 0x8F) return false;
+                    break;
+                default:
+                    if (a < 0x80) return false;
+            }
+        case 1:
+            if (*seq >= 0x80 && *seq < 0xC2) return false;
+            if (*seq > 0xF4) return false;
+    }
+    return true;
+}
+
+/* UTF8 string validation. */
+bool tcisvalidutf8str(const char *str, int len) {
+    if (!str || len < 1) {
+        return false;
+    }
+    int pos = 0;
+    int slen = 1;
+    for (; pos < len; ++pos) {
+        if (str[pos] == '\0' && pos < len - 1) {
+            return false;
+        }
+    }
+    pos = 0;
+    const unsigned char *ustr = (const unsigned char *) str;
+    while (pos < len) {
+        slen = trailingBytesForUTF8[*(ustr + pos)] + 1;
+        if ((pos + slen) > len) {
+            return false;
+        }
+        if (!isvalidutf8seq(ustr + pos, slen)) {
+            return false;
+        }
+        pos += slen;
+    }
+    return true;
+}
+
+
 /* Substitute characters in a string. */
 char *tcstrsubchr(char *str, const char *rstr, const char *sstr) {
     assert(str && rstr && sstr);
diff --git a/src/tcutil/tcutil.h b/src/tcutil/tcutil.h
index 8965289..f38fc15 100644
--- a/src/tcutil/tcutil.h
+++ b/src/tcutil/tcutil.h
@@ -2390,6 +2390,14 @@ EJDB_EXPORT char *tcstrtrim(char *str);
    The return value is the string itself. */
 EJDB_EXPORT char *tcstrsqzspc(char *str);
 
+/*
+ * UTF8 string validation. 
+ * 
+ * Return `true` if a given character buffer 
+ * represents a valid UTF8 string.
+ */
+EJDB_EXPORT bool tcisvalidutf8str(const char *str, int len);
+
 
 /* Substitute characters in a string.
    `str' specifies the string to be converted.