Index support for Freeze dictionaries

6 files changed, 1052 insertions, 84 deletions

diff --git a/cpp/src/Freeze/IndexI.cpp b/cpp/src/Freeze/IndexI.cpp
index 0bef0002e7a..df1137a4269 100644
--- a/cpp/src/Freeze/IndexI.cpp
+++ b/cpp/src/Freeze/IndexI.cpp
@@ -308,13 +308,8 @@ Freeze::IndexI::secondaryKeyCreate(Db* secondary, const Dbt* dbKey,
 {
     Ice::CommunicatorPtr communicator = _store->communicator();
 
-    Ice::Identity ident;
-    Byte* first = static_cast<Byte*>(dbKey->get_data());
-    Key key(first, first + dbKey->get_size());
-    ObjectStore::unmarshal(ident, key, communicator);
-
     ObjectRecord rec;
-    first = static_cast<Byte*>(dbValue->get_data());
+    Byte* first = static_cast<Byte*>(dbValue->get_data());
     Value value(first, first + dbValue->get_size());
     ObjectStore::unmarshal(rec, value, communicator);
 
diff --git a/cpp/src/Freeze/MapI.cpp b/cpp/src/Freeze/MapI.cpp
index 7e39199f794..5470f27d9e0 100644
--- a/cpp/src/Freeze/MapI.cpp
+++ b/cpp/src/Freeze/MapI.cpp
@@ -19,16 +19,50 @@ using namespace Freeze;
 
 
 //
+// MapIndexBase (from Map.h)
+//
+
+Freeze::MapIndexBase::~MapIndexBase()
+{
+}
+
+Freeze::MapIndexBase::MapIndexBase(const string& name) :
+    _name(name),
+    _impl(0),
+    _map(0)
+{
+}
+
+const string&
+MapIndexBase::name() const
+{
+    return _name;
+}
+
+IteratorHelper*
+Freeze::MapIndexBase::untypedFind(const Key& k, bool ro) const
+{
+    return _impl->untypedFind(k, ro, *_map);
+}
+
+int 
+Freeze::MapIndexBase::untypedCount(const Key& k) const
+{
+    return _impl->untypedCount(k, _map->connection());
+}
+
+//
 // MapHelper (from Map.h)
 //
 
 Freeze::MapHelper*
 Freeze::MapHelper::create(const Freeze::ConnectionPtr& connection, 
 			  const string& dbName, 
+			  const std::vector<MapIndexBasePtr>& indices,
 			  bool createDb)
 {
     Freeze::ConnectionIPtr connectionI = Freeze::ConnectionIPtr::dynamicCast(connection);
-    return new MapHelperI(connectionI, dbName, createDb);
+    return new MapHelperI(connectionI, dbName, indices, createDb);
 }
 
 Freeze::MapHelper::~MapHelper()
@@ -45,8 +79,8 @@ Freeze::IteratorHelper::create(const MapHelper& m, bool readOnly)
 {
     const MapHelperI& actualMap = dynamic_cast<const MapHelperI&>(m);
 
-    auto_ptr<IteratorHelperI> r(new IteratorHelperI(actualMap, readOnly));
-    if(r->findFirst())
+    auto_ptr<IteratorHelperI> r(new IteratorHelperI(actualMap, readOnly, 0));
+    if(r->next())
     {
 	return r.release();
     }
@@ -62,21 +96,25 @@ Freeze::IteratorHelper::~IteratorHelper()
 }
 
 
-
 //
 // IteratorHelperI
 //
 
-
-Freeze::IteratorHelperI::IteratorHelperI(const MapHelperI& m, bool readOnly) :
+Freeze::IteratorHelperI::IteratorHelperI(const MapHelperI& m, bool readOnly, 
+					 const MapIndexBasePtr& index) :
     _map(m),
     _dbc(0),
+    _indexed(index != 0),
     _tx(0)
 {
     if(_map._trace >= 2)
     {
 	Trace out(_map._connection->communicator()->getLogger(), "Freeze.Map");
 	out << "opening iterator on Db \"" << _map._dbName << "\"";
+	if(index != 0)
+	{
+	    out << " with index \"" << index->name() << "\"";
+	}
     }
 
     DbTxn* txn = _map._connection->dbTxn();
@@ -92,7 +130,14 @@ Freeze::IteratorHelperI::IteratorHelperI(const MapHelperI& m, bool readOnly) :
 
     try
     {
-	_map._db->cursor(txn, &_dbc, 0);
+	if(index != 0)
+	{
+	    index->_impl->db()->cursor(txn, &_dbc, 0);
+	}
+	else
+	{
+	    _map._db->cursor(txn, &_dbc, 0);
+	}
     }
     catch(const ::DbException& dx)
     {
@@ -100,12 +145,15 @@ Freeze::IteratorHelperI::IteratorHelperI(const MapHelperI& m, bool readOnly) :
 	ex.message = dx.what();
 	throw ex;
     }
+
     _map._iteratorList.push_back(this);
 }
 
+
 Freeze::IteratorHelperI::IteratorHelperI(const IteratorHelperI& it) :
     _map(it._map),
     _dbc(0),
+    _indexed(it._indexed),
     _tx(0)
 {
     if(_map._trace >= 2)
@@ -134,11 +182,6 @@ Freeze::IteratorHelperI::~IteratorHelperI()
     close();
 }
 
-bool 
-Freeze::IteratorHelperI::findFirst() const
-{
-    return next();
-}
 
 bool 
 Freeze::IteratorHelperI::find(const Key& key) const
@@ -217,7 +260,22 @@ Freeze::IteratorHelperI::get(const Key*& key, const Value*& value) const
     {
 	try
 	{
-	    int err = _dbc->get(&dbKey, &dbValue, DB_CURRENT);
+	    int err;
+
+	    if(_indexed)
+	    {
+		//
+		// Not interested in getting the index's key
+		//
+		Dbt iKey;
+		iKey.set_flags(DB_DBT_USERMEM | DB_DBT_PARTIAL);
+	    
+		err = _dbc->pget(&iKey, &dbKey, &dbValue, DB_CURRENT);
+	    }
+	    else
+	    {
+		err = _dbc->get(&dbKey, &dbValue, DB_CURRENT);
+	    }
 
 	    if(err == 0)
 	    {
@@ -314,7 +372,21 @@ Freeze::IteratorHelperI::get() const
     {
 	try
 	{
-	    int err = _dbc->get(&dbKey, &dbValue, DB_CURRENT);
+	    int err;
+	    if(_indexed)
+	    {
+		//
+		// Not interested in getting the index's key
+		//
+		Dbt iKey;
+		iKey.set_flags(DB_DBT_USERMEM | DB_DBT_PARTIAL);
+	    
+		err = _dbc->pget(&iKey, &dbKey, &dbValue, DB_CURRENT);
+	    }
+	    else
+	    {
+		err = _dbc->get(&dbKey, &dbValue, DB_CURRENT);
+	    }
 
 	    if(err == 0)
 	    {
@@ -377,6 +449,13 @@ Freeze::IteratorHelperI::get() const
 void 
 Freeze::IteratorHelperI::set(const Value& value)
 {
+    if(_indexed)
+    {
+	DatabaseException ex(__FILE__, __LINE__);
+	ex.message = "Cannot set an iterator retrieved through an index";
+	throw ex;
+    }
+    
     //
     // key ignored
     //
@@ -463,9 +542,11 @@ Freeze::IteratorHelperI::next() const
     Dbt dbValue;
     dbValue.set_flags(DB_DBT_USERMEM | DB_DBT_PARTIAL);
 
+    int flags = _indexed ? DB_NEXT_DUP : DB_NEXT;
+
     try
     {
-	if(_dbc->get(&dbKey, &dbValue, DB_NEXT) == 0)
+	if(_dbc->get(&dbKey, &dbValue, flags) == 0)
 	{
 	    return true;
 	}
@@ -493,30 +574,6 @@ Freeze::IteratorHelperI::next() const
     }
 }
 
-bool
-Freeze::IteratorHelperI::equals(const IteratorHelper& rhs) const
-{
-    if(this == &rhs)
-    {
-	return true;
-    }
-    else
-    {
-	//
-	// Compare keys
-	//
-	try
-	{
-	    Key rhsKey = *dynamic_cast<const IteratorHelperI&>(rhs).get();
-	    return *get() == rhsKey;
-	}
-	catch(const InvalidPositionException&)
-	{
-	    return false;
-	}
-    }
-}
-
 void
 Freeze::IteratorHelperI::close()
 {
@@ -654,12 +711,26 @@ Freeze::IteratorHelperI::Tx::dead()
 
 Freeze::MapHelperI::MapHelperI(const ConnectionIPtr& connection, 
 			       const std::string& dbName, 
+			       const vector<MapIndexBasePtr>& indices,
 			       bool createDb) :
     _connection(connection),
-    _db(SharedDb::get(connection, dbName, createDb)),
+    _db(SharedDb::get(connection, dbName, indices, createDb)),
     _dbName(dbName),
     _trace(connection->trace())
 { 
+    for(vector<MapIndexBasePtr>::const_iterator p = indices.begin(); 
+	p != indices.end(); ++p)
+    {
+	const MapIndexBasePtr& indexBase = *p;
+	assert(indexBase->_impl != 0);
+	assert(indexBase->_map == 0);
+	bool inserted = 
+	    _indices.insert(IndexMap::value_type(indexBase->name(), indexBase)).second;
+	assert(inserted);
+	indexBase->_map = this;
+	indexBase->_communicator = _connection->communicator();
+    }
+    
     _connection->registerMap(this);
 }
 
@@ -675,7 +746,7 @@ Freeze::MapHelperI::find(const Key& k, bool readOnly) const
     {
 	try
 	{  
-	    auto_ptr<IteratorHelperI> r(new IteratorHelperI(*this, readOnly));
+	    auto_ptr<IteratorHelperI> r(new IteratorHelperI(*this, readOnly, 0));
 	    if(r->find(k))
 	    {
 		return r.release();
@@ -995,6 +1066,19 @@ Freeze::MapHelperI::closeAllIterators()
     }
 }
 
+const MapIndexBasePtr&
+Freeze::MapHelperI::index(const string& name) const
+{
+    IndexMap::const_iterator p = _indices.find(name);
+    if(p == _indices.end())
+    {
+	DatabaseException ex(__FILE__, __LINE__);
+	ex.message = "Cannot find index \"" + name + "\"";
+	throw ex;
+    }
+    return p->second;
+}
+
 void
 Freeze::MapHelperI::close()
 {
@@ -1003,6 +1087,15 @@ Freeze::MapHelperI::close()
 	_connection->unregisterMap(this);
     }
     _db = 0;
+    
+    for(IndexMap::iterator p = _indices.begin(); p != _indices.end(); ++p)
+    {
+	MapIndexBasePtr& indexBase = p->second;
+
+	indexBase->_impl = 0;
+	indexBase->_map = 0;
+    }
+    _indices.clear();
 }
 
 void
@@ -1026,6 +1119,183 @@ Freeze::MapHelperI::closeAllIteratorsExcept(const IteratorHelperI::TxPtr& tx) co
     }
 }
 
+
+//
+// MapIndexI
+//
+
+static int 
+callback(Db* secondary, const Dbt* key, const Dbt* value, Dbt* result)
+{
+    void* indexObj = secondary->get_app_private();
+    MapIndexI* index = static_cast<MapIndexI*>(indexObj);
+    assert(index != 0);
+    return index->secondaryKeyCreate(secondary, key, value, result);
+}
+
+
+Freeze::MapIndexI::MapIndexI(const ConnectionIPtr& connection, SharedDb& db,
+			     DbTxn* txn, bool createDb, const MapIndexBasePtr& index) :
+    _index(index)
+{
+    assert(txn != 0);
+    
+    _db.reset(new Db(connection->dbEnv(), 0));
+    _db->set_flags(DB_DUP | DB_DUPSORT);
+    _db->set_app_private(this);
+
+    u_int32_t flags = 0;
+    if(createDb)
+    {
+	flags = DB_CREATE;
+    }
+
+    _dbName = db.dbName() + "." + _index->name();
+
+    _db->open(txn, _dbName.c_str(), 0, DB_BTREE, flags, FREEZE_DB_MODE);
+
+    //
+    // To populate empty indices
+    //
+    flags = DB_CREATE;
+    db.associate(txn, _db.get(), callback, flags);
+
+    //
+    // Note: caller catch and translates exceptions
+    //
+}
+   
+Freeze::MapIndexI::~MapIndexI()
+{
+    _db->close(0);
+}
+
+IteratorHelper* 
+Freeze::MapIndexI::untypedFind(const Key& k, bool ro, const MapHelperI& map) const
+{
+    auto_ptr<IteratorHelperI> r(new IteratorHelperI(map, ro, _index));
+
+    if(r->find(k))
+    {
+	return r.release();
+    }
+    else
+    {
+	return 0;
+    }
+}
+
+int 
+Freeze::MapIndexI::untypedCount(const Key& k, const ConnectionIPtr& connection) const
+{
+    Dbt dbKey;
+    initializeInDbt(k, dbKey);
+    
+    Dbt dbValue;
+    dbValue.set_flags(DB_DBT_USERMEM | DB_DBT_PARTIAL);
+
+    int result = 0;
+    
+    try
+    {
+	for(;;)
+	{
+	    Dbc* dbc = 0;
+	    
+	    try
+	    {
+		//
+		// Move to the first record
+		// 
+		_db->cursor(0, &dbc, 0);
+		bool found = (dbc->get(&dbKey, &dbValue, DB_SET) == 0);
+		
+		if(found)
+		{
+		    db_recno_t count = 0;
+		    dbc->count(&count, 0);
+		    result = static_cast<int>(count);
+		}
+		
+		Dbc* toClose = dbc;
+		dbc = 0;
+		toClose->close();
+		break; // for (;;)
+	    }
+	    catch(const DbDeadlockException&)
+	    {
+		if(dbc != 0)
+		{
+		    try
+		    {
+			dbc->close();
+		    }
+		    catch(const DbDeadlockException&)
+		    {
+			//
+			// Ignored
+			//
+		    }
+		}
+
+		if(connection->deadlockWarning())
+		{
+		    Warning out(connection->communicator()->getLogger());
+		    out << "Deadlock in Freeze::MapIndexI::untypedCount while searching \"" 
+			<< _dbName << "\"; retrying ...";
+		}
+
+		//
+		// Retry
+		//
+	    }
+	    catch(...)
+	    {
+		if(dbc != 0)
+		{
+		    try
+		    {
+			dbc->close();
+		    }
+		    catch(const DbDeadlockException&)
+		    {
+			//
+			// Ignored
+			//
+		    }
+		}
+		throw;
+	    }
+	}
+    }
+    catch(const DbException& dx)
+    {
+	DatabaseException ex(__FILE__, __LINE__);
+	ex.message = dx.what();
+	throw ex;
+    }
+    
+    return result;
+}
+    
+int
+Freeze::MapIndexI::secondaryKeyCreate(Db* secondary, const Dbt* dbKey, 
+				      const Dbt* dbValue, Dbt* result)
+{
+    Byte* first = static_cast<Byte*>(dbValue->get_data());
+    Value value(first, first + dbValue->get_size());
+    
+    Key bytes;
+    _index->marshalKey(value, bytes);
+
+    result->set_flags(DB_DBT_APPMALLOC);
+    void* data = malloc(bytes.size());
+    memcpy(data, &bytes[0], bytes.size());
+    result->set_data(data);
+    result->set_size(static_cast<u_int32_t>(bytes.size()));
+    return 0;
+}
+
 //
 // Print for the various exception types.
 //
diff --git a/cpp/src/Freeze/MapI.h b/cpp/src/Freeze/MapI.h
index 2bb28cf1b24..6c60c1f55e9 100644
--- a/cpp/src/Freeze/MapI.h
+++ b/cpp/src/Freeze/MapI.h
@@ -23,23 +23,19 @@ class IteratorHelperI : public IteratorHelper
 {  
 public:
 
-    IteratorHelperI(const MapHelperI& m, bool readOnly);
-    
+    IteratorHelperI(const MapHelperI& m, bool readOnly, const MapIndexBasePtr& index);
     IteratorHelperI(const IteratorHelperI&);
 
     virtual 
     ~IteratorHelperI();
-
-    bool
-    findFirst() const;
-
+  
     bool 
     find(const Key& k) const;
 
     virtual IteratorHelper*
     clone() const;
     
-    const Key*
+    virtual const Key*
     get() const;
 
     virtual void
@@ -54,9 +50,6 @@ public:
     virtual bool
     next() const;
 
-    virtual bool
-    equals(const IteratorHelper&) const;
-
     void
     close();
 
@@ -90,9 +83,9 @@ private:
     void
     cleanup();
 
-
     const MapHelperI& _map;
     Dbc* _dbc;
+    const bool _indexed;
     TxPtr _tx;
 
     mutable Key _key;
@@ -105,7 +98,7 @@ class MapHelperI : public MapHelper
 public:
    
     MapHelperI(const ConnectionIPtr& connection, const std::string& dbName, 
-		 bool createDb);
+	       const std::vector<MapIndexBasePtr>&, bool createDb);
 
     virtual ~MapHelperI();
 
@@ -132,16 +125,26 @@ public:
 
     virtual void
     closeAllIterators();
+
+    virtual const MapIndexBasePtr&
+    index(const std::string&) const;
  
     void
     close();
 
+    const ConnectionIPtr& connection() const
+    {
+	return _connection;
+    }
+
+
+    typedef std::map<std::string, MapIndexBasePtr> IndexMap;
+    
 private:
 
     virtual void
     closeAllIteratorsExcept(const IteratorHelperI::TxPtr&) const;
 
-
     friend class IteratorHelperI;
     friend class IteratorHelperI::Tx;
 
@@ -149,9 +152,12 @@ private:
     mutable std::list<IteratorHelperI*> _iteratorList;
     SharedDbPtr _db;
     const std::string _dbName;
+    IndexMap _indices;
+
     Ice::Int _trace;    
 };
 
+
 inline const IteratorHelperI::TxPtr&
 IteratorHelperI::tx() const
 {
diff --git a/cpp/src/Freeze/SharedDb.cpp b/cpp/src/Freeze/SharedDb.cpp
index 62c0e0cd6e3..4cb75641f89 100644
--- a/cpp/src/Freeze/SharedDb.cpp
+++ b/cpp/src/Freeze/SharedDb.cpp
@@ -28,7 +28,9 @@ Freeze::SharedDb::Map* Freeze::SharedDb::sharedDbMap = 0;
 
 Freeze::SharedDbPtr 
 Freeze::SharedDb::get(const ConnectionIPtr& connection, 
-		      const string& dbName, bool createDb)
+		      const string& dbName,
+		      const vector<MapIndexBasePtr>& indices,
+		      bool createDb)
 {
     StaticMutex::Lock lock(_mapMutex);
 
@@ -46,6 +48,7 @@ Freeze::SharedDb::get(const ConnectionIPtr& connection,
 	Map::iterator p = sharedDbMap->find(key);
 	if(p != sharedDbMap->end())
 	{
+	    p->second->connectIndices(indices);
 	    return p->second;
 	}
     }
@@ -53,7 +56,7 @@ Freeze::SharedDb::get(const ConnectionIPtr& connection,
     //
     // MapKey not found, let's create and open a new Db
     //
-    auto_ptr<SharedDb> result(new SharedDb(key, connection, createDb));
+    auto_ptr<SharedDb> result(new SharedDb(key, connection, indices, createDb));
     
     //
     // Insert it into the map
@@ -73,17 +76,7 @@ Freeze::SharedDb::~SharedDb()
 	out << "closing Db \"" << _key.dbName << "\"";
     }
 
-    try
-    {
-	close(0);
-    }
-    catch(const ::DbException& dx)
-    {
-	DatabaseException ex(__FILE__, __LINE__);
-	ex.message = dx.what();
-	throw ex;
-    }
-
+    cleanup(false);
 }
 
 void Freeze::SharedDb::__incRef()
@@ -145,10 +138,10 @@ void Freeze::SharedDb::__decRef()
     }
 }
 
-
   
 Freeze::SharedDb::SharedDb(const MapKey& key, 
 			   const ConnectionIPtr& connection, 
+			   const vector<MapIndexBasePtr>& indices,
 			   bool createDb) :
     Db(connection->dbEnv(), 0),
     _key(key),
@@ -161,19 +154,148 @@ Freeze::SharedDb::SharedDb(const MapKey& key,
 	out << "opening Db \"" << _key.dbName << "\"";
     }
 
+    DbTxn* txn = 0;
+    DbEnv* dbEnv = connection->dbEnv();
+    
     try
     {
-	u_int32_t flags = DB_AUTO_COMMIT | DB_THREAD;
+	dbEnv->txn_begin(0, &txn, 0);
+
+	u_int32_t flags = DB_THREAD;
 	if(createDb)
 	{
 	    flags |= DB_CREATE;
 	}
-	open(0, key.dbName.c_str(), 0, DB_BTREE, flags, FREEZE_DB_MODE);
+	open(txn, key.dbName.c_str(), 0, DB_BTREE, flags, FREEZE_DB_MODE);
+
+	for(vector<MapIndexBasePtr>::const_iterator p = indices.begin();
+	    p != indices.end(); ++p)
+	{
+	    const MapIndexBasePtr& indexBase = *p; 
+		
+	    if(indexBase->_impl != 0)
+	    {
+		DatabaseException ex(__FILE__, __LINE__);
+		ex.message = "Index \"" + indexBase->name() + "\" already initialized!";
+		throw ex;
+	    }
+
+	    auto_ptr<MapIndexI> indexI(new MapIndexI(connection, *this, txn, createDb, indexBase));
+	    
+	    bool inserted = _indices.insert(IndexMap::value_type(indexBase->name(), indexI.get())).second;
+	    if(!inserted)
+	    {
+		DatabaseException ex(__FILE__, __LINE__);
+		ex.message = "Index \"" + indexBase->name() + "\" listed twice!";
+		throw ex;
+	    }
+	    
+	    indexBase->_impl = indexI.release();
+	}
+
+	DbTxn* toCommit = txn;
+	txn = 0;
+	toCommit->commit(0);
     }
     catch(const ::DbException& dx)
     {
+	if(txn != 0)
+	{
+	    try
+	    {
+		txn->abort();
+	    }
+	    catch(...)
+	    {
+	    }
+	}
+
+	cleanup(true);
+
+	if(dx.get_errno() == ENOENT)
+	{
+	    NotFoundException ex(__FILE__, __LINE__);
+	    ex.message = dx.what();
+	    throw ex;
+	}
+	else
+	{
+	    DatabaseException ex(__FILE__, __LINE__);
+	    ex.message = dx.what();
+	    throw ex;
+	}
+
 	DatabaseException ex(__FILE__, __LINE__);
 	ex.message = dx.what();
 	throw ex;
     }
+    catch(...)
+    {
+	if(txn != 0)
+	{
+	    try
+	    {
+		txn->abort();
+	    }
+	    catch(...)
+	    {
+	    }
+	}
+
+	cleanup(true);
+	throw;
+    }
+}
+
+void
+Freeze::SharedDb::connectIndices(const vector<MapIndexBasePtr>& indices) const
+{
+    for(vector<MapIndexBasePtr>::const_iterator p = indices.begin();
+	p != indices.end(); ++p)
+    {
+	const MapIndexBasePtr& indexBase = *p; 
+
+	if(indexBase->_impl != 0)
+	{
+	    DatabaseException ex(__FILE__, __LINE__);
+	    ex.message = "Index \"" + indexBase->name() + "\" already initialized!";
+	    throw ex;
+	}
+
+	IndexMap::const_iterator q = _indices.find(indexBase->name());
+	
+	if(q == _indices.end())
+	{
+	    DatabaseException ex(__FILE__, __LINE__);
+	    ex.message = "\"" + _key.dbName + "\" already opened but without index \"" 
+		+ indexBase->name() +"\"";
+	    throw ex;
+	}
+	
+	indexBase->_impl = q->second;
+    }
+}
+
+void
+Freeze::SharedDb::cleanup(bool noThrow)
+{
+    try
+    {
+	for(IndexMap::iterator p = _indices.begin(); p != _indices.end(); ++p)
+	{
+	    delete p->second;
+	}
+	_indices.clear();
+	
+	close(0);
+    }
+    catch(const ::DbException& dx)
+    {
+	if(!noThrow)
+	{
+	    DatabaseException ex(__FILE__, __LINE__);
+	    ex.message = dx.what();
+	    throw ex;
+	}
+    }
 }
diff --git a/cpp/src/Freeze/SharedDb.h b/cpp/src/Freeze/SharedDb.h
index 9aa2b9a776e..406916541d7 100644
--- a/cpp/src/Freeze/SharedDb.h
+++ b/cpp/src/Freeze/SharedDb.h
@@ -13,6 +13,7 @@
 #include <IceUtil/Config.h>
 #include <db_cxx.h>
 #include <Freeze/ConnectionI.h>
+#include <Freeze/Map.h>
 #include <IceUtil/Handle.h>
 #include <map>
 
@@ -22,13 +23,47 @@ namespace Freeze
 class SharedDb;
 typedef IceUtil::Handle<SharedDb> SharedDbPtr;
 
+class MapIndexI
+{
+public:
+    
+    MapIndexI(const ConnectionIPtr&, SharedDb&,
+	      DbTxn*, bool, const MapIndexBasePtr&);
+
+    ~MapIndexI();
+   
+    IteratorHelper* untypedFind(const Key&, bool, const MapHelperI&) const;
+    int untypedCount(const Key&, const ConnectionIPtr&) const;
+    
+    int
+    secondaryKeyCreate(Db*, const Dbt*, const Dbt*, Dbt*);
+
+    const std::string name() const
+    {
+	return _index->name();
+    }
+    
+    Db* db() const
+    {
+	return _db.get();
+    }
+
+private:
+
+    const MapIndexBasePtr _index;
+    std::auto_ptr<Db> _db;
+    std::string _dbName;
+};
+
+
 class SharedDb : public ::Db
 {
 public:
     
     using Db::get;
 
-    static SharedDbPtr get(const ConnectionIPtr&, const std::string&, bool);
+    static SharedDbPtr get(const ConnectionIPtr&, const std::string&, 
+			   const std::vector<MapIndexBasePtr>&, bool);
 
     ~SharedDb();
 
@@ -47,6 +82,8 @@ public:
     }
 #endif
 
+    typedef std::map<std::string, MapIndexI*> IndexMap;
+
 private:
 
     struct MapKey
@@ -61,12 +98,18 @@ private:
 
     typedef std::map<MapKey, Freeze::SharedDb*> Map;
    
-    SharedDb(const MapKey&, const ConnectionIPtr&, bool);
+    SharedDb(const MapKey&, const ConnectionIPtr&, 
+	     const std::vector<MapIndexBasePtr>&, bool);
     
+    void connectIndices(const std::vector<MapIndexBasePtr>&) const;
+    void cleanup(bool);
+
     MapKey _key;
     int _refCount;
     Ice::Int _trace;
 
+    IndexMap _indices;
+
     static Map* sharedDbMap;
 };
 
diff --git a/cpp/src/slice2freeze/Main.cpp b/cpp/src/slice2freeze/Main.cpp
index 59baaff2c16..457c062a85e 100644
--- a/cpp/src/slice2freeze/Main.cpp
+++ b/cpp/src/slice2freeze/Main.cpp
@@ -15,11 +15,19 @@ using namespace std;
 using namespace IceUtil;
 using namespace Slice;
 
+struct DictIndex
+{
+    string member;
+    bool caseSensitive;
+};
+
 struct Dict
 {
     string name;
     string key;
     string value;
+  
+    vector<DictIndex> indices;
 };
 
 struct Index
@@ -57,6 +65,14 @@ usage(const char* n)
         "                      different names. NAME may be a scoped name.\n"
         "                      When member is a string, the case can be\n"
         "                      sensitive or insensitive (default is sensitive).\n"
+	"--dict-index DICT[,MEMBER][,{case-sensitive|case-insensitive}] \n"
+	"                      Add an index to dictionary DICT. If MEMBER is \n"
+        "                      specified, then DICT's VALUE must be a class or\n"
+	"                      a struct, and MEMBER must designate a member of\n"
+	"                      VALUE. Otherwise, the entire VALUE is used for \n"
+	"                      indexing. When the secondary key is a string, \n"
+	"                      the case can be sensitive or insensitive (default\n"
+	"                      is sensitive).\n"
         "--output-dir DIR      Create files in the directory DIR.\n"
         "-d, --debug           Print debug messages.\n"
         "--ice                 Permit `Ice' prefix (for building Ice source code only)\n"
@@ -110,7 +126,7 @@ printFreezeTypes(Output& out, const vector<Dict>& dicts, const vector<Index>& in
 void
 writeCodecH(const TypePtr& type, const string& name, const string& freezeType, Output& H, const string& dllExport)
 {
-    H << sp << nl << dllExport << "class " << name;
+    H << sp << nl << "class " << dllExport << name;
     H << sb;
     H.dec();
     H << sp << nl << "public:";
@@ -176,8 +192,335 @@ writeCodecC(const TypePtr& type, const string& name, const string& freezeType, b
     C << eb;
 }
 
+void
+writeDictWithIndicesH(const string& name, const Dict& dict, 
+		      const vector<TypePtr> indexTypes, 
+		      const TypePtr& keyType, const TypePtr& valueType,
+		      Output& H, const string& dllExport)
+{
+    
+    string templateParams = string("< ") + typeToString(keyType) + ", "
+	+ typeToString(valueType) + ", " + name + "KeyCodec, " 
+	+ name + "ValueCodec>";
+    
+    vector<string> capitalizedMembers;
+    size_t i;
+    for(i = 0; i < dict.indices.size(); ++i)
+    {
+	const string& member = dict.indices[i].member;
+	if(!member.empty())
+	{
+	    string capitalizedMember = member;
+	    capitalizedMember[0] = toupper(capitalizedMember[0]);
+	    capitalizedMembers.push_back(capitalizedMember);
+	}
+	else
+	{
+	    capitalizedMembers.push_back("Value");
+	}
+    }
+
+    H << sp << nl << "class " << dllExport << name 
+      << " : public Freeze::Map" << templateParams;
+    H << sb;
+    H.dec();
+    H << sp << nl << "public:";
+    H << sp;
+    H.inc();
+
+    //
+    // Typedefs
+    //
+    H << nl << "typedef std::pair<const " << typeToString(keyType)
+      << ", const" << typeToString(valueType) << "> value_type;";
+
+    H << nl << "typedef Freeze::Iterator" << templateParams << " iterator;";
+    H << nl << "typedef Freeze::ConstIterator" << templateParams << " const_iterator;";
+    H << nl << "typedef size_t size_type;";
+    H << nl << "typedef ptrdiff_t difference_type;";
+    
+    //
+    // Nested index classes
+    //
+
+    for(i = 0; i < capitalizedMembers.size(); ++i)
+    {
+	H << sp << nl << "class " << dllExport << capitalizedMembers[i] << "Index"
+	  << " : public Freeze::MapIndexBase";	
+	H << sb;
+
+	H.dec();
+	H << sp << nl << "public:";
+	H << sp;
+	H.inc();
+	H << nl << capitalizedMembers[i] << "Index(const std::string&);";
+	
+	H << sp;
+	H << nl << "static void writeIndex(" << inputTypeToString(indexTypes[i])
+	  << ", Freeze::Key&, const Ice::CommunicatorPtr&);";
+
+
+	H.dec();
+	H << sp << nl << "protected:";
+	H << sp;
+	H.inc();
+
+	H << nl << "virtual void marshalKey(const Freeze::Value&, Freeze::Key&) const;";
+	
+	H << eb << ';';
+    }
+
+    //
+    // Constructors
+    //
+    H << sp;
+    H << nl << name << "(const Freeze::ConnectionPtr&, const std::string&, bool = true);";
+    H << sp;
+    H << nl << "template <class _InputIterator>"
+      << nl << name << "(const Freeze::ConnectionPtr& __connection, const std::string& __dbName, bool __createDb, "
+      << "_InputIterator __first, _InputIterator __last)";
+    H.inc();
+    H << nl << ": Freeze::Map" << templateParams <<"(__connection->getCommunicator())";
+    H.dec();
+    H << sb;
+    H << nl << "std::vector<Freeze::MapIndexBasePtr> __indices;";
+    for(i = 0; i < capitalizedMembers.size(); ++i)
+    {
+	string indexName = dict.indices[i].member;
+	if(indexName.empty())
+	{
+	    indexName = "index";
+	}
+	indexName = string("\"") + indexName + "\"";
+
+	H << nl << "__indices.push_back(new " << capitalizedMembers[i] << "Index(" << indexName << "));";
+    }
+    H << nl << "_helper.reset(Freeze::MapHelper::create(__connection, __dbName, __indices, __createDb));";
+    H << nl << "while(__first != __last)";
+    H << sb;
+    H << nl << "put(*__first);";
+    H << nl << "++__first;";
+    H << eb;
+    H << eb;
+
+    //
+    // Find and count functions
+    //
+    for(i = 0; i < capitalizedMembers.size(); ++i)
+    {
+	H << sp;
+	H << nl << "iterator findBy" << capitalizedMembers[i]
+	  << "(" << inputTypeToString(indexTypes[i]) << ");";
+	H << nl << "const_iterator findBy" << capitalizedMembers[i]
+	  << "(" << inputTypeToString(indexTypes[i]) << ") const;";
+	
+	string countFunction = dict.indices[i].member.empty() ? "valueCount" 
+	    : dict.indices[i].member + "Count";
+
+	H << nl << "int " << countFunction
+	  << "(" << inputTypeToString(indexTypes[i]) << ") const;";
+    }
+    
+    H << eb << ';';
+}
+
+void
+writeDictWithIndicesC(const string& name, const string& absolute, const Dict& dict, 
+		      const vector<TypePtr> indexTypes, 
+		      const TypePtr& keyType, const TypePtr& valueType,
+		      Output& C)
+{
+    string templateParams = string("< ") + typeToString(keyType) + ", "
+	+ typeToString(valueType) + ", " + name + "KeyCodec, " 
+	+ name + "ValueCodec>";
+    
+    vector<string> capitalizedMembers;
+    size_t i;
+    for(i = 0; i < dict.indices.size(); ++i)
+    {
+	const string& member = dict.indices[i].member;
+	if(!member.empty())
+	{
+	    string capitalizedMember = member;
+	    capitalizedMember[0] = toupper(capitalizedMember[0]);
+	    capitalizedMembers.push_back(capitalizedMember);
+	}
+	else
+	{
+	    capitalizedMembers.push_back("Value");
+	}
+    }
+    
+
+    //
+    // Nested index classes
+    //
+    
+    for(i = 0; i < capitalizedMembers.size(); ++i)
+    {
+	string className = capitalizedMembers[i] + "Index";
+
+	C << sp << nl << absolute << "::" << className << "::" << className
+	  << "(const std::string& __name)";
+
+	C.inc();
+	C << nl << ": Freeze::MapIndexBase(__name)";
+	C.dec();
+	C << sb;
+	C << eb;
+
+	C << sp << nl << "void" 
+	  << nl << absolute << "::" << className << "::" 
+	  << "marshalKey(const Freeze::Value& __v, Freeze::Key& __k) const";
+	C << sb;
+	
+	bool optimize = false;
+
+	if(dict.indices[i].member.empty() && dict.indices[i].caseSensitive)
+	{
+	    optimize = true;
+	    C << nl << "__k = __v;";
+	}
+	else
+	{
+	    //
+	    // Can't optimize
+	    //
+	    C << nl << typeToString(valueType) << " __x;";
+	    C << nl << absolute << "ValueCodec::read(__x, __v, _communicator);";
+	    string param = "__x";
+	    
+	    if(!dict.indices[i].member.empty())
+	    {
+		if(ClassDeclPtr::dynamicCast(valueType) != 0)
+		{
+		    param += "->" + dict.indices[i].member;
+		}
+		else
+		{
+		    param += "." + dict.indices[i].member;
+		}
+	    }
+	    C << nl << "writeIndex(" << param << ", __k, _communicator);";
+	}
+	C << eb;
+	
+	C << sp << nl << "void" 
+	  << nl << absolute << "::" << className << "::" 
+	  << "writeIndex(" << inputTypeToString(indexTypes[i])
+	  << " __index, Freeze::Key& __bytes, const Ice::CommunicatorPtr& __communicator)";
+	C << sb;
+	
+	if(optimize)
+	{
+	    C << nl << absolute << "ValueCodec::write(__index, __bytes, __communicator);";
+	}
+	else
+	{
+	    C << nl << "IceInternal::InstancePtr __instance = IceInternal::getInstance(__communicator);";
+	    C << nl << "IceInternal::BasicStream __stream(__instance.get());";
+	    
+	    string valueS;
+	    if(dict.indices[i].caseSensitive)
+	    {
+		valueS = "__index";
+	    }
+	    else
+	    {
+		C << nl << typeToString(indexTypes[i]) << " __lowerCaseIndex = __index;";
+		C << nl << "std::transform(__lowerCaseIndex.begin(), __lowerCaseIndex.end(), __lowerCaseIndex.begin(), tolower);";
+		valueS = "__lowerCaseIndex";
+	    }
+	    
+	    writeMarshalUnmarshalCode(C, indexTypes[i], valueS, true, "__stream", false);
+	    if(indexTypes[i]->usesClasses())
+	    {
+		C << nl << "__stream.writePendingObjects();";
+	    }
+	    C << nl << "__bytes.swap(__stream.b);";
+	}
+	C << eb;
+    }
+
+
+    //
+    // Constructor
+    //
+    
+    C << sp << nl << absolute << "::" << name << "::" << name
+      << "(const Freeze::ConnectionPtr& __connection, const std::string& __dbName , bool __createDb)";
+    
+    C.inc();
+    C << nl << ": Freeze::Map" << templateParams <<"(__connection->getCommunicator())";
+    C.dec();
+    C << sb;
+    C << nl << "std::vector<Freeze::MapIndexBasePtr> __indices;";
+    for(i = 0; i < capitalizedMembers.size(); ++i)
+    {
+	string indexName = dict.indices[i].member;
+	if(indexName.empty())
+	{
+	    indexName = "index";
+	}
+	indexName = string("\"") + indexName + "\"";
+
+	C << nl << "__indices.push_back(new " << capitalizedMembers[i] << "Index(" << indexName << "));";
+    }
+    C << nl << "_helper.reset(Freeze::MapHelper::create(__connection, __dbName, __indices, __createDb));";
+    C << eb;
+
+    //
+    // Find and count functions
+    //
+    for(i = 0; i < capitalizedMembers.size(); ++i)
+    {	
+	string indexClassName = capitalizedMembers[i] + "Index";
+	
+	string indexName = dict.indices[i].member;
+	if(indexName.empty())
+	{
+	    indexName = "index";
+	}
+	indexName = string("\"") + indexName + "\"";
+
+	C << sp << nl << absolute << "::iterator"
+	  << nl << absolute << "::" << "findBy" << capitalizedMembers[i]
+	  << "(" << inputTypeToString(indexTypes[i]) << " __index)";
+	C << sb;
+	C << nl << "Freeze::Key __bytes;";
+	C << nl << indexClassName << "::" << "writeIndex(__index, __bytes, _communicator);";
+	C << nl << "return iterator(_helper->index(" << indexName 
+	  << ")->untypedFind(__bytes, false), _communicator);";
+	C << eb;
+
+	C << sp << nl << absolute << "::const_iterator"
+	  << nl << absolute << "::" << "findBy" << capitalizedMembers[i]
+	  << "(" << inputTypeToString(indexTypes[i]) << " __index) const";
+	C << sb;
+	C << nl << "Freeze::Key __bytes;";
+	C << nl << indexClassName << "::" << "writeIndex(__index, __bytes, _communicator);";
+	C << nl << "return const_iterator(_helper->index(" << indexName 
+	  << ")->untypedFind(__bytes, true), _communicator);";
+	C << eb;
+
+	string countFunction = dict.indices[i].member.empty() ? "valueCount" 
+	    : dict.indices[i].member + "Count";
+
+	C << sp << nl << "int"
+	  << nl << absolute << "::" << countFunction
+	  << "(" << inputTypeToString(indexTypes[i]) << " __index) const";
+	C << sb;
+	C << nl << "Freeze::Key __bytes;";
+	C << nl << indexClassName << "::" << "writeIndex(__index, __bytes, _communicator);";
+	C << nl << "return _helper->index(" << indexName 
+	  << ")->untypedCount(__bytes);";
+	C << eb;
+    }
+}
+
+
 bool
-writeCodecs(const string& n, UnitPtr& u, const Dict& dict, Output& H, Output& C, const string& dllExport)
+writeDict(const string& n, UnitPtr& u, const Dict& dict, Output& H, Output& C, const string& dllExport)
 {
     string absolute = dict.name;
     if(absolute.find("::") == 0)
@@ -232,8 +575,102 @@ writeCodecs(const string& n, UnitPtr& u, const Dict& dict, Output& H, Output& C,
     writeCodecH(keyType, name + "KeyCodec", "Key", H, dllExport);
     writeCodecH(valueType, name + "ValueCodec", "Value", H, dllExport);
 
-    H << sp << nl << "typedef Freeze::Map< " << typeToString(keyType) << ", " << typeToString(valueType) << ", "
-      << name << "KeyCodec, " << name << "ValueCodec> " << name << ";";
+    vector<TypePtr> indexTypes;
+
+    if(dict.indices.size() == 0)
+    {
+	H << sp << nl << "typedef Freeze::Map< " << typeToString(keyType) << ", " << typeToString(valueType) << ", "
+	  << name << "KeyCodec, " << name << "ValueCodec> " << name << ";";
+    }
+    else
+    {
+	for(vector<DictIndex>::const_iterator p = dict.indices.begin();
+	    p != dict.indices.end(); ++p)
+	{
+	    const DictIndex& index = *p;
+	    if(index.member.empty())
+	    {
+		if(dict.indices.size() > 1)
+		{
+		    cerr << n << ": bad index for dictionary `" << dict.name << "'" << endl;
+		    return false;
+		}
+		
+		if(index.caseSensitive == false)
+		{
+		    //
+		    // Let's check value is a string
+		    //
+		
+		    BuiltinPtr builtInType = BuiltinPtr::dynamicCast(valueType);
+		    
+		    if(builtInType == 0 || builtInType->kind() != Builtin::KindString)
+		    {
+			cerr << n << ": VALUE is a `" << dict.value << "', not a string " << endl;
+			return false; 
+		    }
+		}
+		indexTypes.push_back(valueType);
+	    }
+	    else
+	    {
+		DataMemberPtr dataMember = 0;
+		DataMemberList dataMembers;
+		
+		ClassDeclPtr classDecl = ClassDeclPtr::dynamicCast(valueType);
+		if(classDecl != 0)
+		{
+		    dataMembers = classDecl->definition()->allDataMembers();
+		}
+		else
+		{
+		    StructPtr structDecl = StructPtr::dynamicCast(valueType);
+		    if(structDecl == 0)
+		    {
+			cerr << n << ": `" << dict.value << "' is neither a class nor a struct." << endl;
+			return false;
+		    }
+		    dataMembers = structDecl->dataMembers();
+		}
+		DataMemberList::const_iterator q = dataMembers.begin();
+		while(q != dataMembers.end() && dataMember == 0)
+		{
+		    if((*q)->name() == index.member)
+		    {
+			dataMember = *q;
+		    }
+		    else
+		    {
+			++q;
+		    }
+		}
+		
+		if(dataMember == 0)
+		{
+		    cerr << n << ": The value of `" << dict.name << "' has no data member named `" << index.member << "'" << endl;
+		    return false;
+		}
+		
+		TypePtr dataMemberType = dataMember->type();
+
+		if(index.caseSensitive == false)
+		{
+		    //
+		    // Let's check member is a string
+		    //
+		    BuiltinPtr memberType = BuiltinPtr::dynamicCast(dataMemberType);
+		    if(memberType == 0 || memberType->kind() != Builtin::KindString)
+		    {
+			cerr << n << ": `" << index.member << "'is not a string " << endl;
+			return false;
+		    }
+		}
+		indexTypes.push_back(dataMemberType);
+	    }
+	}
+	writeDictWithIndicesH(name, dict, indexTypes, keyType, valueType, H, dllExport);
+    }
+
 
     for(q = scope.begin(); q != scope.end(); ++q)
     {
@@ -243,6 +680,11 @@ writeCodecs(const string& n, UnitPtr& u, const Dict& dict, Output& H, Output& C,
 
     writeCodecC(keyType, absolute + "KeyCodec", "Key", false, C);
     writeCodecC(valueType, absolute + "ValueCodec", "Value", true, C);
+    
+    if(indexTypes.size() > 0)
+    {
+	writeDictWithIndicesC(name, absolute, dict, indexTypes, keyType, valueType, C);
+    }
 
     return true;
 }
@@ -251,7 +693,7 @@ writeCodecs(const string& n, UnitPtr& u, const Dict& dict, Output& H, Output& C,
 void
 writeIndexH(const string& memberTypeString, const string& name, Output& H, const string& dllExport)
 {
-    H << sp << nl << dllExport << "class " << name
+    H << sp << nl << "class " << dllExport << name
       << " : public Freeze::Index";
     H << sb;
     H.dec();
@@ -464,6 +906,9 @@ writeIndex(const string& n, UnitPtr& u, const Index& index, Output& H, Output& C
     return true;
 }
 
+
+
+
 int
 main(int argc, char* argv[])
 {
@@ -648,6 +1093,92 @@ main(int argc, char* argv[])
             }
             argc -= 2;
         }
+	else if(strcmp(argv[idx], "--dict-index") == 0)
+        {
+            if(idx + 1 >= argc || argv[idx + 1][0] == '-')
+            {
+                cerr << argv[0] << ": argument expected for `" << argv[idx] << "'" << endl;
+                usage(argv[0]);
+                return EXIT_FAILURE;
+            }
+
+            string s = argv[idx + 1];
+            s.erase(remove_if(s.begin(), s.end(), ::isspace), s.end());
+            
+	    string dictName;
+	    DictIndex index;
+            string::size_type pos;
+          
+	    string caseString = "case-sensitive";
+	    pos = s.find(',');
+	    if(pos != string::npos)
+            {
+                dictName = s.substr(0, pos);
+                s.erase(0, pos + 1);
+
+		pos = s.find(',');
+		if(pos != string::npos)
+		{
+		    index.member = s.substr(0, pos);
+		    s.erase(0, pos + 1);
+		    caseString = s;
+		}
+		else
+		{
+		    if(s == "case-sensitive" || s == "case-insensitive")
+		    {
+			caseString = s;
+		    }
+		    else
+		    {
+			index.member = s;
+		    }
+		}
+            }
+	    else
+	    {
+		dictName = s;
+	    }
+
+	    if(dictName.empty())
+            {
+                cerr << argv[0] << ": " << argv[idx] << ": no dictionary specified" << endl;
+                usage(argv[0]);
+                return EXIT_FAILURE;
+            }
+
+	    if(caseString != "case-sensitive" && caseString != "case-insensitive")
+            {
+                cerr << argv[0] << ": " << argv[idx]
+		     << ": the case can be `case-sensitive' or `case-insensitive'" << endl;
+                usage(argv[0]);
+                return EXIT_FAILURE;
+            }
+	    index.caseSensitive = (caseString == "case-sensitive");
+
+	    bool found = false;
+	    for(vector<Dict>::iterator p = dicts.begin(); p != dicts.end(); ++p)
+	    {
+		if(p->name == dictName)
+		{
+		    p->indices.push_back(index);
+		    found = true;
+		    break;
+		}
+	    }
+	    if(!found)
+	    {
+		cerr << argv[0] << ": " << argv[idx] << ": unknown dictionary" << endl;
+                usage(argv[0]);
+                return EXIT_FAILURE;
+	    }
+
+            for(int i = idx ; i + 2 < argc ; ++i)
+            {
+                argv[i] = argv[i + 2];
+            }
+            argc -= 2;
+        }
 	else if(strcmp(argv[idx], "-h") == 0 || strcmp(argv[idx], "--help") == 0)
 	{
 	    usage(argv[0]);
@@ -917,7 +1448,7 @@ main(int argc, char* argv[])
 	    {
 		try
 		{
-		    if(!writeCodecs(argv[0], u, *p, H, C, dllExport))
+		    if(!writeDict(argv[0], u, *p, H, C, dllExport))
 		    {
 			u->destroy();
 			return EXIT_FAILURE;
@@ -949,6 +1480,7 @@ main(int argc, char* argv[])
 		    return EXIT_FAILURE;
 		}
 	    }
+
 	}
 
 	H << "\n\n#endif\n";