The default LMDB map size for IceGrid and IceStorm is now 10MB (Windows)

and 100MB (Linux, OS X)

6 files changed, 73 insertions, 15 deletions

diff --git a/cpp/src/IceDB/IceDB.cpp b/cpp/src/IceDB/IceDB.cpp
index 53179092195..ed0180e8d30 100644
--- a/cpp/src/IceDB/IceDB.cpp
+++ b/cpp/src/IceDB/IceDB.cpp
@@ -472,3 +472,23 @@ CursorBase::renew(const ReadOnlyTxn& txn)
 }
 
 
+//
+// On Windows, we use a default LMDB map size of 10MB, whereas on other platforms 
+// (Linux, OS X), we use a default of 100MB.
+//
+// On Windows, LMDB does not use sparse files and allocates immediately the file
+// with the given (max) size. This is why we need a fairly small default map size 
+// on Windows, and a larger value on other platforms.
+
+size_t
+IceDB::getMapSize(int configValue)
+{
+#ifdef _WIN32
+    const size_t defaultMapSize = 10;
+#else
+    const size_t defaultMapSize = 100;
+#endif
+
+   return ((configValue <= 0) ? defaultMapSize : configValue) * 1024 * 1024;
+}
+
diff --git a/cpp/src/IceDB/IceDB.h b/cpp/src/IceDB/IceDB.h
index 4b28c96780e..f0f4ca793e1 100644
--- a/cpp/src/IceDB/IceDB.h
+++ b/cpp/src/IceDB/IceDB.h
@@ -534,6 +534,14 @@ struct Codec<T, IceContext, Ice::OutputStreamPtr>
     }
 };
 
+//
+// Returns computed mapSize in bytes.
+// When the input parameter is <= 0, returns a platform-dependent default
+// (currently 0 on Windows and 100 MB on other platforms).
+// Otherwise, returns input parameter * 1 MB.
+//
+
+ICE_DB_API size_t getMapSize(int);
 }
 
 #endif
diff --git a/cpp/src/IceGrid/Database.cpp b/cpp/src/IceGrid/Database.cpp
index cfdfbb58b8e..75624feb9e4 100644
--- a/cpp/src/IceGrid/Database.cpp
+++ b/cpp/src/IceGrid/Database.cpp
@@ -213,7 +213,7 @@ Database::Database(const Ice::ObjectAdapterPtr& registryAdapter,
     _serverCache(_communicator, _instanceName, _nodeCache, _adapterCache, _objectCache, _allocatableObjectCache),
     _dbLock(_communicator->getProperties()->getProperty("IceGrid.Registry.LMDB.Path") + "/icedb.lock"),
     _env(_communicator->getProperties()->getProperty("IceGrid.Registry.LMDB.Path"), 8,
-         _communicator->getProperties()->getPropertyAsInt("IceGrid.Registry.LMDB.MapSize") * 1024 * 1024),
+         IceDB::getMapSize(_communicator->getProperties()->getPropertyAsInt("IceGrid.Registry.LMDB.MapSize"))),
     _pluginFacade(RegistryPluginFacadeIPtr::dynamicCast(getRegistryPluginFacade())),
     _lock(0)
 {
diff --git a/cpp/src/IceGrid/IceGridDB.cpp b/cpp/src/IceGrid/IceGridDB.cpp
index 3fb35c67c2b..92c95a803b1 100644
--- a/cpp/src/IceGrid/IceGridDB.cpp
+++ b/cpp/src/IceGrid/IceGridDB.cpp
@@ -153,8 +153,10 @@ Client::usage()
         "-v, --version          Display version.\n"
         "--import FILE          Import database from FILE.\n"
         "--export FILE          Export database to FILE.\n"
-        "--dbhome DIR           The database directory.\n"
-        "--server-version VER   Ice version for IceGrid servers (import only).\n"
+        "--dbhome DIR           Source or target database environment.\n"
+        "--dbpath DIR           Source or target database environment.\n"
+        "--mapsize VALUE        Set LMDB map size in MB (optional, import only).\n"
+        "--server-version VER   Set Ice version for IceGrid servers (optional, import only).\n"
         "-d, --debug            Print debug messages."
         ;
 }
@@ -169,6 +171,8 @@ Client::run(int argc, char* argv[])
     opts.addOpt("", "import", IceUtilInternal::Options::NeedArg);
     opts.addOpt("", "export", IceUtilInternal::Options::NeedArg);
     opts.addOpt("", "dbhome", IceUtilInternal::Options::NeedArg);
+    opts.addOpt("", "dbpath", IceUtilInternal::Options::NeedArg);
+    opts.addOpt("", "mapsize", IceUtilInternal::Options::NeedArg);
     opts.addOpt("", "server-version", IceUtilInternal::Options::NeedArg);
 
     vector<string> args;
@@ -201,16 +205,16 @@ Client::run(int argc, char* argv[])
         return EXIT_SUCCESS;
     }
 
-    if((!opts.isSet("import") && !opts.isSet("export")) || (opts.isSet("import") && opts.isSet("export")))
+    if(!(opts.isSet("import") ^ opts.isSet("export")))
     {
         cerr << "Either --import or --export must be set" << endl;
         usage();
         return EXIT_FAILURE;
     }
 
-    if(!opts.isSet("dbhome"))
+    if(!(opts.isSet("dbhome") ^ opts.isSet("dbpath")))
     {
-        cerr << "Database home must be specified" << endl;
+        cerr << "Set the database environment directory with either --dbhome or --dbpath" << endl;
         usage();
         return EXIT_FAILURE;
     }
@@ -218,7 +222,18 @@ Client::run(int argc, char* argv[])
     bool debug = opts.isSet("debug");
     bool import = opts.isSet("import");
     string dbFile = opts.optArg(import ? "import" : "export");
-    string dbPath = opts.optArg("dbhome");
+    string dbPath;
+    if(opts.isSet("dbhome"))
+    {
+        dbPath = opts.optArg("dbhome");
+    }
+    else
+    {
+        dbPath = opts.optArg("dbpath");
+    }
+
+    string mapSizeStr = opts.optArg("mapsize");
+    size_t mapSize = IceDB::getMapSize(atoi(mapSizeStr.c_str()));
     string serverVersion = opts.optArg("server-version");
 
     try
@@ -295,7 +310,7 @@ Client::run(int argc, char* argv[])
             stream->read(data);
 
             {
-                IceDB::Env env(dbPath, 5);
+                IceDB::Env env(dbPath, 5, mapSize);
                 IceDB::ReadWriteTxn txn(env);
 
                 if(debug)
diff --git a/cpp/src/IceStorm/IceStormDB.cpp b/cpp/src/IceStorm/IceStormDB.cpp
index 42a2e3f9b65..d5b8fe416e4 100644
--- a/cpp/src/IceStorm/IceStormDB.cpp
+++ b/cpp/src/IceStorm/IceStormDB.cpp
@@ -53,7 +53,9 @@ Client::usage()
         "-v, --version          Display version.\n"
         "--import FILE          Import database from FILE.\n"
         "--export FILE          Export database to FILE.\n"
-        "--dbhome DIR           The database directory.\n"
+        "--dbhome DIR           Source or target database environment.\n"
+        "--dbpath DIR           Source or target database environment.\n"
+        "--mapsize VALUE        Set LMDB map size in MB (optional, import only).\n"
         "-d, --debug            Print debug messages."
         ;
 }
@@ -68,6 +70,8 @@ Client::run(int argc, char* argv[])
     opts.addOpt("", "import", IceUtilInternal::Options::NeedArg);
     opts.addOpt("", "export", IceUtilInternal::Options::NeedArg);
     opts.addOpt("", "dbhome", IceUtilInternal::Options::NeedArg);
+    opts.addOpt("", "dbpath", IceUtilInternal::Options::NeedArg);
+    opts.addOpt("", "mapsize", IceUtilInternal::Options::NeedArg);
 
     vector<string> args;
     try
@@ -99,16 +103,16 @@ Client::run(int argc, char* argv[])
         return EXIT_SUCCESS;
     }
 
-    if((!opts.isSet("import") && !opts.isSet("export")) || (opts.isSet("import") && opts.isSet("export")))
+    if(!(opts.isSet("import") ^ opts.isSet("export")))
     {
         cerr << "Either --import or --export must be set" << endl;
         usage();
         return EXIT_FAILURE;
     }
 
-    if(!opts.isSet("dbhome"))
+    if(!(opts.isSet("dbhome") ^ opts.isSet("dbpath")))
     {
-        cerr << "Database path must be specified" << endl;
+        cerr << "Set the database environment directory with either --dbhome or --dbpath" << endl;
         usage();
         return EXIT_FAILURE;
     }
@@ -116,7 +120,18 @@ Client::run(int argc, char* argv[])
     bool debug = opts.isSet("debug");
     bool import = opts.isSet("import");
     string dbFile = opts.optArg(import ? "import" : "export");
-    string dbPath = opts.optArg("dbhome");
+    string dbPath;
+    if(opts.isSet("dbhome"))
+    {
+        dbPath = opts.optArg("dbhome");
+    }
+    else
+    {
+        dbPath = opts.optArg("dbpath");
+    }
+
+    string mapSizeStr = opts.optArg("mapsize");
+    size_t mapSize = IceDB::getMapSize(atoi(mapSizeStr.c_str()));
 
     try
     {
@@ -176,7 +191,7 @@ Client::run(int argc, char* argv[])
             stream->read(data);
 
             {
-                IceDB::Env env(dbPath, 2);
+                IceDB::Env env(dbPath, 2, mapSize);
                 IceDB::ReadWriteTxn txn(env);
 
                 if(debug)
diff --git a/cpp/src/IceStorm/Instance.cpp b/cpp/src/IceStorm/Instance.cpp
index 6c02ef06002..4a852e4ca72 100644
--- a/cpp/src/IceStorm/Instance.cpp
+++ b/cpp/src/IceStorm/Instance.cpp
@@ -55,7 +55,7 @@ PersistentInstance::PersistentInstance(
     Instance(instanceName, name, communicator, publishAdapter, topicAdapter, nodeAdapter, nodeProxy),
     _dbLock(communicator->getProperties()->getPropertyWithDefault(name + ".LMDB.Path", name) + "/icedb.lock"),
     _dbEnv(communicator->getProperties()->getPropertyWithDefault(name + ".LMDB.Path", name), 2,
-           communicator->getProperties()->getPropertyAsInt(name + ".LMDB.MapSize") * 1024 * 1024)
+           IceDB::getMapSize(communicator->getProperties()->getPropertyAsInt(name + ".LMDB.MapSize")))
 {
     try
     {