[PATCH-3-5] improve startup performance ...

On Fri, 2012-04-13 at 15:03 +0200, Michael Meeks wrote:
>       The basic idea is to push the iteration over directories down into the
> stoc/ code instead of having it in the cppuhelper code. That lets us
> re-use the same representation avoiding the nesting[1]. I guess it is
> the moral equivalent of concatenating the .rdb files in a given
> directory.

        I had to add some sniffing code and retain the old model for loading
old-style rdb files to make the oovbaapi.rdb load correctly.

        After that it seems to pass 'make check' as well as 'make check' in
general (ie. randomly failing on a11y tests ;-).

        It makes me suspect that the nesting nastiness has a real impact on
type lookups too that'd be great to fix later.

        I'd love to get it into libreoffice-3-5 for 3.5.3 - reviews much
appreciated.

        Thanks,

                Michael.

-- 
michael.meeks@suse.com  <><, Pseudo Engineer, itinerant idiot

From 09524d410bbaad2a0b9b39811cb5cc16621b1396 Mon Sep 17 00:00:00 2001
From: Michael Meeks <michael.meeks@suse.com>
Date: Fri, 13 Apr 2012 15:25:23 +0200
Subject: [PATCH] stoc: accelerate opening of multiple XML .rdb files in a
 directory

Instead of nesting these, we aggregate them into a single non-nested
registry, which saves lots of CPU at startup, sadly we can only do
that for the new-style XML registries, so we have to sniff files,
nevertheless this is still far faster. The merged xml files also
break the XSimpleRegistry::getURL() method - but it appears not
to get called.
---
 cppuhelper/source/bootstrap.cxx                |   60 ++++++++++++---
 stoc/source/simpleregistry/simpleregistry.cxx  |   94 ++++++++++++++++++++----
 stoc/source/simpleregistry/textualservices.cxx |   11 ++-
 stoc/source/simpleregistry/textualservices.hxx |    3 +
 4 files changed, 141 insertions(+), 27 deletions(-)

diff --git a/cppuhelper/source/bootstrap.cxx b/cppuhelper/source/bootstrap.cxx
index 39e44fe..cc64e07 100644
--- a/cppuhelper/source/bootstrap.cxx
+++ b/cppuhelper/source/bootstrap.cxx
@@ -274,16 +274,17 @@ Reference< registry::XSimpleRegistry > readRdbDirectory(
              url),
             css::uno::Reference< css::uno::XInterface >());
     }
-    for (css::uno::Reference< css::registry::XSimpleRegistry > last(
-             lastRegistry);;)
+    std::vector<rtl::OUString> aURLs;
+    css::uno::Reference< css::registry::XSimpleRegistry > last(lastRegistry);
+    for (;;)
     {
         osl::DirectoryItem i;
-        switch (dir.getNextItem(i, SAL_MAX_UINT32)) {
-        case osl::FileBase::E_None:
+        osl::FileBase::RC eResult;
+        eResult = dir.getNextItem(i, SAL_MAX_UINT32);
+        if (eResult == osl::FileBase::E_NOENT)
             break;
-        case osl::FileBase::E_NOENT:
-            return last;
-        default:
+        if (eResult != osl::FileBase::E_None)
+        {
             throw css::uno::RuntimeException(
                 (rtl::OUString(
                     RTL_CONSTASCII_USTRINGPARAM("cannot iterate directory ")) +
@@ -307,12 +308,49 @@ Reference< registry::XSimpleRegistry > readRdbDirectory(
         if (aName.toChar() == '.' || aName.endsWithAsciiL("~", 1))
             continue;
 
-        if (stat.getFileType() != osl::FileStatus::Directory) { //TODO: symlinks
-            last = readRdbFile(
-                stat.getFileURL(), fatalErrors, last, simpleRegistryFactory,
-                nestedRegistryFactory);
+        if (stat.getFileType() != osl::FileStatus::Directory) //TODO: symlinks
+            aURLs.push_back(stat.getFileURL());
+    }
+
+    size_t nXML = 0;
+    for (std::vector<rtl::OUString>::iterator it = aURLs.begin(); it != aURLs.end(); it++)
+    {
+        // Read / sniff the nasty files ...
+        osl::File aIn( *it );
+        if (aIn.open(osl_File_OpenFlag_Read) != osl::FileBase::E_None)
+            continue;
+
+        sal_uInt64 nRead = 0;
+        char buffer[6];
+        bool bIsXML = aIn.read(buffer, 6, nRead) == osl::FileBase::E_None &&
+                      nRead == 6 && !strncmp(buffer, "<?xml ", 6);
+        aIn.close();
+        if (!bIsXML)
+        {
+            OSL_TRACE (OSL_LOG_PREFIX "rdb '%s' is a legacy format\n",
+                       rtl::OUStringToOString( *it, RTL_TEXTENCODING_UTF8 ).getStr());
+            break;
+        }
+        nXML++;
+    }
+    if (nXML == aURLs.size())
+    {
+        OSL_TRACE (OSL_LOG_PREFIX "no legacy rdbs in directory '%s'\n",
+                   rtl::OUStringToOString( url, RTL_TEXTENCODING_UTF8 ).getStr());
+        // read whole directory...
+        last = readRdbFile( url, fatalErrors, last,
+                            simpleRegistryFactory, nestedRegistryFactory);
+    }
+    else
+    {
+        for (std::vector<rtl::OUString>::iterator it = aURLs.begin(); it != aURLs.end(); it++)
+        {
+            // Read / sniff the nasty files ...
+            last = readRdbFile(*it, fatalErrors, last,
+                               simpleRegistryFactory, nestedRegistryFactory);
         }
     }
+    return last;
 }
 
 Reference< registry::XSimpleRegistry > nestRegistries(
diff --git a/stoc/source/simpleregistry/simpleregistry.cxx 
b/stoc/source/simpleregistry/simpleregistry.cxx
index f09b204..9eebb4f 100644
--- a/stoc/source/simpleregistry/simpleregistry.cxx
+++ b/stoc/source/simpleregistry/simpleregistry.cxx
@@ -48,6 +48,7 @@
 #include "cppuhelper/implbase2.hxx"
 #include "cppuhelper/weak.hxx"
 #include "osl/mutex.hxx"
+#include "osl/file.hxx"
 #include "registry/registry.hxx"
 #include "registry/regtype.h"
 #include "rtl/ref.hxx"
@@ -84,6 +85,12 @@ public:
 private:
     virtual rtl::OUString SAL_CALL getURL() throw (css::uno::RuntimeException);
 
+    virtual void SAL_CALL openRdb(
+        rtl::OUString const & rURL, sal_Bool bReadOnly, sal_Bool bCreate)
+        throw (
+            css::registry::InvalidRegistryException,
+            css::uno::RuntimeException);
+
     virtual void SAL_CALL open(
         rtl::OUString const & rURL, sal_Bool bReadOnly, sal_Bool bCreate)
         throw (
@@ -1130,27 +1137,17 @@ rtl::OUString Key::getResolvedName(rtl::OUString const & aKeyName)
     return resolved;
 }
 
-rtl::OUString SimpleRegistry::getURL() throw (css::uno::RuntimeException) {
+rtl::OUString SimpleRegistry::getURL() throw (css::uno::RuntimeException)
+{
     osl::MutexGuard guard(mutex_);
     return textual_.get() == 0 ? registry_.getName() : textual_->getUri();
 }
 
-void SimpleRegistry::open(
+void SimpleRegistry::openRdb(
     rtl::OUString const & rURL, sal_Bool bReadOnly, sal_Bool bCreate)
     throw (css::registry::InvalidRegistryException, css::uno::RuntimeException)
 {
     osl::MutexGuard guard(mutex_);
-    if (textual_.get() != 0) {
-        throw css::registry::InvalidRegistryException(
-            (rtl::OUString(
-                RTL_CONSTASCII_USTRINGPARAM(
-                    "com.sun.star.registry.SimpleRegistry.open(")) +
-             rURL +
-             rtl::OUString(
-                RTL_CONSTASCII_USTRINGPARAM(
-                    "): instance already open"))),
-            static_cast< OWeakObject * >(this));
-    }
     RegError err = (rURL.isEmpty() && bCreate)
         ? REG_REGISTRY_NOT_EXISTS
         : registry_.open(rURL, bReadOnly ? REG_READONLY : REG_READWRITE);
@@ -1162,7 +1159,10 @@ void SimpleRegistry::open(
         break;
     case REG_INVALID_REGISTRY:
         if (bReadOnly && !bCreate) {
-            textual_.reset(new stoc::simpleregistry::TextualServices(rURL));
+            if (!textual_.get())
+                textual_.reset(new stoc::simpleregistry::TextualServices(rURL));
+            else
+                textual_->merge(rURL);
             break;
         }
         // fall through
@@ -1180,6 +1180,72 @@ void SimpleRegistry::open(
     }
 }
 
+void SimpleRegistry::open(
+    rtl::OUString const & rURL, sal_Bool bReadOnly, sal_Bool bCreate)
+    throw (css::registry::InvalidRegistryException, css::uno::RuntimeException)
+{
+    osl::MutexGuard guard(mutex_);
+
+    osl::DirectoryItem aItem;
+    osl::FileBase::RC eErr;
+    osl::FileStatus aStatus(osl_FileStatus_Mask_Type);
+
+    // FIXME: busts the 'create' mode ...
+    if ((eErr = osl::DirectoryItem::get( rURL, aItem )) != osl::FileBase::E_None ||
+        (eErr = aItem.getFileStatus( aStatus )) != osl::FileBase::E_None ||
+        !aStatus.isDirectory())
+    {
+        if (textual_.get() != 0)
+            throw css::registry::InvalidRegistryException(
+                (rtl::OUString("com.sun.star.registry.SimpleRegistry.open(") +
+                 rURL + rtl::OUString("): instance already open")),
+                static_cast< OWeakObject * >(this));
+        openRdb (rURL, bReadOnly, bCreate);
+    }
+    else
+    {
+        osl::Directory dir(rURL);
+        eErr = dir.open();
+        if (eErr != osl::FileBase::E_None)
+            goto err_throw;
+
+        for (;;) {
+            osl::DirectoryItem i;
+            if (dir.getNextItem(i, SAL_MAX_UINT32) != osl::FileBase::E_None)
+                break;
+            osl::FileStatus stat(osl_FileStatus_Mask_Type | osl_FileStatus_Mask_FileName |
+                                 osl_FileStatus_Mask_FileURL);
+            if (i.getFileStatus(stat) != osl::FileBase::E_None)
+                throw css::uno::RuntimeException(
+                        (rtl::OUString("cannot stat in directory ") + rURL ),
+                        css::uno::Reference< css::uno::XInterface >());
+
+            rtl::OUString aName = stat.getFileName();
+
+            // Ignore backup files - to allow people to edit their
+            // services/ without extremely confusing behaviour
+            if (aName.toChar() == '.' || aName.endsWithAsciiL("~", 1))
+                continue;
+
+            if (stat.getFileType() != osl::FileStatus::Directory)
+                openRdb(stat.getFileURL(), bReadOnly, bCreate);
+        }
+    }
+    return;
+
+err_throw:
+        throw css::registry::InvalidRegistryException(
+            (rtl::OUString(
+                RTL_CONSTASCII_USTRINGPARAM(
+                    "com.sun.star.registry.SimpleRegistry.open(")) +
+             rURL +
+             rtl::OUString(
+                RTL_CONSTASCII_USTRINGPARAM(
+                    "): error statting url = ")) +
+             rtl::OUString::valueOf(static_cast< sal_Int32 >(eErr))),
+            static_cast< OWeakObject * >(this));
+}
+
 sal_Bool SimpleRegistry::isValid() throw (css::uno::RuntimeException) {
     osl::MutexGuard guard(mutex_);
     return textual_.get() != 0 || registry_.isValid();
diff --git a/stoc/source/simpleregistry/textualservices.cxx 
b/stoc/source/simpleregistry/textualservices.cxx
index ad24a44..2491f55 100644
--- a/stoc/source/simpleregistry/textualservices.cxx
+++ b/stoc/source/simpleregistry/textualservices.cxx
@@ -1236,6 +1236,15 @@ css::uno::Sequence< rtl::OUString > Key::getChildren() {
 TextualServices::TextualServices(rtl::OUString const & uri):
     uri_(uri), data_(new Data)
 {
+    merge(uri);
+}
+
+TextualServices::~TextualServices() {}
+
+// load and merge registry contents from uri
+void TextualServices::merge(const rtl::OUString &uri)
+        throw (com::sun::star::registry::InvalidRegistryException)
+{
     try {
         Parser(uri, data_);
     } catch (css::container::NoSuchElementException &) {
@@ -1247,8 +1256,6 @@ TextualServices::TextualServices(rtl::OUString const & uri):
     }
 }
 
-TextualServices::~TextualServices() {}
-
 css::uno::Reference< css::registry::XRegistryKey > TextualServices::getRootKey()
 {
     return new Key(data_, std::vector< rtl::OUString >());
diff --git a/stoc/source/simpleregistry/textualservices.hxx 
b/stoc/source/simpleregistry/textualservices.hxx
index 286eb92..0341c15 100644
--- a/stoc/source/simpleregistry/textualservices.hxx
+++ b/stoc/source/simpleregistry/textualservices.hxx
@@ -53,6 +53,9 @@ public:
 
     virtual ~TextualServices();
 
+    void merge(const rtl::OUString &uri)
+        throw (com::sun::star::registry::InvalidRegistryException);
+
     inline rtl::OUString getUri() { return uri_; }
 
     com::sun::star::uno::Reference< com::sun::star::registry::XRegistryKey >
-- 
1.7.9