Re: [Libreoffice] [Proof-of-concept] Drastically speedup the offapi (and oovbaapi) build

Hi Kendy, Bjoern

On 1 September 2011 10:25, Jan Holesovsky <kendy@suse.cz> wrote:
> Hi Matúš, Bjoern,
>
> I've found a way how to speedup the build of offapi (which is a major
> problem on windows - on my box it takes about 10% of the entire build
> time).  The trick is simple - instead of generating the files one by
> one, several .idl's are treated during one idlc call; something that has
> been there before gbuildization of the module.  Of course, it also
> handles the case when everything is built, and just 1 .idl changes - in
> that case, only that one is rebuilt.
That's nice.

> Unfortunately I am heading for vacation, and cannot finish it before I
> go - do you think anybody of you could have a look, please?
So, I had a look, it works that way but is not so much faster.

> With the attached patch, the offapi/UnoApi_offapi.mk (and oovbaapi) has
> to change to something like the following; this is of course only part
> of that, the other rules have to be adapted the same way, but that
> should be ~trivial.  Second part of the problem is that idlc have to be
> patched to bail out with != 0 exit value, according to the comments it
> is not the case right now.
I was not working on the second part yet and I do not intend to do so.
Could someone else do it?
I have no idea what idlc is anyway.

> Enjoy :-)
Well, it took a while, or more. Not so much fun but not really problem.

Bjoern, could you please approve patches?
And how to push them ? First the Kendy's one and then change it?
I have attached just one patch for better reviewing but created two.
(for UnoApiTarget.mk)

Now results.. in offapi:
before:
make -srj 4 => 75s
make -sr => 206s
after:
make -srj 4 => 71s
make -sr => 183s

It's little more interesting for oovbaapi: (make -sr) 49s -> 34s
But that does not take much.

May be for Windows it will be bigger difference but also this is something.

Or is it possible to process even more files at once? Also from
different directories?

And there were side-effects. I have no idea why but I had to add
include directives for a few idl files (also attached).

> Thank you,

You are welcome, thanks for bringing this up.

Matúš

diff --git a/solenv/gbuild/UnoApiTarget.mk b/solenv/gbuild/UnoApiTarget.mk
index 935c5ea..061c2b7 100644
--- a/solenv/gbuild/UnoApiTarget.mk
+++ b/solenv/gbuild/UnoApiTarget.mk
@@ -60,46 +60,55 @@ gb_UnoApiTarget_IDLFILES_$(1) :=
 endef
 
 define gb_UnoApiTarget_add_idlfiles
-$(foreach idl,$(2),$(call gb_UnoApiTarget_add_idlfile,$(1),$(idl)))
+$(foreach idl,$(3),$(call gb_UnoApiTarget_add_idlfile,$(1),$(2),$(idl)))
+
+$(call gb_UnoApiPartTarget_get_target,$(2)/idl.done) : $(foreach repo,$(gb_REPOS),$(foreach 
idl,$(3),$(realpath $(repo)/$(2)/$(idl).idl)))
+       $(gb_UnoApiPartTarget__command)
 
 endef
 
 define gb_UnoApiTarget_add_idlfile
-$(call gb_UnoApiTarget_get_target,$(1)) : $(call gb_UnoApiPartTarget_get_target,$(2).urd)
-$(call gb_UnoApiPartTarget_get_target,$(2).urd) : $(foreach repo,$(gb_REPOS),$(realpath 
$(repo)/$(2).idl))
-gb_UnoApiTarget_HPPFILES_$(1) += $(2).hdl
-gb_UnoApiTarget_HPPFILES_$(1) += $(2).hpp
-gb_UnoApiTarget_IDLFILES_$(1) += $(2).idl
+$(call gb_UnoApiTarget_get_target,$(1)) : $(call gb_UnoApiPartTarget_get_target,$(2)/$(3).urd)
+$(call gb_UnoApiPartTarget_get_target,$(2)/$(3).urd) : $(call 
gb_UnoApiPartTarget_get_target,$(2)/idl.done)
+gb_UnoApiTarget_HPPFILES_$(1) += $(2)/$(3).hdl
+gb_UnoApiTarget_HPPFILES_$(1) += $(2)/$(3).hpp
+gb_UnoApiTarget_IDLFILES_$(1) += $(2)/$(3).idl
 
-$(call gb_UnoApiTarget_get_header_target,)$(2).hpp :| $(call gb_UnoApiTarget_get_target,$(1))
-$(call gb_UnoApiTarget_get_header_target,)$(2).hdl :| $(call gb_UnoApiTarget_get_target,$(1))
+$(call gb_UnoApiTarget_get_header_target,)$(2)/$(3).hpp :| $(call gb_UnoApiTarget_get_target,$(1))
+$(call gb_UnoApiTarget_get_header_target,)$(2)/$(3).hdl :| $(call gb_UnoApiTarget_get_target,$(1))
 
 endef
 
 define gb_UnoApiTarget_add_idlfiles_noheader
-$(foreach idl,$(2),$(call gb_UnoApiTarget_add_idlfile_noheader,$(1),$(idl)))
+$(foreach idl,$(3),$(call gb_UnoApiTarget_add_idlfile_noheader,$(1),$(2),$(idl)))
+
+$(call gb_UnoApiPartTarget_get_target,$(2)/idl_noheader.done) : $(foreach 
repo,$(gb_REPOS),$(foreach idl,$(3),$(realpath $(repo)/$(2)/$(idl).idl)))
+       $(gb_UnoApiPartTarget__command)
 
 endef
 
 define gb_UnoApiTarget_add_idlfile_noheader
-$(call gb_UnoApiTarget_get_target,$(1)) : $(call gb_UnoApiPartTarget_get_target,$(2).urd)
-$(call gb_UnoApiPartTarget_get_target,$(2).urd) : $(foreach repo,$(gb_REPOS),$(realpath 
$(repo)/$(2).idl))
-gb_UnoApiTarget_IDLFILES_$(1) += $(2).idl
+$(call gb_UnoApiTarget_get_target,$(1)) : $(call gb_UnoApiPartTarget_get_target,$(2)/$(3).urd)
+$(call gb_UnoApiPartTarget_get_target,$(2)/$(3).urd) : $(call 
gb_UnoApiPartTarget_get_target,$(2)/idl_noheader.done)
+gb_UnoApiTarget_IDLFILES_$(1) += $(2)/$(3).idl
 
 endef
 
 define gb_UnoApiTarget_add_idlfiles_nohdl
-$(foreach idl,$(2),$(call gb_UnoApiTarget_add_idlfile_nohdl,$(1),$(idl)))
+$(foreach idl,$(3),$(call gb_UnoApiTarget_add_idlfile_nohdl,$(1),$(2),$(idl)))
+
+$(call gb_UnoApiPartTarget_get_target,$(2)/idl_nohdl.done) : $(foreach repo,$(gb_REPOS),$(foreach 
idl,$(3),$(realpath $(repo)/$(2)/$(idl).idl)))
+       $(gb_UnoApiPartTarget__command)
 
 endef
 
 define gb_UnoApiTarget_add_idlfile_nohdl
-$(call gb_UnoApiTarget_get_target,$(1)) : $(call gb_UnoApiPartTarget_get_target,$(2).urd)
-$(call gb_UnoApiPartTarget_get_target,$(2).urd) : $(foreach repo,$(gb_REPOS),$(realpath 
$(repo)/$(2).idl))
-gb_UnoApiTarget_HPPFILES_$(1) += $(2).hpp
-gb_UnoApiTarget_IDLFILES_$(1) += $(2).idl
+$(call gb_UnoApiTarget_get_target,$(1)) : $(call gb_UnoApiPartTarget_get_target,$(2)/$(3).urd)
+$(call gb_UnoApiPartTarget_get_target,$(2)/$(3).urd) : $(call 
gb_UnoApiPartTarget_get_target,$(2)/idl_nohdl.done)
+gb_UnoApiTarget_HPPFILES_$(1) += $(2)/$(3).hpp
+gb_UnoApiTarget_IDLFILES_$(1) += $(2)/$(3).idl
 
-$(call gb_UnoApiTarget_get_header_target,)$(2).hpp :| $(call gb_UnoApiTarget_get_target,$(1))
+$(call gb_UnoApiTarget_get_header_target,)$(2)/$(3).hpp :| $(call gb_UnoApiTarget_get_target,$(1))
 
 endef
 
@@ -152,24 +161,17 @@ $(call gb_UnoApiTarget_get_clean_target,%) :
        -rm -rf $(call gb_UnoApiTarget_get_header_target,$*)\
                        $(call gb_UnoApiPartTarget_get_target,$*)
 
-# idlc doesn't return error codes != 0 in case of an error, so
-# check self 
 define gb_UnoApiPartTarget__command
-mkdir -p $(dir $(1)) && \
-       mkdir -p  $(gb_Helper_MISC) && \
-       RESPONSEFILE=`$(gb_MKTEMP)` && \
-       echo "$(call gb_Helper_convert_native,$(5) $(6) -O $(call 
gb_UnoApiPartTarget_get_target,$(2)) -verbose -C $(4))" > $${RESPONSEFILE} && \
-       $(gb_UnoApiTarget_IDLCCOMMAND) @$${RESPONSEFILE} > /dev/null && \
-       rm -f $${RESPONSEFILE} && \
-       test -f $(@)
+       $$(call gb_Output_announce,$(2),$(true),IDL,2)
+       mkdir -p $(call gb_UnoApiPartTarget_get_target,$(2)) && \
+       RESPONSEFILE=$$(call var2file,$(shell $(gb_MKTEMP)),500,\
+               $(call gb_Helper_convert_native,$$(INCLUDE) $$(DEFS) -O $(call 
gb_UnoApiPartTarget_get_target,$(2)) -verbose -C $$?)) && \
+       $(gb_UnoApiTarget_IDLCCOMMAND) @$$$${RESPONSEFILE} > /dev/null && \
+       rm -f $$$${RESPONSEFILE} && \
+       touch $$@
 
 endef
 
-$(call gb_UnoApiPartTarget_get_target,%) :
-       $(call gb_Output_announce,$*,$(true),IDL,2)
-       $(call gb_UnoApiPartTarget__command,$@,$(dir $*),$<,$?,$(INCLUDE),$(DEFS))
-       
-
 define gb_UnoApiTarget__command
        mkdir -p $(dir $(1)) && \
        RESPONSEFILE=$(call var2file,$(shell $(gb_MKTEMP)),500,$(call gb_Helper_convert_native,$(1) 
$(4) $(5))) && \

From 62aa32494e06fe0c3e34abb236896cdc0c37a0c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mat=C3=BA=C5=A1=20Kukan?= <matus.kukan@gmail.com>
Date: Fri, 2 Sep 2011 03:22:05 +0200
Subject: [PATCH 4/4] include missing header file

---
 offapi/com/sun/star/rendering/XMtfRenderer.idl |    2 ++
 oovbaapi/ooo/vba/excel/XControlObject.idl      |    2 ++
 oovbaapi/ooo/vba/excel/XGraphicObjects.idl     |    2 ++
 3 files changed, 6 insertions(+), 0 deletions(-)

diff --git a/offapi/com/sun/star/rendering/XMtfRenderer.idl 
b/offapi/com/sun/star/rendering/XMtfRenderer.idl
index ac69a9a..619163e 100644
--- a/offapi/com/sun/star/rendering/XMtfRenderer.idl
+++ b/offapi/com/sun/star/rendering/XMtfRenderer.idl
@@ -28,6 +28,8 @@
 #ifndef __com_sun_star_rendering_XMtfRenderer_idl__
 #define __com_sun_star_rendering_XMtfRenderer_idl__
 
+#include <com/sun/star/uno/XInterface.idl>
+
 module com { module sun { module star { module rendering {
 
     interface XMtfRenderer : ::com::sun::star::uno::XInterface
diff --git a/oovbaapi/ooo/vba/excel/XControlObject.idl b/oovbaapi/ooo/vba/excel/XControlObject.idl
index 7a6250d..418a97a 100644
--- a/oovbaapi/ooo/vba/excel/XControlObject.idl
+++ b/oovbaapi/ooo/vba/excel/XControlObject.idl
@@ -28,6 +28,8 @@
 #ifndef __ooo_vba_excel_XControlObject_idl__
 #define __ooo_vba_excel_XControlObject_idl__
 
+#include <com/sun/star/uno/XInterface.idl>
+
 module ooo {  module vba { module excel {
 
 //=============================================================================
diff --git a/oovbaapi/ooo/vba/excel/XGraphicObjects.idl b/oovbaapi/ooo/vba/excel/XGraphicObjects.idl
index 25474fd..5c2e93b 100644
--- a/oovbaapi/ooo/vba/excel/XGraphicObjects.idl
+++ b/oovbaapi/ooo/vba/excel/XGraphicObjects.idl
@@ -28,6 +28,8 @@
 #ifndef __ooo_vba_excel_GraphicObjects_idl__
 #define __ooo_vba_excel_GraphicObjects_idl__
 
+#include <com/sun/star/uno/XInterface.idl>
+
 module ooo {  module vba { module excel {
 
 //=============================================================================
-- 
1.7.1