Merge pull request #139 from Flow-IPC/flow-106_gh-jobs-parallel

GitHub CI pipeline: Parallelize core build step for 2x+ decrease in build time. / Pulling in submodules: <flow: -same-> <ipc_shm+ipc_sal: Eliminate issue in CMake build of test/demo program which made it unreliable in a parallelized build.> / Eliminate issue in CMake build of test/demo programs which made them unreliable in a parallelized build.

Author: ygoldfeld

conanfile.py

@@ -16,6 +16,7 @@
 # permissions and limitations under the License.
 
 from conan import ConanFile
+from conan.tools.build import build_jobs
 from conan.tools.cmake import CMake, cmake_layout, CMakeDeps, CMakeToolchain
 
 def load_version_from_file(version_path):  # Now takes version_path as an argument
@@ -124,11 +125,11 @@ def build(self):
 
         # Cannot use cmake.build(...) because not possible to pass make arguments like --keep-going.
         if self.options.build:
-            self.run("cmake --build . -- --keep-going VERBOSE=1")
+            self.run(f"cmake --build . -- --keep-going VERBOSE=1 -j{build_jobs(self)}")
         if self.options.doc:
             # Note: `flow_doc_public flow_doc_full` could also be added here and work; however
             # we leave that to `flow` and its own Conan setup.
-            self.run("cmake --build . -- ipc_doc_public ipc_doc_full --keep-going VERBOSE=1")
+            self.run(f"cmake --build . -- ipc_doc_public ipc_doc_full --keep-going VERBOSE=1 -j{build_jobs(self)}")
 
     def requirements(self):
         if self.options.build:

flow

@@ -21,6 +21,9 @@ find_package(CapnProto CONFIG REQUIRED)
 set(CAPNPC_IMPORT_DIRS ${FLOW_LIKE_META_ROOT_ipc_transport_structured}/src)
 capnp_generate_cpp(capnp_generated_srcs capnp_generated_hdrs_ignored "schema.capnp")
 
+# Create a custom target that depends on the generated files.
+add_custom_target(perf_demo_schema_generation DEPENDS ${capnp_generated_srcs} ${capnp_generated_hdrs})
+
 function(handle_binary name_sh jem_else_classic)
   if(jem_else_classic)
     set(name_pfx "shm_jemalloc")
@@ -32,6 +35,9 @@ function(handle_binary name_sh jem_else_classic)
   set(name "perf_demo_${name_sh}_${name_pfx}.exec") # Must match common.cpp constant values.
   add_executable(${name} common.cpp "main_${name_sh}.cpp" ${capnp_generated_srcs})
 
+  # Add explicit dependency on schema generation; otherwise things tend to go weird with a parallelized build.
+  add_dependencies(${name} perf_demo_schema_generation)
+
   target_include_directories(${name} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
   target_compile_definitions(${name} PRIVATE "JEM_ELSE_CLASSIC=${def_val}")
 

ipc_shm

@@ -20,7 +20,7 @@ find_package(CapnProto CONFIG REQUIRED)
 
 # Test program: transport_test.
 # It's a compendium of tests and test tools for testing ipc::transport (including ipc::transport::struc)
-# and ipc::session.   SHM modes are tested extensively.
+# and ipc::session.  SHM modes are tested extensively.
 #
 # Please see ./README.txt.
 

ipc_shm_arena_lend

@@ -30,14 +30,25 @@ find_package(CapnProto CONFIG REQUIRED)
 # ipc/shm/arena_lend/jemalloc/test/shm_pool_collection_test.  (TODO: Is that a copy/paste error or accurate?)
 
 # Make those binaries in very similar ways; call this for each.
-function(handle_binary name) # Load SRCS and CAPNP_SCHEMAS before calling.
+function(handle_binary name_root) # Load SRCS and CAPNP_SCHEMAS before calling.
+  set(name "${name_root}.exec")
+
   if(CAPNP_SCHEMAS)
     set(CAPNPC_SRC_PREFIX ${isal_root})
-    capnp_generate_cpp(capnp_generated_srcs capnp_generated_hdrs_ignored ${CAPNP_SCHEMAS})
+    capnp_generate_cpp(capnp_generated_srcs capnp_generated_hdrs ${CAPNP_SCHEMAS})
+
+    # Create a custom target that depends on the generated files.
+    add_custom_target(${name_root}_schema_generation
+                      DEPENDS ${capnp_generated_srcs} ${capnp_generated_hdrs})
   endif()
 
   add_executable(${name} ${SRCS} ${capnp_generated_srcs})
 
+  if(CAPNP_SCHEMAS)
+    # Add explicit dependency on schema generation; otherwise things tend to go weird with a parallelized build.
+    add_dependencies(${name} ${name_root}_schema_generation)
+  endif()
+
   # Test header files are *not* exported for user to include; rather they exist only to be cross-included by
   # each other (namely the support files are).  For that to work add the source dir(s) into the include-path.
   target_include_directories(${name} PRIVATE
@@ -47,7 +58,7 @@ function(handle_binary name) # Load SRCS and CAPNP_SCHEMAS before calling.
                              ${FLOW_LIKE_META_ROOT_ipc_session}/src
                              ${FLOW_LIKE_META_ROOT_ipc_shm}/src
                              ${isal_root}
-                             # So that generated-above .capnp.h are found by SRCS' .cpps' `#include`s.:
+                             # So that generated-above .capnp.h are found by SRCS' .cpps' `#include`s:
                              ${CMAKE_CURRENT_BINARY_DIR})
 
   # Do stuff we've resolved to do on all our targets.
@@ -101,14 +112,14 @@ set(SRCS
 set(CAPNP_SCHEMAS
     ${isal_root}/ipc/session/standalone/shm/arena_lend/jemalloc/test/test_message.capnp
     ${isal_root}/ipc/session/standalone/shm/arena_lend/jemalloc/test/test_session_metadata.capnp)
-handle_binary(libipc_unit_test.exec)
+handle_binary(libipc_unit_test)
 
 set(SRCS
     ${FLOW_LIKE_META_ROOT_flow}/src/flow/test/test_common_util.cpp
     ${isal_root}/ipc/shm/arena_lend/test/test_borrower.cpp
     test_borrower_main.cpp)
 unset(CAPNP_SCHEMAS)
-handle_binary(libipc_test_borrower.exec)
+handle_binary(libipc_test_borrower)
 
 set(SRCS
     ${FLOW_LIKE_META_ROOT_flow}/src/flow/test/test_common_util.cpp
@@ -121,4 +132,4 @@ set(SRCS
 set(CAPNP_SCHEMAS
     ${isal_root}/ipc/session/standalone/shm/arena_lend/jemalloc/test/test_message.capnp
     ${isal_root}/ipc/session/standalone/shm/arena_lend/jemalloc/test/test_session_metadata.capnp)
-handle_binary(libipc_test_jemalloc_shm_session_server.exec)
+handle_binary(libipc_test_jemalloc_shm_session_server)