Merge pull request #108 from APSIS-ANALYSIS/zhao-dev

Zhao dev
APSIS-ANALYSIS · Oct 19, 2023 · 7fffe9e · 7fffe9e
2 parents 3b5277b + a2f8241
commit 7fffe9e
Show file tree

Hide file tree

Showing 13 changed files with 185 additions and 49 deletions.
diff --git a/examples/fsi/CMakeLists.txt b/examples/fsi/CMakeLists.txt
@@ -73,6 +73,7 @@ SET( perigee_preprocess_lib_src
 SET( perigee_analysis_lib_src
   ${perigee_source}/System/PETSc_Tools.cpp
   ${perigee_source}/System/Matrix_PETSc.cpp
+  ${perigee_source}/System/Vector_3.cpp
   ${perigee_source}/System/Tensor2_3D.cpp
   ${perigee_source}/System/SymmTensor2_3D.cpp
   ${perigee_source}/System/Tensor4_3D.cpp
@@ -91,6 +92,7 @@ SET( perigee_analysis_lib_src
   ${perigee_source}/Analysis_Tool/APart_Node.cpp
   ${perigee_source}/Analysis_Tool/APart_Node_FSI.cpp
   ${perigee_source}/Analysis_Tool/FEANode.cpp
+  ${perigee_source}/Element/FE_Tools.cpp
   ${perigee_source}/Element/FEAElement_Triangle3_3D_der0.cpp
   ${perigee_source}/Element/FEAElement_Tet4.cpp
   ${perigee_source}/Model/MaterialModel_NeoHookean_M94_Mixed.cpp
@@ -125,6 +127,9 @@ SET( perigee_analysis_lib_src
 
 # 3. Postprocess cpp
 SET( perigee_postprocess_lib_src
+  ${perigee_source}/Mesh/VTK_Tools.cpp
+  ${perigee_source}/Mesh/Tet_Tools.cpp
+  ${perigee_source}/Mesh/Hex_Tools.cpp
   ${perigee_source}/Postproc_Tool/PostVectSolution.cpp
   ${perigee_source}/Postproc_Tool/Vis_Tools.cpp
   ${perigee_source}/Postproc_Tool/Interpolater.cpp
@@ -150,7 +155,7 @@ TARGET_LINK_LIBRARIES( perigee_preprocess ${EXTRA_LINK_LIBS} )
 
 # 2. Analysis libs
 ADD_LIBRARY( perigee_analysis ${perigee_analysis_lib_src} )
-TARGET_LINK_LIBRARIES( perigee_analysis ${EXTRA_LINK_LIBS} perigee_preprocess )
+TARGET_LINK_LIBRARIES( perigee_analysis ${EXTRA_LINK_LIBS} )
 
 # 3. Postprocess lib
 ADD_LIBRARY( perigee_postprocess ${perigee_postprocess_lib_src} )
@@ -182,4 +187,13 @@ TARGET_LINK_LIBRARIES( vis_fluid ${EXTRA_LINK_LIBS} perigee_postprocess )
 TARGET_LINK_LIBRARIES( vis_solid ${EXTRA_LINK_LIBS} perigee_postprocess )
 TARGET_LINK_LIBRARIES( vis_fsi_wss ${EXTRA_LINK_LIBS} perigee_postprocess )
 
+if(OPENMP_CXX_FOUND)
+  SET_TARGET_PROPERTIES( perigee_preprocess PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -DUSE_OPENMP" )
+  SET_TARGET_PROPERTIES( preprocess_fsi PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -DUSE_OPENMP" )
+  TARGET_INCLUDE_DIRECTORIES( perigee_preprocess PRIVATE ${OpenMP_CXX_INCLUDE_DIR} )
+  TARGET_INCLUDE_DIRECTORIES( preprocess_fsi PRIVATE ${OpenMP_CXX_INCLUDE_DIR} )
+  TARGET_LINK_LIBRARIES( perigee_preprocess ${OpenMP_CXX_LIBRARIES} )
+  TARGET_LINK_LIBRARIES( preprocess_fsi ${OpenMP_CXX_LIBRARIES} )
+endif()
+
 #EOF
diff --git a/examples/fsi/preprocess_fsi.cpp b/examples/fsi/preprocess_fsi.cpp
@@ -24,6 +24,10 @@
 
 int main( int argc, char * argv[] )
 {
+  // Set number of threads and  print info of OpenMP
+  SYS_T::print_omp_info();
+  SYS_T::set_omp_num_threads();
+
   // Remove previously existing hdf5 files
   if( SYS_T::directory_exist("apart") )
   {
@@ -250,7 +254,7 @@ int main( int argc, char * argv[] )
   // IEN for the solid element. If the solid element has node on the fluid-solid
   // interface, it will be mapped to the new index, that is nFunc + ii.
   std::vector<int> vecIEN_p ( vecIEN );
-
+  PERIGEE_OMP_PARALLEL_FOR
   for(int ee=0; ee<nElem; ++ee)
   {
     if( phy_tag[ee] == 1 )
@@ -272,32 +276,52 @@ int main( int argc, char * argv[] )
 
   // Generate the list of nodes for fluid and solid
   std::vector<int> v_node_f, v_node_s; v_node_f.clear(); v_node_s.clear();
-
-  for(int ee=0; ee<nElem; ++ee)
+  PERIGEE_OMP_PARALLEL
   {
-    if( phy_tag[ee] == 0 )
+    std::vector<int> temp_v_node_f {};
+    std::vector<int> temp_v_node_s {};
+    PERIGEE_OMP_FOR
+    for(int ee=0; ee<nElem; ++ee)
     {
-      for(int ii=0; ii<4; ++ii) v_node_f.push_back( IEN_v->get_IEN(ee, ii) );
+      if( phy_tag[ee] == 0 )
+      {
+        for(int ii=0; ii<4; ++ii) temp_v_node_f.push_back( IEN_v->get_IEN(ee, ii) );
+      }
+      else
+      {
+        for(int ii=0; ii<4; ++ii) temp_v_node_s.push_back( IEN_v->get_IEN(ee, ii) );
+      }
     }
-    else
+    PERIGEE_OMP_CRITICAL
     {
-      for(int ii=0; ii<4; ++ii) v_node_s.push_back( IEN_v->get_IEN(ee, ii) );
+      VEC_T::insert_end(v_node_f, temp_v_node_f);
+      VEC_T::insert_end(v_node_s, temp_v_node_s);
     }
   }
 
   VEC_T::sort_unique_resize( v_node_f ); VEC_T::sort_unique_resize( v_node_s );
 
   std::vector<int> p_node_f, p_node_s; p_node_f.clear(); p_node_s.clear();
-
-  for(int ee=0; ee<nElem; ++ee)
+  PERIGEE_OMP_PARALLEL
   {
-    if( phy_tag[ee] == 0 )
+    std::vector<int> temp_p_node_f {};
+    std::vector<int> temp_p_node_s {};
+    PERIGEE_OMP_FOR
+    for(int ee=0; ee<nElem; ++ee)
     {
-      for(int ii=0; ii<4; ++ii) p_node_f.push_back( IEN_p->get_IEN(ee, ii) );
+      if( phy_tag[ee] == 0 )
+      {
+        for(int ii=0; ii<4; ++ii) temp_p_node_f.push_back( IEN_p->get_IEN(ee, ii) );
+      }
+      else
+      {
+        for(int ii=0; ii<4; ++ii) temp_p_node_s.push_back( IEN_p->get_IEN(ee, ii) );
+      }
     }
-    else
+    PERIGEE_OMP_CRITICAL
     {
-      for(int ii=0; ii<4; ++ii) p_node_s.push_back( IEN_p->get_IEN(ee, ii) );
+      VEC_T::insert_end(p_node_f, temp_p_node_f);
+      VEC_T::insert_end(p_node_s, temp_p_node_s);
     }
   }
 

diff --git a/examples/ns/CMakeLists.txt b/examples/ns/CMakeLists.txt
@@ -65,9 +65,13 @@ SET( perigee_preprocess_lib_src
 SET( perigee_analysis_lib_src
   ${perigee_source}/System/PETSc_Tools.cpp
   ${perigee_source}/System/Matrix_PETSc.cpp
+  ${perigee_source}/System/Vector_3.cpp
   ${perigee_source}/System/Tensor2_3D.cpp
   ${perigee_source}/System/SymmTensor2_3D.cpp
+  ${perigee_source}/System/HDF5_Writer.cpp
+  ${perigee_source}/System/HDF5_Reader.cpp
   ${perigee_source}/Mesh/QuadPts_Gauss_Tet.cpp
+  ${perigee_source}/Mesh/QuadPts_Gauss_Triangle.cpp
   ${perigee_source}/Analysis_Tool/AGlobal_Mesh_Info_FEM_3D.cpp
   ${perigee_source}/Analysis_Tool/APart_Basic_Info.cpp
   ${perigee_source}/Analysis_Tool/ALocal_Elem.cpp
@@ -78,8 +82,11 @@ SET( perigee_analysis_lib_src
   ${perigee_source}/Analysis_Tool/ALocal_EBC_outflow.cpp
   ${perigee_source}/Analysis_Tool/APart_Node.cpp
   ${perigee_source}/Analysis_Tool/FEANode.cpp
+  ${perigee_source}/Element/FE_Tools.cpp
   ${perigee_source}/Element/FEAElement_Tet4.cpp
   ${perigee_source}/Element/FEAElement_Tet10_v2.cpp
+  ${perigee_source}/Element/FEAElement_Triangle3_3D_der0.cpp
+  ${perigee_source}/Element/FEAElement_Triangle6_3D_der0.cpp
   ${perigee_source}/Model/GenBC_RCR.cpp
   ${perigee_source}/Model/GenBC_Resistance.cpp
   ${perigee_source}/Model/GenBC_Inductance.cpp
@@ -100,6 +107,9 @@ SET( perigee_analysis_lib_src
   )
 
 SET( perigee_postprocess_lib_src
+  ${perigee_source}/Mesh/VTK_Tools.cpp
+  ${perigee_source}/Mesh/Tet_Tools.cpp
+  ${perigee_source}/Mesh/Hex_Tools.cpp
   ${perigee_source}/Mesh/QuadPts_vis_tri6.cpp
   ${perigee_source}/Mesh/QuadPts_vis_tet4.cpp
   ${perigee_source}/Mesh/QuadPts_vis_tet10_v2.cpp
@@ -121,12 +131,11 @@ TARGET_LINK_LIBRARIES( perigee_preprocess ${EXTRA_LINK_LIBS} )
 
 # 2. Analysis libs
 ADD_LIBRARY( perigee_analysis ${perigee_analysis_lib_src} )
-TARGET_LINK_LIBRARIES( perigee_analysis ${EXTRA_LINK_LIBS} perigee_preprocess )
+TARGET_LINK_LIBRARIES( perigee_analysis ${EXTRA_LINK_LIBS} )
 
 # 3. Postprocess lib
 ADD_LIBRARY( perigee_postprocess ${perigee_postprocess_lib_src} )
-TARGET_LINK_LIBRARIES( perigee_postprocess ${EXTRA_LINK_LIBS} perigee_analysis
-  perigee_preprocess )
+TARGET_LINK_LIBRARIES( perigee_postprocess ${EXTRA_LINK_LIBS} perigee_analysis )
 # -------------------------------------------------------------------
 
 # Print compiler setup
@@ -149,4 +158,13 @@ TARGET_LINK_LIBRARIES( vis_ns perigee_postprocess )
 TARGET_LINK_LIBRARIES( vis_p1_wss perigee_postprocess )
 TARGET_LINK_LIBRARIES( vis_p2_wss perigee_postprocess )
 
+if(OPENMP_CXX_FOUND)
+  SET_TARGET_PROPERTIES( perigee_preprocess PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -DUSE_OPENMP" )
+  SET_TARGET_PROPERTIES( preprocess3d PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -DUSE_OPENMP" )
+  TARGET_INCLUDE_DIRECTORIES( perigee_preprocess PRIVATE ${OpenMP_CXX_INCLUDE_DIR} )
+  TARGET_INCLUDE_DIRECTORIES( preprocess3d PRIVATE ${OpenMP_CXX_INCLUDE_DIR} )
+  TARGET_LINK_LIBRARIES( perigee_preprocess ${OpenMP_CXX_LIBRARIES} )
+  TARGET_LINK_LIBRARIES( preprocess3d ${OpenMP_CXX_LIBRARIES} )
+endif()
+
 # EOF
diff --git a/examples/ns/preprocess_ns.cpp b/examples/ns/preprocess_ns.cpp
@@ -21,6 +21,10 @@
 
 int main( int argc, char * argv[] )
 {
+  // Set number of threads and  print info of OpenMP
+  SYS_T::print_omp_info();
+  SYS_T::set_omp_num_threads();
+
   // Clean the potentially pre-existing hdf5 files in the job folder
   SYS_T::execute("rm -rf part_p*.h5");
   SYS_T::execute("rm -rf preprocessor_cmd.h5");

diff --git a/examples/ruc_fsi/CMakeLists.txt b/examples/ruc_fsi/CMakeLists.txt
@@ -69,10 +69,14 @@ SET( perigee_preprocess_lib_src
 
 SET( perigee_analysis_lib_src
   ${perigee_source}/System/PETSc_Tools.cpp
+  ${perigee_source}/System/Vector_3.cpp
   ${perigee_source}/System/Tensor2_3D.cpp
   ${perigee_source}/System/Matrix_PETSc.cpp
+  ${perigee_source}/System/HDF5_Writer.cpp
+  ${perigee_source}/System/HDF5_Reader.cpp
   ${perigee_source}/System/SymmTensor2_3D.cpp
   ${perigee_source}/Mesh/QuadPts_Gauss_Tet.cpp
+  ${perigee_source}/Mesh/QuadPts_Gauss_Triangle.cpp
   ${perigee_source}/Analysis_Tool/AGlobal_Mesh_Info_FEM_3D.cpp
   ${perigee_source}/Analysis_Tool/APart_Basic_Info.cpp
   ${perigee_source}/Analysis_Tool/ALocal_Elem.cpp
@@ -85,11 +89,14 @@ SET( perigee_analysis_lib_src
   ${perigee_source}/Analysis_Tool/ALocal_EBC_wall.cpp
   ${perigee_source}/Analysis_Tool/APart_Node.cpp
   ${perigee_source}/Analysis_Tool/FEANode.cpp
+  ${perigee_source}/Element/FE_Tools.cpp
   ${perigee_source}/Element/FEAElement_Tet4.cpp
   ${perigee_source}/Element/FEAElement_Tet10_v2.cpp
   ${perigee_source}/Element/FEAElement_Triangle3_membrane.cpp
   ${perigee_source}/Element/FEAElement_Triangle6_membrane.cpp
   ${perigee_source}/Element/FEAElement_Triangle6.cpp  # only for testing
+  ${perigee_source}/Element/FEAElement_Triangle3_3D_der0.cpp
+  ${perigee_source}/Element/FEAElement_Triangle6_3D_der0.cpp
   ${perigee_source}/Model/GenBC_RCR.cpp
   ${perigee_source}/Model/GenBC_Resistance.cpp
   ${perigee_source}/Model/GenBC_Inductance.cpp
@@ -114,6 +121,9 @@ SET( perigee_analysis_lib_src
 )
 
 SET( perigee_postprocess_lib_src
+  ${perigee_source}/Mesh/VTK_Tools.cpp
+  ${perigee_source}/Mesh/Tet_Tools.cpp
+  ${perigee_source}/Mesh/Hex_Tools.cpp
   ${perigee_source}/Mesh/QuadPts_vis_tri6.cpp
   ${perigee_source}/Mesh/QuadPts_vis_tet4.cpp
   ${perigee_source}/Mesh/QuadPts_vis_tet10_v2.cpp
@@ -124,10 +134,6 @@ SET( perigee_postprocess_lib_src
   ${perigee_SOURCE_DIR}/src/VTK_Writer_CMM.cpp
   )
 
-#SET( perigee_test_lib_src
-#  ${perigee_source}/Mesh/QuadPts_debug.cpp
-#  )
-
 INCLUDE_DIRECTORIES( ./include )
 INCLUDE_DIRECTORIES( ../../include )
 
@@ -139,16 +145,11 @@ TARGET_LINK_LIBRARIES( perigee_preprocess ${EXTRA_LINK_LIBS} )
 
 # 2. Analysis libs
 ADD_LIBRARY( perigee_analysis ${perigee_analysis_lib_src} )
-TARGET_LINK_LIBRARIES( perigee_analysis ${EXTRA_LINK_LIBS} perigee_preprocess )
+TARGET_LINK_LIBRARIES( perigee_analysis ${EXTRA_LINK_LIBS} )
 
 # 3. Postprocess lib
 ADD_LIBRARY( perigee_postprocess ${perigee_postprocess_lib_src} )
-TARGET_LINK_LIBRARIES( perigee_postprocess ${EXTRA_LINK_LIBS} perigee_analysis
-  perigee_preprocess )
-
-# 4. Test lib
-#ADD_LIBRARY( perigee_test ${perigee_test_lib_src} )
-#TARGET_LINK_LIBRARIES( perigee_test ${EXTRA_LINK_LIBS} perigee_analysis )
+TARGET_LINK_LIBRARIES( perigee_postprocess ${EXTRA_LINK_LIBS} perigee_analysis )
 # -------------------------------------------------------------------
 
 # Print compiler setup
@@ -165,9 +166,6 @@ ADD_EXECUTABLE( vis_ruc       vis_ruc.cpp)
 ADD_EXECUTABLE( vis_p1_wss    vis_p1_wss.cpp)
 ADD_EXECUTABLE( vis_p2_wss    vis_p2_wss.cpp)
 
-#ADD_EXECUTABLE( test test.cpp )
-#ADD_EXECUTABLE( test_element  test_membrane_element_routine.cpp )
-
 TARGET_LINK_LIBRARIES( preprocess3d  perigee_preprocess )
 TARGET_LINK_LIBRARIES( ruc_tet_3d    perigee_analysis )
 TARGET_LINK_LIBRARIES( wall_solver   perigee_analysis )
@@ -176,7 +174,13 @@ TARGET_LINK_LIBRARIES( vis_ruc       perigee_postprocess )
 TARGET_LINK_LIBRARIES( vis_p1_wss    perigee_postprocess )
 TARGET_LINK_LIBRARIES( vis_p2_wss    perigee_postprocess )
 
-#TARGET_LINK_LIBRARIES( test perigee_preprocess )
-#TARGET_LINK_LIBRARIES( test_element  perigee_test )
+if(OPENMP_CXX_FOUND)
+  SET_TARGET_PROPERTIES( perigee_preprocess PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -DUSE_OPENMP" )
+  SET_TARGET_PROPERTIES( preprocess3d PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -DUSE_OPENMP" )
+  TARGET_INCLUDE_DIRECTORIES( perigee_preprocess PRIVATE ${OpenMP_CXX_INCLUDE_DIR} )
+  TARGET_INCLUDE_DIRECTORIES( preprocess3d PRIVATE ${OpenMP_CXX_INCLUDE_DIR} )
+  TARGET_LINK_LIBRARIES( perigee_preprocess ${OpenMP_CXX_LIBRARIES} )
+  TARGET_LINK_LIBRARIES( preprocess3d ${OpenMP_CXX_LIBRARIES} )
+endif()
 
 # EOF
diff --git a/examples/ruc_fsi/preprocess_tets.cpp b/examples/ruc_fsi/preprocess_tets.cpp
@@ -30,6 +30,10 @@
 
 int main( int argc, char * argv[] )
 {
+  // Set number of threads and  print info of OpenMP
+  SYS_T::print_omp_info();
+  SYS_T::set_omp_num_threads();
+
   // Clean the potentially pre-existing hdf5 files in the job folder
   SYS_T::execute("rm -rf preprocessor_cmd.h5");
   SYS_T::execute("rm -rf apart"); 

diff --git a/include/Sys_Tools.hpp b/include/Sys_Tools.hpp
@@ -21,11 +21,13 @@
 #define PERIGEE_OMP_PARALLEL _Pragma("omp parallel")
 #define PERIGEE_OMP_FOR _Pragma("omp for")
 #define PERIGEE_OMP_CRITICAL _Pragma("omp critical")
+#define PERIGEE_OMP_SINGLE _Pragma("omp single")
 #else
 #define PERIGEE_OMP_PARALLEL_FOR
 #define PERIGEE_OMP_PARALLEL
 #define PERIGEE_OMP_FOR
 #define PERIGEE_OMP_CRITICAL
+#define PERIGEE_OMP_SINGLE
 #endif
 
 #define PETSC_SILENCE_DEPRECATION_WARNINGS_3_19_0
@@ -332,6 +334,32 @@ namespace SYS_T
     }
   }
 
+  // 4. print the number of threads used in openmp
+  inline void print_omp_info()
+  {
+#ifdef _OPENMP
+    PERIGEE_OMP_PARALLEL
+    {
+      PERIGEE_OMP_SINGLE
+      {
+        std::cout<<"The number of threads used: "<<omp_get_num_threads()<<", and ";
+        std::cout<<"the number of processors on the machine: ";
+        std::cout<<omp_get_num_procs()<<".\n";
+      }
+    }
+#else
+    std::cout<<"OpenMP is not invoked.\n";
+#endif
+  }
+
+  // 5. set the number of threads used in openmp
+  inline void set_omp_num_threads()
+  {
+#ifdef _OPENMP
+    omp_set_num_threads( omp_get_num_procs() );
+#endif
+  }
+
   // ================================================================
   // The following are system functions that access the system info.
   // ================================================================

diff --git a/src/Mesh/NBC_Partition.cpp b/src/Mesh/NBC_Partition.cpp
@@ -28,7 +28,7 @@ NBC_Partition::NBC_Partition( const IPart * const &part,
     } // end jj-loop
 
     Num_LPS[ii] = 0; Num_LPM[ii] = 0;
-
+    PERIGEE_OMP_PARALLEL_FOR
     for(unsigned int jj=0; jj<nbc_list[ii]->get_num_per_nodes(); ++jj)
     {
       unsigned int node_ps = nbc_list[ii]->get_per_slave_nodes(jj);
@@ -59,7 +59,7 @@ NBC_Partition::NBC_Partition( const IPart * const &part,
   const int totnode = part->get_nlocghonode();
 
   LID.resize(totnode * dof);
-
+  PERIGEE_OMP_PARALLEL_FOR
   for(int ii=0; ii<dof; ++ii)
   {
     for(int jj=0; jj<totnode; ++jj)
@@ -69,7 +69,7 @@ NBC_Partition::NBC_Partition( const IPart * const &part,
       LID[ii*totnode + jj] = nbc_list[ii]->get_ID(old_index);
     }
   }
-
+  PERIGEE_OMP_PARALLEL_FOR
   for(int ii=0; ii<dof*totnode; ++ii)
   {
     if(LID[ii] != -1) LID[ii] = mnindex->get_old2new(LID[ii]);