Notified RMA counters memory allocation in the shared memory segment for a single and multi rank window.

Signed-off-by: Joseph Antony <jajoseph.antony18@gmail.com>

Author: Joseph Antony

ompi/mca/osc/sm/osc_sm.h

@@ -22,6 +22,7 @@ typedef uint64_t osc_sm_post_type_t;
 typedef opal_atomic_uint64_t osc_sm_post_atomic_type_t;
 #define OSC_SM_POST_BITS 6
 #define OSC_SM_POST_MASK 0x3f
+#define OSC_SM_MAX_NOTIFY_COUNTERS 16
 
 /* data shared across all peers */
 struct ompi_osc_sm_global_state_t {
@@ -79,7 +80,7 @@ struct ompi_osc_sm_module_t {
     size_t *sizes;
     void **bases;
     ptrdiff_t *disp_units;
-    uint64_t **notify_counters;
+    uint64_t *notify_counters;
 
 
     ompi_group_t *start_group;

ompi/mca/osc/sm/osc_sm_component.c

@@ -255,12 +255,17 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis
         module->posts = calloc (1, sizeof(module->posts[0]) + sizeof (module->posts[0][0]));
         if (NULL == module->posts) return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
         module->posts[0] = (osc_sm_post_atomic_type_t *) (module->posts + 1);
+
+        /* allocate notify counters for single process case */
+        module->notify_counters = calloc(OSC_SM_MAX_NOTIFY_COUNTERS, sizeof(uint64_t));
+        if (NULL == module->notify_counters) return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
     } else {
         unsigned long total, *rbuf;
         int i, flag;
         size_t pagesize;
         size_t state_size;
         size_t posts_size, post_size = (comm_size + OSC_SM_POST_MASK) / (OSC_SM_POST_MASK + 1);
+        size_t notify_counters_size;
         size_t data_base_size;
 
         opal_output_verbose(MCA_BASE_VERBOSE_DEBUG, ompi_osc_base_framework.framework_output,
@@ -316,7 +321,9 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis
         state_size += OPAL_ALIGN_PAD_AMOUNT(state_size, 64);
         posts_size = comm_size * post_size * sizeof (module->posts[0][0]);
         posts_size += OPAL_ALIGN_PAD_AMOUNT(posts_size, 64);
-        data_base_size = state_size + posts_size;
+        notify_counters_size = OSC_SM_MAX_NOTIFY_COUNTERS * sizeof(uint64_t);
+        notify_counters_size += OPAL_ALIGN_PAD_AMOUNT(notify_counters_size, 64);
+        data_base_size = state_size + posts_size + notify_counters_size;
         data_base_size += OPAL_ALIGN_PAD_AMOUNT(data_base_size, pagesize);
         if (0 == ompi_comm_rank (module->comm)) {
             char *data_file;
@@ -377,6 +384,12 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis
         module->global_state = (ompi_osc_sm_global_state_t *) (module->posts[0] + comm_size * post_size);
         module->node_states = (ompi_osc_sm_node_state_t *) (module->global_state + 1);
 
+        /* set up notify counters in shared memory after node_states */
+        module->notify_counters = (uint64_t *) ((char *)(module->node_states + comm_size) +
+                                   OPAL_ALIGN_PAD_AMOUNT((uintptr_t)(module->node_states + comm_size), 64));
+        /* zero out notify counters */
+        memset(module->notify_counters, 0, OSC_SM_MAX_NOTIFY_COUNTERS * sizeof(uint64_t));
+
         for (i = 0, total = data_base_size ; i < comm_size ; ++i) {
             if (i > 0) {
                 module->posts[i] = module->posts[i - 1] + post_size;
@@ -555,13 +568,16 @@ ompi_osc_sm_free(struct ompi_win_t *win)
                                           module->comm->c_coll->coll_barrier_module);
 
         opal_shmem_segment_detach (&module->seg_ds);
+        /* notify_counters points into shared memory segment, no separate free needed */
     } else {
         free(module->node_states);
         free(module->global_state);
         if (NULL != module->bases) {
             mca_mpool_base_default_module->mpool_free(mca_mpool_base_default_module,
                                                       module->bases[0]);
         }
+        /* free notify_counters for single process case */
+        free(module->notify_counters);
     }
     free(module->disp_units);
     free(module->outstanding_locks);