First draft and TODOs for notified comm

Signed-off-by: Joseph Schuchart <joseph.schuchart@stonybrook.edu>

Author: devreal

ompi/mca/osc/osc.h

@@ -370,6 +370,8 @@ typedef int (*ompi_osc_base_module_flush_local_all_fn_t)(struct ompi_win_t *win)
  * free to create a structure that inherits this one for use as the
  * module structure.
  */
+
+ // TODO: extend the struct and add pointers to put/get_with_notify functions
 struct ompi_osc_base_module_4_0_0_t {
     ompi_osc_base_module_win_shared_query_fn_t osc_win_shared_query;
 

ompi/mca/osc/sm/osc_sm.h

@@ -79,6 +79,8 @@ struct ompi_osc_sm_module_t {
     size_t *sizes;
     void **bases;
     ptrdiff_t *disp_units;
+    uint64_t **notify_counters;
+
 
     ompi_group_t *start_group;
     ompi_group_t *post_group;
@@ -105,6 +107,8 @@ int ompi_osc_sm_detach(struct ompi_win_t *win, const void *base);
 
 int ompi_osc_sm_free(struct ompi_win_t *win);
 
+// TODO: add put/get_with_notify prototypes
+
 int ompi_osc_sm_put(const void *origin_addr,
                           size_t origin_count,
                           struct ompi_datatype_t *origin_dt,

ompi/mca/osc/sm/osc_sm_comm.c

@@ -267,6 +267,41 @@ ompi_osc_sm_get(void *origin_addr,
 }
 
 
+int
+ompi_osc_sm_get_with_notify(void *origin_addr,
+                size_t origin_count,
+                struct ompi_datatype_t *origin_dt,
+                int target,
+                ptrdiff_t target_disp,
+                size_t target_count,
+                struct ompi_datatype_t *target_dt,
+                int notify,
+                struct ompi_win_t *win)
+{
+    int ret;
+    ompi_osc_sm_module_t *module =
+        (ompi_osc_sm_module_t*) win->w_osc_module;
+    void *remote_address;
+
+    OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
+                         "get: 0x%lx, %zu, %s, %d, %d, %zu, %s, 0x%lx",
+                         (unsigned long) origin_addr, origin_count,
+                         origin_dt->name, target, (int) target_disp,
+                         target_count, target_dt->name,
+                         (unsigned long) win));
+
+    remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp;
+
+    ret = ompi_datatype_sndrcv(remote_address, target_count, target_dt,
+                               origin_addr, origin_count, origin_dt);
+    // TODO: do the same for put_with_notify
+    opal_atomic_rmb();
+    opal_atomic_add(&module->notify_counters[target][notify], 1);
+
+    return ret;
+}
+
+
 int
 ompi_osc_sm_accumulate(const void *origin_addr,
                        size_t origin_count,

ompi/mca/osc/sm/osc_sm_component.c

@@ -70,6 +70,8 @@ ompi_osc_sm_component_t mca_osc_sm_component = {
 MCA_BASE_COMPONENT_INIT(ompi, osc, sm)
 
 
+// TODO: extend the struct and add pointers to put/get_with_notify functions
+// TODO: extend it to rput/rget_with_notify as well
 ompi_osc_sm_module_t ompi_osc_sm_module_template = {
     {
         .osc_win_shared_query = ompi_osc_sm_shared_query,