diff --git a/ompi/errhandler/errcode.c b/ompi/errhandler/errcode.c index 631f1483b12..d04b6f9069c 100644 --- a/ompi/errhandler/errcode.c +++ b/ompi/errhandler/errcode.c @@ -131,6 +131,7 @@ static ompi_mpi_errcode_t ompi_err_value_too_large; static ompi_mpi_errcode_t ompi_err_errhandler; static ompi_mpi_errcode_t ompi_t_err_not_accessible; static ompi_mpi_errcode_t ompi_t_err_not_supported; +static ompi_mpi_errcode_t ompi_err_rma_notification; static void ompi_mpi_errcode_construct(ompi_mpi_errcode_t* errcode); static void ompi_mpi_errcode_destruct(ompi_mpi_errcode_t* errcode); @@ -252,6 +253,7 @@ int ompi_mpi_errcode_init (void) CONSTRUCT_ERRCODE( ompi_err_errhandler, MPI_ERR_ERRHANDLER, "MPI_ERR_ERRHANDLER: Invalid error handler handle" ); CONSTRUCT_ERRCODE( ompi_t_err_not_accessible, MPI_T_ERR_NOT_ACCESSIBLE, "MPI_T_ERR_NOT_ACCESSIBLE: Requested functionality is not accessible" ); CONSTRUCT_ERRCODE( ompi_t_err_not_supported, MPI_T_ERR_NOT_SUPPORTED, "MPI_T_ERR_NOT_SUPPORTED: Requested functionality not supported" ); + CONSTRUCT_ERRCODE( ompi_err_rma_notification, MPI_ERR_RMA_NOTIFICATION, "MPI_ERR_RMA_NOTIFICATION: Invalid notification index passed to MPI call" ); /* Per MPI-3 p353:27-32, MPI_LASTUSEDCODE must be >= MPI_ERR_LASTCODE. So just start it as == MPI_ERR_LASTCODE. */ @@ -373,6 +375,7 @@ int ompi_mpi_errcode_finalize (void) OBJ_DESTRUCT(&ompi_err_errhandler); OBJ_DESTRUCT(&ompi_t_err_not_accessible); OBJ_DESTRUCT(&ompi_t_err_not_supported); + OBJ_DESTRUCT(&ompi_err_rma_notification); OBJ_DESTRUCT(&ompi_mpi_errcodes); ompi_mpi_errcode_lastpredefined = 0; opal_mutex_unlock(&errcode_lock); diff --git a/ompi/include/mpi.h.in b/ompi/include/mpi.h.in index 43e9006257e..71afc4cefbc 100644 --- a/ompi/include/mpi.h.in +++ b/ompi/include/mpi.h.in @@ -767,6 +767,7 @@ enum { #define MPI_ERR_ERRHANDLER 80 #define MPI_T_ERR_NOT_ACCESSIBLE 81 #define MPI_T_ERR_NOT_SUPPORTED 82 +#define MPI_ERR_RMA_NOTIFICATION 83 /* Per MPI-3 p349 47, MPI_ERR_LASTCODE must be >= the last predefined MPI_ERR_ code. Set the last code to allow some room for adding @@ -1921,6 +1922,14 @@ OMPI_DECLSPEC int MPI_Get_c(void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Win win); +OMPI_DECLSPEC int MPI_Get_notify(void *origin_addr, int origin_count, + MPI_Datatype origin_datatype, int target_rank, + MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); +OMPI_DECLSPEC int MPI_Get_notify_c(void *origin_addr, MPI_Count origin_count, + MPI_Datatype origin_datatype, int target_rank, + MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); OMPI_DECLSPEC int MPI_Get_accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, void *result_addr, int result_count, MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp, int target_count, @@ -2184,6 +2193,12 @@ OMPI_DECLSPEC int MPI_Put(const void *origin_addr, int origin_count, MPI_Dataty OMPI_DECLSPEC int MPI_Put_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Win win); +OMPI_DECLSPEC int MPI_Put_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); +OMPI_DECLSPEC int MPI_Put_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); OMPI_DECLSPEC int MPI_Query_thread(int *provided); OMPI_DECLSPEC int MPI_Raccumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, int target_count, @@ -2608,6 +2623,10 @@ OMPI_DECLSPEC int MPI_Win_get_errhandler(MPI_Win win, MPI_Errhandler *errhandle OMPI_DECLSPEC int MPI_Win_get_group(MPI_Win win, MPI_Group *group); OMPI_DECLSPEC int MPI_Win_get_info(MPI_Win win, MPI_Info *info_used); OMPI_DECLSPEC int MPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen); +OMPI_DECLSPEC int MPI_Win_get_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); +OMPI_DECLSPEC int MPI_Win_reset_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); +OMPI_DECLSPEC int MPI_Win_get_num_notify(MPI_Win win, int target_rank, int *num_notifications); +OMPI_DECLSPEC int MPI_Win_set_num_notify(MPI_Win win, MPI_Info info, int num_notifications); OMPI_DECLSPEC int MPI_Win_lock(int lock_type, int rank, int mpi_assert, MPI_Win win); OMPI_DECLSPEC int MPI_Win_lock_all(int mpi_assert, MPI_Win win); OMPI_DECLSPEC int MPI_Win_post(MPI_Group group, int mpi_assert, MPI_Win win); @@ -3095,6 +3114,14 @@ OMPI_DECLSPEC int PMPI_Get_c(void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Win win); +OMPI_DECLSPEC int PMPI_Get_notify(void *origin_addr, int origin_count, + MPI_Datatype origin_datatype, int target_rank, + MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); +OMPI_DECLSPEC int PMPI_Get_notify_c(void *origin_addr, MPI_Count origin_count, + MPI_Datatype origin_datatype, int target_rank, + MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); OMPI_DECLSPEC int PMPI_Get_accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, void *result_addr, int result_count, MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp, int target_count, @@ -3358,6 +3385,12 @@ OMPI_DECLSPEC int PMPI_Put(const void *origin_addr, int origin_count, MPI_Datat OMPI_DECLSPEC int PMPI_Put_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Win win); +OMPI_DECLSPEC int PMPI_Put_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); +OMPI_DECLSPEC int PMPI_Put_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); OMPI_DECLSPEC int PMPI_Query_thread(int *provided); OMPI_DECLSPEC int PMPI_Raccumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, int target_count, @@ -3782,6 +3815,10 @@ OMPI_DECLSPEC int PMPI_Win_get_errhandler(MPI_Win win, MPI_Errhandler *errhandl OMPI_DECLSPEC int PMPI_Win_get_group(MPI_Win win, MPI_Group *group); OMPI_DECLSPEC int PMPI_Win_get_info(MPI_Win win, MPI_Info *info_used); OMPI_DECLSPEC int PMPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen); +OMPI_DECLSPEC int PMPI_Win_get_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); +OMPI_DECLSPEC int PMPI_Win_reset_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); +OMPI_DECLSPEC int PMPI_Win_get_num_notify(MPI_Win win, int target_rank, int *num_notifications); +OMPI_DECLSPEC int PMPI_Win_set_num_notify(MPI_Win win, MPI_Info info, int num_notifications); OMPI_DECLSPEC int PMPI_Win_lock(int lock_type, int rank, int mpi_assert, MPI_Win win); OMPI_DECLSPEC int PMPI_Win_lock_all(int mpi_assert, MPI_Win win); OMPI_DECLSPEC int PMPI_Win_post(MPI_Group group, int mpi_assert, MPI_Win win); diff --git a/ompi/include/mpif-values.py b/ompi/include/mpif-values.py index 7cd50d7e9e4..62cd9c4e421 100755 --- a/ompi/include/mpif-values.py +++ b/ompi/include/mpif-values.py @@ -304,6 +304,7 @@ 'MPI_ERR_ERRHANDLER': 80, 'MPI_T_ERR_NOT_ACCESSIBLE': 81, 'MPI_T_ERR_NOT_SUPPORTED': 82, + 'MPI_ERR_RMA_NOTIFICATION': 83, 'MPI_ERR_LASTCODE': 92, 'MPI_IDENT': 0, 'MPI_CONGRUENT': 1, diff --git a/ompi/mca/osc/osc.h b/ompi/mca/osc/osc.h index 39063ef0914..2f66c2ac107 100644 --- a/ompi/mca/osc/osc.h +++ b/ompi/mca/osc/osc.h @@ -216,6 +216,15 @@ typedef int (*ompi_osc_base_module_put_fn_t)(const void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); +typedef int (*ompi_osc_base_module_put_notify_fn_t)(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win); typedef int (*ompi_osc_base_module_get_fn_t)(void *origin_addr, size_t origin_count, @@ -226,6 +235,36 @@ typedef int (*ompi_osc_base_module_get_fn_t)(void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); +typedef int (*ompi_osc_base_module_get_notify_fn_t)(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win); + +typedef int (*ompi_osc_base_module_win_get_notify_value_fn_t)(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE *value); + +typedef int (*ompi_osc_base_module_win_reset_notify_value_fn_t)(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE *value); + +/* MPI-5.1 section 12.6.1. Blocking, synchronizing collective; sets the number + * of notification counters attached at the calling MPI process to exactly + * num_notifications and resets every counter to zero. */ +typedef int (*ompi_osc_base_module_win_set_num_notify_fn_t)(struct ompi_win_t *win, + struct opal_info_t *info, + int num_notifications); + +/* MPI-5.1 section 12.6.1. Local; returns the number of notification counters + * attached at target_rank. */ +typedef int (*ompi_osc_base_module_win_get_num_notify_fn_t)(struct ompi_win_t *win, + int target_rank, + int *num_notifications); typedef int (*ompi_osc_base_module_accumulate_fn_t)(const void *origin_addr, size_t origin_count, @@ -276,6 +315,17 @@ typedef int (*ompi_osc_base_module_rput_fn_t)(const void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); +typedef int (*ompi_osc_base_module_rput_notify_fn_t)(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); + typedef int (*ompi_osc_base_module_rget_fn_t)(void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -286,6 +336,16 @@ typedef int (*ompi_osc_base_module_rget_fn_t)(void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); +typedef int (*ompi_osc_base_module_rget_notify_fn_t)(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); typedef int (*ompi_osc_base_module_raccumulate_fn_t)(const void *origin_addr, size_t origin_count, @@ -370,6 +430,7 @@ typedef int (*ompi_osc_base_module_flush_local_all_fn_t)(struct ompi_win_t *win) * free to create a structure that inherits this one for use as the * module structure. */ + struct ompi_osc_base_module_4_0_0_t { ompi_osc_base_module_win_shared_query_fn_t osc_win_shared_query; @@ -407,6 +468,14 @@ struct ompi_osc_base_module_4_0_0_t { ompi_osc_base_module_flush_all_fn_t osc_flush_all; ompi_osc_base_module_flush_local_fn_t osc_flush_local; ompi_osc_base_module_flush_local_all_fn_t osc_flush_local_all; + ompi_osc_base_module_put_notify_fn_t osc_put_notify; + ompi_osc_base_module_get_notify_fn_t osc_get_notify; + ompi_osc_base_module_win_get_notify_value_fn_t osc_win_get_notify_value; + ompi_osc_base_module_win_reset_notify_value_fn_t osc_win_reset_notify_value; + ompi_osc_base_module_win_set_num_notify_fn_t osc_win_set_num_notify; + ompi_osc_base_module_win_get_num_notify_fn_t osc_win_get_num_notify; + ompi_osc_base_module_rput_notify_fn_t osc_rput_notify; + ompi_osc_base_module_rget_notify_fn_t osc_rget_notify; }; typedef struct ompi_osc_base_module_4_0_0_t ompi_osc_base_module_4_0_0_t; typedef ompi_osc_base_module_4_0_0_t ompi_osc_base_module_t; diff --git a/ompi/mca/osc/sm/osc_sm.h b/ompi/mca/osc/sm/osc_sm.h index 363d3429a63..88962891ecd 100644 --- a/ompi/mca/osc/sm/osc_sm.h +++ b/ompi/mca/osc/sm/osc_sm.h @@ -23,6 +23,13 @@ typedef opal_atomic_uint64_t osc_sm_post_atomic_type_t; #define OSC_SM_POST_BITS 6 #define OSC_SM_POST_MASK 0x3f +/* Capacity of the per-rank notification counter region reserved in the shared + * segment at window creation, i.e. the effective + * MPI_WIN_NOTIFICATION_NUM_UB. How many of those counters are actually + * *attached* is a separate, per-rank quantity that starts at zero and is set by + * MPI_WIN_SET_NUM_NOTIFY (MPI-5.1 section 12.6.1). */ +#define OSC_SM_MAX_NOTIFY_COUNTERS 16 + /* data shared across all peers */ struct ompi_osc_sm_global_state_t { int use_barrier_for_fence; @@ -47,6 +54,13 @@ struct ompi_osc_sm_node_state_t { opal_atomic_int32_t complete_count; ompi_osc_sm_lock_t lock; opal_atomic_lock_t accumulate_lock; + /* Number of notification counters currently *attached* at this rank. Zero + * until MPI_WIN_SET_NUM_NOTIFY is called (MPI-5.1 section 12.6.1). Lives in + * the shared segment so that an origin can validate a notification index + * against the target's attached count without any communication. */ + uint32_t notify_counter_count; + uint64_t notify_counter_offset; /* offset from segment_base, not raw pointer */ + }; typedef struct ompi_osc_sm_node_state_t ompi_osc_sm_node_state_t; @@ -79,6 +93,11 @@ struct ompi_osc_sm_module_t { size_t *sizes; void **bases; ptrdiff_t *disp_units; + /* Base of the notification counter region. Typed atomic so that plain + * loads are atomic (and never hoisted out of a caller's polling loop) while + * remote origins increment the same location with opal_atomic_add(). */ + opal_atomic_int64_t *notify_counters; + ompi_group_t *start_group; ompi_group_t *post_group; @@ -105,6 +124,7 @@ int ompi_osc_sm_detach(struct ompi_win_t *win, const void *base); int ompi_osc_sm_free(struct ompi_win_t *win); + int ompi_osc_sm_put(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -114,6 +134,16 @@ int ompi_osc_sm_put(const void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); + int ompi_osc_sm_put_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win); + int ompi_osc_sm_get(void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -123,6 +153,32 @@ int ompi_osc_sm_get(void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); +int ompi_osc_sm_get_notify(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win); + +int ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE *value); + +int ompi_osc_sm_win_reset_notify_value(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE *value); + +int ompi_osc_sm_win_set_num_notify(struct ompi_win_t *win, + struct opal_info_t *info, + int num_notifications); + +int ompi_osc_sm_win_get_num_notify(struct ompi_win_t *win, + int target_rank, + int *num_notifications); + int ompi_osc_sm_accumulate(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -172,6 +228,17 @@ int ompi_osc_sm_rput(const void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); +int ompi_osc_sm_rput_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); + int ompi_osc_sm_rget(void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -182,6 +249,17 @@ int ompi_osc_sm_rget(void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); +int ompi_osc_sm_rget_notify(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); + int ompi_osc_sm_raccumulate(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c index bbd5873bf96..a5b27b65fc4 100644 --- a/ompi/mca/osc/sm/osc_sm_comm.c +++ b/ompi/mca/osc/sm/osc_sm_comm.c @@ -17,9 +17,174 @@ #include "ompi/mca/osc/osc.h" #include "ompi/mca/osc/base/base.h" #include "ompi/mca/osc/base/osc_base_obj_convert.h" +#include "ompi/communicator/communicator.h" #include "osc_sm.h" +static inline opal_atomic_int64_t * +osc_sm_target_notify_base(ompi_osc_sm_module_t *module, int target) +{ + if (NULL == module->segment_base) { + /* single-rank path: notify_counters is a regular local allocation */ + return module->notify_counters; + } + + return (opal_atomic_int64_t *) ((char *) module->segment_base + + module->node_states[target].notify_counter_offset); +} + +/* MPI-5.1 section 12.6.1: "The notification counter referenced by a notified + * communication operation must be attached to the window at the target before + * the operation is initiated at the origin. Initiating a notified + * communication operation that references a notification counter that is out of + * range at the target is erroneous." + * + * The check is therefore against the *target's* attached count, and it must + * happen before any data is moved -- otherwise an erroneous call would still + * have overwritten the target window (or, for get, the origin buffer) by the + * time the error is reported. */ +static inline int +osc_sm_check_notify_idx(ompi_osc_sm_module_t *module, int target, int notify) +{ + if (notify < 0 || (uint32_t) notify >= module->node_states[target].notify_counter_count) { + return MPI_ERR_RMA_NOTIFICATION; + } + + return OMPI_SUCCESS; +} + +int +ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE *value) +{ + ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t *) win->w_osc_module; + int rank = ompi_comm_rank(module->comm); + int ret; + + ret = osc_sm_check_notify_idx(module, rank, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Acquire ordering: sample the counter first, then fence, so that window + * loads issued by the caller after this call cannot be satisfied by values + * read before the notification was observed. A barrier placed ahead of the + * counter load would order nothing useful. MPI-5.1 section 12.6.2 requires + * this procedure to synchronize the public and private window copies as if + * MPI_WIN_SYNC had been called; osc/sm is a unified-memory-model component, + * so the barrier is the whole of that synchronization. + * + * The load itself is atomic because notify_counters is opal_atomic_int64_t: + * remote origins bump the same location concurrently, and the standard's + * usage model is to poll this procedure in a loop. */ + *value = (OMPI_MPI_COUNT_TYPE) osc_sm_target_notify_base(module, rank)[notify]; + opal_atomic_rmb(); + + return OMPI_SUCCESS; +} + +int +ompi_osc_sm_win_reset_notify_value(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE *value) +{ + ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t *) win->w_osc_module; + int rank = ompi_comm_rank(module->comm); + int ret; + + ret = osc_sm_check_notify_idx(module, rank, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Atomically swap the counter to 0 and return the previous value. Must be + * a single atomic so that increments arriving from other MPI processes + * between the read and the zeroing are not lost. */ + *value = (OMPI_MPI_COUNT_TYPE) opal_atomic_swap_64( + &osc_sm_target_notify_base(module, rank)[notify], 0); + opal_atomic_rmb(); + + return OMPI_SUCCESS; +} + +int +ompi_osc_sm_win_set_num_notify(struct ompi_win_t *win, + struct opal_info_t *info, + int num_notifications) +{ + ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t *) win->w_osc_module; + int rank = ompi_comm_rank(module->comm); + int ret; + + /* "mpi_assert_same_num_notifications" is an optimization hint only, and + * osc/sm reserves the same fixed number of counters at every rank + * regardless, so there is nothing to specialize on. */ + (void) info; + + if (num_notifications < 0) { + return MPI_ERR_ARG; + } + + /* A fixed region of OSC_SM_MAX_NOTIFY_COUNTERS counters per rank is carved + * out of the shared segment at window creation, so that is the effective + * MPI_WIN_NOTIFICATION_NUM_UB. Asking for more cannot be satisfied without + * re-creating the segment. */ + if (num_notifications > OSC_SM_MAX_NOTIFY_COUNTERS) { + return MPI_ERR_ARG; + } + + /* MPI-5.1 section 12.6.1: "A subsequent call to MPI_WIN_GET_NUM_NOTIFY will + * return the value given to MPI_WIN_SET_NUM_NOTIFY." The count is set to + * exactly what was asked for -- note this differs from earlier drafts of + * the chapter, which forbade decreasing it. + * + * "All notification counters (both existing and newly attached) are reset + * to zero by this call." Zero the whole reserved region rather than just + * the attached prefix, so that counters left over from a previous, larger + * attachment cannot resurface if the count is raised again. It is + * erroneous to call this while an access epoch is open, so no origin can be + * incrementing our counters concurrently and plain stores are sufficient. */ + memset((void *) osc_sm_target_notify_base(module, rank), 0, + OSC_SM_MAX_NOTIFY_COUNTERS * sizeof(int64_t)); + module->node_states[rank].notify_counter_count = (uint32_t) num_notifications; + opal_atomic_wmb(); + + /* "MPI_WIN_SET_NUM_NOTIFY is a blocking, synchronizing collective + * procedure; it will not return until all MPI processes in the group of the + * window have called the function and all processes have adjusted the + * number of notification counters attached to the window." Each rank + * publishes its own count into the shared segment above, so the barrier is + * all that is needed to make every rank's count visible to every other -- + * no counts have to be exchanged. */ + ret = module->comm->c_coll->coll_barrier(module->comm, + module->comm->c_coll->coll_barrier_module); + if (OMPI_SUCCESS != ret) { + return ret; + } + + return OMPI_SUCCESS; +} + +int +ompi_osc_sm_win_get_num_notify(struct ompi_win_t *win, + int target_rank, + int *num_notifications) +{ + ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t *) win->w_osc_module; + + if (target_rank < 0 || target_rank >= ompi_comm_size(module->comm)) { + return MPI_ERR_RANK; + } + + /* MPI-5.1 section 12.6.1: local procedure returning the number of counters + * attached at target_rank. Every rank's count is published in the shared + * segment by MPI_WIN_SET_NUM_NOTIFY, so this is a plain read. */ + *num_notifications = (int) module->node_states[target_rank].notify_counter_count; + + return OMPI_SUCCESS; +} + int ompi_osc_sm_rput(const void *origin_addr, size_t origin_count, @@ -59,6 +224,58 @@ ompi_osc_sm_rput(const void *origin_addr, return OMPI_SUCCESS; } +int +ompi_osc_sm_rput_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **ompi_req) +{ + int ret; + ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; + void *remote_address; + + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rput_notify: 0x%lx, %zu, %s, %d, %d, %zu, %s, %d, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + notify, + (unsigned long) win)); + + ret = osc_sm_check_notify_idx(module, target, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + + ret = ompi_datatype_sndrcv((void *)origin_addr, origin_count, origin_dt, + remote_address, target_count, target_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Release ordering: the data must be visible at the target before the + * notification is (MPI-5.1 section 12.3, "The notification counter will be + * updated at the target only after the completion of the data movement + * operation at the target"). */ + opal_atomic_wmb(); + opal_atomic_add(&osc_sm_target_notify_base(module, target)[notify], 1); + + /* the only valid field of RMA request status is the MPI_ERROR field. + * ompi_request_empty has status MPI_SUCCESS and indicates the request is + * complete. */ + *ompi_req = &ompi_request_empty; + + return OMPI_SUCCESS; +} int ompi_osc_sm_rget(void *origin_addr, @@ -99,6 +316,60 @@ ompi_osc_sm_rget(void *origin_addr, return OMPI_SUCCESS; } +int +ompi_osc_sm_rget_notify(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **ompi_req) +{ + int ret; + ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; + void *remote_address; + + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rget_notify: 0x%lx, %zu, %s, %d, %d, %zu, %s, %d, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + notify, + (unsigned long) win)); + + ret = osc_sm_check_notify_idx(module, target, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + + ret = ompi_datatype_sndrcv(remote_address, target_count, target_dt, + origin_addr, origin_count, origin_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Full barrier, not opal_atomic_rmb(): the notification tells the target + * that this get has read the window, so the loads above must not be + * reordered after the counter increment below -- that is a load-before-store + * constraint, which a load-load fence does not express. opal_atomic_add() + * is relaxed (see opal/include/opal/sys/atomic_stdc.h) and supplies no + * ordering of its own. */ + opal_atomic_mb(); + opal_atomic_add(&osc_sm_target_notify_base(module, target)[notify], 1); + + /* the only valid field of RMA request status is the MPI_ERROR field. + * ompi_request_empty has status MPI_SUCCESS and indicates the request is + * complete. */ + *ompi_req = &ompi_request_empty; + + return OMPI_SUCCESS; +} int ompi_osc_sm_raccumulate(const void *origin_addr, @@ -236,6 +507,53 @@ ompi_osc_sm_put(const void *origin_addr, } +int +ompi_osc_sm_put_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win) +{ + int ret; + ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; + void *remote_address; + + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "put_notify: 0x%lx, %zu, %s, %d, %d, %zu, %s, %d, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + notify, + (unsigned long) win)); + + ret = osc_sm_check_notify_idx(module, target, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + + ret = ompi_datatype_sndrcv((void *)origin_addr, origin_count, origin_dt, + remote_address, target_count, target_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Release ordering: the data must be visible at the target before the + * notification is (MPI-5.1 section 12.3, "The notification counter will be + * updated at the target only after the completion of the data movement + * operation at the target"). */ + opal_atomic_wmb(); + opal_atomic_add(&osc_sm_target_notify_base(module, target)[notify], 1); + + return ret; +} + int ompi_osc_sm_get(void *origin_addr, size_t origin_count, @@ -267,6 +585,50 @@ ompi_osc_sm_get(void *origin_addr, } +int +ompi_osc_sm_get_notify(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win) +{ + int ret; + ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; + void *remote_address; + + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "get: 0x%lx, %zu, %s, %d, %d, %zu, %s, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + (unsigned long) win)); + + ret = osc_sm_check_notify_idx(module, target, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + + ret = ompi_datatype_sndrcv(remote_address, target_count, target_dt, + origin_addr, origin_count, origin_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Full barrier, not opal_atomic_rmb(): see ompi_osc_sm_rget_notify(). */ + opal_atomic_mb(); + opal_atomic_add(&osc_sm_target_notify_base(module, target)[notify], 1); + + return ret; +} + + int ompi_osc_sm_accumulate(const void *origin_addr, size_t origin_count, @@ -437,5 +799,5 @@ ompi_osc_sm_fetch_and_op(const void *origin_addr, done: opal_atomic_unlock(&module->node_states[target].accumulate_lock); - return OMPI_SUCCESS;; + return OMPI_SUCCESS; } diff --git a/ompi/mca/osc/sm/osc_sm_component.c b/ompi/mca/osc/sm/osc_sm_component.c index ad2b3cae25a..9af4530c77c 100644 --- a/ompi/mca/osc/sm/osc_sm_component.c +++ b/ompi/mca/osc/sm/osc_sm_component.c @@ -79,14 +79,22 @@ ompi_osc_sm_module_t ompi_osc_sm_module_template = { .osc_free = ompi_osc_sm_free, .osc_put = ompi_osc_sm_put, + .osc_put_notify = ompi_osc_sm_put_notify, .osc_get = ompi_osc_sm_get, + .osc_get_notify = ompi_osc_sm_get_notify, + .osc_win_get_notify_value = ompi_osc_sm_win_get_notify_value, + .osc_win_reset_notify_value = ompi_osc_sm_win_reset_notify_value, + .osc_win_set_num_notify = ompi_osc_sm_win_set_num_notify, + .osc_win_get_num_notify = ompi_osc_sm_win_get_num_notify, .osc_accumulate = ompi_osc_sm_accumulate, .osc_compare_and_swap = ompi_osc_sm_compare_and_swap, .osc_fetch_and_op = ompi_osc_sm_fetch_and_op, .osc_get_accumulate = ompi_osc_sm_get_accumulate, .osc_rput = ompi_osc_sm_rput, + .osc_rput_notify = ompi_osc_sm_rput_notify, .osc_rget = ompi_osc_sm_rget, + .osc_rget_notify = ompi_osc_sm_rget_notify, .osc_raccumulate = ompi_osc_sm_raccumulate, .osc_rget_accumulate = ompi_osc_sm_rget_accumulate, @@ -251,12 +259,26 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis module->posts = calloc (1, sizeof(module->posts[0]) + sizeof (module->posts[0][0])); if (NULL == module->posts) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; module->posts[0] = (osc_sm_post_atomic_type_t *) (module->posts + 1); + + /* allocate notify counters for single process case. + * + * NOTE: osc/sm pre-attaches the full reserved capacity, whereas osc/ucx + * starts at zero and requires MPI_WIN_SET_NUM_NOTIFY before any counter + * may be referenced. MPI-5.1 section 12.6.1 does not state what the + * initial attached count is, so neither is provably wrong, but the + * divergence means a program that omits MPI_WIN_SET_NUM_NOTIFY works + * here and fails on ucx. Left as-is pending a decision. */ + module->notify_counters = calloc(OSC_SM_MAX_NOTIFY_COUNTERS, sizeof(int64_t)); + if (NULL == module->notify_counters) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; + module->node_states[0].notify_counter_count = OSC_SM_MAX_NOTIFY_COUNTERS; + module->node_states[0].notify_counter_offset = 0; } else { - unsigned long total, *rbuf; + unsigned long total, total_counters, gather_values[2], *rbuf; int i, flag; size_t pagesize; size_t state_size; size_t posts_size, post_size = (comm_size + OSC_SM_POST_MASK) / (OSC_SM_POST_MASK + 1); + size_t notify_counters_size; size_t data_base_size; opal_output_verbose(MCA_BASE_VERBOSE_DEBUG, ompi_osc_base_framework.framework_output, @@ -265,7 +287,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis /* get the pagesize */ pagesize = opal_getpagesize(); - rbuf = malloc(sizeof(unsigned long) * comm_size); + rbuf = malloc(sizeof(unsigned long) * comm_size * 2 ); if (NULL == rbuf) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; /* Note that the alloc_shared_noncontig info key only has @@ -289,9 +311,10 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis "allocating window using contiguous strategy"); } - total = size; - ret = module->comm->c_coll->coll_allgather(&total, 1, MPI_UNSIGNED_LONG, - rbuf, 1, MPI_UNSIGNED_LONG, + gather_values[0] = size; + gather_values[1] = OSC_SM_MAX_NOTIFY_COUNTERS; + ret = module->comm->c_coll->coll_allgather(gather_values, 2, MPI_UNSIGNED_LONG, + rbuf, 2, MPI_UNSIGNED_LONG, module->comm, module->comm->c_coll->coll_allgather_module); if (OMPI_SUCCESS != ret) { @@ -300,8 +323,10 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis } total = 0; + total_counters = 0; for (i = 0 ; i < comm_size ; ++i) { - total += rbuf[i]; + total += rbuf[2 * i]; + total_counters += rbuf[2 * i + 1]; if (module->noncontig) { total += OPAL_ALIGN_PAD_AMOUNT(total, pagesize); } @@ -312,7 +337,9 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis state_size += OPAL_ALIGN_PAD_AMOUNT(state_size, 64); posts_size = comm_size * post_size * sizeof (module->posts[0][0]); posts_size += OPAL_ALIGN_PAD_AMOUNT(posts_size, 64); - data_base_size = state_size + posts_size; + notify_counters_size = total_counters * sizeof(uint64_t); + notify_counters_size += OPAL_ALIGN_PAD_AMOUNT(notify_counters_size, 64); + data_base_size = state_size + posts_size + notify_counters_size; data_base_size += OPAL_ALIGN_PAD_AMOUNT(data_base_size, pagesize); if (0 == ompi_comm_rank (module->comm)) { char *data_file; @@ -373,15 +400,27 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis module->global_state = (ompi_osc_sm_global_state_t *) (module->posts[0] + comm_size * post_size); module->node_states = (ompi_osc_sm_node_state_t *) (module->global_state + 1); - for (i = 0, total = data_base_size ; i < comm_size ; ++i) { + /* set up notify counters in shared memory after node_states */ + module->notify_counters = (opal_atomic_int64_t *) ((char *)(module->node_states + comm_size) + + OPAL_ALIGN_PAD_AMOUNT((uintptr_t)(module->node_states + comm_size), 64)); + /* zero out notify counters */ + memset((void *) module->notify_counters, 0, total_counters * sizeof(int64_t)); + + for (i = 0, total = data_base_size, total_counters = 0 ; i < comm_size ; ++i) { if (i > 0) { module->posts[i] = module->posts[i - 1] + post_size; } - module->sizes[i] = rbuf[i]; + module->node_states[i].notify_counter_count = (uint32_t) rbuf[2 * i + 1]; + module->node_states[i].notify_counter_offset = + (uint64_t) ((char *) (module->notify_counters + total_counters) - + (char *) module->segment_base); + total_counters += rbuf[2 * i + 1]; + + module->sizes[i] = rbuf[2 * i]; if (module->sizes[i] || !module->noncontig) { module->bases[i] = ((char *) module->segment_base) + total; - total += rbuf[i]; + total += rbuf[2 * i]; if (module->noncontig) { total += OPAL_ALIGN_PAD_AMOUNT(total, pagesize); } @@ -395,7 +434,8 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis /* initialize my state shared */ module->my_node_state = &module->node_states[ompi_comm_rank(module->comm)]; - memset (module->my_node_state, 0, sizeof(*module->my_node_state)); + module->my_node_state->complete_count = 0; + memset (&module->my_node_state->lock, 0, sizeof(module->my_node_state->lock)); *base = module->bases[ompi_comm_rank(module->comm)]; @@ -551,6 +591,7 @@ ompi_osc_sm_free(struct ompi_win_t *win) module->comm->c_coll->coll_barrier_module); opal_shmem_segment_detach (&module->seg_ds); + /* notify_counters points into shared memory segment, no separate free needed */ } else { free(module->node_states); free(module->global_state); @@ -558,6 +599,10 @@ ompi_osc_sm_free(struct ompi_win_t *win) mca_mpool_base_default_module->mpool_free(mca_mpool_base_default_module, module->bases[0]); } + /* free notify_counters for single process case */ + /* cast away the atomic/volatile qualifier for free(), as in + * opal/runtime/opal_progress.c */ + free((void *) module->notify_counters); } free(module->disp_units); free(module->outstanding_locks); diff --git a/ompi/mpi/bindings/ompi_bindings/consts.py b/ompi/mpi/bindings/ompi_bindings/consts.py index 7a523d7670d..4a249629079 100644 --- a/ompi/mpi/bindings/ompi_bindings/consts.py +++ b/ompi/mpi/bindings/ompi_bindings/consts.py @@ -100,6 +100,7 @@ 'MPI_T_ERR_PVAR_NO_ATOMIC', 'MPI_T_ERR_NOT_ACCESSIBLE', 'MPI_T_ERR_NOT_SUPPORTED', + 'MPI_ERR_RMA_NOTIFICATION', 'MPI_ERR_LASTCODE', ] diff --git a/ompi/mpi/c/Makefile.am b/ompi/mpi/c/Makefile.am index f532121bf28..50a16c9d134 100644 --- a/ompi/mpi/c/Makefile.am +++ b/ompi/mpi/c/Makefile.am @@ -223,6 +223,7 @@ prototype_sources = \ get_accumulate.c.in \ get_address.c.in \ get.c.in \ + get_notify.c.in \ get_count.c.in \ get_elements.c.in \ get_elements_x.c.in \ @@ -341,6 +342,7 @@ prototype_sources = \ psend_init.c.in \ publish_name.c.in \ put.c.in \ + put_notify.c.in \ query_thread.c.in \ raccumulate.c.in \ recv.c.in \ @@ -484,6 +486,10 @@ prototype_sources = \ win_get_group.c.in \ win_get_info.c.in \ win_get_name.c.in \ + win_get_notify_value.c.in \ + win_reset_notify_value.c.in \ + win_get_num_notify.c.in \ + win_set_num_notify.c.in \ win_lock_all.c.in \ win_lock.c.in \ win_post.c.in \ @@ -698,6 +704,7 @@ interface_profile_sources = \ get_elements_x_generated.c \ get_hw_resource_info_generated.c \ get_library_version_generated.c \ + get_notify_generated.c \ get_processor_name_generated.c \ get_version_generated.c \ graph_create_generated.c \ @@ -811,6 +818,7 @@ interface_profile_sources = \ psend_init_generated.c \ publish_name_generated.c \ put_generated.c \ + put_notify_generated.c \ query_thread_generated.c \ raccumulate_generated.c \ recv_generated.c \ @@ -954,6 +962,10 @@ interface_profile_sources = \ win_get_group_generated.c \ win_get_info_generated.c \ win_get_name_generated.c \ + win_get_notify_value_generated.c \ + win_reset_notify_value_generated.c \ + win_get_num_notify_generated.c \ + win_set_num_notify_generated.c \ win_lock_all_generated.c \ win_lock_generated.c \ win_post_generated.c \ diff --git a/ompi/mpi/c/get_notify.c.in b/ompi/mpi/c/get_notify.c.in new file mode 100644 index 00000000000..d9b8f5b68ef --- /dev/null +++ b/ompi/mpi/c/get_notify.c.in @@ -0,0 +1,77 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2020 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2024 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" +#include + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/communicator/communicator.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" +#include "ompi/datatype/ompi_datatype.h" +#include "ompi/runtime/ompi_spc.h" + +PROTOTYPE ERROR_CLASS get_notify(BUFFER_OUT origin_addr, COUNT origin_count, + DATATYPE origin_datatype, INT target_rank, + AINT target_disp, COUNT target_count, + DATATYPE target_datatype, INT notification_idx, WIN win) +{ + int rc; + + SPC_RECORD(OMPI_SPC_GET_NOTIFY, 1); + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (origin_count < 0 || target_count < 0) { + rc = MPI_ERR_COUNT; + } else if (ompi_win_peer_invalid(win, target_rank) && + (MPI_PROC_NULL != target_rank)) { + rc = MPI_ERR_RANK; + } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) { + rc = MPI_ERR_DISP; + } else if (notification_idx < 0) { + rc = MPI_ERR_RMA_NOTIFICATION; + } else { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count); + if (OMPI_SUCCESS == rc) { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count); + } + } + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (MPI_PROC_NULL == target_rank) return MPI_SUCCESS; + + rc = win->w_osc_module->osc_get_notify(origin_addr, origin_count, origin_datatype, + target_rank, target_disp, target_count, + target_datatype, notification_idx, win); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/put_notify.c.in b/ompi/mpi/c/put_notify.c.in new file mode 100644 index 00000000000..f278e16cb16 --- /dev/null +++ b/ompi/mpi/c/put_notify.c.in @@ -0,0 +1,80 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2020 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2024 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" +#include + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/communicator/communicator.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" +#include "ompi/datatype/ompi_datatype.h" +#include "ompi/runtime/ompi_spc.h" + +PROTOTYPE ERROR_CLASS put_notify(BUFFER origin_addr, COUNT origin_count, DATATYPE origin_datatype, + INT target_rank, AINT target_disp, COUNT target_count, + DATATYPE target_datatype, INT notification_idx, WIN win) +{ + int rc; + + SPC_RECORD(OMPI_SPC_PUT_NOTIFY, 1); + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (origin_count < 0 || target_count < 0) { + rc = MPI_ERR_COUNT; + } else if (ompi_win_peer_invalid(win, target_rank) && + (MPI_PROC_NULL != target_rank)) { + rc = MPI_ERR_RANK; + } else if (NULL == target_datatype || + MPI_DATATYPE_NULL == target_datatype) { + rc = MPI_ERR_TYPE; + } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) { + rc = MPI_ERR_DISP; + } else if (notification_idx < 0) { + rc = MPI_ERR_RMA_NOTIFICATION; + } else { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count); + if (OMPI_SUCCESS == rc) { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count); + } + } + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (MPI_PROC_NULL == target_rank) return MPI_SUCCESS; + + rc = win->w_osc_module->osc_put_notify(origin_addr, origin_count, origin_datatype, + target_rank, target_disp, target_count, + target_datatype, notification_idx, win); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/win_get_notify_value.c.in b/ompi/mpi/c/win_get_notify_value.c.in new file mode 100644 index 00000000000..27df94e1e82 --- /dev/null +++ b/ompi/mpi/c/win_get_notify_value.c.in @@ -0,0 +1,41 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2026 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" + +PROTOTYPE ERROR_CLASS win_get_notify_value(WIN win, INT notification_idx, ELEMENT_COUNT value) +{ + int rc; + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (notification_idx < 0) { + rc = MPI_ERR_RMA_NOTIFICATION; + } else if (NULL == value) { + rc = MPI_ERR_ARG; + } + + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + rc = win->w_osc_module->osc_win_get_notify_value(win, notification_idx, value); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/win_get_num_notify.c.in b/ompi/mpi/c/win_get_num_notify.c.in new file mode 100644 index 00000000000..9ec6b60b5bc --- /dev/null +++ b/ompi/mpi/c/win_get_num_notify.c.in @@ -0,0 +1,43 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2026 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" + +PROTOTYPE ERROR_CLASS win_get_num_notify(WIN win, INT target_rank, INT_OUT num_notifications) +{ + int rc; + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (NULL == num_notifications) { + rc = MPI_ERR_ARG; + } + + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (NULL == win->w_osc_module->osc_win_get_num_notify) { + return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_UNSUPPORTED_OPERATION, FUNC_NAME); + } + + rc = win->w_osc_module->osc_win_get_num_notify(win, target_rank, num_notifications); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/win_reset_notify_value.c.in b/ompi/mpi/c/win_reset_notify_value.c.in new file mode 100644 index 00000000000..68b462a510e --- /dev/null +++ b/ompi/mpi/c/win_reset_notify_value.c.in @@ -0,0 +1,41 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2026 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" + +PROTOTYPE ERROR_CLASS win_reset_notify_value(WIN win, INT notification_idx, ELEMENT_COUNT value) +{ + int rc; + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (notification_idx < 0) { + rc = MPI_ERR_RMA_NOTIFICATION; + } else if (NULL == value) { + rc = MPI_ERR_ARG; + } + + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + rc = win->w_osc_module->osc_win_reset_notify_value(win, notification_idx, value); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/win_set_num_notify.c.in b/ompi/mpi/c/win_set_num_notify.c.in new file mode 100644 index 00000000000..cc1d39a9e77 --- /dev/null +++ b/ompi/mpi/c/win_set_num_notify.c.in @@ -0,0 +1,48 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2026 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/info/info.h" +#include "ompi/mca/osc/osc.h" + +PROTOTYPE ERROR_CLASS win_set_num_notify(WIN win, INFO info, INT num_notifications) +{ + int rc; + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (NULL != info && MPI_INFO_NULL != info && ompi_info_is_freed(info)) { + rc = MPI_ERR_INFO; + } else if (num_notifications < 0) { + rc = MPI_ERR_ARG; + } + + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (NULL == win->w_osc_module->osc_win_set_num_notify) { + return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_UNSUPPORTED_OPERATION, FUNC_NAME); + } + + rc = win->w_osc_module->osc_win_set_num_notify(win, + (NULL != info && MPI_INFO_NULL != info) ? &(info->super) : NULL, + num_notifications); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/runtime/ompi_spc.c b/ompi/runtime/ompi_spc.c index fb097ac6077..e7653e27c39 100644 --- a/ompi/runtime/ompi_spc.c +++ b/ompi/runtime/ompi_spc.c @@ -71,8 +71,10 @@ static const ompi_spc_event_t ompi_spc_events_desc[OMPI_SPC_NUM_COUNTERS] = { SET_COUNTER_ARRAY(OMPI_SPC_SENDRECV, "The number of times MPI_Sendrecv was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_SENDRECV_REPLACE, "The number of times MPI_Sendrecv_replace was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_PUT, "The number of times MPI_Put was called.", false, false), + SET_COUNTER_ARRAY(OMPI_SPC_PUT_NOTIFY, "The number of times MPI_Put_notify was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_RPUT, "The number of times MPI_Rput was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_GET, "The number of times MPI_Get was called.", false, false), + SET_COUNTER_ARRAY(OMPI_SPC_GET_NOTIFY, "The number of times MPI_Get_notify was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_RGET, "The number of times MPI_Rget was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_PROBE, "The number of times MPI_Probe was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_IPROBE, "The number of times MPI_Iprobe was called.", false, false), diff --git a/ompi/runtime/ompi_spc.h b/ompi/runtime/ompi_spc.h index ca61aa8a409..03f58dd2504 100644 --- a/ompi/runtime/ompi_spc.h +++ b/ompi/runtime/ompi_spc.h @@ -58,8 +58,10 @@ typedef enum ompi_spc_counters { OMPI_SPC_SENDRECV, OMPI_SPC_SENDRECV_REPLACE, OMPI_SPC_PUT, + OMPI_SPC_PUT_NOTIFY, OMPI_SPC_RPUT, OMPI_SPC_GET, + OMPI_SPC_GET_NOTIFY, OMPI_SPC_RGET, OMPI_SPC_PROBE, OMPI_SPC_IPROBE,