/**
 * Copyright (C) Mellanox Technologies Ltd. 2015-2017.  ALL RIGHTS RESERVED.
 *
 * See file LICENSE for terms.
 */

#ifndef _SHARP_COLL_API_H
#define _SHARP_COLL_API_H
#include <stdint.h>


/**
 * @brief SHARP coll supported data types
 *
 * The enumeration list describes the data types supported by SHARP coll
 */
#ifdef __cplusplus
extern "C" {
#endif

enum sharp_datatype {
	SHARP_DTYPE_UNSIGNED,		/**< 32-bit unsigned integer. */
	SHARP_DTYPE_INT,		/**< 32-bit integer. */
	SHARP_DTYPE_UNSIGNED_LONG,	/**< 64-bit unsigned integer. */
	SHARP_DTYPE_LONG,		/**< 64-bit integer. */
	SHARP_DTYPE_FLOAT,		/**< 32-bit long floating point number */
	SHARP_DTYPE_DOUBLE,		/**< 64-bit long floating point number. */
	SHARP_DTYPE_NULL		/**< NULL data type */
};

/**
 * @brief SHARP coll supported aggregation operations
 *
 * The enumeration list describes the aggregation operations supported by SHARP coll
 */
enum sharp_reduce_op {
	SHARP_OP_MAX,		/**< maximum. */
	SHARP_OP_MIN,		/**< minimum. */
	SHARP_OP_SUM,		/**< sum. */
	SHARP_OP_PROD,		/**< product. */
	SHARP_OP_LAND,		/**< logical and.*/
	SHARP_OP_BAND,		/**< bit-wise and. */
	SHARP_OP_LOR,		/**< logical or. */
	SHARP_OP_BOR,		/**< bit-wise or. */
	SHARP_OP_LXOR,		/**< logical xor. */
	SHARP_OP_BXOR,		/**< bit-wise xor. */
	SHARP_OP_MAXLOC,	/**< max value and location. */
	SHARP_OP_MINLOC,	/**< min value and location. */
	SHARP_OP_NULL
};

/**
 * @brief SHARP coll status code
 *
 * The enumeration list describes the error codes returned by SHARP coll
 */
enum sharp_error_no {
	SHARP_COLL_SUCCESS	= 0,	/**< Success. */
	SHARP_COLL_ERROR	= -1,	/**< Error. */
	SHARP_COLL_ENOT_SUPP	= -2,	/**< Collective operation not supported. */
	SHARP_COLL_ENOMEM	= -3,	/**< No memory. */
	SHARP_COLL_EGROUP_ALLOC	= -4,	/**< SHARP Group alloc error. */
	SHARP_COLL_ECONN_TREE	= -5,	/**< No connection to sharp tree. */
	SHARP_COLL_EGROUP_JOIN	= -6,	/**< Not able to join sharp grou.*/
	SHARP_COLL_EQUOTA	= -7,	/**< SHARP resource quota error. */
	SHARP_COLL_ESESS_INIT	= -8,	/**< Cannot connect to SHARPD. */
	SHARP_COLL_EDEV		= -9,	/**< SHARP device error. */
	SHARP_COLL_EINVAL	= -10,	/**< Invalid value. */
	SHARP_COLL_EJOB_CREATE	= -11,	/**< Cannot create SHARP job. */
	SHARP_COLL_ETREE_INFO	= -12,	/**< SHARP tree info not found. */
	SHARP_COLL_ENOTREE	= -13,	/**< No available SHARP trees. */
	SHARP_COLL_EGROUP_ID	= -14,	/**< Wrong SHARP group ID. */
	SHARP_COLL_EOOB		= -15,	/**< Out-Of-Band collective error. */
	SHARP_COLL_EGROUP_MCAST = -16,  /**< Multicast target error. */
	SHARP_COLL_EGROUP_TRIM	= -17  /**< SHARP Group trim failed. */
};

/* Forward declarations */
struct sharp_coll_context;
struct sharp_coll_comm;

/*
 * @brief SHARP coll configuration descriptor
 *
 * This descriptor defines the configuration for SHARP coll initialization
 */
struct sharp_coll_config {
	char *ib_dev_list;		/**< IB device name, port list.  */
	int user_progress_num_polls;	/**< Number of polls to do before calling user progress. */
	int coll_timeout;		/**< Timeout (msec) for collective operation, -1 - infinite */
};

/**
 * Default SHARP COLL configuration.
 */
extern const struct sharp_coll_config sharp_coll_default_config;

/*
 * @brief SHARP coll Out-Of-Band collectives descriptor
 *
 * This descriptor defines list of OOB collectives application must implement
 * and provide for SHARP coll initialization
 */
struct sharp_coll_out_of_band_colls {
	/**
	 * @brief out-of-band broadcast
	 *
	 * The pointer refers to application defined out-of-band bcast collective.
	 *
	 * @param [in] context	User-defined context or NULL.
	 * @param [in] buffer	Buffer to send/recv.
	 * @param [in] len	Size of the buffer.
	 * @param [in] root	Root of the broadcast.
	 */
	int (*bcast) (void* context, void* buffer, int len, int root);

	/**
	 * @brief out-of-band barrier
	 *
	 * The pointer refers to application defined out-of-band barrier collective.
	 *
	 * @param [in] context	User-defined context or NULL.
	 */
	int (*barrier) (void* context);

	/**
	 * @brief out-of-band gather
	 *
	 * The pointer refers to application defined out-of-band gather collective.
	 *
	 * @param [in] context	User-defined context or NULL.
	 * @param [in] root	Root of the broadcast.
	 * @param [in] sbuf	Buffer to send.
	 * @param [in] rbuf	Buffer to recv.
	 * @param [in] len	Size of the buffer.
	 */
	int (*gather) (void * context, int root, void *sbuf, void *rbuf, int len);
};

/*
 * @brief SHARP coll group initialization spec descriptor
 *
 * This descriptor defines the list of application specification to create SHARP group.
 *
 */
struct sharp_coll_comm_init_spec {
	int	rank;		/**< Unique process id in the group. */
	int	size;		/**< Size of the SHARP group. */
	int	is_comm_world;	/**< Is universal group (e.g. MPI_COMM_WORLD). */
	void	*oob_ctx;	/**< External group context for OOB functions. */
};

/*
 * @brief SHARP coll initialization spec descriptor
 *
 * This descriptor defines the list of application specification to initialize SHARP coll.
 *
 */
struct sharp_coll_init_spec {
	uint64_t	job_id;				/**< Job unique ID */
	char		*hostlist;			/**< Comma separated hostlist. */
	int		world_rank;			/**< Global unique process id. */
	int		world_size;			/**< Num of processes in the job. */
	int		(*progress_func)(void);		/**< External progress function. */
	int		group_channel_idx;		/**< local group channel index(0 .. (max - 1))*/
	struct sharp_coll_config config;		/**< @ref sharp_coll_config "SHARP COLL Configuration". */
	struct sharp_coll_out_of_band_colls oob_colls;  /**< @ref sharp_coll_out_of_band_colls "List of OOB collectives". */
};


/**
 * @brief SHARP coll buffer types
 *
 * The enumeration list describes the buffer types supported in collective calls.
 * @note Only SHARP_DATA_BUFFER is implemented.
 */
enum sharp_data_buffer_type {
	SHARP_DATA_BUFFER,	/**< Contiguous buffer. */
	SHARP_DATA_STREAM,	/**< Data stream. */
	SHARP_DATA_IOV		/**< Vector input. */
};

/**
 * @brief stream callback
 *
 * Streaming writer callback function.
 *
 * @param buffer    Buffer to read or write the data
 * @param length    Max. number of bytes to copy.
 * @param offset    Offset in the data stream. First time called with offset = 0,
 *                  and it's incremented by the return value.
 * @param context   User-define context, from stream.cb_context;
 *
 * @return Number of bytes processed from the buffer.
 *         For writers - if it's smaller than `length', it means to end the
 *          fragment prematurely, and the callback will be called again until
 *          total length is processed.
 *         For readers, the callback must process the entire buffer, if it returns
 *          a smaller value it means the message is truncated.
 *
 * @note It's guaranteed that (offset + length) is not larger than the declared
 *       stream size (stream.length).
 *
 */
typedef size_t (*sharp_stream_cb)(void *buffer, size_t length,
				    size_t offset, void *context);

/*
 * @brief SHARP coll input buffer description
 *
 * This descriptor defines the buffer description for SHARP coll operations
 *
 */
struct sharp_coll_data_desc {
	enum sharp_data_buffer_type type;
	union {

		/* contiguous buffer */
		struct {
			void *ptr;		/**< contiguous data buffer. */
			size_t length;		/**< Buffer len. */
			void *mem_handle;	/**< memory handle returned from @ref sharp_coll_reg_mr */
		} buffer;

		/* Streaming data */
		struct {
			sharp_stream_cb cb;	/**< Streaming callback. */
			void *cb_context;	/**< Context for stream callback. */
			size_t length;		/**< Total data length. */
		} stream;

		/* Scatter/gather list */
		struct {
			unsigned count;		/**< Number of IOV entries. */
			struct iovec *iov;	/**< IOV entries. */
		} iov;

	};
};

/*
 * @brief SHARP coll reduce collective specification
 *
 * This descriptor defines the input parameters for SHARP coll reduce operation
 *
 */
struct sharp_coll_reduce_spec {
	int				root;		/**< [in] root process id (ignored for allreduce) */
	struct sharp_coll_data_desc	sbuf_desc;	/**< [in] source data buffer desc */
	struct sharp_coll_data_desc	rbuf_desc;	/**< [out] destination data buffer desc */
	enum sharp_datatype		dtype;		/**< [in] data type @ref sharp_datatype */
	int				length;		/**< [in] reduce operation size */
	enum sharp_reduce_op		op;		/**< [in] reduce operator @ref sharp_reduce_op */
};

/*
 * @brief SHARP coll broadcast collective specification
 *
 * This descriptor defines the input paramaters for SHARP coll broadcast operation
 *
 */
struct sharp_coll_bcast_spec {
	int				root;		/**< [in] root process id */
	struct sharp_coll_data_desc	buf_desc;	/**< [in,out] buffer desc to send/recv bcast data */
	int				size;		/**< [in] bcast size */
};

/**
 * @brief SHARP coll context initialization
 *
 * This routine is initialize SHARP coll library and create @ref sharp_coll_context "SHARP coll context".
 * This is a collective, called from all processes of the job.
 *
 * @warning An application cannot call any SHARP coll routine before sharp_coll_init
 *
 * @param [in]	sharp_coll_spec		SHARP coll specification descriptor.
 * @param [out]	sharp_coll_context	Initialized @ref sharp_coll_context "SHARP coll context".
 *
 * @return Error code as defined by @ref sharp_error_no
 */
int sharp_coll_init(struct sharp_coll_init_spec *sharp_coll_spec,
		    struct sharp_coll_context  **sharp_coll_context);

/**
 * @brief SHARP coll context finalize
 *
 * This routine finalizes and releases the resources associated with
 * @ref sharp_coll_context "SHARP coll context". typically done once, just before the process ends.
 *
 * @warning An application cannot call any SHARP coll routine after sharp_coll_finalize
 *
 * @param [in] context	SHARP coll context to cleanup.
 *
 * @return Error code as defined by @ref sharp_error_no
 */
int sharp_coll_finalize(struct sharp_coll_context *context);

/**
 * @brief Progress SHARP coll communication operations.
 *
 * This routine explicitly progresses all SHARP communication operation.
 * For example, this typically called from MPI progress context for MPI case.
 *
 * @param [in] context	SHARP coll context to progress.
 *
 * @return Error code as defined by @ref sharp_error_no
 */
int sharp_coll_progress(struct sharp_coll_context *context);

/**
 * @brief SHARP coll communicator(group) initialization
 *
 * This routine creates @ref sharp_coll_comm "SHARP coll group".
 * This is a collective, called from all processes of the SHARP group.
 *
 * @param [in]	context		Handle to SHARP coll context.
 * @param [in]	spec		Input @ref sharp_coll_comm_init_spec "SHARP coll group specification".
 * @param [out] sharp_coll_comm Handle to SHARP coll communicator(group)
 *
 * @return Error code as defined by @ref sharp_error_no
 */
int sharp_coll_comm_init(struct sharp_coll_context *context,
			 struct sharp_coll_comm_init_spec *spec,
			 struct sharp_coll_comm **sharp_coll_comm);

/**
 * @brief SHARP coll communicator cleanup
 *
 * This routine cleanup SHARP coll communicator handle returned from @ref sharp_coll_comm_init.
 *
 * @param [in] comm   SHARP coll communicator to destroy.
 *
 * @return Error code as defined by @ref sharp_error_no
 */
int sharp_coll_comm_destroy(struct sharp_coll_comm *comm);

/**
 * @brief SHARP coll barrier collective
 *
 * This routine is collective operation blocks until all processes call this routine .
 *
 * @param [in]	comm	SHARP coll communicator to run the barrier.
 *
 * @return Error code as defined by @ref sharp_error_no
 */
int sharp_coll_do_barrier(struct sharp_coll_comm *comm);

/**
 * @brief SHARP coll non-blocking barrier collective
 *
 * This routine is non blocking version of @ref sharp_coll_do_barrier "SHARP coll barrier".
 * The progress of this operation is tracked with return request handle
 *
 * @param [in]	comm	SHARP coll communicator to run the barrier.
 * @param [out] handle  Handle representing the communication operation.
 *
 * @return Error code as defined by @ref sharp_error_no
 *
 */
int sharp_coll_do_barrier_nb(struct sharp_coll_comm *comm, void **handle);

/**
 * @brief SHARP coll allreduce collective
 *
 * This routine aggregates the data from all processes of the group and
 * distributes the result back to all processes.
 *
 * @param [in]	comm	SHARP coll communicator to run the allreduce collective.
 * @param [in]	spec	Allreduce operation specification.
 *
 * @return Error code as defined by @ref sharp_error_no
 */
int sharp_coll_do_allreduce(struct sharp_coll_comm *comm,
			     struct sharp_coll_reduce_spec *spec);

/**
 * @brief SHARP coll non-blocking allreduce collective
 *
 * This routine is non blocking version of @ref sharp_coll_do_allreduce "SHARP coll allreduce".
 * The progress of this operation is tracked with return request handle
 *
 * @param [in]	comm	SHARP coll communicator to run the allreduce collective.
 * @param [in]	spec	Allreduce operation specification.
 * @param [out] handle  Handle representing the communication operation.
 *
 * @return Error code as defined by @ref sharp_error_no
 *
 */
int sharp_coll_do_allreduce_nb(struct sharp_coll_comm *comm,
				struct sharp_coll_reduce_spec *spec, void **handle);

/**
 * @brief SHARP coll reduce collective
 *
 * This routine aggregates the data from all processes of the group to a specific process
 *
 * @param [in]	comm	SHARP coll communicator to run the reduce collective.
 * @param [in]	spec	Reduce operation specification.
 *
 * @return Error code as defined by @ref sharp_error_no
 *
 * @note Not implemented.
 */
int sharp_coll_do_reduce(struct sharp_coll_comm *comm,
			 struct sharp_coll_reduce_spec *spec);

/**
 * @brief SHARP coll non-blocking reduce collective
 *
 * This routine is non blocking version of @ref sharp_coll_do_reduce "SHARP coll Reduce".
 * The progress of this operation is tracked with return request handle
 *
 * @param [in]	comm	SHARP coll communicator to run the reduce collective.
 * @param [in]	spec	Allreduce operation specification.
 * @param [out] handle  Handle representing the communication operation.
 *
 * @return Error code as defined by @ref sharp_error_no
 *
 * @note Not implemented.
 */
int sharp_coll_do_reduce_nb(struct sharp_coll_comm *comm,
			    struct sharp_coll_reduce_spec *spec,
			    void **handle);
/**
 * @brief SHARP coll broadcast collective
 *
 * This routine broadcast data from single process to all processes of the group
 *
 * @param [in]	comm	SHARP coll communicator to run the bcast collective.
 * @param [in]	spec	Bcast operation specification.
 *
 * @return Error code as defined by @ref sharp_error_no
 *
 * @note Not implemented.
 */
int sharp_coll_do_bcast(struct sharp_coll_comm *comm,
			struct sharp_coll_bcast_spec *spec);

/**
 * @brief SHARP coll non blocking broadcast collective
 *
 * This routine is non-blocking version of @ref sharp_coll_do_bcast "SHARP coll Bcast".
 * The progress of this operation is tracked with return request handle
 *
 * @param [in]	comm	SHARP coll communicator to run the bcast collective.
 * @param [in]	spec	Bcast operation specification.
 * @param [out] handle  Handle representing the communication operation.
 *
 * @return Error code as defined by @ref sharp_error_no
 *
 * @note Not implemented.
 */
int sharp_coll_do_bcast_nb(struct sharp_coll_comm *comm,
			   struct sharp_coll_bcast_spec *spec,
			   void **handle);

/**
 * @brief SHARP coll request test
 *
 * This routine tests for the completion of a specific non-blocking coll operation
 *
 * @param [in]	req	SHARP coll request handle
 *
 * @return Error code as defined by @ref sharp_error_no
 */
int sharp_coll_req_test(void *handle);

/**
 * @brief SHARP coll request wait
 *
 * This routine returns when the operation identified by non-blocking collective request is complete.
 * The request object is deallocated by the call to sharp_coll_req_wait and the request handle is set to NULL.
 *
 * @param [in]	req	SHARP coll request handle
 *
 * @return Error code as defined by @ref sharp_error_no
 */

int sharp_coll_req_wait(void *handle);

/**
 * @brief SHARP coll request deallocate
 *
 * This routine deallocates request handle
 *
 * @param [in]	req	SHARP coll request handle
 *
 * @return Error code as defined by @ref sharp_error_no
 */
int sharp_coll_req_free(void *handle);

/**
 * @brief SHARP coll memory registration.
 *
 * This routine registers external mem buffer
 *
 * @param [in] context	SHARP coll context to progress.
 * @param [in] buf	Buffer to register
 * @param [in] size	length of the buffer in bytes
 * @param [out] mr	memory registration handle.
 *
 * @return Error code as defined by @ref sharp_error_no
 *
 * @note Only one outstanding registration supported. no registration cache.
 *
 */
int sharp_coll_reg_mr(struct sharp_coll_context *context,
		      void *buf, int size, void **mr);

/**
 * @brief SHARP coll memory de-registration.
 *
 * This routine de-registers the MR.
 *
 * @param [in] context	SHARP coll context to progress.
 * @param [in] mr	memory registration handle.
 *
 * @return Error code as defined by @ref sharp_error_no
 */
int sharp_coll_dereg_mr(struct sharp_coll_context *context, void *mr);

/**
 * @brief SHARP coll print config flags
 *
 * The enumeration list describes bit masks for different options to print config flags
 */
enum config_print_flags {
	SHARP_COLL_CONFIG_PRINT_CONFIG	= 1,	/**< basic configuration. */
	SHARP_COLL_CONFIG_PRINT_HEADER	= 2,	/**< Print header. */
	SHARP_COLL_CONFIG_PRINT_DOC	= 4,	/**< full description. */
	SHARP_COLL_CONFIG_PRINT_HIDDEN  = 8	/**< hidden options. */
};

/**
 * @brief SHARP coll print configuration
 *
 * This routine prints SHARP coll configuration to a stream.
 *
 * @param [in] stream		Output stream to print to.
 * @param [in] print_flags	Controls how the configuration is printed.
 *
 * @return Error code as defined by @ref sharp_error_no
 */

int sharp_coll_print_config(FILE *stream, enum config_print_flags print_flags);

/**
 * @brief SHARP coll print error string
 *
 * This routine returns error string for a given @ref sharp_error_no "SHARP coll error code".
 *
 * @param [in] error	SHARP coll error code.
 *
 * @return Error string
 */
const char *sharp_coll_strerror(int error);

/**
 * @brief SHARP coll print statistics
 *
 * This routine dumps SHARP coll usage statistics
 *
 * @param [in] context	SHARP coll context to progress.
 *
 * @return Error code as defined by @ref sharp_error_no
 *
 * @note It is expected to Out-Of_Band collectives are operational valid to get
 *	 accumulated stats dumps (SHARP_COLL_STATS_DUMP_MODE=2) during the finalize process.
 *
 */
int sharp_coll_dump_stats(struct sharp_coll_context *context);

#ifdef __cplusplus
}
#endif

#endif
