Compare commits
11 Commits
de1685c3cd
...
69e3c845dd
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
69e3c845dd | ||
|
|
4d3faeeca8 | ||
|
|
6797805567 | ||
|
|
061f12c343 | ||
|
|
316f4e79df | ||
|
|
a1c7de4d52 | ||
|
|
70a0f912d6 | ||
|
|
cdd74f7713 | ||
|
|
e06cdd5a86 | ||
|
|
6257dfdaee | ||
|
|
c4393435e1 |
276
0010-Perftest-Support-selecting-congestion-control-algori.patch
Normal file
276
0010-Perftest-Support-selecting-congestion-control-algori.patch
Normal file
@ -0,0 +1,276 @@
|
||||
From e8164fdc9ce332217a7877ffb6d1535bf0261fb1 Mon Sep 17 00:00:00 2001
|
||||
From: Guofeng Yue <yueguofeng@h-partners.com>
|
||||
Date: Fri, 28 Jun 2024 10:56:09 +0800
|
||||
Subject: [PATCH] Perftest: Support selecting congestion control algorithms
|
||||
|
||||
Support configuring congestion control algorithms with hns direct verbs.
|
||||
|
||||
New option: --congest_type
|
||||
|
||||
Usage example:
|
||||
./ib_send_bw -d hns_0 --congest_type=DCQCN
|
||||
./ib_send_bw -d hns_0 --congest_type=DCQCN 192.168.100.100
|
||||
|
||||
Signed-off-by: Guofeng Yue <yueguofeng@h-partners.com>
|
||||
Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
|
||||
---
|
||||
Makefile.am | 24 ++++++++---------
|
||||
configure.ac | 7 +++++
|
||||
src/perftest_parameters.c | 57 +++++++++++++++++++++++++++++++++++++++
|
||||
src/perftest_parameters.h | 1 +
|
||||
src/perftest_resources.c | 12 +++++++++
|
||||
src/perftest_resources.h | 3 +++
|
||||
6 files changed, 92 insertions(+), 12 deletions(-)
|
||||
|
||||
diff --git a/Makefile.am b/Makefile.am
|
||||
index e6a1132..1f09a61 100755
|
||||
--- a/Makefile.am
|
||||
+++ b/Makefile.am
|
||||
@@ -57,41 +57,41 @@ LIBMLX4=
|
||||
endif
|
||||
|
||||
ib_send_bw_SOURCES = src/send_bw.c src/multicast_resources.c src/multicast_resources.h
|
||||
-ib_send_bw_LDADD = libperftest.a $(LIBUMAD) $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
|
||||
+ib_send_bw_LDADD = libperftest.a $(LIBUMAD) $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
|
||||
|
||||
ib_send_lat_SOURCES = src/send_lat.c src/multicast_resources.c src/multicast_resources.h
|
||||
-ib_send_lat_LDADD = libperftest.a $(LIBUMAD) $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
|
||||
+ib_send_lat_LDADD = libperftest.a $(LIBUMAD) $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
|
||||
|
||||
ib_write_lat_SOURCES = src/write_lat.c
|
||||
-ib_write_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
|
||||
+ib_write_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
|
||||
|
||||
ib_write_bw_SOURCES = src/write_bw.c
|
||||
-ib_write_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
|
||||
+ib_write_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
|
||||
|
||||
ib_read_lat_SOURCES = src/read_lat.c
|
||||
-ib_read_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
|
||||
+ib_read_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
|
||||
|
||||
ib_read_bw_SOURCES = src/read_bw.c
|
||||
-ib_read_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
|
||||
+ib_read_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
|
||||
|
||||
ib_atomic_lat_SOURCES = src/atomic_lat.c
|
||||
-ib_atomic_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
|
||||
+ib_atomic_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
|
||||
|
||||
ib_atomic_bw_SOURCES = src/atomic_bw.c
|
||||
-ib_atomic_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
|
||||
+ib_atomic_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
|
||||
|
||||
if HAVE_RAW_ETH
|
||||
raw_ethernet_bw_SOURCES = src/raw_ethernet_send_bw.c
|
||||
-raw_ethernet_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
|
||||
+raw_ethernet_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
|
||||
|
||||
raw_ethernet_lat_SOURCES = src/raw_ethernet_send_lat.c
|
||||
-raw_ethernet_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
|
||||
+raw_ethernet_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
|
||||
|
||||
raw_ethernet_burst_lat_SOURCES = src/raw_ethernet_send_burst_lat.c
|
||||
-raw_ethernet_burst_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
|
||||
+raw_ethernet_burst_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
|
||||
|
||||
raw_ethernet_fs_rate_SOURCES = src/raw_ethernet_fs_rate.c
|
||||
-raw_ethernet_fs_rate_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA)
|
||||
+raw_ethernet_fs_rate_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5) $(LIBEFA) $(LIBHNS)
|
||||
|
||||
else
|
||||
raw_ethernet_bw_SOURCES =
|
||||
diff --git a/configure.ac b/configure.ac
|
||||
index 2bbc7fc..54fc2cc 100755
|
||||
--- a/configure.ac
|
||||
+++ b/configure.ac
|
||||
@@ -291,6 +291,13 @@ if [test $HAVE_MLX5DV_LIB = yes] && [test $HAVE_MLX5DV = yes]; then
|
||||
AC_SUBST([LIBMLX5])
|
||||
fi
|
||||
|
||||
+AC_CHECK_LIB([hns], [hnsdv_query_device], [HAVE_HNSDV=yes LIBHNS=-lhns], [HAVE_HNSDV=no])
|
||||
+AM_CONDITIONAL([HAVE_HNSDV], [test "x$HAVE_HNSDV" = "xyes"])
|
||||
+if [test $HAVE_HNSDV = yes]; then
|
||||
+ AC_DEFINE([HAVE_HNSDV], [1], [Have hns Direct Verbs support])
|
||||
+ AC_SUBST([LIBHNS])
|
||||
+fi
|
||||
+
|
||||
CFLAGS="-g -Wall -D_GNU_SOURCE -O3 $CFLAGS"
|
||||
LIBS=$LIBS" -lpthread"
|
||||
AC_SUBST([LIBUMAD])
|
||||
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
|
||||
index 034a20e..6fdf0a1 100755
|
||||
--- a/src/perftest_parameters.c
|
||||
+++ b/src/perftest_parameters.c
|
||||
@@ -27,6 +27,9 @@ static const char *portStates[] = {"Nop","Down","Init","Armed","","Active Defer"
|
||||
static const char *qp_state[] = {"OFF","ON"};
|
||||
static const char *exchange_state[] = {"Ethernet","rdma_cm"};
|
||||
static const char *atomicTypesStr[] = {"CMP_AND_SWAP","FETCH_AND_ADD"};
|
||||
+#ifdef HAVE_HNSDV
|
||||
+static const char *congestStr[] = {"DCQCN","LDCP","HC3","DIP"};
|
||||
+#endif
|
||||
|
||||
/******************************************************************************
|
||||
* parse_mac_from_str.
|
||||
@@ -432,6 +435,11 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
|
||||
printf(" --cpu_util ");
|
||||
printf(" Show CPU Utilization in report, valid only in Duration mode \n");
|
||||
|
||||
+ #ifdef HAVE_HNSDV
|
||||
+ printf(" --congest_type=<DCQCN, LDCP, HC3, DIP> ");
|
||||
+ printf(" Use the hnsdv interface to set congestion control algorithm.\n");
|
||||
+ #endif
|
||||
+
|
||||
if (tst != FS_RATE) {
|
||||
printf(" --dlid ");
|
||||
printf(" Set a Destination LID instead of getting it from the other side.\n");
|
||||
@@ -814,6 +822,7 @@ static void init_perftest_params(struct perftest_parameters *user_param)
|
||||
user_param->disable_pcir = 0;
|
||||
user_param->source_ip = NULL;
|
||||
user_param->has_source_ip = 0;
|
||||
+ user_param->congest_type = OFF;
|
||||
}
|
||||
|
||||
static int open_file_write(const char* file_path)
|
||||
@@ -911,6 +920,25 @@ static void change_conn_type(int *cptr, VerbType verb, const char *optarg)
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
+
|
||||
+#ifdef HAVE_HNSDV
|
||||
+static void set_congest_type(int *cgtr, const char *optarg)
|
||||
+{
|
||||
+ if (strcmp(congestStr[0], optarg) == 0) {
|
||||
+ *cgtr = HNSDV_QP_CREATE_ENABLE_DCQCN;
|
||||
+ } else if (strcmp(congestStr[1], optarg) == 0) {
|
||||
+ *cgtr = HNSDV_QP_CREATE_ENABLE_LDCP;
|
||||
+ } else if (strcmp(congestStr[2], optarg) == 0) {
|
||||
+ *cgtr = HNSDV_QP_CREATE_ENABLE_HC3;
|
||||
+ } else if (strcmp(congestStr[3], optarg) == 0) {
|
||||
+ *cgtr = HNSDV_QP_CREATE_ENABLE_DIP;
|
||||
+ } else {
|
||||
+ fprintf(stderr, " Invalid congest type. Please choose from {DCQCN,LDCP,HC3,DIP}\n");
|
||||
+ exit(1);
|
||||
+ }
|
||||
+}
|
||||
+#endif
|
||||
+
|
||||
/******************************************************************************
|
||||
*
|
||||
******************************************************************************/
|
||||
@@ -2057,6 +2085,23 @@ static void ctx_set_max_inline(struct ibv_context *context,struct perftest_param
|
||||
}
|
||||
}
|
||||
|
||||
+ #ifdef HAVE_HNSDV
|
||||
+ if (user_param->congest_type) {
|
||||
+ if (user_param->work_rdma_cm == ON)
|
||||
+ {
|
||||
+ printf(RESULT_LINE);
|
||||
+ fprintf(stderr, "rdma_cm does not support setting congest type.\n");
|
||||
+ exit(1);
|
||||
+ }
|
||||
+
|
||||
+ if (user_param->connection_type == XRC || user_param->connection_type == UD) {
|
||||
+ printf(RESULT_LINE);
|
||||
+ fprintf(stdout, "XRC/UD does not support setting congest type.\n");
|
||||
+ exit(1);
|
||||
+ }
|
||||
+ }
|
||||
+ #endif
|
||||
+
|
||||
return;
|
||||
}
|
||||
/******************************************************************************
|
||||
@@ -2176,6 +2221,9 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
|
||||
static int credentials_path_flag = 0;
|
||||
static int data_enc_key_app_path_flag = 0;
|
||||
#endif
|
||||
+ #ifdef HAVE_HNSDV
|
||||
+ static int congest_type_flag = 0;
|
||||
+ #endif
|
||||
|
||||
char *server_ip = NULL;
|
||||
char *client_ip = NULL;
|
||||
@@ -2323,6 +2371,9 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
|
||||
#if defined HAVE_OOO_ATTR
|
||||
{.name = "use_ooo", .has_arg = 0, .flag = &use_ooo_flag, .val = 1},
|
||||
#endif
|
||||
+ #ifdef HAVE_HNSDV
|
||||
+ { .name = "congest_type", .has_arg = 1, .flag = &congest_type_flag, .val = 1},
|
||||
+ #endif
|
||||
{.name = "source_ip", .has_arg = 1, .flag = &source_ip_flag, .val = 1},
|
||||
{0}
|
||||
};
|
||||
@@ -2569,6 +2620,12 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
|
||||
case 'v': user_param->mac_fwd = ON; break;
|
||||
case 'G': user_param->use_rss = ON; break;
|
||||
case 0: /* required for long options to work. */
|
||||
+ #ifdef HAVE_HNSDV
|
||||
+ if (congest_type_flag) {
|
||||
+ set_congest_type(&user_param->congest_type, optarg);
|
||||
+ congest_type_flag = 0;
|
||||
+ }
|
||||
+ #endif
|
||||
if (pkey_flag) {
|
||||
user_param->pkey_index = strtol(optarg,NULL,0);
|
||||
pkey_flag = 0;
|
||||
diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h
|
||||
index 909d771..ffc19ab 100755
|
||||
--- a/src/perftest_parameters.h
|
||||
+++ b/src/perftest_parameters.h
|
||||
@@ -462,6 +462,7 @@ struct perftest_parameters {
|
||||
int recv_post_list;
|
||||
int duration;
|
||||
int use_srq;
|
||||
+ int congest_type;
|
||||
int use_xrc;
|
||||
int use_rss;
|
||||
int srq_exists;
|
||||
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
|
||||
index bcec080..451f11d 100755
|
||||
--- a/src/perftest_resources.c
|
||||
+++ b/src/perftest_resources.c
|
||||
@@ -2155,6 +2155,9 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
|
||||
struct efadv_qp_init_attr efa_attr = {};
|
||||
#endif
|
||||
#endif
|
||||
+ #ifdef HAVE_HNSDV
|
||||
+ struct hnsdv_qp_init_attr hns_attr = {};
|
||||
+ #endif
|
||||
|
||||
attr.send_cq = ctx->send_cq;
|
||||
attr.recv_cq = (user_param->verb == SEND) ? ctx->recv_cq : ctx->send_cq;
|
||||
@@ -2316,6 +2319,15 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
|
||||
#endif // HAVE_AES_XTS
|
||||
else
|
||||
#endif // HAVE_MLX5DV
|
||||
+
|
||||
+ #ifdef HAVE_HNSDV
|
||||
+ if (user_param->congest_type) {
|
||||
+ hns_attr.comp_mask = HNSDV_QP_INIT_ATTR_MASK_QP_CONGEST_TYPE;
|
||||
+ hns_attr.congest_type = user_param->congest_type;
|
||||
+ qp = hnsdv_create_qp(ctx->context, &attr_ex, &hns_attr);
|
||||
+ }
|
||||
+ else
|
||||
+ #endif //HAVE_HNSDV
|
||||
qp = ibv_create_qp_ex(ctx->context, &attr_ex);
|
||||
}
|
||||
else
|
||||
diff --git a/src/perftest_resources.h b/src/perftest_resources.h
|
||||
index ea4a6c4..cf0502c 100755
|
||||
--- a/src/perftest_resources.h
|
||||
+++ b/src/perftest_resources.h
|
||||
@@ -55,6 +55,9 @@
|
||||
#if defined(HAVE_MLX5DV)
|
||||
#include <infiniband/mlx5dv.h>
|
||||
#endif
|
||||
+#if defined(HAVE_HNSDV)
|
||||
+#include <infiniband/hnsdv.h>
|
||||
+#endif
|
||||
#include <rdma/rdma_cma.h>
|
||||
#include <stdint.h>
|
||||
#if defined(__FreeBSD__)
|
||||
--
|
||||
2.25.1
|
||||
|
||||
71
0011-Perftest-Fix-rx_depth-check-for-XRC.patch
Normal file
71
0011-Perftest-Fix-rx_depth-check-for-XRC.patch
Normal file
@ -0,0 +1,71 @@
|
||||
From c642cd2753b1b9343a5642ffeaaa78ef135f0f6e Mon Sep 17 00:00:00 2001
|
||||
From: Junxian Huang <huangjunxian6@hisilicon.com>
|
||||
Date: Fri, 24 May 2024 17:32:27 +0800
|
||||
Subject: [PATCH] Perftest: Fix rx_depth check for XRC
|
||||
|
||||
When users manually specifies --use_srq in perftest command, the rx_depth
|
||||
will be checked. If rx_depth is less than the number of qps, the process
|
||||
will throw an error and exit.
|
||||
|
||||
For XRC SEND where SRQ is definitely used, users normally don't need to
|
||||
manually specifies --use_srq, since the use_srq flag will be set to on
|
||||
when parsing the XRC parameters. However, the XRC parameters parsing is
|
||||
after the SRQ rx_depth check. If rx_depth is less than the number of qps
|
||||
in this case, it will miss the check, size_per_qp in ctx_set_recv_wqes()
|
||||
will become 0 and ibv_post_srq_recv() won't be called.
|
||||
|
||||
Move the XRC parameters parsing ahead of SRQ rx_depth check and set
|
||||
--use_srq to on in advance so that the rx_depth error can be thrown.
|
||||
|
||||
Fixes: 4c774a951b3c ("Added XRC for ib_send_bw test")
|
||||
Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
|
||||
---
|
||||
src/perftest_parameters.c | 25 +++++++++++++------------
|
||||
1 file changed, 13 insertions(+), 12 deletions(-)
|
||||
|
||||
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
|
||||
index 5d27132..52b21dc 100755
|
||||
--- a/src/perftest_parameters.c
|
||||
+++ b/src/perftest_parameters.c
|
||||
@@ -1098,7 +1098,19 @@ static void force_dependecies(struct perftest_parameters *user_param)
|
||||
exit (1);
|
||||
}
|
||||
|
||||
- if (user_param->use_srq && user_param->num_of_qps > user_param->rx_depth) {
|
||||
+ /* XRC Part */
|
||||
+ if (user_param->connection_type == XRC) {
|
||||
+ if (user_param->work_rdma_cm == ON) {
|
||||
+ printf(RESULT_LINE);
|
||||
+ fprintf(stderr," XRC does not support RDMA_CM\n");
|
||||
+ exit(1);
|
||||
+ }
|
||||
+ user_param->use_xrc = ON;
|
||||
+ user_param->use_srq = ON;
|
||||
+ }
|
||||
+
|
||||
+ if (user_param->use_srq && user_param->verb == SEND &&
|
||||
+ user_param->num_of_qps > user_param->rx_depth) {
|
||||
printf(RESULT_LINE);
|
||||
printf(" Using SRQ depth should be greater than number of QPs.\n");
|
||||
exit (1);
|
||||
@@ -1396,17 +1408,6 @@ static void force_dependecies(struct perftest_parameters *user_param)
|
||||
if (user_param->connection_type == DC && !user_param->use_srq)
|
||||
user_param->use_srq = ON;
|
||||
|
||||
- /* XRC Part */
|
||||
- if (user_param->connection_type == XRC) {
|
||||
- if (user_param->work_rdma_cm == ON) {
|
||||
- printf(RESULT_LINE);
|
||||
- fprintf(stderr," XRC does not support RDMA_CM\n");
|
||||
- exit(1);
|
||||
- }
|
||||
- user_param->use_xrc = ON;
|
||||
- user_param->use_srq = ON;
|
||||
- }
|
||||
-
|
||||
if (!user_param->use_old_post_send)
|
||||
{
|
||||
#ifndef HAVE_IBV_WR_API
|
||||
--
|
||||
2.25.1
|
||||
|
||||
266
0012-Perftest-Add-support-for-TD-lock-free-mode.patch
Normal file
266
0012-Perftest-Add-support-for-TD-lock-free-mode.patch
Normal file
@ -0,0 +1,266 @@
|
||||
From 641ab294a6bc344dffd042019c609d4e9839a2f5 Mon Sep 17 00:00:00 2001
|
||||
From: Guofeng Yue <yueguofeng@h-partners.com>
|
||||
Date: Tue, 13 Aug 2024 21:48:44 +0800
|
||||
Subject: [PATCH] Perftest: Add support for TD lock-free mode
|
||||
|
||||
Add support for TD lock-free mode
|
||||
New option: --no_lock
|
||||
|
||||
Usage example:
|
||||
ib_send_bw -d hns_0 --no_lock
|
||||
ib_send_bw -d hns_0 --no_lock 192.168.100.100
|
||||
|
||||
Signed-off-by: Guofeng Yue <yueguofeng@h-partners.com>
|
||||
Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
|
||||
---
|
||||
configure.ac | 14 +++++++
|
||||
src/perftest_parameters.c | 30 ++++++++++++++-
|
||||
src/perftest_parameters.h | 1 +
|
||||
src/perftest_resources.c | 81 +++++++++++++++++++++++++++++++++++++++
|
||||
src/perftest_resources.h | 4 ++
|
||||
5 files changed, 128 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/configure.ac b/configure.ac
|
||||
index 54fc2cc..d976663 100755
|
||||
--- a/configure.ac
|
||||
+++ b/configure.ac
|
||||
@@ -298,6 +298,20 @@ if [test $HAVE_HNSDV = yes]; then
|
||||
AC_SUBST([LIBHNS])
|
||||
fi
|
||||
|
||||
+AC_TRY_LINK([#include <infiniband/verbs.h>],
|
||||
+ [ibv_cq_ex_to_cq], [HAVE_CQ_EX=yes], [HAVE_CQ_EX=no])
|
||||
+AM_CONDITIONAL([HAVE_CQ_EX], [test "x$HAVE_CQ_EX" = "xyes"])
|
||||
+if [test $HAVE_CQ_EX = yes]; then
|
||||
+ AC_DEFINE([HAVE_CQ_EX], [1], [Have CQ EX API support])
|
||||
+fi
|
||||
+
|
||||
+AC_TRY_LINK([#include <infiniband/verbs.h>],
|
||||
+ [ibv_alloc_td], [HAVE_TD_API=yes], [HAVE_TD_API=no])
|
||||
+AM_CONDITIONAL([HAVE_TD_API], [test "x$HAVE_TD_API" = "xyes"])
|
||||
+if [test $HAVE_TD_API = yes]; then
|
||||
+ AC_DEFINE([HAVE_TD_API], [1], [Have TD API support])
|
||||
+fi
|
||||
+
|
||||
CFLAGS="-g -Wall -D_GNU_SOURCE -O3 $CFLAGS"
|
||||
LIBS=$LIBS" -lpthread"
|
||||
AC_SUBST([LIBUMAD])
|
||||
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
|
||||
index 16af503..4c6ba4f 100755
|
||||
--- a/src/perftest_parameters.c
|
||||
+++ b/src/perftest_parameters.c
|
||||
@@ -464,6 +464,11 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
|
||||
printf(" Use a Shared Receive Queue. --rx-depth controls max-wr size of the SRQ \n");
|
||||
}
|
||||
|
||||
+ #ifdef HAVE_TD_API
|
||||
+ printf(" --no_lock ");
|
||||
+ printf(" No lock in IO, including post send, post recv, post srq recv and poll cq \n");
|
||||
+ #endif
|
||||
+
|
||||
if (connection_type != RawEth) {
|
||||
printf(" --ipv6 ");
|
||||
printf(" Use IPv6 GID. Default is IPv4\n");
|
||||
@@ -2225,6 +2230,9 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
|
||||
#ifdef HAVE_HNSDV
|
||||
static int congest_type_flag = 0;
|
||||
#endif
|
||||
+ #ifdef HAVE_TD_API
|
||||
+ static int no_lock_flag = 0;
|
||||
+ #endif
|
||||
|
||||
char *server_ip = NULL;
|
||||
char *client_ip = NULL;
|
||||
@@ -2300,6 +2308,9 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
|
||||
{ .name = "run_infinitely", .has_arg = 0, .flag = &run_inf_flag, .val = 1 },
|
||||
{ .name = "report_gbits", .has_arg = 0, .flag = &report_fmt_flag, .val = 1},
|
||||
{ .name = "use-srq", .has_arg = 0, .flag = &srq_flag, .val = 1},
|
||||
+ #ifdef HAVE_TD_API
|
||||
+ { .name = "no_lock", .has_arg = 0, .flag = &no_lock_flag, .val = 1},
|
||||
+ #endif
|
||||
{ .name = "report-both", .has_arg = 0, .flag = &report_both_flag, .val = 1},
|
||||
{ .name = "reversed", .has_arg = 0, .flag = &is_reversed_flag, .val = 1},
|
||||
{ .name = "pkey_index", .has_arg = 1, .flag = &pkey_flag, .val = 1},
|
||||
@@ -2944,6 +2955,12 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
|
||||
user_param->use_srq = 1;
|
||||
}
|
||||
|
||||
+ #ifdef HAVE_TD_API
|
||||
+ if (no_lock_flag) {
|
||||
+ user_param->no_lock = 1;
|
||||
+ }
|
||||
+ #endif
|
||||
+
|
||||
if (report_fmt_flag) {
|
||||
user_param->report_fmt = GBS;
|
||||
}
|
||||
@@ -3284,15 +3301,24 @@ void ctx_print_test_info(struct perftest_parameters *user_param)
|
||||
printf(" Number of qps : %d\t\tTransport type : %s\n", user_param->num_of_qps, transport_str(user_param->transport_type));
|
||||
printf(" Connection type : %s\t\tUsing SRQ : %s\n", connStr[user_param->connection_type], user_param->use_srq ? "ON" : "OFF");
|
||||
#ifdef HAVE_RO
|
||||
- printf(" PCIe relax order: %s\n", user_param->disable_pcir ? "OFF" : "ON");
|
||||
+ #ifdef HAVE_TD_API
|
||||
+ printf(" PCIe relax order: %s\t\tLock-free : %s\n", user_param->disable_pcir ? "OFF" : "ON", user_param->no_lock ? "ON" : "OFF");
|
||||
+ #else
|
||||
+ printf(" PCIe relax order: %s\t\tLock-free : %s\n", user_param->disable_pcir ? "OFF" : "ON", "Unsupported");
|
||||
+ #endif //HAVE_TD_API
|
||||
if ((check_pcie_relaxed_ordering_compliant() == false) &&
|
||||
(user_param->disable_pcir == 0)) {
|
||||
printf(" WARNING: CPU is not PCIe relaxed ordering compliant.\n");
|
||||
printf(" WARNING: You should disable PCIe RO with `--disable_pcie_relaxed` for both server and client.\n");
|
||||
}
|
||||
#else
|
||||
- printf(" PCIe relax order: %s\n", "Unsupported");
|
||||
+ #ifdef HAVE_TD_API
|
||||
+ printf(" PCIe relax order: %s\t\tLock-free : %s\n", "Unsupported", user_param->no_lock ? "ON" : "OFF");
|
||||
+ #else
|
||||
+ printf(" PCIe relax order: %s\t\tLock-free : %s\n", "Unsupported", "Unsupported");
|
||||
+ #endif //HAVE_TD_API
|
||||
#endif
|
||||
+
|
||||
printf(" ibv_wr* API : %s\n", user_param->use_old_post_send ? "OFF" : "ON");
|
||||
if (user_param->machine == CLIENT || user_param->duplex) {
|
||||
printf(" TX depth : %d\n",user_param->tx_depth);
|
||||
diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h
|
||||
index ffc19ab..f49fb0f 100755
|
||||
--- a/src/perftest_parameters.h
|
||||
+++ b/src/perftest_parameters.h
|
||||
@@ -462,6 +462,7 @@ struct perftest_parameters {
|
||||
int recv_post_list;
|
||||
int duration;
|
||||
int use_srq;
|
||||
+ int no_lock;
|
||||
int congest_type;
|
||||
int use_xrc;
|
||||
int use_rss;
|
||||
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
|
||||
index 451f11d..0e2b9a3 100755
|
||||
--- a/src/perftest_resources.c
|
||||
+++ b/src/perftest_resources.c
|
||||
@@ -1360,6 +1360,20 @@ int destroy_ctx(struct pingpong_context *ctx,
|
||||
}
|
||||
#endif
|
||||
|
||||
+ #ifdef HAVE_TD_API
|
||||
+ if (user_param->no_lock) {
|
||||
+ if (ibv_dealloc_pd(ctx->pad)) {
|
||||
+ fprintf(stderr, "Failed to deallocate PAD - %s\n", strerror(errno));
|
||||
+ test_result = 1;
|
||||
+ }
|
||||
+
|
||||
+ if (ibv_dealloc_td(ctx->td)) {
|
||||
+ fprintf(stderr, "Failed to deallocate TD - %s\n", strerror(errno));
|
||||
+ test_result = 1;
|
||||
+ }
|
||||
+ }
|
||||
+ #endif
|
||||
+
|
||||
if (ibv_dealloc_pd(ctx->pd)) {
|
||||
fprintf(stderr, "Failed to deallocate PD - %s\n", strerror(errno));
|
||||
test_result = 1;
|
||||
@@ -1502,6 +1516,47 @@ int create_reg_cqs(struct pingpong_context *ctx,
|
||||
struct perftest_parameters *user_param,
|
||||
int tx_buffer_depth, int need_recv_cq)
|
||||
{
|
||||
+ #ifdef HAVE_CQ_EX
|
||||
+ struct ibv_cq_init_attr_ex send_cq_attr = {
|
||||
+ .cqe = tx_buffer_depth * user_param->num_of_qps,
|
||||
+ .cq_context = NULL,
|
||||
+ .channel = ctx->channel,
|
||||
+ .comp_vector = user_param->eq_num,
|
||||
+ };
|
||||
+
|
||||
+ #ifdef HAVE_TD_API
|
||||
+ if (user_param->no_lock) {
|
||||
+ send_cq_attr.parent_domain = ctx->pad;
|
||||
+ send_cq_attr.comp_mask = IBV_CQ_INIT_ATTR_MASK_PD;
|
||||
+ }
|
||||
+ #endif
|
||||
+ ctx->send_cq = ibv_cq_ex_to_cq(ibv_create_cq_ex(ctx->context, &send_cq_attr));
|
||||
+ if (!ctx->send_cq) {
|
||||
+ fprintf(stderr, "Couldn't create CQ\n");
|
||||
+ return FAILURE;
|
||||
+ }
|
||||
+
|
||||
+ if (need_recv_cq) {
|
||||
+ struct ibv_cq_init_attr_ex recv_cq_attr = {
|
||||
+ .cqe = user_param->rx_depth * user_param->num_of_qps,
|
||||
+ .cq_context = NULL,
|
||||
+ .channel = ctx->channel,
|
||||
+ .comp_vector = user_param->eq_num,
|
||||
+ };
|
||||
+ #ifdef HAVE_TD_API
|
||||
+ if (user_param->no_lock) {
|
||||
+ recv_cq_attr.parent_domain = ctx->pad;
|
||||
+ recv_cq_attr.comp_mask = IBV_CQ_INIT_ATTR_MASK_PD;
|
||||
+ }
|
||||
+ #endif
|
||||
+ ctx->recv_cq = ibv_cq_ex_to_cq(ibv_create_cq_ex(ctx->context, &recv_cq_attr));
|
||||
+ if (!ctx->recv_cq) {
|
||||
+ fprintf(stderr, "Couldn't create a receiver CQ\n");
|
||||
+ return FAILURE;
|
||||
+ }
|
||||
+ }
|
||||
+ return SUCCESS;
|
||||
+#else
|
||||
ctx->send_cq = ibv_create_cq(ctx->context,tx_buffer_depth *
|
||||
user_param->num_of_qps, NULL, ctx->channel, user_param->eq_num);
|
||||
if (!ctx->send_cq) {
|
||||
@@ -1519,6 +1574,7 @@ int create_reg_cqs(struct pingpong_context *ctx,
|
||||
}
|
||||
|
||||
return SUCCESS;
|
||||
+#endif
|
||||
}
|
||||
|
||||
/******************************************************************************
|
||||
@@ -1906,6 +1962,31 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
|
||||
fprintf(stderr, "Couldn't allocate PD\n");
|
||||
return FAILURE;
|
||||
}
|
||||
+
|
||||
+ #ifdef HAVE_TD_API
|
||||
+ /* Allocating the Thread domain, Parent domain. */
|
||||
+ if (user_param->no_lock) {
|
||||
+ struct ibv_td_init_attr td_attr = {0};
|
||||
+ ctx->td = ibv_alloc_td(ctx->context, &td_attr);
|
||||
+ if (!ctx->td) {
|
||||
+ fprintf(stderr, "Couldn't allocate TD\n");
|
||||
+ return FAILURE;
|
||||
+ }
|
||||
+
|
||||
+ struct ibv_parent_domain_init_attr pad_attr = {
|
||||
+ .pd = ctx->pd,
|
||||
+ .td = ctx->td,
|
||||
+ .comp_mask = 0,
|
||||
+ };
|
||||
+
|
||||
+ ctx->pad = ibv_alloc_parent_domain(ctx->context, &pad_attr);
|
||||
+ if (!ctx->pad) {
|
||||
+ fprintf(stderr, "Couldn't allocate PAD\n");
|
||||
+ return FAILURE;
|
||||
+ }
|
||||
+ }
|
||||
+ #endif
|
||||
+
|
||||
#ifdef HAVE_AES_XTS
|
||||
if(user_param->aes_xts){
|
||||
struct mlx5dv_dek_init_attr dek_attr = {};
|
||||
diff --git a/src/perftest_resources.h b/src/perftest_resources.h
|
||||
index cf0502c..ba8630b 100755
|
||||
--- a/src/perftest_resources.h
|
||||
+++ b/src/perftest_resources.h
|
||||
@@ -170,6 +170,10 @@ struct pingpong_context {
|
||||
#endif
|
||||
struct ibv_comp_channel *channel;
|
||||
struct ibv_pd *pd;
|
||||
+ #ifdef HAVE_TD_API
|
||||
+ struct ibv_td *td;
|
||||
+ struct ibv_pd *pad;
|
||||
+ #endif
|
||||
struct ibv_mr **mr;
|
||||
struct ibv_cq *send_cq;
|
||||
struct ibv_cq *recv_cq;
|
||||
--
|
||||
2.25.1
|
||||
|
||||
34
0013-Perftest-Fix-TD-lock-free-mode-not-working-for-QP.patch
Normal file
34
0013-Perftest-Fix-TD-lock-free-mode-not-working-for-QP.patch
Normal file
@ -0,0 +1,34 @@
|
||||
From f8f97b189e9af6cff8559b10c749ff289219b754 Mon Sep 17 00:00:00 2001
|
||||
From: Xinghai Cen <cenxinghai@h-partners.com>
|
||||
Date: Wed, 21 Aug 2024 20:06:50 +0800
|
||||
Subject: [PATCH] Perftest: Fix TD lock-free mode not working for QP
|
||||
|
||||
Fix TD lock-free mode not working for QP When
|
||||
creating QP in TD lock-free mode, set attr_ex.pd with ctx->pad instead of
|
||||
ctx->pd, otherwise the lock-free won't work.
|
||||
|
||||
Fixes: "Perftest: Add support for TD lock-free mode"
|
||||
Signed-off-by: Guofeng Yue <yueguofeng@h-partners.com>
|
||||
---
|
||||
src/perftest_resources.c | 4 ++++
|
||||
1 file changed, 4 insertions(+)
|
||||
|
||||
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
|
||||
index 0e2b9a3..1bdf34e 100755
|
||||
--- a/src/perftest_resources.c
|
||||
+++ b/src/perftest_resources.c
|
||||
@@ -2309,7 +2309,11 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
|
||||
else if (opcode == IBV_WR_RDMA_READ)
|
||||
attr_ex.send_ops_flags |= IBV_QP_EX_WITH_RDMA_READ;
|
||||
}
|
||||
+ #ifdef HAVE_TD_API
|
||||
+ attr_ex.pd = user_param->no_lock ? ctx->pad : ctx->pd;
|
||||
+ #else
|
||||
attr_ex.pd = ctx->pd;
|
||||
+ #endif
|
||||
attr_ex.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS | IBV_QP_INIT_ATTR_PD;
|
||||
attr_ex.send_cq = attr.send_cq;
|
||||
attr_ex.recv_cq = attr.recv_cq;
|
||||
--
|
||||
2.25.1
|
||||
|
||||
@ -0,0 +1,64 @@
|
||||
From 3ea6ea2490eb19e5951201804ab9df525338c799 Mon Sep 17 00:00:00 2001
|
||||
From: Ge Hu <huge4@huawei.com>
|
||||
Date: Mon, 2 Sep 2024 19:43:09 +0800
|
||||
Subject: [PATCH] Perftest: Fix failure in creating cq when create cq ex is not
|
||||
supported by providers
|
||||
|
||||
The previous TD lock-free patch introduced ibv_create_cq_ex() to
|
||||
perftest, but it breaks the provider devices which do not support
|
||||
this API yet. For these devices calling this API leads to an errno
|
||||
of EOPNOTSUPP. So add a check of errno, and if it is EOPNOTSUPP,
|
||||
use ibv_create_cq() as a fallback.
|
||||
|
||||
Fixes: b6f957f6bc6c ("Perftest: Add support for TD lock-free mode")
|
||||
Tested-By: Selvin Xavier <selvin.xavier@broadcom.com>
|
||||
Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
|
||||
Signed-off-by: Xinghai Cen <cenxinghai@h-partners.com>
|
||||
---
|
||||
src/perftest_resources.c | 9 ++++++---
|
||||
1 file changed, 6 insertions(+), 3 deletions(-)
|
||||
|
||||
diff --git a/src/perftest_resources.c b/src/perftest_resources.c
|
||||
index 58dbdef..843c45f 100755
|
||||
--- a/src/perftest_resources.c
|
||||
+++ b/src/perftest_resources.c
|
||||
@@ -1516,7 +1516,7 @@ int create_reg_cqs(struct pingpong_context *ctx,
|
||||
struct perftest_parameters *user_param,
|
||||
int tx_buffer_depth, int need_recv_cq)
|
||||
{
|
||||
- #ifdef HAVE_CQ_EX
|
||||
+#ifdef HAVE_CQ_EX
|
||||
struct ibv_cq_init_attr_ex send_cq_attr = {
|
||||
.cqe = tx_buffer_depth * user_param->num_of_qps,
|
||||
.cq_context = NULL,
|
||||
@@ -1532,6 +1532,8 @@ int create_reg_cqs(struct pingpong_context *ctx,
|
||||
#endif
|
||||
ctx->send_cq = ibv_cq_ex_to_cq(ibv_create_cq_ex(ctx->context, &send_cq_attr));
|
||||
if (!ctx->send_cq) {
|
||||
+ if (!user_param->no_lock && errno == EOPNOTSUPP)
|
||||
+ goto cq_ex_not_supported;
|
||||
fprintf(stderr, "Couldn't create CQ\n");
|
||||
return FAILURE;
|
||||
}
|
||||
@@ -1556,7 +1558,9 @@ int create_reg_cqs(struct pingpong_context *ctx,
|
||||
}
|
||||
}
|
||||
return SUCCESS;
|
||||
-#else
|
||||
+
|
||||
+cq_ex_not_supported:
|
||||
+#endif
|
||||
ctx->send_cq = ibv_create_cq(ctx->context,tx_buffer_depth *
|
||||
user_param->num_of_qps, NULL, ctx->channel, user_param->eq_num);
|
||||
if (!ctx->send_cq) {
|
||||
@@ -1574,7 +1578,6 @@ int create_reg_cqs(struct pingpong_context *ctx,
|
||||
}
|
||||
|
||||
return SUCCESS;
|
||||
-#endif
|
||||
}
|
||||
|
||||
/******************************************************************************
|
||||
--
|
||||
2.25.1
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
Name: perftest
|
||||
Version: 4.5
|
||||
Release: 5
|
||||
Release: 11
|
||||
License: GPLv2 or BSD
|
||||
Summary: RDMA Performance Testing Tools
|
||||
Url: https://github.com/linux-rdma/perftest
|
||||
@ -15,9 +15,14 @@ Patch6: 0006-Perftest-Add-new-HNS-roce-device-ROH-to-support-new_.patch
|
||||
Patch7: 0007-add-loongarch-support-for-perftest.patch
|
||||
Patch8: 0008-Get-CPU-MHz-on-RISC-V.patch
|
||||
Patch9: 0009-Get-CPU-cycles-on-RISC-V.patch
|
||||
Patch10: 0010-Perftest-Support-selecting-congestion-control-algori.patch
|
||||
Patch11: 0011-Perftest-Fix-rx_depth-check-for-XRC.patch
|
||||
Patch12: 0012-Perftest-Add-support-for-TD-lock-free-mode.patch
|
||||
Patch13: 0013-Perftest-Fix-TD-lock-free-mode-not-working-for-QP.patch
|
||||
Patch14: 0014-Perftest-Fix-failure-in-creating-cq-when-create-cq-e.patch
|
||||
|
||||
BuildRequires: automake gcc libibverbs-devel >= 1.2.0 librdmacm-devel >= 1.0.21 libibumad-devel >= 1.3.10.2
|
||||
BuildRequires: pciutils-devel
|
||||
BuildRequires: pciutils-devel libibverbs librdmacm libibumad
|
||||
BuildRequires: guile
|
||||
Obsoletes: openib-perftest < 1.3
|
||||
|
||||
@ -28,6 +33,7 @@ Perftest is a collection of simple tools for testing bandwidth and latency over
|
||||
%autosetup -p1
|
||||
|
||||
%build
|
||||
./autogen.sh
|
||||
%configure
|
||||
%make_build CFLAGS+="-fPIC -g -Wall -D_GNU_SOURCE -O3"
|
||||
|
||||
@ -41,6 +47,42 @@ done
|
||||
%_bindir/*
|
||||
|
||||
%changelog
|
||||
* Mon Sep 2 2024 Xinghai Cen <cenxinghai@h-partners.com> - 4.5-11
|
||||
- Type: bugfix
|
||||
- ID: NA
|
||||
- SUG: NA
|
||||
- DESC: Fix failure in creating cq when create cq ex is not
|
||||
|
||||
* Wed Aug 21 2024 Xinghai Cen <cenxinghai@h-partners.com> - 4.5-10
|
||||
- Type: feature
|
||||
- ID: NA
|
||||
- SUG: NA
|
||||
- DESC: Add support for TD lock-free mode
|
||||
|
||||
* Mon Jul 22 2024 liweigang <liweiganga@uniontech.com> - 4.5-9
|
||||
- Type: bugfix
|
||||
- ID: NA
|
||||
- SUG: NA
|
||||
- DESC: fix build error due to automake update 1.17
|
||||
|
||||
* Mon Jul 8 2024 Xinghai Cen <cenxinghai@h-partners.com> - 4.5-8
|
||||
- Type: bugfix
|
||||
- ID: NA
|
||||
- SUG: NA
|
||||
- DESC: fix rx_depth check for XRC
|
||||
|
||||
* Mon Jul 8 2024 Xinghai Cen <cenxinghai@h-partners.com> - 4.5-7
|
||||
- Type: feature
|
||||
- ID: NA
|
||||
- SUG: NA
|
||||
- DESC: support selecting congestion control algorithms
|
||||
|
||||
* Fri Jan 19 2024 Chengchang Tang <tangchengchang@huawei.com> - 4.5-6
|
||||
- Type: bugfix
|
||||
- ID: NA
|
||||
- SUG: NA
|
||||
- DESC: fix missing required libs
|
||||
|
||||
* Mon Jul 03 2023 Xiaoqian Lv <xiaoqian@nj.iscas.ac.cn> - 4.5-5
|
||||
- Type: enhancement
|
||||
- ID: NA
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user