Rev 35: Implemented cm usage. in http://samba.org/~tridge/psomogyi/
psomogyi at gamax.hu
psomogyi at gamax.hu
Wed Dec 6 17:49:47 GMT 2006
------------------------------------------------------------
revno: 35
revision-id: psomogyi at gamax.hu-20061206174946-os4dxhvges06h1m9
parent: psomogyi at gamax.hu-20061205174816-i7ltwane44n034qc
committer: Peter Somogyi <psomogyi at gamax.hu>
branch nick: ctdb
timestamp: Wed 2006-12-06 18:49:46 +0100
message:
Implemented cm usage.
TODO: implement verbs user logic.
modified:
ib/ibwrapper.c ibwrapper.c-20061204130028-0125b4f5a72f4b11
ib/ibwrapper.h ibwrapper.h-20061204130028-32755c6266dd3c49
ib/ibwrapper_internal.h ibwrapper_internal.h-20061204130028-47f0a7e658b16ca2
=== modified file 'ib/ibwrapper.c'
--- a/ib/ibwrapper.c 2006-12-05 17:48:16 +0000
+++ b/ib/ibwrapper.c 2006-12-06 17:49:46 +0000
@@ -51,16 +51,44 @@
ibw_ctx *pctx = talloc_get_type(ctx->internal, ibw_ctx_priv);
assert(pctx!=NULL);
+ /* free memory regions */
+
+ /* destroy verbs */
+ if (pctx->cq) {
+ ibv_destroy_cq(pctx->cq);
+ pctx->cq = NULL;
+ }
+
+ if (pctx->verbs_channel) {
+ ibv_destroy_comp_channel(pctx->verbs_channel);
+ pctx->verbs_channel = NULL;
+ }
+
+ if (pctx->verbs_channel_event) {
+ /* TODO: do we have to do this here? */
+ talloc_free(pctx->verbs_channel_event);
+ pctx->verbs_channel_event = NULL;
+ }
+
+ if (pctx->pd) {
+ ibv_dealloc_pd(pctx->pd);
+ pctx->pd = NULL;
+ }
+
+ /* destroy cm */
+ if (pctx->cm_channel) {
+ rdma_destroy_event_channel(pctx->cm_channel);
+ pctx->cm_channel = NULL;
+ }
+ if (pctx->cm_channel_event) {
+ /* TODO: do we have to do this here? */
+ talloc_free(pctx->cm_channel_event);
+ pctx->cm_channel_event = NULL;
+ }
if (pctx->cm_id) {
rdma_destroy_id(pctx->cm_id);
pctx->cm_id = NULL;
}
- if (pctx->cm_channel) {
- rdma_destroy_event_channel(pctx->cm_channel);
- pctx->cm_channel = NULL;
- }
-
- /* free memory regions */
}
static int ibw_ctx_destruct(void *ptr)
@@ -97,7 +125,7 @@
return 0;
}
-static ibw_conn *ibw_new_conn(ibw_ctx *ctx)
+static ibw_conn *ibw_conn_new(ibw_ctx *ctx)
{
ibw_conn *conn;
ibw_conn_priv *pconn;
@@ -117,7 +145,27 @@
return conn;
}
-static void ibw_process_cm_event(struct event_context *ev,
+static int ibw_manage_connect(struct rdma_cm_id *cma_id)
+{
+ struct rdma_conn_param conn_param;
+ int rc;
+
+ /* TODO: setup verbs... */
+
+ /* cm connect */
+ memset(&conn_param, 0, sizeof conn_param);
+ conn_param.responder_resources = 1;
+ conn_param.initiator_depth = 1;
+ conn_param.retry_count = 10;
+
+ rc = rdma_connect(cma_id, &conn_param);
+ if (rc)
+ sprintf(ibw_lasterr, "rdma_connect error %d\n", rc);
+
+ return rc;
+}
+
+static void ibw_event_handler_cm(struct event_context *ev,
struct fd_event *fde, uint16_t flags, void *private_data)
{
int rc;
@@ -125,8 +173,9 @@
ibw_ctx_priv *pctx = talloc_get_type(ctx->internal, ibw_ctx_priv);
ibw_conn *conn = NULL;
ibw_conn_priv *pconn = NULL;
- struct rdma_cm_id *id = NULL;
+ struct rdma_cm_id *cma_id = NULL;
struct rdma_cm_event *event = NULL;
+ int error = 0;
assert(ctx!=NULL);
@@ -137,42 +186,75 @@
DEBUG(0, ibw_lasterr);
return;
}
- id = event->id;
-
- /* find whose cm_id do we have */
-
-// DEBUG(10, "cma_event type %d cma_id %p (%s)\n", event->event, event->id,
-// (event->id == ctx->cm_id) ? "parent" : "child");
+ cma_id = event->id;
+
+ DEBUG(10, "cma_event type %d cma_id %p (%s)\n", event->event, id,
+ (cma_id == ctx->cm_id) ? "parent" : "child");
switch (event->event) {
case RDMA_CM_EVENT_ADDR_RESOLVED:
+ /* continuing from ibw_connect ... */
assert(pctx->state==IWINT_INIT);
pctx->state = IWINT_ADDR_RESOLVED;
- rc = rdma_resolve_route(event->id, 2000);
+ rc = rdma_resolve_route(cma_id, 2000);
if (rc) {
cb->state = ERROR;
sprintf(ibw_lasterr, "rdma_resolve_route error %d\n", rc);
DEBUG(0, ibw_lasterr);
}
+ /* continued at RDMA_CM_EVENT_ROUTE_RESOLVED */
break;
case RDMA_CM_EVENT_ROUTE_RESOLVED:
+ /* after RDMA_CM_EVENT_ADDR_RESOLVED: */
assert(pctx->state==IWINT_ADDR_RESOLVED);
pctx->state = IWINT_ROUTE_RESOLVED;
+ conn = talloc_get_type(cma_id->context, ibw_conn);
+ pconn = talloc_get_type(conn->internal, ibw_conn_priv);
+
+ rc = ibw_manage_connect(cma_id);
+ if (rc)
+ error = 1;
+
break;
case RDMA_CM_EVENT_CONNECT_REQUEST:
ctx->state = IBWS_CONNECT_REQUEST;
- conn = ibw_new_conn(ctx);
- pconn = talloc_get_type(conn, ibw_conn_priv);
- pconn->cm_id = event->id; /* !!! event will be freed but not id */
- DEBUG(10, "conn->cm_id %p\n", pconn->cm_id);
+ conn = ibw_conn_new(ctx);
+ pconn = talloc_get_type(conn->internal, ibw_conn_priv);
+ pconn->cm_id = cma_id; /* !!! event will be freed but id not */
+ cma_id->context = (void *)conn;
+ DEBUG(10, "pconn->cm_id %p\n", pconn->cm_id);
+
+ conn->state = IBWC_INIT;
+
+ pctx->connstate_func(ctx, conn);
+
+ /* continued at ibw_accept when invoked by the func above */
+ if (!pconn->is_accepted) {
+ talloc_free(conn);
+ DEBUG(10, "pconn->cm_id %p wasn't accepted\n", pconn->cm_id);
+ }
+
+ /* TODO: clarify whether if it's needed by upper layer: */
+ ctx->state = IBWS_READY;
+ pctx->connstate_func(ctx, NULL);
+
+ /* NOTE: more requests can arrive until RDMA_CM_EVENT_ESTABLISHED ! */
break;
case RDMA_CM_EVENT_ESTABLISHED:
+ /* expected after ibw_accept and ibw_connect[not directly] */
DEBUG(0, "ESTABLISHED\n");
ctx->state = IBWS_READY;
- /* TODO */
+ conn = talloc_get_type(cma_id->context, ibw_conn);
+ assert(conn!=NULL); /* important assumption */
+ pconn = talloc_get_type(conn->internal, ibw_conn_priv);
+
+ conn->state = IBWC_CONNECTED;
+
+ /* both ctx and conn have changed */
+ pctx->connstate_func(ctx, conn);
break;
case RDMA_CM_EVENT_ADDR_ERROR:
@@ -180,30 +262,63 @@
case RDMA_CM_EVENT_CONNECT_ERROR:
case RDMA_CM_EVENT_UNREACHABLE:
case RDMA_CM_EVENT_REJECTED:
- DEBUG(0, "cma event %d, error %d\n", event->event,
- event->status);
- ctx->state = IBWS_ERROR;
+ DEBUG(0, "cma event %d, error %d\n", event->event, event->status);
+ error = 1;
break;
case RDMA_CM_EVENT_DISCONNECTED:
- DEBUG(0, "%s DISCONNECT EVENT...\n", cb->server ? "server" : "client");
- /* TODO */
+ if (cma_id!=ctx->cm_id) {
+ DEBUG(0, "client DISCONNECT event\n");
+ conn = talloc_get_type(cma_id->context, ibw_conn);
+ conn->state = IBWC_DISCONNECTED;
+ pctx->connstate_func(NULL, conn);
+
+ talloc_free(conn);
+ } else {
+ DEBUG(0, "server DISCONNECT event\n");
+ ctx->state = IBWS_STOPPED; /* ??? TODO: try it... */
+ pctx->connstate_func(ctx, NULL);
+ }
break;
case RDMA_CM_EVENT_DEVICE_REMOVAL:
DEBUG(0, "cma detected device removal!\n");
+ error = 1;
break;
default:
- DEBUG(0, "oof bad type!\n");
+ DEBUG(0, "unknown event %d\n", event->event);
+ error = 1;
break;
}
+ if (error) {
+ DEBUG(0, ibw_lasterr);
+ if (cma_id!=ctx->cm_id) {
+ conn = talloc_get_type(cma_id->context, ibw_conn);
+ conn->state = IBWC_ERROR;
+ pctx->connstate_func(NULL, conn);
+ } else {
+ ctx->state = IBWS_ERROR;
+ pctx->connstate_func(ctx, NULL);
+ }
+ }
+
if ((rc=rdma_ack_cm_event(event))) {
- DEBUG(0, "rdma_ack_cm_event failed with %d\n", rc);
+ sprintf(ibw_lasterr, "rdma_ack_cm_event failed with %d\n");
+ DEBUG(0, ibw_lasterr, rc);
}
}
+static void ibw_event_handler_verbs(struct event_context *ev,
+ struct fd_event *fde, uint16_t flags, void *private_data)
+{
+ int rc;
+ ibw_ctx *ctx = talloc_get_type(private_data, ibw_ctx);
+ ibw_ctx_priv *pctx = talloc_get_type(ctx->internal, ibw_ctx_priv);
+
+}
+
static int ibw_process_init_attrs(ibw_initattr *attr, int nattr, ibw_opts *opts)
{
int i;
@@ -260,7 +375,7 @@
if (ibw_process_init_attrs(attr, nattr, pctx->opts))
goto cleanup;
- /* initialize CM stuff */
+ /* init cm */
pctx->cm_channel = rdma_create_event_channel();
if (!pctx->cm_channel) {
ret = errno;
@@ -269,7 +384,7 @@
}
pctx->cm_channel_event = event_add_fd(pctx->ectx, pctx,
- pctx->cm_channel->fd, EVENT_FD_READ, ibw_process_cm_event, ctx);
+ pctx->cm_channel->fd, EVENT_FD_READ, ibw_event_handler_cm, ctx);
rc = rdma_create_id(pctx->cm_channel, &pctx->cm_id, cb, RDMA_PS_TCP);
if (rc) {
@@ -279,11 +394,33 @@
}
DEBUG(10, "created cm_id %p\n", pctx->cm_id);
+ /* init verbs */
+ pctx->pd = ibv_alloc_pd(pctx->cmid->verbs);
+ if (!pctx->pd) {
+ sprintf(ibw_lasterr, "ibv_alloc_pd failed %d\n", errno);
+ goto cleanup;
+ }
+ DEBUG(10, "created pd %p\n", pctx->pd);
+
+ pctx->verbs_channel = ibv_create_comp_channel(cm_id->verbs);
+ if (!pctx->verbs_channel) {
+ sprintf(stderr, "ibv_create_comp_channel failed %d\n", errno);
+ goto cleanup;
+ }
+ DEBUG_LOG("created channel %p\n", pctx->channel);
+
+ pctx->verbs_channel_event = event_add_fd(pctx->ectx, pctx,
+ pctx->verbs_channel->fd, EVENT_FD_READ, ibw_event_handler_verbs, ctx);
+
+ pctx->cq = ibv_create_cq(cm_id->verbs, pctx->opts.rx_depth, ctx,
+ ctx->verbs_channel, 0);
+
/* allocate ib memory regions */
return ctx;
cleanup:
+ DEBUG(0, ibw_lasterr);
if (ctx)
talloc_free(ctx);
@@ -301,46 +438,97 @@
ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal;
int rc;
- rc = rdma_bind_addr(cb->cm_id, (struct sockaddr *) &my_addr);
+ rc = rdma_bind_addr(pctx->cm_id, (struct sockaddr *) my_addr);
if (rc) {
sprintf(ibw_lasterr, "rdma_bind_addr error %d\n", rc);
+ DEBUG(0, ibw_lasterr);
return rc;
}
+ DEBUG(10, "rdma_bind_addr successful\n");
return 0;
}
int ibw_listen(ibw_ctx *ctx, int backlog)
{
- ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal;
-
+ ibw_ctx_priv *pctx = talloc_get_type(ctx->internal, ibw_ctx_priv);
+ int rc;
+
+ DEBUG_LOG("rdma_listen...\n");
+ rc = rdma_listen(cb->cm_id, backlog);
+ if (rc) {
+ sprintf(ibw_lasterr, "rdma_listen failed: %d\n", ret);
+ DEBUG(0, ibw_lasterr);
+ return rc;
+ }
+
return 0;
}
-int ibw_accept(ibw_ctx *ctx, void *conn_userdata)
+int ibw_accept(ibw_ctx *ctx, ibw_conn *conn, void *conn_userdata)
{
- ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal;
-
+ ibw_ctx_priv *pctx = talloc_get_type(ctx->internal, ibw_ctx_priv);
+ ibw_conn_priv *pconn = talloc_get_type(conn->internal, ibw_conn_priv);
+ struct rdma_conn_param conn_param;
+
+ memset(&conn_param, 0, sizeof(struct rdma_conn_param));
+ conn_param.responder_resources = 1;
+ conn_param.initiator_depth = 1;
+ rc = rdma_accept(pconn->cm_id, &conn_param);
+ if (rc) {
+ sprintf(ibw_lasterr, "rdma_accept failed %d\n", rc);
+ DEBUG(0, ibw_lasterr);
+ return -1;;
+ }
+
+ pconn->is_accepted = 1;
+
+ /* continued at RDMA_CM_EVENT_ESTABLISHED */
+
return 0;
}
int ibw_connect(ibw_ctx *ctx, struct sockaddr_in *serv_addr, void *conn_userdata)
{
- ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal;
-
+ ibw_ctx_priv *pctx = talloc_get_type(ctx->internal, ibw_ctx_priv);
+ ibw_conn *conn = NULL;
+ int rc;
+
+ conn = ibw_conn_new(ctx);
+ conn->conn_userdata = conn_userdata;
+ pconn = talloc_get_type(conn->internal, ibw_conn_priv);
+
+ rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn, RDMA_PS_TCP);
+ if (rc) {
+ rc = errno;
+ sprintf(ibw_lasterr, "rdma_create_id error %d\n", rc);
+ return rc;
+ }
+
+ assert(ctx->state==IBWS_READY);
+
+ rc = rdma_resolve_addr(pconn->cm_id, NULL, (struct sockaddr *) &serv_addr, 2000);
+ if (rc) {
+ sprintf(ibw_lasterr, "rdma_resolve_addr error %d\n", rc);
+ DEBUG(0, ibw_lasterr);
+ return -1;
+ }
+
+ /* continued at RDMA_CM_EVENT_ADDR_RESOLVED */
+
return 0;
}
void ibw_disconnect(ibw_conn *conn)
{
- ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal;
+ ibw_conn_priv *pconn = talloc_get_type(conn->internal, ibw_conn_priv);
return 0;
}
int ibw_alloc_send_buf(ibw_conn *conn, void **buf, void **key, int n)
{
- ibw_conn_priv *pconn = (ibw_ctx_priv *)ctx->internal;
+ ibw_conn_priv *pconn = talloc_get_type(conn->internal, ibw_conn_priv);
return 0;
}
=== modified file 'ib/ibwrapper.h'
--- a/ib/ibwrapper.h 2006-12-05 17:48:16 +0000
+++ b/ib/ibwrapper.h 2006-12-06 17:49:46 +0000
@@ -148,7 +148,7 @@
*
* Important: you won't get remote IP address (only internal conn info)
*/
-int ibw_accept(ibw_ctx *ctx, void *conn_userdata);
+int ibw_accept(ibw_ctx *ctx, ibw_conn *conn, void *conn_userdata);
/*
* Needs a normal internet address here
=== modified file 'ib/ibwrapper_internal.h'
--- a/ib/ibwrapper_internal.h 2006-12-04 18:48:11 +0000
+++ b/ib/ibwrapper_internal.h 2006-12-06 17:49:46 +0000
@@ -50,12 +50,17 @@
ibw_opts opts;
- struct ibv_context *context;
+ struct rdma_cm_id *cm_id; /* server cm id */
+
+ struct rdma_event_channel *cm_channel;
+ struct fd_event *cm_channel_event;
+
+ struct rdma_event_channel *cm_channel;
+ struct fd_event *cm_channel_event;
+ struct ibv_comp_channel *verbs_channel;
+ struct fd_event *verbs_channel_event;
+
struct ibv_pd *pd;
- struct rdma_cm_id *cm_id; /* server cm id */
-
- struct rdma_event_channel *cm_channel;
- struct fd_event *cm_channel_event;
ibw_connstate_fn_t connstate_func;
ibw_receive_fn_t receive_func;
@@ -66,6 +71,7 @@
struct ibv_qp *qp;
struct rdma_cm_id *cm_id; /* client's cm id */
+ int is_accepted;
} ibw_conn_priv;
/*
More information about the samba-cvs
mailing list