Rev 48: Joining ctdb and ibwrapper (incomplete). in
http://samba.org/~tridge/psomogyi/
psomogyi at gamax.hu
psomogyi at gamax.hu
Tue Jan 2 17:16:40 GMT 2007
------------------------------------------------------------
revno: 48
revision-id: psomogyi at gamax.hu-20070102171639-h0gukz0ahf4kqtyb
parent: psomogyi at gamax.hu-20061221164148-ubmtld0a1bytpjvi
parent: tridge at samba.org-20061220224349-0c7ucwn9mhop9o6t
parent: tridge at samba.org-20061220224349-0c7ucwn9mhop9o6t
parent: tridge at samba.org-20061220224349-0c7ucwn9mhop9o6t
parent: tridge at samba.org-20061220224349-0c7ucwn9mhop9o6t
committer: Peter Somogyi <psomogyi at gamax.hu>
branch nick: ctdb
timestamp: Tue 2007-01-02 18:16:39 +0100
message:
Joining ctdb and ibwrapper (incomplete).
It compiles.
TODO: modify ctdb as well.
added:
ib/ibw_ctdb.c ibw_ctdb.c-20070102171255-7krov7858dqza466-1
ib/ibw_ctdb.h ibw_ctdb.h-20070102171259-nmuvtzt98aqzg7xp-1
ib/ibw_ctdb_init.c ibw_ctdb_init.c-20070102171305-cn2z4k7ibx8141d5-1
modified:
common/ctdb.c ctdb.c-20061127094323-t50f58d65iaao5of-2
ib/config.m4 config.m4-20061215145907-559hmz134o0rplnk-1
------------------------------------------------------------
revno: 44.1.1
merged: tridge at samba.org-20061220224349-0c7ucwn9mhop9o6t
parent: tridge at samba.org-20061219233527-6luodorqow1fms8j
parent: psomogyi at gamax.hu-20061220181630-gfkxkzqfa5v11u43
committer: Andrew Tridgell <tridge at samba.org>
branch nick: tridge
timestamp: Thu 2006-12-21 09:43:49 +1100
message:
merged from peter
=== added file 'ib/ibw_ctdb.c'
--- a/ib/ibw_ctdb.c 1970-01-01 00:00:00 +0000
+++ b/ib/ibw_ctdb.c 2007-01-02 17:16:39 +0000
@@ -0,0 +1,105 @@
+/*
+ * Unix SMB/CIFS implementation.
+ * Join infiniband wrapper and ctdb.
+ *
+ * Copyright (C) Sven Oehme <oehmes at de.ibm.com> 2006
+ *
+ * Major code contributions by Peter Somogyi <psomogyi at gamax.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include "includes.h"
+#include "lib/events/events.h"
+#include <system/network.h>
+#include <assert.h>
+#include "ctdb_private.h"
+#include "ibwrapper.h"
+#include "ibw_ctdb.h"
+
+int ctdb_ibw_connstate_handler(struct ibw_ctx *ctx, struct ibw_conn *conn)
+{
+ if (ctx!=NULL) {
+ /* ctx->state changed */
+ switch(ctx->state) {
+ case IBWS_INIT: /* ctx start - after ibw_init */
+ break;
+ case IBWS_READY: /* after ibw_bind & ibw_listen */
+ break;
+ case IBWS_CONNECT_REQUEST: /* after [IBWS_READY + incoming request] */
+ /* => [(ibw_accept)IBWS_READY | (ibw_disconnect)STOPPED | ERROR] */
+ if (ibw_accept(ctx, conn, NULL)) {
+ DEBUG(0, ("connstate_handler/ibw_accept failed\n"));
+ return -1;
+ } /* else continue in IBWC_CONNECTED */
+ break;
+ case IBWS_STOPPED: /* normal stop <= ibw_disconnect+(IBWS_READY | IBWS_CONNECT_REQUEST) */
+ /* TODO: have a CTDB upcall for which CTDB should wait in a (final) loop */
+ break;
+ case IBWS_ERROR: /* abnormal state; ibw_stop must be called after this */
+ break;
+ default:
+ assert(0);
+ break;
+ }
+ }
+
+ if (conn!=NULL) {
+ /* conn->state changed */
+ switch(conn->state) {
+ case IBWC_INIT: /* conn start - internal state */
+ break;
+ case IBWC_CONNECTED: { /* after ibw_accept or ibw_connect */
+ struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node);
+ if (node!=NULL) /* after ibw_connect */
+ node->ctdb->upcalls->node_connected(node);
+ else { /* after ibw_accept */
+ /* NOP in CTDB case */
+ }
+ } break;
+ case IBWC_DISCONNECTED: /* after ibw_disconnect */
+ /* TODO: have a CTDB upcall */
+ break;
+ case IBWC_ERROR: {
+ struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node);
+ if (node!=NULL)
+ node->ctdb->upcalls->node_connected(node);
+ } break;
+ default:
+ assert(0);
+ break;
+ }
+ }
+
+ return 0;
+}
+
+int ctdb_ibw_receive_handler(struct ibw_conn *conn, void *buf, int n)
+{
+ struct ctdb_context *ctdb = talloc_get_type(conn->ctx->ctx_userdata, struct ctdb_context);
+
+ assert(ctdb!=NULL);
+ assert(conn->state==IBWC_CONNECTED);
+
+ /* TODO: shall I short-circuit this in ibwrapper? */
+ /* maybe when everything go fine... */
+
+ /* TODO2: !!! here I can provide conn->conn_userdata (with no perf. penalty) -
+ * as struct ctdb_node in case the connection
+ * has been built up by ibw_connect !!! */
+ ctdb->upcalls->recv_pkt(ctdb, (uint8_t *)buf, (uint32_t)n);
+
+ return 0;
+}
=== added file 'ib/ibw_ctdb.h'
--- a/ib/ibw_ctdb.h 1970-01-01 00:00:00 +0000
+++ b/ib/ibw_ctdb.h 2007-01-02 17:16:39 +0000
@@ -0,0 +1,25 @@
+/*
+ * Unix SMB/CIFS implementation.
+ * Join infiniband wrapper and ctdb.
+ *
+ * Copyright (C) Sven Oehme <oehmes at de.ibm.com> 2006
+ *
+ * Major code contributions by Peter Somogyi <psomogyi at gamax.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+int ctdb_ibw_connstate_handler(struct ibw_ctx *ctx, struct ibw_conn *conn);
+int ctdb_ibw_receive_handler(struct ibw_conn *conn, void *buf, int n);
=== added file 'ib/ibw_ctdb_init.c'
--- a/ib/ibw_ctdb_init.c 1970-01-01 00:00:00 +0000
+++ b/ib/ibw_ctdb_init.c 2007-01-02 17:16:39 +0000
@@ -0,0 +1,164 @@
+/*
+ * Unix SMB/CIFS implementation.
+ * Join infiniband wrapper and ctdb.
+ *
+ * Copyright (C) Sven Oehme <oehmes at de.ibm.com> 2006
+ *
+ * Major code contributions by Peter Somogyi <psomogyi at gamax.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include "includes.h"
+#include "lib/events/events.h"
+#include <system/network.h>
+#include <assert.h>
+#include "ctdb_private.h"
+#include "ibwrapper.h"
+#include "ibw_ctdb.h"
+
+static int ctdb_ibw_listen(struct ctdb_context *ctdb, int backlog)
+{
+ struct ibw_ctx *ictx = talloc_get_type(ctdb->private, struct ibw_ctx);
+ struct sockaddr_in my_addr;
+
+ assert(ictx!=NULL);
+ memset(&my_addr, 0, sizeof(struct sockaddr_in));
+ my_addr.sin_port = htons(ctdb->address.port);
+ my_addr.sin_family = PF_INET;
+ inet_pton(AF_INET, ctdb->address.address, &my_addr.sin_addr);
+
+ if (ibw_bind(ictx, &my_addr)) {
+ DEBUG(0, ("ctdb_ibw_listen: ibw_bind failed\n"));
+ return -1;
+ }
+
+ if (ibw_listen(ictx, backlog)) {
+ DEBUG(0, ("ctdb_ibw_listen: ibw_listen failed\n"));
+ return -1;
+ }
+
+ return 0;
+}
+
+int ctdb_ibw_node_connect(struct ibw_ctx *ictx, struct ctdb_node *node)
+{
+ struct sockaddr_in sock_out;
+
+ inet_pton(AF_INET, node->address.address, &sock_out.sin_addr);
+ sock_out.sin_port = htons(node->address.port);
+ sock_out.sin_family = PF_INET;
+
+ if (ibw_connect(ictx, &sock_out, node)) {
+ DEBUG(0, ("ctdb_ibw_node_connect: ibw_connect failed\n"));
+ return -1;
+ }
+
+ /* continues at ibw_ctdb.c/IBWC_CONNECTED in good case */
+ return 0;
+}
+
+/*
+ * Start infiniband
+ */
+static int ctdb_ibw_start(struct ctdb_context *ctdb)
+{
+ struct ibw_ctx *ictx = talloc_get_type(ctdb->private, struct ibw_ctx);
+ int i;
+
+ /* listen on our own address */
+ if (ctdb_ibw_listen(ctdb, 10)) /* TODO: backlog as param */
+ return -1;
+
+ /* everything async here */
+ for (i=0;i<ctdb->num_nodes;i++) {
+ struct ctdb_node *node = ctdb->nodes[i];
+ if (!(ctdb->flags & CTDB_FLAG_SELF_CONNECT) &&
+ ctdb_same_address(&ctdb->address, &node->address))
+ continue;
+ ctdb_ibw_node_connect(ictx, node);
+ }
+
+ return 0;
+}
+
+
+/*
+ * initialise ibw portion of a ctdb node
+ */
+static int ctdb_ibw_add_node(struct ctdb_node *node)
+{
+ /* TODO: clarify whether is this necessary for us ?
+ - why not enough doing such thing internally at connect time ? */
+ return 0;
+}
+
+/*
+ * transport packet allocator - allows transport to control memory for packets
+ */
+static void *ctdb_ibw_allocate_pkt(struct ctdb_context *ctdb, size_t size)
+{
+ struct ibw_conn *conn = NULL;
+ void *buf = NULL;
+ void *key; /* TODO: expand the param list with this */
+
+ /* TODO2: !!! I need "node" or ibw_conn here */
+ if (ibw_alloc_send_buf(conn, &buf, &key, (int)size))
+ return NULL;
+
+ return buf;
+}
+
+static int ctdb_ibw_queue_pkt(struct ctdb_node *node, uint8_t *data, uint32_t length)
+{
+ struct ibw_conn *conn = talloc_get_type(node->private, struct ibw_conn);
+ void *key = NULL; /* TODO: expand the param list with this */
+
+ assert(conn!=NULL);
+ return ibw_send(conn, data, key, length);
+}
+
+static const struct ctdb_methods ctdb_ibw_methods = {
+ .start = ctdb_ibw_start,
+ .add_node = ctdb_ibw_add_node,
+ .queue_pkt = ctdb_ibw_queue_pkt,
+ .allocate_pkt = ctdb_ibw_allocate_pkt
+ /* TODO: missing node_disconnect & final_stop upcalls */
+};
+
+/*
+ * initialise ibw portion of ctdb
+ */
+int ctdb_ibw_init(struct ctdb_context *ctdb)
+{
+ struct ibw_ctx *ictx;
+
+ ictx = ibw_init(
+ NULL, //struct ibw_initattr *attr, /* TODO */
+ 0, //int nattr, /* TODO */
+ ctdb,
+ ctdb_ibw_connstate_handler,
+ ctdb_ibw_receive_handler,
+ ctdb->ev);
+
+ if (ictx==NULL) {
+ DEBUG(0, ("ctdb_ibw_init: ibw_init failed\n"));
+ return -1;
+ }
+
+ ctdb->methods = &ctdb_ibw_methods;
+ ctdb->private = ictx;
+ return 0;
+}
=== modified file 'common/ctdb.c'
--- a/common/ctdb.c 2006-12-18 05:01:11 +0000
+++ b/common/ctdb.c 2007-01-02 17:16:39 +0000
@@ -30,10 +30,18 @@
int ctdb_set_transport(struct ctdb_context *ctdb, const char *transport)
{
int ctdb_tcp_init(struct ctdb_context *ctdb);
+#ifdef HAVE_INFINIBAND
+ int ctdb_ibw_init(struct ctdb_context *ctdb);
+#endif /*HAVE_INFINIBAND*/
if (strcmp(transport, "tcp") == 0) {
return ctdb_tcp_init(ctdb);
}
+#ifdef HAVE_INFINIBAND
+ if (strcmp(transport, "ib") == 0) {
+ return ctdb_ibw_init(ctdb);
+ }
+#endif /*HAVE_INFINIBAND*/
ctdb_set_error(ctdb, "Unknown transport '%s'\n", transport);
return -1;
}
=== modified file 'ib/config.m4'
--- a/ib/config.m4 2006-12-15 15:58:49 +0000
+++ b/ib/config.m4 2007-01-02 17:16:39 +0000
@@ -7,7 +7,7 @@
AC_DEFINE(USE_INFINIBAND,1,[Use infiniband])
HAVE_INFINIBAND=yes
- INFINIBAND_WRAPPER_OBJ="ib/ibwrapper.o"
+ INFINIBAND_WRAPPER_OBJ="ib/ibwrapper.o ib/ibw_ctdb.o ib/ibw_ctdb_init.o"
INFINIBAND_LIBS="-lrdmacm -libverbs"
INFINIBAND_BINS="bin/ibwrapper_test"
More information about the samba-cvs
mailing list