corosync  3.0.1
totemknet.c
Go to the documentation of this file.
1 
2 /*
3  * Copyright (c) 2016-2018 Red Hat, Inc.
4  *
5  * All rights reserved.
6  *
7  * Author: Christine Caulfield (ccaulfie@redhat.com)
8 
9  * This software licensed under BSD license, the text of which follows:
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions are met:
13  *
14  * - Redistributions of source code must retain the above copyright notice,
15  * this list of conditions and the following disclaimer.
16  * - Redistributions in binary form must reproduce the above copyright notice,
17  * this list of conditions and the following disclaimer in the documentation
18  * and/or other materials provided with the distribution.
19  * - Neither the name of the MontaVista Software, Inc. nor the names of its
20  * contributors may be used to endorse or promote products derived from this
21  * software without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33  * THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 #include <config.h>
37 
38 #include <assert.h>
39 #include <sys/mman.h>
40 #include <sys/types.h>
41 #include <sys/stat.h>
42 #include <sys/socket.h>
43 #include <netdb.h>
44 #include <sys/un.h>
45 #include <sys/ioctl.h>
46 #include <sys/param.h>
47 #include <netinet/in.h>
48 #include <arpa/inet.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <stdlib.h>
52 #include <stdio.h>
53 #include <errno.h>
54 #include <sched.h>
55 #include <time.h>
56 #include <sys/time.h>
57 #include <sys/poll.h>
58 #include <sys/uio.h>
59 #include <limits.h>
60 
61 #include <qb/qbdefs.h>
62 #include <qb/qbloop.h>
63 
64 #include <corosync/sq.h>
65 #include <corosync/swab.h>
66 #include <corosync/logsys.h>
67 #include <corosync/icmap.h>
68 #include <corosync/totem/totemip.h>
69 #include "totemknet.h"
70 
71 #include "util.h"
72 
73 #include <libknet.h>
75 
76 #ifndef MSG_NOSIGNAL
77 #define MSG_NOSIGNAL 0
78 #endif
79 
80 /* Should match that used by cfg */
81 #define CFG_INTERFACE_STATUS_MAX_LEN 512
82 
84  struct crypto_instance *crypto_inst;
85 
86  qb_loop_t *poll_handle;
87 
88  knet_handle_t knet_handle;
89 
90  int link_mode;
91 
92  void *context;
93 
95  void *context,
96  const void *msg,
97  unsigned int msg_len,
98  const struct sockaddr_storage *system_from);
99 
101  void *context,
102  const struct totem_ip_address *iface_address,
103  unsigned int link_no);
104 
106  void *context,
107  int net_mtu);
108 
110 
111  /*
112  * Function and data used to log messages
113  */
115 
117 
119 
121 
123 
125 
127 
129  int level,
130  int subsys,
131  const char *function,
132  const char *file,
133  int line,
134  const char *format,
135  ...)__attribute__((format(printf, 6, 7)));
136 
138 
139  char iov_buffer[KNET_MAX_PACKET_SIZE];
140 
142 
144 
146 
148 
150 
152 
154 
155  qb_loop_timer_handle timer_netif_check_timeout;
156 
157  qb_loop_timer_handle timer_merge_detect_timeout;
158 
160 
162 
163  int logpipes[2];
164  int knet_fd;
165 };
166 
167 /* Awkward. But needed to get stats from knet */
169 
170 struct work_item {
171  const void *msg;
172  unsigned int msg_len;
174 };
175 
177  void *knet_context);
178 
179 static void totemknet_start_merge_detect_timeout(
180  void *knet_context);
181 
182 static void totemknet_stop_merge_detect_timeout(
183  void *knet_context);
184 
185 static void log_flush_messages (
186  void *knet_context);
187 
188 static void totemknet_instance_initialize (struct totemknet_instance *instance)
189 {
190  memset (instance, 0, sizeof (struct totemknet_instance));
191 }
192 
193 #define knet_log_printf(level, format, args...) \
194 do { \
195  instance->totemknet_log_printf ( \
196  level, instance->totemknet_subsys_id, \
197  __FUNCTION__, __FILE__, __LINE__, \
198  (const char *)format, ##args); \
199 } while (0);
200 
201 #define libknet_log_printf(level, format, args...) \
202 do { \
203  instance->totemknet_log_printf ( \
204  level, instance->knet_subsys_id, \
205  __FUNCTION__, "libknet.h", __LINE__, \
206  (const char *)format, ##args); \
207 } while (0);
208 
209 #define KNET_LOGSYS_PERROR(err_num, level, fmt, args...) \
210 do { \
211  char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
212  const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
213  instance->totemknet_log_printf ( \
214  level, instance->totemknet_subsys_id, \
215  __FUNCTION__, __FILE__, __LINE__, \
216  fmt ": %s (%d)", ##args, _error_ptr, err_num); \
217  } while(0)
218 
219 
220 static int dst_host_filter_callback_fn(void *private_data,
221  const unsigned char *outdata,
222  ssize_t outdata_len,
223  uint8_t tx_rx,
224  knet_node_id_t this_host_id,
225  knet_node_id_t src_host_id,
226  int8_t *channel,
227  knet_node_id_t *dst_host_ids,
228  size_t *dst_host_ids_entries)
229 {
230  struct totem_message_header *header = (struct totem_message_header *)outdata;
231  int res;
232 
233  *channel = 0;
234  if (header->target_nodeid) {
235  dst_host_ids[0] = header->target_nodeid;
236  *dst_host_ids_entries = 1;
237  res = 0; /* unicast message */
238  }
239  else {
240  *dst_host_ids_entries = 0;
241  res = 1; /* multicast message */
242  }
243  return res;
244 }
245 
246 static void socket_error_callback_fn(void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno)
247 {
248  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
249 
250  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet socket ERROR notification called: txrx=%d, error=%d, errorno=%d", tx_rx, error, errorno);
251  if ((error == -1 && errorno != EAGAIN) || (error == 0)) {
252  knet_handle_remove_datafd(instance->knet_handle, datafd);
253  }
254 }
255 
256 static void host_change_callback_fn(void *private_data, knet_node_id_t host_id, uint8_t reachable, uint8_t remote, uint8_t external)
257 {
258  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
259 
260  // TODO: what? if anything.
261  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet host change callback. nodeid: %d reachable: %d", host_id, reachable);
262 }
263 
264 static void pmtu_change_callback_fn(void *private_data, unsigned int data_mtu)
265 {
266  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
267  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet pMTU change: %d", data_mtu);
268 
269  /* We don't need to tell corosync the actual knet MTU */
270 // instance->totemknet_mtu_changed(instance->context, data_mtu);
271 }
272 
274  void *knet_context,
275  const char *cipher_type,
276  const char *hash_type)
277 {
278  return (0);
279 }
280 
281 
282 static inline void ucast_sendmsg (
283  struct totemknet_instance *instance,
284  struct totem_ip_address *system_to,
285  const void *msg,
286  unsigned int msg_len)
287 {
288  int res = 0;
289  struct totem_message_header *header = (struct totem_message_header *)msg;
290  struct msghdr msg_ucast;
291  struct iovec iovec;
292 
293  header->target_nodeid = system_to->nodeid;
294 
295  iovec.iov_base = (void *)msg;
296  iovec.iov_len = msg_len;
297 
298  /*
299  * Build unicast message
300  */
301  memset(&msg_ucast, 0, sizeof(msg_ucast));
302  msg_ucast.msg_iov = (void *)&iovec;
303  msg_ucast.msg_iovlen = 1;
304 #ifdef HAVE_MSGHDR_CONTROL
305  msg_ucast.msg_control = 0;
306 #endif
307 #ifdef HAVE_MSGHDR_CONTROLLEN
308  msg_ucast.msg_controllen = 0;
309 #endif
310 #ifdef HAVE_MSGHDR_FLAGS
311  msg_ucast.msg_flags = 0;
312 #endif
313 #ifdef HAVE_MSGHDR_ACCRIGHTS
314  msg_ucast.msg_accrights = NULL;
315 #endif
316 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
317  msg_ucast.msg_accrightslen = 0;
318 #endif
319 
320  /*
321  * Transmit unicast message
322  * An error here is recovered by totemsrp
323  */
324 
325  res = sendmsg (instance->knet_fd, &msg_ucast, MSG_NOSIGNAL);
326  if (res < 0) {
328  "sendmsg(ucast) failed (non-critical)");
329  }
330 }
331 
332 static inline void mcast_sendmsg (
333  struct totemknet_instance *instance,
334  const void *msg,
335  unsigned int msg_len,
336  int only_active)
337 {
338  int res;
339  struct totem_message_header *header = (struct totem_message_header *)msg;
340  struct msghdr msg_mcast;
341  struct iovec iovec;
342 
343  iovec.iov_base = (void *)msg;
344  iovec.iov_len = msg_len;
345 
346  header->target_nodeid = 0;
347 
348  /*
349  * Build multicast message
350  */
351  memset(&msg_mcast, 0, sizeof(msg_mcast));
352  msg_mcast.msg_iov = (void *)&iovec;
353  msg_mcast.msg_iovlen = 1;
354 #ifdef HAVE_MSGHDR_CONTROL
355  msg_mcast.msg_control = 0;
356 #endif
357 #ifdef HAVE_MSGHDR_CONTROLLEN
358  msg_mcast.msg_controllen = 0;
359 #endif
360 #ifdef HAVE_MSGHDR_FLAGS
361  msg_mcast.msg_flags = 0;
362 #endif
363 #ifdef HAVE_MSGHDR_ACCRIGHTS
364  msg_mcast.msg_accrights = NULL;
365 #endif
366 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
367  msg_mcast.msg_accrightslen = 0;
368 #endif
369 
370 
371 // log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_sendmsg. only_active=%d, len=%d", only_active, msg_len);
372 
373  res = sendmsg (instance->knet_fd, &msg_mcast, MSG_NOSIGNAL);
374  if (res < msg_len) {
375  knet_log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_send sendmsg returned %d", res);
376  }
377 
378  if (!only_active || instance->send_merge_detect_message) {
379  /*
380  * Current message was sent to all nodes
381  */
383  instance->send_merge_detect_message = 0;
384  }
385 }
386 
387 static int node_compare(const void *aptr, const void *bptr)
388 {
389  uint16_t a,b;
390 
391  a = *(uint16_t *)aptr;
392  b = *(uint16_t *)bptr;
393 
394  return a > b;
395 }
396 
397 int totemknet_ifaces_get (void *knet_context,
398  char ***status,
399  unsigned int *iface_count)
400 {
401  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
402  struct knet_link_status link_status;
403  knet_node_id_t host_list[KNET_MAX_HOST];
404  uint8_t link_list[KNET_MAX_LINK];
405  size_t num_hosts;
406  size_t num_links;
407  size_t link_idx;
408  int i,j;
409  char *ptr;
410  int res = 0;
411 
412  /*
413  * Don't do the whole 'link_info' bit if the caller just wants
414  * a count of interfaces.
415  */
416  if (status) {
417 
418  res = knet_host_get_host_list(instance->knet_handle,
419  host_list, &num_hosts);
420  if (res) {
421  return (-1);
422  }
423  qsort(host_list, num_hosts, sizeof(uint16_t), node_compare);
424 
425  for (i=0; i<INTERFACE_MAX; i++) {
426  memset(instance->link_status[i], 'n', CFG_INTERFACE_STATUS_MAX_LEN-1);
427  instance->link_status[i][num_hosts] = '\0';
428  }
429 
430  /* This is all a bit "inside-out" because "status" is a set of strings per link
431  * and knet orders things by host
432  */
433  for (j=0; j<num_hosts; j++) {
434  res = knet_link_get_link_list(instance->knet_handle,
435  host_list[j], link_list, &num_links);
436  if (res) {
437  return (-1);
438  }
439 
440  link_idx = 0;
441  for (i=0; i < num_links; i++) {
442  /*
443  * Skip over links that are unconfigured to corosync. This is basically
444  * link0 if corosync isn't using it for comms, as we will still
445  * have it set up for loopback.
446  */
447  if (!instance->totem_config->interfaces[link_list[i]].configured) {
448  continue;
449  }
450  ptr = instance->link_status[link_idx++];
451 
452  res = knet_link_get_status(instance->knet_handle,
453  host_list[j],
454  link_list[i],
455  &link_status,
456  sizeof(link_status));
457  if (res == 0) {
458  ptr[j] = '0' + (link_status.enabled |
459  link_status.connected<<1 |
460  link_status.dynconnected<<2);
461  }
462  else {
463  ptr[j] = '?';
464  }
465  }
466  }
467  *status = instance->link_status;
468  }
469 
470  *iface_count = INTERFACE_MAX;
471 
472  return (res);
473 }
474 
476  void *knet_context)
477 {
478  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
479  int res = 0;
480  int i,j;
481  static knet_node_id_t nodes[KNET_MAX_HOST]; /* static to save stack */
482  uint8_t links[KNET_MAX_LINK];
483  size_t num_nodes;
484  size_t num_links;
485 
486  knet_log_printf(LOG_DEBUG, "totemknet: finalize");
487 
488  qb_loop_poll_del (instance->poll_handle, instance->logpipes[0]);
489  qb_loop_poll_del (instance->poll_handle, instance->knet_fd);
490 
491  res = knet_host_get_host_list(instance->knet_handle, nodes, &num_nodes);
492  if (res) {
493  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet node list for shutdown: %s", strerror(errno));
494  /* Crash out anyway */
495  goto finalise_error;
496  }
497 
498  /* Tidily shut down all nodes & links. This ensures that the LEAVE message will be sent */
499  for (i=0; i<num_nodes; i++) {
500 
501  res = knet_link_get_link_list(instance->knet_handle, nodes[i], links, &num_links);
502  if (res) {
503  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet link list for node %d: %s", nodes[i], strerror(errno));
504  goto finalise_error;
505  }
506  for (j=0; j<num_links; j++) {
507  res = knet_link_set_enable(instance->knet_handle, nodes[i], links[j], 0);
508  if (res) {
509  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_set_enable(node %d, link %d) failed: %s", nodes[i], links[j], strerror(errno));
510  }
511  res = knet_link_clear_config(instance->knet_handle, nodes[i], links[j]);
512  if (res) {
513  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_clear_config(node %d, link %d) failed: %s", nodes[i], links[j], strerror(errno));
514  }
515  }
516  res = knet_host_remove(instance->knet_handle, nodes[i]);
517  if (res) {
518  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_host_remove(node %d) failed: %s", nodes[i], strerror(errno));
519  }
520  }
521 
522 finalise_error:
523  res = knet_handle_setfwd(instance->knet_handle, 0);
524  if (res) {
525  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_setfwd failed: %s", strerror(errno));
526  }
527  res = knet_handle_free(instance->knet_handle);
528  if (res) {
529  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_free failed: %s", strerror(errno));
530  }
531 
532  totemknet_stop_merge_detect_timeout(instance);
533 
534  log_flush_messages(instance);
535 
536  return (res);
537 }
538 
539 static int log_deliver_fn (
540  int fd,
541  int revents,
542  void *data)
543 {
544  struct totemknet_instance *instance = (struct totemknet_instance *)data;
545  char buffer[sizeof(struct knet_log_msg)*4];
546  char *bufptr = buffer;
547  int done = 0;
548  int len;
549 
550  len = read(fd, buffer, sizeof(buffer));
551  while (done < len) {
552  struct knet_log_msg *msg = (struct knet_log_msg *)bufptr;
553  switch (msg->msglevel) {
554  case KNET_LOG_ERR:
556  knet_log_get_subsystem_name(msg->subsystem),
557  msg->msg);
558  break;
559  case KNET_LOG_WARN:
561  knet_log_get_subsystem_name(msg->subsystem),
562  msg->msg);
563  break;
564  case KNET_LOG_INFO:
566  knet_log_get_subsystem_name(msg->subsystem),
567  msg->msg);
568  break;
569  case KNET_LOG_DEBUG:
571  knet_log_get_subsystem_name(msg->subsystem),
572  msg->msg);
573  break;
574  }
575  bufptr += sizeof(struct knet_log_msg);
576  done += sizeof(struct knet_log_msg);
577  }
578  return 0;
579 }
580 
581 static int data_deliver_fn (
582  int fd,
583  int revents,
584  void *data)
585 {
586  struct totemknet_instance *instance = (struct totemknet_instance *)data;
587  struct msghdr msg_hdr;
588  struct iovec iov_recv;
589  struct sockaddr_storage system_from;
590  ssize_t msg_len;
591  int truncated_packet;
592 
593  iov_recv.iov_base = instance->iov_buffer;
594  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
595 
596  msg_hdr.msg_name = &system_from;
597  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
598  msg_hdr.msg_iov = &iov_recv;
599  msg_hdr.msg_iovlen = 1;
600 #ifdef HAVE_MSGHDR_CONTROL
601  msg_hdr.msg_control = 0;
602 #endif
603 #ifdef HAVE_MSGHDR_CONTROLLEN
604  msg_hdr.msg_controllen = 0;
605 #endif
606 #ifdef HAVE_MSGHDR_FLAGS
607  msg_hdr.msg_flags = 0;
608 #endif
609 #ifdef HAVE_MSGHDR_ACCRIGHTS
610  msg_hdr.msg_accrights = NULL;
611 #endif
612 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
613  msg_hdr.msg_accrightslen = 0;
614 #endif
615 
616  msg_len = recvmsg (fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
617  if (msg_len <= 0) {
618  return (0);
619  }
620 
621  truncated_packet = 0;
622 
623 #ifdef HAVE_MSGHDR_FLAGS
624  if (msg_hdr.msg_flags & MSG_TRUNC) {
625  truncated_packet = 1;
626  }
627 #else
628  /*
629  * We don't have MSGHDR_FLAGS, but we can (hopefully) safely make assumption that
630  * if bytes_received == KNET_MAX_PACKET_SIZE then packet is truncated
631  */
632  if (bytes_received == KNET_MAX_PACKET_SIZE) {
633  truncated_packet = 1;
634  }
635 #endif
636 
637  if (truncated_packet) {
639  "Received too big message. This may be because something bad is happening"
640  "on the network (attack?), or you tried join more nodes than corosync is"
641  "compiled with (%u) or bug in the code (bad estimation of "
642  "the KNET_MAX_PACKET_SIZE). Dropping packet.", PROCESSOR_COUNT_MAX);
643  return (0);
644  }
645 
646  /*
647  * Handle incoming message
648  */
649  instance->totemknet_deliver_fn (
650  instance->context,
651  instance->iov_buffer,
652  msg_len,
653  &system_from);
654 
655  return (0);
656 }
657 
658 static void timer_function_netif_check_timeout (
659  void *data)
660 {
661  struct totemknet_instance *instance = (struct totemknet_instance *)data;
662  int i;
663 
664  for (i=0; i < INTERFACE_MAX; i++) {
665  if (!instance->totem_config->interfaces[i].configured) {
666  continue;
667  }
668  instance->totemknet_iface_change_fn (instance->context,
669  &instance->my_ids[i],
670  i);
671  }
672 }
673 
674 /* NOTE: this relies on the fact that totem_reload_notify() is called first */
675 static void totemknet_refresh_config(
676  int32_t event,
677  const char *key_name,
678  struct icmap_notify_value new_val,
679  struct icmap_notify_value old_val,
680  void *user_data)
681 {
682  uint8_t reloading;
683  uint32_t value;
684  uint32_t link_no;
685  size_t num_nodes;
686  knet_node_id_t host_ids[KNET_MAX_HOST];
687  int i;
688  int err;
689  struct totemknet_instance *instance = (struct totemknet_instance *)user_data;
690 
691  ENTER();
692 
693  /*
694  * If a full reload is in progress then don't do anything until it's done and
695  * can reconfigure it all atomically
696  */
697  if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading) == CS_OK && reloading) {
698  return;
699  }
700 
701  if (icmap_get_uint32("totem.knet_pmtud_interval", &value) == CS_OK) {
702 
704  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_pmtud_interval now %d", value);
705  err = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
706  if (err) {
707  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
708  }
709  }
710 
711  /* Configure link parameters for each node */
712  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_nodes);
713  if (err != 0) {
714  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list failed");
715  }
716 
717  for (i=0; i<num_nodes; i++) {
718  for (link_no = 0; link_no < INTERFACE_MAX; link_no++) {
719  if (host_ids[i] == instance->our_nodeid || !instance->totem_config->interfaces[link_no].configured) {
720  continue;
721  }
722 
723  err = knet_link_set_ping_timers(instance->knet_handle, host_ids[i], link_no,
724  instance->totem_config->interfaces[link_no].knet_ping_interval,
725  instance->totem_config->interfaces[link_no].knet_ping_timeout,
726  instance->totem_config->interfaces[link_no].knet_ping_precision);
727  if (err) {
728  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for node %d link %d failed", host_ids[i], link_no);
729  }
730  err = knet_link_set_pong_count(instance->knet_handle, host_ids[i], link_no,
731  instance->totem_config->interfaces[link_no].knet_pong_count);
732  if (err) {
733  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for node %d link %d failed",host_ids[i], link_no);
734  }
735  err = knet_link_set_priority(instance->knet_handle, host_ids[i], link_no,
736  instance->totem_config->interfaces[link_no].knet_link_priority);
737  if (err) {
738  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for node %d link %d failed", host_ids[i], link_no);
739  }
740 
741  }
742  }
743 
744  LEAVE();
745 }
746 
747 static void totemknet_add_config_notifications(struct totemknet_instance *instance)
748 {
749  icmap_track_t icmap_track_totem = NULL;
750  icmap_track_t icmap_track_reload = NULL;
751 
752  ENTER();
753 
754  icmap_track_add("totem.",
756  totemknet_refresh_config,
757  instance,
758  &icmap_track_totem);
759 
760  icmap_track_add("config.totemconfig_reload_in_progress",
762  totemknet_refresh_config,
763  instance,
764  &icmap_track_reload);
765 
766  LEAVE();
767 }
768 
769 /*
770  * Create an instance
771  */
773  qb_loop_t *poll_handle,
774  void **knet_context,
775  struct totem_config *totem_config,
776  totemsrp_stats_t *stats,
777  void *context,
778 
779  void (*deliver_fn) (
780  void *context,
781  const void *msg,
782  unsigned int msg_len,
783  const struct sockaddr_storage *system_from),
784 
785  void (*iface_change_fn) (
786  void *context,
787  const struct totem_ip_address *iface_address,
788  unsigned int link_no),
789 
790  void (*mtu_changed) (
791  void *context,
792  int net_mtu),
793 
794  void (*target_set_completed) (
795  void *context))
796 {
797  struct totemknet_instance *instance;
798  int8_t channel=0;
799  int res;
800  int i;
801 
802  instance = malloc (sizeof (struct totemknet_instance));
803  if (instance == NULL) {
804  return (-1);
805  }
806 
807  totemknet_instance_initialize (instance);
808 
809  instance->totem_config = totem_config;
810 
811  /*
812  * Configure logging
813  */
814  instance->totemknet_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
821 
822  instance->knet_subsys_id = _logsys_subsys_create("KNET", "libknet.h");
823 
824  /*
825  * Initialize local variables for totemknet
826  */
827 
828  instance->our_nodeid = instance->totem_config->node_id;
829 
830  for (i=0; i< INTERFACE_MAX; i++) {
831  totemip_copy(&instance->my_ids[i], &totem_config->interfaces[i].bindnet);
832  instance->my_ids[i].nodeid = instance->our_nodeid;
833  instance->ip_port[i] = totem_config->interfaces[i].ip_port;
834 
835  /* Needed for totemsrp */
837  }
838 
839  instance->poll_handle = poll_handle;
840 
841  instance->context = context;
842  instance->totemknet_deliver_fn = deliver_fn;
843 
844  instance->totemknet_iface_change_fn = iface_change_fn;
845 
846  instance->totemknet_mtu_changed = mtu_changed;
847 
848  instance->totemknet_target_set_completed = target_set_completed;
849 
850  instance->loopback_link = 0;
851 
852  res = pipe(instance->logpipes);
853  if (res == -1) {
854  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to create pipe for instance->logpipes");
855  goto exit_error;
856  }
857  fcntl(instance->logpipes[0], F_SETFL, O_NONBLOCK);
858  fcntl(instance->logpipes[1], F_SETFL, O_NONBLOCK);
859 
860 #if !defined(KNET_API_VER) || (KNET_API_VER == 1)
861  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG);
862 #endif
863 #if KNET_API_VER == 2
864  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, KNET_HANDLE_FLAG_PRIVILEGED);
865 #endif
866 
867  if (!instance->knet_handle) {
868  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "knet_handle_new failed");
869  goto exit_error;
870  }
871  res = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
872  if (res) {
873  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
874  }
875  res = knet_handle_enable_filter(instance->knet_handle, instance, dst_host_filter_callback_fn);
876  if (res) {
877  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_filter failed");
878  }
879  res = knet_handle_enable_sock_notify(instance->knet_handle, instance, socket_error_callback_fn);
880  if (res) {
881  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_sock_notify failed");
882  }
883  res = knet_host_enable_status_change_notify(instance->knet_handle, instance, host_change_callback_fn);
884  if (res) {
885  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_host_enable_status_change_notify failed");
886  }
887  res = knet_handle_enable_pmtud_notify(instance->knet_handle, instance, pmtu_change_callback_fn);
888  if (res) {
889  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_pmtud_notify failed");
890  }
891  global_instance = instance;
892 
893  /* Get an fd into knet */
894  instance->knet_fd = 0;
895  res = knet_handle_add_datafd(instance->knet_handle, &instance->knet_fd, &channel);
896  if (res) {
897  knet_log_printf(LOG_DEBUG, "knet_handle_add_datafd failed: %s", strerror(errno));
898  goto exit_error;
899  }
900 
901  /* Enable crypto if requested */
902  if (strcmp(instance->totem_config->crypto_cipher_type, "none") != 0) {
903  struct knet_handle_crypto_cfg crypto_cfg;
904 
905  strcpy(crypto_cfg.crypto_model, instance->totem_config->crypto_model);
906  strcpy(crypto_cfg.crypto_cipher_type, instance->totem_config->crypto_cipher_type);
907  strcpy(crypto_cfg.crypto_hash_type, instance->totem_config->crypto_hash_type);
908  memcpy(crypto_cfg.private_key, instance->totem_config->private_key, instance->totem_config->private_key_len);
909  crypto_cfg.private_key_len = instance->totem_config->private_key_len;
910 
911  res = knet_handle_crypto(instance->knet_handle, &crypto_cfg);
912  if (res == -1) {
913  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: %s", strerror(errno));
914  goto exit_error;
915  }
916  if (res == -2) {
917  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: -2");
918  goto exit_error;
919  }
920  knet_log_printf(LOG_INFO, "kronosnet crypto initialized: %s/%s", crypto_cfg.crypto_cipher_type, crypto_cfg.crypto_hash_type);
921  }
922 
923  /* Set up compression */
924  totemknet_reconfigure(instance, instance->totem_config);
925 
926  knet_handle_setfwd(instance->knet_handle, 1);
927 
928  instance->link_mode = KNET_LINK_POLICY_PASSIVE;
929  if (strcmp(instance->totem_config->link_mode, "active")==0) {
930  instance->link_mode = KNET_LINK_POLICY_ACTIVE;
931  }
932  if (strcmp(instance->totem_config->link_mode, "rr")==0) {
933  instance->link_mode = KNET_LINK_POLICY_RR;
934  }
935 
936  for (i=0; i<INTERFACE_MAX; i++) {
937  instance->link_status[i] = malloc(CFG_INTERFACE_STATUS_MAX_LEN);
938  if (!instance->link_status[i]) {
939  goto exit_error;
940  }
941  }
942 
943  qb_loop_poll_add (instance->poll_handle,
944  QB_LOOP_MED,
945  instance->logpipes[0],
946  POLLIN, instance, log_deliver_fn);
947 
948  qb_loop_poll_add (instance->poll_handle,
949  QB_LOOP_HIGH,
950  instance->knet_fd,
951  POLLIN, instance, data_deliver_fn);
952 
953  /*
954  * Upper layer isn't ready to receive message because it hasn't
955  * initialized yet. Add short timer to check the interfaces.
956  */
957  qb_loop_timer_add (instance->poll_handle,
958  QB_LOOP_MED,
959  100*QB_TIME_NS_IN_MSEC,
960  (void *)instance,
961  timer_function_netif_check_timeout,
962  &instance->timer_netif_check_timeout);
963 
964  totemknet_start_merge_detect_timeout(instance);
965 
966  /* Start listening for config changes */
967  totemknet_add_config_notifications(instance);
968 
969  /* Add stats keys to icmap */
971 
972  knet_log_printf (LOGSYS_LEVEL_INFO, "totemknet initialized");
973  *knet_context = instance;
974 
975  return (0);
976 
977 exit_error:
978  log_flush_messages(instance);
979  free(instance);
980  return (-1);
981 }
982 
984 {
985  /* Need to have space for a message AND a struct mcast in case of encapsulated messages */
986  return malloc(KNET_MAX_PACKET_SIZE + 512);
987 }
988 
989 void totemknet_buffer_release (void *ptr)
990 {
991  return free (ptr);
992 }
993 
995  void *knet_context,
996  int processor_count)
997 {
998  return (0);
999 }
1000 
1001 int totemknet_recv_flush (void *knet_context)
1002 {
1003  return (0);
1004 }
1005 
1006 int totemknet_send_flush (void *knet_context)
1007 {
1008  return (0);
1009 }
1010 
1012  void *knet_context,
1013  const void *msg,
1014  unsigned int msg_len)
1015 {
1016  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1017  int res = 0;
1018 
1019  ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
1020 
1021  return (res);
1022 }
1024  void *knet_context,
1025  const void *msg,
1026  unsigned int msg_len)
1027 {
1028  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1029  int res = 0;
1030 
1031  mcast_sendmsg (instance, msg, msg_len, 0);
1032 
1033  return (res);
1034 }
1035 
1037  void *knet_context,
1038  const void *msg,
1039  unsigned int msg_len)
1040 {
1041  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1042  int res = 0;
1043 
1044  mcast_sendmsg (instance, msg, msg_len, 1);
1045 
1046  return (res);
1047 }
1048 
1049 
1051 {
1052  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1053  int res = 0;
1054 
1055  knet_log_printf(LOG_DEBUG, "totemknet: iface_check");
1056 
1057  return (res);
1058 }
1059 
1061 {
1062  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1063 
1064  knet_log_printf(LOG_DEBUG, "totemknet: Returning MTU of %d", totem_config->net_mtu);
1065 }
1066 
1068  void *knet_context,
1069  unsigned int nodeid)
1070 {
1071  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1072  int res = 0;
1073 
1074  instance->token_target.nodeid = nodeid;
1075 
1076  instance->totemknet_target_set_completed (instance->context);
1077 
1078  return (res);
1079 }
1080 
1082  void *knet_context)
1083 {
1084  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1085  unsigned int res;
1086  struct sockaddr_storage system_from;
1087  struct msghdr msg_hdr;
1088  struct iovec iov_recv;
1089  struct pollfd ufd;
1090  int nfds;
1091  int msg_processed = 0;
1092 
1093  iov_recv.iov_base = instance->iov_buffer;
1094  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
1095 
1096  msg_hdr.msg_name = &system_from;
1097  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
1098  msg_hdr.msg_iov = &iov_recv;
1099  msg_hdr.msg_iovlen = 1;
1100 #ifdef HAVE_MSGHDR_CONTROL
1101  msg_hdr.msg_control = 0;
1102 #endif
1103 #ifdef HAVE_MSGHDR_CONTROLLEN
1104  msg_hdr.msg_controllen = 0;
1105 #endif
1106 #ifdef HAVE_MSGHDR_FLAGS
1107  msg_hdr.msg_flags = 0;
1108 #endif
1109 #ifdef HAVE_MSGHDR_ACCRIGHTS
1110  msg_msg_hdr.msg_accrights = NULL;
1111 #endif
1112 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
1113  msg_msg_hdr.msg_accrightslen = 0;
1114 #endif
1115 
1116  do {
1117  ufd.fd = instance->knet_fd;
1118  ufd.events = POLLIN;
1119  nfds = poll (&ufd, 1, 0);
1120  if (nfds == 1 && ufd.revents & POLLIN) {
1121  res = recvmsg (instance->knet_fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
1122  if (res != -1) {
1123  msg_processed = 1;
1124  } else {
1125  msg_processed = -1;
1126  }
1127  }
1128  } while (nfds == 1);
1129 
1130  return (msg_processed);
1131 }
1132 
1133 int totemknet_iface_set (void *knet_context,
1134  const struct totem_ip_address *local_addr,
1135  unsigned short ip_port,
1136  unsigned int iface_no)
1137 {
1138  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1139 
1140  totemip_copy(&instance->my_ids[iface_no], local_addr);
1141 
1142  knet_log_printf(LOG_INFO, "Configured link number %d: local addr: %s, port=%d", iface_no, totemip_print(local_addr), ip_port);
1143 
1144  instance->ip_port[iface_no] = ip_port;
1145 
1146  return 0;
1147 }
1148 
1149 
1151  void *knet_context,
1152  const struct totem_ip_address *local,
1153  const struct totem_ip_address *member,
1154  int link_no)
1155 {
1156  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1157  int err;
1158  int port = instance->ip_port[link_no];
1159  struct sockaddr_storage remote_ss;
1160  struct sockaddr_storage local_ss;
1161  int addrlen;
1162  int i;
1163  int host_found = 0;
1164  knet_node_id_t host_ids[KNET_MAX_HOST];
1165  size_t num_host_ids;
1166 
1167  /* Only create 1 loopback link and use link 0 */
1168  if (member->nodeid == instance->our_nodeid) {
1169  if (!instance->loopback_link) {
1170  link_no = 0;
1171  instance->loopback_link = 1;
1172  } else {
1173  /* Already done */
1174  return 0;
1175  }
1176  }
1177 
1178  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: %d (%s), link=%d", member->nodeid, totemip_print(member), link_no);
1179  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: local: %d (%s)", local->nodeid, totemip_print(local));
1180 
1181 
1182  /* Only add the host if it doesn't already exist in knet */
1183  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_host_ids);
1184  if (err) {
1185  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list");
1186  return -1;
1187  }
1188  for (i=0; i<num_host_ids; i++) {
1189  if (host_ids[i] == member->nodeid) {
1190  host_found = 1;
1191  }
1192  }
1193 
1194  if (!host_found) {
1195  err = knet_host_add(instance->knet_handle, member->nodeid);
1196  if (err != 0 && errno != EEXIST) {
1197  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add");
1198  return -1;
1199  }
1200  } else {
1201  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nodeid %d already added", member->nodeid);
1202  }
1203 
1204 
1205  if (err == 0) {
1206  if (knet_host_set_policy(instance->knet_handle, member->nodeid, instance->link_mode)) {
1207  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_set_policy failed");
1208  return -1;
1209  }
1210  }
1211 
1212  memset(&local_ss, 0, sizeof(local_ss));
1213  /* Casts to remove const */
1214  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port, &remote_ss, &addrlen);
1215  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port, &local_ss, &addrlen);
1216 
1217  if (member->nodeid == instance->our_nodeid) {
1218  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: loopback link is %d\n", link_no);
1219 
1220  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1221  KNET_TRANSPORT_LOOPBACK,
1222  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1223  }
1224  else {
1225  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1226  instance->totem_config->interfaces[link_no].knet_transport,
1227  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1228  }
1229  if (err) {
1230  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_config failed");
1231  return -1;
1232  }
1233 
1234  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: Setting link prio to %d",
1235  instance->totem_config->interfaces[link_no].knet_link_priority);
1236 
1237  err = knet_link_set_priority(instance->knet_handle, member->nodeid, link_no,
1238  instance->totem_config->interfaces[link_no].knet_link_priority);
1239  if (err) {
1240  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for nodeid %d, link %d failed", member->nodeid, link_no);
1241  }
1242 
1243  err = knet_link_set_ping_timers(instance->knet_handle, member->nodeid, link_no,
1244  instance->totem_config->interfaces[link_no].knet_ping_interval,
1245  instance->totem_config->interfaces[link_no].knet_ping_timeout,
1246  instance->totem_config->interfaces[link_no].knet_ping_precision);
1247  if (err) {
1248  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for nodeid %d, link %d failed", member->nodeid, link_no);
1249  }
1250  err = knet_link_set_pong_count(instance->knet_handle, member->nodeid, link_no,
1251  instance->totem_config->interfaces[link_no].knet_pong_count);
1252  if (err) {
1253  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for nodeid %d, link %d failed", member->nodeid, link_no);
1254  }
1255 
1256  err = knet_link_set_enable(instance->knet_handle, member->nodeid, link_no, 1);
1257  if (err) {
1258  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_enable for nodeid %d, link %d failed", member->nodeid, link_no);
1259  return -1;
1260  }
1261 
1262  /* register stats */
1263  stats_knet_add_member(member->nodeid, link_no);
1264  return (0);
1265 }
1266 
1268  void *knet_context,
1269  const struct totem_ip_address *token_target,
1270  int link_no)
1271 {
1272  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1273  int res;
1274  uint8_t link_list[KNET_MAX_LINK];
1275  size_t num_links;
1276 
1277  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_remove: %d, link=%d", token_target->nodeid, link_no);
1278 
1279  /* Don't remove the link with the loopback on it until we shut down */
1280  if (token_target->nodeid == instance->our_nodeid) {
1281  return 0;
1282  }
1283 
1284  /* Tidy stats */
1286 
1287  /* Remove the link first */
1288  res = knet_link_set_enable(instance->knet_handle, token_target->nodeid, link_no, 0);
1289  if (res != 0) {
1290  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set enable(off) for nodeid %d, link %d failed", token_target->nodeid, link_no);
1291  return res;
1292  }
1293 
1294  res = knet_link_clear_config(instance->knet_handle, token_target->nodeid, link_no);
1295  if (res != 0) {
1296  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_clear_config for nodeid %d, link %d failed", token_target->nodeid, link_no);
1297  return res;
1298  }
1299 
1300  /* If this is the last link, then remove the node */
1301  res = knet_link_get_link_list(instance->knet_handle,
1302  token_target->nodeid, link_list, &num_links);
1303  if (res) {
1304  return (0); /* not really failure */
1305  }
1306 
1307  if (num_links == 0) {
1308  res = knet_host_remove(instance->knet_handle, token_target->nodeid);
1309  }
1310  return res;
1311 }
1312 
1314  void *knet_context)
1315 {
1316  return (0);
1317 }
1318 
1320  void *knet_context,
1321  struct totem_config *totem_config)
1322 {
1323  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1324  struct knet_handle_compress_cfg compress_cfg;
1325  int res = 0;
1326 
1328  strcpy(compress_cfg.compress_model, totem_config->knet_compression_model);
1329  compress_cfg.compress_threshold = totem_config->knet_compression_threshold;
1330  compress_cfg.compress_level = totem_config->knet_compression_level;
1331 
1332  res = knet_handle_compress(instance->knet_handle, &compress_cfg);
1333  if (res) {
1334  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_handle_compress failed");
1335  }
1336  }
1337  return (res);
1338 }
1339 
1340 
1342  void *knet_context)
1343 {
1344  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1345 
1346  (void) knet_handle_clear_stats(instance->knet_handle, KNET_CLEARSTATS_HANDLE_AND_LINK);
1347 }
1348 
1349 /* For the stats module */
1351  knet_node_id_t node, uint8_t link_no,
1352  struct knet_link_status *status)
1353 {
1354  int res;
1355  int ret = CS_OK;
1356 
1357  /* We are probably not using knet */
1358  if (!global_instance) {
1359  return CS_ERR_NOT_EXIST;
1360  }
1361 
1362  if (link_no >= INTERFACE_MAX) {
1363  return CS_ERR_NOT_EXIST; /* Invalid link number */
1364  }
1365 
1366  res = knet_link_get_status(global_instance->knet_handle, node, link_no, status, sizeof(struct knet_link_status));
1367  if (res) {
1368  switch (errno) {
1369  case EINVAL:
1370  ret = CS_ERR_INVALID_PARAM;
1371  break;
1372  case EBUSY:
1373  ret = CS_ERR_BUSY;
1374  break;
1375  case EDEADLK:
1376  ret = CS_ERR_TRY_AGAIN;
1377  break;
1378  default:
1379  ret = CS_ERR_LIBRARY;
1380  break;
1381  }
1382  }
1383 
1384  return (ret);
1385 }
1386 
1388  struct knet_handle_stats *stats)
1389 {
1390  /* We are probably not using knet */
1391  if (!global_instance) {
1392  return CS_ERR_NOT_EXIST;
1393  }
1394 
1395  return knet_handle_get_stats(global_instance->knet_handle, stats, sizeof(struct knet_handle_stats));
1396 }
1397 
1398 static void timer_function_merge_detect_timeout (
1399  void *data)
1400 {
1401  struct totemknet_instance *instance = (struct totemknet_instance *)data;
1402 
1403  if (instance->merge_detect_messages_sent_before_timeout == 0) {
1404  instance->send_merge_detect_message = 1;
1405  }
1406 
1408 
1409  totemknet_start_merge_detect_timeout(instance);
1410 }
1411 
1412 static void totemknet_start_merge_detect_timeout(
1413  void *knet_context)
1414 {
1415  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1416 
1417  qb_loop_timer_add(instance->poll_handle,
1418  QB_LOOP_MED,
1419  instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
1420  (void *)instance,
1421  timer_function_merge_detect_timeout,
1422  &instance->timer_merge_detect_timeout);
1423 
1424 }
1425 
1426 static void totemknet_stop_merge_detect_timeout(
1427  void *knet_context)
1428 {
1429  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1430 
1431  qb_loop_timer_del(instance->poll_handle,
1432  instance->timer_merge_detect_timeout);
1433 }
1434 
1435 static void log_flush_messages (void *knet_context)
1436 {
1437  struct pollfd pfd;
1438  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1439  int cont;
1440 
1441  cont = 1;
1442 
1443  while (cont) {
1444  pfd.fd = instance->logpipes[0];
1445  pfd.events = POLLIN;
1446  pfd.revents = 0;
1447 
1448  if ((poll(&pfd, 1, 0) > 0) &&
1449  (pfd.revents & POLLIN) &&
1450  (log_deliver_fn(instance->logpipes[0], POLLIN, instance) == 0)) {
1451  cont = 1;
1452  } else {
1453  cont = 0;
1454  }
1455  }
1456 }
totemknet_ifaces_get
int totemknet_ifaces_get(void *knet_context, char ***status, unsigned int *iface_count)
Definition: totemknet.c:397
knet_log_printf
#define knet_log_printf(level, format, args...)
Definition: totemknet.c:193
totem_logging_configuration::log_level_error
int log_level_error
Definition: totem.h:109
totem_config::node_id
unsigned int node_id
Definition: totem.h:160
value
uint32_t value
Definition: exec/votequorum.c:2
LOGSYS_LEVEL_CRIT
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:71
totemknet_instance::totemknet_log_level_error
int totemknet_log_level_error
Definition: totemknet.c:116
totemknet_mcast_flush_send
int totemknet_mcast_flush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1023
totemknet_instance::totemknet_iface_change_fn
void(* totemknet_iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no)
Definition: totemknet.c:100
totem_interface::knet_ping_precision
int knet_ping_precision
Definition: totem.h:92
totemknet_instance::totemknet_mtu_changed
void(* totemknet_mtu_changed)(void *context, int net_mtu)
Definition: totemknet.c:105
MSG_NOSIGNAL
#define MSG_NOSIGNAL
Definition: totemknet.c:77
totemknet_member_add
int totemknet_member_add(void *knet_context, const struct totem_ip_address *local, const struct totem_ip_address *member, int link_no)
Definition: totemknet.c:1150
work_item::msg_len
unsigned int msg_len
Definition: totemknet.c:172
stats_knet_del_member
void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:624
totemknet_buffer_release
void totemknet_buffer_release(void *ptr)
Definition: totemknet.c:989
totemknet_token_send
int totemknet_token_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1011
CS_ERR_LIBRARY
@ CS_ERR_LIBRARY
Definition: corotypes.h:96
CS_ERR_TRY_AGAIN
@ CS_ERR_TRY_AGAIN
Definition: corotypes.h:100
LOGSYS_LEVEL_ERROR
#define LOGSYS_LEVEL_ERROR
Definition: logsys.h:72
totemknet_crypto_set
int totemknet_crypto_set(void *knet_context, const char *cipher_type, const char *hash_type)
Definition: totemknet.c:273
totemknet_reconfigure
int totemknet_reconfigure(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1319
totemknet_iface_set
int totemknet_iface_set(void *knet_context, const struct totem_ip_address *local_addr, unsigned short ip_port, unsigned int iface_no)
Definition: totemknet.c:1133
totem_config::knet_pmtud_interval
unsigned int knet_pmtud_interval
Definition: totem.h:162
work_item
Definition: totemknet.c:170
LOGSYS_LEVEL_DEBUG
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:76
totem_logging_configuration::log_level_debug
int log_level_debug
Definition: totem.h:112
totemip_print
const char * totemip_print(const struct totem_ip_address *addr)
Definition: totemip.c:237
totemknet_net_mtu_adjust
void totemknet_net_mtu_adjust(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1060
totemsrp_stats_t
Definition: totemstats.h:53
totemknet_instance::poll_handle
qb_loop_t * poll_handle
Definition: totemknet.c:86
totem_logging_configuration::log_level_notice
int log_level_notice
Definition: totem.h:111
CS_ERR_NOT_EXIST
@ CS_ERR_NOT_EXIST
Definition: corotypes.h:106
totem_interface::knet_link_priority
int knet_link_priority
Definition: totem.h:89
KNET_LOGSYS_PERROR
#define KNET_LOGSYS_PERROR(err_num, level, fmt, args...)
Definition: totemknet.c:209
totemknet_instance::totemknet_log_level_notice
int totemknet_log_level_notice
Definition: totemknet.c:120
totemknet_finalize
int totemknet_finalize(void *knet_context)
Definition: totemknet.c:475
libknet_log_printf
#define libknet_log_printf(level, format, args...)
Definition: totemknet.c:201
stats_knet_add_member
void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:614
totemknet_instance::totem_config
struct totem_config * totem_config
Definition: totemknet.c:151
totemknet_instance::context
void * context
Definition: totemknet.c:92
CFG_INTERFACE_STATUS_MAX_LEN
#define CFG_INTERFACE_STATUS_MAX_LEN
Definition: totemknet.c:81
ICMAP_TRACK_PREFIX
#define ICMAP_TRACK_PREFIX
Whole prefix is tracked, instead of key only (so "totem." tracking means that "totem....
Definition: icmap.h:85
totem_ip_address::nodeid
unsigned int nodeid
Definition: coroapi.h:112
totemknet_processor_count_set
int totemknet_processor_count_set(void *knet_context, int processor_count)
Definition: totemknet.c:994
ENTER
#define ENTER
Definition: logsys.h:324
totemknet_instance::iov_buffer
char iov_buffer[KNET_MAX_PACKET_SIZE]
Definition: totemknet.c:139
totemknet_link_get_status
int totemknet_link_get_status(knet_node_id_t node, uint8_t link_no, struct knet_link_status *status)
Definition: totemknet.c:1350
totem_config::merge_timeout
unsigned int merge_timeout
Definition: totem.h:190
totemknet_stats_clear
void totemknet_stats_clear(void *knet_context)
Definition: totemknet.c:1341
swab.h
totem_config::interfaces
struct totem_interface * interfaces
Definition: totem.h:158
totem_interface::knet_transport
int knet_transport
Definition: totem.h:94
totemknet_instance::totemknet_log_level_debug
int totemknet_log_level_debug
Definition: totemknet.c:122
totem_config::knet_compression_level
int knet_compression_level
Definition: totem.h:228
totemip_copy
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition: totemip.c:96
ICMAP_TRACK_ADD
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
icmap.h
header
struct totem_message_header header
Definition: totemsrp.c:0
totemknet_instance::totemknet_log_level_security
int totemknet_log_level_security
Definition: totemknet.c:114
CS_OK
@ CS_OK
Definition: corotypes.h:95
LOGSYS_LEVEL_INFO
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:75
totem_config::net_mtu
unsigned int net_mtu
Definition: totem.h:202
totemip.h
INTERFACE_MAX
#define INTERFACE_MAX
Definition: coroapi.h:88
totemknet_instance::my_ids
struct totem_ip_address my_ids[INTERFACE_MAX]
Definition: totemknet.c:143
totem_config::knet_compression_threshold
uint32_t knet_compression_threshold
Definition: totem.h:226
CS_ERR_BUSY
@ CS_ERR_BUSY
Definition: corotypes.h:104
totem_interface::configured
uint8_t configured
Definition: totem.h:87
totemknet_instance::send_merge_detect_message
int send_merge_detect_message
Definition: totemknet.c:159
__attribute__
typedef __attribute__
totem_interface::boundto
struct totem_ip_address boundto
Definition: totem.h:83
LEAVE
#define LEAVE
Definition: logsys.h:325
totemknet_instance::crypto_inst
struct crypto_instance * crypto_inst
Definition: totemknet.c:84
icmap_track_add
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition: icmap.c:1151
totemknet_instance::knet_fd
int knet_fd
Definition: totemknet.c:164
totemknet_iface_check
int totemknet_iface_check(void *knet_context)
Definition: totemknet.c:1050
totemknet_instance::timer_netif_check_timeout
qb_loop_timer_handle timer_netif_check_timeout
Definition: totemknet.c:155
totem_config::crypto_model
char * crypto_model
Definition: totem.h:218
totem_logging_configuration::log_subsys_id
int log_subsys_id
Definition: totem.h:114
totemknet_instance
Definition: totemknet.c:83
totem_config::link_mode
char link_mode[TOTEM_LINK_MODE_BYTES]
Definition: totem.h:198
totemknet_instance::knet_context
void(*) void knet_context)
Definition: totemknet.c:135
totemknet_recv_flush
int totemknet_recv_flush(void *knet_context)
Definition: totemknet.c:1001
totemknet_instance::ip_port
uint16_t ip_port[INTERFACE_MAX]
Definition: totemknet.c:145
totemknet_instance::our_nodeid
int our_nodeid
Definition: totemknet.c:147
totem_config
Definition: totem.h:152
totemknet_token_target_set
int totemknet_token_target_set(void *knet_context, unsigned int nodeid)
Definition: totemknet.c:1067
totemknet.h
totem_ip_address
The totem_ip_address struct.
Definition: coroapi.h:111
totemknet_instance::token_target
struct totem_ip_address token_target
Definition: totemknet.c:153
sq.h
icmap_get_uint32
cs_error_t icmap_get_uint32(const char *key_name, uint32_t *u32)
Definition: icmap.c:850
totemknet_initialize
int totemknet_initialize(qb_loop_t *poll_handle, void **knet_context, struct totem_config *totem_config, totemsrp_stats_t *stats, void *context, void(*deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from), void(*iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no), void(*mtu_changed)(void *context, int net_mtu), void(*target_set_completed)(void *context))
Definition: totemknet.c:772
global_instance
struct totemknet_instance * global_instance
Definition: totemknet.c:168
totem_message_header
Definition: totem.h:124
totem_interface::knet_pong_count
int knet_pong_count
Definition: totem.h:93
totemknet_member_list_rebind_ip
int totemknet_member_list_rebind_ip(void *knet_context)
Definition: totemknet.c:1313
totem_interface::ip_port
uint16_t ip_port
Definition: totem.h:85
totem_interface::knet_ping_interval
int knet_ping_interval
Definition: totem.h:90
_logsys_subsys_create
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition: logsys.c:433
totemknet_instance::loopback_link
int loopback_link
Definition: totemknet.c:149
totemknet_instance::knet_handle
knet_handle_t knet_handle
Definition: totemknet.c:88
ICMAP_TRACK_DELETE
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
work_item::msg
const void * msg
Definition: totemknet.c:171
totemknet_instance::timer_merge_detect_timeout
qb_loop_timer_handle timer_merge_detect_timeout
Definition: totemknet.c:157
user_data
void * user_data
Definition: sam.c:127
system_from
struct srp_addr system_from
Definition: totemsrp.c:1
ICMAP_TRACK_MODIFY
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
totem_logging_configuration::log_level_warning
int log_level_warning
Definition: totem.h:110
totemknet_recv_mcast_empty
int totemknet_recv_mcast_empty(void *knet_context)
Definition: totemknet.c:1081
totemknet_instance::merge_detect_messages_sent_before_timeout
unsigned int merge_detect_messages_sent_before_timeout
Definition: totemknet.c:161
totemknet_member_remove
int totemknet_member_remove(void *knet_context, const struct totem_ip_address *token_target, int link_no)
Definition: totemknet.c:1267
util.h
totemknet_send_flush
int totemknet_send_flush(void *knet_context)
Definition: totemknet.c:1006
totemknet_mcast_noflush_send
int totemknet_mcast_noflush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1036
totemknet_instance::knet_subsys_id
int knet_subsys_id
Definition: totemknet.c:126
totemknet_instance::totemknet_deliver_fn
void(* totemknet_deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from)
Definition: totemknet.c:94
totemknet_buffer_alloc
void * totemknet_buffer_alloc(void)
Definition: totemknet.c:983
nodeid
unsigned int nodeid
Definition: coroapi.h:0
totem_interface::knet_ping_timeout
int knet_ping_timeout
Definition: totem.h:91
totem_config::private_key
unsigned char private_key[TOTEM_PRIVATE_KEY_LEN_MAX]
Definition: totem.h:167
PROCESSOR_COUNT_MAX
#define PROCESSOR_COUNT_MAX
Definition: coroapi.h:96
config.h
totemknet_handle_get_stats
int totemknet_handle_get_stats(struct knet_handle_stats *stats)
Definition: totemknet.c:1387
CS_ERR_INVALID_PARAM
@ CS_ERR_INVALID_PARAM
Definition: corotypes.h:101
totemknet_instance::totemknet_subsys_id
int totemknet_subsys_id
Definition: totemknet.c:124
logsys.h
totem_config::crypto_hash_type
char * crypto_hash_type
Definition: totem.h:222
totemknet_instance::totemknet_target_set_completed
void(* totemknet_target_set_completed)(void *context)
Definition: totemknet.c:109
totem_logging_configuration::log_printf
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:99
LOGSYS_LEVEL_WARNING
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:73
totem_interface::bindnet
struct totem_ip_address bindnet
Definition: totem.h:82
icmap_track
Definition: icmap.c:61
totem_config::private_key_len
unsigned int private_key_len
Definition: totem.h:169
totemknet_instance::link_mode
int link_mode
Definition: totemknet.c:90
totem_config::crypto_cipher_type
char * crypto_cipher_type
Definition: totem.h:220
totemknet_instance::totemknet_log_printf
void(* totemknet_log_printf)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf
Definition: totemknet.c:128
icmap_notify_value
Structure passed as new_value and old_value in change callback.
Definition: icmap.h:91
stats_knet_add_handle
void stats_knet_add_handle(void)
Definition: stats.c:637
totemip_totemip_to_sockaddr_convert
int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen)
Definition: totemip.c:245
icmap_get_uint8
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
Definition: icmap.c:826
totemknet_instance::totemknet_log_level_warning
int totemknet_log_level_warning
Definition: totemknet.c:118
totemknet_instance::logpipes
int logpipes[2]
Definition: totemknet.c:163
totemstats.h
totem_message_header::target_nodeid
unsigned int target_nodeid
Definition: totem.h:130
work_item::instance
struct totemknet_instance * instance
Definition: totemknet.c:173
totemknet_instance::link_status
char * link_status[INTERFACE_MAX]
Definition: totemknet.c:141
totem_config::knet_compression_model
char * knet_compression_model
Definition: totem.h:224
totem_config::totem_logging_configuration
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:200