View Javadoc

1   package org.lsst.ccs.bus.jgroups;
2   
3   import org.jgroups.*;
4   import org.jgroups.conf.ProtocolConfiguration;
5   import org.jgroups.conf.XmlConfigurator;
6   import org.jgroups.util.UUID;
7   import org.lsst.ccs.bootstrap.resources.BootstrapResourceUtils;
8   import org.lsst.ccs.bus.*;
9   import org.lsst.ccs.utilities.logging.Logger;
10  
11  import java.io.IOException;
12  import java.io.InputStream;
13  import java.io.Serializable;
14  import java.util.*;
15  import java.util.concurrent.ConcurrentHashMap;
16  import java.util.concurrent.CopyOnWriteArrayList;
17  
18  /**
19   * Each Bus is represented by a JGroups cluster.
20   * <p/>
21   * Each agent creates a JChannel, setName to name of agent and connect to corresponding Bus/cluster 
22   * (so each agent may have up to 3 JChannel connected).
23   * <BR>
24   * Each of these 3 channels have a JGroups Address.
25   * <p/>
26   * for sending messages to an agent we have to find its Address in the group 
27   * AND we have to find the addresses of the multiple anonymous agents that will receive a copy
28   * <p/>
29   * There is a dictionary to match agent name to jGroups Address. 
30   * for each agent there is an Address except for anonymous agent that can be matched to multiple addresses.
31   * <p/>
32   * <
33   * p/>
34   * <B>ImPortant</B> every Jchannel in the JVM MUST be closed!
35   */
36  public class JGroupsBusMessagingLayer implements BusMessagingLayer {
37  
38      public static final String DEFAULT_UDP_PROTOCOL = "jgroups:udp_ccs";
39      private Properties jgroupsProperties;
40      private String xmlConfigurationFile = "udp_ccs.xml";
41      private final static String propertyKeyForJGroups = "org.lsst.ccs.jgroups.";
42      private final static String propertyKeyForAllBuses = propertyKeyForJGroups + "ALL.";
43  
44      //added to circumvent problems with wrong view of addresses
45      private boolean tryBroadcast;
46      private static Logger logger = Logger.getLogger("org.lsst.ccs.bus.jgroups");
47      /**
48       * the pair name of agent / array of addresses on the buses. 
49       *  the array index is the bus index.
50       */
51      CopyOnWriteArrayList<AgentAddresses> namedAgentAddresses = new CopyOnWriteArrayList<>();
52      // should be a set!
53      /**
54       * the anonymous agents adresses
55       */
56      private CopyOnWriteArrayList<AgentAddresses> anonymousAddresses = new CopyOnWriteArrayList<>();
57  
58      /**
59       * for a known agent the infos for each bus linked to channels and to codes to handle the message (BusReceiver)
60       */
61      private Map<String, LocalAgentChannels> mapLocalChannels = new ConcurrentHashMap<>();
62      /**
63       *  the membership Listener for each bus. 
64       *  Todo: allow multiple membershipListeners of provide a delegate or decorator
65       */
66      private BusMembershipListener[] membershipListeners = new BusMembershipListener[Bus.values().length];
67      /**
68       * the last known View on a Bus (kept to know if there is a creation or an update)
69       */
70      // accessed from synchronized code :so no volatile
71      private View[] lastViews = new View[Bus.values().length];
72      private volatile boolean closed;
73  
74      /**
75       * for a given agent Name lists the addresses on the corresponding buses,
76       * local agents are also registerd here. (TODO: modiffy that?)
77       */
78      class AgentAddresses {
79  
80          String agentName;
81          Address[] busAddresses = new Address[Bus.values().length];
82      }
83  
84      /**
85       * for a local agent the list of its channels nd of its adapter
86       */
87      class LocalAgentChannels {
88  
89          String agentName;
90          JChannel[] channels = new JChannel[Bus.values().length];
91          BusReceiver[] receivers = new BusReceiver[Bus.values().length];
92      }
93  
94      /**
95       * instance of this class act as a receiver of messages then forward to the BusMessageForwarders registered
96       */
97      class BusReceiver extends ReceiverAdapter {
98  
99          Bus bus;
100         //todo: duplicate information
101         JChannel curChan;
102         //ParallelCommandDispatcher dispatcher;
103         CopyOnWriteArrayList<BusMessageForwarder> listForwarders;
104         //todo: again a duplicate!
105         String agentName;
106 
107         BusReceiver(String agentName, Bus bus, JChannel curChan) {
108             this.agentName = agentName;
109             this.bus = bus;
110             this.curChan = curChan;
111             curChan.setReceiver(this);
112         }
113 
114         public void addForwarder(BusMessageForwarder forwarder) {
115             //TODO: reinstall Parallel ?
116             /*
117              if (dispatcher == null) {
118              dispatcher = new ParallelCommandDispatcher();
119              }
120              dispatcher.addExecutant(forwarder);
121              */
122             if (listForwarders == null) {
123                 listForwarders = new CopyOnWriteArrayList<BusMessageForwarder>();
124             }
125             listForwarders.add(forwarder);
126         }
127 
128         public void removeForwarder(BusMessageForwarder forwarder) {
129             /*
130              if (dispatcher == null) return;
131              dispatcher.removeExecutant(forwarder);
132              */
133             if (listForwarders == null) {
134                 return;
135             }
136             listForwarders.remove(forwarder);
137         }
138 
139         @Override
140         public void receive(Message message) {
141 
142             BusMessage busMessInit;
143             try {
144                 if (listForwarders == null) {
145                     return;
146                 }
147                 if (listForwarders.size() == 0) {
148                     return;
149                 }
150                 busMessInit = (BusMessage) message.getObject();
151             } catch (final RuntimeException exc) {
152                 throw exc;
153                 //TODO: create a generic Message for reporting errors
154                 /* busMessInit = new BusMessage() {
155                  @Override
156                  public String getMessageType() {
157                  return "error" + exc;
158                  }
159                  } ;
160                  */
161             }
162             final BusMessage busMess = busMessInit;
163             if (busMess.getOrigin().equals(agentName)) {
164                 logger.trace("message received by same agent than sender");
165                 return;
166             }
167             if (tryBroadcast) { // we can receive a message which is not for us!
168                 if (!ANONYMOUS_AGENT.equals(agentName)) {
169                     if (busMess instanceof Command) {
170                         String destination = ((Command) busMess).getDestination();
171                         if ((destination == null) || "".equals(destination)
172                                 || "*".equals(destination)) {
173                             //do nothing it's ok
174                         } else {
175                             String[] dests = destination.split(",");
176                             boolean found = false;
177                             for (String dest : dests) {
178                                 if (destination.equalsIgnoreCase(dest)) {
179                                     found = true;
180                                     break;
181                                 }
182                             }
183                             if (!found) {
184                                 return;
185                             }
186                         }
187                     }
188                 }
189             }
190             /*
191              CommandFor<BusMessageForwarder> cmdMessage = new CommandFor<BusMessageForwarder>() {
192              @Override
193              public void invokeOn(BusMessageForwarder instance) {
194              instance.update(busMess);
195              }
196              };
197              dispatcher.dispatchCommand(cmdMessage);
198              */
199             // TODO : reinstall parallel
200             for (BusMessageForwarder forwarder : listForwarders) {
201                 forwarder.update(busMess);
202             }
203 
204         }
205 
206         @Override
207         public void viewAccepted(View view) {
208             updateMapAddressesFromView(bus, curChan, view);
209         }
210 
211         @Override
212         public void suspect(Address address) {
213             //get the address in other
214             int index = bus.ordinal();
215             for (AgentAddresses agentAddresses : namedAgentAddresses) {
216                 Address localAddress = agentAddresses.busAddresses[index];
217                 if (address.equals(localAddress)) {
218                     BusMembershipListener busMembershipListener = membershipListeners[index];
219                     if (busMembershipListener != null) {
220                         String info = "";
221                         if (address instanceof UUID) {
222                             info = ((UUID) address).toStringLong();
223                         }
224                         busMembershipListener.disconnecting(" guess on " + address.toString(), info);
225                     } else {
226                         logger.warn("disconnection suspicion on " + address);
227                     }
228                     agentAddresses.busAddresses[index] = null;
229                     break;
230                 }
231             }
232             // and warn
233         }
234     }
235     /////////////////////////////////////////////// CTOR
236 
237     /**
238      * there should be only one instance perJVM: it's up to the calling code
239      * to check that.
240      */
241     @Deprecated
242     public JGroupsBusMessagingLayer() {
243         this(DEFAULT_UDP_PROTOCOL);
244         //register a close
245         /* Runtime runtime = Runtime.getRuntime() ;
246          runtime.addShutdownHook( new Thread() {
247          public void run() {
248          try {
249          //close() ;
250          } catch (IOException e) {
251          //do nothing
252          }
253          }
254          });
255          */
256 
257     }
258 
259     JGroupsBusMessagingLayer(String protocolString) {
260         //index 1 : is for URL
261         //index 2 :is for String to be passed to JChannel
262         String[] protocolInfos = protocolString.split(":");
263         String rawResource = protocolInfos[1];
264         if (rawResource.length() > 0) {
265             xmlConfigurationFile = String.format("/%s.xml", rawResource);
266         }
267 
268         //The properties file corresponding to the XML file
269         String propertiesConfigurationFile = xmlConfigurationFile.replace(".xml", ".properties");
270         jgroupsProperties = BootstrapResourceUtils.getBootstrapProperties(propertiesConfigurationFile, this.getClass());
271 
272         tryBroadcast = Boolean.parseBoolean(jgroupsProperties.getProperty("org.lsst.ccs.jgroups.trybroadcast"));
273     }
274 
275     //////////////////////////////////////////////
276     ///////////////////////////////////////////// METHODS
277     @Override
278     public void register(String agentName, Bus... buses) throws IOException {
279 
280         StringBuilder allBuses = new StringBuilder();
281         for (Bus bus : buses) {
282             allBuses.append(bus.toString() + " ");
283         }
284         logger.info("### Registering " + agentName + " for buses " + allBuses);
285 
286         if (closed) {
287             // TODO: problem with multiple tests
288             throw new TransportStateException("closing");
289         }
290         if (buses.length == 0) {
291             buses = Bus.values();
292         }
293         if (agentName == null || "".equals(agentName)) {
294             agentName = ANONYMOUS_AGENT;
295         } else {
296 
297             /*
298              is this bus already registered?
299              TODO: stop or not if already connected to the same bus?
300              */
301             for (AgentAddresses addresses : namedAgentAddresses) {
302                 if (agentName.equals(addresses.agentName)) {
303                     for (Bus bus : buses) {
304                         if (addresses.busAddresses[bus.ordinal()] != null) {
305                             DuplicateBusNameException exception = new DuplicateBusNameException(agentName, " already registered on " + bus);
306                             logger.error(exception);
307                             BusMembershipListener listener = membershipListeners[bus.ordinal()];
308                             if (listener != null) {
309                                 listener.anormalEvent(exception);
310                             }
311                         }
312                     }
313                 }
314             }
315         }
316 
317         // is this agent already known to the map
318         LocalAgentChannels localChannels = mapLocalChannels.get(agentName);
319         // if not: we create an entry in the Map
320         if (localChannels == null) {
321             localChannels = new LocalAgentChannels();
322             localChannels.agentName = agentName;
323             mapLocalChannels.put(agentName, localChannels);
324         }
325         // if anonymous registers local channel and add it  to thelist of anonymous
326         for (Bus bus : buses) {
327             int index = bus.ordinal();
328             JChannel channel = localChannels.channels[index];
329             // todo replace isConnected
330             if (channel == null || !channel.isConnected()) {
331                 try {
332 
333                     XmlConfigurator configurator = createXmlConfiguratorForBus(xmlConfigurationFile, jgroupsProperties, bus.toString());
334                     channel = new JChannel(configurator);
335 
336                     // bamade: 4 log messages grouped into one
337                     logger.debug("######################################################\n"
338                             + "JGroup Configuration for bus: " + bus.toString() + "\n"
339                             + channel.getProtocolStack().printProtocolSpecAsXML() + "\n"
340                             + "######################################################"
341                     );
342 
343                     localChannels.channels[index] = channel;
344                     channel.setName(agentName);
345                     channel.connect(bus.toString());
346                     // who is connected before ?
347                     View view = channel.getView();
348                     //todo: check for synchronization problems between receiver and update
349                     updateMapAddressesFromView(bus, channel, view);
350                     // registers a BusReceiver
351                     localChannels.receivers[index] = new BusReceiver(agentName, bus, channel);
352 
353                 } catch (Exception e) {
354                     throw new IOException(e);
355                 }
356             }
357 
358         }
359     }
360 
361     /**
362      * Creates an XmlConfiguration object for a given bus. This object is used to initialize a JChannel. 
363      * The XmlConfiguration object is created from an xml file that is loaded from the bootstrap environment.
364      * The properties object is then scanned to find relevant values to modify XmlConfiguration object.
365      *
366      * The property keys are of the form org.lsst.ccs.jgroups.[BusIdentifier].[ProtocolName].[ProtocolPropertyName]=[ProtocolPropertyValue]
367      *
368      * Where: 
369      * - BusIdentifier can be either COMMAND, LOG, STATUS if the property applied to an individual bus or ALL if it applies to all the buses. 
370      * - ProtocolName is a valid JGroups protocol name as specified at http://www.jgroups.org/manual/html/protlist.html 
371      * - ProtocolPropertyName is a property for the given protocol 
372      * - ProtocolPropertyValue is the value to be given to it
373      *
374      * If the ProtocolName does not exist in the original xml file, a new ProtocolConfiguration object will be added to the 
375      * XmlConfiguration.
376      *
377      *
378      * @param xmlConfigurationFile The name of the xml file to be loaded
379      * @param properties The properties object to be used to overwrite the xml file
380      * @param bus The bus for which the properties are scanned
381      * @return The XmlConfiguration object to be used to create the JChannel
382      *
383      */
384     public static XmlConfigurator createXmlConfiguratorForBus(String xmlConfigurationFile, Properties properties, String bus) {
385 
386         InputStream xmlConfigurationFileInputStream = BootstrapResourceUtils.getBootstrapResource(xmlConfigurationFile, JGroupsBusMessagingLayer.class);
387         if (xmlConfigurationFileInputStream == null) {
388             throw new IllegalArgumentException(xmlConfigurationFile + " not found as a resource");
389         }
390 
391         XmlConfigurator configurator = null;
392         try {
393             configurator = XmlConfigurator.getInstance(xmlConfigurationFileInputStream);
394         } catch (IOException ioe) {
395             throw new RuntimeException("Could not create JGroups xml configuration object for " + xmlConfigurationFile, ioe);
396         }
397 
398         List<ProtocolConfiguration> protocolConfigurationList = configurator.getProtocolStack();
399 
400         //First look for properties that applies to all BUSES, then for bus specific properties.
401         String[] propertyKeys = new String[]{propertyKeyForAllBuses, propertyKeyForJGroups + bus.toUpperCase() + "."};
402 
403         for (String propertyKey : propertyKeys) {
404             // When looping over all the keys of a property object we have to keep in mind that the
405             // properties might be chained. The following is a utility method to get all keys in a
406             // Properties object, including its parents. See https://jira.slac.stanford.edu/browse/LSSTCCS-227
407             Set<Object> keys = BootstrapResourceUtils.getAllKeysInProperties(properties);
408             for (Object key : keys) {
409                 String property = (String) key;
410                 if (property.startsWith(propertyKey)) {
411 
412                     //Remove the property key so that only PROTOCOL.propertyName is left
413                     String shortProperty = property.replace(propertyKey, "");
414                     int divider = shortProperty.lastIndexOf(".");
415                     String protocolPropertyName = shortProperty.substring(divider + 1);
416                     String protocolName = shortProperty.substring(0, divider);
417                     String protocolPropertyValue = properties.getProperty(property);
418 
419                     boolean found = false;
420                     for (ProtocolConfiguration protocolConfiguration : protocolConfigurationList) {
421                         if (protocolConfiguration.getProtocolName().equals(protocolName)) {
422                             protocolConfiguration.getProperties().put(protocolPropertyName, protocolPropertyValue);
423                             found = true;
424                             break;
425                         }
426                     }
427                     if (!found) {
428                         HashMap<String, String> hash = new HashMap<>();
429                         hash.put(protocolPropertyName, protocolPropertyValue);
430                         ProtocolConfiguration newProtocol = new ProtocolConfiguration(protocolName, hash);
431                         protocolConfigurationList.add(newProtocol);
432                     }
433 
434                 }
435             }
436         }
437         return configurator;
438     }
439 
440     /**
441      *  update local data about which are the agents connected to the Clusters.
442      *  this is called in two circumstances :
443      *
444      *      <UL>
445      *          <LI/> when creating an access to a cluster we want to know who is connected and what are their addresses
446      *          <LI/> when we get a <TT>suspect</TT> call due to the fact that a remote agent got 
447      *          disconnect (or is lost!)
448      *      </UL>
449      *  if a remote agent springs to life either ther is not entry for it and we must create one 
450      *  or if there is an empty entry (left by a previous connection) we should fill it.
451      *  <P/>
452      *  if a remote agent is disconnected we must empty its entry.
453      *  <BR/>
454      *  we now must compare the last known view with the current one to know which agent appeared and which diasppeared!
455      *
456      * @param bus
457      * @param channel
458      * @param view
459      */
460     //todo : in a separate thread?
461     private synchronized void updateMapAddressesFromView(Bus bus, JChannel channel, View view) {
462         int index = bus.ordinal();
463         Collection<Address> newMembers = view.getMembers();
464         Collection<Address> addedMembers;
465         Collection<Address> removedMembers = new ArrayList();
466         View lastView = lastViews[index];
467         if (lastView == null) {
468             addedMembers = newMembers;
469         } else {
470             // due to short size complexity is not an issue
471             // so there should be two lists: those added and those removed
472             Collection<Address> oldMembers = lastView.getMembers();
473             addedMembers = new ArrayList<Address>(newMembers);
474             addedMembers.removeAll(oldMembers);
475             removedMembers = new ArrayList(oldMembers);
476             removedMembers.removeAll(newMembers);
477         }
478         lastViews[index] = view;
479         for (Address address : addedMembers) {
480             //TODO: verify which means toString() - is it getName() we mean ?
481             // looks if address is already in table
482             String name = address.toString();
483             if (ANONYMOUS_AGENT.equals(name)) {
484                 // teh problem here is that we don not know which
485                 // anonymous agent is concerned
486                 // so we just add on blindly
487                 // TODO: re-evaluate strategy
488                 AgentAddresses agentAddresses = new AgentAddresses();
489                 agentAddresses.agentName = ANONYMOUS_AGENT;
490                 agentAddresses.busAddresses[index] = address;
491                 anonymousAddresses.add(agentAddresses);
492                 logger.info("adding ====== " + address + " to " + bus);
493                 // TODO: add anonymous connect message ?
494             } else {
495                 // Looking for an agent with the name "name" in namedAgentAddresses
496                 boolean found = false;
497                 for (AgentAddresses agentAddresses : namedAgentAddresses) {
498                     if (agentAddresses.agentName.equals(name)) {
499                         //TODO: if already set?
500                         agentAddresses.busAddresses[index] = address;
501                         logger.info("adding ====== " + address + " to " + bus);
502                         found = true;
503                         break;
504                     }
505                 }
506                 if (!found) { // A new agent is created
507                     AgentAddresses agentAddresses = new AgentAddresses();
508                     agentAddresses.agentName = name;
509                     agentAddresses.busAddresses[index] = address;
510                     namedAgentAddresses.add(agentAddresses);
511                     logger.info("adding ====== " + address + " to " + bus);
512                 }
513                 BusMembershipListener listener = membershipListeners[index];
514                 if (listener != null) {
515                     String info = "";
516                     if (address instanceof UUID) {
517                         info = ((UUID) address).toStringLong();
518                     }
519                     listener.connecting(name, info);
520                 }
521             }
522 
523         }
524 
525         for (Address address : removedMembers) { // Removal
526             // looks if address is already in table
527             String name = address.toString();
528             if (ANONYMOUS_AGENT.equals(name)) {
529                 Iterator<AgentAddresses> it = anonymousAddresses.iterator();
530                 AgentAddresses agentAddressesToRemove = null;
531                 while (it.hasNext()) {
532                     AgentAddresses agentAddresses = it.next();
533                     Address storedAddress = agentAddresses.busAddresses[index];
534                     if (address.equals(storedAddress)) {
535                         //it.remove(); iterator on copyOnwrite does not work!
536                         // TODO: add anonymous remove message?
537                         agentAddressesToRemove = agentAddresses;
538                     }
539                 }
540                 //iterator
541                 if (agentAddressesToRemove != null) {
542                     anonymousAddresses.remove(agentAddressesToRemove);
543                 }
544 
545             } else {
546                 for (AgentAddresses agentAddresses : namedAgentAddresses) {
547                     if (agentAddresses.agentName.equals(name)) {
548                         agentAddresses.busAddresses[index] = null;
549                         BusMembershipListener listener = membershipListeners[index];
550                         if (listener != null) {
551                             String info = "";
552                             if (address instanceof UUID) {
553                                 info = ((UUID) address).toStringLong();
554                             }
555                             listener.disconnecting(name, info);
556                         } else {
557                             logger.info("removing ====== " + address + " from " + bus);
558                         }
559                         break;
560                     }
561                 }
562 
563             }
564         }
565 
566     }
567 
568     @Override
569     public void closeFor(String agentName, Bus... buses) {
570         if (agentName == null || "".equals(agentName)) {
571             agentName = ANONYMOUS_AGENT;
572         }
573         if (buses.length == 0) {
574             buses = Bus.values();
575         }
576         LocalAgentChannels localAgentChannels = mapLocalChannels.get(agentName);
577         if (localAgentChannels != null) {
578             //get the view and if address is local then remove!
579             //Todo! make namedAgentAddresses a Map?
580             AgentAddresses agentAddresses = null;
581             //TODO: this code a shortcut .... you need to be more clever!
582             if (!ANONYMOUS_AGENT.equals(agentName)) {
583                 for (AgentAddresses agAddress : namedAgentAddresses) {
584                     if (agentName.equals(agAddress.agentName)) {
585                         //TODO: check the real address!
586                         agentAddresses = agAddress;
587                         break;
588                     }
589                 }
590             }
591             for (Bus bus : buses) {
592                 int index = bus.ordinal();
593                 JChannel channel = localAgentChannels.channels[index];
594                 if (channel != null) {
595                     channel.close();
596                 }
597                 if (agentAddresses != null) {
598                     agentAddresses.busAddresses[index] = null;
599                 }
600             }
601         }
602     }
603 
604     @Override
605     public void close() throws IOException {
606         closed = true;
607         //for all local agents
608         Iterator<String> itKeys = mapLocalChannels.keySet().iterator();
609         while (itKeys.hasNext()) {
610             String key = itKeys.next();
611             logger.info("******** removing " + key);
612             LocalAgentChannels localAgentChannels = mapLocalChannels.get(key);
613             for (JChannel channel : localAgentChannels.channels) {
614                 if (channel != null) {
615                     channel.close();
616                 }
617             }
618         }
619         //TODO: do not reuse this object! create a volatil boolean closed!
620         //close explicitly all channels
621     }
622 
623     @Override
624     public <T extends BusPayload> void sendMessage(String senderAgent, Bus<T> bus, T message, String... destinations) throws IOException {
625         if (closed) {
626             throw new TransportStateException("closing");
627         }
628         if (senderAgent == null) {
629             throw new IllegalArgumentException("no sender agent");
630         }
631         if (bus == null) {
632             throw new IllegalArgumentException("no bus");
633         }
634         if (message == null) {
635             throw new IllegalArgumentException("no message");
636         }
637         // if explicit list it is initialized, if null -> broadcast
638         ArrayList<Address> listDestinationAddress = null;
639         int index = bus.ordinal();
640         // list of unknown destinations to be kept
641         List<String> failed = new ArrayList<String>();
642         if (destinations.length == 0 || "".equals(destinations[0]) || "*".equals(destinations[0])) {
643             //send to all
644         } else { // explicit list created
645             listDestinationAddress = new ArrayList<Address>();
646             // send to destinations
647             for (String destination : destinations) {
648                 boolean found = false;
649                 for (AgentAddresses agentAddresses : namedAgentAddresses) {
650                     if (agentAddresses.agentName.equals(destination)) {
651                         Address address = agentAddresses.busAddresses[index];
652                         if (address != null) {
653                             listDestinationAddress.add(address);
654                         } else {
655                             if (tryBroadcast) {
656                                 listDestinationAddress.add(null);
657                                 BusMembershipListener listener = membershipListeners[index];
658                                 if (listener != null) {
659                                     listener.anormalEvent(new DuplicateBusNameException(destination, " default broadcast"));
660                                 }
661                                 logger.info("trying broadcast instead of " + agentAddresses.agentName);
662                             } else { /**/
663 
664                                 failed.add(destination);
665                                 logger.info("sending fail (closed)" + agentAddresses.agentName);
666                             } /**/
667 
668                         }
669                         found = true;
670                         break;
671                     }
672                 }
673                 if (!found) {
674                     if (tryBroadcast) {
675                         listDestinationAddress.add(null);
676                         logger.info("trying broadcast instead of " + destination);
677                     } else { /**/
678 
679                         failed.add(destination);
680                         logger.info("sending fail to : " + destination);
681                     }
682                 }
683             }
684             // send to anonymous
685             for (AgentAddresses agentAddresses : anonymousAddresses) {
686                 Address address = agentAddresses.busAddresses[index];
687                 if (address != null) {
688                     listDestinationAddress.add(address);
689                 }
690             }
691         }
692         // now trying to find the proper Channel
693         LocalAgentChannels localAgentChannels = mapLocalChannels.get(senderAgent);
694         if (localAgentChannels == null) {
695             throw new IllegalArgumentException("agent " + senderAgent + " not registered on " + bus);
696         }
697         JChannel channel = localAgentChannels.channels[index];
698         if (channel == null) {
699             throw new IllegalArgumentException("agent " + senderAgent + " not registered on " + bus);
700         }
701         try {
702             //broadcast
703             if (listDestinationAddress == null) {
704                 // versiion 2 channel.send(null, null, (Serializable) message);
705                 channel.send(null, (Serializable) message);
706             } else { //explicit
707                 boolean nullSent = false;
708                 for (Address dest : listDestinationAddress) {
709                     if (dest == null) {
710                         if (nullSent) {
711                             continue;
712                         }
713                         nullSent = true;
714                     }
715                     // version 2 channel.send(dest, null, (Serializable) message);
716                     channel.send(dest, (Serializable) message);
717                 }
718             }
719         } catch (Exception exc) {
720             throw new IOException(exc);
721         }
722         //TODO : what to do if send fails on an existing adresse but there are also fails (unlikely)= getSuppressed?
723         // throwing exception for failed addresses
724         if (failed.size() != 0) {
725             DestinationsException exc = new DestinationsException(senderAgent, failed.toArray());
726             BusMembershipListener listener = membershipListeners[index];
727             if (listener != null) {
728                 listener.anormalEvent(exc);
729             }
730             throw exc;
731         }
732     }
733 
734     @Override
735     public void addMessageListener(String agentName, BusMessageForwarder forwarder, Bus... buses) {
736         if (agentName == null || "".equals(agentName)) {
737             agentName = ANONYMOUS_AGENT;
738         }
739         if (forwarder == null) {
740             throw new IllegalArgumentException("no forwarder");
741         }
742         if (buses.length == 0) {
743             buses = Bus.values();
744         }
745         LocalAgentChannels localAgentChannels = mapLocalChannels.get(agentName);
746         if (localAgentChannels == null) {
747             throw new IllegalArgumentException(" agent " + agentName + "not registered");
748         }
749         for (Bus bus : buses) {
750             BusReceiver receiver = localAgentChannels.receivers[bus.ordinal()];
751             if (receiver == null) {
752                 throw new IllegalArgumentException(" agent " + agentName + "not registered on bus " + bus);
753             }
754             receiver.addForwarder(forwarder);
755         }
756     }
757 
758     @Override
759     public void removeMessageListener(String agentName, BusMessageForwarder forwarder, Bus... buses) {
760         if (agentName == null || "".equals(agentName)) {
761             agentName = ANONYMOUS_AGENT;
762         }
763         if (forwarder == null) {
764             throw new IllegalArgumentException("no forwarder");
765         }
766         if (buses.length == 0) {
767             buses = Bus.values();
768         }
769         LocalAgentChannels localAgentChannels = mapLocalChannels.get(agentName);
770         if (localAgentChannels == null) {
771             return;
772         }
773         for (Bus bus : buses) {
774             BusReceiver receiver = localAgentChannels.receivers[bus.ordinal()];
775             if (receiver == null) {
776                 continue;
777             }
778             receiver.removeForwarder(forwarder);
779         }
780     }
781 
782     @Override
783     public void setMembershipListener(BusMembershipListener listener, Bus... buses) {
784         if (buses.length == 0) {
785             buses = Bus.values();
786         }
787         for (Bus bus : buses) {
788             membershipListeners[bus.ordinal()] = listener;
789         }
790     }
791 
792     /**
793      * gets a list of agent names connected to a bus. 
794      * There is no distinction between agents that are passive or not.
795      * @param bus
796      * @return a List of names (can be empty), anonymous agents are listed with a special Name
797      */
798     @Override
799     public List<String> getConnectedNames(Bus bus) {
800         int index = bus.ordinal();
801         List<String> res = new ArrayList<>() ;
802         for(LocalAgentChannels localAgentChannels : mapLocalChannels.values()) {
803             JChannel channel  = localAgentChannels.channels[index] ;
804             if(channel == null) continue ;
805             // once we get a connected Channel we return the View information on this Channel
806             View view = channel.getView() ;
807             for( Address address : view.getMembers()) {
808                 res.add(channel.getName(address)) ;
809             }
810             return res ;
811         }
812         return res ;
813     }
814 }