/*-
 * election.c --
 *	This functions in this file are responsible for performing the
 *	election algorithm used by the customs daemons.
 *
 * Copyright (c) 1988, 1989 by the Regents of the University of California
 * Copyright (c) 1988, 1989 by Adam de Boor
 * Copyright (c) 1989 by Berkeley Softworks
 *
 * Permission to use, copy, modify, and distribute this
 * software and its documentation for any non-commercial purpose
 * and without fee is hereby granted, provided that the above copyright
 * notice appears in all copies.  The University of California,
 * Berkeley Softworks and Adam de Boor make no representations about
 * the suitability of this software for any purpose.  It is provided
 * "as is" without express or implied warranty.
 */
#ifndef lint
static char *rcsid =
"$Id: election.c,v 1.26 1995/04/22 09:38:01 stolcke Exp $ ICSI (Berkeley)";
#endif /* not lint */

#include    <sys/types.h>
#include    <sys/file.h>
#include    <sys/stat.h>
#include    <errno.h>
extern int errno;

#include    "customsInt.h"

struct sockaddr_in   masterAddr;
			
/*-
 * bool_t
 * CustomsCampaign ()
 *
 * Broadcast by an sca that wants to become the master. Response is TRUE
 * if someone else is also trying to become the master. FALSE if someone else
 * already is the master.
 */

/*-
 * void
 * CustomsNewMaster()
 *
 * Broadcast by the new mca to all scas to inform them that a new master
 * has been elected and the receiving sca should restart.
 */

static enum {
    HAVE_NONE,   	  	/* No response */
    HAVE_CONFLICT,	  	/* Other agent also campaigning */
    HAVE_MASTER  	  	/* Master exists */
} 	    	  	    campaignResponse;

/*
 * A variable to track where we are in an election.
 */
enum {
    NONE,   	    /* No election is taking place */
    WAITING,	    /* We've ok'd someone else's petition, now waiting for
		     * NewMaster message */
    PETITIONING,    /* We've asked to become the master. Waiting for
		     * responses */
    BACKOFF,	    /* Our petition was refused. Waiting to try again. */
    REGISTERING,    /* Registering with the new master */
    YEARNING	    /* We're not allowed to become the master, but we have
		     * no master on the horizon, so wait until someone
		     * speaks up. */
} ElectionState;

#define CANCEL_TOKEN	(-1)	/* Generic token value to force master to
				   give up. */

struct in_addr    lastPetition;	/* Address of last sca whose petition we
				 * accepted. Valid only if WAITING */
Rpc_Event    	  waitEvent;	/* Event for returning to NONE state after
				 * WAITING */

struct timeval	  backOff;  	/* Timeout for exponential backoff */

Rpc_Long	  elect_Token;	/* Token for our network. Set at startup */

static void 	  ElectCampaign(),
		  ElectNewMaster(),
		  ElectForce();
/*-
 *-----------------------------------------------------------------------
 * Elect_Init --
 *	Initialize this module.
 *
 * Results:
 *	None.
 *
 * Side Effects:
 *	The random number generator is randomized, ElectionState set to
 *	NONE, lastPetition set to INADDR_ANY and the election RPC servers
 *	installed on the main udp socket.
 *
 *-----------------------------------------------------------------------
 */
void
Elect_Init ()
{
    struct timeval t;

    /*
     * Randomize for exponential backoff
     */
    gettimeofday(&t, (struct timezone *)NULL);
    srandom((int)(getpid() + localAddr.sin_addr.s_addr + t.tv_sec + t.tv_usec));
    
    ElectionState = NONE;
    lastPetition.s_addr = htonl(INADDR_ANY);

    Rpc_ServerCreate(udpSocket, (Rpc_Proc)CUSTOMS_CAMPAIGN, ElectCampaign,
		     Rpc_SwapLong, Rpc_SwapLong, (Rpc_Opaque)0);
    Rpc_ServerCreate(udpSocket, (Rpc_Proc)CUSTOMS_NEWMASTER, ElectNewMaster,
		     Rpc_SwapLong, Rpc_SwapNull, (Rpc_Opaque)0);
    Rpc_ServerCreate(udpSocket, (Rpc_Proc)CUSTOMS_ELECT, ElectForce,
		     Rpc_SwapNull, Rpc_SwapNull, (Rpc_Opaque)0);
}

/*-
 *-----------------------------------------------------------------------
 * ElectBackOffDone --
 *	Called when the backOff time has expired. Sets the boolean to
 *	which it is pointed to TRUE.
 *
 * Results:
 *	TRUE.
 *
 * Side Effects:
 *	'done' in ElectBackOff is set TRUE.
 *
 *-----------------------------------------------------------------------
 */
static Boolean
ElectBackOffDone(donePtr)
    Boolean *donePtr;
{
    *donePtr = TRUE;
    return(TRUE);
}

/*-
 *-----------------------------------------------------------------------
 * ElectBackOff --
 *	Delay for a random amount of time that increases exponentially
 *	each time this is called. If backOff is 0, a new random time
 *	is selected, else the old time is multiplied by 2.
 *
 * Results:
 *	None.
 *
 * Side Effects:
 *	We delay for a while, handling requests from clients and other
 *	agents as gracefully as possible.
 *
 *-----------------------------------------------------------------------
 */
static void
ElectBackOff()
{
    Rpc_Event	  backEvent;
    Boolean 	  done;
    
    if ((backOff.tv_sec == 0) && (backOff.tv_usec == 0)) {
	int t;

	/*
	 * 1,000,000 ~ 2^20, so take the low twenty bits for microseconds
	 * and the next 3 bits for seconds to get the new backoff amount.
	 */
	t = random();
	backOff.tv_usec = t & 0xfffff;
	backOff.tv_sec = (t >> 20) & 7;
    } else {
	/*
	 * Double the delay.
	 */
	backOff.tv_usec *= 2;
	backOff.tv_sec *= 2;
    }
    /*
     * Normalize the time value
     */
    while (backOff.tv_usec > 1000000) {
	backOff.tv_sec += 1;
	backOff.tv_usec -= 1000000;
    }

    done = FALSE;
    backEvent = Rpc_EventCreate(&backOff, ElectBackOffDone, (Rpc_Opaque)&done);
    if (verbose) {
	xlog (XLOG_DEBUG, "ElectBackOff: waiting %d.%06d seconds",
		backOff.tv_sec, backOff.tv_usec);
    }

    while (!done) {
	Rpc_Wait();
    }
    Rpc_EventDelete(backEvent);
}

/*-
 *-----------------------------------------------------------------------
 * ElectCampaignResponse --
 *	Catch the response to a CAMPAIGN broadcast. Accepts responses
 *	until one comes from the current master.
 *
 * Results:
 *	None.
 *
 * Side Effects:
 *	If the response is FALSE (master saying no), masterAddr is
 *	overwritten and campaignResponse is set to HAVE_MASTER. Else,
 *	campaignResponse is set to HAVE_CONFLICT.
 *
 *-----------------------------------------------------------------------
 */
/*ARGSUSED*/
static Boolean
ElectCampaignResponse(from, len, response)
    struct sockaddr_in	*from;	    /* Source of response */
    int	    	  	len;	    /* Length of response */
    Boolean 	  	*response;  /* Response value */
{
    if (ntohs(from->sin_port) != udpPort) {
	/*
	 * If response from a non-agent, ignore it
	 */
	return(False);
    } else if (*response) {
	if (campaignResponse == HAVE_NONE) {
	    campaignResponse = HAVE_CONFLICT;
	}
    } else if (campaignResponse == HAVE_MASTER) {
	if (from->sin_addr.s_addr != masterAddr.sin_addr.s_addr) {
	    /*
	     * If more than one agent is claiming to be the master, inform
	     * the first one we met of the other one's address. It'll take
	     * care of the rest. Not sure how to actually use this, since
	     * broadcasting isn't multi-threaded enough -- will get responses
	     * back and lose them while this call is in progress.
	     */
	    xlog (XLOG_WARNING, "Warning: duplicate master at %s",
		    InetNtoA(from->sin_addr));
	    (void) Rpc_Call(udpSocket, &masterAddr, (Rpc_Proc)CUSTOMS_CONFLICT,
			    sizeof(struct sockaddr_in), (Rpc_Opaque)from,
			    0, (Rpc_Opaque)0,
			    CUSTOMSINT_NRETRY, &retryTimeOut);
	}
    } else {
	campaignResponse = HAVE_MASTER;
	masterAddr = *from;
	return (True);
    }
    /*
     * We want to broadcast for the entire time....
     */
    return (False);
}

static SockAddr responseAddr;	    /* filled in by CUSTOMS_MASTER */

/*-
 *-----------------------------------------------------------------------
 * ElectMasterResponse --
 *	Catch the response to a MASTER broadcast. For each successful
 *	response, we attempt to register with the master. If that
 *	succeeds, we're happy.
 *
 * Results:
 *	None.
 *
 * Side Effects:
 *	masterAddr is overwritten with the master address returned
 *	by the broadcast.
 *
 *	If the response pans out, ElectionState is reset to NONE.
 *
 *-----------------------------------------------------------------------
 */
/*ARGSUSED*/
static Boolean
ElectMasterResponse(from, len, response)
    struct sockaddr_in	*from;	    /* Source of response */
    int	    	  	len;	    /* Length of response */
    struct sockaddr_in 	*response;  /* Response value */
{
    if (ntohs(from->sin_port) != udpPort) {
	/*
	 * If response from a non-agent, ignore it
	 */
	return(False);
    } else {
	Rpc_Stat    rstat;

	if (verbose) {
	    xlog (XLOG_DEBUG,
		"ElectMasterResponse: received response of %s. registering...",
		InetNtoA(from->sin_addr));
	}

	/*
	 * Reassemble sin_addr structure from portable representation.
	 * As a special case, if the master returned CUSTOMS_FROMADDR as its
	 * address, use the from address, since that is guaranteed to actually
	 * be reachable from our side.
	 */
	if (responseAddr.addr == CUSTOMS_FROMADDR) {
	    masterAddr = *from;
	} else {
	    memset (&masterAddr, 0, sizeof(masterAddr));
#ifdef HAVE_SA_LEN
	    masterAddr.sin_len = sizeof(masterAddr);
#endif
	    masterAddr.sin_family = responseAddr.family;
	    masterAddr.sin_port = responseAddr.port;
	    masterAddr.sin_addr.s_addr = responseAddr.addr;
	}

	rstat = Rpc_Call(udpSocket, &masterAddr,
			 (Rpc_Proc)CUSTOMS_REG,
			 regPacketLen, (Rpc_Opaque)regPacket,
			 0, (Rpc_Opaque)0,
			 CUSTOMSINT_NRETRY, &retryTimeOut);

	if (rstat == RPC_SUCCESS) {
	    if (verbose) {
		xlog (XLOG_DEBUG, "ElectMasterResponse: new master: %s",
			InetNtoA(masterAddr.sin_addr));
	    }
	    ElectionState = NONE;
	    return (True);	/* Stop the broadcast */
	} else if (verbose) {
	    xlog (XLOG_ERROR,
		   "ElectMasterResponse: contacting new master at %s: %s\n",
		   InetNtoA(masterAddr.sin_addr),
		   Rpc_ErrorMessage(rstat));
	}
	/*
	 * Keep searching...
	 */
	return(False);
    }
}

/*-
 *-----------------------------------------------------------------------
 * Elect_GetMaster --
 *	Elect a new master using a broadcast election algorithm. When
 *	this function returns, a new master will have been elected and its
 *	address stored in masterAddr. If we are that new master, amMaster
 *	will be TRUE.
 *
 * Results:
 *	None.
 *
 * Side Effects:
 *	masterAddr is overwritten and amMaster may be changed.
 *
 *-----------------------------------------------------------------------
 */
void
Elect_GetMaster ()
{
    Boolean  	  	conflict;   	    /* Space for Campaign response */
    Rpc_Stat	  	rstat;	    	    /* Status of broadcast */
    struct sockaddr_in 	broadcast;  	    /* Address to which to broadcast */

    timerclear(&backOff);
    broadcast.sin_family = AF_INET;
    broadcast.sin_port = htons(udpPort);
    broadcast.sin_addr.s_addr = htonl(INADDR_ANY);
    
    /*
     * If we're not allowed to become the master, wait until someone comes
     * along to be it. Master agents will occasionally trumpet their mastery
     * so we won't wait forever after a network partition...
     */
    if (!canBeMaster) {
	ElectionState = YEARNING;

	if (verbose) {
	    xlog (XLOG_DEBUG, "Elect_GetMaster: seeking new master...");
	}
	
	/*
	 * First see if anyone else knows the name of a good master.
	 */
	rstat = Rpc_Broadcast(udpSocket, &broadcast,
			      (Rpc_Proc)CUSTOMS_MASTER,
			      sizeof(elect_Token), (Rpc_Opaque)&elect_Token,
			      sizeof(responseAddr), (Rpc_Opaque)&responseAddr,
			      3, &retryTimeOut,
			      ElectMasterResponse, (Rpc_Opaque)0);

	if (verbose && rstat != RPC_SUCCESS) {
	    xlog (XLOG_DEBUG,
		"Elect_GetMaster: no such luck. Waiting for master to appear.");
	}
	    
	/*
	 * If we found one, ElectionState will no longer be YEARNING and
	 * this loop will do nothing. Else we will await the return of
	 * a master...
	 */
	while (ElectionState == YEARNING) {
	    Rpc_Wait();
	}
	return;
    }
    
    while (1) {
	if (verbose) {
	    xlog (XLOG_DEBUG, "Elect_GetMaster: Petitioning...");
	}
	ElectionState = PETITIONING;
	campaignResponse = HAVE_NONE;
	rstat = Rpc_Broadcast(udpSocket, &broadcast,
			      (Rpc_Proc)CUSTOMS_CAMPAIGN,
			      sizeof(elect_Token), (Rpc_Opaque)&elect_Token,
			      sizeof(conflict), (Rpc_Opaque)&conflict,
			      3, &retryTimeOut,
			      ElectCampaignResponse, (Rpc_Opaque)0);
	switch(rstat) {
	    case RPC_SUCCESS:
		/*
		 * Someone objected to our becoming master.
		 */
		if (campaignResponse == HAVE_CONFLICT) {
		    /*
		     * Objected because it was also trying to become the
		     * master. Do exponential backoff in an attempt not
		     * to conflict again.
		     */
		    if (verbose) {
			xlog (XLOG_DEBUG,
				"Elect_GetMaster: CONFLICT: backing off");
		    }
		    ElectionState = BACKOFF;
		    ElectBackOff();
		    break;
		} else if (campaignResponse == HAVE_MASTER) {
		    /*
		     * Objected because it was already the master.
		     * Attempt to contact the agent. If we manage to do so,
		     * register with it. If that fails, try again...
		     */
		    if (verbose) {
			xlog (XLOG_DEBUG,
				"Elect_GetMaster: REFUSED: Contacting new master...");
		    }
		    ElectionState = REGISTERING;
		    rstat = Rpc_Call(udpSocket, &masterAddr,
				     (Rpc_Proc)CUSTOMS_REG,
				     regPacketLen, (Rpc_Opaque)regPacket,
				     0, (Rpc_Opaque)0,
				     CUSTOMSINT_NRETRY, &retryTimeOut);
		    if (rstat == RPC_SUCCESS) {
			if (verbose) {
			    xlog (XLOG_DEBUG,
				    "Elect_GetMaster: new master: %s",
				    InetNtoA(masterAddr.sin_addr));
			}
			ElectionState = NONE;
			return;
		    } else if (verbose) {
			xlog (XLOG_DEBUG,
				"Elect_GetMaster: contacting new: %s",
			        Rpc_ErrorMessage(rstat));
		    }
		    break;
		}
		/*FALLTHRU*/
	    case RPC_TIMEDOUT:
		/*
		 * Noone responded. We are the master.
		 */
		masterAddr = localAddr;
		if (verbose) {
		    xlog (XLOG_DEBUG,
			    "Elect_GetMaster: No one responded: Accepting mastery");
		}
		/*
		 * Become the master -- it will send out the NEWMASTER call
		 * when it's ready.
		 */
		ElectionState = NONE;
		MCA_Init();
		Log_Send(LOG_NEWMASTER, 1, xdr_sockaddr_in, &localAddr);

		return;
	    default: {
		xlog (XLOG_ERROR,
			"Rpc_Broadcast: %s", strerror(errno));
		xlog (XLOG_ERROR, "CUSTOMS_CAMPAIGN: %s",
			Rpc_ErrorMessage(rstat));
		break;
	    }
	}
    }
}

/*-
 *-----------------------------------------------------------------------
 * Elect_InProgress --
 *	Tell if an election is in progress.
 *
 * Results:
 *	TRUE if there's one going. FALSE otherwise.
 *
 * Side Effects:
 *	None.
 *
 *-----------------------------------------------------------------------
 */
Boolean
Elect_InProgress ()
{
    return ((ElectionState == PETITIONING) || (ElectionState == BACKOFF) ||
	    (ElectionState == REGISTERING) || (ElectionState == YEARNING));
}
    

/*-
 *-----------------------------------------------------------------------
 * ElectClearWait --
 *	Return from WAITING to NONE state. Callback for ElectCampaign.
 *
 * Results:
 *	FALSE.
 *
 * Side Effects:
 *	ElectionState is set to NONE if it was WAITING.
 *
 *-----------------------------------------------------------------------
 */
static Boolean
ElectClearWait()
{
    if (ElectionState == WAITING) {
	ElectionState = NONE;
    }
    if (waitEvent) {
	Rpc_EventDelete(waitEvent);
	waitEvent = (Rpc_Event)0;
    }
    return(FALSE);
}
/*-
 *-----------------------------------------------------------------------
 * ElectCampaign --
 *	Stub for CUSTOMS_CAMPAIGN call. To allow for an heterogenous
 *	network, this stub only pays attention to calls for which the
 *	data is elect_Token. elect_Token is set at startup and allows a
 *	network to be partitioned into subnets.
 *
 *	If we have nothing to say, we don't respond at all. We can do that
 *	now there's only the campaigning agent waiting.
 *
 * Results:
 *	None.
 *
 * Side Effects:
 *	A response will be generated if we are (a) campaigning ourselves or
 * 	(b) we are the master.
 *
 *-----------------------------------------------------------------------
 */
static void
ElectCampaign(from, msg, len, data)
    struct sockaddr_in	*from;
    Rpc_Message	  	msg;
    int	    	  	len;
    int	    	  	*data;
{
    Boolean 	  	response;
    
    if (verbose) {
	xlog (XLOG_DEBUG, "ElectCampaign: received CAMPAIGN from %d@%s",
		ntohs(from->sin_port),
		InetNtoA(from->sin_addr));
    }
    if (ntohs(from->sin_port) != udpPort) {
	return;
    }
    if (Local(from)) {
	if (verbose) {
	    xlog (XLOG_DEBUG, "ElectCampaign: talking to myself, again...");
	}
    } else if ((len == sizeof(int)) && (*data == elect_Token)) {
	/*
	 * If machine has same byte-order as we do, then we can play
	 * master/slave with it...
	 */
	if (amMaster) {
	    if (verbose) {
		xlog (XLOG_DEBUG, "ElectCampaign: return(FALSE)");
	    }
	    response = FALSE;
	    Rpc_Return(msg, sizeof(response), (Rpc_Opaque)&response);
	} else if (ElectionState == PETITIONING) {
	    if (verbose) {
		xlog (XLOG_DEBUG, "ElectCampaign: return(TRUE)");
	    }
	    response = TRUE;
	    Rpc_Return(msg, sizeof(response), (Rpc_Opaque)&response);
	} else if ((ElectionState == WAITING) &&
		   (lastPetition.s_addr != from->sin_addr.s_addr))
	{
	    /*
	     * If someone else was campaigning, we refuse to let
	     * this agent become the master. This is in case the
	     * campaigning one missed the broadcast somehow.
	     */
	    if (verbose) {
		xlog (XLOG_DEBUG,
			"ElectCampaign: return (TRUE) -- conflict with %s",
			InetNtoA(lastPetition));
	    }
	    response = TRUE;
	    Rpc_Return(msg, sizeof(response),
		       (Rpc_Opaque)&response);
	} else {
	    /*
	     * It's ok for this agent to become master, as far as we're
	     * concerned. Because petitions are broadcast at startup,
	     * we don't want to stay in the WAITING state forever, or
	     * we'll have to wait for everyone to timeout before we can
	     * elect a master (everyone's petitions will be refused
	     * until then) and that isn't good. So we set an event for
	     * twenty seconds from now to return to the NONE state.
	     */
	    struct timeval waitTimeout;
	    
	    if (verbose) {
		xlog (XLOG_DEBUG, "ElectCampaign: OK");
	    }
	    if (ElectionState == NONE) {
		/*
		 * Only alter the state if no election was in progress.
		 * We must allow petitions if ElectionState == BACKOFF,
		 * but if we set ElectionState to WAITING, the Avail
		 * module might try to send to a non-existent master...
		 */
		ElectionState = WAITING;
	    }
	    lastPetition = from->sin_addr;
	    
	    waitTimeout.tv_sec = 10;
	    waitTimeout.tv_usec = 0;
	    if (waitEvent != (Rpc_Event)NULL) {
		Rpc_EventDelete(waitEvent);
	    }
	    waitEvent = Rpc_EventCreate(&waitTimeout, ElectClearWait,
					(Rpc_Opaque)0);
	}
    } else if (verbose) {
	xlog (XLOG_DEBUG, "ElectCampaign: not my type");
    }
}

/*-
 *-----------------------------------------------------------------------
 * ElectNewMaster --
 *	Stub for the CUSTOMS_NEWMASTER broadcast call.
 *
 * Results:
 *	None.
 *
 * Side Effects:
 *	If we currently are the master, cancels our mastery.
 *
 *-----------------------------------------------------------------------
 */
static void
ElectNewMaster(from, msg, len, data)
    struct sockaddr_in	*from;
    Rpc_Message	  	msg;
    int	    	  	len;
    Rpc_Long	    	*data;
{
    Rpc_Stat	  	rstat;

    if (verbose) {
	xlog (XLOG_DEBUG, "ElectNewMaster: received NEWMASTER from %d@%s",
		ntohs(from->sin_port), InetNtoA(from->sin_addr));
    }
    if (ntohs(from->sin_port) != udpPort) {
	if (verbose) {
	    xlog (XLOG_DEBUG, "ElectNewMaster: wrong port number");
	} else {
	    xlog (XLOG_WARNING, "Bogus NEWMASTER from %d@%s",
		    ntohs(from->sin_port), InetNtoA(from->sin_addr));
	}
	return;
    }
    if ((ElectionState == NONE) || (ElectionState == WAITING) ||
	(ElectionState == YEARNING))
    {
	if (Local(from)) {
	    Rpc_Return(msg, 0, (Rpc_Opaque)0);
	    if (verbose) {
		xlog (XLOG_DEBUG,
			"ElectNewMaster: talking to myself, again...");
	    }
	} else if ((len == sizeof(int)) && (*data == CANCEL_TOKEN)) {
	    /*
	     * Unconditionally cancel our mastery, if any.
	     * We silently invalidate the master address instead of
	     * calling for a new election ourselves.  This will give
	     * the calling party a good chance at becoming the next master.
	     */
	    if (!amMaster) {
		Rpc_Error(msg, RPC_BADARGS);
		return;
	    }

	    Rpc_Return(msg, 0, (Rpc_Opaque)0);

	    (void)ElectClearWait();

	    if (verbose) {
		xlog (XLOG_DEBUG, "ElectNewMaster: giving up mastery");
	    }
	    MCA_Cancel();
	    masterAddr.sin_addr.s_addr = htonl(INADDR_ANY);
	    ElectionState = NONE;
	} else if ((len == sizeof(elect_Token)) && (*data == elect_Token)) {
	    Rpc_Return(msg, 0, (Rpc_Opaque)0);

	    (void)ElectClearWait();

	    /*
	     * We have a master conflict - someone has to give up mastery.
	     * NEWMASTER broadcasts are likely to cross each other, e.g.,
	     * after a network partition goes away.
	     * The master with the lower IP address relinquishes mastery
	     * immediately, the other one is more cautious and checks that
	     * the other guy is actually a master.  That way one of them
	     * gets to keep their status and we avoid a full reelection
	     * which can consume considerable time and network bandwidth.
	     *
	     * We set election state to REGISTERING to avoid giving out
	     * a master adress we're not sure about.  However,
	     * if the NEWMASTER call occurs outside of an election and
	     * the call comes from our old master we don't need to go
	     * into a special state since the master is probably just
	     * his own boasting self.
	     */
	    if (amMaster &&
	        (from->sin_addr.s_addr < localAddr.sin_addr.s_addr)) {

		xlog (verbose ? XLOG_DEBUG : XLOG_INFO,
			"ElectNewMaster: underdog yielding mastery to %s",
			InetNtoA(from->sin_addr));
		MCA_Cancel();
		ElectionState = REGISTERING;
	    } else if (ElectionState != NONE) {
		ElectionState = REGISTERING;
	    } else if (from->sin_addr.s_addr != masterAddr.sin_addr.s_addr) {
		ElectionState = REGISTERING;
	    }

	    rstat = Rpc_Call(udpSocket, from, (Rpc_Proc)CUSTOMS_REG,
			     regPacketLen, (Rpc_Opaque)regPacket,
			     0, (Rpc_Opaque)0,
			     CUSTOMSINT_NRETRY, &retryTimeOut);

	    if (rstat != RPC_SUCCESS) {
		xlog (XLOG_ERROR,
			"ElectNewMaster: registering with new master at %s: %s",
			InetNtoA(masterAddr.sin_addr),
			Rpc_ErrorMessage(rstat));
		if (amMaster) {
		    /*
		     * Retain mastery.
		     */
		    ElectionState = NONE;
		} else {
		    /*
		     * Resets election state
		     */
		    Elect_GetMaster();
		}
	    } else {
		if (amMaster) {
		    /*
		     * Ok, give up. Let the other guy be the master
		     */
		    xlog (verbose ? XLOG_DEBUG : XLOG_INFO,
			    "ElectNewMaster: top dog yielding mastery to %s",
			    InetNtoA(from->sin_addr));
		    MCA_Cancel();
		}
		masterAddr = *from;
		ElectionState = NONE;
	    }
	} else if (verbose) {
	    xlog (XLOG_DEBUG, "ElectNewMaster: not my type");
	}
    } else if (verbose) {
	xlog (XLOG_DEBUG, "ElectNewMaster: ignored");
    }
}

/*-
 *-----------------------------------------------------------------------
 * ElectForce --
 *	Force an election, cancelling our mastery, if we're the master.
 *
 * Results:
 *	Nothing.
 *
 * Side Effects:
 *	
 *-----------------------------------------------------------------------
 */
/*ARGSUSED*/
static void
ElectForce(from, msg, len, data)
    struct sockaddr_in	*from;
    Rpc_Message	  	msg;
    int	    	  	len;
    int	    	  	*data;
{
    Rpc_Stat	  	rstat;
    Rpc_Long		cancelToken = CANCEL_TOKEN;

    /*
     * Forcing an election is a priviledged operation.
     */
    CustomsReserved("ElectForce", from, msg);

    xlog (XLOG_INFO, "ELECT received from %d@%s",
	    ntohs(from->sin_port), InetNtoA(from->sin_addr));

    /*
     * Signal our acceptance
     */
    Rpc_Return(msg, 0, (Rpc_Opaque)0);
    
    /*
     * If we're the master, cancel it before starting the election.
     */
    if (amMaster) {
	if (verbose) {
	    xlog (XLOG_DEBUG, "ElectForce: cancelling mastery");
	}
	MCA_Cancel();
    } else if (canBeMaster) {
	if (verbose) {
	    xlog (XLOG_DEBUG, "ElectForce: telling current master to give up");
	}
	/*
	 * Tell current master to relinquish mastery so we will have a
	 * very good chance at becoming new master.
	 */
	rstat = Rpc_Call(udpSocket, &masterAddr,
			    (Rpc_Proc)CUSTOMS_NEWMASTER,
			    sizeof(cancelToken), (Rpc_Opaque)&cancelToken,
			    0, (Rpc_Opaque)0,
			    CUSTOMSINT_NRETRY, &retryTimeOut);
	if (rstat != RPC_SUCCESS) {
	    xlog (XLOG_WARNING,
		    "ElectForce: failed to cancel old master at %s: %s",
		    InetNtoA(masterAddr.sin_addr),
		    Rpc_ErrorMessage(rstat));
	}
    }

    /*
     * Now force an election.
     */
    Elect_GetMaster();
}
