/*
 * Copyright 2000, International Business Machines Corporation and others.
 * All Rights Reserved.
 *
 * This software has been released under the terms of the IBM Public
 * License.  For details, see the LICENSE file in the top-level source
 * directory or online at http://www.openafs.org/dl/license10.html
 */

#include <afsconfig.h>
#include <afs/param.h>

#include <roken.h>

#include <afs/opr.h>
#ifdef AFS_PTHREAD_ENV
# include <opr/lock.h>
#else
# include <opr/lockstub.h>
#endif
#include <afs/afs_lock.h>
#include <rx/rx.h>
#include <afs/afsutil.h>

#define UBIK_INTERNALS
#include "ubik.h"
#include "ubik_int.h"

/*! \file
 * General Ubik Goal:
 * The goal is to provide reliable operation among N servers, such that any
 * server can crash with the remaining servers continuing operation within a
 * short period of time.  While a \b short outage is acceptable, this time
 * should be order of 3 minutes or less.
 *
 * Theory of operation:
 *
 * Note: #SMALLTIME and #BIGTIME are essentially the same time value, separated
 * only by the clock skew, #MAXSKEW.  In general, if you are making guarantees
 * for someone else, promise them no more than #SMALLTIME seconds of whatever
 * invariant you provide.  If you are waiting to be sure some invariant is now
 * \b false, wait at least #BIGTIME seconds to be sure that #SMALLTIME seconds
 * has passed at the other site.
 *
 * Now, back to the design:
 * One site in the collection is a special site, designated the \b sync site.
 * The sync site sends periodic messages, which can be thought of as
 * keep-alive messages.  When a non-sync site hears from the sync site, it
 * knows that it is getting updates for the next #SMALLTIME seconds from that
 * sync site.
 *
 * If a server does not hear from the sync site in #SMALLTIME seconds, it
 * determines that it no longer is getting updates, and thus refuses to give
 * out potentially out-of-date data.  If a sync site can not muster a majority
 * of servers to agree that it is the sync site, then there is a possibility
 * that a network partition has occurred, allowing another server to claim to
 * be the sync site.  Thus, any time that the sync site has not heard from a
 * majority of the servers in the last #SMALLTIME seconds, it voluntarily
 * relinquishes its role as sync site.
 *
 * While attempting to nominate a new sync site, certain rules apply.  First,
 * a server can not reply "ok" (return 1 from ServBeacon) to two different
 * hosts in less than #BIGTIME seconds; this allows a server that has heard
 * affirmative replies from a majority of the servers to know that no other
 * server in the network has heard enough affirmative replies in the last
 * #BIGTIME seconds to become sync site, too.  The variables #ubik_lastYesTime
 * and #lastYesHost are used by all servers to keep track of which host they
 * have last replied affirmatively to, when queried by a potential new sync
 * site.
 *
 * Once a sync site has become a sync site, it periodically sends beacon
 * messages with a parameter of 1, indicating that it already has determined
 * it is supposed to be the sync site.  The servers treat such a message as a
 * guarantee that no other site will become sync site for the next #SMALLTIME
 * seconds.  In the interim, these servers can answer a query concerning which
 * site is the sync site without any communication with any server.  The
 * variables #lastBeaconArrival and #lastBeaconHost are used by all servers to
 * keep track of which sync site has last contacted them.
 *
 * One complication occurs while nominating a new sync site: each site may be
 * trying to nominate a different site (based on the value of #lastYesHost),
 * yet we must nominate the smallest host (under some order), to prevent this
 * process from looping.  The process could loop by having each server give
 * one vote to another server, but with no server getting a majority of the
 * votes.  To avoid this, we try to withhold our votes for the server with the
 * lowest internet address (an easy-to-generate order).  To this effect, we
 * keep track (in #lowestTime and #lowestHost) of the lowest server trying to
 * become a sync site.  We wait for this server unless there is already a sync
 * site (indicated by ServBeacon's parameter being 1).
 */

afs_int32 ubik_debugFlag = 0;	/*!< print out debugging messages? */

struct vote_data vote_globals;


/*!
 * \brief Decide if we should try to become sync site.
 *
 * The basic rule is that we
 * don't run if there is a valid sync site and it ain't us (we have to run if
 * it is us, in order to keep our votes).  If there is no sync site, then we
 * want to run if we're the lowest numbered host running, otherwise we defer to
 * the lowest host.  However, if the lowest host hasn't been heard from for a
 * while, then we start running again, in case he crashed.
 *
 * \return true if we should run, and false otherwise.
 */
int
uvote_ShouldIRun(void)
{
    afs_int32 now;
    int code = 1; /* default to yes */

    if (amIClone) {
	return 0;		/* if we cannot be the sync-site, do not ask for votes */
    }

    UBIK_VOTE_LOCK;
    now = FT_ApproxTime();
    if (BIGTIME + vote_globals.ubik_lastYesTime < now)
	goto done;
    if (vote_globals.lastYesState && vote_globals.lastYesHost != ubik_host[0]) {
	code = 0;		/* other guy is sync site, leave him alone */
	goto done;
    }
    if (ntohl((afs_uint32)vote_globals.lastYesHost) < ntohl((afs_uint32)ubik_host[0])) {
	code = 0;		/* if someone is valid and better than us, don't run */
	goto done;
    }

done:
    UBIK_VOTE_UNLOCK;
    return code;
}

/*!
 * \brief Return the current synchronization site, if any.
 *
 * Simple approach: if the
 * last guy we voted yes for claims to be the sync site, then we we're happy to
 * use that guy for a sync site until the time his mandate expires.  If the guy
 * does not claim to be sync site, then, of course, there's none.
 *
 * In addition, if we lost the sync, we set #urecovery_syncSite to an invalid
 * value, indicating that we no longer know which version of the dbase is the
 * one we should have.  We'll get a new one when we next hear from the sync
 * site.
 *
 * \return 0 or currently valid sync site.  It can return our own
 * address, if we're the sync site.
 */
afs_int32
uvote_GetSyncSite(void)
{
    afs_int32 now;
    afs_int32 code;

    UBIK_VOTE_LOCK;
    if (!vote_globals.lastYesState)
	code = 0;
    else {
	now = FT_ApproxTime();
	if (SMALLTIME + vote_globals.lastYesClaim < now)
	    code = 0;		/* last guy timed out */
	else
	    code = vote_globals.lastYesHost;
    }
    UBIK_VOTE_UNLOCK;
    return code;
}

/*!
 * \brief called by the sync site to handle vote beacons; if aconn is null, this is a
 * local call
 *
 * \returns 0 or time when the vote was sent.  It returns 0 if we are
 * not voting for this sync site, or the time we actually voted yes, if
 * non-zero.
 */
afs_int32
SVOTE_Beacon(struct rx_call * rxcall, afs_int32 astate,
	     afs_int32 astart, struct ubik_version * avers,
	     struct ubik_tid * atid)
{
    afs_int32 otherHost;
    afs_int32 now;
    afs_int32 vote;
    struct rx_connection *aconn;
    struct rx_peer *rxp;
    struct ubik_server *ts;
    int isClone = 0;
    char hoststr[16];

    if (rxcall) {		/* caller's host */
	aconn = rx_ConnectionOf(rxcall);
	rxp = rx_PeerOf(aconn);
	otherHost = rx_HostOf(rxp);

	/* get the primary interface address for this host.  */
	/* This is the identifier that ubik uses. */
	otherHost = ubikGetPrimaryInterfaceAddr(otherHost);
	if (!otherHost) {
	    ViceLog(5, ("Received beacon from unknown host %s\n",
			afs_inet_ntoa_r(rx_HostOf(rxp), hoststr)));
	    return 0;		/* I don't know about you: vote no */
	}
	for (ts = ubik_servers; ts; ts = ts->next) {
	    if (ts->addr[0] == otherHost)
		break;
	}
	if (!ts)
	    ViceLog(0, ("Unknown host %x has sent a beacon\n", otherHost));
	if (ts && ts->isClone)
	    isClone = 1;
    } else {
	otherHost = ubik_host[0];	/* this host */
	isClone = amIClone;
    }

    ViceLog(5, ("Received beacon type %d from host %s\n", astate,
		afs_inet_ntoa_r(otherHost, hoststr)));

    /* compute the lowest server we've heard from.  We'll try to only vote for
     * this dude if we don't already have a synchronization site.  Also, don't
     * let a very old lowestHost confusing things forever.  We pick a new
     * lowestHost after BIGTIME seconds to limit the damage if this host
     * actually crashes.  Finally, we also count in this computation: don't
     * pick someone else if we're even better!
     *
     * Note that the test below must be <=, not <, so that we keep refreshing
     * lowestTime.  Otherwise it will look like we haven't heard from
     * lowestHost in a while and another host could slip in.  */


    /* First compute the lowest host we've heard from, whether we want them
     * for a sync site or not.  If we haven't heard from a site in BIGTIME
     * seconds, we ignore its presence in lowestHost: it may have crashed.
     * Note that we don't ever let anyone appear in our lowestHost if we're
     * lower than them, 'cause we know we're up. */
    /* But do not consider clones for lowesHost since they never may become
     * sync site */
    UBIK_VOTE_LOCK;
    now = FT_ApproxTime();	/* close to current time */
    if (!isClone
	&& (ntohl((afs_uint32)otherHost) <= ntohl((afs_uint32)vote_globals.lowestHost)
	    || vote_globals.lowestTime + BIGTIME < now)) {
	vote_globals.lowestTime = now;
	vote_globals.lowestHost = otherHost;
    }
    /* why do we need this next check?  Consider the case where each of two
     * servers decides the other is lowestHost.  Each stops sending beacons
     * 'cause the other is there.  Not obvious that this process terminates:
     * i.e. each guy could restart procedure and again think other side is
     * lowest.  Need to prove: if one guy in the system is lowest and knows
     * he's lowest, these loops don't occur.  because if someone knows he's
     * lowest, he will send out beacons telling others to vote for him. */
    if (!amIClone
	&& (ntohl((afs_uint32) ubik_host[0]) <= ntohl((afs_uint32)vote_globals.lowestHost)
	    || vote_globals.lowestTime + BIGTIME < now)) {
	vote_globals.lowestTime = now;
	vote_globals.lowestHost = ubik_host[0];
    }

    /* tell if we've heard from a sync site recently (even if we're not voting
     * for this dude yet).  After a while, time the guy out. */
    if (astate) {		/* this guy is a sync site */
	vote_globals.syncHost = otherHost;
	vote_globals.syncTime = now;
    } else if (vote_globals.syncTime + BIGTIME < now) {
	if (vote_globals.syncHost) {
	    ViceLog(0, ("Ubik: Lost contact with sync-site %s (NOT in quorum)\n",
		 afs_inet_ntoa_r(vote_globals.syncHost, hoststr)));
	}
	vote_globals.syncHost = 0;
    }

    /* decide how to vote */
    vote = 0;			/* start off voting no */

    /* if we this guy isn't a sync site, we don't really have to vote for him.
     * We get to apply some heuristics to try to avoid weird oscillation sates
     * in the voting procedure. */
    if (astate == 0) {
	/* in here only if this guy doesn't claim to be a sync site */

	/* lowestHost is also trying for our votes, then just say no. */
	if (ntohl(vote_globals.lowestHost) != ntohl(otherHost)) {
	    goto done_zero;
	}

	/* someone else *is* a sync site, just say no */
	if (vote_globals.syncHost && vote_globals.syncHost != otherHost)
	    goto done_zero;
    } else if (vote_globals.lastYesHost == 0xffffffff && otherHost == ubik_host[0]) {
	/* fast startup if this is the only non-clone */
	int i = 0;
	for (ts = ubik_servers; ts; ts = ts->next) {
	    if (ts->addr[0] == otherHost)
		continue;
	    if (!ts->isClone)
		i++;
	}
	if (!i)
	    vote_globals.lastYesHost = otherHost;
    }


    if (isClone)
	goto done_zero;		/* clone never can become sync site */

    /* Don't promise sync site support to more than one host every BIGTIME
     * seconds.  This is the heart of our invariants in this system. */
    if (vote_globals.ubik_lastYesTime + BIGTIME < now || otherHost == vote_globals.lastYesHost) {
	if ((vote_globals.ubik_lastYesTime + BIGTIME < now) || (otherHost != vote_globals.lastYesHost)
	    || (vote_globals.lastYesState != astate)) {
	    /* A new vote or a change in the vote or changed quorum */
	    /* XXX This should be at loglevel 0, but the conditionals
	     * ought to be reworked first to prevent excessive logging.
	     */
	    ViceLog(5, ("Ubik: vote 'yes' for %s %s\n",
			afs_inet_ntoa_r(otherHost, hoststr),
			(astate ? "(in quorum)" : "(NOT in quorum)")));
	}

	vote = now;		/* vote yes */
	vote_globals.ubik_lastYesTime = now;	/* remember when we voted yes */
	vote_globals.lastYesClaim = astart;	/* remember for computing when sync site expires */
	vote_globals.lastYesHost = otherHost;	/* and who for */
	vote_globals.lastYesState = astate;	/* remember if site is a sync site */
	vote_globals.ubik_dbVersion = *avers;	/* resync value */
	vote_globals.ubik_dbTid = *atid;	/* transaction id, if any, of active trans */
	UBIK_VOTE_UNLOCK;
	DBHOLD(ubik_dbase);
	urecovery_CheckTid(atid, 0);	/* check if current write trans needs aborted */
	DBRELE(ubik_dbase);
    } else {
	UBIK_VOTE_UNLOCK;
    }
    return vote;
done_zero:
    UBIK_VOTE_UNLOCK;
    return 0;
}

/*!
 * \brief Handle per-server debug command, where 0 is the first server.
 *
 * Basic network debugging hooks.
 */
afs_int32
SVOTE_SDebug(struct rx_call * rxcall, afs_int32 awhich,
	     struct ubik_sdebug * aparm)
{
    afs_int32 code, isClone;
    code = SVOTE_XSDebug(rxcall, awhich, aparm, &isClone);
    return code;
}

afs_int32
SVOTE_XSDebug(struct rx_call * rxcall, afs_int32 awhich,
	      struct ubik_sdebug * aparm, afs_int32 * isclone)
{
    struct ubik_server *ts;
    int i;
    for (ts = ubik_servers; ts; ts = ts->next) {
	if (awhich-- == 0) {
	    /* we're done */
	    aparm->addr = ntohl(ts->addr[0]);	/* primary interface */
	    for (i = 0; i < UBIK_MAX_INTERFACE_ADDR - 1; i++)
		aparm->altAddr[i] = ntohl(ts->addr[i + 1]);
	    aparm->lastVoteTime = ts->lastVoteTime;
	    aparm->lastBeaconSent = ts->lastBeaconSent;
	    memcpy(&aparm->remoteVersion, &ts->version,
		   sizeof(struct ubik_version));
	    aparm->lastVote = ts->lastVote;
	    aparm->up = ts->up;
	    aparm->beaconSinceDown = ts->beaconSinceDown;
	    aparm->currentDB = ts->currentDB;
	    *isclone = ts->isClone;
	    return 0;
	}
    }
    return 2;
}

afs_int32
SVOTE_XDebug(struct rx_call * rxcall, struct ubik_debug * aparm,
	     afs_int32 * isclone)
{
    afs_int32 code;

    code = SVOTE_Debug(rxcall, aparm);
    *isclone = amIClone;
    return code;
}

/*!
 * \brief Handle basic network debug command.  This is the global state dumper.
 */
afs_int32
SVOTE_Debug(struct rx_call * rxcall, struct ubik_debug * aparm)
{
    int i;
    /* fill in the basic debug structure.  Note the the RPC protocol transfers,
     * integers in host order. */

    memset(aparm, 0, sizeof(*aparm));
    aparm->now = FT_ApproxTime();
    aparm->lastYesTime = vote_globals.ubik_lastYesTime;
    aparm->lastYesHost = ntohl(vote_globals.lastYesHost);
    aparm->lastYesState = vote_globals.lastYesState;
    aparm->lastYesClaim = vote_globals.lastYesClaim;
    aparm->lowestHost = ntohl(vote_globals.lowestHost);
    aparm->lowestTime = vote_globals.lowestTime;
    aparm->syncHost = ntohl(vote_globals.syncHost);
    aparm->syncTime = vote_globals.syncTime;
    memcpy(&aparm->syncVersion, &vote_globals.ubik_dbVersion, sizeof(struct ubik_version));
    memcpy(&aparm->syncTid, &vote_globals.ubik_dbTid, sizeof(struct ubik_tid));

    /* fill in all interface addresses of myself in hostbyte order */
    for (i = 0; i < UBIK_MAX_INTERFACE_ADDR; i++)
	aparm->interfaceAddr[i] = ntohl(ubik_host[i]);

    aparm->amSyncSite = beacon_globals.ubik_amSyncSite;
    ubeacon_Debug(aparm);

    udisk_Debug(aparm);

    ulock_Debug(aparm);

    /* Get the recovery state. The label of the database may not have
     * been written yet but set the flag so udebug behavior remains.
     * Defect 9477.
     */
    aparm->recoveryState = urecovery_state;
    if ((urecovery_state & UBIK_RECSYNCSITE)
	&& (urecovery_state & UBIK_RECFOUNDDB)
	&& (urecovery_state & UBIK_RECHAVEDB)) {
	aparm->recoveryState |= UBIK_RECLABELDB;
    }
    aparm->activeWrite = (ubik_dbase->flags & DBWRITING);
    aparm->tidCounter = ubik_dbase->tidCounter;

    if (ubik_currentTrans) {
	aparm->currentTrans = 1;
	aparm->writeTrans = 1;
    } else {
	aparm->currentTrans = 0;
    }

    aparm->epochTime = version_globals.ubik_epochTime;

    return 0;
}

afs_int32
SVOTE_SDebugOld(struct rx_call * rxcall, afs_int32 awhich,
		struct ubik_sdebug_old * aparm)
{
    struct ubik_server *ts;

    for (ts = ubik_servers; ts; ts = ts->next) {
	if (awhich-- == 0) {
	    /* we're done */
	    aparm->addr = ntohl(ts->addr[0]);	/* primary interface */
	    aparm->lastVoteTime = ts->lastVoteTime;
	    aparm->lastBeaconSent = ts->lastBeaconSent;
	    memcpy(&aparm->remoteVersion, &ts->version,
		   sizeof(struct ubik_version));
	    aparm->lastVote = ts->lastVote;
	    aparm->up = ts->up;
	    aparm->beaconSinceDown = ts->beaconSinceDown;
	    aparm->currentDB = ts->currentDB;
	    return 0;
	}
    }
    return 2;
}


/*!
 * \brief Handle basic network debug command.  This is the global state dumper.
 */
afs_int32
SVOTE_DebugOld(struct rx_call * rxcall,
	       struct ubik_debug_old * aparm)
{

    /* fill in the basic debug structure.  Note the the RPC protocol transfers,
     * integers in host order. */

    aparm->now = FT_ApproxTime();
    aparm->lastYesTime = vote_globals.ubik_lastYesTime;
    aparm->lastYesHost = ntohl(vote_globals.lastYesHost);
    aparm->lastYesState = vote_globals.lastYesState;
    aparm->lastYesClaim = vote_globals.lastYesClaim;
    aparm->lowestHost = ntohl(vote_globals.lowestHost);
    aparm->lowestTime = vote_globals.lowestTime;
    aparm->syncHost = ntohl(vote_globals.syncHost);
    aparm->syncTime = vote_globals.syncTime;
    memcpy(&aparm->syncVersion, &vote_globals.ubik_dbVersion, sizeof(struct ubik_version));
    memcpy(&aparm->syncTid, &vote_globals.ubik_dbTid, sizeof(struct ubik_tid));

    aparm->amSyncSite = beacon_globals.ubik_amSyncSite;
    ubeacon_Debug((ubik_debug *)aparm);

    udisk_Debug((ubik_debug *)aparm);

    ulock_Debug((ubik_debug *)aparm);

    /* Get the recovery state. The label of the database may not have
     * been written yet but set the flag so udebug behavior remains.
     * Defect 9477.
     */
    aparm->recoveryState = urecovery_state;
    if ((urecovery_state & UBIK_RECSYNCSITE)
	&& (urecovery_state & UBIK_RECFOUNDDB)
	&& (urecovery_state & UBIK_RECHAVEDB)) {
	aparm->recoveryState |= UBIK_RECLABELDB;
    }
    aparm->activeWrite = (ubik_dbase->flags & DBWRITING);
    aparm->tidCounter = ubik_dbase->tidCounter;

    if (ubik_currentTrans) {
	aparm->currentTrans = 1;
	aparm->writeTrans = 1;
    } else {
	aparm->currentTrans = 0;
    }

    aparm->epochTime = version_globals.ubik_epochTime;

    return 0;
}


/*!
 * \brief Get the sync site; called by remote servers to find where they should go.
 */
afs_int32
SVOTE_GetSyncSite(struct rx_call * rxcall,
		  afs_int32 * ahost)
{
    afs_int32 temp;

    temp = uvote_GetSyncSite();
    *ahost = ntohl(temp);
    return 0;
}

/*!
 * \brief Called once/run to init the vote module
 */
int
uvote_Init(void)
{
    UBIK_VOTE_LOCK;
    /* pretend we just voted for someone else, since we just restarted */
    vote_globals.ubik_lastYesTime = FT_ApproxTime();

    /* Initialize globals */
    vote_globals.lastYesHost = 0xffffffff;
    vote_globals.lastYesClaim = 0;
    vote_globals.lastYesState = 0;
    vote_globals.lowestTime = 0;
    vote_globals.lowestHost = 0xffffffff;
    vote_globals.syncTime = 0;
    vote_globals.syncHost = 0;
    UBIK_VOTE_UNLOCK;

    return 0;
}

void
uvote_set_dbVersion(struct ubik_version version) {
    UBIK_VOTE_LOCK;
    vote_globals.ubik_dbVersion = version;
    UBIK_VOTE_UNLOCK;
}

/* Compare given version to current DB version.  Return true if equal. */
int
uvote_eq_dbVersion(struct ubik_version version) {
    int ret = 0;

    UBIK_VOTE_LOCK;
    if (vote_globals.ubik_dbVersion.epoch == version.epoch && vote_globals.ubik_dbVersion.counter == version.counter) {
	ret = 1;
    }
    UBIK_VOTE_UNLOCK;
    return ret;
}

/*!
 * \brief Check if there is a sync site and whether we have a given db version
 *
 * \return 1 if there is a valid sync site, and the given db version matches the sync site's
 */

int
uvote_HaveSyncAndVersion(struct ubik_version version)
{
    afs_int32 now;
    int code;

    UBIK_VOTE_LOCK;
    now = FT_ApproxTime();
    if (!vote_globals.lastYesState || (SMALLTIME + vote_globals.lastYesClaim < now) ||
			vote_globals.ubik_dbVersion.epoch != version.epoch ||
			vote_globals.ubik_dbVersion.counter != version.counter)
	code = 0;
    else
	code = 1;
    UBIK_VOTE_UNLOCK;
    return code;
}
