equilibrium-data_monitor.c

Go to the documentation of this file.
00001 /*
00002  * $Id: equilibrium-data_monitor.c 346460 2009-11-14 05:06:47Z ssiano $
00003  *
00004  * This code is provided as is by Juniper Networks SDK Developer Support.
00005  * It is provided with no warranties or guarantees, and Juniper Networks
00006  * will not provide support or maintenance of this code in any fashion.
00007  * The code is provided only to help a developer better understand how
00008  * the SDK can be used.
00009  * 
00010  * Copyright (c) 2008, Juniper Networks, Inc.
00011  * All rights reserved.
00012  */
00013 
00021 #include "equilibrium-data_main.h"
00022 #include <sys/socket.h>
00023 #include <unistd.h>
00024 #include "equilibrium-data_config.h"
00025 #include "equilibrium-data_conn.h"
00026 #include "equilibrium-data_packet.h"
00027 #include "equilibrium-data_monitor.h"
00028 
00029 
00030 /*** Constants ***/
00031 
00035 #define LOGM(_level, _fmt...)   \
00036     LOG((_level), "Server Monitor: " _fmt)
00037 
00038 
00039 /*** Data structures ***/
00040 
00041 
00045 typedef struct mon_app_info_s mon_app_info_t;
00046 
00047 
00051 typedef struct mon_server_info_s {
00052     in_addr_t                server_addr;  
00053     uint32_t                 sessions;     
00054     int                      sock;         
00055     evTimerID                test_timer;   
00056     evConnID                 conn_id;      
00057     evFileID                 file_id;      
00058     mon_app_info_t           * app;        
00059     boolean                  is_up;        
00060     uint16_t                 timeouts;     
00061     TAILQ_ENTRY(mon_server_info_s) entries; 
00062 } mon_server_info_t;
00063 
00064 
00068 typedef TAILQ_HEAD(server_info_set_s, mon_server_info_s) server_info_set_t;
00069 
00070 
00074 typedef struct mon_app_key_s {
00075     uint16_t  svc_set_id;           
00076     in_addr_t app_addr;             
00077     uint16_t  app_port;             
00078 } mon_app_key_t;
00079 
00080 
00085 struct mon_app_info_s {
00086     patnode           node;  
00087     mon_app_key_t     key;                  
00088     msp_spinlock_t    app_lock;             
00089     eq_smon_t         * server_mon_params;  
00090     server_info_set_t * up_servers;         
00091     server_info_set_t * down_servers;       
00092 };
00093 
00094 
00095 static patroot apps;        
00096 static boolean doShutdown;  
00097 static evContext mon_ctx;   
00098 static msp_spinlock_t apps_big_lock; 
00099 
00100 /*** STATIC/INTERNAL Functions ***/
00101 
00102 
00107 PATNODE_TO_STRUCT(app_entry, mon_app_info_t, node)
00108 
00109 
00110 
00113 static void
00114 probe_server(evContext c, void * u, struct timespec d, struct timespec i);
00115 
00116 
00124 static void
00125 stop_server_probes(mon_server_info_t * server)
00126 {
00127     if(evTestID(server->test_timer)) {
00128         evClearTimer(mon_ctx, server->test_timer);
00129         evInitID(&server->test_timer);
00130     }
00131     
00132     if(evTestID(server->file_id)) {
00133         evDeselectFD(mon_ctx, server->file_id);
00134         evInitID(&server->file_id);
00135     }
00136 
00137     if(evTestID(server->conn_id)) {
00138         evCancelConn(mon_ctx, server->conn_id);
00139         evInitID(&server->conn_id);
00140     }
00141 
00142     if(server->sock != -1 && close(server->sock) != 0) {
00143        LOGM(LOG_ERR, "%s: Failed to close socket to server: %m", __func__);
00144     }
00145 
00146     server->sock = -1;
00147 }
00148 
00149 
00158 static void
00159 server_probe_failed(mon_server_info_t * server)
00160 {
00161     struct in_addr addr;
00162     mon_app_info_t * app = server->app;
00163     
00164     stop_server_probes(server);
00165     
00166     // check if the timeouts have surpassed the allowed number 
00167     // and we mark the server down if so
00168     if(server->is_up && ++(server->timeouts) <= 
00169             app->server_mon_params->timeouts_allowed) {
00170 
00171         // still ok, so schedule next probe to server with connection_interval
00172         
00173         if(evSetTimer(mon_ctx, probe_server, server, evAddTime(evNowTime(),
00174             evConsTime(app->server_mon_params->connection_interval, 0)),
00175             evConsTime(0, 0), &server->test_timer)) {
00176 
00177             addr.s_addr = server->server_addr; // for inet_ntoa
00178             LOGM(LOG_EMERG, "%s: Failed to initialize a probe timer to probe "
00179                 "server %s (Error: %m)", __func__, inet_ntoa(addr));
00180         }
00181     } else {
00182         // server is down, use down_retry_interval
00183         
00184         if(evSetTimer(mon_ctx, probe_server, server, evAddTime(evNowTime(),
00185             evConsTime(app->server_mon_params->down_retry_interval, 0)),
00186             evConsTime(0, 0), &server->test_timer)) {
00187 
00188             addr.s_addr = server->server_addr; // for inet_ntoa
00189             LOGM(LOG_EMERG, "%s: Failed to initialize a probe timer to probe "
00190                 "server %s (Error: %m)", __func__, inet_ntoa(addr));
00191         }
00192         
00193         // it is was previously up, take it down
00194         if(server->is_up) {
00195             server->is_up = FALSE;
00196             server->sessions = 0;
00197             TAILQ_REMOVE(app->up_servers, server, entries);
00198             TAILQ_INSERT_HEAD(app->down_servers, server, entries);
00199             
00200             notify_server_status(app->key.svc_set_id, app->key.app_addr,
00201                     app->key.app_port, server->server_addr, SERVER_STATUS_DOWN);
00202             
00203             clean_sessions_using_server(app->key.svc_set_id,
00204                     app->key.app_addr, app->key.app_port, server->server_addr);
00205         }
00206     }
00207 }
00208 
00209 
00225 static void
00226 http_read(evContext ctx __unused,
00227           void * uap,
00228           int fd __unused,
00229           int evmask __unused)
00230 {
00231     const uint8_t BUF_LEN  = 64;
00232     mon_server_info_t * server = (mon_server_info_t *)uap;
00233     mon_app_info_t * app;
00234     struct in_addr addr;
00235     char buf[BUF_LEN];
00236     int rc;
00237     
00238     INSIST_ERR(server != NULL);
00239     app = server->app;
00240     bzero(buf, BUF_LEN);
00241     
00242     // Get application lock
00243     INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
00244     
00245     if(app->server_mon_params == NULL) {
00246         addr.s_addr = server->server_addr; // for inet_ntoa
00247         // Release application lock
00248         INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
00249         LOGM(LOG_ERR, "%s: Request to read HTTP response from %s in an "
00250                 "application with no server monitoring parameters",
00251                 __func__, inet_ntoa(addr));
00252         return;
00253     }
00254     
00255     addr.s_addr = server->server_addr; // for inet_ntoa
00256 
00257     // Validate that it is an HTTP response
00258     if((rc = recv(server->sock, buf, BUF_LEN - 1, 0)) < 0) {
00259         
00260         LOGM(LOG_WARNING, "%s: Probe to server %s did not get an HTTP "
00261             "response (Error: %m)", __func__, inet_ntoa(addr));
00262         
00263         server_probe_failed(server);
00264         
00265         // Release application lock
00266         INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
00267         return;
00268         
00269     } else {
00270         if(rc < 3) {
00271             LOGM(LOG_WARNING, "%s: Probe to server %s did not get a long enough"
00272                     "response to tell if it was an HTTP response (Content: %s)",
00273                     __func__, inet_ntoa(addr), buf);
00274             
00275             server_probe_failed(server);
00276             
00277             // Release application lock
00278             INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
00279             return;
00280             
00281         } else if(strnstr(buf, "HTTP", 4) == NULL) {
00282 
00283             LOGM(LOG_WARNING, "%s: Probe to server %s did not get an HTTP "
00284                 "response (Content: %s)", __func__, inet_ntoa(addr), buf);
00285             
00286             server_probe_failed(server);
00287             
00288             // Release application lock
00289             INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
00290             return;
00291         }
00292     }
00293     
00294     LOGM(LOG_INFO, "%s: Probe to server %s returned an HTTP response",
00295             __func__, inet_ntoa(addr));
00296     
00297     stop_server_probes(server);
00298     
00299     // server is ok, so schedule next probe to server with connection_interval
00300     if(evSetTimer(mon_ctx, probe_server, server, evAddTime(evNowTime(),
00301             evConsTime(app->server_mon_params->connection_interval, 0)),
00302             evConsTime(0, 0), &server->test_timer)) {
00303 
00304         addr.s_addr = server->server_addr; // for inet_ntoa
00305         LOGM(LOG_EMERG, "%s: Failed to initialize a probe timer to probe "
00306             "server %s (Error: %m)", __func__, inet_ntoa(addr));
00307         
00308         // Release application lock
00309         INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
00310         return;
00311     }
00312     
00313     // it is was previously down, take it up
00314     if(!server->is_up) {
00315         server->is_up = TRUE;
00316         server->sessions = 0;
00317         TAILQ_REMOVE(app->down_servers, server, entries);
00318         TAILQ_INSERT_HEAD(app->up_servers, server, entries);
00319         
00320         notify_server_status(server->app->key.svc_set_id,
00321                 server->app->key.app_addr, server->app->key.app_port,
00322                 server->server_addr, SERVER_STATUS_UP);
00323     }
00324     
00325     server->timeouts = 0; // reset
00326     
00327     // Release application lock
00328     INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
00329 }
00330 
00331 
00332 
00357 static void
00358 http_connect(evContext ctx __unused, void * uap, int fd,
00359              const void *la __unused, int lalen __unused,
00360              const void *ra __unused, int ralen __unused)
00361 {
00362     const char * GET_REQ = "GET / HTTP/1.1\nHost: %s\n\n";
00363     const uint8_t BUF_LEN = 255;
00364     char buf[BUF_LEN];
00365     mon_server_info_t * server;
00366     mon_app_info_t * app;
00367     struct in_addr addr;
00368     
00369     server = (mon_server_info_t *)uap;
00370     INSIST_ERR(server != NULL);
00371     
00372     app = server->app;
00373     
00374     bzero(buf, BUF_LEN);
00375     
00376     // Get application lock
00377     INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
00378 
00379     addr.s_addr = server->server_addr; // for inet_ntoa
00380     
00381     if(app->server_mon_params == NULL) {
00382         // Release application lock
00383         INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
00384         LOGM(LOG_ERR, "%s: Request to read HTTP response from %s in an "
00385                 "application with no server monitoring parameters",
00386                 __func__, inet_ntoa(addr));
00387         return;
00388     }
00389     
00390     evInitID(&server->conn_id); // reset this
00391     
00392     if(fd == -1) { // error occured and socket has been closed
00393         server->sock = -1;
00394         server_probe_failed(server);
00395         // Release application lock
00396         INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
00397         
00398         LOGM(LOG_ERR, "%s: Connection to server %s failed (Error %m)",
00399                 __func__, inet_ntoa(addr));
00400         return;
00401     }
00402     
00403     LOGM(LOG_INFO, "%s: Connected to server %s. Sending HTTP Request...",
00404             __func__, inet_ntoa(addr));
00405     
00406     // setup reading callback
00407     evInitID(&server->file_id);
00408     if(evSelectFD(mon_ctx, server->sock, EV_READ, http_read, server,
00409             &server->file_id)) {
00410         
00411         LOGM(LOG_ERR, "%s: evSelectFD failed (Error: %m)", __func__);
00412         server_probe_failed(server);
00413 
00414     } else {
00415         // Send HTTP GET Request
00416         
00417         sprintf(buf, GET_REQ, inet_ntoa(addr));
00418         if(send(server->sock, buf, strlen(buf), 0) == -1) {
00419 
00420             LOGM(LOG_ERR, "%s: failed to send HTTP request to %s (Error: %m)",
00421                     __func__, inet_ntoa(addr));
00422             server_probe_failed(server);
00423         } else {
00424             LOGM(LOG_INFO, "%s: Sent HTTP Request to %s",
00425                 __func__, inet_ntoa(addr));
00426         }
00427     }
00428     
00429     // Release application lock
00430     INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
00431 }
00432 
00433 
00449 static void
00450 probe_timeout(evContext ctx __unused,
00451             void * uap,
00452             struct timespec due __unused,
00453             struct timespec inter __unused)
00454 {
00455     mon_server_info_t * server = (mon_server_info_t *)uap;
00456     mon_app_info_t * app;
00457     struct in_addr addr;
00458     
00459     INSIST_ERR(server != NULL);
00460     app = server->app;
00461     
00462     // Get application lock
00463     INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
00464 
00465     addr.s_addr = server->server_addr; // for inet_ntoa
00466     
00467     if(app->server_mon_params == NULL) {
00468         // Release application lock
00469         INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
00470         LOGM(LOG_ERR, "%s: Probe timeout for server %s in an application with "
00471             "no server monitoring parameters", __func__, inet_ntoa(addr));
00472         return;
00473     }
00474 
00475     LOGM(LOG_INFO, "%s: Probe timeout for server %s",
00476             __func__, inet_ntoa(addr));
00477     
00478     server_probe_failed(server);
00479     
00480     // Release application lock
00481     INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
00482 }
00483 
00484 
00494 static status_t
00495 init_http_connection(mon_server_info_t * server)
00496 {
00497     // for socket options
00498     const int sndbuf = 1024;           // send buffer
00499     const int on = 1;                  // to turn an option on
00500     
00501     struct sockaddr_in server_addr;
00502     struct in_addr addr;
00503     
00504     if(server->sock != -1) {
00505         LOGM(LOG_ERR, "%s: socket fd for server is already open", __func__);
00506         return EFAIL;
00507     }
00508     
00509     server->sock = socket(AF_INET, SOCK_STREAM, 0);
00510     if(server->sock == -1) {
00511         LOGM(LOG_ERR, "%s: socket() failed: %m", __func__);
00512         return EFAIL;        
00513     }
00514 
00515     // Set some socket options
00516 
00517     if(setsockopt(server->sock, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on))) {
00518         LOGM(LOG_ERR, "%s: setsockopt SO_REUSEADDR failed: %m", __func__);
00519         close(server->sock);
00520         server->sock = -1;
00521         return EFAIL;
00522     }
00523     if(setsockopt(server->sock, SOL_SOCKET, SO_SNDBUF, &sndbuf,sizeof(sndbuf))){
00524         LOGM(LOG_ERR, "%s: setsockopt SO_SNDBUF failed: %m", __func__);
00525         close(server->sock);
00526         server->sock = -1;
00527         return EFAIL;
00528     }
00529 
00530     bzero(&server_addr, sizeof(struct sockaddr_in));
00531     server_addr.sin_len = sizeof(struct sockaddr_in);
00532     server_addr.sin_family = AF_INET;
00533     server_addr.sin_port = server->app->key.app_port;     // in nw byte order
00534     server_addr.sin_addr.s_addr = server->server_addr;    // in nw byte order
00535 
00536     addr.s_addr = server->server_addr; // for inet_ntoa
00537     LOGM(LOG_INFO,"%s: Connecting to server %s", __func__, inet_ntoa(addr));
00538     
00539     // setup reading callback
00540     evInitID(&server->conn_id);
00541     if(evConnect(mon_ctx, server->sock, (struct sockaddr *)&server_addr,
00542         sizeof(struct sockaddr_in), http_connect, server, &server->conn_id)) {
00543         
00544         LOGM(LOG_ERR, "%s: Connecting to server %s failed (Error %m)",
00545                 __func__, inet_ntoa(addr));
00546         close(server->sock);
00547         server->sock = -1;
00548         return EFAIL;
00549     }
00550     
00551     // schedule timeout for probe to server
00552     if(evSetTimer(mon_ctx, probe_timeout, server, evAddTime(evNowTime(),
00553             evConsTime(server->app->server_mon_params->connection_timeout, 0)),
00554             evConsTime(0, 0), &server->test_timer)) {
00555         
00556         LOGM(LOG_EMERG, "%s: Failed to initialize a probe timeout timer for "
00557             "probe to server %s (Error: %m)", __func__, inet_ntoa(addr));
00558     }
00559     
00560     return SUCCESS;
00561 }
00562 
00563 
00579 static void
00580 probe_server(evContext ctx __unused,
00581             void * uap,
00582             struct timespec due __unused,
00583             struct timespec inter __unused)
00584 {
00585     mon_server_info_t * server = (mon_server_info_t *)uap;
00586     mon_app_info_t * app;
00587     struct in_addr addr;
00588     
00589     INSIST_ERR(server != NULL);
00590     app = server->app;
00591     
00592     // Get application lock
00593     INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
00594     
00595     if(evTestID(server->test_timer)) {
00596         evClearTimer(mon_ctx, server->test_timer);
00597         evInitID(&server->test_timer);
00598     }
00599     
00600     if(app->server_mon_params == NULL) {
00601         addr.s_addr = server->server_addr; // for inet_ntoa
00602         // Release application lock
00603         INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
00604         LOGM(LOG_ERR, "%s: Request to probe server %s in an application with no "
00605             "server monitoring parameters", __func__, inet_ntoa(addr));
00606         return;
00607     }
00608     
00609     if(init_http_connection(server) != SUCCESS) {
00610         server_probe_failed(server);
00611     }
00612     
00613     // Release application lock
00614     INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
00615 }
00616 
00617 
00633 static void
00634 keep_alive(evContext ctx __unused,
00635            void * uap __unused,
00636            struct timespec due __unused,
00637            struct timespec inter __unused)
00638 {
00639 
00640 }
00641 
00642 
00651 static void *
00652 start_monitor(void * params __unused)
00653 {
00654     const int KEEP_ALIVE = 5; //900;
00655     evEvent event;
00656     int rc = 0;
00657     
00658     INSIST_ERR(evCreate(&mon_ctx) != -1);
00659     
00660     // Give the context a dummy event to keep it alive
00661     if(evSetTimer(mon_ctx, keep_alive, NULL, evConsTime(0, 0),
00662             evConsTime(KEEP_ALIVE, 0), NULL)) {
00663         LOGM(LOG_EMERG, "%s: Failed to initialize main loop's keep alive timer",
00664             __func__);
00665         return NULL;
00666     }
00667     
00668     // Go to work, wait for events
00669     // This is exactly like evMainLoop except for doShutdown
00670     while (!doShutdown && ((rc = evGetNext(mon_ctx, &event, EV_WAIT)) == 0)) {
00671         if((rc = evDispatch(mon_ctx, event)) < 0) {
00672             break;
00673         }
00674     }
00675     
00676     evDestroy(mon_ctx);
00677     
00678     if(doShutdown) {
00679         LOGM(LOG_INFO, "Shutting down");
00680     } else {
00681         LOGM(LOG_ERR, "Unexpected exit from main loop: %d", rc);
00682     }
00683     
00684     return NULL;
00685 }
00686 
00687 
00688 /*** GLOBAL/EXTERNAL Functions ***/
00689 
00690 
00703 status_t
00704 init_monitor(evContext ctx, int cpu)
00705 {
00706     pthread_attr_t attr;
00707     pthread_t tid;
00708     
00709     msp_spinlock_init(&apps_big_lock);
00710     
00711     patricia_root_init(&apps, FALSE, sizeof(mon_app_key_t), 0);
00712                    // root, is key ptr, key size, key offset
00713     
00714     if(cpu == MSP_NEXT_END) {
00715         // There's no other user CPU to bind to, so use whatever we are
00716         // currently bound to  
00717         mon_ctx = ctx; // use the main event context
00718         return SUCCESS;
00719     }
00720     
00721     // create a new thread for the monitor and bind it to the available user CPU
00722     
00723     LOG(LOG_INFO, "%s: Starting monitor on user cpu %d", __func__, cpu);
00724     
00725     doShutdown = FALSE;
00726     
00727     // Set up pthread attributes
00728     pthread_attr_init(&attr);
00729     
00730     // schedule with respect to all threads in the system, not process (default)
00731     if(pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM)) {
00732         LOG(LOG_ERR, "%s: pthread_attr_setscope(PTHREAD_SCOPE_SYSTEM) failed",
00733                 __func__);
00734         pthread_attr_destroy(&attr);
00735         return EFAIL;
00736     }
00737     
00738     // bind monitor thread to the user CPU
00739     if(pthread_attr_setcpuaffinity_np(&attr, cpu)) {
00740         LOG(LOG_ERR, "%s: pthread_attr_setcpuaffinity_np on CPU %d failed",
00741                 __func__, cpu);
00742         pthread_attr_destroy(&attr);
00743         return EFAIL;
00744     }
00745     
00746     // create and start thread
00747     if(pthread_create(&tid, &attr, start_monitor, NULL)) {
00748         LOG(LOG_ERR, "%s: pthread_create() on CPU %d failed",
00749                 __func__, cpu);
00750         pthread_attr_destroy(&attr);
00751         return EFAIL;
00752     }
00753     
00754     pthread_attr_destroy(&attr);
00755     
00756     return SUCCESS;
00757 }
00758 
00759 
00763 void
00764 shutdown_monitor(void)
00765 {
00766     mon_app_info_t * app;
00767     mon_server_info_t * server;
00768     
00769     doShutdown = TRUE;
00770     
00771     // delete all applications
00772     
00773     // Get big lock
00774     INSIST_ERR(msp_spinlock_lock(&apps_big_lock) == MSP_OK);
00775    
00776     while((app = app_entry(patricia_find_next(&apps, NULL))) != NULL) {
00777         
00778         // Get application lock
00779         INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
00780         
00781         while((server = TAILQ_FIRST(app->up_servers)) != NULL) {
00782             stop_server_probes(server);
00783             TAILQ_REMOVE(app->up_servers, server, entries);
00784             free(server);
00785         }
00786         free(app->up_servers);
00787         
00788         while((server = TAILQ_FIRST(app->down_servers)) != NULL) {
00789             stop_server_probes(server);
00790             TAILQ_REMOVE(app->up_servers, server, entries);
00791             free(server);
00792         }
00793         free(app->down_servers);
00794         
00795         patricia_delete(&apps, &app->node);
00796         
00797         free(app);
00798     }
00799     
00800     // Release big lock
00801     INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
00802 }
00803 
00804 
00825 void
00826 monitor_add_server(uint16_t ss_id,
00827                    in_addr_t app_addr,
00828                    uint16_t app_port,
00829                    in_addr_t server_addr,
00830                    eq_smon_t * monitor)
00831 {
00832     mon_app_key_t key;
00833     mon_app_info_t * app;
00834     mon_server_info_t * server;
00835     struct in_addr addr;
00836     
00837     bzero(&key, sizeof(mon_app_key_t));
00838     key.svc_set_id = ss_id;
00839     key.app_addr = app_addr;
00840     key.app_port = app_port;
00841     
00842     // Get big lock
00843     INSIST_ERR(msp_spinlock_lock(&apps_big_lock) == MSP_OK);
00844     
00845     // get the application
00846     app = app_entry(patricia_get(&apps, sizeof(key), &key));
00847     
00848     if(app == NULL) { // new app
00849         
00850         app = calloc(1, sizeof(mon_app_info_t));
00851         INSIST_ERR(app != NULL);
00852         
00853         app->key.svc_set_id = ss_id;
00854         app->key.app_addr = app_addr;
00855         app->key.app_port = app_port;
00856         
00857         // init a bit
00858         msp_spinlock_init(&app->app_lock);
00859         // Get application lock
00860         INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
00861 
00862         if(!patricia_add(&apps, &app->node)) {
00863             // Release big lock
00864             INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
00865             
00866             LOG(LOG_ERR, "%s: Failed to add application to monitor config",
00867                     __func__);
00868             free(app);
00869             return;
00870         }
00871         
00872         // Release big lock
00873         INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
00874         
00875         app->server_mon_params = monitor;
00876         
00877         app->down_servers = malloc(sizeof(server_info_set_t));
00878         INSIST_ERR(app->down_servers != NULL);
00879         TAILQ_INIT(app->down_servers);
00880         
00881         app->up_servers = malloc(sizeof(server_info_set_t));
00882         INSIST_ERR(app->up_servers != NULL);
00883         TAILQ_INIT(app->up_servers);
00884         
00885     } else { // app exists already
00886         
00887         // Get application lock
00888         INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
00889         
00890         // Release big lock
00891         INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
00892         
00893         
00894         // check for the server in one of the sets
00895         if(app->up_servers) {
00896             server = TAILQ_FIRST(app->up_servers);
00897             while(server != NULL) {
00898                 if(server->server_addr == server_addr) {
00899                     LOG(LOG_ERR, "%s: Server already exists in monitor config",
00900                             __func__);
00901                     // Release application lock
00902                     INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
00903                     return;
00904                 }
00905                 server = TAILQ_NEXT(server, entries);            
00906             }
00907         }
00908         
00909         // check for the server in one of the sets
00910         if(app->down_servers) {
00911             server = TAILQ_FIRST(app->down_servers);
00912             while(server != NULL) {
00913                 if(server->server_addr == server_addr) {
00914                     LOG(LOG_ERR, "%s: Server already exists in monitor config",
00915                             __func__);
00916                     // Release application lock
00917                     INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
00918                     return;
00919                 }
00920                 server = TAILQ_NEXT(server, entries);            
00921             }
00922         }
00923         
00924         // check monitor
00925         if(monitor != app->server_mon_params) {
00926             // change_monitor_config should have been used first
00927             LOG(LOG_ERR, "%s: monitor configuration does not match existing "
00928                     "monitor configuration for this application", __func__);
00929             // Release application lock
00930             INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
00931             return;
00932         }
00933     }
00934     
00935     // add the server
00936     server = calloc(1, sizeof(mon_server_info_t));
00937     INSIST_ERR(server != NULL);
00938     server->server_addr = server_addr;
00939     server->app = app;
00940     server->sock = -1;
00941     
00942     // if monitor is NULL then it is assumed up, otherwise it starts down
00943     if(monitor != NULL) {
00944         server->is_up = FALSE;
00945         TAILQ_INSERT_TAIL(app->down_servers, server, entries);
00946         
00947         // schedule probe to server immediately
00948         if(evSetTimer(mon_ctx, probe_server, server, evConsTime(0, 0),
00949                 evConsTime(0, 0), &server->test_timer)) {
00950             addr.s_addr = server_addr; // for inet_ntoa
00951             LOG(LOG_EMERG, "%s: Failed to initialize a probe timer to "
00952                 "probe server %s (Error: %m)", __func__, inet_ntoa(addr));
00953         }
00954     } else {
00955         server->is_up = TRUE;
00956         // insert at the head since it is sorted by server->sessions
00957         TAILQ_INSERT_HEAD(app->up_servers, server, entries);
00958         
00959         addr.s_addr = server_addr; // for inet_ntoa
00960         LOG(LOG_INFO, "%s: Added (permanently UP) server %s",
00961                 __func__, inet_ntoa(addr));
00962         
00963         // no probe to setup
00964     }
00965     
00966     // Release application lock
00967     INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
00968 }
00969 
00970 
00986 void
00987 monitor_remove_server(uint16_t ss_id,
00988                       in_addr_t app_addr,
00989                       uint16_t app_port,
00990                       in_addr_t server_addr)
00991 {
00992     boolean found = FALSE;
00993     mon_app_key_t key;
00994     mon_app_info_t * app;
00995     mon_server_info_t * server;
00996     
00997     bzero(&key, sizeof(mon_app_key_t));
00998     key.svc_set_id = ss_id;
00999     key.app_addr = app_addr;
01000     key.app_port = app_port;
01001     
01002     // Get big lock
01003     INSIST_ERR(msp_spinlock_lock(&apps_big_lock) == MSP_OK);
01004     
01005     // get the application
01006     app = app_entry(patricia_get(&apps, sizeof(key), &key));
01007     
01008     if(app == NULL) {
01009         // Release big lock
01010         INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
01011         
01012         LOG(LOG_ERR, "%s: Failed to get application in the monitor config",
01013                 __func__);
01014         return;        
01015     }
01016     
01017     // Get application lock
01018     INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
01019     
01020     // Release big lock
01021     INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
01022     
01023     // check for the server in one of the sets
01024     server = TAILQ_FIRST(app->up_servers);
01025     while(server != NULL) {
01026         if(server->server_addr == server_addr) {
01027             found = TRUE;
01028             TAILQ_REMOVE(app->up_servers, server, entries);
01029             break;
01030         }
01031         server = TAILQ_NEXT(server, entries);            
01032     }
01033     
01034     if(!found) {
01035         server = TAILQ_FIRST(app->down_servers);
01036         while(server != NULL) {
01037             if(server->server_addr == server_addr) {
01038                 found = TRUE;
01039                 TAILQ_REMOVE(app->down_servers, server, entries);
01040                 break;
01041             }
01042             server = TAILQ_NEXT(server, entries);            
01043         }
01044         if(!found) { // still not here
01045             // Release application lock
01046             INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
01047             
01048             LOG(LOG_ERR, "%s: Failed to find the server in the monitor config",
01049                     __func__);
01050             return;
01051         }
01052     }
01053     
01054     stop_server_probes(server);
01055     
01056     clean_sessions_using_server(app->key.svc_set_id,
01057             app->key.app_addr, app->key.app_port, server->server_addr);
01058     
01059     free(server);
01060     
01061     // check if we have any servers left
01062     if(TAILQ_EMPTY(app->up_servers) && TAILQ_EMPTY(app->down_servers)) {
01063         // no servers left, delete app config
01064         
01065         // preserve lock ordering chain of locks (big to small/app)...
01066         INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
01067         INSIST_ERR(msp_spinlock_lock(&apps_big_lock) == MSP_OK);
01068         INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
01069         
01070         if(!patricia_delete(&apps, &app->node)) {
01071             // Release big lock
01072             INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
01073             LOG(LOG_ERR, "%s: Failed to remove app from monitor config",
01074                     __func__);
01075             return;
01076         }
01077 
01078         // Release big lock
01079         INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
01080         
01081         free(app->down_servers);
01082         free(app->up_servers);
01083         free(app);
01084     } else {
01085         // Release application lock
01086         INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
01087     }
01088 }
01089 
01090 
01103 void
01104 monitor_remove_all_servers_in_app(uint16_t ss_id,
01105                                   in_addr_t app_addr,
01106                                   uint16_t app_port)
01107 {
01108     mon_app_key_t key;
01109     mon_app_info_t * app;
01110     mon_server_info_t * server;
01111     
01112     bzero(&key, sizeof(mon_app_key_t));
01113     key.svc_set_id = ss_id;
01114     key.app_addr = app_addr;
01115     key.app_port = app_port;
01116     
01117     // Get big lock
01118     INSIST_ERR(msp_spinlock_lock(&apps_big_lock) == MSP_OK);
01119     
01120     // get the application
01121     app = app_entry(patricia_get(&apps, sizeof(key), &key));
01122     
01123     if(app == NULL) {
01124         // Release big lock
01125         INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
01126         
01127         LOG(LOG_ERR, "%s: Failed to get application in the monitor config",
01128                 __func__);
01129         return;        
01130     }
01131     
01132     // Get application lock
01133     INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
01134     
01135     // get rid of this app from the monitor's config, first so we can unlock
01136     if(!patricia_delete(&apps, &app->node)) {
01137         // Release both locks
01138         INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
01139         INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
01140         
01141         LOG(LOG_ERR, "%s: Failed to remove app from monitor config",
01142                 __func__);
01143         return;
01144     }
01145 
01146     // Release big lock
01147     INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
01148     
01149     // empty both sets (up and down) deleting all servers
01150     while((server = TAILQ_FIRST(app->up_servers)) != NULL) {
01151         stop_server_probes(server);
01152         TAILQ_REMOVE(app->up_servers, server, entries);
01153         free(server);
01154     }
01155     while((server = TAILQ_FIRST(app->down_servers)) != NULL) {
01156         stop_server_probes(server);
01157         TAILQ_REMOVE(app->down_servers, server, entries);
01158         free(server);
01159     }
01160     
01161     clean_sessions_with_app(
01162             app->key.svc_set_id, app->key.app_addr, app->key.app_port);
01163     
01164     free(app->down_servers);
01165     free(app->up_servers);
01166     free(app);
01167 }
01168 
01169 
01176 void
01177 monitor_remove_all_servers_in_service_set(uint16_t ss_id)
01178 {
01179     mon_app_info_t * app;
01180     mon_server_info_t * server;
01181     
01182     struct tmp_app_s {
01183         mon_app_info_t         * app;
01184         TAILQ_ENTRY(tmp_app_s) entries;
01185     }  * tmp_app;
01186 
01187     TAILQ_HEAD(, tmp_app_s) app_list = TAILQ_HEAD_INITIALIZER(app_list);
01188     
01189     // Get big lock
01190     INSIST_ERR(msp_spinlock_lock(&apps_big_lock) == MSP_OK);
01191     
01192     // go thru all the applications with this service set id
01193     while((app = app_entry(patricia_subtree_match(
01194             &apps, sizeof(uint16_t)*8, &ss_id))) != NULL) {
01195 
01196         // Get application lock
01197         INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
01198         
01199         // get rid of this app from the monitor's config
01200         if(!patricia_delete(&apps, &app->node)) {
01201             // Release app lock
01202             INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
01203             LOG(LOG_ERR, "%s: Failed to remove app from monitor config",
01204                     __func__);
01205         }
01206         
01207         tmp_app = malloc(sizeof(struct tmp_app_s));
01208         INSIST_ERR(tmp_app != NULL);
01209         tmp_app->app = app;
01210         TAILQ_INSERT_TAIL(&app_list, tmp_app, entries);
01211     }
01212 
01213     // Release big lock
01214     INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
01215     
01216     if(!TAILQ_EMPTY(&app_list)) {
01217         clean_sessions_with_service_set(ss_id);
01218     }
01219     
01220     // delete all apps in app_list
01221     while((tmp_app = TAILQ_FIRST(&app_list)) != NULL) {
01222         
01223         app = tmp_app->app;
01224         
01225         // empty both sets (up and down) deleting all servers
01226         while((server = TAILQ_FIRST(app->up_servers)) != NULL) {
01227             stop_server_probes(server);
01228             TAILQ_REMOVE(app->up_servers, server, entries);
01229             free(server);
01230         }
01231         while((server = TAILQ_FIRST(app->down_servers)) != NULL) {
01232             stop_server_probes(server);
01233             TAILQ_REMOVE(app->down_servers, server, entries);
01234             free(server);
01235         }
01236         
01237         TAILQ_REMOVE(&app_list, tmp_app, entries);
01238         free(app->down_servers);
01239         free(app->up_servers);
01240         free(app);
01241         free(tmp_app);
01242     }
01243 }
01244 
01245 
01264 void
01265 change_monitoring_config(uint16_t ss_id,
01266                          in_addr_t app_addr,
01267                          uint16_t app_port,
01268                          eq_smon_t * monitor)
01269 {
01270     mon_app_key_t key;
01271     mon_app_info_t * app;
01272     mon_server_info_t * server, * tmp;
01273     struct in_addr addr;
01274     
01275     bzero(&key, sizeof(mon_app_key_t));
01276     key.svc_set_id = ss_id;
01277     key.app_addr = app_addr;
01278     key.app_port = app_port;
01279     
01280     // Get big lock
01281     INSIST_ERR(msp_spinlock_lock(&apps_big_lock) == MSP_OK);
01282     
01283     // get the application
01284     app = app_entry(patricia_get(&apps, sizeof(key), &key));
01285     
01286     if(app == NULL) {
01287         // Release big lock
01288         INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
01289         
01290         LOG(LOG_ERR, "%s: Failed to get application in the monitor config",
01291                 __func__);
01292         return;
01293     }
01294     
01295     // Get application lock
01296     INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
01297     
01298     // Release big lock
01299     INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
01300     
01301     
01302     // switch the server monitoring parameters
01303     // (we don't need to free since the config.c module holds the mem for this)
01304     
01305     app->server_mon_params = monitor;
01306     
01307     // Go through servers, cancel the existing timers and setup the new monitor
01308     // Leaving the servers in whatever set they were already in seems fine and
01309     // causes less disruption
01310     
01311     server = TAILQ_FIRST(app->up_servers);
01312     while(server != NULL) {
01313         // cancel anything currently going on
01314         stop_server_probes(server);
01315         
01316         if(monitor != NULL) {
01317             // schedule probe to server immediately
01318             if(evSetTimer(mon_ctx, probe_server, server, evConsTime(0, 0),
01319                     evConsTime(0, 0), &server->test_timer)) {
01320                 addr.s_addr = server->server_addr; // for inet_ntoa
01321                 LOG(LOG_EMERG, "%s: Failed to initialize a probe timer to probe "
01322                     "server %s (Error: %m)", __func__, inet_ntoa(addr));
01323             }
01324         }
01325         server = TAILQ_NEXT(server, entries);            
01326     }
01327     
01328     server = TAILQ_FIRST(app->down_servers);
01329     while(server != NULL) {
01330         
01331         // cancel anything currently going on
01332         stop_server_probes(server);
01333         
01334         if(monitor != NULL) {
01335             
01336             // schedule probe to server immediately
01337             if(evSetTimer(mon_ctx, probe_server, server, evConsTime(0, 0),
01338                     evConsTime(0, 0), &server->test_timer)) {
01339                 addr.s_addr = server->server_addr; // for inet_ntoa
01340                 LOG(LOG_EMERG, "%s: Failed to initialize a probe timer to probe "
01341                     "server %s (Error: %m)", __func__, inet_ntoa(addr));
01342             }
01343             
01344             server = TAILQ_NEXT(server, entries);
01345 
01346         } else { // no monitor so assume everything is up
01347             
01348             tmp = TAILQ_NEXT(server, entries); // save next
01349             TAILQ_REMOVE(app->down_servers, server, entries);
01350             
01351             // insert at head since sessions should be zero
01352             // (and up_servers is sorted by sessions)
01353             server->sessions = 0; // enforce
01354             server->is_up = TRUE;
01355             TAILQ_INSERT_HEAD(app->up_servers, server, entries);
01356             
01357             addr.s_addr = server->server_addr; // for inet_ntoa
01358             LOG(LOG_INFO, "%s: Added (permanently UP) server %s",
01359                     __func__, inet_ntoa(addr));
01360             
01361             server = tmp; // copy back next server
01362         }    
01363     }
01364     
01365     // Release application lock
01366     INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
01367 }
01368 
01369 
01386 in_addr_t
01387 monitor_get_server_for(uint16_t ss_id,
01388                        in_addr_t app_addr,
01389                        uint16_t app_port)
01390 {
01391     mon_app_key_t key;
01392     mon_app_info_t * app;
01393     mon_server_info_t * server, *tmp, *tmp2;
01394     
01395     bzero(&key, sizeof(mon_app_key_t));
01396     key.svc_set_id = ss_id;
01397     key.app_addr = app_addr;
01398     key.app_port = app_port;
01399     
01400     // Get big lock
01401     INSIST_ERR(msp_spinlock_lock(&apps_big_lock) == MSP_OK);
01402     
01403     // get the application
01404     app = app_entry(patricia_get(&apps, sizeof(key), &key));
01405     
01406     if(app == NULL) {
01407         // Release big lock
01408         INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
01409         return 0;
01410     }
01411     
01412     // Get application lock
01413     INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
01414     
01415     // Release big lock
01416     INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
01417     
01418    
01419     // Get the first server, since it should have the least load
01420     
01421     server = TAILQ_FIRST(app->up_servers);
01422     if(server == NULL) {
01423         return (in_addr_t)-1;
01424     }
01425     ++server->sessions;
01426     
01427     // Go through the up servers and preserve the ordering by load 
01428     
01429     tmp2 = tmp = TAILQ_NEXT(server, entries);
01430     while(tmp != NULL) {
01431         if(tmp->sessions < server->sessions) {
01432             tmp = TAILQ_NEXT(tmp, entries);
01433         } else {
01434             if(tmp != tmp2) {
01435                 // insert server here
01436                 TAILQ_REMOVE(app->up_servers, server, entries);
01437                 TAILQ_INSERT_BEFORE(tmp, server, entries);
01438             }
01439             // else don't need to move it
01440             break;
01441         }
01442     }
01443     // if we went through all of them
01444     if(tmp == NULL) {
01445         // insert server at start
01446         TAILQ_REMOVE(app->up_servers, server, entries);
01447         TAILQ_INSERT_TAIL(app->up_servers, server, entries);
01448     }
01449     
01450     // Release application lock
01451     INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
01452     
01453     return server->server_addr;
01454 }
01455 
01456 
01472 void
01473 monitor_remove_session_for_server(uint16_t ss_id,
01474                                   in_addr_t app_addr,
01475                                   uint16_t app_port,
01476                                   in_addr_t server_addr)
01477 {
01478     mon_app_key_t key;
01479     mon_app_info_t * app;
01480     mon_server_info_t * server, *tmp, *tmp2;
01481     
01482     bzero(&key, sizeof(mon_app_key_t));
01483     key.svc_set_id = ss_id;
01484     key.app_addr = app_addr;
01485     key.app_port = app_port;
01486     
01487     // Get big lock
01488     INSIST_ERR(msp_spinlock_lock(&apps_big_lock) == MSP_OK);
01489     
01490     // get the application
01491     app = app_entry(patricia_get(&apps, sizeof(key), &key));
01492     
01493     if(app == NULL) {
01494         // Release big lock
01495         INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
01496         return;
01497     }
01498     
01499     // Get application lock
01500     INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
01501     
01502     // Release big lock
01503     INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
01504    
01505     server = TAILQ_FIRST(app->up_servers);
01506 
01507     while(server != NULL) {
01508         if(server->server_addr == server_addr) {
01509             --server->sessions;
01510             
01511             // Go through the up servers and preserve the ordering by load
01512             tmp2 = tmp = TAILQ_PREV(server, server_info_set_s, entries);
01513             while(tmp != NULL) {
01514                 if(tmp->sessions > server->sessions) {
01515                     tmp = TAILQ_PREV(tmp, server_info_set_s, entries);
01516                 } else {
01517                     if(tmp != tmp2) {
01518                         // insert server here
01519                         TAILQ_REMOVE(app->up_servers, server, entries);
01520                         TAILQ_INSERT_AFTER(app->up_servers, 
01521                                 tmp, server, entries);
01522                     }
01523                     // else don't need to move it
01524                     break;
01525                 }
01526             }
01527             // if we went through all of them
01528             if(tmp == NULL) {
01529                 // insert server at start
01530                 TAILQ_REMOVE(app->up_servers, server, entries);
01531                 TAILQ_INSERT_HEAD(app->up_servers, server, entries);
01532             }
01533             
01534             break;
01535         }
01536         server = TAILQ_NEXT(server, entries);
01537     }
01538     
01539     // Release application lock
01540     INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
01541 }
01542 
01543 
01547 void
01548 monitor_send_stats(void)
01549 {
01550     mon_app_info_t * app;
01551     mon_server_info_t * server;
01552     uint32_t session_count;
01553     
01554     // Get big lock
01555     INSIST_ERR(msp_spinlock_lock(&apps_big_lock) == MSP_OK);
01556     
01557     app = app_entry(patricia_find_next(&apps, NULL));
01558     
01559     while(app != NULL) { // go thru all apps
01560         
01561         // Get application lock
01562         INSIST_ERR(msp_spinlock_lock(&app->app_lock) == MSP_OK);
01563         
01564         session_count = 0;
01565         
01566         server = TAILQ_FIRST(app->up_servers);
01567         while(server != NULL) { // go thru all servers
01568             
01569             session_count += server->sessions; // add to total for app
01570             
01571             server = TAILQ_NEXT(server, entries);
01572         }
01573         
01574         // report
01575         notify_application_sessions(app->key.svc_set_id,
01576                 app->key.app_addr, app->key.app_port, session_count);
01577         
01578         
01579         // Release application lock
01580         INSIST_ERR(msp_spinlock_unlock(&app->app_lock) == MSP_OK);
01581         
01582         app = app_entry(patricia_find_next(&apps, &app->node));       
01583     }
01584     
01585     // Release big lock
01586     INSIST_ERR(msp_spinlock_unlock(&apps_big_lock) == MSP_OK);
01587 }
01588 

2007-2009 Juniper Networks, Inc. All rights reserved. The information contained herein is confidential information of Juniper Networks, Inc., and may not be used, disclosed, distributed, modified, or copied without the prior written consent of Juniper Networks, Inc. in an express license. This information is subject to change by Juniper Networks, Inc. Juniper Networks, the Juniper Networks logo, and JUNOS are registered trademarks of Juniper Networks, Inc. in the United States and other countries. All other trademarks, service marks, registered trademarks, or registered service marks are the property of their respective owners.
Generated on Sun May 30 20:26:56 2010 for SDK Your Net Corporation Equilibrium Load Balancer Example: equilibrium-data 1.0 by Doxygen 1.5.1