Commit 4ee2c7ad authored by ogonzale's avatar ogonzale

Initial revision

parents
/
0:0 444 etc/logrotate.d/httpd
0:0 444 local/data/httpd/.main/conf/httpd.conf
0:0 444 local/data/httpd/falcon/conf/httpd.conf
0:0 444 local/data/httpd/falcon/htdocs/class_bdd.php
0:0 444 local/data/httpd/falcon/htdocs/common.php
0:0 444 local/data/httpd/falcon/htdocs/consoles.php
0:0 444 local/data/httpd/falcon/htdocs/index.css
0:0 444 local/data/httpd/falcon/htdocs/index.php
0:0 444 local/data/httpd/falcon/htdocs/info.txt
0:0 444 local/data/httpd/falcon/htdocs/jh.php
0:0 444 local/data/httpd/falcon/htdocs/jobs
0:0 444 local/data/httpd/falcon/htdocs/jobs.php
0:0 444 local/data/httpd/falcon/htdocs/man.css
0:0 444 local/data/httpd/falcon/htdocs/man.php
0:0 444 local/data/httpd/falcon/htdocs/sqlite3_blob
0:0 444 local/data/httpd/falcon/htdocs/wiki/swlist.php
0:0 444 local/data/httpd/falcon/htdocs/wiki/swlog.php
0:0 444 local/data/httpd/falcon/htdocs/wiki/swvers.php
0:0 444 local/soft/scripts/rotatelogs.php
Basic skeleton of the web server and the main virtual host:
* the hidden `.main` with the general `httpd.conf` where:
* all common settings are defined
* active virtual hosts are loaded
* use IncludeOptional to load it
* use a wildcard in the name; the more specific is the 1st letter in square brackets
* that should keep the sever running even is a host doesn't exist any more
* the default virtual host, falcon, including web pages
* the script to rotate logs
/local/data/httpd/*/logs/*_log {
weekly
rotate 48
compress
dateext
missingok
notifempty
sharedscripts
postrotate
/local/soft/apps/httpd/current/bin/apachectl restart &> /dev/null || true
endscript
}
###################################################
# installation
####################################################
# set this to the link that points to the active version
ServerRoot /local/soft/apps/httpd/current
# webmaster - this will be ogonzale because user apache is not known by lustre!
User ogonzale
Group apache
ServerAdmin hpcsupport@mpi-cbg.de
# server logs
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
CustomLog "logs/access_log" combined
ErrorLog "logs/error_log"
LogLevel warn
####################################################
# network settings
####################################################
# just in case it can't be determined automatically
ServerName falcon.mpi-cbg.de
# misc
Listen 80
Listen 443
HostnameLookups On
TimeOut 600
# hide banner
ServerTokens ProductOnly
ServerSignature Off
# server-status
<Location /server-status>
SetHandler server-status
Order Deny,Allow
Deny from all
Allow from 127.
</Location>
####################################################
# contents
####################################################
# index files
DirectoryIndex index.html index.php
# deny access everywhere by default
<Directory />
AllowOverride none
Require all denied
</Directory>
# deny access to .ht* and RCS files
<FilesMatch "(^\.ht|,v$)">
Require all denied
</FilesMatch>
# mime
TypesConfig conf/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
AddType application/x-httpd-php .php
# modules
LoadModule authn_file_module modules/mod_authn_file.so
#LoadModule authn_dbm_module modules/mod_authn_dbm.so
#LoadModule authn_anon_module modules/mod_authn_anon.so
#LoadModule authn_dbd_module modules/mod_authn_dbd.so
#LoadModule authn_socache_module modules/mod_authn_socache.so
LoadModule authn_core_module modules/mod_authn_core.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
LoadModule authz_user_module modules/mod_authz_user.so
#LoadModule authz_dbm_module modules/mod_authz_dbm.so
#LoadModule authz_owner_module modules/mod_authz_owner.so
#LoadModule authz_dbd_module modules/mod_authz_dbd.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule auth_basic_module modules/mod_auth_basic.so
#LoadModule auth_form_module modules/mod_auth_form.so
#LoadModule auth_digest_module modules/mod_auth_digest.so
#LoadModule allowmethods_module modules/mod_allowmethods.so
#LoadModule file_cache_module modules/mod_file_cache.so
#LoadModule cache_module modules/mod_cache.so
#LoadModule cache_disk_module modules/mod_cache_disk.so
#LoadModule cache_socache_module modules/mod_cache_socache.so
#LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
#LoadModule socache_dbm_module modules/mod_socache_dbm.so
#LoadModule socache_memcache_module modules/mod_socache_memcache.so
#LoadModule watchdog_module modules/mod_watchdog.so
#LoadModule macro_module modules/mod_macro.so
#LoadModule dbd_module modules/mod_dbd.so
#LoadModule dumpio_module modules/mod_dumpio.so
#LoadModule buffer_module modules/mod_buffer.so
#LoadModule ratelimit_module modules/mod_ratelimit.so
LoadModule reqtimeout_module modules/mod_reqtimeout.so
#LoadModule ext_filter_module modules/mod_ext_filter.so
#LoadModule request_module modules/mod_request.so
#LoadModule include_module modules/mod_include.so
LoadModule filter_module modules/mod_filter.so
#LoadModule substitute_module modules/mod_substitute.so
#LoadModule sed_module modules/mod_sed.so
#LoadModule deflate_module modules/mod_deflate.so
LoadModule mime_module modules/mod_mime.so
LoadModule log_config_module modules/mod_log_config.so
#LoadModule log_debug_module modules/mod_log_debug.so
#LoadModule logio_module modules/mod_logio.so
LoadModule env_module modules/mod_env.so
#LoadModule expires_module modules/mod_expires.so
LoadModule headers_module modules/mod_headers.so
#LoadModule unique_id_module modules/mod_unique_id.so
LoadModule setenvif_module modules/mod_setenvif.so
LoadModule version_module modules/mod_version.so
#LoadModule remoteip_module modules/mod_remoteip.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_connect_module modules/mod_proxy_connect.so
LoadModule proxy_ftp_module modules/mod_proxy_ftp.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule proxy_fcgi_module modules/mod_proxy_fcgi.so
LoadModule proxy_scgi_module modules/mod_proxy_scgi.so
LoadModule proxy_wstunnel_module modules/mod_proxy_wstunnel.so
LoadModule proxy_ajp_module modules/mod_proxy_ajp.so
LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
LoadModule proxy_express_module modules/mod_proxy_express.so
#LoadModule proxy_hcheck_module modules/mod_proxy_hcheck.so
#LoadModule session_module modules/mod_session.so
#LoadModule session_cookie_module modules/mod_session_cookie.so
#LoadModule session_dbd_module modules/mod_session_dbd.so
LoadModule slotmem_shm_module modules/mod_slotmem_shm.so
LoadModule ssl_module modules/mod_ssl.so
#LoadModule lbmethod_byrequests_module modules/mod_lbmethod_byrequests.so
#LoadModule lbmethod_bytraffic_module modules/mod_lbmethod_bytraffic.so
#LoadModule lbmethod_bybusyness_module modules/mod_lbmethod_bybusyness.so
#LoadModule lbmethod_heartbeat_module modules/mod_lbmethod_heartbeat.so
LoadModule unixd_module modules/mod_unixd.so
#LoadModule dav_module modules/mod_dav.so
LoadModule status_module modules/mod_status.so
LoadModule autoindex_module modules/mod_autoindex.so
#LoadModule info_module modules/mod_info.so
#LoadModule cgid_module modules/mod_cgid.so
#LoadModule dav_fs_module modules/mod_dav_fs.so
#LoadModule vhost_alias_module modules/mod_vhost_alias.so
#LoadModule negotiation_module modules/mod_negotiation.so
LoadModule dir_module modules/mod_dir.so
#LoadModule actions_module modules/mod_actions.so
#LoadModule speling_module modules/mod_speling.so
#LoadModule userdir_module modules/mod_userdir.so
LoadModule alias_module modules/mod_alias.so
LoadModule rewrite_module modules/mod_rewrite.so
####################################################
# virtual hosts
####################################################
IncludeOptional /local/data/httpd/[f]alcon/conf/httpd.conf
IncludeOptional /local/data/httpd/[j]upyter/conf/httpd.conf
IncludeOptional /local/data/httpd/[d]vcd/conf/httpd.conf
<VirtualHost *:80>
ServerName falcon.mpi-cbg.de
ServerAlias falcon
ServerAdmin root@localhost
# logs, excluding traffic from the private network. we'd better lose
# some hits than fill the log with connections from the stats script
SetEnvIf Remote_Addr "^172.16." dontlog
CustomLog /local/data/httpd/falcon/logs/access_log combined env=!dontlog
ErrorLog /local/data/httpd/falcon/logs/error_log
SetEnv APACHE_ROOT "/local/data/httpd/falcon"
SetEnv SERVER_NAME "falcon.mpi-cbg.de"
DocumentRoot "/local/data/httpd/falcon/htdocs"
<Directory "/local/data/httpd/falcon/htdocs">
Options FollowSymLinks
AllowOverride None
Require all granted
</Directory>
</VirtualHost>
<VirtualHost *:443>
ServerName falcon.mpi-cbg.de
ServerAlias falcon
# SSL stuff
SSLEngine On
SSLProxyEngine On
SSLCertificateFile /local/soft/etc/ssl/jupyter.crt
SSLCertificateKeyFile /local/soft/etc/ssl/jupyter.key
SSLCACertificateFile /local/soft/etc/ssl/jupyter.ca.crt
Redirect permanent / http://falcon.mpi-cbg.de/
</VirtualHost>
<?php
// a database named XXX will exist in a directory structure like
// XXX/<year>/<month>/<date>.sql and will be linked at the top level
// as XXX.sql
//
// don't mess with the directory structure. no check is done on XXX,
// <year> or <month>. don't create, remove or rename anything. the
// script is in charge and takes for granted everything is correct
//
// as days pass by, the link will be re-done to always point at the
// most current database file. past files must be accessed using full
// path
class BDD extends SQLite3 {
// all databases will be here
private $DATA = "data/";
// lock directory - stupid PHP doesn't allow symbols here. you'll
// have to hard-code $DATA >:(
private $LOCK = "/local/data/httpd/falcon/htdocs/data/.lock";
// set to TRUE if the database is empty
private $empty = FALSE;
function __construct($name) {
error_reporting(E_ALL ^ E_WARNING);
if ( ! mkdir($this->LOCK) ) {
die("Lock directory found\n");
}
error_reporting(E_ALL);
$this->createDir($this->DATA);
chdir($this->DATA);
$d = getdate();
$year = $d['year'];
$month = $d['mon'];
$day = $d['mday'];
$dir = sprintf("%s/%d/%02d",$name,$year,$month);
$path = sprintf("%s/%02d%02d%02d.sql",$dir,$year-2000,$month,$day);
$this->createDir($dir);
if ( ! file_exists($path) || filesize($path) == 0 ) {
$this->empty = TRUE;
}
$this->open($path);
$this->exec("BEGIN TRANSACTION");
$this->createLink($path,"$name.sql");
}
////////////////////////////////////////////////////////////////////
// private methods
private function createDir($name) {
if ( ! file_exists($name) ) {
mkdir($name,0755,TRUE);
}
}
// create or update a link
private function createLink($orig,$dest) {
// we need getcwd() here because readlink() doesn't work with
// relative paths when run within apache. why???
if ( readlink(getcwd()."/$dest") != $orig ) {
if ( file_exists($dest) ) {
unlink($dest);
}
symlink($orig,$dest);
}
}
////////////////////////////////////////////////////////////////////
// public interface
// close the database
public function close() {
$this->exec("END TRANSACTION");
parent::close();
rmdir($this->LOCK);
}
// return TRUE if the database is still empty, FALSE otherwise
public function isEmpty() {
return $this->empty;
}
// create a directory if it doesn't exist already
// table definitions
public function createTables($tables) {
foreach ( $tables as $table=>$def ) {
$cmd = "CREATE TABLE $table ("
. implode(", ",$def)
. ")";
$this->exec($cmd);
}
}
}
?>
This diff is collapsed.
<!doctype html>
<html>
<head>
<title>Admin Interfaces</title>
<style>
body {
background: black;
}
td, th {
font-family: Verdana, Geneva, sans-serif;
color: white;
padding: 0 3px;
}
th {
background: #404040;
text-align: center;
font-size: 12pt;
font-weight: bold;
}
td {
font-size: 10pt;
}
a:link, a:visited {
color: white;
text-decoration: none;
}
</style>
</head>
<body>
<center>
<?php
// temporary, just to access all furiosa's consoles easily
//
// ideally these links should be automatically inserted in DW
// all systems in the private network have their admin interfaces here
$IP = "172.18";
// knowing that furiosa is in 10.3.8.11 onwards, get the names
// from the DNS, sort them and use up to the first number to
// group them. wid is the number of numeric digits in the name
// and pre is the name prefix
$SERVERS = array(
"furiosa" => array( "wid" => 1, "pre" => "" ),
"falcon" => array( "wid" => 1, "pre" => "" ),
"mack" => array( "wid" => 1, "pre" => "" ),
"mds" => array( "wid" => 2, "pre" => "furiosa-" ),
"oss" => array( "wid" => 2, "pre" => "furiosa-" ),
);
// these are 172.18.ipx,ipy. all numbers have 2 digits. when
// enc != 1, ipx is increased when enc nodes have been considered
$BLADES = array(
"thin" => array( "ipx" => 1, "nod" => 105, "enc" => 16, "pre" => "c" ),
);
$RACKS = array(
"fat" => array( "ipx" => 0, "nod" => 4, "pre" => "c" ),
"fat2" => array( "ipx" => 101, "nod" => 4, "pre" => "r" ),
"gpu" => array( "ipx" => 102, "nod" => 22, "pre" => "r" ),
);
function table($h,$r,$c=1) {
echo "<table>"
. trth($h,$c)
. "$r</table><br/>\n";
}
function trth($h,$c) {
$c++;
return "<tr><th colspan=\"$c\">$h</th></tr>";
}
function trtd($n,$i) {
return "<tr>" . td($n) . td($n,$i) . "</tr>";
}
function td($t="",$a="") {
if ( $a == "" ) {
$a = $t;
}
if ( $t != "" ) {
$a = "<a href=\"https://$t\">$a</a>";
}
return "<td>$a</td>";
}
// servers. starting at 1, increment value until a name is not found
// in the DNS server
foreach ( $SERVERS as $sys=>$info ) {
extract($info);
$num = 1;
$rows = "";
$found = TRUE;
while ( $found ) {
$name = sprintf("%s%s%0{$wid}d-console",$pre,$sys,$num++);
$ip = gethostbyname($name);
if ( $ip != $name ) {
$rows .= trtd($name,$ip);
} else {
$found = FALSE;
}
}
echo table($sys,$rows);
}
// blades
foreach ( $BLADES as $sys=>$info ) {
extract($info);
$rows = "";
for ( $c = 1; $c <= ceil($nod/$enc); $c++ ) {
$name = sprintf("%s%02d",$pre,$c);
$ip = sprintf("%s.%d.120",$IP,$c);
$row = td($ip,$name);
for ( $n = 1; $n <= $enc; $n++ ) {
if ( ($c-1) * $enc + $n <= $nod ) {
// node exists
$name = sprintf("%s%02dn%02d",$pre,$c,$n);
$ip = sprintf("%s.%d.%d",$IP,$ipx-1+$c,$n);
$row .= td($ip,$name);
} else {
// node doesn't exist (empty bay)
$row .= td();
}
}
$rows .= "<tr>$row</tr>";
}
echo table($sys,$rows,$enc);
}
// rack-mount
$rows = "";
$span = 0;
foreach ( $RACKS as $sys=>$info ) {
extract($info);
$row = td("",$sys);
$span = max($span,$nod);
for ( $n = 1; $n <= $nod; $n++ ) {
$name = sprintf("%s%02dn%02d",$pre,$ipx-100,$n);
if ( $ipx == 0 ) {
// temporary until we rename the fat nodes from c00 to r01
$name = sprintf("%s%02dn%02d",$pre,$ipx,$n);
}
$ip = sprintf("%s.%d.%d",$IP,$ipx,$n);
$row .= td($ip,$name);
}
$rows .= "<tr>$row</tr>";
}
echo table("rack-mounted",$rows,$span);
?>
</center>
</body>
</html>
body {
font-family: Verdana, Geneva, sans-serif;
}
table {
margin-top: 20px;
margin-bottom: 30px;
}
tr:hover td.tdlist {
background: #f0f0f0;
}
td {
padding: 0 10px 0 10px;
text-align: center;
font-size: 11pt;
font-family: Verdana, Geneva, sans-serif;
}
.head, .foot {
padding: 0 5px 0 5px;
font-weight: bold;
}
.head {
border-bottom: 3px black double;
}
.foot {
border-top: 3px black double;
}
.left {
text-align: left;
}
.right {
text-align: right;
}
.neg {
font-weight: bold;
}
.critical {
font-weight: bold;
color: red;
}
.warning {
font-weight: bold;
color: orange;
}
.tdtitle {
padding-top: 5px;
padding-bottom: 5px;
font-size: 14pt;
background: #e8e8e8;
}
.tdjobs {
padding-left: 10px;
padding-right: 5px;
min-width: 30px;
text-align: right;
}
.tdcores {
padding-left: 5px;
padding-right: 10px;
min-width: 30px;
text-align: right;
}
.tdavail {
padding-left: 6px;
padding-right: 6px;
text-align: right;
}
.tdlist {
padding-left: 6px;
padding-right: 6px;
font-size: 10pt;
}
.tddim {
color: #a0a0a0;
}
.lastexec {
padding-top: 3px;
padding-right: 20px;
border-top: 1px solid gray;
text-align: right;
font-size: 9pt;
}
<!doctype html>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<link rel="stylesheet" href="index.css" />
<script>
function elapsed() {
var sec = Math.floor(Date.now()/1000) - start;
var txt = "";
if ( sec >= 86400 ) {
var days = Math.floor(sec/86400);
sec -= days*86400;
txt += days + " day" + ( days == 1 ? "" : "s" ) + ", ";
}
if ( sec >= 3600 ) {
var hours = Math.floor(sec/3600);
sec -= hours*3600;
txt += hours + " hour" + ( hours == 1 ? "" : "s" ) + ", ";
}
if ( sec >= 60 ) {
var minutes = Math.floor(sec/60);
sec -= minutes*60;
txt += minutes + " minute" + ( minutes == 1 ? "" : "s" ) + ", ";
}
txt += sec + " second" + ( sec == 1 ? "" : "s" );
document.getElementById('elapsed').innerHTML = txt;
var t = setTimeout(elapsed, 1000);
}
</script>
</head>
<body onLoad="startTime()">
<center>
<?php
date_default_timezone_set('Europe/Andorra');
require_once("common.php");
$bdd = new BDD("cluster");
if ( ! $bdd->isEmpty() ) {
$res = $bdd->query("SELECT * FROM slurm ORDER BY ts DESC LIMIT 1");
if ( $row = $res->fetchArray() ) {
$list = json_decode(gzdecode($row['list']),TRUE);
$summary = json_decode(gzdecode($row['summary']),TRUE);
$alloc = json_decode(gzdecode($row['alloc']),TRUE);
echo summaryTable($QUEUE,$summary['running'],"Running jobs");
echo summaryTable($QUEUE,$summary['pending'],"Pending jobs");
echo allocRsc($alloc,"cores",TRUE);
echo allocRsc($alloc,"memory",FALSE,array(25,50));
echo jobList($JOBCOLUMN,$list,$row['ts']);
echo "<p class='lastexec'>"
. "Data recorded <span id=\"elapsed\"></span> ago"
. date(" (D, j M Y H:i:s)",$row['ts'])
. "</p>";
}
}
$bdd->close();
?>
<script>
function startTime() {
start = <?php echo ( isset($row['ts']) ? $row['ts'] : time() ); ?>;
elapsed();
}
</script>
</center>
</body>
</html>
This may be nice to show:
* cluster view: like madmax
* CPU usage per node
* memory usage per node
* processes per node
* SLURM jobs per node
* power consumption per node
* uptime per node
* system view:
* historical of the above
* detailed process view
* overall queue status:
* current tables
* detailed queue status:
* running jobs over time
* used cored over time
* pending jobs over time
* waiting time over time
* somehow, memory requested over memory used