Commit 28f1a3b8 authored by ofgeorg's avatar ofgeorg

some bugfixes - mpi support

parent 81401ea4
......@@ -40,7 +40,11 @@ else
%disp(input_file);
batch = [batch 'mkdir ' out s];
batch = [batch 'tar -x ' out '.txt -f inputs.tar.gz -O > ' out '/input.txt' s];
batch = [batch pathtoCMA ' ' out '/input.txt > ' out '/stdout.txt' s];
if (mpi > 1)
batch = [batch 'ompirun ' pathtoCMA ' ' out '/input.txt > ' out '/stdout.txt' s];
else
batch = [batch pathtoCMA ' ' out '/input.txt > ' out '/stdout.txt' s];
end
batch = [batch 'tar czvf ' out '.tar.gz ' out s];
batch = [batch 'rm -r ' out s];
......
function [something] = Brutus_Benchmarks_final(pathtoCMA,workdir,name,repeats,inopts,queuetime,indbname,inexpt)
function [something] = Brutus_Benchmarks_final(pathtoCMA,workdir,name,repeats,inopts,queuetime,indbname,inexpt,innrproc)
%for this to work brutus must be reachable via ssh without password
%(keypair) and the jdbc driver has to be access able (most likly the path
%must be adjusted)
......@@ -123,7 +123,11 @@ if nargin < 8 || isempty(inexpt) % if time for jobs is given
else
expt = inexpt;
end
if nargin < 9 || isempty(innrproc) % single job or multiple processors
nrproc = 1; %single job default
else
nrproc = innrproc;
end
......@@ -155,7 +159,15 @@ for i = 1:perm_c
if (strcmpi(char(vars(k)),'output_folder') || strcmpi(char(vars(k)),'seed_folder'))
x = [char(vars(k)) ' = output' num2str(i) 'repeat' num2str(j)];
else
x = [char(vars(k)) ' = ' char(output{k,i})];
if (strcmpi(char(vars(k)),'USE_LJC') )
x = ['USE_LJ = ' char(output{k,i})];
else
if (strcmpi(char(vars(k)),'STOPTIMEUSE') )
x = ['STOPTIME = ' char(output{k,i})];
else
x = [char(vars(k)) ' = ' char(output{k,i})];
end
end
end
batch = AddLine(batch,x);
end
......@@ -165,7 +177,7 @@ for i = 1:perm_c
fprintf(fid,'%s',batch);
fclose(fid);
% create the according submit batches
nr_batches = AddtoSub(pathtoCMA,queuetime*60*60,expt,strcat('output',num2str(i),'repeat',num2str(j)),0);
nr_batches = AddtoSub(pathtoCMA,queuetime*60*60,expt,strcat('output',num2str(i),'repeat',num2str(j)),nrproc);
end
end
%flush the final batch
......@@ -202,7 +214,11 @@ if (~isempty(findstr('No such file', result))) %we didnt submit yet
%create the submit script
runme = AddLine('','#!/bin/sh');
runme = AddLine(runme, ['for n in {0..' num2str(batchnumber-1) '}; do']);
runme = AddLine(runme,['bsub -W ' num2str(queuetime) ':00 -o report${n} < ${n}batch.txt > sub${n}']);
if (nrproc < 2)
runme = AddLine(runme,['bsub -W ' num2str(queuetime) ':00 -o report${n} < ${n}batch.txt > sub${n}']);
else
runme = AddLine(runme,['bsub -n ' num2str(nrproc) ' -W ' num2str(queuetime) ':00 -o report${n} < ${n}batch.txt > sub${n}']);
end
runme = AddLine(runme,['done']);
fid = fopen('runme','wt');
......@@ -253,7 +269,7 @@ else
cmd = ['ssh brutus.ethz.ch ''ls ' workdir name '/output*.tar.gz | wc -w'''];
[status,result] = system(cmd);
totalnumber = str2num(result);
FillDB(name,totalnumber/repeats, repeats,dbname);
FillDB(name,totalnumber/repeats, repeats,dbname,nrproc);
cmd = ['ssh brutus.ethz.ch ''echo db_done >' workdir name '/db_done '''];
[status,result] = system(cmd);
else
......
function [something] = FillDB(name,outputs,repeats,indbname)
function [something] = FillDB(name,outputs,repeats,indbname,nrproc)
%% DB Stuff
% Database Server
......@@ -6,9 +6,10 @@ function [something] = FillDB(name,outputs,repeats,indbname)
% Database Username/Password
user = 'root';
password = '';
password = 'montecarlo';
% Database Name
%dbName = 'mysql';
dbName = 'mysql';
% JDBC Parameters
......@@ -46,7 +47,7 @@ function [something] = FillDB(name,outputs,repeats,indbname)
if isconnection(dbConn)
%create settings table
[varnames,varvalue,vartypes,count] = Readsettings('output1repeat1'); %read first file
[varnames,varvalue,vartypes,count] = Readsettings('output1repeat1',nrproc); %read first file
string = AddLine('','CREATE TABLE IF NOT EXISTS `setting` (');
string = AddLine(string,'`id` int(11) NOT NULL AUTO_INCREMENT,');
for i = 1:count
......@@ -63,6 +64,9 @@ function [something] = FillDB(name,outputs,repeats,indbname)
string = 'CREATE TABLE IF NOT EXISTS `result` (';
string = AddLine(string,'`id` int(11) NOT NULL AUTO_INCREMENT,');
string = AddLine(string,'`setting_id` int(11) NOT NULL,');
if (nrproc > 1)
string = AddLine(string,'`proc` int(11) NOT NULL,');
end
string = AddLine(string,'`out_bestever_f` double NOT NULL,');
string = AddLine(string,'`out_bestever_evals` double NOT NULL,');
string = AddLine(string,'`out_countEval` double NOT NULL,');
......@@ -111,9 +115,15 @@ function [something] = FillDB(name,outputs,repeats,indbname)
%now we start filling it
for i = 1:outputs
for j = 1:repeats
for p = 1:nrproc
if (nrproc < 2)
proc_string = '';
else
proc_string = ['_' num2str(p-1)];
end
%insert the complete setting
[varnames,varvalue,vartypes,count] = Readsettings(['output' num2str(i) 'repeat' num2str(j)]);
[varnames,varvalue,vartypes,count] = Readsettings(['output' num2str(i) 'repeat' num2str(j)],nrproc);
string = AddLine('',['INSERT INTO `' indbname '`.`setting` (`id`']);
for k = 1:count
string = AddLine(string,[',`' varnames{k} '`']);
......@@ -129,31 +139,34 @@ function [something] = FillDB(name,outputs,repeats,indbname)
id = result{1};
%insert the results
load 'out_bestever_evals.txt';
load 'out_bestever_f.txt'
load 'out_bestever_x.txt'
out_bestever_evals = load(['out_bestever_evals' proc_string '.txt']);
out_bestever_f = load([ 'out_bestever_f' proc_string '.txt']);
out_bestever_x = load([ 'out_bestever_x' proc_string '.txt']);
%load 'out_BFGSexitcounter.txt'
load 'out_countEvalNaN.txt'
load 'out_countEval.txt'
load 'out_countIter.txt'
load 'out_countOutOfBounds.txt'
out_countEvalNaN = load([ 'out_countEvalNaN' proc_string '.txt']);
out_countEval = load([ 'out_countEval' proc_string '.txt']);
out_countIter = load([ 'out_countIter' proc_string '.txt']);
out_countOutOfBounds = load([ 'out_countOutOfBounds' proc_string '.txt']);
%load 'out_funcName.txt'
load 'out_insigma.txt'
load 'out_lambda.txt'
load 'out_mueff.txt'
load 'out_mu.txt'
load 'out_N.txt'
out_insigma = load([ 'out_insigma' proc_string '.txt']);
out_lambda = load([ 'out_lambda' proc_string '.txt']);
out_mueff = load([ 'out_mueff' proc_string '.txt']);
out_mu = load([ 'out_mu' proc_string '.txt']);
out_N = load([ 'out_N' proc_string '.txt']);
%load 'out_settings.txt'
fid = fopen('out_stopflag.txt','r');
fid = fopen(['out_stopflag' proc_string '.txt'],'r');
out_stopflag = fscanf(fid,'%s' );
fclose(fid);
%load 'out_stopflag.txt'
%load 'out_weights.txt'
load 'out_xstart.txt'
load 'seed.txt'
out_xstart = load([ 'out_xstart' proc_string '.txt']);
seed = load([ 'seed' proc_string '.txt']);
%string = ['INSERT INTO `' name '`.`result` (`id`,
%`setting_id`, `f`, `evals`, `countEval`, `countOutOfBounds`,`countEvalNaN`,`out_countIter`,`out_lambda`,`out_mueff`,`out_mu`,`out_N`,`seed`,`stopflag`) VALUES (NULL, ''' num2str(id) ''', ''' num2str(out_bestever_f) ''', ''' num2str(out_bestever_evals) ''', ''' num2str(out_countEval) ''', ''' num2str(out_countOutOfBounds) ''' , ''' num2str(stopflag) ''');'];
string = ['INSERT INTO `' indbname '`.`result` (`id`, `setting_id`'];
if (nrproc > 1)
string = AddLine(string,',`proc` ');
end
string = AddLine(string,',`out_bestever_f` ');
string = AddLine(string,',`out_bestever_evals` ');
string = AddLine(string,',`out_countEval` ');
......@@ -168,6 +181,9 @@ function [something] = FillDB(name,outputs,repeats,indbname)
string = AddLine(string,',`out_stopflag` ');
string = AddLine(string,') VALUES (NULL');
string = AddLine(string,[',''' num2str(id) '''']);
if (nrproc > 1)
string = AddLine(string,[',''' num2str(p-1) '''']);
end
string = AddLine(string,[',''' num2str(out_bestever_f) '''']);
string = AddLine(string,[',''' num2str(out_bestever_evals) '''']);
string = AddLine(string,[',''' num2str(out_countEval) '''']);
......@@ -219,6 +235,7 @@ function [something] = FillDB(name,outputs,repeats,indbname)
cd ('..');
[status, message, messageid] = rmdir(['output' num2str(i) 'repeat' num2str(j)],'s');
disp(['output' num2str(i) 'repeat' num2str(j) ' added to DB']);
end
end
end
......
function [varnames2,varvalue2,vartypes,count] = Readsettings(name)
function [varnames2,varvalue2,vartypes,count] = Readsettings(name,nrproc)
%mkdir(['dir_' name]);
%cd (['dir_' name]);
%cmd = ['"C:\Program Files\7-Zip\7z.exe" e ../' name '.tar.gz'];
......@@ -7,7 +7,12 @@ function [varnames2,varvalue2,vartypes,count] = Readsettings(name)
cmd = ['tar -xf ' name '.tar.gz'];
[status,result2] = system(cmd );
cd (name);
fid = fopen('out_settings.txt','r');
if (nrproc > 1)
fid = fopen('out_settings_0.txt','r');
else
fid = fopen('out_settings.txt','r');
end
varstat = 1;
count = 0;
varnames = {};
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment