path: root/scilab/modules/development_tools/macros/unit_test_run.sci

//-----------------------------------------------------------------------------
// Pierre MARECHAL
// Scilab team
// Copyright INRIA
// Date : 25/10/2007
//
// Launch unitary tests
//-----------------------------------------------------------------------------

function unit_test_run(varargin)
	
	lhs = argn(1);
	rhs = argn(2);
	
	global test_list;
	global test_count;
	global displayed_txt;
	
	// options
	
	global check_ref;
	global check_error_output;
	global create_ref;
	global launch_mode;

	
	check_ref          = %T;
	check_error_output = %T;
	create_ref         = %F;
	launch_mode        = "-nw";
	just_list_tests    = %F;
	print_help         = %F;
	
	test_count         = 0;
	test_passed_count  = 0;
	test_failed_count  = 0;
	test_skipped_count = 0;
	
	displayed_txt      = '';
	details_failed     = '';
	
	// =======================================================
	// Gestion des tests à lancer
	// =======================================================
	
	if (rhs == 0) ..
				| ((rhs == 1) & (varargin(1)==[])) ..
				| (((rhs == 2)|(rhs == 3)) & (varargin(1)==[]) & (varargin(2)==[])) then
		
		// No input argument
		// unit_test_run()
		// unit_test_run([])
		// => Launch each test of each module
		
		module_list = getmodules();
		for k=1:size(module_list,'*')
			unit_test_add_module(module_list(k));
		end
	
	elseif (rhs == 1) ..
				| ((rhs == 2) & (varargin(2)==[])) ..
				| ((rhs == 3) & (varargin(2)==[])) then
		
		// One input argument
		// unit_test_run(<module_name>)
		// unit_test_run([<module_name_1>,<module_name_2>])
		
		// varargin(1) = [<module_name_1>,<module_name_2>]
		
		module_mat = varargin(1);
		
		[nl,nc] = size(module_mat);
		
		// unit_test_run([<module_name_1>,<module_name_2>])
		
		for i=1:nl
			for j=1:nc
				if( with_module(module_mat(i,j)) ) then
					unit_test_add_module(module_mat(i,j));
				else
					error(sprintf(gettext("%s is not an installed module"),module_mat(i,j)));
				end
			end
		end
		
	elseif (rhs == 2) | (rhs == 3) then
		
		// Two input arguments
		// unit_test_run(<module_name>,<test_name>)
		// unit_test_run(<module_name>,[<test_name_1>,<test_name_2>] )
		
		// varargin(1) = <module_name> ==> string 1x1
		// varargin(2) = <test_name_1> ==> mat nl x nc
		
		module   = varargin(1);
		test_mat = varargin(2);
		
		if ((or(size(module) <> [1,1])) & (test_mat <> [])) then
			example = unit_test_examples();
			err     = ["" ; gettext("error : Input argument sizes are not valid") ; "" ; example ];
			printf("%s\n",err);
			return;
		end
		
		[nl,nc] = size(test_mat);
		
		for i=1:nl
			for j=1:nc
				if( fileinfo(SCI+"/modules/"+module+"/unit_tests/"+test_mat(i,j)+".tst") <> [] ) then
					unit_test_add_onetest(module,test_mat(i,j));
				else
					error(sprintf(gettext("The test ""%s"" is not available from the ""%s"" module"),test_mat(i,j),module));
				end
			end
		end
	else
		error(gettext('Number of parameters incorrect.'));
	end
	
	// =======================================================
	// Gestion des options
	// =======================================================
	
	if rhs == 3 then
		
		option_mat =  varargin(3);
		
		if grep(option_mat,"no_check_ref") <> [] then
			check_ref  = %F;
		end
		
		if grep(option_mat,"no_check_error_output") <> [] then
			check_error_output  = %F;
		end
		
		if grep(option_mat,"create_ref") <> [] then
			create_ref = %T;
			check_ref  = %F;
		end
		
		if grep(option_mat,"mode_nw") <> [] then
			launch_mode = "-nw";
		end
		
		if grep(option_mat,"mode_nwni") <> [] then
			launch_mode = "-nwni";
		end
		
		if grep(option_mat,"list") <> [] then
			just_list_tests    = %T;
		end
		
		if grep(option_mat,"help") <> [] then
			print_help         = %T;
		end
	end

	if print_help then
		
		example = unit_test_examples();
		printf("%s\n",example);
		return;
	
	elseif just_list_tests then
		
		// =======================================================
		// Just list tests
		// =======================================================
		
		for i=1:test_count
			printf("   %02d - ",i);
			printf("[%s] %s\n",test_list(i,1),test_list(i,2));
		end
		
	else
		
		// =======================================================
		// Test launch
		// =======================================================
		
		printf("   TMPDIR = %s\n",TMPDIR);
		printf("\n");
		
		for i=1:test_count
			
			printf("   %02d/%02d - ",i,test_count);
			printf("[%s] %s",test_list(i,1),test_list(i,2));
			for j = length(test_list(i,2) + test_list(i,1)):50
				printf(".");
			end
			
			[status_id,status_msg,status_details] = unit_test_run_onetest(test_list(i,1),test_list(i,2));
			printf("%s \n",status_msg);
			
			// Recencement des tests
			
			if status_id == 0 then
				// passed
				test_passed_count = test_passed_count + 1;
			
			elseif (status_id > 0) & (status_id < 10) then
				// failed
				test_failed_count = test_failed_count + 1;
				details_failed = [ details_failed ; sprintf("   TEST : [%s] %s",test_list(i,1),test_list(i,2))];
				details_failed = [ details_failed ; sprintf("     %s",status_msg) ];
				details_failed = [ details_failed ; status_details ];
				details_failed = [ details_failed ; "" ];
			
			elseif (status_id > 10) & (status_id < 20) then
				// skipped
				test_skipped_count = test_skipped_count + 1;
			end
		end
		
		// Summary
		
		test_passed_percent  = test_passed_count  / test_count * 100;
		test_skipped_percent = test_skipped_count / test_count * 100;
		test_failed_percent  = test_failed_count  / test_count * 100;
		
		printf("\n");
		printf("   ---------------------------------------------------------------------------------------\n");
		printf("   Summary\n\n");
		printf("   tests                     %4d - 100 %% \n",test_count);
		printf("   passed                    %4d - %3d %% \n",test_passed_count ,test_passed_percent);
		printf("   failed                    %4d - %3d %% \n",test_failed_count ,test_failed_percent);
		printf("   skipped                   %4d - %3d %% \n",test_skipped_count,test_skipped_percent);
		printf("   ---------------------------------------------------------------------------------------\n");
		
		if test_failed_count > 0 then
			printf("   Details\n\n");
			printf("%s\n",details_failed);
			printf("\n");
			printf("   ---------------------------------------------------------------------------------------\n");
		end
		
	end
	
endfunction

//-----------------------------------------------------------------------------
// Pierre MARECHAL
// Scilab team
// Copyright INRIA
// Date : 28 oct. 2007
//
// => List all test files in the module <module_mat>
// => Add them to the test_mat matrix
//-----------------------------------------------------------------------------

function unit_test_add_module(module_mat)
	
	module_test_dir = SCI+"/modules/"+module_mat+"/unit_tests";
	test_mat        = basename(listfiles(module_test_dir+"/*.tst"));
	
	nl = size(test_mat,"*");
	for i=1:nl
		unit_test_add_onetest(module_mat,test_mat(i));
	end
	
endfunction

//-----------------------------------------------------------------------------
// Pierre MARECHAL
// Scilab team
// Copyright INRIA
// Date : 28 oct. 2007
//
// => Add the test <test> to the test_mat matrix
//-----------------------------------------------------------------------------

function unit_test_add_onetest(module,test)
	
	global test_list;
	global test_count;
	
	test_count = test_count + 1;
	test_list( test_count , 1 ) = module;
	test_list( test_count , 2 ) = test;
	
endfunction

//-----------------------------------------------------------------------------
// Pierre MARECHAL
// Scilab team
// Copyright INRIA
// Date : 28 oct. 2007
//
// => Run one test
//-----------------------------------------------------------------------------

function [status_id,status_msg,status_details] = unit_test_run_onetest(module,test)
	
	global check_ref;
	global create_ref;
	
	status_id      = 0 ;
	status_msg     = "passed" ;
	status_details = "";
	
	[status_id,status_msg,status_details] = unit_test_run_checkerror(module,test);
	
	if check_ref | create_ref then
		// Check ref or create ref
		if status_id == 0 then
			[status_id,status_msg,status_details] = unit_test_run_proc_ref(module,test);
		end
	end
	
	return;
	
endfunction

//-----------------------------------------------------------------------------
// Pierre MARECHAL
// Scilab team
// Copyright INRIA
// Date : 8 novembre 2007
//
// => Check the test
//-----------------------------------------------------------------------------

function [status_id,status_msg,status_details] = unit_test_run_checkerror(module,test)
	
	status_id      = 0 ;
	status_msg     = "passed" ;
	status_details = "";
	
	global launch_mode;
	global check_error_output;
	
	// Some definitions
	
	tstfile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".tst",%f,%f);
	
	tmp_tstfile = pathconvert(TMPDIR+"/"+test+"_1.tst",%f,%f);
	tmp_diafile = pathconvert(TMPDIR+"/"+test+"_1.dia",%f,%f);
	tmp_resfile = pathconvert(TMPDIR+"/"+test+"_1.res",%f,%f);
	tmp_errfile = pathconvert(TMPDIR+"/"+test+"_1.err",%f,%f);
	
	// Remove the previous tmp files
	if fileinfo(tmp_tstfile) <> [] then
		deletefile(tmp_tstfile)
	end
	
	if fileinfo(tmp_diafile) <> [] then
		deletefile(tmp_diafile)
	end
	
	if fileinfo(tmp_resfile) <> [] then
		deletefile(tmp_resfile)
	end
	
	if fileinfo(tmp_errfile) <> [] then
		deletefile(tmp_errfile)
	end
	
	//Reset standard globals
	rand('seed',0);
	rand('uniform');
	
	// Get the tst file
	txt = mgetl(tstfile);
	
	// Check if it's an interactive test
	if grep(txt,"<-- INTERACTIVE TEST -->") <> [] then
		status_msg = "skipped : interactive test";
		status_id  = 10;
		return;
	end
	
	if (grep(txt,"<-- TEST WITH GRAPHIC -->") <> []) & (launch_mode=="-nwni") then
		status_msg = "skipped : test with graphic";
		status_id  = 11;
		return;
	end
	
	// Do some modification in tst file
	txt = strsubst(txt,'pause,end','bugmes();quit;end');
	txt = strsubst(txt,'-->','@#>'); //to avoid suppression of input --> with prompts
	txt = strsubst(txt,'halt();','');
	
	// Header : the same for each test
	
	head = [	"// <-- HEADER START -->";
				"mode(3);" ;
				"clear;" ;
				"lines(28,72);";
				"lines(0);" ;
				"deff(''[]=bugmes()'',''write(%io(2),''''error on test'''')'');" ;
				"predef(''all'');" ;
				"try";
				"diary(''"+tmp_diafile+"'');";
				"// <-- HEADER END -->"];
				
	tail = [	"// <-- FOOTER START -->";
				"catch";
				"	errmsg = ""<--""+""Error on the test script file""+""-->""";
				"	printf(""%s"",errmsg)";
				"end";
				"diary(0);";
				"exit;";
				"// <-- FOOTER END -->"]
	
	txt = [head;
		txt;
		tail];
	
	// and save it in a temporary file
	mputl(txt,tmp_tstfile);
	
	// Build the command to launch
	if MSDOS then
		unit_test_cmd = "( """+SCI+"\bin\scilex.exe"+""""+" "+launch_mode+" -nb -args -nouserstartup -f """+tmp_tstfile+""" > """+tmp_resfile+""" ) 2> """+tmp_errfile+"""";
	else
		unit_test_cmd = "( "+SCI+"/bin/scilab "+launch_mode+" -nb -args -nouserstartup -f "+tmp_tstfile+" > "+tmp_resfile+" ) 2> "+tmp_errfile;
	end
	
	// Launch the test exec
	host(unit_test_cmd);
	
	// First Check : error output
	if check_error_output then
		tmp_errfile_info = fileinfo(tmp_errfile);
		
		if ( (tmp_errfile_info <> []) & (tmp_errfile_info(1)<>0) ) then
			status_msg = "failed  : error_output not empty"
			status_details = sprintf("     Check the following file : \n     - %s",tmp_errfile);
			status_id  = 5;
			return;
		end
	end
	
	//  Do some modification in  dia file
	dia = mgetl(tmp_diafile);
	
	//Check for execution errors
	if grep(dia,"<--Error on the test scmodules/time/unit_tests/datenum.tstript file-->")<>[] then
		status_msg = "failed  : premature end of the test script";
		status_details = sprintf("     Check the following file : \n     - %s",tmp_diafile);
		status_details = [ status_details ; sprintf("     Or launch the following command : \n     - exec %s;",tstfile) ];
		status_id = 3;
		return;
	end
	
	// Remove Header and Footer
	dia = remove_headers(dia);
	
	//Check for execution errors
	
	if grep(dia,"!--error")<>[] then
		status_msg     = "failed  : the string (!--error) has been detected";
		status_details = sprintf("     Check the following file : \n     - %s",tmp_diafile);
		status_details = [ status_details ; sprintf("     Or launch the following command : \n     - exec %s;",tstfile) ];
		status_id  = 1;
		return;
	end
	
	if grep(dia,"error on test")<>[] then
		status_msg     = "failed  : one or several unit tests failed";
		status_details = sprintf("     Check the following file : \n     - %s",tmp_diafile);
		status_details = [ status_details ; sprintf("     Or launch the following command : \n     - exec %s;",tstfile) ];
		status_id      = 2;
		return;
	end
	
	return;
	
endfunction

//-----------------------------------------------------------------------------
// Pierre MARECHAL
// Scilab team
// Copyright INRIA
// Date : 28 oct. 2007
//
// => Check ref or generate ref
//-----------------------------------------------------------------------------

function [status_id,status_msg,status_details] = unit_test_run_proc_ref(module,test)
	
	global create_ref;
	global check_ref;
	
	status_id      = 0 ;
	status_msg     = "passed" ;
	status_details = "";
	
	// Some definitions
	
	tstfile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".tst",%f,%f);
	diafile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".dia",%f,%f);
	reffile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".dia.ref",%f,%f);
	
	tmp_tstfile = pathconvert(TMPDIR+"/"+test+"_2.tst",%f,%f);
	tmp_diafile = pathconvert(TMPDIR+"/"+test+"_2.dia",%f,%f);
	tmp_resfile = pathconvert(TMPDIR+"/"+test+"_2.res",%f,%f);
	tmp_errfile = pathconvert(TMPDIR+"/"+test+"_2.err",%f,%f);
	
	// On test l'existence de la ref si besoin
	
	if check_ref then
		if fileinfo(reffile) == [] then
			status_msg     = "failed  : the ref file doesn''t exist";
			status_details = "     Add or create the following file"+reffile+" file";
			status_details = sprintf("     Add or create the following file : \n     - %s",reffile);
			status_id      = 5;
			return;
		end
	end
	
	// Remove the previous tmp files
	if fileinfo(tmp_tstfile) <> [] then
		deletefile(tmp_tstfile)
	end
	
	if fileinfo(tmp_diafile) <> [] then
		deletefile(tmp_diafile)
	end
	
	if fileinfo(tmp_resfile) <> [] then
		deletefile(tmp_resfile)
	end
	
	if fileinfo(tmp_errfile) <> [] then
		deletefile(tmp_errfile)
	end
	
	//Reset standard globals
	rand('seed',0);
	rand('uniform');

	// Do some modification in  tst file
	
	txt = mgetl(tstfile);
	txt = strsubst(txt,'pause,end','bugmes();quit;end');
	txt = strsubst(txt,'-->','@#>'); //to avoid suppression of input --> with prompts
	txt = strsubst(txt,'halt()','');
	
	// Header : the same for each test
	
	head = [	"// <-- HEADER START -->";
				"mode(3);" ;
				"clear;" ;
				"lines(28,72);";
				"lines(0);" ;
				"deff(''[]=bugmes()'',''write(%io(2),''''error on test'''')'');" ;
				"predef(''all'');" ;
				"tmpdirToPrint = msprintf(''TMPDIR1=''''%s''''\n'',TMPDIR);" ;
				"diary(''"+tmp_diafile+"'');";
				"write(%io(2),tmpdirToPrint);";
				"// <-- HEADER END -->"];
				
	tail = [	"// <-- FOOTER START -->";
				"diary(0);";
				"exit;";
				"// <-- FOOTER END -->"]
	
	txt = [head;
		txt;
		tail];
	
	// and save it in a temporary file
	mputl(txt,tmp_tstfile);
	
	// Delete previous dia file
	if fileinfo(diafile) <> [] then
		deletefile(diafile)
	end
	
	// Build the command to launch
	if MSDOS then
		unit_test_cmd = "( """+SCI+"\bin\scilex.exe"+""""+" "+launch_mode+" -nb -args -nouserstartup -f """+tmp_tstfile+""" > """+tmp_resfile+""" ) 2> """+tmp_errfile+"""";
	else
		unit_test_cmd = "( "+SCI+"/bin/scilab "+launch_mode+" -nb -args -nouserstartup -f "+tmp_tstfile+" > "+tmp_resfile+" ) 2> "+tmp_errfile;
	end
	
	// Launch the test exec
	host(unit_test_cmd);
	
	// First Check : error output
	if check_error_output then
		tmp_errfile_info = fileinfo(tmp_errfile);
		
		if ( (tmp_errfile_info <> []) & (tmp_errfile_info(1)<>0) ) then
			status_msg = "failed  : error_output not empty"
			status_details = sprintf("     Check the following file : \n     - %s",tmp_errfile);
			status_id  = 5;
			return;
		end
	end
	
	//  Do some modification in  dia file
	dia = mgetl(tmp_diafile);
	
	// To get TMPDIR line
	tmpdir1_line = grep(dia,"TMPDIR1");
	execstr(dia(tmpdir1_line));
	
	// Remove Header and Footer
	dia = remove_headers(dia);
	
	//  Do some modification in  dia file
	dia(grep(dia,"exec("))                     = [];
	dia(grep(dia,"write(%io(2),tmpdirToPrint"))= [];
	dia(grep(dia,"TMPDIR1"))                   = [];
	dia(grep(dia,"diary(0)"))                  = [];
	
	dia = strsubst(dia,TMPDIR,'TMPDIR');
	dia = strsubst(dia,TMPDIR1,'TMPDIR');
	dia = strsubst(dia,TMPDIR1,'TMPDIR');
	dia = strsubst(dia,SCI,'SCI');
	
	//suppress the prompts
	dia = strsubst(dia,'-->','');
	dia = strsubst(dia,'@#>','-->');
	dia = strsubst(dia,'-1->','');
	
	//standardise  number display
	dia = strsubst(strsubst(strsubst(strsubst(dia,' .','0.'),'E+','D+'),'E-','D-'),'-.','-0.');

	//not to change the ref files
	dia=strsubst(dia,'bugmes();return','bugmes();quit');

	if create_ref then
		
		// Delete previous .dia.ref file
		if fileinfo(reffile) <> [] then
			deletefile(reffile)
		end
		
		mputl(dia,reffile);
		
		status_msg = "passed : ref created";
		status_id  = 20;
		
		return;
		
	else
		
		// write down the resulting dia file
		mputl(dia,diafile)
		
		//Check for diff with the .ref file
		
		[u,ierr] = mopen(reffile,'r');
		if ierr== 0 then //ref file exists
			
			ref=mgetl(u);
			mclose(u)
			
			// suppress blank (diff -nw)
			
			dia = strsubst(dia,' ','')
			ref = strsubst(ref,' ','')
			
			dia(find(dia=='')) = [];
			ref(find(ref=='')) = [];
			
			dia(find(dia=='')) = [];
			ref(find(ref=='')) = [];
			
			if or(ref<>dia) then
				if MSDOS then
					status_msg     = "failed  : dia and ref are not equal";
					status_details = sprintf("     Compare the following files : \n     - %s\n     - %s",diafile,reffile);
					status_id      = 4;
				else
					status_msg     = "failed  : dia and ref are not equal";
					status_details = sprintf("     Compare the following files : \n     - %s\n     - %s",diafile,reffile);
					status_id      = 4;
				end
			end
		else
			error(sprintf(gettext("The ref file (%s) doesn''t exist"),reffile));
		end
	end
	
	return;
	
endfunction

//-----------------------------------------------------------------------------
// Pierre MARECHAL
// Scilab team
// Copyright INRIA
// Date : 8 novembre 2007
//
// => remove header from the diary txt
//
//-----------------------------------------------------------------------------

function dia_out = remove_headers(dia_in)
	
	dia_out = dia_in;
	
	body_start = grep(dia_out,"// <-- HEADER END -->");
	
	if body_start<>[] then
		dia_out(1:body_start(1)) = [];
	end
	
	body_end   = grep(dia_out,"// <-- FOOTER START -->");
	
	if body_end<>[] then
		[dia_nl,dia_nc] = size(dia);
		dia_out(body_end(1):dia_nl) = [];
	end
	
	return;
	
endfunction

//-----------------------------------------------------------------------------
// Pierre MARECHAL
// Scilab team
// Copyright INRIA
// Date : 28 oct. 2007
//
// => Check ref or generate ref
//-----------------------------------------------------------------------------

function example = unit_test_examples()
	
	example = [ sprintf("Examples :\n\n") ];

	
	example = [ example ; sprintf("// Launch all tests\n") ];
	example = [ example ; sprintf("unit_test_run();\n") ];
	example = [ example ; sprintf("unit_test_run([]);\n") ];
	example = [ example ; sprintf("unit_test_run([],[]);\n") ];
	example = [ example ; "" ];
	example = [ example ; sprintf("// Test one or several module\n") ];
	example = [ example ; sprintf("unit_test_run(''core'');\n") ];
	example = [ example ; sprintf("unit_test_run(''core'',[]);\n") ];
	example = [ example ; sprintf("unit_test_run([''core'',''string'']);\n") ];
	example = [ example ; "" ];
	example = [ example ; sprintf("// Launch one or several test in a specified module\n") ];
	example = [ example ; sprintf("unit_test_run(''core'',[''trycatch'',''opcode'']);\n") ];
	example = [ example ; "" ];
	example = [ example ; sprintf("// With options\n") ];
	example = [ example ; sprintf("unit_test_run([],[],''no_check_ref'');\n") ];
	example = [ example ; sprintf("unit_test_run([],[],''no_check_error_output'');\n") ];
	example = [ example ; sprintf("unit_test_run([],[],''create_ref'');\n") ];
	example = [ example ; sprintf("unit_test_run([],[],''list'');\n") ];
	example = [ example ; sprintf("unit_test_run([],[],''help'');\n") ];
	example = [ example ; sprintf("unit_test_run([],[],[''no_check_ref'',''mode_nw'']);\n") ];
	example = [ example ; "" ];
	
endfunction