Files
resatennis_tyroltenn/IDrive/.scripts/Restore_Script_IDrive.php

668 lines
21 KiB
PHP
Executable File

<?php
GLOBAL $webDir, $APPDIR, $currentFilePath;
include('sqliteDB.php');
include('generateRestoresetFiles.php');
if(extension_loaded('pcntl') && function_exists('pcntl_signal')) {
// Subroutine that processes SIGINT and SIGTERM signal received by the script during restore
declare(ticks = 1);
pcntl_signal(SIGTERM, 'process_term');// Termination ('kill' was called)
pcntl_signal(SIGINT, 'process_term'); // Interrupted (Ctrl-C is pressed)
}
#create pid.txt if required.
createPid($taskType);
$operationName = RESTORE;
# machine state handling
$evspid = 0;
$opprocpid = 0;
$issignal = FALSE;
# pre cleanup intermediate file and folder.
removeRestoreTempFiles();
checkBinaryExists();
createLogFiles($outputFileName);
writeLogHeader();
$restoreSetEmpty = checkIfSetEmpty($restoreDisplayPath);
if($restoreSetEmpty) {
if(!($logOutputHandle = fopen($logOutputFilePath,"a"))){
traceLog("Could not open file $logOutputFilePath, Reason:".customErrorMsg().LINEFEED);
exit_cleanup();
}
fwrite($logOutputHandle, "Your Restore set is empty.".LINEFEED);
fclose($logOutputHandle);
exit_cleanup();
}
$exec_cores = getSystemCpuCores();
startRestore();
waitForEnginesToFinish();
if($errStr != "")
writeOperationErrorToErrorFile($errStr);
exit_cleanup();
#******************************************************************************************************************
# Subroutine Name : writeLogHeader.
# Objective : This subroutine construct log header message and return constructed message.
# Added By : Vijay Vinoth
#******************************************************************************************************************/
function writeLogHeader()
{
GLOBAL $headerMsg, $logOutputFilePath, $nasUsername;
GLOBAL $username, $hostName;
$errStr = '';
# added by anil for log related issues
verifyLastUserLog(LOGPIDFILE);
# require to open log file to show job in progress as well as to log exclude details
$logOutputHandle = fopen($logOutputFilePath, 'a+');
if($logOutputHandle)
{
# As log file is opened so create logpid file as well
$currPath = trim(getcwd());
if(file_put_contents(LOGPIDFILE,"$currPath/$logOutputFilePath"))
chmod(LOGPIDFILE,PERMISSIONVAL);
else {
$errStr = "Unable to open file ".LOGPIDFILE.", Reason:".customErrorMsg();
fwrite($logOutputHandle,$errStr.PHP_EOL);
fclose($logOutputHandle);
traceLog($errStr);
exit_cleanup();
}
chmod($logOutputFilePath,PERMISSIONVAL);
}
else {
$errStr = "Failed to create $logOutputFilePath, Reason:".customErrorMsg();
traceLog($errStr);
exit_cleanup();
}
//$headerMsg = "Restore Start Time: ".WHITESPACE.date("D M d H:i:s Y").LINEFEED;
$headerMsg = "Restore Start Time: ".WHITESPACE.trim(shell_exec('date +"%a %b %d %T %Y"')).LINEFEED;
$logheader = $headerMsg.LINEFEED."NAS User name: ".$nasUsername.LINEFEED;
$logheader .= "IDrive User name: ".$username.LINEFEED;
$headerMsg .= getHeaderMsg();
$headerMsg .= "Restore Location: $hostName".LINEFEED;
$logheader .= getHeaderMsg();
$logheader .= "Restore Location: $hostName".LINEFEED.LINEFEED;
# write log header.
fwrite($logOutputHandle,$logheader.PHP_EOL);
if($logOutputHandle)
fclose($logOutputHandle);
}
#****************************************************************************************************
# Subroutine Name : startRestore.
# Objective : Start restore operation by reading the content of info_file.
# The info_file is getting created and written by generateRestoresetFiles()
# subroutine. Check the retry condition and perform retry to restore,
# if needed.
# Added By : Avinash Kumar.
#*****************************************************************************************************/
function startRestore($startRestoreInitiateVar = 0)
{
GLOBAL $retrycount,$totalEngineRestore, $totalFiles, $startRestore, $failedfiles_index, $current_source, $processName;
GLOBAL $jobrunningDir, $statusFileHash, $exec_cores, $phpPath, $webDir, $APPDIR, $logOutputFilePath, $RestoreForkchilds;
$engineID = 1;
if($startRestoreInitiateVar == 0){
# creating temp folder for storing intermediate restore set files and info file
if(!is_dir(TEMP_FILES)){
if(!mkdir(TEMP_FILES)){
$errStr = "Unable to create directory ".TEMP_FILES.", Reason:".customErrorMsg();
exitError($errStr);
}
@chmod(TEMP_FILES,PERMISSIONVAL);
}
file_put_contents(ENGINE_LOCKE_FILE,"");
@chmod(ENGINE_LOCKE_FILE,PERMISSIONVAL);
$ourFileHandle = fopen(INFOFILE, 'w+');
if(!$ourFileHandle){
$errStr = "Could not open file ".INFOFILE." to write, Reason:".customErrorMsg();
traceLog($errStr);
return;
}
@chmod(INFOFILE,PERMISSIONVAL);
fclose($ourFileHandle);
if( extension_loaded('pcntl') && function_exists('pcntl_fork') ){
$pidGen = pcntl_fork();
if($pidGen == -1){
$errStr = "Unable to start generateRestoresetFiles operation";
traceLog($errStr);
return;
}
if($pidGen == 0){
$processName = "generate";
generateRestoresetFiles();
if($dbh_LB) {
unset($dbh_LB);
}
exit;
}
} else {
$forkProcessPath = $webDir.PATHSEPARATOR.APPDIR.PATHSEPARATOR.SCRIPTS.PATHSEPARATOR.FORKPROCESSFILE;
$cmd = "$phpPath $forkProcessPath 'restore' '$logOutputFilePath'";
$pidGen = exec("$cmd > /dev/null 2>".EXEC_ERRORFILE." & echo $!");
}
}
$fdRead = fopen(INFOFILE,"r+");
if(!$fdRead){
$errStr = "Could not open file ".INFOFILE." to read, Reason:".customErrorMsg();
traceLog($errStr);
return;
}
while (1)
{
if(!file_exists(PIDFILE))
break;
if(empty($line))
$line = fgets($fdRead);
if(empty($line)){
sleep(1);
fseek($fdRead, 0, 1); #to clear eof flag
continue;
}
if(preg_match('/^TOTALFILES/', $line) === 1)
{
$lineSplit = explode(' ', $line,2);
$totalFiles = trim($lineSplit[1]);
$line = "";
sleep(2);
break;
}
$line = trim($line);
if(($totalEngineRestore > 1) && extension_loaded('pcntl') && function_exists('pcntl_fork'))
{
$retVal = isEngineRunning(PIDFILE."_$engineID");
if($retVal === false){
if(isAnyEngineRunning(ENGINE_LOCKE_FILE)){
#check if any engine is running if running then check down code block or skip
while(1){
if(!file_exists(PIDFILE) or !isAnyEngineRunning(ENGINE_LOCKE_FILE))
break;
$exec_loads = sys_getloadavg();
if($exec_loads[1] > $exec_cores){
sleep(20);
continue;
}
break;
}
}
if(!file_exists(PIDFILE))
break;
if($failedfiles_index != -1) {
$failedfiles_index++;
if($failedfiles_index > 2000000000){
$failedfiles_index = 0;
}
}
$pid_generate_doRestore = pcntl_fork();
if($pid_generate_doRestore == -1){
$errStr = "Unable to start doRestoreOperation. Reason:".customErrorMsg();
exit_cleanup();
}
else if ($pid_generate_doRestore == 0) # this child process will create all backup set files with 1000-1000 files set
{
$processName = 'dorestore';
doRestoreOperation($line,$engineID,$failedfiles_index);
exit;
}
else{
$RestoreForkchilds[] = $pid_generate_doRestore;
if(isset($exec_loads[1]) && ($exec_loads[1] > $exec_cores))
sleep(2);
else
sleep(1);
}
$line = "";
}
$engineID++;
if($engineID > $totalEngineRestore){
$engineID = 1;
sleep(2);
}
foreach($RestoreForkchilds as $key => $pid) {
$res = pcntl_waitpid($pid, $status, WNOHANG);
// If the process has already exited
if($res == -1 || $res > 0){
unset($RestoreForkchilds[$key]);
}
}
}
else{
if($failedfiles_index != -1) {
$failedfiles_index++;
if($failedfiles_index > 2000000000){
$failedfiles_index = 0;
}
}
doRestoreOperation($line,1,$failedfiles_index);
$line = "";
}
}
waitForProcess($pidGen,NOSCANERRORFILE);
waitForEnginesToFinish();
$line = fgets($fdRead);
if(preg_match('/^FAILEDCOUNT/', $line) === 1){
$lineSplit = explode(' ', $line,2);
$failedFilesCount = trim($lineSplit[1]);
}
fclose($fdRead);
if($totalFiles == 0){
$fdRead = fopen(INFOFILE,"r");
if(!$fdRead){
$errStr = "Could not open file ".INFOFILE." to read, Reason:".customErrorMsg();
traceLog($errStr);
return;
}
fclose($fdRead);
$file_content = file(INFOFILE);
foreach($file_content as $file_content_data)
{
if(preg_match('/^TOTALFILES/', $file_content_data) === 1){
$split_totalFiles = explode(' ',$file_content_data,2);
$totalFiles = trim($split_totalFiles[1]);
}
if(preg_match('/^FAILEDCOUNT/', $file_content_data) === 1){
$lineSplit = explode(' ', $file_content_data,2);
$failedFilesCount = trim($lineSplit[1]);
}
}
if($totalFiles == 0)
traceLog("Unable to find total file count from info file.");
}
if(file_exists(RETRY_INFOFILE) && @customFileSize(RETRY_INFOFILE) > 0 && $retrycount < MAX_RETRY && file_exists(PIDFILE))
{
rename(RETRY_INFOFILE, INFOFILE);
if (file_exists(INFOFILE))
{
$fd_write = fopen(INFOFILE, 'a');
if(!$fd_write){
$errStr = "Could not open file ".INFOFILE." to read in retry case, Reason:".customErrorMsg();
traceLog($errStr);
return;
}
fwrite($fd_write, "TOTALFILES $totalFiles\n FAILEDCOUNT $failedFilesCount");
fclose($fd_write);
@chmod(INFOFILE,PERMISSIONVAL);
}
updateRetryCount();
if($retrycount == MAX_RETRY - 1)
$failedfiles_index = -1;
startRestore(1);
}
}
#****************************************************************************************************
# Subroutine Name : doRestoreOperation.
# Objective : This subroutine performs the actual task of backing up files. It creates
# a child process which executes the restore command. It also creates a process
# which continuously monitors the temporary output file. At the end of restore,
# it inspects the temporary error file if present. It then deletes the temporary
# output file, temporary error file and the temporary directory created by
# idevsutil binary.
# Modified By : Vijay Vinoth, Sabin Cheruvattil
#*****************************************************************************************************/
function doRestoreOperation($line, $engineID, $failedfiles_index)
{
GLOBAL $isLocalRestore, $idevsutilBinaryPath, $dedupDetailFile, $webDir, $phpPath, $localBackupLocationFile, $logOutputFilePath, $filesonly, $operationStartTimeRestore;
GLOBAL $errStr, $restoreUtfFile, $retrycount, $hostNameLB, $username, $evspid, $opprocpid;
$pid_OutputProcess = '';
$startPidFileLock = fopen(ENGINE_LOCKE_FILE, 'a+');
$locked = flock($startPidFileLock, LOCK_SH|LOCK_NB);
// Creating PID based on engine
$engineFp = fopen(PIDFILE."_$engineID", 'w+');
fwrite($engineFp," ");
$isLocked = flock($engineFp, LOCK_EX|LOCK_NB);
if(!$isLocked){
return;
}
@chmod(PIDFILE."_$engineID",PERMISSIONVAL);
$opType = RESTORE;
$parameters = $line;
$parameter_list = explode(' ', $parameters, SPLIT_LIMIT_INFO_LINE);
$source = '';
if($isLocalRestore == 1){
$opType = LOCALRESTORE;
if(file_exists($localBackupLocationFile) && file_exists($dedupDetailFile)){
$dbpaths = $hostNameLB.'/'.$username.'/dbpaths.xml';
$serverRoot = getServerRoot($dbpaths,$localBackupLocationFile);
$source = isset($parameter_list[2])?'/'.$serverRoot:'/'.'/'.$serverRoot;
}
}
if($source == '')
$source = isset($parameter_list[2])?$parameter_list[2]:'/';
$restoreUtfFile = getOperationFile(array("opType" => $opType,
"backupsetFile" => $parameter_list[0],
"backupType" => $parameter_list[1],
"source" => $source,
"engineID" => $engineID));
# EVS command to execute for restore
$idevsutilCommandLine = $idevsutilBinaryPath.WHITESPACE.IDEVSUTIL_ARG.ASSIGNMENTOP."\"$restoreUtfFile\"";
if(file_exists(PIDFILE)) {
$pid = exec("$idevsutilCommandLine > /dev/null 2>".EXEC_ERRORFILE." & echo $!");
$evspid = $pid;
# restore child process for output file parsing
$fileChildProcessPath = $webDir.PATHSEPARATOR.APPDIR.PATHSEPARATOR.SCRIPTS.PATHSEPARATOR.RESTORE_CHILDPROCESSFILE;
# fix for source which contains special characters like "", '' etc.
if($retrycount == 0)
$source = preg_quote($source);
$childCmd = "$phpPath $fileChildProcessPath \"$logOutputFilePath\" \"$parameter_list[0]\" \"$parameter_list[1]\" \"$source\" \"$filesonly\" \"$operationStartTimeRestore\" \"$engineID\" \"$failedfiles_index\" > /dev/null 2>".EXEC_ERRORFILE." & echo $!";
$pid_OutputProcess = exec($childCmd);
$opprocpid = $pid_OutputProcess;
waitForEVS($pid, SCANERRORFILE, $engineID);
$evspid = 0;
}
# update server address if cmd failed due to wrong evs server address
checkErrorAndUpdateServerAddr("_$engineID");
$outputFile = fopen(OUTPUTFILE."_$engineID", 'a+');
if($outputFile) {
fwrite($outputFile,"CHILD_PROCESS_COMPLETED".PHP_EOL);
fclose($outputFile);
} else {
traceLog("Unable to open file ".OUTPUTFILE.", Reason: ".customErrorMsg());
@unlink(UTFFILE."_$engineID");
fclose($engineFp);
@unlink(PIDFILE."_$engineID");
flock($startPidFileLock, LOCK_UN);
fclose($startPidFileLock);
return 0;
}
if($pid_OutputProcess != '') {
waitForEVS($pid_OutputProcess, NOSCANERRORFILE);
$opprocpid = 0;
}
@unlink(UTFFILE."_$engineID");
fclose($engineFp);
@unlink(PIDFILE."_$engineID");
flock($startPidFileLock, LOCK_UN);
fclose($startPidFileLock);
return RESTORE_SUCCESS;
}
#****************************************************************************************************
# Subroutine Name : checkErrorFile.
# Objective : This function will compare error.txt content with error msgs and terminate the job if required.
# Added By : Vijay Vinoth.
#*****************************************************************************************************/
function checkErrorFile(&$prevErrorSize, $engineID)
{
GLOBAL $ErrorArgumentsExit, $phpPath, $fileJobTerminationPath;
#GLOBAL $dedupDetailFile, $dedupRestoreDeviceID;
#checking if error.txt has size
if(!customFileSize(ERRORFILE."_$engineID"))
{
return 0;
}
$errorFileContents = file(ERRORFILE."_$engineID");
if(!$errorFileContents)
{
traceLog("Unable to open file ".ERRORFILE.", Reason:".customErrorMsg());
return 0;
}
$linesRestoreErrorFile = $errorFileContents;
$errorline = "idevs error";
$linesRestoreErrorFileCount = count($linesRestoreErrorFile);
for($i = 0; $i<= $linesRestoreErrorFileCount; $i++)
{
if($linesRestoreErrorFile[$i] == "" or strpos( $linesRestoreErrorFile[$i], $errorline ) !== false)
{
continue;
}
$ErrorArgumentsExitCount = count($ErrorArgumentsExit);
for($j=0; $j<=$ErrorArgumentsExitCount; $j++)
{
if(strpos( $linesRestoreErrorFile[$i], $ErrorArgumentsExit[$j] ) !== false)
{
checkDedupDeviceError($linesRestoreErrorFile[$i]);
$errStr = "Operation could not be completed. Reason : $ErrorArgumentsExit[$j] \ntotalFiles : $totalFiles \n";
exec("$phpPath $fileJobTerminationPath 2 '".$ErrorArgumentsExit[$j]."'");
unlink(ERRORFILE."_$engineID");
return 1;
}
}
}
return 0;
}
#****************************************************************************************************
# Subroutine Name : appendEnumerationError.
# Objective : Enumeration of a folder from restore set file gets fail then write
# proper error message to log file.
# Added By : Vijay Vinoth.
#*****************************************************************************************************/
function appendEnumerationError($fileName)
{
GLOBAL $jobrunningDir, $outputErrorFile, $errorFile;
GLOBAL $failedFilesCount, $totalFiles;
$errorFile = $jobrunningDir.PATHSEPARATOR.ERROR_DIR.PATHSEPARATOR.TEMP_FILES.PATHSEPARATOR.FAILEDFILES;
if($outputErrorFile)
fwrite($outputErrorFile,"[".trim(shell_exec('date +"%a %b %d %T %Y"'))."] [FAILED] [$fileName] Reason : No such file or directory.".LINEFEED);
$failedFilesCount++;
$totalFiles++;
}
#****************************************************************************************************
# Subroutine Name : exit_cleanup.
# Objective : This function will execute the major functions required at the time of exit
# Added By : Deepak Chaurasia
# Modified By : Avinash Kumar, Sabin Cheruvattil
#*****************************************************************************************************/
function exit_cleanup()
{
GLOBAL $jobrunningDir, $restoreUtfFile, $errStr, $successFiles, $syncedFiles, $failedFiles, $cancelFlag, $status;
GLOBAL $cancelFlag, $exit_flag, $issignal, $processName, $issignal;
list($successFiles, $syncedFiles, $failedFiles, $exit_flag) = getParameterValueFromStatusFile(array('FILES_COUNT_INDEX', 'SYNC_COUNT_FILES_INDEX', 'FAILED_COUNT_FILES_INDEX', 'EXIT_FLAG_INDEX'));
/*if(file_exists(EVS_TEMP))
removeItems(EVS_TEMP);*/
if(file_exists(PIDFILE)) {
unlink(PIDFILE);
} else if(!$issignal) {
# In childprocess, if we exit due to some exit scenario, then this exit_flag will be true with error msg
$exit = explode("-",$exit_flag,2);
$cancelFlag = true;
if($exit[0] && $exit[1] != ""){
$errStr = $exit[1];
checkDedupDeviceError($errStr);
} else
$errStr = readFromUserCancel();
traceLog($errStr);
} else if($issignal) {
$cancelFlag = TRUE;
}
writeOperationSummary(RESTORE);
@unlink($jobrunningDir."/error.txt");
getOperationStatus();
updateLogFileName();
sendMail();
@unlink($restoreUtfFile);
@unlink(OUTPUTFILE);
removeItems(OUTPUTFILE."*");
removeItems(UTFFILE."*");
removeRestoreTempFiles();
//@fclose($fhsPid);
@unlink(STARTPIDFILE);
@unlink(CANCEL_PATH);
if(file_exists(ENGINE_LOCKE_FILE))
@unlink(ENGINE_LOCKE_FILE);
if(file_exists("operation_".ERROR_FILE))
@unlink("operation_".ERROR_FILE);
if(file_exists("operation_".OUTPUTFILE))
@unlink("operation_".OUTPUTFILE);
if(file_exists("operation_".UTF8_FILE))
@unlink("operation_".UTF8_FILE);
if(file_exists(TEMP_OPERATION_FILE))
@unlink(TEMP_OPERATION_FILE);
if(file_exists(OPERATION_SUMMERY_ERROR_FILE))
@unlink(OPERATION_SUMMERY_ERROR_FILE);
exit;
}
#****************************************************************************************************
# Subroutine Name : process_term.
# Objective : The signal handler invoked when SIGINT or SIGTERM signal is received by the script
# Added By : Arnab Gupta
# Modified by : Vijay Vinoth, Sabin Cheruvattil
#*****************************************************************************************************/
function process_term($signal)
{
GLOBAL $isErrorFileOpen, $processName, $pidGen, $pid_OutputProcess, $errStr, $dbh, $FD_WRITE, $bacSync, $issignal;
GLOBAL $totalFiles, $failedFilesCount, $outputFile, $openRestoreFileOrg, $filehandle, $outputErrorFile, $evspid, $opprocpid;
$issignal = TRUE;
if($processName == "main") {
if(file_exists(PIDFILE)) {
@unlink(PIDFILE);
}
if(file_exists(STARTPIDFILE)) {
@unlink(STARTPIDFILE);
}
waitForProcess($pidGen, NOSCANERRORFILE);
/* if($opprocpid) {
waitForEVS($opprocpid, NOSCANERRORFILE);
} */
waitForEnginesToFinish();
$filecontent = file(INFOFILE);
for($i = count($filecontent)-1; $i >= 0; $i--) {
if(preg_match("/^FAILEDCOUNT/",$filecontent[$i])) {
$lineSplit = preg_split('/ /', $filecontent[$i],2);
$failedFilesCount = $lineSplit[1];
$failedFilesCount = trim($failedFilesCount);
}
if(preg_match("/^TOTALFILES/",$filecontent[$i])) {
$split_totalFiles = preg_split('/ /',$filecontent[$i],2);
$totalFiles = $split_totalFiles[1];
$totalFiles = trim($totalFiles);
}
}
$errStr = "Operation could not be completed. Reason: Process has been killed due to restart/shutdown.";
exit_cleanup();
} else if($processName == "dorestore") {
if($evspid) {
`kill $evspid`;
}
if($opprocpid) {
waitForEVS($opprocpid, NOSCANERRORFILE);
}
// waitForEnginesToFinish();
# engines lock will release
if(file_exists(ENGINE_LOCKE_FILE)) {
@unlink(ENGINE_LOCKE_FILE);
}
} else if($processName == "generate") {
fwrite($FD_WRITE, "TOTALFILES $totalFiles\n");
fwrite($FD_WRITE, "FAILEDCOUNT $failedFilesCount");
fclose($FD_WRITE);
if($isErrorFileOpen){
@fclose($outputErrorFile);
}
if($outputFile){
@fclose($outputFile);
}
if($openRestoreFileOrg){
@fclose($openRestoreFileOrg);
}
if($filehandle){
@fclose($filehandle);
}
# close DB connection for offline restore
if($dbh_LB){
unset($dbh_LB);
}
if($errStr != ""){
# write $errStr to status file.
writeOperationErrorToErrorFile($errStr);
}
}
}
#****************************************************************************************************
# Subroutine Name : waitForEnginesToFinish.
# Objective : wait for fork process to complete
# Added By : Vijay Vinoth
#*****************************************************************************************************/
function waitForEnginesToFinish()
{
GLOBAL $RestoreForkchilds;
while(count($RestoreForkchilds) > 0) {
foreach($RestoreForkchilds as $key => $pid) {
$res = pcntl_waitpid($pid, $status, WNOHANG);
// If the process has already exited
if($res == -1 || $res > 0){
unset($RestoreForkchilds[$key]);
}
}
sleep(1);
}
while(isAnyEngineRunning(ENGINE_LOCKE_FILE))
sleep(1);
return;
}
?>