/dev/null 2>".EXEC_ERRORFILE." & echo $!"); } } $fdRead = fopen(INFOFILE,"r+"); if(!$fdRead){ $errStr = "Could not open file ".INFOFILE." to read, Reason:".customErrorMsg(); traceLog($errStr); return; } while (1) { if(!file_exists(PIDFILE)) break; if(empty($line)) $line = fgets($fdRead); if(empty($line)){ sleep(1); fseek($fdRead, 0, 1); #to clear eof flag continue; } if(preg_match('/^TOTALFILES/', $line) === 1) { $lineSplit = explode(' ', $line,2); $totalFiles = trim($lineSplit[1]); $line = ""; sleep(2); break; } $line = trim($line); if(($totalEngineRestore > 1) && extension_loaded('pcntl') && function_exists('pcntl_fork')) { $retVal = isEngineRunning(PIDFILE."_$engineID"); if($retVal === false){ if(isAnyEngineRunning(ENGINE_LOCKE_FILE)){ #check if any engine is running if running then check down code block or skip while(1){ if(!file_exists(PIDFILE) or !isAnyEngineRunning(ENGINE_LOCKE_FILE)) break; $exec_loads = sys_getloadavg(); if($exec_loads[1] > $exec_cores){ sleep(20); continue; } break; } } if(!file_exists(PIDFILE)) break; if($failedfiles_index != -1) { $failedfiles_index++; if($failedfiles_index > 2000000000){ $failedfiles_index = 0; } } $pid_generate_doRestore = pcntl_fork(); if($pid_generate_doRestore == -1){ $errStr = "Unable to start doRestoreOperation. Reason:".customErrorMsg(); exit_cleanup(); } else if ($pid_generate_doRestore == 0) # this child process will create all backup set files with 1000-1000 files set { $processName = 'dorestore'; doRestoreOperation($line,$engineID,$failedfiles_index); exit; } else{ $RestoreForkchilds[] = $pid_generate_doRestore; if(isset($exec_loads[1]) && ($exec_loads[1] > $exec_cores)) sleep(2); else sleep(1); } $line = ""; } $engineID++; if($engineID > $totalEngineRestore){ $engineID = 1; sleep(2); } foreach($RestoreForkchilds as $key => $pid) { $res = pcntl_waitpid($pid, $status, WNOHANG); // If the process has already exited if($res == -1 || $res > 0){ unset($RestoreForkchilds[$key]); } } } else{ if($failedfiles_index != -1) { $failedfiles_index++; if($failedfiles_index > 2000000000){ $failedfiles_index = 0; } } doRestoreOperation($line,1,$failedfiles_index); $line = ""; } } waitForProcess($pidGen,NOSCANERRORFILE); waitForEnginesToFinish(); $line = fgets($fdRead); if(preg_match('/^FAILEDCOUNT/', $line) === 1){ $lineSplit = explode(' ', $line,2); $failedFilesCount = trim($lineSplit[1]); } fclose($fdRead); if($totalFiles == 0){ $fdRead = fopen(INFOFILE,"r"); if(!$fdRead){ $errStr = "Could not open file ".INFOFILE." to read, Reason:".customErrorMsg(); traceLog($errStr); return; } fclose($fdRead); $file_content = file(INFOFILE); foreach($file_content as $file_content_data) { if(preg_match('/^TOTALFILES/', $file_content_data) === 1){ $split_totalFiles = explode(' ',$file_content_data,2); $totalFiles = trim($split_totalFiles[1]); } if(preg_match('/^FAILEDCOUNT/', $file_content_data) === 1){ $lineSplit = explode(' ', $file_content_data,2); $failedFilesCount = trim($lineSplit[1]); } } if($totalFiles == 0) traceLog("Unable to find total file count from info file."); } if(file_exists(RETRY_INFOFILE) && @customFileSize(RETRY_INFOFILE) > 0 && $retrycount < MAX_RETRY && file_exists(PIDFILE)) { rename(RETRY_INFOFILE, INFOFILE); if (file_exists(INFOFILE)) { $fd_write = fopen(INFOFILE, 'a'); if(!$fd_write){ $errStr = "Could not open file ".INFOFILE." to read in retry case, Reason:".customErrorMsg(); traceLog($errStr); return; } fwrite($fd_write, "TOTALFILES $totalFiles\n FAILEDCOUNT $failedFilesCount"); fclose($fd_write); @chmod(INFOFILE,PERMISSIONVAL); } updateRetryCount(); if($retrycount == MAX_RETRY - 1) $failedfiles_index = -1; startRestore(1); } } #**************************************************************************************************** # Subroutine Name : doRestoreOperation. # Objective : This subroutine performs the actual task of backing up files. It creates # a child process which executes the restore command. It also creates a process # which continuously monitors the temporary output file. At the end of restore, # it inspects the temporary error file if present. It then deletes the temporary # output file, temporary error file and the temporary directory created by # idevsutil binary. # Modified By : Vijay Vinoth, Sabin Cheruvattil #*****************************************************************************************************/ function doRestoreOperation($line, $engineID, $failedfiles_index) { GLOBAL $isLocalRestore, $idevsutilBinaryPath, $dedupDetailFile, $webDir, $phpPath, $localBackupLocationFile, $logOutputFilePath, $filesonly, $operationStartTimeRestore; GLOBAL $errStr, $restoreUtfFile, $retrycount, $hostNameLB, $username, $evspid, $opprocpid; $pid_OutputProcess = ''; $startPidFileLock = fopen(ENGINE_LOCKE_FILE, 'a+'); $locked = flock($startPidFileLock, LOCK_SH|LOCK_NB); // Creating PID based on engine $engineFp = fopen(PIDFILE."_$engineID", 'w+'); fwrite($engineFp," "); $isLocked = flock($engineFp, LOCK_EX|LOCK_NB); if(!$isLocked){ return; } @chmod(PIDFILE."_$engineID",PERMISSIONVAL); $opType = RESTORE; $parameters = $line; $parameter_list = explode(' ', $parameters, SPLIT_LIMIT_INFO_LINE); $source = ''; if($isLocalRestore == 1){ $opType = LOCALRESTORE; if(file_exists($localBackupLocationFile) && file_exists($dedupDetailFile)){ $dbpaths = $hostNameLB.'/'.$username.'/dbpaths.xml'; $serverRoot = getServerRoot($dbpaths,$localBackupLocationFile); $source = isset($parameter_list[2])?'/'.$serverRoot:'/'.'/'.$serverRoot; } } if($source == '') $source = isset($parameter_list[2])?$parameter_list[2]:'/'; $restoreUtfFile = getOperationFile(array("opType" => $opType, "backupsetFile" => $parameter_list[0], "backupType" => $parameter_list[1], "source" => $source, "engineID" => $engineID)); # EVS command to execute for restore $idevsutilCommandLine = $idevsutilBinaryPath.WHITESPACE.IDEVSUTIL_ARG.ASSIGNMENTOP."\"$restoreUtfFile\""; if(file_exists(PIDFILE)) { $pid = exec("$idevsutilCommandLine > /dev/null 2>".EXEC_ERRORFILE." & echo $!"); $evspid = $pid; # restore child process for output file parsing $fileChildProcessPath = $webDir.PATHSEPARATOR.APPDIR.PATHSEPARATOR.SCRIPTS.PATHSEPARATOR.RESTORE_CHILDPROCESSFILE; # fix for source which contains special characters like "", '' etc. if($retrycount == 0) $source = preg_quote($source); $childCmd = "$phpPath $fileChildProcessPath \"$logOutputFilePath\" \"$parameter_list[0]\" \"$parameter_list[1]\" \"$source\" \"$filesonly\" \"$operationStartTimeRestore\" \"$engineID\" \"$failedfiles_index\" > /dev/null 2>".EXEC_ERRORFILE." & echo $!"; $pid_OutputProcess = exec($childCmd); $opprocpid = $pid_OutputProcess; waitForEVS($pid, SCANERRORFILE, $engineID); $evspid = 0; } # update server address if cmd failed due to wrong evs server address checkErrorAndUpdateServerAddr("_$engineID"); $outputFile = fopen(OUTPUTFILE."_$engineID", 'a+'); if($outputFile) { fwrite($outputFile,"CHILD_PROCESS_COMPLETED".PHP_EOL); fclose($outputFile); } else { traceLog("Unable to open file ".OUTPUTFILE.", Reason: ".customErrorMsg()); @unlink(UTFFILE."_$engineID"); fclose($engineFp); @unlink(PIDFILE."_$engineID"); flock($startPidFileLock, LOCK_UN); fclose($startPidFileLock); return 0; } if($pid_OutputProcess != '') { waitForEVS($pid_OutputProcess, NOSCANERRORFILE); $opprocpid = 0; } @unlink(UTFFILE."_$engineID"); fclose($engineFp); @unlink(PIDFILE."_$engineID"); flock($startPidFileLock, LOCK_UN); fclose($startPidFileLock); return RESTORE_SUCCESS; } #**************************************************************************************************** # Subroutine Name : checkErrorFile. # Objective : This function will compare error.txt content with error msgs and terminate the job if required. # Added By : Vijay Vinoth. #*****************************************************************************************************/ function checkErrorFile(&$prevErrorSize, $engineID) { GLOBAL $ErrorArgumentsExit, $phpPath, $fileJobTerminationPath; #GLOBAL $dedupDetailFile, $dedupRestoreDeviceID; #checking if error.txt has size if(!customFileSize(ERRORFILE."_$engineID")) { return 0; } $errorFileContents = file(ERRORFILE."_$engineID"); if(!$errorFileContents) { traceLog("Unable to open file ".ERRORFILE.", Reason:".customErrorMsg()); return 0; } $linesRestoreErrorFile = $errorFileContents; $errorline = "idevs error"; $linesRestoreErrorFileCount = count($linesRestoreErrorFile); for($i = 0; $i<= $linesRestoreErrorFileCount; $i++) { if($linesRestoreErrorFile[$i] == "" or strpos( $linesRestoreErrorFile[$i], $errorline ) !== false) { continue; } $ErrorArgumentsExitCount = count($ErrorArgumentsExit); for($j=0; $j<=$ErrorArgumentsExitCount; $j++) { if(strpos( $linesRestoreErrorFile[$i], $ErrorArgumentsExit[$j] ) !== false) { checkDedupDeviceError($linesRestoreErrorFile[$i]); $errStr = "Operation could not be completed. Reason : $ErrorArgumentsExit[$j] \ntotalFiles : $totalFiles \n"; exec("$phpPath $fileJobTerminationPath 2 '".$ErrorArgumentsExit[$j]."'"); unlink(ERRORFILE."_$engineID"); return 1; } } } return 0; } #**************************************************************************************************** # Subroutine Name : appendEnumerationError. # Objective : Enumeration of a folder from restore set file gets fail then write # proper error message to log file. # Added By : Vijay Vinoth. #*****************************************************************************************************/ function appendEnumerationError($fileName) { GLOBAL $jobrunningDir, $outputErrorFile, $errorFile; GLOBAL $failedFilesCount, $totalFiles; $errorFile = $jobrunningDir.PATHSEPARATOR.ERROR_DIR.PATHSEPARATOR.TEMP_FILES.PATHSEPARATOR.FAILEDFILES; if($outputErrorFile) fwrite($outputErrorFile,"[".trim(shell_exec('date +"%a %b %d %T %Y"'))."] [FAILED] [$fileName] Reason : No such file or directory.".LINEFEED); $failedFilesCount++; $totalFiles++; } #**************************************************************************************************** # Subroutine Name : exit_cleanup. # Objective : This function will execute the major functions required at the time of exit # Added By : Deepak Chaurasia # Modified By : Avinash Kumar, Sabin Cheruvattil #*****************************************************************************************************/ function exit_cleanup() { GLOBAL $jobrunningDir, $restoreUtfFile, $errStr, $successFiles, $syncedFiles, $failedFiles, $cancelFlag, $status; GLOBAL $cancelFlag, $exit_flag, $issignal, $processName, $issignal; list($successFiles, $syncedFiles, $failedFiles, $exit_flag) = getParameterValueFromStatusFile(array('FILES_COUNT_INDEX', 'SYNC_COUNT_FILES_INDEX', 'FAILED_COUNT_FILES_INDEX', 'EXIT_FLAG_INDEX')); /*if(file_exists(EVS_TEMP)) removeItems(EVS_TEMP);*/ if(file_exists(PIDFILE)) { unlink(PIDFILE); } else if(!$issignal) { # In childprocess, if we exit due to some exit scenario, then this exit_flag will be true with error msg $exit = explode("-",$exit_flag,2); $cancelFlag = true; if($exit[0] && $exit[1] != ""){ $errStr = $exit[1]; checkDedupDeviceError($errStr); } else $errStr = readFromUserCancel(); traceLog($errStr); } else if($issignal) { $cancelFlag = TRUE; } writeOperationSummary(RESTORE); @unlink($jobrunningDir."/error.txt"); getOperationStatus(); updateLogFileName(); sendMail(); @unlink($restoreUtfFile); @unlink(OUTPUTFILE); removeItems(OUTPUTFILE."*"); removeItems(UTFFILE."*"); removeRestoreTempFiles(); //@fclose($fhsPid); @unlink(STARTPIDFILE); @unlink(CANCEL_PATH); if(file_exists(ENGINE_LOCKE_FILE)) @unlink(ENGINE_LOCKE_FILE); if(file_exists("operation_".ERROR_FILE)) @unlink("operation_".ERROR_FILE); if(file_exists("operation_".OUTPUTFILE)) @unlink("operation_".OUTPUTFILE); if(file_exists("operation_".UTF8_FILE)) @unlink("operation_".UTF8_FILE); if(file_exists(TEMP_OPERATION_FILE)) @unlink(TEMP_OPERATION_FILE); if(file_exists(OPERATION_SUMMERY_ERROR_FILE)) @unlink(OPERATION_SUMMERY_ERROR_FILE); exit; } #**************************************************************************************************** # Subroutine Name : process_term. # Objective : The signal handler invoked when SIGINT or SIGTERM signal is received by the script # Added By : Arnab Gupta # Modified by : Vijay Vinoth, Sabin Cheruvattil #*****************************************************************************************************/ function process_term($signal) { GLOBAL $isErrorFileOpen, $processName, $pidGen, $pid_OutputProcess, $errStr, $dbh, $FD_WRITE, $bacSync, $issignal; GLOBAL $totalFiles, $failedFilesCount, $outputFile, $openRestoreFileOrg, $filehandle, $outputErrorFile, $evspid, $opprocpid; $issignal = TRUE; if($processName == "main") { if(file_exists(PIDFILE)) { @unlink(PIDFILE); } if(file_exists(STARTPIDFILE)) { @unlink(STARTPIDFILE); } waitForProcess($pidGen, NOSCANERRORFILE); /* if($opprocpid) { waitForEVS($opprocpid, NOSCANERRORFILE); } */ waitForEnginesToFinish(); $filecontent = file(INFOFILE); for($i = count($filecontent)-1; $i >= 0; $i--) { if(preg_match("/^FAILEDCOUNT/",$filecontent[$i])) { $lineSplit = preg_split('/ /', $filecontent[$i],2); $failedFilesCount = $lineSplit[1]; $failedFilesCount = trim($failedFilesCount); } if(preg_match("/^TOTALFILES/",$filecontent[$i])) { $split_totalFiles = preg_split('/ /',$filecontent[$i],2); $totalFiles = $split_totalFiles[1]; $totalFiles = trim($totalFiles); } } $errStr = "Operation could not be completed. Reason: Process has been killed due to restart/shutdown."; exit_cleanup(); } else if($processName == "dorestore") { if($evspid) { `kill $evspid`; } if($opprocpid) { waitForEVS($opprocpid, NOSCANERRORFILE); } // waitForEnginesToFinish(); # engines lock will release if(file_exists(ENGINE_LOCKE_FILE)) { @unlink(ENGINE_LOCKE_FILE); } } else if($processName == "generate") { fwrite($FD_WRITE, "TOTALFILES $totalFiles\n"); fwrite($FD_WRITE, "FAILEDCOUNT $failedFilesCount"); fclose($FD_WRITE); if($isErrorFileOpen){ @fclose($outputErrorFile); } if($outputFile){ @fclose($outputFile); } if($openRestoreFileOrg){ @fclose($openRestoreFileOrg); } if($filehandle){ @fclose($filehandle); } # close DB connection for offline restore if($dbh_LB){ unset($dbh_LB); } if($errStr != ""){ # write $errStr to status file. writeOperationErrorToErrorFile($errStr); } } } #**************************************************************************************************** # Subroutine Name : waitForEnginesToFinish. # Objective : wait for fork process to complete # Added By : Vijay Vinoth #*****************************************************************************************************/ function waitForEnginesToFinish() { GLOBAL $RestoreForkchilds; while(count($RestoreForkchilds) > 0) { foreach($RestoreForkchilds as $key => $pid) { $res = pcntl_waitpid($pid, $status, WNOHANG); // If the process has already exited if($res == -1 || $res > 0){ unset($RestoreForkchilds[$key]); } } sleep(1); } while(isAnyEngineRunning(ENGINE_LOCKE_FILE)) sleep(1); return; } ?>