$item){ $currentRestoresetValue = trim($item,"\n"); $current_RestoresetHash[$currentRestoresetValue] = 0; $RestoresetHashOrig_Encrypted[$currentRestoresetValue] = trim($currentRestoresetEncrypted[$key],"\n"); } } #**************************************************************************************************** # Subroutine Name : loadToHash. # Objective : This function manage the hash for failed file check. # Added By : Vijay Vinoth #*****************************************************************************************************/ function loadToHash($current_restoresetfile) { GLOBAL $current_RestoresetHash, $currentRestoreset, $engineID; $currentRestoreset = file($current_restoresetfile); if(empty($currentRestoreset)) { $errStr = "Unable to open current restore file $current_restoresetfile, Reason:".customErrorMsg()."\n"; traceLog($errStr); writeBackupErrorMsgToStatusFile($errStr,$engineID); exit; } $current_restoresetfile_size = customFileSize($current_restoresetfile); if($current_restoresetfile_size>0) { # only one file will be there for version restore $currentRestoreset[0] = preg_replace('/_IBVER+\d$/', '', $currentRestoreset[0]); reset($currentRestoreset); foreach($currentRestoreset as $key => $item){ $item = trim($item,"\n"); if(empty($item)) continue; $current_RestoresetHash[$item] = 0; } } } #**************************************************************************************************** # Subroutine Name : subOutputProcess. # Objective : This function monitors and parse the evs output file and creates the App # log file which is shown to user. # Modified By : Deepak Chaurasia, Vijay Vinoth. #*****************************************************************************************************/ function subOutputProcess() { GLOBAL $logOutputFilePath, $operationComplete, $opStartTimeRestore, $filesonly, $isLocalRestore, $skip, $restore_error; GLOBAL $current_RestoresetHash, $engineID, $xtrafileRestoreCount, $xtrafileSyncCount; $Restoretype = ""; # Flags to determine the status of OUTFILE $fileOpened = 0; $termData = "CHILD_PROCESS_COMPLETED"; $fields_in_progress = 10; $byteRead = 0; $lastLine = ""; # Waiting for EVS output file to open while(!$fileOpened) { if(!file_exists(PIDFILE)) break; if(file_exists(OUTPUTFILE."_$engineID")){ if($tempOutputFile = fopen(OUTPUTFILE."_$engineID",'r')){ $fileOpened = 1; } else { traceLog("Could not open file : ".OUTPUTFILE.", Reason : ".customErrorMsg()); break; } } sleep(2); } # As while is an indefinite loop for now so this condition will not make any sense but we may use it later if($fileOpened === 0) { traceLog("Unable to open file ".OUTPUTFILE); return; } # App log file open for parsed output $logOutputHandle = fopen($logOutputFilePath,"a"); if(!$logOutputHandle) { traceLog("Unable to open file $logOutputFilePath, Reason:".customErrorMsg()); fclose($tempOutputFile); fclose($logOutputHandle); chmod($logOutputFilePath,PERMISSIONVAL); return; } $prevTransferedSize = 0; $totalTransfered = 0; if(file_exists(PROGRESS_PATH."_$engineID") && customFileSize(PROGRESS_PATH."_$engineID")){ $progressContent = file(PROGRESS_PATH."_$engineID"); $totalTransfered = isset($progressContent[3]) ? trim($progressContent[3]):''; unset($progressContent); } $progressPath = PROGRESS_PATH."_$engineID"; $extendedAsciiChars = getExtendedASCIIChrOrd(); while(1) { $buffer = fread($tempOutputFile, LIMIT); if(empty($buffer)){ if(!file_exists(PIDFILE)) break; sleep(2); fseek($tempOutputFile, 0, 1); #to clear eof flag continue; } //need to check appending partial record to packet or to first line of packet if(!empty($lastLine)) $buffer = $lastLine.$buffer; if(!preg_match('/\n$/', $buffer)){ $lastNewlinePos = strrpos($buffer, "\n"); $resultList = substr($buffer,0,$lastNewlinePos); $lastLine = substr($buffer,$lastNewlinePos); } else { $resultList = $buffer; $lastLine = ''; } unset($buffer); $noOfLines = substr_count($resultList,"\n"); $resultListXML = "$resultList"; $xml = simplexml_load_string($resultListXML); if(!$xml){ $resultListXML = checkAndConstructXML($resultList); $resultListXML = "$resultListXML"; $xml = simplexml_load_string($resultListXML); } if($isLocalRestore !== 1 && !$xml) { $resultListXML = str_replace($extendedAsciiChars['chr'], $extendedAsciiChars['ord'], $resultListXML); $xml = simplexml_load_string($resultListXML); } $toCheck = 0; reset($xml); foreach ($xml->item as $value) { $filename = trim($value['fname'],"\n"); $trf_type = trim($value['trf_type']); $size = trim($value['size']); $tottrf_sz = (int) trim($value['tottrf_sz']); $rate_trf = trim($value['rate_trf']); $per = (string) trim($value['per']); $changedName = html_entity_decode($filename); if($changedName !== (string)$filename) { if(isset($current_RestoresetHash[$changedName])) { $current_RestoresetHash[$changedName] = 1; } else { $filename = $changedName; } } if($tottrf_sz === 0 && $trf_type === "FILE IN SYNC"){ $totalTransfered += $size; } else { $totalTransfered += $tottrf_sz - $prevTransferedSize; $prevTransferedSize = $tottrf_sz; } updateProgressFile($trf_type, $filename, $size, $totalTransfered, $rate_trf, $opStartTimeRestore, $size, $per, $progressPath ); if($per === $operationComplete ){ $Restoretype = ""; $keyString = $filename; if($isLocalRestore !== 1) $keyString = PATHSEPARATOR.$keyString; # check if file in sync if($trf_type === "FILE IN SYNC") $Restoretype = "Sync Restore"; else if($trf_type === "FULL") $Restoretype = "Full Restore"; else if($trf_type === "INCREMENTAL") $Restoretype = "Incremental Restore"; if(!empty($Restoretype)) { if(isset($current_RestoresetHash[$keyString])) $current_RestoresetHash[$keyString] = (($Restoretype === "Sync Restore")? 2 : 1); else { $tempFullPath = getFullPathofFile($keyString); if($tempFullPath != '' && isset($current_RestoresetHash[$tempFullPath])) $current_RestoresetHash[$tempFullPath] = (($Restoretype === 'Sync Restore')? 2 : 1); else { if($Restoretype === 'Sync Restore') $xtrafileSyncCount++; else $xtrafileRestoreCount++; } } if($Restoretype !== "Sync Restore"){ $restoreFinishTime = trim(exec('date +"%a %b %d %T %Y"')); $fileSize = convertFileSize($size); if($filesonly === 1 && strrpos($filename, "/") >1) $filename = substr($filename,strrpos($filename, "/")+1); #log this message to App logs if($isLocalRestore !== 1) $filename = PATHSEPARATOR.$filename; $outputArray[] = "[$restoreFinishTime] [$Restoretype] [SUCCESS] [$filename] [$fileSize] ".LINEFEED; } } } ++$toCheck; unset($filename,$trf_type,$size,$tottrf_sz,$rate_trf,$per,$keyString,$restoreFinishTime,$fileSize); } if($outputArray) { $isLocked = flock($logOutputHandle,LOCK_EX); if($isLocked){ fwrite($logOutputHandle, implode("",$outputArray)); flock($logOutputHandle,LOCK_UN); } else { fwrite($logOutputHandle, implode("",$outputArray)); traceLog("Unable to lock LOG file"); } } unset($xml,$outputArray); if(strpos($resultList,$termData) !== FALSE) { $skip = 1; break; } else if($noOfLines > $toCheck){ $exp = explode("\n",$resultList); foreach($exp as $key=>$data){ $data = trim($data); if(!empty($data) && strpos($data," 0 && $failedfiles_index != -1 && file_exists(PIDFILE)){ if(!checkretryAttempt($individual_errorfile)){ getFinalErrorFile(); updateFailedFileCount(); return; } } else { getFinalErrorFile(); updateFailedFileCount(); return; } if(!$retryAttempt || $failedfiles_index == -1) getFinalErrorFile(); else { if($failedfiles_count > 0) { $failedfiles = TEMP_FILES."/failedfiles.txt".$failedfiles_index; $oldfile_error = $jobrunningDir."/".ERROR_DIR."/$restore_error"; $newfile_error = $jobrunningDir."/".ERROR_DIR."/$failedfiles"."_ERROR"; if(file_exists($oldfile_error) || file_exists($oldfile_error."_FINAL")) rename($oldfile_error, $newfile_error); if($isLocalRestore === 1){ $failedfileOrig = $failedfiles."_org"; $FD = fopen($failedfileOrig, 'w'); if(!$FD){ traceLog("Unable to open file $failedfileOrig, Reason:".customErrorMsg()); updateFailedFileCount(); return; } $arrayContent = join("\n", $failedfiles_array); fwrite($FD,"$arrayContent\n"); fclose($FD); $failedfiles_array = array(); $failedfiles_array = $failedfiles_array_Encrypted; } $FAILEDLIST = fopen($failedfiles, 'w'); if(!$FAILEDLIST){ traceLog("Unable to open file $failedfiles, Reason:".customErrorMsg()); updateFailedFileCount(); return; } /* To Test Retry $i = 0; foreach ($failedfiles_array as $key => $val) { if($i %2 == 0) $failedfiles_array[$key] = str_replace("_ODD", "", $val); $i++; } */ $arrayContent = join("\n", $failedfiles_array); fwrite($FAILEDLIST,"$arrayContent\n"); fclose($FAILEDLIST); } if(file_exists($failedfiles)){ $RETRYINFO = fopen(RETRY_INFOFILE, 'a'); if(!$RETRYINFO){ traceLog("Unable to open file ".RETRY_INFOFILE.", Reason:".customErrorMsg()); updateFailedFileCount(); return; } flock($RETRYINFO, LOCK_EX); fwrite($RETRYINFO,"$failedfiles $relative $current_source\n"); fflush($RETRYINFO); flock($RETRYINFO, LOCK_UN); fclose($RETRYINFO); } } updateFailedFileCount(); } #**************************************************************************************************** # Subroutine Name : getFailedFileCount. # Objective : This function calculate failed file count by checking current restore # hash value zero for failed file. # Added By : Vijay Vinoth #*****************************************************************************************************/ function getFailedFileCount() { GLOBAL $currentRestoreset, $failedfiles_array, $failedfiles_count, $current_RestoresetHash, $isLocalRestore, $failedfiles_array_Encrypted; GLOBAL $RestoresetHashOrig_Encrypted, $fileOperationCount, $fileSyncCount, $xtrafileRestoreCount, $xtrafileSyncCount, $xtraSuccessCounter; $j = 0; reset($currentRestoreset); foreach($currentRestoreset as $key => $item){ $tempCurrentRestoreset = trim($item,"\n"); if(empty($tempCurrentRestoreset)) continue; if($current_RestoresetHash[$tempCurrentRestoreset] === 0){ $failedfiles_array[$j] = $tempCurrentRestoreset; if($isLocalRestore === 1) $failedfiles_array_Encrypted[$j] = $RestoresetHashOrig_Encrypted[$tempCurrentRestoreset]; $j++; } elseif($current_RestoresetHash[$tempCurrentRestoreset] === 1) $fileOperationCount++; elseif($current_RestoresetHash[$tempCurrentRestoreset] === 2) $fileSyncCount++; } $failedfiles_count = $j - ($xtrafileRestoreCount + $xtrafileSyncCount); $fileOperationCount += $xtrafileRestoreCount; $fileSyncCount += $xtrafileSyncCount; $xtraSuccessCounter = $xtrafileRestoreCount + $xtrafileSyncCount; } #**************************************************************************************************************************** # Subroutine Name : getFullPathofFile # Objective : This subroutine gets the full path of files that have backed up # Added By : Vijay Vinoth # Modified By : Sabin Cheruvattil #******************************************************************************************************************************/ function getFullPathofFile($fileToCheck) { GLOBAL $currentRestoreset, $current_RestoresetHash; $returnItem = ''; $retStr = trim($fileToCheck, "\n"); reset($currentRestoreset); $currentRestoresetReverse = array_reverse($currentRestoreset); reset($currentRestoresetReverse); foreach($currentRestoresetReverse as $key => $item) { $item = trim($item, "\n"); $currentRestoresetTemp = ((strrpos($retStr, '/') === 0)? strrpos($item, '/') : strpos($item, '/')); $match = substr($item, $currentRestoresetTemp); if($retStr === $match) { $returnItem = $item; break; } } unset($currentRestoresetReverse); return $returnItem; } #**************************************************************************************************** # Subroutine Name : checkretryAttempt. # Objective : This function checks whether restore has to retry # Added By : Vijay Vinoth #*****************************************************************************************************/ function checkretryAttempt($individual_errorfile) { GLOBAL $ErrorArgumentsExit, $ErrorArgumentsRetry, $retryAttempt, $phpPath; GLOBAL $webDir,$engineID; #GLOBAL $dedupDetailFile, $dedupRestoreDeviceID; $errorline = "idevs error"; $linesRestoreErrorFile = file($individual_errorfile); #To check the individual error file for exit scenarios reset($linesRestoreErrorFile); foreach($linesRestoreErrorFile as $key => $item){ if(empty($item) || strpos($item,$errorline) !== FALSE) continue; reset($ErrorArgumentsExit); foreach($ErrorArgumentsExit as $index => $val){ if(strpos($item,$val) !== FALSE) { @unlink(OUTPUTFILE."_$engineID"); /*if(file_exists(EVS_TEMP)) removeItems(EVS_TEMP);*/ #kill evs and then exit $fileJobTerminationPath = $webDir.PATHSEPARATOR.APPDIR.PATHSEPARATOR.SCRIPTS.PATHSEPARATOR.JOBTERMINATIONFILE; $cmd = "$phpPath $fileJobTerminationPath 2 '$val'"; exec($cmd); if(($val == "account has been cancelled")|| ($val == "account is under maintenance") || ($val == "account has been blocked") || ($val == "account has been expired")){ //update user status to account settings file updateUserStatus($val); } return 0; } } } #To check the individual error file for retry case reset($linesRestoreErrorFile); foreach($linesRestoreErrorFile as $key => $item){ if(empty($item) || strpos($item,$errorline) !== FALSE) continue; reset($ErrorArgumentsRetry); foreach($ErrorArgumentsRetry as $inx => $val){ if(strpos($item,$val) !== FALSE){ $retryAttempt = true; break; } } if($retryAttempt){ reduceRetryFileSizeInprogress(); break; } } return 1; } #**************************************************************************************************** # Subroutine Name : reduceRetryFileSizeInprogress. # Objective : This function is used to reduce the file size of files which got failed. # Added By : Anil Kumar # Modified By : #***************************************************************************************************** function reduceRetryFileSizeInprogress() { GLOBAL $engineID; $progressPath = PROGRESS_PATH."_$engineID"; $progressFile = file_get_contents($progressPath); $progressData = explode("\n", $progressFile); if($progressData[8] != '100%' && $progressData[7] > 0){ $progressData[2] -= $progressData[7]; } updateProgressFile($progressData[0], $progressData[1], $progressData[2], $progressData[3], $progressData[4], $progressData[6], $progressData[7], $progressData[8], $progressPath); } #**************************************************************************************************** # Subroutine Name : updateFailedFileCount. # Objective : This function manage the counts of failed files during retry case. # Added By : Vijay Vinoth #*****************************************************************************************************/ function updateFailedFileCount() { GLOBAL $currentRestoreset, $failedfiles_count, $restore_error, $engineID; list($orig_count) = getParameterValueFromStatusFile(array("FAILED_COUNT_FILES_INDEX"), $engineID); if(strpos( $restore_error,"failedfiles.txt" ) !== false){ $size_Restoreset = count($currentRestoreset); $newcount = ($orig_count - $size_Restoreset) + $failedfiles_count; $failedfiles_count = $newcount; } else{ $orig_count += $failedfiles_count; $failedfiles_count = $orig_count; } } #**************************************************************************************************************************** # Subroutine Name : getFinalErrorFile # Objective : This subroutine creates a final ERROR file for each backupset, which has to be displayed in LOGS # Modified By : Vijay Vinoth #******************************************************************************************************************************/ function getFinalErrorFile() { GLOBAL $jobrunningDir, $restore_error, $exit_flag; GLOBAL $failedfiles_count, $failedfiles_array, $RestoresetHashOrig_Encrypted, $hostNameLB,$username; GLOBAL $current_source, $isLocalRestore, $engineID, $xtraSuccessCounter, $webDir; $restoreErrors = ''; $cancel = 0; $failedWithReason = 0; $individual_errorfile = $jobrunningDir."/".ERROR_DIR."/$restore_error"; if(!$exit_flag){ if(!file_exists(PIDFILE)) $cancel = 1; } if($failedfiles_count > 0){ $ERROR_FINAL = fopen($individual_errorfile."_FINAL", 'w+'); if(!$ERROR_FINAL){ traceLog("Unable to open file $individual_errorfile"."_FINAL, Reason:".customErrorMsg()); return; } else{ if(!file_exists($individual_errorfile)){ if($exit_flag or $cancel !== 0) $failedfiles_count = $failedWithReason; if($cancel===1) traceLog("Unable to open $individual_errorfile."); } else $restoreErrors = file($individual_errorfile); } if($restoreErrors){ sort($failedfiles_array); $j = 0; $last_index = count($restoreErrors); $destPath = trim($hostNameLB."/$username$current_source"); $failedfiles_arrayCount = count($failedfiles_array); for($i = 0; $i < $failedfiles_arrayCount; $i++){ $matched = 0; #reset the initial and last limit for internal for loop for new item if required if($j > $last_index){ $j = 0; $last_index = count($restoreErrors); } #fill the last matched index for later use $index = $j; if($isLocalRestore === 1) { $failedfile = $failedfiles_array[$i]; } else { $failedfile = substr($failedfiles_array[$i],1); } #try to find a match between start and end point of error file for($j=0;$j < $last_index; $j++){ if(!empty($restoreErrors[$j])){ if(strpos( $restoreErrors[$j], $failedfile )) { $failedfile = phpQuoteMeta($failedfile); $Reason = trim(preg_replace("/.*$failedfile.*:/", '', $restoreErrors[$j])); if($Reason == '-3') { $Reason = "Decryption failed"; // need to run auth list to confirm and logout $res = validatePvtKeyUsingAuthList(); if(!$res){ $needToLogout = "$webDir/".APPDIR.PATHSEPARATOR."logout.txt"; traceLog("logout trigger".$needToLogout); $errDetails = DECRYPTIONFAILED; writeOperationErrorToErrorFile($errDetails); traceLog($errDetails); if(file_put_contents($needToLogout, DECRYPTIONFAILED)) chmod($needToLogout,0777); } } fwrite($ERROR_FINAL,"[".trim(exec('date +"%a %b %d %T %Y"'))."] [FAILED] [$failedfiles_array[$i]]. Reason: $Reason ".LINEFEED); $matched = 1; $failedWithReason++; #got a match so resetting the last index $last_index = count($restoreErrors); $j++; break; } } #if no match till last item and intial index is not zero try for remaining error file content if($j === $last_index && $index !== 0) { $j = 0; $last_index = $index; $index = $j; } } if($matched === 0 && !$exit_flag && $cancel === 0) { if($xtraSuccessCounter <= 0) { $errorContent = "[".trim(exec('date +"%a %b %d %T %Y"'))."] [FAILED] [$failedfiles_array[$i]] ".LINEFEED; } else { $xtraSuccessCounter--; } } } } else if($cancel === 0){ $localtime = trim(exec('date +"%a %b %d %T %Y"')); foreach($failedfiles_array as $failedItem){ if($xtraSuccessCounter <= 0) { $failedfile = substr($failedItem,1); $errorContent = "[$localtime] [FAILED] [$failedfile] ".LINEFEED ; } else { $xtraSuccessCounter--; } } } if(!$exit_flag and $cancel === 0) fwrite($ERROR_FINAL, $errorContent); fclose($ERROR_FINAL); } if($exit_flag or $cancel !== 0) $failedfiles_count = $failedWithReason; @unlink($individual_errorfile); } #**************************************************************************************************** # Subroutine Name : process_term # Objective : The signal handler invoked when SIGTERM signal is received by the script # Added By : Vijay Vinoth #*****************************************************************************************************/ function process_term() { GLOBAL $tempOutputFile, $logOutputHandle, $logOutputFilePath,$engineID; if($tempOutputFile) fclose($tempOutputFile); if($logOutputHandle) fclose($logOutputHandle); chmod($logOutputFilePath,PERMISSIONVAL); writeParameterValuesToStatusFile($engineID); exit; } ?>