$item){ $item = trim($item,"\n"); if(!empty($item)) { $current_BackupsetHash[$item]['status'] = 0; $current_BackupsetHash[$item]['detail'] = 0; $current_BackupsetHash[$item]['size'] = 0; } } # signal handlers if(extension_loaded('pcntl') && function_exists('pcntl_signal')){ declare(ticks = 1); //To handle signals (Kill/Term/..etc) pcntl_signal(SIGINT, 'process_term'); pcntl_signal(SIGTERM, 'process_term'); } if($isLocalBackup === 0 ){ #Online backup for two tier if(!createDB()){ $errStr = "Create DB failed. Reason: ".$e->getMessage(); traceLog($errStr); writeBackupErrorMsgToStatusFile($errStr,$engineID); exit; } try{ $dbh_insert = $dbh->prepare("INSERT INTO IDriveLocalTable (FILEPATH, MOD_DT, FILESIZE, LEVELCOUNT) values (?,?,?,?)") ; $dbh_update = $dbh->prepare("UPDATE IDriveLocalTable SET MOD_DT = ?, FILESIZE = ? , PRESENTDATE = ? , LEVELCOUNT = ? WHERE FILEPATH = ?"); } catch(PDOException $e){ $errStr = "Error: Prepare statement failed for insert/update. Reason: ".$e->getMessage(); traceLog($errStr); writeBackupErrorMsgToStatusFile($errStr,$engineID); unset($dbh); exit; } subOutputProcess(); unset($dbh); subErrorRoutine(); } else if($isLocalBackup === 1) { #local backup for NAS subXMLOutputProcess(); subErrorRoutine(); } else{ #Online backup for other NASs subOutputProcess(); subErrorRoutine(); } # need to write method to update it // Commenting update retained by anil updateUserLogForBackupRestoreFiles(); writeParameterValuesToStatusFile($engineID); #**************************************************************************************************** # Subroutine Name : subOutputProcess. # Objective : This function monitors and parse the evs output file and creates the user # log file. # Added By : Senthil Pandian [11/25/2016] # Modified By : #*****************************************************************************************************/ function subOutputProcess() { GLOBAL $dbh, $skip, $opStartTime, $isLocalBackup, $logOutputFilePath, $backup_error, $operationComplete, $current_backupsetfile; GLOBAL $backupLocation, $fileSyncCount, $fileOperationCount, $current_BackupsetHash, $dedupDetailFile; GLOBAL $lastLine, $engineID, $databaseLB, $extFileSyncCount, $extFileBackupCount; $backupLocation1 = $backupLocation; if(file_exists($dedupDetailFile)) $backupLocation1 = ''; # Flags to determine the status of OUTFILE $fileOpened = 0; $termData = "CHILD_PROCESS_COMPLETED"; $buffer = ""; $byteRead = 0; $lastLine = ""; $fields_in_progress = 10; # Waiting for EVS output file to open while(!$fileOpened){ if(file_exists(OUTPUTFILE."_$engineID")){ if($handle = fopen(OUTPUTFILE."_$engineID",'r+')) $fileOpened = 1; else break; } if(!file_exists(PIDFILE)) break; sleep(2); } if(!$fileOpened){ traceLog("Could not open file : ".OUTPUTFILE.customErrorMsg()); return; } $Backuptype =""; if($isLocalBackup === 0) $dbh->beginTransaction(); $prevTransferedSize = 0; $totalTransfered = 0; $fileCount = 0; if(file_exists(PROGRESS_PATH."_$engineID")){ if(filesize(PROGRESS_PATH."_$engineID")>0){ $progressContent = file(PROGRESS_PATH."_$engineID"); $totalTransfered = isset($progressContent[3])?(double)$progressContent[3]:''; unset($progressContent); } } else{ file_put_contents(PROGRESS_PATH."_$engineID",""); chmod(PROGRESS_PATH."_$engineID",PERMISSIONVAL); } $progressPath = PROGRESS_PATH."_$engineID"; $cnt = 0; $extendedAsciiChars = getExtendedASCIIChrOrd(); while(1){ ++$cnt; if($cnt%100 === 0) usleep(2000); $buffer = fread($handle, LIMIT); if(empty($buffer)){ if(!file_exists(PIDFILE)){ break; } sleep(WAIT_FOR); fseek($handle, 0, 1); //to clear eof flag continue; } //need to check appending partial record to packet or to first line of packet if(!empty($lastLine)) $buffer = $lastLine.$buffer; if(!preg_match('/\n$/', $buffer)){ $lastNewlinePos = strrpos($buffer, "\n"); $resultList = substr($buffer,0,$lastNewlinePos); $lastLine = substr($buffer,$lastNewlinePos); } else { $resultList = $buffer; $lastLine = ''; } unset($buffer); $noOfLines = substr_count($resultList,"\n"); $resultListXML = "$resultList"; #$resultListXML = str_replace($extendedAsciiChars['chr'], $extendedAsciiChars['ord'], $resultListXML); $xml = simplexml_load_string($resultListXML); if(!$xml) { $resultListXML = str_replace($extendedAsciiChars['chr'], $extendedAsciiChars['ord'], $resultListXML); $xml = simplexml_load_string($resultListXML); } if(!$xml){ $resultListXML = checkAndConstructXML($resultList); $resultListXML = "$resultListXML"; $xml = simplexml_load_string($resultListXML); } $toCheck = 0; reset($xml); foreach ($xml->item as $value) { $extFileFoundInHash = FALSE; $type = trim($value['trf_type']); $fileName = trim($value['fname'],"\n"); $size = trim($value['size']); $transferedSize = (double) trim($value['tottrf_sz']); $rateOfTransfer = trim($value['rate_trf']); $percentage = (string)trim($value['per']); $offset = 0; if($value['offset']) $offset = trim($value['offset']); else $offset = $size; // added by anil $keyString = PATHSEPARATOR.$fileName; # calculate transfered file size for specific file if($transferedSize == 0 && $type === "FILE IN SYNC"){ $totalTransfered += $size; //$prevTransferedSize = 0; } else if($type != ''){ if(isset($current_BackupsetHash[$keyString]['status']) && ($current_BackupsetHash[$keyString]['status'] == 1)) { //continue; }else { $totalTransfered = (double)$totalTransfered + (double)($transferedSize - $prevTransferedSize); $prevTransferedSize = (double)$transferedSize; } } else{ continue; } updateProgressFile($type, $fileName, $size, $totalTransfered, $rateOfTransfer, $opStartTime, $offset, $percentage, $progressPath); if($percentage === $operationComplete) { $keyString = PATHSEPARATOR.$fileName; if($isLocalBackup == 1) $keyString = $fileName; if($isLocalBackup === 0){ $localFilePath = getLocalFilePath($keyString); # $size = customFileSizeFromLS($localFilePath); $modtime = filemtime($localFilePath); $fileInsert = "$backupLocation1$keyString"; } $Backuptype =""; if($type === "FILE IN SYNC") $Backuptype = "Sync"; else if($type === "FULL") $Backuptype = "Full Backup"; else if($type === "INCREMENTAL") $Backuptype = "Incremental Backup"; if(!empty($Backuptype)) { if(isset($current_BackupsetHash[$keyString]['status'])) $current_BackupsetHash[$keyString]['status'] = (($Backuptype === "Sync")? 2 : 1); else if(isset($current_BackupsetHash[utf8_decode($keyString)]['status'])) $current_BackupsetHash[utf8_decode($keyString)]['status'] = (($Backuptype == "Sync")? 2 : 1); else if(isset($current_BackupsetHash[html_entity_decode($keyString)]['status'])) $current_BackupsetHash[html_entity_decode($keyString)]['status'] = (($Backuptype == "Sync")? 2 : 1); else { $tempFullPath = getFullPathofFile($keyString); if($tempFullPath != "" && isset($current_BackupsetHash[$tempFullPath]['status'])) $current_BackupsetHash[$tempFullPath]['status'] = (($Backuptype == "Sync")? 2 : 1); else if($tempFullPath != "" && isset($current_BackupsetHash[utf8_decode($tempFullPath)]['status'])) $current_BackupsetHash[utf8_decode($tempFullPath)]['status'] = (($Backuptype == "Sync")? 2 : 1); else { if($Backuptype == 'Sync') $extFileSyncCount++; else $extFileBackupCount++; } } $fileCount++; if($isLocalBackup === 0){ $levelCount = substr_count($fileInsert,"/"); if($Backuptype === "Full Backup") insert(1, $fileInsert,$modtime,$size, $levelCount-1); else update(1, $modtime, $size, time(), $levelCount-1, $fileInsert); } if($Backuptype !== "Sync"){ //$backupFinishTime = date("D M d H:i:s Y"); //Ex: Sat Feb 16 06:50:45 2013 $backupFinishTime = trim(exec('date +"%a %b %d %T %Y"')); //Ex: Sat Feb 16 06:50:45 2013 $fileSize = convertFileSize($size); if(strpos($current_backupsetfile,"_onlyFiles") !== FALSE){ $pos = strrpos($fileName, '/'); $fileName = substr($fileName,$pos+1); } # // Commenting update retained by anil $current_BackupsetHash[$keyString]['detail'] = "[$backupFinishTime] [$Backuptype]"; $current_BackupsetHash[$keyString]['size'] = $fileSize; // $outputArray[] = "[$backupFinishTime] [$Backuptype] [SUCCESS] [".PATHSEPARATOR."$fileName] [$fileSize]".LINEFEED; } if($fileCount % 1000 === 0){ if($isLocalBackup === 0){ $dbh->commit(); $dbh->beginTransaction(); } $fileCount = 0; } } } $toCheck++; unset($filename,$type,$size,$transferedSize,$rateOfTransfer,$percentage,$keyString,$backupFinishTime,$fileSize); } if(strpos($resultList,$termData) !== FALSE){ break; } else if($noOfLines > $toCheck){ $exp = explode("\n",$resultList); reset($exp); foreach($exp as $key=>$data){ $data = trim($data); if(!empty($data) && (strpos($data," 0){ $dbh->commit(); } } #**************************************************************************************************** # Subroutine Name : subXMLOutputProcess. # Objective : This function monitors and parse the evs output file(XML content) and creates the App # log file which is shown to user. # Added By : Senthil Pandian [11/30/2016] # Modified By : #*****************************************************************************************************/ function subXMLOutputProcess() { GLOBAL $logOutputFilePath, $opStartTime, $isLocalBackup, $current_BackupsetHash; GLOBAL $skip, $current_backupsetfile, $backup_error, $operationComplete, $engineID; GLOBAL $databaseLB, $dbIndexCount, $textDbFile, $localDBCommitFreq, $extFileSyncCount, $extFileBackupCount; # Flags to determine the status of OUTFILE $fileOpened = 0; $termData = "CHILD_PROCESS_COMPLETED"; $buffer = ""; $byteRead = 0; $lastLine = ""; //$fields_in_progress = 20; $fileVersion = "file_ver="; $fileName = ""; $fileCount = 0; $xml_search_str = array('"', "&"); $xml_replace_str = array('\"', "&"); # Waiting for EVS output file to open while(!$fileOpened){ if(file_exists(OUTPUTFILE."_$engineID")){ if($handle = fopen(OUTPUTFILE."_$engineID",'r')) $fileOpened = 1; else break; sleep(2); } if(!file_exists(PIDFILE)) break; } if(!$fileOpened){ traceLog("Could not open ".OUTPUTFILE."_$engineID".customErrorMsg()); return; } $Backuptype =""; $prevTransferedSize = 0; $totalTransfered = 0; $fileCount = 0; if(file_exists(PROGRESS_PATH."_$engineID") && customFileSize(PROGRESS_PATH."_$engineID")>0){ $progressContent = file_get_contents(PROGRESS_PATH."_$engineID"); $progressContent = explode("\n",$progressContent); $totalTransfered = isset($progressContent[3])?(double)$progressContent[3]:''; } $progressPath = PROGRESS_PATH."_$engineID"; $dbpaths = TEMP_FILES."/DbFiles/textdb_".$engineID."_".$dbIndexCount.".xml"; $textDbFile = fopen($dbpaths,"w+"); fwrite($textDbFile,''.LINEFEED.''.LINEFEED); @chmod($dbpaths,PERMISSIONVAL); $extendedAsciiChars = getExtendedASCIIChrOrd(); while(1) { $buffer = fread($handle, LIMIT); $byteRead = strlen($buffer); if($byteRead == 0){ if(!file_exists(PIDFILE)){ break; } sleep(WAIT_FOR); fseek($handle, 0, 1); //to clear eof flag continue; } //need to check appending partial record to packet or to first line of packet if("" != $lastLine){ $buffer = $lastLine.$buffer; } if(!preg_match('/\n$/', $buffer)){ $lastNewlinePos = strrpos($buffer, "\n"); $resultList = substr($buffer,0,$lastNewlinePos+1); $lastLine = substr($buffer,$lastNewlinePos); } else { $resultList = $buffer; $lastLine = ''; } $noOfLines = substr_count($resultList, "\n"); $resultListXML = "$resultList"; #$resultListXML = str_replace($extendedAsciiChars['chr'], $extendedAsciiChars['ord'], $resultListXML); $xml = simplexml_load_string($resultListXML); if(!$xml) { $resultListXML = str_replace($extendedAsciiChars['chr'], $extendedAsciiChars['ord'], $resultListXML); $xml = simplexml_load_string($resultListXML); } if(!$xml){ $resultListXML = checkAndConstructXML($resultList); $resultListXML = "$resultListXML"; $xml = simplexml_load_string($resultListXML); } $toCheck = 0; foreach($xml->item as $value) { $fieldSize = (double)$value['size']; $fieldTRFType = $value['trf_type']; $fieldFolderId = $value['folder_id']; $fieldMPC = $value['mpc']; $fieldEncName = $value['enc_name']; $fieldFileName = $value['fname']; $percentage = $value['per']; $rateOfTransfer = $value['rate_trf']; $transferedSize = (double)$value['tottrf_sz']; $keyString = $fieldFileName; $offset = 0; if($value['offset']) $offset = trim($value['offset']); else $offset = $size; $changedName = html_entity_decode($keyString); if($changedName !== (string)$keyString) { if(isset($current_BackupsetHash[$changedName]['status'])) { $current_BackupsetHash[$changedName]['status'] = 1; } else { $keyString = $changedName; } } # calculate transfered file size for specfic file if($transferedSize == 0 && $fieldTRFType == "FILE IN SYNC") { $totalTransfered += $fieldSize; } else { $totalTransfered = (double)$totalTransfered + (double)($transferedSize - $prevTransferedSize); $prevTransferedSize = (double)$transferedSize; } updateProgressFile($fieldTRFType, $keyString, $fieldSize, $totalTransfered, $rateOfTransfer, $opStartTime, $offset, $percentage, $progressPath); if($percentage == $operationComplete) { $fileName = $keyString; $modtime = date("Y/m/d h:i:s A", filemtime($fileName)); $Backuptype =""; if($fieldTRFType == "FILE IN SYNC") $Backuptype = "Sync"; else if($fieldTRFType == "FULL") $Backuptype = "Full Backup"; else if($fieldTRFType == "INCREMENTAL") $Backuptype = "Incremental Backup"; else{ if($percentage == "") addtionalErrorInfo($resultListXML,$backup_error); } if($Backuptype != '') { $size = convertFileSize(trim($fieldSize)); if(isset($current_BackupsetHash[$keyString]['status'])) { $current_BackupsetHash[$keyString]['status'] = (($Backuptype === "Sync")? 2 : 1); $current_BackupsetHash[$keyString]['detail'] = "[$modtime] [$Backuptype]"; $current_BackupsetHash[$keyString]['size'] = $size; } else if(isset($current_BackupsetHash[utf8_decode($keyString)]['status'])) { $current_BackupsetHash[utf8_decode($keyString)]['status'] = (($Backuptype == "Sync")? 2 : 1); $current_BackupsetHash[utf8_decode($keyString)]['detail'] = "[$modtime] [$Backuptype]"; $current_BackupsetHash[utf8_decode($keyString)]['size'] = $size; } else if(isset($current_BackupsetHash[html_entity_decode($keyString)]['status'])) { $current_BackupsetHash[html_entity_decode($keyString)]['status'] = (($Backuptype == "Sync")? 2 : 1); $current_BackupsetHash[html_entity_decode($keyString)]['detail'] = "[$modtime] [$Backuptype]"; $current_BackupsetHash[html_entity_decode($keyString)]['size'] = $size; } else { $tempFullPath = getFullPathofFile($keyString); if($tempFullPath != "" && isset($current_BackupsetHash[$tempFullPath]['status'])) { $current_BackupsetHash[$tempFullPath]['status'] = (($Backuptype == "Sync")? 2 : 1); $current_BackupsetHash[$tempFullPath]['detail'] = "[$modtime] [$Backuptype]"; $current_BackupsetHash[$tempFullPath]['size'] = $size; } else if($tempFullPath != "" && isset($current_BackupsetHash[utf8_decode($tempFullPath)]['status'])) { $current_BackupsetHash[utf8_decode($tempFullPath)]['status'] = (($Backuptype == "Sync")? 2 : 1); $current_BackupsetHash[utf8_decode($tempFullPath)]['detail'] = "[$modtime] [$Backuptype]"; $current_BackupsetHash[utf8_decode($tempFullPath)]['size'] = $size; } else { if($Backuptype == 'Sync') $extFileSyncCount++; else $extFileBackupCount++; } } $fileCount++; $fileName = html_entity_decode($fileName); $fileName1 = "'$fileName'"; if($Backuptype == "Full Backup"){ $keyStringData = substr($keyString,0,strrpos($keyString, '/'))."/"; $fileName1 = str_replace($keyStringData,'',$fileName1); $fileDetail = " fileName = \"".str_replace($xml_search_str,$xml_replace_str,$fileName1)."\""; $fileDetail .= " operation = \"insert\" dbtable = \"file\""; $fileDetail .= " modtime = \"$modtime\""; $fileDetail .= " fieldSize = \"$fieldSize\""; $fileDetail .= " bksetLMD = \"$bksetLMD\""; $fileDetail .= " syncDate = \"$syncDate\""; $fileDetail .= " bksetName = \"$bksetName\""; $fileDetail .= " hashValue = \"$hashValue\""; $fileDetail .= " fieldFolderId = \"$fieldFolderId\""; $fileDetail .= " fieldMPC = \"$fieldMPC\""; $fileDetail .= " fieldEncName = \"$fieldEncName\""; $fileDetail .= " fileOrFolder = \"1\""; $fileDetail .= " backupStatus = \"1\""; $fileDetail .= " keyString = \"".str_replace($xml_search_str,$xml_replace_str,$keyString)."\""; fwrite($textDbFile,"".LINEFEED); } else { $keyStringData = substr($keyString, 0, strrpos($keyString, '/'))."/"; $fileName1 = str_replace($keyStringData, '', $fileName1); $fileDetail = " fieldFolderId = \"$fieldFolderId\""; $fileDetail .= " fileName = \"".str_replace($xml_search_str,$xml_replace_str,$fileName1)."\""; $fileDetail .= " operation = \"update\" dbtable = \"file\""; $fileDetail .= " fieldSize = \"$fieldSize\""; $fileDetail .= " modtime = \"$modtime\""; $fileDetail .= " keyString = \"".str_replace($xml_search_str,$xml_replace_str,$keyString)."\""; fwrite($textDbFile,"".LINEFEED); } if($Backuptype != "Sync"){ $backupFinishTime = trim(shell_exec('date +"%a %b %d %T %Y"')); //Ex: Sat Feb 16 06:50:45 2013 $fileSize = convertFileSize(trim($fieldSize)); if(strstr($current_backupsetfile, "_onlyFiles")) { $pos = strrpos($fileName, '/'); $fileName = substr($fileName,$pos+1); } $current_BackupsetHash[$keyString]['detail'] = "[$backupFinishTime] [$Backuptype]"; $current_BackupsetHash[$keyString]['size'] = $size; } } } $toCheck++; } if(strstr($resultList,$termData)) { break; } else if($noOfLines > $toCheck){ $exp = explode("\n",$resultList); foreach($exp as $data){ $data = trim($data); //if(!empty($data) && strpos($data," 0 && $failedfiles_index != -1 && file_exists(PIDFILE)){ if(!checkretryAttempt($individual_errorfile)){ getFinalErrorFile($failed_count); updateTotalFailedFileCount($failed_count); return; } } else{ getFinalErrorFile($failed_count); updateTotalFailedFileCount($failed_count); return; } if(!$retryAttempt or $failedfiles_index == -1){ getFinalErrorFile($failed_count); } else{ if(count($failedfiles_array) > 0){ getFinalErrorFile($failed_count); $failedfilesName = TEMP_FILES."/failedfiles.txt".$failedfiles_index; $oldfile_error = $jobrunningDir."/".ERROR_DIR."/$backup_error"."_FINAL"; $newfile_error = $jobrunningDir."/".ERROR_DIR."/$failedfilesName"."_ERROR_FINAL"; if(file_exists($oldfile_error) or file_exists($oldfile_error."_FINAL")){ exec("mv '$oldfile_error' '$newfile_error'"); } if(!$handle = fopen($failedfilesName,'w')){ traceLog("Unable to open file $failedfiles, Reason : ".customErrorMsg()); updateTotalFailedFileCount($failed_count); return; } /* To Test Retry $i = 0; foreach ($failedfiles_array as $key => $val) { if($i %2 == 0) $failedfiles_array[$key] = str_replace("_ODD", "", $val); $i++; } */ $arrayContent = join("\n", $failedfiles_array); fwrite($handle,$arrayContent."\n"); fclose($handle); } if(file_exists($failedfilesName)){ $handle = fopen(RETRY_INFOFILE,'a+'); if(!$handle){ traceLog("not open retry file. Reason: ".customErrorMsg()); } else{ flock($handle, LOCK_EX); fwrite($handle,"$failedfilesName $relative $current_source\n"); fflush($handle); flock($handle, LOCK_UN); fclose($handle); } } } updateTotalFailedFileCount($failed_count); } #**************************************************************************************************** # Subroutine Name : checkretryAttempt. # Objective : This function checks whether backup has to retry # Added By : Senthil Pandian [12/01/2016] # Modified By : #***************************************************************************************************** function checkretryAttempt($individual_errorfile) { GLOBAL $ErrorArgumentsExit,$ErrorArgumentsRetry,$ErrorArgumentsMinRetry,$retryAttempt,$dedupDetailFile,$engineID,$webDir; $errorline = "idevs error"; $linesBackupErrorFile = file($individual_errorfile); #To check the individual error file for exit scenarios if(count($linesBackupErrorFile)>0){ foreach($linesBackupErrorFile as $key => $item){ $tempLinesBackupErrorFile = trim($item); if(empty($tempLinesBackupErrorFile) or stripos($tempLinesBackupErrorFile,$errorline) !== FALSE){ continue; } if($retryAttempt == false){ //Added by anil on 12 Aug 2019 $errorTxt = "Network is unreachable"; if(stripos($tempLinesBackupErrorFile,$errorTxt) !== FALSE){ traceLog("Retry due to : ".$tempLinesBackupErrorFile); checkForInternetAndWaitAccord(); $retryAttempt = true; break; } // Moves the internal pointer to the first element of the array reset($ErrorArgumentsMinRetry); foreach($ErrorArgumentsMinRetry as $inx => $val){ if(stripos($tempLinesBackupErrorFile,$val) !== FALSE){ // $engineID file_put_contents("LimitedRetry.txt", $val, FILE_APPEND); $retryAttempt = true; break; } } if($retryAttempt == false){ reset($ErrorArgumentsRetry); foreach($ErrorArgumentsRetry as $inx => $val){ if(stripos($tempLinesBackupErrorFile,$val) !== FALSE){ $retryAttempt = true; checkErrorAndUpdateServerAddr("_$engineID"); traceLog("Retry due to : ".$tempLinesBackupErrorFile); break; } } } } // Moves the internal pointer to the first element of the array reset($ErrorArgumentsExit); foreach($ErrorArgumentsExit as $inx => $val){ if(stripos($tempLinesBackupErrorFile,$val) !== FALSE) { checkDedupDeviceError($tempLinesBackupErrorFile); //@unlink(ERRORFILE."_$engineID"); #commented as per deepak advice /*if(is_dir(EVS_TEMP)) removeItems(EVS_TEMP);*/ $errStr = "Operation could not be completed. Reason : $val \n"; traceLog($errStr); jobTerminate($val); if(($val == "account has been cancelled")|| ($val == "account is under maintenance") || ($val == "account has been blocked") || ($val == "account has been expired")){ //update user status to account settings file updateUserStatus($val); } if($val == "encryption verification failed"){ $needToLogout = "$webDir/".APPDIR.PATHSEPARATOR."logout.txt"; if(file_put_contents($needToLogout, DECRYPTIONFAILED)) chmod($needToLogout,0777); } return false; } } } } reduceRetryFileSizeInprogress(); return true; } #**************************************************************************************************** # Subroutine Name : reduceRetryFileSizeInprogress. # Objective : This function is used to reduce the file size of files which got failed. # Added By : Anil Kumar # Modified By : #***************************************************************************************************** function reduceRetryFileSizeInprogress() { GLOBAL $engineID; $progressPath = PROGRESS_PATH."_$engineID"; $progressFile = file_get_contents($progressPath); $progressData = explode("\n", $progressFile); if($progressData[8] != '100%' && $progressData[7] > 0){ $progressData[3] -= $progressData[7]; } updateProgressFile($progressData[0], $progressData[1], $progressData[2], $progressData[3], $progressData[4], $progressData[6], $progressData[7], $progressData[8], $progressPath); } #**************************************************************************************************** # Subroutine Name : process_term # Objective : The signal handler invoked when SIGTERM signal is received by the script # Added By : Senthil Pandian [12/01/2016] # Modified By : #***************************************************************************************************** function process_term() { GLOBAL $dbh,$fileCount,$engineID; if($fileCount > 0){ if($dbh){ $dbh->commit(); unset($dbh); } } writeParameterValuesToStatusFile($engineID); exit; } #**************************************************************************************************** # Subroutine Name : writeParameterValuesToStatusFile # Objective : This subroutine writes the value of the specified parameter to the Status File # Added By : Senthil Pandian [12/01/2016] # Modified By : #***************************************************************************************************** /*function writeParameterValuesToStatusFile() { GLOBAL $fileOperationCount, $fileSyncCount, $failedfiles_count, $exit_flag, $engineID; $Backupcount= 0; $Synccount = 0; $failedcount = 0; $exitFlagRead = 0; list($Backupcount, $Synccount, $failedcount, $exitFlagRead) = getParameterValueFromStatusFile(array('FILES_COUNT_INDEX', 'SYNC_COUNT_FILES_INDEX', 'FAILED_COUNT_FILES_INDEX', 'EXIT_FLAG_INDEX'),$engineID); try { # Calculate the backup, sync and error count based on new values $Backupcount += $fileOperationCount; $Synccount += $fileSyncCount; $failedcount = $failedfiles_count; $content = "FILES_COUNT_INDEX = $Backupcount \n"; $content .= "SYNC_COUNT_FILES_INDEX = $Synccount \n"; $content .= "FAILED_COUNT_FILES_INDEX = $failedcount\n"; if(!$exitFlagRead or !empty($exitFlagRead)) $content .= "EXIT_FLAG_INDEX = $exit_flag "; else $content .= "EXIT_FLAG_INDEX = $exitFlagRead "; file_put_contents(STATUSFILE."_$engineID",$content); @chmod(STATUSFILE."_$engineID",PERMISSIONVAL); } catch (Exception $e){ traceLog("Unable to open file ".STATUSFILE.". Reason:".$e->getMessage()); } } */ #************************************************************************************************************************* # Subroutine Name : getFullPathofFile # Objective : This subroutine gets the full path of files that have backed up # Added By : Senthil Pandian [11/30/2016] # Modified By : Sabin Cheruvattil #************************************************************************************************************************* function getFullPathofFile($fileToCheck) { GLOBAL $currentBackupset, $relative; $returnItem = ''; if($relative == RELATIVE) return ''; $fileToCheck = html_entity_decode($fileToCheck); reset($currentBackupset); $currentBackupsetReverse = array_reverse($currentBackupset); foreach($currentBackupsetReverse as $key => $item) { $item = trim($item, "\n"); $a = strrpos($item, '/'); $match = substr($item, $a); if($fileToCheck !== $match) continue; $returnItem = $item; break; } unset($currentBackupsetReverse); return $returnItem; } #************************************************************************************************************************* # Subroutine Name : getFinalErrorFile # Objective : This subroutine creates a final ERROR file for each backupset, which has to be displayed in LOGS # Added By : Senthil Pandian [12/01/2016] # Modified By : #************************************************************************************************************************** function getFinalErrorFile(&$failed_count) { GLOBAL $exit_flag, $jobrunningDir, $backup_error, $failedfiles_array, $xtraSuccessCounter, $dedupDetailFile, $ErrorArgumentsForNoRetry; $individual_errorfileContents = ''; $failedWithReason = 0; $errorContent = ""; $cancel = 0; $serverRoot = ""; if($exit_flag == 0 && !file_exists(PIDFILE)) $cancel = 1; #In case of abort only failed with reason must be shown if($exit_flag != 0 or $cancel !== 0) $failed_count = $failedWithReason; $failedfiles_arrayCount = count($failedfiles_array); #If no failed files then return if($failedfiles_arrayCount <= 0) return; $individual_errorfile = $jobrunningDir."/".ERROR_DIR."/$backup_error"; #If unable to open final error file to write then return if(!$errorFinalHandle = fopen($individual_errorfile."_FINAL",'w')){ traceLog("Unable to open $individual_errorfile"."_FINAL"."Reason : ".customErrorMsg()); return; } #Load error file content in an array if(file_exists($individual_errorfile)){ $individual_errorfileContents = file($individual_errorfile); @unlink($individual_errorfile); } $errStringCount = count($individual_errorfileContents); # If no error file and backup completes then dump failed files without error reason if($errStringCount <= 0 && $cancel === 0){ $localtime = trim(exec('date +"%a %b %d %T %Y"')); foreach($failedfiles_array as $failedItem){ if($xtraSuccessCounter > 0){ $xtraSuccessCounter--; continue; } $failedfile = substr($failedItem,1); $errorContent .= "[$localtime] [FAILED] [$failedfile] ".LINEFEED; } if($errorContent != "") fwrite($errorFinalHandle, $errorContent); fclose($errorFinalHandle); return; } #Fetch server root for dedup accounts if(file_exists($dedupDetailFile)){ $deviceArray = json_decode(file_get_contents($dedupDetailFile), true); $deviceID = trim($deviceArray['deviceID']); $deviceID = preg_replace("/^5c0b/", '', preg_replace("/4b5z$/", '', $deviceID, 1), 1); $serverRoot = "/".trim($deviceArray['serverRoot'])."/"; } for($i = 0; $i < $failedfiles_arrayCount; $i++){ $errString = ""; $failedfile = phpQuoteMeta(substr($failedfiles_array[$i],1)); $noRetryFileContent = ""; for($j = 0; $j < $errStringCount; $j++){ if($individual_errorfileContents[$j] == '') continue; if(preg_match("/$failedfile/", $individual_errorfileContents[$j])) { foreach($ErrorArgumentsForNoRetry as $val){ if(strpos($individual_errorfileContents[$j],$val) !== FALSE){ $noRetryFileContent .= "[".trim(exec('date +"%a %b %d %T %Y"'))."] [FAILED] [$failedfiles_array[$i]] Reason : $val ".LINEFEED; } } if($noRetryFileContent != "") { $noRetryErrorFile = $individual_errorfile."_noretry.txt"; file_put_contents($noRetryErrorFile, $noRetryFileContent, FILE_APPEND); $errString = $noRetryFileContent; $failedWithReason++; break; } $errString = $individual_errorfileContents[$j]; $errString = trim(preg_replace("/\[.*\]\,/", '', $errString)); $errString = preg_replace("/$failedfile/", '', $errString); $errString = preg_replace("/\[\/\]/", '', $errString); $errString = str_replace('Number of files failed to backup 0.', '', $errString); //added for the update retained issue. if($serverRoot != ""){ $errString = preg_replace("/ERROR: /", '', $errString, 1); $errString = str_replace($serverRoot, '', $errString); } $updateRetainedText = "read errors mapping"; if(($errString != '') && preg_match("/$updateRetainedText/", $errString)) { $errString = "failed verification -- update retained"; } if(!file_exists($dedupDetailFile) && preg_match("/$updateRetainednonDedup/", $errString)){ $updateRetainednonDedup = "(will try again)"; if( preg_match("/$updateRetainednonDedup/", $errString)) $errString = "failed verification -- update retained"; } $errorContent .= "[".trim(exec('date +"%a %b %d %T %Y"'))."] [FAILED] [$failedfiles_array[$i]] Reason : $errString ".LINEFEED; $failedWithReason++; break; } } if($errString == "" and $exit_flag == 0 and $cancel === 0){ if($xtraSuccessCounter > 0){ $xtraSuccessCounter--; continue; } $errorContent .= "[".trim(exec('date +"%a %b %d %T %Y"'))."] [FAILED] [$failedfiles_array[$i]] ".LINEFEED; } } #In case of abort only failed with reason must be shown if($exit_flag != 0 or $cancel !== 0) $failed_count = $failedWithReason; if($errorContent != "") fwrite($errorFinalHandle, $errorContent); fclose($errorFinalHandle); } #************************************************************************************************************************ # Subroutine Name : getFailedFileCount # Objective : This subroutine gets the failed files count from the failedfiles array # Added By : Senthil Pandian [12/01/2016] # Modified By : #************************************************************************************************************************ function getFailedFileCount() { GLOBAL $currentBackupset,$current_BackupsetHash,$failedfiles_array, $fileOperationCount, $fileSyncCount, $xtraSuccessCounter; GLOBAL $extFileBackupCount, $extFileSyncCount; $failed = 0; reset($currentBackupset); foreach($currentBackupset as $key => $item){ $tempCurrentBackupset = trim($item,"\n"); if(empty($tempCurrentBackupset)) continue; if($current_BackupsetHash[$tempCurrentBackupset]['status'] === 0){ $failedfiles_array[$failed] = $tempCurrentBackupset; $failed++; } elseif($current_BackupsetHash[$tempCurrentBackupset]['status'] === 1) $fileOperationCount++; elseif($current_BackupsetHash[$tempCurrentBackupset]['status'] === 2) $fileSyncCount++; } $failed_count = $failed - ($extFileBackupCount + $extFileSyncCount); $fileOperationCount += $extFileBackupCount; $fileSyncCount += $extFileSyncCount; $xtraSuccessCounter = $extFileBackupCount + $extFileSyncCount; return $failed_count; } #************************************************************************************************************************ # Subroutine Name : updateTotalFailedFileCount # Objective : This subroutine gets the updated failed files count incase retry backup is in process # Added By : Senthil Pandian [12/01/2016] # Modified By : #************************************************************************************************************************* function updateTotalFailedFileCount($failed_count) { GLOBAL $currentBackupset,$backup_error,$failedfiles_count, $engineID; list($orig_count) = getParameterValueFromStatusFile(array("FAILED_COUNT_FILES_INDEX"),$engineID); if(strpos($backup_error,'failedfiles.txt') !== FALSE){ $size_Backupset = count($currentBackupset); $newcount = $orig_count - $size_Backupset + $failed_count; $failedfiles_count = $newcount; }else{ $orig_count += $failed_count; $failedfiles_count = $orig_count; } } #*********************************************************************************************************************** # Subroutine Name : jobTerminate # Objective : This subroutine terminates the job in case of error # Added By : Senthil Pandian [12/01/2016] # Modified By : #************************************************************************************************************************ function jobTerminate($reason) { GLOBAL $phpPath,$fileJobTerminationPath; $cmd = "$phpPath $fileJobTerminationPath 2 '$reason'"; exec($cmd); } #*********************************************************************************************************************** # Subroutine Name : getUpdateRetainedFilesList # Objective : Change the status from success to failure if failed verification -- update retained data exists. # Added By : Anil Kumar #************************************************************************************************************************ function getUpdateRetainedFilesList($errorFilePath) { GLOBAL $current_BackupsetHash, $xtraSuccessCounter, $backupLocation, $dedupDetailFile, $engineID; $linesBackupErrorFile = array(); if(file_exists($errorFilePath)) { # hash used is $current_BackupsetHash $linesBackupErrorFile = file($errorFilePath); } $size = 0; $serverRoot = ""; if(file_exists($dedupDetailFile)){ $deviceJSON = file_get_contents($dedupDetailFile); $deviceArray = json_decode($deviceJSON, true); $deviceID = trim($deviceArray['deviceID']); $deviceID = preg_replace("/^5c0b/", '', preg_replace("/4b5z$/", '', $deviceID, 1), 1); $serverRoot = "/".trim($deviceArray['serverRoot']); } for($i = 0; $i <= count($linesBackupErrorFile); $i++){ $error = "failed verification -- update retained"; if(preg_match("/failed verification -- update retained/", $linesBackupErrorFile[$i])) { if(file_exists($dedupDetailFile)){ #Dedup $linesBackupErrorFile[$i] = preg_replace('/\s+/', ' ', trim($linesBackupErrorFile[$i])); $linesBackupErrorFile[$i] = preg_replace("/ERROR: /", '', $linesBackupErrorFile[$i], 1); $linesBackupErrorFile[$i] = preg_replace("/ failed verification -- update retained./", '', $linesBackupErrorFile[$i], 1); $linesBackupErrorFile[$i] = str_replace($serverRoot, '', $linesBackupErrorFile[$i]); } else { #Non dedup $backupLocation = readBackupLocation(); $linesBackupErrorFile[$i] = preg_replace('/\s+/', ' ', trim($linesBackupErrorFile[$i])); $linesBackupErrorFile[$i] = preg_replace("/WARNING: /", '', $linesBackupErrorFile[$i], 1); $linesBackupErrorFile[$i] = str_replace(" failed verification -- update retained (will try again).", '', $linesBackupErrorFile[$i]); $linesBackupErrorFile[$i] = str_replace($backupLocation, '', $linesBackupErrorFile[$i]); } if($current_BackupsetHash{$linesBackupErrorFile[$i]}{'status'} == 1){ $fileOperationCount--; //$xtraSuccessCounter--; $current_BackupsetHash{$linesBackupErrorFile[$i]}{'status'} = 0; # this is to update the size value in the progress path. $size += convertToBytes($current_BackupsetHash{$linesBackupErrorFile[$i]}{'size'}); } } # As EVS team is replacing error message "Directory not empty", we are changing it. if(preg_match("/Directory not empty/", $linesBackupErrorFile[$i])) { $linesBackupErrorFile[$i] = preg_replace("/Directory not empty/", 'A directory name with the same file name already exists ', $linesBackupErrorFile[$i], 1); } } # this is to update the size value in the progress path. ################################################################# $progressPath = PROGRESS_PATH."_$engineID"; $progressContent = array(); if(file_exists($progressPath)){ if(filesize($progressPath)>0){ $progressContent = file($progressPath); $size = (double)trim($progressContent[3],"\n") - $size; } } else{ file_put_contents($progressPath,""); chmod($progressPath,PERMISSIONVAL); } updateProgressFile(trim($progressContent[0],"\n"), trim($progressContent[1],"\n"), trim($progressContent[2],"\n"), $size, trim($progressContent[4],"\n"), trim($progressContent[6],"\n"), trim($progressContent[7],"\n"),trim($progressContent[8],"\n"), $progressPath); } #*********************************************************************************************************************** # Subroutine Name : updateUserLogForBackupRestoreFiles # Objective : Writes the data in user log after one backup set complete. # Added By : Anil Kumar #************************************************************************************************************************ function updateUserLogForBackupRestoreFiles() { GLOBAL $logOutputFilePath, $current_BackupsetHash, $outputArray; if(!$logOutputHandle = fopen($logOutputFilePath,'a')){ traceLog("Could not open $logOutputFilePath, Reason : ".customErrorMsg()); return; } foreach($current_BackupsetHash as $key => $item){ $keyString = trim($key,"\n"); if(empty($keyString)) continue; if($current_BackupsetHash[$keyString]['status'] == 1) { $outputArray[] = $current_BackupsetHash[$keyString]['detail'] ." [SUCCESS] [".$keyString."] [".$current_BackupsetHash[$keyString]['size']."]".LINEFEED; } } if($outputArray) { $isLocked = flock($logOutputHandle,LOCK_EX); if($isLocked){ fwrite($logOutputHandle, implode("",$outputArray)); flock($logOutputHandle,LOCK_UN); } else { fwrite($logOutputHandle, implode("",$outputArray)); traceLog("Unable to lock LOG file"); } } //need to check unset($outputArray); fclose($logOutputHandle); } function convertToBytes($p_sFormatted) { $aUnits = array('B'=>0, 'KB'=>1, 'MB'=>2, 'GB'=>3, 'TB'=>4, 'PB'=>5, 'EB'=>6, 'ZB'=>7, 'YB'=>8); $sUnit = strtoupper(trim(substr($p_sFormatted, -2))); if (intval($sUnit) !== 0) { $sUnit = 'B'; } if (!in_array($sUnit, array_keys($aUnits))) { return false; } $iUnits = trim(substr($p_sFormatted, 0, strlen($p_sFormatted) - 2)); if (!intval($iUnits) == $iUnits) { return false; } return $iUnits * pow(1024, $aUnits[$sUnit]); } #*********************************************************************************************************************** # Subroutine Name : checkForInternetAndWaitAccord # Objective : This method is used to check internet availability and sleep for 30 seconds incase if it is not connected # Added By : Anil Kumar #************************************************************************************************************************ function checkForInternetAndWaitAccord() { GLOBAL $webDir; $pingPath = $webDir.PATHSEPARATOR.APPDIR.PATHSEPARATOR."ping"; $str = ""; $str = `$pingPath -c 1 8.8.8.8 2>&1`; if (strpos($str, '0% packet loss')) { traceLog("Internet connection available."); } else { traceLog("Internet Connectivity not available."); if(!file_exists(PIDFILE)) return; sleep(30); checkForInternetAndWaitAccord(); } }