From 5b440368ca3251ba78309c92b0f0a5255062a7a2 Mon Sep 17 00:00:00 2001 From: dhanabalan Date: Fri, 6 Nov 2026 20:56:10 +0530 Subject: [PATCH] Changed logic in sftp process files --- app/Console/Commands/ProcessSftpFiles.php | 562 +++++++++++++++++----- 1 file changed, 448 insertions(+), 114 deletions(-) diff --git a/app/Console/Commands/ProcessSftpFiles.php b/app/Console/Commands/ProcessSftpFiles.php index 57872fbc..6149f1b7 100644 --- a/app/Console/Commands/ProcessSftpFiles.php +++ b/app/Console/Commands/ProcessSftpFiles.php @@ -13,6 +13,7 @@ use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Mail; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Facades\Schema; +use Illuminate\Support\Facades\File; class ProcessSftpFiles extends Command { @@ -27,229 +28,562 @@ class ProcessSftpFiles extends Command $this->sftpService = $sftpService; } + // public function handle() + // { + + // // $files = Storage::disk('ftp')->allFiles(); + + // $folders = Storage::disk('ftp')->directories(); + + // $allFiles = []; + + // foreach ($folders as $folder) { + + // Log::info("Checking folder: $folder"); + + // // Get files inside the current folder + // $files = Storage::disk('ftp')->files($folder); + + // foreach ($files as $file) { + + // $allFiles[] = $file; + // } + // } + + // if (empty($allFiles)) { + // $this->info('No files to process.'); + // return; + // } + + // $logMessages = []; + + // foreach ($allFiles as $file) + // { + // $this->info("Processing file: $file"); + + // $folderName = explode('/', $file)[0]; + + // $plant = Plant::where('code', $folderName)->first(); + + // if (!$plant) { + // $this->warn("No plant found for folder: $folderName. Skipping file."); + // $logMessages[] = $folderName; + // continue; + // } + + // $plantId = $plant->id; + + // $plantCode = $plant->code ?? null; + + // $fileName = basename($file); // GMGLAS03-1818199-1.txt + // $workCenterCode = explode('-', $fileName)[0]; // GMGLAS03 + + // $machine = Machine::where('work_center', $workCenterCode)->first(); + // if (!$machine) { + // $message = "Machine '$workCenterCode' not found and file name '$file'"; + // $logMessages[] = $message; + // continue; + // } + + // $machineAgaPlant = Machine::where('plant_id', $plantId)->where('work_center', $workCenterCode)->first(); + // if (!$machineAgaPlant) { + // $message = "Machine '$workCenterCode' not found for plant code '$plantCode' and file name '$file'"; + // $logMessages[] = $message; + // continue; + // } + + // $machineId = $machineAgaPlant->id; + + // $content = $this->sftpService->getFileContent($file); + + // $lines = array_filter(preg_split("/\r\n|\n|\r/", $content)); + + // $headers = array_map(fn($header) => strtolower(trim($header)), str_getcsv(array_shift($lines))); + + // $dbColumns = Schema::getColumnListing('production_temps'); + + // $missingColumns = array_diff($headers, $dbColumns); + + // if (!empty($missingColumns)) { + + // $message = "Skipping file: $file. The following columns do not exist in the database. Please add them to the database: " . implode(', ', $missingColumns); + + // $logMessages[] = $message; + + // $this->warn($message); + + // continue; + // } + + // $expectedColumns = count($headers); + // $isValidFile = true; + // $isserialNotEmptyFile = true; + + // foreach ($lines as $lineNumber => $line) { + // $data = str_getcsv($line); + + // if (!empty($data[0])) { + // if (count($data) != $expectedColumns) { + // $message = "Column mismatch in file: $file at line " . ($lineNumber + 2) . + // ". Expected $expectedColumns, got " . count($data); + + // $logMessages[] = $message; + + // $isValidFile = false; + // break; + // } + // } + // } + + // if (!$isValidFile) { + // $message = "Skipping entire file due to invalid column mismatch: $file"; + // $logMessages[] = $message; + // continue; + // } + // else if (!$isserialNotEmptyFile) { + // $message = "serial number not found in file name $file"; + // $logMessages[] = $message; + // continue; + // } + + // $rows = []; + + // // foreach ($lines as $line) { + // // $data = str_getcsv($line); + + // // if (!empty($data[0])) { + // // $row = []; + // // foreach ($headers as $index => $column) { + // // $row[$column] = isset($data[$index]) ? trim($data[$index]) : null; + // // } + + // // $row['plant_id'] = $plantId; + // // $row['machine_id'] = $machineId; + // // $row['created_at'] = now(); + // // $row['updated_at'] = now(); + + // // // $rows[] = $row; + // // if (!empty($row['gernr'])) { + // // $rows[] = $row; + // // } + // // } + // // } + + // foreach ($lines as $line) { + + // $data = str_getcsv($line); + + // if (!empty($data[0])) { + + // $row = []; + + // foreach ($headers as $index => $column) { + // $row[$column] = isset($data[$index]) + // ? trim($data[$index]) + // : null; + // } + + // $row['plant_id'] = $plantId; + // $row['machine_id'] = $machineId; + // $row['created_at'] = now(); + // $row['updated_at'] = now(); + + // if (empty($row['gernr'])) { + // $message = "serial number not found in file name $file"; + // $logMessages[] = $message; + // continue; + // } + + // $gamng = (float) ($row['gamng'] ?? 0); + // $lmnga = (float) ($row['lmnga'] ?? 0); + + // $existingRecords = ProductionTemp::where('plant_id', $plantId) + // ->where('aufnr', ($row['aufnr'])) + // ->get(); + + + // $existingRecord = $existingRecords->where('gernr', $row['gernr'])->first(); + + // $lastStatus = $existingRecords->max('pending_released_status'); + + // if ($gamng == $lmnga) { + + // $row['pending_released_status'] = 0; + + // $this->info("Production order {$row['aufnr']} completed. " ."Serial: {$row['gernr']}. " ."GAMNG: {$row['gamng']}, LMNGA: {$row['lmnga']}. " ."pending_released_status = 0" ); + + // } elseif ($existingRecord) { + + // $row['pending_released_status'] = (int) $existingRecord->pending_released_status; + + // $this->info("Same serial already exists. " ."Production order: {$row['aufnr']}. " ."Serial: {$row['gernr']}. " ."Keeping pending_released_status = {$row['pending_released_status']}"); + + // } else { + + // if ($existingRecords->isEmpty()) { + + // // First release + // $row['pending_released_status'] = 0; + + // } else { + // $row['pending_released_status'] = ((int) $lastStatus) + 1; + // } + // $this->info("Production order {$row['aufnr']} pending. " ."New serial: {$row['gernr']}. " . "GAMNG: {$row['gamng']}, LMNGA: {$row['lmnga']}. " ."pending_released_status = {$row['pending_released_status']}"); + // } + + // $rows[] = $row; + // } + // } + + // if (!empty($rows)) { + // ProductionTemp::upsert($rows, ['plant_id', 'gernr'], array_keys($rows[0])); //(update + insert) + // } + // } + + // if (!empty($logMessages)) { + + // $content = implode(PHP_EOL, $logMessages); + + // Mail::raw('Please find attached machine error logs.', function ($message) use ($content) { + // $message->to(env('ERROR_LOG_MAIL')) + // ->subject('Machine Error Logs') + // ->attachData($content, 'machine_errors.txt', [ + // 'mime' => 'text/plain', + // ]); + // }); + // } + // } + public function handle() { + $basePath = '/srv/pds.iotsignin.com/www/ftpdata'; - $files = Storage::disk('ftp')->allFiles(); - - $folders = Storage::disk('ftp')->directories(); + // $basePath = '/home/iot-dev/projects/pds/ftpdata'; $allFiles = []; + $logMessages = []; - foreach ($folders as $folder) { + // Check whether data2 exists + if (!File::isDirectory($basePath)) { + $this->error("Directory not found: {$basePath}"); + return; + } - Log::info("Checking folder: $folder"); + $folders = Plant::pluck('code'); - // Get files inside the current folder - $files = Storage::disk('ftp')->files($folder); + foreach ($folders as $folderName) { + + $folderPath = $basePath . '/' . $folderName; + + if (!File::isDirectory($folderPath)) { + + File::makeDirectory($folderPath, 0755, true); + + // Set ownership + exec("chown www-data:www-data " . escapeshellarg($folderPath)); + + $this->info("Created missing folder: {$folderName}"); + } + } + + // Get plant folders: 1000, 1010, etc. + $folders = File::directories($basePath); + + if (empty($folders)) { + $this->info('No plant folders found.'); + return; + } + + // Collect TXT files from each plant folder + foreach ($folders as $folderPath) { + + $folderName = basename($folderPath); + + Log::info("Checking plant folder: {$folderName}"); + + // Get files inside the plant folder + $files = File::files($folderPath); foreach ($files as $file) { - $allFiles[] = $file; + if (strtolower($file->getExtension()) !== 'txt') { + continue; + } + + $allFiles[] = [ + 'path' => $file->getPathname(), + 'folder' => $folderName, + ]; } } if (empty($allFiles)) { - $this->info('No files to process.'); + $this->info('No TXT files to process.'); return; } - $logMessages = []; + $dbColumns = Schema::getColumnListing('production_temps'); - foreach ($allFiles as $file) - { - $this->info("Processing file: $file"); + // Process each TXT file + foreach ($allFiles as $fileDetails) { - $folderName = explode('/', $file)[0]; + $file = $fileDetails['path']; + $folderName = $fileDetails['folder']; + $this->info("Processing file: {$file}"); + + // Find plant using folder name $plant = Plant::where('code', $folderName)->first(); if (!$plant) { - $this->warn("No plant found for folder: $folderName. Skipping file."); - $logMessages[] = $folderName; + + $message = "No plant found for folder: {$folderName}. File: {$file}"; + + $this->warn($message); + $logMessages[] = $message; + continue; } $plantId = $plant->id; + $plantCode = $plant->code; - $plantCode = $plant->code ?? null; + // Get filename and work center + // Example: GMGLAS03-1818199-1.txt + $fileName = basename($file); - $fileName = basename($file); // GMGLAS03-1818199-1.txt - $workCenterCode = explode('-', $fileName)[0]; // GMGLAS03 + $workCenterCode = explode('-', $fileName)[0]; + + // Find machine belonging to this plant + $machine = Machine::where('plant_id', $plantId) + ->where('work_center', $workCenterCode) + ->first(); - $machine = Machine::where('work_center', $workCenterCode)->first(); if (!$machine) { - $message = "Machine '$workCenterCode' not found and file name '$file'"; + + $message = "Machine '{$workCenterCode}' not found for plant '{$plantCode}'. File: {$file}"; + + $this->warn($message); $logMessages[] = $message; + continue; } - $machineAgaPlant = Machine::where('plant_id', $plantId)->where('work_center', $workCenterCode)->first(); - if (!$machineAgaPlant) { - $message = "Machine '$workCenterCode' not found for plant code '$plantCode' and file name '$file'"; + $machineId = $machine->id; + + // Read local TXT file + $content = File::get($file); + + // Split file into lines + $lines = preg_split("/\r\n|\n|\r/", $content); + + // Remove empty lines and reindex + $lines = array_values(array_filter($lines, fn($line) => trim($line) !== '')); + + if (empty($lines)) { + + $message = "File is empty: {$file}"; + + $this->warn($message); $logMessages[] = $message; + continue; } - $machineId = $machineAgaPlant->id; + // Read CSV headers + $headers = array_map( + fn($header) => strtolower(trim($header)), + str_getcsv(array_shift($lines)) + ); - $content = $this->sftpService->getFileContent($file); + if (empty($headers)) { - $lines = array_filter(preg_split("/\r\n|\n|\r/", $content)); + $message = "Headers not found in file: {$file}"; - $headers = array_map(fn($header) => strtolower(trim($header)), str_getcsv(array_shift($lines))); + $this->warn($message); + $logMessages[] = $message; - $dbColumns = Schema::getColumnListing('production_temps'); + continue; + } + // Validate CSV columns against database columns $missingColumns = array_diff($headers, $dbColumns); if (!empty($missingColumns)) { - $message = "Skipping file: $file. The following columns do not exist in the database. Please add them to the database: " . implode(', ', $missingColumns); - - $logMessages[] = $message; + $message = "Skipping file: {$file}. The following columns do not exist in the database: " + . implode(', ', $missingColumns); $this->warn($message); + $logMessages[] = $message; continue; } $expectedColumns = count($headers); $isValidFile = true; - $isserialNotEmptyFile = true; + // Validate column count in every row foreach ($lines as $lineNumber => $line) { + $data = str_getcsv($line); - if (!empty($data[0])) { - if (count($data) != $expectedColumns) { - $message = "Column mismatch in file: $file at line " . ($lineNumber + 2) . - ". Expected $expectedColumns, got " . count($data); + if (!empty($data[0]) && count($data) != $expectedColumns) { - $logMessages[] = $message; + $message = "Column mismatch in file: {$file} at line " + . ($lineNumber + 2) + . ". Expected {$expectedColumns}, got " + . count($data); - $isValidFile = false; - break; - } + $logMessages[] = $message; + + $this->warn($message); + + $isValidFile = false; + + break; } } if (!$isValidFile) { - $message = "Skipping entire file due to invalid column mismatch: $file"; - $logMessages[] = $message; - continue; - } - else if (!$isserialNotEmptyFile) { - $message = "serial number not found in file name $file"; + + $message = "Skipping entire file due to invalid column mismatch: {$file}"; + + $this->warn($message); $logMessages[] = $message; + continue; } $rows = []; - // foreach ($lines as $line) { - // $data = str_getcsv($line); - - // if (!empty($data[0])) { - // $row = []; - // foreach ($headers as $index => $column) { - // $row[$column] = isset($data[$index]) ? trim($data[$index]) : null; - // } - - // $row['plant_id'] = $plantId; - // $row['machine_id'] = $machineId; - // $row['created_at'] = now(); - // $row['updated_at'] = now(); - - // // $rows[] = $row; - // if (!empty($row['gernr'])) { - // $rows[] = $row; - // } - // } - // } - + // Process each CSV row foreach ($lines as $line) { $data = str_getcsv($line); - if (!empty($data[0])) { + if (empty($data[0])) { + continue; + } - $row = []; + $row = []; - foreach ($headers as $index => $column) { - $row[$column] = isset($data[$index]) - ? trim($data[$index]) - : null; - } + foreach ($headers as $index => $column) { - $row['plant_id'] = $plantId; - $row['machine_id'] = $machineId; - $row['created_at'] = now(); - $row['updated_at'] = now(); + $row[$column] = isset($data[$index]) + ? trim($data[$index]) + : null; + } - if (empty($row['gernr'])) { - $message = "serial number not found in file name $file"; - $logMessages[] = $message; - continue; - } + $row['plant_id'] = $plantId; + $row['machine_id'] = $machineId; + $row['created_at'] = now(); + $row['updated_at'] = now(); - $gamng = (float) ($row['gamng'] ?? 0); - $lmnga = (float) ($row['lmnga'] ?? 0); + // Validate serial number + if (empty($row['gernr'])) { - $existingRecords = ProductionTemp::where('plant_id', $plantId) - ->where('aufnr', ($row['aufnr'])) - ->get(); + $message = "Serial number not found in file {$file}"; + $logMessages[] = $message; - $existingRecord = $existingRecords->where('gernr', $row['gernr'])->first(); + continue; + } - $lastStatus = $existingRecords->max('pending_released_status'); + $gamng = (float) ($row['gamng'] ?? 0); + $lmnga = (float) ($row['lmnga'] ?? 0); - if ($gamng == $lmnga) { + // Get existing production order records + $existingRecords = ProductionTemp::where('plant_id', $plantId) + ->where('aufnr', $row['aufnr'] ?? null) + ->get(); + // Check whether this serial already exists + $existingRecord = $existingRecords + ->where('gernr', $row['gernr']) + ->first(); + + $lastStatus = $existingRecords->max('pending_released_status'); + + if ($gamng == $lmnga) { + + // Production order completed + $row['pending_released_status'] = 0; + + $this->info( + "Production order {$row['aufnr']} completed. " + . "Serial: {$row['gernr']}. " + . "GAMNG: {$gamng}, LMNGA: {$lmnga}. " + . "pending_released_status = 0" + ); + + } elseif ($existingRecord) { + + // Keep existing serial status + $row['pending_released_status'] = + (int) $existingRecord->pending_released_status; + + $this->info( + "Same serial already exists. " + . "Production order: {$row['aufnr']}. " + . "Serial: {$row['gernr']}. " + . "Keeping pending_released_status = {$row['pending_released_status']}" + ); + + } else { + + if ($existingRecords->isEmpty()) { + + // First release $row['pending_released_status'] = 0; - $this->info("Production order {$row['aufnr']} completed. " ."Serial: {$row['gernr']}. " ."GAMNG: {$row['gamng']}, LMNGA: {$row['lmnga']}. " ."pending_released_status = 0" ); - - } elseif ($existingRecord) { - - $row['pending_released_status'] = (int) $existingRecord->pending_released_status; - - $this->info("Same serial already exists. " ."Production order: {$row['aufnr']}. " ."Serial: {$row['gernr']}. " ."Keeping pending_released_status = {$row['pending_released_status']}"); - } else { - if ($existingRecords->isEmpty()) { - - // First release - $row['pending_released_status'] = 0; - - } else { - $row['pending_released_status'] = ((int) $lastStatus) + 1; - } - $this->info("Production order {$row['aufnr']} pending. " ."New serial: {$row['gernr']}. " . "GAMNG: {$row['gamng']}, LMNGA: {$row['lmnga']}. " ."pending_released_status = {$row['pending_released_status']}"); + // Increment pending release status + $row['pending_released_status'] = ((int) $lastStatus) + 1; } - $rows[] = $row; + $this->info( + "Production order {$row['aufnr']} pending. " + . "New serial: {$row['gernr']}. " + . "GAMNG: {$gamng}, LMNGA: {$lmnga}. " + . "pending_released_status = {$row['pending_released_status']}" + ); } + + $rows[] = $row; } + // Insert or update records if (!empty($rows)) { - ProductionTemp::upsert($rows, ['plant_id', 'gernr'], array_keys($rows[0])); //(update + insert) + + ProductionTemp::upsert( + $rows, + ['plant_id', 'gernr'], + array_keys($rows[0]) + ); + + $this->info( + count($rows) . " records inserted/updated from {$file}" + ); } } + // Send error logs through email if (!empty($logMessages)) { - $content = implode(PHP_EOL, $logMessages); + $emailContent = implode(PHP_EOL, $logMessages); + + Mail::raw('Please find attached machine error logs.', function ($message) use ($emailContent) { - Mail::raw('Please find attached machine error logs.', function ($message) use ($content) { $message->to(env('ERROR_LOG_MAIL')) ->subject('Machine Error Logs') - ->attachData($content, 'machine_errors.txt', [ + ->attachData($emailContent, 'machine_errors.txt', [ 'mime' => 'text/plain', ]); }); } + + $this->info('Production TXT file processing completed.'); } } -- 2.49.1