ï»¿
Your IP : 216.73.217.78

Current File : /home/seto/indexator.pm/bulk_import_worker.php
<?php
/**
 * Bulk import worker — spawned by BulkImportController.
 * Usage: php bulk_import_worker.php <jobId>
 */

define('BASE_PATH', __DIR__);
require __DIR__ . '/bootstrap.php';

$jobId = (int) ($argv[1] ?? 0);
if (!$jobId) {
    exit(1);
}

$db = \App\Core\Database::getInstance();

$db->execute(
    "UPDATE bulk_import_jobs SET status='running', started_at=datetime('now') WHERE id=?",
    [$jobId]
);

$job = $db->fetchOne("SELECT * FROM bulk_import_jobs WHERE id=?", [$jobId]);
if (!$job) {
    exit(1);
}

$domains     = json_decode($job['domains'], true) ?? [];
$sitemapPath = $job['sitemap_path'];

$projectRepo = new \App\Projects\ProjectRepository($db);
$urlRepo     = new \App\Urls\UrlRepository($db);
$queue       = new \App\Queue\JobQueue($db);
$validator   = new \App\Urls\UrlValidator();
$normalizer  = new \App\Urls\UrlNormalizer();
$importer    = new \App\Import\SitemapImporter([]);

$existing        = $db->fetchAll("SELECT domain FROM projects");
$existingDomains = array_map(
    fn($d) => preg_replace('/^www\./i', '', strtolower($d)),
    array_column($existing, 'domain')
);

$created            = 0;
$skipped            = 0;
$errors             = 0;
$createdProjectIds  = [];

function appendLog(int $jobId, \App\Core\Database $db, string $line): void
{
    $db->execute(
        "UPDATE bulk_import_jobs SET log = log || ? WHERE id = ?",
        [$line . "\n", $jobId]
    );
}

foreach ($domains as $domain) {
    $domain = strtolower(trim($domain));
    if ($domain === '') {
        continue;
    }

    $domainNorm = preg_replace('/^www\./i', '', $domain);
    if (in_array($domainNorm, $existingDomains, true)) {
        appendLog($jobId, $db, "SKIP\t$domain\talready exists");
        $skipped++;
        continue;
    }

    $sitemapUrl = 'https://' . $domain . $sitemapPath;

    try {
        $rawUrls = $importer->fetch($sitemapUrl);
    } catch (\Throwable $e) {
        appendLog($jobId, $db, "ERROR\t$domain\t" . $e->getMessage());
        $errors++;
        continue;
    }

    $seen     = [];
    $toInsert = [];
    foreach ($rawUrls as $raw) {
        $raw = trim($raw);
        if (!$validator->isValid($raw)) {
            continue;
        }
        $normalized = $normalizer->normalize($raw);
        if (isset($seen[$normalized])) {
            continue;
        }
        $seen[$normalized] = true;
        $toInsert[]        = ['url' => $raw, 'normalized_url' => $normalized];
    }

    $projectId = $projectRepo->create($domain, $domain, 'sitemap', $sitemapUrl);
    $urlRepo->bulkInsert($projectId, $toInsert, 'sitemap');
    $existingDomains[]   = $domainNorm;
    $createdProjectIds[] = $projectId;

    appendLog($jobId, $db, "OK\t$domain\t#$projectId\t" . count($toInsert) . " URLs");
    $created++;
}

// Auto-start indexation check for all newly created projects
foreach ($createdProjectIds as $pid) {
    $urlIds = $urlRepo->findIdsByStatus($pid, 'new');
    if (!empty($urlIds)) {
        $queue->enqueueBatch($pid, $urlIds);
        $projectRepo->updateStatus($pid, 'checking');
    }
}
if (!empty($createdProjectIds)) {
    $config  = $GLOBALS['config'];
    $candidates = [
        '/opt/cpanel/ea-php81/root/usr/bin/php',
        '/opt/cpanel/ea-php80/root/usr/bin/php',
        '/usr/local/bin/php',
        '/usr/bin/php',
        'php',
    ];
    $phpBin = 'php';
    foreach ($candidates as $p) {
        if ($p === 'php' || is_executable($p)) { $phpBin = $p; break; }
    }
    $workerPath = __DIR__ . '/worker.php';
    $logPath    = __DIR__ . '/worker_run.log';
    exec(sprintf('%s %s >> %s 2>&1 &',
        escapeshellcmd($phpBin),
        escapeshellarg($workerPath),
        escapeshellarg($logPath)
    ));
}

$summary = json_encode(['created' => $created, 'skipped' => $skipped, 'errors' => $errors]);
$db->execute(
    "UPDATE bulk_import_jobs SET status='done', finished_at=datetime('now'), summary=? WHERE id=?",
    [$summary, $jobId]
);