PDO::ERRMODE_EXCEPTION, PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, ] ); $insertSql = <<<'SQL' INSERT INTO `minorplanets_mpc` ( `desig_packed`, `mp_number`, `absolute_magnitude_h`, `slope_g`, `epoch_packed`, `mean_anomaly_deg`, `arg_perihelion_deg`, `ascending_node_deg`, `inclination_deg`, `eccentricity`, `mean_motion_deg_per_day`, `semimajor_axis_au`, `uncertainty_u`, `reference_code`, `num_observations`, `num_oppositions`, `arc_text`, `rms_residual`, `perturbers_1`, `perturbers_2`, `orbit_computer`, `hex_flags`, `designation_text`, `last_obs_compact`, `last_obs_date`, `raw_record` ) VALUES ( :desig_packed, :mp_number, :absolute_magnitude_h, :slope_g, :epoch_packed, :mean_anomaly_deg, :arg_perihelion_deg, :ascending_node_deg, :inclination_deg, :eccentricity, :mean_motion_deg_per_day, :semimajor_axis_au, :uncertainty_u, :reference_code, :num_observations, :num_oppositions, :arc_text, :rms_residual, :perturbers_1, :perturbers_2, :orbit_computer, :hex_flags, :designation_text, :last_obs_compact, :last_obs_date, :raw_record ) SQL; $statement = $pdo->prepare($insertSql); $isCli = PHP_SAPI === 'cli' || PHP_SAPI === 'phpdbg'; $statePath = $rootDir . DIRECTORY_SEPARATOR . 'scripts' . DIRECTORY_SEPARATOR . 'import' . DIRECTORY_SEPARATOR . 'import_mpc_minorplanets.state.json'; if ($isCli) { runFullImport($pdo, $statement, $datPath); exit(0); } runWebImportChunk($pdo, $statement, $datPath, $statePath); exit(0); function runFullImport(PDO $pdo, PDOStatement $statement, string $datPath): void { writeMessage('Stelle sicher, dass desig_packed case-sensitive gespeichert wird...'); ensureCaseSensitiveDesigPackedColumn($pdo); $minorplanetAssignments = backupUserMinorplanetAssignments($pdo); writeMessage('Gesicherte Nutzer-Kleinplaneten-Zuordnungen: ' . count($minorplanetAssignments)); writeMessage('Leere app_user_minorplanets und minorplanets_mpc und setze AUTO_INCREMENT zurueck...'); clearTableWithRetry($pdo, 'app_user_minorplanets'); resetAutoIncrementWithRetry($pdo, 'app_user_minorplanets'); clearTableWithRetry($pdo, 'minorplanets_mpc'); resetAutoIncrementWithRetry($pdo, 'minorplanets_mpc'); $state = [ 'processed' => 0, 'imported' => 0, 'skipped' => 0, 'in_data_section' => false, 'byte_offset' => 0, 'started_at' => microtime(true), 'total_data_lines' => countMpcorbDataLines($datPath), 'minorplanet_assignments' => $minorplanetAssignments, ]; if ($state['total_data_lines'] > 0) { writeMessage("Import startet. Erwartete Datensaetze: {$state['total_data_lines']}."); } else { writeMessage('Import startet. Gesamtzahl der Datensaetze konnte nicht ermittelt werden.'); } $result = processImportChunk($pdo, $statement, $datPath, $state, PHP_INT_MAX, 10000, 5000); if (!$result['completed']) { writeMessage('Import wurde unerwartet nicht abgeschlossen.', true); exit(1); } writeMessage("Import abgeschlossen. Verarbeitet: {$result['state']['processed']}, importiert: {$result['state']['imported']}, uebersprungen: {$result['state']['skipped']}."); restoreUserMinorplanetAssignments($pdo, $result['state']['minorplanet_assignments'] ?? []); writeDatabaseCountSummary($pdo, $result['state']['imported']); } function runWebImportChunk(PDO $pdo, PDOStatement $statement, string $datPath, string $statePath): void { $resetRequested = isset($_GET['reset']) && $_GET['reset'] === '1'; $batchLineLimit = max(1000, min(50000, (int) ($_GET['batch'] ?? 20000))); $progressInterval = max(1000, min($batchLineLimit, (int) ($_GET['progress'] ?? 5000))); $commitInterval = max(1000, min($batchLineLimit, (int) ($_GET['commit'] ?? 5000))); if ($resetRequested || !is_file($statePath)) { writeMessage('Initialisiere Web-Import...'); ensureCaseSensitiveDesigPackedColumn($pdo); $minorplanetAssignments = backupUserMinorplanetAssignments($pdo); writeMessage('Gesicherte Nutzer-Kleinplaneten-Zuordnungen: ' . count($minorplanetAssignments)); clearTableWithRetry($pdo, 'app_user_minorplanets'); resetAutoIncrementWithRetry($pdo, 'app_user_minorplanets'); clearTableWithRetry($pdo, 'minorplanets_mpc'); resetAutoIncrementWithRetry($pdo, 'minorplanets_mpc'); $state = [ 'processed' => 0, 'imported' => 0, 'skipped' => 0, 'in_data_section' => false, 'byte_offset' => 0, 'started_at' => microtime(true), 'total_data_lines' => countMpcorbDataLines($datPath), 'status' => 'running', 'minorplanet_assignments' => $minorplanetAssignments, ]; saveImportState($statePath, $state); writeMessage("Web-Import vorbereitet. Erwartete Datensaetze: {$state['total_data_lines']}."); } else { $state = loadImportState($statePath); if (($state['status'] ?? 'running') === 'completed') { writeMessage('Vorheriger Import ist bereits abgeschlossen. Mit ?reset=1 neu starten.'); writeDatabaseCountSummary($pdo, (int) ($state['imported'] ?? 0)); return; } writeMessage("Setze Web-Import fort bei Datensatz {$state['processed']}..."); } $result = processImportChunk($pdo, $statement, $datPath, $state, $batchLineLimit, $progressInterval, $commitInterval); $state = $result['state']; if ($result['completed']) { restoreUserMinorplanetAssignments($pdo, $state['minorplanet_assignments'] ?? []); $state['status'] = 'completed'; saveImportState($statePath, $state); writeMessage("Import abgeschlossen. Verarbeitet: {$state['processed']}, importiert: {$state['imported']}, uebersprungen: {$state['skipped']}."); writeDatabaseCountSummary($pdo, (int) $state['imported']); writeRestartHint(); return; } $state['status'] = 'running'; saveImportState($statePath, $state); writeMessage("Chunk fertig. Naechster Start bei Datensatz {$state['processed']}."); writeMessage('Naechster Chunk wird automatisch geladen...'); writeAutoContinueScript($batchLineLimit, $progressInterval, $commitInterval); } function processImportChunk( PDO $pdo, PDOStatement $statement, string $datPath, array $state, int $lineLimit, int $progressInterval, int $commitInterval ): array { $handle = fopen($datPath, 'rb'); if ($handle === false) { writeMessage("MPCORB.DAT konnte nicht geoeffnet werden: {$datPath}", true); exit(1); } if (($state['byte_offset'] ?? 0) > 0) { fseek($handle, (int) $state['byte_offset']); } $processedInChunk = 0; $batchProcessed = 0; $completed = true; $pdo->beginTransaction(); try { while (($line = fgets($handle)) !== false) { $state['byte_offset'] = ftell($handle); $line = rtrim($line, "\r\n"); if ($line === '') { continue; } if (!($state['in_data_section'] ?? false)) { if (preg_match('/^-{20,}$/', trim($line)) === 1) { $state['in_data_section'] = true; } continue; } $state['processed']++; $processedInChunk++; $parseResult = parseMpcorbLine($line); if ($parseResult['record'] === null) { $state['skipped']++; writeMessage(formatSkipDetail((int) $state['processed'], $line, $parseResult['designation'], $parseResult['reason']), true); } else { try { $statement->execute($parseResult['record']); $state['imported']++; $batchProcessed++; } catch (Throwable $rowException) { $state['skipped']++; writeMessage(formatSkipDetail( (int) $state['processed'], $line, $parseResult['designation'], 'DB-Fehler: ' . $rowException->getMessage() ), true); } } if ($batchProcessed >= $commitInterval) { $pdo->commit(); writeMessage("Zwischenstand gespeichert bei {$state['processed']} verarbeiteten Datensaetzen."); $pdo->beginTransaction(); $batchProcessed = 0; } if ($state['processed'] % $progressInterval === 0) { writeMessage(buildProgressMessage( (int) $state['processed'], (int) $state['imported'], (int) $state['skipped'], (int) ($state['total_data_lines'] ?? 0), (float) ($state['started_at'] ?? microtime(true)) )); } if ($processedInChunk >= $lineLimit) { $completed = false; break; } } if ($pdo->inTransaction()) { $pdo->commit(); } } catch (Throwable $e) { fclose($handle); if ($pdo->inTransaction()) { $pdo->rollBack(); } writeMessage('Importfehler: ' . $e->getMessage(), true); exit(1); } if (feof($handle)) { $completed = true; } fclose($handle); return [ 'completed' => $completed, 'state' => $state, ]; } function loadImportState(string $statePath): array { $json = @file_get_contents($statePath); $data = is_string($json) ? json_decode($json, true) : null; if (!is_array($data)) { return [ 'processed' => 0, 'imported' => 0, 'skipped' => 0, 'in_data_section' => false, 'byte_offset' => 0, 'started_at' => microtime(true), 'total_data_lines' => 0, 'status' => 'running', ]; } return $data; } function saveImportState(string $statePath, array $state): void { file_put_contents( $statePath, json_encode($state, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES), LOCK_EX ); } function writeDatabaseCountSummary(PDO $pdo, int $imported): void { $dbRowCount = (int) $pdo->query('SELECT COUNT(*) FROM `minorplanets_mpc`')->fetchColumn(); writeMessage("Kontrolle Datenbank: minorplanets_mpc enthaelt {$dbRowCount} Zeilen."); if ($dbRowCount !== $imported) { writeMessage( "WARNUNG: Importzaehler ({$imported}) und echte Datenbankzeilen ({$dbRowCount}) unterscheiden sich.", true ); } } function backupUserMinorplanetAssignments(PDO $pdo): array { $sql = <<<'SQL' SELECT aum.`user_id`, aum.`is_favorite`, aum.`created_at`, mp.`desig_packed` FROM `app_user_minorplanets` aum INNER JOIN `minorplanets_mpc` mp ON mp.`id` = aum.`minorplanet_id` ORDER BY aum.`user_id` ASC, aum.`minorplanet_id` ASC SQL; return $pdo->query($sql)->fetchAll() ?: []; } function restoreUserMinorplanetAssignments(PDO $pdo, array $assignments): void { if ($assignments === []) { writeMessage('Keine Nutzer-Kleinplaneten-Zuordnungen zum Wiederherstellen vorhanden.'); return; } $lookupStatement = $pdo->prepare('SELECT `id` FROM `minorplanets_mpc` WHERE `desig_packed` = :desig_packed LIMIT 1'); $insertStatement = $pdo->prepare( 'INSERT INTO `app_user_minorplanets` (`user_id`, `minorplanet_id`, `is_favorite`, `created_at`) VALUES (:user_id, :minorplanet_id, :is_favorite, :created_at)' ); $restored = 0; $missing = 0; foreach ($assignments as $assignment) { $desigPacked = (string) ($assignment['desig_packed'] ?? ''); if ($desigPacked === '') { $missing++; continue; } $lookupStatement->execute([ ':desig_packed' => $desigPacked, ]); $minorplanetId = $lookupStatement->fetchColumn(); if ($minorplanetId === false) { $missing++; writeMessage("Zuordnung konnte nicht wiederhergestellt werden: {$desigPacked}", true); continue; } $insertStatement->execute([ ':user_id' => (int) $assignment['user_id'], ':minorplanet_id' => (int) $minorplanetId, ':is_favorite' => (int) $assignment['is_favorite'], ':created_at' => (string) $assignment['created_at'], ]); $restored++; } writeMessage("Nutzer-Kleinplaneten-Zuordnungen wiederhergestellt: {$restored}."); if ($missing > 0) { writeMessage("Nicht wiederherstellbare Zuordnungen: {$missing}.", true); } } function writeAutoContinueScript(int $batchLineLimit, int $progressInterval, int $commitInterval): void { $query = http_build_query([ 'batch' => $batchLineLimit, 'progress' => $progressInterval, 'commit' => $commitInterval, ]); echo '' . "\n"; echo '' . "\n"; } function writeRestartHint(): void { echo '

Import neu starten

' . "\n"; } function countMpcorbDataLines(string $datPath): int { $handle = fopen($datPath, 'rb'); if ($handle === false) { return 0; } $inDataSection = false; $count = 0; while (($line = fgets($handle)) !== false) { $line = rtrim($line, "\r\n"); if ($line === '') { continue; } if (!$inDataSection) { if (preg_match('/^-{20,}$/', trim($line)) === 1) { $inDataSection = true; } continue; } $count++; } fclose($handle); return $count; } function clearTableWithRetry(PDO $pdo, string $tableName, int $maxAttempts = 5, int $waitMilliseconds = 1500): void { $sql = "DELETE FROM `{$tableName}`"; for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) { try { $pdo->exec($sql); return; } catch (PDOException $e) { if (!isLockTimeoutException($e) || $attempt === $maxAttempts) { throw $e; } writeMessage("Warte auf Sperre beim Leeren von {$tableName} (Versuch {$attempt}/{$maxAttempts})...", true); usleep($waitMilliseconds * 1000); } } } function ensureCaseSensitiveDesigPackedColumn(PDO $pdo): void { $sql = <<<'SQL' ALTER TABLE `minorplanets_mpc` MODIFY `desig_packed` VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT 'Spalte Des''n aus MPCORB.DAT, z. B. 00001 oder K07Tf8A' SQL; $pdo->exec($sql); } function resetAutoIncrementWithRetry(PDO $pdo, string $tableName, int $maxAttempts = 5, int $waitMilliseconds = 1500): void { $sql = "ALTER TABLE `{$tableName}` AUTO_INCREMENT = 1"; for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) { try { $pdo->exec($sql); return; } catch (PDOException $e) { if (!isLockTimeoutException($e) || $attempt === $maxAttempts) { throw $e; } writeMessage("Warte auf Sperre beim Zuruecksetzen von AUTO_INCREMENT fuer {$tableName} (Versuch {$attempt}/{$maxAttempts})...", true); usleep($waitMilliseconds * 1000); } } } function isLockTimeoutException(PDOException $e): bool { $message = $e->getMessage(); return str_contains($message, '1205') || stripos($message, 'Lock wait timeout exceeded') !== false; } function buildProgressMessage(int $processed, int $imported, int $skipped, int $totalDataLines, float $startedAt): string { $elapsedSeconds = max(microtime(true) - $startedAt, 0.001); $rate = $processed / $elapsedSeconds; $percentage = $totalDataLines > 0 ? ($processed / $totalDataLines) * 100 : null; $remainingSeconds = ($totalDataLines > 0 && $rate > 0) ? max(($totalDataLines - $processed) / $rate, 0) : null; $parts = [ 'Status', "verarbeitet {$processed}" . ($totalDataLines > 0 ? " von {$totalDataLines}" : ''), ]; if ($percentage !== null) { $parts[] = sprintf('%.2f %%', $percentage); } $parts[] = "importiert {$imported}"; $parts[] = "uebersprungen {$skipped}"; $parts[] = sprintf('%.0f Datensaetze/s', $rate); $parts[] = 'Laufzeit ' . formatDuration($elapsedSeconds); if ($remainingSeconds !== null) { $parts[] = 'Rest ca. ' . formatDuration($remainingSeconds); } return implode(' | ', $parts); } function formatDuration(float $seconds): string { $seconds = (int) round($seconds); $hours = intdiv($seconds, 3600); $minutes = intdiv($seconds % 3600, 60); $secs = $seconds % 60; if ($hours > 0) { return sprintf('%02d:%02d:%02d', $hours, $minutes, $secs); } return sprintf('%02d:%02d', $minutes, $secs); } function parseMpcorbLine(string $line): array { $trimmed = rtrim($line); $padded = str_pad($trimmed, 202, ' '); $desigPacked = readFixedField($padded, 1, 7); $epochPacked = readFixedField($padded, 21, 25); if ($desigPacked === null || $epochPacked === null) { return [ 'record' => null, 'designation' => guessDesignationFromRawLine($trimmed), 'reason' => 'Pflichtfelder in den festen MPC-Spalten fehlen', ]; } $designationText = readFixedField($padded, 167, 194); return [ 'record' => [ ':desig_packed' => $desigPacked, ':mp_number' => extractMinorPlanetNumber((string) $designationText, $desigPacked), ':absolute_magnitude_h' => toNullableDecimal((string) readFixedField($padded, 9, 13)), ':slope_g' => toNullableDecimal((string) readFixedField($padded, 15, 19)), ':epoch_packed' => $epochPacked, ':mean_anomaly_deg' => (float) readFixedField($padded, 27, 35), ':arg_perihelion_deg' => (float) readFixedField($padded, 38, 46), ':ascending_node_deg' => (float) readFixedField($padded, 49, 57), ':inclination_deg' => (float) readFixedField($padded, 60, 68), ':eccentricity' => (float) readFixedField($padded, 71, 79), ':mean_motion_deg_per_day' => (float) readFixedField($padded, 81, 91), ':semimajor_axis_au' => (float) readFixedField($padded, 93, 103), ':uncertainty_u' => nullableTrim(readFixedField($padded, 106, 106)), ':reference_code' => nullableTrim(readFixedField($padded, 108, 116)), ':num_observations' => toNullableInt(readFixedField($padded, 118, 122)), ':num_oppositions' => toNullableInt(readFixedField($padded, 124, 126)), ':arc_text' => nullableTrim(readFixedField($padded, 128, 136)), ':rms_residual' => toNullableDecimal((string) readFixedField($padded, 138, 141)), ':perturbers_1' => nullableTrim(readFixedField($padded, 143, 145)), ':perturbers_2' => nullableTrim(readFixedField($padded, 147, 149)), ':orbit_computer' => nullableTrim(readFixedField($padded, 151, 160)), ':hex_flags' => nullableTrim(readFixedField($padded, 162, 165)), ':designation_text' => $designationText, ':last_obs_compact' => readFixedField($padded, 195, 202), ':last_obs_date' => compactDateToSql(readFixedField($padded, 195, 202)), ':raw_record' => $trimmed, ], 'designation' => $designationText ?? guessDesignationFromRawLine($trimmed), 'reason' => null, ]; } function readFixedField(string $line, int $startColumn, int $endColumn): ?string { $offset = $startColumn - 1; $length = $endColumn - $startColumn + 1; $value = substr($line, $offset, $length); if ($value === false) { return null; } $value = trim($value); return $value === '' ? null : $value; } function guessDesignationFromRawLine(string $line): string { if ($line === '') { return '[leere Zeile]'; } if (preg_match('/\((\d+)\)\s+([^\r\n]+?)\s+\d{8}$/', $line, $matches) === 1) { return '(' . $matches[1] . ') ' . trim($matches[2]); } if (preg_match('/([A-Za-z0-9\-\(\)\/ ]+)\s+\d{8}$/', $line, $matches) === 1) { return trim($matches[1]); } return substr($line, 0, 80); } function formatSkipDetail(int $lineNumber, string $rawLine, ?string $designation, ?string $reason): string { $objectLabel = $designation !== null && trim($designation) !== '' ? trim($designation) : guessDesignationFromRawLine(rtrim($rawLine)); $reasonText = $reason !== null && trim($reason) !== '' ? trim($reason) : 'Unbekannter Grund'; return "Datensatz {$lineNumber}: {$objectLabel} -> {$reasonText}"; } function extractMinorPlanetNumber(string $designationText, string $desigPacked): ?int { if (preg_match('/^\((\d+)\)/', $designationText, $matches) === 1) { return (int) $matches[1]; } if (ctype_digit($desigPacked)) { return (int) ltrim($desigPacked, '0'); } return null; } function compactDateToSql(?string $value): ?string { if ($value === null || preg_match('/^\d{8}$/', $value) !== 1) { return null; } $year = substr($value, 0, 4); $month = substr($value, 4, 2); $day = substr($value, 6, 2); if (!checkdate((int) $month, (int) $day, (int) $year)) { return null; } return $year . '-' . $month . '-' . $day; } function toNullableDecimal(string $value): ?float { $value = trim($value); if ($value === '') { return null; } return is_numeric($value) ? (float) $value : null; } function toNullableInt(?string $value): ?int { if ($value === null) { return null; } $value = trim($value); if ($value === '' || !preg_match('/^-?\d+$/', $value)) { return null; } return (int) $value; } function nullableTrim(?string $value): ?string { if ($value === null) { return null; } $value = trim($value); return $value === '' ? null : $value; } function normalizeDbConfig(array $config): array { if (isset($config['password']) && !isset($config['pass'])) { $config['pass'] = $config['password']; } if (isset($config['database']) && !isset($config['dbname'])) { $config['dbname'] = $config['database']; } if (isset($config['db']) && !isset($config['dbname'])) { $config['dbname'] = $config['db']; } if (!isset($config['port'])) { $config['port'] = 3306; } if (!isset($config['charset'])) { $config['charset'] = 'utf8mb4'; } return $config; } function writeMessage(string $message, bool $isError = false): void { $line = $message . PHP_EOL; $isCli = PHP_SAPI === 'cli' || PHP_SAPI === 'phpdbg'; if ($isCli && $isError && defined('STDERR')) { fwrite(STDERR, $line); if (function_exists('fflush')) { fflush(STDERR); } return; } if ($isCli && !$isError && defined('STDOUT')) { fwrite(STDOUT, $line); if (function_exists('fflush')) { fflush(STDOUT); } return; } echo htmlspecialchars($message, ENT_QUOTES, 'UTF-8') . "
\n"; if (function_exists('ob_flush')) { @ob_flush(); } if (function_exists('flush')) { flush(); } }