From 4eff1f02a3195ac41ecfc0ccc02ebc86eea0eed0 Mon Sep 17 00:00:00 2001 From: Eskimue Date: Fri, 10 Apr 2026 19:27:09 +0200 Subject: [PATCH] Kemetenimport verbessert --- config/skyview.sql | 4 +- scripts/import/import_mpc_comets.php | 270 ++++++++++++++++++++++++--- 2 files changed, 251 insertions(+), 23 deletions(-) diff --git a/config/skyview.sql b/config/skyview.sql index d943e7b..0944c24 100644 --- a/config/skyview.sql +++ b/config/skyview.sql @@ -136,6 +136,7 @@ CREATE TABLE IF NOT EXISTS `comets_mpc` ( `designation_and_name` varchar(255) NOT NULL COMMENT 'Lesbare Bezeichnung, z. B. C/1995 O1 (Hale-Bopp)', `reference_text` varchar(64) DEFAULT NULL COMMENT 'Referenz aus der MPC-Datei, z. B. MPEC 2026-FC3', `raw_record` varchar(255) DEFAULT NULL COMMENT 'Originale Fixed-width-Zeile aus Soft00Cmt.txt', + `source_hash` char(40) DEFAULT NULL COMMENT 'Hash ueber die relevanten MPC-Quelldaten zur Aenderungserkennung', `source_url` varchar(255) DEFAULT NULL COMMENT 'Quelle des letzten Imports', `imported_at` datetime NOT NULL DEFAULT current_timestamp() COMMENT 'Zeitpunkt des letzten Imports', `created_at` datetime NOT NULL DEFAULT current_timestamp(), @@ -145,7 +146,8 @@ CREATE TABLE IF NOT EXISTS `comets_mpc` ( KEY `idx_mpc_comets_orbit_type` (`orbit_type`), KEY `idx_mpc_comets_perihelion_year_month` (`year_of_perihelion`,`month_of_perihelion`), KEY `idx_mpc_comets_epoch_compact` (`epoch_compact`), - KEY `idx_mpc_comets_designation_and_name` (`designation_and_name`) + KEY `idx_mpc_comets_designation_and_name` (`designation_and_name`), + KEY `idx_mpc_comets_source_hash` (`source_hash`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='Kometenelemente aus Soft00Cmt.txt des Minor Planet Center'; -- Daten-Export vom Benutzer nicht ausgewählt diff --git a/scripts/import/import_mpc_comets.php b/scripts/import/import_mpc_comets.php index cdea286..2fef7f8 100644 --- a/scripts/import/import_mpc_comets.php +++ b/scripts/import/import_mpc_comets.php @@ -49,13 +49,17 @@ $pdo = new PDO( ] ); +ensureImportRunsTableExists($pdo); +ensureCometsTableExists($pdo); + $recordsReceived = 0; -$recordsImported = 0; +$recordsInserted = 0; +$recordsUpdated = 0; +$recordsUnchanged = 0; $recordsSkipped = 0; +$importRunId = startImportRun($pdo, 'mpc-comets', $localSourcePath !== null && trim((string) $localSourcePath) !== '' ? (string) $localSourcePath : $sourceUrl); try { - rebuildCometsTable($pdo); - [$lines, $resolvedSource] = resolveSourceLines($sourceUrl, $localSourcePath); $statement = $pdo->prepare( 'INSERT INTO `comets_mpc` ( @@ -76,6 +80,7 @@ try { `designation_and_name`, `reference_text`, `raw_record`, + `source_hash`, `source_url`, `imported_at`, `created_at`, @@ -98,14 +103,37 @@ try { :designation_and_name, :reference_text, :raw_record, + :source_hash, :source_url, NOW(), NOW(), NOW() - )' + ) + ON DUPLICATE KEY UPDATE + `orbit_type` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`orbit_type`), `orbit_type`), + `year_of_perihelion` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`year_of_perihelion`), `year_of_perihelion`), + `month_of_perihelion` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`month_of_perihelion`), `month_of_perihelion`), + `day_of_perihelion` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`day_of_perihelion`), `day_of_perihelion`), + `perihelion_dist_au` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`perihelion_dist_au`), `perihelion_dist_au`), + `eccentricity` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`eccentricity`), `eccentricity`), + `arg_perihelion_deg` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`arg_perihelion_deg`), `arg_perihelion_deg`), + `ascending_node_deg` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`ascending_node_deg`), `ascending_node_deg`), + `inclination_deg` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`inclination_deg`), `inclination_deg`), + `epoch_compact` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`epoch_compact`), `epoch_compact`), + `epoch_date` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`epoch_date`), `epoch_date`), + `absolute_magnitude_h` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`absolute_magnitude_h`), `absolute_magnitude_h`), + `slope_parameter_g` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`slope_parameter_g`), `slope_parameter_g`), + `designation_and_name` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`designation_and_name`), `designation_and_name`), + `reference_text` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`reference_text`), `reference_text`), + `raw_record` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`raw_record`), `raw_record`), + `source_hash` = VALUES(`source_hash`), + `source_url` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`source_url`), `source_url`), + `imported_at` = IF(`source_hash` <> VALUES(`source_hash`), NOW(), `imported_at`), + `updated_at` = IF(`source_hash` <> VALUES(`source_hash`), NOW(), `updated_at`)' ); $pdo->beginTransaction(); + $seenDesignations = []; foreach ($lines as $lineNumber => $line) { $trimmedLine = rtrim($line, "\r\n"); @@ -118,26 +146,66 @@ try { try { $record = parseSoft00CmtLine($trimmedLine); $record[':source_url'] = $resolvedSource; + $seenDesignations[] = $record[':designation_packed']; $statement->execute($record); - $recordsImported++; + $affectedRows = (int) $statement->rowCount(); + if ($affectedRows === 1) { + $recordsInserted++; + } elseif ($affectedRows >= 2) { + $recordsUpdated++; + } else { + $recordsUnchanged++; + } } catch (Throwable $rowError) { $recordsSkipped++; echo 'Uebersprungen Zeile ' . ($lineNumber + 1) . ': ' . $rowError->getMessage() . "\n"; } } + if ($seenDesignations !== []) { + $seenDesignations = array_values(array_unique($seenDesignations)); + $placeholders = implode(', ', array_fill(0, count($seenDesignations), '?')); + $deleteStaleStatement = $pdo->prepare(sprintf( + 'DELETE FROM `comets_mpc` WHERE `designation_packed` NOT IN (%s)', + $placeholders + )); + $deleteStaleStatement->execute($seenDesignations); + } + $pdo->commit(); echo "Import erfolgreich\n"; echo "Quelle: {$resolvedSource}\n"; echo "Empfangen: {$recordsReceived}\n"; - echo "Importiert: {$recordsImported}\n"; + echo "Neu: {$recordsInserted}\n"; + echo "Aktualisiert: {$recordsUpdated}\n"; + echo "Unveraendert: {$recordsUnchanged}\n"; echo "Uebersprungen: {$recordsSkipped}\n"; + + finishImportRun( + $pdo, + $importRunId, + $recordsReceived, + $recordsInserted, + $recordsUpdated, + 'success', + $recordsSkipped > 0 ? "Es wurden {$recordsSkipped} Datensaetze uebersprungen." : null + ); } catch (Throwable $e) { if ($pdo->inTransaction()) { $pdo->rollBack(); } + finishImportRun( + $pdo, + $importRunId, + $recordsReceived, + $recordsInserted, + $recordsUpdated, + 'error', + substr($e->getMessage(), 0, 65000) + ); + writeError('Fehler: ' . $e->getMessage()); exit(1); } @@ -401,28 +469,67 @@ function parseSoft00CmtLine(string $line): array $designationPacked = requireStringField(readFixedField($padded, 1, 12), 'designation_packed'); $designationAndName = requireStringField(readFixedField($padded, 103, 158), 'designation_and_name'); $epochCompact = toNullableCompactDate(readFixedField($padded, 82, 89)); + $orbitType = deriveOrbitType($designationAndName, $designationPacked); + $yearOfPerihelion = requireIntField(readFixedField($padded, 15, 18), 'year_of_perihelion'); + $monthOfPerihelion = requireIntField(readFixedField($padded, 20, 21), 'month_of_perihelion'); + $dayOfPerihelion = requireFloatField(readFixedField($padded, 23, 29), 'day_of_perihelion'); + $perihelionDistAu = requireFloatField(readFixedField($padded, 32, 39), 'perihelion_dist_au'); + $eccentricity = requireFloatField(readFixedField($padded, 42, 49), 'eccentricity'); + $argPerihelionDeg = requireFloatField(readFixedField($padded, 52, 59), 'arg_perihelion_deg'); + $ascendingNodeDeg = requireFloatField(readFixedField($padded, 62, 69), 'ascending_node_deg'); + $inclinationDeg = requireFloatField(readFixedField($padded, 72, 79), 'inclination_deg'); + $absoluteMagnitudeH = toNullableFloat(readFixedField($padded, 91, 95)); + $slopeParameterG = toNullableFloat(readFixedField($padded, 97, 101)); + $referenceText = toNullableString(readFixedField($padded, 160, 200)); return [ ':designation_packed' => $designationPacked, - ':orbit_type' => deriveOrbitType($designationAndName, $designationPacked), - ':year_of_perihelion' => requireIntField(readFixedField($padded, 15, 18), 'year_of_perihelion'), - ':month_of_perihelion' => requireIntField(readFixedField($padded, 20, 21), 'month_of_perihelion'), - ':day_of_perihelion' => requireFloatField(readFixedField($padded, 23, 29), 'day_of_perihelion'), - ':perihelion_dist_au' => requireFloatField(readFixedField($padded, 32, 39), 'perihelion_dist_au'), - ':eccentricity' => requireFloatField(readFixedField($padded, 42, 49), 'eccentricity'), - ':arg_perihelion_deg' => requireFloatField(readFixedField($padded, 52, 59), 'arg_perihelion_deg'), - ':ascending_node_deg' => requireFloatField(readFixedField($padded, 62, 69), 'ascending_node_deg'), - ':inclination_deg' => requireFloatField(readFixedField($padded, 72, 79), 'inclination_deg'), + ':orbit_type' => $orbitType, + ':year_of_perihelion' => $yearOfPerihelion, + ':month_of_perihelion' => $monthOfPerihelion, + ':day_of_perihelion' => $dayOfPerihelion, + ':perihelion_dist_au' => $perihelionDistAu, + ':eccentricity' => $eccentricity, + ':arg_perihelion_deg' => $argPerihelionDeg, + ':ascending_node_deg' => $ascendingNodeDeg, + ':inclination_deg' => $inclinationDeg, ':epoch_compact' => $epochCompact, ':epoch_date' => $epochCompact !== null ? compactDateToSql($epochCompact) : null, - ':absolute_magnitude_h' => toNullableFloat(readFixedField($padded, 91, 95)), - ':slope_parameter_g' => toNullableFloat(readFixedField($padded, 97, 101)), + ':absolute_magnitude_h' => $absoluteMagnitudeH, + ':slope_parameter_g' => $slopeParameterG, ':designation_and_name' => $designationAndName, - ':reference_text' => toNullableString(readFixedField($padded, 160, 200)), + ':reference_text' => $referenceText, ':raw_record' => $line, + ':source_hash' => buildSourceHash([ + $designationPacked, + $orbitType, + $yearOfPerihelion, + $monthOfPerihelion, + $dayOfPerihelion, + $perihelionDistAu, + $eccentricity, + $argPerihelionDeg, + $ascendingNodeDeg, + $inclinationDeg, + $epochCompact, + $absoluteMagnitudeH, + $slopeParameterG, + $designationAndName, + $referenceText, + ]), ]; } +function buildSourceHash(array $parts): string +{ + $normalized = array_map( + static fn(mixed $value): string => $value === null ? '' : trim((string) $value), + $parts + ); + + return sha1(implode('|', $normalized)); +} + function deriveOrbitType(string $designationAndName, string $designationPacked): string { if (preg_match('/^\d+([A-Z])\//', $designationAndName, $matches) === 1) { @@ -550,11 +657,87 @@ function normalizeDbConfig(array $config): array ]; } -function rebuildCometsTable(PDO $pdo): void +function ensureImportRunsTableExists(PDO $pdo): void +{ + $pdo->exec(" + CREATE TABLE IF NOT EXISTS `app_import_runs` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `source` varchar(32) NOT NULL, + `source_url` varchar(500) DEFAULT NULL, + `started_at` datetime NOT NULL, + `finished_at` datetime DEFAULT NULL, + `records_received` int(11) NOT NULL DEFAULT 0, + `records_inserted` int(11) NOT NULL DEFAULT 0, + `records_updated` int(11) NOT NULL DEFAULT 0, + `status` varchar(32) NOT NULL DEFAULT 'success', + `message` text DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `idx_app_import_runs_source` (`source`), + KEY `idx_app_import_runs_started_at` (`started_at`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + "); +} + +function startImportRun(PDO $pdo, string $source, ?string $sourceUrl): int +{ + $stmt = $pdo->prepare(" + INSERT INTO `app_import_runs` ( + `source`, + `source_url`, + `started_at`, + `status`, + `message` + ) VALUES ( + :source, + :source_url, + NOW(), + 'running', + NULL + ) + "); + $stmt->execute([ + ':source' => $source, + ':source_url' => $sourceUrl, + ]); + + return (int) $pdo->lastInsertId(); +} + +function finishImportRun( + PDO $pdo, + int $runId, + int $recordsReceived, + int $recordsInserted, + int $recordsUpdated, + string $status, + ?string $message +): void { + $stmt = $pdo->prepare(" + UPDATE `app_import_runs` + SET + `finished_at` = NOW(), + `records_received` = :records_received, + `records_inserted` = :records_inserted, + `records_updated` = :records_updated, + `status` = :status, + `message` = :message + WHERE `id` = :id + LIMIT 1 + "); + $stmt->execute([ + ':records_received' => $recordsReceived, + ':records_inserted' => $recordsInserted, + ':records_updated' => $recordsUpdated, + ':status' => $status, + ':message' => $message, + ':id' => $runId, + ]); +} + +function ensureCometsTableExists(PDO $pdo): void { - $pdo->exec('DROP TABLE IF EXISTS `comets_mpc`'); $pdo->exec(<<<'SQL' -CREATE TABLE `comets_mpc` ( +CREATE TABLE IF NOT EXISTS `comets_mpc` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, `designation_packed` varchar(12) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT 'Fixed-width MPC-Packbezeichnung aus Soft00Cmt.txt, inkl. Fragmentkennung', `orbit_type` char(1) NOT NULL COMMENT 'Abgeleiteter Kometentyp, z. B. C, P oder I', @@ -573,6 +756,7 @@ CREATE TABLE `comets_mpc` ( `designation_and_name` varchar(255) NOT NULL COMMENT 'Lesbare Bezeichnung, z. B. C/1995 O1 (Hale-Bopp)', `reference_text` varchar(64) DEFAULT NULL COMMENT 'Referenz aus der MPC-Datei, z. B. MPEC 2026-FC3', `raw_record` varchar(255) DEFAULT NULL COMMENT 'Originale Fixed-width-Zeile aus Soft00Cmt.txt', + `source_hash` char(40) DEFAULT NULL COMMENT 'Hash ueber die relevanten MPC-Quelldaten zur Aenderungserkennung', `source_url` varchar(255) DEFAULT NULL COMMENT 'Quelle des letzten Imports', `imported_at` datetime NOT NULL DEFAULT current_timestamp() COMMENT 'Zeitpunkt des letzten Imports', `created_at` datetime NOT NULL DEFAULT current_timestamp(), @@ -582,8 +766,50 @@ CREATE TABLE `comets_mpc` ( KEY `idx_mpc_comets_orbit_type` (`orbit_type`), KEY `idx_mpc_comets_perihelion_year_month` (`year_of_perihelion`,`month_of_perihelion`), KEY `idx_mpc_comets_epoch_compact` (`epoch_compact`), - KEY `idx_mpc_comets_designation_and_name` (`designation_and_name`) + KEY `idx_mpc_comets_designation_and_name` (`designation_and_name`), + KEY `idx_mpc_comets_source_hash` (`source_hash`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='Kometenelemente aus Soft00Cmt.txt des Minor Planet Center' SQL ); + + ensureColumnExists($pdo, 'comets_mpc', 'source_hash', "ALTER TABLE `comets_mpc` ADD COLUMN `source_hash` char(40) DEFAULT NULL COMMENT 'Hash ueber die relevanten MPC-Quelldaten zur Aenderungserkennung' AFTER `raw_record`"); + ensureIndexExists($pdo, 'comets_mpc', 'idx_mpc_comets_source_hash', 'ALTER TABLE `comets_mpc` ADD KEY `idx_mpc_comets_source_hash` (`source_hash`)'); +} + +function ensureColumnExists(PDO $pdo, string $tableName, string $columnName, string $alterSql): void +{ + $stmt = $pdo->prepare(' + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = :table_name + AND COLUMN_NAME = :column_name + '); + $stmt->execute([ + ':table_name' => $tableName, + ':column_name' => $columnName, + ]); + + if ((int) $stmt->fetchColumn() === 0) { + $pdo->exec($alterSql); + } +} + +function ensureIndexExists(PDO $pdo, string $tableName, string $indexName, string $alterSql): void +{ + $stmt = $pdo->prepare(' + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = :table_name + AND INDEX_NAME = :index_name + '); + $stmt->execute([ + ':table_name' => $tableName, + ':index_name' => $indexName, + ]); + + if ((int) $stmt->fetchColumn() === 0) { + $pdo->exec($alterSql); + } }