Kemetenimport verbessert

config/skyview.sql

@@ -136,6 +136,7 @@ CREATE TABLE IF NOT EXISTS `comets_mpc` (
   `designation_and_name` varchar(255) NOT NULL COMMENT 'Lesbare Bezeichnung, z. B. C/1995 O1 (Hale-Bopp)',
   `reference_text` varchar(64) DEFAULT NULL COMMENT 'Referenz aus der MPC-Datei, z. B. MPEC 2026-FC3',
   `raw_record` varchar(255) DEFAULT NULL COMMENT 'Originale Fixed-width-Zeile aus Soft00Cmt.txt',
+  `source_hash` char(40) DEFAULT NULL COMMENT 'Hash ueber die relevanten MPC-Quelldaten zur Aenderungserkennung',
   `source_url` varchar(255) DEFAULT NULL COMMENT 'Quelle des letzten Imports',
   `imported_at` datetime NOT NULL DEFAULT current_timestamp() COMMENT 'Zeitpunkt des letzten Imports',
   `created_at` datetime NOT NULL DEFAULT current_timestamp(),
@@ -145,7 +146,8 @@ CREATE TABLE IF NOT EXISTS `comets_mpc` (
   KEY `idx_mpc_comets_orbit_type` (`orbit_type`),
   KEY `idx_mpc_comets_perihelion_year_month` (`year_of_perihelion`,`month_of_perihelion`),
   KEY `idx_mpc_comets_epoch_compact` (`epoch_compact`),
-  KEY `idx_mpc_comets_designation_and_name` (`designation_and_name`)
+  KEY `idx_mpc_comets_designation_and_name` (`designation_and_name`),
+  KEY `idx_mpc_comets_source_hash` (`source_hash`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='Kometenelemente aus Soft00Cmt.txt des Minor Planet Center';
 
 -- Daten-Export vom Benutzer nicht ausgewählt

scripts/import/import_mpc_comets.php

@@ -49,13 +49,17 @@ $pdo = new PDO(
     ]
 );
 
+ensureImportRunsTableExists($pdo);
+ensureCometsTableExists($pdo);
+
 $recordsReceived = 0;
-$recordsImported = 0;
+$recordsInserted = 0;
+$recordsUpdated = 0;
+$recordsUnchanged = 0;
 $recordsSkipped = 0;
+$importRunId = startImportRun($pdo, 'mpc-comets', $localSourcePath !== null && trim((string) $localSourcePath) !== '' ? (string) $localSourcePath : $sourceUrl);
 
 try {
-    rebuildCometsTable($pdo);
-
     [$lines, $resolvedSource] = resolveSourceLines($sourceUrl, $localSourcePath);
     $statement = $pdo->prepare(
         'INSERT INTO `comets_mpc` (
@@ -76,6 +80,7 @@ try {
             `designation_and_name`,
             `reference_text`,
             `raw_record`,
+            `source_hash`,
             `source_url`,
             `imported_at`,
             `created_at`,
@@ -98,14 +103,37 @@ try {
             :designation_and_name,
             :reference_text,
             :raw_record,
+            :source_hash,
             :source_url,
             NOW(),
             NOW(),
             NOW()
-        )'
+        )
+        ON DUPLICATE KEY UPDATE
+            `orbit_type` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`orbit_type`), `orbit_type`),
+            `year_of_perihelion` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`year_of_perihelion`), `year_of_perihelion`),
+            `month_of_perihelion` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`month_of_perihelion`), `month_of_perihelion`),
+            `day_of_perihelion` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`day_of_perihelion`), `day_of_perihelion`),
+            `perihelion_dist_au` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`perihelion_dist_au`), `perihelion_dist_au`),
+            `eccentricity` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`eccentricity`), `eccentricity`),
+            `arg_perihelion_deg` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`arg_perihelion_deg`), `arg_perihelion_deg`),
+            `ascending_node_deg` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`ascending_node_deg`), `ascending_node_deg`),
+            `inclination_deg` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`inclination_deg`), `inclination_deg`),
+            `epoch_compact` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`epoch_compact`), `epoch_compact`),
+            `epoch_date` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`epoch_date`), `epoch_date`),
+            `absolute_magnitude_h` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`absolute_magnitude_h`), `absolute_magnitude_h`),
+            `slope_parameter_g` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`slope_parameter_g`), `slope_parameter_g`),
+            `designation_and_name` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`designation_and_name`), `designation_and_name`),
+            `reference_text` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`reference_text`), `reference_text`),
+            `raw_record` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`raw_record`), `raw_record`),
+            `source_hash` = VALUES(`source_hash`),
+            `source_url` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`source_url`), `source_url`),
+            `imported_at` = IF(`source_hash` <> VALUES(`source_hash`), NOW(), `imported_at`),
+            `updated_at` = IF(`source_hash` <> VALUES(`source_hash`), NOW(), `updated_at`)'
     );
 
     $pdo->beginTransaction();
+    $seenDesignations = [];
 
     foreach ($lines as $lineNumber => $line) {
         $trimmedLine = rtrim($line, "\r\n");
@@ -118,26 +146,66 @@ try {
         try {
             $record = parseSoft00CmtLine($trimmedLine);
             $record[':source_url'] = $resolvedSource;
+            $seenDesignations[] = $record[':designation_packed'];
             $statement->execute($record);
-            $recordsImported++;
+            $affectedRows = (int) $statement->rowCount();
+            if ($affectedRows === 1) {
+                $recordsInserted++;
+            } elseif ($affectedRows >= 2) {
+                $recordsUpdated++;
+            } else {
+                $recordsUnchanged++;
+            }
         } catch (Throwable $rowError) {
             $recordsSkipped++;
             echo 'Uebersprungen Zeile ' . ($lineNumber + 1) . ': ' . $rowError->getMessage() . "\n";
         }
     }
 
+    if ($seenDesignations !== []) {
+        $seenDesignations = array_values(array_unique($seenDesignations));
+        $placeholders = implode(', ', array_fill(0, count($seenDesignations), '?'));
+        $deleteStaleStatement = $pdo->prepare(sprintf(
+            'DELETE FROM `comets_mpc` WHERE `designation_packed` NOT IN (%s)',
+            $placeholders
+        ));
+        $deleteStaleStatement->execute($seenDesignations);
+    }
+
     $pdo->commit();
 
     echo "Import erfolgreich\n";
     echo "Quelle: {$resolvedSource}\n";
     echo "Empfangen: {$recordsReceived}\n";
-    echo "Importiert: {$recordsImported}\n";
+    echo "Neu: {$recordsInserted}\n";
+    echo "Aktualisiert: {$recordsUpdated}\n";
+    echo "Unveraendert: {$recordsUnchanged}\n";
     echo "Uebersprungen: {$recordsSkipped}\n";
+
+    finishImportRun(
+        $pdo,
+        $importRunId,
+        $recordsReceived,
+        $recordsInserted,
+        $recordsUpdated,
+        'success',
+        $recordsSkipped > 0 ? "Es wurden {$recordsSkipped} Datensaetze uebersprungen." : null
+    );
 } catch (Throwable $e) {
     if ($pdo->inTransaction()) {
         $pdo->rollBack();
     }
 
+    finishImportRun(
+        $pdo,
+        $importRunId,
+        $recordsReceived,
+        $recordsInserted,
+        $recordsUpdated,
+        'error',
+        substr($e->getMessage(), 0, 65000)
+    );
+
     writeError('Fehler: ' . $e->getMessage());
     exit(1);
 }
@@ -401,28 +469,67 @@ function parseSoft00CmtLine(string $line): array
     $designationPacked = requireStringField(readFixedField($padded, 1, 12), 'designation_packed');
     $designationAndName = requireStringField(readFixedField($padded, 103, 158), 'designation_and_name');
     $epochCompact = toNullableCompactDate(readFixedField($padded, 82, 89));
+    $orbitType = deriveOrbitType($designationAndName, $designationPacked);
+    $yearOfPerihelion = requireIntField(readFixedField($padded, 15, 18), 'year_of_perihelion');
+    $monthOfPerihelion = requireIntField(readFixedField($padded, 20, 21), 'month_of_perihelion');
+    $dayOfPerihelion = requireFloatField(readFixedField($padded, 23, 29), 'day_of_perihelion');
+    $perihelionDistAu = requireFloatField(readFixedField($padded, 32, 39), 'perihelion_dist_au');
+    $eccentricity = requireFloatField(readFixedField($padded, 42, 49), 'eccentricity');
+    $argPerihelionDeg = requireFloatField(readFixedField($padded, 52, 59), 'arg_perihelion_deg');
+    $ascendingNodeDeg = requireFloatField(readFixedField($padded, 62, 69), 'ascending_node_deg');
+    $inclinationDeg = requireFloatField(readFixedField($padded, 72, 79), 'inclination_deg');
+    $absoluteMagnitudeH = toNullableFloat(readFixedField($padded, 91, 95));
+    $slopeParameterG = toNullableFloat(readFixedField($padded, 97, 101));
+    $referenceText = toNullableString(readFixedField($padded, 160, 200));
 
     return [
         ':designation_packed' => $designationPacked,
-        ':orbit_type' => deriveOrbitType($designationAndName, $designationPacked),
+        ':orbit_type' => $orbitType,
-        ':year_of_perihelion' => requireIntField(readFixedField($padded, 15, 18), 'year_of_perihelion'),
+        ':year_of_perihelion' => $yearOfPerihelion,
-        ':month_of_perihelion' => requireIntField(readFixedField($padded, 20, 21), 'month_of_perihelion'),
+        ':month_of_perihelion' => $monthOfPerihelion,
-        ':day_of_perihelion' => requireFloatField(readFixedField($padded, 23, 29), 'day_of_perihelion'),
+        ':day_of_perihelion' => $dayOfPerihelion,
-        ':perihelion_dist_au' => requireFloatField(readFixedField($padded, 32, 39), 'perihelion_dist_au'),
+        ':perihelion_dist_au' => $perihelionDistAu,
-        ':eccentricity' => requireFloatField(readFixedField($padded, 42, 49), 'eccentricity'),
+        ':eccentricity' => $eccentricity,
-        ':arg_perihelion_deg' => requireFloatField(readFixedField($padded, 52, 59), 'arg_perihelion_deg'),
+        ':arg_perihelion_deg' => $argPerihelionDeg,
-        ':ascending_node_deg' => requireFloatField(readFixedField($padded, 62, 69), 'ascending_node_deg'),
+        ':ascending_node_deg' => $ascendingNodeDeg,
-        ':inclination_deg' => requireFloatField(readFixedField($padded, 72, 79), 'inclination_deg'),
+        ':inclination_deg' => $inclinationDeg,
         ':epoch_compact' => $epochCompact,
         ':epoch_date' => $epochCompact !== null ? compactDateToSql($epochCompact) : null,
-        ':absolute_magnitude_h' => toNullableFloat(readFixedField($padded, 91, 95)),
+        ':absolute_magnitude_h' => $absoluteMagnitudeH,
-        ':slope_parameter_g' => toNullableFloat(readFixedField($padded, 97, 101)),
+        ':slope_parameter_g' => $slopeParameterG,
         ':designation_and_name' => $designationAndName,
-        ':reference_text' => toNullableString(readFixedField($padded, 160, 200)),
+        ':reference_text' => $referenceText,
         ':raw_record' => $line,
+        ':source_hash' => buildSourceHash([
+            $designationPacked,
+            $orbitType,
+            $yearOfPerihelion,
+            $monthOfPerihelion,
+            $dayOfPerihelion,
+            $perihelionDistAu,
+            $eccentricity,
+            $argPerihelionDeg,
+            $ascendingNodeDeg,
+            $inclinationDeg,
+            $epochCompact,
+            $absoluteMagnitudeH,
+            $slopeParameterG,
+            $designationAndName,
+            $referenceText,
+        ]),
     ];
 }
 
+function buildSourceHash(array $parts): string
+{
+    $normalized = array_map(
+        static fn(mixed $value): string => $value === null ? '' : trim((string) $value),
+        $parts
+    );
+
+    return sha1(implode('|', $normalized));
+}
+
 function deriveOrbitType(string $designationAndName, string $designationPacked): string
 {
     if (preg_match('/^\d+([A-Z])\//', $designationAndName, $matches) === 1) {
@@ -550,11 +657,87 @@ function normalizeDbConfig(array $config): array
     ];
 }
 
-function rebuildCometsTable(PDO $pdo): void
+function ensureImportRunsTableExists(PDO $pdo): void
+{
+    $pdo->exec("
+        CREATE TABLE IF NOT EXISTS `app_import_runs` (
+          `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
+          `source` varchar(32) NOT NULL,
+          `source_url` varchar(500) DEFAULT NULL,
+          `started_at` datetime NOT NULL,
+          `finished_at` datetime DEFAULT NULL,
+          `records_received` int(11) NOT NULL DEFAULT 0,
+          `records_inserted` int(11) NOT NULL DEFAULT 0,
+          `records_updated` int(11) NOT NULL DEFAULT 0,
+          `status` varchar(32) NOT NULL DEFAULT 'success',
+          `message` text DEFAULT NULL,
+          PRIMARY KEY (`id`),
+          KEY `idx_app_import_runs_source` (`source`),
+          KEY `idx_app_import_runs_started_at` (`started_at`)
+        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
+    ");
+}
+
+function startImportRun(PDO $pdo, string $source, ?string $sourceUrl): int
+{
+    $stmt = $pdo->prepare("
+        INSERT INTO `app_import_runs` (
+            `source`,
+            `source_url`,
+            `started_at`,
+            `status`,
+            `message`
+        ) VALUES (
+            :source,
+            :source_url,
+            NOW(),
+            'running',
+            NULL
+        )
+    ");
+    $stmt->execute([
+        ':source' => $source,
+        ':source_url' => $sourceUrl,
+    ]);
+
+    return (int) $pdo->lastInsertId();
+}
+
+function finishImportRun(
+    PDO $pdo,
+    int $runId,
+    int $recordsReceived,
+    int $recordsInserted,
+    int $recordsUpdated,
+    string $status,
+    ?string $message
+): void {
+    $stmt = $pdo->prepare("
+        UPDATE `app_import_runs`
+        SET
+            `finished_at` = NOW(),
+            `records_received` = :records_received,
+            `records_inserted` = :records_inserted,
+            `records_updated` = :records_updated,
+            `status` = :status,
+            `message` = :message
+        WHERE `id` = :id
+        LIMIT 1
+    ");
+    $stmt->execute([
+        ':records_received' => $recordsReceived,
+        ':records_inserted' => $recordsInserted,
+        ':records_updated' => $recordsUpdated,
+        ':status' => $status,
+        ':message' => $message,
+        ':id' => $runId,
+    ]);
+}
+
+function ensureCometsTableExists(PDO $pdo): void
 {
-    $pdo->exec('DROP TABLE IF EXISTS `comets_mpc`');
     $pdo->exec(<<<'SQL'
-CREATE TABLE `comets_mpc` (
+CREATE TABLE IF NOT EXISTS `comets_mpc` (
   `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
   `designation_packed` varchar(12) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT 'Fixed-width MPC-Packbezeichnung aus Soft00Cmt.txt, inkl. Fragmentkennung',
   `orbit_type` char(1) NOT NULL COMMENT 'Abgeleiteter Kometentyp, z. B. C, P oder I',
@@ -573,6 +756,7 @@ CREATE TABLE `comets_mpc` (
   `designation_and_name` varchar(255) NOT NULL COMMENT 'Lesbare Bezeichnung, z. B. C/1995 O1 (Hale-Bopp)',
   `reference_text` varchar(64) DEFAULT NULL COMMENT 'Referenz aus der MPC-Datei, z. B. MPEC 2026-FC3',
   `raw_record` varchar(255) DEFAULT NULL COMMENT 'Originale Fixed-width-Zeile aus Soft00Cmt.txt',
+  `source_hash` char(40) DEFAULT NULL COMMENT 'Hash ueber die relevanten MPC-Quelldaten zur Aenderungserkennung',
   `source_url` varchar(255) DEFAULT NULL COMMENT 'Quelle des letzten Imports',
   `imported_at` datetime NOT NULL DEFAULT current_timestamp() COMMENT 'Zeitpunkt des letzten Imports',
   `created_at` datetime NOT NULL DEFAULT current_timestamp(),
@@ -582,8 +766,50 @@ CREATE TABLE `comets_mpc` (
   KEY `idx_mpc_comets_orbit_type` (`orbit_type`),
   KEY `idx_mpc_comets_perihelion_year_month` (`year_of_perihelion`,`month_of_perihelion`),
   KEY `idx_mpc_comets_epoch_compact` (`epoch_compact`),
-  KEY `idx_mpc_comets_designation_and_name` (`designation_and_name`)
+  KEY `idx_mpc_comets_designation_and_name` (`designation_and_name`),
+  KEY `idx_mpc_comets_source_hash` (`source_hash`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='Kometenelemente aus Soft00Cmt.txt des Minor Planet Center'
 SQL
     );
+
+    ensureColumnExists($pdo, 'comets_mpc', 'source_hash', "ALTER TABLE `comets_mpc` ADD COLUMN `source_hash` char(40) DEFAULT NULL COMMENT 'Hash ueber die relevanten MPC-Quelldaten zur Aenderungserkennung' AFTER `raw_record`");
+    ensureIndexExists($pdo, 'comets_mpc', 'idx_mpc_comets_source_hash', 'ALTER TABLE `comets_mpc` ADD KEY `idx_mpc_comets_source_hash` (`source_hash`)');
+}
+
+function ensureColumnExists(PDO $pdo, string $tableName, string $columnName, string $alterSql): void
+{
+    $stmt = $pdo->prepare('
+        SELECT COUNT(*)
+        FROM information_schema.COLUMNS
+        WHERE TABLE_SCHEMA = DATABASE()
+          AND TABLE_NAME = :table_name
+          AND COLUMN_NAME = :column_name
+    ');
+    $stmt->execute([
+        ':table_name' => $tableName,
+        ':column_name' => $columnName,
+    ]);
+
+    if ((int) $stmt->fetchColumn() === 0) {
+        $pdo->exec($alterSql);
+    }
+}
+
+function ensureIndexExists(PDO $pdo, string $tableName, string $indexName, string $alterSql): void
+{
+    $stmt = $pdo->prepare('
+        SELECT COUNT(*)
+        FROM information_schema.STATISTICS
+        WHERE TABLE_SCHEMA = DATABASE()
+          AND TABLE_NAME = :table_name
+          AND INDEX_NAME = :index_name
+    ');
+    $stmt->execute([
+        ':table_name' => $tableName,
+        ':index_name' => $indexName,
+    ]);
+
+    if ((int) $stmt->fetchColumn() === 0) {
+        $pdo->exec($alterSql);
+    }
 }