Eskimue/skyview.astronomiemuseum.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Kemetenimport verbessert

Browse Source
This commit is contained in:
Eskimue
2026-04-10 19:27:09 +02:00
parent 92c45a60af
commit 4eff1f02a3
2 changed files with 251 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/import/import_mpc_comets.php
+248 -22
View File
@@ -49,13 +49,17 @@ $pdo = new PDO(
]
);
ensureImportRunsTableExists($pdo);
ensureCometsTableExists($pdo);
$recordsReceived = 0;
$recordsImported = 0;
$recordsInserted = 0;
$recordsUpdated = 0;
$recordsUnchanged = 0;
$recordsSkipped = 0;
$importRunId = startImportRun($pdo, 'mpc-comets', $localSourcePath !== null && trim((string) $localSourcePath) !== '' ? (string) $localSourcePath : $sourceUrl);
try {
rebuildCometsTable($pdo);
[$lines, $resolvedSource] = resolveSourceLines($sourceUrl, $localSourcePath);
$statement = $pdo->prepare(
'INSERT INTO `comets_mpc` (
@@ -76,6 +80,7 @@ try {
`designation_and_name`,
`reference_text`,
`raw_record`,
`source_hash`,
`source_url`,
`imported_at`,
`created_at`,
@@ -98,14 +103,37 @@ try {
:designation_and_name,
:reference_text,
:raw_record,
:source_hash,
:source_url,
NOW(),
NOW(),
NOW()
)'
)
ON DUPLICATE KEY UPDATE
`orbit_type` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`orbit_type`), `orbit_type`),
`year_of_perihelion` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`year_of_perihelion`), `year_of_perihelion`),
`month_of_perihelion` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`month_of_perihelion`), `month_of_perihelion`),
`day_of_perihelion` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`day_of_perihelion`), `day_of_perihelion`),
`perihelion_dist_au` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`perihelion_dist_au`), `perihelion_dist_au`),
`eccentricity` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`eccentricity`), `eccentricity`),
`arg_perihelion_deg` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`arg_perihelion_deg`), `arg_perihelion_deg`),
`ascending_node_deg` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`ascending_node_deg`), `ascending_node_deg`),
`inclination_deg` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`inclination_deg`), `inclination_deg`),
`epoch_compact` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`epoch_compact`), `epoch_compact`),
`epoch_date` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`epoch_date`), `epoch_date`),
`absolute_magnitude_h` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`absolute_magnitude_h`), `absolute_magnitude_h`),
`slope_parameter_g` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`slope_parameter_g`), `slope_parameter_g`),
`designation_and_name` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`designation_and_name`), `designation_and_name`),
`reference_text` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`reference_text`), `reference_text`),
`raw_record` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`raw_record`), `raw_record`),
`source_hash` = VALUES(`source_hash`),
`source_url` = IF(`source_hash` <> VALUES(`source_hash`), VALUES(`source_url`), `source_url`),
`imported_at` = IF(`source_hash` <> VALUES(`source_hash`), NOW(), `imported_at`),
`updated_at` = IF(`source_hash` <> VALUES(`source_hash`), NOW(), `updated_at`)'
);
$pdo->beginTransaction();
$seenDesignations = [];
foreach ($lines as $lineNumber => $line) {
$trimmedLine = rtrim($line, "\r\n");
@@ -118,26 +146,66 @@ try {
try {
$record = parseSoft00CmtLine($trimmedLine);
$record[':source_url'] = $resolvedSource;
$seenDesignations[] = $record[':designation_packed'];
$statement->execute($record);
$recordsImported++;
$affectedRows = (int) $statement->rowCount();
if ($affectedRows === 1) {
$recordsInserted++;
} elseif ($affectedRows >= 2) {
$recordsUpdated++;
} else {
$recordsUnchanged++;
}
} catch (Throwable $rowError) {
$recordsSkipped++;
echo 'Uebersprungen Zeile ' . ($lineNumber + 1) . ': ' . $rowError->getMessage() . "\n";
}
}
if ($seenDesignations !== []) {
$seenDesignations = array_values(array_unique($seenDesignations));
$placeholders = implode(', ', array_fill(0, count($seenDesignations), '?'));
$deleteStaleStatement = $pdo->prepare(sprintf(
'DELETE FROM `comets_mpc` WHERE `designation_packed` NOT IN (%s)',
$placeholders
));
$deleteStaleStatement->execute($seenDesignations);
}
$pdo->commit();
echo "Import erfolgreich\n";
echo "Quelle: {$resolvedSource}\n";
echo "Empfangen: {$recordsReceived}\n";
echo "Importiert: {$recordsImported}\n";
echo "Neu: {$recordsInserted}\n";
echo "Aktualisiert: {$recordsUpdated}\n";
echo "Unveraendert: {$recordsUnchanged}\n";
echo "Uebersprungen: {$recordsSkipped}\n";
finishImportRun(
$pdo,
$importRunId,
$recordsReceived,
$recordsInserted,
$recordsUpdated,
'success',
$recordsSkipped > 0 ? "Es wurden {$recordsSkipped} Datensaetze uebersprungen." : null
);
} catch (Throwable $e) {
if ($pdo->inTransaction()) {
$pdo->rollBack();
}
finishImportRun(
$pdo,
$importRunId,
$recordsReceived,
$recordsInserted,
$recordsUpdated,
'error',
substr($e->getMessage(), 0, 65000)
);
writeError('Fehler: ' . $e->getMessage());
exit(1);
}
@@ -401,28 +469,67 @@ function parseSoft00CmtLine(string $line): array
$designationPacked = requireStringField(readFixedField($padded, 1, 12), 'designation_packed');
$designationAndName = requireStringField(readFixedField($padded, 103, 158), 'designation_and_name');
$epochCompact = toNullableCompactDate(readFixedField($padded, 82, 89));
$orbitType = deriveOrbitType($designationAndName, $designationPacked);
$yearOfPerihelion = requireIntField(readFixedField($padded, 15, 18), 'year_of_perihelion');
$monthOfPerihelion = requireIntField(readFixedField($padded, 20, 21), 'month_of_perihelion');
$dayOfPerihelion = requireFloatField(readFixedField($padded, 23, 29), 'day_of_perihelion');
$perihelionDistAu = requireFloatField(readFixedField($padded, 32, 39), 'perihelion_dist_au');
$eccentricity = requireFloatField(readFixedField($padded, 42, 49), 'eccentricity');
$argPerihelionDeg = requireFloatField(readFixedField($padded, 52, 59), 'arg_perihelion_deg');
$ascendingNodeDeg = requireFloatField(readFixedField($padded, 62, 69), 'ascending_node_deg');
$inclinationDeg = requireFloatField(readFixedField($padded, 72, 79), 'inclination_deg');
$absoluteMagnitudeH = toNullableFloat(readFixedField($padded, 91, 95));
$slopeParameterG = toNullableFloat(readFixedField($padded, 97, 101));
$referenceText = toNullableString(readFixedField($padded, 160, 200));
return [
':designation_packed' => $designationPacked,
':orbit_type' => deriveOrbitType($designationAndName, $designationPacked),
':year_of_perihelion' => requireIntField(readFixedField($padded, 15, 18), 'year_of_perihelion'),
':month_of_perihelion' => requireIntField(readFixedField($padded, 20, 21), 'month_of_perihelion'),
':day_of_perihelion' => requireFloatField(readFixedField($padded, 23, 29), 'day_of_perihelion'),
':perihelion_dist_au' => requireFloatField(readFixedField($padded, 32, 39), 'perihelion_dist_au'),
':eccentricity' => requireFloatField(readFixedField($padded, 42, 49), 'eccentricity'),
':arg_perihelion_deg' => requireFloatField(readFixedField($padded, 52, 59), 'arg_perihelion_deg'),
':ascending_node_deg' => requireFloatField(readFixedField($padded, 62, 69), 'ascending_node_deg'),
':inclination_deg' => requireFloatField(readFixedField($padded, 72, 79), 'inclination_deg'),
':orbit_type' => $orbitType,
':year_of_perihelion' => $yearOfPerihelion,
':month_of_perihelion' => $monthOfPerihelion,
':day_of_perihelion' => $dayOfPerihelion,
':perihelion_dist_au' => $perihelionDistAu,
':eccentricity' => $eccentricity,
':arg_perihelion_deg' => $argPerihelionDeg,
':ascending_node_deg' => $ascendingNodeDeg,
':inclination_deg' => $inclinationDeg,
':epoch_compact' => $epochCompact,
':epoch_date' => $epochCompact !== null ? compactDateToSql($epochCompact) : null,
':absolute_magnitude_h' => toNullableFloat(readFixedField($padded, 91, 95)),
':slope_parameter_g' => toNullableFloat(readFixedField($padded, 97, 101)),
':absolute_magnitude_h' => $absoluteMagnitudeH,
':slope_parameter_g' => $slopeParameterG,
':designation_and_name' => $designationAndName,
':reference_text' => toNullableString(readFixedField($padded, 160, 200)),
':reference_text' => $referenceText,
':raw_record' => $line,
':source_hash' => buildSourceHash([
$designationPacked,
$orbitType,
$yearOfPerihelion,
$monthOfPerihelion,
$dayOfPerihelion,
$perihelionDistAu,
$eccentricity,
$argPerihelionDeg,
$ascendingNodeDeg,
$inclinationDeg,
$epochCompact,
$absoluteMagnitudeH,
$slopeParameterG,
$designationAndName,
$referenceText,
]),
];
}
function buildSourceHash(array $parts): string
{
$normalized = array_map(
static fn(mixed $value): string => $value === null ? '' : trim((string) $value),
$parts
);
return sha1(implode('|', $normalized));
}
function deriveOrbitType(string $designationAndName, string $designationPacked): string
{
if (preg_match('/^\d+([A-Z])\//', $designationAndName, $matches) === 1) {
@@ -550,11 +657,87 @@ function normalizeDbConfig(array $config): array
];
}
function rebuildCometsTable(PDO $pdo): void
function ensureImportRunsTableExists(PDO $pdo): void
{
$pdo->exec("
CREATE TABLE IF NOT EXISTS `app_import_runs` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`source` varchar(32) NOT NULL,
`source_url` varchar(500) DEFAULT NULL,
`started_at` datetime NOT NULL,
`finished_at` datetime DEFAULT NULL,
`records_received` int(11) NOT NULL DEFAULT 0,
`records_inserted` int(11) NOT NULL DEFAULT 0,
`records_updated` int(11) NOT NULL DEFAULT 0,
`status` varchar(32) NOT NULL DEFAULT 'success',
`message` text DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_app_import_runs_source` (`source`),
KEY `idx_app_import_runs_started_at` (`started_at`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
");
}
function startImportRun(PDO $pdo, string $source, ?string $sourceUrl): int
{
$stmt = $pdo->prepare("
INSERT INTO `app_import_runs` (
`source`,
`source_url`,
`started_at`,
`status`,
`message`
) VALUES (
:source,
:source_url,
NOW(),
'running',
NULL
)
");
$stmt->execute([
':source' => $source,
':source_url' => $sourceUrl,
]);
return (int) $pdo->lastInsertId();
}
function finishImportRun(
PDO $pdo,
int $runId,
int $recordsReceived,
int $recordsInserted,
int $recordsUpdated,
string $status,
?string $message
): void {
$stmt = $pdo->prepare("
UPDATE `app_import_runs`
SET
`finished_at` = NOW(),
`records_received` = :records_received,
`records_inserted` = :records_inserted,
`records_updated` = :records_updated,
`status` = :status,
`message` = :message
WHERE `id` = :id
LIMIT 1
");
$stmt->execute([
':records_received' => $recordsReceived,
':records_inserted' => $recordsInserted,
':records_updated' => $recordsUpdated,
':status' => $status,
':message' => $message,
':id' => $runId,
]);
}
function ensureCometsTableExists(PDO $pdo): void
{
$pdo->exec('DROP TABLE IF EXISTS `comets_mpc`');
$pdo->exec(<<<'SQL'
CREATE TABLE `comets_mpc` (
CREATE TABLE IF NOT EXISTS `comets_mpc` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`designation_packed` varchar(12) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT 'Fixed-width MPC-Packbezeichnung aus Soft00Cmt.txt, inkl. Fragmentkennung',
`orbit_type` char(1) NOT NULL COMMENT 'Abgeleiteter Kometentyp, z. B. C, P oder I',
@@ -573,6 +756,7 @@ CREATE TABLE `comets_mpc` (
`designation_and_name` varchar(255) NOT NULL COMMENT 'Lesbare Bezeichnung, z. B. C/1995 O1 (Hale-Bopp)',
`reference_text` varchar(64) DEFAULT NULL COMMENT 'Referenz aus der MPC-Datei, z. B. MPEC 2026-FC3',
`raw_record` varchar(255) DEFAULT NULL COMMENT 'Originale Fixed-width-Zeile aus Soft00Cmt.txt',
`source_hash` char(40) DEFAULT NULL COMMENT 'Hash ueber die relevanten MPC-Quelldaten zur Aenderungserkennung',
`source_url` varchar(255) DEFAULT NULL COMMENT 'Quelle des letzten Imports',
`imported_at` datetime NOT NULL DEFAULT current_timestamp() COMMENT 'Zeitpunkt des letzten Imports',
`created_at` datetime NOT NULL DEFAULT current_timestamp(),
@@ -582,8 +766,50 @@ CREATE TABLE `comets_mpc` (
KEY `idx_mpc_comets_orbit_type` (`orbit_type`),
KEY `idx_mpc_comets_perihelion_year_month` (`year_of_perihelion`,`month_of_perihelion`),
KEY `idx_mpc_comets_epoch_compact` (`epoch_compact`),
KEY `idx_mpc_comets_designation_and_name` (`designation_and_name`)
KEY `idx_mpc_comets_designation_and_name` (`designation_and_name`),
KEY `idx_mpc_comets_source_hash` (`source_hash`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='Kometenelemente aus Soft00Cmt.txt des Minor Planet Center'
SQL
);
ensureColumnExists($pdo, 'comets_mpc', 'source_hash', "ALTER TABLE `comets_mpc` ADD COLUMN `source_hash` char(40) DEFAULT NULL COMMENT 'Hash ueber die relevanten MPC-Quelldaten zur Aenderungserkennung' AFTER `raw_record`");
ensureIndexExists($pdo, 'comets_mpc', 'idx_mpc_comets_source_hash', 'ALTER TABLE `comets_mpc` ADD KEY `idx_mpc_comets_source_hash` (`source_hash`)');
}
function ensureColumnExists(PDO $pdo, string $tableName, string $columnName, string $alterSql): void
{
$stmt = $pdo->prepare('
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = :table_name
AND COLUMN_NAME = :column_name
');
$stmt->execute([
':table_name' => $tableName,
':column_name' => $columnName,
]);
if ((int) $stmt->fetchColumn() === 0) {
$pdo->exec($alterSql);
}
}
function ensureIndexExists(PDO $pdo, string $tableName, string $indexName, string $alterSql): void
{
$stmt = $pdo->prepare('
SELECT COUNT(*)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = :table_name
AND INDEX_NAME = :index_name
');
$stmt->execute([
':table_name' => $tableName,
':index_name' => $indexName,
]);
if ((int) $stmt->fetchColumn() === 0) {
$pdo->exec($alterSql);
}
}