Eskimue/skyview.astronomiemuseum.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TCRB und Datenbank umgebaut für Imort Runs

Browse Source
This commit is contained in:
Eskimue
2026-04-05 09:43:19 +02:00
parent cbeced3a2a
commit 39685ac989
9 changed files with 2012 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/import/import_celestrak_active.php
+7 -4
View File
@@ -9,7 +9,7 @@ declare(strict_types=1);
* Tabellen:
* - sat_satellites
* - sat_tle_current
* - sat_import_runs
* - app_import_runs
*/
ini_set('display_errors', '1');
@@ -182,13 +182,15 @@ $recordsUpdated = 0;
try {
$stmtImportStart = $pdo->prepare("
INSERT INTO sat_import_runs (
INSERT INTO app_import_runs (
source,
source_url,
started_at,
status,
message
) VALUES (
:source,
:source_url,
NOW(),
'running',
NULL
@@ -196,6 +198,7 @@ try {
");
$stmtImportStart->execute([
':source' => $sourceName,
':source_url' => $sourceUrl,
]);
$importRunId = (int)$pdo->lastInsertId();
@@ -411,7 +414,7 @@ try {
$pdo->commit();
$stmtImportFinish = $pdo->prepare("
UPDATE sat_import_runs
UPDATE app_import_runs
SET
finished_at = NOW(),
records_received = :records_received,
@@ -440,7 +443,7 @@ try {
if ($importRunId !== null) {
$stmtImportError = $pdo->prepare("
UPDATE sat_import_runs
UPDATE app_import_runs
SET
finished_at = NOW(),
records_received = :records_received,
scripts/import/import_tcrb.php
+704
View File
@@ -0,0 +1,704 @@
<?php
declare(strict_types=1);
/**
* Importiert aktuelle T-CrB-Beobachtungen aus AAVSO WebObs in die Tabelle
* `aavso_tcrb_observations`.
*
* Nutzung:
* php scripts/import/import_tcrb.php
* php scripts/import/import_tcrb.php --pages=3
* php scripts/import/import_tcrb.php --html=C:\pfad\webobs.html --dry-run
* php scripts/import/import_tcrb.php --dir=C:\Users\ich\Desktop\TCrB
*
* Typischer Cron-Aufruf:
* php /pfad/zum/projekt/scripts/import/import_tcrb.php --pages=2
*/
error_reporting(E_ALL);
ini_set('display_errors', '1');
set_time_limit(0);
const DEFAULT_TCRB_URL = 'https://apps.aavso.org/webobs/results/?star=t+CRB&num_results=25&obs_types=ccd&page=1';
const DEFAULT_TABLE_NAME = 'aavso_tcrb_observations';
const DEFAULT_IMPORT_SOURCE = 'aavso-tcrb';
$rootDir = dirname(__DIR__, 2);
$config = require $rootDir . DIRECTORY_SEPARATOR . 'config' . DIRECTORY_SEPARATOR . 'database.php';
if (!is_array($config)) {
writeMessage('config/database.php muss ein Array mit DB-Zugangsdaten zurueckgeben.', true);
exit(1);
}
$config = normalizeDbConfig($config);
foreach (['host', 'dbname', 'user', 'pass'] as $requiredKey) {
if (!array_key_exists($requiredKey, $config) || $config[$requiredKey] === '') {
writeMessage("DB-Konfiguration unvollstaendig: {$requiredKey} fehlt.", true);
exit(1);
}
}
$options = parseCliArguments($argv);
if (($options['help'] ?? false) === true) {
writeUsage();
exit(0);
}
$dsn = sprintf(
'mysql:host=%s;port=%d;dbname=%s;charset=%s',
$config['host'],
(int) $config['port'],
$config['dbname'],
$config['charset']
);
$pdo = new PDO(
$dsn,
$config['user'],
$config['pass'],
[
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
]
);
$tableName = DEFAULT_TABLE_NAME;
if (!tableExists($pdo, $tableName)) {
writeMessage("Die Zieltabelle `{$tableName}` existiert noch nicht. Wir koennen sie im naechsten Schritt gemeinsam anlegen.", true);
exit(1);
}
ensureImportRunsTableExists($pdo);
$insertSql = <<<SQL
INSERT INTO `{$tableName}` (
`source_row_key`,
`star_name`,
`jd`,
`observed_at_utc`,
`calendar_date_text`,
`magnitude`,
`magnitude_error`,
`filter_code`,
`observer_code`,
`notes_raw`,
`imported_at`
) VALUES (
:source_row_key,
:star_name,
:jd,
:observed_at_utc,
:calendar_date_text,
:magnitude,
:magnitude_error,
:filter_code,
:observer_code,
:notes_raw,
NOW()
)
ON DUPLICATE KEY UPDATE
`star_name` = VALUES(`star_name`),
`observed_at_utc` = VALUES(`observed_at_utc`),
`calendar_date_text` = VALUES(`calendar_date_text`),
`magnitude` = VALUES(`magnitude`),
`magnitude_error` = VALUES(`magnitude_error`),
`filter_code` = VALUES(`filter_code`),
`observer_code` = VALUES(`observer_code`),
`notes_raw` = VALUES(`notes_raw`),
`imported_at` = NOW(),
`updated_at` = NOW()
SQL;
$statement = $pdo->prepare($insertSql);
$pagesToImport = max(1, (int) ($options['pages'] ?? 1));
$sourceHtmlPath = $options['html'] ?? null;
$sourceDirectoryPath = $options['dir'] ?? null;
$dryRun = (bool) ($options['dry-run'] ?? false);
$baseUrl = (string) ($options['url'] ?? DEFAULT_TCRB_URL);
$totalImported = 0;
$totalUpserts = 0;
$recordsInserted = 0;
$recordsUpdated = 0;
$importRunId = null;
if ($sourceHtmlPath !== null && $sourceDirectoryPath !== null) {
writeMessage('Bitte entweder --html oder --dir verwenden, nicht beides gleichzeitig.', true);
exit(1);
}
$importSources = [];
if ($sourceDirectoryPath !== null) {
foreach (findHtmlFilesInDirectory($sourceDirectoryPath) as $filePath) {
$importSources[] = [
'label' => basename($filePath),
'html' => loadHtmlFromFile($filePath),
'is_local_file' => true,
];
}
if (empty($importSources)) {
writeMessage("Im Ordner wurden keine importierbaren HTML/TXT-Dateien gefunden: {$sourceDirectoryPath}", true);
exit(1);
}
} elseif ($sourceHtmlPath !== null) {
$importSources[] = [
'label' => basename($sourceHtmlPath),
'html' => loadHtmlFromFile($sourceHtmlPath),
'is_local_file' => true,
];
} else {
for ($pageNumber = 1; $pageNumber <= $pagesToImport; $pageNumber++) {
$importSources[] = [
'label' => 'Seite ' . $pageNumber,
'html' => downloadUrl(buildPagedUrl($baseUrl, $pageNumber)),
'is_local_file' => false,
];
}
}
if (!$dryRun) {
$stmtImportStart = $pdo->prepare("
INSERT INTO app_import_runs (
source,
source_url,
started_at,
status,
message
) VALUES (
:source,
:source_url,
NOW(),
'running',
NULL
)
");
$stmtImportStart->execute([
':source' => DEFAULT_IMPORT_SOURCE,
':source_url' => $sourceDirectoryPath !== null ? $sourceDirectoryPath : ($sourceHtmlPath !== null ? $sourceHtmlPath : $baseUrl),
]);
$importRunId = (int) $pdo->lastInsertId();
}
foreach ($importSources as $sourceIndex => $source) {
$pageNumber = $sourceIndex + 1;
$html = (string) $source['html'];
$pageData = parseWebObsPage($html);
if (empty($pageData['observations'])) {
writeMessage(($source['label'] ?? ('Seite ' . $pageNumber)) . ': keine Beobachtungen gefunden.');
if (($source['is_local_file'] ?? false) === true) {
continue;
}
continue;
}
writeMessage(sprintf(
'%s: %d Beobachtungen fuer %s gefunden.',
(string) ($source['label'] ?? ('Seite ' . $pageNumber)),
count($pageData['observations']),
$pageData['target_name'] !== '' ? $pageData['target_name'] : 'T CrB'
));
if ($dryRun) {
$totalImported += count($pageData['observations']);
continue;
}
$pdo->beginTransaction();
try {
foreach ($pageData['observations'] as $observation) {
$statement->execute([
':source_row_key' => $observation['source_row_key'],
':star_name' => $observation['star_name'],
':jd' => $observation['jd'],
':observed_at_utc' => $observation['observed_at_utc'],
':calendar_date_text' => $observation['calendar_date_text'],
':magnitude' => $observation['magnitude'],
':magnitude_error' => $observation['magnitude_error'],
':filter_code' => $observation['filter_code'],
':observer_code' => $observation['observer_code'],
':notes_raw' => $observation['notes_raw'],
]);
$affectedRows = (int) $statement->rowCount();
$totalUpserts += $affectedRows;
if ($affectedRows === 1) {
$recordsInserted++;
} elseif ($affectedRows >= 2) {
$recordsUpdated++;
}
$totalImported++;
}
$pdo->commit();
} catch (Throwable $e) {
if ($pdo->inTransaction()) {
$pdo->rollBack();
}
if ($importRunId !== null) {
$stmtImportError = $pdo->prepare("
UPDATE app_import_runs
SET
finished_at = NOW(),
records_received = :records_received,
records_inserted = :records_inserted,
records_updated = :records_updated,
status = 'error',
message = :message
WHERE id = :id
");
$stmtImportError->execute([
':records_received' => $totalImported,
':records_inserted' => $recordsInserted,
':records_updated' => $recordsUpdated,
':message' => substr($e->getMessage(), 0, 65000),
':id' => $importRunId,
]);
}
writeMessage('Import fehlgeschlagen bei ' . (string) ($source['label'] ?? ('Seite ' . $pageNumber)) . ': ' . $e->getMessage(), true);
exit(1);
}
}
if ($dryRun) {
writeMessage("Dry-Run abgeschlossen. Parsbare Beobachtungen: {$totalImported}.");
exit(0);
}
if ($importRunId !== null) {
$stmtImportFinish = $pdo->prepare("
UPDATE app_import_runs
SET
finished_at = NOW(),
records_received = :records_received,
records_inserted = :records_inserted,
records_updated = :records_updated,
status = 'success',
message = NULL
WHERE id = :id
");
$stmtImportFinish->execute([
':records_received' => $totalImported,
':records_inserted' => $recordsInserted,
':records_updated' => $recordsUpdated,
':id' => $importRunId,
]);
}
writeMessage("Import abgeschlossen. Verarbeitete Beobachtungen: {$totalImported}. DB-Upserts: {$totalUpserts}.");
exit(0);
function parseCliArguments(array $argv): array
{
$options = [
'pages' => 1,
'dry-run' => false,
'help' => false,
];
foreach (array_slice($argv, 1) as $argument) {
if ($argument === '--dry-run') {
$options['dry-run'] = true;
continue;
}
if ($argument === '--help' || $argument === '-h') {
$options['help'] = true;
continue;
}
if (str_starts_with($argument, '--pages=')) {
$options['pages'] = (int) substr($argument, 8);
continue;
}
if (str_starts_with($argument, '--url=')) {
$options['url'] = (string) substr($argument, 6);
continue;
}
if (str_starts_with($argument, '--html=')) {
$options['html'] = (string) substr($argument, 7);
continue;
}
if (str_starts_with($argument, '--dir=')) {
$options['dir'] = (string) substr($argument, 6);
continue;
}
}
return $options;
}
function writeUsage(): void
{
$usageLines = [
'Nutzung:',
' php scripts/import/import_tcrb.php',
' php scripts/import/import_tcrb.php --pages=3',
' php scripts/import/import_tcrb.php --html=C:\pfad\webobs.html --dry-run',
' php scripts/import/import_tcrb.php --dir=C:\Users\ich\Desktop\TCrB',
'',
'Optionen:',
' --pages=N Anzahl der zu ladenden Ergebnisseiten, Standard: 1',
' --url=URL Basis-URL fuer den Abruf, Standard: AAVSO T CrB CCD',
' --html=PFAD Liest die erste Seite aus einer lokalen HTML-Datei statt aus dem Web',
' --dir=PFAD Liest alle HTML/TXT-Dateien aus einem Ordner in Dateinamen-Reihenfolge ein',
' --dry-run Parst nur und schreibt nichts in die DB',
' --help Diese Hilfe anzeigen',
];
foreach ($usageLines as $line) {
writeMessage($line);
}
}
function normalizeDbConfig(array $config): array
{
$config['port'] = isset($config['port']) && (int) $config['port'] > 0 ? (int) $config['port'] : 3306;
$config['charset'] = isset($config['charset']) && $config['charset'] !== '' ? (string) $config['charset'] : 'utf8mb4';
return $config;
}
function tableExists(PDO $pdo, string $tableName): bool
{
$stmt = $pdo->prepare('
SELECT COUNT(*)
FROM information_schema.TABLES
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = :table_name
');
$stmt->execute([':table_name' => $tableName]);
return (int) $stmt->fetchColumn() > 0;
}
function ensureImportRunsTableExists(PDO $pdo): void
{
$pdo->exec("
CREATE TABLE IF NOT EXISTS `app_import_runs` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`source` varchar(32) NOT NULL,
`source_url` varchar(500) DEFAULT NULL,
`started_at` datetime NOT NULL,
`finished_at` datetime DEFAULT NULL,
`records_received` int(11) NOT NULL DEFAULT 0,
`records_inserted` int(11) NOT NULL DEFAULT 0,
`records_updated` int(11) NOT NULL DEFAULT 0,
`status` varchar(32) NOT NULL,
`message` text DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `idx_app_import_runs_source` (`source`),
KEY `idx_app_import_runs_started_at` (`started_at`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
");
}
function loadHtmlFromFile(string $path): string
{
if (!is_file($path)) {
throw new RuntimeException("HTML-Datei nicht gefunden: {$path}");
}
$html = file_get_contents($path);
if ($html === false || trim($html) === '') {
throw new RuntimeException("HTML-Datei konnte nicht gelesen werden: {$path}");
}
return $html;
}
function findHtmlFilesInDirectory(string $directoryPath): array
{
if (!is_dir($directoryPath)) {
throw new RuntimeException("Ordner nicht gefunden: {$directoryPath}");
}
$entries = scandir($directoryPath);
if ($entries === false) {
throw new RuntimeException("Ordner konnte nicht gelesen werden: {$directoryPath}");
}
$filePaths = [];
foreach ($entries as $entry) {
if ($entry === '.' || $entry === '..') {
continue;
}
$fullPath = rtrim($directoryPath, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $entry;
if (!is_file($fullPath)) {
continue;
}
$extension = strtolower((string) pathinfo($fullPath, PATHINFO_EXTENSION));
if (!in_array($extension, ['html', 'htm', 'txt'], true)) {
continue;
}
$filePaths[] = $fullPath;
}
natsort($filePaths);
return array_values($filePaths);
}
function buildPagedUrl(string $baseUrl, int $pageNumber): string
{
$parts = parse_url($baseUrl);
if ($parts === false) {
throw new RuntimeException('Ungueltige URL: ' . $baseUrl);
}
$query = [];
parse_str((string) ($parts['query'] ?? ''), $query);
$query['page'] = $pageNumber;
$scheme = $parts['scheme'] ?? 'https';
$host = $parts['host'] ?? '';
$path = $parts['path'] ?? '';
$queryString = http_build_query($query);
return $scheme . '://' . $host . $path . ($queryString !== '' ? '?' . $queryString : '');
}
function downloadUrl(string $url): string
{
if (DIRECTORY_SEPARATOR === '\\' && function_exists('shell_exec')) {
$urlBase64 = base64_encode($url);
$powerShellScript = str_replace(
'__URL_BASE64__',
$urlBase64,
<<<'PS'
$ErrorActionPreference = 'Stop'
$ProgressPreference = 'SilentlyContinue'
$url = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String('__URL_BASE64__'))
$response = Invoke-WebRequest -Uri $url -UseBasicParsing -Headers @{ 'User-Agent' = 'SkyView-TCrB-Importer/1.0' }
[Console]::OutputEncoding = [System.Text.Encoding]::UTF8
$response.Content
PS
);
$encodedCommandBinary = function_exists('iconv')
? iconv('UTF-8', 'UTF-16LE', $powerShellScript)
: $powerShellScript;
$encodedCommand = base64_encode($encodedCommandBinary === false ? $powerShellScript : $encodedCommandBinary);
$command = 'powershell -NoProfile -ExecutionPolicy Bypass -EncodedCommand ' . $encodedCommand;
$output = shell_exec($command);
if ($output !== null && trim($output) !== '') {
return $output;
}
}
if (function_exists('curl_init')) {
$ch = curl_init($url);
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_CONNECTTIMEOUT => 15,
CURLOPT_TIMEOUT => 60,
CURLOPT_USERAGENT => 'SkyView-TCrB-Importer/1.0',
]);
$response = curl_exec($ch);
if ($response === false) {
$error = curl_error($ch);
curl_close($ch);
throw new RuntimeException('Download fehlgeschlagen: ' . $error);
}
$httpCode = (int) curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
curl_close($ch);
if ($httpCode < 200 || $httpCode >= 300) {
throw new RuntimeException("Download fehlgeschlagen. HTTP-Status: {$httpCode}");
}
return (string) $response;
}
$context = stream_context_create([
'http' => [
'method' => 'GET',
'header' => implode("\r\n", [
'User-Agent: SkyView-TCrB-Importer/1.0',
]),
'timeout' => 60,
'ignore_errors' => true,
],
]);
$response = @file_get_contents($url, false, $context);
if ($response === false) {
throw new RuntimeException('Download fehlgeschlagen: keine Antwort erhalten.');
}
return $response;
}
function parseWebObsPage(string $html): array
{
libxml_use_internal_errors(true);
$dom = new DOMDocument();
$loaded = $dom->loadHTML($html);
libxml_clear_errors();
if ($loaded === false) {
throw new RuntimeException('HTML konnte nicht geparst werden.');
}
$xpath = new DOMXPath($dom);
$targetName = '';
$obsInfoNode = $xpath->query("//p[@id='obsinfo']")->item(0);
if ($obsInfoNode instanceof DOMNode) {
$obsInfoText = normalizeWhitespace($obsInfoNode->textContent);
if (preg_match('/for\s+(.+?)\s+from\s+/i', $obsInfoText, $matches) === 1) {
$targetName = trim($matches[1]);
}
}
$observations = [];
$rowNodes = $xpath->query("//table[contains(@class,'observations')]/tbody/tr[contains(@class,'obs')]");
if ($rowNodes === false) {
return [
'target_name' => $targetName,
'observations' => [],
];
}
foreach ($rowNodes as $rowNode) {
if (!$rowNode instanceof DOMElement) {
continue;
}
$rowId = trim((string) $rowNode->getAttribute('id'));
$tdNodes = $xpath->query('./td', $rowNode);
if ($tdNodes === false || $tdNodes->length < 8) {
continue;
}
$starName = normalizeWhitespace($tdNodes->item(1)?->textContent ?? '');
$jdText = normalizeWhitespace($tdNodes->item(2)?->textContent ?? '');
$calendarDateText = normalizeWhitespace($tdNodes->item(3)?->textContent ?? '');
$magnitudeText = normalizeWhitespace($tdNodes->item(4)?->textContent ?? '');
$magnitudeErrorText = normalizeWhitespace($tdNodes->item(5)?->textContent ?? '');
$filterCode = normalizeWhitespace($tdNodes->item(6)?->textContent ?? '');
$observerCode = normalizeWhitespace($tdNodes->item(7)?->textContent ?? '');
if ($jdText === '' || $magnitudeText === '') {
continue;
}
$detailRow = $xpath->query("//tr[@id='{$rowId}-detail']")->item(0);
$notesRaw = null;
if ($detailRow instanceof DOMElement) {
$detailCells = $xpath->query(".//table[contains(@class,'obs-detail')]/tbody/tr/td", $detailRow);
if ($detailCells !== false && $detailCells->length >= 6) {
$notesRaw = normalizeWhitespace($detailCells->item(5)?->textContent ?? '');
$notesRaw = $notesRaw !== '' && $notesRaw !== '—' ? $notesRaw : null;
}
}
$jd = (float) str_replace(',', '.', $jdText);
$observedAtUtc = jdToUtcString($jd);
$observations[] = [
'source_row_key' => $rowId !== '' ? $rowId : null,
'star_name' => $starName !== '' ? $starName : ($targetName !== '' ? $targetName : 'T CrB'),
'jd' => number_format($jd, 6, '.', ''),
'observed_at_utc' => $observedAtUtc,
'calendar_date_text' => $calendarDateText !== '' ? $calendarDateText : null,
'magnitude' => normalizeNumeric($magnitudeText),
'magnitude_error' => $magnitudeErrorText !== '' ? normalizeNumeric($magnitudeErrorText) : null,
'filter_code' => $filterCode !== '' ? $filterCode : null,
'observer_code' => $observerCode !== '' ? $observerCode : null,
'notes_raw' => $notesRaw,
];
}
return [
'target_name' => $targetName,
'observations' => $observations,
];
}
function normalizeWhitespace(string $value): string
{
$value = html_entity_decode($value, ENT_QUOTES | ENT_HTML5, 'UTF-8');
$value = preg_replace('/\s+/u', ' ', trim($value));
return $value ?? '';
}
function normalizeNumeric(string $value): string
{
return str_replace(',', '.', trim($value));
}
function jdToUtcString(float $jd): string
{
$jd += 0.5;
$z = (int) floor($jd);
$f = $jd - $z;
if ($z >= 2299161) {
$alpha = (int) floor(($z - 1867216.25) / 36524.25);
$a = $z + 1 + $alpha - (int) floor($alpha / 4);
} else {
$a = $z;
}
$b = $a + 1524;
$c = (int) floor(($b - 122.1) / 365.25);
$d = (int) floor(365.25 * $c);
$e = (int) floor(($b - $d) / 30.6001);
$day = $b - $d - (int) floor(30.6001 * $e) + $f;
$month = $e < 14 ? $e - 1 : $e - 13;
$year = $month > 2 ? $c - 4716 : $c - 4715;
$dayInteger = (int) floor($day);
$dayFraction = $day - $dayInteger;
$totalSeconds = (int) round($dayFraction * 86400);
if ($totalSeconds >= 86400) {
$totalSeconds = 86399;
}
$hours = intdiv($totalSeconds, 3600);
$minutes = intdiv($totalSeconds % 3600, 60);
$seconds = $totalSeconds % 60;
return sprintf(
'%04d-%02d-%02d %02d:%02d:%02d',
$year,
$month,
$dayInteger,
$hours,
$minutes,
$seconds
);
}
function writeMessage(string $message, bool $isError = false): void
{
$prefix = '[' . date('Y-m-d H:i:s') . '] ';
$stream = $isError ? STDERR : STDOUT;
fwrite($stream, $prefix . $message . PHP_EOL);
}