# -*- coding: utf-8 -*- from __future__ import print_function import itertools import os import errno import sys from fnmatch import fnmatch from pkgutil import get_data from time import time import certifi import numpy as np from .data import iers from .curvelib import Splines from .functions import load_bundled_npy from .io_timescale import ( _build_legacy_data, parse_deltat_data, parse_deltat_preds, parse_leap_seconds, ) from .jpllib import SpiceKernel from .sgp4lib import EarthSatellite from .timelib import Timescale try: from io import BytesIO except: from StringIO import StringIO as BytesIO if sys.version_info >= (3, 3): _replace = os.replace else: _replace = os.rename # Raises OSError on Windows if destination exists try: from ssl import create_default_context except ImportError: create_default_context = None try: from urllib.parse import urlparse from urllib.request import urlopen except: from urlparse import urlparse from urllib2 import urlopen try: urlopen('', cafile=None) except TypeError: _supports_cafile_argument = False except ValueError: # Expected when the URL is an empty string. _supports_cafile_argument = True # If we are running under the built-in IDLE development environment, we # cannot use '\r' to keep repainting the current line as a progress bar: _running_IDLE = (sys.stderr.__class__.__name__ == 'PseudoOutputFile') def _filename_of(url): """Return the last path component of a url.""" return urlparse(url).path.split('/')[-1] _IERS = 'https://hpiers.obspm.fr/iers/bul/bulc/' _IERS2 = 'https://datacenter.iers.org/products/eop/rapid/standard/' _JPL = 'https://ssd.jpl.nasa.gov/ftp/eph/planets/bsp/' _NAIF_KERNELS = 'https://naif.jpl.nasa.gov/pub/naif/generic_kernels/' _NAIF = 'https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/satellites/' def _open_binary(path): return open(path, mode='rb') class Loader(object): """A tool for downloading and opening astronomical data files. A default `Loader` that saves data files to the current working directory can be imported directly from the Skyfield API:: from skyfield.api import load But users can also create a `Loader` of their own, if there is another directory they want data files saved to, or if they want to specify different options. The directory is created automatically if it does not yet exist:: from skyfield.api import Loader load = Loader('~/skyfield-data') The options are: ``verbose`` If set to ``False``, then the loader will not print a progress bar to the screen each time it downloads a file. (If the standard output is not a TTY, then no progress bar is printed anyway.) ``expire`` (This option is no longer supported.) Once a `Loader` is created, it can be called like a function to open, or else to download and open, a file whose name it recognizes:: planets = load('de405.bsp') Each loader also supports an attribute and a few methods. """ def __init__(self, directory, verbose=True, expire=False): self.directory = os.path.expanduser(directory) self.verbose = verbose self.events = [] try: os.makedirs(self.directory) except OSError as e: if e.errno != errno.EEXIST and not os.path.isdir(self.directory): raise # Each instance gets its own copy of these data structures, # instead of sharing a single copy, so users can edit them # without changing the behavior of other Loader objects: self.urls = { 'finals2000A.all': _IERS2, 'Leap_Second.dat': _IERS, 'moon_080317.tf': _NAIF_KERNELS + 'fk/satellites/', 'moon_pa_de421_1900-2050.bpc': _NAIF_KERNELS + 'pck/', 'pck00008.tpc': _NAIF_KERNELS + 'pck/a_old_versions/', '.bsp': [ ('jup*.bsp', _NAIF), ('*.bsp', _JPL), ], } self.parsers = { 'deltat.data': parse_deltat_data, 'deltat.preds': parse_deltat_preds, 'Leap_Second.dat': parse_leap_seconds, } self.openers = { # Old-fashioned: auto-create objects, leaving readers and # code tools guessing what kind of object we have returned. '.bsp': [ ('*.bsp', SpiceKernel), ], # New approach: just return open files, which callers can # then pass to the right class, making the class visible in # the code to both human readers and their IDEs. '.bpc': [('*', _open_binary)], '.tpc': [('*', _open_binary)], '.tf': [('*', _open_binary)], } def path_to(self, filename): """Return the path to ``filename`` in this loader's directory.""" if self.directory == '.': return filename return os.path.join(self.directory, filename) def days_old(self, filename): """Return how recently ``filename`` was modified, measured in days.""" mtime = os.stat(self.path_to(filename)).st_mtime seconds = time() - mtime return seconds / 86400.0 def exists(self, filename): return os.path.exists(self.path_to(filename)) def __call__(self, filename, reload=False, backup=False, builtin=False): """Open the given file, downloading it first if necessary.""" if '://' in filename: url = filename filename = urlparse(url).path.split('/')[-1] # Should this API accept full path names? It might look like: # elif os.sep in filename: # os.path.expanduser(directory) # path = filename # filename = os.path.basename(path) # url = _search(self.urls, filename) # directory = else: url = _search(self.urls, filename) if url: url += filename parser = _search(self.parsers, filename) opener = _search(self.openers, filename) if (parser is None) and (opener is None): raise ValueError('Skyfield does not know how to open a file' ' named {0!r}'.format(filename)) if builtin: self._log('{0}\n Parsing builtin file with {1}()', filename, parser.__name__) f = BytesIO(get_data('skyfield.data', filename)) return parser(f) path = self._assure(url, filename, reload, backup) if parser is not None: self._log(' Parsing with {0}()', parser.__name__) with open(path, 'rb') as f: return parser(f) else: self._log(' Opening with {0}', opener.__name__) return opener(path) def _assure(self, url, filename, reload, backup): path = self.path_to(filename) exists = os.path.exists(path) self._log(path) if exists: self._log(' File already exists') if (not exists) or reload: if url is None: raise ValueError('Skyfield does not know where to download {!r}' .format(filename)) self._log(' Downloading {0}', url) download(url, path, self.verbose, backup=backup) return path def _log(self, message, *args): self.events.append(message.format(*args)) def build_url(self, filename): """Return the URL Skyfield will try downloading for a given filename. Raises ``ValueError`` if Skyfield doesn't know where to get the file based on its name. """ base = _search(self.urls, filename) if base: return base + filename raise ValueError("Skyfield doesn't know the URL of {0!r}" .format(filename)) def tle(self, url, reload=False, filename=None): """Load and parse a satellite TLE file. DEPRECATED: in a misguided attempt to be overly convenient, this routine builds an unweildy dictionary of satellites with keys of two different Python types: integer keys for satellite numbers, and string keys for satellite names. It even lists satellites like ``ISS (ZARYA)`` twice, in case the user wants to look them up by a single name like ``ZARYA``. What a mess. Users should instead call the simple ``tle_file()`` method, and themselves build any dictionaries they need. See the :meth:`~skyfield.iokit.Loader.open()` documentation for the meaning of the ``reload`` and ``filename`` parameters. """ d = {} with self.open(url, reload=reload, filename=filename) as f: for names, sat in parse_tle(f): d[sat.model.satnum] = sat for name in names: d[name] = sat return d def tle_file(self, url, reload=False, filename=None, ts=None, skip_names=False): """Load and parse a TLE file, returning a list of Earth satellites. Given a URL or local path to an ASCII text file, this loads a series of TLE “Two-Line Element” sets and returns a list of :class:`~skyfield.sgp4lib.EarthSatellite` objects for them. See :doc:`earth-satellites`. See the :meth:`~skyfield.iokit.Loader.open()` method for the meaning of the ``reload`` and ``filename`` parameters. See the :meth:`parse_tle_file()` function for the meaning of the ``ts`` and ``skip_names`` parameters. """ with self.open(url, reload=reload, filename=filename) as f: return list(parse_tle_file(f, ts, skip_names)) def download(self, url, filename=None, backup=False): """Download a file, even if it’s already on disk; return its path. You can specify the local ``filename`` to which the file will be saved; the default is to use the final component of ``url``. Set ``backup`` to ``True`` if you want an already-existing file moved out of the way instead of overwritten. Your operating system may raise any of several errors during a download: hostname lookup failure (this is the usual symptom if you are disconnected from the Internet); the server refusing the connection; and the connection closing mid-download. Skyfield makes no attempt to intercept or interpret these errors — which vary by operating system — so your application itself should catch network errors if it needs to avoid printing raw Python exceptions, or if you want to retry failed downloads. """ if '://' not in url: url = self.build_url(url) if filename is None: filename = urlparse(url).path.split('/')[-1] path = self.path_to(filename) download(url, path, self.verbose, backup=backup) return path def open(self, url, mode='rb', reload=False, filename=None, backup=False): """Open a file, downloading it first if it does not yet exist. Unlike when you call a loader directly like ``my_loader()``, this ``my_loader.open()`` method does not attempt to parse or interpret the file; it simply returns an open file object. The ``url`` can be either an external URL, or else the path to a file on the current filesystem. A relative path will be assumed to be relative to the base directory of this loader object. If a URL was provided and the ``reload`` parameter is true, then any existing file will be removed before the download starts. The ``filename`` parameter lets you specify an alternative local filename instead of having the filename extracted from the final component of the URL. """ if '://' not in url: path_that_might_be_relative = url path = os.path.join(self.directory, path_that_might_be_relative) return open(path, mode) if filename is None: filename = urlparse(url).path.split('/')[-1] path = self._assure(url, filename, reload, backup) return open(path, mode) def timescale(self, delta_t=None, builtin=True): """Return a `Timescale` built using official Earth rotation data. ``delta_t`` — Lets you override the standard ∆T tables by providing your own ∆T offset in seconds. For details, see :ref:`custom-delta-t`. ``builtin`` — By default, Skyfield uses ∆T and leap second tables that it carries internally; to instead load this data from files, set this option to ``False``. For compatibility with Skyfield ≤ 1.30, if you have on disk the three files ``deltat.data``, ``deltat.preds``, and ``Leap_Second.dat``, then Skyfield will load them. Otherwise, Skyfield will download and use ``finals2000A.all`` from the International Earth Rotation Service. For details, see :ref:`downloading-timescale-files`. """ e = self.exists if builtin: # See "build_arrays.py" for a notes on how these are stored. arrays = load_bundled_npy('iers.npz') daily_tt = arrays['tt_jd_minus_arange'] daily_tt += np.arange(len(daily_tt)) daily_delta_t = (arrays['delta_t_1e7'] / 1e7).round(7) delta_t_recent = daily_tt, daily_delta_t leap_dates = arrays['leap_dates'] leap_offsets = arrays['leap_offsets'] elif e('deltat.data') and e('deltat.preds') and e('Leap_Second.dat'): # Avoid changing the meaning of "builtin=False" and # surprising the user with a file download, if their # previous version of Skyfield already downloaded the three # old files we used to rely on. deltat_data = self('deltat.data') deltat_preds = self('deltat.preds') _, leap_second_dat = self('Leap_Second.dat') delta_t_recent, leap_dates, leap_offsets = _build_legacy_data( deltat_data, deltat_preds, leap_second_dat) else: url = self.build_url('finals2000A.all') with self.open(url) as f: utc_mjd, dut1 = iers.parse_dut1_from_finals_all(f) daily_tt, daily_delta_t, leap_dates, leap_offsets = ( iers.build_timescale_arrays(utc_mjd, dut1)) delta_t_recent = daily_tt, daily_delta_t if delta_t is not None: delta_t_recent = Splines([0, 1, delta_t]) return Timescale(delta_t_recent, leap_dates, leap_offsets) @property def log(self): return '\n'.join(self.events) def _search(mapping, filename): """Search a Loader data structure for a filename.""" result = mapping.get(filename) if result is not None: return result name, ext = os.path.splitext(filename) result = mapping.get(ext) if result is not None: for pattern, result2 in result: if fnmatch(filename, pattern): return result2 return None def load_file(path): """Open a file on your local drive, using its extension to guess its type. This routine only works on ``.bsp`` ephemeris files right now, but will gain support for additional file types in the future. :: from skyfield.api import load_file planets = load_file('~/Downloads/de421.bsp') """ path = os.path.expanduser(path) base, ext = os.path.splitext(path) if ext == '.bsp': return SpiceKernel(path) raise ValueError('unrecognized file extension: {}'.format(path)) def parse_tle(fileobj): """DEPRECATED: call the simpler `parse_tle_file()` routine instead.""" for sat in parse_tle_file(fileobj): names = () if sat.name: names = [sat.name] if ' (' in sat.name: # A name like `HTV-6 (KOUNOTORI 6)`. short_name, secondary_name = sat.name.split(' (') secondary_name = secondary_name.rstrip(')') names.append(short_name) names.append(secondary_name) yield names, sat def parse_tle_file(lines, ts=None, skip_names=False): """Parse lines of TLE satellite data, yielding a sequence of satellites. Given a sequence ``lines`` of byte strings (which can be an open binary file, which acts like a sequence of lines in Python), this routine yields an :class:`~skyfield.sgp4lib.EarthSatellite` for each pair of adjacent lines that start with ``"1 "`` and ``"2 "`` and have 69 or more characters each. If the line preceding a TLE is not part of another TLE, it is used as the satellite’s ``.name``. If you pass a ``ts`` timescale, Skyfield will use it to build the ``.epoch`` date attribute on each satellite; otherwise a timescale derived from Skyfield’s built-in leap second files will be used. If for a particular file you see random lines of text being interpreted as satellite names, set ``skip_names`` to ``True`` and Skyfield will not try to store satellite names. See :doc:`earth-satellites` for details. An exception is raised if the attempt to parse a pair of candidate lines as TLE lines fails. """ b0 = b1 = b'' for b2 in lines: if (b2.startswith(b'2 ') and len(b2) >= 69 and b1.startswith(b'1 ') and len(b1) >= 69): if not skip_names and b0: b0 = b0.rstrip(b' \n\r') if b0.startswith(b'0 '): b0 = b0[2:] # Spacetrack 3-line format name = b0.decode('ascii') else: name = None line1 = b1.decode('ascii') line2 = b2.decode('ascii') yield EarthSatellite(line1, line2, name, ts) b0 = b1 = b'' # don't accidentally use line 2 as next sat's name else: b0 = b1 b1 = b2 def download(url, path, verbose=None, blocksize=128*1024, backup=False): """Download a file from a URL, possibly displaying a progress bar. Saves the output to the file named by `path`. If the URL cannot be downloaded or the file cannot be written, an ``IOError`` is raised. Normally, if the standard error output is a terminal, then a progress bar is displayed to keep the user entertained. Specify `verbose=True` or `verbose=False` to override this behavior. """ try: if create_default_context is not None: ssl_context = create_default_context(cafile=certifi.where()) connection = urlopen(url, context=ssl_context) elif _supports_cafile_argument: connection = urlopen(url, cafile=certifi.where()) else: connection = urlopen(url) # Very old Python: no certificate check. except Exception as e: e2 = IOError('cannot download {0} because {1}'.format(url, e)) e2.__cause__ = None raise e2 if verbose is None: verbose = sys.stderr.isatty() bar = None if verbose: if _running_IDLE: print('Downloading {0} ...'.format(os.path.basename(path)), file=sys.stderr) else: bar = ProgressBar(path) content_length = int(connection.headers.get('content-length', -1)) # Claim our own unique download filename. tempbase = tempname = path + '.download' flags = getattr(os, 'O_BINARY', 0) | os.O_CREAT | os.O_EXCL | os.O_RDWR i = 1 while True: try: fd = os.open(tempname, flags, 0o666) except OSError as e: # "FileExistsError" is not supported by Python 2 if e.errno != errno.EEXIST: raise i += 1 tempname = '{0}{1}'.format(tempbase, i) else: break # Download to the temporary filename. with os.fdopen(fd, 'wb') as w: try: length = 0 while True: data = connection.read(blocksize) if not data: break w.write(data) length += len(data) if bar is not None: bar.report(length, content_length) w.flush() except Exception as e: raise IOError('error getting {0} - {1}'.format(url, e)) # Move the original out of the way, if requested. if os.path.exists(path) and backup: _rename_original(path) # Rename the temporary file to the destination name. try: _replace(tempname, path) except Exception as e: raise IOError('error renaming {0} to {1} - {2}'.format( tempname, path, e)) def _rename_original(path): for n in itertools.count(1): prefix, suffix = path.rsplit('.', 1) backup_path = '{0}.old{1}.{2}'.format(prefix, n, suffix) if not os.path.exists(backup_path): break os.rename(path, backup_path) class ProgressBar(object): def __init__(self, path): self.filename = os.path.basename(path) self.t0 = 0 def report(self, bytes_so_far, bytes_total): if bytes_total < 0: return percent = 100 * bytes_so_far // bytes_total if (percent != 100) and (time() - self.t0 < 0.5): return self.t0 = time() bar = '#' * (percent // 3) print('\r[{0:33}] {1:3}% {2}'.format(bar, percent, self.filename), end='\n' if (percent == 100) else '', file=sys.stderr) sys.stderr.flush()