Source code for astroNN.gaia.downloader

# ---------------------------------------------------------#
#   astroNN.gaia.downloader: download gaia files
# ---------------------------------------------------------#

import os
import urllib.request

import numpy as np
from astropy.io import fits

import astroNN
import astroNN.data
from astroNN.gaia.gaia_shared import gaia_env, gaia_default_dr
from astroNN.shared.custom_warnings import deprecated
from astroNN.shared.downloader_tools import TqdmUpTo, filehash

currentdir = os.getcwd()


[docs]def tgas(flag=None): """ Get path to the Gaia TGAS DR1 files, download if files not found :return: List of file path :rtype: list :History: 2017-Oct-13 - Written - Henry Leung (University of Toronto) """ # Check if dr arguement is provided, if none then use default fulllist = [] # Check if directory exists folderpath = os.path.join(gaia_env(), 'Gaia/gdr1/tgas_source/fits/') urlbase = 'http://cdn.gea.esac.esa.int/Gaia/gdr1/tgas_source/fits/' if not os.path.exists(folderpath): os.makedirs(folderpath) hash_filename = 'MD5SUM.txt' full_hash_filename = os.path.join(folderpath, hash_filename) if not os.path.isfile(full_hash_filename): urllib.request.urlretrieve(urlbase + hash_filename, full_hash_filename) hash_list = np.loadtxt(full_hash_filename, dtype='str').T for i in range(0, 16, 1): filename = f'TgasSource_000-000-0{i:0{2}d}.fits' fullfilename = os.path.join(folderpath, filename) urlstr = urlbase + filename file_hash = (hash_list[0])[np.argwhere(hash_list[1] == filename)] # Check if files exists if os.path.isfile(fullfilename) and flag is None: checksum = filehash(fullfilename, algorithm='md5') # In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too if checksum != file_hash and len(file_hash) != 0: print(checksum) print(file_hash) print('File corruption detected, astroNN is attempting to download again') tgas(flag=1) else: print(fullfilename + ' was found!') elif not os.path.isfile(fullfilename) or flag == 1: # progress bar with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=urlstr.split('/')[-1]) as t: # Download urllib.request.urlretrieve(urlstr, fullfilename, reporthook=t.update_to) checksum = filehash(fullfilename, algorithm='md5') if checksum != file_hash and len(file_hash) != 0: print('File corruption detected, astroNN is attempting to download again') tgas(flag=1) print(f'Downloaded Gaia DR1 TGAS ({i:d} of 15) file catalog successfully to {fullfilename}') fulllist.extend([fullfilename]) return fulllist
[docs]def tgas_load(cuts=True): """ To load useful parameters from multiple TGAS DR1 files :param cuts: Whether to cut bad data (negative parallax and percentage error more than 20%, or a custom cut percentage) :type cuts: Union[boolean, 0.2] :return: Dictionary of parameters :rtype: dict :History: 2017-Dec-17 - Written - Henry Leung (University of Toronto) """ tgas_list = tgas() ra = np.array([]) dec = np.array([]) pmra_gaia = np.array([]) pmdec_gaia = np.array([]) parallax_gaia = np.array([]) parallax_error_gaia = np.array([]) g_band_gaia = np.array([]) for i in tgas_list: gaia = fits.open(i) ra = np.concatenate((ra, gaia[1].data['RA'])) dec = np.concatenate((dec, gaia[1].data['DEC'])) pmra_gaia = np.concatenate((pmra_gaia, gaia[1].data['PMRA'])) pmdec_gaia = np.concatenate((pmdec_gaia, gaia[1].data['PMDEC'])) parallax_gaia = np.concatenate((parallax_gaia, gaia[1].data['parallax'])) parallax_error_gaia = np.concatenate((parallax_error_gaia, gaia[1].data['parallax_error'])) g_band_gaia = np.concatenate((g_band_gaia, gaia[1].data['phot_g_mean_mag'])) gaia.close() if cuts is True or isinstance(cuts, float): filtered_index = [(parallax_error_gaia / parallax_gaia < (0.2 if isinstance(cuts, bool) else cuts)) & (parallax_gaia > 0.)] ra = ra[filtered_index] dec = dec[filtered_index] pmra_gaia = pmra_gaia[filtered_index] pmdec_gaia = pmdec_gaia[filtered_index] parallax_gaia = parallax_gaia[filtered_index] parallax_error_gaia = parallax_error_gaia[filtered_index] g_band_gaia = g_band_gaia[filtered_index] return {'ra': ra, 'dec': dec, 'pmra': pmra_gaia, 'pmdec': pmdec_gaia, 'parallax': parallax_gaia, 'parallax_err': parallax_error_gaia, 'gmag': g_band_gaia}
@deprecated def gaia_source(dr=None, flag=None): """ NAME: gaia_source PURPOSE: download the gaia_source files INPUT: dr (int): Gaia DR, example dr=1 flag (int): 0: normal, 1: force to re-download OUTPUT: list of file path HISTORY: 2017-Oct-13 - Written - Henry Leung (University of Toronto) 2017-Nov-26 - Update - Henry Leung (University of Toronto) """ dr = gaia_default_dr(dr=dr) fulllist = [] if dr == 1: # Check if directory exists folderpath = os.path.join(gaia_env(), 'Gaia/gdr1/gaia_source/fits/') urlbase = 'http://cdn.gea.esac.esa.int/Gaia/gdr1/gaia_source/fits/' if not os.path.exists(folderpath): os.makedirs(folderpath) hash_filename = 'MD5SUM.txt' full_hash_filename = os.path.join(folderpath, hash_filename) if not os.path.isfile(full_hash_filename): urllib.request.urlretrieve(urlbase + hash_filename, full_hash_filename) hash_list = np.loadtxt(full_hash_filename, dtype='str').T for j in range(0, 20, 1): for i in range(0, 256, 1): filename = f'GaiaSource_000-0{j:0{2}d}-{i:0{3}d}.fits' urlstr = urlbase + filename fullfilename = os.path.join(folderpath, filename) file_hash = (hash_list[0])[np.argwhere(hash_list[1] == filename)] # Check if files exists if os.path.isfile(fullfilename) and flag is None: checksum = filehash(fullfilename, algorithm='md5') # In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too if checksum != file_hash and len(file_hash) != 0: print(checksum) print(file_hash) print('File corruption detected, astroNN is attempting to download again') gaia_source(dr=dr, flag=1) else: print(fullfilename + ' was found!') elif not os.path.isfile(fullfilename) or flag == 1: # progress bar with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=urlstr.split('/')[-1]) as t: urllib.request.urlretrieve(urlstr, fullfilename, reporthook=t.update_to) checksum = filehash(fullfilename, algorithm='md5') if checksum != file_hash and len(file_hash) != 0: print('File corruption detected, astroNN is attempting to download again') gaia_source(dr=dr, flag=1) print(f'Downloaded Gaia DR{dr} Gaia Source ({(j * 256 + i):d} of {(256 * 20 + 112):d}) ' f'file catalog successfully to {fullfilename}') fulllist.extend([fullfilename]) for i in range(0, 111, 1): filename = f'GaiaSource_000-020-{i:0{3}d}.fits' urlstr = urlbase + filename fullfilename = os.path.join(folderpath, filename) file_hash = (hash_list[0])[np.argwhere(hash_list[1] == filename)] # Check if files exists if os.path.isfile(fullfilename) and flag is None: checksum = filehash(fullfilename, algorithm='md5') # In some rare case, the hash cant be found, so during checking, check len(file_has)!=0 too if checksum != file_hash and len(file_hash) != 0: print(checksum) print(file_hash) print('File corruption detected, astroNN is attempting to download again') gaia_source(dr=dr, flag=1) else: print(fullfilename + ' was found!') elif not os.path.isfile(fullfilename) or flag == 1: # progress bar with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=urlstr.split('/')[-1]) as t: urllib.request.urlretrieve(urlstr, fullfilename, reporthook=t.update_to) checksum = filehash(fullfilename, algorithm='md5') if checksum != file_hash and len(file_hash) != 0: print('File corruption detected, astroNN is attempting to download again') gaia_source(dr=dr, flag=1) print(f'Downloaded Gaia DR{dr} Gaia Source ({(20 * 256 + i):d} of {(256 * 20 + 112):d}) file ' f'catalog successfully to {fullfilename}') fulllist.extend([fullfilename]) else: raise ValueError('gaia_source() only supports Gaia DR1 Gaia Source') return fulllist @deprecated def anderson_2017_parallax(cuts=True): """ NAME: anderson_2017_parallax PURPOSE: load pre-compiled Anderson et al 2017 improved parallax from data-driven stars model INPUT: cuts (boolean): whether to cut those parallax err larger than 20% or not OUTPUT: ra (ndarray) dec (ndarray) parallax (ndarray): parallax in mas parallax_err (ndarray): 1-standard derivation parallax error in mas HISTORY: 2017-Dec-22 - Written - Henry Leung (University of Toronto) """ fullfilename = os.path.join(astroNN.data.datapath(), 'anderson_2017_dr14_parallax.npz') print('\nOriginal dataset at: http://voms.simonsfoundation.org:50013/8kM7XXPCJleK2M02B9E7YIYmvu5l2rh/ServedFiles/') print('Please be advised starting from 26 April 2018, anderson2017 in astroNN was reduced to parallax cross ' 'matched with APOGEE DR14 only') print('If you see this message, anderson2017 in this astroNN version is reduced. Moreover, anderson2017 will be ' 'removed in the future\n') hdu = np.load(fullfilename) ra = hdu['ra'] dec = hdu['dec'] parallax = hdu['parallax'] parallax_err = hdu['parallax_err'] if cuts is True: good_index = np.where(parallax_err / parallax < 0.2)[0] ra = ra[good_index] dec = dec[good_index] parallax = parallax[good_index] parallax_err = parallax_err[good_index] return ra, dec, parallax, parallax_err
[docs]@deprecated def gaiadr2_parallax(cuts=True, keepdims=False, offset=False): """ Load Gaia DR2 - APOGEE DR14 matches, indices corresponds to APOGEE allstar DR14 file :param cuts: Whether to cut bad data (negative parallax and percentage error more than 20%), or a float to set the threshold :type cuts: Union[boolean, float] :param keepdims: Whether to preserve indices the same as APOGEE allstar DR14, no effect when cuts=False, set to -9999 for bad indices when cuts=True keepdims=True :type keepdims: boolean :param offset: Whether to correction Gaia DR2 zero point offset - False to assume no offset correction - True to assume 52.8-4.21(G-12.2) - "leungbovy2019" for leung & bovy 2019 offset correction - a float to assume a float offset globally :type offset: Union[boolean, float, str] :return: numpy array of ra, dec, parallax, parallax_error :rtype: ndarrays :History: 2018-Apr-26 - Written - Henry Leung (University of Toronto) """ fullfilename = os.path.join(astroNN.data.datapath(), 'gaiadr2_apogeedr14_parallax.npz') print('This is Gaia DR2 - APOGEE DR14 matched parallax, RA DEC in J2015.5, parallax in mas') hdu = np.load(fullfilename) ra = np.array(hdu['RA']) dec = np.array(hdu['DEC']) parallax = np.array(hdu['parallax']) parallax_err = np.array(hdu['parallax_error']) gmag = np.array(hdu['g']) if (cuts is True or isinstance(cuts, float)) and keepdims is False: good_idx = ((parallax_err / parallax < (0.2 if cuts is True else cuts)) & (parallax > 0.)) ra = ra[good_idx] dec = dec[good_idx] parallax = parallax[good_idx] parallax_err = parallax_err[good_idx] gmag = gmag[good_idx] elif (cuts is True or isinstance(cuts, float)) and keepdims is True: print("Moreover, indices correspond to APOGEE allstar DR14 file") # Not magic_number because this should be apogee style bad_idx = ((parallax_err / parallax > (0.2 if cuts is True else cuts)) | (parallax < 0.)) parallax[bad_idx] = -9999. parallax_err[bad_idx] = -9999. else: # no cuts so do nothing pass if offset is True: parallax[parallax != -9999.] += (0.0528 - 0.0421 * (gmag[parallax != -9999.] - 12.2)) elif offset is False: pass elif isinstance(offset, float): parallax[parallax != -9999.] += offset elif offset == 'leungbovy2019': def bias(x): bias = 0.056 - 0.00574 * x - 0.0096 * x ** 2 return bias parallax[(parallax != -9999.) & (parallax < 2.)] += bias(parallax[(parallax != -9999.) & (parallax < 2.)]) else: raise ValueError("Unknown offset option") return ra, dec, parallax, parallax_err