Skip to content

Download

goes_download(start_date, end_date=None, start_time='00:00:00', end_time='23:59:00', daily_window_t0='00:00:00', daily_window_t1='23:59:00', time_step=None, satellite_number=16, save_dir='.', instrument='ABI', processing_level='L1b', data_product='Rad', domain='F', bands='all', check_bands_downloaded=False)

Downloads GOES satellite data for a specified time period and set of bands.

Parameters:

Name Type Description Default
start_date str

The start date of the data download in the format 'YYYY-MM-DD'.

required
end_date str

The end date of the data download in the format 'YYYY-MM-DD'. If not provided, the end date will be the same as the start date.

None
start_time str

The start time of the data download in the format 'HH:MM:SS'. Default is '00:00:00'.

'00:00:00'
end_time str

The end time of the data download in the format 'HH:MM:SS'. Default is '23:59:00'.

'23:59:00'
daily_window_t0 str

The start time of the daily window in the format 'HH:MM:SS'. Default is '00:00:00'. Used if e.g., only day/night measurements are required.

'00:00:00'
daily_window_t1 str

The end time of the daily window in the format 'HH:MM:SS'. Default is '23:59:00'. Used if e.g., only day/night measurements are required.

'23:59:00'
time_step str

The time step between each data download in the format 'HH:MM:SS'. If not provided, the default is 1 hour.

None
satellite_number int

The satellite number. Default is 16.

16
save_dir str

The directory where the downloaded files will be saved. Default is the current directory.

'.'
instrument str

The instrument name. Default is 'ABI'.

'ABI'
processing_level str

The processing level of the data. Default is 'L1b'.

'L1b'
data_product str

The data product to download. Default is 'Rad'.

'Rad'
domain str

The domain of the data. Default is 'F' - Full Disk.

'F'
bands str

The bands to download. Default is 'all'.

'all'
check_bands_downloaded bool

Whether to check if all bands were successfully downloaded for each time step. Default is False.

False

Returns:

Name Type Description
list

A list of file paths for the downloaded files.

Examples:

=========================

GOES LEVEL 1B Test Cases

=========================

custom day

python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01

custom day + end points

python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00

custom day + end points + time window

python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00

custom day + end points + time window + timestep

python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00 --time-step 06:00:00

===================================

GOES LEVEL 2 CLOUD MASK Test Cases

===================================

python scripts/goes-download.py 2020-10-01 --start-time 10:00:00 --end-time 11:00:00 --processing-level L2 --data-product ACM

====================

FAILURE TEST CASES

====================

python scripts/goes-download.py 2018-10-01 --end-date 2018-10-01 --daily-window-t0 17:00:00 --daily-window-t1 17:14:00 --time-step 00:15:00 --save-dir /home/juanjohn/data/ python scripts/goes-download.py 2018-10-01 --end-date 2018-10-01 --daily-window-t0 17:00:00 --daily-window-t1 17:14:00 --time-step 00:15:00 --save-dir /home/juanjohn/data/ --check-bands-downloaded

Source code in rs_tools/_src/data/goes/download.py
def goes_download(
    start_date: str,
    end_date: Optional[str]=None,
    start_time: Optional[str]='00:00:00',
    end_time: Optional[str]='23:59:00',
    daily_window_t0: Optional[str]='00:00:00',
    daily_window_t1: Optional[str]='23:59:00',
    time_step: Optional[str]=None,
    satellite_number: int=16,
    save_dir: Optional[str] = ".",
    instrument: str = "ABI",
    processing_level: str = 'L1b',
    data_product: str = 'Rad',
    domain: str = 'F',
    bands: Optional[str] = "all",
    check_bands_downloaded: bool = False,
):
    """
    Downloads GOES satellite data for a specified time period and set of bands.

    Args:
        start_date (str): The start date of the data download in the format 'YYYY-MM-DD'.
        end_date (str, optional): The end date of the data download in the format 'YYYY-MM-DD'. If not provided, the end date will be the same as the start date.
        start_time (str, optional): The start time of the data download in the format 'HH:MM:SS'. Default is '00:00:00'.
        end_time (str, optional): The end time of the data download in the format 'HH:MM:SS'. Default is '23:59:00'.
        daily_window_t0 (str, optional): The start time of the daily window in the format 'HH:MM:SS'. Default is '00:00:00'. Used if e.g., only day/night measurements are required.
        daily_window_t1 (str, optional): The end time of the daily window in the format 'HH:MM:SS'. Default is '23:59:00'. Used if e.g., only day/night measurements are required.
        time_step (str, optional): The time step between each data download in the format 'HH:MM:SS'. If not provided, the default is 1 hour.
        satellite_number (int, optional): The satellite number. Default is 16.
        save_dir (str, optional): The directory where the downloaded files will be saved. Default is the current directory.
        instrument (str, optional): The instrument name. Default is 'ABI'.
        processing_level (str, optional): The processing level of the data. Default is 'L1b'.
        data_product (str, optional): The data product to download. Default is 'Rad'.
        domain (str, optional): The domain of the data. Default is 'F' - Full Disk.
        bands (str, optional): The bands to download. Default is 'all'.
        check_bands_downloaded (bool, optional): Whether to check if all bands were successfully downloaded for each time step. Default is False.

    Returns:
        list: A list of file paths for the downloaded files.

    Examples:
        # =========================
        # GOES LEVEL 1B Test Cases
        # =========================
        # custom day
        python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01
        # custom day + end points
        python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00
        # custom day + end points + time window
        python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00
        # custom day + end points + time window + timestep
        python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00 --time-step 06:00:00
        # ===================================
        # GOES LEVEL 2 CLOUD MASK Test Cases
        # ===================================
        python scripts/goes-download.py 2020-10-01 --start-time 10:00:00 --end-time 11:00:00 --processing-level L2 --data-product ACM

        # ====================
        # FAILURE TEST CASES
        # ====================
        python scripts/goes-download.py 2018-10-01 --end-date 2018-10-01 --daily-window-t0 17:00:00 --daily-window-t1 17:14:00 --time-step 00:15:00 --save-dir /home/juanjohn/data/
        python scripts/goes-download.py 2018-10-01 --end-date 2018-10-01 --daily-window-t0 17:00:00 --daily-window-t1 17:14:00 --time-step 00:15:00 --save-dir /home/juanjohn/data/ --check-bands-downloaded
    """

    # run checks
    # check satellite details
    _check_input_processing_level(processing_level=processing_level)
    _check_instrument(instrument=instrument)
    _check_satellite_number(satellite_number=satellite_number)
    logger.info(f"Satellite Number: {satellite_number}")
    _check_domain(domain=domain)
    # compile bands
    if processing_level == 'L1b':
        list_of_bands = _check_bands(bands=bands)
    elif processing_level == 'L2':
        list_of_bands = None
    else:
        raise ValueError('bands not correctly specified for given processing level')
    # check data product
    data_product = f"{instrument}-{processing_level}-{data_product}{domain}"
    logger.info(f"Data Product: {data_product}")
    _check_data_product_name(data_product=data_product)

    # check start/end dates/times
    if end_date is None:
        end_date = start_date

    # combine date and time information
    start_datetime_str = start_date + ' ' + start_time
    end_datetime_str = end_date + ' ' + end_time
    _check_datetime_format(start_datetime_str, end_datetime_str)
    # datetime conversion 
    start_datetime = datetime.strptime(start_datetime_str, "%Y-%m-%d %H:%M:%S")
    end_datetime = datetime.strptime(end_datetime_str, "%Y-%m-%d %H:%M:%S")
    _check_start_end_times(start_datetime=start_datetime, end_datetime=end_datetime)

    # define time step for data query                       
    if time_step is None: 
        time_step = '1:00:00'
        logger.info("No timedelta specified. Default is 1 hour.")
    _check_timedelta_format(time_delta=time_step)

    # convert str to datetime object
    hours, minutes, seconds = convert_str2time(time=time_step)
    time_delta = timedelta(hours=hours, minutes=minutes, seconds=seconds)

    _check_timedelta(time_delta=time_delta, domain=domain)

    # Compile list of dates/times
    list_of_dates = np.arange(start_datetime, end_datetime + time_delta, time_delta).astype(datetime)
    print('Times to check: ',list_of_dates[0], list_of_dates[-1])

    window_date = '1991-10-19' # Add arbitrary date to convert into proper datetime object
    start_datetime_window_str = window_date + ' ' + str(daily_window_t0)
    end_datetime_window_str = window_date + ' ' + str(daily_window_t1)
    _check_start_end_times(start_datetime=start_datetime, end_datetime=end_datetime)
    # datetime conversion 
    daily_window_t0_datetime = datetime.strptime(start_datetime_window_str, "%Y-%m-%d %H:%M:%S")
    daily_window_t1_datetime = datetime.strptime(end_datetime_window_str, "%Y-%m-%d %H:%M:%S")
    _check_start_end_times(start_datetime=daily_window_t0_datetime, end_datetime=daily_window_t1_datetime)

    # filter function - check that query times fall within desired time window
    def is_in_between(date):
       return daily_window_t0_datetime.time() <= date.time() <= daily_window_t1_datetime.time()

    # compile new list of dates within desired time window
    list_of_dates = list(filter(is_in_between, list_of_dates))
    if list_of_dates == []:
        msg = f"No times within the specified time window {daily_window_t0_datetime.time()} - {daily_window_t1_datetime.time()} for time step {time_step}"
        msg += f"\n adjust daily window times or time step"
        raise ValueError(msg)

    # check if save_dir is valid before attempting to download
    _check_save_dir(save_dir=save_dir)

    files = []

    # create progress bars for dates and bands
    pbar_time = tqdm.tqdm(list_of_dates)

    for itime in pbar_time:

        pbar_time.set_description(f"Time - {itime}")

        if processing_level == 'L1b':
            sub_files_list = _goes_level1_download(
                time=itime, 
                list_of_bands=list_of_bands,
                satellite_number=satellite_number,
                data_product=data_product,
                domain=domain,
                save_dir=save_dir,
                check_bands_downloaded=check_bands_downloaded,
                )
        elif processing_level == 'L2':
            sub_files_list = _goes_level2_download(
                time=itime, 
                satellite_number=satellite_number,
                data_product=data_product,
                domain=domain,
                save_dir=save_dir)
        else:
            raise ValueError(f"Unrecognized processing level: {processing_level}")

        files += sub_files_list

    return files