Skip to content

Download

goes_download(start_date, end_date=None, start_time='00:00:00', end_time='23:59:00', daily_window_t0='00:00:00', daily_window_t1='23:59:00', time_step=None, satellite_number=16, save_dir='.', instrument='ABI', processing_level='L1b', data_product='Rad', domain='F', bands='all', check_bands_downloaded=False)

Downloads GOES satellite data for a specified time period and set of bands.

Parameters:

Name Type Description Default
start_date str

The start date of the data download in the format 'YYYY-MM-DD'.

 required
end_date str

The end date of the data download in the format 'YYYY-MM-DD'. If not provided, the end date will be the same as the start date.

 None
start_time str

The start time of the data download in the format 'HH:MM:SS'. Default is '00:00:00'.

 '00:00:00'
end_time str

The end time of the data download in the format 'HH:MM:SS'. Default is '23:59:00'.

 '23:59:00'
daily_window_t0 str

The start time of the daily window in the format 'HH:MM:SS'. Default is '00:00:00'. Used if e.g., only day/night measurements are required.

 '00:00:00'
daily_window_t1 str

The end time of the daily window in the format 'HH:MM:SS'. Default is '23:59:00'. Used if e.g., only day/night measurements are required.

 '23:59:00'
time_step str

The time step between each data download in the format 'HH:MM:SS'. If not provided, the default is 1 hour.

 None
satellite_number int

The satellite number. Default is 16.

 16
save_dir str

The directory where the downloaded files will be saved. Default is the current directory.

 '.'
instrument str

The instrument name. Default is 'ABI'.

 'ABI'
processing_level str

The processing level of the data. Default is 'L1b'.

 'L1b'
data_product str

The data product to download. Default is 'Rad'.

 'Rad'
domain str

The domain of the data. Default is 'F' - Full Disk.

 'F'
bands str

The bands to download. Default is 'all'.

 'all'
check_bands_downloaded bool

Whether to check if all bands were successfully downloaded for each time step. Default is False.

 False

Returns:

Name Type Description
list

A list of file paths for the downloaded files.

Examples:

=========================

GOES LEVEL 1B Test Cases

=========================

custom day

python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01

custom day + end points

python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00

custom day + end points + time window

python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00

custom day + end points + time window + timestep

python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00 --time-step 06:00:00

===================================

GOES LEVEL 2 CLOUD MASK Test Cases

===================================

python scripts/goes-download.py 2020-10-01 --start-time 10:00:00 --end-time 11:00:00 --processing-level L2 --data-product ACM

====================

FAILURE TEST CASES

====================

python scripts/goes-download.py 2018-10-01 --end-date 2018-10-01 --daily-window-t0 17:00:00 --daily-window-t1 17:14:00 --time-step 00:15:00 --save-dir /home/juanjohn/data/ python scripts/goes-download.py 2018-10-01 --end-date 2018-10-01 --daily-window-t0 17:00:00 --daily-window-t1 17:14:00 --time-step 00:15:00 --save-dir /home/juanjohn/data/ --check-bands-downloaded

Source code in rs_tools/_src/data/goes/download.py 
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
def goes_download(
    start_date: str,
    end_date: Optional[str]=None,
    start_time: Optional[str]='00:00:00',
    end_time: Optional[str]='23:59:00',
    daily_window_t0: Optional[str]='00:00:00',
    daily_window_t1: Optional[str]='23:59:00',
    time_step: Optional[str]=None,
    satellite_number: int=16,
    save_dir: Optional[str] = ".",
    instrument: str = "ABI",
    processing_level: str = 'L1b',
    data_product: str = 'Rad',
    domain: str = 'F',
    bands: Optional[str] = "all",
    check_bands_downloaded: bool = False,
):
    """
    Downloads GOES satellite data for a specified time period and set of bands.

    Args:
        start_date (str): The start date of the data download in the format 'YYYY-MM-DD'.
        end_date (str, optional): The end date of the data download in the format 'YYYY-MM-DD'. If not provided, the end date will be the same as the start date.
        start_time (str, optional): The start time of the data download in the format 'HH:MM:SS'. Default is '00:00:00'.
        end_time (str, optional): The end time of the data download in the format 'HH:MM:SS'. Default is '23:59:00'.
        daily_window_t0 (str, optional): The start time of the daily window in the format 'HH:MM:SS'. Default is '00:00:00'. Used if e.g., only day/night measurements are required.
        daily_window_t1 (str, optional): The end time of the daily window in the format 'HH:MM:SS'. Default is '23:59:00'. Used if e.g., only day/night measurements are required.
        time_step (str, optional): The time step between each data download in the format 'HH:MM:SS'. If not provided, the default is 1 hour.
        satellite_number (int, optional): The satellite number. Default is 16.
        save_dir (str, optional): The directory where the downloaded files will be saved. Default is the current directory.
        instrument (str, optional): The instrument name. Default is 'ABI'.
        processing_level (str, optional): The processing level of the data. Default is 'L1b'.
        data_product (str, optional): The data product to download. Default is 'Rad'.
        domain (str, optional): The domain of the data. Default is 'F' - Full Disk.
        bands (str, optional): The bands to download. Default is 'all'.
        check_bands_downloaded (bool, optional): Whether to check if all bands were successfully downloaded for each time step. Default is False.

    Returns:
        list: A list of file paths for the downloaded files.

    Examples:
        # =========================
        # GOES LEVEL 1B Test Cases
        # =========================
        # custom day
        python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01
        # custom day + end points
        python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00
        # custom day + end points + time window
        python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00
        # custom day + end points + time window + timestep
        python scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00 --time-step 06:00:00
        # ===================================
        # GOES LEVEL 2 CLOUD MASK Test Cases
        # ===================================
        python scripts/goes-download.py 2020-10-01 --start-time 10:00:00 --end-time 11:00:00 --processing-level L2 --data-product ACM

        # ====================
        # FAILURE TEST CASES
        # ====================
        python scripts/goes-download.py 2018-10-01 --end-date 2018-10-01 --daily-window-t0 17:00:00 --daily-window-t1 17:14:00 --time-step 00:15:00 --save-dir /home/juanjohn/data/
        python scripts/goes-download.py 2018-10-01 --end-date 2018-10-01 --daily-window-t0 17:00:00 --daily-window-t1 17:14:00 --time-step 00:15:00 --save-dir /home/juanjohn/data/ --check-bands-downloaded
    """

    # run checks
    # check satellite details
    _check_input_processing_level(processing_level=processing_level)
    _check_instrument(instrument=instrument)
    _check_satellite_number(satellite_number=satellite_number)
    logger.info(f"Satellite Number: {satellite_number}")
    _check_domain(domain=domain)
    # compile bands
    if processing_level == 'L1b':
        list_of_bands = _check_bands(bands=bands)
    elif processing_level == 'L2':
        list_of_bands = None
    else:
        raise ValueError('bands not correctly specified for given processing level')
    # check data product
    data_product = f"{instrument}-{processing_level}-{data_product}{domain}"
    logger.info(f"Data Product: {data_product}")
    _check_data_product_name(data_product=data_product)

    # check start/end dates/times
    if end_date is None:
        end_date = start_date

    # combine date and time information
    start_datetime_str = start_date + ' ' + start_time
    end_datetime_str = end_date + ' ' + end_time
    _check_datetime_format(start_datetime_str, end_datetime_str)
    # datetime conversion 
    start_datetime = datetime.strptime(start_datetime_str, "%Y-%m-%d %H:%M:%S")
    end_datetime = datetime.strptime(end_datetime_str, "%Y-%m-%d %H:%M:%S")
    _check_start_end_times(start_datetime=start_datetime, end_datetime=end_datetime)

    # define time step for data query                       
    if time_step is None: 
        time_step = '1:00:00'
        logger.info("No timedelta specified. Default is 1 hour.")
    _check_timedelta_format(time_delta=time_step)

    # convert str to datetime object
    hours, minutes, seconds = convert_str2time(time=time_step)
    time_delta = timedelta(hours=hours, minutes=minutes, seconds=seconds)

    _check_timedelta(time_delta=time_delta, domain=domain)

    # Compile list of dates/times
    list_of_dates = np.arange(start_datetime, end_datetime + time_delta, time_delta).astype(datetime)
    print('Times to check: ',list_of_dates[0], list_of_dates[-1])

    window_date = '1991-10-19' # Add arbitrary date to convert into proper datetime object
    start_datetime_window_str = window_date + ' ' + str(daily_window_t0)
    end_datetime_window_str = window_date + ' ' + str(daily_window_t1)
    _check_start_end_times(start_datetime=start_datetime, end_datetime=end_datetime)
    # datetime conversion 
    daily_window_t0_datetime = datetime.strptime(start_datetime_window_str, "%Y-%m-%d %H:%M:%S")
    daily_window_t1_datetime = datetime.strptime(end_datetime_window_str, "%Y-%m-%d %H:%M:%S")
    _check_start_end_times(start_datetime=daily_window_t0_datetime, end_datetime=daily_window_t1_datetime)

    # filter function - check that query times fall within desired time window
    def is_in_between(date):
       return daily_window_t0_datetime.time() <= date.time() <= daily_window_t1_datetime.time()

    # compile new list of dates within desired time window
    list_of_dates = list(filter(is_in_between, list_of_dates))
    if list_of_dates == []:
        msg = f"No times within the specified time window {daily_window_t0_datetime.time()} - {daily_window_t1_datetime.time()} for time step {time_step}"
        msg += f"\n adjust daily window times or time step"
        raise ValueError(msg)

    # check if save_dir is valid before attempting to download
    _check_save_dir(save_dir=save_dir)

    files = []

    # create progress bars for dates and bands
    pbar_time = tqdm.tqdm(list_of_dates)

    for itime in pbar_time:

        pbar_time.set_description(f"Time - {itime}")

        if processing_level == 'L1b':
            sub_files_list = _goes_level1_download(
                time=itime, 
                list_of_bands=list_of_bands,
                satellite_number=satellite_number,
                data_product=data_product,
                domain=domain,
                save_dir=save_dir,
                check_bands_downloaded=check_bands_downloaded,
                )
        elif processing_level == 'L2':
            sub_files_list = _goes_level2_download(
                time=itime, 
                satellite_number=satellite_number,
                data_product=data_product,
                domain=domain,
                save_dir=save_dir)
        else:
            raise ValueError(f"Unrecognized processing level: {processing_level}")

        files += sub_files_list

    return files