Skip to content

Download

msg_download(start_date, end_date=None, start_time='00:00:00', end_time='23:59:00', daily_window_t0='00:00:00', daily_window_t1='23:59:00', time_step=None, satellite='MSG', instrument='HRSEVIRI', processing_level='L1', save_dir='.', eumdac_key='', eumdac_secret='')

Downloads MSG satellite data for a specified time period and set of bands.

Parameters:

Name Type Description Default
start_date str

The start date of the data download in the format 'YYYY-MM-DD'.

 required
end_date str

The end date of the data download in the format 'YYYY-MM-DD'. If not provided, the end date will be the same as the start date.

 None
start_time str

The start time of the data download in the format 'HH:MM:SS'. Default is '00:00:00'.

 '00:00:00'
end_time str

The end time of the data download in the format 'HH:MM:SS'. Default is '23:59:00'.

 '23:59:00'
daily_window_t0 str

The start time of the daily window in the format 'HH:MM:SS'. Default is '00:00:00'. Used if e.g., only day/night measurements are required.

 '00:00:00'
daily_window_t1 str

The end time of the daily window in the format 'HH:MM:SS'. Default is '23:59:00'. Used if e.g., only day/night measurements are required.

 '23:59:00'
time_step str

The time step between each data download in the format 'HH:MM:SS'. If not provided, the default is 1 hour.

 None
satellite str

The satellite. Default is MSG.

 'MSG'
instrument str

The data product to download. Default is 'HRSEVIRI', also implemented for Cloud Mask (CLM).

 'HRSEVIRI'
processing_level str

The processing level of the data. Default is 'L1'.

 'L1'
save_dir str

The directory where the downloaded files will be saved. Default is the current directory.

 '.'
eumdac_key str

The EUMETSAT Data Centre (EUMDAC) API key. If not provided, the user will be prompted to enter the key.

 ''
eumdac_secret str

The EUMETSAT Data Centre (EUMDAC) API secret. If not provided, the user will be prompted to enter the secret.

 ''

Returns:

Name Type Description
list

A list of file paths for the downloaded files.

Examples:

=========================

MSG LEVEL 1 Test Cases

=========================

custom day

python scripts/msg-download.py 2018-10-01 python scripts/msg-download.py 2018-10-01 --end-date 2018-10-01

custom day + end points

python scripts/msg-download.py 2018-10-01 --end-date 2018-10-05 python scripts/msg-download.py 2018-10-01 --end-date 2018-10-01 --start-time 09:00:00 --end-time 12:00:00 python scripts/msg-download.py 2018-10-01 --end-date 2018-10-05 --start-time 09:05:00 --end-time 12:05:00

custom day + end points + time window

scripts/msg-download.py 2018-10-01 --end-date 2018-10-01 --start-time 00:05:00 --end-time 23:54:00 --daily-window-t0 09:00:00 --daily-window-t1 12:00:00

custom day + end points + time window + timestep

python scripts/msg-download.py 2018-10-01 --end-date 2018-10-01 --start-time 00:05:00 --end-time 23:54:00 --daily-window-t0 09:00:00 --daily-window-t1 12:00:00 --time-step 00:15:00 python scripts/msg-download.py 2018-10-01 --end-date 2018-10-01 --start-time 00:05:00 --end-time 23:54:00 --daily-window-t0 09:00:00 --daily-window-t1 12:00:00 --time-step 00:25:00

===================================

MSG CLOUD MASK Test Cases

===================================

custom day

python scripts/msg-download.py 2018-10-01 --instrument=CLM

custom day + end points

python scripts/msg-download.py 2018-10-01 --end-date 2018-10-05 --instrument=CLM

custom day + end points + time window

python scripts/msg-download.py 2018-10-01 --end-date 2018-10-05 --start-time 09:00:00 --end-time 12:00:00 --instrument=CLM

custom day + end points + time window + timestep

python scripts/msg-download.py 2018-10-01 --end-date 2018-10-05 --start-time 09:00:00 --end-time 12:00:00 --time-step 00:25:00 --instrument=CLM

====================

FAILURE TEST CASES

====================

Source code in rs_tools/_src/data/msg/download.py 
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
def msg_download(
    start_date: str,
    end_date: Optional[str]=None,
    start_time: Optional[str]='00:00:00', # EUMDAC did not find any data for 00:00:00
    end_time: Optional[str]='23:59:00', # EUMDAC did not find any data for 23:59:00
    daily_window_t0: Optional[str]='00:00:00',
    daily_window_t1: Optional[str]='23:59:00',
    time_step: Optional[str]=None,
    satellite: str="MSG",
    instrument: str ="HRSEVIRI",
    processing_level: Optional[str] = "L1",
    save_dir: Optional[str] = ".",
    eumdac_key: Optional[str]="",
    eumdac_secret: Optional[str]="",
):
    """
    Downloads MSG satellite data for a specified time period and set of bands.

    Args:
        start_date (str): The start date of the data download in the format 'YYYY-MM-DD'.
        end_date (str, optional): The end date of the data download in the format 'YYYY-MM-DD'. If not provided, the end date will be the same as the start date.
        start_time (str, optional): The start time of the data download in the format 'HH:MM:SS'. Default is '00:00:00'.
        end_time (str, optional): The end time of the data download in the format 'HH:MM:SS'. Default is '23:59:00'.
        daily_window_t0 (str, optional): The start time of the daily window in the format 'HH:MM:SS'. Default is '00:00:00'. Used if e.g., only day/night measurements are required.
        daily_window_t1 (str, optional): The end time of the daily window in the format 'HH:MM:SS'. Default is '23:59:00'. Used if e.g., only day/night measurements are required.
        time_step (str, optional): The time step between each data download in the format 'HH:MM:SS'. If not provided, the default is 1 hour.
        satellite (str): The satellite. Default is MSG.
        instrument (str): The data product to download. Default is 'HRSEVIRI', also implemented for Cloud Mask (CLM).
        processing_level (str, optional): The processing level of the data. Default is 'L1'.
        save_dir (str, optional): The directory where the downloaded files will be saved. Default is the current directory.
        eumdac_key (str, optional): The EUMETSAT Data Centre (EUMDAC) API key. If not provided, the user will be prompted to enter the key.
        eumdac_secret (str, optional): The EUMETSAT Data Centre (EUMDAC) API secret. If not provided, the user will be prompted to enter the secret.

    Returns:
        list: A list of file paths for the downloaded files.

    Examples:
        # =========================
        # MSG LEVEL 1 Test Cases
        # =========================
        # custom day
        python scripts/msg-download.py 2018-10-01
        python scripts/msg-download.py 2018-10-01 --end-date 2018-10-01
        # custom day + end points
        python scripts/msg-download.py 2018-10-01 --end-date 2018-10-05
        python scripts/msg-download.py 2018-10-01 --end-date 2018-10-01 --start-time 09:00:00 --end-time 12:00:00
        python scripts/msg-download.py 2018-10-01 --end-date 2018-10-05 --start-time 09:05:00 --end-time 12:05:00
        # custom day + end points + time window
        scripts/msg-download.py 2018-10-01 --end-date 2018-10-01 --start-time 00:05:00 --end-time 23:54:00 --daily-window-t0 09:00:00 --daily-window-t1 12:00:00 
        # custom day + end points + time window + timestep
        python scripts/msg-download.py 2018-10-01 --end-date 2018-10-01 --start-time 00:05:00 --end-time 23:54:00 --daily-window-t0 09:00:00 --daily-window-t1 12:00:00 --time-step 00:15:00
        python scripts/msg-download.py 2018-10-01 --end-date 2018-10-01 --start-time 00:05:00 --end-time 23:54:00 --daily-window-t0 09:00:00 --daily-window-t1 12:00:00 --time-step 00:25:00
        # ===================================
        # MSG CLOUD MASK Test Cases
        # ===================================
        # custom day
        python scripts/msg-download.py 2018-10-01 --instrument=CLM
        # custom day + end points
        python scripts/msg-download.py 2018-10-01 --end-date 2018-10-05 --instrument=CLM 
        # custom day + end points + time window
        python scripts/msg-download.py 2018-10-01 --end-date 2018-10-05 --start-time 09:00:00 --end-time 12:00:00 --instrument=CLM 
        # custom day + end points + time window + timestep
        python scripts/msg-download.py 2018-10-01 --end-date 2018-10-05 --start-time 09:00:00 --end-time 12:00:00 --time-step 00:25:00 --instrument=CLM
        # ====================
        # FAILURE TEST CASES
        # ====================
    """

    # run checks
    # check if eumdac login is available
    token = _check_eumdac_login(eumdac_key=eumdac_key, eumdac_secret=eumdac_secret)
    datastore = eumdac.DataStore(token)

    # check if netcdf4 backend is available
    _check_netcdf4_backend()

    # check satellite details
    _check_input_processing_level(processing_level=processing_level)
    _check_instrument(instrument=instrument)
    # check data product
    data_product = f"EO:EUM:DAT:{satellite}:{instrument}"
    logger.info(f"Data Product: {data_product}")
    _check_data_product_name(data_product=data_product)

    # check start/end dates/times
    if end_date is None:
        end_date = start_date

    # combine date and time information
    start_datetime_str = start_date + ' ' + start_time
    end_datetime_str = end_date + ' ' + end_time
    _check_datetime_format(start_datetime_str, end_datetime_str)
    # datetime conversion 
    start_datetime = datetime.strptime(start_datetime_str, "%Y-%m-%d %H:%M:%S")
    end_datetime = datetime.strptime(end_datetime_str, "%Y-%m-%d %H:%M:%S")
    _check_start_end_times(start_datetime=start_datetime, end_datetime=end_datetime)

    # define time step for data query                       
    if time_step is None: 
        time_step = '1:00:00'
        logger.info("No timedelta specified. Default is 1 hour.")
    _check_timedelta_format(time_delta=time_step)

    # convert str to datetime object
    hours, minutes, seconds = convert_str2time(time=time_step)
    time_delta = timedelta(hours=hours, minutes=minutes, seconds=seconds)

    _check_timedelta(time_delta=time_delta)

    # Compile list of dates/times
    list_of_dates = np.arange(start_datetime, end_datetime, time_delta).astype(datetime)
    print('Times to check: ',list_of_dates[0], list_of_dates[-1])

    window_date = '1991-10-19' # Add arbitrary date to convert into proper datetime object
    start_datetime_window_str = window_date + ' ' + daily_window_t0
    end_datetime_window_str = window_date + ' ' + daily_window_t1
    _check_start_end_times(start_datetime=start_datetime, end_datetime=end_datetime)
    # datetime conversion 
    daily_window_t0_datetime = datetime.strptime(start_datetime_window_str, "%Y-%m-%d %H:%M:%S")
    daily_window_t1_datetime = datetime.strptime(end_datetime_window_str, "%Y-%m-%d %H:%M:%S")
    _check_start_end_times(start_datetime=daily_window_t0_datetime, end_datetime=daily_window_t1_datetime)

    # filter function - check that query times fall within desired time window
    def is_in_between(date):
       return daily_window_t0_datetime.time() <= date.time() <= daily_window_t1_datetime.time()

    # compile new list of dates within desired time window
    list_of_dates = list(filter(is_in_between, list_of_dates))

    # check if save_dir is valid before attempting to download
    _check_save_dir(save_dir=save_dir)

    files = []

    # create progress bars for dates and bands
    pbar_time = tqdm.tqdm(list_of_dates)

    for itime in pbar_time:

        pbar_time.set_description(f"Time - {itime}")

        sub_files_list = _download(time=itime, data_product=data_product, save_dir=save_dir, datastore=datastore)
        if sub_files_list is None:
            if itime != list_of_dates[-1]:
                logger.info(f"Could not find data for time {itime}. Trying to add 5 mins to timestamp.")
                time_delta = timedelta(hours=0, minutes=5, seconds=0)
                itime_5 = itime+ time_delta
                sub_files_list = _download(time=itime_5, data_product=data_product, save_dir=save_dir, datastore=datastore)
        if sub_files_list is None:
            if itime != list_of_dates[-1]:
                logger.info(f"Could not find data for time {itime}. Trying to add 10 mins to timestamp.")
                time_delta = timedelta(hours=0, minutes=10, seconds=0)
                itime_10 = itime+ time_delta
                sub_files_list = _download(time=itime_10, data_product=data_product, save_dir=save_dir, datastore=datastore)

        if sub_files_list is None:
            logger.info(f"Could not find data for time {itime}. Skipping to next time.")
        else:
            files += sub_files_list

    return files