Download

get_daily_window(daily_start, end_time)

computes tuple of start and end date/time for each day for earthaccess call

Source code in rs_tools/_src/data/modis/download.py

def get_daily_window(daily_start, end_time):
    """computes tuple of start and end date/time for each day for earthaccess call"""
    day = daily_start.strftime("%Y-%m-%d")
    daily_end = day + ' ' + end_time
    return (daily_start.strftime("%Y-%m-%d %H:%M:%S"), daily_end)

modis_download(start_date, end_date=None, start_time='00:00:00', end_time='23:59:00', day_step=1, satellite='Terra', save_dir='.', processing_level='L1b', resolution='1KM', bounding_box=(-180, -90, 180, 90), earthdata_username='', earthdata_password='', day_night_flag=None, identifier='02')

Downloads MODIS satellite data for a specified time period and location.

Parameters:

Name	Type	Description	Default
start_date	str	The start date of the data download in the format 'YYYY-MM-DD'.	required
end_date	str	The end date of the data download in the format 'YYYY-MM-DD'. If not provided, the end date will be the same as the start date.	None
start_time	str	The start time of the data download in the format 'HH:MM:SS'. Default is '00:00:00'.	'00:00:00'
end_time	str	The end time of the data download in the format 'HH:MM:SS'. Default is '23:59:00'.	'23:59:00'
day_step	int	The time step (in days) between downloads. This is to allow the user to download data every e.g. 2 days. If not provided, the default is daily downloads.	1
satellite	str	The satellite. Options are "Terra" and "Aqua", with "Terra" as default.	'Terra'
save_dir	str	The directory where the downloaded files will be saved. Default is the current directory.	'.'
processing_level	str	The processing level of the data. Default is 'L1b'.	'L1b'
resolution	str	The resolution of the data. Options are "QKM" (250m), "HKM (500m), "1KM" (1000m), with "1KM" as default. Not all bands are measured at all resolutions.	'1KM'
bounding_box	tuple	The region to be downloaded.	(-180, -90, 180, 90)
earthdata_username	str	Username associated with the NASA Earth Data login. Required for download.	''
earthdata_password	str	Password associated with the NASA Earth Data login. Required for download.	''
day_night_flag	str	The time of day for the data. Options are "day" and "night". If not provided, both day and night data will be downloaded.	None
identifier	str	The MODIS data product identifier. Options are "02" and "35". Default is "02".	'02'

Returns: list: A list of file paths for the downloaded files.

Examples:

=========================

MODIS LEVEL 1B Test Cases

=========================

one day - successfully downloaded 4 granules (all nighttime)

python scripts/modis-download.py 2018-10-01 --start-time 08:00:00 --end-time 8:10:00 --save-dir ./notebooks/modisdata/test_script/

multiple days - finds 62 granules, stopped download for times sake but seemed to work

python scripts/modis-download.py 2018-10-01 --end-date 2018-10-9 --day-step 3 --start-time 08:00:00 --end-time 13:00:00 --save-dir ./notebooks/modisdata/test_script/

test bounding box - successfully downloaded 4 files (all daytime)

python scripts/modis-download.py 2018-10-01 --start-time 08:00:00 --end-time 13:00:00 --save-dir ./notebooks/modisdata/test_script/ --bounding-box -10 -10 20 5

test day/night flag - successfully downloaded 1 file (daytime only)

python scripts/modis-download.py 2018-10-15 --save-dir ./notebooks/modisdata/test_script/ --bounding-box -10 10 -5 15 --day-night-flag day

=========================

MODIS LEVEL 2 CLOUD MASK Test Cases

=========================

one day - successfully downloaded 4 granules (all nighttime)

python scripts/modis-download.py 2018-10-01 --start-time 08:00:00 --end-time 8:10:00 --save-dir ./notebooks/modisdata/ --processing-level L2 --identifier 35

====================

FAILURE TEST CASES

====================

bounding box input invalid - throws error as expected

python scripts/modis-download.py 2018-10-01 --bounding-box a b c d

end date before start date - throws error as expected

python scripts/modis-download.py 2018-10-01 --end-date 2018-09-01

empty results - warns user as expected

python scripts/modis-download.py 2018-10-01 --start-time 07:00:00 --end-time 7:10:00 --save-dir ./notebooks/modisdata/test_script/ --bounding-box -10 -10 -5 -5

Source code in rs_tools/_src/data/modis/download.py

def modis_download(
    start_date: str,
    end_date: Optional[str]=None,
    start_time: Optional[str]='00:00:00', # used for daily window
    end_time: Optional[str]='23:59:00', # used for daily window
    day_step: Optional[int]=1, 
    satellite: str='Terra',
    save_dir: Optional[str]=".",
    processing_level: str = 'L1b',
    resolution: str = "1KM",
    bounding_box: Optional[tuple[float, float, float, float]]=(-180, -90, 180, 90), # TODO: Add polygon option
    earthdata_username: Optional[str]="",
    earthdata_password: Optional[str]="",
    day_night_flag: Optional[str]=None, 
    identifier: Optional[str] = "02"
):
    """
    Downloads MODIS satellite data for a specified time period and location.

    Args:        
        start_date (str): The start date of the data download in the format 'YYYY-MM-DD'.
        end_date (str, optional): The end date of the data download in the format 'YYYY-MM-DD'. If not provided, the end date will be the same as the start date.
        start_time (str, optional): The start time of the data download in the format 'HH:MM:SS'. Default is '00:00:00'.
        end_time (str, optional): The end time of the data download in the format 'HH:MM:SS'. Default is '23:59:00'.
        day_step (int, optional): The time step (in days) between downloads. This is to allow the user to download data every e.g. 2 days. If not provided, the default is daily downloads.
        satellite (str, optional): The satellite. Options are "Terra" and "Aqua", with "Terra" as default.
        save_dir (str, optional): The directory where the downloaded files will be saved. Default is the current directory.
        processing_level (str, optional): The processing level of the data. Default is 'L1b'.
        resolution (str, optional): The resolution of the data. Options are "QKM" (250m), "HKM (500m), "1KM" (1000m), with "1KM" as default. Not all bands are measured at all resolutions.
        bounding_box (tuple, optional): The region to be downloaded.
        earthdata_username (str): Username associated with the NASA Earth Data login. Required for download.
        earthdata_password (str): Password associated with the NASA Earth Data login. Required for download.
        day_night_flag (str, optional): The time of day for the data. Options are "day" and "night". If not provided, both day and night data will be downloaded.
        identifier (str, optional): The MODIS data product identifier. Options are "02" and "35". Default is "02".
    Returns:
        list: A list of file paths for the downloaded files.

    Examples:
    # =========================
    # MODIS LEVEL 1B Test Cases
    # =========================
    # one day - successfully downloaded 4 granules (all nighttime)
    python scripts/modis-download.py 2018-10-01 --start-time 08:00:00 --end-time 8:10:00 --save-dir ./notebooks/modisdata/test_script/

    # multiple days - finds 62 granules, stopped download for times sake but seemed to work
    python scripts/modis-download.py 2018-10-01 --end-date 2018-10-9 --day-step 3 --start-time 08:00:00 --end-time 13:00:00 --save-dir ./notebooks/modisdata/test_script/

    # test bounding box - successfully downloaded 4 files (all daytime)
    python scripts/modis-download.py 2018-10-01 --start-time 08:00:00 --end-time 13:00:00 --save-dir ./notebooks/modisdata/test_script/ --bounding-box -10 -10 20 5

    # test day/night flag - successfully downloaded 1 file (daytime only)
    python scripts/modis-download.py 2018-10-15 --save-dir ./notebooks/modisdata/test_script/ --bounding-box -10 10 -5 15 --day-night-flag day

    # =========================
    # MODIS LEVEL 2 CLOUD MASK Test Cases
    # =========================

    # one day - successfully downloaded 4 granules (all nighttime)
    python scripts/modis-download.py 2018-10-01 --start-time 08:00:00 --end-time 8:10:00 --save-dir ./notebooks/modisdata/ --processing-level L2 --identifier 35

    # ====================
    # FAILURE TEST CASES
    # ====================
    # bounding box input invalid - throws error as expected
    python scripts/modis-download.py 2018-10-01 --bounding-box a b c d

    # end date before start date - throws error as expected
    python scripts/modis-download.py 2018-10-01  --end-date 2018-09-01 

    # empty results - warns user as expected
    python scripts/modis-download.py 2018-10-01 --start-time 07:00:00 --end-time 7:10:00 --save-dir ./notebooks/modisdata/test_script/ --bounding-box -10 -10 -5 -5

    """
    # check if earthdata login is available
    _check_earthdata_login(earthdata_username=earthdata_username, earthdata_password=earthdata_password)

    # check if netcdf4 backend is available
    _check_netcdf4_backend()

    # run checks
    # translate str inputs to modis specific names
    _check_input_processing_level(processing_level=processing_level)
    _check_identifier(identifier=identifier)
    satellite_code = _check_satellite(satellite=satellite)
    resolution_code = _check_resolution(resolution=resolution)
    logger.info(f"Satellite: {satellite}")
    # check data product
    if processing_level == 'L1b':
        data_product = f"{satellite_code}{identifier}{resolution_code}"
    elif processing_level == 'L2':
        # TODO: Implement other level-2 products or allow passing in data_product?
        # NOTE: Resolution argument not needed for cloud mask download
        data_product = f"{satellite_code}{identifier}_{processing_level}"
    else:
        raise ValueError("Incorrect processing level, downloader only implemented for 'L1b' and 'L2'")

    logger.info(f"Data Product: {data_product}")
    _check_data_product_name(data_product=data_product)

    # check start/end dates/times
    if end_date is None:
        end_date = start_date

    # combine date and time information
    start_datetime_str = start_date + ' ' + start_time
    end_datetime_str = end_date + ' ' + end_time
    _check_datetime_format(start_datetime_str=start_datetime_str, end_datetime_str=end_datetime_str) 
    # datetime conversion
    start_datetime = datetime.strptime(start_datetime_str, "%Y-%m-%d %H:%M:%S")
    end_datetime = datetime.strptime(end_datetime_str, "%Y-%m-%d %H:%M:%S")
    _check_start_end_dates(start_datetime=start_datetime, end_datetime=end_datetime)

    # compile list of dates/times 
    day_delta = timedelta(days=day_step)
    list_of_dates = np.arange(start_datetime, end_datetime, day_delta).astype(datetime)

    list_of_daily_windows = [get_daily_window(daily_start, end_time) for daily_start in list_of_dates]

    # check if save_dir is valid before attempting to download
    _check_save_dir(save_dir=save_dir)

    # check that bounding box is valid
    # TODO: Add option to add multiple location requests
    # NOTE: earthaccess allows other ways to specify spatial extent, e.g. polygon, point 
    # NOTE: extend to allow these options
    _check_bounding_box(bounding_box=bounding_box)

    # create dictionary of earthaccess search parameters
    search_params = {
        "short_name": data_product,
        "bounding_box": bounding_box,
    }

    # if day_night_flag was provided, check that day_night_flag is valid
    if day_night_flag: 
        _check_day_night_flag(day_night_flag=day_night_flag)
        # add day_night_flag to search parameters
        search_params["day_night_flag"] = day_night_flag

    # TODO: remove - logging search_params for testing
    logger.info(f"Search parameters: {search_params}")

    files = []

    # create progress bar for dates
    pbar_time = tqdm.tqdm(list_of_daily_windows)

    for itime in pbar_time:
        pbar_time.set_description(f"Time - {itime[0]} to {itime[1]}")
        success_flag = True

        # add daytime window to search parameters
        search_params["temporal"] = itime

        # search for data
        results_day = earthaccess.search_data(**search_params)

        # check if any results were returned
        if not results_day:
            # if not: log warning and continue to next date
            success_flag = False
            warn = f"No data found for {itime[0]} to {itime[1]} in the specified bounding box"
            if day_night_flag: warn += f" for {day_night_flag}-time measurements only"
            logger.warning(warn)
            continue

        files_day = earthaccess.download(results_day, save_dir) 
        # TODO: can this fail? if yes, use try / except to prevent the programme from crashing
        # TODO: check file sizes - if less than X MB (ca 70MB) the download failed
        if success_flag:
            files += files_day

    return files