EnMAP¶
This notebook requires the cloudsen12_models package
pip install georeader-spaceml cloudsen12_models fsspec
# cloudsen12_models is not a georeader dependency (it depends on georeader),
# so install it in the running kernel.
%pip install cloudsen12_models
import os
from pathlib import Path
from georeader.readers import enmap
from cloudsen12_models import cloudsen12
from georeader import plot
# Resolve the repo-level examples/ folder (works regardless of where the notebook runs from).
EXAMPLES_DIR = next(
(p / "examples" for p in [Path.cwd(), *Path.cwd().parents] if (p / "examples").is_dir()),
Path("examples"),
)
# EnMAP L1B tile used by this notebook. If it is not already in examples/,
# download it from the Azure container. Provide these as environment
# variables (or CI secrets); see examples/README.md.
SAS_TOKEN = os.environ.get("SAS_TOKEN", "")
AZURE_STORAGE_ACCOUNT = os.environ.get("AZURE_STORAGE_ACCOUNT", "")
CONTAINER = os.environ.get("CONTAINER_NAME", "")
tile = "ENMAP01-____L1B-DT0000074101_20240511T080843Z_001_V010402_20240514T093550Z"
folder_dest_tile = EXAMPLES_DIR / "EnMAP" / tile
xml_file = os.fspath(folder_dest_tile / f"{tile}-METADATA.XML")
PRODUCT_FOLDERS = {
"SPECTRAL_IMAGE_SWIR": "swir",
"SPECTRAL_IMAGE_VNIR": "vnir",
"QL_QUALITY_CLOUDSHADOW": "ql/cloudshadow",
"QL_PIXELMASK_SWIR": "ql/pixelmask_swir",
"QL_QUALITY_CIRRUS": "ql/cirrus",
"QL_QUALITY_SNOW": "ql/snow",
"QL_PIXELMASK_VNIR": "ql/pixelmask_vnir",
"QL_QUALITY_TESTFLAGS_SWIR": "ql/testflags_swir",
"QL_QUALITY_HAZE": "ql/haze",
"QL_QUALITY_TESTFLAGS_VNIR": "ql/testflags_vnir",
"QL_QUALITY_CLASSES": "ql/classes",
"QL_SWIR": "ql/swir",
"QL_QUALITY_CLOUD": "ql/cloud",
"QL_VNIR": "ql/vnir",
}
if not os.path.exists(xml_file):
import adlfs
fs = adlfs.AzureBlobFileSystem(
account_name=AZURE_STORAGE_ACCOUNT, sas_token=SAS_TOKEN, container=CONTAINER
)
os.makedirs(folder_dest_tile, exist_ok=True)
path_metadata = f"az://{CONTAINER}/EnMAP/metadata/{tile}.xml"
if not fs.exists(path_metadata):
raise FileNotFoundError(f"File {path_metadata} does not exist")
fs.get(path_metadata, xml_file)
for extension, folder in PRODUCT_FOLDERS.items():
path = f"az://{CONTAINER}/EnMAP/{folder}/{tile}.tif"
path_dst = os.fspath(folder_dest_tile / f"{tile}-{extension}.TIF")
if os.path.exists(path_dst):
continue
if not fs.exists(path):
print(f"File {path} not found, skipping")
continue
print(f"Downloading {path} ...")
fs.get(path, path_dst)
enmap_reader = enmap.EnMAP(xml_file, by_folder=False)
enmap_reader
%%time
rgb = enmap_reader.load_rgb(apply_rpcs=False)
plot.show(rgb)
from georeader.readers import S2_SAFE_reader
srf = S2_SAFE_reader.read_srf("S2A")
srf
%%time
# Load swir in enmap_reader.units (i.e. radiance units)
swir = enmap_reader.load_product("SPECTRAL_IMAGE_SWIR")
from georeader import reflectance
bands_s2_swir = ["B09","B10","B11","B12"]
s2bandsswir = reflectance.transform_to_srf(swir,
srf[bands_s2_swir],
wavelengths_hyperspectral=enmap_reader.wl_center["swir"],
as_reflectance=True,
observation_date_corr_factor=enmap_reader.observation_date_correction_factor,
verbose=True,
units=enmap_reader.units)
s2bandsswir
import matplotlib.pyplot as plt
fig, ax =plt.subplots(2,2,figsize=(12,5), tight_layout=True)
ax =ax.flatten()
for i,b in enumerate(bands_s2_swir):
ax[i].hist(s2bandsswir.values[i].ravel())
ax[i].set_title(b)
vnir = enmap_reader.load_product("SPECTRAL_IMAGE_VNIR")
bands_s2_vnir = ["B01","B02","B03","B04","B05","B06","B07","B08","B8A"]
s2bandsvnir = reflectance.transform_to_srf(vnir,
srf[bands_s2_vnir],
wavelengths_hyperspectral=enmap_reader.wl_center["vnir"],
as_reflectance=True,
observation_date_corr_factor=enmap_reader.observation_date_correction_factor,
verbose=True,
extrapolate=True,
units=enmap_reader.units)
s2bandsvnir
fig, ax =plt.subplots(4,2,figsize=(12,10), tight_layout=True)
ax =ax.flatten()
for i,b in enumerate(bands_s2_vnir[:-1]):
ax[i].hist(s2bandsvnir.values[i].ravel())
ax[i].set_title(b)
rgb_s2 = s2bandsvnir.isel({"band": [3,2,1]})
plot.show(rgb_s2)
from georeader.geotensor import GeoTensor
import numpy as np
s2_image = GeoTensor(np.concatenate([s2bandsvnir.values, s2bandsswir.values],axis=0),
transform=s2bandsswir.transform, crs=s2bandsswir.crs,
fill_value_default=s2bandsswir.fill_value_default)
s2_image
swir_nir_red = (s2_image.isel({"band": [S2_SAFE_reader.BANDS_S2_L1C.index(b) for b in ["B11", "B08", "B04"]]}) / .45).clip(0,1)
plot.show(swir_nir_red)
model_4bands = cloudsen12.load_model_by_name(name="dtacs4bands", weights_folder="cloudsen12_models")
cloudmask = model_4bands.predict(s2_image.isel({"band": [S2_SAFE_reader.BANDS_S2_L1C.index(b) for b in model_4bands.bands]}))
fig, ax = plt.subplots(1,2,figsize=(14,5),sharey=True, tight_layout=True)
plot.show(swir_nir_red,ax=ax[0])
cloudsen12.plot_cloudSEN12mask(cloudmask, ax=ax[1])
model = cloudsen12.load_model_by_name(name="UNetMobV2_V2", weights_folder="cloudsen12_models")
cloudmask = model.predict(s2_image)
fig, ax = plt.subplots(1,2,figsize=(14,5),sharey=True, tight_layout=True)
plot.show(swir_nir_red,ax=ax[0])
cloudsen12.plot_cloudSEN12mask(cloudmask, ax=ax[1])
Licence¶
The cloudsen12_models package is published under a GNU Lesser GPL v3 licence
The CloudSEN12 database and all pre-trained models are released under a Creative Commons non-commercial licence. For using the models in comercial pipelines written consent by the authors must be provided.
georeader tutorials and notebooks are released under a Creative Commons non-commercial licence.
If you find this work useful please cite:
@article{aybar_cloudsen12_2024,
title = {{CloudSEN12}+: {The} largest dataset of expert-labeled pixels for cloud and cloud shadow detection in {Sentinel}-2},
issn = {2352-3409},
url = {https://www.sciencedirect.com/science/article/pii/S2352340924008163},
doi = {10.1016/j.dib.2024.110852},
journal = {Data in Brief},
author = {Aybar, Cesar and Bautista, Lesly and Montero, David and Contreras, Julio and Ayala, Daryl and Prudencio, Fernando and Loja, Jhomira and Ysuhuaylas, Luis and Herrera, Fernando and Gonzales, Karen and Valladares, Jeanett and Flores, Lucy A. and Mamani, Evelin and Quiñonez, Maria and Fajardo, Rai and Espinoza, Wendy and Limas, Antonio and Yali, Roy and Alcántara, Alejandro and Leyva, Martin and Loayza-Muro, Rau´l and Willems, Bram and Mateo-GarcĂa, Gonzalo and GĂłmez-Chova, Luis},
month = aug,
year = {2024},
pages = {110852},
}