# -*- coding: utf-8 -*-
# Copyright 2018, CS GROUP - France, https://www.csgroup.eu/
#
# This file is part of EODAG project
#     https://www.github.com/CS-SI/EODAG
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations

import logging
import os
import re
import shutil
import tarfile
import zipfile
from email.message import Message
from itertools import chain
from json import JSONDecodeError
from pathlib import Path
from typing import TYPE_CHECKING, Any, Iterator, Literal, Optional, Union, cast
from urllib.parse import parse_qs, urlparse

import geojson
import requests
from concurrent.futures import ThreadPoolExecutor, as_completed
from lxml import etree
from requests import RequestException
from requests.auth import AuthBase
from requests.structures import CaseInsensitiveDict
from typing_extensions import TypedDict
from zipstream import ZipStream

from eodag.api.product.metadata_mapping import (
    NOT_AVAILABLE,
    OFFLINE_STATUS,
    ONLINE_STATUS,
    STAGING_STATUS,
    mtd_cfg_as_conversion_and_querypath,
    properties_from_json,
    properties_from_xml,
)
from eodag.plugins.download.base import Download
from eodag.utils import (
    DEFAULT_DOWNLOAD_TIMEOUT,
    DEFAULT_DOWNLOAD_WAIT,
    DEFAULT_STREAM_REQUESTS_TIMEOUT,
    HTTP_REQ_TIMEOUT,
    USER_AGENT,
    ProgressCallback,
    StreamResponse,
    flatten_top_directories,
    guess_extension,
    guess_file_type,
    parse_header,
    path_to_uri,
    rename_with_version,
    sanitize,
    string_to_jsonpath,
    uri_to_path,
)
from eodag.utils.exceptions import (
    AuthenticationError,
    DownloadError,
    MisconfiguredError,
    NotAvailableError,
    QuotaExceededError,
    TimeOutError,
    ValidationError,
)

if TYPE_CHECKING:
    from jsonpath_ng import JSONPath
    from mypy_boto3_s3 import S3ServiceResource
    from requests import Response

    from eodag.api.product import Asset, EOProduct  # type: ignore
    from eodag.config import PluginConfig
    from eodag.types.download_args import DownloadConf
    from eodag.utils import Unpack

logger = logging.getLogger("eodag.download.http")


class HTTPDownload(Download):
    """HTTPDownload plugin. Handles product download over HTTP protocol

    :param provider: provider name
    :param config: Download plugin configuration:

        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): ``HTTPDownload``
        * :attr:`~eodag.config.PluginConfig.base_uri` (``str``): default endpoint url
        * :attr:`~eodag.config.PluginConfig.method` (``str``): HTTP request method for the download request (``GET`` or
          ``POST``); default: ``GET``
        * :attr:`~eodag.config.PluginConfig.extract` (``bool``): if the content of the downloaded file should be
          extracted; default: ``True``
        * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an
          authentication error
        * :attr:`~eodag.config.PluginConfig.dl_url_params` (``dict[str, Any]``): parameters to be
          added to the query params of the request
        * :attr:`~eodag.config.PluginConfig.archive_depth` (``int``): level in extracted path tree where to find data;
          default: ``1``
        * :attr:`~eodag.config.PluginConfig.flatten_top_dirs` (``bool``): if the directory structure should be
          flattened; default: ``True``
        * :attr:`~eodag.config.PluginConfig.ignore_assets` (``bool``): ignore assets and download using
          eodag:download_link;
          default: ``False``
        * :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds;
          default: ``5``
        * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in
          requests; default: ``True``
        * :attr:`~eodag.config.PluginConfig.no_auth_download` (``bool``): if the download should be done without
          authentication; default: ``True``
        * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to be ordered to download it;
          if this parameter is set to true, a mapping for ``eodag:order_link`` has to be added to the metadata mapping
          of the search plugin used for the provider; default: ``False``
        * :attr:`~eodag.config.PluginConfig.order_method` (``str``): HTTP request method for the order request (``GET``
          or ``POST``); default: ``GET``
        * :attr:`~eodag.config.PluginConfig.order_headers` (``[dict[str, str]]``): headers to be added to the order
          request
        * :attr:`~eodag.config.PluginConfig.order_on_response` (:class:`~eodag.config.PluginConfig.OrderOnResponse`):
          a typed dictionary containing the key ``metadata_mapping`` which can be used to add new product properties
          based on the data in response to the order request
        * :attr:`~eodag.config.PluginConfig.order_status` (:class:`~eodag.config.PluginConfig.OrderStatus`):
          configuration to handle the order status; contains information which method to use, how the response data is
          interpreted, which status corresponds to success, ordered and error and what should be done on success.
        * :attr:`~eodag.config.PluginConfig.products` (``dict[str, dict[str, Any]``): collection specific config; the
          keys are the collections, the values are dictionaries which can contain the key
          :attr:`~eodag.config.PluginConfig.extract` to overwrite the provider config for a specific collection

    """

    def __init__(self, provider: str, config: PluginConfig) -> None:
        super(HTTPDownload, self).__init__(provider, config)

    def _should_ignore_assets(self, product: EOProduct) -> bool:
        """Get ignore_assets value with product-level override support."""
        product_conf = getattr(self.config, "products", {}).get(product.collection, {})
        return product_conf.get(
            "ignore_assets", getattr(self.config, "ignore_assets", False)
        )

    def _order(
        self,
        product: EOProduct,
        auth: Optional[AuthBase] = None,
        **kwargs: Unpack[DownloadConf],
    ) -> Optional[dict[str, Any]]:
        """Send product order request.

        It will be executed once before the download retry loop, if the product is orderable
        and has ``eodag:order_link`` in its properties.
        Product ordering can be configured using the following download plugin parameters:

            - :attr:`~eodag.config.PluginConfig.order_enabled`: Wether order is enabled or not (may not use this method
              if no ``eodag:order_link`` exists)

            - :attr:`~eodag.config.PluginConfig.order_method`: (optional) HTTP request method, GET (default) or POST

            - :attr:`~eodag.config.PluginConfig.order_on_response`: (optional) things to do with obtained order
              response:

              - *metadata_mapping*: edit or add new product propoerties properties

        Product properties used for order:

            - **eodag:order_link**: order request URL

        :param product: The EO product to order
        :param auth: (optional) authenticated object
        :param kwargs: download additional kwargs
        :returns: the returned json status response
        """
        product.properties["order:status"] = STAGING_STATUS

        order_method = getattr(self.config, "order_method", "GET").upper()
        ssl_verify = getattr(self.config, "ssl_verify", True)
        timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
        OrderKwargs = TypedDict(
            "OrderKwargs", {"json": dict[str, Union[Any, list[str]]]}, total=False
        )
        order_kwargs: OrderKwargs = {}
        if order_method == "POST":
            # separate url & parameters
            parts = urlparse(str(product.properties["eodag:order_link"]))
            query_dict = {}
            # `parts.query` may be a JSON with query strings as one of values. If `parse_qs` is executed as first step,
            # the resulting `query_dict` would be erroneous.
            try:
                query_dict = geojson.loads(parts.query)
            except JSONDecodeError:
                try:
                    # due to the fact how metadata formatting works, there might be 2 {} around the string
                    query_dict = geojson.loads(parts.query[1:-1])
                except JSONDecodeError:
                    if parts.query:
                        query_dict = parse_qs(parts.query)
            order_url = parts._replace(query="").geturl()
            if query_dict:
                order_kwargs["json"] = query_dict
        else:
            order_url = product.properties["eodag:order_link"]
            order_kwargs = {}

        headers = {**getattr(self.config, "order_headers", {}), **USER_AGENT}
        try:
            with requests.request(
                method=order_method,
                url=order_url,
                auth=auth,
                timeout=timeout,
                headers=headers,
                verify=ssl_verify,
                **order_kwargs,
            ) as response:
                logger.debug(f"{order_method} {order_url} {headers} {order_kwargs}")
                try:
                    response.raise_for_status()
                    ordered_message = response.text
                    logger.debug(ordered_message)
                    product.properties["order:status"] = STAGING_STATUS
                except RequestException as e:
                    self._check_auth_exception(e)
                    QuotaExceededError.raise_if_quota_exceeded(e, self.provider)
                    msg = f"{product.properties['title']} could not be ordered"
                    if e.response is not None and e.response.status_code == 400:
                        raise ValidationError.from_error(e, msg) from e
                    else:
                        raise DownloadError.from_error(e, msg) from e

                return self.order_response_process(response, product)
        except requests.exceptions.Timeout as exc:
            raise TimeOutError(exc, timeout=timeout) from exc

    def order_response_process(
        self, response: Response, product: EOProduct
    ) -> Optional[dict[str, Any]]:
        """Process order response

        :param response: The order response
        :param product: The orderd EO product
        :returns: the returned json status response
        """
        on_response_mm = getattr(self.config, "order_on_response", {}).get(
            "metadata_mapping", {}
        )
        if not on_response_mm:
            return None

        logger.debug("Parsing order response to update product metada-mapping")
        on_response_mm_jsonpath = mtd_cfg_as_conversion_and_querypath(
            on_response_mm,
        )

        json_response = response.json()

        properties_update = properties_from_json(
            {"json": json_response, "headers": response.headers},
            on_response_mm_jsonpath,
        )
        product.properties.update(
            {k: v for k, v in properties_update.items() if v != NOT_AVAILABLE}
        )
        # the job id becomes the product id for EcmwfSearch products
        if "ORDERABLE" in product.properties.get("id", ""):
            product.properties["id"] = product.properties.get(
                "eodag:order_id", product.properties["id"]
            )
            product.properties["title"] = (
                (product.collection or product.provider).upper()
                + "_"
                + product.properties["id"]
            )
        if "eodag:download_link" in product.properties:
            product.remote_location = product.location = product.properties[
                "eodag:download_link"
            ]
            logger.debug(f"Product location updated to {product.location}")

        return json_response

    def _order_status(
        self,
        product: EOProduct,
        auth: Optional[AuthBase] = None,
    ) -> None:
        """Send product order status request.

        It will be executed before each download retry.
        Product order status request can be configured using the following download plugin parameters:

            - :attr:`~eodag.config.PluginConfig.order_status`: :class:`~eodag.config.PluginConfig.OrderStatus`

        Product properties used for order status:

            - **eodag:status_link**: order status request URL

        :param product: The ordered EO product
        :param auth: (optional) authenticated object
        :param kwargs: download additional kwargs
        """

        status_config = getattr(self.config, "order_status", {})
        success_code: Optional[int] = status_config.get("success", {}).get("http_code")

        timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)

        def _request(
            url: str,
            method: str = "GET",
            headers: Optional[dict[str, Any]] = None,
            json: Optional[Any] = None,
            timeout: int = HTTP_REQ_TIMEOUT,
        ) -> Response:
            """Send request and handle allow redirects"""

            logger.debug(f"{method} {url} {headers} {json}")
            try:
                response = requests.request(
                    method=method,
                    url=url,
                    auth=auth,
                    timeout=timeout,
                    headers={**(headers or {}), **USER_AGENT},
                    allow_redirects=False,  # Redirection is manually handled
                    json=json,
                )
                logger.debug(
                    f"Order download status request responded with {response.status_code}"
                )

                response.raise_for_status()  # Raise an exception if status code indicates an error

                # Handle redirection (if needed)
                if (
                    300 <= response.status_code < 400
                    and response.status_code != success_code
                ):
                    # cf: https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other
                    if response.status_code == 303:
                        method = "GET"
                    if new_url := response.headers.get("Location"):
                        return _request(new_url, method, headers, json, timeout)
                return response
            except requests.exceptions.Timeout as exc:
                raise TimeOutError(exc, timeout=timeout) from exc

        status_request: dict[str, Any] = status_config.get("request", {})
        status_request_method = str(status_request.get("method", "GET")).upper()

        if status_request_method == "POST":
            # separate url & parameters
            parts = urlparse(str(product.properties["eodag:status_link"]))
            status_url = parts._replace(query="").geturl()
            query_dict = parse_qs(parts.query)
            if not query_dict and parts.query:
                query_dict = geojson.loads(parts.query)
            json_data = query_dict if query_dict else None
        else:
            status_url = product.properties["eodag:status_link"]
            json_data = None

        # check header for success before full status request
        skip_parsing_status_response = False
        status_dict: dict[str, Any] = {}
        config_on_success: dict[str, Any] = status_config.get("on_success", {})
        on_success_mm = config_on_success.get("metadata_mapping", {})

        status_response_content_needed = (
            False
            if not any([v.startswith("$.json.") for v in on_success_mm.values()])
            else True
        )

        if success_code:
            try:
                response = _request(
                    status_url,
                    "HEAD",
                    status_request.get("headers"),
                    json_data,
                    timeout,
                )
                if (
                    response.status_code == success_code
                    and not status_response_content_needed
                ):
                    # success and no need to get status response content
                    skip_parsing_status_response = True
            except RequestException as e:
                logger.debug(e)

        if not skip_parsing_status_response:
            # status request
            try:
                response = _request(
                    status_url,
                    status_request_method,
                    status_request.get("headers"),
                    json_data,
                    timeout,
                )
                if (
                    response.status_code == success_code
                    and not status_response_content_needed
                ):
                    # success and no need to get status response content
                    skip_parsing_status_response = True
            except RequestException as e:
                msg = (
                    f"{product.properties.get('title') or product.properties.get('id') or product} "
                    "order status could not be checked"
                )
                if e.response is not None and e.response.status_code == 400:
                    raise ValidationError.from_error(e, msg) from e
                else:
                    raise DownloadError.from_error(e, msg) from e

        if not skip_parsing_status_response:
            # status request
            json_response = response.json()
            if not isinstance(json_response, dict):
                raise RequestException("response content is not a dict")
            status_dict = json_response

            status_mm = status_config.get("metadata_mapping", {})
            status_mm_jsonpath = (
                mtd_cfg_as_conversion_and_querypath(
                    status_mm,
                )
                if status_mm
                else {}
            )
            logger.debug("Parsing order status response")
            status_dict = properties_from_json(
                {"json": response.json(), "headers": response.headers},
                status_mm_jsonpath,
            )

            # display progress percentage
            if "eodag:order_percent" in status_dict:
                status_percent = str(status_dict["eodag:order_percent"])
                if status_percent.isdigit():
                    status_percent += "%"
                logger.info(
                    f"{product.properties['title']} order status: {status_percent}"
                )

            product.properties.update(
                {k: v for k, v in status_dict.items() if v != NOT_AVAILABLE}
            )

            order_status = status_dict.get("eodag:order_status")
            product.properties["eodag:order_status"] = order_status

            status_message = status_dict.get(
                "eodag:order_message", f"request status: {order_status}"
            )

            # handle status error
            errors: dict[str, Any] = status_config.get("error", {})
            if errors and errors.items() <= status_dict.items():
                raise DownloadError(
                    f"Provider {product.provider} returned: {status_dict.get('error_message', status_message)}"
                )

        product.properties["order:status"] = STAGING_STATUS

        success_status: dict[str, Any] = status_config.get("success", {}).get(
            "eodag:order_status"
        )
        # if not success
        if (
            success_status and success_status != status_dict.get("eodag:order_status")
        ) or (success_code and success_code != response.status_code):
            # Remove the download link if the order has not been completed or was not successful
            product.properties.pop("eodag:download_link", None)
            return None

        product.properties["order:status"] = ONLINE_STATUS

        if not config_on_success:
            # Nothing left to do
            return None

        # need search on success ?
        if config_on_success.get("need_search"):
            logger.debug(
                f"Search for new location: {product.properties['eodag:search_link']}"
            )
            try:
                response = _request(
                    product.properties["eodag:search_link"], timeout=timeout
                )
            except RequestException as e:
                logger.warning(
                    "%s order status could not be checked, request returned %s",
                    product.properties["title"],
                    e,
                )
                msg = f"{product.properties['title']} order status could not be checked"
                if e.response is not None and e.response.status_code == 400:
                    raise ValidationError.from_error(e, msg) from e
                else:
                    raise DownloadError.from_error(e, msg) from e

        result_type = config_on_success.get("result_type", "json")
        result_entry = config_on_success.get("results_entry")

        on_success_mm_querypath = (
            # append product.properties as input for on success response parsing
            mtd_cfg_as_conversion_and_querypath(on_success_mm)
            if on_success_mm
            else {}
        )
        try:
            if result_type == "xml":
                if not result_entry:
                    raise MisconfiguredError(
                        '"result_entry" is required with "result_type" "xml"'
                        'in "order_status.on_success"'
                    )
                root_node = etree.fromstring(response.content)
                namespaces = {k or "ns": v for k, v in root_node.nsmap.items()}
                results = [
                    etree.tostring(entry)
                    for entry in root_node.xpath(
                        result_entry,
                        namespaces=namespaces,
                    )
                ]
                if len(results) != 1:
                    raise DownloadError(
                        "Could not get a single result after order success for "
                        f"{product.properties['eodag:search_link']} request. "
                        f"Please search and download {product} again"
                    )
                assert isinstance(results, list), "results must be in a list"
                # single result
                result = results[0]
                if on_success_mm_querypath:
                    properties_update = properties_from_xml(
                        result,
                        on_success_mm_querypath,
                    )
                else:
                    properties_update = {}
            else:
                json_response = (
                    response.json()
                    if status_response_content_needed
                    and (
                        "application/json" in response.headers.get("Content-Type", "")
                        or "application/geo+json"
                        in response.headers.get("Content-Type", "")
                    )
                    else {}
                )
                if result_entry:
                    entry_jsonpath: JSONPath = string_to_jsonpath(
                        result_entry, force=True
                    )
                    json_response = entry_jsonpath.find(json_response)
                    raise NotImplementedError(
                        'result_entry in config.on_success is not yet supported for result_type "json"'
                    )
                if on_success_mm_querypath:
                    logger.debug(
                        "Parsing on-success metadata-mapping using order status response"
                    )
                    properties_update = properties_from_json(
                        {"json": json_response, "headers": response.headers},
                        on_success_mm_querypath,
                    )
                    # only keep properties to update (remove product.properties added for parsing)
                    properties_update = {
                        k: v for k, v in properties_update.items() if k in on_success_mm
                    }
                else:
                    properties_update = {}
        except Exception as e:
            if isinstance(e, DownloadError):
                raise
            logger.debug(e)
            raise DownloadError(
                f"Could not parse result after order success. Please search and download {product} again"
            ) from e

        # update product
        product.properties.update(properties_update)
        if "eodag:download_link" in properties_update:
            product.location = product.remote_location = product.properties[
                "eodag:download_link"
            ]
        else:
            self.order_response_process(response, product)

    def download(
        self,
        product: EOProduct,
        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
        progress_callback: Optional[ProgressCallback] = None,
        executor: Optional[ThreadPoolExecutor] = None,
        wait: float = DEFAULT_DOWNLOAD_WAIT,
        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
        **kwargs: Unpack[DownloadConf],
    ) -> Optional[str]:
        """Download a product using HTTP protocol.

        The downloaded product is assumed to be a Zip file. If it is not,
        the user is warned, it is renamed to remove the zip extension and
        no further treatment is done (no extraction)
        """
        if auth is not None and not isinstance(auth, AuthBase):
            raise MisconfiguredError(f"Incompatible auth plugin: {type(auth)}")

        if progress_callback is None:
            logger.info(
                "Progress bar unavailable, please call product.download() instead of plugin.download()"
            )
            progress_callback = ProgressCallback(disable=True)

        fs_path, record_filename = self._prepare_download(
            product,
            progress_callback=progress_callback,
            **kwargs,
        )
        if not fs_path or not record_filename:
            if fs_path:
                product.location = path_to_uri(fs_path)
            return fs_path

        # download assets if exist instead of remote_location
        if len(product.assets) > 0 and (
            not self._should_ignore_assets(product) or kwargs.get("asset") is not None
        ):
            try:
                fs_path = self._download_assets(
                    product,
                    fs_path,
                    record_filename,
                    auth,
                    progress_callback,
                    executor,
                    **kwargs,
                )
                if kwargs.get("asset") is None:
                    product.location = path_to_uri(fs_path)
                return fs_path
            except NotAvailableError as e:
                if kwargs.get("asset") is not None:
                    raise NotAvailableError(e).with_traceback(e.__traceback__)
                else:
                    pass

        url = product.remote_location

        @self._order_download_retry(product, wait, timeout)
        def download_request(
            product: EOProduct,
            auth: AuthBase,
            progress_callback: ProgressCallback,
            wait: float,
            timeout: float,
            **kwargs: Unpack[DownloadConf],
        ) -> os.PathLike:
            is_empty = True
            chunk_iterator = self._raw_stream_download(
                product, auth, progress_callback, **kwargs
            )
            if fs_path is not None:
                ext = Path(product.filename).suffix
                path = Path(fs_path).with_suffix(ext)
                if "ORDERABLE" in path.stem and product.properties.get("title"):
                    path = path.with_stem(sanitize(product.properties["title"]))

                with open(path, "wb") as fhandle:
                    for chunk in chunk_iterator:
                        is_empty = False
                        progress_callback(len(chunk))
                        fhandle.write(chunk)
                product._stream.close()  # Closing response stream

                if is_empty:
                    raise DownloadError(f"product {product.properties['id']} is empty")
                else:
                    # make sure storage status is online
                    product.properties["order:status"] = ONLINE_STATUS

                return path
            else:
                raise DownloadError(
                    f"download of product {product.properties['id']} failed"
                )

        path = download_request(
            product, auth, progress_callback, wait, timeout, **kwargs
        )

        with open(record_filename, "w") as fh:
            fh.write(url)
        logger.debug("Download recorded in %s", record_filename)

        if os.path.isfile(path) and not (
            zipfile.is_zipfile(path) or tarfile.is_tarfile(path)
        ):
            new_fs_path = os.path.join(
                os.path.dirname(path),
                sanitize(product.properties["title"]),
            )
            if os.path.isfile(new_fs_path):
                rename_with_version(new_fs_path)
            if not os.path.isdir(new_fs_path):
                os.makedirs(new_fs_path)
            shutil.move(path, new_fs_path)
            product.location = path_to_uri(new_fs_path)
            return new_fs_path

        product_path = self._finalize(
            str(path),
            progress_callback=progress_callback,
            **kwargs,
        )
        product.location = path_to_uri(product_path)
        return product_path

    def _check_stream_size(self, product: EOProduct) -> int:
        stream_size = int(product._stream.headers.get("content-length", 0))
        if (
            stream_size == 0
            and "order:status" in product.properties
            and product.properties["order:status"] != ONLINE_STATUS
        ):
            raise NotAvailableError(
                "%s(initially %s) ordered, got: %s"
                % (
                    product.properties["title"],
                    product.properties["order:status"],
                    product._stream.reason,
                )
            )
        return stream_size

    def _check_product_filename(self, product: EOProduct) -> str:
        filename = None
        asset_content_disposition = product._stream.headers.get("Content-Disposition")
        if asset_content_disposition:
            filename = cast(
                Optional[str],
                parse_header(asset_content_disposition).get_param("filename", None),
            )
        if not filename:
            # default filename extracted from path
            filename = str(os.path.basename(product._stream.url))

            # remove http get params and anchors
            filename = filename.split("?")[0]
            filename = filename.split("#")[0]

            filename_extension = os.path.splitext(filename)[1]
            if not filename_extension:
                if content_type := getattr(product, "headers", {}).get("Content-Type"):
                    ext = guess_extension(content_type)
                    if ext:
                        filename += ext
        return filename

    def stream_download(
        self,
        product: EOProduct,
        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
        byte_range: tuple[Optional[int], Optional[int]] = (None, None),
        compress: Literal["zip", "raw", "auto"] = "auto",
        wait: float = DEFAULT_DOWNLOAD_WAIT,
        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
        **kwargs: Unpack[DownloadConf],
    ) -> StreamResponse:
        r"""
        Returns dictionary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments.
        It contains a generator to streamed download chunks and the response headers.

        :param product: The EO product to download
        :param auth: (optional) authenticated object
        :param progress_callback: (optional) A progress callback
        :param wait: (optional) If download fails, wait time in minutes between two download tries
        :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
                        to download
        :param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
                        and `dl_url_params` (dict) can be provided as additional kwargs
                        and will override any other values defined in a configuration
                        file or with environment variables.
        :returns: Dictionary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments
        """
        if auth is not None and not isinstance(auth, AuthBase):
            raise MisconfiguredError(f"Incompatible auth plugin: {type(auth)}")

        # download assets if exist instead of remote_location
        if len(product.assets) > 0 and (
            not self._should_ignore_assets(product) or kwargs.get("asset") is not None
        ):
            executor = ThreadPoolExecutor(
                max_workers=getattr(self.config, "max_workers", None)
            )
            try:
                assets_values = product.assets.get_values(kwargs.get("asset") or "")
                with executor:
                    assets_stream_list = self._raw_stream_download_assets(
                        product,
                        executor,
                        auth,
                        None,
                        assets_values,
                        **kwargs,
                    )

                # single asset
                if len(assets_stream_list) == 1:
                    asset_stream = assets_stream_list[0]
                    if assets_values[0].get("type"):
                        asset_stream.headers["Content-Type"] = assets_values[0]["type"]
                    return asset_stream

                # multiple assets in zip
                else:
                    outputs_filename = (
                        sanitize(product.properties["title"])
                        if "title" in product.properties
                        else sanitize(product.properties.get("id", "download"))
                    )

                    # do not use global size if one of the assets has no size
                    missing_length = any(not (asset.size) for asset in assets_values)

                    zip_stream = (
                        ZipStream(sized=True) if not missing_length else ZipStream()
                    )
                    for asset_stream in assets_stream_list:
                        zip_stream.add(
                            asset_stream.content,
                            arcname=asset_stream.arcname,
                            size=asset_stream.size,
                        )

                    zip_length = len(zip_stream) if not missing_length else None

                    return StreamResponse(
                        content=zip_stream,
                        media_type="application/zip",
                        filename=f"{outputs_filename}.zip",
                        size=zip_length,
                    )
            except NotAvailableError as e:
                if kwargs.get("asset") is not None:
                    raise NotAvailableError(e).with_traceback(e.__traceback__)
                else:
                    pass

        chunk_iterator = self._raw_stream_download(product, auth, None, **kwargs)

        # start reading chunks to set product.headers
        try:
            first_chunk = next(chunk_iterator)
        except StopIteration:
            # product is empty file
            logger.error("product %s is empty", product.properties["id"])
            raise NotAvailableError(f"product {product.properties['id']} is empty")

        return StreamResponse(
            content=chain(iter([first_chunk]), chunk_iterator),
            headers=getattr(product, "headers", {}),
            filename=getattr(product, "filename", None),
            size=getattr(product, "size", None),
        )

    def _check_auth_exception(self, e: Optional[RequestException]) -> None:
        # check if error is identified as auth_error in provider conf
        auth_errors = getattr(self.config, "auth_error_code", [None])
        if not isinstance(auth_errors, list):
            auth_errors = [auth_errors]
        response_text = (
            e.response.text.strip() if e is not None and e.response is not None else ""
        )
        if (
            e is not None
            and e.response is not None
            and e.response.status_code in auth_errors
        ):
            raise AuthenticationError(
                f"Please check your credentials for {self.provider}.",
                f"HTTP Error {e.response.status_code} returned.",
                response_text,
            )

    def _process_exception(
        self, e: Optional[RequestException], product: EOProduct, ordered_message: str
    ) -> None:
        self._check_auth_exception(e)
        if e is not None:
            QuotaExceededError.raise_if_quota_exceeded(e, self.provider)
        response_text = (
            e.response.text.strip() if e is not None and e.response is not None else ""
        )
        # product not available
        if product.properties.get("order:status", ONLINE_STATUS) != ONLINE_STATUS:
            msg = (
                ordered_message
                if ordered_message and not response_text
                else response_text
            )

            raise NotAvailableError(
                "%s(initially %s) requested, returned: %s"
                % (
                    product.properties["title"],
                    product.properties["order:status"],
                    msg,
                )
            )
        else:
            import traceback as tb

            if e:
                logger.error(
                    "Error while getting resource :\n%s\n%s",
                    tb.format_exc(),
                    response_text,
                )
            else:
                logger.error("Error while getting resource :\n%s", tb.format_exc())

    def _order_request(
        self,
        product: EOProduct,
        auth: Optional[AuthBase],
    ) -> None:
        if (
            "eodag:order_link" in product.properties
            and product.properties.get("order:status") == OFFLINE_STATUS
            and not product.properties.get("eodag:order_status")
        ):
            self._order(product=product, auth=auth)

        if (
            product.properties.get("eodag:status_link")
            and product.properties.get("order:status") != ONLINE_STATUS
        ):
            self._order_status(product=product, auth=auth)

    def order(
        self,
        product: EOProduct,
        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
        wait: float = DEFAULT_DOWNLOAD_WAIT,
        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
    ) -> None:
        """
        Order product and poll to check its status

        :param product: The EO product to download
        :param auth: (optional) authenticated object
        :param wait: (optional) Wait time in minutes between two order status check
        :param timeout: (optional) Maximum time in minutes before stop checking
                        order status
        """
        self._order_download_retry(product, wait, timeout)(self._order_request)(
            product, auth
        )

    def _raw_stream_download(
        self,
        product: EOProduct,
        auth: Optional[AuthBase] = None,
        progress_callback: Optional[ProgressCallback] = None,
        **kwargs: Unpack[DownloadConf],
    ) -> Iterator[Any]:
        """
        Fetches a zip file containing the assets of a given product as a stream
        and returns a generator yielding the chunks of the file

        :param product: product for which the assets should be downloaded
        :param auth: The configuration of a plugin of type Authentication
        :param progress_callback: A method or a callable object
                                  which takes a current size and a maximum
                                  size as inputs and handle progress bar
                                  creation and update to give the user a
                                  feedback on the download progress
        :param kwargs: additional arguments
        """
        if progress_callback is None:
            logger.info("Progress bar unavailable, please call product.download()")
            progress_callback = ProgressCallback(disable=True)

        ssl_verify = getattr(self.config, "ssl_verify", True)

        ordered_message = ""
        # retry handled at download level
        self._order_request(product, auth)

        params = kwargs.pop("dl_url_params", None) or getattr(
            self.config, "dl_url_params", {}
        )

        req_method = (
            product.properties.get("eodag:download_method", "").lower()
            or getattr(self.config, "method", "GET").lower()
        )
        url = product.remote_location
        if req_method == "post":
            # separate url & parameters
            parts = urlparse(url)
            query_dict = parse_qs(parts.query)
            if not query_dict and parts.query:
                query_dict = geojson.loads(parts.query)
            req_url = parts._replace(query="").geturl()
            req_kwargs: dict[str, Any] = {"json": query_dict} if query_dict else {}
        else:
            req_url = url
            req_kwargs = {}

        if req_url.startswith(NOT_AVAILABLE):
            raise NotAvailableError("Download link is not available")

        if getattr(self.config, "no_auth_download", False):
            auth = None

        s = requests.Session()
        try:
            product._stream = s.request(
                req_method,
                req_url,
                stream=True,
                auth=auth,
                params=params,
                headers=USER_AGENT,
                timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
                verify=ssl_verify,
                **req_kwargs,
            )
        except requests.exceptions.MissingSchema:
            # location is not a valid url -> product is not available yet
            raise NotAvailableError("Product is not available yet")
        try:
            product._stream.raise_for_status()
        except requests.exceptions.Timeout as exc:
            raise TimeOutError(exc, timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT) from exc
        except RequestException as e:
            self._process_exception(e, product, ordered_message)
            raise DownloadError(
                f"download of {product.properties['id']} is empty"
            ) from e
        else:
            # check if product was ordered

            if getattr(
                product._stream, "status_code", None
            ) is not None and product._stream.status_code == getattr(
                self.config, "order_status", {}
            ).get(
                "ordered", {}
            ).get(
                "http_code"
            ):
                product.properties["order:status"] = "ORDERED"
                self._process_exception(None, product, ordered_message)
            stream_size = self._check_stream_size(product) or None

            product.headers = product._stream.headers
            filename = self._check_product_filename(product)
            content_type = product.headers.get("Content-Type")
            guessed_content_type = (
                guess_file_type(filename) if filename and not content_type else None
            )
            if guessed_content_type is not None:
                product.headers["Content-Type"] = guessed_content_type

            progress_callback.reset(total=stream_size)
            product.size = stream_size

            product.filename = filename
            return product._stream.iter_content(chunk_size=64 * 1024)

    def _raw_stream_download_assets(
        self,
        product: EOProduct,
        executor: ThreadPoolExecutor,
        auth: Optional[AuthBase] = None,
        progress_callback: Optional[ProgressCallback] = None,
        assets_values: list[Asset] = [],
        **kwargs: Unpack[DownloadConf],
    ) -> list[StreamResponse]:
        """Stream download assets as a zip file."""

        if progress_callback is None:
            logger.info("Progress bar unavailable, please call product.download()")
            progress_callback = ProgressCallback(disable=True)

        # get extra parameters to pass to the query
        params = kwargs.pop("dl_url_params", None) or getattr(
            self.config, "dl_url_params", {}
        )

        total_size = self._get_asset_sizes(assets_values, executor, auth, params)

        progress_callback.reset(total=total_size or None)

        # loop for assets paths and get common_subdir
        asset_rel_paths_list = []
        for asset in assets_values:
            asset_rel_path_parts = urlparse(asset["href"]).path.strip("/").split("/")
            asset_rel_path_parts_sanitized = [
                sanitize(part) for part in asset_rel_path_parts
            ]
            asset.rel_path = os.path.join(*asset_rel_path_parts_sanitized)
            asset_rel_paths_list.append(asset.rel_path)

        assets_common_subdir = ""
        if isinstance(asset_rel_paths_list, list) and len(asset_rel_paths_list) > 1:
            # If only one asset, common subdir must not == asset dir
            assets_common_subdir = os.path.commonpath(asset_rel_paths_list)

        # product conf overrides provider conf for "flatten_top_dirs"
        product_conf = getattr(self.config, "products", {}).get(product.collection, {})
        flatten_top_dirs = product_conf.get(
            "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True)
        )
        ssl_verify = getattr(self.config, "ssl_verify", True)
        matching_url = (
            getattr(product.downloader_auth.config, "matching_url", "")
            if product.downloader_auth
            else ""
        )
        matching_conf = (
            getattr(product.downloader_auth.config, "matching_conf", None)
            if product.downloader_auth
            else None
        )

        def get_chunks_generator(asset: Asset) -> Iterator[bytes]:
            """Create a generator function that will be called by ZipStream when needed."""

            asset_href = asset.get("href")
            # This function will be called by zipstream when it needs the data
            if not asset_href or asset_href.startswith("file:"):
                logger.info(f"Local asset detected. Download skipped for {asset_href}")
                return

            # Determine auth
            if matching_conf or (matching_url and re.match(matching_url, asset_href)):
                auth_object = auth
            else:
                auth_object = None

            # Make the request inside the generator
            try:
                with requests.get(
                    asset_href,
                    stream=True,
                    auth=auth_object,
                    params=params,
                    headers=USER_AGENT,
                    timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
                    verify=ssl_verify,
                ) as stream:
                    stream.raise_for_status()

                    # Process asset path
                    asset_rel_path = asset.rel_path
                    if flatten_top_dirs:
                        asset_rel_path = asset_rel_path.replace(
                            assets_common_subdir, ""
                        ).strip(os.sep)
                    asset_rel_dir = os.path.dirname(asset_rel_path)

                    if not getattr(asset, "filename", None):
                        # try getting filename in GET header if was not found in HEAD result
                        asset_content_disposition = stream.headers.get(
                            "Content-Disposition", None
                        )
                        if asset_content_disposition is not None:
                            asset.filename = cast(
                                Optional[str],
                                parse_header(asset_content_disposition).get_param(
                                    "filename", None
                                ),
                            )

                    if not getattr(asset, "filename", None):
                        # default filename extracted from path
                        asset.filename = os.path.basename(asset.rel_path)

                    asset.rel_path = os.path.join(
                        asset_rel_dir, cast(str, asset.filename)
                    )

                    for chunk in stream.iter_content(chunk_size=64 * 1024):
                        if chunk:
                            progress_callback(len(chunk))
                            yield chunk

            except requests.exceptions.Timeout as exc:
                raise TimeOutError(
                    exc, timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT
                ) from exc
            except RequestException as e:
                is_single_asset = len(assets_values) == 1
                self._handle_asset_exception(e, asset, is_single_asset)

        assets_stream_list = []

        # Process each asset
        for asset in assets_values:
            asset_chunks = get_chunks_generator(asset)
            try:
                # start reading chunks to set assets attributes
                first_chunk = next(asset_chunks)
                asset_chunks = chain(iter([first_chunk]), asset_chunks)
            except StopIteration:
                # Empty generator
                asset_chunks = iter([])

            assets_stream_list.append(
                StreamResponse(
                    content=asset_chunks,
                    filename=getattr(asset, "filename", None),
                    arcname=getattr(asset, "rel_path", None),
                    size=getattr(asset, "size", 0) or None,
                )
            )

        return assets_stream_list

    def _download_assets(
        self,
        product: EOProduct,
        fs_dir_path: str,
        record_filename: str,
        auth: Optional[AuthBase] = None,
        progress_callback: Optional[ProgressCallback] = None,
        executor: Optional[ThreadPoolExecutor] = None,
        **kwargs: Unpack[DownloadConf],
    ) -> str:
        """Download product assets if they exist"""
        if progress_callback is None:
            logger.info("Progress bar unavailable, please call product.download()")
            progress_callback = ProgressCallback(disable=True)

        # create an executor if not given and anticipate the possible need to shut it down
        executor, shutdown_executor = (
            (ThreadPoolExecutor(), True) if executor is None else (executor, False)
        )
        self._config_executor(executor)

        assets_urls = [
            a["href"] for a in getattr(product, "assets", {}).values() if "href" in a
        ]
        if not assets_urls:
            raise NotAvailableError("No assets available for %s" % product)

        assets_values = product.assets.get_values(kwargs.get("asset") or "")

        assets_stream_list = self._raw_stream_download_assets(
            product, executor, auth, progress_callback, assets_values, **kwargs
        )

        # remove existing incomplete file
        if os.path.isfile(fs_dir_path):
            os.remove(fs_dir_path)
        # create product dest dir
        if not os.path.isdir(fs_dir_path):
            os.makedirs(fs_dir_path)

        # product conf overrides provider conf for "flatten_top_dirs"
        product_conf = getattr(self.config, "products", {}).get(product.collection, {})
        flatten_top_dirs = product_conf.get(
            "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True)
        )

        # count local assets
        local_assets_count = 0
        for asset in assets_values:
            if asset["href"].startswith("file:"):
                local_assets_count += 1
                continue

        def download_asset(asset_stream: StreamResponse) -> None:
            asset_chunks = asset_stream.content
            asset_path = cast(str, asset_stream.arcname)
            asset_abs_path = os.path.join(fs_dir_path, asset_path)
            asset_abs_path_temp = asset_abs_path + "~"
            # create asset subdir if not exist
            asset_abs_path_dir = os.path.dirname(asset_abs_path)
            os.makedirs(asset_abs_path_dir, exist_ok=True)
            # remove temporary file
            if os.path.isfile(asset_abs_path_temp):
                os.remove(asset_abs_path_temp)
            if not os.path.isfile(asset_abs_path):
                logger.debug("Downloading to temporary file '%s'", asset_abs_path_temp)
                with open(asset_abs_path_temp, "wb") as fhandle:
                    for chunk in asset_chunks:
                        if chunk:
                            fhandle.write(chunk)
                logger.debug(
                    "Download completed. Renaming temporary file '%s' to '%s'",
                    os.path.basename(asset_abs_path_temp),
                    os.path.basename(asset_abs_path),
                )
                os.rename(asset_abs_path_temp, asset_abs_path)
            else:
                skipped_size = os.path.getsize(asset_abs_path)
                logger.debug(
                    "Asset already exists at '%s', skipping download", asset_abs_path
                )
                progress_callback(skipped_size)
            return

        # use parallelization if possible
        # when products are already downloaded in parallel but the executor has only one worker,
        # we avoid submitting nested tasks to the executor to prevent deadlocks
        if (
            executor._thread_name_prefix == "eodag-download-all"
            and executor._max_workers == 1
        ):
            for asset_stream in assets_stream_list:
                download_asset(asset_stream)
        else:
            futures = (
                executor.submit(download_asset, asset_stream)
                for asset_stream in assets_stream_list
            )
            [f.result() for f in as_completed(futures)]

        if shutdown_executor:
            executor.shutdown(wait=True)

        # only one local asset
        if local_assets_count == len(assets_urls) and local_assets_count == 1:
            # remove empty {fs_dir_path}
            shutil.rmtree(fs_dir_path)
            # and return assets_urls[0] path
            fs_dir_path = uri_to_path(assets_urls[0])
            # do not flatten dir
            flatten_top_dirs = False
        # several local assets
        elif local_assets_count == len(assets_urls) and local_assets_count > 0:
            common_path = os.path.commonpath([uri_to_path(uri) for uri in assets_urls])
            # remove empty {fs_dir_path}
            shutil.rmtree(fs_dir_path)
            # and return assets_urls common path
            fs_dir_path = common_path
            # do not flatten dir
            flatten_top_dirs = False
        # no assets downloaded but some should have been
        elif len(os.listdir(fs_dir_path)) == 0:
            raise NotAvailableError("No assets could be downloaded")

        # flatten directory structure
        if flatten_top_dirs:
            flatten_top_directories(fs_dir_path)

        if kwargs.get("asset") is None:
            # save hash/record file
            with open(record_filename, "w") as fh:
                fh.write(product.remote_location)
            logger.debug("Download recorded in %s", record_filename)

        return fs_dir_path

    def _handle_asset_exception(
        self, e: RequestException, asset: Asset, is_single_asset: bool
    ) -> None:
        # check if error is identified as auth_error or quota_exceeded in provider conf
        self._check_auth_exception(e)
        QuotaExceededError.raise_if_quota_exceeded(e, self.provider)
        logger.error("Unexpected error at download of asset %s: %s", asset["href"], e)
        if (
            e
            and e.response is not None
            and e.response.status_code == 404
            and is_single_asset
        ):
            raise NotAvailableError("Asset {asset['href'] not found.}")
        raise DownloadError(e)

    def _get_asset_sizes(
        self,
        assets_values: list[Asset],
        executor: ThreadPoolExecutor,
        auth: Optional[AuthBase],
        params: Optional[dict[str, str]],
        zipped: bool = False,
    ) -> int:

        # Try to get sizes from metadata first
        sizes_from_metadata = [
            int(asset["file:size"])
            for asset in assets_values
            if asset.get("file:size") is not None
        ]
        # If available, use the sum of sizes from metadata to avoid making HEAD requests for each asset
        if sizes_from_metadata:
            return sum(sizes_from_metadata)

        total_size = 0

        timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
        ssl_verify = getattr(self.config, "ssl_verify", True)

        # loop for assets size & filename in parallel
        def fetch_asset_size(asset: Asset) -> None:
            nonlocal total_size

            if asset["href"] and not asset["href"].startswith("file:"):
                # HEAD request for size & filename
                try:
                    asset_headers_resp = requests.head(
                        asset["href"],
                        auth=auth,
                        params=params,
                        headers=USER_AGENT,
                        timeout=timeout,
                        verify=ssl_verify,
                    )
                    asset_headers_resp.raise_for_status()
                    asset_headers = asset_headers_resp.headers
                except RequestException as e:
                    logger.debug(f"HEAD request failed: {str(e)}")
                    asset_headers = CaseInsensitiveDict()

                if not getattr(asset, "size", 0):
                    # size from HEAD header / Content-Length
                    asset.size = int(asset_headers.get("Content-Length", 0))

                header_content_disposition = Message()
                if not getattr(asset, "size", 0) or not getattr(asset, "filename", 0):
                    # header Content-Disposition
                    header_content_disposition = parse_header(
                        asset_headers.get("Content-Disposition", "")
                    )
                if not getattr(asset, "size", 0):
                    # size from HEAD header / Content-Disposition / size
                    size_str = str(header_content_disposition.get_param("size", 0))
                    asset.size = int(size_str) if size_str.isdigit() else 0
                if not getattr(asset, "filename", 0):
                    # filename from HEAD header / Content-Disposition / size
                    asset_filename = header_content_disposition.get_param(
                        "filename", None
                    )
                    asset.filename = str(asset_filename) if asset_filename else None

                if not getattr(asset, "size", 0):
                    # GET request for size
                    with requests.get(
                        asset["href"],
                        stream=True,
                        auth=auth,
                        params=params,
                        headers=USER_AGENT,
                        timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
                        verify=ssl_verify,
                    ) as stream:
                        # size from GET header / Content-length
                        asset.size = int(stream.headers.get("Content-Length", 0))
                        if not getattr(asset, "size", 0):
                            # size from GET header / Content-Disposition / size
                            size_str = str(
                                parse_header(
                                    stream.headers.get("Content-Disposition", "")
                                ).get_param("size", 0)
                            )
                            asset.size = int(size_str) if size_str.isdigit() else 0

                total_size += asset.size

        # use parallelization if possible
        # when products are already downloaded in parallel but the executor has only one worker,
        # we avoid submitting nested tasks to the executor to prevent deadlocks
        if (
            executor._thread_name_prefix == "eodag-download-all"
            and executor._max_workers == 1
        ):
            for asset in assets_values:
                fetch_asset_size(asset)
        else:
            futures = (
                executor.submit(fetch_asset_size, asset) for asset in assets_values
            )
            [f.result() for f in as_completed(futures)]

        return total_size


__all__ = ["HTTPDownload"]
