Базовый интерфейс

Bases: BaseConnection

Implements generic methods for reading and writing dataframe as files.

.. versionadded:: 0.9.0

Source code in onetl/base/base_file_df_connection.py

class BaseFileDFConnection(BaseConnection):
    """
    Implements generic methods for reading  and writing dataframe as files.

    .. versionadded:: 0.9.0
    """

    spark: SparkSession

    @abstractmethod
    def check_if_format_supported(
        self,
        format: BaseReadableFileFormat | BaseWritableFileFormat,  # noqa: WPS125
    ) -> None:
        """
        Validate if specific file format is supported. |support_hooks|

        .. versionadded:: 0.9.0

        Raises
        ------
        RuntimeError
            If file format is not supported.
        """

    @abstractmethod
    def path_from_string(self, path: os.PathLike | str) -> PurePathProtocol:
        """
        Convert path from string to object. |support_hooks|

        .. versionadded:: 0.9.0
        """

    @property
    @abstractmethod
    def instance_url(self) -> str:
        """Instance URL.

        .. versionadded:: 0.9.0
        """

    @abstractmethod
    def read_files_as_df(
        self,
        paths: list[PurePathProtocol],
        format: BaseReadableFileFormat,  # noqa: WPS125
        root: PurePathProtocol | None = None,
        df_schema: StructType | None = None,
        options: FileDFReadOptions | None = None,
    ) -> DataFrame:
        """
        Read files in some paths list as dataframe. |support_hooks|

        .. versionadded:: 0.9.0
        """

    @abstractmethod
    def write_df_as_files(
        self,
        df: DataFrame,
        path: PurePathProtocol,
        format: BaseWritableFileFormat,  # noqa: WPS125
        options: FileDFWriteOptions | None = None,
    ) -> None:
        """
        Write dataframe as files in some path. |support_hooks|

        .. versionadded:: 0.9.0
        """

check_if_format_supported(format) abstractmethod

Validate if specific file format is supported. |support_hooks|

.. versionadded:: 0.9.0

Raises

RuntimeError If file format is not supported.

Source code in onetl/base/base_file_df_connection.py

@abstractmethod
def check_if_format_supported(
    self,
    format: BaseReadableFileFormat | BaseWritableFileFormat,  # noqa: WPS125
) -> None:
    """
    Validate if specific file format is supported. |support_hooks|

    .. versionadded:: 0.9.0

    Raises
    ------
    RuntimeError
        If file format is not supported.
    """

read_files_as_df(paths, format, root=None, df_schema=None, options=None) abstractmethod

Read files in some paths list as dataframe. |support_hooks|

.. versionadded:: 0.9.0

Source code in onetl/base/base_file_df_connection.py

@abstractmethod
def read_files_as_df(
    self,
    paths: list[PurePathProtocol],
    format: BaseReadableFileFormat,  # noqa: WPS125
    root: PurePathProtocol | None = None,
    df_schema: StructType | None = None,
    options: FileDFReadOptions | None = None,
) -> DataFrame:
    """
    Read files in some paths list as dataframe. |support_hooks|

    .. versionadded:: 0.9.0
    """

write_df_as_files(df, path, format, options=None) abstractmethod

Write dataframe as files in some path. |support_hooks|

.. versionadded:: 0.9.0

Source code in onetl/base/base_file_df_connection.py

@abstractmethod
def write_df_as_files(
    self,
    df: DataFrame,
    path: PurePathProtocol,
    format: BaseWritableFileFormat,  # noqa: WPS125
    options: FileDFWriteOptions | None = None,
) -> None:
    """
    Write dataframe as files in some path. |support_hooks|

    .. versionadded:: 0.9.0
    """