Skip to content

Базовый интерфейс

Bases: ABC

Representation of readable file format.

.. versionadded:: 0.9.0

Source code in onetl/base/base_file_format.py 
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
class BaseReadableFileFormat(ABC):
    """
    Representation of readable file format.

    .. versionadded:: 0.9.0
    """

    @abstractmethod
    def check_if_supported(self, spark: SparkSession) -> None:
        """
        Check if Spark session does support this file format. |support_hooks|

        .. versionadded:: 0.9.0

        Raises
        ------
        RuntimeError
            If file format is not supported.
        """

    @abstractmethod
    def apply_to_reader(self, reader: DataFrameReader) -> DataFrameReader:
        """
        Apply provided format to :obj:`pyspark.sql.DataFrameReader`. |support_hooks|

        .. versionadded:: 0.9.0

        Returns
        -------
        :obj:`pyspark.sql.DataFrameReader`
            DataFrameReader with options applied.
        """

check_if_supported(spark) abstractmethod

Check if Spark session does support this file format. |support_hooks|

.. versionadded:: 0.9.0

Raises

RuntimeError If file format is not supported.

Source code in onetl/base/base_file_format.py 
19
20
21
22
23
24
25
26
27
28
29
30
@abstractmethod
def check_if_supported(self, spark: SparkSession) -> None:
    """
    Check if Spark session does support this file format. |support_hooks|

    .. versionadded:: 0.9.0

    Raises
    ------
    RuntimeError
        If file format is not supported.
    """

apply_to_reader(reader) abstractmethod

Apply provided format to :obj:pyspark.sql.DataFrameReader. |support_hooks|

.. versionadded:: 0.9.0

Returns

:obj:pyspark.sql.DataFrameReader DataFrameReader with options applied.

Source code in onetl/base/base_file_format.py 
32
33
34
35
36
37
38
39
40
41
42
43
@abstractmethod
def apply_to_reader(self, reader: DataFrameReader) -> DataFrameReader:
    """
    Apply provided format to :obj:`pyspark.sql.DataFrameReader`. |support_hooks|

    .. versionadded:: 0.9.0

    Returns
    -------
    :obj:`pyspark.sql.DataFrameReader`
        DataFrameReader with options applied.
    """

Bases: ABC

Representation of writable file format.

.. versionadded:: 0.9.0

Source code in onetl/base/base_file_format.py 
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
class BaseWritableFileFormat(ABC):
    """
    Representation of writable file format.

    .. versionadded:: 0.9.0
    """

    @abstractmethod
    def check_if_supported(self, spark: SparkSession) -> None:
        """
        Check if Spark session does support this file format. |support_hooks|

        .. versionadded:: 0.9.0

        Raises
        ------
        RuntimeError
            If file format is not supported.
        """

    @abstractmethod
    def apply_to_writer(self, writer: DataFrameWriter) -> DataFrameWriter:
        """
        Apply provided format to :obj:`pyspark.sql.DataFrameWriter`. |support_hooks|

        .. versionadded:: 0.9.0

        Returns
        -------
        :obj:`pyspark.sql.DataFrameWriter`
            DataFrameWriter with options applied.
        """

check_if_supported(spark) abstractmethod

Check if Spark session does support this file format. |support_hooks|

.. versionadded:: 0.9.0

Raises

RuntimeError If file format is not supported.

Source code in onetl/base/base_file_format.py 
53
54
55
56
57
58
59
60
61
62
63
64
@abstractmethod
def check_if_supported(self, spark: SparkSession) -> None:
    """
    Check if Spark session does support this file format. |support_hooks|

    .. versionadded:: 0.9.0

    Raises
    ------
    RuntimeError
        If file format is not supported.
    """

apply_to_writer(writer) abstractmethod

Apply provided format to :obj:pyspark.sql.DataFrameWriter. |support_hooks|

.. versionadded:: 0.9.0

Returns

:obj:pyspark.sql.DataFrameWriter DataFrameWriter with options applied.

Source code in onetl/base/base_file_format.py 
66
67
68
69
70
71
72
73
74
75
76
77
@abstractmethod
def apply_to_writer(self, writer: DataFrameWriter) -> DataFrameWriter:
    """
    Apply provided format to :obj:`pyspark.sql.DataFrameWriter`. |support_hooks|

    .. versionadded:: 0.9.0

    Returns
    -------
    :obj:`pyspark.sql.DataFrameWriter`
        DataFrameWriter with options applied.
    """