Bases: GenericOptions
File uploading options.
.. versionadded:: 0.3.0
Examples
.. code:: python
from onetl.file import FileUploader
options = FileUploader.Options(
if_exists="replace_entire_directory",
delete_local=True,
workers=4,
)
Source code in onetl/file/file_uploader/options.py
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85 | class FileUploaderOptions(GenericOptions):
"""File uploading options.
.. versionadded:: 0.3.0
Examples
--------
.. code:: python
from onetl.file import FileUploader
options = FileUploader.Options(
if_exists="replace_entire_directory",
delete_local=True,
workers=4,
)
"""
if_exists: FileExistBehavior = Field( # type: ignore[literal-required]
default=FileExistBehavior.ERROR,
alias=avoid_alias("mode"),
)
"""
How to handle existing files in the target directory.
Possible values:
* ``error`` (default) - mark file as failed
* ``ignore`` - mark file as skipped
* ``replace_file`` - replace existing file with a new one
* ``replace_entire_directory`` - delete local directory content before downloading files
.. versionchanged:: 0.9.0
Renamed ``mode`` → ``if_exists``
"""
delete_local: bool = False
"""
If ``True``, remove local file after successful download.
If download failed, file will left intact.
.. versionadded:: 0.2.0
.. versionchanged:: 0.3.0
Move ``FileUploader.delete_local`` to ``FileUploaderOptions``
"""
workers: int = Field(default=1, ge=1)
"""
Number of workers to create for parallel file upload.
1 (default) means files will me uploaded sequentially.
2 or more means files will be uploaded in parallel workers.
Recommended value is ``min(32, os.cpu_count() + 4)``, e.g. ``5``.
.. versionadded:: 0.8.1
"""
@root_validator(pre=True)
def _mode_is_deprecated(cls, values):
if "mode" in values:
warnings.warn(
"Option `FileUploader.Options(mode=...)` is deprecated since v0.9.0 and will be removed in v1.0.0. "
"Use `FileUploader.Options(if_exists=...)` instead",
category=UserWarning,
stacklevel=3,
)
return values
|