structure saas with tools
This commit is contained in:
Davidson Gomes
307
.venv/lib/python3.10/site-packages/huggingface_hub/utils/tqdm.py
Normal file
307
.venv/lib/python3.10/site-packages/huggingface_hub/utils/tqdm.py
Normal file
@@ -0,0 +1,307 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License
|
||||
"""Utility helpers to handle progress bars in `huggingface_hub`.
|
||||
|
||||
Example:
|
||||
1. Use `huggingface_hub.utils.tqdm` as you would use `tqdm.tqdm` or `tqdm.auto.tqdm`.
|
||||
2. To disable progress bars, either use `disable_progress_bars()` helper or set the
|
||||
environment variable `HF_HUB_DISABLE_PROGRESS_BARS` to 1.
|
||||
3. To re-enable progress bars, use `enable_progress_bars()`.
|
||||
4. To check whether progress bars are disabled, use `are_progress_bars_disabled()`.
|
||||
|
||||
NOTE: Environment variable `HF_HUB_DISABLE_PROGRESS_BARS` has the priority.
|
||||
|
||||
Example:
|
||||
```py
|
||||
>>> from huggingface_hub.utils import are_progress_bars_disabled, disable_progress_bars, enable_progress_bars, tqdm
|
||||
|
||||
# Disable progress bars globally
|
||||
>>> disable_progress_bars()
|
||||
|
||||
# Use as normal `tqdm`
|
||||
>>> for _ in tqdm(range(5)):
|
||||
... pass
|
||||
|
||||
# Still not showing progress bars, as `disable=False` is overwritten to `True`.
|
||||
>>> for _ in tqdm(range(5), disable=False):
|
||||
... pass
|
||||
|
||||
>>> are_progress_bars_disabled()
|
||||
True
|
||||
|
||||
# Re-enable progress bars globally
|
||||
>>> enable_progress_bars()
|
||||
|
||||
# Progress bar will be shown !
|
||||
>>> for _ in tqdm(range(5)):
|
||||
... pass
|
||||
100%|███████████████████████████████████████| 5/5 [00:00<00:00, 117817.53it/s]
|
||||
```
|
||||
|
||||
Group-based control:
|
||||
```python
|
||||
# Disable progress bars for a specific group
|
||||
>>> disable_progress_bars("peft.foo")
|
||||
|
||||
# Check state of different groups
|
||||
>>> assert not are_progress_bars_disabled("peft"))
|
||||
>>> assert not are_progress_bars_disabled("peft.something")
|
||||
>>> assert are_progress_bars_disabled("peft.foo"))
|
||||
>>> assert are_progress_bars_disabled("peft.foo.bar"))
|
||||
|
||||
# Enable progress bars for a subgroup
|
||||
>>> enable_progress_bars("peft.foo.bar")
|
||||
|
||||
# Check if enabling a subgroup affects the parent group
|
||||
>>> assert are_progress_bars_disabled("peft.foo"))
|
||||
>>> assert not are_progress_bars_disabled("peft.foo.bar"))
|
||||
|
||||
# No progress bar for `name="peft.foo"`
|
||||
>>> for _ in tqdm(range(5), name="peft.foo"):
|
||||
... pass
|
||||
|
||||
# Progress bar will be shown for `name="peft.foo.bar"`
|
||||
>>> for _ in tqdm(range(5), name="peft.foo.bar"):
|
||||
... pass
|
||||
100%|███████████████████████████████████████| 5/5 [00:00<00:00, 117817.53it/s]
|
||||
|
||||
```
|
||||
"""
|
||||
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import warnings
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from pathlib import Path
|
||||
from typing import ContextManager, Dict, Iterator, Optional, Union
|
||||
|
||||
from tqdm.auto import tqdm as old_tqdm
|
||||
|
||||
from ..constants import HF_HUB_DISABLE_PROGRESS_BARS
|
||||
|
||||
|
||||
# The `HF_HUB_DISABLE_PROGRESS_BARS` environment variable can be True, False, or not set (None),
|
||||
# allowing for control over progress bar visibility. When set, this variable takes precedence
|
||||
# over programmatic settings, dictating whether progress bars should be shown or hidden globally.
|
||||
# Essentially, the environment variable's setting overrides any code-based configurations.
|
||||
#
|
||||
# If `HF_HUB_DISABLE_PROGRESS_BARS` is not defined (None), it implies that users can manage
|
||||
# progress bar visibility through code. By default, progress bars are turned on.
|
||||
|
||||
|
||||
progress_bar_states: Dict[str, bool] = {}
|
||||
|
||||
|
||||
def disable_progress_bars(name: Optional[str] = None) -> None:
|
||||
"""
|
||||
Disable progress bars either globally or for a specified group.
|
||||
|
||||
This function updates the state of progress bars based on a group name.
|
||||
If no group name is provided, all progress bars are disabled. The operation
|
||||
respects the `HF_HUB_DISABLE_PROGRESS_BARS` environment variable's setting.
|
||||
|
||||
Args:
|
||||
name (`str`, *optional*):
|
||||
The name of the group for which to disable the progress bars. If None,
|
||||
progress bars are disabled globally.
|
||||
|
||||
Raises:
|
||||
Warning: If the environment variable precludes changes.
|
||||
"""
|
||||
if HF_HUB_DISABLE_PROGRESS_BARS is False:
|
||||
warnings.warn(
|
||||
"Cannot disable progress bars: environment variable `HF_HUB_DISABLE_PROGRESS_BARS=0` is set and has priority."
|
||||
)
|
||||
return
|
||||
|
||||
if name is None:
|
||||
progress_bar_states.clear()
|
||||
progress_bar_states["_global"] = False
|
||||
else:
|
||||
keys_to_remove = [key for key in progress_bar_states if key.startswith(f"{name}.")]
|
||||
for key in keys_to_remove:
|
||||
del progress_bar_states[key]
|
||||
progress_bar_states[name] = False
|
||||
|
||||
|
||||
def enable_progress_bars(name: Optional[str] = None) -> None:
|
||||
"""
|
||||
Enable progress bars either globally or for a specified group.
|
||||
|
||||
This function sets the progress bars to enabled for the specified group or globally
|
||||
if no group is specified. The operation is subject to the `HF_HUB_DISABLE_PROGRESS_BARS`
|
||||
environment setting.
|
||||
|
||||
Args:
|
||||
name (`str`, *optional*):
|
||||
The name of the group for which to enable the progress bars. If None,
|
||||
progress bars are enabled globally.
|
||||
|
||||
Raises:
|
||||
Warning: If the environment variable precludes changes.
|
||||
"""
|
||||
if HF_HUB_DISABLE_PROGRESS_BARS is True:
|
||||
warnings.warn(
|
||||
"Cannot enable progress bars: environment variable `HF_HUB_DISABLE_PROGRESS_BARS=1` is set and has priority."
|
||||
)
|
||||
return
|
||||
|
||||
if name is None:
|
||||
progress_bar_states.clear()
|
||||
progress_bar_states["_global"] = True
|
||||
else:
|
||||
keys_to_remove = [key for key in progress_bar_states if key.startswith(f"{name}.")]
|
||||
for key in keys_to_remove:
|
||||
del progress_bar_states[key]
|
||||
progress_bar_states[name] = True
|
||||
|
||||
|
||||
def are_progress_bars_disabled(name: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Check if progress bars are disabled globally or for a specific group.
|
||||
|
||||
This function returns whether progress bars are disabled for a given group or globally.
|
||||
It checks the `HF_HUB_DISABLE_PROGRESS_BARS` environment variable first, then the programmatic
|
||||
settings.
|
||||
|
||||
Args:
|
||||
name (`str`, *optional*):
|
||||
The group name to check; if None, checks the global setting.
|
||||
|
||||
Returns:
|
||||
`bool`: True if progress bars are disabled, False otherwise.
|
||||
"""
|
||||
if HF_HUB_DISABLE_PROGRESS_BARS is True:
|
||||
return True
|
||||
|
||||
if name is None:
|
||||
return not progress_bar_states.get("_global", True)
|
||||
|
||||
while name:
|
||||
if name in progress_bar_states:
|
||||
return not progress_bar_states[name]
|
||||
name = ".".join(name.split(".")[:-1])
|
||||
|
||||
return not progress_bar_states.get("_global", True)
|
||||
|
||||
|
||||
def is_tqdm_disabled(log_level: int) -> Optional[bool]:
|
||||
"""
|
||||
Determine if tqdm progress bars should be disabled based on logging level and environment settings.
|
||||
|
||||
see https://github.com/huggingface/huggingface_hub/pull/2000 and https://github.com/huggingface/huggingface_hub/pull/2698.
|
||||
"""
|
||||
if log_level == logging.NOTSET:
|
||||
return True
|
||||
if os.getenv("TQDM_POSITION") == "-1":
|
||||
return False
|
||||
return None
|
||||
|
||||
|
||||
class tqdm(old_tqdm):
|
||||
"""
|
||||
Class to override `disable` argument in case progress bars are globally disabled.
|
||||
|
||||
Taken from https://github.com/tqdm/tqdm/issues/619#issuecomment-619639324.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
name = kwargs.pop("name", None) # do not pass `name` to `tqdm`
|
||||
if are_progress_bars_disabled(name):
|
||||
kwargs["disable"] = True
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def __delattr__(self, attr: str) -> None:
|
||||
"""Fix for https://github.com/huggingface/huggingface_hub/issues/1603"""
|
||||
try:
|
||||
super().__delattr__(attr)
|
||||
except AttributeError:
|
||||
if attr != "_lock":
|
||||
raise
|
||||
|
||||
|
||||
@contextmanager
|
||||
def tqdm_stream_file(path: Union[Path, str]) -> Iterator[io.BufferedReader]:
|
||||
"""
|
||||
Open a file as binary and wrap the `read` method to display a progress bar when it's streamed.
|
||||
|
||||
First implemented in `transformers` in 2019 but removed when switched to git-lfs. Used in `huggingface_hub` to show
|
||||
progress bar when uploading an LFS file to the Hub. See github.com/huggingface/transformers/pull/2078#discussion_r354739608
|
||||
for implementation details.
|
||||
|
||||
Note: currently implementation handles only files stored on disk as it is the most common use case. Could be
|
||||
extended to stream any `BinaryIO` object but we might have to debug some corner cases.
|
||||
|
||||
Example:
|
||||
```py
|
||||
>>> with tqdm_stream_file("config.json") as f:
|
||||
>>> requests.put(url, data=f)
|
||||
config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s]
|
||||
```
|
||||
"""
|
||||
if isinstance(path, str):
|
||||
path = Path(path)
|
||||
|
||||
with path.open("rb") as f:
|
||||
total_size = path.stat().st_size
|
||||
pbar = tqdm(
|
||||
unit="B",
|
||||
unit_scale=True,
|
||||
total=total_size,
|
||||
initial=0,
|
||||
desc=path.name,
|
||||
)
|
||||
|
||||
f_read = f.read
|
||||
|
||||
def _inner_read(size: Optional[int] = -1) -> bytes:
|
||||
data = f_read(size)
|
||||
pbar.update(len(data))
|
||||
return data
|
||||
|
||||
f.read = _inner_read # type: ignore
|
||||
|
||||
yield f
|
||||
|
||||
pbar.close()
|
||||
|
||||
|
||||
def _get_progress_bar_context(
|
||||
*,
|
||||
desc: str,
|
||||
log_level: int,
|
||||
total: Optional[int] = None,
|
||||
initial: int = 0,
|
||||
unit: str = "B",
|
||||
unit_scale: bool = True,
|
||||
name: Optional[str] = None,
|
||||
_tqdm_bar: Optional[tqdm] = None,
|
||||
) -> ContextManager[tqdm]:
|
||||
if _tqdm_bar is not None:
|
||||
return nullcontext(_tqdm_bar)
|
||||
# ^ `contextlib.nullcontext` mimics a context manager that does nothing
|
||||
# Makes it easier to use the same code path for both cases but in the later
|
||||
# case, the progress bar is not closed when exiting the context manager.
|
||||
|
||||
return tqdm(
|
||||
unit=unit,
|
||||
unit_scale=unit_scale,
|
||||
total=total,
|
||||
initial=initial,
|
||||
desc=desc,
|
||||
disable=is_tqdm_disabled(log_level=log_level),
|
||||
name=name,
|
||||
)
|
||||
Reference in New Issue
Block a user