initial commit

This commit is contained in:
Davidson Gomes
2024-10-30 11:19:09 -03:00
commit 8654a31a4d
3744 changed files with 585542 additions and 0 deletions

View File

@@ -0,0 +1,49 @@
"""Top-level module for Twine.
The contents of this package are not a public API. For more details, see
https://github.com/pypa/twine/issues/194 and https://github.com/pypa/twine/issues/665.
"""
# Copyright 2018 Donald Stufft and individual contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
__all__ = (
"__title__",
"__summary__",
"__uri__",
"__version__",
"__author__",
"__email__",
"__license__",
"__copyright__",
)
__copyright__ = "Copyright 2019 Donald Stufft and individual contributors"
import email
import importlib_metadata
metadata = importlib_metadata.metadata("twine")
__title__ = metadata["name"]
__summary__ = metadata["summary"]
__uri__ = next(
entry.split(", ")[1]
for entry in metadata.get_all("Project-URL", ())
if entry.startswith("Homepage")
)
__version__ = metadata["version"]
__author__, __email__ = email.utils.parseaddr(metadata["author-email"])
__license__ = None

View File

@@ -0,0 +1,54 @@
#!/usr/bin/env python3
# Copyright 2013 Donald Stufft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import http
import logging
import sys
from typing import Any, cast
import requests
from twine import cli
from twine import exceptions
logger = logging.getLogger(__name__)
def main() -> Any:
# Ensure that all errors are logged, even before argparse
cli.configure_output()
try:
error = cli.dispatch(sys.argv[1:])
except requests.HTTPError as exc:
# Assuming this response will never be None
response = cast(requests.Response, exc.response)
error = True
status_code = response.status_code
status_phrase = http.HTTPStatus(status_code).phrase
logger.error(
f"{exc.__class__.__name__}: {status_code} {status_phrase} "
f"from {response.url}\n"
f"{response.reason}"
)
except exceptions.TwineException as exc:
error = True
logger.error(f"{exc.__class__.__name__}: {exc.args[0]}")
return error
if __name__ == "__main__":
sys.exit(main())

View File

@@ -0,0 +1,117 @@
import functools
import getpass
import logging
from typing import Callable, Optional, Type, cast
import keyring
from twine import exceptions
from twine import utils
logger = logging.getLogger(__name__)
class CredentialInput:
def __init__(
self, username: Optional[str] = None, password: Optional[str] = None
) -> None:
self.username = username
self.password = password
class Resolver:
def __init__(self, config: utils.RepositoryConfig, input: CredentialInput) -> None:
self.config = config
self.input = input
@classmethod
def choose(cls, interactive: bool) -> Type["Resolver"]:
return cls if interactive else Private
@property
@functools.lru_cache()
def username(self) -> Optional[str]:
if cast(str, self.config["repository"]).startswith(
(utils.DEFAULT_REPOSITORY, utils.TEST_REPOSITORY)
):
# As of 2024-01-01, PyPI requires API tokens for uploads, meaning
# that the username is invariant.
return "__token__"
return utils.get_userpass_value(
self.input.username,
self.config,
key="username",
prompt_strategy=self.username_from_keyring_or_prompt,
)
@property
@functools.lru_cache()
def password(self) -> Optional[str]:
return utils.get_userpass_value(
self.input.password,
self.config,
key="password",
prompt_strategy=self.password_from_keyring_or_prompt,
)
@property
def system(self) -> Optional[str]:
return self.config["repository"]
def get_username_from_keyring(self) -> Optional[str]:
try:
system = cast(str, self.system)
logger.info("Querying keyring for username")
creds = keyring.get_credential(system, None)
if creds:
return cast(str, creds.username)
except AttributeError:
# To support keyring prior to 15.2
pass
except Exception as exc:
logger.warning("Error getting username from keyring", exc_info=exc)
return None
def get_password_from_keyring(self) -> Optional[str]:
try:
system = cast(str, self.system)
username = cast(str, self.username)
logger.info("Querying keyring for password")
return cast(str, keyring.get_password(system, username))
except Exception as exc:
logger.warning("Error getting password from keyring", exc_info=exc)
return None
def username_from_keyring_or_prompt(self) -> str:
username = self.get_username_from_keyring()
if username:
logger.info("username set from keyring")
return username
return self.prompt("username", input)
def password_from_keyring_or_prompt(self) -> str:
password = self.get_password_from_keyring()
if password:
logger.info("password set from keyring")
return password
# As of 2024-01-01, PyPI requires API tokens for uploads;
# specialize the prompt to clarify that an API token must be provided.
if cast(str, self.config["repository"]).startswith(
(utils.DEFAULT_REPOSITORY, utils.TEST_REPOSITORY)
):
prompt = "API token"
else:
prompt = "password"
return self.prompt(prompt, getpass.getpass)
def prompt(self, what: str, how: Callable[..., str]) -> str:
return how(f"Enter your {what}: ")
class Private(Resolver):
def prompt(self, what: str, how: Optional[Callable[..., str]] = None) -> str:
raise exceptions.NonInteractive(f"Credential not found for {what}.")

View File

@@ -0,0 +1,123 @@
# Copyright 2013 Donald Stufft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import logging.config
from typing import Any, List, Tuple
import importlib_metadata
import rich
import rich.highlighter
import rich.logging
import rich.theme
import twine
args = argparse.Namespace()
def configure_output() -> None:
# Configure the global Console, available via rich.get_console().
# https://rich.readthedocs.io/en/latest/reference/init.html
# https://rich.readthedocs.io/en/latest/console.html
rich.reconfigure(
# Setting force_terminal makes testing easier by ensuring color codes. This
# could be based on FORCE_COLORS or PY_COLORS in os.environ, since Rich
# doesn't support that (https://github.com/Textualize/rich/issues/343).
force_terminal=True,
no_color=getattr(args, "no_color", False),
highlight=False,
theme=rich.theme.Theme(
{
"logging.level.debug": "green",
"logging.level.info": "blue",
"logging.level.warning": "yellow",
"logging.level.error": "red",
"logging.level.critical": "reverse red",
}
),
)
# Using dictConfig to override existing loggers, which prevents failures in
# test_main.py due to capsys not being cleared.
logging.config.dictConfig(
{
"disable_existing_loggers": False,
"version": 1,
"handlers": {
"console": {
"class": "rich.logging.RichHandler",
"show_time": False,
"show_path": False,
"highlighter": rich.highlighter.NullHighlighter(),
}
},
"root": {
"handlers": ["console"],
},
}
)
def list_dependencies_and_versions() -> List[Tuple[str, str]]:
deps = (
"importlib-metadata",
"keyring",
"pkginfo",
"requests",
"requests-toolbelt",
"urllib3",
)
return [(dep, importlib_metadata.version(dep)) for dep in deps]
def dep_versions() -> str:
return ", ".join(
"{}: {}".format(*dependency) for dependency in list_dependencies_and_versions()
)
def dispatch(argv: List[str]) -> Any:
registered_commands = importlib_metadata.entry_points(
group="twine.registered_commands"
)
parser = argparse.ArgumentParser(prog="twine")
parser.add_argument(
"--version",
action="version",
version=f"%(prog)s version {twine.__version__} ({dep_versions()})",
)
parser.add_argument(
"--no-color",
default=False,
required=False,
action="store_true",
help="disable colored output",
)
parser.add_argument(
"command",
choices=registered_commands.names,
)
parser.add_argument(
"args",
help=argparse.SUPPRESS,
nargs=argparse.REMAINDER,
)
parser.parse_args(argv, namespace=args)
configure_output()
main = registered_commands[args.command].load()
return main(args.args)

View File

@@ -0,0 +1,54 @@
"""Module containing the logic for the ``twine`` sub-commands.
The contents of this package are not a public API. For more details, see
https://github.com/pypa/twine/issues/194 and https://github.com/pypa/twine/issues/665.
"""
# Copyright 2013 Donald Stufft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import glob
import os.path
from typing import List
from twine import exceptions
__all__: List[str] = []
def _group_wheel_files_first(files: List[str]) -> List[str]:
if not any(fname for fname in files if fname.endswith(".whl")):
# Return early if there's no wheel files
return files
files.sort(key=lambda x: -1 if x.endswith(".whl") else 0)
return files
def _find_dists(dists: List[str]) -> List[str]:
uploads = []
for filename in dists:
if os.path.exists(filename):
uploads.append(filename)
continue
# The filename didn't exist so it may be a glob
files = glob.glob(filename)
# If nothing matches, files is []
if not files:
raise exceptions.InvalidDistribution(
"Cannot find file (or expand pattern): '%s'" % filename
)
# Otherwise, files will be filenames that exist
uploads.extend(files)
return _group_wheel_files_first(uploads)

View File

@@ -0,0 +1,194 @@
"""Module containing the logic for ``twine check``."""
# Copyright 2018 Dustin Ingram
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import email.message
import io
import logging
import re
from typing import Dict, List, Optional, Tuple, cast
import readme_renderer.rst
from rich import print
from twine import commands
from twine import package as package_file
logger = logging.getLogger(__name__)
_RENDERERS = {
None: readme_renderer.rst, # Default if description_content_type is None
"text/plain": None, # Rendering cannot fail
"text/x-rst": readme_renderer.rst,
"text/markdown": None, # Rendering cannot fail
}
# Regular expression used to capture and reformat docutils warnings into
# something that a human can understand. This is loosely borrowed from
# Sphinx: https://github.com/sphinx-doc/sphinx/blob
# /c35eb6fade7a3b4a6de4183d1dd4196f04a5edaf/sphinx/util/docutils.py#L199
_REPORT_RE = re.compile(
r"^<string>:(?P<line>(?:\d+)?): "
r"\((?P<level>DEBUG|INFO|WARNING|ERROR|SEVERE)/(\d+)?\) "
r"(?P<message>.*)",
re.DOTALL | re.MULTILINE,
)
class _WarningStream(io.StringIO):
def write(self, text: str) -> int:
matched = _REPORT_RE.search(text)
if matched:
line = matched.group("line")
level_text = matched.group("level").capitalize()
message = matched.group("message").rstrip("\r\n")
text = f"line {line}: {level_text}: {message}\n"
return super().write(text)
def __str__(self) -> str:
return self.getvalue().strip()
def _parse_content_type(value: str) -> Tuple[str, Dict[str, str]]:
"""Implement logic of deprecated cgi.parse_header().
From https://docs.python.org/3.11/library/cgi.html#cgi.parse_header.
"""
msg = email.message.EmailMessage()
msg["content-type"] = value
return msg.get_content_type(), msg["content-type"].params
def _check_file(
filename: str, render_warning_stream: _WarningStream
) -> Tuple[List[str], bool]:
"""Check given distribution."""
warnings = []
is_ok = True
package = package_file.PackageFile.from_filename(filename, comment=None)
metadata = package.metadata_dictionary()
description = cast(Optional[str], metadata["description"])
description_content_type = cast(Optional[str], metadata["description_content_type"])
if description_content_type is None:
warnings.append(
"`long_description_content_type` missing. defaulting to `text/x-rst`."
)
description_content_type = "text/x-rst"
content_type, params = _parse_content_type(description_content_type)
renderer = _RENDERERS.get(content_type, _RENDERERS[None])
if description is None or description.rstrip() == "UNKNOWN":
warnings.append("`long_description` missing.")
elif renderer:
rendering_result = renderer.render(
description, stream=render_warning_stream, **params
)
if rendering_result is None:
is_ok = False
return warnings, is_ok
def check(
dists: List[str],
strict: bool = False,
) -> bool:
"""Check that a distribution will render correctly on PyPI and display the results.
This is currently only validates ``long_description``, but more checks could be
added; see https://github.com/pypa/twine/projects/2.
:param dists:
The distribution files to check.
:param output_stream:
The destination of the resulting output.
:param strict:
If ``True``, treat warnings as errors.
:return:
``True`` if there are rendering errors, otherwise ``False``.
"""
uploads = [i for i in commands._find_dists(dists) if not i.endswith(".asc")]
if not uploads: # Return early, if there are no files to check.
logger.error("No files to check.")
return False
failure = False
for filename in uploads:
print(f"Checking {filename}: ", end="")
render_warning_stream = _WarningStream()
warnings, is_ok = _check_file(filename, render_warning_stream)
# Print the status and/or error
if not is_ok:
failure = True
print("[red]FAILED[/red]")
logger.error(
"`long_description` has syntax errors in markup"
" and would not be rendered on PyPI."
f"\n{render_warning_stream}"
)
elif warnings:
if strict:
failure = True
print("[red]FAILED due to warnings[/red]")
else:
print("[yellow]PASSED with warnings[/yellow]")
else:
print("[green]PASSED[/green]")
# Print warnings after the status and/or error
for message in warnings:
logger.warning(message)
return failure
def main(args: List[str]) -> bool:
"""Execute the ``check`` command.
:param args:
The command-line arguments.
:return:
The exit status of the ``check`` command.
"""
parser = argparse.ArgumentParser(prog="twine check")
parser.add_argument(
"dists",
nargs="+",
metavar="dist",
help="The distribution files to check, usually dist/*",
)
parser.add_argument(
"--strict",
action="store_true",
default=False,
required=False,
help="Fail on warnings",
)
parsed_args = parser.parse_args(args)
# Call the check function with the arguments from the command line
return check(parsed_args.dists, strict=parsed_args.strict)

View File

@@ -0,0 +1,87 @@
"""Module containing the logic for ``twine register``."""
# Copyright 2015 Ian Cordasco
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import os.path
from typing import List, cast
from rich import print
from twine import exceptions
from twine import package as package_file
from twine import settings
def register(register_settings: settings.Settings, package: str) -> None:
"""Pre-register a package name with a repository before uploading a distribution.
Pre-registration is not supported on PyPI, so the ``register`` command is only
necessary if you are using a different repository that requires it.
:param register_settings:
The configured options relating to repository registration.
:param package:
The path of the distribution to use for package metadata.
:raises twine.exceptions.TwineException:
The registration failed due to a configuration error.
:raises requests.HTTPError:
The repository responded with an error.
"""
repository_url = cast(str, register_settings.repository_config["repository"])
print(f"Registering package to {repository_url}")
repository = register_settings.create_repository()
if not os.path.exists(package):
raise exceptions.PackageNotFound(
f'"{package}" does not exist on the file system.'
)
resp = repository.register(
package_file.PackageFile.from_filename(package, register_settings.comment)
)
repository.close()
if resp.is_redirect:
raise exceptions.RedirectDetected.from_args(
repository_url,
resp.headers["location"],
)
resp.raise_for_status()
def main(args: List[str]) -> None:
"""Execute the ``register`` command.
:param args:
The command-line arguments.
"""
parser = argparse.ArgumentParser(
prog="twine register",
description="register operation is not required with PyPI.org",
)
settings.Settings.register_argparse_arguments(parser)
parser.add_argument(
"package",
metavar="package",
help="File from which we read the package metadata.",
)
parsed_args = parser.parse_args(args)
register_settings = settings.Settings.from_argparse(parsed_args)
# Call the register function with the args from the command line
register(register_settings, parsed_args.package)

View File

@@ -0,0 +1,298 @@
"""Module containing the logic for ``twine upload``."""
# Copyright 2013 Donald Stufft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import fnmatch
import logging
import os.path
from typing import Dict, List, NamedTuple, cast
import requests
from rich import print
from twine import commands
from twine import exceptions
from twine import package as package_file
from twine import settings
from twine import utils
logger = logging.getLogger(__name__)
def skip_upload(
response: requests.Response, skip_existing: bool, package: package_file.PackageFile
) -> bool:
"""Determine if a failed upload is an error or can be safely ignored.
:param response:
The response from attempting to upload ``package`` to a repository.
:param skip_existing:
If ``True``, use the status and content of ``response`` to determine if the
package already exists on the repository. If so, then a failed upload is safe
to ignore.
:param package:
The package that was being uploaded.
:return:
``True`` if a failed upload can be safely ignored, otherwise ``False``.
"""
if not skip_existing:
return False
status = response.status_code
reason = getattr(response, "reason", "").lower()
text = getattr(response, "text", "").lower()
# NOTE(sigmavirus24): PyPI presently returns a 400 status code with the
# error message in the reason attribute. Other implementations return a
# 403 or 409 status code.
return (
# pypiserver (https://pypi.org/project/pypiserver)
status == 409
# PyPI / TestPyPI / GCP Artifact Registry
or (status == 400 and any("already exist" in x for x in [reason, text]))
# Nexus Repository OSS (https://www.sonatype.com/nexus-repository-oss)
or (status == 400 and any("updating asset" in x for x in [reason, text]))
# Artifactory (https://jfrog.com/artifactory/)
or (status == 403 and "overwrite artifact" in text)
# Gitlab Enterprise Edition (https://about.gitlab.com)
or (status == 400 and "already been taken" in text)
)
def _make_package(
filename: str,
signatures: Dict[str, str],
attestations: List[str],
upload_settings: settings.Settings,
) -> package_file.PackageFile:
"""Create and sign a package, based off of filename, signatures, and settings.
Additionally, any supplied attestations are attached to the package when
the settings indicate to do so.
"""
package = package_file.PackageFile.from_filename(filename, upload_settings.comment)
signed_name = package.signed_basefilename
if signed_name in signatures:
package.add_gpg_signature(signatures[signed_name], signed_name)
elif upload_settings.sign:
package.sign(upload_settings.sign_with, upload_settings.identity)
# Attestations are only attached if explicitly requested with `--attestations`.
if upload_settings.attestations:
# Passing `--attestations` without any actual attestations present
# indicates user confusion, so we fail rather than silently allowing it.
if not attestations:
raise exceptions.InvalidDistribution(
"Upload with attestations requested, but "
f"{filename} has no associated attestations"
)
package.add_attestations(attestations)
file_size = utils.get_file_size(package.filename)
logger.info(f"{package.filename} ({file_size})")
if package.gpg_signature:
logger.info(f"Signed with {package.signed_filename}")
return package
class Inputs(NamedTuple):
"""Represents structured user inputs."""
dists: List[str]
signatures: Dict[str, str]
attestations_by_dist: Dict[str, List[str]]
def _split_inputs(
inputs: List[str],
) -> Inputs:
"""
Split the unstructured list of input files provided by the user into groups.
Three groups are returned: upload files (i.e. dists), signatures, and attestations.
Upload files are returned as a linear list, signatures are returned as a
dict of ``basename -> path``, and attestations are returned as a dict of
``dist-path -> [attestation-path]``.
"""
signatures = {os.path.basename(i): i for i in fnmatch.filter(inputs, "*.asc")}
attestations = fnmatch.filter(inputs, "*.*.attestation")
dists = [
dist
for dist in inputs
if dist not in (set(signatures.values()) | set(attestations))
]
attestations_by_dist = {}
for dist in dists:
dist_basename = os.path.basename(dist)
attestations_by_dist[dist] = [
a for a in attestations if os.path.basename(a).startswith(dist_basename)
]
return Inputs(dists, signatures, attestations_by_dist)
def upload(upload_settings: settings.Settings, dists: List[str]) -> None:
"""Upload one or more distributions to a repository, and display the progress.
If a package already exists on the repository, most repositories will return an
error response. However, if ``upload_settings.skip_existing`` is ``True``, a message
will be displayed and any remaining distributions will be uploaded.
For known repositories (like PyPI), the web URLs of successfully uploaded packages
will be displayed.
:param upload_settings:
The configured options related to uploading to a repository.
:param dists:
The distribution files to upload to the repository. This can also include
``.asc`` and ``.attestation`` files, which will be added to their respective
file uploads.
:raises twine.exceptions.TwineException:
The upload failed due to a configuration error.
:raises requests.HTTPError:
The repository responded with an error.
"""
upload_settings.check_repository_url()
repository_url = cast(str, upload_settings.repository_config["repository"])
# Attestations are only supported on PyPI and TestPyPI at the moment.
# We warn instead of failing to allow twine to be used in local testing
# setups (where the PyPI deployment doesn't have a well-known domain).
if upload_settings.attestations and not repository_url.startswith(
(utils.DEFAULT_REPOSITORY, utils.TEST_REPOSITORY)
):
logger.warning(
"Only PyPI and TestPyPI support attestations; "
"if you experience failures, remove the --attestations flag and "
"re-try this command"
)
dists = commands._find_dists(dists)
# Determine if the user has passed in pre-signed distributions or any attestations.
uploads, signatures, attestations_by_dist = _split_inputs(dists)
print(f"Uploading distributions to {utils.sanitize_url(repository_url)}")
packages_to_upload = [
_make_package(
filename, signatures, attestations_by_dist[filename], upload_settings
)
for filename in uploads
]
if any(p.gpg_signature for p in packages_to_upload):
if repository_url.startswith((utils.DEFAULT_REPOSITORY, utils.TEST_REPOSITORY)):
# Warn the user if they're trying to upload a PGP signature to PyPI
# or TestPyPI, which will (as of May 2023) ignore it.
# This warning is currently limited to just those indices, since other
# indices may still support PGP signatures.
logger.warning(
"One or more packages has an associated PGP signature; "
"these will be silently ignored by the index"
)
else:
# On other indices, warn the user that twine is considering
# removing PGP support outright.
logger.warning(
"One or more packages has an associated PGP signature; "
"a future version of twine may silently ignore these. "
"See https://github.com/pypa/twine/issues/1009 for more "
"information"
)
repository = upload_settings.create_repository()
uploaded_packages = []
if signatures and not packages_to_upload:
raise exceptions.InvalidDistribution(
"Cannot upload signed files by themselves, must upload with a "
"corresponding distribution file."
)
for package in packages_to_upload:
skip_message = (
f"Skipping {package.basefilename} because it appears to already exist"
)
# Note: The skip_existing check *needs* to be first, because otherwise
# we're going to generate extra HTTP requests against a hardcoded
# URL for no reason.
if upload_settings.skip_existing and repository.package_is_uploaded(package):
logger.warning(skip_message)
continue
resp = repository.upload(package)
logger.info(f"Response from {resp.url}:\n{resp.status_code} {resp.reason}")
if resp.text:
logger.info(resp.text)
# Bug 92. If we get a redirect we should abort because something seems
# funky. The behaviour is not well defined and redirects being issued
# by PyPI should never happen in reality. This should catch malicious
# redirects as well.
if resp.is_redirect:
raise exceptions.RedirectDetected.from_args(
utils.sanitize_url(repository_url),
utils.sanitize_url(resp.headers["location"]),
)
if skip_upload(resp, upload_settings.skip_existing, package):
logger.warning(skip_message)
continue
utils.check_status_code(resp, upload_settings.verbose)
uploaded_packages.append(package)
release_urls = repository.release_urls(uploaded_packages)
if release_urls:
print("\n[green]View at:")
for url in release_urls:
print(url)
# Bug 28. Try to silence a ResourceWarning by clearing the connection
# pool.
repository.close()
def main(args: List[str]) -> None:
"""Execute the ``upload`` command.
:param args:
The command-line arguments.
"""
parser = argparse.ArgumentParser(prog="twine upload")
settings.Settings.register_argparse_arguments(parser)
parser.add_argument(
"dists",
nargs="+",
metavar="dist",
help="The distribution files to upload to the repository "
"(package index). Usually dist/* . May additionally contain "
"a .asc file to include an existing signature with the "
"file upload.",
)
parsed_args = parser.parse_args(args)
upload_settings = settings.Settings.from_argparse(parsed_args)
# Call the upload function with the arguments from the command line
return upload(upload_settings, parsed_args.dists)

View File

@@ -0,0 +1,125 @@
"""Module containing exceptions raised by twine."""
# Copyright 2015 Ian Stapleton Cordasco
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class TwineException(Exception):
"""Base class for all exceptions raised by twine."""
pass
class RedirectDetected(TwineException):
"""A redirect was detected that the user needs to resolve.
In some cases, requests refuses to issue a new POST request after a
redirect. In order to prevent a confusing user experience, we raise this
exception to allow users to know the index they're uploading to is
redirecting them.
"""
@classmethod
def from_args(cls, repository_url: str, redirect_url: str) -> "RedirectDetected":
if redirect_url == f"{repository_url}/":
return cls(
f"{repository_url} attempted to redirect to {redirect_url}.\n"
f"Your repository URL is missing a trailing slash. "
"Please add it and try again.",
)
return cls(
f"{repository_url} attempted to redirect to {redirect_url}.\n"
f"If you trust these URLs, set {redirect_url} as your repository URL "
"and try again.",
)
class PackageNotFound(TwineException):
"""A package file was provided that could not be found on the file system.
This is only used when attempting to register a package_file.
"""
pass
class UploadToDeprecatedPyPIDetected(TwineException):
"""An upload attempt was detected to deprecated PyPI domains.
The sites pypi.python.org and testpypi.python.org are deprecated.
"""
@classmethod
def from_args(
cls, target_url: str, default_url: str, test_url: str
) -> "UploadToDeprecatedPyPIDetected":
"""Return an UploadToDeprecatedPyPIDetected instance."""
return cls(
"You're trying to upload to the legacy PyPI site '{}'. "
"Uploading to those sites is deprecated. \n "
"The new sites are pypi.org and test.pypi.org. Try using "
"{} (or {}) to upload your packages instead. "
"These are the default URLs for Twine now. \n More at "
"https://packaging.python.org/guides/migrating-to-pypi-org/"
" .".format(target_url, default_url, test_url)
)
class UnreachableRepositoryURLDetected(TwineException):
"""An upload attempt was detected to a URL without a protocol prefix.
All repository URLs must have a protocol (e.g., ``https://``).
"""
pass
class InvalidSigningConfiguration(TwineException):
"""Both the sign and identity parameters must be present."""
pass
class InvalidSigningExecutable(TwineException):
"""Signing executable must be installed on system."""
pass
class InvalidConfiguration(TwineException):
"""Raised when configuration is invalid."""
pass
class InvalidDistribution(TwineException):
"""Raised when a distribution is invalid."""
pass
class NonInteractive(TwineException):
"""Raised in non-interactive mode when credentials could not be found."""
pass
class InvalidPyPIUploadURL(TwineException):
"""Repository configuration tries to use PyPI with an incorrect URL.
For example, https://pypi.org instead of https://upload.pypi.org/legacy.
"""
pass

View File

@@ -0,0 +1,332 @@
# Copyright 2015 Ian Cordasco
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import hashlib
import io
import json
import logging
import os
import re
import subprocess
from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Tuple, Union, cast
import importlib_metadata
import pkginfo
from rich import print
from twine import exceptions
from twine import wheel
from twine import wininst
DIST_TYPES = {
"bdist_wheel": wheel.Wheel,
"bdist_wininst": wininst.WinInst,
"bdist_egg": pkginfo.BDist,
"sdist": pkginfo.SDist,
}
DIST_EXTENSIONS = {
".whl": "bdist_wheel",
".exe": "bdist_wininst",
".egg": "bdist_egg",
".tar.bz2": "sdist",
".tar.gz": "sdist",
".zip": "sdist",
}
MetadataValue = Union[Optional[str], Sequence[str], Tuple[str, bytes]]
logger = logging.getLogger(__name__)
def _safe_name(name: str) -> str:
"""Convert an arbitrary string to a standard distribution name.
Any runs of non-alphanumeric/. characters are replaced with a single '-'.
Copied from pkg_resources.safe_name for compatibility with warehouse.
See https://github.com/pypa/twine/issues/743.
"""
return re.sub("[^A-Za-z0-9.]+", "-", name)
class PackageFile:
def __init__(
self,
filename: str,
comment: Optional[str],
metadata: pkginfo.Distribution,
python_version: Optional[str],
filetype: Optional[str],
) -> None:
self.filename = filename
self.basefilename = os.path.basename(filename)
self.comment = comment
self.metadata = metadata
self.python_version = python_version
self.filetype = filetype
self.safe_name = _safe_name(metadata.name)
self.signed_filename = self.filename + ".asc"
self.signed_basefilename = self.basefilename + ".asc"
self.gpg_signature: Optional[Tuple[str, bytes]] = None
self.attestations: Optional[List[Dict[Any, str]]] = None
hasher = HashManager(filename)
hasher.hash()
hexdigest = hasher.hexdigest()
self.md5_digest = hexdigest.md5
self.sha2_digest = hexdigest.sha2
self.blake2_256_digest = hexdigest.blake2
@classmethod
def from_filename(cls, filename: str, comment: Optional[str]) -> "PackageFile":
# Extract the metadata from the package
for ext, dtype in DIST_EXTENSIONS.items():
if filename.endswith(ext):
try:
meta = DIST_TYPES[dtype](filename)
except EOFError:
raise exceptions.InvalidDistribution(
"Invalid distribution file: '%s'" % os.path.basename(filename)
)
else:
break
else:
raise exceptions.InvalidDistribution(
"Unknown distribution format: '%s'" % os.path.basename(filename)
)
# If pkginfo encounters a metadata version it doesn't support, it may give us
# back empty metadata. At the very least, we should have a name and version,
# which could also be empty if, for example, a MANIFEST.in doesn't include
# setup.cfg.
missing_fields = [
f.capitalize() for f in ["name", "version"] if not getattr(meta, f)
]
if missing_fields:
supported_metadata = list(pkginfo.distribution.HEADER_ATTRS)
raise exceptions.InvalidDistribution(
"Metadata is missing required fields: "
f"{', '.join(missing_fields)}.\n"
"Make sure the distribution includes the files where those fields "
"are specified, and is using a supported Metadata-Version: "
f"{', '.join(supported_metadata)}."
)
py_version: Optional[str]
if dtype == "bdist_egg":
(dist,) = importlib_metadata.Distribution.discover(path=[filename])
py_version = dist.metadata["Version"]
elif dtype == "bdist_wheel":
py_version = cast(wheel.Wheel, meta).py_version
elif dtype == "bdist_wininst":
py_version = cast(wininst.WinInst, meta).py_version
else:
py_version = None
return cls(filename, comment, meta, py_version, dtype)
def metadata_dictionary(self) -> Dict[str, MetadataValue]:
"""Merge multiple sources of metadata into a single dictionary.
Includes values from filename, PKG-INFO, hashers, and signature.
"""
meta = self.metadata
data: Dict[str, MetadataValue] = {
# identify release
"name": self.safe_name,
"version": meta.version,
# file content
"filetype": self.filetype,
"pyversion": self.python_version,
# additional meta-data
"metadata_version": meta.metadata_version,
"summary": meta.summary,
"home_page": meta.home_page,
"author": meta.author,
"author_email": meta.author_email,
"maintainer": meta.maintainer,
"maintainer_email": meta.maintainer_email,
"license": meta.license,
"description": meta.description,
"keywords": meta.keywords,
"platform": meta.platforms,
"classifiers": meta.classifiers,
"download_url": meta.download_url,
"supported_platform": meta.supported_platforms,
"comment": self.comment,
"sha256_digest": self.sha2_digest,
# PEP 314
"provides": meta.provides,
"requires": meta.requires,
"obsoletes": meta.obsoletes,
# Metadata 1.2
"project_urls": meta.project_urls,
"provides_dist": meta.provides_dist,
"obsoletes_dist": meta.obsoletes_dist,
"requires_dist": meta.requires_dist,
"requires_external": meta.requires_external,
"requires_python": meta.requires_python,
# Metadata 2.1
"provides_extra": meta.provides_extras,
"description_content_type": meta.description_content_type,
# Metadata 2.2
"dynamic": meta.dynamic,
}
if self.gpg_signature is not None:
data["gpg_signature"] = self.gpg_signature
if self.attestations is not None:
data["attestations"] = json.dumps(self.attestations)
# FIPS disables MD5 and Blake2, making the digest values None. Some package
# repositories don't allow null values, so this only sends non-null values.
# See also: https://github.com/pypa/twine/issues/775
if self.md5_digest:
data["md5_digest"] = self.md5_digest
if self.blake2_256_digest:
data["blake2_256_digest"] = self.blake2_256_digest
return data
def add_attestations(self, attestations: List[str]) -> None:
loaded_attestations = []
for attestation in attestations:
with open(attestation, "rb") as att:
try:
loaded_attestations.append(json.load(att))
except json.JSONDecodeError:
raise exceptions.InvalidDistribution(
f"invalid JSON in attestation: {attestation}"
)
self.attestations = loaded_attestations
def add_gpg_signature(
self, signature_filepath: str, signature_filename: str
) -> None:
if self.gpg_signature is not None:
raise exceptions.InvalidDistribution("GPG Signature can only be added once")
with open(signature_filepath, "rb") as gpg:
self.gpg_signature = (signature_filename, gpg.read())
def sign(self, sign_with: str, identity: Optional[str]) -> None:
print(f"Signing {self.basefilename}")
gpg_args: Tuple[str, ...] = (sign_with, "--detach-sign")
if identity:
gpg_args += ("--local-user", identity)
gpg_args += ("-a", self.filename)
self.run_gpg(gpg_args)
self.add_gpg_signature(self.signed_filename, self.signed_basefilename)
@classmethod
def run_gpg(cls, gpg_args: Tuple[str, ...]) -> None:
try:
subprocess.check_call(gpg_args)
return
except FileNotFoundError:
if gpg_args[0] != "gpg":
raise exceptions.InvalidSigningExecutable(
f"{gpg_args[0]} executable not available."
)
logger.warning("gpg executable not available. Attempting fallback to gpg2.")
try:
subprocess.check_call(("gpg2",) + gpg_args[1:])
except FileNotFoundError:
raise exceptions.InvalidSigningExecutable(
"'gpg' or 'gpg2' executables not available.\n"
"Try installing one of these or specifying an executable "
"with the --sign-with flag."
)
class Hexdigest(NamedTuple):
md5: Optional[str]
sha2: Optional[str]
blake2: Optional[str]
class HashManager:
"""Manage our hashing objects for simplicity.
This will also allow us to better test this logic.
"""
def __init__(self, filename: str) -> None:
"""Initialize our manager and hasher objects."""
self.filename = filename
self._md5_hasher = None
try:
self._md5_hasher = hashlib.md5()
except ValueError:
# FIPs mode disables MD5
pass
self._sha2_hasher = hashlib.sha256()
self._blake_hasher = None
try:
self._blake_hasher = hashlib.blake2b(digest_size=256 // 8)
except (ValueError, TypeError, AttributeError):
# FIPS mode disables blake2
pass
def _md5_update(self, content: bytes) -> None:
if self._md5_hasher is not None:
self._md5_hasher.update(content)
def _md5_hexdigest(self) -> Optional[str]:
if self._md5_hasher is not None:
return self._md5_hasher.hexdigest()
return None
def _sha2_update(self, content: bytes) -> None:
if self._sha2_hasher is not None:
self._sha2_hasher.update(content)
def _sha2_hexdigest(self) -> Optional[str]:
if self._sha2_hasher is not None:
return self._sha2_hasher.hexdigest()
return None
def _blake_update(self, content: bytes) -> None:
if self._blake_hasher is not None:
self._blake_hasher.update(content)
def _blake_hexdigest(self) -> Optional[str]:
if self._blake_hasher is not None:
return self._blake_hasher.hexdigest()
return None
def hash(self) -> None:
"""Hash the file contents."""
with open(self.filename, "rb") as fp:
for content in iter(lambda: fp.read(io.DEFAULT_BUFFER_SIZE), b""):
self._md5_update(content)
self._sha2_update(content)
self._blake_update(content)
def hexdigest(self) -> Hexdigest:
"""Return the hexdigest for the file."""
return Hexdigest(
self._md5_hexdigest(),
self._sha2_hexdigest(),
self._blake_hexdigest(),
)

View File

View File

@@ -0,0 +1,250 @@
# Copyright 2015 Ian Cordasco
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from typing import Any, Dict, List, Optional, Set, Tuple, cast
import requests
import requests_toolbelt
import rich.progress
import urllib3
from requests import adapters
from requests_toolbelt.utils import user_agent
from rich import print
import twine
from twine import package as package_file
KEYWORDS_TO_NOT_FLATTEN = {"gpg_signature", "attestations", "content"}
LEGACY_PYPI = "https://pypi.python.org/"
LEGACY_TEST_PYPI = "https://testpypi.python.org/"
WAREHOUSE = "https://upload.pypi.org/"
OLD_WAREHOUSE = "https://upload.pypi.io/"
TEST_WAREHOUSE = "https://test.pypi.org/"
WAREHOUSE_WEB = "https://pypi.org/"
logger = logging.getLogger(__name__)
class Repository:
def __init__(
self,
repository_url: str,
username: Optional[str],
password: Optional[str],
disable_progress_bar: bool = False,
) -> None:
self.url = repository_url
self.session = requests.session()
# requests.Session.auth should be Union[None, Tuple[str, str], ...]
# But username or password could be None
# See TODO for utils.RepositoryConfig
self.session.auth = (
(username or "", password or "") if username or password else None
)
logger.info(f"username: {username if username else '<empty>'}")
logger.info(f"password: <{'hidden' if password else 'empty'}>")
self.session.headers["User-Agent"] = self._make_user_agent_string()
for scheme in ("http://", "https://"):
self.session.mount(scheme, self._make_adapter_with_retries())
# Working around https://github.com/python/typing/issues/182
self._releases_json_data: Dict[str, Dict[str, Any]] = {}
self.disable_progress_bar = disable_progress_bar
@staticmethod
def _make_adapter_with_retries() -> adapters.HTTPAdapter:
retry = urllib3.Retry(
allowed_methods=["GET"],
connect=5,
total=10,
status_forcelist=[500, 501, 502, 503],
)
return adapters.HTTPAdapter(max_retries=retry)
@staticmethod
def _make_user_agent_string() -> str:
user_agent_string = (
user_agent.UserAgentBuilder("twine", twine.__version__)
.include_implementation()
.build()
)
return cast(str, user_agent_string)
def close(self) -> None:
self.session.close()
@staticmethod
def _convert_data_to_list_of_tuples(data: Dict[str, Any]) -> List[Tuple[str, Any]]:
data_to_send = []
for key, value in data.items():
if key in KEYWORDS_TO_NOT_FLATTEN or not isinstance(value, (list, tuple)):
data_to_send.append((key, value))
else:
for item in value:
data_to_send.append((key, item))
return data_to_send
def set_certificate_authority(self, cacert: Optional[str]) -> None:
if cacert:
self.session.verify = cacert
def set_client_certificate(self, clientcert: Optional[str]) -> None:
if clientcert:
self.session.cert = clientcert
def register(self, package: package_file.PackageFile) -> requests.Response:
data = package.metadata_dictionary()
data.update({":action": "submit", "protocol_version": "1"})
print(f"Registering {package.basefilename}")
data_to_send = self._convert_data_to_list_of_tuples(data)
encoder = requests_toolbelt.MultipartEncoder(data_to_send)
resp = self.session.post(
self.url,
data=encoder,
allow_redirects=False,
headers={"Content-Type": encoder.content_type},
)
# Bug 28. Try to silence a ResourceWarning by releasing the socket.
resp.close()
return resp
def _upload(self, package: package_file.PackageFile) -> requests.Response:
data = package.metadata_dictionary()
data.update(
{
# action
":action": "file_upload",
"protocol_version": "1",
}
)
data_to_send = self._convert_data_to_list_of_tuples(data)
print(f"Uploading {package.basefilename}")
with open(package.filename, "rb") as fp:
data_to_send.append(
("content", (package.basefilename, fp, "application/octet-stream"))
)
encoder = requests_toolbelt.MultipartEncoder(data_to_send)
with rich.progress.Progress(
"[progress.percentage]{task.percentage:>3.0f}%",
rich.progress.BarColumn(),
rich.progress.DownloadColumn(),
"",
rich.progress.TimeRemainingColumn(
compact=True,
elapsed_when_finished=True,
),
"",
rich.progress.TransferSpeedColumn(),
disable=self.disable_progress_bar,
) as progress:
task_id = progress.add_task("", total=encoder.len)
monitor = requests_toolbelt.MultipartEncoderMonitor(
encoder,
lambda monitor: progress.update(
task_id,
completed=monitor.bytes_read,
),
)
resp = self.session.post(
self.url,
data=monitor,
allow_redirects=False,
headers={"Content-Type": monitor.content_type},
)
return resp
def upload(
self, package: package_file.PackageFile, max_redirects: int = 5
) -> requests.Response:
number_of_redirects = 0
while number_of_redirects < max_redirects:
resp = self._upload(package)
if resp.status_code == requests.codes.OK:
return resp
if 500 <= resp.status_code < 600:
number_of_redirects += 1
logger.warning(
f'Received "{resp.status_code}: {resp.reason}"'
"\nPackage upload appears to have failed."
f" Retry {number_of_redirects} of {max_redirects}."
)
else:
return resp
return resp
def package_is_uploaded(
self, package: package_file.PackageFile, bypass_cache: bool = False
) -> bool:
# NOTE(sigmavirus24): Not all indices are PyPI and pypi.io doesn't
# have a similar interface for finding the package versions.
if not self.url.startswith((LEGACY_PYPI, WAREHOUSE, OLD_WAREHOUSE)):
return False
safe_name = package.safe_name
releases = None
if not bypass_cache:
releases = self._releases_json_data.get(safe_name)
if releases is None:
url = f"{LEGACY_PYPI}pypi/{safe_name}/json"
headers = {"Accept": "application/json"}
response = self.session.get(url, headers=headers)
if response.status_code == 200:
releases = response.json()["releases"]
else:
releases = {}
self._releases_json_data[safe_name] = releases
packages = releases.get(package.metadata.version, [])
for uploaded_package in packages:
if uploaded_package["filename"] == package.basefilename:
return True
return False
def release_urls(self, packages: List[package_file.PackageFile]) -> Set[str]:
if self.url.startswith(WAREHOUSE):
url = WAREHOUSE_WEB
elif self.url.startswith(TEST_WAREHOUSE):
url = TEST_WAREHOUSE
else:
return set()
return {
f"{url}project/{package.safe_name}/{package.metadata.version}/"
for package in packages
}
def verify_package_integrity(self, package: package_file.PackageFile) -> None:
# TODO(sigmavirus24): Add a way for users to download the package and
# check its hash against what it has locally.
pass

View File

@@ -0,0 +1,341 @@
"""Module containing logic for handling settings."""
# Copyright 2018 Ian Stapleton Cordasco
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import contextlib
import logging
from typing import Any, Optional, cast
from twine import auth
from twine import exceptions
from twine import repository
from twine import utils
class Settings:
"""Object that manages the configuration for Twine.
This object can only be instantiated with keyword arguments.
For example,
.. code-block:: python
Settings(True, username='fakeusername')
Will raise a :class:`TypeError`. Instead, you would want
.. code-block:: python
Settings(sign=True, username='fakeusername')
"""
def __init__(
self,
*,
attestations: bool = False,
sign: bool = False,
sign_with: str = "gpg",
identity: Optional[str] = None,
username: Optional[str] = None,
password: Optional[str] = None,
non_interactive: bool = False,
comment: Optional[str] = None,
config_file: str = utils.DEFAULT_CONFIG_FILE,
skip_existing: bool = False,
cacert: Optional[str] = None,
client_cert: Optional[str] = None,
repository_name: str = "pypi",
repository_url: Optional[str] = None,
verbose: bool = False,
disable_progress_bar: bool = False,
**ignored_kwargs: Any,
) -> None:
"""Initialize our settings instance.
:param attestations:
Whether the package file should be uploaded with attestations.
:param sign:
Configure whether the package file should be signed.
:param sign_with:
The name of the executable used to sign the package with.
:param identity:
The GPG identity that should be used to sign the package file.
:param username:
The username used to authenticate to the repository (package
index).
:param password:
The password used to authenticate to the repository (package
index).
:param non_interactive:
Do not interactively prompt for username/password if the required
credentials are missing.
:param comment:
The comment to include with each distribution file.
:param config_file:
The path to the configuration file to use.
:param skip_existing:
Specify whether twine should continue uploading files if one
of them already exists. This primarily supports PyPI. Other
package indexes may not be supported.
:param cacert:
The path to the bundle of certificates used to verify the TLS
connection to the package index.
:param client_cert:
The path to the client certificate used to perform authentication to the
index. This must be a single file that contains both the private key and
the PEM-encoded certificate.
:param repository_name:
The name of the repository (package index) to interact with. This
should correspond to a section in the config file.
:param repository_url:
The URL of the repository (package index) to interact with. This
will override the settings inferred from ``repository_name``.
:param verbose:
Show verbose output.
:param disable_progress_bar:
Disable the progress bar.
"""
self.config_file = config_file
self.comment = comment
self.verbose = verbose
self.disable_progress_bar = disable_progress_bar
self.skip_existing = skip_existing
self._handle_repository_options(
repository_name=repository_name,
repository_url=repository_url,
)
self.attestations = attestations
self._handle_package_signing(
sign=sign,
sign_with=sign_with,
identity=identity,
)
# _handle_certificates relies on the parsed repository config
self._handle_certificates(cacert, client_cert)
self.auth = auth.Resolver.choose(not non_interactive)(
self.repository_config,
auth.CredentialInput(username, password),
)
@property
def username(self) -> Optional[str]:
return self.auth.username
@property
def password(self) -> Optional[str]:
with self._allow_noninteractive():
return self.auth.password
def _allow_noninteractive(self) -> "contextlib.AbstractContextManager[None]":
"""Bypass NonInteractive error when client cert is present."""
suppressed = (exceptions.NonInteractive,) if self.client_cert else ()
return contextlib.suppress(*suppressed)
@property
def verbose(self) -> bool:
return self._verbose
@verbose.setter
def verbose(self, verbose: bool) -> None:
"""Initialize a logger based on the --verbose option."""
self._verbose = verbose
twine_logger = logging.getLogger("twine")
twine_logger.setLevel(logging.INFO if verbose else logging.WARNING)
@staticmethod
def register_argparse_arguments(parser: argparse.ArgumentParser) -> None:
"""Register the arguments for argparse."""
parser.add_argument(
"-r",
"--repository",
action=utils.EnvironmentDefault,
env="TWINE_REPOSITORY",
default="pypi",
help="The repository (package index) to upload the package to. "
"Should be a section in the config file (default: "
"%(default)s). (Can also be set via %(env)s environment "
"variable.)",
)
parser.add_argument(
"--repository-url",
action=utils.EnvironmentDefault,
env="TWINE_REPOSITORY_URL",
default=None,
required=False,
help="The repository (package index) URL to upload the package to."
" This overrides --repository. "
"(Can also be set via %(env)s environment variable.)",
)
parser.add_argument(
"--attestations",
action="store_true",
default=False,
help="Upload each file's associated attestations.",
)
parser.add_argument(
"-s",
"--sign",
action="store_true",
default=False,
help="Sign files to upload using GPG.",
)
parser.add_argument(
"--sign-with",
default="gpg",
help="GPG program used to sign uploads (default: %(default)s).",
)
parser.add_argument(
"-i",
"--identity",
help="GPG identity used to sign files.",
)
parser.add_argument(
"-u",
"--username",
action=utils.EnvironmentDefault,
env="TWINE_USERNAME",
required=False,
help="The username to authenticate to the repository "
"(package index) as. (Can also be set via "
"%(env)s environment variable.)",
)
parser.add_argument(
"-p",
"--password",
action=utils.EnvironmentDefault,
env="TWINE_PASSWORD",
required=False,
help="The password to authenticate to the repository "
"(package index) with. (Can also be set via "
"%(env)s environment variable.)",
)
parser.add_argument(
"--non-interactive",
action=utils.EnvironmentFlag,
env="TWINE_NON_INTERACTIVE",
help="Do not interactively prompt for username/password if the "
"required credentials are missing. (Can also be set via "
"%(env)s environment variable.)",
)
parser.add_argument(
"-c",
"--comment",
help="The comment to include with the distribution file.",
)
parser.add_argument(
"--config-file",
default=utils.DEFAULT_CONFIG_FILE,
help="The .pypirc config file to use.",
)
parser.add_argument(
"--skip-existing",
default=False,
action="store_true",
help="Continue uploading files if one already exists. (Only valid "
"when uploading to PyPI. Other implementations may not "
"support this.)",
)
parser.add_argument(
"--cert",
action=utils.EnvironmentDefault,
env="TWINE_CERT",
default=None,
required=False,
metavar="path",
help="Path to alternate CA bundle (can also be set via %(env)s "
"environment variable).",
)
parser.add_argument(
"--client-cert",
metavar="path",
help="Path to SSL client certificate, a single file containing the"
" private key and the certificate in PEM format.",
)
parser.add_argument(
"--verbose",
default=False,
required=False,
action="store_true",
help="Show verbose output.",
)
parser.add_argument(
"--disable-progress-bar",
default=False,
required=False,
action="store_true",
help="Disable the progress bar.",
)
@classmethod
def from_argparse(cls, args: argparse.Namespace) -> "Settings":
"""Generate the Settings from parsed arguments."""
settings = vars(args)
settings["repository_name"] = settings.pop("repository")
settings["cacert"] = settings.pop("cert")
return cls(**settings)
def _handle_package_signing(
self, sign: bool, sign_with: str, identity: Optional[str]
) -> None:
if not sign and identity:
raise exceptions.InvalidSigningConfiguration(
"sign must be given along with identity"
)
self.sign = sign
self.sign_with = sign_with
self.identity = identity
def _handle_repository_options(
self, repository_name: str, repository_url: Optional[str]
) -> None:
self.repository_config = utils.get_repository_from_config(
self.config_file,
repository_name,
repository_url,
)
def _handle_certificates(
self, cacert: Optional[str], client_cert: Optional[str]
) -> None:
self.cacert = utils.get_cacert(cacert, self.repository_config)
self.client_cert = utils.get_clientcert(client_cert, self.repository_config)
def check_repository_url(self) -> None:
"""Verify we are not using legacy PyPI.
:raises twine.exceptions.UploadToDeprecatedPyPIDetected:
The configured repository URL is for legacy PyPI.
"""
repository_url = cast(str, self.repository_config["repository"])
if repository_url.startswith(
(repository.LEGACY_PYPI, repository.LEGACY_TEST_PYPI)
):
raise exceptions.UploadToDeprecatedPyPIDetected.from_args(
repository_url, utils.DEFAULT_REPOSITORY, utils.TEST_REPOSITORY
)
def create_repository(self) -> repository.Repository:
"""Create a new repository for uploading."""
repo = repository.Repository(
cast(str, self.repository_config["repository"]),
self.username,
self.password,
self.disable_progress_bar,
)
repo.set_certificate_authority(self.cacert)
repo.set_client_certificate(self.client_cert)
return repo

View File

@@ -0,0 +1,354 @@
# Copyright 2013 Donald Stufft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import collections
import configparser
import functools
import logging
import os
import os.path
import unicodedata
from typing import Any, Callable, DefaultDict, Dict, Optional, Sequence, Union, cast
from urllib.parse import urlparse
from urllib.parse import urlunparse
import requests
import rfc3986
from twine import exceptions
# Shim for input to allow testing.
input_func = input
DEFAULT_REPOSITORY = "https://upload.pypi.org/legacy/"
TEST_REPOSITORY = "https://test.pypi.org/legacy/"
DEFAULT_CONFIG_FILE = "~/.pypirc"
# TODO: In general, it seems to be assumed that the values retrieved from
# instances of this type aren't None, except for username and password.
# Type annotations would be cleaner if this were Dict[str, str], but that
# requires reworking the username/password handling, probably starting with
# get_userpass_value.
RepositoryConfig = Dict[str, Optional[str]]
logger = logging.getLogger(__name__)
def get_config(path: str) -> Dict[str, RepositoryConfig]:
"""Read repository configuration from a file (i.e. ~/.pypirc).
Format: https://packaging.python.org/specifications/pypirc/
If the default config file doesn't exist, return a default configuration for
pypyi and testpypi.
"""
realpath = os.path.realpath(os.path.expanduser(path))
parser = configparser.RawConfigParser()
try:
with open(realpath) as f:
parser.read_file(f)
logger.info(f"Using configuration from {realpath}")
except FileNotFoundError:
# User probably set --config-file, but the file can't be read
if path != DEFAULT_CONFIG_FILE:
raise
# server-login is obsolete, but retained for backwards compatibility
defaults: RepositoryConfig = {
"username": parser.get("server-login", "username", fallback=None),
"password": parser.get("server-login", "password", fallback=None),
}
config: DefaultDict[str, RepositoryConfig]
config = collections.defaultdict(lambda: defaults.copy())
index_servers = parser.get(
"distutils", "index-servers", fallback="pypi testpypi"
).split()
# Don't require users to manually configure URLs for these repositories
config["pypi"]["repository"] = DEFAULT_REPOSITORY
if "testpypi" in index_servers:
config["testpypi"]["repository"] = TEST_REPOSITORY
# Optional configuration values for individual repositories
for repository in index_servers:
for key in [
"username",
"repository",
"password",
"ca_cert",
"client_cert",
]:
if parser.has_option(repository, key):
config[repository][key] = parser.get(repository, key)
# Convert the defaultdict to a regular dict to prevent surprising behavior later on
return dict(config)
def sanitize_url(url: str) -> str:
"""Sanitize a URL.
Sanitize URLs, removing any user:password combinations and replacing them with
asterisks. Returns the original URL if the string is a non-matching pattern.
:param url:
str containing a URL to sanitize.
return:
str either sanitized or as entered depending on pattern match.
"""
uri = rfc3986.urlparse(url)
if uri.userinfo:
return cast(str, uri.copy_with(userinfo="*" * 8).unsplit())
return url
def _validate_repository_url(repository_url: str) -> None:
"""Validate the given url for allowed schemes and components."""
# Allowed schemes are http and https, based on whether the repository
# supports TLS or not, and scheme and host must be present in the URL
validator = (
rfc3986.validators.Validator()
.allow_schemes("http", "https")
.require_presence_of("scheme", "host")
)
try:
validator.validate(rfc3986.uri_reference(repository_url))
except rfc3986.exceptions.RFC3986Exception as exc:
raise exceptions.UnreachableRepositoryURLDetected(
f"Invalid repository URL: {exc.args[0]}."
)
def get_repository_from_config(
config_file: str,
repository: str,
repository_url: Optional[str] = None,
) -> RepositoryConfig:
"""Get repository config command-line values or the .pypirc file."""
# Prefer CLI `repository_url` over `repository` or .pypirc
if repository_url:
_validate_repository_url(repository_url)
return _config_from_repository_url(repository_url)
try:
config = get_config(config_file)[repository]
except OSError as exc:
raise exceptions.InvalidConfiguration(str(exc))
except KeyError:
raise exceptions.InvalidConfiguration(
f"Missing '{repository}' section from {config_file}.\n"
f"More info: https://packaging.python.org/specifications/pypirc/ "
)
config["repository"] = normalize_repository_url(cast(str, config["repository"]))
return config
_HOSTNAMES = {
"pypi.python.org",
"testpypi.python.org",
"upload.pypi.org",
"test.pypi.org",
}
def _config_from_repository_url(url: str) -> RepositoryConfig:
parsed = urlparse(url)
config = {"repository": url, "username": None, "password": None}
if parsed.username:
config["username"] = parsed.username
config["password"] = parsed.password
config["repository"] = cast(
str, rfc3986.urlparse(url).copy_with(userinfo=None).unsplit()
)
config["repository"] = normalize_repository_url(cast(str, config["repository"]))
return config
def normalize_repository_url(url: str) -> str:
parsed = urlparse(url)
if parsed.netloc in _HOSTNAMES:
return urlunparse(("https",) + parsed[1:])
return urlunparse(parsed)
def get_file_size(filename: str) -> str:
"""Return the size of a file in KB, or MB if >= 1024 KB."""
file_size = os.path.getsize(filename) / 1024
size_unit = "KB"
if file_size > 1024:
file_size = file_size / 1024
size_unit = "MB"
return f"{file_size:.1f} {size_unit}"
def check_status_code(response: requests.Response, verbose: bool) -> None:
"""Generate a helpful message based on the response from the repository.
Raise a custom exception for recognized errors. Otherwise, print the
response content (based on the verbose option) before re-raising the
HTTPError.
"""
if response.status_code == 410 and "pypi.python.org" in response.url:
raise exceptions.UploadToDeprecatedPyPIDetected(
f"It appears you're uploading to pypi.python.org (or "
f"testpypi.python.org). You've received a 410 error response. "
f"Uploading to those sites is deprecated. The new sites are "
f"pypi.org and test.pypi.org. Try using {DEFAULT_REPOSITORY} (or "
f"{TEST_REPOSITORY}) to upload your packages instead. These are "
f"the default URLs for Twine now. More at "
f"https://packaging.python.org/guides/migrating-to-pypi-org/."
)
elif response.status_code == 405 and "pypi.org" in response.url:
raise exceptions.InvalidPyPIUploadURL(
f"It appears you're trying to upload to pypi.org but have an "
f"invalid URL. You probably want one of these two URLs: "
f"{DEFAULT_REPOSITORY} or {TEST_REPOSITORY}. Check your "
f"--repository-url value."
)
try:
response.raise_for_status()
except requests.HTTPError as err:
if not verbose:
logger.warning(
"Error during upload. "
"Retry with the --verbose option for more details."
)
raise err
def get_userpass_value(
cli_value: Optional[str],
config: RepositoryConfig,
key: str,
prompt_strategy: Optional[Callable[[], str]] = None,
) -> Optional[str]:
"""Get a credential (e.g. a username or password) from the configuration.
Uses the following rules:
1. If ``cli_value`` is specified, use that.
2. If ``config[key]`` is specified, use that.
3. If ``prompt_strategy`` is specified, use its return value.
4. Otherwise return ``None``
:param cli_value:
The value supplied from the command line.
:param config:
A dictionary of repository configuration values.
:param key:
The credential to look up in ``config``, e.g. ``"username"`` or ``"password"``.
:param prompt_strategy:
An argumentless function to get the value, e.g. from keyring or by prompting
the user.
:return:
The credential value, i.e. the username or password.
"""
if cli_value is not None:
logger.info(f"{key} set by command options")
return cli_value
elif config.get(key) is not None:
logger.info(f"{key} set from config file")
return config[key]
elif prompt_strategy:
warning = ""
value = prompt_strategy()
if not value:
warning = f"Your {key} is empty"
elif any(unicodedata.category(c).startswith("C") for c in value):
# See https://www.unicode.org/reports/tr44/#General_Category_Values
# Most common case is "\x16" when pasting in Windows Command Prompt
warning = f"Your {key} contains control characters"
if warning:
logger.warning(f"{warning}. Did you enter it correctly?")
logger.warning(
"See https://twine.readthedocs.io/#entering-credentials "
"for more information."
)
return value
else:
return None
#: Get the CA bundle via :func:`get_userpass_value`.
get_cacert = functools.partial(get_userpass_value, key="ca_cert")
#: Get the client certificate via :func:`get_userpass_value`.
get_clientcert = functools.partial(get_userpass_value, key="client_cert")
class EnvironmentDefault(argparse.Action):
"""Get values from environment variable."""
def __init__(
self,
env: str,
required: bool = True,
default: Optional[str] = None,
**kwargs: Any,
) -> None:
default = os.environ.get(env, default)
self.env = env
if default:
required = False
super().__init__(default=default, required=required, **kwargs)
def __call__(
self,
parser: argparse.ArgumentParser,
namespace: argparse.Namespace,
values: Union[str, Sequence[Any], None],
option_string: Optional[str] = None,
) -> None:
setattr(namespace, self.dest, values)
class EnvironmentFlag(argparse.Action):
"""Set boolean flag from environment variable."""
def __init__(self, env: str, **kwargs: Any) -> None:
default = self.bool_from_env(os.environ.get(env))
self.env = env
super().__init__(default=default, nargs=0, **kwargs)
def __call__(
self,
parser: argparse.ArgumentParser,
namespace: argparse.Namespace,
values: Union[str, Sequence[Any], None],
option_string: Optional[str] = None,
) -> None:
setattr(namespace, self.dest, True)
@staticmethod
def bool_from_env(val: Optional[str]) -> bool:
"""Allow '0' and 'false' and 'no' to be False."""
falsey = {"0", "false", "no"}
return bool(val and val.lower() not in falsey)

View File

@@ -0,0 +1,99 @@
# Copyright 2013 Donald Stufft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import io
import os
import re
import zipfile
from typing import List, Optional
from typing import cast as type_cast
from pkginfo import distribution
from twine import exceptions
# Monkeypatch Metadata 2.0 support
distribution.HEADER_ATTRS_2_0 = distribution.HEADER_ATTRS_1_2
distribution.HEADER_ATTRS.update({"2.0": distribution.HEADER_ATTRS_2_0})
wheel_file_re = re.compile(
r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>\d.+?))?)
((-(?P<build>\d.*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
\.whl|\.dist-info)$""",
re.VERBOSE,
)
class Wheel(distribution.Distribution):
def __init__(self, filename: str, metadata_version: Optional[str] = None) -> None:
self.filename = filename
self.basefilename = os.path.basename(self.filename)
self.metadata_version = metadata_version
self.extractMetadata()
@property
def py_version(self) -> str:
wheel_info = wheel_file_re.match(self.basefilename)
if wheel_info is None:
return "any"
else:
return wheel_info.group("pyver")
@staticmethod
def find_candidate_metadata_files(names: List[str]) -> List[List[str]]:
"""Filter files that may be METADATA files."""
tuples = [x.split("/") for x in names if "METADATA" in x]
return [x[1] for x in sorted((len(x), x) for x in tuples)]
def read(self) -> bytes:
fqn = os.path.abspath(os.path.normpath(self.filename))
if not os.path.exists(fqn):
raise exceptions.InvalidDistribution("No such file: %s" % fqn)
if fqn.endswith(".whl"):
archive = zipfile.ZipFile(fqn)
names = archive.namelist()
def read_file(name: str) -> bytes:
return archive.read(name)
else:
raise exceptions.InvalidDistribution(
"Not a known archive format for file: %s" % fqn
)
searched_files: List[str] = []
try:
for path in self.find_candidate_metadata_files(names):
candidate = "/".join(path)
data = read_file(candidate)
if b"Metadata-Version" in data:
return data
searched_files.append(candidate)
finally:
archive.close()
raise exceptions.InvalidDistribution(
"No METADATA in archive or METADATA missing 'Metadata-Version': "
"%s (searched %s)" % (fqn, ",".join(searched_files))
)
def parse(self, data: bytes) -> None:
super().parse(data)
fp = io.StringIO(data.decode("utf-8", errors="replace"))
# msg is ``email.message.Message`` which is a legacy API documented
# here: https://docs.python.org/3/library/email.compat32-message.html
msg = distribution.parse(fp)
self.description = type_cast(str, msg.get_payload())

View File

@@ -0,0 +1,59 @@
import os
import re
import zipfile
from typing import Optional
from pkginfo import distribution
from twine import exceptions
wininst_file_re = re.compile(r".*py(?P<pyver>\d+\.\d+)\.exe$")
class WinInst(distribution.Distribution):
def __init__(self, filename: str, metadata_version: Optional[str] = None) -> None:
self.filename = filename
self.metadata_version = metadata_version
self.extractMetadata()
@property
def py_version(self) -> str:
m = wininst_file_re.match(self.filename)
if m is None:
return "any"
else:
return m.group("pyver")
def read(self) -> bytes:
fqn = os.path.abspath(os.path.normpath(self.filename))
if not os.path.exists(fqn):
raise exceptions.InvalidDistribution("No such file: %s" % fqn)
if fqn.endswith(".exe"):
archive = zipfile.ZipFile(fqn)
names = archive.namelist()
def read_file(name: str) -> bytes:
return archive.read(name)
else:
raise exceptions.InvalidDistribution(
"Not a known archive format for file: %s" % fqn
)
try:
tuples = [
x.split("/") for x in names if x.endswith((".egg-info", "PKG-INFO"))
]
schwarz = sorted((len(x), x) for x in tuples)
for path in [x[1] for x in schwarz]:
candidate = "/".join(path)
data = read_file(candidate)
if b"Metadata-Version" in data:
return data
finally:
archive.close()
raise exceptions.InvalidDistribution(
"No PKG-INFO/.egg-info in archive: %s" % fqn
)