Skip to content

Commit f53bdbe

Browse files
LeonLuttenbergerLeonLuttenberger
authored
chore: ruff improvements (#2571)
1 parent b0bf30e commit f53bdbe

21 files changed

+82
-75
lines changed

‎.github/workflows/static-checking.yml

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,10 @@ jobs:
3232
python -m pip install poetry
3333
poetry config virtualenvs.create false --local
3434
poetry install --all-extras -vvv
35-
- name: Run ruff format
35+
- name: ruff format check
3636
run: ruff format --check .
37-
- name: Run ruff check on source code
38-
run: ruff awswrangler --output-format=github
39-
- name: Run ruff check on tests and tutorials
40-
run: ruff . --ignore "PL" --ignore "D" --output-format=github
37+
- name: ruff check
38+
run: ruff check --output-format=github .
4139
- name: mypy check
4240
run: mypy --install-types --non-interactive awswrangler
4341
- name: Pylint Lint

‎awswrangler/__init__.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
import logging as _logging
99

10-
from awswrangler import ( # noqa
10+
from awswrangler import (
1111
athena,
1212
catalog,
1313
chime,
@@ -34,9 +34,9 @@
3434
timestream,
3535
typing,
3636
)
37-
from awswrangler.__metadata__ import __description__, __license__, __title__, __version__ # noqa
38-
from awswrangler._config import config # noqa
39-
from awswrangler._distributed import EngineEnum, MemoryFormatEnum, engine, memory_format # noqa
37+
from awswrangler.__metadata__ import __description__, __license__, __title__, __version__
38+
from awswrangler._config import config
39+
from awswrangler._distributed import EngineEnum, MemoryFormatEnum, engine, memory_format
4040

4141
engine.register()
4242

‎awswrangler/athena/__init__.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,13 @@
1212
delete_prepared_statement,
1313
list_prepared_statements,
1414
)
15-
from awswrangler.athena._read import ( # noqa
15+
from awswrangler.athena._read import (
1616
get_query_results,
1717
read_sql_query,
1818
read_sql_table,
1919
unload,
2020
)
21-
from awswrangler.athena._utils import ( # noqa
21+
from awswrangler.athena._utils import (
2222
create_athena_bucket,
2323
create_ctas_table,
2424
describe_table,

‎awswrangler/catalog/__init__.py

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,7 @@
77
add_orc_partitions,
88
add_parquet_partitions,
99
)
10-
11-
# noqa
12-
from awswrangler.catalog._create import ( # noqa
10+
from awswrangler.catalog._create import (
1311
_create_csv_table,
1412
_create_json_table,
1513
_create_parquet_table,
@@ -21,14 +19,14 @@
2119
overwrite_table_parameters,
2220
upsert_table_parameters,
2321
)
24-
from awswrangler.catalog._delete import ( # noqa
22+
from awswrangler.catalog._delete import (
2523
delete_all_partitions,
2624
delete_column,
2725
delete_database,
2826
delete_partitions,
2927
delete_table_if_exists,
3028
)
31-
from awswrangler.catalog._get import ( # noqa
29+
from awswrangler.catalog._get import (
3230
_get_table_input,
3331
databases,
3432
get_columns_comments,
@@ -48,7 +46,7 @@
4846
table,
4947
tables,
5048
)
51-
from awswrangler.catalog._utils import ( # noqa
49+
from awswrangler.catalog._utils import (
5250
does_table_exist,
5351
drop_duplicated_columns,
5452
extract_athena_types,

‎awswrangler/distributed/ray/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
"""Ray Module."""
22

3-
from awswrangler.distributed.ray._core import RayLogger, initialize_ray, ray_get, ray_logger, ray_remote # noqa
3+
from awswrangler.distributed.ray._core import RayLogger, initialize_ray, ray_get, ray_logger, ray_remote
44

55
__all__ = [
66
"RayLogger",

‎awswrangler/distributed/ray/datasources/arrow_parquet_datasource.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
import numpy as np
1414

1515
# fs required to implicitly trigger S3 subsystem initialization
16-
import pyarrow.fs # noqa: F401 pylint: disable=unused-import
16+
import pyarrow.fs # pylint: disable=unused-import
1717
from pyarrow.dataset import ParquetFileFragment
1818
from pyarrow.lib import Schema
1919
from ray import cloudpickle

‎awswrangler/emr.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ def _build_cluster_args(**pars: Any) -> Dict[str, Any]: # pylint: disable=too-m
245245
{
246246
"Classification": "spark-hive-site",
247247
"Properties": {
248-
"hive.metastore.client.factory.class": "com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory" # noqa
248+
"hive.metastore.client.factory.class": "com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory"
249249
},
250250
"Configurations": [],
251251
}

‎awswrangler/lakeformation/__init__.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
"""Amazon Lake Formation Module."""
22

3-
from awswrangler.lakeformation._read import read_sql_query, read_sql_table # noqa
4-
from awswrangler.lakeformation._utils import ( # noqa
3+
from awswrangler.lakeformation._read import read_sql_query, read_sql_table
4+
from awswrangler.lakeformation._utils import (
55
_build_table_objects,
66
_get_table_objects,
77
_update_table_objects,

‎awswrangler/pandas/__init__.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
from pandas import * # noqa: F403
99

1010
# Explicit import because mypy doesn't support forward references to a star import
11-
from pandas import ( # noqa: F401
11+
from pandas import (
1212
DataFrame,
1313
Series,
1414
concat,
@@ -24,7 +24,7 @@
2424
from modin.pandas import * # noqa: F403
2525

2626
# Explicit import because mypy doesn't support forward references to a star import
27-
from modin.pandas import ( # noqa: F401
27+
from modin.pandas import (
2828
DataFrame,
2929
Series,
3030
concat,

‎awswrangler/quicksight/__init__.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
"""Amazon QuickSight Module."""
22

3-
from awswrangler.quicksight._cancel import cancel_ingestion # noqa
4-
from awswrangler.quicksight._create import create_athena_data_source, create_athena_dataset, create_ingestion # noqa
5-
from awswrangler.quicksight._delete import ( # noqa
3+
from awswrangler.quicksight._cancel import cancel_ingestion
4+
from awswrangler.quicksight._create import create_athena_data_source, create_athena_dataset, create_ingestion
5+
from awswrangler.quicksight._delete import (
66
delete_all_dashboards,
77
delete_all_data_sources,
88
delete_all_datasets,
@@ -12,14 +12,14 @@
1212
delete_dataset,
1313
delete_template,
1414
)
15-
from awswrangler.quicksight._describe import ( # noqa
15+
from awswrangler.quicksight._describe import (
1616
describe_dashboard,
1717
describe_data_source,
1818
describe_data_source_permissions,
1919
describe_dataset,
2020
describe_ingestion,
2121
)
22-
from awswrangler.quicksight._get_list import ( # noqa
22+
from awswrangler.quicksight._get_list import (
2323
get_dashboard_id,
2424
get_dashboard_ids,
2525
get_data_source_arn,

‎awswrangler/s3/__init__.py

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,23 @@
11
"""Amazon S3 Read Module."""
22

3-
from awswrangler.s3._copy import copy_objects, merge_datasets # noqa
4-
from awswrangler.s3._delete import delete_objects # noqa
5-
from awswrangler.s3._describe import describe_objects, get_bucket_region, size_objects # noqa
6-
from awswrangler.s3._download import download # noqa
7-
from awswrangler.s3._list import does_object_exist, list_buckets, list_directories, list_objects # noqa
8-
from awswrangler.s3._read_deltalake import read_deltalake # noqa
9-
from awswrangler.s3._read_excel import read_excel # noqa
10-
from awswrangler.s3._read_orc import read_orc, read_orc_metadata, read_orc_table # noqa
11-
from awswrangler.s3._read_parquet import read_parquet, read_parquet_metadata, read_parquet_table # noqa
12-
from awswrangler.s3._read_text import read_csv, read_fwf, read_json # noqa
3+
from awswrangler.s3._copy import copy_objects, merge_datasets
4+
from awswrangler.s3._delete import delete_objects
5+
from awswrangler.s3._describe import describe_objects, get_bucket_region, size_objects
6+
from awswrangler.s3._download import download
7+
from awswrangler.s3._list import does_object_exist, list_buckets, list_directories, list_objects
8+
from awswrangler.s3._read_deltalake import read_deltalake
9+
from awswrangler.s3._read_excel import read_excel
10+
from awswrangler.s3._read_orc import read_orc, read_orc_metadata, read_orc_table
11+
from awswrangler.s3._read_parquet import read_parquet, read_parquet_metadata, read_parquet_table
12+
from awswrangler.s3._read_text import read_csv, read_fwf, read_json
1313
from awswrangler.s3._select import select_query
14-
from awswrangler.s3._upload import upload # noqa
15-
from awswrangler.s3._wait import wait_objects_exist, wait_objects_not_exist # noqa
16-
from awswrangler.s3._write_deltalake import to_deltalake # noqa
17-
from awswrangler.s3._write_excel import to_excel # noqa
18-
from awswrangler.s3._write_orc import to_orc # noqa
19-
from awswrangler.s3._write_parquet import store_parquet_metadata, to_parquet # noqa
20-
from awswrangler.s3._write_text import to_csv, to_json # noqa
14+
from awswrangler.s3._upload import upload
15+
from awswrangler.s3._wait import wait_objects_exist, wait_objects_not_exist
16+
from awswrangler.s3._write_deltalake import to_deltalake
17+
from awswrangler.s3._write_excel import to_excel
18+
from awswrangler.s3._write_orc import to_orc
19+
from awswrangler.s3._write_parquet import store_parquet_metadata, to_parquet
20+
from awswrangler.s3._write_text import to_csv, to_json
2121

2222
__all__ = [
2323
"copy_objects",

‎awswrangler/s3/_write_dataset.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ def _to_partitions(
155155
subgroup.drop(
156156
columns=[col for col in partition_cols if col in subgroup.columns],
157157
inplace=True,
158-
) # noqa: PLW2901
158+
)
159159
# Drop index levels if partitioning by index columns
160160
subgroup = subgroup.droplevel( # noqa: PLW2901
161161
level=[col for col in partition_cols if col in subgroup.index.names]

‎fix.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22
set -ex
33

44
ruff format .
5-
ruff --fix awswrangler
5+
ruff check --fix .

‎pyproject.toml

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -132,9 +132,9 @@ requires = ["poetry-core>=1.0.0"]
132132
build-backend = "poetry.core.masonry.api"
133133

134134
[tool.ruff]
135-
select = ["D", "E", "F", "I001", "I002", "PL", "W"]
135+
select = ["D", "E", "F", "I", "PL", "RUF100", "W"]
136136
ignore = ["E501", "PLR2004", "PLR0911", "PLR0912", "PLR0913", "PLR0915"]
137-
fixable = ["I001", "I002", "W291"]
137+
fixable = ["ALL"]
138138
extend-include = ["*.ipynb"]
139139
exclude = [
140140
".eggs",
@@ -152,14 +152,17 @@ exclude = [
152152
line-length = 120
153153
target-version = "py38"
154154

155-
[tool.ruff.lint]
156-
exclude = ["*.ipynb"]
155+
[tool.ruff.lint.per-file-ignores]
156+
"docs/*" = ["D"]
157+
"test_infra/*" = ["D"]
158+
"tests/*" = ["PL", "D"]
159+
"tutorials/*" = ["D", "E402", "F401", "F811", "F821"]
157160

158161
[tool.ruff.pydocstyle]
159162
convention = "numpy"
160163

161164
[tool.mypy]
162-
python_version = 3.8
165+
python_version = "3.8"
163166
strict = true
164167
ignore_missing_imports = true
165168
warn_unused_ignores = true

‎test_infra/stacks/databases_stack.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -139,11 +139,11 @@ def _set_catalog_encryption(self) -> None:
139139
self,
140140
"aws-sdk-pandas-catalog-encryption",
141141
catalog_id=f"{Aws.ACCOUNT_ID}",
142-
data_catalog_encryption_settings=CfnDataCatalogEncryptionSettings.DataCatalogEncryptionSettingsProperty( # noqa: E501
142+
data_catalog_encryption_settings=CfnDataCatalogEncryptionSettings.DataCatalogEncryptionSettingsProperty(
143143
encryption_at_rest=CfnDataCatalogEncryptionSettings.EncryptionAtRestProperty(
144144
catalog_encryption_mode="DISABLED",
145145
),
146-
connection_password_encryption=CfnDataCatalogEncryptionSettings.ConnectionPasswordEncryptionProperty( # noqa: E501
146+
connection_password_encryption=CfnDataCatalogEncryptionSettings.ConnectionPasswordEncryptionProperty(
147147
kms_key_id=self.key.key_id,
148148
return_connection_password_encrypted=True,
149149
),
@@ -270,7 +270,7 @@ def _setup_redshift(self) -> None:
270270
type=glue.ConnectionType.JDBC,
271271
connection_name="aws-sdk-pandas-redshift",
272272
properties={
273-
"JDBC_CONNECTION_URL": f"jdbc:redshift://{redshift_cluster.cluster_endpoint.hostname}:{port}/{database}", # noqa: E501
273+
"JDBC_CONNECTION_URL": f"jdbc:redshift://{redshift_cluster.cluster_endpoint.hostname}:{port}/{database}",
274274
"USERNAME": self.db_username,
275275
"PASSWORD": self.db_password,
276276
},
@@ -663,7 +663,7 @@ def _setup_sqlserver(self) -> None:
663663
type=glue.ConnectionType.JDBC,
664664
connection_name="aws-sdk-pandas-sqlserver",
665665
properties={
666-
"JDBC_CONNECTION_URL": f"jdbc:sqlserver://{sqlserver.instance_endpoint.hostname}:{port};databaseName={database}", # noqa: E501
666+
"JDBC_CONNECTION_URL": f"jdbc:sqlserver://{sqlserver.instance_endpoint.hostname}:{port};databaseName={database}",
667667
"USERNAME": self.db_username,
668668
"PASSWORD": self.db_password,
669669
},
@@ -725,7 +725,7 @@ def _setup_oracle(self) -> None:
725725
type=glue.ConnectionType.JDBC,
726726
connection_name="aws-sdk-pandas-oracle",
727727
properties={
728-
"JDBC_CONNECTION_URL": f"jdbc:oracle:thin://@{oracle.instance_endpoint.hostname}:{port}/{database}", # noqa: E501
728+
"JDBC_CONNECTION_URL": f"jdbc:oracle:thin://@{oracle.instance_endpoint.hostname}:{port}/{database}",
729729
"USERNAME": self.db_username,
730730
"PASSWORD": self.db_password,
731731
},

‎tests/unit/test_athena_parquet.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -779,12 +779,12 @@ def test_cast_decimal(path, glue_table, glue_database):
779779
def test_splits():
780780
s = "a:struct<id:string,name:string>,b:struct<id:string,name:string>"
781781
assert list(_split_fields(s)) == ["a:struct<id:string,name:string>", "b:struct<id:string,name:string>"]
782-
s = "a:struct<a:struct<id:string,name:string>,b:struct<id:string,name:string>>,b:struct<a:struct<id:string,name:string>,b:struct<id:string,name:string>>" # noqa
782+
s = "a:struct<a:struct<id:string,name:string>,b:struct<id:string,name:string>>,b:struct<a:struct<id:string,name:string>,b:struct<id:string,name:string>>"
783783
assert list(_split_fields(s)) == [
784784
"a:struct<a:struct<id:string,name:string>,b:struct<id:string,name:string>>",
785785
"b:struct<a:struct<id:string,name:string>,b:struct<id:string,name:string>>",
786786
]
787-
s = "a:struct<id:string,name:string>,b:struct<id:string,name:string>,c:struct<id:string,name:string>,d:struct<id:string,name:string>" # noqa
787+
s = "a:struct<id:string,name:string>,b:struct<id:string,name:string>,c:struct<id:string,name:string>,d:struct<id:string,name:string>"
788788
assert list(_split_fields(s)) == [
789789
"a:struct<id:string,name:string>",
790790
"b:struct<id:string,name:string>",

‎tutorials/017 - Partition Projection.ipynb

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@
5454
},
5555
"outputs": [
5656
{
57-
"name": "stdin",
57+
"name": "stdout",
5858
"output_type": "stream",
5959
"text": [
6060
" ···········································\n"
@@ -255,7 +255,7 @@
255255
}
256256
],
257257
"source": [
258-
"wr.athena.read_sql_query(f\"SELECT * FROM table_integer\", database=\"default\")"
258+
"wr.athena.read_sql_query(\"SELECT * FROM table_integer\", database=\"default\")"
259259
]
260260
},
261261
{
@@ -439,7 +439,7 @@
439439
}
440440
],
441441
"source": [
442-
"wr.athena.read_sql_query(f\"SELECT * FROM table_enum\", database=\"default\")"
442+
"wr.athena.read_sql_query(\"SELECT * FROM table_enum\", database=\"default\")"
443443
]
444444
},
445445
{
@@ -524,8 +524,13 @@
524524
}
525525
],
526526
"source": [
527-
"ts = lambda x: datetime.strptime(x, \"%Y-%m-%d %H:%M:%S\")\n",
528-
"dt = lambda x: datetime.strptime(x, \"%Y-%m-%d\").date()\n",
527+
"def ts(x):\n",
528+
" return datetime.strptime(x, \"%Y-%m-%d %H:%M:%S\")\n",
529+
"\n",
530+
"\n",
531+
"def dt(x):\n",
532+
" return datetime.strptime(x, \"%Y-%m-%d\").date()\n",
533+
"\n",
529534
"\n",
530535
"df = pd.DataFrame(\n",
531536
" {\n",
@@ -636,7 +641,7 @@
636641
}
637642
],
638643
"source": [
639-
"wr.athena.read_sql_query(f\"SELECT * FROM table_date\", database=\"default\")"
644+
"wr.athena.read_sql_query(\"SELECT * FROM table_date\", database=\"default\")"
640645
]
641646
},
642647
{
@@ -812,7 +817,7 @@
812817
],
813818
"source": [
814819
"wr.athena.read_sql_query(\n",
815-
" sql=f\"SELECT * FROM table_injected WHERE uuid='b89ed095-8179-4635-9537-88592c0f6bc3'\", database=\"default\"\n",
820+
" sql=\"SELECT * FROM table_injected WHERE uuid='b89ed095-8179-4635-9537-88592c0f6bc3'\", database=\"default\"\n",
816821
")"
817822
]
818823
},
@@ -892,4 +897,4 @@
892897
},
893898
"nbformat": 4,
894899
"nbformat_minor": 4
895-
}
900+
}

‎tutorials/022 - Writing Partitions Concurrently.ipynb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
"metadata": {},
4141
"outputs": [
4242
{
43-
"name": "stdin",
43+
"name": "stdout",
4444
"output_type": "stream",
4545
"text": [
4646
" ············\n"

0 commit comments

Comments
 (0)