Skip to content

Commit e1c30db

Browse files
Haris-mslmHaris-mslm
committed
added line limit for code
1 parent 8bffe46 commit e1c30db

File tree

15 files changed

+164
-53
lines changed

15 files changed

+164
-53
lines changed

‎scripts/fmt.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@
44
isort src
55

66
# Run black on the src directory
7-
black src
7+
black --line-length 79 src

‎src/oxylabs/internal/internal.py

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,10 @@ def __init__(self, base_url: str, api_credentials: APICredentials) -> None:
117117
}
118118

119119
async def get_job_id(
120-
self, payload: dict, user_session: aiohttp.ClientSession, request_timeout: int
120+
self,
121+
payload: dict,
122+
user_session: aiohttp.ClientSession,
123+
request_timeout: int,
121124
) -> str:
122125
try:
123126
async with user_session.post(
@@ -130,11 +133,15 @@ async def get_job_id(
130133
response.raise_for_status()
131134
return data["id"]
132135
except aiohttp.ClientResponseError as e:
133-
print(f"HTTP error occurred: {e.status} - {e.message} - {data['message']}")
136+
print(
137+
f"HTTP error occurred: {e.status} - {e.message} - {data['message']}"
138+
)
134139
except aiohttp.ClientConnectionError as e:
135140
print(f"Connection error occurred: {e}")
136141
except asyncio.TimeoutError:
137-
print(f"Timeout error. The request to {self.base_url} has timed out.")
142+
print(
143+
f"Timeout error. The request to {self.base_url} has timed out."
144+
)
138145
except Exception as e:
139146
print(f"Error occurred: {str(e)}")
140147
return None
@@ -187,12 +194,16 @@ async def get_http_resp(
187194
"""
188195
result_url = f"{self.base_url}/{job_id}/results"
189196
try:
190-
async with user_session.get(result_url, headers=self.headers) as response:
197+
async with user_session.get(
198+
result_url, headers=self.headers
199+
) as response:
191200
data = await response.json()
192201
response.raise_for_status()
193202
return data
194203
except aiohttp.ClientResponseError as e:
195-
print(f"HTTP error occurred: {e.status} - {e.message} - {data['message']}")
204+
print(
205+
f"HTTP error occurred: {e.status} - {e.message} - {data['message']}"
206+
)
196207
except aiohttp.ClientConnectionError as e:
197208
print(f"Connection error occurred: {e}")
198209
except asyncio.TimeoutError:

‎src/oxylabs/proxy/proxy.py

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,11 @@
33

44
import requests
55

6-
from src.oxylabs.utils.defaults import NON_UNIVERSAL_DOMAINS, PROXY_BASE_URL, PROXY_PORT
6+
from src.oxylabs.utils.defaults import (
7+
NON_UNIVERSAL_DOMAINS,
8+
PROXY_BASE_URL,
9+
PROXY_PORT,
10+
)
711
from src.oxylabs.utils.utils import prepare_config
812

913

@@ -52,7 +56,9 @@ def get(
5256
try:
5357
config = prepare_config(request_timeout=request_timeout)
5458
self._url_to_scrape = url
55-
response = self._session.get(url, timeout=config["request_timeout"])
59+
response = self._session.get(
60+
url, timeout=config["request_timeout"]
61+
)
5662
response.raise_for_status()
5763
return response
5864
except requests.exceptions.Timeout:
@@ -120,7 +126,9 @@ def add_parse_header(
120126
if parse or parsing_instructions:
121127
self._session.headers["x-oxylabs-parse"] = "1"
122128
if self._is_universal_source():
123-
self._session.headers["x-oxylabs-parser-type"] = "universal_ecommerce"
129+
self._session.headers["x-oxylabs-parser-type"] = (
130+
"universal_ecommerce"
131+
)
124132
else:
125133
self._session.headers.pop("x-oxylabs-parser-type", None)
126134
else:
@@ -135,7 +143,8 @@ def _is_universal_source(self) -> bool:
135143
"""
136144
parsed_url = urlparse(self._url_to_scrape)
137145
if any(
138-
domain in parsed_url.netloc.decode() for domain in NON_UNIVERSAL_DOMAINS
146+
domain in parsed_url.netloc.decode()
147+
for domain in NON_UNIVERSAL_DOMAINS
139148
):
140149
return False
141150

‎src/oxylabs/sources/ecommerce/amazon/amazon.py

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -341,7 +341,9 @@ async def scrape_search(
341341
async_integration=True,
342342
)
343343
payload = self._prepare_search_payload(query, opts)
344-
response = await self._ecommerce_async_instance._get_resp(payload, config)
344+
response = await self._ecommerce_async_instance._get_resp(
345+
payload, config
346+
)
345347
return response
346348

347349
async def scrape_url(
@@ -382,7 +384,9 @@ async def scrape_url(
382384
async_integration=True,
383385
)
384386
payload = self._prepare_url_payload(url, opts)
385-
response = await self._ecommerce_async_instance._get_resp(payload, config)
387+
response = await self._ecommerce_async_instance._get_resp(
388+
payload, config
389+
)
386390
return response
387391

388392
async def scrape_product(
@@ -425,7 +429,9 @@ async def scrape_product(
425429
async_integration=True,
426430
)
427431
payload = self._prepare_product_payload(query, opts)
428-
response = await self._ecommerce_async_instance._get_resp(payload, config)
432+
response = await self._ecommerce_async_instance._get_resp(
433+
payload, config
434+
)
429435
return response
430436

431437
async def scrape_pricing(
@@ -469,7 +475,9 @@ async def scrape_pricing(
469475
async_integration=True,
470476
)
471477
payload = self._prepare_pricing_payload(query, opts)
472-
response = await self._ecommerce_async_instance._get_resp(payload, config)
478+
response = await self._ecommerce_async_instance._get_resp(
479+
payload, config
480+
)
473481
return response
474482

475483
async def scrape_reviews(
@@ -513,7 +521,9 @@ async def scrape_reviews(
513521
async_integration=True,
514522
)
515523
payload = self._prepare_reviews_payload(query, opts)
516-
response = await self._ecommerce_async_instance._get_resp(payload, config)
524+
response = await self._ecommerce_async_instance._get_resp(
525+
payload, config
526+
)
517527
return response
518528

519529
async def scrape_questions(
@@ -555,7 +565,9 @@ async def scrape_questions(
555565
async_integration=True,
556566
)
557567
payload = self._prepare_questions_payload(query, opts)
558-
response = await self._ecommerce_async_instance._get_resp(payload, config)
568+
response = await self._ecommerce_async_instance._get_resp(
569+
payload, config
570+
)
559571
return response
560572

561573
async def scrape_bestsellers(
@@ -599,7 +611,9 @@ async def scrape_bestsellers(
599611
async_integration=True,
600612
)
601613
payload = self._prepare_bestseller_payload(query, opts)
602-
response = await self._ecommerce_async_instance._get_resp(payload, config)
614+
response = await self._ecommerce_async_instance._get_resp(
615+
payload, config
616+
)
603617
return response
604618

605619
async def scrape_sellers(
@@ -641,5 +655,7 @@ async def scrape_sellers(
641655
async_integration=True,
642656
)
643657
payload = self._prepare_seller_payload(query, opts)
644-
response = await self._ecommerce_async_instance._get_resp(payload, config)
658+
response = await self._ecommerce_async_instance._get_resp(
659+
payload, config
660+
)
645661
return response

‎src/oxylabs/sources/ecommerce/amazon/amazon_base.py

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,9 @@ class AmazonPricingOpts(BaseEcommerceOpts):
7777
Represents the product pricing options for AmazonPricing.
7878
"""
7979

80-
def __init__(self, domain=None, start_page=None, pages=None, parse=None, **kwargs):
80+
def __init__(
81+
self, domain=None, start_page=None, pages=None, parse=None, **kwargs
82+
):
8183
super().__init__(**kwargs)
8284
self.domain = domain
8385
self.start_page = start_page
@@ -99,7 +101,9 @@ class AmazonReviewsOpts(BaseEcommerceOpts):
99101
Represents the product review options for AmazonReview.
100102
"""
101103

102-
def __init__(self, domain=None, start_page=None, pages=None, parse=None, **kwargs):
104+
def __init__(
105+
self, domain=None, start_page=None, pages=None, parse=None, **kwargs
106+
):
103107
super().__init__(**kwargs)
104108
self.domain = domain
105109
self.start_page = start_page
@@ -139,7 +143,9 @@ class AmazonBestsellerOpts(BaseEcommerceOpts):
139143
Represents the bestseller options for AmazonBestseller.
140144
"""
141145

142-
def __init__(self, domain=None, start_page=None, pages=None, parse=None, **kwargs):
146+
def __init__(
147+
self, domain=None, start_page=None, pages=None, parse=None, **kwargs
148+
):
143149
super().__init__(**kwargs)
144150
self.domain = domain
145151
self.start_page = start_page
@@ -217,7 +223,9 @@ def _prepare_search_payload(
217223

218224
return payload
219225

220-
def _prepare_url_payload(self, url: str, user_opts: Optional[dict] = None) -> dict:
226+
def _prepare_url_payload(
227+
self, url: str, user_opts: Optional[dict] = None
228+
) -> dict:
221229
"""
222230
Prepares the payload for making a request to Amazon URL.
223231
@@ -376,7 +384,9 @@ def _prepare_questions_payload(
376384
Raises:
377385
ValueError: If the provided options are invalid.
378386
"""
379-
opts = AmazonQuestionsOpts(**user_opts if user_opts is not None else {})
387+
opts = AmazonQuestionsOpts(
388+
**user_opts if user_opts is not None else {}
389+
)
380390

381391
opts.check_parameter_validity()
382392

@@ -410,7 +420,9 @@ def _prepare_bestseller_payload(
410420
Returns:
411421
dict: The prepared payload for the request.
412422
"""
413-
opts = AmazonBestsellerOpts(**user_opts if user_opts is not None else {})
423+
opts = AmazonBestsellerOpts(
424+
**user_opts if user_opts is not None else {}
425+
)
414426

415427
opts.check_parameter_validity()
416428

‎src/oxylabs/sources/ecommerce/ecommerce.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,10 @@
33
from src.oxylabs.utils.defaults import ASYNC_BASE_URL, SYNC_BASE_URL
44

55
from .amazon.amazon import Amazon, AmazonAsync
6-
from .google_shopping.google_shopping import GoogleShopping, GoogleShoppingAsync
6+
from .google_shopping.google_shopping import (
7+
GoogleShopping,
8+
GoogleShoppingAsync,
9+
)
710
from .universal.universal import Universal, UniversalAsync
811
from .wayfair.wayfair import Wayfair, WayfairAsync
912

‎src/oxylabs/sources/ecommerce/google_shopping/google_shopping.py

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,9 @@ async def scrape_shopping_search(
202202
async_integration=True,
203203
)
204204
payload = self._prepare_shopping_search_payload(query, opts)
205-
response = await self._ecommerce_async_instance._get_resp(payload, config)
205+
response = await self._ecommerce_async_instance._get_resp(
206+
payload, config
207+
)
206208
return response
207209

208210
async def scrape_shopping_url(
@@ -240,7 +242,9 @@ async def scrape_shopping_url(
240242
async_integration=True,
241243
)
242244
payload = self._prepare_shopping_url_payload(url, opts)
243-
response = await self._ecommerce_async_instance._get_resp(payload, config)
245+
response = await self._ecommerce_async_instance._get_resp(
246+
payload, config
247+
)
244248
return response
245249

246250
async def scrape_shopping_products(
@@ -281,7 +285,9 @@ async def scrape_shopping_products(
281285
async_integration=True,
282286
)
283287
payload = self._prepare_shopping_product_payload(query, opts)
284-
response = await self._ecommerce_async_instance._get_resp(payload, config)
288+
response = await self._ecommerce_async_instance._get_resp(
289+
payload, config
290+
)
285291
return response
286292

287293
async def scrape_product_pricing(
@@ -324,5 +330,7 @@ async def scrape_product_pricing(
324330
async_integration=True,
325331
)
326332
payload = self._prepare_shopping_product_pricing_payload(query, opts)
327-
response = await self._ecommerce_async_instance._get_resp(payload, config)
333+
response = await self._ecommerce_async_instance._get_resp(
334+
payload, config
335+
)
328336
return response

‎src/oxylabs/sources/ecommerce/google_shopping/google_shopping_base.py

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,12 @@ class GoogleShoppingProductOpts(BaseEcommerceOpts):
7474
"""
7575

7676
def __init__(
77-
self, domain=None, locale=None, results_language=None, parse=None, **kwargs
77+
self,
78+
domain=None,
79+
locale=None,
80+
results_language=None,
81+
parse=None,
82+
**kwargs
7883
):
7984
super().__init__(**kwargs)
8085
self.domain = domain
@@ -138,7 +143,9 @@ def _prepare_shopping_search_payload(
138143
dict: The prepared search payload.
139144
140145
"""
141-
opts = GoogleShoppingSearchOpts(**user_opts if user_opts is not None else {})
146+
opts = GoogleShoppingSearchOpts(
147+
**user_opts if user_opts is not None else {}
148+
)
142149

143150
opts.check_parameter_validity()
144151

@@ -178,7 +185,9 @@ def _prepare_shopping_url_payload(
178185
dict: The prepared payload for the request.
179186
"""
180187
validate_url(url, "google")
181-
opts = GoogleShoppingUrlOpts(**user_opts if user_opts is not None else {})
188+
opts = GoogleShoppingUrlOpts(
189+
**user_opts if user_opts is not None else {}
190+
)
182191

183192
opts.check_parameter_validity()
184193

@@ -211,7 +220,9 @@ def _prepare_shopping_product_payload(
211220
Returns:
212221
dict: The prepared payload for the request.
213222
"""
214-
opts = GoogleShoppingProductOpts(**user_opts if user_opts is not None else {})
223+
opts = GoogleShoppingProductOpts(
224+
**user_opts if user_opts is not None else {}
225+
)
215226

216227
opts.check_parameter_validity()
217228

@@ -247,7 +258,9 @@ def _prepare_shopping_product_pricing_payload(
247258
Returns:
248259
dict: The prepared payload for the request.
249260
"""
250-
opts = GoogleProductPricingOpts(**user_opts if user_opts is not None else {})
261+
opts = GoogleProductPricingOpts(
262+
**user_opts if user_opts is not None else {}
263+
)
251264

252265
opts.check_parameter_validity()
253266

‎src/oxylabs/sources/ecommerce/universal/universal.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@ def __init__(self, ecommerce_instance) -> None:
1616
self._ecommerce_instance = ecommerce_instance
1717

1818
def scrape_url(
19-
self, url: str, opts: Optional[dict] = None, request_timeout: int = None
19+
self,
20+
url: str,
21+
opts: Optional[dict] = None,
22+
request_timeout: int = None,
2023
) -> dict:
2124
"""
2225
Scrapes Universal search results for a given URL.
@@ -101,5 +104,7 @@ async def scrape_url(
101104
async_integration=True,
102105
)
103106
payload = self._prepare_url_payload(url, opts)
104-
response = await self._ecommerce_async_instance._get_resp(payload, config)
107+
response = await self._ecommerce_async_instance._get_resp(
108+
payload, config
109+
)
105110
return response

0 commit comments

Comments
 (0)