Merge branch 'master' into bug/work-times-undef

myrontuttle · web-flow · commit 699757583cad · 2023-05-10T18:26:47.000-06:00
diff --git a/README.md b/README.md
@@ -59,6 +59,21 @@ First, you must set your chromedriver location by
 export CHROMEDRIVER=~/chromedriver
 ```
 
+## Sponsor
+[![rds-cost](https://raw.githubusercontent.com/joeyism/linkedin_scraper/master/docs/proxycurl.png)](https://nubela.co/proxycurl/?utm_campaign=influencer%20marketing&utm_source=github&utm_medium=social&utm_term=-&utm_content=joeyism)
+
+Scrape public LinkedIn profile data at scale with [Proxycurl APIs](https://nubela.co/proxycurl/?utm_campaign=influencer%20marketing&utm_source=github&utm_medium=social&utm_term=-&utm_content=joeyism).
+
+• Scraping Public profiles are battle tested in court in HiQ VS LinkedIn case.<br/>
+• GDPR, CCPA, SOC2 compliant<br/>
+• High rate limit - 300 requests/minute<br/>
+• Fast - APIs respond in ~2s<br/>
+• Fresh data - 88% of data is scraped real-time, other 12% are not older than 29 days<br/>
+• High accuracy<br/>
+• Tons of data points returned per profile
+
+Built for developers, by developers.
+
 ## Usage
 To use it, just create the class.
 
diff --git a/README.rst b/README.rst
@@ -76,6 +76,25 @@ First, you must set your chromedriver location by
 
    export CHROMEDRIVER=~/chromedriver
 
+Sponsor 
+-----
+
+.. image:: https://raw.githubusercontent.com/joeyism/linkedin_scraper/master/docs/proxycurl.png
+   :alt: Proxycurl API
+   :target: https://nubela.co/proxycurl/?utm_campaign=influencer%20marketing&utm_source=github&utm_medium=social&utm_term=-&utm_content=joeyism
+
+Scrape public LinkedIn profile data at scale with `Proxycurl APIs <https://nubela.co/proxycurl/?utm_campaign=influencer%20marketing&utm_source=github&utm_medium=social&utm_term=-&utm_content=joeyism>`_.
+
+• Scraping Public profiles are battle tested in court in HiQ VS LinkedIn case.
+• GDPR, CCPA, SOC2 compliant
+• High rate limit - 300 requests/minute
+• Fast - APIs respond in ~2s
+• Fresh data - 88% of data is scraped real-time, other 12% are not older than 29 days
+• High accuracy
+• Tons of data points returned per profile
+
+Built for developers, by developers.
+
 Usage
 -----
 
diff --git a/docs/proxycurl.png b/docs/proxycurl.png
diff --git a/linkedin_scraper/__init__.py b/linkedin_scraper/__init__.py
@@ -5,7 +5,7 @@
 from .jobs import Job
 from .job_search import JobSearch
 
-__version__ = "2.11.0"
+__version__ = "2.11.1"
 
 import glob
 modules = glob.glob(dirname(__file__)+"/*.py")
diff --git a/linkedin_scraper/job_search.py b/linkedin_scraper/job_search.py
@@ -36,8 +36,8 @@ def scrape_job_card(self, base_element) -> Job:
         job_div = self.wait_for_element_to_load(name="job-card-list__title", base=base_element)
         job_title = job_div.text.strip()
         linkedin_url = job_div.get_attribute("href")
-        company = base_element.find_element_by_class_name("job-card-container__primary-description")
-        location = base_element.find_element_by_class_name("job-card-container__metadata-item")
+        company = base_element.find_element_by_class_name("job-card-container__primary-description").text
+        location = base_element.find_element_by_class_name("job-card-container__metadata-item").text
         job = Job(linkedin_url=linkedin_url, job_title=job_title, company=company, location=location, scrape=False, driver=self.driver)
         return job
 
diff --git a/linkedin_scraper/jobs.py b/linkedin_scraper/jobs.py
@@ -40,14 +40,27 @@ def __init__(
             self.scrape(close_on_complete)
 
     def __repr__(self):
-        return f"{self.job_title} {self.company}"
+        return f"<Job {self.job_title} {self.company}>"
 
     def scrape(self, close_on_complete=True):
         if self.is_signed_in():
             self.scrape_logged_in(close_on_complete=close_on_complete)
         else:
             raise NotImplemented("This part is not implemented yet")
 
+    def to_dict(self):
+        return {
+            "linkedin_url": self.linkedin_url,
+            "job_title": self.job_title,
+            "company": self.company,
+            "company_linkedin_url": self.company_linkedin_url,
+            "location": self.location,
+            "posted_date": self.posted_date,
+            "applicant_count": self.applicant_count,
+            "job_description": self.job_description,
+            "benefits": self.benefits
+        }
+
 
     def scrape_logged_in(self, close_on_complete=True):
         driver = self.driver
@@ -63,8 +76,15 @@ def scrape_logged_in(self, close_on_complete=True):
             self.applicant_count = self.wait_for_element_to_load(name="jobs-unified-top-card__applicant-count").text.strip()
         except TimeoutException:
             self.applicant_count = 0
-        self.job_description = self.wait_for_element_to_load(name="jobs-description").text.strip()
-        self.benefits = self.wait_for_element_to_load(name="jobs-unified-description__salary-main-rail-card").text.strip()
+        job_description_elem = self.wait_for_element_to_load(name="jobs-description")
+        self.mouse_click(job_description_elem.find_element_by_tag_name("button"))
+        job_description_elem = self.wait_for_element_to_load(name="jobs-description")
+        job_description_elem.find_element_by_tag_name("button").click()
+        self.job_description = job_description_elem.text.strip()
+        try:
+            self.benefits = self.wait_for_element_to_load(name="jobs-unified-description__salary-main-rail-card").text.strip()
+        except TimeoutException:
+            self.benefits = None
 
         if close_on_complete:
             driver.close()
diff --git a/linkedin_scraper/objects.py b/linkedin_scraper/objects.py
@@ -5,6 +5,7 @@
 
 from . import constants as c
 
+from selenium import webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support.wait import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
@@ -72,6 +73,10 @@ def focus(self):
         self.driver.execute_script('alert("Focus window")')
         self.driver.switch_to.alert.accept()
 
+    def mouse_click(self, elem):
+        action = webdriver.ActionChains(self.driver)
+        action.move_to_element(elem).perform()
+
     def wait_for_element_to_load(self, by=By.CLASS_NAME, name="pv-top-card", base=None):
         base = base or self.driver
         return WebDriverWait(base, self.WAIT_FOR_ELEMENT_TIMEOUT).until(
diff --git a/linkedin_scraper/person.py b/linkedin_scraper/person.py
@@ -102,67 +102,66 @@ def _click_see_more_by_class_name(self, class_name):
 
     def is_open_to_work(self):
         try:
-            return "#OPEN_TO_WORK" in self.driver.find_element_by_class_name("pv-top-card-profile-picture").find_element_by_tag_name("img").get_attribute("title")
+            return "#OPEN_TO_WORK" in self.driver.find_element(By.CLASS_NAME,"pv-top-card-profile-picture").find_element(By.TAG_NAME,"img").get_attribute("title")
         except:
             return False
 
     def get_experiences(self):
         url = os.path.join(self.linkedin_url, "details/experience")
         self.driver.get(url)
         self.focus()
-        main = self.wait_for_element_to_load(by=By.ID, name="main")
+        main = self.wait_for_element_to_load(by=By.TAG_NAME, name="main")
         self.scroll_to_half()
         self.scroll_to_bottom()
         main_list = self.wait_for_element_to_load(name="pvs-list", base=main)
-        for position in main_list.find_elements_by_xpath("li"):
-            position = position.find_element_by_class_name("pvs-entity")
-            company_logo_elem, position_details = position.find_elements_by_xpath("*")
+        for position in main_list.find_elements(By.XPATH,"li"):
+            position = position.find_element(By.CLASS_NAME,"pvs-entity")
+            company_logo_elem, position_details = position.find_elements(By.XPATH,"*")
 
             # company elem
-            company_linkedin_url = company_logo_elem.find_element_by_xpath("*").get_attribute("href")
+            company_linkedin_url = company_logo_elem.find_element(By.XPATH,"*").get_attribute("href")
 
             # position details
-            position_details_list = position_details.find_elements_by_xpath("*")
+            position_details_list = position_details.find_elements(By.XPATH,"*")
             position_summary_details = position_details_list[0] if len(position_details_list) > 0 else None
             position_summary_text = position_details_list[1] if len(position_details_list) > 1 else None
-            outer_positions = position_summary_details.find_element_by_xpath("*").find_elements_by_xpath("*")
+            outer_positions = position_summary_details.find_element(By.XPATH,"*").find_elements(By.XPATH,"*")
 
             if len(outer_positions) == 4:
-                position_title = outer_positions[0].find_element_by_tag_name("span").find_element_by_tag_name("span").text
-                company = outer_positions[1].find_element_by_tag_name("span").text
-                work_times = outer_positions[2].find_element_by_tag_name("span").text
-                location = outer_positions[3].find_element_by_tag_name("span").text
+                position_title = outer_positions[0].find_element(By.TAG_NAME,"span").find_element(By.TAG_NAME,"span").text
+                company = outer_positions[1].find_element(By.TAG_NAME,"span").text
+                work_times = outer_positions[2].find_element(By.TAG_NAME,"span").text
+                location = outer_positions[3].find_element(By.TAG_NAME,"span").text
             elif len(outer_positions) == 3:
                 if "·" in outer_positions[2].text:
-                    position_title = outer_positions[0].find_element_by_tag_name("span").find_element_by_tag_name("span").text
-                    company = outer_positions[1].find_element_by_tag_name("span").text
-                    work_times = outer_positions[2].find_element_by_tag_name("span").text
+                    position_title = outer_positions[0].find_element(By.TAG_NAME,"span").find_element(By.TAG_NAME,"span").text
+                    company = outer_positions[1].find_element(By.TAG_NAME,"span").text
+                    work_times = outer_positions[2].find_element(By.TAG_NAME,"span").text
                     location = ""
                 else:
                     position_title = ""
-                    company = outer_positions[0].find_element_by_tag_name("span").find_element_by_tag_name("span").text
+                    company = outer_positions[0].find_element(By.TAG_NAME,"span").find_element(By.TAG_NAME,"span").text
                     company = company[:company.find("\n")]
-                    work_times = outer_positions[1].find_element_by_tag_name("span").text
-                    location = outer_positions[2].find_element_by_tag_name("span").text
+                    work_times = outer_positions[1].find_element(By.TAG_NAME,"span").text
+                    location = outer_positions[2].find_element(By.TAG_NAME,"span").text
             else:
                 # len(outer_positions) == 2
-                company = outer_positions[0].find_element_by_tag_name("span").text
-                work_times = outer_positions[1].find_element_by_tag_name("span").text
+                company = outer_positions[0].find_element(By.TAG_NAME,"span").text
+                work_times = outer_positions[1].find_element(By.TAG_NAME,"span").text
                 position_title = ""
                 location = ""
 
-            if position_summary_text and len(
-                    position_summary_text.find_element_by_class_name("pvs-list").find_element_by_class_name("pvs-list").find_elements_by_xpath("li")) > 1:
-                descriptions = position_summary_text.find_element_by_class_name("pvs-list").find_element_by_class_name("pvs-list").find_elements_by_xpath("li")
+            if position_summary_text and len(position_summary_text.find_element(By.CLASS_NAME,"pvs-list").find_element(By.CLASS_NAME,"pvs-list").find_elements(By.XPATH,"li")) > 1:
+                descriptions = position_summary_text.find_element(By.CLASS_NAME,"pvs-list").find_element(By.CLASS_NAME,"pvs-list").find_elements(By.XPATH,"li")
                 for description in descriptions:
-                    res = description.find_element_by_tag_name("a").find_elements_by_xpath("*")
+                    res = description.find_element(By.TAG_NAME,"a").find_elements(By.XPATH,"*")
                     position_title_elem = res[0] if len(res) > 0 else None
                     work_times_elem = res[1] if len(res) > 1 else None
                     location_elem = res[2] if len(res) > 2 else None
 
-                    location = location_elem.find_element_by_xpath("*").text if location_elem else None
-                    position_title = position_title_elem.find_element_by_xpath("*").find_element_by_tag_name("*").text if position_title_elem else ""
-                    work_times = work_times_elem.find_element_by_xpath("*").text if work_times_elem else ""
+                    location = location_elem.find_element(By.XPATH,"*").text if location_elem else None
+                    position_title = position_title_elem.find_element(By.XPATH,"*").find_element(By.TAG_NAME,"*").text if position_title_elem else ""
+                    work_times = work_times_elem.find_element(By.XPATH,"*").text if work_times_elem else ""
                     times = work_times.split("·")[0].strip() if work_times else ""
                     duration = work_times.split("·")[1].strip() if len(work_times.split("·")) > 1 else None
                     from_date = " ".join(times.split(" ")[:2]) if times else ""
@@ -204,27 +203,27 @@ def get_educations(self):
         url = os.path.join(self.linkedin_url, "details/education")
         self.driver.get(url)
         self.focus()
-        main = self.wait_for_element_to_load(by=By.ID, name="main")
+        main = self.wait_for_element_to_load(by=By.TAG_NAME, name="main")
         self.scroll_to_half()
         self.scroll_to_bottom()
         main_list = self.wait_for_element_to_load(name="pvs-list", base=main)
-        for position in main_list.find_elements_by_class_name("pvs-entity"):
-            institution_logo_elem, position_details = position.find_elements_by_xpath("*")
+        for position in main_list.find_elements(By.CLASS_NAME,"pvs-entity"):
+            institution_logo_elem, position_details = position.find_elements(By.XPATH,"*")
 
             # company elem
-            institution_linkedin_url = institution_logo_elem.find_element_by_xpath("*").get_attribute("href")
+            institution_linkedin_url = institution_logo_elem.find_element(By.XPATH,"*").get_attribute("href")
 
             # position details
-            position_details_list = position_details.find_elements_by_xpath("*")
+            position_details_list = position_details.find_elements(By.XPATH,"*")
             position_summary_details = position_details_list[0] if len(position_details_list) > 0 else None
             position_summary_text = position_details_list[1] if len(position_details_list) > 1 else None
-            outer_positions = position_summary_details.find_element_by_xpath("*").find_elements_by_xpath("*")
+            outer_positions = position_summary_details.find_element(By.XPATH,"*").find_elements(By.XPATH,"*")
 
-            institution_name = outer_positions[0].find_element_by_tag_name("span").find_element_by_tag_name("span").text
-            degree = outer_positions[1].find_element_by_tag_name("span").text
+            institution_name = outer_positions[0].find_element(By.TAG_NAME,"span").find_element(By.TAG_NAME,"span").text
+            degree = outer_positions[1].find_element(By.TAG_NAME,"span").text
 
             if len(outer_positions) > 2:
-                times = outer_positions[2].find_element_by_tag_name("span").text
+                times = outer_positions[2].find_element(By.TAG_NAME,"span").text
 
                 from_date = " ".join(times.split(" ")[:2])
                 to_date = " ".join(times.split(" ")[3:])
@@ -247,14 +246,14 @@ def get_educations(self):
             self.add_education(education)
 
     def get_name_and_location(self):
-        top_panels = self.driver.find_elements_by_class_name("pv-text-details__left-panel")
-        self.name = top_panels[0].find_elements_by_xpath("*")[0].text
-        self.location = top_panels[1].find_element_by_tag_name("span").text
+        top_panels = self.driver.find_elements(By.CLASS_NAME,"pv-text-details__left-panel")
+        self.name = top_panels[0].find_elements(By.XPATH,"*")[0].text
+        self.location = top_panels[1].find_element(By.TAG_NAME,"span").text
 
 
     def get_about(self):
         try:
-            about = self.driver.find_element_by_id("about").find_element_by_xpath("..").find_element_by_class_name("display-flex").text
+            about = self.driver.find_element(By.ID,"about").find_element(By.XPATH,"..").find_element(By.CLASS_NAME,"display-flex").text
         except NoSuchElementException :
             about=None
         self.about = about
@@ -390,7 +389,7 @@ def job_title(self):
             return None
 
     def __repr__(self):
-        return "{name}\n\nAbout\n{about}\n\nExperience\n{exp}\n\nEducation\n{edu}\n\nInterest\n{int}\n\nAccomplishments\n{acc}\n\nContacts\n{conn}".format(
+        return "<Person {name}\n\nAbout\n{about}\n\nExperience\n{exp}\n\nEducation\n{edu}\n\nInterest\n{int}\n\nAccomplishments\n{acc}\n\nContacts\n{conn}>".format(
             name=self.name,
             about=self.about,
             exp=self.experiences,
