In [ ]:
In [1]:
import selenium
from selenium import webdriver
import json
import time
all_links = [
"https://www.researchgate.net/topic/Financial-Risk-Management/publications",
"https://www.researchgate.net/topic/Financial-Management/publications",
"https://www.researchgate.net/topic/Financial-Markets/publications",
"https://www.researchgate.net/topic/Financial-Modelling/publications",
"https://www.researchgate.net/topic/Financial-History/publications",
"https://www.researchgate.net/topic/Financial-Statement-Analysis/publications",
"https://www.researchgate.net/topic/Financial-Economics/publications",
"https://www.researchgate.net/topic/Finance-Management/publications",
"https://www.researchgate.net/topic/Financial-Accounting/publications",
"https://www.researchgate.net/topic/Financial-Mathematics/publications",
"https://www.researchgate.net/topic/Financial-Crises/publications",
"https://www.researchgate.net/topic/Financial-Engineering/publications",
"https://www.researchgate.net/topic/Financial-Services/publications",
"https://www.researchgate.net/topic/Financial-Valuation/publications",
"https://www.researchgate.net/topic/Financial-Risk-Sharing/publications",
"https://www.researchgate.net/topic/Financial-Law/publications",
"https://www.researchgate.net/topic/Financial-Services-industry/publications",
"https://www.researchgate.net/topic/Financial-Reporting/publications",
"https://www.researchgate.net/topic/Financial-Integration/publications",
"https://www.researchgate.net/topic/Financial-institutions/publications",
"https://www.researchgate.net/topic/Financial-Flows/publications",
"https://www.researchgate.net/topic/Financial-Inclusion/publications",
"https://www.researchgate.net/topic/Financial-Support/publications",
"https://www.researchgate.net/topic/Finance/publications",
"https://www.researchgate.net/topic/Financial-Analysis/publications",
"https://www.researchgate.net/topic/Financial-Econometrics/publications",
"https://www.researchgate.net/topic/Financial-Accounting-Theory/publications",
"https://www.researchgate.net/topic/Financial-Intermediation/publications",
"https://www.researchgate.net/topic/Financial-Modeling/publications",
"https://www.researchgate.net/topic/Financial-Development/publications",
"https://www.researchgate.net/topic/financial-statements/publications",
"https://www.researchgate.net/topic/Financial-Stability/publications",
"https://www.researchgate.net/topic/Financial-Crisis/publications",
"https://www.researchgate.net/topic/Financial-Criminology/publications",
"https://www.researchgate.net/topic/Financial-market-regulation/publications",
"https://www.researchgate.net/topic/Financial-Physics/publications",
"https://www.researchgate.net/topic/Economics-of-Science/publications",
"https://www.researchgate.net/topic/Environmental-Economics/publications",
"https://www.researchgate.net/topic/Economic-Development/publications",
"https://www.researchgate.net/topic/Economic-Evaluation/publications",
"https://www.researchgate.net/topic/Economic-Theory/publications",
"https://www.researchgate.net/topic/Economy/publications",
"https://www.researchgate.net/topic/Economic-Forecasting/publications",
"https://www.researchgate.net/topic/Economic-Policy-Analysis/publications",
"https://www.researchgate.net/topic/Economics-Analysis/publications",
"https://www.researchgate.net/topic/Econometric-Methods/publications",
"https://www.researchgate.net/topic/Experimental-Economics/publications",
"https://www.researchgate.net/topic/Economic-Geology/publications",
"https://www.researchgate.net/topic/Economic-History/publications",
"https://www.researchgate.net/topic/Economic-Policy/publications",
"https://www.researchgate.net/topic/Economic-Integration/publications",
"https://www.researchgate.net/topic/Emerging-Economies/publications",
"https://www.researchgate.net/topic/Economics-of-Climate-Adaptation/publications",
"https://www.researchgate.net/topic/Economic-Bubble/publications",
"https://www.researchgate.net/topic/Economic-Sanctions/publications",
"https://www.researchgate.net/topic/econometric-models/publications",
"https://www.researchgate.net/topic/Economic-Recession/publications",
"https://www.researchgate.net/topic/Economical-Psychology/publications",
"https://www.researchgate.net/topic/Economics-of-Innovation/publications",
"https://www.researchgate.net/topic/Evolutionary-Economic-Geography/publications",
"https://www.researchgate.net/topic/Econometric-Modeling/publications",
"https://www.researchgate.net/topic/Economics-of-Technology/publications",
"https://www.researchgate.net/topic/Economic-Growth/publications",
"https://www.researchgate.net/topic/Economic-Indicators/publications",
"https://www.researchgate.net/topic/Economic-Analysis/publications",
"https://www.researchgate.net/topic/Economic-Sociology/publications",
"https://www.researchgate.net/topic/Econometric-Techniques/publications",
"https://www.researchgate.net/topic/Economic-Crisis/publications",
"https://www.researchgate.net/topic/Ecological-Economics/publications",
"https://www.researchgate.net/topic/Economic-Modeling/publications",
"https://www.researchgate.net/topic/Economic-Valuation/publications",
"https://www.researchgate.net/topic/Economic-Geography/publications",
"https://www.researchgate.net/topic/Eco-Social-Market-Economy/publications",
"https://www.researchgate.net/topic/Evolutionary-Economics/publications",
"https://www.researchgate.net/topic/Economic-Security/publications",
"https://www.researchgate.net/topic/Engineering-Economy/publications",
"https://www.researchgate.net/topic/Entrepreneurial-Economics/publications",
"https://www.researchgate.net/topic/Economic-Inequality/publications",
"https://www.researchgate.net/topic/Economic-Dynamics/publications",
"https://www.researchgate.net/topic/Economic-Psychology/publications",
"https://www.researchgate.net/topic/Economic-Criminology/publications",
"https://www.researchgate.net/topic/Energy-Economics/publications",
"https://www.researchgate.net/topic/Econometrics-Packages/publications",
"https://www.researchgate.net/topic/Economic-dispatch/publications",
"https://www.researchgate.net/topic/Economic-models/publications",
"https://www.researchgate.net/topic/Economic-Inflation/publications",
"https://www.researchgate.net/topic/Economics-of-Regulation/publications",
"https://www.researchgate.net/topic/European-Economy/publications",
"https://www.researchgate.net/topic/Econometrics/publications",
"https://www.researchgate.net/topic/Economic-Efficiency/publications",
"https://www.researchgate.net/topic/Accounting-History/publications",
"https://www.researchgate.net/topic/Accounting-Software/publications",
"https://www.researchgate.net/topic/Accounting-Education/publications",
"https://www.researchgate.net/topic/Accounting-Scholarship/publications",
"https://www.researchgate.net/topic/Accountability/publications",
"https://www.researchgate.net/topic/Accounting-Research/publications",
"https://www.researchgate.net/topic/Accounts/publications",
"https://www.researchgate.net/topic/Accounting/publications",
"https://www.researchgate.net/topic/Accounting-Standard-Setting/publications",
"https://www.researchgate.net/topic/Account-Books/publications",
"https://www.researchgate.net/topic/Accounting-Information-Systems/publications",
"https://www.researchgate.net/topic/Accounting-Processes/publications",
"https://www.researchgate.net/topic/Accounting-Theory/publications",
"https://www.researchgate.net/topic/auditory_steady-state_response_assr/publications",
"https://www.researchgate.net/topic/Auditing/publications",
"https://www.researchgate.net/topic/Audit/publications",
"https://www.researchgate.net/topic/Auditory-System/publications",
"https://www.researchgate.net/topic/Auditory-Neuroscience/publications",
"https://www.researchgate.net/topic/Auditory-Evoked-Potentials/publications",
"https://www.researchgate.net/topic/Audition/publications",
"https://www.researchgate.net/topic/Auditory-Interfaces/publications",
"https://www.researchgate.net/topic/Auditory-Brainstem-Response/publications",
"https://www.researchgate.net/topic/Auditory-Hallucinations/publications",
"https://www.researchgate.net/topic/auditory-cortex/publications",
"https://www.researchgate.net/topic/Auditory-Sustained-Attention/publications",
"https://www.researchgate.net/topic/Auditory-Brain-Stem-Implantation/publications",
"https://www.researchgate.net/topic/Auditory-Perception/publications",
"https://www.researchgate.net/topic/Marketing-Management/publications",
"https://www.researchgate.net/topic/Managerial-Economics/publications",
"https://www.researchgate.net/topic/Management-Development/publications",
"https://www.researchgate.net/topic/Management-studies/publications",
"https://www.researchgate.net/topic/Maintenance-Management/publications",
"https://www.researchgate.net/topic/Marine-Management/publications",
"https://www.researchgate.net/topic/Management-Education/publications",
"https://www.researchgate.net/topic/Management-System/publications",
"https://www.researchgate.net/topic/Management-Theory/publications",
"https://www.researchgate.net/topic/Museum-Management/publications",
"https://www.researchgate.net/topic/Management-Consultancy/publications",
"https://www.researchgate.net/topic/Management-Control/publications",
"https://www.researchgate.net/topic/Management-Literature/publications",
"https://www.researchgate.net/topic/Management-Service-Organizations/publications",
"https://www.researchgate.net/topic/Marine-Resource-Management/publications",
"https://www.researchgate.net/topic/Management-Consulting/publications",
"https://www.researchgate.net/topic/Management-Of-Change/publications",
"https://www.researchgate.net/topic/Managed-Relocation/publications",
"https://www.researchgate.net/topic/Micro-Manager/publications",
"https://www.researchgate.net/topic/Medical-Practice-Management/publications",
"https://www.researchgate.net/topic/Manure-Management/publications",
"https://www.researchgate.net/topic/Management-Quality-Circles/publications",
"https://www.researchgate.net/topic/Management-Information-Systems/publications",
"https://www.researchgate.net/topic/Management-Accounting/publications",
"https://www.researchgate.net/topic/Management-Engineering/publications",
"https://www.researchgate.net/topic/Management-Accounting-Techniques/publications",
"https://www.researchgate.net/topic/Commercialization-Of-Products-Services/publications",
"https://www.researchgate.net/topic/Commercialization/publications"
]
len(all_links)
Out[1]:
145
In [2]:
import re
pattern = re.compile(r"publication/(\d+)_")
link = all_links[2]
# has_downloaded = json.load(open("d:/has_downloaded.json",encoding="utf-8"))
topic_ = link.split("/")[-2]
option = webdriver.ChromeOptions()
option.add_experimental_option("excludeSwitches", ['enable-automation'])
option.add_argument("--disable-javascript")
option.add_argument(f"--user-data-dir=d:/chrome_for_Citations_{topic_}")
option.add_argument("--disable-cache")
option.add_experimental_option(
'prefs',
{
"download.default_directory": f"E:\\Citations",
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"plugins.always_open_pdf_externally": True, # 这句配置很重要
"profile.managed_default_content_settings.images": 2,
"profile.managed_default_content_settings.media_stream": 2
}
)
# driver = webdriver.Chrome(
# executable_path="d:/chromedriver.exe",
# chrome_options=option
# )
# driver.implicitly_wait(2)
In [3]:
has_downloaded = json.load(open(fr'd:/out_pdfs/{topic_}_cite_download_urls.json',"r", encoding="utf-8"))
has_downloaded = list(map(lambda it: pattern.findall(it)[0], has_downloaded))
len(has_downloaded), has_downloaded[:2]
# has_downloaded = json.load(open("d:/has_downloaded.json",encoding="utf-8"))
# len(has_downloaded), has_downloaded[:2]
Out[3]:
(9999, ['370637680', '371120919'])
In [4]:
uids = set(map(lambda it: it.split("_")[0], has_downloaded))
walked = set()
In [5]:
import os
walked = set()
directory = 'E:\\Citations'
for file in os.listdir(directory):
walked.add(file.split(".")[0].split("-")[-1])
print(len(walked), list(walked)[:2])
19481 ['341056937', '300027856']
In [6]:
len(uids - walked)
Out[6]:
3456
In [7]:
ind = 0
driver = webdriver.Chrome(
executable_path="d:/chromedriver.exe",
chrome_options=option
)
driver.implicitly_wait(2)
for uid in uids:
if uid in walked:
print("skip", end="\r")
continue
url = 'https://www.researchgate.net/lite.publication.PublicationDownloadCitationModal.downloadCitation.html?publicationUid={}&fileType=BibTeX&citation=citation'
driver.get(url.format(uid))
walked.add(uid)
time.sleep(6)
ind += 1
if ind > 10:
ind = 0
driver.quit()
time.sleep(15)
driver = webdriver.Chrome(
executable_path="d:/chromedriver.exe",
chrome_options=option
)
driver.implicitly_wait(2)
d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\ipykernel_launcher.py:4: DeprecationWarning: use options instead of chrome_options after removing the cwd from sys.path.
skip
d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\ipykernel_launcher.py:23: DeprecationWarning: use options instead of chrome_options
skip
--------------------------------------------------------------------------- KeyboardInterrupt Traceback (most recent call last) <ipython-input-7-6e2f4b3819b4> in <module> 18 ind = 0 19 driver.quit() ---> 20 time.sleep(15) 21 driver = webdriver.Chrome( 22 executable_path="d:/chromedriver.exe", KeyboardInterrupt: