0. 定义与导包¶
In [1]:
all_links = [
"https://www.researchgate.net/topic/Financial-Risk-Management/publications",
"https://www.researchgate.net/topic/Financial-Management/publications",
"https://www.researchgate.net/topic/Financial-Markets/publications",
"https://www.researchgate.net/topic/Financial-Modelling/publications",
"https://www.researchgate.net/topic/Financial-History/publications",
"https://www.researchgate.net/topic/Financial-Statement-Analysis/publications",
"https://www.researchgate.net/topic/Financial-Economics/publications",
"https://www.researchgate.net/topic/Finance-Management/publications",
"https://www.researchgate.net/topic/Financial-Accounting/publications",
"https://www.researchgate.net/topic/Financial-Mathematics/publications",
"https://www.researchgate.net/topic/Financial-Crises/publications",
"https://www.researchgate.net/topic/Financial-Engineering/publications",
"https://www.researchgate.net/topic/Financial-Services/publications",
"https://www.researchgate.net/topic/Financial-Valuation/publications",
"https://www.researchgate.net/topic/Financial-Risk-Sharing/publications",
"https://www.researchgate.net/topic/Financial-Law/publications",
"https://www.researchgate.net/topic/Financial-Services-industry/publications",
"https://www.researchgate.net/topic/Financial-Reporting/publications",
"https://www.researchgate.net/topic/Financial-Integration/publications",
"https://www.researchgate.net/topic/Financial-institutions/publications",
"https://www.researchgate.net/topic/Financial-Flows/publications",
"https://www.researchgate.net/topic/Financial-Inclusion/publications",
"https://www.researchgate.net/topic/Financial-Support/publications",
"https://www.researchgate.net/topic/Finance/publications",
"https://www.researchgate.net/topic/Financial-Analysis/publications",
"https://www.researchgate.net/topic/Financial-Econometrics/publications",
"https://www.researchgate.net/topic/Financial-Accounting-Theory/publications",
"https://www.researchgate.net/topic/Financial-Intermediation/publications",
"https://www.researchgate.net/topic/Financial-Modeling/publications",
"https://www.researchgate.net/topic/Financial-Development/publications",
"https://www.researchgate.net/topic/financial-statements/publications",
"https://www.researchgate.net/topic/Financial-Stability/publications",
"https://www.researchgate.net/topic/Financial-Crisis/publications",
"https://www.researchgate.net/topic/Financial-Criminology/publications",
"https://www.researchgate.net/topic/Financial-market-regulation/publications",
"https://www.researchgate.net/topic/Financial-Physics/publications",
"https://www.researchgate.net/topic/Economics-of-Science/publications",
"https://www.researchgate.net/topic/Environmental-Economics/publications",
"https://www.researchgate.net/topic/Economic-Development/publications",
"https://www.researchgate.net/topic/Economic-Evaluation/publications",
"https://www.researchgate.net/topic/Economic-Theory/publications",
"https://www.researchgate.net/topic/Economy/publications",
"https://www.researchgate.net/topic/Economic-Forecasting/publications",
"https://www.researchgate.net/topic/Economic-Policy-Analysis/publications",
"https://www.researchgate.net/topic/Economics-Analysis/publications",
"https://www.researchgate.net/topic/Econometric-Methods/publications",
"https://www.researchgate.net/topic/Experimental-Economics/publications",
"https://www.researchgate.net/topic/Economic-Geology/publications",
"https://www.researchgate.net/topic/Economic-History/publications",
"https://www.researchgate.net/topic/Economic-Policy/publications",
"https://www.researchgate.net/topic/Economic-Integration/publications",
"https://www.researchgate.net/topic/Emerging-Economies/publications",
"https://www.researchgate.net/topic/Economics-of-Climate-Adaptation/publications",
"https://www.researchgate.net/topic/Economic-Bubble/publications",
"https://www.researchgate.net/topic/Economic-Sanctions/publications",
"https://www.researchgate.net/topic/econometric-models/publications",
"https://www.researchgate.net/topic/Economic-Recession/publications",
"https://www.researchgate.net/topic/Economical-Psychology/publications",
"https://www.researchgate.net/topic/Economics-of-Innovation/publications",
"https://www.researchgate.net/topic/Evolutionary-Economic-Geography/publications",
"https://www.researchgate.net/topic/Econometric-Modeling/publications",
"https://www.researchgate.net/topic/Economics-of-Technology/publications",
"https://www.researchgate.net/topic/Economic-Growth/publications",
"https://www.researchgate.net/topic/Economic-Indicators/publications",
"https://www.researchgate.net/topic/Economic-Analysis/publications",
"https://www.researchgate.net/topic/Economic-Sociology/publications",
"https://www.researchgate.net/topic/Econometric-Techniques/publications",
"https://www.researchgate.net/topic/Economic-Crisis/publications",
"https://www.researchgate.net/topic/Ecological-Economics/publications",
"https://www.researchgate.net/topic/Economic-Modeling/publications",
"https://www.researchgate.net/topic/Economic-Valuation/publications",
"https://www.researchgate.net/topic/Economic-Geography/publications",
"https://www.researchgate.net/topic/Eco-Social-Market-Economy/publications",
"https://www.researchgate.net/topic/Evolutionary-Economics/publications",
"https://www.researchgate.net/topic/Economic-Security/publications",
"https://www.researchgate.net/topic/Engineering-Economy/publications",
"https://www.researchgate.net/topic/Entrepreneurial-Economics/publications",
"https://www.researchgate.net/topic/Economic-Inequality/publications",
"https://www.researchgate.net/topic/Economic-Dynamics/publications",
"https://www.researchgate.net/topic/Economic-Psychology/publications",
"https://www.researchgate.net/topic/Economic-Criminology/publications",
"https://www.researchgate.net/topic/Energy-Economics/publications",
"https://www.researchgate.net/topic/Econometrics-Packages/publications",
"https://www.researchgate.net/topic/Economic-dispatch/publications",
"https://www.researchgate.net/topic/Economic-models/publications",
"https://www.researchgate.net/topic/Economic-Inflation/publications",
"https://www.researchgate.net/topic/Economics-of-Regulation/publications",
"https://www.researchgate.net/topic/European-Economy/publications",
"https://www.researchgate.net/topic/Econometrics/publications",
"https://www.researchgate.net/topic/Economic-Efficiency/publications",
"https://www.researchgate.net/topic/Accounting-History/publications",
"https://www.researchgate.net/topic/Accounting-Software/publications",
"https://www.researchgate.net/topic/Accounting-Education/publications",
"https://www.researchgate.net/topic/Accounting-Scholarship/publications",
"https://www.researchgate.net/topic/Accountability/publications",
"https://www.researchgate.net/topic/Accounting-Research/publications",
"https://www.researchgate.net/topic/Accounts/publications",
"https://www.researchgate.net/topic/Accounting/publications",
"https://www.researchgate.net/topic/Accounting-Standard-Setting/publications",
"https://www.researchgate.net/topic/Account-Books/publications",
"https://www.researchgate.net/topic/Accounting-Information-Systems/publications",
"https://www.researchgate.net/topic/Accounting-Processes/publications",
"https://www.researchgate.net/topic/Accounting-Theory/publications",
"https://www.researchgate.net/topic/auditory_steady-state_response_assr/publications",
"https://www.researchgate.net/topic/Auditing/publications",
"https://www.researchgate.net/topic/Audit/publications",
"https://www.researchgate.net/topic/Auditory-System/publications",
"https://www.researchgate.net/topic/Auditory-Neuroscience/publications",
"https://www.researchgate.net/topic/Auditory-Evoked-Potentials/publications",
"https://www.researchgate.net/topic/Audition/publications",
"https://www.researchgate.net/topic/Auditory-Interfaces/publications",
"https://www.researchgate.net/topic/Auditory-Brainstem-Response/publications",
"https://www.researchgate.net/topic/Auditory-Hallucinations/publications",
"https://www.researchgate.net/topic/auditory-cortex/publications",
"https://www.researchgate.net/topic/Auditory-Sustained-Attention/publications",
"https://www.researchgate.net/topic/Auditory-Brain-Stem-Implantation/publications",
"https://www.researchgate.net/topic/Auditory-Perception/publications",
"https://www.researchgate.net/topic/Marketing-Management/publications",
"https://www.researchgate.net/topic/Managerial-Economics/publications",
"https://www.researchgate.net/topic/Management-Development/publications",
"https://www.researchgate.net/topic/Management-studies/publications",
"https://www.researchgate.net/topic/Maintenance-Management/publications",
"https://www.researchgate.net/topic/Marine-Management/publications",
"https://www.researchgate.net/topic/Management-Education/publications",
"https://www.researchgate.net/topic/Management-System/publications",
"https://www.researchgate.net/topic/Management-Theory/publications",
"https://www.researchgate.net/topic/Museum-Management/publications",
"https://www.researchgate.net/topic/Management-Consultancy/publications",
"https://www.researchgate.net/topic/Management-Control/publications",
"https://www.researchgate.net/topic/Management-Literature/publications",
"https://www.researchgate.net/topic/Management-Service-Organizations/publications",
"https://www.researchgate.net/topic/Marine-Resource-Management/publications",
"https://www.researchgate.net/topic/Management-Consulting/publications",
"https://www.researchgate.net/topic/Management-Of-Change/publications",
"https://www.researchgate.net/topic/Managed-Relocation/publications",
"https://www.researchgate.net/topic/Micro-Manager/publications",
"https://www.researchgate.net/topic/Medical-Practice-Management/publications",
"https://www.researchgate.net/topic/Manure-Management/publications",
"https://www.researchgate.net/topic/Management-Quality-Circles/publications",
"https://www.researchgate.net/topic/Management-Information-Systems/publications",
"https://www.researchgate.net/topic/Management-Accounting/publications",
"https://www.researchgate.net/topic/Management-Engineering/publications",
"https://www.researchgate.net/topic/Management-Accounting-Techniques/publications",
"https://www.researchgate.net/topic/Commercialization-Of-Products-Services/publications",
"https://www.researchgate.net/topic/Commercialization/publications"
]
len(all_links)
Out[1]:
145
In [5]:
import os
import json
json_dir = "d:/out_pdfs/"
data_dict = {}
for name in os.listdir(json_dir):
if 'pdf_download_page_urls.json' in name:
file_path =os.path.join(json_dir, name)
topic = name.replace('_pdf_download_page_urls.json', '')
print(topic)
with open(file_path, encoding='utf-8') as fr:
data_dict[topic] = len(set(json.load(fr)))
Account-Books Accountability Accounting-Education Accounting-History Accounting-Information-Systems Accounting-Processes Accounting-Research Accounting-Scholarship Accounting-Software Accounting-Standard-Setting Accounting-Theory Accounting Accounts Auditing Audition Auditory-Brain-Stem-Implantation Auditory-Brainstem-Response auditory-cortex Auditory-Evoked-Potentials Auditory-Hallucinations Auditory-Interfaces Auditory-Neuroscience Auditory-Perception Auditory-Sustained-Attention Auditory-System auditory_steady-state_response_assr Audit Commercialization-Of-Products-Services Commercialization Eco-Social-Market-Economy Ecological-Economics Econometric-Methods Econometric-Modeling econometric-models Econometric-Techniques Econometrics-Packages Econometrics Economic-Analysis Economic-Bubble Economic-Criminology Economic-Crisis Economic-Development Economic-dispatch Economic-Dynamics Economic-Efficiency Economic-Evaluation Economic-Forecasting Economic-Geography Economic-Geology Economic-Growth Economic-History Economic-Indicators Economic-Inequality Economic-Inflation Economic-Integration Economic-Modeling Economic-models Economic-Policy-Analysis Economic-Policy Economic-Psychology Economic-Recession Economic-Sanctions Economic-Security Economic-Sociology Economic-Theory Economic-Valuation Economical-Psychology Economics-Analysis Economics-of-Climate-Adaptation Economics-of-Innovation Economics-of-Regulation Economics-of-Science Economics-of-Technology Economy Emerging-Economies Energy-Economics Engineering-Economy Entrepreneurial-Economics Environmental-Economics European-Economy Evolutionary-Economic-Geography Evolutionary-Economics Experimental-Economics Finance-Management Finance Financial-Accounting-Theory Financial-Accounting Financial-Analysis Financial-Criminology Financial-Crises Financial-Crisis Financial-Development Financial-Econometrics Financial-Economics Financial-Engineering Financial-Flows Financial-History Financial-Inclusion Financial-institutions Financial-Integration Financial-Intermediation Financial-Law Financial-market-regulation Financial-Markets Financial-Mathematics Financial-Modeling Financial-Modelling Financial-Physics Financial-Reporting Financial-Risk-Sharing Financial-Services-industry Financial-Services Financial-Stability Financial-Statement-Analysis financial-statements Financial-Support Financial-Valuation Maintenance-Management Managed-Relocation Management-Accounting-Techniques Management-Accounting Management-Consultancy Management-Consulting Management-Control Management-Development Management-Education Management-Engineering Management-Information-Systems Management-Literature Management-Of-Change Management-Quality-Circles Management-Service-Organizations Management-studies Management-System Management-Theory Managerial-Economics Manure-Management Marine-Management Marine-Resource-Management Marketing-Management Medical-Practice-Management Micro-Manager Museum-Management pdf_download_page_urls.json
In [ ]:
In [9]:
json.dumps(data_dict)
import pandas as pd
from pandas import DataFrame as df
data_frame = df(data_dict.items())
data_frame.to_excel("topic_expected.xlsx")
In [4]:
import selenium
from bs4 import BeautifulSoup as bsoup
from selenium import webdriver
import time
import re
import json
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait #WebDriverWait注意大小写
from selenium.webdriver.common.by import By
TOTAL_NUM_PATTERN = re.compile(r"\([\d,]+\)")
option = webdriver.ChromeOptions()
option.add_argument("--disable-cache")
option.add_argument("--user-data-dir=d:/chrome_for_PDFS2")
option.add_argument("--disable-javascript")
option.set_capability('pageLoadStrategy', 'none')
option.add_experimental_option('excludeSwitches', ['enable-automation'])
prefs = {
"download.default_directory": "D:\\out_pdfs",
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"plugins.always_open_pdf_externally": True,
"profile.managed_default_content_settings.images": 2,
"profile.managed_default_content_settings.media_stream": 2
}
option.add_experimental_option('prefs', prefs)
# driver = webdriver.Chrome(
# executable_path="d:/chromedriver.exe", chrome_options=option)
def get_driver():
driver = webdriver.Chrome(executable_path="d:/chromedriver.exe", chrome_options=option)
driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument',
{'source': 'Object.defineProperty(navigator, "webdriver", {get: () => undefined})'})
driver.implicitly_wait(5)
return driver
1. 获取 100 条书籍下载网页地址¶
In [5]:
all_links[4:10][0].split("/")[-2]
Out[5]:
'Financial-History'
In [6]:
for link in all_links[131:]:
print(all_links.index(link), "="*10, link)
topic = link.split("/")[-2]
base_dir_str = 'D:/out_pdfs'
pdf_download_page_urls_path = f'{base_dir_str}/{topic}_pdf_download_page_urls.json'
page_to_download_list_path = f'{base_dir_str}/{topic}_page_to_download_list.json'
cite_download_urls_path = f'{base_dir_str}/{topic}_cite_download_urls.json'
start_page = 1
cite_url_preffix = f"https://www.researchgate.net/topic/{topic}/publications"
pdf_download_page_urls = []
topic, total_num = None, None
page_to_download_list = {}
error_pages = []
cite_download_urls = []
for i in range(max(1, start_page), start_page + 100):
cite_url = cite_url_preffix + \
("" if i == 1 else f"/{i}")
print(cite_url)
driver = get_driver()
# page_to_download_list[cite_url] = None
if i == 1:
driver.get(cite_url)
try:
driver.implicitly_wait(5)
time.sleep(6)
# driver.get(cite_url)
driver.get(cite_url)
for i in range(5):
try:
WebDriverWait(driver,10).until(EC.presence_of_element_located((By.CSS_SELECTOR,'#lite-page > main > section.lite-page__content')))
for i in range(3):
# 向下滚动屏幕
driver.execute_script("window.scrollBy(0, 500);")
time.sleep(0.5)
# driver.find_elements_by_css_selector(
# "#lite-page > main > section.lite-page__content > div:nth-child(4) > div > div > div:nth-child(100) > div > div > div > div:nth-child(1) > div a")[-1]
except:
time.sleep(15)
driver.get(cite_url_preffix + ("" if i == 1 else f"/{i+1}"))
time.sleep(15)
driver.get(cite_url_preffix + ("" if i == 1 else f"/{i}"))
try:
if '需要首先验证您的连接安全性' in driver.page_source:
print("@" * 30)
driver.switch_to_frame(
driver.find_element_by_css_selector("iframe"))
driver.find_element_by_css_selector(
"input[type='checkbox']").click()
driver.switch_to_default_content()
except:
pass
soup = bsoup(driver.page_source)
spans = soup.select("span.nova-legacy-c-button__label")
time_out_second = 5
while len(spans) < 1 and time_out_second > 0:
time.sleep(0.5)
time_out_second -= 0.5
soup = bsoup(driver.page_source)
spans = soup.select("span.nova-legacy-c-button__label")
if i == 1:
topic = bsoup(driver.page_source).select(
"#lite-page > main > section.lite-page__content div.nova-legacy-c-qualifier__labels div:nth-child(1)")[0].text
total_num = eval(TOTAL_NUM_PATTERN.findall(topic)
[0].replace(",", ""))
print(topic, total_num)
temp_urls = []
for span in soup.select("span.nova-legacy-c-button__label"):
if 'Download full-text' in span.text:
pdf_download_page_urls.append(span.parent.attrs["href"])
temp_urls.append(span.parent.attrs["href"])
cite_ls =soup.select("div.nova-legacy-v-publication-item__footer-button-group")
cite_download_urls.extend(map(str, cite_ls))
page_to_download_list[driver.current_url] = temp_urls
print("\n", driver.current_url, "pdf_download_page_urls count:",
len(set(pdf_download_page_urls)), len(temp_urls))
json.dump(pdf_download_page_urls,
open(pdf_download_page_urls_path, "w", encoding="utf-8"), ensure_ascii=False)
json.dump(cite_download_urls,
open(cite_download_urls_path, "w", encoding="utf-8"), ensure_ascii=False)
time.sleep(6)
if len(temp_urls) == 0:
break
except Exception as e:
print(e)
error_pages.append(i)
break
finally:
# pass
driver.close()
driver.quit()
print(topic, total_num, len(pdf_download_page_urls))
json.dump(pdf_download_page_urls, open(pdf_download_page_urls_path, "w", encoding="utf-8"), ensure_ascii=False)
json.dump(page_to_download_list, open(page_to_download_list_path, "w", encoding="utf-8"), ensure_ascii=False)
json.dump(error_pages, open(f"d:/out_pdfs/{topic}_error_pages.json","w", encoding="utf-8"), ensure_ascii=False)
json.dump(cite_download_urls, open(cite_download_urls_path, "w", encoding="utf-8"), ensure_ascii=False)
131 ========== https://www.researchgate.net/topic/Marine-Resource-Management/publications https://www.researchgate.net/topic/Marine-Resource-Management/publications
d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\ipykernel_launcher.py:34: DeprecationWarning: use options instead of chrome_options
https://www.researchgate.net/topic/Marine-Resource-Management/publications pdf_download_page_urls count: 100 100 https://www.researchgate.net/topic/Marine-Resource-Management/publications/2 https://www.researchgate.net/topic/Marine-Resource-Management/publications/2 pdf_download_page_urls count: 200 100 https://www.researchgate.net/topic/Marine-Resource-Management/publications/3 https://www.researchgate.net/topic/Marine-Resource-Management/publications/3 pdf_download_page_urls count: 300 100 https://www.researchgate.net/topic/Marine-Resource-Management/publications/4 https://www.researchgate.net/topic/Marine-Resource-Management/publications/4 pdf_download_page_urls count: 398 98 https://www.researchgate.net/topic/Marine-Resource-Management/publications/5 https://www.researchgate.net/topic/Marine-Resource-Management/publications/5 pdf_download_page_urls count: 447 49 https://www.researchgate.net/topic/Marine-Resource-Management/publications/6 https://www.researchgate.net/topic/Marine-Resource-Management/publications/6 pdf_download_page_urls count: 447 0 None None 447 132 ========== https://www.researchgate.net/topic/Management-Consulting/publications https://www.researchgate.net/topic/Management-Consulting/publications https://www.researchgate.net/topic/Management-Consulting/publications pdf_download_page_urls count: 100 100 https://www.researchgate.net/topic/Management-Consulting/publications/2 https://www.researchgate.net/topic/Management-Consulting/publications/2 pdf_download_page_urls count: 200 100 https://www.researchgate.net/topic/Management-Consulting/publications/3 https://www.researchgate.net/topic/Management-Consulting/publications/3 pdf_download_page_urls count: 300 100 https://www.researchgate.net/topic/Management-Consulting/publications/4 https://www.researchgate.net/topic/Management-Consulting/publications/4 pdf_download_page_urls count: 400 100 https://www.researchgate.net/topic/Management-Consulting/publications/5 https://www.researchgate.net/topic/Management-Consulting/publications/5 pdf_download_page_urls count: 500 100 https://www.researchgate.net/topic/Management-Consulting/publications/6 https://www.researchgate.net/topic/Management-Consulting/publications/6 pdf_download_page_urls count: 598 98 https://www.researchgate.net/topic/Management-Consulting/publications/7 https://www.researchgate.net/topic/Management-Consulting/publications/7 pdf_download_page_urls count: 659 61 https://www.researchgate.net/topic/Management-Consulting/publications/8 https://www.researchgate.net/topic/Management-Consulting/publications/8 pdf_download_page_urls count: 659 0 None None 659 133 ========== https://www.researchgate.net/topic/Management-Of-Change/publications https://www.researchgate.net/topic/Management-Of-Change/publications https://www.researchgate.net/topic/Management-Of-Change/publications pdf_download_page_urls count: 100 100 https://www.researchgate.net/topic/Management-Of-Change/publications/2 https://www.researchgate.net/topic/Management-Of-Change/publications/2 pdf_download_page_urls count: 200 100 https://www.researchgate.net/topic/Management-Of-Change/publications/3 https://www.researchgate.net/topic/Management-Of-Change/publications/3 pdf_download_page_urls count: 300 100 https://www.researchgate.net/topic/Management-Of-Change/publications/4 https://www.researchgate.net/topic/Management-Of-Change/publications/4 pdf_download_page_urls count: 400 100 https://www.researchgate.net/topic/Management-Of-Change/publications/5 https://www.researchgate.net/topic/Management-Of-Change/publications/5 pdf_download_page_urls count: 500 100 https://www.researchgate.net/topic/Management-Of-Change/publications/6 https://www.researchgate.net/topic/Management-Of-Change/publications/6 pdf_download_page_urls count: 600 100 https://www.researchgate.net/topic/Management-Of-Change/publications/7 https://www.researchgate.net/topic/Management-Of-Change/publications/7 pdf_download_page_urls count: 700 100 https://www.researchgate.net/topic/Management-Of-Change/publications/8 https://www.researchgate.net/topic/Management-Of-Change/publications/8 pdf_download_page_urls count: 800 100 https://www.researchgate.net/topic/Management-Of-Change/publications/9 https://www.researchgate.net/topic/Management-Of-Change/publications/9 pdf_download_page_urls count: 900 100 https://www.researchgate.net/topic/Management-Of-Change/publications/10 https://www.researchgate.net/topic/Management-Of-Change/publications/10 pdf_download_page_urls count: 1000 100 https://www.researchgate.net/topic/Management-Of-Change/publications/11 https://www.researchgate.net/topic/Management-Of-Change/publications/11 pdf_download_page_urls count: 1100 100 https://www.researchgate.net/topic/Management-Of-Change/publications/12 https://www.researchgate.net/topic/Management-Of-Change/publications/12 pdf_download_page_urls count: 1200 100 https://www.researchgate.net/topic/Management-Of-Change/publications/13 https://www.researchgate.net/topic/Management-Of-Change/publications/13 pdf_download_page_urls count: 1300 100 https://www.researchgate.net/topic/Management-Of-Change/publications/14 https://www.researchgate.net/topic/Management-Of-Change/publications/14 pdf_download_page_urls count: 1400 100 https://www.researchgate.net/topic/Management-Of-Change/publications/15 https://www.researchgate.net/topic/Management-Of-Change/publications/15 pdf_download_page_urls count: 1500 100 https://www.researchgate.net/topic/Management-Of-Change/publications/16 https://www.researchgate.net/topic/Management-Of-Change/publications/16 pdf_download_page_urls count: 1600 100 https://www.researchgate.net/topic/Management-Of-Change/publications/17 https://www.researchgate.net/topic/Management-Of-Change/publications/17 pdf_download_page_urls count: 1699 99 https://www.researchgate.net/topic/Management-Of-Change/publications/18 https://www.researchgate.net/topic/Management-Of-Change/publications/18 pdf_download_page_urls count: 1799 100 https://www.researchgate.net/topic/Management-Of-Change/publications/19 https://www.researchgate.net/topic/Management-Of-Change/publications/19 pdf_download_page_urls count: 1898 99 https://www.researchgate.net/topic/Management-Of-Change/publications/20 https://www.researchgate.net/topic/Management-Of-Change/publications/20 pdf_download_page_urls count: 1998 100 https://www.researchgate.net/topic/Management-Of-Change/publications/21 https://www.researchgate.net/topic/Management-Of-Change/publications/21 pdf_download_page_urls count: 2098 100 https://www.researchgate.net/topic/Management-Of-Change/publications/22 https://www.researchgate.net/topic/Management-Of-Change/publications/22 pdf_download_page_urls count: 2198 100 https://www.researchgate.net/topic/Management-Of-Change/publications/23 https://www.researchgate.net/topic/Management-Of-Change/publications/23 pdf_download_page_urls count: 2297 99 https://www.researchgate.net/topic/Management-Of-Change/publications/24 https://www.researchgate.net/topic/Management-Of-Change/publications/24 pdf_download_page_urls count: 2397 100 https://www.researchgate.net/topic/Management-Of-Change/publications/25 https://www.researchgate.net/topic/Management-Of-Change/publications/25 pdf_download_page_urls count: 2497 100 https://www.researchgate.net/topic/Management-Of-Change/publications/26 https://www.researchgate.net/topic/Management-Of-Change/publications/26 pdf_download_page_urls count: 2597 100 https://www.researchgate.net/topic/Management-Of-Change/publications/27 https://www.researchgate.net/topic/Management-Of-Change/publications/27 pdf_download_page_urls count: 2697 100 https://www.researchgate.net/topic/Management-Of-Change/publications/28 https://www.researchgate.net/topic/Management-Of-Change/publications/28 pdf_download_page_urls count: 2796 99 https://www.researchgate.net/topic/Management-Of-Change/publications/29 https://www.researchgate.net/topic/Management-Of-Change/publications/29 pdf_download_page_urls count: 2896 100 https://www.researchgate.net/topic/Management-Of-Change/publications/30 https://www.researchgate.net/topic/Management-Of-Change/publications/30 pdf_download_page_urls count: 2996 100 https://www.researchgate.net/topic/Management-Of-Change/publications/31 https://www.researchgate.net/topic/Management-Of-Change/publications/31 pdf_download_page_urls count: 3096 100 https://www.researchgate.net/topic/Management-Of-Change/publications/32 https://www.researchgate.net/topic/Management-Of-Change/publications/32 pdf_download_page_urls count: 3195 99 https://www.researchgate.net/topic/Management-Of-Change/publications/33 https://www.researchgate.net/topic/Management-Of-Change/publications/33 pdf_download_page_urls count: 3294 99 https://www.researchgate.net/topic/Management-Of-Change/publications/34 https://www.researchgate.net/topic/Management-Of-Change/publications/34 pdf_download_page_urls count: 3394 100 https://www.researchgate.net/topic/Management-Of-Change/publications/35 https://www.researchgate.net/topic/Management-Of-Change/publications/35 pdf_download_page_urls count: 3493 99 https://www.researchgate.net/topic/Management-Of-Change/publications/36 https://www.researchgate.net/topic/Management-Of-Change/publications/36 pdf_download_page_urls count: 3593 100 https://www.researchgate.net/topic/Management-Of-Change/publications/37 https://www.researchgate.net/topic/Management-Of-Change/publications/37 pdf_download_page_urls count: 3692 99 https://www.researchgate.net/topic/Management-Of-Change/publications/38 https://www.researchgate.net/topic/Management-Of-Change/publications/38 pdf_download_page_urls count: 3789 97 https://www.researchgate.net/topic/Management-Of-Change/publications/39 https://www.researchgate.net/topic/Management-Of-Change/publications/39 pdf_download_page_urls count: 3868 79 https://www.researchgate.net/topic/Management-Of-Change/publications/40 https://www.researchgate.net/topic/Management-Of-Change/publications/40 pdf_download_page_urls count: 3868 0 None None 3868 134 ========== https://www.researchgate.net/topic/Managed-Relocation/publications https://www.researchgate.net/topic/Managed-Relocation/publications https://www.researchgate.net/topic/Managed-Relocation/publications pdf_download_page_urls count: 74 74 https://www.researchgate.net/topic/Managed-Relocation/publications/2 https://www.researchgate.net/topic/Managed-Relocation/publications/2 pdf_download_page_urls count: 74 0 None None 74 135 ========== https://www.researchgate.net/topic/Micro-Manager/publications https://www.researchgate.net/topic/Micro-Manager/publications https://www.researchgate.net/topic/Micro-Manager/publications pdf_download_page_urls count: 100 100 https://www.researchgate.net/topic/Micro-Manager/publications/2 https://www.researchgate.net/topic/Micro-Manager/publications/2 pdf_download_page_urls count: 199 99 https://www.researchgate.net/topic/Micro-Manager/publications/3 https://www.researchgate.net/topic/Micro-Manager/publications/3 pdf_download_page_urls count: 233 34 https://www.researchgate.net/topic/Micro-Manager/publications/4 https://www.researchgate.net/topic/Micro-Manager/publications/4 pdf_download_page_urls count: 233 0 None None 233 136 ========== https://www.researchgate.net/topic/Medical-Practice-Management/publications https://www.researchgate.net/topic/Medical-Practice-Management/publications https://www.researchgate.net/topic/Medical-Practice-Management/publications pdf_download_page_urls count: 99 99 https://www.researchgate.net/topic/Medical-Practice-Management/publications/2 https://www.researchgate.net/topic/Medical-Practice-Management/publications/2 pdf_download_page_urls count: 198 99 https://www.researchgate.net/topic/Medical-Practice-Management/publications/3 https://www.researchgate.net/topic/Medical-Practice-Management/publications/3 pdf_download_page_urls count: 297 99 https://www.researchgate.net/topic/Medical-Practice-Management/publications/4 https://www.researchgate.net/topic/Medical-Practice-Management/publications/4 pdf_download_page_urls count: 396 99 https://www.researchgate.net/topic/Medical-Practice-Management/publications/5 https://www.researchgate.net/topic/Medical-Practice-Management/publications/5 pdf_download_page_urls count: 396 0 None None 396 137 ========== https://www.researchgate.net/topic/Manure-Management/publications https://www.researchgate.net/topic/Manure-Management/publications https://www.researchgate.net/topic/Manure-Management/publications pdf_download_page_urls count: 100 100 https://www.researchgate.net/topic/Manure-Management/publications/2 https://www.researchgate.net/topic/Manure-Management/publications/2 pdf_download_page_urls count: 200 100 https://www.researchgate.net/topic/Manure-Management/publications/3 https://www.researchgate.net/topic/Manure-Management/publications/3 pdf_download_page_urls count: 300 100 https://www.researchgate.net/topic/Manure-Management/publications/4 https://www.researchgate.net/topic/Manure-Management/publications/4 pdf_download_page_urls count: 399 99 https://www.researchgate.net/topic/Manure-Management/publications/5 https://www.researchgate.net/topic/Manure-Management/publications/5 pdf_download_page_urls count: 499 100 https://www.researchgate.net/topic/Manure-Management/publications/6 https://www.researchgate.net/topic/Manure-Management/publications/6 pdf_download_page_urls count: 599 100 https://www.researchgate.net/topic/Manure-Management/publications/7 https://www.researchgate.net/topic/Manure-Management/publications/7 pdf_download_page_urls count: 699 100 https://www.researchgate.net/topic/Manure-Management/publications/8 https://www.researchgate.net/topic/Manure-Management/publications/8 pdf_download_page_urls count: 799 100 https://www.researchgate.net/topic/Manure-Management/publications/9 https://www.researchgate.net/topic/Manure-Management/publications/9 pdf_download_page_urls count: 898 99 https://www.researchgate.net/topic/Manure-Management/publications/10 https://www.researchgate.net/topic/Manure-Management/publications/10 pdf_download_page_urls count: 998 100 https://www.researchgate.net/topic/Manure-Management/publications/11 https://www.researchgate.net/topic/Manure-Management/publications/11 pdf_download_page_urls count: 1098 100 https://www.researchgate.net/topic/Manure-Management/publications/12 https://www.researchgate.net/topic/Manure-Management/publications/12 pdf_download_page_urls count: 1197 99 https://www.researchgate.net/topic/Manure-Management/publications/13 https://www.researchgate.net/topic/Manure-Management/publications/13 pdf_download_page_urls count: 1295 98 https://www.researchgate.net/topic/Manure-Management/publications/14 https://www.researchgate.net/topic/Manure-Management/publications/14 pdf_download_page_urls count: 1340 45 https://www.researchgate.net/topic/Manure-Management/publications/15 https://www.researchgate.net/topic/Manure-Management/publications/15 pdf_download_page_urls count: 1340 0 None None 1340 138 ========== https://www.researchgate.net/topic/Management-Quality-Circles/publications https://www.researchgate.net/topic/Management-Quality-Circles/publications https://www.researchgate.net/topic/Management-Quality-Circles/publications pdf_download_page_urls count: 99 99 https://www.researchgate.net/topic/Management-Quality-Circles/publications/2 https://www.researchgate.net/topic/Management-Quality-Circles/publications/2 pdf_download_page_urls count: 198 99 https://www.researchgate.net/topic/Management-Quality-Circles/publications/3 https://www.researchgate.net/topic/Management-Quality-Circles/publications/3 pdf_download_page_urls count: 293 95 https://www.researchgate.net/topic/Management-Quality-Circles/publications/4 https://www.researchgate.net/topic/Management-Quality-Circles/publications/4 pdf_download_page_urls count: 339 46 https://www.researchgate.net/topic/Management-Quality-Circles/publications/5 https://www.researchgate.net/topic/Management-Quality-Circles/publications/5 pdf_download_page_urls count: 342 3 https://www.researchgate.net/topic/Management-Quality-Circles/publications/6 https://www.researchgate.net/topic/Management-Quality-Circles/publications/6 pdf_download_page_urls count: 344 2 https://www.researchgate.net/topic/Management-Quality-Circles/publications/7 https://www.researchgate.net/topic/Management-Quality-Circles/publications/7 pdf_download_page_urls count: 345 1 https://www.researchgate.net/topic/Management-Quality-Circles/publications/8 https://www.researchgate.net/topic/Management-Quality-Circles/publications/8 pdf_download_page_urls count: 346 1 https://www.researchgate.net/topic/Management-Quality-Circles/publications/9 https://www.researchgate.net/topic/Management-Quality-Circles/publications/9 pdf_download_page_urls count: 346 0 None None 346 139 ========== https://www.researchgate.net/topic/Management-Information-Systems/publications https://www.researchgate.net/topic/Management-Information-Systems/publications https://www.researchgate.net/topic/Management-Information-Systems/publications pdf_download_page_urls count: 100 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/2 https://www.researchgate.net/topic/Management-Information-Systems/publications/2 pdf_download_page_urls count: 200 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/3 https://www.researchgate.net/topic/Management-Information-Systems/publications/3 pdf_download_page_urls count: 300 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/4 https://www.researchgate.net/topic/Management-Information-Systems/publications/4 pdf_download_page_urls count: 400 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/5 https://www.researchgate.net/topic/Management-Information-Systems/publications/5 pdf_download_page_urls count: 500 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/6 https://www.researchgate.net/topic/Management-Information-Systems/publications/6 pdf_download_page_urls count: 600 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/7 https://www.researchgate.net/topic/Management-Information-Systems/publications/7 pdf_download_page_urls count: 700 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/8 https://www.researchgate.net/topic/Management-Information-Systems/publications/8 pdf_download_page_urls count: 800 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/9 https://www.researchgate.net/topic/Management-Information-Systems/publications/9 pdf_download_page_urls count: 900 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/10 https://www.researchgate.net/topic/Management-Information-Systems/publications/10 pdf_download_page_urls count: 999 99 https://www.researchgate.net/topic/Management-Information-Systems/publications/11 https://www.researchgate.net/topic/Management-Information-Systems/publications/11 pdf_download_page_urls count: 1099 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/12 https://www.researchgate.net/topic/Management-Information-Systems/publications/12 pdf_download_page_urls count: 1199 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/13 https://www.researchgate.net/topic/Management-Information-Systems/publications/13 pdf_download_page_urls count: 1299 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/14 https://www.researchgate.net/topic/Management-Information-Systems/publications/14 pdf_download_page_urls count: 1399 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/15 https://www.researchgate.net/topic/Management-Information-Systems/publications/15 pdf_download_page_urls count: 1499 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/16 https://www.researchgate.net/topic/Management-Information-Systems/publications/16 pdf_download_page_urls count: 1599 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/17 https://www.researchgate.net/topic/Management-Information-Systems/publications/17 pdf_download_page_urls count: 1699 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/18 https://www.researchgate.net/topic/Management-Information-Systems/publications/18 pdf_download_page_urls count: 1799 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/19 https://www.researchgate.net/topic/Management-Information-Systems/publications/19 pdf_download_page_urls count: 1899 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/20 https://www.researchgate.net/topic/Management-Information-Systems/publications/20 pdf_download_page_urls count: 1999 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/21 https://www.researchgate.net/topic/Management-Information-Systems/publications/21 pdf_download_page_urls count: 2099 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/22 https://www.researchgate.net/topic/Management-Information-Systems/publications/22 pdf_download_page_urls count: 2198 99 https://www.researchgate.net/topic/Management-Information-Systems/publications/23 https://www.researchgate.net/topic/Management-Information-Systems/publications/23 pdf_download_page_urls count: 2297 99 https://www.researchgate.net/topic/Management-Information-Systems/publications/24 https://www.researchgate.net/topic/Management-Information-Systems/publications/24 pdf_download_page_urls count: 2397 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/25 https://www.researchgate.net/topic/Management-Information-Systems/publications/25 pdf_download_page_urls count: 2496 99 https://www.researchgate.net/topic/Management-Information-Systems/publications/26 https://www.researchgate.net/topic/Management-Information-Systems/publications/26 pdf_download_page_urls count: 2596 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/27 https://www.researchgate.net/topic/Management-Information-Systems/publications/27 pdf_download_page_urls count: 2693 97 https://www.researchgate.net/topic/Management-Information-Systems/publications/28 https://www.researchgate.net/topic/Management-Information-Systems/publications/28 pdf_download_page_urls count: 2791 98 https://www.researchgate.net/topic/Management-Information-Systems/publications/29 https://www.researchgate.net/topic/Management-Information-Systems/publications/29 pdf_download_page_urls count: 2891 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/30 https://www.researchgate.net/topic/Management-Information-Systems/publications/30 pdf_download_page_urls count: 2991 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/31 https://www.researchgate.net/topic/Management-Information-Systems/publications/31 pdf_download_page_urls count: 3091 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/32 https://www.researchgate.net/topic/Management-Information-Systems/publications/32 pdf_download_page_urls count: 3190 99 https://www.researchgate.net/topic/Management-Information-Systems/publications/33 https://www.researchgate.net/topic/Management-Information-Systems/publications/33 pdf_download_page_urls count: 3290 100 https://www.researchgate.net/topic/Management-Information-Systems/publications/34 https://www.researchgate.net/topic/Management-Information-Systems/publications/34 pdf_download_page_urls count: 3388 98 https://www.researchgate.net/topic/Management-Information-Systems/publications/35 https://www.researchgate.net/topic/Management-Information-Systems/publications/35 pdf_download_page_urls count: 3388 0 None None 3388 140 ========== https://www.researchgate.net/topic/Management-Accounting/publications https://www.researchgate.net/topic/Management-Accounting/publications https://www.researchgate.net/topic/Management-Accounting/publications pdf_download_page_urls count: 100 100 https://www.researchgate.net/topic/Management-Accounting/publications/2 https://www.researchgate.net/topic/Management-Accounting/publications/2 pdf_download_page_urls count: 200 100 https://www.researchgate.net/topic/Management-Accounting/publications/3 https://www.researchgate.net/topic/Management-Accounting/publications/3 pdf_download_page_urls count: 300 100 https://www.researchgate.net/topic/Management-Accounting/publications/4 https://www.researchgate.net/topic/Management-Accounting/publications/4 pdf_download_page_urls count: 400 100 https://www.researchgate.net/topic/Management-Accounting/publications/5 https://www.researchgate.net/topic/Management-Accounting/publications/5 pdf_download_page_urls count: 500 100 https://www.researchgate.net/topic/Management-Accounting/publications/6 https://www.researchgate.net/topic/Management-Accounting/publications/6 pdf_download_page_urls count: 600 100 https://www.researchgate.net/topic/Management-Accounting/publications/7 https://www.researchgate.net/topic/Management-Accounting/publications/7 pdf_download_page_urls count: 700 100 https://www.researchgate.net/topic/Management-Accounting/publications/8 https://www.researchgate.net/topic/Management-Accounting/publications/8 pdf_download_page_urls count: 800 100 https://www.researchgate.net/topic/Management-Accounting/publications/9 https://www.researchgate.net/topic/Management-Accounting/publications/9 pdf_download_page_urls count: 900 100 https://www.researchgate.net/topic/Management-Accounting/publications/10 https://www.researchgate.net/topic/Management-Accounting/publications/10 pdf_download_page_urls count: 1000 100 https://www.researchgate.net/topic/Management-Accounting/publications/11 https://www.researchgate.net/topic/Management-Accounting/publications/11 pdf_download_page_urls count: 1100 100 https://www.researchgate.net/topic/Management-Accounting/publications/12 https://www.researchgate.net/topic/Management-Accounting/publications/12 pdf_download_page_urls count: 1200 100 https://www.researchgate.net/topic/Management-Accounting/publications/13 https://www.researchgate.net/topic/Management-Accounting/publications/13 pdf_download_page_urls count: 1300 100 https://www.researchgate.net/topic/Management-Accounting/publications/14 https://www.researchgate.net/topic/Management-Accounting/publications/14 pdf_download_page_urls count: 1399 99 https://www.researchgate.net/topic/Management-Accounting/publications/15 https://www.researchgate.net/topic/Management-Accounting/publications/15 pdf_download_page_urls count: 1499 100 https://www.researchgate.net/topic/Management-Accounting/publications/16 https://www.researchgate.net/topic/Management-Accounting/publications/16 pdf_download_page_urls count: 1599 100 https://www.researchgate.net/topic/Management-Accounting/publications/17 https://www.researchgate.net/topic/Management-Accounting/publications/17 pdf_download_page_urls count: 1699 100 https://www.researchgate.net/topic/Management-Accounting/publications/18 https://www.researchgate.net/topic/Management-Accounting/publications/18 pdf_download_page_urls count: 1798 99 https://www.researchgate.net/topic/Management-Accounting/publications/19 https://www.researchgate.net/topic/Management-Accounting/publications/19 pdf_download_page_urls count: 1897 99 https://www.researchgate.net/topic/Management-Accounting/publications/20 https://www.researchgate.net/topic/Management-Accounting/publications/20 pdf_download_page_urls count: 1997 100 https://www.researchgate.net/topic/Management-Accounting/publications/21 https://www.researchgate.net/topic/Management-Accounting/publications/21 pdf_download_page_urls count: 2097 100 https://www.researchgate.net/topic/Management-Accounting/publications/22 https://www.researchgate.net/topic/Management-Accounting/publications/22 pdf_download_page_urls count: 2197 100 https://www.researchgate.net/topic/Management-Accounting/publications/23 https://www.researchgate.net/topic/Management-Accounting/publications/23 pdf_download_page_urls count: 2297 100 https://www.researchgate.net/topic/Management-Accounting/publications/24 https://www.researchgate.net/topic/Management-Accounting/publications/24 pdf_download_page_urls count: 2397 100 https://www.researchgate.net/topic/Management-Accounting/publications/25 https://www.researchgate.net/topic/Management-Accounting/publications/25 pdf_download_page_urls count: 2497 100 https://www.researchgate.net/topic/Management-Accounting/publications/26 https://www.researchgate.net/topic/Management-Accounting/publications/26 pdf_download_page_urls count: 2597 100 https://www.researchgate.net/topic/Management-Accounting/publications/27 https://www.researchgate.net/topic/Management-Accounting/publications/27 pdf_download_page_urls count: 2697 100 https://www.researchgate.net/topic/Management-Accounting/publications/28 https://www.researchgate.net/topic/Management-Accounting/publications/28 pdf_download_page_urls count: 2797 100 https://www.researchgate.net/topic/Management-Accounting/publications/29 https://www.researchgate.net/topic/Management-Accounting/publications/29 pdf_download_page_urls count: 2897 100 https://www.researchgate.net/topic/Management-Accounting/publications/30 https://www.researchgate.net/topic/Management-Accounting/publications/30 pdf_download_page_urls count: 2997 100 https://www.researchgate.net/topic/Management-Accounting/publications/31 https://www.researchgate.net/topic/Management-Accounting/publications/31 pdf_download_page_urls count: 3096 99 https://www.researchgate.net/topic/Management-Accounting/publications/32 https://www.researchgate.net/topic/Management-Accounting/publications/32 pdf_download_page_urls count: 3196 100 https://www.researchgate.net/topic/Management-Accounting/publications/33 https://www.researchgate.net/topic/Management-Accounting/publications/33 pdf_download_page_urls count: 3296 100 https://www.researchgate.net/topic/Management-Accounting/publications/34 https://www.researchgate.net/topic/Management-Accounting/publications/34 pdf_download_page_urls count: 3396 100 https://www.researchgate.net/topic/Management-Accounting/publications/35 https://www.researchgate.net/topic/Management-Accounting/publications/35 pdf_download_page_urls count: 3496 100 https://www.researchgate.net/topic/Management-Accounting/publications/36 https://www.researchgate.net/topic/Management-Accounting/publications/36 pdf_download_page_urls count: 3596 100 https://www.researchgate.net/topic/Management-Accounting/publications/37 https://www.researchgate.net/topic/Management-Accounting/publications/37 pdf_download_page_urls count: 3696 100 https://www.researchgate.net/topic/Management-Accounting/publications/38 https://www.researchgate.net/topic/Management-Accounting/publications/38 pdf_download_page_urls count: 3796 100 https://www.researchgate.net/topic/Management-Accounting/publications/39 https://www.researchgate.net/topic/Management-Accounting/publications/39 pdf_download_page_urls count: 3896 100 https://www.researchgate.net/topic/Management-Accounting/publications/40 https://www.researchgate.net/topic/Management-Accounting/publications/40 pdf_download_page_urls count: 3996 100 https://www.researchgate.net/topic/Management-Accounting/publications/41 https://www.researchgate.net/topic/Management-Accounting/publications/41 pdf_download_page_urls count: 4096 100 https://www.researchgate.net/topic/Management-Accounting/publications/42 https://www.researchgate.net/topic/Management-Accounting/publications/42 pdf_download_page_urls count: 4196 100 https://www.researchgate.net/topic/Management-Accounting/publications/43 https://www.researchgate.net/topic/Management-Accounting/publications/43 pdf_download_page_urls count: 4296 100 https://www.researchgate.net/topic/Management-Accounting/publications/44 https://www.researchgate.net/topic/Management-Accounting/publications/44 pdf_download_page_urls count: 4396 100 https://www.researchgate.net/topic/Management-Accounting/publications/45 https://www.researchgate.net/topic/Management-Accounting/publications/45 pdf_download_page_urls count: 4496 100 https://www.researchgate.net/topic/Management-Accounting/publications/46 https://www.researchgate.net/topic/Management-Accounting/publications/46 pdf_download_page_urls count: 4596 100 https://www.researchgate.net/topic/Management-Accounting/publications/47 https://www.researchgate.net/topic/Management-Accounting/publications/47 pdf_download_page_urls count: 4695 99 https://www.researchgate.net/topic/Management-Accounting/publications/48 https://www.researchgate.net/topic/Management-Accounting/publications/48 pdf_download_page_urls count: 4795 100 https://www.researchgate.net/topic/Management-Accounting/publications/49 https://www.researchgate.net/topic/Management-Accounting/publications/49 pdf_download_page_urls count: 4894 99 https://www.researchgate.net/topic/Management-Accounting/publications/50 https://www.researchgate.net/topic/Management-Accounting/publications/50 pdf_download_page_urls count: 4994 100 https://www.researchgate.net/topic/Management-Accounting/publications/51 https://www.researchgate.net/topic/Management-Accounting/publications/51 pdf_download_page_urls count: 5094 100 https://www.researchgate.net/topic/Management-Accounting/publications/52 https://www.researchgate.net/topic/Management-Accounting/publications/52 pdf_download_page_urls count: 5193 99 https://www.researchgate.net/topic/Management-Accounting/publications/53 https://www.researchgate.net/topic/Management-Accounting/publications/53 pdf_download_page_urls count: 5242 49 https://www.researchgate.net/topic/Management-Accounting/publications/54 https://www.researchgate.net/topic/Management-Accounting/publications/54 pdf_download_page_urls count: 5242 0 None None 5242 141 ========== https://www.researchgate.net/topic/Management-Engineering/publications https://www.researchgate.net/topic/Management-Engineering/publications https://www.researchgate.net/topic/Management-Engineering/publications pdf_download_page_urls count: 100 100 https://www.researchgate.net/topic/Management-Engineering/publications/2 https://www.researchgate.net/topic/Management-Engineering/publications/2 pdf_download_page_urls count: 200 100 https://www.researchgate.net/topic/Management-Engineering/publications/3 https://www.researchgate.net/topic/Management-Engineering/publications/3 pdf_download_page_urls count: 264 64 https://www.researchgate.net/topic/Management-Engineering/publications/4 https://www.researchgate.net/topic/Management-Engineering/publications/4 pdf_download_page_urls count: 264 0 None None 264 142 ========== https://www.researchgate.net/topic/Management-Accounting-Techniques/publications https://www.researchgate.net/topic/Management-Accounting-Techniques/publications https://www.researchgate.net/topic/Management-Accounting-Techniques/publications pdf_download_page_urls count: 100 100 https://www.researchgate.net/topic/Management-Accounting-Techniques/publications/2 https://www.researchgate.net/topic/Management-Accounting-Techniques/publications/2 pdf_download_page_urls count: 200 100 https://www.researchgate.net/topic/Management-Accounting-Techniques/publications/3 https://www.researchgate.net/topic/Management-Accounting-Techniques/publications/3 pdf_download_page_urls count: 205 5 https://www.researchgate.net/topic/Management-Accounting-Techniques/publications/4 https://www.researchgate.net/topic/Management-Accounting-Techniques/publications/4 pdf_download_page_urls count: 205 0 None None 205 143 ========== https://www.researchgate.net/topic/Commercialization-Of-Products-Services/publications https://www.researchgate.net/topic/Commercialization-Of-Products-Services/publications https://www.researchgate.net/topic/Commercialization-Of-Products-Services/publications pdf_download_page_urls count: 9 9 https://www.researchgate.net/topic/Commercialization-Of-Products-Services/publications/2 https://www.researchgate.net/topic/Commercialization-Of-Products-Services/publications/2 pdf_download_page_urls count: 9 0 None None 9 144 ========== https://www.researchgate.net/topic/Commercialization/publications https://www.researchgate.net/topic/Commercialization/publications https://www.researchgate.net/topic/Commercialization/publications pdf_download_page_urls count: 100 100 https://www.researchgate.net/topic/Commercialization/publications/2 https://www.researchgate.net/topic/Commercialization/publications/2 pdf_download_page_urls count: 200 100 https://www.researchgate.net/topic/Commercialization/publications/3 https://www.researchgate.net/topic/Commercialization/publications/3 pdf_download_page_urls count: 300 100 https://www.researchgate.net/topic/Commercialization/publications/4 https://www.researchgate.net/topic/Commercialization/publications/4 pdf_download_page_urls count: 400 100 https://www.researchgate.net/topic/Commercialization/publications/5 https://www.researchgate.net/topic/Commercialization/publications/5 pdf_download_page_urls count: 500 100 https://www.researchgate.net/topic/Commercialization/publications/6 https://www.researchgate.net/topic/Commercialization/publications/6 pdf_download_page_urls count: 600 100 https://www.researchgate.net/topic/Commercialization/publications/7 https://www.researchgate.net/topic/Commercialization/publications/7 pdf_download_page_urls count: 700 100 https://www.researchgate.net/topic/Commercialization/publications/8 https://www.researchgate.net/topic/Commercialization/publications/8 pdf_download_page_urls count: 800 100 https://www.researchgate.net/topic/Commercialization/publications/9 https://www.researchgate.net/topic/Commercialization/publications/9 pdf_download_page_urls count: 900 100 https://www.researchgate.net/topic/Commercialization/publications/10 https://www.researchgate.net/topic/Commercialization/publications/10 pdf_download_page_urls count: 1000 100 https://www.researchgate.net/topic/Commercialization/publications/11 https://www.researchgate.net/topic/Commercialization/publications/11 pdf_download_page_urls count: 1100 100 https://www.researchgate.net/topic/Commercialization/publications/12 https://www.researchgate.net/topic/Commercialization/publications/12 pdf_download_page_urls count: 1199 99 https://www.researchgate.net/topic/Commercialization/publications/13 https://www.researchgate.net/topic/Commercialization/publications/13 pdf_download_page_urls count: 1299 100 https://www.researchgate.net/topic/Commercialization/publications/14 https://www.researchgate.net/topic/Commercialization/publications/14 pdf_download_page_urls count: 1399 100 https://www.researchgate.net/topic/Commercialization/publications/15 https://www.researchgate.net/topic/Commercialization/publications/15 pdf_download_page_urls count: 1499 100 https://www.researchgate.net/topic/Commercialization/publications/16 https://www.researchgate.net/topic/Commercialization/publications/16 pdf_download_page_urls count: 1599 100 https://www.researchgate.net/topic/Commercialization/publications/17 https://www.researchgate.net/topic/Commercialization/publications/17 pdf_download_page_urls count: 1699 100 https://www.researchgate.net/topic/Commercialization/publications/18 https://www.researchgate.net/topic/Commercialization/publications/18 pdf_download_page_urls count: 1799 100 https://www.researchgate.net/topic/Commercialization/publications/19 https://www.researchgate.net/topic/Commercialization/publications/19 pdf_download_page_urls count: 1899 100 https://www.researchgate.net/topic/Commercialization/publications/20 https://www.researchgate.net/topic/Commercialization/publications/20 pdf_download_page_urls count: 1999 100 https://www.researchgate.net/topic/Commercialization/publications/21 https://www.researchgate.net/topic/Commercialization/publications/21 pdf_download_page_urls count: 2099 100 https://www.researchgate.net/topic/Commercialization/publications/22 https://www.researchgate.net/topic/Commercialization/publications/22 pdf_download_page_urls count: 2199 100 https://www.researchgate.net/topic/Commercialization/publications/23 https://www.researchgate.net/topic/Commercialization/publications/23 pdf_download_page_urls count: 2299 100 https://www.researchgate.net/topic/Commercialization/publications/24 https://www.researchgate.net/topic/Commercialization/publications/24 pdf_download_page_urls count: 2399 100 https://www.researchgate.net/topic/Commercialization/publications/25 https://www.researchgate.net/topic/Commercialization/publications/25 pdf_download_page_urls count: 2499 100 https://www.researchgate.net/topic/Commercialization/publications/26 https://www.researchgate.net/topic/Commercialization/publications/26 pdf_download_page_urls count: 2599 100 https://www.researchgate.net/topic/Commercialization/publications/27 https://www.researchgate.net/topic/Commercialization/publications/27 pdf_download_page_urls count: 2699 100 https://www.researchgate.net/topic/Commercialization/publications/28 https://www.researchgate.net/topic/Commercialization/publications/28 pdf_download_page_urls count: 2799 100 https://www.researchgate.net/topic/Commercialization/publications/29 https://www.researchgate.net/topic/Commercialization/publications/29 pdf_download_page_urls count: 2899 100 https://www.researchgate.net/topic/Commercialization/publications/30 https://www.researchgate.net/topic/Commercialization/publications/30 pdf_download_page_urls count: 2999 100 https://www.researchgate.net/topic/Commercialization/publications/31 https://www.researchgate.net/topic/Commercialization/publications/31 pdf_download_page_urls count: 3099 100 https://www.researchgate.net/topic/Commercialization/publications/32 https://www.researchgate.net/topic/Commercialization/publications/32 pdf_download_page_urls count: 3199 100 https://www.researchgate.net/topic/Commercialization/publications/33 https://www.researchgate.net/topic/Commercialization/publications/33 pdf_download_page_urls count: 3299 100 https://www.researchgate.net/topic/Commercialization/publications/34 https://www.researchgate.net/topic/Commercialization/publications/34 pdf_download_page_urls count: 3399 100 https://www.researchgate.net/topic/Commercialization/publications/35 https://www.researchgate.net/topic/Commercialization/publications/35 pdf_download_page_urls count: 3499 100 https://www.researchgate.net/topic/Commercialization/publications/36 https://www.researchgate.net/topic/Commercialization/publications/36 pdf_download_page_urls count: 3599 100 https://www.researchgate.net/topic/Commercialization/publications/37 https://www.researchgate.net/topic/Commercialization/publications/37 pdf_download_page_urls count: 3699 100 https://www.researchgate.net/topic/Commercialization/publications/38 https://www.researchgate.net/topic/Commercialization/publications/38 pdf_download_page_urls count: 3799 100 https://www.researchgate.net/topic/Commercialization/publications/39 https://www.researchgate.net/topic/Commercialization/publications/39 pdf_download_page_urls count: 3899 100 https://www.researchgate.net/topic/Commercialization/publications/40 https://www.researchgate.net/topic/Commercialization/publications/40 pdf_download_page_urls count: 3999 100 https://www.researchgate.net/topic/Commercialization/publications/41 https://www.researchgate.net/topic/Commercialization/publications/41 pdf_download_page_urls count: 4099 100 https://www.researchgate.net/topic/Commercialization/publications/42 https://www.researchgate.net/topic/Commercialization/publications/42 pdf_download_page_urls count: 4199 100 https://www.researchgate.net/topic/Commercialization/publications/43 https://www.researchgate.net/topic/Commercialization/publications/43 pdf_download_page_urls count: 4299 100 https://www.researchgate.net/topic/Commercialization/publications/44 https://www.researchgate.net/topic/Commercialization/publications/44 pdf_download_page_urls count: 4399 100 https://www.researchgate.net/topic/Commercialization/publications/45 https://www.researchgate.net/topic/Commercialization/publications/45 pdf_download_page_urls count: 4499 100 https://www.researchgate.net/topic/Commercialization/publications/46 https://www.researchgate.net/topic/Commercialization/publications/46 pdf_download_page_urls count: 4599 100 https://www.researchgate.net/topic/Commercialization/publications/47 https://www.researchgate.net/topic/Commercialization/publications/47 pdf_download_page_urls count: 4699 100 https://www.researchgate.net/topic/Commercialization/publications/48 https://www.researchgate.net/topic/Commercialization/publications/48 pdf_download_page_urls count: 4799 100 https://www.researchgate.net/topic/Commercialization/publications/49 https://www.researchgate.net/topic/Commercialization/publications/49 pdf_download_page_urls count: 4899 100 https://www.researchgate.net/topic/Commercialization/publications/50 https://www.researchgate.net/topic/Commercialization/publications/50 pdf_download_page_urls count: 4999 100 https://www.researchgate.net/topic/Commercialization/publications/51 https://www.researchgate.net/topic/Commercialization/publications/51 pdf_download_page_urls count: 5099 100 https://www.researchgate.net/topic/Commercialization/publications/52 https://www.researchgate.net/topic/Commercialization/publications/52 pdf_download_page_urls count: 5199 100 https://www.researchgate.net/topic/Commercialization/publications/53 https://www.researchgate.net/topic/Commercialization/publications/53 pdf_download_page_urls count: 5299 100 https://www.researchgate.net/topic/Commercialization/publications/54 https://www.researchgate.net/topic/Commercialization/publications/54 pdf_download_page_urls count: 5399 100 https://www.researchgate.net/topic/Commercialization/publications/55 https://www.researchgate.net/topic/Commercialization/publications/55 pdf_download_page_urls count: 5499 100 https://www.researchgate.net/topic/Commercialization/publications/56 https://www.researchgate.net/topic/Commercialization/publications/56 pdf_download_page_urls count: 5599 100 https://www.researchgate.net/topic/Commercialization/publications/57 https://www.researchgate.net/topic/Commercialization/publications/57 pdf_download_page_urls count: 5699 100 https://www.researchgate.net/topic/Commercialization/publications/58 https://www.researchgate.net/topic/Commercialization/publications/58 pdf_download_page_urls count: 5799 100 https://www.researchgate.net/topic/Commercialization/publications/59 https://www.researchgate.net/topic/Commercialization/publications/59 pdf_download_page_urls count: 5899 100 https://www.researchgate.net/topic/Commercialization/publications/60 https://www.researchgate.net/topic/Commercialization/publications/60 pdf_download_page_urls count: 5999 100 https://www.researchgate.net/topic/Commercialization/publications/61 https://www.researchgate.net/topic/Commercialization/publications/61 pdf_download_page_urls count: 6099 100 https://www.researchgate.net/topic/Commercialization/publications/62 https://www.researchgate.net/topic/Commercialization/publications/62 pdf_download_page_urls count: 6199 100 https://www.researchgate.net/topic/Commercialization/publications/63 https://www.researchgate.net/topic/Commercialization/publications/63 pdf_download_page_urls count: 6299 100 https://www.researchgate.net/topic/Commercialization/publications/64 https://www.researchgate.net/topic/Commercialization/publications/64 pdf_download_page_urls count: 6399 100 https://www.researchgate.net/topic/Commercialization/publications/65 https://www.researchgate.net/topic/Commercialization/publications/65 pdf_download_page_urls count: 6499 100 https://www.researchgate.net/topic/Commercialization/publications/66 https://www.researchgate.net/topic/Commercialization/publications/66 pdf_download_page_urls count: 6599 100 https://www.researchgate.net/topic/Commercialization/publications/67 https://www.researchgate.net/topic/Commercialization/publications/67 pdf_download_page_urls count: 6699 100 https://www.researchgate.net/topic/Commercialization/publications/68 https://www.researchgate.net/topic/Commercialization/publications/68 pdf_download_page_urls count: 6799 100 https://www.researchgate.net/topic/Commercialization/publications/69 https://www.researchgate.net/topic/Commercialization/publications/69 pdf_download_page_urls count: 6899 100 https://www.researchgate.net/topic/Commercialization/publications/70 https://www.researchgate.net/topic/Commercialization/publications/70 pdf_download_page_urls count: 6999 100 https://www.researchgate.net/topic/Commercialization/publications/71 https://www.researchgate.net/topic/Commercialization/publications/71 pdf_download_page_urls count: 7099 100 https://www.researchgate.net/topic/Commercialization/publications/72 https://www.researchgate.net/topic/Commercialization/publications/72 pdf_download_page_urls count: 7199 100 https://www.researchgate.net/topic/Commercialization/publications/73 https://www.researchgate.net/topic/Commercialization/publications/73 pdf_download_page_urls count: 7299 100 https://www.researchgate.net/topic/Commercialization/publications/74 https://www.researchgate.net/topic/Commercialization/publications/74 pdf_download_page_urls count: 7399 100 https://www.researchgate.net/topic/Commercialization/publications/75 https://www.researchgate.net/topic/Commercialization/publications/75 pdf_download_page_urls count: 7499 100 https://www.researchgate.net/topic/Commercialization/publications/76 https://www.researchgate.net/topic/Commercialization/publications/76 pdf_download_page_urls count: 7599 100 https://www.researchgate.net/topic/Commercialization/publications/77 https://www.researchgate.net/topic/Commercialization/publications/77 pdf_download_page_urls count: 7699 100 https://www.researchgate.net/topic/Commercialization/publications/78 https://www.researchgate.net/topic/Commercialization/publications/78 pdf_download_page_urls count: 7799 100 https://www.researchgate.net/topic/Commercialization/publications/79 https://www.researchgate.net/topic/Commercialization/publications/79 pdf_download_page_urls count: 7899 100 https://www.researchgate.net/topic/Commercialization/publications/80 https://www.researchgate.net/topic/Commercialization/publications/80 pdf_download_page_urls count: 7999 100 https://www.researchgate.net/topic/Commercialization/publications/81 https://www.researchgate.net/topic/Commercialization/publications/81 pdf_download_page_urls count: 8099 100 https://www.researchgate.net/topic/Commercialization/publications/82 https://www.researchgate.net/topic/Commercialization/publications/82 pdf_download_page_urls count: 8199 100 https://www.researchgate.net/topic/Commercialization/publications/83 https://www.researchgate.net/topic/Commercialization/publications/83 pdf_download_page_urls count: 8299 100 https://www.researchgate.net/topic/Commercialization/publications/84 https://www.researchgate.net/topic/Commercialization/publications/84 pdf_download_page_urls count: 8399 100 https://www.researchgate.net/topic/Commercialization/publications/85 https://www.researchgate.net/topic/Commercialization/publications/85 pdf_download_page_urls count: 8499 100 https://www.researchgate.net/topic/Commercialization/publications/86 https://www.researchgate.net/topic/Commercialization/publications/86 pdf_download_page_urls count: 8599 100 https://www.researchgate.net/topic/Commercialization/publications/87 https://www.researchgate.net/topic/Commercialization/publications/87 pdf_download_page_urls count: 8699 100 https://www.researchgate.net/topic/Commercialization/publications/88 https://www.researchgate.net/topic/Commercialization/publications/88 pdf_download_page_urls count: 8799 100 https://www.researchgate.net/topic/Commercialization/publications/89 https://www.researchgate.net/topic/Commercialization/publications/89 pdf_download_page_urls count: 8899 100 https://www.researchgate.net/topic/Commercialization/publications/90 https://www.researchgate.net/topic/Commercialization/publications/90 pdf_download_page_urls count: 8999 100 https://www.researchgate.net/topic/Commercialization/publications/91 https://www.researchgate.net/topic/Commercialization/publications/91 pdf_download_page_urls count: 9099 100 https://www.researchgate.net/topic/Commercialization/publications/92 https://www.researchgate.net/topic/Commercialization/publications/92 pdf_download_page_urls count: 9199 100 https://www.researchgate.net/topic/Commercialization/publications/93 https://www.researchgate.net/topic/Commercialization/publications/93 pdf_download_page_urls count: 9299 100 https://www.researchgate.net/topic/Commercialization/publications/94 https://www.researchgate.net/topic/Commercialization/publications/94 pdf_download_page_urls count: 9399 100 https://www.researchgate.net/topic/Commercialization/publications/95 https://www.researchgate.net/topic/Commercialization/publications/95 pdf_download_page_urls count: 9499 100 https://www.researchgate.net/topic/Commercialization/publications/96 https://www.researchgate.net/topic/Commercialization/publications/96 pdf_download_page_urls count: 9599 100 https://www.researchgate.net/topic/Commercialization/publications/97 https://www.researchgate.net/topic/Commercialization/publications/97 pdf_download_page_urls count: 9699 100 https://www.researchgate.net/topic/Commercialization/publications/98 https://www.researchgate.net/topic/Commercialization/publications/98 pdf_download_page_urls count: 9799 100 https://www.researchgate.net/topic/Commercialization/publications/99 https://www.researchgate.net/topic/Commercialization/publications/99 pdf_download_page_urls count: 9899 100 https://www.researchgate.net/topic/Commercialization/publications/100 https://www.researchgate.net/topic/Commercialization/publications/100 pdf_download_page_urls count: 9999 100 None None 9999
In [11]:
str(cite_download_urls[0])
Out[11]:
'<div class="nova-legacy-o-pack__item nova-legacy-v-publication-item__footer-button-group"><div class="nova-legacy-c-button-group nova-legacy-c-button-group--gutter-m nova-legacy-c-button-group--orientation-horizontal nova-legacy-c-button-group--width-auto nova-legacy-v-publication-item__actions"><div class="nova-legacy-c-button-group__item"><a class="nova-legacy-c-button nova-legacy-c-button--align-center nova-legacy-c-button--radius-m nova-legacy-c-button--size-s nova-legacy-c-button--color-blue nova-legacy-c-button--theme-bare nova-legacy-v-publication-item__action" href="publication/23738978_The_Human_Mind_as_a_Barrier_to_Wiser_Environmental_Agreements/citation/download"><span class="nova-legacy-c-button__label">Cite</span></a></div><div class="nova-legacy-c-button-group__item"><a class="nova-legacy-c-button nova-legacy-c-button--align-center nova-legacy-c-button--radius-m nova-legacy-c-button--size-s nova-legacy-c-button--color-blue nova-legacy-c-button--theme-ghost nova-legacy-v-publication-item__action" href="publication/23738978_The_Human_Mind_as_a_Barrier_to_Wiser_Environmental_Agreements/download"><span class="nova-legacy-c-button__label">Download full-text</span></a></div></div></div>'
2. 获取pdf url¶
In [2]:
import selenium
from selenium import webdriver
import json
import time
option = webdriver.ChromeOptions()
option.add_experimental_option("excludeSwitches", ['enable-automation'])
option.add_argument("--disable-javascript")
option.add_argument("--disable-cache")
option.add_experimental_option(
'prefs',
{
"download.default_directory": "D:\\out_pdfs",
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"plugins.always_open_pdf_externally": True, # 这句配置很重要
"profile.managed_default_content_settings.images": 2,
"profile.managed_default_content_settings.media_stream": 2
}
)
driver = webdriver.Chrome(
executable_path="d:/chromedriver.exe",
chrome_options=option
)
driver.implicitly_wait(5)
pdf_download_page_urls = json.load(open(r'D:\out_pdfs\pdf_download_page_urls_2023-07-06.json',"r", encoding="utf-8"))
pdf_urls = {}
print("pdf_download_page_urls", len(set(pdf_download_page_urls)))
driver.get("https://www.researchgate.net/" + pdf_download_page_urls[0])
for download_page_url in set(pdf_download_page_urls):
# driver.get("https://www.researchgate.net/" + download_page_url.replace("/download", ""))
driver.get("https://www.researchgate.net/" + download_page_url)
try:
pdf_url = driver.find_element_by_css_selector(
"a[data-lite]").get_attribute("href")
# pdf_detail = None
# try:
# pdf_detail = driver.find_element_by_css_selector("div.research-detail-header-section__ie11").get_attribute("innerHTML")
# except:
# print("获取DOI出错:", pdf_url)
# print(pdf_url, pdf_detail)
# if pdf_url:
# pdf_urls[download_page_url] = (pdf_url, pdf_detail)
except Exception as e:
print("ERROR", e, end="\r")
pdf_urls[download_page_url] = None
time.sleep(0.5)
d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\ipykernel_launcher.py:23: DeprecationWarning: use options instead of chrome_options
pdf_download_page_urls 300
ERROR Message: no such element: Unable to locate element: {"method":"css selector","selector":"a[data-lite]"}
(Session info: chrome=114.0.5735.199)
ERROR Message: no such element: Unable to locate element: {"method":"css selector","selector":"a[data-lite]"}
(Session info: chrome=114.0.5735.199)
ERROR Message: no such element: Unable to locate element: {"method":"css selector","selector":"a[data-lite]"}
(Session info: chrome=114.0.5735.199)
ERROR Message: no such element: Unable to locate element: {"method":"css selector","selector":"a[data-lite]"}
(Session info: chrome=114.0.5735.199)
--------------------------------------------------------------------------- KeyboardInterrupt Traceback (most recent call last) <ipython-input-2-33485c74a7a1> in <module> 34 try: 35 pdf_url = driver.find_element_by_css_selector( ---> 36 "a[data-lite]").get_attribute("href") 37 # pdf_detail = None 38 # try: d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\selenium\webdriver\remote\webdriver.py in find_element_by_css_selector(self, css_selector) 596 element = driver.find_element_by_css_selector('#foo') 597 """ --> 598 return self.find_element(by=By.CSS_SELECTOR, value=css_selector) 599 600 def find_elements_by_css_selector(self, css_selector): d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\selenium\webdriver\remote\webdriver.py in find_element(self, by, value) 976 return self.execute(Command.FIND_ELEMENT, { 977 'using': by, --> 978 'value': value})['value'] 979 980 def find_elements(self, by=By.ID, value=None): d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params) 317 318 params = self._wrap_value(params) --> 319 response = self.command_executor.execute(driver_command, params) 320 if response: 321 self.error_handler.check_response(response) d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\selenium\webdriver\remote\remote_connection.py in execute(self, command, params) 372 data = utils.dump_json(params) 373 url = '%s%s' % (self._url, path) --> 374 return self._request(command_info[0], url, body=data) 375 376 def _request(self, method, url, body=None): d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\selenium\webdriver\remote\remote_connection.py in _request(self, method, url, body) 395 396 if self.keep_alive: --> 397 resp = self._conn.request(method, url, body=body, headers=headers) 398 399 statuscode = resp.status d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\urllib3\request.py in request(self, method, url, fields, headers, **urlopen_kw) 77 else: 78 return self.request_encode_body( ---> 79 method, url, fields=fields, headers=headers, **urlopen_kw 80 ) 81 d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\urllib3\request.py in request_encode_body(self, method, url, fields, headers, encode_multipart, multipart_boundary, **urlopen_kw) 168 extra_kw.update(urlopen_kw) 169 --> 170 return self.urlopen(method, url, **extra_kw) d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\urllib3\poolmanager.py in urlopen(self, method, url, redirect, **kw) 373 response = conn.urlopen(method, url, **kw) 374 else: --> 375 response = conn.urlopen(method, u.request_uri, **kw) 376 377 redirect_location = redirect and response.get_redirect_location() d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw) 704 body=body, 705 headers=headers, --> 706 chunked=chunked, 707 ) 708 d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\urllib3\connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw) 443 # Python 3 (including for exceptions like SystemExit). 444 # Otherwise it looks like a bug in the code. --> 445 six.raise_from(e, None) 446 except (SocketTimeout, BaseSSLError, SocketError) as e: 447 self._raise_timeout(err=e, url=url, timeout_value=read_timeout) d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\urllib3\packages\six.py in raise_from(value, from_value) d:\Users\Administrator\anaconda3\envs\crawler\lib\site-packages\urllib3\connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw) 438 # Python 3 439 try: --> 440 httplib_response = conn.getresponse() 441 except BaseException as e: 442 # Remove the TypeError from the exception chain in d:\Users\Administrator\anaconda3\envs\crawler\lib\http\client.py in getresponse(self) 1377 try: 1378 try: -> 1379 response.begin() 1380 except ConnectionError: 1381 self.close() d:\Users\Administrator\anaconda3\envs\crawler\lib\http\client.py in begin(self) 309 # read until we get a non-100 response 310 while True: --> 311 version, status, reason = self._read_status() 312 if status != CONTINUE: 313 break d:\Users\Administrator\anaconda3\envs\crawler\lib\http\client.py in _read_status(self) 270 271 def _read_status(self): --> 272 line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") 273 if len(line) > _MAXLINE: 274 raise LineTooLong("status line") d:\Users\Administrator\anaconda3\envs\crawler\lib\socket.py in readinto(self, b) 584 while True: 585 try: --> 586 return self._sock.recv_into(b) 587 except timeout: 588 self._timeout_occurred = True KeyboardInterrupt:
3. 下载pdf¶
In [ ]:
import selenium
from selenium import webdriver
option = webdriver.ChromeOptions()
option.add_argument("--disable-javascript")
option.add_experimental_option("excludeSwitches", ['enable-automation'])
option.add_experimental_option(
'prefs',
{
"download.default_directory": "D:\\out_pdfs",
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"plugins.always_open_pdf_externally": True, # 这句配置很重要
"profile.managed_default_content_settings.images": 2,
"profile.managed_default_content_settings.media_stream": 2
}
)
driver = webdriver.Chrome(
executable_path="d:/chromedriver.exe",
chrome_options=option
)
for key, val in pdf_urls.items():
if val:
url = val[0]
driver.get(url)
driver.get("chrome://downloads")
time.sleep(1)
dl_files = driver.execute_script(
"return document.querySelector(\"downloads-manager\").shadowRoot.querySelector(\"#downloadsList\").items")
print("dl_files", len(dl_files))
json.dump(dl_files, open("d:/out_pdfs/download_info.json", "w", encoding="utf-8"), ensure_ascii=False)
# driver.quit()
In [ ]:
dl_files = driver.execute_script(
"return document.querySelector(\"downloads-manager\").shadowRoot.querySelector(\"#downloadsList\").items")
print("dl_files", len(dl_files))
json.dump(dl_files, open("d:/out_pdfs/download_info.json", "w", encoding="utf-8"), ensure_ascii=False)
# driver.quit()
In [ ]:
pdf_urls = list(map(lambda it:(it["url"], it["fileName"]), dl_files))
json.dump(pdf_urls, open("d:/out_pdfs/downloaded_files_info.json", "w", encoding="utf-8"), ensure_ascii=False)
In [ ]:
to_download_set = set(map(lambda it: it.split("/")[1], pdf_download_page_urls))
downloaded_set = set(map(lambda it: it[0][it[0].index("publication"):].split("/")[1], pdf_urls))
In [ ]:
driver.get("https://sci-hub.hkvisa.net/10.58567/jea02030006")
从 api.crossref.org 获取数据¶
In [ ]:
import requests
headers = {
'accept': 'application/json',
}
doi = '10.2139/ssrn.1623526'.replace("/", "%2F")
resp = requests.get(f"https://api.crossref.org/works/{doi}", headers=headers).json()
url = resp["message"]["link"][0]["URL"]
print("url", url)
requests.get(url).content
获取CITE_URL¶
In [1]:
import os
walked = set()
directory = 'E:\\Citations'
for file in os.listdir(directory):
walked.add(file.split(".")[0].split("-")[-1])
print(len(walked), list(walked)[:2])
28119 ['356169192', '349772167']
In [2]:
import re
import json
uids = []
pattern = re.compile(r"publication/(\d+)_")
for file in os.listdir("d:/out_pdfs/"):
if 'pdf_download_page_urls' in file:
need_download = json.load(open(fr'd:/out_pdfs/{file}',"r", encoding="utf-8"))
need_download = list(map(lambda it: pattern.findall(it)[0], need_download))
print(file, len(need_download))
uids.extend(need_download)
print(len(uids))
Account-Books_pdf_download_page_urls.json 337 Accountability_pdf_download_page_urls.json 9988 Accounting-Education_pdf_download_page_urls.json 783 Accounting-History_pdf_download_page_urls.json 317 Accounting-Information-Systems_pdf_download_page_urls.json 1161 Accounting-Processes_pdf_download_page_urls.json 1067 Accounting-Research_pdf_download_page_urls.json 1840 Accounting-Scholarship_pdf_download_page_urls.json 30 Accounting-Software_pdf_download_page_urls.json 394 Accounting-Standard-Setting_pdf_download_page_urls.json 130 Accounting-Theory_pdf_download_page_urls.json 880 Accounting_pdf_download_page_urls.json 9984 Accounts_pdf_download_page_urls.json 9999 Auditing_pdf_download_page_urls.json 10000 Audition_pdf_download_page_urls.json 3416 Auditory-Brain-Stem-Implantation_pdf_download_page_urls.json 27 Auditory-Brainstem-Response_pdf_download_page_urls.json 2801 auditory-cortex_pdf_download_page_urls.json 6079 Auditory-Evoked-Potentials_pdf_download_page_urls.json 3476 Auditory-Hallucinations_pdf_download_page_urls.json 1660 Auditory-Interfaces_pdf_download_page_urls.json 144 Auditory-Neuroscience_pdf_download_page_urls.json 103 Auditory-Perception_pdf_download_page_urls.json 7527 Auditory-Sustained-Attention_pdf_download_page_urls.json 3 Auditory-System_pdf_download_page_urls.json 5988 auditory_steady-state_response_assr_pdf_download_page_urls.json 289 Audit_pdf_download_page_urls.json 10000 Commercialization-Of-Products-Services_pdf_download_page_urls.json 9 Commercialization_pdf_download_page_urls.json 9999 Eco-Social-Market-Economy_pdf_download_page_urls.json 11 Ecological-Economics_pdf_download_page_urls.json 4532 Econometric-Methods_pdf_download_page_urls.json 2385 Econometric-Modeling_pdf_download_page_urls.json 1737 econometric-models_pdf_download_page_urls.json 8673 Econometric-Techniques_pdf_download_page_urls.json 0 Econometrics-Packages_pdf_download_page_urls.json 38 Econometrics_pdf_download_page_urls.json 3600 Economic-Analysis_pdf_download_page_urls.json 9984 Economic-Bubble_pdf_download_page_urls.json 107 Economic-Criminology_pdf_download_page_urls.json 6 Economic-Crisis_pdf_download_page_urls.json 9993 Economic-Development_pdf_download_page_urls.json 10000 Economic-dispatch_pdf_download_page_urls.json 2508 Economic-Dynamics_pdf_download_page_urls.json 2208 Economic-Efficiency_pdf_download_page_urls.json 948 Economic-Evaluation_pdf_download_page_urls.json 2910 Economic-Forecasting_pdf_download_page_urls.json 1700 Economic-Geography_pdf_download_page_urls.json 3361 Economic-Geology_pdf_download_page_urls.json 513 Economic-Growth_pdf_download_page_urls.json 10000 Economic-History_pdf_download_page_urls.json 5174 Economic-Indicators_pdf_download_page_urls.json 418 Economic-Inequality_pdf_download_page_urls.json 1092 Economic-Inflation_pdf_download_page_urls.json 3798 Economic-Integration_pdf_download_page_urls.json 1136 Economic-Modeling_pdf_download_page_urls.json 2001 Economic-models_pdf_download_page_urls.json 9970 Economic-Policy-Analysis_pdf_download_page_urls.json 123 Economic-Policy_pdf_download_page_urls.json 9988 Economic-Psychology_pdf_download_page_urls.json 944 Economic-Recession_pdf_download_page_urls.json 0 Economic-Sanctions_pdf_download_page_urls.json 1443 Economic-Security_pdf_download_page_urls.json 4203 Economic-Sociology_pdf_download_page_urls.json 2466 Economic-Theory_pdf_download_page_urls.json 9952 Economic-Valuation_pdf_download_page_urls.json 2641 Economical-Psychology_pdf_download_page_urls.json 0 Economics-Analysis_pdf_download_page_urls.json 963 Economics-of-Climate-Adaptation_pdf_download_page_urls.json 1 Economics-of-Innovation_pdf_download_page_urls.json 96 Economics-of-Regulation_pdf_download_page_urls.json 0 Economics-of-Science_pdf_download_page_urls.json 2541 Economics-of-Technology_pdf_download_page_urls.json 7079 Economy_pdf_download_page_urls.json 9999 Emerging-Economies_pdf_download_page_urls.json 9988 Energy-Economics_pdf_download_page_urls.json 2601 Engineering-Economy_pdf_download_page_urls.json 30 Entrepreneurial-Economics_pdf_download_page_urls.json 83 Environmental-Economics_pdf_download_page_urls.json 9377 European-Economy_pdf_download_page_urls.json 2066 Evolutionary-Economic-Geography_pdf_download_page_urls.json 278 Evolutionary-Economics_pdf_download_page_urls.json 977 Experimental-Economics_pdf_download_page_urls.json 1576 Finance-Management_pdf_download_page_urls.json 1751 Finance_pdf_download_page_urls.json 9999 Financial-Accounting-Theory_pdf_download_page_urls.json 8 Financial-Accounting_pdf_download_page_urls.json 3894 Financial-Analysis_pdf_download_page_urls.json 4716 Financial-Criminology_pdf_download_page_urls.json 6 Financial-Crises_pdf_download_page_urls.json 7302 Financial-Crisis_pdf_download_page_urls.json 9996 Financial-Development_pdf_download_page_urls.json 6524 Financial-Econometrics_pdf_download_page_urls.json 258 Financial-Economics_pdf_download_page_urls.json 1193 Financial-Engineering_pdf_download_page_urls.json 293 Financial-Flows_pdf_download_page_urls.json 1832 Financial-History_pdf_download_page_urls.json 214 Financial-Inclusion_pdf_download_page_urls.json 2409 Financial-institutions_pdf_download_page_urls.json 9997 Financial-Integration_pdf_download_page_urls.json 364 Financial-Intermediation_pdf_download_page_urls.json 9980 Financial-Law_pdf_download_page_urls.json 346 Financial-market-regulation_pdf_download_page_urls.json 211 Financial-Markets_pdf_download_page_urls.json 9994 Financial-Mathematics_pdf_download_page_urls.json 906 Financial-Modeling_pdf_download_page_urls.json 1837 Financial-Modelling_pdf_download_page_urls.json 287 Financial-Physics_pdf_download_page_urls.json 329 Financial-Reporting_pdf_download_page_urls.json 9997 Financial-Risk-Sharing_pdf_download_page_urls.json 91 Financial-Services-industry_pdf_download_page_urls.json 1164 Financial-Services_pdf_download_page_urls.json 9988 Financial-Stability_pdf_download_page_urls.json 7064 Financial-Statement-Analysis_pdf_download_page_urls.json 504 financial-statements_pdf_download_page_urls.json 9992 Financial-Support_pdf_download_page_urls.json 9994 Financial-Valuation_pdf_download_page_urls.json 286 Maintenance-Management_pdf_download_page_urls.json 2597 Managed-Relocation_pdf_download_page_urls.json 74 Management-Accounting-Techniques_pdf_download_page_urls.json 205 Management-Accounting_pdf_download_page_urls.json 5242 Management-Consultancy_pdf_download_page_urls.json 1079 Management-Consulting_pdf_download_page_urls.json 659 Management-Control_pdf_download_page_urls.json 5021 Management-Development_pdf_download_page_urls.json 3134 Management-Education_pdf_download_page_urls.json 1549 Management-Engineering_pdf_download_page_urls.json 264 Management-Information-Systems_pdf_download_page_urls.json 3388 Management-Literature_pdf_download_page_urls.json 4047 Management-Of-Change_pdf_download_page_urls.json 3868 Management-Quality-Circles_pdf_download_page_urls.json 346 Management-Service-Organizations_pdf_download_page_urls.json 26 Management-studies_pdf_download_page_urls.json 4786 Management-System_pdf_download_page_urls.json 9996 Management-Theory_pdf_download_page_urls.json 4248 Managerial-Economics_pdf_download_page_urls.json 249 Manure-Management_pdf_download_page_urls.json 1340 Marine-Management_pdf_download_page_urls.json 744 Marine-Resource-Management_pdf_download_page_urls.json 447 Marketing-Management_pdf_download_page_urls.json 5467 Medical-Practice-Management_pdf_download_page_urls.json 396 Micro-Manager_pdf_download_page_urls.json 233 Museum-Management_pdf_download_page_urls.json 281 pdf_download_page_urls.json 9879 pdf_download_page_urls_2023-07-06.json 7600 pdf_download_page_urls_p100.json 9879 497419
In [24]:
uid_set = set(uids)
len(uid_set), len(uid_set - walked)
Out[24]:
(420383, 403631)
In [25]:
with open("cite_urls.txt", 'w', encoding='utf-8') as fpw:
for uid in (uid_set - walked):
url = f'https://www.researchgate.net/lite.publication.PublicationDownloadCitationModal.downloadCitation.html?publicationUid={uid}&fileType=BibTeX&citation=citation'
fpw.write(url+"\n")
解压¶
In [12]:
import zipfile
import os
def unzip_file(zip_file_path, extract_path):
with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
zip_ref.extractall(extract_path)
zip_dir = r'E:\researchgate'
extract_path = r'E:\Citations'
# 示例用法
for name in os.listdir(zip_dir):
if 'researchgate' in name:
zip_file_path = os.path.join(zip_dir, name)
print(zip_file_path)
unzip_file(zip_file_path, extract_path)
E:\researchgate\researchgate_10.150.108.15.zip E:\researchgate\researchgate_10.150.108.16.zip E:\researchgate\researchgate_10.150.108.160.zip E:\researchgate\researchgate_10.150.108.17.zip
In [11]:
import zipfile
import os
process_file = r'D:\WORK\UCP\ResearchGate\cite_urls.txt'
def get_uid(url):
# url = 'https://www.researchgate.net/lite.publication.PublicationDownloadCitationModal.downloadCitation.html?publicationUid=366367227&fileType=BibTeX&citation=citation'
uid = url[url.index("publicationUid="):url.index("&fileType")].replace('publicationUid=', '')
return uid
uids = set()
with open(process_file, 'r', encoding='utf-8') as fpr:
for line in fpr:
uids.add(get_uid(line))
print("uids", len(uids))
uids 403631
In [ ]:
In [1]:
import time
prefix = 'researchgate_cite'
process_file = r'D:\WORK\UCP\ResearchGate\cite_urls.txt'
line_count = 0
with open(process_file, 'r', encoding='utf-8', buffering=1024*1024*4) as fr:
for line in fr:
line_count += 1
out_file_count = int(line_count / 999)+1
count_per_file = int(line_count // out_file_count) + 1
print('process_file', process_file, 'line_count', line_count,
'out_file_count', out_file_count, 'count_per_file', count_per_file)
file_ind = 0
seg_file_fp = open(f'e:/{prefix}_sub_{file_ind}.txt', 'w', encoding='utf-8')
with open(process_file, 'r', encoding='utf-8', buffering=1024*1024*4) as fr:
line_no = 0
for line in fr:
line_no += 1
seg_file_fp.write(line.strip() + '\n')
if line_no > count_per_file:
line_no = 0
file_ind += 1
seg_file_fp.close()
seg_file_fp = open(
f'e:/{prefix}_sub_{file_ind}.txt', 'w', encoding='utf-8')
seg_file_fp.close()
print("success!")
process_file D:\WORK\UCP\ResearchGate\cite_urls.txt line_count 403631 out_file_count 405 count_per_file 997 success!
In [11]:
try:
import selenium
except ImportError:
import pip
pip.main(['install','-i','https://pypi.tuna.tsinghua.edu.cn/simple', 'selenium'])
import selenium
import selenium
from selenium import webdriver
import json
import time
import re
import os
def mkdir(path):
if not os.path.exists(path):
os.makedirs(path)
print("mkdir", path)
urls_dir = "d:/researchgate"
download_dir = "d:\\cites"
mkdir(urls_dir)
mkdir(download_dir)
option = webdriver.ChromeOptions()
option.add_experimental_option("excludeSwitches", ['enable-automation'])
option.add_argument("--disable-javascript")
option.add_argument(f"--user-data-dir={urls_dir}/chrome_for_Citations")
option.add_argument("--disable-cache")
option.add_experimental_option(
'prefs',
{
"download.default_directory": download_dir,
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"plugins.always_open_pdf_externally": True, # 这句配置很重要
"profile.managed_default_content_settings.images": 2,
"profile.managed_default_content_settings.media_stream": 2
}
)
walked = set()
for file in os.listdir(download_dir):
uid = file.split(".")[0].split("-")[-1]
url = f'https://www.researchgate.net/lite.publication.PublicationDownloadCitationModal.downloadCitation.html?publicationUid={uid}&fileType=BibTeX&citation=citation'
walked.add(url)
need_download_urls = set()
for file in os.listdir(urls_dir):
if "researchgate_cite" in file:
with open(os.path.join(urls_dir, file), 'r', encoding='utf-8') as fpr:
need_download_urls |= set(map(str.strip, fpr.readlines()))
print(f"all: {len(need_download_urls)}")
need_download_urls -= walked
print(f"need_download_urls: {len(need_download_urls)}, has_downloaded: {len(walked)}")
def get_driver():
import platform
if "3.6" in str(platform.python_version()):
driver = webdriver.Chrome(
executable_path="d:/chromedriver.exe",
chrome_options=option
)
else:
driver = webdriver.Chrome(options=option)
driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument',
{'source': 'Object.defineProperty(navigator, "webdriver", {get: () => undefined})'})
driver.implicitly_wait(2)
return driver
driver = get_driver()
ind = 0
for url in need_download_urls:
if url in walked:
print("skip", end="\r")
continue
print(ind, url, end="\r")
driver.get(url)
walked.add(url)
time.sleep(4)
ind += 1
if ind % 10 == 0:
driver.quit()
time.sleep(10)
driver = get_driver()
driver.quit()
all: 8074 need_download_urls: 8074, has_downloaded: 0
d:\Users\Administrator\anaconda3\envs\python3.6\lib\site-packages\ipykernel_launcher.py:56: DeprecationWarning: use options instead of chrome_options
4 https://www.researchgate.net/lite.publication.PublicationDownloadCitationModal.downloadCitation.html?publicationUid=367292969&fileType=BibTeX&citation=citation
--------------------------------------------------------------------------- NoSuchWindowException Traceback (most recent call last) <ipython-input-11-f06d2c4e6563> in <module> 70 continue 71 print(ind, url, end="\r") ---> 72 driver.get(url) 73 walked.add(url) 74 time.sleep(4) d:\Users\Administrator\anaconda3\envs\python3.6\lib\site-packages\selenium\webdriver\remote\webdriver.py in get(self, url) 331 Loads a web page in the current browser session. 332 """ --> 333 self.execute(Command.GET, {'url': url}) 334 335 @property d:\Users\Administrator\anaconda3\envs\python3.6\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params) 319 response = self.command_executor.execute(driver_command, params) 320 if response: --> 321 self.error_handler.check_response(response) 322 response['value'] = self._unwrap_value( 323 response.get('value', None)) d:\Users\Administrator\anaconda3\envs\python3.6\lib\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response) 240 alert_text = value['alert'].get('text') 241 raise exception_class(message, screen, stacktrace, alert_text) --> 242 raise exception_class(message, screen, stacktrace) 243 244 def _value_or_default(self, obj, key, default): NoSuchWindowException: Message: no such window: target window already closed from unknown error: web view not found (Session info: chrome=115.0.5790.110)
In [6]:
try:
import selenium
except ImportError:
import pip
pip.main(['install','-i','https://pypi.tuna.tsinghua.edu.cn/simple', 'selenium'])
import selenium
import selenium
from selenium import webdriver
import json
import time
import re
import os
def mkdir(path):
if not os.path.exists(path):
os.makedirs(path)
print("mkdir", path)
# urls_dir = "d:/researchgate"
# download_dir = "d:\\cites"
urls_dir = "d:/researchgate"
download_dir = "e:\\Citations"
mkdir(urls_dir)
mkdir(download_dir)
option = webdriver.ChromeOptions()
option.add_experimental_option("excludeSwitches", ['enable-automation'])
option.add_argument("--disable-javascript")
option.add_argument(f"--user-data-dir={urls_dir}/chrome_for_Citations")
option.add_argument("--disable-cache")
option.add_experimental_option(
'prefs',
{
"download.default_directory": download_dir,
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"plugins.always_open_pdf_externally": True, # 这句配置很重要
"profile.managed_default_content_settings.images": 2,
"profile.managed_default_content_settings.media_stream": 2
}
)
while True:
walked = set()
for file in os.listdir(download_dir):
uid = file.split(".")[0].split("-")[-1]
url = f'https://www.researchgate.net/lite.publication.PublicationDownloadCitationModal.downloadCitation.html?publicationUid={uid}&fileType=BibTeX&citation=citation'
walked.add(url)
need_download_urls = set()
for file in os.listdir(urls_dir):
if ".txt" in file:
with open(os.path.join(urls_dir, file), 'r', encoding='utf-8') as fpr:
need_download_urls |= set(map(str.strip, fpr.readlines()))
print(f"all: {len(need_download_urls)}")
need_download_urls -= walked
print(f"need_download_urls: {len(need_download_urls)}, has_downloaded: {len(walked)}")
def get_driver():
import platform
if "3.6" in str(platform.python_version()):
driver = webdriver.Chrome(
executable_path="d:/chromedriver.exe",
chrome_options=option
)
else:
driver = webdriver.Chrome(options=option)
driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument',
{'source': 'Object.defineProperty(navigator, "webdriver", {get: () => undefined})'})
driver.implicitly_wait(2)
return driver
driver = get_driver()
ind = 0
if len(need_download_urls) == 0:
break
for url in need_download_urls:
if url in walked:
print("skip", end="\r")
continue
driver.get(url)
walked.add(url)
time.sleep(1)
ind += 1
if ind % 5 == 0:
time.sleep(1.5)
print(ind,len(os.listdir(download_dir)), url, end="\r")
driver.quit()
# import shutil
# shutil.rmtree(r"D:\researchgate\chrome_for_Citations")
driver = get_driver()
driver.quit()
all: 16257 need_download_urls: 8, has_downloaded: 38054
d:\Users\Administrator\anaconda3\envs\python3.6\lib\site-packages\ipykernel_launcher.py:66: DeprecationWarning: use options instead of chrome_options
all: 16257tps://www.researchgate.net/lite.publication.PublicationDownloadCitationModal.downloadCitation.html?publicationUid=350660924&fileType=BibTeX&citation=citation need_download_urls: 4, has_downloaded: 38058
--------------------------------------------------------------------------- WebDriverException Traceback (most recent call last) <ipython-input-6-ef70c50f039a> in <module> 73 return driver 74 ---> 75 driver = get_driver() 76 ind = 0 77 if len(need_download_urls) == 0: <ipython-input-6-ef70c50f039a> in get_driver() 64 driver = webdriver.Chrome( 65 executable_path="d:/chromedriver.exe", ---> 66 chrome_options=option 67 ) 68 else: d:\Users\Administrator\anaconda3\envs\python3.6\lib\site-packages\selenium\webdriver\chrome\webdriver.py in __init__(self, executable_path, port, options, service_args, desired_capabilities, service_log_path, chrome_options, keep_alive) 79 remote_server_addr=self.service.service_url, 80 keep_alive=keep_alive), ---> 81 desired_capabilities=desired_capabilities) 82 except Exception: 83 self.quit() d:\Users\Administrator\anaconda3\envs\python3.6\lib\site-packages\selenium\webdriver\remote\webdriver.py in __init__(self, command_executor, desired_capabilities, browser_profile, proxy, keep_alive, file_detector, options) 155 warnings.warn("Please use FirefoxOptions to set browser profile", 156 DeprecationWarning, stacklevel=2) --> 157 self.start_session(capabilities, browser_profile) 158 self._switch_to = SwitchTo(self) 159 self._mobile = Mobile(self) d:\Users\Administrator\anaconda3\envs\python3.6\lib\site-packages\selenium\webdriver\remote\webdriver.py in start_session(self, capabilities, browser_profile) 250 parameters = {"capabilities": w3c_caps, 251 "desiredCapabilities": capabilities} --> 252 response = self.execute(Command.NEW_SESSION, parameters) 253 if 'sessionId' not in response: 254 response = response['value'] d:\Users\Administrator\anaconda3\envs\python3.6\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params) 319 response = self.command_executor.execute(driver_command, params) 320 if response: --> 321 self.error_handler.check_response(response) 322 response['value'] = self._unwrap_value( 323 response.get('value', None)) d:\Users\Administrator\anaconda3\envs\python3.6\lib\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response) 240 alert_text = value['alert'].get('text') 241 raise exception_class(message, screen, stacktrace, alert_text) --> 242 raise exception_class(message, screen, stacktrace) 243 244 def _value_or_default(self, obj, key, default): WebDriverException: Message: unknown error: Chrome failed to start: exited normally. (unknown error: DevToolsActivePort file doesn't exist) (The process started from chrome location C:\Users\Administrator\AppData\Local\Google\Chrome\Application\chrome.exe is no longer running, so ChromeDriver is assuming that Chrome has crashed.)
In [8]:
import zipfile
file_dir = r'E:\Citations'
file_extension = "bib"
# 压缩文件的名称
zip_name = 'researchgate.zip'
# 要压缩的文件列表
files = []
for name in os.listdir(file_dir):
if file_extension in name:
files.append(os.path.join(file_dir, name))
def create_zip(files, zip_name):
ind = 0
with zipfile.ZipFile(zip_name, 'w', compression=zipfile.ZIP_DEFLATED) as zipf:
for file in files:
ind += 1
print(ind, file, end='\r')
zipf.write(file, os.path.basename(file))
print(" "*100)
print(">>",zip_name, "total files count:", ind)
create_zip(files, zip_name)
>> researchgate.zip total files count: 38057btations\citation-223385882.bib E:\Citations\citation-228102848.bib E:\Citations\citation-23510808.bib E:\Citations\citation-2475639.bib E:\Citations\citation-256021248.bib E:\Citations\citation-263845401.bib E:\Citations\citation-278026202.bib E:\Citations\citation-289925494.bib E:\Citations\citation-299707569.bib E:\Citations\citation-308856328.bib E:\Citations\citation-321385442.bib E:\Citations\citation-326365773.bibE:\Citations\citation-329136193.bib E:\Citations\citation-338564012.bib E:\Citations\citation-340066144.bib E:\Citations\citation-340835515.bib E:\Citations\citation-343486595.bib E:\Citations\citation-346033489.bib E:\Citations\citation-346800672.bib E:\Citations\citation-348000394.bib E:\Citations\citation-348631139.bib E:\Citations\citation-349350863.bib E:\Citations\citation-352011216.bib E:\Citations\citation-357392736.bibE:\Citations\citation-361075914.bib E:\Citations\citation-361563863.bib E:\Citations\citation-361956956.bib E:\Citations\citation-362888219.bib E:\Citations\citation-363367708.bib E:\Citations\citation-363881405.bibE:\Citations\citation-364349877.bib E:\Citations\citation-364957045.bib E:\Citations\citation-367194036.bib E:\Citations\citation-367690844.bib E:\Citations\citation-368457510.bib E:\Citations\citation-369424168.bib E:\Citations\citation-369834258.bib E:\Citations\citation-370262093.bib E:\Citations\citation-371136044.bibE:\Citations\citation-47727321.bib E:\Citations\citation-51725942.bib