From da720a61318b5abb7f803ccd836b3e1e97f036be Mon Sep 17 00:00:00 2001 From: radvanyimome <97281689+radvanyimome@users.noreply.github.com> Date: Tue, 4 Apr 2023 15:37:22 +0200 Subject: [PATCH] added wos selenium crawler slightly updated WOS data processing --- WOS/wos_extract/wos_query_generator.ipynb | 230 +++++++++-- WOS/wos_extract/wossel_miners.py | 266 ++++++++++++ WOS/wos_processing.ipynb | 479 +++++++++++++++++----- 3 files changed, 855 insertions(+), 120 deletions(-) create mode 100644 WOS/wos_extract/wossel_miners.py diff --git a/WOS/wos_extract/wos_query_generator.ipynb b/WOS/wos_extract/wos_query_generator.ipynb index db1cf5f..bda3936 100644 --- a/WOS/wos_extract/wos_query_generator.ipynb +++ b/WOS/wos_extract/wos_query_generator.ipynb @@ -2,19 +2,21 @@ "cells": [ { "cell_type": "code", - "execution_count": 50, + "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ + "import os\n", + "\n", "import pandas as pd\n", "focal_countries_list = [\"Peoples R china\", \"Hong Kong\"]" ] }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 2, "outputs": [], "source": [ "country_mode = \"CU\" #CU-country-region AU-address" @@ -28,7 +30,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 3, "outputs": [], "source": [ "# (TS=(\"artificial intelligence\") OR TS=(\"machine learning\") OR TS=(\"neural network\") OR TS=(\"big data\") OR TS=(\"deep learning\") OR TS=(\"computer vision\") OR TS=(\"pattern recognition\")) AND" @@ -42,13 +44,13 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 4, "outputs": [ { "data": { "text/plain": "'TS=(\"artificial intelligence\") OR TS=(\"machine learning\") OR TS=(\"neural network\") OR TS=(\"big data\") OR TS=(\"deep learning\") OR TS=(\"pattern recognition\") OR TS=(\"computer vision\") OR TS=(\"image classification\") OR TS=(\"reinforcement learning\") OR TS=(\"support vector machines\") OR TS=(\"recommender system\") OR TS=(\"random forest\") OR TS=(\"ensemble model\") OR TS=(\"image processing\") OR TS=(\"generative network\") OR TS=(\"ai ethic\") OR TS=(\"natural language processing\") OR TS=(\"clustering algorithm\") OR TS=(\"feature extraction\") OR TS=(\"time series forecast\") OR TS=(\"anomaly detection\") OR TS=(\"identity fraud detection\") OR TS=(\"dimensionality reduction\") OR TS=(\"feature elicitation\") OR TS=(\"chatbot\") OR TS=(\"clustering\") OR TS=(\"unsupervised learning\") OR TS=(\"supervised learning\") OR TS=(\"convolutional network\") OR TS=(\"adversarial network\")'" }, - "execution_count": 53, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -60,7 +62,7 @@ "\n", "keywords = [c.strip() for c in keywords[0].split(\",\")]\n", "\n", - "keywords_str = ' OR '.join('TS=(\"'+k+'\")' for k in keywords)\n", + "keywords_str = ' OR '.join('TS=(\\\"'+k+'\\\")' for k in keywords)\n", "keywords_str" ], "metadata": { @@ -72,17 +74,8 @@ }, { "cell_type": "code", - "execution_count": 54, - "outputs": [ - { - "data": { - "text/plain": "'CU=AUSTRIA OR CU=BELGIUM OR CU=BULGARIA OR CU=CROATIA OR CU=CYPRUS OR CU=CZECH REPUBLIC OR CU=DENMARK OR CU=ESTONIA OR CU=FINLAND OR CU=FRANCE OR CU=GERMANY OR CU=GREECE OR CU=HUNGARY OR CU=IRELAND OR CU=ITALY OR CU=LATVIA OR CU=LITHUANIA OR CU=LUXEMBOURG OR CU=MALTA OR CU=NETHERLANDS OR CU=POLAND OR CU=PORTUGAL OR CU=ROMANIA OR CU=SLOVAKIA OR CU=SLOVENIA OR CU=SPAIN OR CU=SWEDEN OR CU=NORWAY OR CU=SWITZERLAND OR CU=UNITED KINGDOM OR CU=ENGLAND OR CU=WALES OR CU=SCOTLAND OR CU=N IRELAND'" - }, - "execution_count": 54, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": 5, + "outputs": [], "source": [ "scope_country_source = r'..\\eu_scope_countries.txt'\n", "\n", @@ -90,11 +83,58 @@ " coop_countries = f.readlines()\n", "coop_countries = [c.strip().upper() for c in coop_countries[0].split(\",\")]\n", "focal_countries = [c.strip().upper() for c in focal_countries_list]\n", + "eu_countries = coop_countries[0:-7]\n", + "assoc_countries = coop_countries[-7:]\n", + "\n", + "nor_c = [coop_countries[-7],]\n", + "swi_c = [coop_countries[-6],]\n", + "uk_c = coop_countries[-5:]\n", "\n", "foc_str = ' OR '.join([country_mode+'='+c for c in focal_countries])\n", "coop_str = ' OR '.join([country_mode+'='+c for c in coop_countries])\n", + "eu_str = ' OR '.join([country_mode+'='+c for c in eu_countries])\n", + "assoc_str = ' OR '.join([country_mode+'='+c for c in assoc_countries])\n", "\n", - "coop_str" + "nor_str =' OR '.join([country_mode+'='+c for c in nor_c])\n", + "swi_str =' OR '.join([country_mode+'='+c for c in swi_c])\n", + "uk_str =' OR '.join([country_mode+'='+c for c in uk_c])\n", + "eu_sub_str = eu_str.split(' OR ')\n", + "# eu_sub_str" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 5, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 6, + "outputs": [ + { + "data": { + "text/plain": "['UNITED KINGDOM', 'ENGLAND', 'WALES', 'SCOTLAND', 'N IRELAND']" + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "coop_countries[-5:]" ], "metadata": { "collapsed": false, @@ -105,13 +145,13 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 7, "outputs": [ { "data": { "text/plain": "'CU=PEOPLES R CHINA OR CU=HONG KONG'" }, - "execution_count": 55, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -128,19 +168,19 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 8, "outputs": [ { "data": { - "text/plain": "'(CU=PEOPLES R CHINA OR CU=HONG KONG) AND (CU=AUSTRIA OR CU=BELGIUM OR CU=BULGARIA OR CU=CROATIA OR CU=CYPRUS OR CU=CZECH REPUBLIC OR CU=DENMARK OR CU=ESTONIA OR CU=FINLAND OR CU=FRANCE OR CU=GERMANY OR CU=GREECE OR CU=HUNGARY OR CU=IRELAND OR CU=ITALY OR CU=LATVIA OR CU=LITHUANIA OR CU=LUXEMBOURG OR CU=MALTA OR CU=NETHERLANDS OR CU=POLAND OR CU=PORTUGAL OR CU=ROMANIA OR CU=SLOVAKIA OR CU=SLOVENIA OR CU=SPAIN OR CU=SWEDEN OR CU=NORWAY OR CU=SWITZERLAND OR CU=UNITED KINGDOM OR CU=ENGLAND OR CU=WALES OR CU=SCOTLAND OR CU=N IRELAND) AND (TS=(\"artificial intelligence\") OR TS=(\"machine learning\") OR TS=(\"neural network\") OR TS=(\"big data\") OR TS=(\"deep learning\") OR TS=(\"pattern recognition\") OR TS=(\"computer vision\") OR TS=(\"image classification\") OR TS=(\"reinforcement learning\") OR TS=(\"support vector machines\") OR TS=(\"recommender system\") OR TS=(\"random forest\") OR TS=(\"ensemble model\") OR TS=(\"image processing\") OR TS=(\"generative network\") OR TS=(\"ai ethic\") OR TS=(\"natural language processing\") OR TS=(\"clustering algorithm\") OR TS=(\"feature extraction\") OR TS=(\"time series forecast\") OR TS=(\"anomaly detection\") OR TS=(\"identity fraud detection\") OR TS=(\"dimensionality reduction\") OR TS=(\"feature elicitation\") OR TS=(\"chatbot\") OR TS=(\"clustering\") OR TS=(\"unsupervised learning\") OR TS=(\"supervised learning\") OR TS=(\"convolutional network\") OR TS=(\"adversarial network\"))'" + "text/plain": "'(CU=PEOPLES R CHINA OR CU=HONG KONG) AND (CU=AUSTRIA OR CU=BELGIUM OR CU=BULGARIA OR CU=CROATIA OR CU=CYPRUS OR CU=CZECH REPUBLIC OR CU=DENMARK OR CU=ESTONIA OR CU=FINLAND OR CU=FRANCE OR CU=GERMANY OR CU=GREECE OR CU=HUNGARY OR CU=IRELAND OR CU=ITALY OR CU=LATVIA OR CU=LITHUANIA OR CU=LUXEMBOURG OR CU=MALTA OR CU=NETHERLANDS OR CU=POLAND OR CU=PORTUGAL OR CU=ROMANIA OR CU=SLOVAKIA OR CU=SLOVENIA OR CU=SPAIN OR CU=SWEDEN OR CU=NORWAY OR CU=SWITZERLAND OR CU=UNITED KINGDOM OR CU=ENGLAND OR CU=WALES OR CU=SCOTLAND OR CU=N IRELAND) AND (TS=(\"artificial intelligence\") OR TS=(\"machine learning\") OR TS=(\"neural network\") OR TS=(\"big data\") OR TS=(\"deep learning\") OR TS=(\"pattern recognition\") OR TS=(\"computer vision\") OR TS=(\"image classification\") OR TS=(\"reinforcement learning\") OR TS=(\"support vector machines\") OR TS=(\"recommender system\") OR TS=(\"random forest\") OR TS=(\"ensemble model\") OR TS=(\"image processing\") OR TS=(\"generative network\") OR TS=(\"ai ethic\") OR TS=(\"natural language processing\") OR TS=(\"clustering algorithm\") OR TS=(\"feature extraction\") OR TS=(\"time series forecast\") OR TS=(\"anomaly detection\") OR TS=(\"identity fraud detection\") OR TS=(\"dimensionality reduction\") OR TS=(\"feature elicitation\") OR TS=(\"chatbot\") OR TS=(\"clustering\") OR TS=(\"unsupervised learning\") OR TS=(\"supervised learning\") OR TS=(\"convolutional network\") OR TS=(\"adversarial network\")) AND PY=(2011-2022)'" }, - "execution_count": 58, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "scope_query = f'({foc_str}) AND ({coop_str}) AND ({keywords_str})'\n", + "scope_query = f'({foc_str}) AND ({coop_str}) AND ({keywords_str}) AND PY=(2011-2022)'\n", "scope_query" ], "metadata": { @@ -152,19 +192,19 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 9, "outputs": [ { "data": { - "text/plain": "'(CU=AUSTRIA OR CU=BELGIUM OR CU=BULGARIA OR CU=CROATIA OR CU=CYPRUS OR CU=CZECH REPUBLIC OR CU=DENMARK OR CU=ESTONIA OR CU=FINLAND OR CU=FRANCE OR CU=GERMANY OR CU=GREECE OR CU=HUNGARY OR CU=IRELAND OR CU=ITALY OR CU=LATVIA OR CU=LITHUANIA OR CU=LUXEMBOURG OR CU=MALTA OR CU=NETHERLANDS OR CU=POLAND OR CU=PORTUGAL OR CU=ROMANIA OR CU=SLOVAKIA OR CU=SLOVENIA OR CU=SPAIN OR CU=SWEDEN OR CU=NORWAY OR CU=SWITZERLAND OR CU=UNITED KINGDOM OR CU=ENGLAND OR CU=WALES OR CU=SCOTLAND OR CU=N IRELAND) AND (TS=(\"artificial intelligence\") OR TS=(\"machine learning\") OR TS=(\"neural network\") OR TS=(\"big data\") OR TS=(\"deep learning\") OR TS=(\"pattern recognition\") OR TS=(\"computer vision\") OR TS=(\"image classification\") OR TS=(\"reinforcement learning\") OR TS=(\"support vector machines\") OR TS=(\"recommender system\") OR TS=(\"random forest\") OR TS=(\"ensemble model\") OR TS=(\"image processing\") OR TS=(\"generative network\") OR TS=(\"ai ethic\") OR TS=(\"natural language processing\") OR TS=(\"clustering algorithm\") OR TS=(\"feature extraction\") OR TS=(\"time series forecast\") OR TS=(\"anomaly detection\") OR TS=(\"identity fraud detection\") OR TS=(\"dimensionality reduction\") OR TS=(\"feature elicitation\") OR TS=(\"chatbot\") OR TS=(\"clustering\") OR TS=(\"unsupervised learning\") OR TS=(\"supervised learning\") OR TS=(\"convolutional network\") OR TS=(\"adversarial network\"))'" + "text/plain": "'(CU=PEOPLES R CHINA OR CU=HONG KONG) AND (TS=(\"artificial intelligence\") OR TS=(\"machine learning\") OR TS=(\"neural network\") OR TS=(\"big data\") OR TS=(\"deep learning\") OR TS=(\"pattern recognition\") OR TS=(\"computer vision\") OR TS=(\"image classification\") OR TS=(\"reinforcement learning\") OR TS=(\"support vector machines\") OR TS=(\"recommender system\") OR TS=(\"random forest\") OR TS=(\"ensemble model\") OR TS=(\"image processing\") OR TS=(\"generative network\") OR TS=(\"ai ethic\") OR TS=(\"natural language processing\") OR TS=(\"clustering algorithm\") OR TS=(\"feature extraction\") OR TS=(\"time series forecast\") OR TS=(\"anomaly detection\") OR TS=(\"identity fraud detection\") OR TS=(\"dimensionality reduction\") OR TS=(\"feature elicitation\") OR TS=(\"chatbot\") OR TS=(\"clustering\") OR TS=(\"unsupervised learning\") OR TS=(\"supervised learning\") OR TS=(\"convolutional network\") OR TS=(\"adversarial network\"))'" }, - "execution_count": 60, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "ch_scope_query = f'({coop_str}) AND ({keywords_str})'\n", + "ch_scope_query = f'({foc_str}) AND ({keywords_str})'\n", "ch_scope_query" ], "metadata": { @@ -173,6 +213,140 @@ "name": "#%%\n" } } + }, + { + "cell_type": "code", + "execution_count": 10, + "outputs": [ + { + "data": { + "text/plain": "'(CU=AUSTRIA OR CU=BELGIUM OR CU=BULGARIA OR CU=CROATIA OR CU=CYPRUS OR CU=CZECH REPUBLIC OR CU=DENMARK OR CU=ESTONIA OR CU=FINLAND OR CU=FRANCE OR CU=GERMANY OR CU=GREECE OR CU=HUNGARY OR CU=IRELAND OR CU=ITALY OR CU=LATVIA OR CU=LITHUANIA OR CU=LUXEMBOURG OR CU=MALTA OR CU=NETHERLANDS OR CU=POLAND OR CU=PORTUGAL OR CU=ROMANIA OR CU=SLOVAKIA OR CU=SLOVENIA OR CU=SPAIN OR CU=SWEDEN) AND (TS=(\"artificial intelligence\") OR TS=(\"machine learning\") OR TS=(\"neural network\") OR TS=(\"big data\") OR TS=(\"deep learning\") OR TS=(\"pattern recognition\") OR TS=(\"computer vision\") OR TS=(\"image classification\") OR TS=(\"reinforcement learning\") OR TS=(\"support vector machines\") OR TS=(\"recommender system\") OR TS=(\"random forest\") OR TS=(\"ensemble model\") OR TS=(\"image processing\") OR TS=(\"generative network\") OR TS=(\"ai ethic\") OR TS=(\"natural language processing\") OR TS=(\"clustering algorithm\") OR TS=(\"feature extraction\") OR TS=(\"time series forecast\") OR TS=(\"anomaly detection\") OR TS=(\"identity fraud detection\") OR TS=(\"dimensionality reduction\") OR TS=(\"feature elicitation\") OR TS=(\"chatbot\") OR TS=(\"clustering\") OR TS=(\"unsupervised learning\") OR TS=(\"supervised learning\") OR TS=(\"convolutional network\") OR TS=(\"adversarial network\"))'" + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eu_scope_query = f'({eu_str}) AND ({keywords_str})'\n", + "eu_scope_query" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 11, + "outputs": [], + "source": [ + "sub_queries = [f'PY=(2011-2022) AND ({i_str}) AND ({keywords_str})' for i_str in [foc_str,eu_str,assoc_str,nor_str,swi_str,uk_str]+eu_sub_str]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 12, + "outputs": [], + "source": [ + "from wossel_miners import wos_fetch_entries,wos_fetch_yearly_output" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 13, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 33/33 [12:49<00:00, 23.31s/it]\n" + ] + } + ], + "source": [ + "wos_fetch_yearly_output(query_str_list=sub_queries)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 14, + "outputs": [ + { + "data": { + "text/plain": "'(CU=PEOPLES R CHINA OR CU=HONG KONG) AND (CU=AUSTRIA OR CU=BELGIUM OR CU=BULGARIA OR CU=CROATIA OR CU=CYPRUS OR CU=CZECH REPUBLIC OR CU=DENMARK OR CU=ESTONIA OR CU=FINLAND OR CU=FRANCE OR CU=GERMANY OR CU=GREECE OR CU=HUNGARY OR CU=IRELAND OR CU=ITALY OR CU=LATVIA OR CU=LITHUANIA OR CU=LUXEMBOURG OR CU=MALTA OR CU=NETHERLANDS OR CU=POLAND OR CU=PORTUGAL OR CU=ROMANIA OR CU=SLOVAKIA OR CU=SLOVENIA OR CU=SPAIN OR CU=SWEDEN OR CU=NORWAY OR CU=SWITZERLAND OR CU=UNITED KINGDOM OR CU=ENGLAND OR CU=WALES OR CU=SCOTLAND OR CU=N IRELAND) AND (TS=(\"artificial intelligence\") OR TS=(\"machine learning\") OR TS=(\"neural network\") OR TS=(\"big data\") OR TS=(\"deep learning\") OR TS=(\"pattern recognition\") OR TS=(\"computer vision\") OR TS=(\"image classification\") OR TS=(\"reinforcement learning\") OR TS=(\"support vector machines\") OR TS=(\"recommender system\") OR TS=(\"random forest\") OR TS=(\"ensemble model\") OR TS=(\"image processing\") OR TS=(\"generative network\") OR TS=(\"ai ethic\") OR TS=(\"natural language processing\") OR TS=(\"clustering algorithm\") OR TS=(\"feature extraction\") OR TS=(\"time series forecast\") OR TS=(\"anomaly detection\") OR TS=(\"identity fraud detection\") OR TS=(\"dimensionality reduction\") OR TS=(\"feature elicitation\") OR TS=(\"chatbot\") OR TS=(\"clustering\") OR TS=(\"unsupervised learning\") OR TS=(\"supervised learning\") OR TS=(\"convolutional network\") OR TS=(\"adversarial network\")) AND PY=(2011-2022)'" + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scope_query" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 16, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hoooold...\n", + "27672 records found! Here we go in 93 steps...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 92/92 [09:38<00:00, 6.29s/it]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "final batch of 27601-27672\n" + ] + } + ], + "source": [ + "wos_fetch_entries(query_str=scope_query)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } } ], "metadata": { diff --git a/WOS/wos_extract/wossel_miners.py b/WOS/wos_extract/wossel_miners.py new file mode 100644 index 0000000..d926672 --- /dev/null +++ b/WOS/wos_extract/wossel_miners.py @@ -0,0 +1,266 @@ +import os +import glob +import pytest +import time +from datetime import datetime +import json +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.common.action_chains import ActionChains +from selenium.webdriver.support import expected_conditions +from selenium.webdriver.support.wait import WebDriverWait +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +# from selenium.webdriver.chrome.options import Options +from selenium.webdriver.firefox.options import Options + +from tqdm import tqdm +import random + +def close_pendo_windows(driver): + '''Close guiding windows''' + # Cookies + try: + driver.find_element(By.XPATH, '//*[@id="onetrust-accept-btn-handler"]').click() + except: + pass + # "Got it" + try: + driver.find_element(By.XPATH, '//button[contains(@class, "_pendo-button-primaryButton")]').click() + except: + pass + # "No thanks" + try: + driver.find_element(By.XPATH, '//button[contains(@class, "_pendo-button-secondaryButton")]').click() + except: + pass + # What was it... I forgot... + try: + driver.find_element(By.XPATH, '//span[contains(@class, "_pendo-close-guide")').click() + except: + pass + # Overlay + try: + driver.find_element(By.XPATH, '//div[contains(@class, "cdk-overlay-container")').click() + except: + pass + + +def wos_fetch_entries(query_str="TS=\"web of science\" AND PY=(2008-2010)", + wait_mu=1, wait_sigma=0.2, debug=False): + + now = datetime.now() # current date and time + date_time = now.strftime("%Y-%m-%d-%H-%M-%S-%f")+"save" + + options = Options() + + # init directory + download_path = fr'C:\Users\radvanyi\PycharmProjects\ZSI_analytics\WOS\wos_extract\wos_downloads\entry_batches\{date_time}' + os.makedirs(download_path, exist_ok=True) + files = glob.glob(fr'{download_path}\*') + for f in files: + os.remove(f) + + options.set_preference("browser.download.folderList", 2) + options.set_preference("browser.download.manager.showWhenStarting", False) + options.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv/xls") + options.set_preference("browser.download.dir", download_path) + + with open(fr'{download_path}\query.txt', "w") as f: + f.write(query_str) + + # options.headless = True + if debug==False: + options.add_argument('--headless') + driver = webdriver.Firefox(options=options) + driver.get("https://www.webofscience.com/") + driver.set_window_size(974, 1040) + try: + WebDriverWait(driver, 30).until( + expected_conditions.visibility_of_element_located((By.ID, "onetrust-reject-all-handler"))) + driver.find_element(By.ID, "onetrust-reject-all-handler").click() + except: + close_pendo_windows(driver) + WebDriverWait(driver, 30).until( + expected_conditions.visibility_of_element_located((By.LINK_TEXT, "Advanced Search"))) + WebDriverWait(driver, 30).until( + expected_conditions.invisibility_of_element_located((By.ID, "onetrust-pc-dark-filter ot-fade-in"))) + + print("Hoooold...") + time.sleep(2) + WebDriverWait(driver, 30).until(expected_conditions.element_to_be_clickable((By.LINK_TEXT, "Advanced Search"))) + driver.find_element(By.LINK_TEXT, "Advanced Search").click() + + WebDriverWait(driver, 30).until(expected_conditions.element_to_be_clickable((By.ID, "advancedSearchInputArea"))) + driver.find_element(By.ID, "advancedSearchInputArea").click() + driver.find_element(By.ID, "advancedSearchInputArea").send_keys(query_str) + driver.find_element(By.CSS_SELECTOR, ".mat-menu-trigger > svg").click() + driver.find_element(By.CSS_SELECTOR, ".cdk-focused > span").click() + + WebDriverWait(driver, 30).until(expected_conditions.visibility_of_element_located((By.CSS_SELECTOR, ".brand-blue"))) + driver.execute_script("window.scrollTo(0,0)") + count_str = driver.find_element(By.CSS_SELECTOR, ".brand-blue").text + count_int = int(count_str.replace(",", "").replace(".", "").strip()) + print(f'{count_int} records found! Here we go in {int(count_int / 300) + 1} steps...') + for i in tqdm(range(1, count_int - 300, 300), position=0, leave=True): + # print(f'records {i}-{i+299}') + if i == 1: + driver.find_element(By.XPATH, "//app-export-menu/div/button").click() + # driver.find_element(By.ID, "exportToExcelButton").click() + driver.find_element(By.ID, "exportToTabWinButton").click() + driver.find_element(By.CSS_SELECTOR, "#radio3 .mat-radio-outer-circle").click() + driver.find_element(By.NAME, "markTo").clear() + driver.find_element(By.NAME, "markTo").send_keys("300") + driver.find_element(By.CSS_SELECTOR, ".margin-top-5 > .dropdown").click() + driver.find_element(By.XPATH, "//span[contains(.,\'Full Record\')]").click() + driver.find_element(By.CSS_SELECTOR, ".mat-flat-button .ng-star-inserted").click() + WebDriverWait(driver, 30).until(expected_conditions.invisibility_of_element_located( + (By.CSS_SELECTOR, ".mat-flat-button .ng-star-inserted"))) + time.sleep(random.gauss(wait_mu, wait_sigma)) + else: + WebDriverWait(driver, 30).until(expected_conditions.invisibility_of_element_located( + (By.CSS_SELECTOR, ".mat-flat-button .ng-star-inserted"))) + + driver.find_element(By.XPATH, "//app-export-menu/div/button").click() + # driver.find_element(By.ID, "exportToExcelButton").click() + driver.find_element(By.ID, "exportToTabWinButton").click() + driver.find_element(By.CSS_SELECTOR, "#radio3 .mat-radio-container").click() + driver.find_element(By.NAME, "markFrom").clear() + driver.find_element(By.NAME, "markFrom").send_keys(f"{i}") + driver.find_element(By.NAME, "markTo").clear() + driver.find_element(By.NAME, "markTo").send_keys(f"{i + 299}") + driver.find_element(By.CSS_SELECTOR, ".margin-top-5 > .dropdown").click() + driver.find_element(By.XPATH, "//span[contains(.,\'Full Record\')]").click() + driver.find_element(By.CSS_SELECTOR, ".mat-flat-button .ng-star-inserted").click() + + WebDriverWait(driver, 30).until(expected_conditions.invisibility_of_element_located( + (By.CSS_SELECTOR, ".mat-flat-button .ng-star-inserted"))) + time.sleep(random.gauss(wait_mu, wait_sigma)) + + # Absolute path of a file + old_name = fr"{download_path}\savedrecs.txt" + new_name = fr"{download_path}\records_{i}_{i + 299}.txt" + + # Renaming the file + os.rename(old_name, new_name) + + if (i + 299) % count_int != 0: + print(f'final batch of {i + 300}-{count_int}') + WebDriverWait(driver, 30).until(expected_conditions.invisibility_of_element_located( + (By.CSS_SELECTOR, ".mat-flat-button .ng-star-inserted"))) + + driver.find_element(By.XPATH, "//app-export-menu/div/button").click() + # driver.find_element(By.ID, "exportToExcelButton").click() + driver.find_element(By.ID, "exportToTabWinButton").click() + driver.find_element(By.CSS_SELECTOR, "#radio3 .mat-radio-container").click() + driver.find_element(By.NAME, "markFrom").clear() + driver.find_element(By.NAME, "markFrom").send_keys(f"{i + 300}") + driver.find_element(By.NAME, "markTo").clear() + driver.find_element(By.NAME, "markTo").send_keys(f"{count_int}") + driver.find_element(By.CSS_SELECTOR, ".margin-top-5 > .dropdown").click() + driver.find_element(By.XPATH, "//span[contains(.,\'Full Record\')]").click() + driver.find_element(By.CSS_SELECTOR, ".mat-flat-button .ng-star-inserted").click() + + WebDriverWait(driver, 30).until(expected_conditions.invisibility_of_element_located( + (By.CSS_SELECTOR, ".mat-flat-button .ng-star-inserted"))) + time.sleep(random.gauss(wait_mu, wait_sigma)) + + # Absolute path of a file + old_name = fr"{download_path}\savedrecs.txt" + new_name = fr"{download_path}\records_{i + 300}_{count_int}.txt" + + # Renaming the file + time.sleep(0.1) + os.rename(old_name, new_name) + + time.sleep(2) + time.sleep(random.gauss(wait_mu, wait_sigma)) + driver.close() + +def wos_fetch_yearly_output(query_str_list = ( + "TS=\"web of science\" AND PY=(2008-2010)", + "TS=\"artificial intelligence\" AND PY=(2011-2022)"), + wait_mu=1, wait_sigma=0.2,debug=False): + + # if isinstance(query_iterable,tuple) or + + for query_str in tqdm(query_str_list): + options = Options() + + # query_file_str = query_str.replace('"', '``') + + now = datetime.now() # current date and time + date_time = now.strftime("%Y-%m-%d-%H-%M-%S-%f")+"save" + + # init directory + download_path = fr'C:\Users\radvanyi\PycharmProjects\ZSI_analytics\WOS\wos_extract\wos_downloads\aggregated\{date_time}' + os.makedirs(download_path, exist_ok=True) + files = glob.glob(fr'{download_path}\*') + for f in files: + os.remove(f) + + options.set_preference("browser.download.folderList", 2) + options.set_preference("browser.download.manager.showWhenStarting", False) + options.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv/xls") + options.set_preference("browser.download.dir", download_path) + + with open(fr'{download_path}\query.txt', "w") as f: + f.write(query_str) + + # options.headless = True + if debug == False: + options.add_argument('--headless') + driver = webdriver.Firefox(options=options) + driver.get("https://www.webofscience.com/") + driver.set_window_size(974, 1040) + try: + WebDriverWait(driver, 30).until( + expected_conditions.visibility_of_element_located((By.ID, "onetrust-reject-all-handler"))) + driver.find_element(By.ID, "onetrust-reject-all-handler").click() + except: + close_pendo_windows(driver) + WebDriverWait(driver, 30).until( + expected_conditions.visibility_of_element_located((By.LINK_TEXT, "Advanced Search"))) + WebDriverWait(driver, 30).until( + expected_conditions.invisibility_of_element_located((By.ID, "onetrust-pc-dark-filter ot-fade-in"))) + + # print("Hoooold...") + time.sleep(2) + WebDriverWait(driver, 30).until(expected_conditions.element_to_be_clickable((By.LINK_TEXT, "Advanced Search"))) + driver.find_element(By.LINK_TEXT, "Advanced Search").click() + + WebDriverWait(driver, 30).until(expected_conditions.element_to_be_clickable((By.ID, "advancedSearchInputArea"))) + driver.find_element(By.ID, "advancedSearchInputArea").click() + driver.find_element(By.ID, "advancedSearchInputArea").send_keys(query_str) + driver.find_element(By.CSS_SELECTOR, ".mat-menu-trigger > svg").click() + driver.find_element(By.CSS_SELECTOR, ".cdk-focused > span").click() + + WebDriverWait(driver, 30).until( + expected_conditions.visibility_of_element_located((By.CSS_SELECTOR, ".brand-blue"))) + driver.execute_script("window.scrollTo(0,0)") + count_str = driver.find_element(By.CSS_SELECTOR, ".brand-blue").text + count_int = int(count_str.replace(",", "").replace(".", "").strip()) + # print(f'{count_int} records found!') + + driver.find_element(By.XPATH, "//span[contains(.,\'Analyze Results\')]").click() + # element = driver.find_element(By.CSS_SELECTOR, ".search-terms") + # actions = ActionChains(driver) + # actions.move_to_element(element).perform() + driver.find_element(By.CSS_SELECTOR, "#snSelectCategories svg").click() + driver.find_element(By.XPATH, "//span[contains(.,\'Publication Years\')]").click() + driver.find_element(By.XPATH, "//mat-radio-button[@id=\'mat-radio-3\']/label/span/span").click() + driver.find_element(By.XPATH, "//span[contains(.,\'Download data table\')]").click() + + # Absolute path of a file + old_name = fr"{download_path}\analyze.txt" + new_name = fr'{download_path}\analyze_PY_{date_time}_.txt' + + # Renaming the file + time.sleep(2) + os.rename(old_name, new_name) + time.sleep(random.gauss(wait_mu, wait_sigma)) + driver.close() + +if __name__ == '__main__': + wos_fetch_entries(debug=False) + wos_fetch_yearly_output(debug=False) \ No newline at end of file diff --git a/WOS/wos_processing.ipynb b/WOS/wos_processing.ipynb index 0fe5e0b..03e92de 100644 --- a/WOS/wos_processing.ipynb +++ b/WOS/wos_processing.ipynb @@ -2,20 +2,196 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, - "metadata": {}, + "execution_count": 35, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "import os\n", "import shutil\n", - "from flashgeotext.geotext import GeoText" - ] + "from flashgeotext.geotext import GeoText\n", + "import re\n", + "import spacy" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 20, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "I like salty fries and hamburgers. <-> Fast food tastes very good. 0.691649353055761\n", + "salty fries <-> hamburgers 0.6938489675521851\n" + ] + } + ], + "source": [ + "import spacy\n", + "\n", + "nlp = spacy.load(\"en_core_web_md\") # make sure to use larger package!\n", + "doc1 = nlp(\"I like salty fries and hamburgers.\")\n", + "doc2 = nlp(\"Fast food tastes very good.\")\n", + "\n", + "# Similarity of two documents\n", + "print(doc1, \"<->\", doc2, doc1.similarity(doc2))\n", + "# Similarity of tokens and spans\n", + "french_fries = doc1[2:4]\n", + "burgers = doc1[5]\n", + "print(french_fries, \"<->\", burgers, french_fries.similarity(burgers))" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 21, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "I\n", + "salty fry\n", + "hamburger\n" + ] + }, + { + "data": { + "text/plain": "[None, None, None]" + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "[print(i.lemma_) for i in doc1.noun_chunks]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 22, + "outputs": [], + "source": [ + "doc_test = nlp(\"On the inevitability of neural networks and other tasty topics of the 21st century\")" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 23, + "outputs": [ + { + "data": { + "text/plain": "['the inevitability',\n 'neural network',\n 'other tasty topic',\n 'the 21st century']" + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "[i.lemma_ for i in doc_test.noun_chunks]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 24, + "outputs": [ + { + "data": { + "text/plain": "(300,)" + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "doc1.vector.shape" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 25, + "outputs": [ + { + "data": { + "text/plain": "\"tokens = []\\nlemma = []\\npos = []\\n\\nfor doc in nlp.pipe(df['species'].astype('unicode').values, batch_size=50,\\n n_threads=3):\\n if doc.is_parsed:\\n tokens.append([n.text for n in doc])\\n lemma.append([n.lemma_ for n in doc])\\n pos.append([n.pos_ for n in doc])\\n else:\\n # We want to make sure that the lists of parsed results have the\\n # same number of entries of the original Dataframe, so add some blanks in case the parse fails\\n tokens.append(None)\\n lemma.append(None)\\n pos.append(None)\\n\\ndf['species_tokens'] = tokens\\ndf['species_lemma'] = lemma\\ndf['species_pos'] = pos\"" + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#spacy pipe example\n", + "\"\"\"tokens = []\n", + "lemma = []\n", + "pos = []\n", + "\n", + "for doc in nlp.pipe(df['species'].astype('unicode').values, batch_size=50,\n", + " n_threads=3):\n", + " if doc.is_parsed:\n", + " tokens.append([n.text for n in doc])\n", + " lemma.append([n.lemma_ for n in doc])\n", + " pos.append([n.pos_ for n in doc])\n", + " else:\n", + " # We want to make sure that the lists of parsed results have the\n", + " # same number of entries of the original Dataframe, so add some blanks in case the parse fails\n", + " tokens.append(None)\n", + " lemma.append(None)\n", + " pos.append(None)\n", + "\n", + "df['species_tokens'] = tokens\n", + "df['species_lemma'] = lemma\n", + "df['species_pos'] = pos\"\"\"" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -34,7 +210,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -43,7 +219,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -66,14 +242,14 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/plain": "0 Publication Type\n1 Authors\n2 Book Authors\n3 Book Editors\n4 Book Group Authors\n ... \n76 SubField_English\n77 2.00 SEQ\n78 Source_title\n79 srcid\n80 issn_type\nLength: 81, dtype: object" }, - "execution_count": 5, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -84,14 +260,14 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 30, "metadata": {}, "outputs": [ { "data": { "text/plain": "0 Salucci, Marco/S-8654-2016; Arrebola, Manuel/L...\n9714 Huang, Yu/AAY-5464-2020\n9697 Kakavand, Mohammad Reza Azadi/X-9556-2019; Fen...\n9699 Dong, Sheng/AAE-3619-2021; Soares, Carlos Gued...\n9701 Han, Guoqi/T-7365-2019; Nan, Yang/HKD-9687-202...\n ... \n3066 ; Liotta, Antonio/G-9532-2014\n5097 , 卢帅/AAK-2185-2020; Popp, József/AFN-1250-2022\n11369 NaN\n11368 Rossiter, D G/D-3842-2009\n11362 Jin, Shuanggen/B-8094-2008\nName: Researcher Ids, Length: 9889, dtype: object" }, - "execution_count": 6, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } @@ -102,7 +278,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 31, "metadata": {}, "outputs": [ { @@ -110,7 +286,7 @@ "text/plain": " Publication Type Authors \n16979 J Zhang, MS; Huang, J; Cao, Y; Xiong, CH; Mohamm... \\\n1880 J Zhang, MS; Huang, J; Cao, Y; Xiong, CH; Mohamm... \n\n Book Authors Book Editors Book Group Authors \n16979 NaN NaN NaN \\\n1880 NaN NaN NaN \n\n Author Full Names \n16979 Zhang, Mengshi; Huang, Jian; Cao, Yu; Xiong, C... \\\n1880 Zhang, Mengshi; Huang, Jian; Cao, Yu; Xiong, C... \n\n Book Author Full Names Group Authors \n16979 NaN NaN \\\n1880 NaN NaN \n\n Article Title \n16979 Echo State Network-Enhanced Super-Twisting Con... \\\n1880 Echo State Network-Enhanced Super-Twisting Con... \n\n Source Title ... Web of Science Record \n16979 IEEE-ASME TRANSACTIONS ON MECHATRONICS ... 0 \\\n1880 IEEE-ASME TRANSACTIONS ON MECHATRONICS ... 0 \n\n issn_var issn Domain_English Field_English \n16979 issn 10834435 Applied Sciences Engineering \\\n1880 issn 10834435 Applied Sciences Engineering \n\n SubField_English 2.00 SEQ \n16979 Industrial Engineering & Automation 27 \\\n1880 Industrial Engineering & Automation 27 \n\n Source_title srcid issn_type \n16979 IEEE/ASME Transactions on Mechatronics 19113.0 issn1 \n1880 IEEE/ASME Transactions on Mechatronics 19113.0 issn1 \n\n[2 rows x 81 columns]", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
Publication TypeAuthorsBook AuthorsBook EditorsBook Group AuthorsAuthor Full NamesBook Author Full NamesGroup AuthorsArticle TitleSource Title...Web of Science Recordissn_varissnDomain_EnglishField_EnglishSubField_English2.00 SEQSource_titlesrcidissn_type
16979JZhang, MS; Huang, J; Cao, Y; Xiong, CH; Mohamm...NaNNaNNaNZhang, Mengshi; Huang, Jian; Cao, Yu; Xiong, C...NaNNaNEcho State Network-Enhanced Super-Twisting Con...IEEE-ASME TRANSACTIONS ON MECHATRONICS...0issn10834435Applied SciencesEngineeringIndustrial Engineering & Automation27IEEE/ASME Transactions on Mechatronics19113.0issn1
1880JZhang, MS; Huang, J; Cao, Y; Xiong, CH; Mohamm...NaNNaNNaNZhang, Mengshi; Huang, Jian; Cao, Yu; Xiong, C...NaNNaNEcho State Network-Enhanced Super-Twisting Con...IEEE-ASME TRANSACTIONS ON MECHATRONICS...0issn10834435Applied SciencesEngineeringIndustrial Engineering & Automation27IEEE/ASME Transactions on Mechatronics19113.0issn1
\n

2 rows × 81 columns

\n
" }, - "execution_count": 7, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -121,14 +297,14 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 32, "outputs": [ { "data": { "text/plain": " UT (Unique WOS ID) Keywords Plus \n0 WOS:000852293800024 CONVOLUTIONAL NEURAL-NETWORK; DEEP LEARNING FR... \\\n9714 WOS:000540750000002 STATE-SPACE RECONSTRUCTION; SURFACE AIR-TEMPER... \n9697 WOS:000600708400002 COMPRESSIVE STRENGTH; MODELS; ADABOOST.RT; DUC... \n9699 WOS:000511965100005 STRUCTURAL RELIABILITY; FAILURE MODES \n9701 WOS:000663142500003 REFLECTED GPS SIGNALS; SOIL-MOISTURE; OCEAN; S... \n... ... ... \n3066 WOS:000528727500074 LOCAL SEARCH; ALGORITHM; VARIANCE; MODEL \n5097 WOS:000596139400001 INDUSTRY 4.0; MANAGEMENT; RISK; ANALYTICS; CHA... \n11369 WOS:000436774300069 NaN \n11368 WOS:000846290700001 PARTIAL LEAST-SQUARES; INFRARED-SPECTROSCOPY; ... \n11362 WOS:000480527800025 MICROWAVE DIELECTRIC BEHAVIOR; GPS SIGNALS; RE... \n\n Author Keywords \n0 Imaging; Three-dimensional displays; Electroma... \\\n9714 NaN \n9697 Plastic hinge length; RC columns; Machine lear... \n9699 system reliability; jacket platform; beta-unzi... \n9701 Cyclone GNSS (CYGNSS); Sea surface wind speed;... \n... ... \n3066 sea surface temperature; sea surface temperatu... \n5097 Big data finance; Big data in financial servic... \n11369 planetary gear; fault diagnosis; VMD; center f... \n11368 soil fertility class; reflectance spectroscopy... \n11362 global navigation satellite system (GNSS)-refl... \n\n Article Title \n0 Artificial Intelligence: New Frontiers in Real... \\\n9714 Detecting causality from time series in a mach... \n9697 Data-Driven Approach to Predict the Plastic Hi... \n9699 System Reliability Analysis of an Offshore Jac... \n9701 Analysis of coastal wind speed retrieval from ... \n... ... \n3066 Improved Particle Swarm Optimization for Sea S... \n5097 Current landscape and influence of big data on... \n11369 Planetary Gear Fault Diagnosis via Feature Ima... \n11368 How Well Can Reflectance Spectroscopy Allocate... \n11362 GNSS-R Soil Moisture Retrieval Based on a XGbo... \n\n Abstract \n0 In recent years, artificial intelligence (AI) ... \n9714 Detecting causality from observational data is... \n9697 Inelastic response of reinforced concrete colu... \n9699 This study investigates strategies for solving... \n9701 This paper demonstrates the capability and per... \n... ... \n3066 The Sea Surface Temperature (SST) is one of th... \n5097 Big data is one of the most recent business an... \n11369 Poor working environment leads to frequent fai... \n11368 Fertilization decisions depend on the measurem... \n11362 Global navigation satellite system (GNSS)-refl... \n\n[9889 rows x 5 columns]", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
UT (Unique WOS ID)Keywords PlusAuthor KeywordsArticle TitleAbstract
0WOS:000852293800024CONVOLUTIONAL NEURAL-NETWORK; DEEP LEARNING FR...Imaging; Three-dimensional displays; Electroma...Artificial Intelligence: New Frontiers in Real...In recent years, artificial intelligence (AI) ...
9714WOS:000540750000002STATE-SPACE RECONSTRUCTION; SURFACE AIR-TEMPER...NaNDetecting causality from time series in a mach...Detecting causality from observational data is...
9697WOS:000600708400002COMPRESSIVE STRENGTH; MODELS; ADABOOST.RT; DUC...Plastic hinge length; RC columns; Machine lear...Data-Driven Approach to Predict the Plastic Hi...Inelastic response of reinforced concrete colu...
9699WOS:000511965100005STRUCTURAL RELIABILITY; FAILURE MODESsystem reliability; jacket platform; beta-unzi...System Reliability Analysis of an Offshore Jac...This study investigates strategies for solving...
9701WOS:000663142500003REFLECTED GPS SIGNALS; SOIL-MOISTURE; OCEAN; S...Cyclone GNSS (CYGNSS); Sea surface wind speed;...Analysis of coastal wind speed retrieval from ...This paper demonstrates the capability and per...
..................
3066WOS:000528727500074LOCAL SEARCH; ALGORITHM; VARIANCE; MODELsea surface temperature; sea surface temperatu...Improved Particle Swarm Optimization for Sea S...The Sea Surface Temperature (SST) is one of th...
5097WOS:000596139400001INDUSTRY 4.0; MANAGEMENT; RISK; ANALYTICS; CHA...Big data finance; Big data in financial servic...Current landscape and influence of big data on...Big data is one of the most recent business an...
11369WOS:000436774300069NaNplanetary gear; fault diagnosis; VMD; center f...Planetary Gear Fault Diagnosis via Feature Ima...Poor working environment leads to frequent fai...
11368WOS:000846290700001PARTIAL LEAST-SQUARES; INFRARED-SPECTROSCOPY; ...soil fertility class; reflectance spectroscopy...How Well Can Reflectance Spectroscopy Allocate...Fertilization decisions depend on the measurem...
11362WOS:000480527800025MICROWAVE DIELECTRIC BEHAVIOR; GPS SIGNALS; RE...global navigation satellite system (GNSS)-refl...GNSS-R Soil Moisture Retrieval Based on a XGbo...Global navigation satellite system (GNSS)-refl...
\n

9889 rows × 5 columns

\n
" }, - "execution_count": 11, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } @@ -145,14 +321,14 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 68, "outputs": [ { "data": { - "text/plain": " UT (Unique WOS ID) level_1 keyword\n0 WOS:000209536100003 11117 NaN\n1 WOS:000297893800037 10831 ADAPTIVE DYNAMIC SURFACE CONTROL\n2 WOS:000297893800037 10831 NEURAL COMPENSATOR\n3 WOS:000297893800037 10831 BUCK CONVERTER\n4 WOS:000297893800037 10831 FINITE-TIME IDENTIFIER\n... ... ... ...\n94060 WOS:000947693400001 240 EXPRESSION\n94061 WOS:000947693400001 240 RNALOCATE\n94062 WOS:000947693400001 240 PROTEINS\n94063 WOS:000947693400001 240 RESOURCE\n94064 WOS:000947693400001 240 CELLS\n\n[94065 rows x 3 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
UT (Unique WOS ID)level_1keyword
0WOS:00020953610000311117NaN
1WOS:00029789380003710831ADAPTIVE DYNAMIC SURFACE CONTROL
2WOS:00029789380003710831NEURAL COMPENSATOR
3WOS:00029789380003710831BUCK CONVERTER
4WOS:00029789380003710831FINITE-TIME IDENTIFIER
............
94060WOS:000947693400001240EXPRESSION
94061WOS:000947693400001240RNALOCATE
94062WOS:000947693400001240PROTEINS
94063WOS:000947693400001240RESOURCE
94064WOS:000947693400001240CELLS
\n

94065 rows × 3 columns

\n
" + "text/plain": " UT (Unique WOS ID) keyword_all\n1 WOS:000297893800037 ADAPTIVE DYNAMIC SURFACE CONTROL\n2 WOS:000297893800037 NEURAL COMPENSATOR\n3 WOS:000297893800037 BUCK CONVERTER\n4 WOS:000297893800037 FINITE-TIME IDENTIFIER\n5 WOS:000301090100061 TEMPORAL CONJUNCTION\n.. ... ...\n99 WOS:000309409400280 SCIENTIFIC DATA CLOUD\n100 WOS:000309409400280 VIRTUAL DATASPACES\n101 WOS:000309409400280 SEMANTIC INTEGRATION\n102 WOS:000309409400280 ONTOLOGY\n103 WOS:000309409400280 PAY-AS-YOU-GO\n\n[100 rows x 2 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
UT (Unique WOS ID)keyword_all
1WOS:000297893800037ADAPTIVE DYNAMIC SURFACE CONTROL
2WOS:000297893800037NEURAL COMPENSATOR
3WOS:000297893800037BUCK CONVERTER
4WOS:000297893800037FINITE-TIME IDENTIFIER
5WOS:000301090100061TEMPORAL CONJUNCTION
.........
99WOS:000309409400280SCIENTIFIC DATA CLOUD
100WOS:000309409400280VIRTUAL DATASPACES
101WOS:000309409400280SEMANTIC INTEGRATION
102WOS:000309409400280ONTOLOGY
103WOS:000309409400280PAY-AS-YOU-GO
\n

100 rows × 2 columns

\n
" }, - "execution_count": 22, + "execution_count": 68, "metadata": {}, "output_type": "execute_result" } @@ -161,9 +337,11 @@ "kw_df = pd.DataFrame()\n", "for c in [\"Keywords Plus\",\"Author Keywords\"]:\n", " kwp = wos.groupby(record_col)[c].apply(lambda x: x.str.split(';')).explode().str.strip().str.upper()\n", - " kwp.name = 'keyword'\n", + " kwp.name = 'keyword_all'\n", " kw_df = pd.concat([kwp.reset_index(),kw_df],ignore_index=True)\n", - "kw_df" + "kw_df = kw_df[~kw_df[\"keyword_all\"].isna()].copy().drop(columns=\"level_1\").drop_duplicates()\n", + "kw_df[\"keyword_all\"] = kw_df[\"keyword_all\"].apply(lambda x: re.sub(\"[\\(\\[].*?[\\)\\]]\", \"\", x))\n", + "kw_df.head(100)" ], "metadata": { "collapsed": false, @@ -174,7 +352,32 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 69, + "outputs": [ + { + "data": { + "text/plain": " UT (Unique WOS ID) keyword_all\n0 WOS:000297893800037 ADAPTIVE DYNAMIC SURFACE CONTROL; NEURAL COMPE...\n1 WOS:000301090100061 TEMPORAL CONJUNCTION; CAUDATE NUCLEUS; PREFRON...\n2 WOS:000301155300013 AUTOMATIC INCIDENT DETECTION; DATA CLEANSING; ...\n3 WOS:000301973200015 TRACHEO-BRONCHIAL; LUNG; INNERVATION; ESOPHAGE...\n4 WOS:000302289400006 LINGUISTIC ANNOTATION; ANNOTATION TOOLS; INTER...", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
UT (Unique WOS ID)keyword_all
0WOS:000297893800037ADAPTIVE DYNAMIC SURFACE CONTROL; NEURAL COMPE...
1WOS:000301090100061TEMPORAL CONJUNCTION; CAUDATE NUCLEUS; PREFRON...
2WOS:000301155300013AUTOMATIC INCIDENT DETECTION; DATA CLEANSING; ...
3WOS:000301973200015TRACHEO-BRONCHIAL; LUNG; INNERVATION; ESOPHAGE...
4WOS:000302289400006LINGUISTIC ANNOTATION; ANNOTATION TOOLS; INTER...
\n
" + }, + "execution_count": 69, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "wos_kwd_concat = kw_df.groupby(record_col, as_index=False).agg({'keyword_all': '; '.join})\n", + "wos_kwd_concat.head()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 34, "outputs": [ { "data": { @@ -182,59 +385,102 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "43fb040512964c61bdcca3e35d4e9778" + "model_id": "0d9a3ff741694ac895a40780392c62fe" } }, "metadata": {}, "output_type": "display_data" }, { - "ename": "ChunkedEncodingError", - "evalue": "(\"Connection broken: ConnectionResetError(10054, 'A létező kapcsolatot a távoli állomás kényszerítetten bezárta', None, 10054, None)\", ConnectionResetError(10054, 'A létező kapcsolatot a távoli állomás kényszerítetten bezárta', None, 10054, None))", - "output_type": "error", - "traceback": [ - "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[1;31mConnectionResetError\u001B[0m Traceback (most recent call last)", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\urllib3\\response.py:444\u001B[0m, in \u001B[0;36mHTTPResponse._error_catcher\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m 443\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m--> 444\u001B[0m \u001B[38;5;28;01myield\u001B[39;00m\n\u001B[0;32m 446\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m SocketTimeout:\n\u001B[0;32m 447\u001B[0m \u001B[38;5;66;03m# FIXME: Ideally we'd like to include the url in the ReadTimeoutError but\u001B[39;00m\n\u001B[0;32m 448\u001B[0m \u001B[38;5;66;03m# there is yet no clean way to get at it from this context.\u001B[39;00m\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\urllib3\\response.py:567\u001B[0m, in \u001B[0;36mHTTPResponse.read\u001B[1;34m(self, amt, decode_content, cache_content)\u001B[0m\n\u001B[0;32m 566\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_error_catcher():\n\u001B[1;32m--> 567\u001B[0m data \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_fp_read\u001B[49m\u001B[43m(\u001B[49m\u001B[43mamt\u001B[49m\u001B[43m)\u001B[49m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m fp_closed \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;124mb\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m 568\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m amt \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\urllib3\\response.py:533\u001B[0m, in \u001B[0;36mHTTPResponse._fp_read\u001B[1;34m(self, amt)\u001B[0m\n\u001B[0;32m 531\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m 532\u001B[0m \u001B[38;5;66;03m# StringIO doesn't like amt=None\u001B[39;00m\n\u001B[1;32m--> 533\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_fp\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mread\u001B[49m\u001B[43m(\u001B[49m\u001B[43mamt\u001B[49m\u001B[43m)\u001B[49m \u001B[38;5;28;01mif\u001B[39;00m amt \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_fp\u001B[38;5;241m.\u001B[39mread()\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\http\\client.py:463\u001B[0m, in \u001B[0;36mHTTPResponse.read\u001B[1;34m(self, amt)\u001B[0m\n\u001B[0;32m 462\u001B[0m b \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mbytearray\u001B[39m(amt)\n\u001B[1;32m--> 463\u001B[0m n \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mreadinto\u001B[49m\u001B[43m(\u001B[49m\u001B[43mb\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 464\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mmemoryview\u001B[39m(b)[:n]\u001B[38;5;241m.\u001B[39mtobytes()\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\http\\client.py:507\u001B[0m, in \u001B[0;36mHTTPResponse.readinto\u001B[1;34m(self, b)\u001B[0m\n\u001B[0;32m 504\u001B[0m \u001B[38;5;66;03m# we do not use _safe_read() here because this may be a .will_close\u001B[39;00m\n\u001B[0;32m 505\u001B[0m \u001B[38;5;66;03m# connection, and the user is reading more bytes than will be provided\u001B[39;00m\n\u001B[0;32m 506\u001B[0m \u001B[38;5;66;03m# (for example, reading in 1k chunks)\u001B[39;00m\n\u001B[1;32m--> 507\u001B[0m n \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfp\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mreadinto\u001B[49m\u001B[43m(\u001B[49m\u001B[43mb\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 508\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m n \u001B[38;5;129;01mand\u001B[39;00m b:\n\u001B[0;32m 509\u001B[0m \u001B[38;5;66;03m# Ideally, we would raise IncompleteRead if the content-length\u001B[39;00m\n\u001B[0;32m 510\u001B[0m \u001B[38;5;66;03m# wasn't satisfied, but it might break compatibility.\u001B[39;00m\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\socket.py:704\u001B[0m, in \u001B[0;36mSocketIO.readinto\u001B[1;34m(self, b)\u001B[0m\n\u001B[0;32m 703\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m--> 704\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_sock\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrecv_into\u001B[49m\u001B[43m(\u001B[49m\u001B[43mb\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 705\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m timeout:\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\ssl.py:1242\u001B[0m, in \u001B[0;36mSSLSocket.recv_into\u001B[1;34m(self, buffer, nbytes, flags)\u001B[0m\n\u001B[0;32m 1239\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\n\u001B[0;32m 1240\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001B[39m\u001B[38;5;132;01m%s\u001B[39;00m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;241m%\u001B[39m\n\u001B[0;32m 1241\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m\u001B[38;5;18m__class__\u001B[39m)\n\u001B[1;32m-> 1242\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mread\u001B[49m\u001B[43m(\u001B[49m\u001B[43mnbytes\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mbuffer\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 1243\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\ssl.py:1100\u001B[0m, in \u001B[0;36mSSLSocket.read\u001B[1;34m(self, len, buffer)\u001B[0m\n\u001B[0;32m 1099\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m buffer \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m-> 1100\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_sslobj\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mread\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mlen\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mbuffer\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 1101\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n", - "\u001B[1;31mConnectionResetError\u001B[0m: [WinError 10054] A létező kapcsolatot a távoli állomás kényszerítetten bezárta", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001B[1;31mProtocolError\u001B[0m Traceback (most recent call last)", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\requests\\models.py:816\u001B[0m, in \u001B[0;36mResponse.iter_content..generate\u001B[1;34m()\u001B[0m\n\u001B[0;32m 815\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m--> 816\u001B[0m \u001B[38;5;28;01myield from\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mraw\u001B[38;5;241m.\u001B[39mstream(chunk_size, decode_content\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m)\n\u001B[0;32m 817\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m ProtocolError \u001B[38;5;28;01mas\u001B[39;00m e:\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\urllib3\\response.py:628\u001B[0m, in \u001B[0;36mHTTPResponse.stream\u001B[1;34m(self, amt, decode_content)\u001B[0m\n\u001B[0;32m 627\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m is_fp_closed(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_fp):\n\u001B[1;32m--> 628\u001B[0m data \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mread\u001B[49m\u001B[43m(\u001B[49m\u001B[43mamt\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mamt\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdecode_content\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mdecode_content\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 630\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m data:\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\urllib3\\response.py:593\u001B[0m, in \u001B[0;36mHTTPResponse.read\u001B[1;34m(self, amt, decode_content, cache_content)\u001B[0m\n\u001B[0;32m 584\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39menforce_content_length \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mlength_remaining \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;129;01min\u001B[39;00m (\n\u001B[0;32m 585\u001B[0m \u001B[38;5;241m0\u001B[39m,\n\u001B[0;32m 586\u001B[0m \u001B[38;5;28;01mNone\u001B[39;00m,\n\u001B[1;32m (...)\u001B[0m\n\u001B[0;32m 591\u001B[0m \u001B[38;5;66;03m# raised during streaming, so all calls with incorrect\u001B[39;00m\n\u001B[0;32m 592\u001B[0m \u001B[38;5;66;03m# Content-Length are caught.\u001B[39;00m\n\u001B[1;32m--> 593\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m IncompleteRead(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_fp_bytes_read, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mlength_remaining)\n\u001B[0;32m 595\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m data:\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\contextlib.py:137\u001B[0m, in \u001B[0;36m_GeneratorContextManager.__exit__\u001B[1;34m(self, typ, value, traceback)\u001B[0m\n\u001B[0;32m 136\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m--> 137\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgen\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mthrow\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtyp\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mvalue\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mtraceback\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 138\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mStopIteration\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m exc:\n\u001B[0;32m 139\u001B[0m \u001B[38;5;66;03m# Suppress StopIteration *unless* it's the same exception that\u001B[39;00m\n\u001B[0;32m 140\u001B[0m \u001B[38;5;66;03m# was passed to throw(). This prevents a StopIteration\u001B[39;00m\n\u001B[0;32m 141\u001B[0m \u001B[38;5;66;03m# raised inside the \"with\" statement from being suppressed.\u001B[39;00m\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\urllib3\\response.py:461\u001B[0m, in \u001B[0;36mHTTPResponse._error_catcher\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m 459\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (HTTPException, SocketError) \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[0;32m 460\u001B[0m \u001B[38;5;66;03m# This includes IncompleteRead.\u001B[39;00m\n\u001B[1;32m--> 461\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m ProtocolError(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mConnection broken: \u001B[39m\u001B[38;5;132;01m%r\u001B[39;00m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;241m%\u001B[39m e, e)\n\u001B[0;32m 463\u001B[0m \u001B[38;5;66;03m# If no exception is thrown, we should avoid cleaning up\u001B[39;00m\n\u001B[0;32m 464\u001B[0m \u001B[38;5;66;03m# unnecessarily.\u001B[39;00m\n", - "\u001B[1;31mProtocolError\u001B[0m: (\"Connection broken: ConnectionResetError(10054, 'A létező kapcsolatot a távoli állomás kényszerítetten bezárta', None, 10054, None)\", ConnectionResetError(10054, 'A létező kapcsolatot a távoli állomás kényszerítetten bezárta', None, 10054, None))", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001B[1;31mChunkedEncodingError\u001B[0m Traceback (most recent call last)", - "Cell \u001B[1;32mIn[39], line 7\u001B[0m\n\u001B[0;32m 4\u001B[0m \u001B[38;5;66;03m# Uses stopwords for english from NLTK, and all puntuation characters by\u001B[39;00m\n\u001B[0;32m 5\u001B[0m \u001B[38;5;66;03m# default\u001B[39;00m\n\u001B[0;32m 6\u001B[0m r \u001B[38;5;241m=\u001B[39m Rake()\n\u001B[1;32m----> 7\u001B[0m kw_model \u001B[38;5;241m=\u001B[39m \u001B[43mKeyBERT\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmodel\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mall-mpnet-base-v2\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m)\u001B[49m\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\keybert\\_model.py:55\u001B[0m, in \u001B[0;36mKeyBERT.__init__\u001B[1;34m(self, model)\u001B[0m\n\u001B[0;32m 39\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m__init__\u001B[39m(\u001B[38;5;28mself\u001B[39m, model\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mall-MiniLM-L6-v2\u001B[39m\u001B[38;5;124m\"\u001B[39m):\n\u001B[0;32m 40\u001B[0m \u001B[38;5;124;03m\"\"\"KeyBERT initialization\u001B[39;00m\n\u001B[0;32m 41\u001B[0m \n\u001B[0;32m 42\u001B[0m \u001B[38;5;124;03m Arguments:\u001B[39;00m\n\u001B[1;32m (...)\u001B[0m\n\u001B[0;32m 53\u001B[0m \u001B[38;5;124;03m * https://www.sbert.net/docs/pretrained_models.html\u001B[39;00m\n\u001B[0;32m 54\u001B[0m \u001B[38;5;124;03m \"\"\"\u001B[39;00m\n\u001B[1;32m---> 55\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mmodel \u001B[38;5;241m=\u001B[39m \u001B[43mselect_backend\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmodel\u001B[49m\u001B[43m)\u001B[49m\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\keybert\\backend\\_utils.py:49\u001B[0m, in \u001B[0;36mselect_backend\u001B[1;34m(embedding_model)\u001B[0m\n\u001B[0;32m 47\u001B[0m \u001B[38;5;66;03m# Create a Sentence Transformer model based on a string\u001B[39;00m\n\u001B[0;32m 48\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(embedding_model, \u001B[38;5;28mstr\u001B[39m):\n\u001B[1;32m---> 49\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mSentenceTransformerBackend\u001B[49m\u001B[43m(\u001B[49m\u001B[43membedding_model\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 51\u001B[0m \u001B[38;5;66;03m# Hugging Face embeddings\u001B[39;00m\n\u001B[0;32m 52\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(embedding_model, Pipeline):\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\keybert\\backend\\_sentencetransformers.py:42\u001B[0m, in \u001B[0;36mSentenceTransformerBackend.__init__\u001B[1;34m(self, embedding_model)\u001B[0m\n\u001B[0;32m 40\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39membedding_model \u001B[38;5;241m=\u001B[39m embedding_model\n\u001B[0;32m 41\u001B[0m \u001B[38;5;28;01melif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(embedding_model, \u001B[38;5;28mstr\u001B[39m):\n\u001B[1;32m---> 42\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39membedding_model \u001B[38;5;241m=\u001B[39m \u001B[43mSentenceTransformer\u001B[49m\u001B[43m(\u001B[49m\u001B[43membedding_model\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 43\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m 44\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\n\u001B[0;32m 45\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mPlease select a correct SentenceTransformers model: \u001B[39m\u001B[38;5;130;01m\\n\u001B[39;00m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m 46\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m`from sentence_transformers import SentenceTransformer` \u001B[39m\u001B[38;5;130;01m\\n\u001B[39;00m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m 47\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m`model = SentenceTransformer(\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mall-MiniLM-L6-v2\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124m)`\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m 48\u001B[0m )\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\sentence_transformers\\SentenceTransformer.py:87\u001B[0m, in \u001B[0;36mSentenceTransformer.__init__\u001B[1;34m(self, model_name_or_path, modules, device, cache_folder, use_auth_token)\u001B[0m\n\u001B[0;32m 83\u001B[0m model_path \u001B[38;5;241m=\u001B[39m os\u001B[38;5;241m.\u001B[39mpath\u001B[38;5;241m.\u001B[39mjoin(cache_folder, model_name_or_path\u001B[38;5;241m.\u001B[39mreplace(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m/\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m_\u001B[39m\u001B[38;5;124m\"\u001B[39m))\n\u001B[0;32m 85\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m os\u001B[38;5;241m.\u001B[39mpath\u001B[38;5;241m.\u001B[39mexists(os\u001B[38;5;241m.\u001B[39mpath\u001B[38;5;241m.\u001B[39mjoin(model_path, \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mmodules.json\u001B[39m\u001B[38;5;124m'\u001B[39m)):\n\u001B[0;32m 86\u001B[0m \u001B[38;5;66;03m# Download from hub with caching\u001B[39;00m\n\u001B[1;32m---> 87\u001B[0m \u001B[43msnapshot_download\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmodel_name_or_path\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 88\u001B[0m \u001B[43m \u001B[49m\u001B[43mcache_dir\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcache_folder\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 89\u001B[0m \u001B[43m \u001B[49m\u001B[43mlibrary_name\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43msentence-transformers\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[0;32m 90\u001B[0m \u001B[43m \u001B[49m\u001B[43mlibrary_version\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m__version__\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 91\u001B[0m \u001B[43m \u001B[49m\u001B[43mignore_files\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m[\u001B[49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mflax_model.msgpack\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mrust_model.ot\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mtf_model.h5\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 92\u001B[0m \u001B[43m \u001B[49m\u001B[43muse_auth_token\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43muse_auth_token\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 94\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m os\u001B[38;5;241m.\u001B[39mpath\u001B[38;5;241m.\u001B[39mexists(os\u001B[38;5;241m.\u001B[39mpath\u001B[38;5;241m.\u001B[39mjoin(model_path, \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mmodules.json\u001B[39m\u001B[38;5;124m'\u001B[39m)): \u001B[38;5;66;03m#Load as SentenceTransformer model\u001B[39;00m\n\u001B[0;32m 95\u001B[0m modules \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_load_sbert_model(model_path)\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\sentence_transformers\\util.py:491\u001B[0m, in \u001B[0;36msnapshot_download\u001B[1;34m(repo_id, revision, cache_dir, library_name, library_version, user_agent, ignore_files, use_auth_token)\u001B[0m\n\u001B[0;32m 486\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m version\u001B[38;5;241m.\u001B[39mparse(huggingface_hub\u001B[38;5;241m.\u001B[39m__version__) \u001B[38;5;241m>\u001B[39m\u001B[38;5;241m=\u001B[39m version\u001B[38;5;241m.\u001B[39mparse(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m0.8.1\u001B[39m\u001B[38;5;124m\"\u001B[39m):\n\u001B[0;32m 487\u001B[0m \u001B[38;5;66;03m# huggingface_hub v0.8.1 introduces a new cache layout. We sill use a manual layout\u001B[39;00m\n\u001B[0;32m 488\u001B[0m \u001B[38;5;66;03m# And need to pass legacy_cache_layout=True to avoid that a warning will be printed\u001B[39;00m\n\u001B[0;32m 489\u001B[0m cached_download_args[\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mlegacy_cache_layout\u001B[39m\u001B[38;5;124m'\u001B[39m] \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n\u001B[1;32m--> 491\u001B[0m path \u001B[38;5;241m=\u001B[39m cached_download(\u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mcached_download_args)\n\u001B[0;32m 493\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m os\u001B[38;5;241m.\u001B[39mpath\u001B[38;5;241m.\u001B[39mexists(path \u001B[38;5;241m+\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m.lock\u001B[39m\u001B[38;5;124m\"\u001B[39m):\n\u001B[0;32m 494\u001B[0m os\u001B[38;5;241m.\u001B[39mremove(path \u001B[38;5;241m+\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m.lock\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\huggingface_hub\\utils\\_validators.py:120\u001B[0m, in \u001B[0;36mvalidate_hf_hub_args.._inner_fn\u001B[1;34m(*args, **kwargs)\u001B[0m\n\u001B[0;32m 117\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m check_use_auth_token:\n\u001B[0;32m 118\u001B[0m kwargs \u001B[38;5;241m=\u001B[39m smoothly_deprecate_use_auth_token(fn_name\u001B[38;5;241m=\u001B[39mfn\u001B[38;5;241m.\u001B[39m\u001B[38;5;18m__name__\u001B[39m, has_token\u001B[38;5;241m=\u001B[39mhas_token, kwargs\u001B[38;5;241m=\u001B[39mkwargs)\n\u001B[1;32m--> 120\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m fn(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\huggingface_hub\\file_download.py:780\u001B[0m, in \u001B[0;36mcached_download\u001B[1;34m(url, library_name, library_version, cache_dir, user_agent, force_download, force_filename, proxies, etag_timeout, resume_download, token, local_files_only, legacy_cache_layout)\u001B[0m\n\u001B[0;32m 777\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m temp_file_manager() \u001B[38;5;28;01mas\u001B[39;00m temp_file:\n\u001B[0;32m 778\u001B[0m logger\u001B[38;5;241m.\u001B[39minfo(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mdownloading \u001B[39m\u001B[38;5;132;01m%s\u001B[39;00m\u001B[38;5;124m to \u001B[39m\u001B[38;5;132;01m%s\u001B[39;00m\u001B[38;5;124m\"\u001B[39m, url, temp_file\u001B[38;5;241m.\u001B[39mname)\n\u001B[1;32m--> 780\u001B[0m \u001B[43mhttp_get\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m 781\u001B[0m \u001B[43m \u001B[49m\u001B[43murl_to_download\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 782\u001B[0m \u001B[43m \u001B[49m\u001B[43mtemp_file\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 783\u001B[0m \u001B[43m \u001B[49m\u001B[43mproxies\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mproxies\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 784\u001B[0m \u001B[43m \u001B[49m\u001B[43mresume_size\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mresume_size\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 785\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m 786\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m 788\u001B[0m logger\u001B[38;5;241m.\u001B[39minfo(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mstoring \u001B[39m\u001B[38;5;132;01m%s\u001B[39;00m\u001B[38;5;124m in cache at \u001B[39m\u001B[38;5;132;01m%s\u001B[39;00m\u001B[38;5;124m\"\u001B[39m, url, cache_path)\n\u001B[0;32m 789\u001B[0m _chmod_and_replace(temp_file\u001B[38;5;241m.\u001B[39mname, cache_path)\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\huggingface_hub\\file_download.py:538\u001B[0m, in \u001B[0;36mhttp_get\u001B[1;34m(url, temp_file, proxies, resume_size, headers, timeout, max_retries)\u001B[0m\n\u001B[0;32m 528\u001B[0m displayed_name \u001B[38;5;241m=\u001B[39m \u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m(…)\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mdisplayed_name[\u001B[38;5;241m-\u001B[39m\u001B[38;5;241m20\u001B[39m:]\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m 530\u001B[0m progress \u001B[38;5;241m=\u001B[39m tqdm(\n\u001B[0;32m 531\u001B[0m unit\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mB\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[0;32m 532\u001B[0m unit_scale\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m,\n\u001B[1;32m (...)\u001B[0m\n\u001B[0;32m 536\u001B[0m disable\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mbool\u001B[39m(logger\u001B[38;5;241m.\u001B[39mgetEffectiveLevel() \u001B[38;5;241m==\u001B[39m logging\u001B[38;5;241m.\u001B[39mNOTSET),\n\u001B[0;32m 537\u001B[0m )\n\u001B[1;32m--> 538\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m chunk \u001B[38;5;129;01min\u001B[39;00m r\u001B[38;5;241m.\u001B[39miter_content(chunk_size\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m10\u001B[39m \u001B[38;5;241m*\u001B[39m \u001B[38;5;241m1024\u001B[39m \u001B[38;5;241m*\u001B[39m \u001B[38;5;241m1024\u001B[39m):\n\u001B[0;32m 539\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m chunk: \u001B[38;5;66;03m# filter out keep-alive new chunks\u001B[39;00m\n\u001B[0;32m 540\u001B[0m progress\u001B[38;5;241m.\u001B[39mupdate(\u001B[38;5;28mlen\u001B[39m(chunk))\n", - "File \u001B[1;32m~\\Anaconda3\\envs\\MOME_BIGDATA\\lib\\site-packages\\requests\\models.py:818\u001B[0m, in \u001B[0;36mResponse.iter_content..generate\u001B[1;34m()\u001B[0m\n\u001B[0;32m 816\u001B[0m \u001B[38;5;28;01myield from\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mraw\u001B[38;5;241m.\u001B[39mstream(chunk_size, decode_content\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m)\n\u001B[0;32m 817\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m ProtocolError \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m--> 818\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m ChunkedEncodingError(e)\n\u001B[0;32m 819\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m DecodeError \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[0;32m 820\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m ContentDecodingError(e)\n", - "\u001B[1;31mChunkedEncodingError\u001B[0m: (\"Connection broken: ConnectionResetError(10054, 'A létező kapcsolatot a távoli állomás kényszerítetten bezárta', None, 10054, None)\", ConnectionResetError(10054, 'A létező kapcsolatot a távoli állomás kényszerítetten bezárta', None, 10054, None))" - ] + "data": { + "text/plain": "Downloading (…)nce_bert_config.json: 0%| | 0.00/53.0 [00:00 1\u001B[0m \u001B[43mRake\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mextract_keywords_from_text\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mmy time to shine\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mget_ranked_phrases\u001B[49m()\n", - "\u001B[1;31mAttributeError\u001B[0m: 'NoneType' object has no attribute 'get_ranked_phrases'" - ] + "data": { + "text/plain": "'ELECTROMAGNETIC IMAGING; INVERSE SCATTERING; SCATTERING ELECTROMAGNETIC'" + }, + "execution_count": 78, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "Rake().extract_keywords_from_text(\"my time to shine\").get_ranked_phrases()" + "def kwd_extract(text):\n", + " keywords = kw_model.extract_keywords(text,\n", + "\n", + " keyphrase_ngram_range=(1, 2),\n", + "\n", + " stop_words='english',\n", + "\n", + " highlight=False,\n", + "\n", + " top_n=3)\n", + " return \"; \".join([i[0].upper() for i in keywords])\n", + "\n", + "kwd_extract(text=\"Artificial Intelligence: New Frontiers in Real-Time Inverse Scattering and Electromagnetic Imaging - In recent years, artificial intelligence (AI) techniques have been developed rapidly. With the ...\")" ], "metadata": { "collapsed": false, @@ -287,16 +542,45 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 61, + "outputs": [ + { + "data": { + "text/plain": "'ELECTROMAGNETIC IMAGING; INVERSE SCATTERING; SCATTERING ELECTROMAGNETIC; SCATTERING; AI; ELECTROMAGNETIC; IMAGING; ARTIFICIAL INTELLIGENCE'" + }, + "execution_count": 61, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 86, "outputs": [], "source": [ - "def kwd_rake(text):\n", - " r = Rake()\n", - " r.extract_keywords_from_sentences(text)\n", - " return r.get_ranked_phrases()\n", + "wos_nlp = wos[[record_col,\"Article Title\",\"Abstract\"]]\n", + "wos_nlp = wos_nlp.merge(wos_kwd_concat, on = record_col)\n", + "wos_nlp[\"Document\"] = wos_nlp[\"Article Title\"].str.cat(wos_nlp[[\"Abstract\",\"keyword_all\"]].fillna(\"\"), sep=' - ')\n", + "# wos_kwd_test[\"BERT_KWDS\"] = wos_kwd_test[\"Document\"].map(kwd_extract)\n", + "\n", + "vectors = list()\n", + "vector_norms = list()\n", + "\n", + "for doc in nlp.pipe(wos_nlp['Document'].astype('unicode').values, batch_size=100,\n", + " n_process=4):\n", + " vectors.append(doc.vector)\n", + " vector_norms.append(doc.vector_norm)\n", "\n", - "kwds_rake = wos[\"Abstract\"].fillna(\"\").map(kwd_rake)\n", - "# kwds_bert = wos[\"A\"]\n" + "wos_nlp['vector'] = vectors\n", + "wos_nlp['vector_norm'] = vector_norms" ], "metadata": { "collapsed": false, @@ -307,19 +591,27 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 84, "outputs": [ { "data": { - "text/plain": "0 [brief summary could help us better understand...\n9714 [known phase space reconstruction based causal...\n9697 [column behavior requires accurate plastic hin...\n9699 [approach needs excessive computational effort...\n9701 [proposed ann model achieves good wind speed r...\n ... \n3066 [key factors affecting ocean climate change, r...\n5097 [big data influences different financial secto...\n11369 [planetary gear fault diagnosis via feature im...\n11368 [simultaneously predict various soil fertility...\n11362 [recently developed ensemble machine learning ...\nName: Abstract, Length: 9889, dtype: object" + "text/plain": "" }, - "execution_count": 38, + "execution_count": 84, "metadata": {}, "output_type": "execute_result" + }, + { + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAGdCAYAAADzOWwgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAuYElEQVR4nO3dfVRVdb7H8Q+KICoHfOLpikqjqeRDqV09y4c7JiMlzaqkWVmaTlFeGyyVzIc7XWusCUeXmq5KehqxVY7p3KzUq8Zo4pT4REM+lKRpFw0OeEfhiAki7PtHl708YZZH5MD5vV9r7bU8+/c9+3x//TI+7bP3JsCyLEsAAAAGa+brBgAAAHyNQAQAAIxHIAIAAMYjEAEAAOMRiAAAgPEIRAAAwHgEIgAAYDwCEQAAMF6grxtoCmpqalRYWKjQ0FAFBAT4uh0AAPAzWJals2fPKiYmRs2aXfkcEIHoZygsLFRsbKyv2wAAAF44ceKEOnXqdMUaAtHPEBoaKun7f6AOh8PH3QAAgJ/D7XYrNjbW/jl+JQSin6H2azKHw0EgAgCgifk5l7twUTUAADAegQgAABiPQAQAAIxHIAIAAMYjEAEAAOMRiAAAgPEIRAAAwHgEIgAAYDwCEQAAMB6BCAAAGI9ABAAAjEcgAgAAxiMQAQAA4xGIAACA8QJ93QCapq6zN/q6hav2zfwkX7cAAGikOEMEAACMRyACAADGIxABAADjEYgAAIDxCEQAAMB4BCIAAGA8AhEAADAegQgAABiPQAQAAIxHIAIAAMYjEAEAAOMRiAAAgPEIRAAAwHgEIgAAYDwCEQAAMB6BCAAAGI9ABAAAjEcgAgAAxiMQAQAA4xGIAACA8QhEAADAeAQiAABgPAIRAAAwHoEIAAAYz+eB6Ntvv9X48ePVvn17hYSEqE+fPtq3b589blmW5s6dq+joaIWEhCghIUFHjhzxOMbp06c1btw4ORwOhYeHKyUlReXl5R41+/fv17Bhw9SyZUvFxsZqwYIFDTI/AADQ+Pk0EJ05c0ZDhgxRixYttGnTJn3xxRdatGiR2rZta9csWLBAy5YtU0ZGhnbv3q3WrVsrMTFRFRUVds24ceN06NAhZWVlacOGDdqxY4cmTZpkj7vdbo0aNUpdunRRbm6uFi5cqGeffVavvfZag84XAAA0TgGWZVm++vDZs2fr008/1d///vfLjluWpZiYGD355JOaMWOGJKmsrEyRkZHKzMzU2LFj9eWXXyo+Pl579+7VwIEDJUmbN2/W6NGjdfLkScXExGj58uX6/e9/L5fLpaCgIPuz33//fR0+fPgn+3S73QoLC1NZWZkcDkc9zb5p6zp7o69buGrfzE/ydQsAgAZ0NT+/fXqG6MMPP9TAgQP1m9/8RhEREbrlllv0+uuv2+PHjx+Xy+VSQkKCvS8sLEyDBg1STk6OJCknJ0fh4eF2GJKkhIQENWvWTLt377Zrhg8fbochSUpMTFR+fr7OnDlTp6/Kykq53W6PDQAA+C+fBqJjx45p+fLl6t69u7Zs2aLHHntMTzzxhFauXClJcrlckqTIyEiP90VGRtpjLpdLERERHuOBgYFq166dR83ljnHpZ1wqPT1dYWFh9hYbG1sPswUAAI2VTwNRTU2N+vfvrxdeeEG33HKLJk2apEcffVQZGRm+bEtz5sxRWVmZvZ04ccKn/QAAgOvLp4EoOjpa8fHxHvt69eqlgoICSVJUVJQkqbi42KOmuLjYHouKilJJSYnH+MWLF3X69GmPmssd49LPuFRwcLAcDofHBgAA/JdPA9GQIUOUn5/vse+rr75Sly5dJElxcXGKiorS1q1b7XG3263du3fL6XRKkpxOp0pLS5Wbm2vXbNu2TTU1NRo0aJBds2PHDlVVVdk1WVlZ6tGjh8cdbQAAwEw+DUTTp0/Xrl279MILL+jo0aNatWqVXnvtNaWmpkqSAgICNG3aND3//PP68MMPdeDAAU2YMEExMTG6++67JX1/Run222/Xo48+qj179ujTTz/VlClTNHbsWMXExEiSHnjgAQUFBSklJUWHDh3Su+++q6VLlyotLc1XUwcAAI1IoC8//NZbb9W6des0Z84czZs3T3FxcXrxxRc1btw4u2bmzJk6d+6cJk2apNLSUg0dOlSbN29Wy5Yt7Zp33nlHU6ZM0ciRI9WsWTMlJydr2bJl9nhYWJg++ugjpaamasCAAerQoYPmzp3r8awiAABgLp8+h6ip4DlEdfEcIgBAY9dknkMEAADQGBCIAACA8QhEAADAeAQiAABgPAIRAAAwHoEIAAAYj0AEAACMRyACAADGIxABAADjEYgAAIDxCEQAAMB4BCIAAGA8AhEAADAegQgAABiPQAQAAIxHIAIAAMYjEAEAAOMRiAAAgPEIRAAAwHgEIgAAYDwCEQAAMB6BCAAAGI9ABAAAjEcgAgAAxiMQAQAA4xGIAACA8QhEAADAeAQiAABgPAIRAAAwHoEIAAAYj0AEAACMRyACAADGIxABAADjEYgAAIDxCEQAAMB4BCIAAGA8AhEAADAegQgAABiPQAQAAIxHIAIAAMYjEAEAAOMRiAAAgPEIRAAAwHg+DUTPPvusAgICPLaePXva4xUVFUpNTVX79u3Vpk0bJScnq7i42OMYBQUFSkpKUqtWrRQREaGnnnpKFy9e9KjZvn27+vfvr+DgYHXr1k2ZmZkNMT0AANBE+PwM0U033aSioiJ7++STT+yx6dOna/369Vq7dq2ys7NVWFioMWPG2OPV1dVKSkrShQsXtHPnTq1cuVKZmZmaO3euXXP8+HElJSVpxIgRysvL07Rp0/TII49oy5YtDTpPAADQeAX6vIHAQEVFRdXZX1ZWpjfffFOrVq3SbbfdJklasWKFevXqpV27dmnw4MH66KOP9MUXX+hvf/ubIiMjdfPNN+u5557TrFmz9OyzzyooKEgZGRmKi4vTokWLJEm9evXSJ598oiVLligxMbFB5woAABonn58hOnLkiGJiYnTDDTdo3LhxKigokCTl5uaqqqpKCQkJdm3Pnj3VuXNn5eTkSJJycnLUp08fRUZG2jWJiYlyu906dOiQXXPpMWprao9xOZWVlXK73R4bAADwXz4NRIMGDVJmZqY2b96s5cuX6/jx4xo2bJjOnj0rl8uloKAghYeHe7wnMjJSLpdLkuRyuTzCUO147diVatxut86fP3/ZvtLT0xUWFmZvsbGx9TFdAADQSPn0K7M77rjD/nPfvn01aNAgdenSRWvWrFFISIjP+pozZ47S0tLs1263m1AEAIAf8/lXZpcKDw/XjTfeqKNHjyoqKkoXLlxQaWmpR01xcbF9zVFUVFSdu85qX/9UjcPh+NHQFRwcLIfD4bEBAAD/1agCUXl5ub7++mtFR0drwIABatGihbZu3WqP5+fnq6CgQE6nU5LkdDp14MABlZSU2DVZWVlyOByKj4+3ay49Rm1N7TEAAAB8GohmzJih7OxsffPNN9q5c6fuueceNW/eXPfff7/CwsKUkpKitLQ0ffzxx8rNzdVDDz0kp9OpwYMHS5JGjRql+Ph4Pfjgg/r888+1ZcsWPf3000pNTVVwcLAkafLkyTp27Jhmzpypw4cP65VXXtGaNWs0ffp0X04dAAA0Ij69hujkyZO6//779c9//lMdO3bU0KFDtWvXLnXs2FGStGTJEjVr1kzJycmqrKxUYmKiXnnlFfv9zZs314YNG/TYY4/J6XSqdevWmjhxoubNm2fXxMXFaePGjZo+fbqWLl2qTp066Y033uCWewAAYAuwLMvydRONndvtVlhYmMrKyrie6P91nb3R1y1ctW/mJ/m6BQBAA7qan9+N6hoiAAAAXyAQAQAA4xGIAACA8QhEAADAeAQiAABgPAIRAAAwHoEIAAAYj0AEAACMRyACAADGIxABAADjEYgAAIDxCEQAAMB4BCIAAGA8AhEAADAegQgAABiPQAQAAIxHIAIAAMYjEAEAAOMRiAAAgPEIRAAAwHgEIgAAYDwCEQAAMB6BCAAAGI9ABAAAjEcgAgAAxiMQAQAA4xGIAACA8QhEAADAeAQiAABgPAIRAAAwHoEIAAAYj0AEAACMRyACAADGIxABAADjEYgAAIDxCEQAAMB4BCIAAGA8AhEAADAegQgAABiPQAQAAIxHIAIAAMYjEAEAAOMRiAAAgPEaTSCaP3++AgICNG3aNHtfRUWFUlNT1b59e7Vp00bJyckqLi72eF9BQYGSkpLUqlUrRURE6KmnntLFixc9arZv367+/fsrODhY3bp1U2ZmZgPMCAAANBWNIhDt3btXr776qvr27euxf/r06Vq/fr3Wrl2r7OxsFRYWasyYMfZ4dXW1kpKSdOHCBe3cuVMrV65UZmam5s6da9ccP35cSUlJGjFihPLy8jRt2jQ98sgj2rJlS4PNDwAANG5eBaJjx47VWwPl5eUaN26cXn/9dbVt29beX1ZWpjfffFOLFy/WbbfdpgEDBmjFihXauXOndu3aJUn66KOP9MUXX+jtt9/WzTffrDvuuEPPPfecXn75ZV24cEGSlJGRobi4OC1atEi9evXSlClTdO+992rJkiX1NgcAANC0eRWIunXrphEjRujtt99WRUXFNTWQmpqqpKQkJSQkeOzPzc1VVVWVx/6ePXuqc+fOysnJkSTl5OSoT58+ioyMtGsSExPldrt16NAhu+aHx05MTLSPcTmVlZVyu90eGwAA8F9eBaLPPvtMffv2VVpamqKiovTv//7v2rNnz1UfZ/Xq1frss8+Unp5eZ8zlcikoKEjh4eEe+yMjI+VyueyaS8NQ7Xjt2JVq3G63zp8/f9m+0tPTFRYWZm+xsbFXPTcAANB0eBWIbr75Zi1dulSFhYX685//rKKiIg0dOlS9e/fW4sWLderUqZ88xokTJzR16lS98847atmypTdtXDdz5sxRWVmZvZ04ccLXLQEAgOvomi6qDgwM1JgxY7R27Vr96U9/0tGjRzVjxgzFxsZqwoQJKioq+tH35ubmqqSkRP3791dgYKACAwOVnZ2tZcuWKTAwUJGRkbpw4YJKS0s93ldcXKyoqChJUlRUVJ27zmpf/1SNw+FQSEjIZXsLDg6Ww+Hw2AAAgP+6pkC0b98+/e53v1N0dLQWL16sGTNm6Ouvv1ZWVpYKCwt11113/eh7R44cqQMHDigvL8/eBg4cqHHjxtl/btGihbZu3Wq/Jz8/XwUFBXI6nZIkp9OpAwcOqKSkxK7JysqSw+FQfHy8XXPpMWprao8BAAAQ6M2bFi9erBUrVig/P1+jR4/WW2+9pdGjR6tZs+/zVVxcnDIzM9W1a9cfPUZoaKh69+7tsa9169Zq3769vT8lJUVpaWlq166dHA6HHn/8cTmdTg0ePFiSNGrUKMXHx+vBBx/UggUL5HK59PTTTys1NVXBwcGSpMmTJ+ull17SzJkz9fDDD2vbtm1as2aNNm7c6M3UAQCAH/IqEC1fvlwPP/ywfvvb3yo6OvqyNREREXrzzTevqbklS5aoWbNmSk5OVmVlpRITE/XKK6/Y482bN9eGDRv02GOPyel0qnXr1po4caLmzZtn18TFxWnjxo2aPn26li5dqk6dOumNN95QYmLiNfUGAAD8R4BlWZavm2js3G63wsLCVFZWxvVE/6/r7KZ3hu2b+Um+bgEA0ICu5ue3V9cQrVixQmvXrq2zf+3atVq5cqU3hwQAAPAZrwJRenq6OnToUGd/RESEXnjhhWtuCgAAoCF5FYgKCgoUFxdXZ3+XLl1UUFBwzU0BAAA0JK8CUUREhPbv319n/+eff6727dtfc1MAAAANyatAdP/99+uJJ57Qxx9/rOrqalVXV2vbtm2aOnWqxo4dW989AgAAXFde3Xb/3HPP6ZtvvtHIkSMVGPj9IWpqajRhwgSuIQIAAE2OV4EoKChI7777rp577jl9/vnnCgkJUZ8+fdSlS5f67g8AAOC68yoQ1brxxht144031lcvAAAAPuFVIKqurlZmZqa2bt2qkpIS1dTUeIxv27atXpoDAABoCF4FoqlTpyozM1NJSUnq3bu3AgIC6rsvAACABuNVIFq9erXWrFmj0aNH13c/AAAADc6r2+6DgoLUrVu3+u4FAADAJ7wKRE8++aSWLl0qfi8sAADwB159ZfbJJ5/o448/1qZNm3TTTTepRYsWHuPvvfdevTQHAADQELwKROHh4brnnnvquxcAAACf8CoQrVixor77AAAA8BmvriGSpIsXL+pvf/ubXn31VZ09e1aSVFhYqPLy8nprDgAAoCF4dYbof/7nf3T77beroKBAlZWV+tWvfqXQ0FD96U9/UmVlpTIyMuq7TwAAgOvGqzNEU6dO1cCBA3XmzBmFhITY+++55x5t3bq13poDAABoCF6dIfr73/+unTt3KigoyGN/165d9e2339ZLYwAAAA3FqzNENTU1qq6urrP/5MmTCg0NveamAAAAGpJXgWjUqFF68cUX7dcBAQEqLy/XM888w6/zAAAATY5XX5ktWrRIiYmJio+PV0VFhR544AEdOXJEHTp00F/+8pf67hEAAOC68ioQderUSZ9//rlWr16t/fv3q7y8XCkpKRo3bpzHRdYAAABNgVeBSJICAwM1fvz4+uwFAADAJ7wKRG+99dYVxydMmOBVMwAAAL7gVSCaOnWqx+uqqip99913CgoKUqtWrQhEAACgSfHqLrMzZ854bOXl5crPz9fQoUO5qBoAADQ5Xv8usx/q3r275s+fX+fsEQAAQGNXb4FI+v5C68LCwvo8JAAAwHXn1TVEH374ocdry7JUVFSkl156SUOGDKmXxgAAABqKV4Ho7rvv9ngdEBCgjh076rbbbtOiRYvqoy8AAIAG41Ugqqmpqe8+AAAAfKZeryECAABoirw6Q5SWlvazaxcvXuzNRwAAADQYrwLRP/7xD/3jH/9QVVWVevToIUn66quv1Lx5c/Xv39+uCwgIqJ8uAQAAriOvAtGvf/1rhYaGauXKlWrbtq2k7x/W+NBDD2nYsGF68skn67VJAACA68mra4gWLVqk9PR0OwxJUtu2bfX8889zlxkAAGhyvApEbrdbp06dqrP/1KlTOnv27DU3BQAA0JC8CkT33HOPHnroIb333ns6efKkTp48qf/6r/9SSkqKxowZU989AgAAXFdeXUOUkZGhGTNm6IEHHlBVVdX3BwoMVEpKihYuXFivDQIAAFxvXgWiVq1a6ZVXXtHChQv19ddfS5J+8YtfqHXr1vXaHAAAQEO4pgczFhUVqaioSN27d1fr1q1lWVZ99QUAANBgvApE//znPzVy5EjdeOONGj16tIqKiiRJKSkpV3XL/fLly9W3b185HA45HA45nU5t2rTJHq+oqFBqaqrat2+vNm3aKDk5WcXFxR7HKCgoUFJSklq1aqWIiAg99dRTunjxokfN9u3b1b9/fwUHB6tbt27KzMz0ZtoAAMBPeRWIpk+frhYtWqigoECtWrWy9993333avHnzzz5Op06dNH/+fOXm5mrfvn267bbbdNddd+nQoUP256xfv15r165Vdna2CgsLPS7arq6uVlJSki5cuKCdO3dq5cqVyszM1Ny5c+2a48ePKykpSSNGjFBeXp6mTZumRx55RFu2bPFm6gAAwA8FWF58zxUVFaUtW7aoX79+Cg0N1eeff64bbrhBx44dU9++fVVeXu51Q+3atdPChQt17733qmPHjlq1apXuvfdeSdLhw4fVq1cv5eTkaPDgwdq0aZPuvPNOFRYWKjIyUtL3F3zPmjVLp06dUlBQkGbNmqWNGzfq4MGD9meMHTtWpaWlPzu8ud1uhYWFqaysTA6Hw+u5+ZOuszf6uoWr9s38JF+3AABoQFfz89urM0Tnzp3zODNU6/Tp0woODvbmkKqurtbq1at17tw5OZ1O5ebmqqqqSgkJCXZNz5491blzZ+Xk5EiScnJy1KdPHzsMSVJiYqLcbrd9liknJ8fjGLU1tce4nMrKSrndbo8NAAD4L68C0bBhw/TWW2/ZrwMCAlRTU6MFCxZoxIgRV3WsAwcOqE2bNgoODtbkyZO1bt06xcfHy+VyKSgoSOHh4R71kZGRcrlckiSXy+URhmrHa8euVON2u3X+/PnL9pSenq6wsDB7i42Nvao5AQCApsWr2+4XLFigkSNHat++fbpw4YJmzpypQ4cO6fTp0/r000+v6lg9evRQXl6eysrK9Ne//lUTJ05Udna2N23Vmzlz5igtLc1+7Xa7CUUAAPgxrwJR79699dVXX+mll15SaGioysvLNWbMGKWmpio6OvqqjhUUFKRu3bpJkgYMGKC9e/dq6dKluu+++3ThwgWVlpZ6nCUqLi5WVFSUpO+vZdqzZ4/H8WrvQru05od3phUXF8vhcCgkJOSyPQUHB3v91R8AAGh6rjoQVVVV6fbbb1dGRoZ+//vf13tDNTU1qqys1IABA9SiRQtt3bpVycnJkqT8/HwVFBTI6XRKkpxOp/74xz+qpKREERERkqSsrCw5HA7Fx8fbNf/93//t8RlZWVn2MQAAAK46ELVo0UL79++vlw+fM2eO7rjjDnXu3Flnz57VqlWrtH37dm3ZskVhYWFKSUlRWlqa2rVrJ4fDoccff1xOp1ODBw+WJI0aNUrx8fF68MEHtWDBArlcLj399NNKTU21z/BMnjxZL730kmbOnKmHH35Y27Zt05o1a7RxY9O7SwoAAFwfXl1UPX78eL355pvX/OElJSWaMGGCevTooZEjR2rv3r3asmWLfvWrX0mSlixZojvvvFPJyckaPny4oqKi9N5779nvb968uTZs2KDmzZvL6XRq/PjxmjBhgubNm2fXxMXFaePGjcrKylK/fv20aNEivfHGG0pMTLzm/gEAgH/w6jlEjz/+uN566y11795dAwYMqPM7zBYvXlxvDTYGPIeoLp5DBABo7K7m5/dVfWV27Ngxde3aVQcPHlT//v0lSV999ZVHTUBAwFW2CwAA4FtXFYi6d++uoqIiffzxx5K+/1Udy5Ytq/OcHwAAgKbkqq4h+uG3a5s2bdK5c+fqtSEAAICG5tVF1bW8uPwIAACg0bmqQBQQEFDnGiGuGQIAAE3dVV1DZFmWfvvb39rP+KmoqNDkyZPr3GV26a3xAAAAjd1VBaKJEyd6vB4/fny9NgMAAOALVxWIVqxYcb36AAAA8JlruqgaAADAHxCIAACA8QhEAADAeAQiAABgPAIRAAAwHoEIAAAYj0AEAACMRyACAADGIxABAADjEYgAAIDxCEQAAMB4BCIAAGA8AhEAADAegQgAABiPQAQAAIxHIAIAAMYjEAEAAOMRiAAAgPEIRAAAwHgEIgAAYDwCEQAAMB6BCAAAGI9ABAAAjEcgAgAAxiMQAQAA4xGIAACA8QhEAADAeAQiAABgPAIRAAAwHoEIAAAYj0AEAACMRyACAADGIxABAADjEYgAAIDxCEQAAMB4Pg1E6enpuvXWWxUaGqqIiAjdfffdys/P96ipqKhQamqq2rdvrzZt2ig5OVnFxcUeNQUFBUpKSlKrVq0UERGhp556ShcvXvSo2b59u/r376/g4GB169ZNmZmZ13t6AACgifBpIMrOzlZqaqp27dqlrKwsVVVVadSoUTp37pxdM336dK1fv15r165Vdna2CgsLNWbMGHu8urpaSUlJunDhgnbu3KmVK1cqMzNTc+fOtWuOHz+upKQkjRgxQnl5eZo2bZoeeeQRbdmypUHnCwAAGqcAy7IsXzdR69SpU4qIiFB2draGDx+usrIydezYUatWrdK9994rSTp8+LB69eqlnJwcDR48WJs2bdKdd96pwsJCRUZGSpIyMjI0a9YsnTp1SkFBQZo1a5Y2btyogwcP2p81duxYlZaWavPmzT/Zl9vtVlhYmMrKyuRwOK7P5JuYrrM3+rqFq/bN/CRftwAAaEBX8/O7UV1DVFZWJklq166dJCk3N1dVVVVKSEiwa3r27KnOnTsrJydHkpSTk6M+ffrYYUiSEhMT5Xa7dejQIbvm0mPU1tQe44cqKyvldrs9NgAA4L8aTSCqqanRtGnTNGTIEPXu3VuS5HK5FBQUpPDwcI/ayMhIuVwuu+bSMFQ7Xjt2pRq3263z58/X6SU9PV1hYWH2FhsbWy9zBAAAjVOjCUSpqak6ePCgVq9e7etWNGfOHJWVldnbiRMnfN0SAAC4jgJ93YAkTZkyRRs2bNCOHTvUqVMne39UVJQuXLig0tJSj7NExcXFioqKsmv27Nnjcbzau9AurfnhnWnFxcVyOBwKCQmp009wcLCCg4PrZW4AAKDx8+kZIsuyNGXKFK1bt07btm1TXFycx/iAAQPUokULbd261d6Xn5+vgoICOZ1OSZLT6dSBAwdUUlJi12RlZcnhcCg+Pt6uufQYtTW1xwAAAGbz6Rmi1NRUrVq1Sh988IFCQ0Pta37CwsIUEhKisLAwpaSkKC0tTe3atZPD4dDjjz8up9OpwYMHS5JGjRql+Ph4Pfjgg1qwYIFcLpeefvpppaam2md5Jk+erJdeekkzZ87Uww8/rG3btmnNmjXauLHp3SkFAADqn0/PEC1fvlxlZWX65S9/qejoaHt799137ZolS5bozjvvVHJysoYPH66oqCi999579njz5s21YcMGNW/eXE6nU+PHj9eECRM0b948uyYuLk4bN25UVlaW+vXrp0WLFumNN95QYmJig84XAAA0To3qOUSNFc8hqovnEAEAGrsm+xwiAAAAXyAQAQAA4xGIAACA8RrFc4iAhsB1TwCAH8MZIgAAYDwCEQAAMB6BCAAAGI9ABAAAjEcgAgAAxiMQAQAA4xGIAACA8QhEAADAeAQiAABgPAIRAAAwHoEIAAAYj0AEAACMRyACAADGIxABAADjEYgAAIDxCEQAAMB4BCIAAGA8AhEAADAegQgAABiPQAQAAIxHIAIAAMYjEAEAAOMRiAAAgPEIRAAAwHgEIgAAYDwCEQAAMB6BCAAAGI9ABAAAjEcgAgAAxiMQAQAA4xGIAACA8QhEAADAeAQiAABgPAIRAAAwHoEIAAAYj0AEAACMRyACAADGIxABAADj+TQQ7dixQ7/+9a8VExOjgIAAvf/++x7jlmVp7ty5io6OVkhIiBISEnTkyBGPmtOnT2vcuHFyOBwKDw9XSkqKysvLPWr279+vYcOGqWXLloqNjdWCBQuu99QAAEAT4tNAdO7cOfXr108vv/zyZccXLFigZcuWKSMjQ7t371br1q2VmJioiooKu2bcuHE6dOiQsrKytGHDBu3YsUOTJk2yx91ut0aNGqUuXbooNzdXCxcu1LPPPqvXXnvtus8PAAA0DQGWZVm+bkKSAgICtG7dOt19992Svj87FBMToyeffFIzZsyQJJWVlSkyMlKZmZkaO3asvvzyS8XHx2vv3r0aOHCgJGnz5s0aPXq0Tp48qZiYGC1fvly///3v5XK5FBQUJEmaPXu23n//fR0+fPhn9eZ2uxUWFqaysjI5HI76n3wT1HX2Rl+3YIRv5if5ugUAaLKu5ud3o72G6Pjx43K5XEpISLD3hYWFadCgQcrJyZEk5eTkKDw83A5DkpSQkKBmzZpp9+7dds3w4cPtMCRJiYmJys/P15kzZy772ZWVlXK73R4bAADwX402ELlcLklSZGSkx/7IyEh7zOVyKSIiwmM8MDBQ7dq186i53DEu/YwfSk9PV1hYmL3FxsZe+4QAAECj1WgDkS/NmTNHZWVl9nbixAlftwQAAK6jRhuIoqKiJEnFxcUe+4uLi+2xqKgolZSUeIxfvHhRp0+f9qi53DEu/YwfCg4OlsPh8NgAAID/arSBKC4uTlFRUdq6dau9z+12a/fu3XI6nZIkp9Op0tJS5ebm2jXbtm1TTU2NBg0aZNfs2LFDVVVVdk1WVpZ69Oihtm3bNtBsAABAY+bTQFReXq68vDzl5eVJ+v5C6ry8PBUUFCggIEDTpk3T888/rw8//FAHDhzQhAkTFBMTY9+J1qtXL91+++169NFHtWfPHn366aeaMmWKxo4dq5iYGEnSAw88oKCgIKWkpOjQoUN69913tXTpUqWlpflo1gAAoLEJ9OWH79u3TyNGjLBf14aUiRMnKjMzUzNnztS5c+c0adIklZaWaujQodq8ebNatmxpv+edd97RlClTNHLkSDVr1kzJyclatmyZPR4WFqaPPvpIqampGjBggDp06KC5c+d6PKsIAACYrdE8h6gx4zlEdfEcoobBc4gAwHt+8RwiAACAhkIgAgAAxiMQAQAA4xGIAACA8QhEAADAeAQiAABgPJ8+hwjf4xZ2AAB8izNEAADAeAQiAABgPAIRAAAwHoEIAAAYj0AEAACMRyACAADGIxABAADjEYgAAIDxCEQAAMB4BCIAAGA8AhEAADAegQgAABiPQAQAAIxHIAIAAMYjEAEAAOMRiAAAgPEIRAAAwHgEIgAAYDwCEQAAMB6BCAAAGI9ABAAAjEcgAgAAxiMQAQAA4xGIAACA8QhEAADAeAQiAABgPAIRAAAwHoEIAAAYj0AEAACMRyACAADGIxABAADjEYgAAIDxCEQAAMB4BCIAAGC8QF83AAAA6lfX2Rt93cJV+2Z+kk8/nzNEAADAeEYFopdfflldu3ZVy5YtNWjQIO3Zs8fXLQEAgEbAmED07rvvKi0tTc8884w+++wz9evXT4mJiSopKfF1awAAwMeMuYZo8eLFevTRR/XQQw9JkjIyMrRx40b9+c9/1uzZs33cHXB5XAcAAA3DiEB04cIF5ebmas6cOfa+Zs2aKSEhQTk5OXXqKysrVVlZab8uKyuTJLnd7uvSX03ld9fluIAvXK+/JwB+vqb4c+V6/Lej9piWZf1krRGB6H//939VXV2tyMhIj/2RkZE6fPhwnfr09HT94Q9/qLM/Njb2uvUI+IuwF33dAYCm6Hr+t+Ps2bMKCwu7Yo0RgehqzZkzR2lpafbrmpoanT59Wu3bt1dAQIAPO7syt9ut2NhYnThxQg6Hw9ftNAjmzJz9kWnzlZizCXP2xXwty9LZs2cVExPzk7VGBKIOHTqoefPmKi4u9thfXFysqKioOvXBwcEKDg722BceHn49W6xXDofDiL9cl2LOZjBtzqbNV2LOJmjo+f7UmaFaRtxlFhQUpAEDBmjr1q32vpqaGm3dulVOp9OHnQEAgMbAiDNEkpSWlqaJEydq4MCB+td//Ve9+OKLOnfunH3XGQAAMJcxgei+++7TqVOnNHfuXLlcLt18883avHlznQutm7Lg4GA988wzdb7u82fM2Qymzdm0+UrM2QSNfb4B1s+5Fw0AAMCPGXENEQAAwJUQiAAAgPEIRAAAwHgEIgAAYDwCUROUnp6uW2+9VaGhoYqIiNDdd9+t/Px8j5pf/vKXCggI8NgmT57so46vzfLly9W3b1/7YV5Op1ObNm2yxysqKpSamqr27durTZs2Sk5OrvMQzqbmp+bsT+v7Y+bPn6+AgABNmzbN3uePa13rcvP1t3V+9tln68ynZ8+e9rg/ru9Pzdnf1rjWt99+q/Hjx6t9+/YKCQlRnz59tG/fPnvcsizNnTtX0dHRCgkJUUJCgo4cOeLDjglETVJ2drZSU1O1a9cuZWVlqaqqSqNGjdK5c+c86h599FEVFRXZ24IFC3zU8bXp1KmT5s+fr9zcXO3bt0+33Xab7rrrLh06dEiSNH36dK1fv15r165Vdna2CgsLNWbMGB93fW1+as6S/6zv5ezdu1evvvqq+vbt67HfH9da+vH5Sv63zjfddJPHfD755BN7zF/X90pzlvxvjc+cOaMhQ4aoRYsW2rRpk7744gstWrRIbdu2tWsWLFigZcuWKSMjQ7t371br1q2VmJioiooK3zVuockrKSmxJFnZ2dn2vn/7t3+zpk6d6rumrrO2bdtab7zxhlVaWmq1aNHCWrt2rT325ZdfWpKsnJwcH3ZY/2rnbFn+vb5nz561unfvbmVlZXnM01/X+sfma1n+t87PPPOM1a9fv8uO+ev6XmnOluV/a2xZljVr1ixr6NChPzpeU1NjRUVFWQsXLrT3lZaWWsHBwdZf/vKXhmjxsjhD5AfKysokSe3atfPY/84776hDhw7q3bu35syZo++++84X7dWr6upqrV69WufOnZPT6VRubq6qqqqUkJBg1/Ts2VOdO3dWTk6ODzutPz+ccy1/XF9JSk1NVVJSkseaSvLbtf6x+dbyt3U+cuSIYmJidMMNN2jcuHEqKCiQ5L/rK/34nGv52xp/+OGHGjhwoH7zm98oIiJCt9xyi15//XV7/Pjx43K5XB5rHRYWpkGDBvl0rY15UrW/qqmp0bRp0zRkyBD17t3b3v/AAw+oS5cuiomJ0f79+zVr1izl5+frvffe82G33jtw4ICcTqcqKirUpk0brVu3TvHx8crLy1NQUFCdX74bGRkpl8vlm2bryY/NWfK/9a21evVqffbZZ9q7d2+dMZfL5XdrfaX5Sv63zoMGDVJmZqZ69OihoqIi/eEPf9CwYcN08OBBv1xf6cpzDg0N9bs1lqRjx45p+fLlSktL03/8x39o7969euKJJxQUFKSJEyfa6/nD3xTh67UmEDVxqampOnjwYJ3vpCdNmmT/uU+fPoqOjtbIkSP19ddf6xe/+EVDt3nNevTooby8PJWVlemvf/2rJk6cqOzsbF+3dV392Jzj4+P9bn0l6cSJE5o6daqysrLUsmVLX7dz3f2c+frbOt9xxx32n/v27atBgwapS5cuWrNmjUJCQnzY2fVzpTmnpKT43RpL3/+P+sCBA/XCCy9Ikm655RYdPHhQGRkZmjhxoo+7+3F8ZdaETZkyRRs2bNDHH3+sTp06XbF20KBBkqSjR482RGv1LigoSN26ddOAAQOUnp6ufv36aenSpYqKitKFCxdUWlrqUV9cXKyoqCjfNFtPfmzOl9PU11f6/iuTkpIS9e/fX4GBgQoMDFR2draWLVumwMBARUZG+tVa/9R8q6ur67zHH9b5UuHh4brxxht19OhRv/67fKlL53w5/rDG0dHR9tnsWr169bK/Kqxdzx/eQejrtSYQNUGWZWnKlClat26dtm3bpri4uJ98T15enqTv/0X1BzU1NaqsrNSAAQPUokULbd261R7Lz89XQUGBx/U2/qB2zpfjD+s7cuRIHThwQHl5efY2cOBAjRs3zv6zP631T823efPmdd7jD+t8qfLycn399deKjo425u/ypXO+HH9Y4yFDhtR5FMxXX32lLl26SJLi4uIUFRXlsdZut1u7d+/27Vr77HJueO2xxx6zwsLCrO3bt1tFRUX29t1331mWZVlHjx615s2bZ+3bt886fvy49cEHH1g33HCDNXz4cB937p3Zs2db2dnZ1vHjx639+/dbs2fPtgICAqyPPvrIsizLmjx5stW5c2dr27Zt1r59+yyn02k5nU4fd31trjRnf1vfK/nhHTj+uNaXunS+/rjOTz75pLV9+3br+PHj1qeffmolJCRYHTp0sEpKSizL8s/1vdKc/XGNLcuy9uzZYwUGBlp//OMfrSNHjljvvPOO1apVK+vtt9+2a+bPn2+Fh4dbH3zwgbV//37rrrvusuLi4qzz58/7rG8CURMk6bLbihUrLMuyrIKCAmv48OFWu3btrODgYKtbt27WU089ZZWVlfm2cS89/PDDVpcuXaygoCCrY8eO1siRI+0wZFmWdf78eet3v/ud1bZtW6tVq1bWPffcYxUVFfmw42t3pTn72/peyQ8DkT+u9aUuna8/rvN9991nRUdHW0FBQda//Mu/WPfdd5919OhRe9wf1/dKc/bHNa61fv16q3fv3lZwcLDVs2dP67XXXvMYr6mpsf7zP//TioyMtIKDg62RI0da+fn5Pur2ewGWZVm+Oz8FAADge1xDBAAAjEcgAgAAxiMQAQAA4xGIAACA8QhEAADAeAQiAABgPAIRAAAwHoEIAAAYj0AEAACMRyACAADGIxABAADjEYgAAIDx/g+jeRoM2MvdtgAAAABJRU5ErkJggg==\n" + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "kwds_rake" + "wos_nlp['vector_norm'].plot(kind=\"hist\")" ], "metadata": { "collapsed": false, @@ -330,19 +622,20 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 85, "outputs": [ { "data": { - "text/plain": "0 None\n9714 None\n9697 None\n9699 None\n9701 None\n ... \n3066 None\n5097 None\n11369 None\n11368 None\n11362 None\nName: Abstract, Length: 9889, dtype: object" + "text/plain": " UT (Unique WOS ID) Article Title \n79 WOS:000852230800001 Deep Learning-Based Object Tracking in Satelli... \\\n87 WOS:000732189300001 Hyperspectral Anomaly Detection: A Survey \n248 WOS:000446451700003 New Frontiers in Spectral-Spatial Hyperspectra... \n276 WOS:000728108300001 IEEE ACCESS SPECIAL SECTION EDITORIAL: BIG DAT... \n365 WOS:000376531500004 Future Perspectives and Challenges of Fungal S... \n... ... ... \n8855 WOS:000541900700064 Statistical Machine Learning for Human Behavio... \n8869 WOS:000756384800001 Editorial: Artificial Intelligence in Positron... \n8891 WOS:000638348600001 Advancing Grid-Connected Renewable Generation ... \n8918 WOS:000859103100001 Editorial: Systems biology approach for the me... \n8925 WOS:000885247700001 Editorial: Investigation of brain functional c... \n\n Abstract keyword_all Document \n79 NaN FEATURE EXTRACTION; TRANSFORMERS; OBJECT TRACK... NaN \\\n87 NaN ANOMALY DETECTION; HYPERSPECTRAL IMAGING; PRIN... NaN \n248 NaN LOOPY BELIEF PROPAGATION; EXTINCTION PROFILES;... NaN \n276 NaN FEATURE-SELECTION; NEURAL-NETWORKS; STOCK-MARK... NaN \n365 NaN INTERNAL TRANSCRIBED SPACER; ASPERGILLUS-NIGER... NaN \n... ... ... ... \n8855 NaN ACTION RECOGNITION; EMOTION RECOGNITION; PRIVA... NaN \n8869 NaN ARTIFICIAL INTELLIGENCE; MOLECULAR IMAGING; PO... NaN \n8891 NaN RENEWABLE ENERGY SOURCES ; POWER QUALITY; VIRT... NaN \n8918 NaN CHRONIC LIVER DISEASE; OMICS; SYSTEMATIC BIOLO... NaN \n8925 NaN BRAIN; FUNCTIONAL CONNECTIVITY; EEG; ELECTROEN... NaN \n\n vector vector_norm \n79 [0.53393, -0.6493, -3.1156, -0.04664, -2.6227,... 60.533962 \n87 [0.53393, -0.6493, -3.1156, -0.04664, -2.6227,... 60.533962 \n248 [0.53393, -0.6493, -3.1156, -0.04664, -2.6227,... 60.533962 \n276 [0.53393, -0.6493, -3.1156, -0.04664, -2.6227,... 60.533962 \n365 [0.53393, -0.6493, -3.1156, -0.04664, -2.6227,... 60.533962 \n... ... ... \n8855 [0.53393, -0.6493, -3.1156, -0.04664, -2.6227,... 60.533962 \n8869 [0.53393, -0.6493, -3.1156, -0.04664, -2.6227,... 60.533962 \n8891 [0.53393, -0.6493, -3.1156, -0.04664, -2.6227,... 60.533962 \n8918 [0.53393, -0.6493, -3.1156, -0.04664, -2.6227,... 60.533962 \n8925 [0.53393, -0.6493, -3.1156, -0.04664, -2.6227,... 60.533962 \n\n[121 rows x 7 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
UT (Unique WOS ID)Article TitleAbstractkeyword_allDocumentvectorvector_norm
79WOS:000852230800001Deep Learning-Based Object Tracking in Satelli...NaNFEATURE EXTRACTION; TRANSFORMERS; OBJECT TRACK...NaN[0.53393, -0.6493, -3.1156, -0.04664, -2.6227,...60.533962
87WOS:000732189300001Hyperspectral Anomaly Detection: A SurveyNaNANOMALY DETECTION; HYPERSPECTRAL IMAGING; PRIN...NaN[0.53393, -0.6493, -3.1156, -0.04664, -2.6227,...60.533962
248WOS:000446451700003New Frontiers in Spectral-Spatial Hyperspectra...NaNLOOPY BELIEF PROPAGATION; EXTINCTION PROFILES;...NaN[0.53393, -0.6493, -3.1156, -0.04664, -2.6227,...60.533962
276WOS:000728108300001IEEE ACCESS SPECIAL SECTION EDITORIAL: BIG DAT...NaNFEATURE-SELECTION; NEURAL-NETWORKS; STOCK-MARK...NaN[0.53393, -0.6493, -3.1156, -0.04664, -2.6227,...60.533962
365WOS:000376531500004Future Perspectives and Challenges of Fungal S...NaNINTERNAL TRANSCRIBED SPACER; ASPERGILLUS-NIGER...NaN[0.53393, -0.6493, -3.1156, -0.04664, -2.6227,...60.533962
........................
8855WOS:000541900700064Statistical Machine Learning for Human Behavio...NaNACTION RECOGNITION; EMOTION RECOGNITION; PRIVA...NaN[0.53393, -0.6493, -3.1156, -0.04664, -2.6227,...60.533962
8869WOS:000756384800001Editorial: Artificial Intelligence in Positron...NaNARTIFICIAL INTELLIGENCE; MOLECULAR IMAGING; PO...NaN[0.53393, -0.6493, -3.1156, -0.04664, -2.6227,...60.533962
8891WOS:000638348600001Advancing Grid-Connected Renewable Generation ...NaNRENEWABLE ENERGY SOURCES ; POWER QUALITY; VIRT...NaN[0.53393, -0.6493, -3.1156, -0.04664, -2.6227,...60.533962
8918WOS:000859103100001Editorial: Systems biology approach for the me...NaNCHRONIC LIVER DISEASE; OMICS; SYSTEMATIC BIOLO...NaN[0.53393, -0.6493, -3.1156, -0.04664, -2.6227,...60.533962
8925WOS:000885247700001Editorial: Investigation of brain functional c...NaNBRAIN; FUNCTIONAL CONNECTIVITY; EEG; ELECTROEN...NaN[0.53393, -0.6493, -3.1156, -0.04664, -2.6227,...60.533962
\n

121 rows × 7 columns

\n
" }, - "execution_count": 28, + "execution_count": 85, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "kwds_rake" + "wos_nlp[wos_nlp['vector_norm']>50]" ], "metadata": { "collapsed": false, @@ -353,18 +646,20 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 77, "outputs": [ { "data": { - "text/plain": "'Keywords Plus'" + "text/plain": "array([[-1.8670139 , -1.6925758 , 0.48349068, ..., -2.2703056 ,\n -1.4200605 , 0.46284062],\n [-1.7312453 , -0.4499114 , -0.54250187, ..., -2.2334094 ,\n -1.3671577 , 0.7283594 ],\n [-2.3378334 , -0.424522 , -0.82274777, ..., -2.2994597 ,\n -1.5955478 , 0.26363412],\n ...,\n [-2.3435452 , -0.34531432, -1.1484123 , ..., -2.3003943 ,\n -1.8553756 , -0.31917948],\n [-2.8046715 , -1.7071993 , 0.55096555, ..., -2.709951 ,\n -0.621031 , 0.46265596],\n [-2.2533355 , -0.8899313 , 0.08667578, ..., -2.5975435 ,\n -1.0989579 , 1.2003326 ]], dtype=float32)" }, - "execution_count": 19, + "execution_count": 77, "metadata": {}, "output_type": "execute_result" } ], - "source": [], + "source": [ + "np.array(wos_nlp[\"vector\"].to_list())" + ], "metadata": { "collapsed": false, "pycharm": {