ZSI_Reconnect_China/WOS/wos_extract/wos_query_generator_simplesyntax.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "focal_countries_list = [\"Peoples R china\", \"Hong Kong\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "outputs": [],
   "source": [
    "country_mode = \"CU\" #CU-country-region AU-address"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "outputs": [],
   "source": [
    "# (TS=(\"artificial intelligence\") OR TS=(\"machine learning\") OR TS=(\"neural network\") OR TS=(\"big data\") OR TS=(\"deep learning\") OR TS=(\"computer vision\") OR TS=(\"pattern recognition\")) AND"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "outputs": [],
   "source": [
    "keywords_source = r'..\\ai_scope_keywords.txt'\n",
    "with open(keywords_source,'r') as f:\n",
    "    keywords = \"\\n\".join(f.readlines()).replace('\\n','')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "outputs": [],
   "source": [
    "keywords_source_suggest = r'..\\ai_scope_keywords_suggestions.txt'\n",
    "with open(keywords_source_suggest,'r') as f:\n",
    "    keywords_suggest = \"\\n\".join(f.readlines()).replace('\\n','')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "outputs": [],
   "source": [
    "keywords_oklist_source= r'..\\kw_token_ranked_bibliometrics_okset.xlsx'\n",
    "keyword_df = pd.read_excel(keywords_oklist_source)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "outputs": [
    {
     "data": {
      "text/plain": "'neural network*,machine* learn*,deep learn*,clustering,remote sensing,convolutional neural,Internet of Things,feature extraction,genetic algorithm*,big data*,artificial intelligence*,data driven*,support vector machine*,classifier,logistic regression &! p=,optimization algorithm*,principal component analysis,artificial neural network*,swarm optimization,regularization,linear regression &! p=,optimization algorithm,random forest,cloud computing,reinforcement learning,computer vision,kalman filter*,image processing,data mining,evolutionary algorithm*,edge computing,*supervised learning,computational modeling,pattern recognition,image classification,long short-term memor*,robotics,image segmentation,convex optimization,covariance matri*,attention mechanism*,markov chain,object detection &! brain,clustering algorithm*,recurrent neural network*,data augmentation,transfer learning,loss function*,adversarial network*,decision tree*,multi agent system*,fuzzy set*,convolutional network*,image reconstruction,data* analytic*,smart grid,autoencoder*,fuzzy logic,radial basis function,Bayesian network*,dimensionality reduction,face recognition &! brain,gaussian process,anomaly detection,k-nearest neighbor*,natural language processing,monte carlo method,large$ dataset*,gradient descent,support vector regression,extreme learning machine*,perceptron*,model selection,ensemble learning,representation learning,recommender system*,target tracking,singular value decomposition,KNN,feature learning,smart city,sentiment analy*,markov decision process,k-means clustering,independent component analysis,brain computer interface,human-computer interaction,markov chain monte carlo,hierarchical clustering,semantic web*,semi-supervised learning,human-robot interact*,knowledge graph*,speech recognition &! brain,ensemble model*,fog computing,map$reduce,evolutionary computation*,data science*,text mining,generative model*,active learning,swarm intelligence,multi-task learning,language model*,collaborative filtering,backpropagation,machine vision,computer-aided diagnosis,gated recurrent unit*,lagrange multiplier,expert system*,learning rate*,hadoop*,markov process,nonlinear optimization,learning system,self-organizing map*,smart manufacturing,smart home,few shot learning,few-shot learning,meta-learning,meta learning,adversarial training,zero-shot learning,word embedding*,expectation maximization algorithm*,stochastic gradient descent,ridge regression,deep belief network*,non-negative matrix factorization,affective computing,latent dirichlet allocation,kernel method,kernel learning,feature engineering,variational inference,image representation,manifold learning,t5,adversarial example*,knowledge distillation,time series forecast*,variational autoencoder*,lasso regression,smart energy,dbscan,multi-label classification,intelligent robot*,ubiquitous computing,gaussian mixture models,smart technolog*,boltzmann machine*,smart buildings,predictive analytic*,pervasive computing,smart agriculture,capsule network*,human-in-the-loop,intelligent agent*,ai applications,word vector*,transformer model*,facial recognition,unstructured data*,restricted boltzmann machine*,albert,lifelong learning,autonomous agents,chatbot*,Cholesky decomposition,no$sql,nosql,explainable AI,seq2seq,probabilistic graphical model*,QR decomposition,L? regulari*,unsupervised deep learning,data warehouse*,quantum machine learning,continual learning,smart environment,multimodal learning,smart health,artificial immune system*,swarm robotics,kernel machine*,latent factor model*,eigendecomposition,adversarial machine,adversarial machine learning,smart mobility,sequence-to-sequence model*,eigen decomposition,adversarial robustness,smart parking,adversarial neural,roberta,bidirectional encoder representations from transformer*,locally linear embedding*,Hebbian learning,one-shot learning,multimodal representation,smart tourism,entity extraction,adaptive moment estimation,ontology learning,topic modeling*'"
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keywords = \",\".join(keyword_df[keyword_df[\"u_Priority (done)\"].isin([\"High\",\"Medium\"])][\"kw_token\"].str.replace('\"','').tolist()).replace(\"NOT\",\"&!\")\n",
    "keywords"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "outputs": [],
   "source": [
    "keywords = [c.strip() for c in keywords.split(\",\")]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "outputs": [],
   "source": [
    "def wos_kw_formatter(text):\n",
    "    if \"&!\" in text:\n",
    "        return('('+' NOT '.join('\\\"'+sub_text.strip()+'\\\"' for sub_text in text.split('&!'))+')')\n",
    "    else:\n",
    "        return '\\\"'+text+'\\\"'\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "outputs": [
    {
     "data": {
      "text/plain": "'\"neural network*\" OR \"machine* learn*\" OR \"deep learn*\" OR \"clustering\" OR \"remote sensing\" OR \"convolutional neural\" OR \"Internet of Things\" OR \"feature extraction\" OR \"genetic algorithm*\" OR \"big data*\" OR \"artificial intelligence*\" OR \"data driven*\" OR \"support vector machine*\" OR \"classifier\" OR (\"logistic regression\" NOT \"p=\") OR \"optimization algorithm*\" OR \"principal component analysis\" OR \"artificial neural network*\" OR \"swarm optimization\" OR \"regularization\" OR (\"linear regression\" NOT \"p=\") OR \"optimization algorithm\" OR \"random forest\" OR \"cloud computing\" OR \"reinforcement learning\" OR \"computer vision\" OR \"kalman filter*\" OR \"image processing\" OR \"data mining\" OR \"evolutionary algorithm*\" OR \"edge computing\" OR \"*supervised learning\" OR \"computational modeling\" OR \"pattern recognition\" OR \"image classification\" OR \"long short-term memor*\" OR \"robotics\" OR \"image segmentation\" OR \"convex optimization\" OR \"covariance matri*\" OR \"attention mechanism*\" OR \"markov chain\" OR (\"object detection\" NOT \"brain\") OR \"clustering algorithm*\" OR \"recurrent neural network*\" OR \"data augmentation\" OR \"transfer learning\" OR \"loss function*\" OR \"adversarial network*\" OR \"decision tree*\" OR \"multi agent system*\" OR \"fuzzy set*\" OR \"convolutional network*\" OR \"image reconstruction\" OR \"data* analytic*\" OR \"smart grid\" OR \"autoencoder*\" OR \"fuzzy logic\" OR \"radial basis function\" OR \"Bayesian network*\" OR \"dimensionality reduction\" OR (\"face recognition\" NOT \"brain\") OR \"gaussian process\" OR \"anomaly detection\" OR \"k-nearest neighbor*\" OR \"natural language processing\" OR \"monte carlo method\" OR \"large$ dataset*\" OR \"gradient descent\" OR \"support vector regression\" OR \"extreme learning machine*\" OR \"perceptron*\" OR \"model selection\" OR \"ensemble learning\" OR \"representation learning\" OR \"recommender system*\" OR \"target tracking\" OR \"singular value decomposition\" OR \"KNN\" OR \"feature learning\" OR \"smart city\" OR \"sentiment analy*\" OR \"markov decision process\" OR \"k-means clustering\" OR \"independent component analysis\" OR \"brain computer interface\" OR \"human-computer interaction\" OR \"markov chain monte carlo\" OR \"hierarchical clustering\" OR \"semantic web*\" OR \"semi-supervised learning\" OR \"human-robot interact*\" OR \"knowledge graph*\" OR (\"speech recognition\" NOT \"brain\") OR \"ensemble model*\" OR \"fog computing\" OR \"map$reduce\" OR \"evolutionary computation*\" OR \"data science*\" OR \"text mining\" OR \"generative model*\" OR \"active learning\" OR \"swarm intelligence\" OR \"multi-task learning\" OR \"language model*\" OR \"collaborative filtering\" OR \"backpropagation\" OR \"machine vision\" OR \"computer-aided diagnosis\" OR \"gated recurrent unit*\" OR \"lagrange multiplier\" OR \"expert system*\" OR \"learning rate*\" OR \"hadoop*\" OR \"markov process\" OR \"nonlinear optimization\" OR \"learning system\" OR \"self-organizing map*\" OR \"smart manufacturing\" OR \"smart home\" OR \"few shot learning\" OR \"few-shot learning\" OR \"meta-learning\" OR \"meta learning\" OR \"adversarial training\" OR \"zero-shot learning\" OR \"word embedding*\" OR \"expectation maximization algorithm*\" OR \"stochastic gradient descent\" OR \"ridge regression\" OR \"deep belief network*\" OR \"non-negative matrix factorization\" OR \"affective computing\" OR \"latent dirichlet allocation\" OR \"kernel method\" OR \"kernel learning\" OR \"feature engineering\" OR \"variational inference\" OR \"image representation\" OR \"manifold learning\" OR \"t5\" OR \"adversarial example*\" OR \"knowledge distillation\" OR \"time series forecast*\" OR \"variational autoencoder*\" OR \"lasso regression\" OR \"smart energy\" OR \"dbscan\" OR \"multi-label classification\" OR \"intelligent robot*\" OR \"ubiquitous computing\" OR \"gaussian mixture models\" OR \"smart technolog*\" OR \"boltzmann machine*\" OR \"smart buildings\" OR \"predi
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keywords_str = ' OR '.join(wos_kw_formatter(k) for k in keywords)\n",
    "keywords_str"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "outputs": [
    {
     "data": {
      "text/plain": "210"
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(keywords)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "outputs": [],
   "source": [
    "scope_country_source = r'..\\eu_scope_countries.txt'\n",
    "\n",
    "with open(scope_country_source,'r') as f:\n",
    "    coop_countries = f.readlines()\n",
    "coop_countries = [c.strip().upper() for c in coop_countries[0].split(\",\")]\n",
    "focal_countries = [c.strip().upper() for c in focal_countries_list]\n",
    "eu_countries = coop_countries[0:-7]\n",
    "assoc_countries = coop_countries[-7:]\n",
    "\n",
    "nor_c = [coop_countries[-7],]\n",
    "swi_c = [coop_countries[-6],]\n",
    "uk_c = coop_countries[-5:]\n",
    "\n",
    "foc_str = ' OR '.join([c for c in focal_countries])\n",
    "coop_str = ' OR '.join([c for c in coop_countries])\n",
    "eu_str = ' OR '.join([c for c in eu_countries])\n",
    "assoc_str = ' OR '.join([c for c in assoc_countries])\n",
    "\n",
    "nor_str =' OR '.join([c for c in nor_c])\n",
    "swi_str =' OR '.join([c for c in swi_c])\n",
    "uk_str =' OR '.join([c for c in uk_c])\n",
    "eu_sub_str = eu_str.split(' OR ')\n",
    "# eu_sub_str"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "outputs": [
    {
     "data": {
      "text/plain": "'AUSTRIA OR BELGIUM OR BULGARIA OR CROATIA OR CYPRUS OR CZECH REPUBLIC OR DENMARK OR ESTONIA OR FINLAND OR FRANCE OR GERMANY OR GREECE OR HUNGARY OR IRELAND OR ITALY OR LATVIA OR LITHUANIA OR LUXEMBOURG OR MALTA OR NETHERLANDS OR POLAND OR PORTUGAL OR ROMANIA OR SLOVAKIA OR SLOVENIA OR SPAIN OR SWEDEN OR NORWAY OR SWITZERLAND OR UNITED KINGDOM OR ENGLAND OR WALES OR SCOTLAND OR N IRELAND'"
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eu_assoc = ' OR '.join([eu_str,nor_str,swi_str,uk_str])\n",
    "eu_assoc"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "outputs": [
    {
     "data": {
      "text/plain": "'PEOPLES R CHINA OR HONG KONG'"
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "foc_str"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "outputs": [],
   "source": [
    "p_regex='(\"p\"  NEAR/0 \"0.0*\" OR \"p$value*\" OR \"p-value*\"))'"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "outputs": [],
   "source": [
    "scope_query = f'CU=({foc_str}) AND CU=({eu_assoc}) AND TS=({keywords_str}) AND PY=(2011-2022)'\n",
    "scope_query = scope_query.replace('\"p=\")',p_regex).replace('OR \"momentum\"','')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "outputs": [
    {
     "data": {
      "text/plain": "'CU=(PEOPLES R CHINA OR HONG KONG) AND CU=(AUSTRIA OR BELGIUM OR BULGARIA OR CROATIA OR CYPRUS OR CZECH REPUBLIC OR DENMARK OR ESTONIA OR FINLAND OR FRANCE OR GERMANY OR GREECE OR HUNGARY OR IRELAND OR ITALY OR LATVIA OR LITHUANIA OR LUXEMBOURG OR MALTA OR NETHERLANDS OR POLAND OR PORTUGAL OR ROMANIA OR SLOVAKIA OR SLOVENIA OR SPAIN OR SWEDEN OR NORWAY OR SWITZERLAND OR UNITED KINGDOM OR ENGLAND OR WALES OR SCOTLAND OR N IRELAND) AND TS=(\"neural network*\" OR \"machine* learn*\" OR \"deep learn*\" OR \"clustering\" OR \"remote sensing\" OR \"convolutional neural\" OR \"Internet of Things\" OR \"feature extraction\" OR \"genetic algorithm*\" OR \"big data*\" OR \"artificial intelligence*\" OR \"data driven*\" OR \"support vector machine*\" OR \"classifier\" OR (\"logistic regression\" NOT (\"p\"  NEAR/0 \"0.0*\" OR \"p$value*\" OR \"p-value*\")) OR \"optimization algorithm*\" OR \"principal component analysis\" OR \"artificial neural network*\" OR \"swarm optimization\" OR \"regularization\" OR (\"linear regression\" NOT (\"p\"  NEAR/0 \"0.0*\" OR \"p$value*\" OR \"p-value*\")) OR \"optimization algorithm\" OR \"random forest\" OR \"cloud computing\" OR \"reinforcement learning\" OR \"computer vision\" OR \"kalman filter*\" OR \"image processing\" OR \"data mining\" OR \"evolutionary algorithm*\" OR \"edge computing\" OR \"*supervised learning\" OR \"computational modeling\" OR \"pattern recognition\" OR \"image classification\" OR \"long short-term memor*\" OR \"robotics\" OR \"image segmentation\" OR \"convex optimization\" OR \"covariance matri*\" OR \"attention mechanism*\" OR \"markov chain\" OR (\"object detection\" NOT \"brain\") OR \"clustering algorithm*\" OR \"recurrent neural network*\" OR \"data augmentation\" OR \"transfer learning\" OR \"loss function*\" OR \"adversarial network*\" OR \"decision tree*\" OR \"multi agent system*\" OR \"fuzzy set*\" OR \"convolutional network*\" OR \"image reconstruction\" OR \"data* analytic*\" OR \"smart grid\" OR \"autoencoder*\" OR \"fuzzy logic\" OR \"radial basis function\" OR \"Bayesian network*\" OR \"dimensionality reduction\" OR (\"face recognition\" NOT \"brain\") OR \"gaussian process\" OR \"anomaly detection\" OR \"k-nearest neighbor*\" OR \"natural language processing\" OR \"monte carlo method\" OR \"large$ dataset*\" OR \"gradient descent\" OR \"support vector regression\" OR \"extreme learning machine*\" OR \"perceptron*\" OR \"model selection\" OR \"ensemble learning\" OR \"representation learning\" OR \"recommender system*\" OR \"target tracking\" OR \"singular value decomposition\" OR \"KNN\" OR \"feature learning\" OR \"smart city\" OR \"sentiment analy*\" OR \"markov decision process\" OR \"k-means clustering\" OR \"independent component analysis\" OR \"brain computer interface\" OR \"human-computer interaction\" OR \"markov chain monte carlo\" OR \"hierarchical clustering\" OR \"semantic web*\" OR \"semi-supervised learning\" OR \"human-robot interact*\" OR \"knowledge graph*\" OR (\"speech recognition\" NOT \"brain\") OR \"ensemble model*\" OR \"fog computing\" OR \"map$reduce\" OR \"evolutionary computation*\" OR \"data science*\" OR \"text mining\" OR \"generative model*\" OR \"active learning\" OR \"swarm intelligence\" OR \"multi-task learning\" OR \"language model*\" OR \"collaborative filtering\" OR \"backpropagation\" OR \"machine vision\" OR \"computer-aided diagnosis\" OR \"gated recurrent unit*\" OR \"lagrange multiplier\" OR \"expert system*\" OR \"learning rate*\" OR \"hadoop*\" OR \"markov process\" OR \"nonlinear optimization\" OR \"learning system\" OR \"self-organizing map*\" OR \"smart manufacturing\" OR \"smart home\" OR \"few shot learning\" OR \"few-shot learning\" OR \"meta-learning\" OR \"meta learning\" OR \"adversarial training\" OR \"zero-shot learning\" OR \"word embedding*\" OR \"expectation maximization algorithm*\" OR \"stochastic gradient descent\" OR \"ridge regression\" OR \"deep belief network*\" OR \"non-negative matrix factorization\" OR \"affective computing\" OR \"latent dirichlet allocation\" O
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scope_query"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "outputs": [],
   "source": [
    "from wossel_miners import wos_fetch_entries,wos_fetch_yearly_output"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hoooold...\n",
      "57534 records found! Here we go in 192 steps...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 191/191 [24:55<00:00,  7.83s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "final batch of 57301-57534\n"
     ]
    }
   ],
   "source": [
    "wos_fetch_entries(query_str=scope_query)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "outputs": [],
   "source": [
    "keywords_sub_str = keywords_str.split(' OR ')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "outputs": [
    {
     "data": {
      "text/plain": "['\"kalman filter*\"',\n '\"target tracking\"',\n '\"learning rate*\"',\n '\"covariance matri*\"',\n '\"loss function*\"']"
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# keywords_sub_str[343:]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('TS=(\"kalman filter*\") AND PY=(2011-2022)', 'CU=(PEOPLES R CHINA OR HONG KONG) AND CU=(AUSTRIA OR BELGIUM OR BULGARIA OR CROATIA OR CYPRUS OR CZECH REPUBLIC OR DENMARK OR ESTONIA OR FINLAND OR FRANCE OR GERMANY OR GREECE OR HUNGARY OR IRELAND OR ITALY OR LATVIA OR LITHUANIA OR LUXEMBOURG OR MALTA OR NETHERLANDS OR POLAND OR PORTUGAL OR ROMANIA OR SLOVAKIA OR SLOVENIA OR SPAIN OR SWEDEN OR NORWAY OR SWITZERLAND OR UNITED KINGDOM OR ENGLAND OR WALES OR SCOTLAND OR N IRELAND) AND TS=(\"kalman filter*\") AND PY=(2011-2022)')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:56<00:00, 28.31s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('TS=(\"target tracking\") AND PY=(2011-2022)', 'CU=(PEOPLES R CHINA OR HONG KONG) AND CU=(AUSTRIA OR BELGIUM OR BULGARIA OR CROATIA OR CYPRUS OR CZECH REPUBLIC OR DENMARK OR ESTONIA OR FINLAND OR FRANCE OR GERMANY OR GREECE OR HUNGARY OR IRELAND OR ITALY OR LATVIA OR LITHUANIA OR LUXEMBOURG OR MALTA OR NETHERLANDS OR POLAND OR PORTUGAL OR ROMANIA OR SLOVAKIA OR SLOVENIA OR SPAIN OR SWEDEN OR NORWAY OR SWITZERLAND OR UNITED KINGDOM OR ENGLAND OR WALES OR SCOTLAND OR N IRELAND) AND TS=(\"target tracking\") AND PY=(2011-2022)')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:55<00:00, 27.63s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('TS=(\"learning rate*\") AND PY=(2011-2022)', 'CU=(PEOPLES R CHINA OR HONG KONG) AND CU=(AUSTRIA OR BELGIUM OR BULGARIA OR CROATIA OR CYPRUS OR CZECH REPUBLIC OR DENMARK OR ESTONIA OR FINLAND OR FRANCE OR GERMANY OR GREECE OR HUNGARY OR IRELAND OR ITALY OR LATVIA OR LITHUANIA OR LUXEMBOURG OR MALTA OR NETHERLANDS OR POLAND OR PORTUGAL OR ROMANIA OR SLOVAKIA OR SLOVENIA OR SPAIN OR SWEDEN OR NORWAY OR SWITZERLAND OR UNITED KINGDOM OR ENGLAND OR WALES OR SCOTLAND OR N IRELAND) AND TS=(\"learning rate*\") AND PY=(2011-2022)')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:50<00:00, 25.22s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('TS=(\"covariance matri*\") AND PY=(2011-2022)', 'CU=(PEOPLES R CHINA OR HONG KONG) AND CU=(AUSTRIA OR BELGIUM OR BULGARIA OR CROATIA OR CYPRUS OR CZECH REPUBLIC OR DENMARK OR ESTONIA OR FINLAND OR FRANCE OR GERMANY OR GREECE OR HUNGARY OR IRELAND OR ITALY OR LATVIA OR LITHUANIA OR LUXEMBOURG OR MALTA OR NETHERLANDS OR POLAND OR PORTUGAL OR ROMANIA OR SLOVAKIA OR SLOVENIA OR SPAIN OR SWEDEN OR NORWAY OR SWITZERLAND OR UNITED KINGDOM OR ENGLAND OR WALES OR SCOTLAND OR N IRELAND) AND TS=(\"covariance matri*\") AND PY=(2011-2022)')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:55<00:00, 27.74s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('TS=(\"loss function*\") AND PY=(2011-2022)', 'CU=(PEOPLES R CHINA OR HONG KONG) AND CU=(AUSTRIA OR BELGIUM OR BULGARIA OR CROATIA OR CYPRUS OR CZECH REPUBLIC OR DENMARK OR ESTONIA OR FINLAND OR FRANCE OR GERMANY OR GREECE OR HUNGARY OR IRELAND OR ITALY OR LATVIA OR LITHUANIA OR LUXEMBOURG OR MALTA OR NETHERLANDS OR POLAND OR PORTUGAL OR ROMANIA OR SLOVAKIA OR SLOVENIA OR SPAIN OR SWEDEN OR NORWAY OR SWITZERLAND OR UNITED KINGDOM OR ENGLAND OR WALES OR SCOTLAND OR N IRELAND) AND TS=(\"loss function*\") AND PY=(2011-2022)')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:46<00:00, 23.18s/it]\n"
     ]
    }
   ],
   "source": [
    "# from selenium.common.exceptions import TimeoutException\n",
    "#\n",
    "#\n",
    "# for kw in keywords_sub_str[343:]:\n",
    "#     test_query_all = f'TS=({kw}) AND PY=(2011-2022)'\n",
    "#     test_query_coop = f'CU=({foc_str}) AND CU=({eu_assoc}) AND TS=({kw}) AND PY=(2011-2022)'\n",
    "#     sub_queries = tuple([test_query_all,test_query_coop])\n",
    "#     print(sub_queries)\n",
    "#     try:\n",
    "#         wos_fetch_yearly_output(query_str_list=sub_queries)\n",
    "#     except TimeoutException:\n",
    "#         print('No results')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('CU=(PEOPLES R CHINA OR HONG KONG) AND CU=(AUSTRIA OR BELGIUM OR BULGARIA OR CROATIA OR CYPRUS OR CZECH REPUBLIC OR DENMARK OR ESTONIA OR FINLAND OR FRANCE OR GERMANY OR GREECE OR HUNGARY OR IRELAND OR ITALY OR LATVIA OR LITHUANIA OR LUXEMBOURG OR MALTA OR NETHERLANDS OR POLAND OR PORTUGAL OR ROMANIA OR SLOVAKIA OR SLOVENIA OR SPAIN OR SWEDEN OR NORWAY OR SWITZERLAND OR UNITED KINGDOM OR ENGLAND OR WALES OR SCOTLAND OR N IRELAND) AND TS=(\"artificial intelligence*\" OR \"machine* learn*\" OR \"neural network*\" OR \"big data*\" OR \"deep learn*\" OR \"pattern recognition\" OR \"computer vision\" OR \"image classification\" OR \"reinforcement learning\" OR \"support vector machine*\" OR \"recommender system*\" OR \"random forest*\" OR \"ensemble model*\" OR \"image processing\" OR \"generative network*\" OR \"ai ethic*\" OR \"natural language processing\" OR \"clustering algorithm*\" OR \"feature extraction\" OR \"time series forecast*\" OR \"anomaly detection\" OR \"identity fraud detection\" OR \"dimensionality reduction\" OR \"feature elicitation\" OR \"chatbot*\" OR \"clustering\" OR \"*supervised learning\" OR \"convolutional network*\" OR \"convolutional neural\" OR \"adversarial network*\" OR \"adversarial neural\" OR \"adversarial machine*\" OR \"autoencoder*\" OR \"gated recurrent unit*\" OR \"perceptron*\" OR \"feature learning\" OR \"feature engineering\" OR \"long short-term memor*\" OR \"word embedding*\" OR \"word vector*\" OR \"gradient descent\" OR \"k-nearest neighbor*\" OR \"naive bayes\" OR \"transfer learning\" OR \"fuzzy logic*\" OR \"backpropagation\" OR \"computational modeling\" OR \"computational statistic*\" OR \"intelligent agent*\" OR \"expert system*\" OR \"decision tree*\" OR \"Bayesian network*\" OR \"genetic algorithm*\" OR \"swarm intelligence\" OR \"cognitive computing\" OR \"artificial neural network*\" OR \"convolutional neural network*\" OR \"recurrent neural network*\" OR \"ensemble learning\" OR \"data mining\" OR \"artificial general intelligence\" OR \"artificial consciousness\" OR \"evolutionary algorithm*\" OR \"self-organizing map*\" OR \"deep reinforcement learning\" OR \"adversarial machine learning\" OR \"machine vision\" OR \"neural-symbolic integration\" OR \"probabilistic graphical model*\" OR \"hybrid intelligent system*\" OR \"machine creativity\" OR \"explainable AI\" OR \"interactive machine learning\" OR \"artificial emotional intelligence\" OR \"evolutionary computation*\" OR \"human-in-the-loop\" OR \"unsupervised deep learning\" OR \"deep belief network*\" OR \"quantum machine learning\" OR \"artificial immune system*\" OR \"swarm robotics\" OR \"autonomous agent*\" OR \"machine ethic*\" OR \"collaborative filtering\" OR \"content based filtering\" OR \"pervasive computing\" OR \"ubiquitous computing\" OR \"human-computer interaction\" OR \"cloud computing\" OR \"Internet of Things\" OR \"artificial cognition\" OR \"computational creativity\" OR \"sentiment analy*\" OR \"robotics\" OR \"boltzmann machine*\" OR \"kernel machine*\" OR \"Hopfield network*\" OR \"Hebbian learning\" OR \"latent factor model*\" OR \"non-negative matrix factorization\" OR \"independent component analysis\" OR \"principal component analysis\" OR \"data augmentation\" OR \"image segmentation\" OR \"autoregressive language model*\" OR \"generative pre-trained transformer*\" OR \"smart city\" OR \"smart home\" OR \"smart grid\" OR \"smart health\" OR \"smart manufacturing\" OR \"smart agriculture\" OR \"smart environment\" OR \"smart energy\" OR \"smart mobility\" OR \"smart buildings\" OR \"smart tourism\" OR \"smart logistics\" OR \"smart supply chain\" OR \"smart retail\" OR \"smart waste management\" OR \"smart parking\" OR \"smart governance\" OR \"smart education\" OR \"smart technolog*\" OR \"smart diagnostic*\" OR \"data* analytic*\" OR \"hadoop*\" OR \"mapreduce\" OR \"map$reduce\" OR \"large$ dataset*\" OR \"data warehouse*\" OR \"predictive analytic*\" OR \"no$sql\" OR \"nosql\" OR \"no sql\" OR \"unstructured data*\" OR \"data science*\" OR \"facial recognition\" OR \"t$S
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/2 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CU=(PEOPLES R CHINA OR HONG KONG) AND CU=(AUSTRIA OR BELGIUM OR BULGARIA OR CROATIA OR CYPRUS OR CZECH REPUBLIC OR DENMARK OR ESTONIA OR FINLAND OR FRANCE OR GERMANY OR GREECE OR HUNGARY OR IRELAND OR ITALY OR LATVIA OR LITHUANIA OR LUXEMBOURG OR MALTA OR NETHERLANDS OR POLAND OR PORTUGAL OR ROMANIA OR SLOVAKIA OR SLOVENIA OR SPAIN OR SWEDEN OR NORWAY OR SWITZERLAND OR UNITED KINGDOM OR ENGLAND OR WALES OR SCOTLAND OR N IRELAND) AND TS=(\"artificial intelligence*\" OR \"machine* learn*\" OR \"neural network*\" OR \"big data*\" OR \"deep learn*\" OR \"pattern recognition\" OR \"computer vision\" OR \"image classification\" OR \"reinforcement learning\" OR \"support vector machine*\" OR \"recommender system*\" OR \"random forest*\" OR \"ensemble model*\" OR \"image processing\" OR \"generative network*\" OR \"ai ethic*\" OR \"natural language processing\" OR \"clustering algorithm*\" OR \"feature extraction\" OR \"time series forecast*\" OR \"anomaly detection\" OR \"identity fraud detection\" OR \"dimensionality reduction\" OR \"feature elicitation\" OR \"chatbot*\" OR \"clustering\" OR \"*supervised learning\" OR \"convolutional network*\" OR \"convolutional neural\" OR \"adversarial network*\" OR \"adversarial neural\" OR \"adversarial machine*\" OR \"autoencoder*\" OR \"gated recurrent unit*\" OR \"perceptron*\" OR \"feature learning\" OR \"feature engineering\" OR \"long short-term memor*\" OR \"word embedding*\" OR \"word vector*\" OR \"gradient descent\" OR \"k-nearest neighbor*\" OR \"naive bayes\" OR \"transfer learning\" OR \"fuzzy logic*\" OR \"backpropagation\" OR \"computational modeling\" OR \"computational statistic*\" OR \"intelligent agent*\" OR \"expert system*\" OR \"decision tree*\" OR \"Bayesian network*\" OR \"genetic algorithm*\" OR \"swarm intelligence\" OR \"cognitive computing\" OR \"artificial neural network*\" OR \"convolutional neural network*\" OR \"recurrent neural network*\" OR \"ensemble learning\" OR \"data mining\" OR \"artificial general intelligence\" OR \"artificial consciousness\" OR \"evolutionary algorithm*\" OR \"self-organizing map*\" OR \"deep reinforcement learning\" OR \"adversarial machine learning\" OR \"machine vision\" OR \"neural-symbolic integration\" OR \"probabilistic graphical model*\" OR \"hybrid intelligent system*\" OR \"machine creativity\" OR \"explainable AI\" OR \"interactive machine learning\" OR \"artificial emotional intelligence\" OR \"evolutionary computation*\" OR \"human-in-the-loop\" OR \"unsupervised deep learning\" OR \"deep belief network*\" OR \"quantum machine learning\" OR \"artificial immune system*\" OR \"swarm robotics\" OR \"autonomous agent*\" OR \"machine ethic*\" OR \"collaborative filtering\" OR \"content based filtering\" OR \"pervasive computing\" OR \"ubiquitous computing\" OR \"human-computer interaction\" OR \"cloud computing\" OR \"Internet of Things\" OR \"artificial cognition\" OR \"computational creativity\" OR \"sentiment analy*\" OR \"robotics\" OR \"boltzmann machine*\" OR \"kernel machine*\" OR \"Hopfield network*\" OR \"Hebbian learning\" OR \"latent factor model*\" OR \"non-negative matrix factorization\" OR \"independent component analysis\" OR \"principal component analysis\" OR \"data augmentation\" OR \"image segmentation\" OR \"autoregressive language model*\" OR \"generative pre-trained transformer*\" OR \"smart city\" OR \"smart home\" OR \"smart grid\" OR \"smart health\" OR \"smart manufacturing\" OR \"smart agriculture\" OR \"smart environment\" OR \"smart energy\" OR \"smart mobility\" OR \"smart buildings\" OR \"smart tourism\" OR \"smart logistics\" OR \"smart supply chain\" OR \"smart retail\" OR \"smart waste management\" OR \"smart parking\" OR \"smart governance\" OR \"smart education\" OR \"smart technolog*\" OR \"smart diagnostic*\" OR \"data* analytic*\" OR \"hadoop*\" OR \"mapreduce\" OR \"map$reduce\" OR \"large$ dataset*\" OR \"data warehouse*\" OR \"predictive analytic*\" OR \"no$sql\" OR \"nosql\" OR \"no sql\" OR \"unstructured data*\" OR \"data science*\" OR \"facial recognition\" OR \"t$SNE
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 1/2 [01:16<01:16, 76.44s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TS=(\"artificial intelligence*\" OR \"machine* learn*\" OR \"neural network*\" OR \"big data*\" OR \"deep learn*\" OR \"pattern recognition\" OR \"computer vision\" OR \"image classification\" OR \"reinforcement learning\" OR \"support vector machine*\" OR \"recommender system*\" OR \"random forest*\" OR \"ensemble model*\" OR \"image processing\" OR \"generative network*\" OR \"ai ethic*\" OR \"natural language processing\" OR \"clustering algorithm*\" OR \"feature extraction\" OR \"time series forecast*\" OR \"anomaly detection\" OR \"identity fraud detection\" OR \"dimensionality reduction\" OR \"feature elicitation\" OR \"chatbot*\" OR \"clustering\" OR \"*supervised learning\" OR \"convolutional network*\" OR \"convolutional neural\" OR \"adversarial network*\" OR \"adversarial neural\" OR \"adversarial machine*\" OR \"autoencoder*\" OR \"gated recurrent unit*\" OR \"perceptron*\" OR \"feature learning\" OR \"feature engineering\" OR \"long short-term memor*\" OR \"word embedding*\" OR \"word vector*\" OR \"gradient descent\" OR \"k-nearest neighbor*\" OR \"naive bayes\" OR \"transfer learning\" OR \"fuzzy logic*\" OR \"backpropagation\" OR \"computational modeling\" OR \"computational statistic*\" OR \"intelligent agent*\" OR \"expert system*\" OR \"decision tree*\" OR \"Bayesian network*\" OR \"genetic algorithm*\" OR \"swarm intelligence\" OR \"cognitive computing\" OR \"artificial neural network*\" OR \"convolutional neural network*\" OR \"recurrent neural network*\" OR \"ensemble learning\" OR \"data mining\" OR \"artificial general intelligence\" OR \"artificial consciousness\" OR \"evolutionary algorithm*\" OR \"self-organizing map*\" OR \"deep reinforcement learning\" OR \"adversarial machine learning\" OR \"machine vision\" OR \"neural-symbolic integration\" OR \"probabilistic graphical model*\" OR \"hybrid intelligent system*\" OR \"machine creativity\" OR \"explainable AI\" OR \"interactive machine learning\" OR \"artificial emotional intelligence\" OR \"evolutionary computation*\" OR \"human-in-the-loop\" OR \"unsupervised deep learning\" OR \"deep belief network*\" OR \"quantum machine learning\" OR \"artificial immune system*\" OR \"swarm robotics\" OR \"autonomous agent*\" OR \"machine ethic*\" OR \"collaborative filtering\" OR \"content based filtering\" OR \"pervasive computing\" OR \"ubiquitous computing\" OR \"human-computer interaction\" OR \"cloud computing\" OR \"Internet of Things\" OR \"artificial cognition\" OR \"computational creativity\" OR \"sentiment analy*\" OR \"robotics\" OR \"boltzmann machine*\" OR \"kernel machine*\" OR \"Hopfield network*\" OR \"Hebbian learning\" OR \"latent factor model*\" OR \"non-negative matrix factorization\" OR \"independent component analysis\" OR \"principal component analysis\" OR \"data augmentation\" OR \"image segmentation\" OR \"autoregressive language model*\" OR \"generative pre-trained transformer*\" OR \"smart city\" OR \"smart home\" OR \"smart grid\" OR \"smart health\" OR \"smart manufacturing\" OR \"smart agriculture\" OR \"smart environment\" OR \"smart energy\" OR \"smart mobility\" OR \"smart buildings\" OR \"smart tourism\" OR \"smart logistics\" OR \"smart supply chain\" OR \"smart retail\" OR \"smart waste management\" OR \"smart parking\" OR \"smart governance\" OR \"smart education\" OR \"smart technolog*\" OR \"smart diagnostic*\" OR \"data* analytic*\" OR \"hadoop*\" OR \"mapreduce\" OR \"map$reduce\" OR \"large$ dataset*\" OR \"data warehouse*\" OR \"predictive analytic*\" OR \"no$sql\" OR \"nosql\" OR \"no sql\" OR \"unstructured data*\" OR \"data science*\" OR \"facial recognition\" OR \"t$SNE\" OR \"KNN\" OR \"singular value decomposition\" OR \"regularization\" OR \"turing test\" OR \"computational learning theory\" OR \"backward chaining\" OR \"forward chaining\" OR \"entity annotation\" OR \"entity extraction\" OR \"scalable computing\" OR \"expectation maximization algorithm*\" OR \"markov chain\" OR \"markov process\" OR \"markov decision process\" OR \"monte carlo method\" OR \"bayesian inference\" OR \"kernel meth
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [02:26<00:00, 73.43s/it]\n"
     ]
    }
   ],
   "source": [
    "# scope_region_query = f'CU=({foc_str}) AND CU=({eu_assoc}) AND TS=({keywords_str}) AND PY=(2011-2022)'\n",
    "# ww_query = f'TS=({keywords_str}) AND PY=(2011-2022)'\n",
    "#\n",
    "#\n",
    "# sub_queries = tuple([scope_region_query,ww_query])\n",
    "# print(sub_queries)\n",
    "# try:\n",
    "#     wos_fetch_yearly_output(query_str_list=sub_queries)\n",
    "# except TimeoutException:\n",
    "#     print('No results')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# scope_region_query"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# wos_fetch_yearly_output(query_str_list=sub_queries)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hoooold...\n",
      "57838 records found! Here we go in 193 steps...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 192/192 [24:34<00:00,  7.68s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "final batch of 57601-57838\n"
     ]
    }
   ],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}