diff --git a/WOS/wos_univ_normalizer.ipynb b/WOS/wos_univ_normalizer.ipynb index 8089d5e..70ba15c 100644 --- a/WOS/wos_univ_normalizer.ipynb +++ b/WOS/wos_univ_normalizer.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 23, + "execution_count": 191, "outputs": [], "source": [ "import pandas as pd\n", @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 192, "outputs": [], "source": [ "def wikinorm(univ_string):\n", @@ -43,7 +43,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 193, "outputs": [], "source": [ "def replace_uppercase_words(text):\n", @@ -66,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 194, "outputs": [ { "name": "stdout", @@ -85,7 +85,7 @@ "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, - "model_id": "ee2cde76498b4a46a2e87ea6c971aed9" + "model_id": "8551fdcfc52a43108a78c1e91915c681" } }, "metadata": {}, @@ -115,14 +115,14 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 195, "outputs": [ { "data": { - "text/plain": " UT (Unique WOS ID) Institution Country \n84810 WOS:000571399800004 Univ Birmingham United Kingdom \\\n122264 WOS:000732918800001 Univ Southampton United Kingdom \n135675 WOS:000799234000004 UCL United Kingdom \n153134 WOS:000900724501058 Kore Univ Enna Italy \n51445 WOS:000455277600005 Univ Sheffield United Kingdom \n... ... ... ... \n21043 WOS:000372583700005 Vrije Univ Amsterdam Netherlands \n1938 WOS:000297611600011 Univ Essex United Kingdom \n64691 WOS:000490430500091 Xian Jiaotong Liverpool Univ China \n25740 WOS:000386793200001 Chinese Acad Sci China \n112682 WOS:000696110800001 Dalian Univ Technol China \n\n Institution_harm \n84810 Univ Birmingham \n122264 Univ Southampton \n135675 UCL \n153134 Kore Univ Enna \n51445 Univ Sheffield \n... ... \n21043 Vrije Univ Amsterdam \n1938 Univ Essex \n64691 Xian Jiaotong Liverpool Univ \n25740 Chinese Acad Sci \n112682 Dalian Univ Technol \n\n[100 rows x 4 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
UT (Unique WOS ID)InstitutionCountryInstitution_harm
84810WOS:000571399800004Univ BirminghamUnited KingdomUniv Birmingham
122264WOS:000732918800001Univ SouthamptonUnited KingdomUniv Southampton
135675WOS:000799234000004UCLUnited KingdomUCL
153134WOS:000900724501058Kore Univ EnnaItalyKore Univ Enna
51445WOS:000455277600005Univ SheffieldUnited KingdomUniv Sheffield
...............
21043WOS:000372583700005Vrije Univ AmsterdamNetherlandsVrije Univ Amsterdam
1938WOS:000297611600011Univ EssexUnited KingdomUniv Essex
64691WOS:000490430500091Xian Jiaotong Liverpool UnivChinaXian Jiaotong Liverpool Univ
25740WOS:000386793200001Chinese Acad SciChinaChinese Acad Sci
112682WOS:000696110800001Dalian Univ TechnolChinaDalian Univ Technol
\n

100 rows × 4 columns

\n
" + "text/plain": " UT (Unique WOS ID) Institution \n1094 WOS:000292330300050 Hong Kong Polytech Univ \\\n21547 WOS:000374363900001 Guangdong Univ Technol \n53778 WOS:000459846300019 Aarhus Univ \n153776 WOS:000907044000014 Univ Siena \n81562 WOS:000554591602038 China Natl Elect Import Export Corp \n... ... ... \n29206 WOS:000397047200002 Univ Duisburg Essen \n21658 WOS:000374617600020 Univ Southampton \n43289 WOS:000434742800004 Univ Strathclyde \n37200 WOS:000418525100013 Goethe Univ Frankfurt \n95964 WOS:000616310200013 Eindhoven Univ Technol \n\n Country Institution_harm \n1094 China Hong Kong Polytech Univ \n21547 China Guangdong Univ Technol \n53778 Denmark Aarhus Univ \n153776 Italy Univ Siena \n81562 China China Natl Elect Import Export Corp \n... ... ... \n29206 Germany Univ Duisburg Essen \n21658 United Kingdom Univ Southampton \n43289 United Kingdom Univ Strathclyde \n37200 Germany Goethe Univ Frankfurt \n95964 Netherlands Eindhoven Univ Technol \n\n[100 rows x 4 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
UT (Unique WOS ID)InstitutionCountryInstitution_harm
1094WOS:000292330300050Hong Kong Polytech UnivChinaHong Kong Polytech Univ
21547WOS:000374363900001Guangdong Univ TechnolChinaGuangdong Univ Technol
53778WOS:000459846300019Aarhus UnivDenmarkAarhus Univ
153776WOS:000907044000014Univ SienaItalyUniv Siena
81562WOS:000554591602038China Natl Elect Import Export CorpChinaChina Natl Elect Import Export Corp
...............
29206WOS:000397047200002Univ Duisburg EssenGermanyUniv Duisburg Essen
21658WOS:000374617600020Univ SouthamptonUnited KingdomUniv Southampton
43289WOS:000434742800004Univ StrathclydeUnited KingdomUniv Strathclyde
37200WOS:000418525100013Goethe Univ FrankfurtGermanyGoethe Univ Frankfurt
95964WOS:000616310200013Eindhoven Univ TechnolNetherlandsEindhoven Univ Technol
\n

100 rows × 4 columns

\n
" }, - "execution_count": 39, + "execution_count": 195, "metadata": {}, "output_type": "execute_result" } @@ -136,14 +136,14 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 196, "outputs": [ { "data": { - "text/plain": " Country Institution_harm \n7736 France Yncrea Ouest \\\n13752 Spain Univ Carlos \n15855 United Kingdom Northumbria Univ Newcastle Upon Tyne \n12514 Norway Nord Univ \n602 China Henan Polytech Univ \n... ... ... \n11620 Italy Deep Blue Srl \n11183 Italy Univ Giustino Fortunato \n7433 Estonia Platinum Software Dev Co \n5129 China State & Local Joint Engn Lab Estuarine Hydraul Te \n6799 China MOA \n\n count \n7736 9 \n13752 1 \n15855 1 \n12514 1 \n602 87 \n... ... \n11620 1 \n11183 3 \n7433 1 \n5129 1 \n6799 1 \n\n[100 rows x 3 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
CountryInstitution_harmcount
7736FranceYncrea Ouest9
13752SpainUniv Carlos1
15855United KingdomNorthumbria Univ Newcastle Upon Tyne1
12514NorwayNord Univ1
602ChinaHenan Polytech Univ87
............
11620ItalyDeep Blue Srl1
11183ItalyUniv Giustino Fortunato3
7433EstoniaPlatinum Software Dev Co1
5129ChinaState & Local Joint Engn Lab Estuarine Hydraul Te1
6799ChinaMOA1
\n

100 rows × 3 columns

\n
" + "text/plain": " Country Institution_harm count\n12655 Poland Space Res Ctr 6\n12940 Portugal Ctr Invest Energia State Grid 1\n616 China Minist Nat Resources 78\n5561 China PowerChina Huadong Engn Corp Ltd 1\n514 China Chongqing Univ 478\n... ... ... ...\n476 Bulgaria Tech Univ 1\n12454 Norway Stavanger Univ Hosp 9\n5489 China Shanghai Sports Sch 1\n768 China Hubei Univ 25\n13527 Spain Jimenez Diaz Univ Hosp 2\n\n[100 rows x 3 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
CountryInstitution_harmcount
12655PolandSpace Res Ctr6
12940PortugalCtr Invest Energia State Grid1
616ChinaMinist Nat Resources78
5561ChinaPowerChina Huadong Engn Corp Ltd1
514ChinaChongqing Univ478
............
476BulgariaTech Univ1
12454NorwayStavanger Univ Hosp9
5489ChinaShanghai Sports Sch1
768ChinaHubei Univ25
13527SpainJimenez Diaz Univ Hosp2
\n

100 rows × 3 columns

\n
" }, - "execution_count": 71, + "execution_count": 196, "metadata": {}, "output_type": "execute_result" } @@ -159,7 +159,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 197, "outputs": [], "source": [ "# from pandarallel import pandarallel\n", @@ -173,27 +173,7 @@ }, { "cell_type": "code", - "execution_count": 73, - "outputs": [ - { - "data": { - "text/plain": "['Austria', 'Belgium']" - }, - "execution_count": 73, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "list(univ_norm[\"Country\"].unique())[0:2]" - ], - "metadata": { - "collapsed": false - } - }, - { - "cell_type": "code", - "execution_count": 95, + "execution_count": 198, "outputs": [], "source": [ "def ngrams(string, n=3):\n", @@ -296,26 +276,27 @@ }, { "cell_type": "code", - "execution_count": 130, + "execution_count": 199, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "30it [00:00, 53.27it/s]\n" + "100%|██████████| 31/31 [00:00<00:00, 32.89it/s]\n" ] } ], "source": [ "merger = pd.DataFrame()\n", "\n", - "for i in tqdm(filter(lambda c: c!=\"China\", list(univ_norm[\"Country\"].unique()))):\n", + "# for i in tqdm(filter(lambda c: c!=\"China\", list(univ_norm[\"Country\"].unique()))):\n", + "for i in tqdm(list(univ_norm[\"Country\"].unique())):\n", " sub_inst = univ_norm[univ_norm[\"Country\"]==i].reset_index()\n", " types = sub_inst['Institution_harm']\n", " vectorizer = TfidfVectorizer(min_df=1, analyzer=ngrams)\n", " tf_idf_matrix = vectorizer.fit_transform(types)\n", " t1 = time.time()\n", - " matches = awesome_cossim_top(tf_idf_matrix, tf_idf_matrix.transpose(), 10, 0.8)\n", + " matches = awesome_cossim_top(tf_idf_matrix, tf_idf_matrix.transpose(), 10, 0.8 if i!=\"China\" else 0.9)\n", " t = time.time()-t1\n", "\n", " # store the matches into new dataframe called matched_df and printing 10 samples\n", @@ -353,20 +334,20 @@ }, { "cell_type": "code", - "execution_count": 131, + "execution_count": 200, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "1192\n" + "1538\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "1192it [00:44, 26.68it/s]\n" + "1538it [01:04, 23.79it/s]\n" ] } ], @@ -387,6 +368,27 @@ "collapsed": false } }, + { + "cell_type": "code", + "execution_count": 201, + "outputs": [ + { + "data": { + "text/plain": " Country low_side \n0 China Logist Univ Chinese Peoples Armed Police Forces \\\n9 China Flight Automat Control Res Inst \n10 China Northwest Elect Power Design Inst Co Ltd \n11 China Northwest Elect Power Design Inst Co Ltd China \n12 China Northwest Inst Ecoenvironm & Resources \n... ... ... \n1531 China Chinese Univ Hong Kong Hong \n1532 China Huazhong Univ Sci & Techno \n1533 China Hong Kong Polytech Univ Hong Kong \n1534 China Kong Kong Polytech Univ \n1537 China Univ Elect Sci & Technol Chin \n\n high_side high_count \n0 Logist Univ Chinese Peoples Armed Police Force 1 \\\n9 Xian Flight Automat Control Res Inst 1 \n10 Northwest Elect Power Design Inst Co Ltd China 1 \n11 Northwest Elect Power Design Inst Co Ltd 1 \n12 Northwest Inst Ecoenvironm & Resources Chinese Ac 1 \n... ... ... \n1531 Chinese Univ Hong Kong 728 \n1532 Huazhong Univ Sci & Technol 729 \n1533 Hong Kong Polytech Univ 809 \n1534 Hong Kong Polytech Univ 809 \n1537 Univ Elect Sci & Technol China 1076 \n\n low_count similarity \n0 1 0.988072 \n9 1 0.905747 \n10 1 0.926984 \n11 1 0.926984 \n12 1 0.910630 \n... ... ... \n1531 1 0.935944 \n1532 1 0.989260 \n1533 1 0.917345 \n1534 1 0.939416 \n1537 1 0.983258 \n\n[346 rows x 6 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
Countrylow_sidehigh_sidehigh_countlow_countsimilarity
0ChinaLogist Univ Chinese Peoples Armed Police ForcesLogist Univ Chinese Peoples Armed Police Force110.988072
9ChinaFlight Automat Control Res InstXian Flight Automat Control Res Inst110.905747
10ChinaNorthwest Elect Power Design Inst Co LtdNorthwest Elect Power Design Inst Co Ltd China110.926984
11ChinaNorthwest Elect Power Design Inst Co Ltd ChinaNorthwest Elect Power Design Inst Co Ltd110.926984
12ChinaNorthwest Inst Ecoenvironm & ResourcesNorthwest Inst Ecoenvironm & Resources Chinese Ac110.910630
.....................
1531ChinaChinese Univ Hong Kong HongChinese Univ Hong Kong72810.935944
1532ChinaHuazhong Univ Sci & TechnoHuazhong Univ Sci & Technol72910.989260
1533ChinaHong Kong Polytech Univ Hong KongHong Kong Polytech Univ80910.917345
1534ChinaKong Kong Polytech UnivHong Kong Polytech Univ80910.939416
1537ChinaUniv Elect Sci & Technol ChinUniv Elect Sci & Technol China107610.983258
\n

346 rows × 6 columns

\n
" + }, + "execution_count": 201, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# fuzzymerger[fuzzymerger[\"Country\"]==\"China\"]" + ], + "metadata": { + "collapsed": false + } + }, { "cell_type": "markdown", "source": [], @@ -396,20 +398,64 @@ }, { "cell_type": "code", - "execution_count": 120, + "execution_count": 202, + "outputs": [ + { + "data": { + "text/plain": " UT (Unique WOS ID) Institution Country \n244 WOS:000286472300003 Univ Trent Italy \\\n364 WOS:000287586100011 Univ Trent Italy \n410 WOS:000287939200011 Abdus Salam Int Ctr Theoret Phys Italy \n765 WOS:000290996200002 Univ Trent Italy \n907 WOS:000291698400013 INFN Sez Roma 1 Italy \n... ... ... ... \n153063 WOS:000900129900175 Univ Rome Campus Biomed Aquila Italy \n154775 WOS:000929737300001 Prevent & Res Inst Italy \n154813 WOS:000929737300001 Ist Super Sanit Italy \n154855 WOS:000933331200004 Univ Federio II Italy \n154857 WOS:000933331200004 INAF Osservatorio Astron Capodimonte Italy \n\n Institution_harm merge_iter \n244 Univ Trento 1 \n364 Univ Trento 1 \n410 Abdus Salaam Int Ctr Theoret Phys 1 \n765 Univ Trento 1 \n907 Sez Roma 1 \n... ... ... \n153063 Univ Rome Campus Biomed Aquila 2 \n154775 Prevent & Res Inst 2 \n154813 Ist Super Sanita 1 \n154855 Univ Federico 1 \n154857 Osserv Astron Capodimonte 1 \n\n[375 rows x 5 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
UT (Unique WOS ID)InstitutionCountryInstitution_harmmerge_iter
244WOS:000286472300003Univ TrentItalyUniv Trento1
364WOS:000287586100011Univ TrentItalyUniv Trento1
410WOS:000287939200011Abdus Salam Int Ctr Theoret PhysItalyAbdus Salaam Int Ctr Theoret Phys1
765WOS:000290996200002Univ TrentItalyUniv Trento1
907WOS:000291698400013INFN Sez Roma 1ItalySez Roma1
..................
153063WOS:000900129900175Univ Rome Campus Biomed AquilaItalyUniv Rome Campus Biomed Aquila2
154775WOS:000929737300001Prevent & Res InstItalyPrevent & Res Inst2
154813WOS:000929737300001Ist Super SanitItalyIst Super Sanita1
154855WOS:000933331200004Univ Federio IIItalyUniv Federico1
154857WOS:000933331200004INAF Osservatorio Astron CapodimonteItalyOsserv Astron Capodimonte1
\n

375 rows × 5 columns

\n
" + }, + "execution_count": 202, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# univ_harm[((univ_harm[\"merge_iter\"]>0) & (univ_harm[\"Country\"]==\"Italy\"))]" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 208, + "outputs": [], + "source": [ + "univ_harm.loc[((univ_harm[\"merge_iter\"]>0) & (univ_harm[\"Country\"]==\"Italy\")&\n", + " (univ_harm[\"Institution\"].str.lower().str.contains(\"sapien\"))&\n", + " (univ_harm[\"Institution\"].str.lower().str.contains(\"univ\"))), \"Institution_harm\"] = \"Sapienza Univ Rome\"" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 209, "outputs": [ { "data": { - "text/plain": " UT (Unique WOS ID) Institution Country \n244 WOS:000286472300003 Univ Trent Italy \\\n364 WOS:000287586100011 Univ Trent Italy \n410 WOS:000287939200011 Abdus Salam Int Ctr Theoret Phys Italy \n765 WOS:000290996200002 Univ Trent Italy \n907 WOS:000291698400013 INFN Sez Roma 1 Italy \n... ... ... ... \n153063 WOS:000900129900175 Univ Rome Campus Biomed Aquila Italy \n154775 WOS:000929737300001 Prevent & Res Inst Italy \n154813 WOS:000929737300001 Ist Super Sanit Italy \n154855 WOS:000933331200004 Univ Federio II Italy \n154857 WOS:000933331200004 INAF Osservatorio Astron Capodimonte Italy \n\n Institution_harm merge_iter \n244 Univ Trento 1 \n364 Univ Trento 1 \n410 Abdus Salaam Int Ctr Theoret Phys 1 \n765 Univ Trento 1 \n907 Sez Roma 1 \n... ... ... \n153063 Univ Rome Campus Biomed LAquila 1 \n154775 Prevent & Res Inst 2 \n154813 Ist Super Sanita 1 \n154855 Univ Federio 2 \n154857 Osserv Astron Capodimonte 1 \n\n[375 rows x 5 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
UT (Unique WOS ID)InstitutionCountryInstitution_harmmerge_iter
244WOS:000286472300003Univ TrentItalyUniv Trento1
364WOS:000287586100011Univ TrentItalyUniv Trento1
410WOS:000287939200011Abdus Salam Int Ctr Theoret PhysItalyAbdus Salaam Int Ctr Theoret Phys1
765WOS:000290996200002Univ TrentItalyUniv Trento1
907WOS:000291698400013INFN Sez Roma 1ItalySez Roma1
..................
153063WOS:000900129900175Univ Rome Campus Biomed AquilaItalyUniv Rome Campus Biomed LAquila1
154775WOS:000929737300001Prevent & Res InstItalyPrevent & Res Inst2
154813WOS:000929737300001Ist Super SanitItalyIst Super Sanita1
154855WOS:000933331200004Univ Federio IIItalyUniv Federio2
154857WOS:000933331200004INAF Osservatorio Astron CapodimonteItalyOsserv Astron Capodimonte1
\n

375 rows × 5 columns

\n
" + "text/plain": "Institution 17083\nInstitution_harm 14449\ndtype: int64" }, - "execution_count": 120, + "execution_count": 209, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "univ_harm[((univ_harm[\"merge_iter\"]>0) & (univ_harm[\"Country\"]==\"Italy\"))]" + "univ_harm[[\"Institution\",\"Institution_harm\"]].nunique()" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 210, + "outputs": [], + "source": [ + "univ_harm.to_excel(f\"{outdir}/wos_institution_locations_harmonized.xlsx\", index=False)" ], "metadata": { "collapsed": false