diff --git a/.gitignore b/.gitignore index 435d880..0cf5551 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -/PATSTAT/EU_CH_scope/cpc_defs.csv /misc_code/ /PATSTAT/appln_data.xlsx /PATSTAT/person_data.xlsx diff --git a/PATSTAT/WESTERN_CH_scope/cpc_defs.csv b/PATSTAT/WESTERN_CH_scope/cpc_defs.csv new file mode 100644 index 0000000..f6dc1fe --- /dev/null +++ b/PATSTAT/WESTERN_CH_scope/cpc_defs.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab0f4acc10a622f8a162a9f1f2aaf39b06799f65feab44412aed2dc2d6f27cf8 +size 159305379 diff --git a/PATSTAT/WESTERN_CH_scope/scope_cpc_defs.csv b/PATSTAT/WESTERN_CH_scope/scope_cpc_defs.csv new file mode 100644 index 0000000..a062eab --- /dev/null +++ b/PATSTAT/WESTERN_CH_scope/scope_cpc_defs.csv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76479394378a76774399904f0aa8104a2fdf0d2ec39d22a928a0c07eb80e6e0c +size 209293 diff --git a/PATSTAT/patstat_cpc_parse.ipynb b/PATSTAT/patstat_cpc_parse.ipynb index 7302714..057f87c 100644 --- a/PATSTAT/patstat_cpc_parse.ipynb +++ b/PATSTAT/patstat_cpc_parse.ipynb @@ -244,30 +244,99 @@ }, { "cell_type": "code", - "execution_count": 22, - "id": "6c3baa5b", - "metadata": {}, + "execution_count": 49, + "outputs": [ + { + "data": { + "text/plain": "'neural network|machine learn|deep learn|remote sensing|convolutional neural|internet of things|feature extraction|genetic algorithm|big data|artificial intelligence|data driven|support vector machine|logistic regression not p=|optimization algorithm|principal component analysis|artificial neural network|swarm optimization|regularization|linear regression not p=|optimization algorithm|random forest|cloud computing|reinforcement learning|computer vision|kalman filter|image processing|data mining|evolutionary algorithm|edge computing|supervised learning|computational modeling|pattern recognition|image classification|long short-term memor|robotics|image segmentation|convex optimization|covariance matri|attention mechanism|markov chain|object detection not brain|clustering algorithm|recurrent neural network|data augmentation|transfer learning|adversarial network|decision tree|multi agent system|fuzzy set|convolutional network|image reconstruction|data analytic|smart grid|autoencoder|fuzzy logic|radial basis function|bayesian network|dimensionality reduction|face recognition not brain|gaussian process|anomaly detection|k-nearest neighbor|natural language processing|monte carlo method|large dataset|gradient descent|support vector regression|extreme learning machine|perceptron|model selection|ensemble learning|representation learning|recommender system|target tracking|singular value decomposition|feature learning|smart city|sentiment analy|markov decision process|k-means clustering|independent component analysis|brain computer interface|human-computer interaction|markov chain monte carlo|hierarchical clustering|semantic web|semi-supervised learning|human-robot interact|knowledge graph|speech recognition not brain|ensemble model|fog computing|mapreduce|evolutionary computation|data science|text mining|generative model|active learning|swarm intelligence|multi-task learning|language model|collaborative filtering|backpropagation|machine vision|computer-aided diagnosis|gated recurrent unit|lagrange multiplier|expert system|learning rate|hadoop|markov process|nonlinear optimization|learning system|self-organizing map|smart manufacturing|smart home|few shot learning|few-shot learning|meta-learning|meta learning|adversarial training|zero-shot learning|word embedding|expectation maximization algorithm|stochastic gradient descent|ridge regression|deep belief network|non-negative matrix factorization|affective computing|latent dirichlet allocation|kernel method|kernel learning|feature engineering|variational inference|image representation|manifold learning|adversarial example|knowledge distillation|time series forecast|variational autoencoder|lasso regression|smart energy|dbscan|multi-label classification|intelligent robot|ubiquitous computing|gaussian mixture models|smart technolog|boltzmann machine|smart buildings|predictive analytic|pervasive computing|smart agriculture|capsule network|human-in-the-loop|intelligent agent|ai applications|word vector|transformer model|facial recognition|unstructured data|restricted boltzmann machine|albert|lifelong learning|autonomous agents|chatbot|cholesky decomposition|nosql|nosql|explainable ai|seq2seq|probabilistic graphical model|qr decomposition|unsupervised deep learning|data warehouse|quantum machine learning|continual learning|smart environment|multimodal learning|smart health|artificial immune system|swarm robotics|kernel machine|latent factor model|eigendecomposition|adversarial machine|adversarial machine learning|smart mobility|sequence-to-sequence model|eigen decomposition|adversarial robustness|smart parking|adversarial neural|roberta|bidirectional encoder representations from transformer|locally linear embedding|hebbian learning|one-shot learning|multimodal representation|smart tourism|entity extraction|adaptive moment estimation|ontology learning|topic modeling|relational database'" + }, + "execution_count": 49, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "keywords_oklist_source= r'..\\WOS\\kw_token_ranked_bibliometrics_okset.xlsx'\n", + "keyword_df = pd.read_excel(keywords_oklist_source)\n", + "keywords = keyword_df[keyword_df[\"u_Priority (done)\"].isin([\"High\",\"Medium\"])][\"kw_token\"].str.replace('\"','').tolist()\n", + "keywords = [kw.replace(\"*\",\"\").replace(\"$\",\"\").lower() for kw in keywords if (\"?\" not in kw and len(kw)>3)]\n", + "keywords = \"|\".join([kw for kw in keywords if kw not in [\"classifier\",\"clustering\",\"loss function\",'classification']]+[\"relational database\"])\n", + "keywords" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 54, "outputs": [ { "data": { "text/plain": " cpc_id cpc_name \n12725 A61B1/000096 {using artificial intelligence} \\\n13764 A61B5/7264 {Classification of physiological signals or da... \n13897 A61B6/52 {Devices using data or image processing specia... \n14016 A61B8/52 {Devices using data or image processing specia... \n15252 A61B2018/0069 {using fuzzy logic} \n... ... ... \n250685 Y10S707/99946 Object-oriented database structure network \n250686 Y10S707/99947 Object-oriented database structure reference \n250687 Y10S707/99948 Application of database or data structure, e.g... \n250688 Y10S707/99951 File or database maintenance \n250703 Y10S715/968 interface for database querying and retrieval \n\n section class subclass group main_group cpc_version \n12725 A 61 B 1 000096 2023 \\\n13764 A 61 B 5 7264 2023 \n13897 A 61 B 6 52 2023 \n14016 A 61 B 8 52 2023 \n15252 A 61 B 2018 0069 2023 \n... ... ... ... ... ... ... \n250685 Y 10 S 707 99946 2023 \n250686 Y 10 S 707 99947 2023 \n250687 Y 10 S 707 99948 2023 \n250688 Y 10 S 707 99951 2023 \n250703 Y 10 S 715 968 2023 \n\n version https://git-lfs.github.com/spec/v1 \n12725 NaN \\\n13764 NaN \n13897 NaN \n14016 NaN \n15252 NaN \n... ... \n250685 NaN \n250686 NaN \n250687 NaN \n250688 NaN \n250703 NaN \n\n cpc_taxonomy \n12725 [(A, HUMAN NECESSITIES), (A61, MEDICAL OR VETE... \n13764 [(A, HUMAN NECESSITIES), (A61, MEDICAL OR VETE... \n13897 [(A, HUMAN NECESSITIES), (A61, MEDICAL OR VETE... \n14016 [(A, HUMAN NECESSITIES), (A61, MEDICAL OR VETE... \n15252 [(A, HUMAN NECESSITIES), (A61, MEDICAL OR VETE... \n... ... \n250685 [(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE... \n250686 [(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE... \n250687 [(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE... \n250688 [(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE... \n250703 [(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE... \n\n[317 rows x 10 columns]", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
cpc_idcpc_namesectionclasssubclassgroupmain_groupcpc_versionversion https://git-lfs.github.com/spec/v1cpc_taxonomy
12725A61B1/000096{using artificial intelligence}A61B10000962023NaN[(A, HUMAN NECESSITIES), (A61, MEDICAL OR VETE...
13764A61B5/7264{Classification of physiological signals or da...A61B572642023NaN[(A, HUMAN NECESSITIES), (A61, MEDICAL OR VETE...
13897A61B6/52{Devices using data or image processing specia...A61B6522023NaN[(A, HUMAN NECESSITIES), (A61, MEDICAL OR VETE...
14016A61B8/52{Devices using data or image processing specia...A61B8522023NaN[(A, HUMAN NECESSITIES), (A61, MEDICAL OR VETE...
15252A61B2018/0069{using fuzzy logic}A61B201800692023NaN[(A, HUMAN NECESSITIES), (A61, MEDICAL OR VETE...
.................................
250685Y10S707/99946Object-oriented database structure networkY10S707999462023NaN[(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE...
250686Y10S707/99947Object-oriented database structure referenceY10S707999472023NaN[(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE...
250687Y10S707/99948Application of database or data structure, e.g...Y10S707999482023NaN[(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE...
250688Y10S707/99951File or database maintenanceY10S707999512023NaN[(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE...
250703Y10S715/968interface for database querying and retrievalY10S7159682023NaN[(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE...
\n

317 rows × 10 columns

\n
" }, - "execution_count": 22, + "execution_count": 54, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "cpc_ids[cpc_ids[\"cpc_name\"].str.lower().str.contains(\"machine learn|neural network|deep learn|deep network|artificial intel*| big data|database|recommender system|computer vision|image processing|language model|language processing|fuzzy logic|principal component|image classification|video classification\", regex=True, na=False)]" - ] + "scope_df = cpc_ids[cpc_ids[\"cpc_name\"].str.lower().str.contains(\"machine learn|neural network|deep learn|deep network|artificial intel*| big data|database|recommender system|computer vision|image processing|language model|language processing|fuzzy logic|principal component|image classification|video classification\", regex=True, na=False)]\n", + "scope_df" + ], + "metadata": { + "collapsed": false + } }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 58, + "id": "6c3baa5b", + "metadata": {}, "outputs": [], "source": [ + "scope_ids = scope_df[\"cpc_id\"].unique()\n", + "cpc_ids[\"data_scope\"] = cpc_ids[\"cpc_id\"].isin(scope_ids)\n", "cpc_ids.to_csv(f\"{outdir}/cpc_defs.csv\", index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "outputs": [ + { + "data": { + "text/plain": "'WESTERN_CH_scope'" + }, + "execution_count": 59, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "outdir" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 56, + "outputs": [ + { + "data": { + "text/plain": " cpc_id cpc_name \n0 A HUMAN NECESSITIES \\\n1 A01 AGRICULTURE; FORESTRY; ANIMAL HUSBANDRY; HUNTI... \n2 A01B SOIL WORKING IN AGRICULTURE OR FORESTRY; PARTS... \n3 A01B1/00 Hand tools (edge trimmers for lawns A01G3/06 ... \n4 A01B1/02 Spades; Shovels {(hand-operated dredgers E02F3... \n... ... ... \n260486 Y10T483/1873 Indexing matrix \n260487 Y10T483/1882 Rotary disc \n260488 Y10T483/1891 Chain or belt \n260489 Y10T483/19 Miscellaneous \n260490 NaN NaN \n\n section class subclass group main_group cpc_version \n0 A None None None None 2023 \\\n1 A 01 None None None 2023 \n2 A 01 B None None 2023 \n3 A 01 B 1 00 2023 \n4 A 01 B 1 02 2023 \n... ... ... ... ... ... ... \n260486 Y 10 T 483 1873 2023 \n260487 Y 10 T 483 1882 2023 \n260488 Y 10 T 483 1891 2023 \n260489 Y 10 T 483 19 2023 \n260490 NaN NaN NaN NaN NaN 2022 \n\n version https://git-lfs.github.com/spec/v1 \n0 NaN \\\n1 NaN \n2 NaN \n3 NaN \n4 NaN \n... ... \n260486 NaN \n260487 NaN \n260488 NaN \n260489 NaN \n260490 oid sha256:f138d6bdf2939ba576b96b633d81366123b... \n\n cpc_taxonomy data_scope \n0 [(A, HUMAN NECESSITIES)] False \n1 [(A, HUMAN NECESSITIES), (A01, AGRICULTURE; FO... False \n2 [(A, HUMAN NECESSITIES), (A01, AGRICULTURE; FO... False \n3 [(A, HUMAN NECESSITIES), (A01, AGRICULTURE; FO... False \n4 [(A, HUMAN NECESSITIES), (A01, AGRICULTURE; FO... False \n... ... ... \n260486 [(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE... False \n260487 [(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE... False \n260488 [(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE... False \n260489 [(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE... False \n260490 [] False \n\n[260491 rows x 11 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
cpc_idcpc_namesectionclasssubclassgroupmain_groupcpc_versionversion https://git-lfs.github.com/spec/v1cpc_taxonomydata_scope
0AHUMAN NECESSITIESANoneNoneNoneNone2023NaN[(A, HUMAN NECESSITIES)]False
1A01AGRICULTURE; FORESTRY; ANIMAL HUSBANDRY; HUNTI...A01NoneNoneNone2023NaN[(A, HUMAN NECESSITIES), (A01, AGRICULTURE; FO...False
2A01BSOIL WORKING IN AGRICULTURE OR FORESTRY; PARTS...A01BNoneNone2023NaN[(A, HUMAN NECESSITIES), (A01, AGRICULTURE; FO...False
3A01B1/00Hand tools (edge trimmers for lawns A01G3/06 ...A01B1002023NaN[(A, HUMAN NECESSITIES), (A01, AGRICULTURE; FO...False
4A01B1/02Spades; Shovels {(hand-operated dredgers E02F3...A01B1022023NaN[(A, HUMAN NECESSITIES), (A01, AGRICULTURE; FO...False
....................................
260486Y10T483/1873Indexing matrixY10T48318732023NaN[(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE...False
260487Y10T483/1882Rotary discY10T48318822023NaN[(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE...False
260488Y10T483/1891Chain or beltY10T48318912023NaN[(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE...False
260489Y10T483/19MiscellaneousY10T483192023NaN[(Y, GENERAL TAGGING OF NEW TECHNOLOGICAL DEVE...False
260490NaNNaNNaNNaNNaNNaNNaN2022oid sha256:f138d6bdf2939ba576b96b633d81366123b...[]False
\n

260491 rows × 11 columns

\n
" + }, + "execution_count": 56, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cpc_ids" ], "metadata": { "collapsed": false