You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
376 lines
12 KiB
Plaintext
376 lines
12 KiB
Plaintext
2 years ago
|
{
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 1,
|
||
|
|
"metadata": {
|
||
|
|
"collapsed": true
|
||
|
|
},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"import pandas as pd\n",
|
||
|
|
"import pyarrow\n",
|
||
|
|
"from tqdm import tqdm\n",
|
||
|
|
"import os\n",
|
||
|
|
"import shutil"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 2,
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"text/plain": "'2.0.0rc1'"
|
||
|
|
},
|
||
|
|
"execution_count": 2,
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "execute_result"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"pd.__version__"
|
||
|
|
],
|
||
|
|
"metadata": {
|
||
|
|
"collapsed": false
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 2,
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"C:\\Users\\radvanyi\\PycharmProjects\\ZSI_analytics\\PATSTAT\n",
|
||
|
|
"D:\\PATSTAT\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"import os\n",
|
||
|
|
"print(os.getcwd()) # Prints the current working directory\n",
|
||
|
|
"\n",
|
||
|
|
"workdir_path=r\"D:\\PATSTAT\"\n",
|
||
|
|
"os.chdir(workdir_path)\n",
|
||
|
|
"print(os.getcwd())"
|
||
|
|
],
|
||
|
|
"metadata": {
|
||
|
|
"collapsed": false
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": null,
|
||
|
|
"outputs": [],
|
||
|
|
"source": [],
|
||
|
|
"metadata": {
|
||
|
|
"collapsed": false
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 3,
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"CONCAT: ['tls201_part01.csv', 'tls201_part02.csv', 'tls201_part03.csv'] TO D:\\PATSTAT\\table_tls201.csv\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"1000it [08:05, 2.06it/s]\n",
|
||
|
|
"1000it [08:23, 1.99it/s]\n",
|
||
|
|
"506it [04:04, 2.07it/s]\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"deleting\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_01\\tls201_part01\\tls201_part01.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_01\\tls201_part02\\tls201_part02.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_01\\tls201_part03\\tls201_part03.csv\n",
|
||
|
|
"MOVE: D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_01\\tls202_part01\\tls202_part01.csv TO D:\\PATSTAT\\table_tls202.csv\n",
|
||
|
|
"MOVE: D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_05\\tls204_part01\\tls204_part01.csv TO D:\\PATSTAT\\table_tls204.csv\n",
|
||
|
|
"MOVE: D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_05\\tls205_part01\\tls205_part01.csv TO D:\\PATSTAT\\table_tls205.csv\n",
|
||
|
|
"CONCAT: ['tls206_part01.csv', 'tls206_part02.csv'] TO D:\\PATSTAT\\table_tls206.csv\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"1000it [08:15, 2.02it/s]\n",
|
||
|
|
"777it [06:38, 1.95it/s]\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"deleting\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_05\\tls206_part01\\tls206_part01.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_06\\tls206_part02\\tls206_part02.csv\n",
|
||
|
|
"MOVE: D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_06\\tls207_part01\\tls207_part01.csv TO D:\\PATSTAT\\table_tls207.csv\n",
|
||
|
|
"CONCAT: ['tls209_part01.csv', 'tls209_part02.csv', 'tls209_part03.csv'] TO D:\\PATSTAT\\table_tls209.csv\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"3200it [07:58, 6.69it/s]\n",
|
||
|
|
"3200it [07:54, 6.75it/s]\n",
|
||
|
|
"217it [00:31, 6.96it/s]\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"deleting\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_06\\tls209_part01\\tls209_part01.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_06\\tls209_part02\\tls209_part02.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_06\\tls209_part03\\tls209_part03.csv\n",
|
||
|
|
"MOVE: D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_06\\tls210_part01\\tls210_part01.csv TO D:\\PATSTAT\\table_tls210.csv\n",
|
||
|
|
"CONCAT: ['tls211_part01.csv', 'tls211_part02.csv'] TO D:\\PATSTAT\\table_tls211.csv\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"2500it [08:18, 5.01it/s]\n",
|
||
|
|
"463it [01:43, 4.46it/s]\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"deleting\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_07\\tls211_part01\\tls211_part01.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_07\\tls211_part02\\tls211_part02.csv\n",
|
||
|
|
"CONCAT: ['tls212_part01.csv', 'tls212_part02.csv', 'tls212_part03.csv'] TO D:\\PATSTAT\\table_tls212.csv\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"4000it [10:46, 6.19it/s]\n",
|
||
|
|
"4000it [10:54, 6.11it/s]\n",
|
||
|
|
"1764it [04:47, 6.13it/s]\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"deleting\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_07\\tls212_part01\\tls212_part01.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_07\\tls212_part02\\tls212_part02.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_08\\tls212_part03\\tls212_part03.csv\n",
|
||
|
|
"MOVE: D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_08\\tls214_part01\\tls214_part01.csv TO D:\\PATSTAT\\table_tls214.csv\n",
|
||
|
|
"CONCAT: ['tls215_part01.csv', 'tls215_part02.csv', 'tls215_part03.csv'] TO D:\\PATSTAT\\table_tls215.csv\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"8000it [12:35, 10.60it/s]\n",
|
||
|
|
"8000it [12:24, 10.74it/s]\n",
|
||
|
|
"3101it [04:50, 10.66it/s]\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"deleting\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_08\\tls215_part01\\tls215_part01.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_08\\tls215_part02\\tls215_part02.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_09\\tls215_part03\\tls215_part03.csv\n",
|
||
|
|
"MOVE: D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_09\\tls216_part01\\tls216_part01.csv TO D:\\PATSTAT\\table_tls216.csv\n",
|
||
|
|
"CONCAT: ['tls222_part01.csv', 'tls222_part02.csv'] TO D:\\PATSTAT\\table_tls222.csv\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"5000it [08:34, 9.71it/s]\n",
|
||
|
|
"2489it [04:16, 9.70it/s]\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"deleting\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_09\\tls222_part01\\tls222_part01.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_09\\tls222_part02\\tls222_part02.csv\n",
|
||
|
|
"CONCAT: ['tls224_part01.csv', 'tls224_part02.csv', 'tls224_part03.csv'] TO D:\\PATSTAT\\table_tls224.csv\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"3000it [04:24, 11.35it/s]\n",
|
||
|
|
"3000it [04:27, 11.20it/s]\n",
|
||
|
|
"1442it [02:07, 11.29it/s]\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"deleting\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_09\\tls224_part01\\tls224_part01.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_09\\tls224_part02\\tls224_part02.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_09\\tls224_part03\\tls224_part03.csv\n",
|
||
|
|
"CONCAT: ['tls225_part01.csv', 'tls225_part02.csv'] TO D:\\PATSTAT\\table_tls225.csv\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"3000it [08:29, 5.89it/s]\n",
|
||
|
|
"718it [02:02, 5.88it/s]\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"deleting\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_09\\tls225_part01\\tls225_part01.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_09\\tls225_part02\\tls225_part02.csv\n",
|
||
|
|
"CONCAT: ['tls226_part01.csv', 'tls226_part02.csv'] TO D:\\PATSTAT\\table_tls226.csv\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"1500it [10:40, 2.34it/s]\n",
|
||
|
|
"664it [04:56, 2.24it/s]\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"deleting\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_09\\tls226_part01\\tls226_part01.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_10\\tls226_part02\\tls226_part02.csv\n",
|
||
|
|
"CONCAT: ['tls227_part01.csv', 'tls227_part02.csv'] TO D:\\PATSTAT\\table_tls227.csv\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"8000it [12:42, 10.49it/s]\n",
|
||
|
|
"862it [01:22, 10.39it/s]\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"deleting\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_10\\tls227_part01\\tls227_part01.csv\n",
|
||
|
|
"D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_10\\tls227_part02\\tls227_part02.csv\n",
|
||
|
|
"MOVE: D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_10\\tls228_part01\\tls228_part01.csv TO D:\\PATSTAT\\table_tls228.csv\n",
|
||
|
|
"MOVE: D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_10\\tls229_part01\\tls229_part01.csv TO D:\\PATSTAT\\table_tls229.csv\n",
|
||
|
|
"MOVE: D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_10\\tls230_part01\\tls230_part01.csv TO D:\\PATSTAT\\table_tls230.csv\n",
|
||
|
|
"MOVE: D:\\PATSTAT\\PATSTAT_spring\\data_PATSTAT_Global_2023_Spring_11\\tls801_part01\\tls801_part01.csv TO D:\\PATSTAT\\table_tls801.csv\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"file_path_dict=dict()\n",
|
||
|
|
"\n",
|
||
|
|
"# iterate over files in\n",
|
||
|
|
"# that directory\n",
|
||
|
|
"for root, dirs, files in os.walk(workdir_path):\n",
|
||
|
|
" for filename in files:\n",
|
||
|
|
" if filename.endswith(\".csv\") and not filename.startswith(\"table\"):\n",
|
||
|
|
" path= os.path.join(root, filename)\n",
|
||
|
|
" file_path_dict[filename] = path\n",
|
||
|
|
"\n",
|
||
|
|
"complete_file_set = set()\n",
|
||
|
|
"for fname in file_path_dict.keys():\n",
|
||
|
|
" complete_file_set.add(fname.split(\"_\")[0])\n",
|
||
|
|
"complete_file_set = sorted(complete_file_set)\n",
|
||
|
|
"\n",
|
||
|
|
"for complete_file in complete_file_set:\n",
|
||
|
|
" file_list = [file for file in file_path_dict.keys() if complete_file in file]\n",
|
||
|
|
"\n",
|
||
|
|
" outfile_path = os.path.join(workdir_path,\"table_\"+complete_file+\".csv\")\n",
|
||
|
|
" # print(outfile_path,file_list)\n",
|
||
|
|
"\n",
|
||
|
|
" if len(file_list)==1:\n",
|
||
|
|
" file_path = file_path_dict.get(file_list[0])\n",
|
||
|
|
" print(\"MOVE:\",file_path ,\"TO\",outfile_path)\n",
|
||
|
|
" shutil.move(file_path, outfile_path)\n",
|
||
|
|
" else:\n",
|
||
|
|
" print(\"CONCAT:\",file_list ,\"TO\",outfile_path)\n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
" CHUNK_SIZE = 50000\n",
|
||
|
|
" with_header = True\n",
|
||
|
|
" first_one = True\n",
|
||
|
|
" for csv_file_name in file_list:\n",
|
||
|
|
" csv_file_path = file_path_dict.get(csv_file_name)\n",
|
||
|
|
" chunk_container = pd.read_csv(csv_file_path, chunksize=CHUNK_SIZE,low_memory=False)\n",
|
||
|
|
" for chunk in tqdm(chunk_container):\n",
|
||
|
|
" chunk.to_csv(outfile_path, mode=\"a\", index=False, header=with_header)\n",
|
||
|
|
" with_header=False\n",
|
||
|
|
" first_one = False\n",
|
||
|
|
"\n",
|
||
|
|
" print(\"deleting\")\n",
|
||
|
|
" for csv_file_name in file_list:\n",
|
||
|
|
" csv_file_path = file_path_dict.get(csv_file_name)\n",
|
||
|
|
" print(csv_file_path)\n",
|
||
|
|
" os.remove(csv_file_path)"
|
||
|
|
],
|
||
|
|
"metadata": {
|
||
|
|
"collapsed": false
|
||
|
|
}
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"metadata": {
|
||
|
|
"kernelspec": {
|
||
|
|
"display_name": "Python 3",
|
||
|
|
"language": "python",
|
||
|
|
"name": "python3"
|
||
|
|
},
|
||
|
|
"language_info": {
|
||
|
|
"codemirror_mode": {
|
||
|
|
"name": "ipython",
|
||
|
|
"version": 2
|
||
|
|
},
|
||
|
|
"file_extension": ".py",
|
||
|
|
"mimetype": "text/x-python",
|
||
|
|
"name": "python",
|
||
|
|
"nbconvert_exporter": "python",
|
||
|
|
"pygments_lexer": "ipython2",
|
||
|
|
"version": "2.7.6"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"nbformat": 4,
|
||
|
|
"nbformat_minor": 0
|
||
|
|
}
|