{"id":1292,"date":"2021-09-18T11:10:34","date_gmt":"2021-09-18T11:10:34","guid":{"rendered":"https:\/\/huzaifazoom.org\/?p=1292"},"modified":"2021-09-18T11:10:34","modified_gmt":"2021-09-18T11:10:34","slug":"automated-data-update","status":"publish","type":"post","link":"https:\/\/observatory.huzaifazoom.org\/ar\/2021\/09\/18\/automated-data-update\/","title":{"rendered":"3. Automated Data Update"},"content":{"rendered":"<p>The two Python\/Pandas scripts below are called by the corresponding (UN Comtrade or UNIDO) front-end programs which in turn are invoked from the <a href=\"https:\/\/huzaifazoom.org\/?page_id=1027\">Data Update page<\/a>. These two scripts are located in the \/var\/www\/wordpress\/jordan\/prototype\/scripts\/ folder. They are included here to give an idea of the data structures and manipulation involved in displaying the charts.<\/p>\n<p>The front-end programs are HTML\/Javascripts\/CSS based and they are to be found at:<\/p>\n<ul>\n<li>&lt;Wordpress root folder&gt;\/jordan\/prototype\/admin\/comtrade_update.html<\/li>\n<li>&lt;Wordpress root folder&gt;\/var\/www\/wordpress\/jordan\/prototype\/admin\/unido_update.html<\/li>\n<\/ul>\n<p><a>UN Comtrade<\/a><\/p>\n<pre><code>\n#!\/home\/huzaifa\/anaconda3\/bin\/python3\n#\/usr\/bin\/python3\n# Name: update_comtrade.py\n# Created By: Huzaifa Zoomkawala\n# Last Modified: August 10, 2021\nimport cgi\nimport cgitb; cgitb.enable()  # for troubleshooting\nimport os\nimport os.path\nimport datetime\nimport json\nimport pandas as pd\nimport sys\n#------------------------------------ Global Variables\nA = ['001', '011', '012', '022', '025', '034', '036', '041', '042', '043', '044', '045', '054', '057', '071', '072', '074', '075', '081', '121', '211', '212', '222', '223', '231', '244', '245', '246', '261', '263', '268', '269', '272', '273', '274', '277', '278', '291', '292', '321', '325', '333', '343', '681', '682', '683', '684', '685', '686', '687']\nB = ['016', '017', '023', '024', '035', '037', '046', '047', '048', '056', '058', '059', '061', '062', '073', '091', '098', '111', '112', '122', '232', '247', '248', '251', '264', '265', '421', '422', '431']\nC = ['281', '282', '283', '284', '285', '286', '287', '288', '289', '322', '334', '335', '342', '344', '345', '411', '511', '514', '515', '516', '522', '523', '524', '531', '532', '551', '592', '621', '625', '629', '633', '634', '635', '641', '661', '662', '663', '664', '667', '689']\nD = ['611', '612', '613', '651', '652', '654', '655', '656', '657', '658', '659', '831', '841', '842', '843', '844', '845', '846', '848', '851']\nE = ['642', '665', '666', '673', '674', '675', '676', '677', '679', '691', '692', '693', '694', '695', '696', '697', '699', '821', '893', '894', '895', '897', '898', '899']\nF = ['713', '781', '782', '783', '784', '785']\nG = ['266', '267', '512', '513', '533', '553', '554', '562', '571', '572', '573', '574', '575', '579', '581', '582', '583', '591', '593', '597', '598', '653', '671', '672', '678', '786', '791', '882']\nH = ['711', '712', '714', '721', '722', '723', '724', '725', '726', '727', '728', '731', '733', '735', '737', '741', '742', '743', '744', '745', '746', '747', '748', '749', '761', '762', '763', '772', '773', '775', '778', '793', '811', '812', '813', '872', '873', '884', '885']\nI = ['751', '752', '759', '764', '771', '774', '776', '891']\nJ = ['525', '541', '542', '716', '718', '792', '871', '874', '881']\nK = ['351', '883', '892', '896', '961', '971']\nRB = B+C\nLT = D+E\nMT = F+G+H\nHT = I+J\nmanuf = RB+LT+MT+HT\nMHT = MT+HT\nyear_col='Year'\ncode_col='Commodity Code'\nvalue_col='Trade Value (US$)'\naggr_col='Aggregate Level'\ncountries = {\n\t'12': 'Algeria',\n\t'48': 'Bahrain',\n\t'818': 'Egypt',\n\t'275': 'Gaza and West Bank',\n\t'364': 'Iran',\n\t'368': 'Iraq',\n\t'376': 'Israel',\n\t'400': 'Jordan',\n\t'414': 'Kuwait',\n\t'422': 'Lebanon',\n\t'434': 'Libya',\n\t'504': 'Morocco',\n\t'512': 'Oman',\n\t'682': 'Saudi Arabia',\n\t'760': 'Syria',\n\t'788': 'Tunisia',\n\t'784': 'United Arab Emirates',\n\t'887': 'Yemen'\n}\none_dgt = {\n\t\"1\": {\"name\": \"Food, Beverages and Tobacco\", \"codes\":['016', '017' '023', '024', '035',\n\t\t'037', '046', '047', '048', '056', '058', '059', '061', '062', '073', '091', '098',\n\t\t'111', '112', '122', '411', '421', '422', '431']},\n\t\"2\": {\"name\": \"Textiles, wearing apparel and leather products\", 'codes':['264','265',\n\t\t'266', '267', '611', '612', '613', '651', '652', '653', '654', '655', '656', '657',\n\t\t'658', '659', '831', '841', '842', '843', '844', '845', '846', '847', '848', '851']},\n\t\"3\": {\"name\": \"Wood and Paper products\", 'codes':['247', '248', '633', '634', '635',\n\t\t'641', '642', '251']},\n\t\"4\": {\"name\": \"Metal products\", 'codes':['281', '282', '283', '284', '285', '286', '287',\n\t\t'288', '289', '671', '672', '673', '674', '675', '676', '677', '678', '679', '678',\n\t\t'689', '691', '692', '693', '694', '695', '696', '697', '698', '699']},\n\t\"5\": {\"name\": \"Coke, refined petroleum, non-metallic mineral products and rubber\",\n\t\t'codes':['232', '322', '334', '335', '621', '625', '629', '661', '662', '663', '664',\n\t\t'665', '666', '667', '342', '344', '345']},\n\t\"6\": {\"name\": \"Machinery, equipment and telecommunications\", 'codes':['711', '712', '713',\n\t\t'714','716', '718', '721', '722', '723', '724', '725', '726', '727', '728', '731',\n\t\t'733', '735', '737', '741', '742', '743', '744', '745', '746', '747', '748', '749',\n\t\t'751', '752', '759', '761', '762', '763', '764', '771', '772', '773', '774', '775',\n\t\t'776', '778', '871', '872', '873', '874', '881', '882', '883', '884', '885']},\n\t\"7\": {\"name\": \"Transport equipment\", 'codes':['781', '782', '783', '784', '785', '786',\n\t\t'791', '792', '793']},\n\t\"8\": {\"name\": \"Chemical and plastic products\", 'codes':['512', '513', '525', '533', '511',\n\t\t'514', '515', '516', '522', '523', '524', '531', '532', '541', '542', '551', '553',\n\t\t'554', '562', '571', '572', '573', '574', '575', '579', '581', '582', '583', '591',\n\t\t'592', '593', '597', '598', '893']},\n\t\"9\": {\"name\": \"N.E.S. (incl. furniture, recycling and publishing and printing )\",\n\t\t'codes':['811', '812', '813', '899', '894', '895', '896', '897', '898', '821', '891']}\n}\nisos = {\n\t'12': 'DZA',\n\t'48': 'BHR',\n\t'818': 'EGY',\n\t'275': 'PSE',\n\t'364': 'IRN',\n\t'368': 'IRQ',\n\t'376': 'ISR',\n\t'400': 'JOR',\n\t'414': 'KWT',\n\t'422': 'LBN',\n\t'434': 'LBY',\n\t'504': 'MAR',\n\t'512': 'OMN',\n\t'682': 'SAU',\n\t'760': 'SYR',\n\t'788': 'TUN',\n\t'784': 'ARE',\n\t'887': 'YEM'\n}\nfltrd = {\n\t'682': 'Saudi Arabia',\n\t'760': 'Syria',\n\t'788': 'Tunisia',\n\t'784': 'United Arab Emirates',\n\t'887': 'Yemen'\n}\n#--------------------------------- backup\ndef backup(typ):\n\ttimestamp = str(datetime.datetime.now()).split('.')\n\tno_ms = timestamp[0] # remove milliseconds\n\tday, time = no_ms.split(' ')\n\tif typ == 'comtrade':\n\t\tfolders = ('dataframe',)\n\telse:\n\t\tfolders = ('mht_by_manuf', 'primary_vs_manuf', 'trends_by_sector', 'trends_by_tech')\n\tout['output'] = []\n\tdestfolder = '..\/trade\/backup\/'+day\n\ttry:\n\t\tif not os.path.isdir(destfolder):\n\t\t\tos.makedirs(destfolder)\n\t\tfor folder in folders:\n\t\t\tcommand = 'zip -r '+destfolder+'\/'+folder+'.zip ..\/trade\/'+folder+'\/'\n\t\t\tpopen = os.popen(command)\n\t\t\tout['output'] += popen.readlines()\n\t\t\tstatus = popen.close()\n\t\t\tif status != None:\n\t\t\t\tbreak\n\t\tif status is None:\n\t\t\tout['status'] = 'success'\n\t\telse:\n\t\t\tout['status'] = status\n\texcept:\n\t\tout['status'] = 'error'\n\t\tout['error'] = str(sys.exc_info()[0])\n#------------------------------------- fetchComtradeAPIData((r='400', rg='2', cc='TOTAL')\n# Use Comtrade API (https:\/\/comtrade.un.org\/data\/doc\/api\/#DataRequests) to fetch data directly (via URL) to a DataFrame\n#\n# Export all 3 digit codes - https:\/\/comtrade.un.org\/api\/get?r=400&amp;px=S3&amp;ps=ALL&amp;p=0&amp;rg=2&amp;cc=AG3&amp;fmt=CSV&amp;type=C&amp;freq=A\n# Export Total - https:\/\/comtrade.un.org\/api\/get?r=400&amp;px=S3&amp;ps=ALL&amp;p=0&amp;rg=2&amp;cc=TOTAL&amp;fmt=CSV&amp;type=C&amp;freq=A\n# Re-export all 3 digit codes - https:\/\/comtrade.un.org\/api\/get?r=400&amp;px=S3&amp;ps=ALL&amp;p=0&amp;rg=3&amp;cc=AG3&amp;fmt=CSV&amp;type=C&amp;freq=A\n# Re-export Total - https:\/\/comtrade.un.org\/api\/get?r=400&amp;px=S3&amp;ps=ALL&amp;p=0&amp;rg=3&amp;cc=TOTAL&amp;fmt=CSV&amp;type=C&amp;freq=A\n#\n# r: reporting area - https:\/\/comtrade.un.org\/Data\/cache\/reporterAreas.json\n# rg: trade regime (2: exports, 3: re-exports)\n# cc: classification code (AG3: 3 digit SITC Rev 3, TOTAL: total)\ndef fetchComtradeAPIData(r='400', rg='2', cc='TOTAL'):\n\turl = 'https:\/\/comtrade.un.org\/api\/get?r='+r+'&amp;px=S3&amp;ps=ALL&amp;p=0&amp;rg='+rg+'&amp;cc='+cc+'&amp;fmt=CSV&amp;type=C&amp;freq=A'\n\t#print(url)\n\ttry:\n\t\tdf = pd.read_csv(url, dtype={code_col: str})\n\t\tdf[year_col]=df[year_col].astype(str)\n\t\treturn df\n\texcept:\n\t\tout['status'] = 'error'\n\t\tout['error'] = str(sys.exc_info()[0])\n\t\treturn None\n#------------------------------------- saveIfOk\ndef saveIfOk(sizes, df, country_title, filename):\n\tif df is not None:\n\t\tread_size = df.shape[0]\n\t\texp_size = sizes[country_title][filename]\n\t\tif (read_size == 1) and (exp_size &gt; 1):\n\t\t\tout['status'] = 'error'\n\t\t\tout['error'] = filename+' for '+country_title+' should not be empty'\n\t\t\treturn False\n\t\telse:\n\t\t\ttoCSV(df, '..\/trade\/dataframe\/'+country_title+'\/'+filename)\n\t\t\treturn True\n\telse:\n\t\treturn False # no need to set out['status'] as reaching this point means it has been already set by fetchComtradeAPIData\n#------------------------------------- fetchExportData(country)\ndef fetchExportData(country):\n\t#global df_exports_tot, df_exports_3_dgt, df_reexports_tot, df_reexports_3_dgt\n\tsizes = json.load(open('dataframe_sizes.json'))\n\tout['status'] = 'success'\n\tif country != 'none':\n\t\tcountry_title = countries[country]\n\t\tif (not os.path.isdir('..\/trade\/dataframe\/'+country_title)):\n\t\t\tos.makedirs('..\/trade\/dataframe\/'+country_title)\n\t\tdf_exports_tot = fetchComtradeAPIData(country, '2')\n\t\tif saveIfOk(sizes, df_exports_tot, country_title, 'exports_tot.csv'):\n\t\t\tdf_exports_3_dgt = fetchComtradeAPIData(country, '2', 'AG3')\n\t\t\tif saveIfOk(sizes, df_exports_3_dgt, country_title, 'exports_3_dgt.csv'):\n\t\t\t\tdf_reexports_tot = fetchComtradeAPIData(country, '3')\n\t\t\t\tif saveIfOk(sizes, df_reexports_tot, country_title, 'reexports_tot.csv'):\n\t\t\t\t\tdf_reexports_3_dgt = fetchComtradeAPIData(country, '3', 'AG3')\n\t\t\t\t\tsaveIfOk(sizes, df_reexports_3_dgt, country_title, 'reexports_3_dgt.csv')\n#------------------------------------- fetchImportData(country)\ndef fetchImportData(country):\n\t#global df_imports_tot, df_imports_3_dgt, df_reimports_tot, df_reimports_3_dgt\n\tsizes = json.load(open('dataframe_sizes.json'))\n\tout['status'] = 'success'\n\tif country != 'none':\n\t\tcountry_title = countries[country]\n\t\tif (not os.path.isdir('..\/trade\/dataframe\/'+country_title)):\n\t\t\tos.makedirs('..\/trade\/dataframe\/'+country_title)\n\t\tdf_imports_tot = fetchComtradeAPIData(country, '1')\n\t\tif saveIfOk(sizes, df_imports_tot, country_title, 'imports_tot.csv'):\n\t\t\tdf_imports_3_dgt = fetchComtradeAPIData(country, '1', 'AG3')\n\t\t\tif saveIfOk(sizes, df_imports_3_dgt, country_title, 'imports_3_dgt.csv'):\n\t\t\t\tdf_reimports_tot = fetchComtradeAPIData(country, '4')\n\t\t\t\tif saveIfOk(sizes, df_reimports_tot, country_title, 'reimports_tot.csv'):\n\t\t\t\t\tdf_reimports_3_dgt = fetchComtradeAPIData(country, '4', 'AG3')\n\t\t\t\t\tsaveIfOk(sizes, df_reimports_3_dgt, country_title, 'reimports_3_dgt.csv')\n#------------------------------------- getFltrd(df_3_dgt, fltr_on)\ndef getFltrd(df_3_dgt, fltr_on):\n\tdf_fltr = df_3_dgt[df_3_dgt[code_col].isin(fltr_on)]\n\treturn df_fltr[[year_col, value_col]].groupby(year_col).sum()\n#------------------------------------- getPct(df_tot, df_3_dgt, fltr_on, tot)\ndef getPct(df_tot, df_3_dgt, fltr_on, tot):\n\tdf_fltr_nmrtr = df_3_dgt[df_3_dgt[code_col].isin(fltr_on)]\n\t# TOTAL\n\tif tot == 'TOTAL':\n\t\tdf_fltr_denom = df_tot\n\telse:\n\t\tdf_fltr_denom = df_3_dgt[df_3_dgt[code_col].isin(tot)]\n\tglobal df_nmrtr, df_denom\n\tdf_nmrtr = df_fltr_nmrtr[[year_col, value_col]].groupby(year_col).sum()\n\tdf_denom = df_fltr_denom[[year_col, value_col]].groupby(year_col).sum()\n\t# group and sum\n\tdf_pct = df_nmrtr\/df_denom * 100\n\treturn df_nmrtr, df_denom, df_pct\n################################ FIGURE 1: Manufactured Goods against Primary Goods ########################################\n#------------------------------------- getFigure1Data(country)\ndef getFigure1Data(country):\n\tglobal df_exports_tot, df_exports_3_dgt, df_reexports_tot, df_reexports_3_dgt\n\tfolder = '..\/trade\/dataframe\/'+countries[country]+'\/'\n\tdf_exports_tot = pd.read_csv(folder+'exports_tot.csv')\n\tdf_exports_3_dgt = pd.read_csv(folder+'exports_3_dgt.csv', dtype={code_col: str})\n\tdf_reexports_tot = pd.read_csv(folder+'reexports_tot.csv')\n\tdf_reexports_3_dgt = pd.read_csv(folder+'reexports_3_dgt.csv', dtype={code_col: str})\n#------------------------------------- computeFigure1All(fltr_on, tot)\ndef computeFigure1All(fltr_on, tot):\n\tdf_nmrtr_exports, df_denom_exports, df_pct_exports = getPct(df_exports_tot, df_exports_3_dgt, fltr_on, tot)\n\tif df_reexports_tot.shape[0] &gt; 1:\n\t\tdf_nmrtr_reexports, df_denom_reexports, df_pct_reexports = getPct(df_reexports_tot, df_reexports_3_dgt, fltr_on, tot)\n\t\tdf_pct_netexports = df_nmrtr_exports.subtract(df_nmrtr_reexports, fill_value=0)\/df_denom_exports.subtract(df_denom_reexports, fill_value=0) * 100\n\telse:\n\t\tdf_pct_reexports = None\n\t\tdf_pct_netexports = df_pct_exports\n\treturn df_pct_exports, df_pct_reexports, df_pct_netexports\n#------------------------------------- saveFigure1Files\ndef saveFigure1Files(country):\n\tiso = isos[country]\n\tgetFigure1Data(country)\n\tout['status'] = 'success'\n\tdf_pct_exports_prmry_vs_tot, df_pct_reexports_prmry_vs_tot, df_pct_netexports_prmry_vs_tot =\n\t\tcomputeFigure1All(A, 'TOTAL')\n\tdf_pct_exports_manuf_vs_tot, df_pct_reexports_manuf_vs_tot, df_pct_netexports_manuf_vs_tot =\n\t\tcomputeFigure1All(manuf, 'TOTAL')\n\tdest='..\/trade\/primary_vs_manuf'\n\tfolder = dest+'\/'+iso\n\tif (not os.path.isdir(folder)):\n\t\tos.makedirs(folder)\n\ttoCSV(df_pct_exports_prmry_vs_tot, folder+'\/pct_exports_prmry_vs_tot.csv', True) # index=True\n\ttoCSV(df_pct_exports_manuf_vs_tot, folder+'\/pct_exports_manuf_vs_tot.csv', True)\n\tif (df_pct_reexports_prmry_vs_tot is not None) and (df_pct_reexports_prmry_vs_tot.shape[0] &gt; 1):\n\t\ttoCSV(df_pct_reexports_prmry_vs_tot, folder+'\/pct_reexports_prmry_vs_tot.csv', True)\n\t\ttoCSV(df_pct_netexports_prmry_vs_tot, folder+'\/pct_netexports_prmry_vs_tot.csv', True)\n\t\ttoCSV(df_pct_reexports_manuf_vs_tot, folder+'\/pct_reexports_manuf_vs_tot.csv', True)\n\t\ttoCSV(df_pct_netexports_manuf_vs_tot, folder+'\/pct_netexports_manuf_vs_tot.csv', True)\n################################ Figure 2 ######################################\n#------------------------------------- getFigure2Data(country)\ndef getFigure2Data(country):\n\tglobal df_exports_tot, df_exports_3_dgt, df_reexports_tot, df_reexports_3_dgt\n\tfolder = '..\/trade\/dataframe\/'+countries[country]+'\/'\n\tdf_exports_tot = pd.read_csv(folder+'exports_tot.csv')\n\tdf_exports_3_dgt = pd.read_csv(folder+'exports_3_dgt.csv', dtype={code_col: str})\n\tdf_reexports_tot = pd.read_csv(folder+'reexports_tot.csv')\n\tdf_reexports_3_dgt = pd.read_csv(folder+'reexports_3_dgt.csv', dtype={code_col: str})\n#------------------------------------- computeFigure2\ndef computeFigure2(fltr_on, tot):\n\tdf_nmrtr_exports, df_denom_exports, df_pct_exports = getPct(df_exports_tot, df_exports_3_dgt, fltr_on, tot)\n\tif df_reexports_tot.shape[0] &gt; 1:\n\t\tdf_nmrtr_reexports, df_denom_reexports, df_pct_reexports = getPct(df_reexports_tot, df_reexports_3_dgt, fltr_on, tot)\n\t\tdf_pct_netexports = df_nmrtr_exports.subtract(df_nmrtr_reexports, fill_value=0)\/df_denom_exports.subtract(df_denom_reexports, fill_value=0) * 100\n\telse:\n\t\tdf_pct_reexports = None\n\t\tdf_pct_netexports = df_pct_exports\n\treturn df_pct_exports, df_pct_reexports, df_pct_netexports\n#------------------------------------- computeFigure2Group\ndef computeFigure2Group(fltrd):\n\tfigure2 = {}\n\t# First get the data sets for each country, then compute the division (MHT\/manuf) and finally rename their \"Trade Value (US$)\" columns to country name\n\tfor country, iso in fltrd.items():\n\t\t#print(country, iso)\n\t\tgetFigure2Data(country)\n\t\tdf_exports_mht_vs_manuf, df_reexports_mht_vs_manuf, df_netexports_mht_vs_manuf =\n\t\t\tcomputeFigure2(MHT, manuf)\n\t\tdf_exports_mht_vs_manuf.rename(columns={\"Trade Value (US$)\": fltrd[country]}, inplace=True)\n\t\tif isinstance(df_reexports_mht_vs_manuf, pd.DataFrame) and (df_reexports_mht_vs_manuf.shape[0]&gt;1):\n\t\t\tdf_reexports_mht_vs_manuf.rename(columns={\"Trade Value (US$)\": fltrd[country]}, inplace=True)\n\t\t\tdf_netexports_mht_vs_manuf.rename(columns={\"Trade Value (US$)\": fltrd[country]}, inplace=True)\n\t\telse:\n\t\t\tdf_reexports_mht_vs_manuf = None\n\t\t\tdf_netexports_mht_vs_manuf = df_exports_mht_vs_manuf\n\t\tfigure2[country] = {\n\t\t\t'df_exports_mht_vs_manuf': df_exports_mht_vs_manuf,\n\t\t\t'df_reexports_mht_vs_manuf': df_reexports_mht_vs_manuf,\n\t\t\t'df_netexports_mht_vs_manuf': df_netexports_mht_vs_manuf\n\t\t}\n\t# Now combine the datasets for each country for chart\n\tkeys = list(figure2.keys())\n\tkeys.reverse\n\tkey = keys.pop()\n\tdf_exports_mht_vs_manuf = figure2[key]['df_exports_mht_vs_manuf']\n\tdf_reexports_mht_vs_manuf = figure2[key]['df_reexports_mht_vs_manuf']\n\tdf_netexports_mht_vs_manuf = figure2[key]['df_netexports_mht_vs_manuf']\n\tfor key in keys:\n\t\tdf_exports_mht_vs_manuf = df_exports_mht_vs_manuf.join(figure2[key]['df_exports_mht_vs_manuf'], how='outer')\n\t\tdf_netexports_mht_vs_manuf = df_netexports_mht_vs_manuf.join(figure2[key]['df_netexports_mht_vs_manuf'], how='outer')\n\t\tdf_reexports_mht_vs_manuf = joinDataframes(df_reexports_mht_vs_manuf, figure2[key]['df_reexports_mht_vs_manuf'])\n\t\t#if isinstance(figure2[key]['df_reexports_mht_vs_manuf'], pd.DataFrame) and\n\t\t#\t(figure2[key]['df_reexports_mht_vs_manuf'].shape[0]&gt;1):\n\t\t#\tdf_reexports_mht_vs_manuf = df_reexports_mht_vs_manuf.\n\t\t#\t\tjoin(figure2[key]['df_reexports_mht_vs_manuf'], how='outer')\n\treturn df_exports_mht_vs_manuf, df_reexports_mht_vs_manuf, df_netexports_mht_vs_manuf\n#------------------------------------- joinDataFrames(df1, df2)\ndef joinDataframes(df1, df2):\n\tdf1_ok = isinstance(df1, pd.DataFrame) and (df1.shape[0] &gt; 1)\n\tdf2_ok = isinstance(df2, pd.DataFrame) and (df2.shape[0] &gt; 1)\n\tif df1_ok and df2_ok:\n\t\treturn df1.join(df2, how='outer')\n\telif df1_ok:\n\t\treturn df1\n\telif df2_ok:\n\t\treturn df2\n\telse:\n\t\treturn None\n#------------------------------------- saveFigure2Files\ndef saveFigure2Files():\n\tout['status'] = 'success'\n\tdf_exports_mht_vs_manuf, df_reexports_mht_vs_manuf, df_netexports_mht_vs_manuf = computeFigure2Group(isos)\n\tfolder='..\/trade\/mht_by_manuf'\n\tif (not os.path.isdir(folder)):\n\t\tos.makedirs(folder)\n\ttoCSV(df_exports_mht_vs_manuf.T, folder+'\/exports_mht_vs_manuf.csv', 'iso')\n\ttoCSV(df_reexports_mht_vs_manuf.T, folder+'\/reexports_mht_vs_manuf.csv', 'iso')\n\ttoCSV(df_netexports_mht_vs_manuf.T, folder+'\/netexports_mht_vs_manuf.csv', 'iso')\n################################ Figure 3: #####################################\n#------------------------------------- getFigure3Data(country)\ndef getFigure3Data(country):\n\tglobal df_exports_3_dgt, df_reexports_3_dgt\n\tfolder = '..\/trade\/dataframe\/'+countries[country]+'\/'\n\tout['status'] = 'success'\n\tdf_exports_3_dgt = pd.read_csv(folder+'exports_3_dgt.csv', dtype={code_col: str})\n\tdf_reexports_3_dgt = pd.read_csv(folder+'reexports_3_dgt.csv', dtype={code_col: str})\n#------------------------------------- computeFigure3Group(fltrd=countries)\ndef computeFigure3Group(fltrd=countries):\n\tfigure3 = {}\n\tcategories = {'RB': RB, 'LT': LT, 'MT': MT, 'HT': HT}\n\tfor country in fltrd:\n\t\tfigure3[country]={ 'exports': {}, 'reexports': {}, 'netexports': {}}\n\t\tgetFigure3Data(country)\n\t\tfor key, category in categories.items():\n\t\t\tfigure3[country]['exports'][key] = getFltrd(df_exports_3_dgt, category)\n\t\t\tif df_reexports_3_dgt.shape[0] &gt; 1:\n\t\t\t\tfigure3[country]['reexports'][key] = getFltrd(df_reexports_3_dgt, category)\n\t\t\t\tfigure3[country]['netexports'][key] = getFltrd(df_exports_3_dgt, category).\n\t\t\t\t\tsubtract(getFltrd(df_reexports_3_dgt, category))\n\t\t\telse:\n\t\t\t\tfigure3[country]['reexports'][key] = None\n\t\t\t\tfigure3[country]['netexports'][key] = getFltrd(df_exports_3_dgt, category)\n\treturn figure3\n#------------------------------------- saveFigure3Files\ndef saveFigure3Files():\n\tfigure3 = computeFigure3Group(isos)\n\ttech_sctrs = ['RB', 'LT', 'MT', 'HT']\n\tout['status'] = 'success'\n\tfor country, datahash in figure3.items():\n\t\tiso = isos[country]\n\t\t#print(iso)\n\t\tfor typ in datahash.keys():\n\t\t\tif datahash[typ]['RB'] is not None:\n\t\t\t\t#print(typ)\n\t\t\t\tdest='..\/trade\/trends_by_tech'\n\t\t\t\tfolder = dest+'\/'+iso+'\/'+typ\n\t\t\t\tif (not os.path.isdir(folder)):\n\t\t\t\t\tos.makedirs(folder)\n\t\t\t\tfor tech_sctr in tech_sctrs:\n\t\t\t\t\ttoCSV(datahash[typ][tech_sctr], folder+'\/'+tech_sctr+'.csv', True)\n\t\t\t\t\t#datahash[typ][tech_sctr].to_csv(folder+'\/'+tech_sctr+'.csv')\n################################ Figure 4: #####################################\n#------------------------------------- getFigure4Data(country)\ndef getFigure4Data(country):\n\tglobal df_exports_3_dgt, df_reexports_3_dgt, df_imports_3_dgt, df_reimports_3_dgt\n\tfolder = '..\/trade\/dataframe\/'+countries[country]+'\/'\n\tdf_exports_3_dgt = pd.read_csv(folder+'exports_3_dgt.csv', dtype={code_col: str})\n\tdf_reexports_3_dgt = pd.read_csv(folder+'reexports_3_dgt.csv', dtype={code_col: str})\n\tdf_imports_3_dgt = pd.read_csv(folder+'imports_3_dgt.csv', dtype={code_col: str})\n\tdf_reimports_3_dgt = pd.read_csv(folder+'reimports_3_dgt.csv', dtype={code_col: str})\n#------------------------------------- computeFigure4Group(fltrd=countries)\ndef computeFigure4Group(fltrd=countries):\n\tfigure4 = {}\n\tcategories = {}\n\tfor category in one_dgt.keys():\n\t\tcategories[category] = one_dgt[category]['codes']\n\tfor country in fltrd:\n\t\tfigure4[country]={ 'exports': {}, 'reexports': {}, 'netexports': {},\n\t\t\t'imports': {}, 'reimports': {}, 'netimports': {}}\n\t\tgetFigure4Data(country)\n\t\tfor key, category in categories.items():\n\t\t\tfigure4[country]['exports'][key] = getFltrd(df_exports_3_dgt, category)\n\t\t\tif df_reexports_3_dgt.shape[0] &gt; 1:\n\t\t\t\tfigure4[country]['reexports'][key] = getFltrd(df_reexports_3_dgt, category)\n\t\t\t\tfigure4[country]['netexports'][key] = getFltrd(df_exports_3_dgt, category).\n\t\t\t\t\tsubtract(getFltrd(df_reexports_3_dgt, category))\n\t\t\telse:\n\t\t\t\tfigure4[country]['reexports'][key] = None\n\t\t\t\tfigure4[country]['netexports'][key] = getFltrd(df_exports_3_dgt, category)\n\t\t\tfigure4[country]['imports'][key] = getFltrd(df_imports_3_dgt, category)\n\t\t\tif df_reimports_3_dgt.shape[0] &gt; 1:\n\t\t\t\tfigure4[country]['reimports'][key] = getFltrd(df_reimports_3_dgt, category)\n\t\t\t\tfigure4[country]['netimports'][key] = getFltrd(df_imports_3_dgt, category).\n\t\t\t\t\tsubtract(getFltrd(df_reimports_3_dgt, category))\n\t\t\telse:\n\t\t\t\tfigure4[country]['reimports'][key] = None\n\t\t\t\tfigure4[country]['netimports'][key] = getFltrd(df_imports_3_dgt, category)\n\treturn figure4\n#------------------------------------- saveFigure4Files\ndef saveFigure4Files():\n\tout['status'] = 'success'\n\tfigure4 = computeFigure4Group(isos)\n\tone_dgt_sctrs = list(map(str, range(1,10)))\n\tfor country, datahash in figure4.items():\n\t\tiso = isos[country]\n\t\t#print(iso)\n\t\tfor typ in datahash.keys():\n\t\t\tif datahash[typ]['1'] is not None:\n\t\t\t\t#print(typ)\n\t\t\t\tdest='..\/trade\/trends_by_sector'\n\t\t\t\tfolder = dest+'\/'+iso+'\/'+typ\n\t\t\t\tif (not os.path.isdir(folder)):\n\t\t\t\t\tos.makedirs(folder)\n\t\t\t\tfor sctr in one_dgt_sctrs:\n\t\t\t\t\ttoCSV(datahash[typ][sctr], folder+'\/'+sctr+'.csv', True)\n\t\t\t\t\t#datahash[typ][sctr].to_csv(folder+'\/'+sctr+'.csv')\n#------------------------------------- toCSV\ndef toCSV(df, fil, index=False):\n\tif isinstance(df, pd.DataFrame):\n\t\tcheckAndRemove(fil)\n\t\tif type(index) is bool:\n\t\t\tdf.to_csv(fil, index=index)\n\t\telse: # if index is not True\/False it is the index_label\n\t\t\tdf.to_csv(fil, index_label=index)\n#------------------------------------- checkAndRemove(fil)\ndef checkAndRemove(fil):\n\tif os.path.exists(fil):\n\t\tos.remove(fil)\n#------------------------------------- saveStatus()\ndef saveStatus():\n\tstatus = {}\n\tstatus['status'] = form.getvalue('status', 'none')\n\tstatus['date'] = str(datetime.datetime.today()).split(' ')[0]\n\tif not status['status'] in ('none', 'success'):\n\t\tstatus['country'] = form.getvalue('country', 'none')\n\ttry:\n\t\tjson.dump(status, open('comtrade_status.json','w'))\n\texcept:\n\t\tout['status'] = 'error'\n\t\tout['error'] = str(sys.exc_info()[0])\n#--------------------------------- MAIN\nprint(\"Content-type: text\/json\")\nprint()\nform = cgi.FieldStorage(keep_blank_values=True)\naction = form.getvalue('action', 'none')\n#folder = form.getvalue('folder','..\/instance')\nout= { 'action': action }\nif action == 'backup_comtrade':\n\tbackup('comtrade')\nelif action == 'backup_chartdata':\n\tbackup('chartdata')\nelif action == 'fetch_exports':\n\tcountry = form.getvalue('country', 'none')\n\tout['country'] = country\n\tfetchExportData(country)\nelif action == 'fetch_imports':\n\tcountry = form.getvalue('country', 'none')\n\tout['country'] = country\n\tfetchImportData(country)\nelif action == 'fig1':\n\tcountry = form.getvalue('country', 'none')\n\tout['country'] = country\n\tsaveFigure1Files(country)\nelif action == 'fig2':\n\tsaveFigure2Files()\nelif action == 'fig3':\n\tsaveFigure3Files()\nelif action == 'fig4':\n\tsaveFigure4Files()\nelif action == 'save_status':\n\tsaveStatus()\nelse:\n\tout['error'] = \"No valid action specified: \"+action\nprint(json.dumps(out, indent=3))\n<\/code><\/pre>\n<p><a>UNIDO<\/a><\/p>\n<pre><code>\n#!\/home\/huzaifa\/anaconda3\/bin\/python3\n#\/usr\/bin\/python3\n# Name: update_unido.py\n# Created By: Huzaifa Zoomkawala\n# Last Modified: September 4, 2021\nimport cgi\nimport cgitb; cgitb.enable()  # for troubleshooting\nimport datetime\nimport os\nimport os.path\nimport time\nimport json\nimport pandas as pd\nimport shutil\nimport sys\n#------------------------------------ Global Variables\ncolset = {\n\t'indstat': {'Table Code', 'Country Code', 'Year', 'ISIC', 'ISIC Combination', 'Value', 'Table Definition Code', 'Source Code', 'Unit'},\n\t'mva': {'Variable Code', 'Country Code', 'Year', 'Value', 'Unit'}\n}\nvarcols = { 'indstat': 'Table Code', 'mva': 'Variable Code' }\nvarset = {\n\t'indstat': {'01', '04', '20', '31'},\n\t'mva': {'GdpCud', 'MvaCud', 'GdpCod', 'MvaCod', 'Pop'}\n}\nunits = { '#', '$'}\nminsize = { 'indstat2' : 50e3, 'indstat4': 100e3, 'indstat4_4': 48e3, 'mva': 3e3 }\ncountries = {\n\t'indstat': {'048', '368', '422', '504', '414', '012', '434', '760', '792', '682',\n\t'275', '512', '887', '784', '400', '788', '376', '818', '364', '634'},\n\t'mva': {'048', '368', '422', '504', '414', '012', '434', '760', '792', 'WOR',\n\t\t'682', '275', '512', '887', '784', '400', '788', '376', '818', '364', '634'}\n}\nisic_set = {\n\t'indstat2': {'22', '26', '25', '33', '19', '36', '29', '34', '35', '15', '20',\n\t'30', '16', '37', 'D', '28', '23', '27', '18', '31', '32', '24', '21', '17'},\n\t'indstat4_4': {'2811', '2520', '3319', '1702', '1062', '1075', '1313', '2310',\n\t'3091', '1061', '2652', '2750', '239', '2030', '2660', '2815', '2610', '2910',\n\t'1623', '1104', '2591', '2431', '321', '1511', '2393', '1071', '243', '2930',\n\t'1512', '162', '251', '2211', '2816', '3012', '309', '2821', '1920', '301',\n\t'2100', '259', '2822', '1010', '1073', '3020', '1311', '2731', '2710', '273',\n\t'2732', '1391', '1811', '2012', '3290', '1080', '3311', '3211', '2513', '2511',\n\t'281', '2029', '106', '3212', '1520', '3250', '3312', '3313', '3092', '1709',\n\t'2817', '1050', '2512', '2640', '1910', '2396', '170', '2219', '1102', '181',\n\t'1072', '1393', '1200', '1410', '131', '139', '3011', '1622', '3100', '2825',\n\t'2392', '2733', '1101', '282', '110', '2021', '2395', '1392', '1040', '2829',\n\t'2399', '3030', '1621', '2630', '1312', '3230', '331', '2818', '2013', '151',\n\t'2599', '221', '201', '2819', '1430', '3320', '2011', '1610', '2432', '2790',\n\t'2420', 'C', '1820', '2220', '1394', '3220', '2826', '1103', '1812', '2391',\n\t'2920', '3315', '1020', '2022', '1079', '2410', '3240', '2720', '3040', '3099',\n\t'1420', '3314', '2620', '2813', '1629', '2812', '107', '1701', '2651', '2823',\n\t'2592', '2593', '2824', '2023', '1030', '265', '2680', '202', '1074', '1399',\n\t'2394', '2740', '2670', '2814'},\n\t'indstat4': {'2694', '1553', '3430', '2924', '2811', '1532', '2520', '3190',\n\t'2925', '2310', '2610', '2330', '2413', '2919', '1511', '2930', '1512', '251',\n\t'3693', '2211', '242', '2926', 'D', '1920', '3511', '1911', '2109', '1730',\n\t'1810', '2731', '191', '2913', '2710', '273', '2732', '3520', '1711', '3610',\n\t'3311', '3140', '3120', '359', '2511', '3591', '3699', '281', '292', '210',\n\t'1549', '1721', '2029', '1722', '1552', '1520', '2320', '2693', '3312', '3313',\n\t'2102', '3530', '241', '3210', '3599', '2691', '2915', '369', '2219', '2922',\n\t'153', '2912', '2101', '3130', '2230', '2222', '2927', '2421', '1729', '1531',\n\t'1543', '3420', '3692', '2010', '3710', '2021', '2411', '2412', '2929', '3694',\n\t'222', '2892', '3230', '331', '151', '1912', '1600', '221', '2914', '172',\n\t'3320', '2692', '2696', '3512', '3150', '1820', '1723', '2921', '2891',\n\t'1514', '3220', '2422', '2429', '3720', '2695', '3410', '1533', '1712', '2423',\n\t'3000', '1541', '3592', '2022', '2893', '2430', '2923', '2424', '3691', '2221',\n\t'2519', '2720', '269', '1551', '351', '3330', '1554', '2212', '2813', '2899',\n\t'155', '171', '2812', '1542', '154', '2023', '1513', '291', '202', '2699',\n\t'3110', '289', '2911', '1544', '2213'}\n}\n# for convertTech and related routines\nisic_indstat = dict(\n\tindstat2 = dict(\n\t\tlt = ['15', '16', '17', '18', '19', '20', '21', '22', '36', '37'],\n\t\tml = ['23', '25', '26', '27', '28'],\n\t\tmht = ['24', '29', '30', '31', '32', '33', '34'],\n\t\tlt_incl = [],\n\t\tml_incl = [],\n\t\tmht_incl = [],\n\t\tlt_excl = [],\n\t\tml_excl = [],\n\t\tmht_excl = []\n\t),\n\tindstat4 = dict(\n\t\tlt = ['15', '16', '17', '18', '19', '20', '21', '22', '36', '37'],\n\t\tml = ['23', '25', '26', '27', '28'],\n\t\tmht = ['24', '29', '30', '31', '32', '33', '34', '35'],\n\t\tlt_incl = [],\n\t\tml_incl = ['351'],\n\t\tmht_incl = [],\n\t\tlt_excl = [],\n\t\tml_excl = [],\n\t\tmht_excl = ['351']\n\t),\n\tindstat4_4 = dict(\n\t\tlt = ['10', '11', '12', '13', '14', '15', '16', '17', '18', '25', '31'],\n\t\tml = ['22', '23', '24', '32', '33'],\n\t\tmht = ['20', '21', '26', '27', '28', '29', '30'],\n\t\tlt_incl = [],\n\t\tml_incl = ['301'],\n\t\tmht_incl = ['252', '325'],\n\t\tlt_excl = ['252'],\n\t\tml_excl = ['325'],\n\t\tmht_excl = ['301']\n\t)\n)\n#------------------------------------ checkDB\ndef checkDB(dbtype, df):\n\tout['status'] = 'error'\n\tout['error'] = 'unspecified'\n\tif dbtype.startswith('indstat'):\n\t\trefcols = colset['indstat']\n\t\tvarcol = varcols['indstat']\n\t\tvarlist = varset['indstat']\n\t\trefcountries = countries['indstat']\n\telse:\n\t\trefcols = colset['mva']\n\t\tvarcol = varcols['mva']\n\t\tvarlist = varset['mva']\n\t\trefcountries = countries['mva']\n\tcols = set(df.columns)\n\tif refcols.issubset(cols):\n\t\tif minsize[dbtype] &lt; df.shape[0]: df_countries = set(df['Country Code']) if dbtype == 'indstat4_4': countries_cond = df_countries.issubset(refcountries) else: countries_cond = (df_countries == refcountries) if countries_cond: df_vars = set(df[varcol]) if df_vars == varlist: df_units = set(df['Unit']) if df_units == units: if dbtype == 'mva': out['status'] = 'success' else: df_isics = set(df['ISIC']) refisics = isic_set[dbtype] len_common = len(refisics.intersection(df_isics)) len_all = len(refisics) pct_intersect = round(len_common\/len_all * 100) if pct_intersect &gt; 85:\n\t\t\t\t\t\t\t\tout['status'] = 'success'\n\t\t\t\t\t\t\telse:\n\t\t\t\t\t\t\t\tout['error'] = 'ISIC codes in data file do not match those for '+dbtype\n\t\t\t\t\telse:\n\t\t\t\t\t\tout['error'] =\t'Unit for data file is not correct. Make sure US Dollar is chosen instead of Local Currency'\n\t\t\t\telse:\n\t\t\t\t\tout['error'] =\t'Variables in data file are not chosen according to specification for '+dbtype\n\t\t\telse:\n\t\t\t\tif dbtype == 'mva':\n\t\t\t\t\tout['error'] = 'Countries in data set should only be from MENA region or the World (as a region)'\n\t\t\t\telse:\n\t\t\t\t\tout['error'] = 'Countries in data set should only be from MENA region'\n\t\telse:\n\t\t\tout['error'] =\tf'Too few rows in data file: Expected minimum {minsize[dbtype]:,.0f} and found {df.shape[0]:,.0f}'\n\telse:\n\t\tout['error'] = 'Columns in data file not correct: please check specification for '+dbtype\n#--------------------------------- saveAsCSV\ndef saveAsCSV(dbtype, df):\n\tdf_country = pd.read_csv('country.csv', dtype={'numeric': str})\n\tdf_country.set_index('numeric', inplace=True)\n\tout['status'] == 'success'\n\tdf_with_iso = df.merge(df_country, left_on='Country Code', right_on=df_country.index, how='left')\n\tif dbtype == 'mva':\n\t\tdf_mva = df_with_iso[['Year', 'iso', 'Variable Code', 'Value']]\n\t\tdf_mva.rename(columns={'Year': 'year', 'Variable Code': 'var', 'Value': 'value'}, inplace=True)\n\t\t# recode 'var\/Variable Code'\n\t\tdf_mva.loc[df_mva['var']=='GdpCud', 'var']='gdp'\n\t\tdf_mva.loc[df_mva['var']=='MvaCud', 'var']='mva'\n\t\tdf_mva.loc[df_mva['var']=='GdpCod', 'var']='gdpcons'\n\t\tdf_mva.loc[df_mva['var']=='MvaCod', 'var']='mvacons'\n\t\tdf_mva.loc[df_mva['var']=='Pop', 'var']='pop'\n\t\tdf_mva.to_csv(datafolder+'\/mva.csv', index=False)\n\telse: # indstat2 or indstat4 or indstat4_4\n\t\tif dbtype == 'indstat2':\n\t\t\tisic = 'isic2'\n\t\telse:\n\t\t\tisic = 'isic4'\n\t\tdf_indstat = df_with_iso[['Year', 'iso', 'Table Code', 'ISIC', 'Value']]\n\t\tdf_indstat.rename(columns={'Year': 'year', 'Table Code': 'var', 'ISIC': isic, 'Value': 'value'}, inplace=True)\n\t\t# recode 'var\/Variable Code'\n\t\tdf_indstat.loc[df_indstat['var']=='01','var']='est'\n\t\tdf_indstat.loc[df_indstat['var']=='04','var']='emp'\n\t\tdf_indstat.loc[df_indstat['var']=='20','var']='va'\n\t\tdf_indstat.loc[df_indstat['var']=='31','var']='fem'\n\t\tdf_indstat.to_csv(datafolder+'\/'+dbtype+'.csv', index=False)\n#--------------------------------- listFiles\ndef listFiles():\n\tout['files'] = { fil.split('.csv')[0]: time.strftime('%m\/%d\/%Y', time.gmtime(os.path.getmtime(datafolder+'\/'+fil)))\n\t\tfor fil in filter(lambda x: x.endswith(\"csv\"), os.listdir(datafolder)) }\n\tout['status'] = 'success'\n#--------------------------------- get_specifics (for convertTech)\ndef get_specifics(indstat, group):\n\tif indstat == '2':\n\t\tdf_indstat = df_indstat2\n\t\toverall_group = 'D'\n\t\tisic_col = 'isic2'\n\t\tisic_maxlen = 2\n\telif indstat == '4':\n\t\tdf_indstat = df_indstat4\n\t\toverall_group = 'D'\n\t\tisic_col = 'isic4'\n\t\tisic_maxlen = 4\n\telse: # indstat == '4_4':\n\t\tdf_indstat = df_indstat4_4\n\t\toverall_group = 'C'\n\t\tisic_col = 'isic4'\n\t\tisic_maxlen = 4\n\tif group == 'overall':\n\t\tcond = df_indstat[isic_col] == overall_group\n\t\tcond_add = None\n\telse: # group == 'lt' or 'mt' or 'mht':\n\t\tcond = df_indstat[isic_col].str[:2].isin(isic_indstat['indstat'+indstat][group]) &amp;\n\t\t\t   (df_indstat[isic_col].str.len() == isic_maxlen) &amp;\n\t\t\t   ~df_indstat[isic_col].str[:3].isin(isic_indstat['indstat'+indstat][group+'_excl'])\n\t\tcond_add = df_indstat[isic_col].isin(isic_indstat['indstat'+indstat][group+'_incl'])\n\treturn df_indstat, cond, cond_add, isic_col\n#--------------------------------- calc_shr_add_indemp (for ConvertTech)\n### additional (included) ISICs\ndef calc_shr_add_indemp(df_indstat, cond_add, isic_col, ind='fem'):\n\tdf_add_ind = df_indstat[(df_indstat['iso'] != 'WLD') &amp; (df_indstat['var'] == ind) &amp; cond_add]\n\tdf_add_emp = df_indstat[(df_indstat['iso'] != 'WLD') &amp; (df_indstat['var'] == 'emp') &amp; cond_add]\n\tdf_add_ind.rename(columns = {isic_col:'isic'}, inplace = True)\n\tdf_add_emp.rename(columns = {isic_col:'isic'}, inplace = True)\n\tdf_add_ind = df_add_ind[['year', 'iso', 'isic', 'value']].set_index(['year', 'iso', 'isic'])\n\tdf_add_emp = df_add_emp[['year', 'iso', 'isic', 'value']].set_index(['year', 'iso', 'isic'])\n\treturn df_add_ind, df_add_emp\n#----------------------------------- makeTSCopy\ndef makeTSCopy(folder, filename):\n\ttimestamp = str(datetime.datetime.now()).split('.')\n\tno_ms = timestamp[0] # remove milliseconds\n\tday, time = no_ms.split(' ')\n\tminute = ':'.join(time.split(':')[:2])\n\tdest = folder+'\/backup\/'+day+' '+minute\n\tif not os.path.isdir(dest):\n\t\tos.makedirs(dest)\n\tif os.path.isfile(folder+'\/'+filename):\n\t\tshutil.copyfile(folder+'\/'+filename, dest+'\/'+filename)\n#--------------------------------- calc_shr_indemp (for ConvertTech)\ndef calc_shr_indemp(indstat, group, ind='fem', pct=1):\n\tdf_indstat, cond, cond_add, isic_col = get_specifics(indstat, group)\n\tdf_ind = df_indstat[(df_indstat['iso'] != 'WLD') &amp; (df_indstat['var'] == ind) &amp; cond]\n\tdf_emp = df_indstat[(df_indstat['iso'] != 'WLD') &amp; (df_indstat['var'] == 'emp') &amp; cond]\n\tif group == 'overall':\n\t\tfields_include = ['year', 'iso', 'value']\n\t\tfields_index = ['year', 'iso']\n\t\tdf_ind = df_ind[fields_include].set_index(fields_index)\n\t\tdf_emp = df_emp[fields_include].set_index(fields_index)\n\telse:\n\t\tdf_ind['isic'] = df_ind[isic_col].str[:2]\n\t\tdf_ind = df_ind.groupby(['year', 'iso', 'isic']).sum()\n\t\tdf_emp['isic'] = df_emp[isic_col].str[:2]\n\t\tdf_emp = df_emp.groupby(['year', 'iso', 'isic']).sum()\n\tdf_shr_indemp = ((df_ind\/df_emp) * pct).dropna()\n\tif group != 'overall':\n\t\tdf_add_ind, df_add_emp = calc_shr_add_indemp(df_indstat, cond_add, isic_col, ind)\n\t\tif df_add_ind.shape[0] &gt; 0:\n\t\t\tdf_shr_indemp = df_shr_indemp.append(((df_add_ind\/df_add_emp) * pct).dropna())\n\t\t\tdf_ind = df_ind.append(df_add_ind)\n\t\t\tdf_emp = df_emp.append(df_add_emp)\n\t\t# Add group total\n\t\tdf_shr_indemp_tot = (df_ind.groupby(['year', 'iso']).sum()\/df_emp.groupby(['year', 'iso']).sum()*pct).dropna()\n\t\tdf_shr_indemp_tot['isic'] = group\n\t\tdf_shr_indemp_tot.reset_index(inplace=True)\n\t\tdf_shr_indemp_tot.set_index(['year','iso', 'isic'], inplace=True)\n\t\tdf_shr_indemp = df_shr_indemp.append(df_shr_indemp_tot)\n\tdestfolder = tblnfolder+'\/indstat'+indstat\n\tif not os.path.isdir(destfolder):\n\t\tos.makedirs(destfolder)\n\tfilename = ind+'_emp_'+group+'.csv'\n\tmakeTSCopy(destfolder, filename)\n\tdf_shr_indemp.to_csv(destfolder+'\/'+filename)\n\t#return df_shr_indemp\n#--------------------------------- convertTech\ndef convertTech():\n\tindstats = ('2', '4', '4_4')\n\tgroups = ('overall', 'lt', 'ml', 'mht')\n\tfor ind in ('va', 'fem'):\n\t\tif ind == 'va':\n\t\t\tpct = 1\n\t\telse:\n\t\t\tpct = 100\n\t\tfor indstat in indstats:\n\t\t\tfor group in groups:\n\t\t\t\tcalc_shr_indemp(indstat, group, ind, pct)\n#--------------------------------- MVA_MVACONS (convertNoTech)\ndef MVA_MVACONS(destfolder):\n\tfor var in ['mva', 'mvacons']:\n\t\tdf_mva_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == var)]\n\t\tfilename = var+'.csv'\n\t\tmakeTSCopy(destfolder, filename)\n\t\tdf_mva_fltrd[['year', 'iso', 'value']].to_csv(destfolder+'\/'+filename, index=None)\n#--------------------------------- MVA_GrowthRate (convertNoTech)\ndef MVA_GrowthRate(destfolder):\n\tdf_mva_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'mvacons')]\n\tdf_mva_fltrd = df_mva_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\tdf_growth = (df_mva_fltrd \/ df_mva_fltrd.shift() * 100).dropna()\n\tdf_growth.reset_index(inplace=True)\n\tmin_year = min(df_growth['year'])\n\tdf_growth_shift = df_growth[df_growth['year']!=min_year]\n\tfilename = 'mva_growth.csv'\n\tmakeTSCopy(destfolder, filename)\n\tdf_growth_shift.to_csv(destfolder+'\/'+filename, index=False)\n#--------------------------------- GDP_GrowthRate (convertNoTech)\ndef GDP_GrowthRate(destfolder):\n\tdf_gdp_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'gdpcons')]\n\tdf_gdp_fltrd = df_gdp_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\tdf_growth = (df_gdp_fltrd \/ df_gdp_fltrd.shift() * 100).dropna()\n\tdf_growth.reset_index(inplace=True)\n\tmin_year = min(df_growth['year'])\n\tdf_growth_shift = df_growth[df_growth['year']!=min_year]\n\tfilename = 'gdp_growth.csv'\n\tmakeTSCopy(destfolder, filename)\n\tdf_growth_shift.to_csv(destfolder+'\/'+filename, index=False)\n#--------------------------------- Pop_GrowthRate (convertNoTech)\ndef Pop_GrowthRate(destfolder):\n\tdf_pop_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'pop')]\n\tdf_pop_fltrd = df_pop_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\tdf_growth = (df_pop_fltrd \/ df_pop_fltrd.shift() * 100).dropna()\n\tdf_growth.reset_index(inplace=True)\n\tmin_year = min(df_growth['year'])\n\tdf_growth_shift = df_growth[df_growth['year']!=min_year]\n\tfilename = 'pop_growth.csv'\n\tmakeTSCopy(destfolder, filename)\n\tdf_growth_shift.to_csv(destfolder+'\/'+filename, index=False)\n#--------------------------------- ShareofMVAinGDP (convertNoTech)\ndef ShareofMVAinGDP(destfolder):\n\tdf_mva_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'mva')]\n\tdf_gdp_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'gdp')]\n\tdf_mva_fltrd = df_mva_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\tdf_gdp_fltrd = df_gdp_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\tdf_shr_mva_gdp = df_mva_fltrd\/df_gdp_fltrd * 100\n\tfilename = 'share_mva_gdp.csv'\n\tmakeTSCopy(destfolder, filename)\n\tdf_shr_mva_gdp.to_csv(destfolder+'\/'+filename)\n#--------------------------------- MVAperCapita (convertNoTech)\ndef MVAperCapita(destfolder):\n\tdf_mva_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'mva')]\n\tdf_pop_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'pop')]\n\tdf_mva_fltrd = df_mva_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\tdf_pop_fltrd = df_pop_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\tdf_mva_pcap = df_mva_fltrd\/df_pop_fltrd\n\tfilename = 'mva_pcap.csv'\n\tmakeTSCopy(destfolder, filename)\n\tdf_mva_pcap.to_csv(destfolder+'\/'+filename)\n#--------------------------------- MVACONSperCapita (convertNoTech)\ndef MVACONSperCapita(destfolder):\n\tdf_mva_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'mvacons')]\n\tdf_pop_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'pop')]\n\tdf_mva_fltrd = df_mva_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\tdf_pop_fltrd = df_pop_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\tdf_mva_pcap = df_mva_fltrd\/df_pop_fltrd\n\tfilename = 'mvacons_pcap.csv'\n\tmakeTSCopy(destfolder, filename)\n\tdf_mva_pcap.to_csv(destfolder+'\/'+filename)\n#--------------------------------- GDPperCapita (convertNoTech)\ndef GDPperCapita(destfolder):\n\tdf_gdp_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'gdp')]\n\tdf_pop_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'pop')]\n\tdf_gdp_fltrd = df_gdp_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\tdf_pop_fltrd = df_pop_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\tdf_gdp_pcap = df_gdp_fltrd\/df_pop_fltrd\n\tfilename = 'gdp_pcap.csv'\n\tmakeTSCopy(destfolder, filename)\n\tdf_gdp_pcap.to_csv(destfolder+'\/'+filename)\n#--------------------------------- GDPCONSperCapita (convertNoTech)\ndef GDPCONSperCapita(destfolder):\n\tdf_gdp_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'gdpcons')]\n\tdf_pop_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'pop')]\n\tdf_gdp_fltrd = df_gdp_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\tdf_pop_fltrd = df_pop_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\tdf_gdp_pcap = df_gdp_fltrd\/df_pop_fltrd\n\tfilename = 'gdpcons_pcap.csv'\n\tmakeTSCopy(destfolder, filename)\n\tdf_gdp_pcap.to_csv(destfolder+'\/'+filename)\n#--------------------------------- ShareWorldMVA (convertNoTech)\ndef ShareWorldMVA(destfolder):\n\tdf_mva_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'mva')]\n\tdf_mva_wld = df_mva[(df_mva['iso'] == 'WLD') &amp; (df_mva['var'] == 'mva')]\n\tdf_shr_mva_wld = pd.DataFrame(columns = ['year', 'iso', 'value'])\n\tdf_mva_wld = df_mva_wld[['year', 'value']].set_index(['year'])\n\tisos = df_mva_fltrd.iso.unique()\n\tfor iso in isos:\n\t\tdf_mva_iso = df_mva_fltrd[df_mva_fltrd.iso == iso][['year', 'value']].set_index(['year'])\n\t\tdf_share_wld_iso = df_mva_iso\/df_mva_wld * 100\n\t\tdf_share_wld_iso['iso'] = iso\n\t\tdf_share_wld_iso.reset_index(inplace=True)\n\t\tdf_shr_mva_wld = df_shr_mva_wld.append(df_share_wld_iso)\n\tfilename = 'mva_share_world.csv'\n\tmakeTSCopy(destfolder, filename)\n\tdf_shr_mva_wld.to_csv(destfolder+'\/'+filename, index=False)\n#--------------------------------- ShareWorldGDP (convertNoTech)\ndef ShareWorldGDP(destfolder):\n\tdf_gdp_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'gdp')]\n\tdf_gdp_wld = df_mva[(df_mva['iso'] == 'WLD') &amp; (df_mva['var'] == 'gdp')]\n\tdf_shr_gdp_wld = pd.DataFrame(columns = ['year', 'iso', 'value'])\n\tdf_gdp_wld = df_gdp_wld[['year', 'value']].set_index(['year'])\n\tisos = df_gdp_fltrd.iso.unique()\n\tfor iso in isos:\n\t\tdf_gdp_iso = df_gdp_fltrd[df_gdp_fltrd.iso == iso][['year', 'value']].set_index(['year'])\n\t\tdf_share_wld_iso = df_gdp_iso\/df_gdp_wld * 100\n\t\tdf_share_wld_iso['iso'] = iso\n\t\tdf_share_wld_iso.reset_index(inplace=True)\n\t\tdf_shr_gdp_wld = df_shr_gdp_wld.append(df_share_wld_iso)\n\tfilename = 'gdp_share_world.csv'\n\tmakeTSCopy(destfolder, filename)\n\tdf_shr_gdp_wld.to_csv(destfolder+'\/'+filename, index=False)\n#--------------------------------- ShareWorldPop (convertNoTech)\ndef ShareWorldPop(destfolder):\n\tdf_pop_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == 'pop')]\n\tdf_pop_wld = df_mva[(df_mva['iso'] == 'WLD') &amp; (df_mva['var'] == 'pop')]\n\tdf_shr_pop_wld = pd.DataFrame(columns = ['year', 'iso', 'value'])\n\tdf_pop_wld = df_pop_wld[['year', 'value']].set_index(['year'])\n\tisos = df_pop_fltrd.iso.unique()\n\tfor iso in isos:\n\t\tdf_pop_iso = df_pop_fltrd[df_pop_fltrd.iso == iso][['year', 'value']].set_index(['year'])\n\t\tdf_share_wld_iso = df_pop_iso\/df_pop_wld * 100\n\t\tdf_share_wld_iso['iso'] = iso\n\t\tdf_share_wld_iso.reset_index(inplace=True)\n\t\tdf_shr_pop_wld = df_shr_pop_wld.append(df_share_wld_iso)\n\tfilename = 'pop_share_world.csv'\n\tmakeTSCopy(destfolder, filename)\n\tdf_shr_pop_wld.to_csv(destfolder+'\/'+filename, index=False)\n#--------------------------------- MVA_MVACONSperEmp (convertNoTech)\ndef MVA_MVACONSperEmp(tblnfolder):\n\tfor var in ['mva', 'mvacons']:\n\t\tfilename = var+'_emp.csv'\n\t\t## indstat2\n\t\tdf_mva_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == var)]\n\t\tdf_emp_indstat2 = df_indstat2[(df_indstat2['iso'] != 'WLD') &amp; (df_indstat2['var'] == 'emp') &amp;\n\t\t\t\t\t\t\t\t\t  (df_indstat2['isic2'] == 'D')]\n\t\tdf_emp_indstat2=df_emp_indstat2[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\t\tdf_mva_fltrd = df_mva_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\t\tdestfolder = tblnfolder+'\/indstat2'\n\t\tmakeTSCopy(destfolder, filename)\n\t\t(df_mva_fltrd\/df_emp_indstat2).to_csv(destfolder+'\/'+filename)\n\t\t## indstat4\n\t\tdf_mva_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == var)]\n\t\tdf_emp_indstat4 = df_indstat4[(df_indstat4['iso'] != 'WLD') &amp; (df_indstat4['var'] == 'emp') &amp;\n\t\t\t\t\t\t\t\t\t  (df_indstat4['isic4'] == 'D')]\n\t\tdf_emp_indstat4=df_emp_indstat4[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\t\tdf_mva_fltrd = df_mva_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\t\tdestfolder = tblnfolder+'\/indstat4'\n\t\tmakeTSCopy(destfolder, filename)\n\t\t(df_mva_fltrd\/df_emp_indstat4).to_csv(destfolder+'\/'+filename)\n\t\t## indstat4_4\n\t\tdf_mva_fltrd = df_mva[(df_mva['iso'] != 'WLD') &amp; (df_mva['var'] == var)]\n\t\tdf_emp_indstat4_4 = df_indstat4_4[(df_indstat4_4['iso'] != 'WLD') &amp; (df_indstat4_4['var'] == 'emp') &amp;\n\t\t\t\t\t\t\t\t\t  (df_indstat4_4['isic4'] == 'C')]\n\t\tdf_emp_indstat4_4=df_emp_indstat4_4[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\t\tdf_mva_fltrd = df_mva_fltrd[['year', 'iso', 'value']].set_index(['year', 'iso'])\n\t\tdestfolder = tblnfolder+'\/indstat4'\n\t\tmakeTSCopy(destfolder, filename)\n\t\t(df_mva_fltrd\/df_emp_indstat4_4).to_csv(destfolder+'\/'+filename)\n#--------------------------------- convertNoTech\ndef convertNoTech():\n\tdestfolder = tblnfolder+'\/mva'\n\tMVA_MVACONS(destfolder)\n\tMVA_GrowthRate(destfolder)\n\tGDP_GrowthRate(destfolder)\n\tPop_GrowthRate(destfolder)\n\tShareofMVAinGDP(destfolder)\n\tMVAperCapita(destfolder)\n\tMVACONSperCapita(destfolder)\n\tGDPperCapita(destfolder)\n\tGDPCONSperCapita(destfolder)\n\tShareWorldMVA(destfolder)\n\tShareWorldGDP(destfolder)\n\tShareWorldPop(destfolder)\n\tMVA_MVACONSperEmp(tblnfolder)\n#--------------------------------- transferToDB\ndef transferToDB():\n\tcommand = 'cd '+datafolder+';mysql -ujordanio -pjordaniopwd jordanio &lt; import.sql'\n\tpopen = os.popen(command)\n\tstatus = popen.close()\n\tif status is None:\n\t\tout['status'] = 'success'\n\telse:\n\t\tout['status'] = 'error'\n\t\tout['error'] = status\n#--------------------------------- MAIN\nprint(\"Content-type: text\/json\")\nprint()\nform = cgi.FieldStorage(keep_blank_values=True)\naction = form.getvalue('action', 'none')\nout= { 'action': action }\ndatafolder = '..\/production\/data'\ntblnfolder = '..\/production\/tabulation'\nif action == 'import':\n\tfileitem = form['datafile']\n\ttry:\n\t\tdf = pd.read_excel(fileitem.file, dtype={'Table Code': str, 'Country Code': str}, na_values=['...'])\n\t\tdbtype = form.getvalue('db_type')\n\t\tcheckDB(dbtype, df)\n\t\tif out['status'] != 'error':\n\t\t\tsaveAsCSV(dbtype, df)\n\texcept:\n\t\tout['status'] = 'error'\n\t\tout['error'] = str(sys.exc_info()[0])\nelif action == 'listfiles':\n\tlistFiles()\nelif action == 'convert':\n\tout['status'] = 'success'\n\ttry:\n\t\tdf_mva = pd.read_csv(datafolder+'\/'+'mva.csv')\n\t\tdf_indstat2 = pd.read_csv(datafolder+'\/'+'indstat2.csv')\n\t\tdf_indstat4 = pd.read_csv(datafolder+'\/'+'indstat4.csv')\n\t\tdf_indstat4_4 = pd.read_csv(datafolder+'\/'+'indstat4_4.csv')\n\t\tconvertTech()\n\t\tconvertNoTech()\n\t\ttransferToDB()\n\texcept:\n\t\tout['status'] = 'error'\n\t\tout['error'] = str(sys.exc_info()[0])\nelse:\n\tout['error'] = \"No valid action specified: \"+action\nprint(json.dumps(out, indent=3))<\/code><\/pre>\n<p>The above Python scripts create files in certain folders. If the Observatory application is being initialized from scratch on a new host, those folders (and their descendants) need to have write permissions set as follows:<code><\/code><\/p>\n<pre><code>chmod -R 777 \/var\/www\/wordpress\/jordan\/prototype\/production\nchmod -R 777 \/var\/www\/wordpress\/jordan\/prototype\/scripts\nchmod -R 777 \/var\/www\/wordpress\/jordan\/prototype\/trade\n<\/code><\/pre>\n<p>The two Python\/Pandas scripts below are called by the corresponding<br \/>(UN Comtrade or UNIDO) front-end programs which in turn are invoked from<br \/>the <a href=\"https:\/\/huzaifazoom.org\/?page_id=1027\">Data Update page<\/a>.<br \/>These two scripts are located in the &lt;Wordpress root folder&gt;\/jordan\/prototype\/scripts\/ folder:<\/p>\n<p>\u00a0<\/p>\n<ul>\n<li>&lt;Wordpress root folder&gt;\/jordan\/prototype\/scripts\/update_comtrade.py<\/li>\n<li>&lt;Wordpress root folder&gt;\/jordan\/prototype\/scripts\/update_unido.py<\/li>\n<\/ul>\n<p>The front-end programs that call these scripts are HTML\/Javascripts\/CSS based and they are to be found at:<\/p>\n<ul>\n<li>&lt;Wordpress root folder&gt;\/jordan\/prototype\/admin\/comtrade_update.html<\/li>\n<li>&lt;Wordpress root folder&gt;\/jordan\/prototype\/admin\/unido_update.html<\/li>\n<\/ul>\n","protected":false},"excerpt":{"rendered":"<p>The two Python\/Pandas scripts below are called by the corresponding (UN Comtrade or UNIDO) front-end programs which in turn are invoked from the Data Update page. These two scripts are located in the \/var\/www\/wordpress\/jordan\/prototype\/scripts\/ folder. They are included here to give an idea of the data structures and manipulation involved in displaying the charts. The &hellip;<\/p>\n<p class=\"read-more\"> <a class=\"\" href=\"https:\/\/observatory.huzaifazoom.org\/ar\/2021\/09\/18\/automated-data-update\/\"> <span class=\"screen-reader-text\">3. Automated Data Update<\/span> \u0642\u0631\u0627\u0621\u0629 \u0627\u0644\u0645\u0632\u064a\u062f &raquo;<\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"inline_featured_image":false,"site-sidebar-layout":"default","site-content-layout":"default","ast-global-header-display":"","ast-main-header-display":"","ast-hfb-above-header-display":"","ast-hfb-below-header-display":"","ast-hfb-mobile-header-display":"","site-post-title":"","ast-breadcrumbs-content":"","ast-featured-img":"","footer-sml-layout":"","theme-transparent-header-meta":"","adv-header-id-meta":"","stick-header-meta":"","header-above-stick-meta":"","header-main-stick-meta":"","header-below-stick-meta":""},"categories":[15],"tags":[],"_links":{"self":[{"href":"https:\/\/observatory.huzaifazoom.org\/ar\/wp-json\/wp\/v2\/posts\/1292"}],"collection":[{"href":"https:\/\/observatory.huzaifazoom.org\/ar\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/observatory.huzaifazoom.org\/ar\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/observatory.huzaifazoom.org\/ar\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/observatory.huzaifazoom.org\/ar\/wp-json\/wp\/v2\/comments?post=1292"}],"version-history":[{"count":0,"href":"https:\/\/observatory.huzaifazoom.org\/ar\/wp-json\/wp\/v2\/posts\/1292\/revisions"}],"wp:attachment":[{"href":"https:\/\/observatory.huzaifazoom.org\/ar\/wp-json\/wp\/v2\/media?parent=1292"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/observatory.huzaifazoom.org\/ar\/wp-json\/wp\/v2\/categories?post=1292"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/observatory.huzaifazoom.org\/ar\/wp-json\/wp\/v2\/tags?post=1292"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}