{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n# Bayesian Hyerparameter tuning\n\nExperiment to utilize the bayesian hyperparameter tuning for the algorithms. \n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "try: import cPickle as pickle\nexcept: import pickle\nfrom time import time\nfrom argparse import ArgumentParser\nimport importlib\nimport json\n# import cPickle\nimport networkx as nx\nimport itertools\nimport pdb\nimport sys\nimport numpy as np\nimport pandas as pd\n# sys.path.insert(0, './')\nimport os\nfrom gemben.utils      import graph_util, plot_util\nfrom gemben.evaluation import visualize_embedding as viz\nfrom gemben.evaluation.evaluate_graph_reconstruction import expGR\nfrom gemben.evaluation.evaluate_link_prediction import expLP, expLPT\nfrom gemben.evaluation.evaluate_node_classification import expNC\nfrom gemben.evaluation.visualize_embedding import expVis\n\nfrom gemben.utils.bayesian_opt import BayesianOpt\n\n\nmethClassMap = {\"gf\": \"GraphFactorization\",\n                \"hope\": \"HOPE\",\n                \"lap\": \"LaplacianEigenmaps\",\n                \"node2vec\": \"node2vec\",\n                \"sdne\": \"SDNE\",\n                \"pa\": \"PreferentialAttachment\",\n                \"rand\": \"RandomEmb\",\n                \"cn\": \"CommonNeighbors\",\n                \"aa\": \"AdamicAdar\",\n                \"jc\": \"JaccardCoefficient\"}\nexpMap = {\"gf\": \"GF MAP\", \"lp\": \"LP MAP\",\n          \"nc\": \"NC MAP\"}\n\n\ndef learn_emb(MethObj, di_graph, params, res_pre, m_summ):\n    if params[\"experiments\"] == [\"lp\"]:\n        X = None\n    else:\n        print('Learning Embedding: %s' % m_summ)\n        if not bool(int(params[\"load_emb\"])):\n            X, learn_t = MethObj.learn_embedding(graph=di_graph,\n                                                 edge_f=None,\n                                                 no_python=True)\n            print('\\tTime to learn embedding: %f sec' % learn_t)\n            pickle.dump(X, open('%s_%s.emb' % (res_pre, m_summ), 'wb'))\n            pickle.dump(learn_t,\n                        open('%s_%s.learnT' % (res_pre, m_summ), 'wb'))\n        else:\n            X = pickle.load(open('%s_%s.emb' % (res_pre, m_summ),\n                                 'rb'))\n            try:\n                learn_t = pickle.load(open('%s_%s.learnT' % (res_pre, m_summ),\n                                           'rb'))\n                print('\\tTime to learn emb.: %f sec' % learn_t)\n            except IOError:\n                print('\\tTime info not found')\n    return X\n\n\ndef run_exps(MethObj, meth, dim, di_graph, data_set, node_labels, params):\n    m_summ = '%s_%d' % (meth, dim)\n    res_pre = \"gemben/results/%s\" % data_set\n    n_r = params[\"rounds\"]\n    X = learn_emb(MethObj, di_graph, params, res_pre, m_summ)\n    gr, lp, nc = [0] * n_r, [0] * n_r, [0] * n_r\n    if \"gr\" in params[\"experiments\"]:\n        gr = expGR(di_graph, MethObj,\n                   X, params[\"n_sample_nodes\"].split(\",\"),\n                   n_r, res_pre,\n                   m_summ, is_undirected=params[\"is_undirected\"],\n                   sampling_scheme=params[\"samp_scheme\"])\n    if \"lpt\" in params[\"experiments\"]:\n        expLPT(di_graph, MethObj, res_pre, m_summ,\n               is_undirected=params[\"is_undirected\"])\n    if \"lp\" in params[\"experiments\"]:\n        lp = expLP(di_graph, MethObj,\n                   params[\"n_sample_nodes\"].split(\",\"),\n                   n_r, res_pre,\n                   m_summ, is_undirected=params[\"is_undirected\"],\n                   sampling_scheme=params[\"samp_scheme\"])\n    if \"nc\" in params[\"experiments\"]:\n        if \"nc_test_ratio_arr\" not in params:\n            print('NC test ratio not provided')\n        else:\n            nc = expNC(X, node_labels, params[\"nc_test_ratio_arr\"],\n                       n_r, res_pre,\n                       m_summ)\n    if \"viz\" in params[\"experiments\"]:\n        if MethObj.get_method_name() == 'hope_gsvd':\n            d = X.shape[1] / 2\n            expVis(X[:, :d], res_pre, m_summ,\n                   node_labels=node_labels, di_graph=di_graph)\n        else:\n            expVis(X, res_pre, m_summ,\n                   node_labels=node_labels, di_graph=di_graph)\n    return gr, lp, nc\n\n\ndef get_max(val, val_max, idx, idx_max):\n    if val > val_max:\n        return val, idx\n    else:\n        return val_max, idx_max\n\n\ndef bayesian_optimization(data_set, di_graph, node_labels, params):\n    # Load range of hyper parameters \n    try:\n        model_hyp_range = json.load(\n            open('gemben/experiments/config/%s_hypRange.conf' % data_set, 'r')\n        )\n    except IOError:\n        model_hyp_range = json.load(\n            open('gemben/experiments/config/default_hypRange.conf', 'r')\n        )\n  \n    params['model_hyp_range'] = model_hyp_range\n    params['data_set'] = data_set\n    params['di_graph'] = di_graph\n\n    ## test for gf\n    #pdb.set_trace()\n    bayesian_opt = BayesianOpt(**params)\n    bayesian_opt.optimize()\n\n    \n\n\ndef choose_best_hyp(data_set, di_graph, node_labels, params):\n    # Load range of hyper parameters to test on\n    try:\n        model_hyp_range = json.load(\n            open('gemben/experiments/config/%s_hypRange.conf' % data_set, 'r')\n        )\n    except IOError:\n        model_hyp_range = json.load(\n            open('gemben/experiments/config/default_hypRange.conf', 'r')\n        )\n    try:\n        os.makedirs(\"gemben/temp_hyp_res\")\n    except:\n        pass\n    # Test each hyperparameter for each method and store the best\n    for meth in params[\"methods\"]:\n        dim = int(params[\"dimensions\"][0])\n        MethClass = getattr(\n            importlib.import_module(\"gem.embedding.%s\" % meth),\n            methClassMap[meth]\n        )\n        meth_hyp_range = model_hyp_range[meth]\n        gr_max, lp_max, nc_max = 0, 0, 0\n        gr_hyp, lp_hyp, nc_hyp = 0, 0, 0\n        gr_hyp, lp_hyp, nc_hyp = {meth: {}}, {meth: {}}, {meth: {}}\n\n        # Test each hyperparameter\n        ev_cols = [\"GR MAP\", \"LP MAP\", \"NC F1 score\"]\n        hyp_df = pd.DataFrame(\n            columns=list(meth_hyp_range.keys()) + ev_cols + [\"Round Id\"]\n        )\n        hyp_r_idx = 0\n        for hyp in itertools.product(*meth_hyp_range.values()):\n            hyp_d = {\"d\": dim}\n            hyp_d.update(dict(zip(meth_hyp_range.keys(), hyp)))\n            print(hyp_d)\n            if meth == \"sdne\":\n                hyp_d.update({\n                    \"modelfile\": [\n                        \"gemben/intermediate/enc_mdl_%s_%d.json\" % (data_set, dim),\n                        \"gemben/intermediate/dec_mdl_%s_%d.json\" % (data_set, dim)\n                    ],\n                    \"weightfile\": [\n                        \"gemben/intermediate/enc_wts_%s_%d.hdf5\" % (data_set, dim),\n                        \"gemben/intermediate/dec_wts_%s_%d.hdf5\" % (data_set, dim)\n                    ]\n                })\n            elif meth == \"gf\" or meth == \"node2vec\":\n                hyp_d.update({\"data_set\": data_set})\n            MethObj = MethClass(hyp_d)\n            gr, lp, nc = run_exps(MethObj, meth, dim, di_graph,\n                                  data_set, node_labels, params)\n            gr_m, lp_m, nc_m = np.mean(gr), np.mean(lp), np.mean(nc)\n            gr_max, gr_hyp[meth] = get_max(gr_m, gr_max, hyp_d, gr_hyp[meth])\n            lp_max, lp_hyp[meth] = get_max(lp_m, lp_max, hyp_d, lp_hyp[meth])\n            nc_max, nc_hyp[meth] = get_max(nc_m, nc_max, hyp_d, nc_hyp[meth])\n            hyp_df_row = dict(zip(meth_hyp_range.keys(), hyp))\n            f_hyp_temp = open(\"gemben/temp_hyp_res/%s_%s.txt\" % (data_set, meth), \"a\")\n            hyp_str = '_'.join(\"%s=%s\" % (key, str(val).strip(\"'\")) for (key, val) in hyp_d.items())\n            f_hyp_temp.write('%s: MAP: %f\\n' % (hyp_str, lp_max))\n            f_hyp_temp.close()\n            for r_id in range(params[\"rounds\"]):\n                hyp_df.loc[hyp_r_idx, meth_hyp_range.keys()] = \\\n                    pd.Series(hyp_df_row)\n                hyp_df.loc[hyp_r_idx, ev_cols + [\"Round Id\"]] = \\\n                    [gr[min(r_id, len(gr) -1)], lp[r_id], nc[r_id], r_id]\n                hyp_r_idx += 1\n        exp_param = params[\"experiments\"]\n        for exp in exp_param:\n            hyp_df.to_hdf(\n                \"gemben/intermediate/%s_%s_%s_%s_hyp.h5\" % (data_set, meth,\n                                                         exp,\n                                                         params[\"samp_scheme\"]),\n                \"df\"\n            )\n        ###plot_util.plot_hyp(meth_hyp_range.keys(), exp_param,\n           ##:                meth, data_set, s_sch=params[\"samp_scheme\"])\n\n        # Store the best hyperparameter\n        ####### put the file into synthetic\n        opt_hyp_f_pre = 'gemben/experiments/config/synthetic/%s_%s_%s' % (\n            data_set,\n            meth,\n            params[\"samp_scheme\"]\n        )\n        if gr_max:\n            with open('%s_gr.conf' % opt_hyp_f_pre, 'w') as f:\n                f.write(json.dumps(gr_hyp, indent=4))\n        if lp_max:\n            with open('%s_lp.conf' % opt_hyp_f_pre, 'w') as f:\n                f.write(json.dumps(lp_hyp, indent=4))\n        if nc_max:\n            with open('%s_nc.conf' % opt_hyp_f_pre, 'w') as f:\n                f.write(json.dumps(nc_hyp, indent=4))\n\n\ndef call_plot_hyp(data_set, params):\n    # Load range of hyper parameters tested on to plot\n    try:\n        model_hyp_range = json.load(\n            open('gemben/experiments/config/%s_hypRange.conf' % data_set, 'r')\n        )\n    except IOError:\n        model_hyp_range = json.load(\n            open('gemben/experiments/config/default_hypRange.conf', 'r')\n        )\n    for meth in params[\"methods\"]:\n            meth_hyp_range = model_hyp_range[meth]\n            exp_param = params[\"experiments\"]\n            plot_util.plot_hyp(meth_hyp_range.keys(), exp_param,\n                               meth, data_set,\n                               s_sch=params[\"samp_scheme\"])\n\n\ndef call_plot_hyp_all(data_sets, params):\n    # Load range of hyper parameters tested on to plot\n    try:\n        model_hyp_range = json.load(\n            open('gemben/experiments/config/%s_hypRange.conf' % data_sets[0], 'r')\n        )\n    except IOError:\n        model_hyp_range = json.load(\n            open('gemben/experiments/config/default_hypRange.conf', 'r')\n        )\n    for meth in params[\"methods\"]:\n            meth_hyp_range = model_hyp_range[meth]\n            exp_param = params[\"experiments\"]\n            plot_util.plot_hyp_all(meth_hyp_range.keys(), exp_param,\n                                   meth, data_sets,\n                                   s_sch=params[\"samp_scheme\"])\n\n\ndef call_exps(params, data_set):\n    # Load Dataset\n    print('Dataset: %s' % data_set)\n\n\n\n\n########  for SBM, r_mat, hyperbolic\n    #if data_set[10:13] == 'r_m' or data_set[10:13] == 'sto' or data_set[10:13] == 'hyp':\n     #   di_graph = nx.read_gpickle('gem/data/%s/graph.gpickle' % data_set)[0]\n    #else:\n   \n    #di_graph = nx.read_gpickle('gem/data/%s/graph.gpickle' % data_set)[0]\n    di_graph = nx.read_gpickle('gemben/data/%s/graph.gpickle' % data_set)\n    \n    di_graph, nodeListMap = graph_util.get_lcc(di_graph)\n    graph_util.print_graph_stats(di_graph)\n\n\n    # Load node labels if given\n    if bool(params[\"node_labels\"]):\n        node_labels = cPickle.load(\n            open('gemben/data/%s/node_labels.pickle' % data_set, 'rb')\n        )\n        node_labels_gc = np.zeros(\n            (di_graph.number_of_nodes(), node_labels.shape[1]))\n        for k, v in nodeListMap.iteritems():\n            try:\n                node_labels_gc[v, :] = node_labels[k, :].toarray()\n            # Already a numpy array\n            except AttributeError:\n                node_labels_gc[v, :] = node_labels[k, :]\n        node_labels = node_labels_gc\n    else:\n        node_labels = None\n\n    # Search through the hyperparameter space\n    if params[\"find_hyp\"]:\n        #choose_best_hyp(data_set, di_graph, node_labels, params)\n        bayesian_optimization(data_set, di_graph, node_labels, params)\n\n\n    # Load best hyperparameter and test it again on new test data\n    for d, meth, exp in itertools.product(\n        params[\"dimensions\"],\n        params[\"methods\"],\n        params[\"experiments\"]\n    ):\n        dim = int(d)\n        MethClass = getattr(\n            importlib.import_module(\"gemben.embedding.%s\" % meth),\n            methClassMap[meth]\n        )\n        opt_hyp_f_pre = 'gemben/experiments/config/synthetic/%s_%s_%s' % (\n            data_set,\n            meth,\n            params[\"samp_scheme\"]\n        )\n        try:\n            if exp != \"viz\":\n                if exp == 'lpt':\n                    model_hyp = json.load(\n                        open('%s_lp.conf' % opt_hyp_f_pre, 'r')\n                    )\n                else:\n                    model_hyp = json.load(\n                        open('%s_%s.conf' % (opt_hyp_f_pre, exp), 'r')\n                    )\n            else:\n                model_hyp = json.load(\n                    open(\n                        '%s_%s.conf' % (opt_hyp_f_pre, params[\"viz_params\"]), 'r'\n                    )\n                )\n        except IOError:\n            print('Default hyperparameter of the method chosen')\n            model_hyp = json.load(\n                open('gemben/experiments/config/%s.conf' % meth, 'r')\n            )\n        hyp = {}\n        hyp.update(model_hyp[meth])\n        hyp.update({\"d\": dim})\n        if meth == \"sdne\":\n                hyp.update({\n                    \"modelfile\": [\n                        \"gemben/intermediate/en_mdl_%s_%d.json\" % (data_set, dim),\n                        \"gemben/intermediate/dec_mdl_%s_%d.json\" % (data_set, dim)\n                    ],\n                    \"weightfile\": [\n                        \"gemben/intermediate/enc_wts_%s_%d.hdf5\" % (data_set, dim),\n                        \"gemben/intermediate/dec_wts_%s_%d.hdf5\" % (data_set, dim)\n                    ]\n                })\n        elif meth == \"gf\" or meth == \"node2vec\":\n            hyp.update({\"data_set\": data_set})\n        MethObj = MethClass(hyp)\n        run_exps(MethObj, meth, dim, di_graph, data_set, node_labels, params)\n\n\nif __name__ == '__main__':\n    ''' Sample usage\n    python experiments/exp.py -data sbm -dim 128 -meth sdne -exp gr,lp\n    '''\n    t1 = time()\n    parser = ArgumentParser(description='Graph Embedding Experiments')\n    parser.add_argument('-data', '--data_sets',\n                        help='dataset names (default: sbm)')\n    parser.add_argument('-dim', '--dimensions',\n                        help='embedding dimensions list(default: 2^1 to 2^8)')\n    parser.add_argument('-meth', '--methods',\n                        help='method list (default: all methods)')\n    parser.add_argument('-exp', '--experiments',\n                        help='exp list (default: gr,lp,viz,nc)')\n    parser.add_argument('-lemb', '--load_emb',\n                        help='load saved embeddings (default: False)')\n    parser.add_argument('-lexp', '--load_exp',\n                        help='load saved experiment results (default: False)')\n    parser.add_argument('-node_labels', '--node_labels',\n                        help='node labels available or not (default: False)')\n    parser.add_argument('-rounds', '--rounds',\n                        help='number of rounds (default: 5)')\n    parser.add_argument('-plot', '--plot',\n                        help='plot the results (default: True)')\n    parser.add_argument('-plot_d', '--plot_d',\n                        help='plot the results wrt dims(default: True)')\n    parser.add_argument('-hyp_plot', '--hyp_plot',\n                        help='plot the hyperparameter results (default: True)')\n    parser.add_argument('-hyp_plot_all', '--hyp_plot_all',\n                        help='plot the hyperparameter results (all) (default: True)')\n    parser.add_argument('-viz_params', '--viz_params',\n                        help='which params to use for viz (default: gr)')\n    parser.add_argument('-find_hyp', '--find_hyp',\n                        help='find best hyperparameters (default: False)')\n    parser.add_argument('-saveMAP', '--save_MAP',\n                        help='save MAP in a latex table (default: False)')\n    parser.add_argument('-n_samples', '--n_sample_nodes',\n                        help='number of sampled nodes (default: 1024)')\n    parser.add_argument('-s_sch', '--samp_scheme',\n                        help='sampling scheme (default: u_rand)')\n\n    params = json.load(open('gemben/experiments/config/params.conf', 'r'))\n\n    args = vars(parser.parse_args())\n\n    for k, v in args.items():\n        if v is not None:\n            params[k] = v\n\n    params[\"experiments\"] = params[\"experiments\"].split(',')\n    params[\"data_sets\"] = params[\"data_sets\"].split(',')\n    params[\"experiments\"] = list(set(params[\"experiments\"]))\n    params[\"data_sets\"] = list(set(params[\"data_sets\"]))\n    params[\"rounds\"] = int(params[\"rounds\"])\n    params[\"node_labels\"] = int(params[\"node_labels\"])\n    # params[\"n_sample_nodes\"] = int(params[\"n_sample_nodes\"])\n    params[\"is_undirected\"] = bool(int(params[\"is_undirected\"]))\n   \n\n    \n    params[\"plot_d\"] = bool(int(params[\"plot_d\"]))\n    params[\"plot\"] = bool(int(params[\"plot\"]))\n    params[\"hyp_plot\"] = bool(int(params[\"hyp_plot\"]))\n    params[\"hyp_plot_all\"] = bool(int(params[\"hyp_plot_all\"]))\n    params[\"find_hyp\"] = bool(int(params[\"find_hyp\"]))\n    \n    if params[\"methods\"] == \"all\":\n        params[\"methods\"] = methClassMap.keys()\n    else:\n        params[\"methods\"] = params[\"methods\"].split(',')\n    params[\"methods\"] = list(set(params[\"methods\"]))       \n        \n    params[\"dimensions\"] = params[\"dimensions\"].split(',')\n    params[\"dimensions\"] = list(set(params[\"dimensions\"]))\n    if \"nc_test_ratio_arr\" in params:\n        params[\"nc_test_ratio_arr\"] = params[\"nc_test_ratio_arr\"].split(',')\n        params[\"nc_test_ratio_arr\"] = \\\n            [float(ratio) for ratio in params[\"nc_test_ratio_arr\"]]\n    try:\n      os.makedirs(\"gemben/intermediate\")\n    except:\n      pass\n    try:\n      os.makedirs(\"gemben/results\")\n    except:\n      pass\n\n    for data_set in params[\"data_sets\"]:\n        if not int(params[\"load_exp\"]):\n            call_exps(params, data_set)\n        if int(params[\"plot\"]):\n            res_pre = \"gemben/results/%s\" % data_set\n            plot_util.plotExpRes(res_pre, params[\"methods\"],\n                                 params[\"experiments\"], params[\"dimensions\"],\n                                 'gemben/plots/%s_%s' % (data_set, params[\"samp_scheme\"]),\n                                 params[\"rounds\"], params[\"plot_d\"],\n                                 params[\"samp_scheme\"])\n        if int(params[\"hyp_plot\"]):\n            call_plot_hyp(data_set, params)\n    if int(params[\"hyp_plot_all\"]):\n            call_plot_hyp_all(params[\"data_sets\"], params)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.8"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}