{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n# Experiment with Benchmark\n\nExample to run the benchmark across all the baseline embedding algorithms.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "from subprocess import call\nimport itertools\ntry: import cPickle as pickle\nexcept: import pickle\nimport json\nfrom argparse import ArgumentParser\nimport networkx as nx\nimport pandas as pd\nimport pdb\nimport os\nimport sys\nfrom time import time\n# sys.path.insert(0, './')\nfrom gemben.utils import graph_gens\n\nmethClassMap = {\"gf\": \"GraphFactorization\",\n                \"hope\": \"HOPE\",\n                \"lap\": \"LaplacianEigenmaps\",\n                \"node2vec\": \"node2vec\",\n                \"sdne\": \"SDNE\",\n                \"pa\": \"PreferentialAttachment\",\n                \"rand\": \"RandomEmb\",\n                \"cn\": \"CommonNeighbors\",\n                \"aa\": \"AdamicAdar\",\n                \"jc\": \"JaccardCoefficient\"}\n\n\nif __name__ == \"__main__\":\n    ''' Sample usage\n    python experiments/exp_synthetic.py -syn_names all -plot_hyp_data 1 -meths all\n    '''\n    t1 = time()\n    parser = ArgumentParser(description='Graph Embedding Benchmark Experiments')\n    parser.add_argument('-domain', '--domain_name',\n                        help='domain name (default: social)')\n    parser.add_argument('-graph', '--graphs',\n                        help='graph name (default: all)')\n    parser.add_argument('-meth', '--methods',\n                        help='method list (default: all)')\n    parser.add_argument('-plot_hyp_data', '--plot_hyp_data',\n                        help='plot the hyperparameter results (default: False)')\n    parser.add_argument('-rounds', '--rounds',\n                        help='number of rounds (default: 20)')\n    parser.add_argument('-s_sch', '--samp_scheme',\n                        help='sampling scheme (default: rw)')\n    parser.add_argument('-lexp', '--lexp',\n                        help='load experiment (default: False)')\n    params = json.load(\n        open('gemben/experiments/config/params_benchmark.conf', 'r')\n    )\n    args = vars(parser.parse_args())\n    print (args)\n    domain_graph_map = json.load(\n        open('gemben/experiments/config/domain_graph_map.conf', 'r')\n    )\n    # graph_hyp_range: {N: [128, 256, 512, 1024], deg: [4, 6,8, 10, 12]}\n    graph_hyp_range = json.load(\n        open('gemben/experiments/config/graph_hyp_range.conf', 'r')\n    )\n    # def_graph_hyps: {N: 1024, deg: 8, dia: None, dim: 128}\n    def_graph_hyps = json.load(\n        open('gemben/experiments/config/def_graph_hyps.conf', 'r')\n    )\n    for k, v in args.items():\n        if v is not None:\n            params[k] = v\n    params[\"rounds\"] = int(params[\"rounds\"])\n    #params[\"domain_name\"] = params[\"domain_name\"].split(',')\n    if params[\"graphs\"] == \"all\":\n        params[\"graphs\"] = domain_graph_map[params[\"domain_name\"]]\n    else:\n        params[\"graphs\"] = params[\"graphs\"].split(',')\n    params[\"lexp\"] = bool(int(params[\"lexp\"]))\n    params[\"plot_hyp_data\"] = bool(int(params[\"plot_hyp_data\"]))\n    if params[\"methods\"] == \"all\":\n        params[\"methods\"] = methClassMap.keys()\n    else:\n        params[\"methods\"] = params[\"methods\"].split(',')\n    samp_scheme = params[\"samp_scheme\"]\n\n\n\n\n\n    \n\n    try:\n      os.makedirs(\"gemben/intermediate\")\n    except:\n      pass\n    try:\n      os.makedirs(\"gemben/results\")\n    except:\n      pass\n    try:\n      os.makedirs(\"gemben/temp\")\n    except:\n      pass\n#     if not os.path.exists(\"gem/intermediate\"):\n#         os.makedirs(\"gem/intermediate\")\n#     if not os.path.exists(\"gem/results\"):\n#         os.makedirs(\"gem/results\")\n\n    graph_hyp_keys = list(graph_hyp_range.keys())\n    ev_cols = [\"LP MAP\", \"LP P@100\"]\n    for meth , graph in itertools.product(*[params[\"methods\"],params[\"graphs\"]]):\n        hyp_df = pd.DataFrame(\n                columns=graph_hyp_keys + ev_cols + [\"Round Id\"]\n            )\n        hyp_r_idx = 0\n        for hyp_key in graph_hyp_keys:\n           \n            for curr_hyp_key_range, r_id in itertools.product(\n                *[graph_hyp_range[hyp_key], range(params[\"rounds\"])]\n            ):\n                \n                \n                ##### first round to find the best parameter for each methods\n                if r_id == 0: \n                    f_hyp = 1\n                else:\n                    f_hyp = 0\n                \n                \n                \n                curr_hyps = def_graph_hyps.copy()\n            \n                curr_hyps[hyp_key] = curr_hyp_key_range\n                curr_hyps[\"domain\"] = params[\"domain_name\"]\n                hyp_str = '_'.join(\n                            \"%s=%s\" % (key, str(val).strip(\"'\")) for (key, val) in curr_hyps.items()\n                        )\n                \n                hyp_str_graph_name = '_'.join(\n                            \"%s=%s\" % (key, str(val).strip(\"'\")) for (key, val) in curr_hyps.items() if key != 'dim' \n                        )\n                \n                syn_data_folder = 'benchmark_%s_%s_%s' % (graph, hyp_str_graph_name, r_id)\n                \n                graphClass = getattr(graph_gens, graph)\n                \n                try:\n                    nx.read_gpickle(\n                            'gemben/data/%s/graph.gpickle' % syn_data_folder\n                  )\n                except:\n                    flag =  1\n                    ##### flag = 0 means the labels are continous on lcc\n                    while flag:\n                        print(\"Graph is generating...\")\n                        G = graphClass(**curr_hyps)[0]\n                        if len(set(G.nodes())) == G.number_of_nodes() and list(G.nodes())[-1] == G.number_of_nodes() -1:\n                            flag = 0\n                    if G:\n                        if not os.path.exists(\"gemben/data/%s\" % syn_data_folder):\n                            os.makedirs(\"gemben/data/%s\" % syn_data_folder)\n                        nx.write_gpickle(\n                                G, 'gemben/data/%s/graph.gpickle' % syn_data_folder\n                    )\n                perf_exp = not params[\"lexp\"]      \n                if params[\"lexp\"]:\n                  try:\n                    MAP, prec, n_samps = pickle.load(\n                      open('gemben/results/%s_%s_%d_%s.lp' % (\n                          syn_data_folder, meth, \n                          curr_hyps[\"dim\"], samp_scheme), 'rb'))\n                  except:   \n                      perf_exp = 1\n                       ##### only find the best hyp for first round\n                if perf_exp:\n                    os.system(\n                      \"python3 gemben/experiments/exp.py -data %s -meth %s -dim %d -rounds 1 -find_hyp %d -s_sch %s -exp lp\" % (\n                          syn_data_folder,\n                          meth,\n                          curr_hyps[\"dim\"],\n                          f_hyp,\n                          samp_scheme\n                      )\n                    )\n                MAP, prec, n_samps = pickle.load(\n                open('gemben/results/%s_%s_%d_%s.lp' % (\n                    syn_data_folder, meth, \n                    curr_hyps[\"dim\"], samp_scheme), 'rb'))        \n                hyp_df.loc[hyp_r_idx, graph_hyp_keys] = \\\n                pd.Series(curr_hyps)\n                #prec_100 = prec[int(n_samps[0])][0][100]\n                try:\n                  prec_100 = list(prec.values())[0][0][100]\n                except:\n                  pdb.set_trace()\n                f_temp = open(\"gemben/temp/%s_%s_%s_lp_%s_data_hyp.txt\" % (\n                  params[\"domain_name\"], graph, meth, samp_scheme), 'a')\n                f_temp.write('%s: round: %d, MAP: %f, prec_100: %f' % (hyp_str, r_id, list(MAP.values())[0][0], prec_100))\n                f_temp.close()\n                hyp_df.loc[hyp_r_idx, ev_cols + [\"Round Id\"]] = \\\n                [list(MAP.values())[0][0], prec_100, r_id]\n                #[MAP[int(n_samps[0])][0], prec_100, r_id]\n                hyp_r_idx += 1\n\n        hyp_df.to_hdf(\n            \"gemben/intermediate/%s_%s_%s_lp_%s_data_hyp.h5\" % (\n                params[\"domain_name\"], graph, meth, samp_scheme),\n            \"df\"\n        )\n        print('Experiments done for %s, %s' % (graph, meth))"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.8"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}