Experiment with Benchmark

Example to run the benchmark across all the baseline embedding algorithms.

from subprocess import call
import itertools
try: import cPickle as pickle
except: import pickle
import json
from argparse import ArgumentParser
import networkx as nx
import pandas as pd
import pdb
import os
import sys
from time import time
# sys.path.insert(0, './')
from gemben.utils import graph_gens

methClassMap = {"gf": "GraphFactorization",
                "hope": "HOPE",
                "lap": "LaplacianEigenmaps",
                "node2vec": "node2vec",
                "sdne": "SDNE",
                "pa": "PreferentialAttachment",
                "rand": "RandomEmb",
                "cn": "CommonNeighbors",
                "aa": "AdamicAdar",
                "jc": "JaccardCoefficient"}


if __name__ == "__main__":
    ''' Sample usage
    python experiments/exp_synthetic.py -syn_names all -plot_hyp_data 1 -meths all
    '''
    t1 = time()
    parser = ArgumentParser(description='Graph Embedding Benchmark Experiments')
    parser.add_argument('-domain', '--domain_name',
                        help='domain name (default: social)')
    parser.add_argument('-graph', '--graphs',
                        help='graph name (default: all)')
    parser.add_argument('-meth', '--methods',
                        help='method list (default: all)')
    parser.add_argument('-plot_hyp_data', '--plot_hyp_data',
                        help='plot the hyperparameter results (default: False)')
    parser.add_argument('-rounds', '--rounds',
                        help='number of rounds (default: 20)')
    parser.add_argument('-s_sch', '--samp_scheme',
                        help='sampling scheme (default: rw)')
    parser.add_argument('-lexp', '--lexp',
                        help='load experiment (default: False)')
    params = json.load(
        open('gemben/experiments/config/params_benchmark.conf', 'r')
    )
    args = vars(parser.parse_args())
    print (args)
    domain_graph_map = json.load(
        open('gemben/experiments/config/domain_graph_map.conf', 'r')
    )
    # graph_hyp_range: {N: [128, 256, 512, 1024], deg: [4, 6,8, 10, 12]}
    graph_hyp_range = json.load(
        open('gemben/experiments/config/graph_hyp_range.conf', 'r')
    )
    # def_graph_hyps: {N: 1024, deg: 8, dia: None, dim: 128}
    def_graph_hyps = json.load(
        open('gemben/experiments/config/def_graph_hyps.conf', 'r')
    )
    for k, v in args.items():
        if v is not None:
            params[k] = v
    params["rounds"] = int(params["rounds"])
    #params["domain_name"] = params["domain_name"].split(',')
    if params["graphs"] == "all":
        params["graphs"] = domain_graph_map[params["domain_name"]]
    else:
        params["graphs"] = params["graphs"].split(',')
    params["lexp"] = bool(int(params["lexp"]))
    params["plot_hyp_data"] = bool(int(params["plot_hyp_data"]))
    if params["methods"] == "all":
        params["methods"] = methClassMap.keys()
    else:
        params["methods"] = params["methods"].split(',')
    samp_scheme = params["samp_scheme"]







    try:
      os.makedirs("gemben/intermediate")
    except:
      pass
    try:
      os.makedirs("gemben/results")
    except:
      pass
    try:
      os.makedirs("gemben/temp")
    except:
      pass
#     if not os.path.exists("gem/intermediate"):
#         os.makedirs("gem/intermediate")
#     if not os.path.exists("gem/results"):
#         os.makedirs("gem/results")

    graph_hyp_keys = list(graph_hyp_range.keys())
    ev_cols = ["LP MAP", "LP P@100"]
    for meth , graph in itertools.product(*[params["methods"],params["graphs"]]):
        hyp_df = pd.DataFrame(
                columns=graph_hyp_keys + ev_cols + ["Round Id"]
            )
        hyp_r_idx = 0
        for hyp_key in graph_hyp_keys:

            for curr_hyp_key_range, r_id in itertools.product(
                *[graph_hyp_range[hyp_key], range(params["rounds"])]
            ):


                ##### first round to find the best parameter for each methods
                if r_id == 0:
                    f_hyp = 1
                else:
                    f_hyp = 0



                curr_hyps = def_graph_hyps.copy()

                curr_hyps[hyp_key] = curr_hyp_key_range
                curr_hyps["domain"] = params["domain_name"]
                hyp_str = '_'.join(
                            "%s=%s" % (key, str(val).strip("'")) for (key, val) in curr_hyps.items()
                        )

                hyp_str_graph_name = '_'.join(
                            "%s=%s" % (key, str(val).strip("'")) for (key, val) in curr_hyps.items() if key != 'dim'
                        )

                syn_data_folder = 'benchmark_%s_%s_%s' % (graph, hyp_str_graph_name, r_id)

                graphClass = getattr(graph_gens, graph)

                try:
                    nx.read_gpickle(
                            'gemben/data/%s/graph.gpickle' % syn_data_folder
                  )
                except:
                    flag =  1
                    ##### flag = 0 means the labels are continous on lcc
                    while flag:
                        print("Graph is generating...")
                        G = graphClass(**curr_hyps)[0]
                        if len(set(G.nodes())) == G.number_of_nodes() and list(G.nodes())[-1] == G.number_of_nodes() -1:
                            flag = 0
                    if G:
                        if not os.path.exists("gemben/data/%s" % syn_data_folder):
                            os.makedirs("gemben/data/%s" % syn_data_folder)
                        nx.write_gpickle(
                                G, 'gemben/data/%s/graph.gpickle' % syn_data_folder
                    )
                perf_exp = not params["lexp"]
                if params["lexp"]:
                  try:
                    MAP, prec, n_samps = pickle.load(
                      open('gemben/results/%s_%s_%d_%s.lp' % (
                          syn_data_folder, meth,
                          curr_hyps["dim"], samp_scheme), 'rb'))
                  except:
                      perf_exp = 1
                       ##### only find the best hyp for first round
                if perf_exp:
                    os.system(
                      "python3 gemben/experiments/exp.py -data %s -meth %s -dim %d -rounds 1 -find_hyp %d -s_sch %s -exp lp" % (
                          syn_data_folder,
                          meth,
                          curr_hyps["dim"],
                          f_hyp,
                          samp_scheme
                      )
                    )
                MAP, prec, n_samps = pickle.load(
                open('gemben/results/%s_%s_%d_%s.lp' % (
                    syn_data_folder, meth,
                    curr_hyps["dim"], samp_scheme), 'rb'))
                hyp_df.loc[hyp_r_idx, graph_hyp_keys] = \
                pd.Series(curr_hyps)
                #prec_100 = prec[int(n_samps[0])][0][100]
                try:
                  prec_100 = list(prec.values())[0][0][100]
                except:
                  pdb.set_trace()
                f_temp = open("gemben/temp/%s_%s_%s_lp_%s_data_hyp.txt" % (
                  params["domain_name"], graph, meth, samp_scheme), 'a')
                f_temp.write('%s: round: %d, MAP: %f, prec_100: %f' % (hyp_str, r_id, list(MAP.values())[0][0], prec_100))
                f_temp.close()
                hyp_df.loc[hyp_r_idx, ev_cols + ["Round Id"]] = \
                [list(MAP.values())[0][0], prec_100, r_id]
                #[MAP[int(n_samps[0])][0], prec_100, r_id]
                hyp_r_idx += 1

        hyp_df.to_hdf(
            "gemben/intermediate/%s_%s_%s_lp_%s_data_hyp.h5" % (
                params["domain_name"], graph, meth, samp_scheme),
            "df"
        )
        print('Experiments done for %s, %s' % (graph, meth))