diff --git a/build.sh b/build.sh index 42051ed..c576465 100644 --- a/build.sh +++ b/build.sh @@ -3,7 +3,8 @@ export BUILDROOT="$PWD" export BTCPATH="$PWD/btcroot" export GOPATH="$PWD/goroot" export PATH="$PATH:$GOPATH/bin" -export BRANCH='new-stats' +export BRANCH='clean-waterfilling-dp' +#export BRANCH='pari-lp-routing-debug' BUILD_LND='false' BUILD_EXPCTRL='false' @@ -28,6 +29,7 @@ if [ "$BUILD_LND" == "true" ] || [ "$BUILD_EXPCTRL" == "true" ] ; then go get -u github.com/golang/dep/cmd/dep go get -d github.com/urfave/cli go get -d github.com/vibhaa/lnd + go get gopkg.in/zabawaba99/firego.v1 rm -rf $GOPATH/src/github.com/lightningnetwork/lnd mv $GOPATH/src/github.com/vibhaa/lnd $GOPATH/src/github.com/lightningnetwork @@ -96,7 +98,7 @@ if [ "$BUILD_BITCOIND" == "true" ]; then cp $BTCPATH/bitcoin/src/bitcoin-tx $BUILDROOT/binaries/ fi -echo "Cleaning up build files" +#echo "Cleaning up build files" rm -rf $GOPATH rm -rf $BTCPATH rm -rf $BUILDROOT/downloads diff --git a/expctrl/main.go b/expctrl/main.go index 0748b4e..59d26a3 100644 --- a/expctrl/main.go +++ b/expctrl/main.go @@ -7,8 +7,11 @@ import ( "time" "sync" "strconv" + "gopkg.in/zabawaba99/firego.v1" ) +var FIREBASE_URL string = "https://spider2.firebaseio.com/" + func main() { etcd := getEtcdKeyClient() @@ -36,6 +39,28 @@ func main() { numTot := 0 numSucc := 0 + // Logging gorouting + go func () { + fmt.Printf("started new routine, from %s to %s\n", demand.Source, demand.Destination) + src := demand.Source + dst := demand.Destination + EXP_NAME := os.Getenv("SPIDER_EXP_NAME") + fb := firego.New(FIREBASE_URL + EXP_NAME + "/aggregateStats/" + src + "/" + dst, nil) + fmt.Printf("started firebase, EXP_NAME = %s\n", EXP_NAME) + for { + //fmt.Printf("src: %s, dst: %s\n", src, dst) + curVals := make(map[string] string) + curVals["attempted"] = fmt.Sprintf("%d", numTot) + curVals["successful"] = fmt.Sprintf("%d", numSucc) + go func() { + if _, err := fb.Push(curVals); err != nil { + fmt.Println("error when logging to firebase in main.go") + } + }() + time.Sleep(time.Duration(1) * time.Second) + } + }() + for { resp, _ := etcdwatch.Next(context.Background()) pr := resp.Node.Value @@ -75,6 +100,5 @@ func main() { } recverwg.Wait() senderwg.Wait() - } diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 9044202..70884bc 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -6,6 +6,7 @@ export PAYMENT_SIZE=${PAYMENT_SIZE-100} # default paymen export SPIDER_LOG_FIREBASE=${SPIDER_LOG_FIREBASE-1} # enable firebase log by default export SPIDER_QUEUE=${SPIDER_QUEUE-0} # disable queue by default export EXP_TIME=${EXP_TIME-120} # default duration is 120s +export SPIDER_LP_ROUTING=${SPIDER_LP_ROUTING-0} # disable LP routing by default mkdir -p /root/log /root/scripts/runexp.sh &> /root/log/main.log & diff --git a/tools/fix_json.py b/tools/fix_json.py index af2c93f..885e92a 100755 --- a/tools/fix_json.py +++ b/tools/fix_json.py @@ -31,8 +31,9 @@ else: print("Invalid channel capacity") -for dm in dt['demands']: - dm['rate'] = int(dm['rate']) +for ch in dt['demands']: + if isinstance(ch['rate'], float): + ch['rate'] = int(ch['rate']) with open(filepath, 'w') as f: json.dump(dt, f, indent=4) diff --git a/tools/run_all.py b/tools/run_all.py new file mode 100644 index 0000000..7aefd54 --- /dev/null +++ b/tools/run_all.py @@ -0,0 +1,104 @@ +import argparse +import subprocess as sp +import time + +CMD_TEMPLATE = "./run.sh start-exp {topo_file} {exp_name} {time}" +TOPO_NAMES = ["two_node_circ", "three_node_circ", "four_node_circ", + "five_node_circ"] +TOPO_NODES = [4, 4, 4, 5] + +# TOPO_NAMES = ["three_node_circ", "four_node_circ", + # "five_node_circ"] +# TOPO_NODES = [3, 4, 5] + +FIREBASE_TEMPLATE = "python ~/lightning_routing/lnd-vis/main.py -exp_name {exp_name}" + +''' +sequence of commands: + - stop instances + - start instances N + - init docker + - init image + - sync + - rebuild + - repackage +''' + +def read_flags(): + parser = argparse.ArgumentParser() + parser.add_argument("-topos", type=int, required=False, + default=0, help="index of topo to run. -1 runs all.") + parser.add_argument("-exp_time", type=int, required=False, + default=60, help="") + parser.add_argument("-reps", type=int, required=False, + default=1, help="") + parser.add_argument("-exp_prefix", type=str, required=True, + help="") + parser.add_argument("-stop_instances", action='store_true', + help="") + parser.add_argument("-restart_instances", action='store_true', + help="") + return parser.parse_args() + +def run_exp(exp_num, exp_suffix): + topo_name = TOPO_NAMES[exp_num] + if args.restart_instances: + p = sp.Popen("./run.sh stop-instances", shell=True) + p.wait() + p = sp.Popen("./run.sh start-instances " + str(TOPO_NODES[exp_num]), + shell=True) + p.wait() + # sleep extra for instances to start + time.sleep(180) + p = sp.Popen("./run.sh init-docker", shell=True) + p.wait() + time.sleep(2) + p = sp.Popen("./run.sh init-image", shell=True) + p.wait() + time.sleep(4) + p = sp.Popen("./run.sh rebuild-binary lnd expctrl", shell=True) + p.wait() + print("rebuilt binaries") + time.sleep(5) + p = sp.Popen("./run.sh repackage-image", shell=True) + p.wait() + time.sleep(5) + + print("going to run exp: ", topo_name) + exp_name = args.exp_prefix + "-" + topo_name + "-" + exp_suffix + cmd = CMD_TEMPLATE.format(topo_file = "../topology/" + topo_name + ".json", + exp_name = exp_name, + time = args.exp_time) + print("cmd: ", cmd) + + sp.Popen(cmd, shell=True) + # sleep a bit extra, because why not + if args.exp_time <= 100: + time.sleep(args.exp_time*2) + else: + time.sleep(args.exp_time + 600) + p = sp.Popen("./run.sh stop-exp", shell=True) + p.wait() + time.sleep(5) + # test it. + cmd = FIREBASE_TEMPLATE.format(exp_name = exp_name) + try: + p = sp.Popen(cmd, shell=True) + p.wait() + except: + pass + # nothing + # time.sleep(10) + +args = read_flags() +for i in range(args.reps): + if args.topos == -1: + for j, _ in enumerate(TOPO_NAMES): + run_exp(j, str(i)) + else: + run_exp(args.topos, str(i)) + +# shut down it all +if args.stop_instances: + sp.Popen("./run.sh stop-instances", shell=True) + time.sleep(5)