mostly just minor changes

build.sh

@@ -3,7 +3,8 @@ export BUILDROOT="$PWD"
 export BTCPATH="$PWD/btcroot"
 export GOPATH="$PWD/goroot"
 export PATH="$PATH:$GOPATH/bin"
-export BRANCH='new-stats'
+export BRANCH='clean-waterfilling-dp'
+#export BRANCH='pari-lp-routing-debug'
 
 BUILD_LND='false'
 BUILD_EXPCTRL='false'
@@ -28,6 +29,7 @@ if [ "$BUILD_LND" == "true" ] || [ "$BUILD_EXPCTRL" == "true" ] ; then
 	go get -u github.com/golang/dep/cmd/dep
 	go get -d github.com/urfave/cli
 	go get -d github.com/vibhaa/lnd
+	go get gopkg.in/zabawaba99/firego.v1
 
 	rm -rf $GOPATH/src/github.com/lightningnetwork/lnd
 	mv $GOPATH/src/github.com/vibhaa/lnd $GOPATH/src/github.com/lightningnetwork
@@ -96,7 +98,7 @@ if [ "$BUILD_BITCOIND" == "true" ]; then
 	cp $BTCPATH/bitcoin/src/bitcoin-tx $BUILDROOT/binaries/
 fi
 
-echo "Cleaning up build files"
+#echo "Cleaning up build files"
 rm -rf $GOPATH
 rm -rf $BTCPATH
 rm -rf $BUILDROOT/downloads

expctrl/main.go

@@ -7,8 +7,11 @@ import (
 	"time"
 	"sync"
 	"strconv"
+	"gopkg.in/zabawaba99/firego.v1"
 )
 
+var FIREBASE_URL string = "https://spider2.firebaseio.com/"
+
 func main() {
 	etcd := getEtcdKeyClient()
 
@@ -36,6 +39,28 @@ func main() {
 				numTot := 0
 				numSucc := 0
 
+				// Logging gorouting
+				go func () {
+					fmt.Printf("started new routine, from %s to %s\n", demand.Source, demand.Destination)
+					src := demand.Source
+					dst := demand.Destination
+					EXP_NAME := os.Getenv("SPIDER_EXP_NAME")
+					fb := firego.New(FIREBASE_URL + EXP_NAME + "/aggregateStats/" + src + "/" + dst, nil)
+					fmt.Printf("started firebase, EXP_NAME = %s\n", EXP_NAME)
+					for {
+						//fmt.Printf("src: %s, dst: %s\n", src, dst)
+						curVals := make(map[string] string)
+						curVals["attempted"] = fmt.Sprintf("%d", numTot)
+						curVals["successful"] = fmt.Sprintf("%d", numSucc)
+						go func() {
+							if _, err := fb.Push(curVals); err != nil {
+								fmt.Println("error when logging to firebase in main.go")
+							}
+						}()
+						time.Sleep(time.Duration(1) * time.Second)
+					}
+				}()
+
 				for {
 					resp, _ := etcdwatch.Next(context.Background())
 					pr := resp.Node.Value
@@ -75,6 +100,5 @@ func main() {
 	}
 	recverwg.Wait()
 	senderwg.Wait()
-
 }
 

scripts/entrypoint.sh

@@ -6,6 +6,7 @@ export PAYMENT_SIZE=${PAYMENT_SIZE-100}                         # default paymen
 export SPIDER_LOG_FIREBASE=${SPIDER_LOG_FIREBASE-1}             # enable firebase log by default
 export SPIDER_QUEUE=${SPIDER_QUEUE-0}                           # disable queue by default
 export EXP_TIME=${EXP_TIME-120}                                 # default duration is 120s
+export SPIDER_LP_ROUTING=${SPIDER_LP_ROUTING-0}									# disable LP routing by default
 
 mkdir -p /root/log
 /root/scripts/runexp.sh &> /root/log/main.log &

tools/fix_json.py

@@ -31,8 +31,9 @@
     else:
         print("Invalid channel capacity")
 
-for dm in dt['demands']:
-    dm['rate'] = int(dm['rate'])
+for ch in dt['demands']:
+    if isinstance(ch['rate'], float):
+        ch['rate'] = int(ch['rate'])
 
 with open(filepath, 'w') as f:
     json.dump(dt, f, indent=4)

tools/run_all.py

@@ -0,0 +1,104 @@
+import argparse
+import subprocess as sp
+import time
+
+CMD_TEMPLATE = "./run.sh start-exp {topo_file} {exp_name} {time}"
+TOPO_NAMES = ["two_node_circ", "three_node_circ", "four_node_circ",
+        "five_node_circ"]
+TOPO_NODES = [4, 4, 4, 5]
+
+# TOPO_NAMES = ["three_node_circ", "four_node_circ",
+        # "five_node_circ"]
+# TOPO_NODES = [3, 4, 5]
+
+FIREBASE_TEMPLATE = "python ~/lightning_routing/lnd-vis/main.py -exp_name {exp_name}"
+
+'''
+sequence of commands:
+    - stop instances
+    - start instances N
+    - init docker
+    - init image
+    - sync 
+    - rebuild 
+    - repackage
+'''
+
+def read_flags():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-topos", type=int, required=False,
+            default=0, help="index of topo to run. -1 runs all.")
+    parser.add_argument("-exp_time", type=int, required=False,
+            default=60, help="")
+    parser.add_argument("-reps", type=int, required=False,
+            default=1, help="")
+    parser.add_argument("-exp_prefix", type=str, required=True,
+                                help="")
+    parser.add_argument("-stop_instances", action='store_true',
+                                help="")
+    parser.add_argument("-restart_instances", action='store_true',
+                                help="")
+    return parser.parse_args()
+
+def run_exp(exp_num, exp_suffix):
+    topo_name = TOPO_NAMES[exp_num]
+    if args.restart_instances:
+        p = sp.Popen("./run.sh stop-instances", shell=True)
+        p.wait()
+        p = sp.Popen("./run.sh start-instances " + str(TOPO_NODES[exp_num]),
+                shell=True)
+        p.wait()
+        # sleep extra for instances to start
+        time.sleep(180)
+        p = sp.Popen("./run.sh init-docker", shell=True) 
+        p.wait()
+        time.sleep(2)
+        p = sp.Popen("./run.sh init-image", shell=True) 
+        p.wait()
+        time.sleep(4)
+        p = sp.Popen("./run.sh rebuild-binary lnd expctrl", shell=True)
+        p.wait()
+        print("rebuilt binaries")
+        time.sleep(5)
+        p = sp.Popen("./run.sh repackage-image", shell=True)
+        p.wait()
+        time.sleep(5)
+
+    print("going to run exp: ", topo_name)
+    exp_name = args.exp_prefix + "-" + topo_name + "-" + exp_suffix
+    cmd = CMD_TEMPLATE.format(topo_file = "../topology/" + topo_name + ".json",
+                              exp_name = exp_name,
+                              time = args.exp_time)
+    print("cmd: ", cmd)
+
+    sp.Popen(cmd, shell=True)
+    # sleep a bit extra, because why not
+    if args.exp_time <= 100:
+        time.sleep(args.exp_time*2)
+    else:
+        time.sleep(args.exp_time + 600)
+    p = sp.Popen("./run.sh stop-exp", shell=True)
+    p.wait()
+    time.sleep(5)
+    # test it.
+    cmd = FIREBASE_TEMPLATE.format(exp_name = exp_name)
+    try:
+        p = sp.Popen(cmd, shell=True)
+        p.wait()
+    except:
+        pass
+        # nothing
+    # time.sleep(10)
+
+args = read_flags()
+for i in range(args.reps):
+    if args.topos == -1:
+        for j, _ in enumerate(TOPO_NAMES):
+            run_exp(j, str(i))
+    else:
+        run_exp(args.topos, str(i))
+
+# shut down it all
+if args.stop_instances:
+    sp.Popen("./run.sh stop-instances", shell=True)
+    time.sleep(5)