new index service

.gitignore

@@ -15,3 +15,6 @@ bin/*
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+main
+.vscode
+*.log

cmd/main.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"os"
+
+	"github.com/iotexproject/iotex-core/pkg/log"
+	"github.com/urfave/cli/v2"
+	"go.uber.org/zap"
+
+	"github.com/iotexproject/iotex-analytics/cmd/server"
+)
+
+var (
+	version = "0.1.0-dev"
+)
+
+func main() {
+	app := cli.NewApp()
+	app.Name = "iotex-analytics"
+	app.Usage = "iotex-analytics is Analytics Platform for Iotex Smart Chain"
+	app.Version = version
+	app.Commands = []*cli.Command{
+		server.Commands,
+	}
+	if err := app.Run(os.Args); err != nil {
+		log.L().Fatal("Failed to start application", zap.Error(err))
+	}
+}

cmd/server/server.go

@@ -0,0 +1,127 @@
+package server
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"time"
+
+	"github.com/iotexproject/iotex-analytics/config"
+	"github.com/iotexproject/iotex-analytics/indexer"
+	"github.com/iotexproject/iotex-analytics/indexservice"
+	"github.com/iotexproject/iotex-analytics/sql"
+	"github.com/iotexproject/iotex-core/action/protocol"
+	"github.com/iotexproject/iotex-core/blockchain/blockdao"
+	"github.com/iotexproject/iotex-core/blockchain/genesis"
+	"github.com/iotexproject/iotex-core/pkg/log"
+	"github.com/iotexproject/iotex-proto/golang/iotexapi"
+	"github.com/urfave/cli/v2"
+	"go.uber.org/zap"
+	"google.golang.org/grpc"
+)
+
+var Commands = &cli.Command{
+	Name:        "server",
+	Usage:       "start server",
+	Description: `start server`,
+	Action:      runServer,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "config",
+			Aliases: []string{"c"},
+			Value:   config.FindDefaultConfigPath(),
+			Usage:   "Load configuration from `FILE`",
+		},
+	},
+}
+
+func runServer(c *cli.Context) error {
+	var err error
+	var cfg *config.Config
+	if c.String("config") == "" {
+		log.L().Fatal("Cannot determine default configuration path.",
+			zap.Any("DefaultConfigFiles", config.DefaultConfigFiles),
+			zap.Any("DefaultConfigDirs", config.DefaultConfigDirs))
+	}
+
+	cfg, err = config.New(c.String("config"))
+	if err != nil {
+		return err
+	}
+	if err := log.InitLoggers(cfg.Log, cfg.SubLogs); err != nil {
+		fmt.Fprintf(os.Stderr, "ERROR: Failed to init logger: %v\n", err)
+		os.Exit(1)
+	}
+
+	store := sql.CreateMySQLStore(
+		cfg.Mysql.UserName,
+		cfg.Mysql.Password,
+		cfg.Mysql.Host,
+		cfg.Mysql.Port,
+		cfg.Mysql.DBName,
+	)
+	go func() {
+		if err := store.Start(c.Context); err != nil {
+			log.L().Fatal("Failed to start mysql store", zap.Error(err))
+		}
+	}()
+	ctx := sql.WithStore(c.Context, store)
+
+	grpcCtx1, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	conn1, err := grpc.DialContext(grpcCtx1, cfg.Iotex.ChainEndPoint, grpc.WithBlock(), grpc.WithInsecure())
+	if err != nil {
+		log.L().Error("Failed to connect to chain's API server.")
+	}
+	chainClient := iotexapi.NewAPIServiceClient(conn1)
+
+	var indexers []blockdao.BlockIndexer
+	var dao blockdao.BlockDAO
+	if false {
+		dao = blockdao.NewBlockDAOInMemForTest(indexers)
+	} else {
+		dao = blockdao.NewBlockDAO(indexers, cfg.BlockDB)
+	}
+	var tip protocol.TipInfo
+	ctx = protocol.WithBlockchainCtx(
+		ctx,
+		protocol.BlockchainCtx{
+			Genesis: genesis.Default,
+			Tip:     tip,
+		},
+	)
+	var asyncindexers []indexer.AsyncIndexer
+
+	asyncindexers = append(asyncindexers, indexer.NewBlockIndexer(store))
+	ids := indexservice.NewIndexService(chainClient, 64, dao, asyncindexers)
+	go func() {
+		if err := ids.Start(ctx); err != nil {
+			log.L().Fatal("Failed to start the indexer", zap.Error(err))
+		}
+	}()
+	handleShutdown(store, ids)
+
+	return nil
+}
+
+type Stopper interface {
+	Stop(context.Context) error
+}
+
+func handleShutdown(service ...Stopper) {
+	stop := make(chan os.Signal)
+	signal.Notify(stop, os.Interrupt, os.Kill)
+
+	// wait INT or KILL
+	<-stop
+	log.L().Info("shutting down ...")
+	ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
+	defer cancel()
+	for _, s := range service {
+		if err := s.Stop(ctx); err != nil {
+			log.L().Error("shutdown", zap.Error(err))
+		}
+	}
+	return
+}

config/config.go

@@ -0,0 +1,113 @@
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	coreconfig "github.com/iotexproject/iotex-core/config"
+	"github.com/iotexproject/iotex-core/pkg/log"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+)
+
+var (
+	// Default is the default config
+	Default = Config{
+		Server: Server{
+			Host: "127.0.0.1",
+			Port: "8113",
+		},
+		BlockDB: coreconfig.DB{
+			NumRetries:            3,
+			MaxCacheSize:          64,
+			BlockStoreBatchSize:   16,
+			V2BlocksToSplitDB:     1000000,
+			Compressor:            "Snappy",
+			CompressLegacy:        false,
+			SplitDBSizeMB:         0,
+			SplitDBHeight:         900000,
+			HistoryStateRetention: 2000,
+		},
+		SubLogs: make(map[string]log.GlobalConfig),
+	}
+)
+
+type (
+	Server struct {
+		Host string `yaml:"host"`
+		Port string `yaml:"port"`
+	}
+	Mysql struct {
+		DBName   string `yaml:"dbname"`
+		UserName string `yaml:"username"`
+		Password string `yaml:"password"`
+		Host     string `yaml:"host"`
+		Port     string `yaml:"port"`
+	}
+	Iotex struct {
+		ChainEndPoint string `yaml:"chainEndPoint"`
+	}
+	Config struct {
+		Server  Server                      `yaml:"server"`
+		Mysql   Mysql                       `yaml:"mysql"`
+		Iotex   Iotex                       `yaml:"iotex"`
+		BlockDB coreconfig.DB               `yaml:"blockDB"`
+		Log     log.GlobalConfig            `yaml:"log"`
+		SubLogs map[string]log.GlobalConfig `yaml:"subLogs" json:"subLogs"`
+	}
+)
+
+func New(path string) (cfg *Config, err error) {
+	body, err := ioutil.ReadFile(path)
+	if err != nil {
+		return cfg, errors.Wrap(err, "failed to read config content")
+	}
+	cfg = &Default
+	if err = yaml.Unmarshal(body, cfg); err != nil {
+		return cfg, errors.Wrap(err, "failed to unmarshal config to struct")
+	}
+	return
+}
+
+var (
+	// File names from which we attempt to read configuration.
+	DefaultConfigFiles = []string{"config.yml", "config.yaml"}
+
+	// Launchd doesn't set root env variables, so there is default
+	DefaultConfigDirs = []string{"~/.iotex-analytics", "/usr/local/etc/iotex-analytics", "/etc/iotex-analytics"}
+)
+
+// FileExists checks to see if a file exist at the provided path.
+func FileExists(path string) (bool, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// ignore missing files
+			return false, nil
+		}
+		return false, err
+	}
+	f.Close()
+	return true, nil
+}
+
+// FindDefaultConfigPath returns the first path that contains a config file.
+// If none of the combination of DefaultConfigDirs and DefaultConfigFiles
+// contains a config file, return empty string.
+func FindDefaultConfigPath() string {
+	for _, configDir := range DefaultConfigDirs {
+		for _, configFile := range DefaultConfigFiles {
+			dirPath, err := homedir.Expand(configDir)
+			if err != nil {
+				continue
+			}
+			path := filepath.Join(dirPath, configFile)
+			if ok, _ := FileExists(path); ok {
+				return path
+			}
+		}
+	}
+	return ""
+}

config_new.yaml

@@ -0,0 +1,23 @@
+server:
+  port: 8883
+mysql:
+  host: 127.0.0.1
+  port: 3306
+  username: root
+  password: root
+  dbname: iotex_analytics
+iotex:
+  chainEndPoint: api.testnet.iotex.one:80
+blockDB:
+  dbPath: chain.db
+log:
+  zap:
+    development: true
+    level: debug
+    encoding: console
+    disableCaller: true
+    disableStacktrace: false
+    outputPaths: ["stdout"]
+    errorOutputPaths: ["stderr"]
+  stderrRedirectFile: error.log
+  stdLogRedirect: false

go.mod

@@ -9,22 +9,24 @@ require (
 	github.com/ethereum/go-ethereum v1.9.5
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/golang/mock v1.4.4
-	github.com/golang/protobuf v1.4.2
-	github.com/iotexproject/go-pkgs v0.1.2-0.20200523040337-5f1d9ddaa8ee
-	github.com/iotexproject/iotex-address v0.2.2
-	github.com/iotexproject/iotex-core v1.1.3
+	github.com/golang/protobuf v1.4.3
+	github.com/iotexproject/go-pkgs v0.1.5-0.20210105202208-2dc9b27250a6
+	github.com/iotexproject/iotex-address v0.2.4
+	github.com/iotexproject/iotex-core v1.1.4-0.20210309070336-f9809474d1f4
 	github.com/iotexproject/iotex-election v0.3.5-0.20201031050050-c3ab4f339a54
-	github.com/iotexproject/iotex-proto v0.4.4-0.20201029172022-a8466422b0f1
+	github.com/iotexproject/iotex-proto v0.4.7
+	github.com/mitchellh/go-homedir v1.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.3.0
 	github.com/rs/zerolog v1.18.0
-	github.com/stretchr/testify v1.5.1
+	github.com/stretchr/testify v1.6.1
+	github.com/urfave/cli/v2 v2.3.0
 	github.com/vektah/gqlparser v1.1.2
 	go.uber.org/zap v1.10.0
-	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
 	golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566 // indirect
-	google.golang.org/grpc v1.28.0
-	gopkg.in/yaml.v2 v2.2.8
+	google.golang.org/grpc v1.33.1
+	gopkg.in/yaml.v2 v2.4.0
 )
 
 replace github.com/ethereum/go-ethereum => github.com/iotexproject/go-ethereum v0.3.1