ads_handler.go

@@ -100,6 +100,7 @@ func adsHandler(c *gin.Context) {
 			Androidid    string `json:"android_id"`
 			ShortMessage string `json:"short_message"`
 			ReqSource    string `json:"req_source"`
+			Hehe         string `json:"hehe"`
 		}{
 			Ip:           ip,
 			Imei:         request.Imei,
@@ -113,6 +114,7 @@ func adsHandler(c *gin.Context) {
 			Androidid:    request.Androidid,
 			ShortMessage: "ads_api_log",
 			ReqSource:    "zhiku",
+			Hehe:         "heheheheh",
 		}
 
 		graylog.Log(grayLogData)
@@ -506,6 +508,7 @@ func adsHandler(c *gin.Context) {
 			// 先跑定投
 			customAdData, err := addata.GetCustomAdsInfos(&dspInfo, 0, 0, xiaomiResponseFlag)
 			if err != nil {
+				log.WithField("request_id", dspInfo.RequestId).Infof("get custom ads info failed: %s", err)
 				c.String(404, "get custom ads info failed: %s", err.Error())
 				return
 			}
@@ -515,6 +518,7 @@ func adsHandler(c *gin.Context) {
 			if customAdData == nil {
 				customAdData, err = addata.GetCustomAdsInfos(&dspInfo, 0, 1, xiaomiResponseFlag)
 				if err != nil {
+					log.WithField("request_id", dspInfo.RequestId).Errorf("get fix custom ads info failed: %s", err)
 					c.String(404, "get fix custom ads info failed: %s", err.Error())
 					return
 				}
@@ -603,6 +607,7 @@ func adsHandler(c *gin.Context) {
 					log.WithField("request_id", dspInfo.RequestId).Infof("rsp1: %+v", customAdData)
 					jsonBytes, err := json.Marshal(customAdData)
 					if err != nil {
+						log.WithField("request_id", dspInfo.RequestId).Errorf("marshal rsp1 custom ad failed: %s", err)
 						c.String(404, "marsha custom addata failed: %s", err.Error())
 						return
 					}
@@ -617,6 +622,7 @@ func adsHandler(c *gin.Context) {
 					graylog.ReportGrayLog(request, dspInfo, 0, 0)
 					rspBytes, err := json.Marshal(response)
 					if err != nil {
+						log.WithField("request_id", dspInfo.RequestId).Errorf("marshal rsp1 failed: %s", err)
 						c.String(404, "marshal Response failed: %s", err.Error())
 						return
 					}
@@ -629,9 +635,11 @@ func adsHandler(c *gin.Context) {
 					graylog.ReportGrayLog(request, dspInfo, 0, 0)
 					rspBytes, err := json.Marshal(customAdData)
 					if err != nil {
+						log.WithField("request_id", dspInfo.RequestId).Errorf("marshal rsp2 failed: %s", err)
 						c.String(404, err.Error())
 						return
 					}
+					log.WithField("request_id", dspInfo.RequestId).Infof("rsp2: %s", rspBytes)
 					c.String(200, string(rspBytes))
 					return
 				}
@@ -641,6 +649,7 @@ func adsHandler(c *gin.Context) {
 	redis_data.SetAdsRealRequestNum(advertiser)
 	rspBytes, err := json.Marshal(response)
 	if err != nil {
+		log.WithField("request_id", dspInfo.RequestId).Errorf("marshal rsp3 failed: %s", err)
 		c.String(404, err.Error())
 		return
 	}

adslib/addata/custom.go

@@ -54,6 +54,8 @@ func GetOneAds(dsp *utils.DspParam, orderType int, fixFlag int) (*redis_data.AdO
 	if err != nil {
 		return nil, err
 	}
+
+	log.WithField("request_id", dsp.RequestId).Infof("get one ads, order len: %d", len(orders))
 	if len(orders) == 0 {
 		return nil, nil
 	}
@@ -150,11 +152,12 @@ func GetCustomAdsInfos(dsp *utils.DspParam, orderType int, fixFlag int, xiaomiHa
 		log.WithField("request_id", dsp.RequestId).Errorf("get one ads failed: %s", err)
 		return nil, err
 	}
+	log.WithField("request_id", dsp.RequestId).Infof("lat get one ad info, order: %+v, xiaomi has flag: %d", order, xiaomiHasFlag)
+
 	if order == nil {
 		return nil, nil
 	}
 
-	log.WithField("request_id", dsp.RequestId).Infof("begin got custom ad info, xiaomi has flag: %d", xiaomiHasFlag)
 	if xiaomiHasFlag == 1 {
 		if strings.Index(order.Title, "_ios") > 0 {
 			return nil, nil
@@ -186,16 +189,17 @@ func GetCustomAdsInfos(dsp *utils.DspParam, orderType int, fixFlag int, xiaomiHa
 	rate := float32(order.ShowKpi) / float32(needDispatchCnt)
 
 	lineValue := 0
+	key := ""
 	if order.LineType == 1 {
 		// 订单定制曲线
-		lineValue, err = redis_data.GetOrderPerMinuteNeedDispatchCnt(order.OrderID, curTime)
+		lineValue, key, err = redis_data.GetOrderPerMinuteNeedDispatchCnt(order.OrderID, curTime)
 	} else {
-		lineValue, err = redis_data.GetPerMinuteNeedDispatchCnt(curTime)
+		lineValue, key, err = redis_data.GetPerMinuteNeedDispatchCnt(curTime)
 	}
 	if err != nil {
 		return nil, err
 	}
-	log.WithField("request_id", dsp.RequestId).Infof("line value: %d, rate: %f", lineValue, rate)
+	log.WithField("request_id", dsp.RequestId).Infof("key: %s, line value: %d, rate: %f", key, lineValue, rate)
 
 	// 当前分钟需要放出去的数量
 	curMinuteNeedDispatchCnt := int(float32(lineValue) * rate)
@@ -205,11 +209,10 @@ func GetCustomAdsInfos(dsp *utils.DspParam, orderType int, fixFlag int, xiaomiHa
 
 	redis_data.SetOrderPlanDispatchCount(order.OrderID, "show", curMinuteNeedDispatchCnt, curTime)
 
-	log.WithField("request_id", dsp.RequestId).Infof("tt thirds: %+v", order)
 	// 获取当前分钟已经完成的下发
-	curMinuteFinishedDispatchCnt, err := redis_data.GetPreMinuteFinishedDispatchCount(order.OrderID, "show", curTime)
+	curMinuteFinishedDispatchCnt, key, err := redis_data.GetPreMinuteFinishedDispatchCount(order.OrderID, "show", curTime)
 
-	log.WithField("request_id", dsp.RequestId).Infof("o value: %d, w value: %d", curMinuteFinishedDispatchCnt, curMinuteNeedDispatchCnt)
+	log.WithField("request_id", dsp.RequestId).Infof("o value: %s, %d, w value: %d", key, curMinuteFinishedDispatchCnt, curMinuteNeedDispatchCnt)
 	if curMinuteFinishedDispatchCnt < curMinuteNeedDispatchCnt {
 		data := AdData{
 			Duration:  5,
@@ -219,11 +222,11 @@ func GetCustomAdsInfos(dsp *utils.DspParam, orderType int, fixFlag int, xiaomiHa
 		}
 
 		//放量
-		_, err := redis_data.IncrFinishedDispatchCount(order.OrderID, "show", 1, curTime)
+		_, err := redis_data.IncrFinishedDispatchCount(order.OrderID, "show", curTime)
 		if err != nil {
 			return nil, err
 		}
-		_, err = redis_data.IncrPreMinuteFinishedDispatchCount(order.OrderID, "show", 1, curTime)
+		_, err = redis_data.IncrPreMinuteFinishedDispatchCount(order.OrderID, "show", curTime)
 		if err != nil {
 			return nil, err
 		}
@@ -326,7 +329,7 @@ func GetCustomAdsInfos(dsp *utils.DspParam, orderType int, fixFlag int, xiaomiHa
 			addi = genMonitorAction("CLICK", order.Title, dsp.ReqSource, clickUrl)
 			data.TargetAddition = append(data.TargetAddition, *addi)
 			md5Skip := utils.Md5(targetUrl)
-			_, err = redis_data.IncrFinishedDispatchCount(order.OrderID, "click", 1, curTime)
+			_, err = redis_data.IncrFinishedDispatchCount(order.OrderID, "click", curTime)
 			if err != nil {
 				log.WithField("request_id", dsp.RequestId).Errorf("incr finished dispatch count failed: %s", err)
 			}

adslib/ads_redis/redis.go

@@ -14,9 +14,9 @@ func Setup() error {
 	host := adslib.GetConf().RedisHost
 	password := adslib.GetConf().RedisPassword
 	RedisConn = &redis.Pool{
-		MaxIdle:     10,
-		MaxActive:   10,
-		IdleTimeout: 50,
+		MaxIdle:     20,
+		MaxActive:   50,
+		IdleTimeout: 3 * time.Minute,
 		Dial: func() (redis.Conn, error) {
 			c, err := redis.Dial("tcp", host)
 			if err != nil {
@@ -32,11 +32,6 @@ func Setup() error {
 			}
 			return c, err
 		},
-		TestOnBorrow: func(c redis.Conn, t time.Time) error {
-			_, err := c.Do("PING")
-			fmt.Println(err)
-			return err
-		},
 	}
 
 	return nil

adslib/config.go

@@ -26,6 +26,7 @@ type SvrConf struct {
 	LogPath            string                                `toml:"log_path"`
 	RedisHost          string                                `toml:"redis_host"`
 	RedisPassword      string                                `toml:"redis_password"`
+	LogLevel           string                                `toml:"log_level"`
 }
 type FlogControlItem struct {
 	Percent  int `toml:"percent"`

adslib/graylog/graylog.go

@@ -16,9 +16,9 @@ var grayLog *gelf.Gelf
 
 func init() {
 	grayLog = gelf.New(gelf.Config{
-		GraylogPort:     62001,
-		GraylogHostname: "172.18.138.7",
-		Connection:      "wan",
+		GraylogPort:     62002,
+		GraylogHostname: "172.18.199.72",
+		Connection:      "lan",
 		MaxChunkSizeWan: 42,
 		MaxChunkSizeLan: 1337,
 	})

adslib/redis_data/order.go

@@ -113,6 +113,7 @@ func GetOrderInfos(dsp *utils.DspParam, fixFlag int) ([]AdOrderInfo, error) {
 					}
 				}
 				if !isValidChannel {
+					log.WithField("request_id", dsp.RequestId).Tracef("not valid channel ad: %+v", order)
 					continue
 				}
 			}
@@ -153,7 +154,7 @@ func GetOrderInfos(dsp *utils.DspParam, fixFlag int) ([]AdOrderInfo, error) {
 		}
 
 		if !needInsert {
-			return []AdOrderInfo{}, nil
+			continue
 		}
 
 		if dsp != nil {

adslib/redis_data/redis_data.go

@@ -312,12 +312,12 @@ func GetFinishedDispatchCount(orderId int64, dispatchType string, curTime time.T
 
 // 增加已经投放的量
 // dispatchType: "click", "show"
-func IncrFinishedDispatchCount(orderId int64, dispatchType string, incrCnt int, curTime time.Time) (int, error) {
+func IncrFinishedDispatchCount(orderId int64, dispatchType string, curTime time.Time) (int, error) {
 	conn := ads_redis.RedisConn.Get()
 	defer conn.Close()
 
 	key := fmt.Sprintf("order_kpi_%d_%d%d%d_%s_kpi", orderId, curTime.Year(), curTime.Month(), curTime.Day(), dispatchType)
-	rsp, err := conn.Do("INCR", key, incrCnt)
+	rsp, err := conn.Do("INCR", key)
 	if err != nil {
 		return 0, err
 	}
@@ -328,35 +328,34 @@ func IncrFinishedDispatchCount(orderId int64, dispatchType string, incrCnt int,
 
 // 获取当前分钟已经投放的量
 // dispatchType: "click", "show"
-func GetPreMinuteFinishedDispatchCount(orderId int64, dispatchType string, curTime time.Time) (int, error) {
+func GetPreMinuteFinishedDispatchCount(orderId int64, dispatchType string, curTime time.Time) (int, string, error) {
 	conn := ads_redis.RedisConn.Get()
 	defer conn.Close()
 
 	curMinutes := curTime.Hour()*60 + curTime.Minute()
 	key := fmt.Sprintf("order_kpi_%d_%d%d%d_%d_%s_kpi", orderId, curTime.Year(), curTime.Month(), curTime.Day(), curMinutes, dispatchType)
 	rsp, err := conn.Do("GET", key)
-	log.Infof("%s %s", key, rsp)
 	if err != nil {
-		return 0, err
+		return 0, key, err
 	}
 
 	if rsp == nil {
-		return 0, nil
+		return 0, key, nil
 	}
 
 	count, _ := redis.Int(rsp, err)
-	return count, nil
+	return count, key, nil
 }
 
 // 增加当前分钟已经投放的量
 // dispatchType: "click", "show"
-func IncrPreMinuteFinishedDispatchCount(orderId int64, dispatchType string, incrCnt int, curTime time.Time) (int, error) {
+func IncrPreMinuteFinishedDispatchCount(orderId int64, dispatchType string, curTime time.Time) (int, error) {
 	conn := ads_redis.RedisConn.Get()
 	defer conn.Close()
 
 	curMinutes := curTime.Hour()*60 + curTime.Minute()
 	key := fmt.Sprintf("order_kpi_%d_%d%d%d_%d_%s_kpi", orderId, curTime.Year(), curTime.Month(), curTime.Day(), curMinutes, dispatchType)
-	rsp, err := conn.Do("INCR", key, incrCnt)
+	rsp, err := conn.Do("INCR", key)
 	if err != nil {
 		return 0, err
 	}
@@ -386,7 +385,7 @@ func SetOrderPlanDispatchCount(orderId int64, dispatchType string, cnt int, curT
 }
 
 // 获取总量每分钟要投放的量
-func GetPerMinuteNeedDispatchCnt(curTime time.Time) (int, error) {
+func GetPerMinuteNeedDispatchCnt(curTime time.Time) (int, string, error) {
 	conn := ads_redis.RedisConn.Get()
 	defer conn.Close()
 
@@ -395,19 +394,19 @@ func GetPerMinuteNeedDispatchCnt(curTime time.Time) (int, error) {
 	rsp, err := conn.Do("GET", key)
 
 	if err != nil {
-		return 0, err
+		return 0, key, err
 	}
 
 	if rsp == nil {
-		return 0, nil
+		return 0, key, nil
 	}
 
 	count, _ := redis.Int(rsp, err)
-	return count, nil
+	return count, key, nil
 }
 
 // 获取订单每分钟要投放的量
-func GetOrderPerMinuteNeedDispatchCnt(orderId int64, curTime time.Time) (int, error) {
+func GetOrderPerMinuteNeedDispatchCnt(orderId int64, curTime time.Time) (int, string, error) {
 	conn := ads_redis.RedisConn.Get()
 	defer conn.Close()
 
@@ -416,15 +415,15 @@ func GetOrderPerMinuteNeedDispatchCnt(orderId int64, curTime time.Time) (int, er
 	rsp, err := conn.Do("GET", key)
 
 	if err != nil {
-		return 0, err
+		return 0, key, err
 	}
 
 	if rsp == nil {
-		return 0, nil
+		return 0, key, nil
 	}
 
 	count, _ := redis.Int(rsp, err)
-	return count, nil
+	return count, key, nil
 }
 
 // 获取ios的ua和imei
@@ -464,8 +463,11 @@ func GetIosUaImei(ip string) (string, string, error) {
 		Imei string
 		Ua   string
 	}
+
 	rspBytes, _ := redis.Bytes(rsp, err)
-	err = json.Unmarshal(rspBytes, &uaImeiInfo)
+	rspStr := strings.ReplaceAll(string(rspBytes), "'", "\"")
+
+	err = json.Unmarshal([]byte(rspStr), &uaImeiInfo)
 	if err != nil {
 		return "", "", err
 	}

conf/config.toml

@@ -12,6 +12,7 @@ ip2region_file = "../conf/ip2region_20200113.db"
 log_path = "../log"
 redis_host="r-wz9sbx1faq7j6zxxgt.redis.rds.aliyuncs.com:6379"
 redis_password="3iXAJFuI$J"
+log_level=trace
 
 
 ## 流量占比控制

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
 	github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 // indirect
 	github.com/fvbock/endless v0.0.0-20170109170031-447134032cb6
+	github.com/gin-contrib/pprof v1.3.0
 	github.com/gin-gonic/gin v1.6.3
 	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/jarcoal/httpmock v1.0.5

go.sum

@@ -11,8 +11,11 @@ github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 h1:Ghm4eQYC0nEPnSJ
 github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239/go.mod h1:Gdwt2ce0yfBxPvZrHkprdPPTTS3N5rwmLE8T22KBXlw=
 github.com/fvbock/endless v0.0.0-20170109170031-447134032cb6 h1:6VSn3hB5U5GeA6kQw4TwWIWbOhtvR2hmbBJnTOtqTWc=
 github.com/fvbock/endless v0.0.0-20170109170031-447134032cb6/go.mod h1:YxOVT5+yHzKvwhsiSIWmbAYM3Dr9AEEbER2dVayfBkg=
+github.com/gin-contrib/pprof v1.3.0 h1:G9eK6HnbkSqDZBYbzG4wrjCsA4e+cvYAHUZw6W+W9K0=
+github.com/gin-contrib/pprof v1.3.0/go.mod h1:waMjT1H9b179t3CxuG1cV3DHpga6ybizwfBaM5OXaB0=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.6.2/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=

main_linux.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/fvbock/endless"
+	"github.com/gin-contrib/pprof"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -40,6 +41,7 @@ func setupLogger(logName string, logLevel log.Level) {
 
 func setupRouter() *gin.Engine {
 	r := gin.Default()
+	pprof.Register(r) // 性能
 
 	r.POST("/ads", adsHandler)
 	return r
@@ -47,8 +49,13 @@ func setupRouter() *gin.Engine {
 
 func main() {
 	ads_redis.Setup()
-	adslib.GetConf()
-	setupLogger(adslib.GetConf().LogPath+"/ads.log", log.TraceLevel)
+	conf := adslib.GetConf()
+	logLevel, err := log.ParseLevel(conf.LogLevel)
+	if err != nil {
+		log.Errorf("parse log level failed, use info level, log str: %s, err: %s", conf.LogLevel, err)
+		logLevel = log.InfoLevel
+	}
+	setupLogger(conf.LogPath+"/ads.log", logLevel)
 
 	r := setupRouter()
 	endless.ListenAndServe(":1111", r)