added actual notification through signal api

main.go

@@ -22,13 +22,13 @@ var indexHTML = filepath.Join("templates", "index.html")
 var newWatchHTML = filepath.Join("templates", "newWatch.html")
 
 type Web struct {
-	//Bot *tgbotapi.BotAPI
 	router    *gin.Engine
 	templates multitemplate.Renderer
 	cron      *cron.Cron
 	urlCache  map[string]string
 	cronWatch map[uint]cron.Entry
 	db        *gorm.DB
+	Bot       *tgbotapi.BotAPI
 }
 
 func newWeb() *Web {
@@ -45,6 +45,7 @@ func (web *Web) init() {
 
 	web.initRouter()
 	web.initCronJobs()
+	web.initNotifiers()
 }
 
 func (web *Web) initDB() {
@@ -105,6 +106,36 @@ func (web *Web) initCronJobs() {
 	web.cron.Start()
 }
 
+func (web *Web) initNotifiers() {
+	bot, _ := tgbotapi.NewBotAPI(viper.GetString("telegram.token"))
+	bot.Debug = true
+	web.Bot = bot
+	log.Printf("Authorized on account %s", bot.Self.UserName)
+	go web.passiveBot()
+}
+
+func (web *Web) passiveBot() {
+	u := tgbotapi.NewUpdate(0)
+	u.Timeout = 60
+	updates := web.Bot.GetUpdatesChan(u)
+
+	for update := range updates {
+		if update.Message != nil { // If we got a message
+			log.Printf("[%s] %s", update.Message.From.UserName, update.Message.Text)
+
+			msg := tgbotapi.NewMessage(update.Message.Chat.ID, update.Message.Text)
+			msg.ReplyToMessageID = update.Message.MessageID
+
+			web.Bot.Send(msg)
+		}
+	}
+}
+
+func (web *Web) notify(message string) {
+	msg := tgbotapi.NewMessage(viper.GetInt64("telegram.chat"), message)
+	web.Bot.Send(msg)
+}
+
 func (web *Web) run() {
 	web.router.Run("0.0.0.0:8080")
 }
@@ -115,8 +146,6 @@ type WatchEntry struct {
 }
 
 func (web *Web) index(c *gin.Context) {
-	//msg := tgbotapi.NewMessage(viper.GetInt64("telegram.chat"), message)
-	//web.Bot.Send(msg)
 	watches := []Watch{}
 	web.db.Find(&watches)
 
@@ -243,7 +272,7 @@ func (web *Web) watchEdit(c *gin.Context) {
 	web.db.Model(&FilterOutput{}).Where("watch_id = ?", watch.ID).Find(&values)
 
 	buildFilterTree(filters, connections)
-	processFilters(filters, web, &watch, true, true)
+	processFilters(filters, web, &watch, true)
 
 	c.HTML(http.StatusOK, "watchEdit", gin.H{
 		"Watch":       watch,
@@ -328,23 +357,6 @@ func (web *Web) cacheClear(c *gin.Context) {
 	c.Redirect(http.StatusSeeOther, "/cache/view")
 }
 
-func passiveBot(bot *tgbotapi.BotAPI) {
-	u := tgbotapi.NewUpdate(0)
-	u.Timeout = 60
-	updates := bot.GetUpdatesChan(u)
-
-	for update := range updates {
-		if update.Message != nil { // If we got a message
-			log.Printf("[%s] %s", update.Message.From.UserName, update.Message.Text)
-
-			msg := tgbotapi.NewMessage(update.Message.Chat.ID, update.Message.Text)
-			msg.ReplyToMessageID = update.Message.MessageID
-
-			bot.Send(msg)
-		}
-	}
-}
-
 func main() {
 	viper.SetConfigName("config")
 	viper.SetConfigType("yaml")
@@ -356,14 +368,6 @@ func main() {
 		log.Fatalln("Could not load config file")
 	}
 
-	//bot, _ := tgbotapi.NewBotAPI(viper.GetString("telegram.token"))
-
-	//bot.Debug = true
-
-	//log.Printf("Authorized on account %s", bot.Self.UserName)
-
-	//go passiveBot(bot)
-
 	web := newWeb()
 	web.run()
 }

scraping.go

@@ -20,7 +20,7 @@ import (
 	"gorm.io/gorm"
 )
 
-func processFilters(filters []Filter, web *Web, watch *Watch, useCache bool, setCache bool) {
+func processFilters(filters []Filter, web *Web, watch *Watch, debug bool) {
 	allFilters := filters
 	processedMap := make(map[uint]bool, len(filters))
 	for len(filters) > 0 {
@@ -37,20 +37,20 @@ func processFilters(filters []Filter, web *Web, watch *Watch, useCache bool, set
 			filters = append(filters, *filter)
 			continue
 		}
-		getFilterResult(allFilters, filter, watch, web, useCache, setCache)
+		getFilterResult(allFilters, filter, watch, web, debug)
 		processedMap[filter.ID] = true
 	}
 }
 
-func getFilterResult(filters []Filter, filter *Filter, watch *Watch, web *Web, useCache bool, setCache bool) {
+func getFilterResult(filters []Filter, filter *Filter, watch *Watch, web *Web, debug bool) {
 	switch {
 	case filter.Type == "gurl":
 		{
-			getFilterResultURL(filter, web.urlCache, useCache, setCache)
+			getFilterResultURL(filter, web.urlCache, debug)
 		}
 	case filter.Type == "gurls":
 		{
-			getFilterResultURLs(filter, web.urlCache, useCache, setCache)
+			getFilterResultURLs(filter, web.urlCache, debug)
 		}
 	case filter.Type == "xpath":
 		{
@@ -115,7 +115,7 @@ func getFilterResult(filters []Filter, filter *Filter, watch *Watch, web *Web, u
 		}
 	case filter.Type == "notify":
 		{
-			notifyFilter(filters, filter, watch, web.db)
+			notifyFilter(filters, filter, watch, web, debug)
 		}
 	case filter.Type == "cron":
 		{
@@ -159,10 +159,10 @@ func getFilterResult(filters []Filter, filter *Filter, watch *Watch, web *Web, u
 	}
 }
 
-func getFilterResultURL(filter *Filter, urlCache map[string]string, useCache bool, setCache bool) {
+func getFilterResultURL(filter *Filter, urlCache map[string]string, debug bool) {
 	url := filter.Var1
 	val, exists := urlCache[url]
-	if useCache && exists {
+	if debug && exists {
 		filter.Results = append(filter.Results, val)
 		return
 	}
@@ -179,17 +179,17 @@ func getFilterResultURL(filter *Filter, urlCache map[string]string, useCache boo
 	}
 	str := string(body)
 	filter.Results = append(filter.Results, str)
-	if setCache {
+	if debug {
 		urlCache[url] = str
 	}
 }
 
-func getFilterResultURLs(filter *Filter, urlCache map[string]string, useCache bool, setCache bool) {
+func getFilterResultURLs(filter *Filter, urlCache map[string]string, debug bool) {
 	for _, parent := range filter.Parents {
 		for _, result := range parent.Results {
 			url := result
 			val, exists := urlCache[url]
-			if useCache && exists {
+			if debug && exists {
 				filter.Results = append(filter.Results, val)
 				continue
 			}
@@ -206,7 +206,7 @@ func getFilterResultURLs(filter *Filter, urlCache map[string]string, useCache bo
 			}
 			str := string(body)
 			filter.Results = append(filter.Results, str)
-			if setCache {
+			if debug {
 				urlCache[url] = str
 			}
 		}
@@ -736,7 +736,7 @@ func getFilterResultConditionHigherThan(filter *Filter) {
 	}
 }
 
-func notifyFilter(filters []Filter, filter *Filter, watch *Watch, db *gorm.DB) {
+func notifyFilter(filters []Filter, filter *Filter, watch *Watch, web *Web, debug bool) {
 	haveResults := false
 	for _, parent := range filter.Parents {
 		if len(parent.Results) > 0 {
@@ -762,8 +762,11 @@ func notifyFilter(filters []Filter, filter *Filter, watch *Watch, db *gorm.DB) {
 
 	var buffer bytes.Buffer
 	tmpl.Execute(&buffer, dataMap)
-
+	if debug {
-	log.Print(buffer.String())
+		log.Println(buffer.String())
+	} else {
+		web.notify(buffer.String())
+	}
 }
 
 func triggerSchedule(watchID uint, web *Web) {
@@ -777,5 +780,5 @@ func triggerSchedule(watchID uint, web *Web) {
 	web.db.Model(&FilterConnection{}).Where("watch_id = ?", watch.ID).Find(&connections)
 
 	buildFilterTree(filters, connections)
-	processFilters(filters, web, watch, true, true)
+	processFilters(filters, web, watch, false)
 }

todo.md

@@ -1,3 +1,4 @@
 # Todo
 - refactor getFilterResult so it's just part of web ?
 - add timer to run watch ?
+- make generic 'notifier' interface