broodjeaap/go-watch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

more fiddling/refactoring

Browse source
This commit is contained in:
BroodjeAap 2022-10-07 17:27:23 +00:00
parent 43814652f3
commit b2644e429d
2 changed files with 21 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
main.go
View file

@ -92,7 +92,7 @@ func (web *Web) initCronJobs() {
web.cronWatch = make(map[uint]cron.Entry, len(cronFilters)) web.cronWatch = make(map[uint]cron.Entry, len(cronFilters))
web.cron = cron.New() web.cron = cron.New()
for _, cronFilter := range cronFilters { for _, cronFilter := range cronFilters {
entryID, err := web.cron.AddFunc(cronFilter.Var1, func() { triggerSchedule(cronFilter.WatchID, web.db) }) entryID, err := web.cron.AddFunc(cronFilter.Var1, func() { triggerSchedule(cronFilter.WatchID, web) })
if err != nil { if err != nil {
log.Println("Could not start job for Watch: ", cronFilter.WatchID) log.Println("Could not start job for Watch: ", cronFilter.WatchID)
continue continue
@ -162,7 +162,7 @@ func (web *Web) watchView(c *gin.Context) {
web.db.Model(&FilterOutput{}).Where("watch_id = ?", watch.ID).Find(&values) web.db.Model(&FilterOutput{}).Where("watch_id = ?", watch.ID).Find(&values)
buildFilterTree(filters, connections) buildFilterTree(filters, connections)
processFilters(filters, web.db, web.urlCache, true, true) processFilters(filters, web, &watch, true, true)
c.HTML(http.StatusOK, "watchView", gin.H{ c.HTML(http.StatusOK, "watchView", gin.H{
"Watch": watch, "Watch": watch,

42
scraping.go
View file

@ -20,7 +20,7 @@ import (
"gorm.io/gorm" "gorm.io/gorm"
) )
func processFilters(filters []Filter, db *gorm.DB, urlCache map[string]string, useCache bool, setCache bool) { func processFilters(filters []Filter, web *Web, watch *Watch, useCache bool, setCache bool) {
allFilters := filters allFilters := filters
processedMap := make(map[uint]bool, len(filters)) processedMap := make(map[uint]bool, len(filters))
for len(filters) > 0 { for len(filters) > 0 {
@ -37,20 +37,20 @@ func processFilters(filters []Filter, db *gorm.DB, urlCache map[string]string, u
filters = append(filters, *filter) filters = append(filters, *filter)
continue continue
} }
getFilterResult(allFilters, filter, db, urlCache, useCache, setCache) getFilterResult(allFilters, filter, watch, web, useCache, setCache)
processedMap[filter.ID] = true processedMap[filter.ID] = true
} }
} }
func getFilterResult(filters []Filter, filter *Filter, db *gorm.DB, urlCache map[string]string, useCache bool, setCache bool) { func getFilterResult(filters []Filter, filter *Filter, watch *Watch, web *Web, useCache bool, setCache bool) {
switch { switch {
case filter.Type == "gurl": case filter.Type == "gurl":
{ {
getFilterResultURL(filter, urlCache, useCache, setCache) getFilterResultURL(filter, web.urlCache, useCache, setCache)
} }
case filter.Type == "gurls": case filter.Type == "gurls":
{ {
getFilterResultURLs(filter, urlCache, useCache, setCache) getFilterResultURLs(filter, web.urlCache, useCache, setCache)
} }
case filter.Type == "xpath": case filter.Type == "xpath":
{ {
@ -111,11 +111,11 @@ func getFilterResult(filters []Filter, filter *Filter, db *gorm.DB, urlCache map
} }
case filter.Type == "store": case filter.Type == "store":
{ {
storeFilterResult(filter, db) storeFilterResult(filter, web.db)
} }
case filter.Type == "notify": case filter.Type == "notify":
{ {
notifyFilter(filters, filter, db) notifyFilter(filters, filter, watch, web.db)
} }
case filter.Type == "cron": case filter.Type == "cron":
{ {
@ -126,15 +126,15 @@ func getFilterResult(filters []Filter, filter *Filter, db *gorm.DB, urlCache map
switch filter.Var1 { switch filter.Var1 {
case "diff": case "diff":
{ {
getFilterResultConditionDiff(filter, db) getFilterResultConditionDiff(filter, web.db)
} }
case "lowerl": case "lowerl":
{ {
getFilterResultConditionLowerLast(filter, db) getFilterResultConditionLowerLast(filter, web.db)
} }
case "lowest": case "lowest":
{ {
getFilterResultConditionLowest(filter, db) getFilterResultConditionLowest(filter, web.db)
} }
case "lowert": case "lowert":
{ {
@ -142,11 +142,11 @@ func getFilterResult(filters []Filter, filter *Filter, db *gorm.DB, urlCache map
} }
case "higherl": case "higherl":
{ {
getFilterResultConditionHigherLast(filter, db) getFilterResultConditionHigherLast(filter, web.db)
} }
case "highest": case "highest":
{ {
getFilterResultConditionHighest(filter, db) getFilterResultConditionHighest(filter, web.db)
} }
case "highert": case "highert":
{ {
@ -736,7 +736,7 @@ func getFilterResultConditionHigherThan(filter *Filter) {
} }
} }
func notifyFilter(filters []Filter, filter *Filter, db *gorm.DB) { func notifyFilter(filters []Filter, filter *Filter, watch *Watch, db *gorm.DB) {
haveResults := false haveResults := false
for _, parent := range filter.Parents { for _, parent := range filter.Parents {
if len(parent.Results) > 0 { if len(parent.Results) > 0 {
@ -758,10 +758,6 @@ func notifyFilter(filters []Filter, filter *Filter, db *gorm.DB) {
dataMap[f.Name] = html.UnescapeString(strings.Join(f.Results, ", ")) dataMap[f.Name] = html.UnescapeString(strings.Join(f.Results, ", "))
} }
id := filter.WatchID
var watch Watch
db.Model(&Watch{}).First(&watch, id)
dataMap["Watch"] = watch dataMap["Watch"] = watch
var buffer bytes.Buffer var buffer bytes.Buffer
@ -770,16 +766,16 @@ func notifyFilter(filters []Filter, filter *Filter, db *gorm.DB) {
log.Print(buffer.String()) log.Print(buffer.String())
} }
func triggerSchedule(watchID uint, db *gorm.DB) { func triggerSchedule(watchID uint, web *Web) {
var watch Watch var watch *Watch
db.Model(&Watch{}).First(&watch, watchID) web.db.Model(&Watch{}).First(&watch, watchID)
var filters []Filter var filters []Filter
db.Model(&Filter{}).Where("watch_id = ?", watch.ID).Find(&filters) web.db.Model(&Filter{}).Where("watch_id = ?", watch.ID).Find(&filters)
var connections []FilterConnection var connections []FilterConnection
db.Model(&FilterConnection{}).Where("watch_id = ?", watch.ID).Find(&connections) web.db.Model(&FilterConnection{}).Where("watch_id = ?", watch.ID).Find(&connections)
buildFilterTree(filters, connections) buildFilterTree(filters, connections)
processFilters(filters, db, make(map[string]string, 0), true, true) processFilters(filters, web, watch, true, true)
} }