cron filters give a warning when theres an error in var1
This commit is contained in:
parent
e694853366
commit
01f3393968
2 changed files with 27 additions and 19 deletions
45
scraping.go
45
scraping.go
|
@ -15,6 +15,7 @@ import (
|
||||||
|
|
||||||
"github.com/andybalholm/cascadia"
|
"github.com/andybalholm/cascadia"
|
||||||
"github.com/antchfx/htmlquery"
|
"github.com/antchfx/htmlquery"
|
||||||
|
"github.com/robfig/cron/v3"
|
||||||
"github.com/tidwall/gjson"
|
"github.com/tidwall/gjson"
|
||||||
lualibs "github.com/vadv/gopher-lua-libs"
|
lualibs "github.com/vadv/gopher-lua-libs"
|
||||||
lua "github.com/yuin/gopher-lua"
|
lua "github.com/yuin/gopher-lua"
|
||||||
|
@ -65,31 +66,32 @@ func processFilters(filters []Filter, web *Web, watch *Watch, debug bool, schedu
|
||||||
nextFilters := make([]*Filter, 0, len(currentFilters))
|
nextFilters := make([]*Filter, 0, len(currentFilters))
|
||||||
for i := range currentFilters {
|
for i := range currentFilters {
|
||||||
filter := currentFilters[i]
|
filter := currentFilters[i]
|
||||||
if filter.Type == "store" {
|
if filter.Type == "store" {
|
||||||
storeFilters = append(storeFilters, filter)
|
storeFilters = append(storeFilters, filter)
|
||||||
processedMap[filter.ID] = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if debug && filter.Type == "cron" {
|
|
||||||
processedMap[filter.ID] = true
|
processedMap[filter.ID] = true
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if len(filter.Parents) == 0 && !debug {
|
if debug && filter.Type == "cron" {
|
||||||
continue
|
processedMap[filter.ID] = true
|
||||||
}
|
getCronDebugResult(filter)
|
||||||
var allParentsProcessed = true
|
continue
|
||||||
for _, parent := range filter.Parents {
|
|
||||||
if _, contains := processedMap[parent.ID]; !contains {
|
|
||||||
allParentsProcessed = false
|
|
||||||
break
|
|
||||||
}
|
}
|
||||||
}
|
if len(filter.Parents) == 0 && !debug {
|
||||||
if !allParentsProcessed {
|
continue
|
||||||
|
}
|
||||||
|
var allParentsProcessed = true
|
||||||
|
for _, parent := range filter.Parents {
|
||||||
|
if _, contains := processedMap[parent.ID]; !contains {
|
||||||
|
allParentsProcessed = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !allParentsProcessed {
|
||||||
nextFilters = append(nextFilters, filter)
|
nextFilters = append(nextFilters, filter)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
getFilterResult(filters, filter, watch, web, debug)
|
getFilterResult(filters, filter, watch, web, debug)
|
||||||
processedMap[filter.ID] = true
|
processedMap[filter.ID] = true
|
||||||
}
|
}
|
||||||
if len(nextFilters) == 0 {
|
if len(nextFilters) == 0 {
|
||||||
break
|
break
|
||||||
|
@ -850,6 +852,13 @@ func triggerSchedule(watchID uint, web *Web, scheduleID *uint) {
|
||||||
processFilters(filters, web, watch, false, scheduleID)
|
processFilters(filters, web, watch, false, scheduleID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getCronDebugResult(filter *Filter) {
|
||||||
|
_, err := cron.ParseStandard(filter.Var1)
|
||||||
|
if err != nil {
|
||||||
|
filter.log(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func getFilterResultLua(filter *Filter) {
|
func getFilterResultLua(filter *Filter) {
|
||||||
L := lua.NewState()
|
L := lua.NewState()
|
||||||
defer L.Close()
|
defer L.Close()
|
||||||
|
|
1
todo.md
1
todo.md
|
@ -11,5 +11,4 @@
|
||||||
- log things to db for cron runs
|
- log things to db for cron runs
|
||||||
- comments
|
- comments
|
||||||
- db prune job
|
- db prune job
|
||||||
- add var1 parsing test for cron jobs
|
|
||||||
- add full buildFilterTree/processFilters test
|
- add full buildFilterTree/processFilters test
|
Loading…
Add table
Reference in a new issue