broodjeaap/go-watch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

filter.Var2 is now a value instead of a pointer

Browse source
This commit is contained in:
BroodjeAap 2023-03-27 19:08:51 +00:00
parent 0c52878b08
commit 7d377e905a
6 changed files with 36 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
models/filter.go
View file

@ -17,13 +17,12 @@ type Filter struct {
Y int `form:"y" yaml:"y" json:"y" validate:"default=0"` Y int `form:"y" yaml:"y" json:"y" validate:"default=0"`
Type FilterType `form:"filter_type" yaml:"filter_type" json:"filter_type" binding:"required" validate:"oneof=url xpath json css replace match substring math store condition cron"` Type FilterType `form:"filter_type" yaml:"filter_type" json:"filter_type" binding:"required" validate:"oneof=url xpath json css replace match substring math store condition cron"`
Var1 string `form:"var1" yaml:"var1" json:"var1" binding:"required"` Var1 string `form:"var1" yaml:"var1" json:"var1" binding:"required"`
Var2 *string `form:"var2" yaml:"var2" json:"var2"` Var2 string `form:"var2" yaml:"var2" json:"var2"`
Parents []*Filter `gorm:"-:all"` Parents []*Filter `gorm:"-:all"`
Children []*Filter `gorm:"-:all"` Children []*Filter `gorm:"-:all"`
Results []string `gorm:"-:all"` Results []string `gorm:"-:all"`
Logs []string `gorm:"-:all"` Logs []string `gorm:"-:all"`
CronEntry *cron.Entry `gorm:"-:all"` CronEntry *cron.Entry `gorm:"-:all"`
Enabled string `gorm:"-:all"` // lazy fix for not being able to pointer deref in golang template....
} }
func (filter *Filter) Log(v ...any) { func (filter *Filter) Log(v ...any) {

3
todo.md
View file

@ -1,5 +1,4 @@
# Todo # Todo
- add time 'jitter' to schedule filter - add time 'jitter' to schedule filter
- var2 with another duration string, var1 + (var2 * random) duration - var2 with another duration string, var1 + (var2 * random) duration
- remove startup delay? - remove startup delay?
- change Filter.Var2 from pointer to just value?

62
web/scraping.go
View file

@ -97,7 +97,7 @@ func ProcessFilters(filters []Filter, web *Web, watch *Watch, debug bool, schedu
// for schedule filters during editing, just check the schedule string and set to processed // for schedule filters during editing, just check the schedule string and set to processed
if debug && filter.Type == "cron" { if debug && filter.Type == "cron" {
if filter.Var2 != nil && *filter.Var2 == "no" { if filter.Var2 == "no" {
filter.Log("Schedule is disabled") filter.Log("Schedule is disabled")
} }
processedMap[filter.ID] = true processedMap[filter.ID] = true
@ -380,11 +380,7 @@ func getURLContent(filter *Filter, fetchURL string) (string, error) {
// getFilterResultBrowserlessURL Fetches the given URL and outputs the HTTP response through Browserless // getFilterResultBrowserlessURL Fetches the given URL and outputs the HTTP response through Browserless
func getFilterResultBrowserlessURL(filter *Filter, urlCache map[string]string, debug bool) { func getFilterResultBrowserlessURL(filter *Filter, urlCache map[string]string, debug bool) {
if filter.Var2 == nil { fetchURL := filter.Var2
filter.Log("filter.Var2 == nil")
return
}
fetchURL := *filter.Var2
val, exists := urlCache["b"+fetchURL] val, exists := urlCache["b"+fetchURL]
if debug && exists { if debug && exists {
filter.Results = append(filter.Results, val) filter.Results = append(filter.Results, val)
@ -499,10 +495,7 @@ func getBrowserlessFunctionContent(filter *Filter, result string) (string, error
return "", errors.New("browserless.url not set") return "", errors.New("browserless.url not set")
} }
browserlessURL := viper.GetString("browserless.url") browserlessURL := viper.GetString("browserless.url")
if filter.Var2 == nil { code := filter.Var2
return "", errors.New("filter.Var2 == nil")
}
code := *filter.Var2
data := struct { data := struct {
Code string `json:"code"` Code string `json:"code"`
Context BrowserlessContext `json:"context"` Context BrowserlessContext `json:"context"`
@ -533,10 +526,7 @@ func getBrowserlessFunctionContent(filter *Filter, result string) (string, error
// getFilterResultXPath Filters the results of its parents based on an XPath query // getFilterResultXPath Filters the results of its parents based on an XPath query
func getFilterResultXPath(filter *Filter) { func getFilterResultXPath(filter *Filter) {
selectType := "node" selectType := filter.Var2
if filter.Var2 != nil {
selectType = *filter.Var2
}
for _, parent := range filter.Parents { for _, parent := range filter.Parents {
for _, result := range parent.Results { for _, result := range parent.Results {
doc, err := htmlquery.Parse(strings.NewReader(result)) doc, err := htmlquery.Parse(strings.NewReader(result))
@ -596,10 +586,7 @@ func getFilterResultJSON(filter *Filter) {
// getFilterResultCSS Filters the results of its parents based on an CSS query // getFilterResultCSS Filters the results of its parents based on an CSS query
func getFilterResultCSS(filter *Filter) { func getFilterResultCSS(filter *Filter) {
selectType := "node" selectType := filter.Var2
if filter.Var2 != nil {
selectType = *filter.Var2
}
for _, parent := range filter.Parents { for _, parent := range filter.Parents {
for _, result := range parent.Results { for _, result := range parent.Results {
doc, err := html.Parse(strings.NewReader(result)) doc, err := html.Parse(strings.NewReader(result))
@ -659,11 +646,7 @@ func getFilterResultReplace(filter *Filter) {
} }
for _, parent := range filter.Parents { for _, parent := range filter.Parents {
for _, result := range parent.Results { for _, result := range parent.Results {
if filter.Var2 == nil { filter.Results = append(filter.Results, r.ReplaceAllString(result, filter.Var2))
filter.Results = append(filter.Results, r.ReplaceAllString(result, ""))
} else {
filter.Results = append(filter.Results, r.ReplaceAllString(result, *filter.Var2))
}
} }
} }
} }
@ -759,7 +742,7 @@ func getFilterResultSubstring(filter *Filter) {
// getFilterResultContains performs a regex contains on all the results of its parents // getFilterResultContains performs a regex contains on all the results of its parents
func getFilterResultContains(filter *Filter) { func getFilterResultContains(filter *Filter) {
r, err := regexp.Compile(filter.Var1) r, err := regexp.Compile(filter.Var1)
invert, err := strconv.ParseBool(*filter.Var2) invert, err := strconv.ParseBool(filter.Var2)
if err != nil { if err != nil {
invert = false invert = false
} }
@ -893,13 +876,9 @@ func roundFloat(val float64, precision uint) float64 {
// getFilterResultRound outputs the rounded value of every numerical input // getFilterResultRound outputs the rounded value of every numerical input
func getFilterResultRound(filter *Filter) { func getFilterResultRound(filter *Filter) {
var decimals int64 = 0 var decimals int64 = 0
if filter.Var2 != nil { d, err := strconv.ParseInt(filter.Var2, 10, 32)
d, err := strconv.ParseInt(*filter.Var2, 10, 32) if err == nil {
if err != nil { decimals = d
decimals = 0
} else {
decimals = d
}
} }
for _, parent := range filter.Parents { for _, parent := range filter.Parents {
@ -1023,13 +1002,9 @@ func getFilterResultConditionLowest(filter *Filter, db *gorm.DB) {
// getFilterResultConditionLowerThan outputs any input that is lower than a given value // getFilterResultConditionLowerThan outputs any input that is lower than a given value
func getFilterResultConditionLowerThan(filter *Filter) { func getFilterResultConditionLowerThan(filter *Filter) {
if filter.Var2 == nil { threshold, err := strconv.ParseFloat(filter.Var2, 64)
filter.Log("No threshold given")
return
}
threshold, err := strconv.ParseFloat(*filter.Var2, 64)
if err != nil { if err != nil {
filter.Log("Could not convert convert threshold to number: '", *filter.Var2, "'") filter.Log("Could not convert convert threshold to number: '", filter.Var2, "'")
return return
} }
for _, parent := range filter.Parents { for _, parent := range filter.Parents {
@ -1117,13 +1092,9 @@ func getFilterResultConditionHighest(filter *Filter, db *gorm.DB) {
// getFilterResultConditionHigherThan outputs any input that is higher than a given value // getFilterResultConditionHigherThan outputs any input that is higher than a given value
func getFilterResultConditionHigherThan(filter *Filter) { func getFilterResultConditionHigherThan(filter *Filter) {
if filter.Var2 == nil { threshold, err := strconv.ParseFloat(filter.Var2, 64)
filter.Log("No threshold given for Higher Than Filter")
return
}
threshold, err := strconv.ParseFloat(*filter.Var2, 64)
if err != nil { if err != nil {
filter.Log("Could not convert convert threshold to number: '", *filter.Var2, "'") filter.Log("Could not convert convert threshold to number: '", filter.Var2, "'")
return return
} }
for _, parent := range filter.Parents { for _, parent := range filter.Parents {
@ -1195,10 +1166,7 @@ func notifyFilter(filters []Filter, filter *Filter, watch *Watch, web *Web, debu
filter.Results = append(filter.Results, buffer.String()) filter.Results = append(filter.Results, buffer.String())
} else { } else {
notifier := filter.Var2 notifier := filter.Var2
if notifier == nil { web.notify(notifier, buffer.String())
return
}
web.notify(*notifier, buffer.String())
} }
} }

38
web/scraping_test.go
View file

@ -55,7 +55,6 @@ func DeepEqualStringSlice(a []string, b []string) bool {
} }
func TestFilterXPathNode(t *testing.T) { func TestFilterXPathNode(t *testing.T) {
var2 := "node"
var tests = []struct { var tests = []struct {
Query string Query string
Want []string Want []string
@ -79,7 +78,7 @@ func TestFilterXPathNode(t *testing.T) {
{Results: []string{HTML_STRING}}, {Results: []string{HTML_STRING}},
}, },
Var1: test.Query, Var1: test.Query,
Var2: &var2, Var2: "node",
} }
getFilterResultXPath( getFilterResultXPath(
&filter, &filter,
@ -92,7 +91,6 @@ func TestFilterXPathNode(t *testing.T) {
} }
func TestFilterXPathInnerHTML(t *testing.T) { func TestFilterXPathInnerHTML(t *testing.T) {
var2 := "inner"
var tests = []struct { var tests = []struct {
Query string Query string
Want []string Want []string
@ -115,7 +113,7 @@ func TestFilterXPathInnerHTML(t *testing.T) {
{Results: []string{HTML_STRING}}, {Results: []string{HTML_STRING}},
}, },
Var1: test.Query, Var1: test.Query,
Var2: &var2, Var2: "inner",
} }
getFilterResultXPath( getFilterResultXPath(
&filter, &filter,
@ -128,7 +126,6 @@ func TestFilterXPathInnerHTML(t *testing.T) {
} }
func TestFilterXPathAttributes(t *testing.T) { func TestFilterXPathAttributes(t *testing.T) {
var2 := "attr"
var tests = []struct { var tests = []struct {
Query string Query string
Want []string Want []string
@ -150,7 +147,7 @@ func TestFilterXPathAttributes(t *testing.T) {
{Results: []string{HTML_STRING}}, {Results: []string{HTML_STRING}},
}, },
Var1: test.Query, Var1: test.Query,
Var2: &var2, Var2: "attr",
} }
getFilterResultXPath( getFilterResultXPath(
&filter, &filter,
@ -194,7 +191,6 @@ func TestFilterJSON(t *testing.T) {
} }
func TestFilterCSSNode(t *testing.T) { func TestFilterCSSNode(t *testing.T) {
var2 := "node"
var tests = []struct { var tests = []struct {
Query string Query string
Want []string Want []string
@ -216,7 +212,7 @@ func TestFilterCSSNode(t *testing.T) {
{Results: []string{HTML_STRING}}, {Results: []string{HTML_STRING}},
}, },
Var1: test.Query, Var1: test.Query,
Var2: &var2, Var2: "node",
} }
getFilterResultCSS( getFilterResultCSS(
&filter, &filter,
@ -229,7 +225,6 @@ func TestFilterCSSNode(t *testing.T) {
} }
func TestFilterCSSInnerHTML(t *testing.T) { func TestFilterCSSInnerHTML(t *testing.T) {
var2 := "inner"
var tests = []struct { var tests = []struct {
Query string Query string
Want []string Want []string
@ -251,7 +246,7 @@ func TestFilterCSSInnerHTML(t *testing.T) {
{Results: []string{HTML_STRING}}, {Results: []string{HTML_STRING}},
}, },
Var1: test.Query, Var1: test.Query,
Var2: &var2, Var2: "inner",
} }
getFilterResultCSS( getFilterResultCSS(
&filter, &filter,
@ -264,7 +259,6 @@ func TestFilterCSSInnerHTML(t *testing.T) {
} }
func TestFilterCSSAttributes(t *testing.T) { func TestFilterCSSAttributes(t *testing.T) {
var2 := "attr"
var tests = []struct { var tests = []struct {
Query string Query string
Want []string Want []string
@ -286,7 +280,7 @@ func TestFilterCSSAttributes(t *testing.T) {
{Results: []string{HTML_STRING}}, {Results: []string{HTML_STRING}},
}, },
Var1: test.Query, Var1: test.Query,
Var2: &var2, Var2: "attr",
} }
getFilterResultCSS( getFilterResultCSS(
&filter, &filter,
@ -342,7 +336,7 @@ func TestFilterReplace(t *testing.T) {
{Results: []string{test.Input}}, {Results: []string{test.Input}},
}, },
Var1: test.Var1, Var1: test.Var1,
Var2: &test.Var2, Var2: test.Var2,
} }
getFilterResultReplace( getFilterResultReplace(
&filter, &filter,
@ -486,7 +480,7 @@ func TestFilterContains(t *testing.T) {
{Results: test.Input}, {Results: test.Input},
}, },
Var1: test.Query, Var1: test.Query,
Var2: &test.Invert, Var2: test.Invert,
} }
getFilterResultContains( getFilterResultContains(
&filter, &filter,
@ -685,7 +679,7 @@ func TestFilterRound(t *testing.T) {
Parents: []*Filter{ Parents: []*Filter{
{Results: test.Input}, {Results: test.Input},
}, },
Var2: &test.Query, Var2: test.Query,
} }
getFilterResultRound( getFilterResultRound(
&filter, &filter,
@ -791,7 +785,7 @@ func TestConditionDiff(t *testing.T) {
filter := Filter{ filter := Filter{
WatchID: test.WatchID, WatchID: test.WatchID,
Name: "Test", Name: "Test",
Var2: &testName, Var2: testName,
Parents: []*Filter{ Parents: []*Filter{
{Results: test.Input}, {Results: test.Input},
}, },
@ -911,7 +905,7 @@ func TestConditionLowerLast(t *testing.T) {
filter := Filter{ filter := Filter{
WatchID: test.WatchID, WatchID: test.WatchID,
Name: testName, Name: testName,
Var2: &testName, Var2: testName,
Parents: []*Filter{ Parents: []*Filter{
{Results: test.Input}, {Results: test.Input},
}, },
@ -1018,7 +1012,7 @@ func TestConditionLowest(t *testing.T) {
filter := Filter{ filter := Filter{
WatchID: test.WatchID, WatchID: test.WatchID,
Name: testName, Name: testName,
Var2: &testName, Var2: testName,
Parents: []*Filter{ Parents: []*Filter{
{Results: test.Input}, {Results: test.Input},
}, },
@ -1060,7 +1054,7 @@ func TestFilterLowerThan(t *testing.T) {
Results: test.Input, Results: test.Input,
}, },
}, },
Var2: &test.Threshold, Var2: test.Threshold,
} }
getFilterResultConditionLowerThan( getFilterResultConditionLowerThan(
&filter, &filter,
@ -1173,7 +1167,7 @@ func TestConditionHigherLast(t *testing.T) {
filter := Filter{ filter := Filter{
WatchID: test.WatchID, WatchID: test.WatchID,
Name: testName, Name: testName,
Var2: &testName, Var2: testName,
Parents: []*Filter{ Parents: []*Filter{
{Results: test.Input}, {Results: test.Input},
}, },
@ -1280,7 +1274,7 @@ func TestConditionHighest(t *testing.T) {
filter := Filter{ filter := Filter{
WatchID: test.WatchID, WatchID: test.WatchID,
Name: testName, Name: testName,
Var2: &testName, Var2: testName,
Parents: []*Filter{ Parents: []*Filter{
{Results: test.Input}, {Results: test.Input},
}, },
@ -1322,7 +1316,7 @@ func TestFilterHigherThan(t *testing.T) {
Results: test.Input, Results: test.Input,
}, },
}, },
Var2: &test.Threshold, Var2: test.Threshold,
} }
getFilterResultConditionHigherThan( getFilterResultConditionHigherThan(
&filter, &filter,

2
web/templates/schedules.html
View file

@ -28,7 +28,7 @@ GoWatch
{{ else }} {{ else }}
<td colspan="2" class="h3">Not scheduled</td> <td colspan="2" class="h3">Not scheduled</td>
{{ end }} {{ end }}
{{ if eq $scheduleFilter.Enabled "yes" }} {{ if eq $scheduleFilter.Var2 "yes" }}
<td class="h2"> <td class="h2">
<input class="form-check-input" id="schedules_{{ $scheduleFilter.ID }}" type="checkbox" value="{{ $scheduleFilter.ID }}" name="schedules" checked> <input class="form-check-input" id="schedules_{{ $scheduleFilter.ID }}" type="checkbox" value="{{ $scheduleFilter.ID }}" name="schedules" checked>
</td> </td>

6
web/web.go
View file

@ -88,7 +88,7 @@ func (web *Web) addCronJobIfCronFilter(filter *Filter, startup bool) {
if filter.Type != "cron" { if filter.Type != "cron" {
return return
} }
if filter.Var2 != nil && *filter.Var2 == "no" { if filter.Var2 == "no" {
return return
} }
scheduleSplit := strings.Split(filter.Var1, "+") scheduleSplit := strings.Split(filter.Var1, "+")
@ -538,7 +538,6 @@ func (web *Web) schedulesView(c *gin.Context) {
entry := web.cron.Entry(entryID) entry := web.cron.Entry(entryID)
filter.CronEntry = &entry filter.CronEntry = &entry
} }
filter.Enabled = *filter.Var2
watch := watchMap[filter.WatchID] watch := watchMap[filter.WatchID]
watchSchedules[watch] = append(watchSchedules[watch], filter) watchSchedules[watch] = append(watchSchedules[watch], filter)
} }
@ -569,8 +568,7 @@ func (web *Web) schedulesUpdate(c *gin.Context) {
web.cron.Remove(entryID) web.cron.Remove(entryID)
delete(web.cronWatch, cronFilter.ID) delete(web.cronWatch, cronFilter.ID)
} else if !exist && checked { } else if !exist && checked {
yes := "yes" cronFilter.Var2 = "yes"
cronFilter.Var2 = &yes
web.addCronJobIfCronFilter(cronFilter, false) web.addCronJobIfCronFilter(cronFilter, false)
} }
} }